]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
tracing: update sample with TRACE_INCLUDE_FILE
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
2cadf913 14#include <linux/ring_buffer.h>
bc0c38d1 15#include <linux/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
4c11d7ae 23#include <linux/pagemap.h>
bc0c38d1
SR
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
2cadf913 27#include <linux/kprobes.h>
bc0c38d1
SR
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
2cadf913 31#include <linux/splice.h>
3f5a54e3 32#include <linux/kdebug.h>
5f0c6c03 33#include <linux/string.h>
bc0c38d1
SR
34#include <linux/ctype.h>
35#include <linux/init.h>
2a2cc8f7 36#include <linux/poll.h>
bc0c38d1
SR
37#include <linux/gfp.h>
38#include <linux/fs.h>
86387f7e 39
bc0c38d1 40#include "trace.h"
f0868d1e 41#include "trace_output.h"
bc0c38d1 42
3928a8a2
SR
43#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
44
745b1626 45unsigned long __read_mostly tracing_max_latency;
bc0c38d1
SR
46unsigned long __read_mostly tracing_thresh;
47
73c5162a
SR
48/*
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
51 */
52static int ring_buffer_expanded;
53
8e1b82e0
FW
54/*
55 * We need to change this state when a selftest is running.
ff32504f
FW
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
5e1607a0 58 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
59 * at the same time, giving false positive or negative results.
60 */
8e1b82e0 61static bool __read_mostly tracing_selftest_running;
ff32504f 62
b2821ae6
SR
63/*
64 * If a tracer is running, we do not want to run SELFTEST.
65 */
66static bool __read_mostly tracing_selftest_disabled;
67
adf9f195
FW
68/* For tracers that don't implement custom flags */
69static struct tracer_opt dummy_tracer_opt[] = {
70 { }
71};
72
73static struct tracer_flags dummy_tracer_flags = {
74 .val = 0,
75 .opts = dummy_tracer_opt
76};
77
78static int dummy_set_flag(u32 old_flags, u32 bit, int set)
79{
80 return 0;
81}
0f048701
SR
82
83/*
84 * Kill all tracing for good (never come back).
85 * It is initialized to 1 but will turn to zero if the initialization
86 * of the tracer is successful. But that is the only place that sets
87 * this back to zero.
88 */
4fd27358 89static int tracing_disabled = 1;
0f048701 90
d769041f
SR
91static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
92
93static inline void ftrace_disable_cpu(void)
94{
95 preempt_disable();
96 local_inc(&__get_cpu_var(ftrace_cpu_disabled));
97}
98
99static inline void ftrace_enable_cpu(void)
100{
101 local_dec(&__get_cpu_var(ftrace_cpu_disabled));
102 preempt_enable();
103}
104
9e01c1b7 105static cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 106
b04cc6b1
FW
107/* Define which cpu buffers are currently read in trace_pipe */
108static cpumask_var_t tracing_reader_cpumask;
109
ab46428c 110#define for_each_tracing_cpu(cpu) \
9e01c1b7 111 for_each_cpu(cpu, tracing_buffer_mask)
ab46428c 112
944ac425
SR
113/*
114 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
115 *
116 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
117 * is set, then ftrace_dump is called. This will output the contents
118 * of the ftrace buffers to the console. This is very useful for
119 * capturing traces that lead to crashes and outputing it to a
120 * serial console.
121 *
122 * It is default off, but you can enable it with either specifying
123 * "ftrace_dump_on_oops" in the kernel command line, or setting
124 * /proc/sys/kernel/ftrace_dump_on_oops to true.
125 */
126int ftrace_dump_on_oops;
127
b2821ae6
SR
128static int tracing_set_tracer(const char *buf);
129
130#define BOOTUP_TRACER_SIZE 100
131static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
132static char *default_bootup_tracer;
d9e54076
PZ
133
134static int __init set_ftrace(char *str)
135{
b2821ae6
SR
136 strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
137 default_bootup_tracer = bootup_tracer_buf;
73c5162a
SR
138 /* We are using ftrace early, expand it */
139 ring_buffer_expanded = 1;
d9e54076
PZ
140 return 1;
141}
b2821ae6 142__setup("ftrace=", set_ftrace);
d9e54076 143
944ac425
SR
144static int __init set_ftrace_dump_on_oops(char *str)
145{
146 ftrace_dump_on_oops = 1;
147 return 1;
148}
149__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 150
cf8e3474 151unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
152{
153 nsec += 500;
154 do_div(nsec, 1000);
155 return nsec;
156}
157
4fcdae83
SR
158/*
159 * The global_trace is the descriptor that holds the tracing
160 * buffers for the live tracing. For each CPU, it contains
161 * a link list of pages that will store trace entries. The
162 * page descriptor of the pages in the memory is used to hold
163 * the link list by linking the lru item in the page descriptor
164 * to each of the pages in the buffer per CPU.
165 *
166 * For each active CPU there is a data field that holds the
167 * pages for the buffer for that CPU. Each CPU has the same number
168 * of pages allocated for its buffer.
169 */
bc0c38d1
SR
170static struct trace_array global_trace;
171
172static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
173
eb02ce01
TZ
174int filter_current_check_discard(struct ftrace_event_call *call, void *rec,
175 struct ring_buffer_event *event)
176{
177 return filter_check_discard(call, rec, global_trace.buffer, event);
178}
17c873ec 179EXPORT_SYMBOL_GPL(filter_current_check_discard);
eb02ce01 180
37886f6a
SR
181cycle_t ftrace_now(int cpu)
182{
183 u64 ts;
184
185 /* Early boot up does not have a buffer yet */
186 if (!global_trace.buffer)
187 return trace_clock_local();
188
189 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
190 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
191
192 return ts;
193}
bc0c38d1 194
4fcdae83
SR
195/*
196 * The max_tr is used to snapshot the global_trace when a maximum
197 * latency is reached. Some tracers will use this to store a maximum
198 * trace while it continues examining live traces.
199 *
200 * The buffers for the max_tr are set up the same as the global_trace.
201 * When a snapshot is taken, the link list of the max_tr is swapped
202 * with the link list of the global_trace and the buffers are reset for
203 * the global_trace so the tracing can continue.
204 */
bc0c38d1
SR
205static struct trace_array max_tr;
206
207static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
208
4fcdae83 209/* tracer_enabled is used to toggle activation of a tracer */
26994ead 210static int tracer_enabled = 1;
4fcdae83 211
9036990d
SR
212/**
213 * tracing_is_enabled - return tracer_enabled status
214 *
215 * This function is used by other tracers to know the status
216 * of the tracer_enabled flag. Tracers may use this function
217 * to know if it should enable their features when starting
218 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
219 */
220int tracing_is_enabled(void)
221{
222 return tracer_enabled;
223}
224
4fcdae83 225/*
3928a8a2
SR
226 * trace_buf_size is the size in bytes that is allocated
227 * for a buffer. Note, the number of bytes is always rounded
228 * to page size.
3f5a54e3
SR
229 *
230 * This number is purposely set to a low number of 16384.
231 * If the dump on oops happens, it will be much appreciated
232 * to not have to wait for all that output. Anyway this can be
233 * boot time and run time configurable.
4fcdae83 234 */
3928a8a2 235#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 236
3928a8a2 237static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 238
4fcdae83 239/* trace_types holds a link list of available tracers. */
bc0c38d1 240static struct tracer *trace_types __read_mostly;
4fcdae83
SR
241
242/* current_trace points to the tracer that is currently active */
bc0c38d1 243static struct tracer *current_trace __read_mostly;
4fcdae83
SR
244
245/*
246 * max_tracer_type_len is used to simplify the allocating of
247 * buffers to read userspace tracer names. We keep track of
248 * the longest tracer name registered.
249 */
bc0c38d1
SR
250static int max_tracer_type_len;
251
4fcdae83
SR
252/*
253 * trace_types_lock is used to protect the trace_types list.
254 * This lock is also used to keep user access serialized.
255 * Accesses from userspace will grab this lock while userspace
256 * activities happen inside the kernel.
257 */
bc0c38d1 258static DEFINE_MUTEX(trace_types_lock);
4fcdae83
SR
259
260/* trace_wait is a waitqueue for tasks blocked on trace_poll */
4e655519
IM
261static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
262
ee6bce52 263/* trace_flags holds trace_options default values */
12ef7d44 264unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a
SR
265 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
266 TRACE_ITER_GRAPH_TIME;
4e655519 267
4fcdae83
SR
268/**
269 * trace_wake_up - wake up tasks waiting for trace input
270 *
271 * Simply wakes up any task that is blocked on the trace_wait
272 * queue. These is used with trace_poll for tasks polling the trace.
273 */
4e655519
IM
274void trace_wake_up(void)
275{
017730c1
IM
276 /*
277 * The runqueue_is_locked() can fail, but this is the best we
278 * have for now:
279 */
280 if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
4e655519
IM
281 wake_up(&trace_wait);
282}
bc0c38d1 283
3928a8a2 284static int __init set_buf_size(char *str)
bc0c38d1 285{
3928a8a2 286 unsigned long buf_size;
c6caeeb1
SR
287 int ret;
288
bc0c38d1
SR
289 if (!str)
290 return 0;
3928a8a2 291 ret = strict_strtoul(str, 0, &buf_size);
c6caeeb1 292 /* nr_entries can not be zero */
3928a8a2 293 if (ret < 0 || buf_size == 0)
c6caeeb1 294 return 0;
3928a8a2 295 trace_buf_size = buf_size;
bc0c38d1
SR
296 return 1;
297}
3928a8a2 298__setup("trace_buf_size=", set_buf_size);
bc0c38d1 299
57f50be1
SR
300unsigned long nsecs_to_usecs(unsigned long nsecs)
301{
302 return nsecs / 1000;
303}
304
4fcdae83 305/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
306static const char *trace_options[] = {
307 "print-parent",
308 "sym-offset",
309 "sym-addr",
310 "verbose",
f9896bf3 311 "raw",
5e3ca0ec 312 "hex",
cb0f12aa 313 "bin",
2a2cc8f7 314 "block",
86387f7e 315 "stacktrace",
4ac3ba41 316 "sched-tree",
5e1607a0 317 "trace_printk",
b2a866f9 318 "ftrace_preempt",
9f029e83 319 "branch",
12ef7d44 320 "annotate",
02b67518 321 "userstacktrace",
b54d3de9 322 "sym-userobj",
66896a85 323 "printk-msg-only",
c4a8e8be 324 "context-info",
c032ef64 325 "latency-format",
af4617bd 326 "global-clock",
be6f164a 327 "sleep-time",
a2a16d6a 328 "graph-time",
bc0c38d1
SR
329 NULL
330};
331
4fcdae83
SR
332/*
333 * ftrace_max_lock is used to protect the swapping of buffers
334 * when taking a max snapshot. The buffers themselves are
335 * protected by per_cpu spinlocks. But the action of the swap
336 * needs its own lock.
337 *
338 * This is defined as a raw_spinlock_t in order to help
339 * with performance when lockdep debugging is enabled.
340 */
92205c23
SR
341static raw_spinlock_t ftrace_max_lock =
342 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
bc0c38d1
SR
343
344/*
345 * Copy the new maximum trace into the separate maximum-trace
346 * structure. (this way the maximum trace is permanently saved,
347 * for later retrieval via /debugfs/tracing/latency_trace)
348 */
e309b41d 349static void
bc0c38d1
SR
350__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
351{
352 struct trace_array_cpu *data = tr->data[cpu];
353
354 max_tr.cpu = cpu;
355 max_tr.time_start = data->preempt_timestamp;
356
357 data = max_tr.data[cpu];
358 data->saved_latency = tracing_max_latency;
359
360 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
361 data->pid = tsk->pid;
b6dff3ec 362 data->uid = task_uid(tsk);
bc0c38d1
SR
363 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
364 data->policy = tsk->policy;
365 data->rt_priority = tsk->rt_priority;
366
367 /* record this tasks comm */
af513098 368 tracing_record_cmdline(tsk);
bc0c38d1
SR
369}
370
6c6c2796
PP
371ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
372{
373 int len;
374 int ret;
375
2dc5d12b
SR
376 if (!cnt)
377 return 0;
378
6c6c2796
PP
379 if (s->len <= s->readpos)
380 return -EBUSY;
381
382 len = s->len - s->readpos;
383 if (cnt > len)
384 cnt = len;
385 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
2dc5d12b 386 if (ret == cnt)
6c6c2796
PP
387 return -EFAULT;
388
2dc5d12b
SR
389 cnt -= ret;
390
e74da523 391 s->readpos += cnt;
6c6c2796 392 return cnt;
214023c3
SR
393}
394
b8b94265 395static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
396{
397 int len;
398 void *ret;
399
400 if (s->len <= s->readpos)
401 return -EBUSY;
402
403 len = s->len - s->readpos;
404 if (cnt > len)
405 cnt = len;
406 ret = memcpy(buf, s->buffer + s->readpos, cnt);
407 if (!ret)
408 return -EFAULT;
409
e74da523 410 s->readpos += cnt;
3c56819b
EGM
411 return cnt;
412}
413
4fcdae83
SR
414/**
415 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
416 * @tr: tracer
417 * @tsk: the task with the latency
418 * @cpu: The cpu that initiated the trace.
419 *
420 * Flip the buffers between the @tr and the max_tr and record information
421 * about which task was the cause of this latency.
422 */
e309b41d 423void
bc0c38d1
SR
424update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
425{
3928a8a2 426 struct ring_buffer *buf = tr->buffer;
bc0c38d1 427
4c11d7ae 428 WARN_ON_ONCE(!irqs_disabled());
92205c23 429 __raw_spin_lock(&ftrace_max_lock);
3928a8a2
SR
430
431 tr->buffer = max_tr.buffer;
432 max_tr.buffer = buf;
433
d769041f 434 ftrace_disable_cpu();
3928a8a2 435 ring_buffer_reset(tr->buffer);
d769041f 436 ftrace_enable_cpu();
bc0c38d1
SR
437
438 __update_max_tr(tr, tsk, cpu);
92205c23 439 __raw_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
440}
441
442/**
443 * update_max_tr_single - only copy one trace over, and reset the rest
444 * @tr - tracer
445 * @tsk - task with the latency
446 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
447 *
448 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 449 */
e309b41d 450void
bc0c38d1
SR
451update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
452{
3928a8a2 453 int ret;
bc0c38d1 454
4c11d7ae 455 WARN_ON_ONCE(!irqs_disabled());
92205c23 456 __raw_spin_lock(&ftrace_max_lock);
bc0c38d1 457
d769041f
SR
458 ftrace_disable_cpu();
459
3928a8a2
SR
460 ring_buffer_reset(max_tr.buffer);
461 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
462
d769041f
SR
463 ftrace_enable_cpu();
464
97b17efe 465 WARN_ON_ONCE(ret && ret != -EAGAIN);
bc0c38d1
SR
466
467 __update_max_tr(tr, tsk, cpu);
92205c23 468 __raw_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
469}
470
4fcdae83
SR
471/**
472 * register_tracer - register a tracer with the ftrace system.
473 * @type - the plugin for the tracer
474 *
475 * Register a new plugin tracer.
476 */
bc0c38d1 477int register_tracer(struct tracer *type)
e7669b8e
HE
478__releases(kernel_lock)
479__acquires(kernel_lock)
bc0c38d1
SR
480{
481 struct tracer *t;
482 int len;
483 int ret = 0;
484
485 if (!type->name) {
486 pr_info("Tracer must have a name\n");
487 return -1;
488 }
489
86fa2f60
IM
490 /*
491 * When this gets called we hold the BKL which means that
492 * preemption is disabled. Various trace selftests however
493 * need to disable and enable preemption for successful tests.
494 * So we drop the BKL here and grab it after the tests again.
495 */
496 unlock_kernel();
bc0c38d1 497 mutex_lock(&trace_types_lock);
86fa2f60 498
8e1b82e0
FW
499 tracing_selftest_running = true;
500
bc0c38d1
SR
501 for (t = trace_types; t; t = t->next) {
502 if (strcmp(type->name, t->name) == 0) {
503 /* already found */
504 pr_info("Trace %s already registered\n",
505 type->name);
506 ret = -1;
507 goto out;
508 }
509 }
510
adf9f195
FW
511 if (!type->set_flag)
512 type->set_flag = &dummy_set_flag;
513 if (!type->flags)
514 type->flags = &dummy_tracer_flags;
515 else
516 if (!type->flags->opts)
517 type->flags->opts = dummy_tracer_opt;
6eaaa5d5
FW
518 if (!type->wait_pipe)
519 type->wait_pipe = default_wait_pipe;
520
adf9f195 521
60a11774 522#ifdef CONFIG_FTRACE_STARTUP_TEST
b2821ae6 523 if (type->selftest && !tracing_selftest_disabled) {
60a11774 524 struct tracer *saved_tracer = current_trace;
60a11774 525 struct trace_array *tr = &global_trace;
60a11774 526 int i;
ff32504f 527
60a11774
SR
528 /*
529 * Run a selftest on this tracer.
530 * Here we reset the trace buffer, and set the current
531 * tracer to be this tracer. The tracer can then run some
532 * internal tracing to verify that everything is in order.
533 * If we fail, we do not register this tracer.
534 */
86fa2f60 535 for_each_tracing_cpu(i)
3928a8a2 536 tracing_reset(tr, i);
86fa2f60 537
60a11774 538 current_trace = type;
60a11774
SR
539 /* the test is responsible for initializing and enabling */
540 pr_info("Testing tracer %s: ", type->name);
541 ret = type->selftest(type, tr);
542 /* the test is responsible for resetting too */
543 current_trace = saved_tracer;
60a11774
SR
544 if (ret) {
545 printk(KERN_CONT "FAILED!\n");
546 goto out;
547 }
1d4db00a 548 /* Only reset on passing, to avoid touching corrupted buffers */
86fa2f60 549 for_each_tracing_cpu(i)
3928a8a2 550 tracing_reset(tr, i);
86fa2f60 551
60a11774
SR
552 printk(KERN_CONT "PASSED\n");
553 }
554#endif
555
bc0c38d1
SR
556 type->next = trace_types;
557 trace_types = type;
558 len = strlen(type->name);
559 if (len > max_tracer_type_len)
560 max_tracer_type_len = len;
60a11774 561
bc0c38d1 562 out:
8e1b82e0 563 tracing_selftest_running = false;
bc0c38d1
SR
564 mutex_unlock(&trace_types_lock);
565
dac74940
SR
566 if (ret || !default_bootup_tracer)
567 goto out_unlock;
568
569 if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
570 goto out_unlock;
571
572 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
573 /* Do we want this tracer to start on bootup? */
574 tracing_set_tracer(type->name);
575 default_bootup_tracer = NULL;
576 /* disable other selftests, since this will break it. */
577 tracing_selftest_disabled = 1;
b2821ae6 578#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
579 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
580 type->name);
b2821ae6 581#endif
b2821ae6 582
dac74940 583 out_unlock:
b2821ae6 584 lock_kernel();
bc0c38d1
SR
585 return ret;
586}
587
588void unregister_tracer(struct tracer *type)
589{
590 struct tracer **t;
591 int len;
592
593 mutex_lock(&trace_types_lock);
594 for (t = &trace_types; *t; t = &(*t)->next) {
595 if (*t == type)
596 goto found;
597 }
598 pr_info("Trace %s not registered\n", type->name);
599 goto out;
600
601 found:
602 *t = (*t)->next;
b5db03c4
ACM
603
604 if (type == current_trace && tracer_enabled) {
605 tracer_enabled = 0;
606 tracing_stop();
607 if (current_trace->stop)
608 current_trace->stop(&global_trace);
609 current_trace = &nop_trace;
610 }
611
bc0c38d1
SR
612 if (strlen(type->name) != max_tracer_type_len)
613 goto out;
614
615 max_tracer_type_len = 0;
616 for (t = &trace_types; *t; t = &(*t)->next) {
617 len = strlen((*t)->name);
618 if (len > max_tracer_type_len)
619 max_tracer_type_len = len;
620 }
621 out:
622 mutex_unlock(&trace_types_lock);
623}
624
3928a8a2 625void tracing_reset(struct trace_array *tr, int cpu)
bc0c38d1 626{
d769041f 627 ftrace_disable_cpu();
3928a8a2 628 ring_buffer_reset_cpu(tr->buffer, cpu);
d769041f 629 ftrace_enable_cpu();
bc0c38d1
SR
630}
631
213cc060
PE
632void tracing_reset_online_cpus(struct trace_array *tr)
633{
634 int cpu;
635
636 tr->time_start = ftrace_now(tr->cpu);
637
638 for_each_online_cpu(cpu)
639 tracing_reset(tr, cpu);
640}
641
bc0c38d1 642#define SAVED_CMDLINES 128
2c7eea4c 643#define NO_CMDLINE_MAP UINT_MAX
bc0c38d1
SR
644static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
645static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
646static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
647static int cmdline_idx;
efed792d 648static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED;
25b0b44a 649
25b0b44a 650/* temporary disable recording */
4fd27358 651static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1
SR
652
653static void trace_init_cmdlines(void)
654{
2c7eea4c
TG
655 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
656 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
bc0c38d1
SR
657 cmdline_idx = 0;
658}
659
0f048701
SR
660static int trace_stop_count;
661static DEFINE_SPINLOCK(tracing_start_lock);
662
69bb54ec
SR
663/**
664 * ftrace_off_permanent - disable all ftrace code permanently
665 *
666 * This should only be called when a serious anomally has
667 * been detected. This will turn off the function tracing,
668 * ring buffers, and other tracing utilites. It takes no
669 * locks and can be called from any context.
670 */
671void ftrace_off_permanent(void)
672{
673 tracing_disabled = 1;
674 ftrace_stop();
675 tracing_off_permanent();
676}
677
0f048701
SR
678/**
679 * tracing_start - quick start of the tracer
680 *
681 * If tracing is enabled but was stopped by tracing_stop,
682 * this will start the tracer back up.
683 */
684void tracing_start(void)
685{
686 struct ring_buffer *buffer;
687 unsigned long flags;
688
689 if (tracing_disabled)
690 return;
691
692 spin_lock_irqsave(&tracing_start_lock, flags);
b06a8301
SR
693 if (--trace_stop_count) {
694 if (trace_stop_count < 0) {
695 /* Someone screwed up their debugging */
696 WARN_ON_ONCE(1);
697 trace_stop_count = 0;
698 }
0f048701
SR
699 goto out;
700 }
701
702
703 buffer = global_trace.buffer;
704 if (buffer)
705 ring_buffer_record_enable(buffer);
706
707 buffer = max_tr.buffer;
708 if (buffer)
709 ring_buffer_record_enable(buffer);
710
711 ftrace_start();
712 out:
713 spin_unlock_irqrestore(&tracing_start_lock, flags);
714}
715
716/**
717 * tracing_stop - quick stop of the tracer
718 *
719 * Light weight way to stop tracing. Use in conjunction with
720 * tracing_start.
721 */
722void tracing_stop(void)
723{
724 struct ring_buffer *buffer;
725 unsigned long flags;
726
727 ftrace_stop();
728 spin_lock_irqsave(&tracing_start_lock, flags);
729 if (trace_stop_count++)
730 goto out;
731
732 buffer = global_trace.buffer;
733 if (buffer)
734 ring_buffer_record_disable(buffer);
735
736 buffer = max_tr.buffer;
737 if (buffer)
738 ring_buffer_record_disable(buffer);
739
740 out:
741 spin_unlock_irqrestore(&tracing_start_lock, flags);
742}
743
e309b41d 744void trace_stop_cmdline_recording(void);
bc0c38d1 745
e309b41d 746static void trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 747{
a635cf04 748 unsigned pid, idx;
bc0c38d1
SR
749
750 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
751 return;
752
753 /*
754 * It's not the end of the world if we don't get
755 * the lock, but we also don't want to spin
756 * nor do we want to disable interrupts,
757 * so if we miss here, then better luck next time.
758 */
efed792d 759 if (!__raw_spin_trylock(&trace_cmdline_lock))
bc0c38d1
SR
760 return;
761
762 idx = map_pid_to_cmdline[tsk->pid];
2c7eea4c 763 if (idx == NO_CMDLINE_MAP) {
bc0c38d1
SR
764 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
765
a635cf04
CE
766 /*
767 * Check whether the cmdline buffer at idx has a pid
768 * mapped. We are going to overwrite that entry so we
769 * need to clear the map_pid_to_cmdline. Otherwise we
770 * would read the new comm for the old pid.
771 */
772 pid = map_cmdline_to_pid[idx];
773 if (pid != NO_CMDLINE_MAP)
774 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 775
a635cf04 776 map_cmdline_to_pid[idx] = tsk->pid;
bc0c38d1
SR
777 map_pid_to_cmdline[tsk->pid] = idx;
778
779 cmdline_idx = idx;
780 }
781
782 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
783
efed792d 784 __raw_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
785}
786
4ca53085 787void trace_find_cmdline(int pid, char comm[])
bc0c38d1 788{
bc0c38d1
SR
789 unsigned map;
790
4ca53085
SR
791 if (!pid) {
792 strcpy(comm, "<idle>");
793 return;
794 }
bc0c38d1 795
4ca53085
SR
796 if (pid > PID_MAX_DEFAULT) {
797 strcpy(comm, "<...>");
798 return;
799 }
bc0c38d1 800
4ca53085 801 __raw_spin_lock(&trace_cmdline_lock);
bc0c38d1 802 map = map_pid_to_cmdline[pid];
50d88758
TG
803 if (map != NO_CMDLINE_MAP)
804 strcpy(comm, saved_cmdlines[map]);
805 else
806 strcpy(comm, "<...>");
bc0c38d1 807
4ca53085 808 __raw_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
809}
810
e309b41d 811void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 812{
18aecd36
TG
813 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
814 !tracing_is_on())
bc0c38d1
SR
815 return;
816
817 trace_save_cmdline(tsk);
818}
819
45dcd8b8 820void
38697053
SR
821tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
822 int pc)
bc0c38d1
SR
823{
824 struct task_struct *tsk = current;
bc0c38d1 825
777e208d
SR
826 entry->preempt_count = pc & 0xff;
827 entry->pid = (tsk) ? tsk->pid : 0;
ef18012b 828 entry->tgid = (tsk) ? tsk->tgid : 0;
777e208d 829 entry->flags =
9244489a 830#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 831 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
832#else
833 TRACE_FLAG_IRQS_NOSUPPORT |
834#endif
bc0c38d1
SR
835 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
836 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
837 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
838}
839
51a763dd 840struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
7a4f453b 841 int type,
51a763dd
ACM
842 unsigned long len,
843 unsigned long flags, int pc)
844{
845 struct ring_buffer_event *event;
846
847 event = ring_buffer_lock_reserve(tr->buffer, len);
848 if (event != NULL) {
849 struct trace_entry *ent = ring_buffer_event_data(event);
850
851 tracing_generic_entry_update(ent, flags, pc);
852 ent->type = type;
853 }
854
855 return event;
856}
857static void ftrace_trace_stack(struct trace_array *tr,
858 unsigned long flags, int skip, int pc);
859static void ftrace_trace_userstack(struct trace_array *tr,
860 unsigned long flags, int pc);
861
07edf712
FW
862static inline void __trace_buffer_unlock_commit(struct trace_array *tr,
863 struct ring_buffer_event *event,
864 unsigned long flags, int pc,
865 int wake)
51a763dd
ACM
866{
867 ring_buffer_unlock_commit(tr->buffer, event);
868
869 ftrace_trace_stack(tr, flags, 6, pc);
870 ftrace_trace_userstack(tr, flags, pc);
07edf712
FW
871
872 if (wake)
873 trace_wake_up();
874}
875
876void trace_buffer_unlock_commit(struct trace_array *tr,
877 struct ring_buffer_event *event,
878 unsigned long flags, int pc)
879{
880 __trace_buffer_unlock_commit(tr, event, flags, pc, 1);
51a763dd
ACM
881}
882
ef5580d0 883struct ring_buffer_event *
7a4f453b 884trace_current_buffer_lock_reserve(int type, unsigned long len,
ef5580d0
SR
885 unsigned long flags, int pc)
886{
887 return trace_buffer_lock_reserve(&global_trace,
888 type, len, flags, pc);
889}
94487d6d 890EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0
SR
891
892void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
893 unsigned long flags, int pc)
894{
77d9f465 895 __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1);
07edf712 896}
94487d6d 897EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712
FW
898
899void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event,
900 unsigned long flags, int pc)
901{
77d9f465
SR
902 __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0);
903}
94487d6d 904EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
77d9f465
SR
905
906void trace_current_buffer_discard_commit(struct ring_buffer_event *event)
907{
908 ring_buffer_discard_commit(global_trace.buffer, event);
ef5580d0 909}
12acd473 910EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 911
e309b41d 912void
7be42151 913trace_function(struct trace_array *tr,
38697053
SR
914 unsigned long ip, unsigned long parent_ip, unsigned long flags,
915 int pc)
bc0c38d1 916{
e1112b4d 917 struct ftrace_event_call *call = &event_function;
3928a8a2 918 struct ring_buffer_event *event;
777e208d 919 struct ftrace_entry *entry;
bc0c38d1 920
d769041f
SR
921 /* If we are reading the ring buffer, don't trace */
922 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
923 return;
924
51a763dd
ACM
925 event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
926 flags, pc);
3928a8a2
SR
927 if (!event)
928 return;
929 entry = ring_buffer_event_data(event);
777e208d
SR
930 entry->ip = ip;
931 entry->parent_ip = parent_ip;
e1112b4d 932
eb02ce01
TZ
933 if (!filter_check_discard(call, entry, tr->buffer, event))
934 ring_buffer_unlock_commit(tr->buffer, event);
bc0c38d1
SR
935}
936
fb52607a 937#ifdef CONFIG_FUNCTION_GRAPH_TRACER
16185369 938static int __trace_graph_entry(struct trace_array *tr,
287b6e68
FW
939 struct ftrace_graph_ent *trace,
940 unsigned long flags,
941 int pc)
942{
e1112b4d 943 struct ftrace_event_call *call = &event_funcgraph_entry;
287b6e68
FW
944 struct ring_buffer_event *event;
945 struct ftrace_graph_ent_entry *entry;
287b6e68
FW
946
947 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
16185369 948 return 0;
287b6e68 949
51a763dd
ACM
950 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
951 sizeof(*entry), flags, pc);
287b6e68 952 if (!event)
16185369 953 return 0;
287b6e68 954 entry = ring_buffer_event_data(event);
287b6e68 955 entry->graph_ent = *trace;
eb02ce01
TZ
956 if (!filter_current_check_discard(call, entry, event))
957 ring_buffer_unlock_commit(global_trace.buffer, event);
16185369
FW
958
959 return 1;
287b6e68
FW
960}
961
962static void __trace_graph_return(struct trace_array *tr,
fb52607a 963 struct ftrace_graph_ret *trace,
15e6cb36
FW
964 unsigned long flags,
965 int pc)
966{
e1112b4d 967 struct ftrace_event_call *call = &event_funcgraph_exit;
15e6cb36 968 struct ring_buffer_event *event;
287b6e68 969 struct ftrace_graph_ret_entry *entry;
15e6cb36
FW
970
971 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
972 return;
973
51a763dd
ACM
974 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
975 sizeof(*entry), flags, pc);
15e6cb36
FW
976 if (!event)
977 return;
978 entry = ring_buffer_event_data(event);
287b6e68 979 entry->ret = *trace;
eb02ce01
TZ
980 if (!filter_current_check_discard(call, entry, event))
981 ring_buffer_unlock_commit(global_trace.buffer, event);
15e6cb36
FW
982}
983#endif
984
e309b41d 985void
2e0f5761 986ftrace(struct trace_array *tr, struct trace_array_cpu *data,
38697053
SR
987 unsigned long ip, unsigned long parent_ip, unsigned long flags,
988 int pc)
2e0f5761
IM
989{
990 if (likely(!atomic_read(&data->disabled)))
7be42151 991 trace_function(tr, ip, parent_ip, flags, pc);
2e0f5761
IM
992}
993
53614991 994static void __ftrace_trace_stack(struct trace_array *tr,
53614991
SR
995 unsigned long flags,
996 int skip, int pc)
86387f7e 997{
c2c80529 998#ifdef CONFIG_STACKTRACE
e1112b4d 999 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1000 struct ring_buffer_event *event;
777e208d 1001 struct stack_entry *entry;
86387f7e
IM
1002 struct stack_trace trace;
1003
51a763dd
ACM
1004 event = trace_buffer_lock_reserve(tr, TRACE_STACK,
1005 sizeof(*entry), flags, pc);
3928a8a2
SR
1006 if (!event)
1007 return;
1008 entry = ring_buffer_event_data(event);
777e208d 1009 memset(&entry->caller, 0, sizeof(entry->caller));
86387f7e
IM
1010
1011 trace.nr_entries = 0;
1012 trace.max_entries = FTRACE_STACK_ENTRIES;
1013 trace.skip = skip;
777e208d 1014 trace.entries = entry->caller;
86387f7e
IM
1015
1016 save_stack_trace(&trace);
eb02ce01
TZ
1017 if (!filter_check_discard(call, entry, tr->buffer, event))
1018 ring_buffer_unlock_commit(tr->buffer, event);
c2c80529 1019#endif
f0a920d5
IM
1020}
1021
53614991 1022static void ftrace_trace_stack(struct trace_array *tr,
53614991
SR
1023 unsigned long flags,
1024 int skip, int pc)
1025{
1026 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1027 return;
1028
7be42151 1029 __ftrace_trace_stack(tr, flags, skip, pc);
53614991
SR
1030}
1031
38697053 1032void __trace_stack(struct trace_array *tr,
38697053 1033 unsigned long flags,
53614991 1034 int skip, int pc)
38697053 1035{
7be42151 1036 __ftrace_trace_stack(tr, flags, skip, pc);
38697053
SR
1037}
1038
02b67518 1039static void ftrace_trace_userstack(struct trace_array *tr,
7be42151 1040 unsigned long flags, int pc)
02b67518 1041{
c7425acb 1042#ifdef CONFIG_STACKTRACE
e1112b4d 1043 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1044 struct ring_buffer_event *event;
02b67518
TE
1045 struct userstack_entry *entry;
1046 struct stack_trace trace;
02b67518
TE
1047
1048 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1049 return;
1050
51a763dd
ACM
1051 event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1052 sizeof(*entry), flags, pc);
02b67518
TE
1053 if (!event)
1054 return;
1055 entry = ring_buffer_event_data(event);
02b67518
TE
1056
1057 memset(&entry->caller, 0, sizeof(entry->caller));
1058
1059 trace.nr_entries = 0;
1060 trace.max_entries = FTRACE_STACK_ENTRIES;
1061 trace.skip = 0;
1062 trace.entries = entry->caller;
1063
1064 save_stack_trace_user(&trace);
eb02ce01
TZ
1065 if (!filter_check_discard(call, entry, tr->buffer, event))
1066 ring_buffer_unlock_commit(tr->buffer, event);
c7425acb 1067#endif
02b67518
TE
1068}
1069
4fd27358
HE
1070#ifdef UNUSED
1071static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1072{
7be42151 1073 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1074}
4fd27358 1075#endif /* UNUSED */
02b67518 1076
38697053 1077static void
7be42151 1078ftrace_trace_special(void *__tr,
38697053
SR
1079 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1080 int pc)
a4feb834 1081{
3928a8a2 1082 struct ring_buffer_event *event;
a4feb834 1083 struct trace_array *tr = __tr;
777e208d 1084 struct special_entry *entry;
a4feb834 1085
51a763dd
ACM
1086 event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1087 sizeof(*entry), 0, pc);
3928a8a2
SR
1088 if (!event)
1089 return;
1090 entry = ring_buffer_event_data(event);
777e208d
SR
1091 entry->arg1 = arg1;
1092 entry->arg2 = arg2;
1093 entry->arg3 = arg3;
51a763dd 1094 trace_buffer_unlock_commit(tr, event, 0, pc);
a4feb834
IM
1095}
1096
38697053
SR
1097void
1098__trace_special(void *__tr, void *__data,
1099 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1100{
7be42151 1101 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
38697053
SR
1102}
1103
e309b41d 1104void
bc0c38d1 1105tracing_sched_switch_trace(struct trace_array *tr,
86387f7e
IM
1106 struct task_struct *prev,
1107 struct task_struct *next,
38697053 1108 unsigned long flags, int pc)
bc0c38d1 1109{
e1112b4d 1110 struct ftrace_event_call *call = &event_context_switch;
3928a8a2 1111 struct ring_buffer_event *event;
777e208d 1112 struct ctx_switch_entry *entry;
bc0c38d1 1113
51a763dd
ACM
1114 event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1115 sizeof(*entry), flags, pc);
3928a8a2
SR
1116 if (!event)
1117 return;
1118 entry = ring_buffer_event_data(event);
777e208d
SR
1119 entry->prev_pid = prev->pid;
1120 entry->prev_prio = prev->prio;
1121 entry->prev_state = prev->state;
1122 entry->next_pid = next->pid;
1123 entry->next_prio = next->prio;
1124 entry->next_state = next->state;
1125 entry->next_cpu = task_cpu(next);
e1112b4d 1126
eb02ce01
TZ
1127 if (!filter_check_discard(call, entry, tr->buffer, event))
1128 trace_buffer_unlock_commit(tr, event, flags, pc);
bc0c38d1
SR
1129}
1130
57422797
IM
1131void
1132tracing_sched_wakeup_trace(struct trace_array *tr,
86387f7e
IM
1133 struct task_struct *wakee,
1134 struct task_struct *curr,
38697053 1135 unsigned long flags, int pc)
57422797 1136{
e1112b4d 1137 struct ftrace_event_call *call = &event_wakeup;
3928a8a2 1138 struct ring_buffer_event *event;
777e208d 1139 struct ctx_switch_entry *entry;
57422797 1140
51a763dd
ACM
1141 event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1142 sizeof(*entry), flags, pc);
3928a8a2
SR
1143 if (!event)
1144 return;
1145 entry = ring_buffer_event_data(event);
777e208d
SR
1146 entry->prev_pid = curr->pid;
1147 entry->prev_prio = curr->prio;
1148 entry->prev_state = curr->state;
1149 entry->next_pid = wakee->pid;
1150 entry->next_prio = wakee->prio;
1151 entry->next_state = wakee->state;
1152 entry->next_cpu = task_cpu(wakee);
6eaaa5d5 1153
eb02ce01
TZ
1154 if (!filter_check_discard(call, entry, tr->buffer, event))
1155 ring_buffer_unlock_commit(tr->buffer, event);
6eaaa5d5
FW
1156 ftrace_trace_stack(tr, flags, 6, pc);
1157 ftrace_trace_userstack(tr, flags, pc);
57422797
IM
1158}
1159
4902f884
SR
1160void
1161ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1162{
1163 struct trace_array *tr = &global_trace;
1164 struct trace_array_cpu *data;
5aa1ba6a 1165 unsigned long flags;
4902f884 1166 int cpu;
38697053 1167 int pc;
4902f884 1168
c76f0694 1169 if (tracing_disabled)
4902f884
SR
1170 return;
1171
38697053 1172 pc = preempt_count();
5aa1ba6a 1173 local_irq_save(flags);
4902f884
SR
1174 cpu = raw_smp_processor_id();
1175 data = tr->data[cpu];
4902f884 1176
5aa1ba6a 1177 if (likely(atomic_inc_return(&data->disabled) == 1))
7be42151 1178 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
4902f884 1179
5aa1ba6a
SR
1180 atomic_dec(&data->disabled);
1181 local_irq_restore(flags);
4902f884
SR
1182}
1183
fb52607a 1184#ifdef CONFIG_FUNCTION_GRAPH_TRACER
e49dc19c 1185int trace_graph_entry(struct ftrace_graph_ent *trace)
15e6cb36
FW
1186{
1187 struct trace_array *tr = &global_trace;
1188 struct trace_array_cpu *data;
1189 unsigned long flags;
1190 long disabled;
16185369 1191 int ret;
15e6cb36
FW
1192 int cpu;
1193 int pc;
1194
804a6851
SR
1195 if (!ftrace_trace_task(current))
1196 return 0;
1197
ea4e2bc4
SR
1198 if (!ftrace_graph_addr(trace->func))
1199 return 0;
1200
a5e25883 1201 local_irq_save(flags);
15e6cb36
FW
1202 cpu = raw_smp_processor_id();
1203 data = tr->data[cpu];
1204 disabled = atomic_inc_return(&data->disabled);
1205 if (likely(disabled == 1)) {
1206 pc = preempt_count();
16185369
FW
1207 ret = __trace_graph_entry(tr, trace, flags, pc);
1208 } else {
1209 ret = 0;
287b6e68 1210 }
ea4e2bc4
SR
1211 /* Only do the atomic if it is not already set */
1212 if (!test_tsk_trace_graph(current))
1213 set_tsk_trace_graph(current);
16185369 1214
287b6e68 1215 atomic_dec(&data->disabled);
a5e25883 1216 local_irq_restore(flags);
e49dc19c 1217
16185369 1218 return ret;
287b6e68
FW
1219}
1220
1221void trace_graph_return(struct ftrace_graph_ret *trace)
1222{
1223 struct trace_array *tr = &global_trace;
1224 struct trace_array_cpu *data;
1225 unsigned long flags;
1226 long disabled;
1227 int cpu;
1228 int pc;
1229
a5e25883 1230 local_irq_save(flags);
287b6e68
FW
1231 cpu = raw_smp_processor_id();
1232 data = tr->data[cpu];
1233 disabled = atomic_inc_return(&data->disabled);
1234 if (likely(disabled == 1)) {
1235 pc = preempt_count();
7be42151 1236 __trace_graph_return(tr, trace, flags, pc);
15e6cb36 1237 }
ea4e2bc4
SR
1238 if (!trace->depth)
1239 clear_tsk_trace_graph(current);
15e6cb36 1240 atomic_dec(&data->disabled);
a5e25883 1241 local_irq_restore(flags);
15e6cb36 1242}
fb52607a 1243#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
15e6cb36 1244
769b0441
FW
1245
1246/**
48ead020 1247 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
1248 *
1249 */
40ce74f1 1250int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 1251{
80370cb7
SR
1252 static raw_spinlock_t trace_buf_lock =
1253 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
769b0441
FW
1254 static u32 trace_buf[TRACE_BUF_SIZE];
1255
e1112b4d 1256 struct ftrace_event_call *call = &event_bprint;
769b0441
FW
1257 struct ring_buffer_event *event;
1258 struct trace_array *tr = &global_trace;
1259 struct trace_array_cpu *data;
48ead020 1260 struct bprint_entry *entry;
769b0441 1261 unsigned long flags;
3189cdb3 1262 int disable;
769b0441
FW
1263 int resched;
1264 int cpu, len = 0, size, pc;
1265
1266 if (unlikely(tracing_selftest_running || tracing_disabled))
1267 return 0;
1268
1269 /* Don't pollute graph traces with trace_vprintk internals */
1270 pause_graph_tracing();
1271
1272 pc = preempt_count();
1273 resched = ftrace_preempt_disable();
1274 cpu = raw_smp_processor_id();
1275 data = tr->data[cpu];
1276
3189cdb3
SR
1277 disable = atomic_inc_return(&data->disabled);
1278 if (unlikely(disable != 1))
769b0441
FW
1279 goto out;
1280
80370cb7
SR
1281 /* Lockdep uses trace_printk for lock tracing */
1282 local_irq_save(flags);
1283 __raw_spin_lock(&trace_buf_lock);
769b0441
FW
1284 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1285
1286 if (len > TRACE_BUF_SIZE || len < 0)
1287 goto out_unlock;
1288
1289 size = sizeof(*entry) + sizeof(u32) * len;
48ead020 1290 event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc);
769b0441
FW
1291 if (!event)
1292 goto out_unlock;
1293 entry = ring_buffer_event_data(event);
1294 entry->ip = ip;
769b0441
FW
1295 entry->fmt = fmt;
1296
1297 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
eb02ce01
TZ
1298 if (!filter_check_discard(call, entry, tr->buffer, event))
1299 ring_buffer_unlock_commit(tr->buffer, event);
769b0441
FW
1300
1301out_unlock:
80370cb7
SR
1302 __raw_spin_unlock(&trace_buf_lock);
1303 local_irq_restore(flags);
769b0441
FW
1304
1305out:
3189cdb3 1306 atomic_dec_return(&data->disabled);
769b0441
FW
1307 ftrace_preempt_enable(resched);
1308 unpause_graph_tracing();
1309
1310 return len;
1311}
48ead020
FW
1312EXPORT_SYMBOL_GPL(trace_vbprintk);
1313
40ce74f1 1314int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
48ead020
FW
1315{
1316 static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
1317 static char trace_buf[TRACE_BUF_SIZE];
1318
e1112b4d 1319 struct ftrace_event_call *call = &event_print;
48ead020
FW
1320 struct ring_buffer_event *event;
1321 struct trace_array *tr = &global_trace;
1322 struct trace_array_cpu *data;
1323 int cpu, len = 0, size, pc;
1324 struct print_entry *entry;
1325 unsigned long irq_flags;
3189cdb3 1326 int disable;
48ead020
FW
1327
1328 if (tracing_disabled || tracing_selftest_running)
1329 return 0;
1330
1331 pc = preempt_count();
1332 preempt_disable_notrace();
1333 cpu = raw_smp_processor_id();
1334 data = tr->data[cpu];
1335
3189cdb3
SR
1336 disable = atomic_inc_return(&data->disabled);
1337 if (unlikely(disable != 1))
48ead020
FW
1338 goto out;
1339
1340 pause_graph_tracing();
1341 raw_local_irq_save(irq_flags);
1342 __raw_spin_lock(&trace_buf_lock);
1343 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1344
1345 len = min(len, TRACE_BUF_SIZE-1);
1346 trace_buf[len] = 0;
1347
1348 size = sizeof(*entry) + len + 1;
1349 event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
1350 if (!event)
1351 goto out_unlock;
1352 entry = ring_buffer_event_data(event);
1353 entry->ip = ip;
48ead020
FW
1354
1355 memcpy(&entry->buf, trace_buf, len);
1356 entry->buf[len] = 0;
eb02ce01
TZ
1357 if (!filter_check_discard(call, entry, tr->buffer, event))
1358 ring_buffer_unlock_commit(tr->buffer, event);
48ead020
FW
1359
1360 out_unlock:
1361 __raw_spin_unlock(&trace_buf_lock);
1362 raw_local_irq_restore(irq_flags);
1363 unpause_graph_tracing();
1364 out:
3189cdb3 1365 atomic_dec_return(&data->disabled);
48ead020
FW
1366 preempt_enable_notrace();
1367
1368 return len;
1369}
769b0441
FW
1370EXPORT_SYMBOL_GPL(trace_vprintk);
1371
bc0c38d1
SR
1372enum trace_file_type {
1373 TRACE_FILE_LAT_FMT = 1,
12ef7d44 1374 TRACE_FILE_ANNOTATE = 2,
bc0c38d1
SR
1375};
1376
e2ac8ef5 1377static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 1378{
d769041f
SR
1379 /* Don't allow ftrace to trace into the ring buffers */
1380 ftrace_disable_cpu();
1381
5a90f577 1382 iter->idx++;
d769041f
SR
1383 if (iter->buffer_iter[iter->cpu])
1384 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1385
1386 ftrace_enable_cpu();
5a90f577
SR
1387}
1388
e309b41d 1389static struct trace_entry *
3928a8a2 1390peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
dd0e545f 1391{
3928a8a2
SR
1392 struct ring_buffer_event *event;
1393 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
dd0e545f 1394
d769041f
SR
1395 /* Don't allow ftrace to trace into the ring buffers */
1396 ftrace_disable_cpu();
1397
1398 if (buf_iter)
1399 event = ring_buffer_iter_peek(buf_iter, ts);
1400 else
1401 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1402
1403 ftrace_enable_cpu();
1404
3928a8a2 1405 return event ? ring_buffer_event_data(event) : NULL;
dd0e545f 1406}
d769041f 1407
dd0e545f 1408static struct trace_entry *
3928a8a2 1409__find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
bc0c38d1 1410{
3928a8a2 1411 struct ring_buffer *buffer = iter->tr->buffer;
bc0c38d1 1412 struct trace_entry *ent, *next = NULL;
b04cc6b1 1413 int cpu_file = iter->cpu_file;
3928a8a2 1414 u64 next_ts = 0, ts;
bc0c38d1
SR
1415 int next_cpu = -1;
1416 int cpu;
1417
b04cc6b1
FW
1418 /*
1419 * If we are in a per_cpu trace file, don't bother by iterating over
1420 * all cpu and peek directly.
1421 */
1422 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1423 if (ring_buffer_empty_cpu(buffer, cpu_file))
1424 return NULL;
1425 ent = peek_next_entry(iter, cpu_file, ent_ts);
1426 if (ent_cpu)
1427 *ent_cpu = cpu_file;
1428
1429 return ent;
1430 }
1431
ab46428c 1432 for_each_tracing_cpu(cpu) {
dd0e545f 1433
3928a8a2
SR
1434 if (ring_buffer_empty_cpu(buffer, cpu))
1435 continue;
dd0e545f 1436
3928a8a2 1437 ent = peek_next_entry(iter, cpu, &ts);
dd0e545f 1438
cdd31cd2
IM
1439 /*
1440 * Pick the entry with the smallest timestamp:
1441 */
3928a8a2 1442 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
1443 next = ent;
1444 next_cpu = cpu;
3928a8a2 1445 next_ts = ts;
bc0c38d1
SR
1446 }
1447 }
1448
1449 if (ent_cpu)
1450 *ent_cpu = next_cpu;
1451
3928a8a2
SR
1452 if (ent_ts)
1453 *ent_ts = next_ts;
1454
bc0c38d1
SR
1455 return next;
1456}
1457
dd0e545f 1458/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
1459struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1460 int *ent_cpu, u64 *ent_ts)
bc0c38d1 1461{
3928a8a2 1462 return __find_next_entry(iter, ent_cpu, ent_ts);
dd0e545f
SR
1463}
1464
1465/* Find the next real entry, and increment the iterator to the next entry */
1466static void *find_next_entry_inc(struct trace_iterator *iter)
1467{
3928a8a2 1468 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
dd0e545f 1469
3928a8a2 1470 if (iter->ent)
e2ac8ef5 1471 trace_iterator_increment(iter);
dd0e545f 1472
3928a8a2 1473 return iter->ent ? iter : NULL;
b3806b43 1474}
bc0c38d1 1475
e309b41d 1476static void trace_consume(struct trace_iterator *iter)
b3806b43 1477{
d769041f
SR
1478 /* Don't allow ftrace to trace into the ring buffers */
1479 ftrace_disable_cpu();
3928a8a2 1480 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
d769041f 1481 ftrace_enable_cpu();
bc0c38d1
SR
1482}
1483
e309b41d 1484static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
1485{
1486 struct trace_iterator *iter = m->private;
bc0c38d1 1487 int i = (int)*pos;
4e3c3333 1488 void *ent;
bc0c38d1
SR
1489
1490 (*pos)++;
1491
1492 /* can't go backwards */
1493 if (iter->idx > i)
1494 return NULL;
1495
1496 if (iter->idx < 0)
1497 ent = find_next_entry_inc(iter);
1498 else
1499 ent = iter;
1500
1501 while (ent && iter->idx < i)
1502 ent = find_next_entry_inc(iter);
1503
1504 iter->pos = *pos;
1505
bc0c38d1
SR
1506 return ent;
1507}
1508
d7350c3f
FW
1509/*
1510 * No necessary locking here. The worst thing which can
1511 * happen is loosing events consumed at the same time
1512 * by a trace_pipe reader.
1513 * Other than that, we don't risk to crash the ring buffer
1514 * because it serializes the readers.
1515 *
1516 * The current tracer is copied to avoid a global locking
1517 * all around.
1518 */
bc0c38d1
SR
1519static void *s_start(struct seq_file *m, loff_t *pos)
1520{
1521 struct trace_iterator *iter = m->private;
d7350c3f 1522 static struct tracer *old_tracer;
b04cc6b1 1523 int cpu_file = iter->cpu_file;
bc0c38d1
SR
1524 void *p = NULL;
1525 loff_t l = 0;
3928a8a2 1526 int cpu;
bc0c38d1 1527
d7350c3f 1528 /* copy the tracer to avoid using a global lock all around */
bc0c38d1 1529 mutex_lock(&trace_types_lock);
d7350c3f
FW
1530 if (unlikely(old_tracer != current_trace && current_trace)) {
1531 old_tracer = current_trace;
1532 *iter->trace = *current_trace;
d15f57f2 1533 }
d7350c3f 1534 mutex_unlock(&trace_types_lock);
bc0c38d1
SR
1535
1536 atomic_inc(&trace_record_cmdline_disabled);
1537
bc0c38d1
SR
1538 if (*pos != iter->pos) {
1539 iter->ent = NULL;
1540 iter->cpu = 0;
1541 iter->idx = -1;
1542
d769041f
SR
1543 ftrace_disable_cpu();
1544
b04cc6b1
FW
1545 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1546 for_each_tracing_cpu(cpu)
1547 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1548 } else
1549 ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1550
bc0c38d1 1551
d769041f
SR
1552 ftrace_enable_cpu();
1553
bc0c38d1
SR
1554 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1555 ;
1556
1557 } else {
4c11d7ae 1558 l = *pos - 1;
bc0c38d1
SR
1559 p = s_next(m, p, &l);
1560 }
1561
1562 return p;
1563}
1564
1565static void s_stop(struct seq_file *m, void *p)
1566{
bc0c38d1 1567 atomic_dec(&trace_record_cmdline_disabled);
bc0c38d1
SR
1568}
1569
e309b41d 1570static void print_lat_help_header(struct seq_file *m)
bc0c38d1 1571{
a6168353
ME
1572 seq_puts(m, "# _------=> CPU# \n");
1573 seq_puts(m, "# / _-----=> irqs-off \n");
1574 seq_puts(m, "# | / _----=> need-resched \n");
1575 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1576 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1577 seq_puts(m, "# |||| / \n");
1578 seq_puts(m, "# ||||| delay \n");
1579 seq_puts(m, "# cmd pid ||||| time | caller \n");
1580 seq_puts(m, "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
1581}
1582
e309b41d 1583static void print_func_help_header(struct seq_file *m)
bc0c38d1 1584{
a6168353
ME
1585 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1586 seq_puts(m, "# | | | | |\n");
bc0c38d1
SR
1587}
1588
1589
e309b41d 1590static void
bc0c38d1
SR
1591print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1592{
1593 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1594 struct trace_array *tr = iter->tr;
1595 struct trace_array_cpu *data = tr->data[tr->cpu];
1596 struct tracer *type = current_trace;
3928a8a2
SR
1597 unsigned long total;
1598 unsigned long entries;
bc0c38d1
SR
1599 const char *name = "preemption";
1600
1601 if (type)
1602 name = type->name;
1603
3928a8a2
SR
1604 entries = ring_buffer_entries(iter->tr->buffer);
1605 total = entries +
1606 ring_buffer_overruns(iter->tr->buffer);
bc0c38d1 1607
888b55dc 1608 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 1609 name, UTS_RELEASE);
888b55dc 1610 seq_puts(m, "# -----------------------------------"
bc0c38d1 1611 "---------------------------------\n");
888b55dc 1612 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 1613 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 1614 nsecs_to_usecs(data->saved_latency),
bc0c38d1 1615 entries,
4c11d7ae 1616 total,
bc0c38d1
SR
1617 tr->cpu,
1618#if defined(CONFIG_PREEMPT_NONE)
1619 "server",
1620#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1621 "desktop",
b5c21b45 1622#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
1623 "preempt",
1624#else
1625 "unknown",
1626#endif
1627 /* These are reserved for later use */
1628 0, 0, 0, 0);
1629#ifdef CONFIG_SMP
1630 seq_printf(m, " #P:%d)\n", num_online_cpus());
1631#else
1632 seq_puts(m, ")\n");
1633#endif
888b55dc
KM
1634 seq_puts(m, "# -----------------\n");
1635 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1
SR
1636 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1637 data->comm, data->pid, data->uid, data->nice,
1638 data->policy, data->rt_priority);
888b55dc 1639 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
1640
1641 if (data->critical_start) {
888b55dc 1642 seq_puts(m, "# => started at: ");
214023c3
SR
1643 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1644 trace_print_seq(m, &iter->seq);
888b55dc 1645 seq_puts(m, "\n# => ended at: ");
214023c3
SR
1646 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1647 trace_print_seq(m, &iter->seq);
888b55dc 1648 seq_puts(m, "#\n");
bc0c38d1
SR
1649 }
1650
888b55dc 1651 seq_puts(m, "#\n");
bc0c38d1
SR
1652}
1653
a309720c
SR
1654static void test_cpu_buff_start(struct trace_iterator *iter)
1655{
1656 struct trace_seq *s = &iter->seq;
1657
12ef7d44
SR
1658 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1659 return;
1660
1661 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1662 return;
1663
4462344e 1664 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
1665 return;
1666
4462344e 1667 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
1668
1669 /* Don't print started cpu buffer for the first entry of the trace */
1670 if (iter->idx > 1)
1671 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1672 iter->cpu);
a309720c
SR
1673}
1674
2c4f035f 1675static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 1676{
214023c3 1677 struct trace_seq *s = &iter->seq;
bc0c38d1 1678 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 1679 struct trace_entry *entry;
f633cef0 1680 struct trace_event *event;
bc0c38d1 1681
4e3c3333 1682 entry = iter->ent;
dd0e545f 1683
a309720c
SR
1684 test_cpu_buff_start(iter);
1685
c4a8e8be 1686 event = ftrace_find_event(entry->type);
bc0c38d1 1687
c4a8e8be 1688 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
27d48be8
SR
1689 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1690 if (!trace_print_lat_context(iter))
1691 goto partial;
1692 } else {
1693 if (!trace_print_context(iter))
1694 goto partial;
1695 }
c4a8e8be 1696 }
bc0c38d1 1697
268ccda0 1698 if (event)
d9793bd8
ACM
1699 return event->trace(iter, sym_flags);
1700
1701 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1702 goto partial;
02b67518 1703
2c4f035f 1704 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
1705partial:
1706 return TRACE_TYPE_PARTIAL_LINE;
bc0c38d1
SR
1707}
1708
2c4f035f 1709static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
1710{
1711 struct trace_seq *s = &iter->seq;
1712 struct trace_entry *entry;
f633cef0 1713 struct trace_event *event;
f9896bf3
IM
1714
1715 entry = iter->ent;
dd0e545f 1716
c4a8e8be 1717 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
d9793bd8
ACM
1718 if (!trace_seq_printf(s, "%d %d %llu ",
1719 entry->pid, iter->cpu, iter->ts))
1720 goto partial;
c4a8e8be 1721 }
f9896bf3 1722
f633cef0 1723 event = ftrace_find_event(entry->type);
268ccda0 1724 if (event)
d9793bd8
ACM
1725 return event->raw(iter, 0);
1726
1727 if (!trace_seq_printf(s, "%d ?\n", entry->type))
1728 goto partial;
777e208d 1729
2c4f035f 1730 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
1731partial:
1732 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3
IM
1733}
1734
2c4f035f 1735static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
1736{
1737 struct trace_seq *s = &iter->seq;
1738 unsigned char newline = '\n';
1739 struct trace_entry *entry;
f633cef0 1740 struct trace_event *event;
5e3ca0ec
IM
1741
1742 entry = iter->ent;
dd0e545f 1743
c4a8e8be
FW
1744 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1745 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1746 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1747 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1748 }
5e3ca0ec 1749
f633cef0 1750 event = ftrace_find_event(entry->type);
268ccda0 1751 if (event) {
ae7462b4 1752 enum print_line_t ret = event->hex(iter, 0);
d9793bd8
ACM
1753 if (ret != TRACE_TYPE_HANDLED)
1754 return ret;
1755 }
7104f300 1756
5e3ca0ec
IM
1757 SEQ_PUT_FIELD_RET(s, newline);
1758
2c4f035f 1759 return TRACE_TYPE_HANDLED;
5e3ca0ec
IM
1760}
1761
2c4f035f 1762static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
1763{
1764 struct trace_seq *s = &iter->seq;
1765 struct trace_entry *entry;
f633cef0 1766 struct trace_event *event;
cb0f12aa
IM
1767
1768 entry = iter->ent;
dd0e545f 1769
c4a8e8be
FW
1770 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1771 SEQ_PUT_FIELD_RET(s, entry->pid);
1830b52d 1772 SEQ_PUT_FIELD_RET(s, iter->cpu);
c4a8e8be
FW
1773 SEQ_PUT_FIELD_RET(s, iter->ts);
1774 }
cb0f12aa 1775
f633cef0 1776 event = ftrace_find_event(entry->type);
268ccda0 1777 return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
cb0f12aa
IM
1778}
1779
bc0c38d1
SR
1780static int trace_empty(struct trace_iterator *iter)
1781{
bc0c38d1
SR
1782 int cpu;
1783
9aba60fe
SR
1784 /* If we are looking at one CPU buffer, only check that one */
1785 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1786 cpu = iter->cpu_file;
1787 if (iter->buffer_iter[cpu]) {
1788 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1789 return 0;
1790 } else {
1791 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1792 return 0;
1793 }
1794 return 1;
1795 }
1796
ab46428c 1797 for_each_tracing_cpu(cpu) {
d769041f
SR
1798 if (iter->buffer_iter[cpu]) {
1799 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1800 return 0;
1801 } else {
1802 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1803 return 0;
1804 }
bc0c38d1 1805 }
d769041f 1806
797d3712 1807 return 1;
bc0c38d1
SR
1808}
1809
2c4f035f 1810static enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 1811{
2c4f035f
FW
1812 enum print_line_t ret;
1813
1814 if (iter->trace && iter->trace->print_line) {
1815 ret = iter->trace->print_line(iter);
1816 if (ret != TRACE_TYPE_UNHANDLED)
1817 return ret;
1818 }
72829bc3 1819
48ead020
FW
1820 if (iter->ent->type == TRACE_BPRINT &&
1821 trace_flags & TRACE_ITER_PRINTK &&
1822 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 1823 return trace_print_bprintk_msg_only(iter);
48ead020 1824
66896a85
FW
1825 if (iter->ent->type == TRACE_PRINT &&
1826 trace_flags & TRACE_ITER_PRINTK &&
1827 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 1828 return trace_print_printk_msg_only(iter);
66896a85 1829
cb0f12aa
IM
1830 if (trace_flags & TRACE_ITER_BIN)
1831 return print_bin_fmt(iter);
1832
5e3ca0ec
IM
1833 if (trace_flags & TRACE_ITER_HEX)
1834 return print_hex_fmt(iter);
1835
f9896bf3
IM
1836 if (trace_flags & TRACE_ITER_RAW)
1837 return print_raw_fmt(iter);
1838
f9896bf3
IM
1839 return print_trace_fmt(iter);
1840}
1841
bc0c38d1
SR
1842static int s_show(struct seq_file *m, void *v)
1843{
1844 struct trace_iterator *iter = v;
1845
1846 if (iter->ent == NULL) {
1847 if (iter->tr) {
1848 seq_printf(m, "# tracer: %s\n", iter->trace->name);
1849 seq_puts(m, "#\n");
1850 }
8bba1bf5
MM
1851 if (iter->trace && iter->trace->print_header)
1852 iter->trace->print_header(m);
1853 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
bc0c38d1
SR
1854 /* print nothing if the buffers are empty */
1855 if (trace_empty(iter))
1856 return 0;
1857 print_trace_header(m, iter);
1858 if (!(trace_flags & TRACE_ITER_VERBOSE))
1859 print_lat_help_header(m);
1860 } else {
1861 if (!(trace_flags & TRACE_ITER_VERBOSE))
1862 print_func_help_header(m);
1863 }
1864 } else {
f9896bf3 1865 print_trace_line(iter);
214023c3 1866 trace_print_seq(m, &iter->seq);
bc0c38d1
SR
1867 }
1868
1869 return 0;
1870}
1871
1872static struct seq_operations tracer_seq_ops = {
4bf39a94
IM
1873 .start = s_start,
1874 .next = s_next,
1875 .stop = s_stop,
1876 .show = s_show,
bc0c38d1
SR
1877};
1878
e309b41d 1879static struct trace_iterator *
85a2f9b4 1880__tracing_open(struct inode *inode, struct file *file)
bc0c38d1 1881{
b04cc6b1 1882 long cpu_file = (long) inode->i_private;
85a2f9b4 1883 void *fail_ret = ERR_PTR(-ENOMEM);
bc0c38d1 1884 struct trace_iterator *iter;
3928a8a2 1885 struct seq_file *m;
85a2f9b4 1886 int cpu, ret;
bc0c38d1 1887
85a2f9b4
SR
1888 if (tracing_disabled)
1889 return ERR_PTR(-ENODEV);
60a11774 1890
bc0c38d1 1891 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
85a2f9b4
SR
1892 if (!iter)
1893 return ERR_PTR(-ENOMEM);
bc0c38d1 1894
d7350c3f
FW
1895 /*
1896 * We make a copy of the current tracer to avoid concurrent
1897 * changes on it while we are reading.
1898 */
bc0c38d1 1899 mutex_lock(&trace_types_lock);
d7350c3f 1900 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 1901 if (!iter->trace)
d7350c3f 1902 goto fail;
85a2f9b4 1903
d7350c3f
FW
1904 if (current_trace)
1905 *iter->trace = *current_trace;
1906
b0dfa978
FW
1907 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL))
1908 goto fail;
1909
1910 cpumask_clear(iter->started);
1911
bc0c38d1
SR
1912 if (current_trace && current_trace->print_max)
1913 iter->tr = &max_tr;
1914 else
b04cc6b1 1915 iter->tr = &global_trace;
bc0c38d1 1916 iter->pos = -1;
d7350c3f 1917 mutex_init(&iter->mutex);
b04cc6b1 1918 iter->cpu_file = cpu_file;
bc0c38d1 1919
8bba1bf5
MM
1920 /* Notify the tracer early; before we stop tracing. */
1921 if (iter->trace && iter->trace->open)
a93751ca 1922 iter->trace->open(iter);
8bba1bf5 1923
12ef7d44
SR
1924 /* Annotate start of buffers if we had overruns */
1925 if (ring_buffer_overruns(iter->tr->buffer))
1926 iter->iter_flags |= TRACE_FILE_ANNOTATE;
1927
b04cc6b1
FW
1928 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1929 for_each_tracing_cpu(cpu) {
12ef7d44 1930
b04cc6b1
FW
1931 iter->buffer_iter[cpu] =
1932 ring_buffer_read_start(iter->tr->buffer, cpu);
b04cc6b1
FW
1933 }
1934 } else {
1935 cpu = iter->cpu_file;
3928a8a2 1936 iter->buffer_iter[cpu] =
b04cc6b1 1937 ring_buffer_read_start(iter->tr->buffer, cpu);
3928a8a2
SR
1938 }
1939
bc0c38d1 1940 /* TODO stop tracer */
85a2f9b4
SR
1941 ret = seq_open(file, &tracer_seq_ops);
1942 if (ret < 0) {
1943 fail_ret = ERR_PTR(ret);
3928a8a2 1944 goto fail_buffer;
85a2f9b4 1945 }
bc0c38d1 1946
3928a8a2
SR
1947 m = file->private_data;
1948 m->private = iter;
bc0c38d1 1949
3928a8a2 1950 /* stop the trace while dumping */
9036990d 1951 tracing_stop();
3928a8a2 1952
bc0c38d1
SR
1953 mutex_unlock(&trace_types_lock);
1954
bc0c38d1 1955 return iter;
3928a8a2
SR
1956
1957 fail_buffer:
1958 for_each_tracing_cpu(cpu) {
1959 if (iter->buffer_iter[cpu])
1960 ring_buffer_read_finish(iter->buffer_iter[cpu]);
1961 }
b0dfa978 1962 free_cpumask_var(iter->started);
d7350c3f 1963 fail:
3928a8a2 1964 mutex_unlock(&trace_types_lock);
d7350c3f 1965 kfree(iter->trace);
0bb943c7 1966 kfree(iter);
3928a8a2 1967
85a2f9b4 1968 return fail_ret;
bc0c38d1
SR
1969}
1970
1971int tracing_open_generic(struct inode *inode, struct file *filp)
1972{
60a11774
SR
1973 if (tracing_disabled)
1974 return -ENODEV;
1975
bc0c38d1
SR
1976 filp->private_data = inode->i_private;
1977 return 0;
1978}
1979
4fd27358 1980static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1
SR
1981{
1982 struct seq_file *m = (struct seq_file *)file->private_data;
4acd4d00 1983 struct trace_iterator *iter;
3928a8a2 1984 int cpu;
bc0c38d1 1985
4acd4d00
SR
1986 if (!(file->f_mode & FMODE_READ))
1987 return 0;
1988
1989 iter = m->private;
1990
bc0c38d1 1991 mutex_lock(&trace_types_lock);
3928a8a2
SR
1992 for_each_tracing_cpu(cpu) {
1993 if (iter->buffer_iter[cpu])
1994 ring_buffer_read_finish(iter->buffer_iter[cpu]);
1995 }
1996
bc0c38d1
SR
1997 if (iter->trace && iter->trace->close)
1998 iter->trace->close(iter);
1999
2000 /* reenable tracing if it was previously enabled */
9036990d 2001 tracing_start();
bc0c38d1
SR
2002 mutex_unlock(&trace_types_lock);
2003
2004 seq_release(inode, file);
d7350c3f 2005 mutex_destroy(&iter->mutex);
b0dfa978 2006 free_cpumask_var(iter->started);
d7350c3f 2007 kfree(iter->trace);
bc0c38d1
SR
2008 kfree(iter);
2009 return 0;
2010}
2011
2012static int tracing_open(struct inode *inode, struct file *file)
2013{
85a2f9b4
SR
2014 struct trace_iterator *iter;
2015 int ret = 0;
bc0c38d1 2016
4acd4d00
SR
2017 /* If this file was open for write, then erase contents */
2018 if ((file->f_mode & FMODE_WRITE) &&
2019 !(file->f_flags & O_APPEND)) {
2020 long cpu = (long) inode->i_private;
bc0c38d1 2021
4acd4d00
SR
2022 if (cpu == TRACE_PIPE_ALL_CPU)
2023 tracing_reset_online_cpus(&global_trace);
2024 else
2025 tracing_reset(&global_trace, cpu);
2026 }
bc0c38d1 2027
4acd4d00
SR
2028 if (file->f_mode & FMODE_READ) {
2029 iter = __tracing_open(inode, file);
2030 if (IS_ERR(iter))
2031 ret = PTR_ERR(iter);
2032 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2033 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2034 }
bc0c38d1
SR
2035 return ret;
2036}
2037
e309b41d 2038static void *
bc0c38d1
SR
2039t_next(struct seq_file *m, void *v, loff_t *pos)
2040{
2041 struct tracer *t = m->private;
2042
2043 (*pos)++;
2044
2045 if (t)
2046 t = t->next;
2047
2048 m->private = t;
2049
2050 return t;
2051}
2052
2053static void *t_start(struct seq_file *m, loff_t *pos)
2054{
2055 struct tracer *t = m->private;
2056 loff_t l = 0;
2057
2058 mutex_lock(&trace_types_lock);
2059 for (; t && l < *pos; t = t_next(m, t, &l))
2060 ;
2061
2062 return t;
2063}
2064
2065static void t_stop(struct seq_file *m, void *p)
2066{
2067 mutex_unlock(&trace_types_lock);
2068}
2069
2070static int t_show(struct seq_file *m, void *v)
2071{
2072 struct tracer *t = v;
2073
2074 if (!t)
2075 return 0;
2076
2077 seq_printf(m, "%s", t->name);
2078 if (t->next)
2079 seq_putc(m, ' ');
2080 else
2081 seq_putc(m, '\n');
2082
2083 return 0;
2084}
2085
2086static struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
2087 .start = t_start,
2088 .next = t_next,
2089 .stop = t_stop,
2090 .show = t_show,
bc0c38d1
SR
2091};
2092
2093static int show_traces_open(struct inode *inode, struct file *file)
2094{
2095 int ret;
2096
60a11774
SR
2097 if (tracing_disabled)
2098 return -ENODEV;
2099
bc0c38d1
SR
2100 ret = seq_open(file, &show_traces_seq_ops);
2101 if (!ret) {
2102 struct seq_file *m = file->private_data;
2103 m->private = trace_types;
2104 }
2105
2106 return ret;
2107}
2108
4acd4d00
SR
2109static ssize_t
2110tracing_write_stub(struct file *filp, const char __user *ubuf,
2111 size_t count, loff_t *ppos)
2112{
2113 return count;
2114}
2115
5e2336a0 2116static const struct file_operations tracing_fops = {
4bf39a94
IM
2117 .open = tracing_open,
2118 .read = seq_read,
4acd4d00 2119 .write = tracing_write_stub,
4bf39a94
IM
2120 .llseek = seq_lseek,
2121 .release = tracing_release,
bc0c38d1
SR
2122};
2123
5e2336a0 2124static const struct file_operations show_traces_fops = {
c7078de1
IM
2125 .open = show_traces_open,
2126 .read = seq_read,
2127 .release = seq_release,
2128};
2129
36dfe925
IM
2130/*
2131 * Only trace on a CPU if the bitmask is set:
2132 */
9e01c1b7 2133static cpumask_var_t tracing_cpumask;
36dfe925
IM
2134
2135/*
2136 * The tracer itself will not take this lock, but still we want
2137 * to provide a consistent cpumask to user-space:
2138 */
2139static DEFINE_MUTEX(tracing_cpumask_update_lock);
2140
2141/*
2142 * Temporary storage for the character representation of the
2143 * CPU bitmask (and one more byte for the newline):
2144 */
2145static char mask_str[NR_CPUS + 1];
2146
c7078de1
IM
2147static ssize_t
2148tracing_cpumask_read(struct file *filp, char __user *ubuf,
2149 size_t count, loff_t *ppos)
2150{
36dfe925 2151 int len;
c7078de1
IM
2152
2153 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 2154
9e01c1b7 2155 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
36dfe925
IM
2156 if (count - len < 2) {
2157 count = -EINVAL;
2158 goto out_err;
2159 }
2160 len += sprintf(mask_str + len, "\n");
2161 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2162
2163out_err:
c7078de1
IM
2164 mutex_unlock(&tracing_cpumask_update_lock);
2165
2166 return count;
2167}
2168
2169static ssize_t
2170tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2171 size_t count, loff_t *ppos)
2172{
36dfe925 2173 int err, cpu;
9e01c1b7
RR
2174 cpumask_var_t tracing_cpumask_new;
2175
2176 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2177 return -ENOMEM;
c7078de1
IM
2178
2179 mutex_lock(&tracing_cpumask_update_lock);
9e01c1b7 2180 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 2181 if (err)
36dfe925
IM
2182 goto err_unlock;
2183
a5e25883 2184 local_irq_disable();
92205c23 2185 __raw_spin_lock(&ftrace_max_lock);
ab46428c 2186 for_each_tracing_cpu(cpu) {
36dfe925
IM
2187 /*
2188 * Increase/decrease the disabled counter if we are
2189 * about to flip a bit in the cpumask:
2190 */
9e01c1b7
RR
2191 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2192 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
36dfe925
IM
2193 atomic_inc(&global_trace.data[cpu]->disabled);
2194 }
9e01c1b7
RR
2195 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2196 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
36dfe925
IM
2197 atomic_dec(&global_trace.data[cpu]->disabled);
2198 }
2199 }
92205c23 2200 __raw_spin_unlock(&ftrace_max_lock);
a5e25883 2201 local_irq_enable();
36dfe925 2202
9e01c1b7 2203 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
2204
2205 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 2206 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
2207
2208 return count;
36dfe925
IM
2209
2210err_unlock:
2211 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 2212 free_cpumask_var(tracing_cpumask);
36dfe925
IM
2213
2214 return err;
c7078de1
IM
2215}
2216
5e2336a0 2217static const struct file_operations tracing_cpumask_fops = {
c7078de1
IM
2218 .open = tracing_open_generic,
2219 .read = tracing_cpumask_read,
2220 .write = tracing_cpumask_write,
bc0c38d1
SR
2221};
2222
2223static ssize_t
ee6bce52 2224tracing_trace_options_read(struct file *filp, char __user *ubuf,
bc0c38d1
SR
2225 size_t cnt, loff_t *ppos)
2226{
d8e83d26
SR
2227 struct tracer_opt *trace_opts;
2228 u32 tracer_flags;
2229 int len = 0;
bc0c38d1
SR
2230 char *buf;
2231 int r = 0;
d8e83d26 2232 int i;
adf9f195 2233
bc0c38d1 2234
c3706f00 2235 /* calculate max size */
bc0c38d1
SR
2236 for (i = 0; trace_options[i]; i++) {
2237 len += strlen(trace_options[i]);
5c6a3ae1 2238 len += 3; /* "no" and newline */
bc0c38d1
SR
2239 }
2240
d8e83d26
SR
2241 mutex_lock(&trace_types_lock);
2242 tracer_flags = current_trace->flags->val;
2243 trace_opts = current_trace->flags->opts;
2244
adf9f195
FW
2245 /*
2246 * Increase the size with names of options specific
2247 * of the current tracer.
2248 */
2249 for (i = 0; trace_opts[i].name; i++) {
2250 len += strlen(trace_opts[i].name);
5c6a3ae1 2251 len += 3; /* "no" and newline */
adf9f195
FW
2252 }
2253
bc0c38d1
SR
2254 /* +2 for \n and \0 */
2255 buf = kmalloc(len + 2, GFP_KERNEL);
d8e83d26
SR
2256 if (!buf) {
2257 mutex_unlock(&trace_types_lock);
bc0c38d1 2258 return -ENOMEM;
d8e83d26 2259 }
bc0c38d1
SR
2260
2261 for (i = 0; trace_options[i]; i++) {
2262 if (trace_flags & (1 << i))
5c6a3ae1 2263 r += sprintf(buf + r, "%s\n", trace_options[i]);
bc0c38d1 2264 else
5c6a3ae1 2265 r += sprintf(buf + r, "no%s\n", trace_options[i]);
bc0c38d1
SR
2266 }
2267
adf9f195
FW
2268 for (i = 0; trace_opts[i].name; i++) {
2269 if (tracer_flags & trace_opts[i].bit)
5c6a3ae1 2270 r += sprintf(buf + r, "%s\n",
adf9f195
FW
2271 trace_opts[i].name);
2272 else
5c6a3ae1 2273 r += sprintf(buf + r, "no%s\n",
adf9f195
FW
2274 trace_opts[i].name);
2275 }
d8e83d26 2276 mutex_unlock(&trace_types_lock);
adf9f195 2277
bc0c38d1
SR
2278 WARN_ON(r >= len + 2);
2279
36dfe925 2280 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2281
2282 kfree(buf);
bc0c38d1
SR
2283 return r;
2284}
2285
adf9f195
FW
2286/* Try to assign a tracer specific option */
2287static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2288{
2289 struct tracer_flags *trace_flags = trace->flags;
2290 struct tracer_opt *opts = NULL;
2291 int ret = 0, i = 0;
2292 int len;
2293
2294 for (i = 0; trace_flags->opts[i].name; i++) {
2295 opts = &trace_flags->opts[i];
2296 len = strlen(opts->name);
2297
2298 if (strncmp(cmp, opts->name, len) == 0) {
2299 ret = trace->set_flag(trace_flags->val,
2300 opts->bit, !neg);
2301 break;
2302 }
2303 }
2304 /* Not found */
2305 if (!trace_flags->opts[i].name)
2306 return -EINVAL;
2307
2308 /* Refused to handle */
2309 if (ret)
2310 return ret;
2311
2312 if (neg)
2313 trace_flags->val &= ~opts->bit;
2314 else
2315 trace_flags->val |= opts->bit;
2316
2317 return 0;
2318}
2319
af4617bd
SR
2320static void set_tracer_flags(unsigned int mask, int enabled)
2321{
2322 /* do nothing if flag is already set */
2323 if (!!(trace_flags & mask) == !!enabled)
2324 return;
2325
2326 if (enabled)
2327 trace_flags |= mask;
2328 else
2329 trace_flags &= ~mask;
2330
2331 if (mask == TRACE_ITER_GLOBAL_CLK) {
2332 u64 (*func)(void);
2333
2334 if (enabled)
2335 func = trace_clock_global;
2336 else
2337 func = trace_clock_local;
2338
2339 mutex_lock(&trace_types_lock);
2340 ring_buffer_set_clock(global_trace.buffer, func);
2341
2342 if (max_tr.buffer)
2343 ring_buffer_set_clock(max_tr.buffer, func);
2344 mutex_unlock(&trace_types_lock);
2345 }
2346}
2347
bc0c38d1 2348static ssize_t
ee6bce52 2349tracing_trace_options_write(struct file *filp, const char __user *ubuf,
bc0c38d1
SR
2350 size_t cnt, loff_t *ppos)
2351{
2352 char buf[64];
2353 char *cmp = buf;
2354 int neg = 0;
adf9f195 2355 int ret;
bc0c38d1
SR
2356 int i;
2357
cffae437
SR
2358 if (cnt >= sizeof(buf))
2359 return -EINVAL;
bc0c38d1
SR
2360
2361 if (copy_from_user(&buf, ubuf, cnt))
2362 return -EFAULT;
2363
2364 buf[cnt] = 0;
2365
2366 if (strncmp(buf, "no", 2) == 0) {
2367 neg = 1;
2368 cmp += 2;
2369 }
2370
2371 for (i = 0; trace_options[i]; i++) {
2372 int len = strlen(trace_options[i]);
2373
2374 if (strncmp(cmp, trace_options[i], len) == 0) {
af4617bd 2375 set_tracer_flags(1 << i, !neg);
bc0c38d1
SR
2376 break;
2377 }
2378 }
adf9f195
FW
2379
2380 /* If no option could be set, test the specific tracer options */
2381 if (!trace_options[i]) {
d8e83d26 2382 mutex_lock(&trace_types_lock);
adf9f195 2383 ret = set_tracer_option(current_trace, cmp, neg);
d8e83d26 2384 mutex_unlock(&trace_types_lock);
adf9f195
FW
2385 if (ret)
2386 return ret;
2387 }
bc0c38d1
SR
2388
2389 filp->f_pos += cnt;
2390
2391 return cnt;
2392}
2393
5e2336a0 2394static const struct file_operations tracing_iter_fops = {
c7078de1 2395 .open = tracing_open_generic,
ee6bce52
SR
2396 .read = tracing_trace_options_read,
2397 .write = tracing_trace_options_write,
bc0c38d1
SR
2398};
2399
7bd2f24c
IM
2400static const char readme_msg[] =
2401 "tracing mini-HOWTO:\n\n"
2402 "# mkdir /debug\n"
2403 "# mount -t debugfs nodev /debug\n\n"
2404 "# cat /debug/tracing/available_tracers\n"
bc2b6871 2405 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
7bd2f24c 2406 "# cat /debug/tracing/current_tracer\n"
bc2b6871 2407 "nop\n"
7bd2f24c
IM
2408 "# echo sched_switch > /debug/tracing/current_tracer\n"
2409 "# cat /debug/tracing/current_tracer\n"
2410 "sched_switch\n"
ee6bce52 2411 "# cat /debug/tracing/trace_options\n"
7bd2f24c 2412 "noprint-parent nosym-offset nosym-addr noverbose\n"
ee6bce52 2413 "# echo print-parent > /debug/tracing/trace_options\n"
7bd2f24c
IM
2414 "# echo 1 > /debug/tracing/tracing_enabled\n"
2415 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2416 "echo 0 > /debug/tracing/tracing_enabled\n"
2417;
2418
2419static ssize_t
2420tracing_readme_read(struct file *filp, char __user *ubuf,
2421 size_t cnt, loff_t *ppos)
2422{
2423 return simple_read_from_buffer(ubuf, cnt, ppos,
2424 readme_msg, strlen(readme_msg));
2425}
2426
5e2336a0 2427static const struct file_operations tracing_readme_fops = {
c7078de1
IM
2428 .open = tracing_open_generic,
2429 .read = tracing_readme_read,
7bd2f24c
IM
2430};
2431
69abe6a5
AP
2432static ssize_t
2433tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2434 size_t cnt, loff_t *ppos)
2435{
2436 char *buf_comm;
2437 char *file_buf;
2438 char *buf;
2439 int len = 0;
2440 int pid;
2441 int i;
2442
2443 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2444 if (!file_buf)
2445 return -ENOMEM;
2446
2447 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2448 if (!buf_comm) {
2449 kfree(file_buf);
2450 return -ENOMEM;
2451 }
2452
2453 buf = file_buf;
2454
2455 for (i = 0; i < SAVED_CMDLINES; i++) {
2456 int r;
2457
2458 pid = map_cmdline_to_pid[i];
2459 if (pid == -1 || pid == NO_CMDLINE_MAP)
2460 continue;
2461
2462 trace_find_cmdline(pid, buf_comm);
2463 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2464 buf += r;
2465 len += r;
2466 }
2467
2468 len = simple_read_from_buffer(ubuf, cnt, ppos,
2469 file_buf, len);
2470
2471 kfree(file_buf);
2472 kfree(buf_comm);
2473
2474 return len;
2475}
2476
2477static const struct file_operations tracing_saved_cmdlines_fops = {
2478 .open = tracing_open_generic,
2479 .read = tracing_saved_cmdlines_read,
2480};
2481
bc0c38d1
SR
2482static ssize_t
2483tracing_ctrl_read(struct file *filp, char __user *ubuf,
2484 size_t cnt, loff_t *ppos)
2485{
bc0c38d1
SR
2486 char buf[64];
2487 int r;
2488
9036990d 2489 r = sprintf(buf, "%u\n", tracer_enabled);
4e3c3333 2490 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2491}
2492
2493static ssize_t
2494tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2495 size_t cnt, loff_t *ppos)
2496{
2497 struct trace_array *tr = filp->private_data;
bc0c38d1 2498 char buf[64];
5e39841c 2499 unsigned long val;
c6caeeb1 2500 int ret;
bc0c38d1 2501
cffae437
SR
2502 if (cnt >= sizeof(buf))
2503 return -EINVAL;
bc0c38d1
SR
2504
2505 if (copy_from_user(&buf, ubuf, cnt))
2506 return -EFAULT;
2507
2508 buf[cnt] = 0;
2509
c6caeeb1
SR
2510 ret = strict_strtoul(buf, 10, &val);
2511 if (ret < 0)
2512 return ret;
bc0c38d1
SR
2513
2514 val = !!val;
2515
2516 mutex_lock(&trace_types_lock);
9036990d
SR
2517 if (tracer_enabled ^ val) {
2518 if (val) {
bc0c38d1 2519 tracer_enabled = 1;
9036990d
SR
2520 if (current_trace->start)
2521 current_trace->start(tr);
2522 tracing_start();
2523 } else {
bc0c38d1 2524 tracer_enabled = 0;
9036990d
SR
2525 tracing_stop();
2526 if (current_trace->stop)
2527 current_trace->stop(tr);
2528 }
bc0c38d1
SR
2529 }
2530 mutex_unlock(&trace_types_lock);
2531
2532 filp->f_pos += cnt;
2533
2534 return cnt;
2535}
2536
2537static ssize_t
2538tracing_set_trace_read(struct file *filp, char __user *ubuf,
2539 size_t cnt, loff_t *ppos)
2540{
2541 char buf[max_tracer_type_len+2];
2542 int r;
2543
2544 mutex_lock(&trace_types_lock);
2545 if (current_trace)
2546 r = sprintf(buf, "%s\n", current_trace->name);
2547 else
2548 r = sprintf(buf, "\n");
2549 mutex_unlock(&trace_types_lock);
2550
4bf39a94 2551 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2552}
2553
b6f11df2
ACM
2554int tracer_init(struct tracer *t, struct trace_array *tr)
2555{
2556 tracing_reset_online_cpus(tr);
2557 return t->init(tr);
2558}
2559
73c5162a
SR
2560static int tracing_resize_ring_buffer(unsigned long size)
2561{
2562 int ret;
2563
2564 /*
2565 * If kernel or user changes the size of the ring buffer
a123c52b
SR
2566 * we use the size that was given, and we can forget about
2567 * expanding it later.
73c5162a
SR
2568 */
2569 ring_buffer_expanded = 1;
2570
2571 ret = ring_buffer_resize(global_trace.buffer, size);
2572 if (ret < 0)
2573 return ret;
2574
2575 ret = ring_buffer_resize(max_tr.buffer, size);
2576 if (ret < 0) {
2577 int r;
2578
2579 r = ring_buffer_resize(global_trace.buffer,
2580 global_trace.entries);
2581 if (r < 0) {
a123c52b
SR
2582 /*
2583 * AARGH! We are left with different
2584 * size max buffer!!!!
2585 * The max buffer is our "snapshot" buffer.
2586 * When a tracer needs a snapshot (one of the
2587 * latency tracers), it swaps the max buffer
2588 * with the saved snap shot. We succeeded to
2589 * update the size of the main buffer, but failed to
2590 * update the size of the max buffer. But when we tried
2591 * to reset the main buffer to the original size, we
2592 * failed there too. This is very unlikely to
2593 * happen, but if it does, warn and kill all
2594 * tracing.
2595 */
73c5162a
SR
2596 WARN_ON(1);
2597 tracing_disabled = 1;
2598 }
2599 return ret;
2600 }
2601
2602 global_trace.entries = size;
2603
2604 return ret;
2605}
2606
1852fcce
SR
2607/**
2608 * tracing_update_buffers - used by tracing facility to expand ring buffers
2609 *
2610 * To save on memory when the tracing is never used on a system with it
2611 * configured in. The ring buffers are set to a minimum size. But once
2612 * a user starts to use the tracing facility, then they need to grow
2613 * to their default size.
2614 *
2615 * This function is to be called when a tracer is about to be used.
2616 */
2617int tracing_update_buffers(void)
2618{
2619 int ret = 0;
2620
1027fcb2 2621 mutex_lock(&trace_types_lock);
1852fcce
SR
2622 if (!ring_buffer_expanded)
2623 ret = tracing_resize_ring_buffer(trace_buf_size);
1027fcb2 2624 mutex_unlock(&trace_types_lock);
1852fcce
SR
2625
2626 return ret;
2627}
2628
577b785f
SR
2629struct trace_option_dentry;
2630
2631static struct trace_option_dentry *
2632create_trace_option_files(struct tracer *tracer);
2633
2634static void
2635destroy_trace_option_files(struct trace_option_dentry *topts);
2636
b2821ae6 2637static int tracing_set_tracer(const char *buf)
bc0c38d1 2638{
577b785f 2639 static struct trace_option_dentry *topts;
bc0c38d1
SR
2640 struct trace_array *tr = &global_trace;
2641 struct tracer *t;
d9e54076 2642 int ret = 0;
bc0c38d1 2643
1027fcb2
SR
2644 mutex_lock(&trace_types_lock);
2645
73c5162a
SR
2646 if (!ring_buffer_expanded) {
2647 ret = tracing_resize_ring_buffer(trace_buf_size);
2648 if (ret < 0)
59f586db 2649 goto out;
73c5162a
SR
2650 ret = 0;
2651 }
2652
bc0c38d1
SR
2653 for (t = trace_types; t; t = t->next) {
2654 if (strcmp(t->name, buf) == 0)
2655 break;
2656 }
c2931e05
FW
2657 if (!t) {
2658 ret = -EINVAL;
2659 goto out;
2660 }
2661 if (t == current_trace)
bc0c38d1
SR
2662 goto out;
2663
9f029e83 2664 trace_branch_disable();
bc0c38d1
SR
2665 if (current_trace && current_trace->reset)
2666 current_trace->reset(tr);
2667
577b785f
SR
2668 destroy_trace_option_files(topts);
2669
bc0c38d1 2670 current_trace = t;
577b785f
SR
2671
2672 topts = create_trace_option_files(current_trace);
2673
1c80025a 2674 if (t->init) {
b6f11df2 2675 ret = tracer_init(t, tr);
1c80025a
FW
2676 if (ret)
2677 goto out;
2678 }
bc0c38d1 2679
9f029e83 2680 trace_branch_enable(tr);
bc0c38d1
SR
2681 out:
2682 mutex_unlock(&trace_types_lock);
2683
d9e54076
PZ
2684 return ret;
2685}
2686
2687static ssize_t
2688tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2689 size_t cnt, loff_t *ppos)
2690{
2691 char buf[max_tracer_type_len+1];
2692 int i;
2693 size_t ret;
e6e7a65a
FW
2694 int err;
2695
2696 ret = cnt;
d9e54076
PZ
2697
2698 if (cnt > max_tracer_type_len)
2699 cnt = max_tracer_type_len;
2700
2701 if (copy_from_user(&buf, ubuf, cnt))
2702 return -EFAULT;
2703
2704 buf[cnt] = 0;
2705
2706 /* strip ending whitespace. */
2707 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2708 buf[i] = 0;
2709
e6e7a65a
FW
2710 err = tracing_set_tracer(buf);
2711 if (err)
2712 return err;
d9e54076 2713
e6e7a65a 2714 filp->f_pos += ret;
bc0c38d1 2715
c2931e05 2716 return ret;
bc0c38d1
SR
2717}
2718
2719static ssize_t
2720tracing_max_lat_read(struct file *filp, char __user *ubuf,
2721 size_t cnt, loff_t *ppos)
2722{
2723 unsigned long *ptr = filp->private_data;
2724 char buf[64];
2725 int r;
2726
cffae437 2727 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 2728 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
2729 if (r > sizeof(buf))
2730 r = sizeof(buf);
4bf39a94 2731 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2732}
2733
2734static ssize_t
2735tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2736 size_t cnt, loff_t *ppos)
2737{
5e39841c 2738 unsigned long *ptr = filp->private_data;
bc0c38d1 2739 char buf[64];
5e39841c 2740 unsigned long val;
c6caeeb1 2741 int ret;
bc0c38d1 2742
cffae437
SR
2743 if (cnt >= sizeof(buf))
2744 return -EINVAL;
bc0c38d1
SR
2745
2746 if (copy_from_user(&buf, ubuf, cnt))
2747 return -EFAULT;
2748
2749 buf[cnt] = 0;
2750
c6caeeb1
SR
2751 ret = strict_strtoul(buf, 10, &val);
2752 if (ret < 0)
2753 return ret;
bc0c38d1
SR
2754
2755 *ptr = val * 1000;
2756
2757 return cnt;
2758}
2759
b3806b43
SR
2760static int tracing_open_pipe(struct inode *inode, struct file *filp)
2761{
b04cc6b1 2762 long cpu_file = (long) inode->i_private;
b3806b43 2763 struct trace_iterator *iter;
b04cc6b1 2764 int ret = 0;
b3806b43
SR
2765
2766 if (tracing_disabled)
2767 return -ENODEV;
2768
b04cc6b1
FW
2769 mutex_lock(&trace_types_lock);
2770
2771 /* We only allow one reader per cpu */
2772 if (cpu_file == TRACE_PIPE_ALL_CPU) {
2773 if (!cpumask_empty(tracing_reader_cpumask)) {
2774 ret = -EBUSY;
2775 goto out;
2776 }
2777 cpumask_setall(tracing_reader_cpumask);
2778 } else {
2779 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2780 cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2781 else {
2782 ret = -EBUSY;
2783 goto out;
2784 }
b3806b43
SR
2785 }
2786
2787 /* create a buffer to store the information to pass to userspace */
2788 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
2789 if (!iter) {
2790 ret = -ENOMEM;
2791 goto out;
2792 }
b3806b43 2793
d7350c3f
FW
2794 /*
2795 * We make a copy of the current tracer to avoid concurrent
2796 * changes on it while we are reading.
2797 */
2798 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2799 if (!iter->trace) {
2800 ret = -ENOMEM;
2801 goto fail;
2802 }
2803 if (current_trace)
2804 *iter->trace = *current_trace;
2805
4462344e 2806 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 2807 ret = -ENOMEM;
d7350c3f 2808 goto fail;
4462344e
RR
2809 }
2810
a309720c 2811 /* trace pipe does not show start of buffer */
4462344e 2812 cpumask_setall(iter->started);
a309720c 2813
b04cc6b1 2814 iter->cpu_file = cpu_file;
b3806b43 2815 iter->tr = &global_trace;
d7350c3f 2816 mutex_init(&iter->mutex);
b3806b43
SR
2817 filp->private_data = iter;
2818
107bad8b
SR
2819 if (iter->trace->pipe_open)
2820 iter->trace->pipe_open(iter);
107bad8b 2821
b04cc6b1
FW
2822out:
2823 mutex_unlock(&trace_types_lock);
2824 return ret;
d7350c3f
FW
2825
2826fail:
2827 kfree(iter->trace);
2828 kfree(iter);
2829 mutex_unlock(&trace_types_lock);
2830 return ret;
b3806b43
SR
2831}
2832
2833static int tracing_release_pipe(struct inode *inode, struct file *file)
2834{
2835 struct trace_iterator *iter = file->private_data;
2836
b04cc6b1
FW
2837 mutex_lock(&trace_types_lock);
2838
2839 if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2840 cpumask_clear(tracing_reader_cpumask);
2841 else
2842 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2843
2844 mutex_unlock(&trace_types_lock);
2845
4462344e 2846 free_cpumask_var(iter->started);
d7350c3f
FW
2847 mutex_destroy(&iter->mutex);
2848 kfree(iter->trace);
b3806b43 2849 kfree(iter);
b3806b43
SR
2850
2851 return 0;
2852}
2853
2a2cc8f7
SSP
2854static unsigned int
2855tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2856{
2857 struct trace_iterator *iter = filp->private_data;
2858
2859 if (trace_flags & TRACE_ITER_BLOCK) {
2860 /*
2861 * Always select as readable when in blocking mode
2862 */
2863 return POLLIN | POLLRDNORM;
afc2abc0 2864 } else {
2a2cc8f7
SSP
2865 if (!trace_empty(iter))
2866 return POLLIN | POLLRDNORM;
2867 poll_wait(filp, &trace_wait, poll_table);
2868 if (!trace_empty(iter))
2869 return POLLIN | POLLRDNORM;
2870
2871 return 0;
2872 }
2873}
2874
6eaaa5d5
FW
2875
2876void default_wait_pipe(struct trace_iterator *iter)
2877{
2878 DEFINE_WAIT(wait);
2879
2880 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
2881
2882 if (trace_empty(iter))
2883 schedule();
2884
2885 finish_wait(&trace_wait, &wait);
2886}
2887
2888/*
2889 * This is a make-shift waitqueue.
2890 * A tracer might use this callback on some rare cases:
2891 *
2892 * 1) the current tracer might hold the runqueue lock when it wakes up
2893 * a reader, hence a deadlock (sched, function, and function graph tracers)
2894 * 2) the function tracers, trace all functions, we don't want
2895 * the overhead of calling wake_up and friends
2896 * (and tracing them too)
2897 *
2898 * Anyway, this is really very primitive wakeup.
2899 */
2900void poll_wait_pipe(struct trace_iterator *iter)
2901{
2902 set_current_state(TASK_INTERRUPTIBLE);
2903 /* sleep for 100 msecs, and try again. */
2904 schedule_timeout(HZ / 10);
2905}
2906
ff98781b
EGM
2907/* Must be called with trace_types_lock mutex held. */
2908static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
2909{
2910 struct trace_iterator *iter = filp->private_data;
b3806b43 2911
b3806b43 2912 while (trace_empty(iter)) {
2dc8f095 2913
107bad8b 2914 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 2915 return -EAGAIN;
107bad8b 2916 }
2dc8f095 2917
d7350c3f 2918 mutex_unlock(&iter->mutex);
107bad8b 2919
6eaaa5d5 2920 iter->trace->wait_pipe(iter);
b3806b43 2921
d7350c3f 2922 mutex_lock(&iter->mutex);
107bad8b 2923
6eaaa5d5 2924 if (signal_pending(current))
ff98781b 2925 return -EINTR;
b3806b43
SR
2926
2927 /*
2928 * We block until we read something and tracing is disabled.
2929 * We still block if tracing is disabled, but we have never
2930 * read anything. This allows a user to cat this file, and
2931 * then enable tracing. But after we have read something,
2932 * we give an EOF when tracing is again disabled.
2933 *
2934 * iter->pos will be 0 if we haven't read anything.
2935 */
2936 if (!tracer_enabled && iter->pos)
2937 break;
b3806b43
SR
2938 }
2939
ff98781b
EGM
2940 return 1;
2941}
2942
2943/*
2944 * Consumer reader.
2945 */
2946static ssize_t
2947tracing_read_pipe(struct file *filp, char __user *ubuf,
2948 size_t cnt, loff_t *ppos)
2949{
2950 struct trace_iterator *iter = filp->private_data;
d7350c3f 2951 static struct tracer *old_tracer;
ff98781b
EGM
2952 ssize_t sret;
2953
2954 /* return any leftover data */
2955 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2956 if (sret != -EBUSY)
2957 return sret;
2958
f9520750 2959 trace_seq_init(&iter->seq);
ff98781b 2960
d7350c3f 2961 /* copy the tracer to avoid using a global lock all around */
ff98781b 2962 mutex_lock(&trace_types_lock);
d7350c3f
FW
2963 if (unlikely(old_tracer != current_trace && current_trace)) {
2964 old_tracer = current_trace;
2965 *iter->trace = *current_trace;
2966 }
2967 mutex_unlock(&trace_types_lock);
2968
2969 /*
2970 * Avoid more than one consumer on a single file descriptor
2971 * This is just a matter of traces coherency, the ring buffer itself
2972 * is protected.
2973 */
2974 mutex_lock(&iter->mutex);
ff98781b
EGM
2975 if (iter->trace->read) {
2976 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2977 if (sret)
2978 goto out;
2979 }
2980
2981waitagain:
2982 sret = tracing_wait_pipe(filp);
2983 if (sret <= 0)
2984 goto out;
2985
b3806b43 2986 /* stop when tracing is finished */
ff98781b
EGM
2987 if (trace_empty(iter)) {
2988 sret = 0;
107bad8b 2989 goto out;
ff98781b 2990 }
b3806b43
SR
2991
2992 if (cnt >= PAGE_SIZE)
2993 cnt = PAGE_SIZE - 1;
2994
53d0aa77 2995 /* reset all but tr, trace, and overruns */
53d0aa77
SR
2996 memset(&iter->seq, 0,
2997 sizeof(struct trace_iterator) -
2998 offsetof(struct trace_iterator, seq));
4823ed7e 2999 iter->pos = -1;
b3806b43 3000
088b1e42 3001 while (find_next_entry_inc(iter) != NULL) {
2c4f035f 3002 enum print_line_t ret;
088b1e42
SR
3003 int len = iter->seq.len;
3004
f9896bf3 3005 ret = print_trace_line(iter);
2c4f035f 3006 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
3007 /* don't print partial lines */
3008 iter->seq.len = len;
b3806b43 3009 break;
088b1e42 3010 }
b91facc3
FW
3011 if (ret != TRACE_TYPE_NO_CONSUME)
3012 trace_consume(iter);
b3806b43
SR
3013
3014 if (iter->seq.len >= cnt)
3015 break;
b3806b43
SR
3016 }
3017
b3806b43 3018 /* Now copy what we have to the user */
6c6c2796
PP
3019 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3020 if (iter->seq.readpos >= iter->seq.len)
f9520750 3021 trace_seq_init(&iter->seq);
9ff4b974
PP
3022
3023 /*
3024 * If there was nothing to send to user, inspite of consuming trace
3025 * entries, go back to wait for more entries.
3026 */
6c6c2796 3027 if (sret == -EBUSY)
9ff4b974 3028 goto waitagain;
b3806b43 3029
107bad8b 3030out:
d7350c3f 3031 mutex_unlock(&iter->mutex);
107bad8b 3032
6c6c2796 3033 return sret;
b3806b43
SR
3034}
3035
3c56819b
EGM
3036static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3037 struct pipe_buffer *buf)
3038{
3039 __free_page(buf->page);
3040}
3041
3042static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3043 unsigned int idx)
3044{
3045 __free_page(spd->pages[idx]);
3046}
3047
3048static struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998
SR
3049 .can_merge = 0,
3050 .map = generic_pipe_buf_map,
3051 .unmap = generic_pipe_buf_unmap,
3052 .confirm = generic_pipe_buf_confirm,
3053 .release = tracing_pipe_buf_release,
3054 .steal = generic_pipe_buf_steal,
3055 .get = generic_pipe_buf_get,
3c56819b
EGM
3056};
3057
34cd4998 3058static size_t
fa7c7f6e 3059tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
3060{
3061 size_t count;
3062 int ret;
3063
3064 /* Seq buffer is page-sized, exactly what we need. */
3065 for (;;) {
3066 count = iter->seq.len;
3067 ret = print_trace_line(iter);
3068 count = iter->seq.len - count;
3069 if (rem < count) {
3070 rem = 0;
3071 iter->seq.len -= count;
3072 break;
3073 }
3074 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3075 iter->seq.len -= count;
3076 break;
3077 }
3078
3079 trace_consume(iter);
3080 rem -= count;
3081 if (!find_next_entry_inc(iter)) {
3082 rem = 0;
3083 iter->ent = NULL;
3084 break;
3085 }
3086 }
3087
3088 return rem;
3089}
3090
3c56819b
EGM
3091static ssize_t tracing_splice_read_pipe(struct file *filp,
3092 loff_t *ppos,
3093 struct pipe_inode_info *pipe,
3094 size_t len,
3095 unsigned int flags)
3096{
3097 struct page *pages[PIPE_BUFFERS];
3098 struct partial_page partial[PIPE_BUFFERS];
3099 struct trace_iterator *iter = filp->private_data;
3100 struct splice_pipe_desc spd = {
34cd4998
SR
3101 .pages = pages,
3102 .partial = partial,
3103 .nr_pages = 0, /* This gets updated below. */
3104 .flags = flags,
3105 .ops = &tracing_pipe_buf_ops,
3106 .spd_release = tracing_spd_release_pipe,
3c56819b 3107 };
d7350c3f 3108 static struct tracer *old_tracer;
3c56819b 3109 ssize_t ret;
34cd4998 3110 size_t rem;
3c56819b
EGM
3111 unsigned int i;
3112
d7350c3f 3113 /* copy the tracer to avoid using a global lock all around */
3c56819b 3114 mutex_lock(&trace_types_lock);
d7350c3f
FW
3115 if (unlikely(old_tracer != current_trace && current_trace)) {
3116 old_tracer = current_trace;
3117 *iter->trace = *current_trace;
3118 }
3119 mutex_unlock(&trace_types_lock);
3120
3121 mutex_lock(&iter->mutex);
3c56819b
EGM
3122
3123 if (iter->trace->splice_read) {
3124 ret = iter->trace->splice_read(iter, filp,
3125 ppos, pipe, len, flags);
3126 if (ret)
34cd4998 3127 goto out_err;
3c56819b
EGM
3128 }
3129
3130 ret = tracing_wait_pipe(filp);
3131 if (ret <= 0)
34cd4998 3132 goto out_err;
3c56819b
EGM
3133
3134 if (!iter->ent && !find_next_entry_inc(iter)) {
3135 ret = -EFAULT;
34cd4998 3136 goto out_err;
3c56819b
EGM
3137 }
3138
3139 /* Fill as many pages as possible. */
3140 for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3141 pages[i] = alloc_page(GFP_KERNEL);
34cd4998
SR
3142 if (!pages[i])
3143 break;
3c56819b 3144
fa7c7f6e 3145 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
3146
3147 /* Copy the data into the page, so we can start over. */
3148 ret = trace_seq_to_buffer(&iter->seq,
3149 page_address(pages[i]),
3150 iter->seq.len);
3151 if (ret < 0) {
3152 __free_page(pages[i]);
3153 break;
3154 }
3155 partial[i].offset = 0;
3156 partial[i].len = iter->seq.len;
3157
f9520750 3158 trace_seq_init(&iter->seq);
3c56819b
EGM
3159 }
3160
d7350c3f 3161 mutex_unlock(&iter->mutex);
3c56819b
EGM
3162
3163 spd.nr_pages = i;
3164
3165 return splice_to_pipe(pipe, &spd);
3166
34cd4998 3167out_err:
d7350c3f 3168 mutex_unlock(&iter->mutex);
3c56819b
EGM
3169
3170 return ret;
3171}
3172
a98a3c3f
SR
3173static ssize_t
3174tracing_entries_read(struct file *filp, char __user *ubuf,
3175 size_t cnt, loff_t *ppos)
3176{
3177 struct trace_array *tr = filp->private_data;
db526ca3 3178 char buf[96];
a98a3c3f
SR
3179 int r;
3180
db526ca3
SR
3181 mutex_lock(&trace_types_lock);
3182 if (!ring_buffer_expanded)
3183 r = sprintf(buf, "%lu (expanded: %lu)\n",
3184 tr->entries >> 10,
3185 trace_buf_size >> 10);
3186 else
3187 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3188 mutex_unlock(&trace_types_lock);
3189
a98a3c3f
SR
3190 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3191}
3192
3193static ssize_t
3194tracing_entries_write(struct file *filp, const char __user *ubuf,
3195 size_t cnt, loff_t *ppos)
3196{
3197 unsigned long val;
3198 char buf[64];
bf5e6519 3199 int ret, cpu;
a98a3c3f 3200
cffae437
SR
3201 if (cnt >= sizeof(buf))
3202 return -EINVAL;
a98a3c3f
SR
3203
3204 if (copy_from_user(&buf, ubuf, cnt))
3205 return -EFAULT;
3206
3207 buf[cnt] = 0;
3208
c6caeeb1
SR
3209 ret = strict_strtoul(buf, 10, &val);
3210 if (ret < 0)
3211 return ret;
a98a3c3f
SR
3212
3213 /* must have at least 1 entry */
3214 if (!val)
3215 return -EINVAL;
3216
3217 mutex_lock(&trace_types_lock);
3218
c76f0694 3219 tracing_stop();
a98a3c3f 3220
bf5e6519
SR
3221 /* disable all cpu buffers */
3222 for_each_tracing_cpu(cpu) {
3223 if (global_trace.data[cpu])
3224 atomic_inc(&global_trace.data[cpu]->disabled);
3225 if (max_tr.data[cpu])
3226 atomic_inc(&max_tr.data[cpu]->disabled);
3227 }
3228
1696b2b0
SR
3229 /* value is in KB */
3230 val <<= 10;
3231
3928a8a2 3232 if (val != global_trace.entries) {
73c5162a 3233 ret = tracing_resize_ring_buffer(val);
3928a8a2
SR
3234 if (ret < 0) {
3235 cnt = ret;
3eefae99
SR
3236 goto out;
3237 }
a98a3c3f
SR
3238 }
3239
3240 filp->f_pos += cnt;
3241
19384c03
SR
3242 /* If check pages failed, return ENOMEM */
3243 if (tracing_disabled)
3244 cnt = -ENOMEM;
a98a3c3f 3245 out:
bf5e6519
SR
3246 for_each_tracing_cpu(cpu) {
3247 if (global_trace.data[cpu])
3248 atomic_dec(&global_trace.data[cpu]->disabled);
3249 if (max_tr.data[cpu])
3250 atomic_dec(&max_tr.data[cpu]->disabled);
3251 }
3252
c76f0694 3253 tracing_start();
a98a3c3f
SR
3254 max_tr.entries = global_trace.entries;
3255 mutex_unlock(&trace_types_lock);
3256
3257 return cnt;
3258}
3259
5bf9a1ee
PP
3260static int mark_printk(const char *fmt, ...)
3261{
3262 int ret;
3263 va_list args;
3264 va_start(args, fmt);
40ce74f1 3265 ret = trace_vprintk(0, fmt, args);
5bf9a1ee
PP
3266 va_end(args);
3267 return ret;
3268}
3269
3270static ssize_t
3271tracing_mark_write(struct file *filp, const char __user *ubuf,
3272 size_t cnt, loff_t *fpos)
3273{
3274 char *buf;
3275 char *end;
5bf9a1ee 3276
c76f0694 3277 if (tracing_disabled)
5bf9a1ee
PP
3278 return -EINVAL;
3279
3280 if (cnt > TRACE_BUF_SIZE)
3281 cnt = TRACE_BUF_SIZE;
3282
3283 buf = kmalloc(cnt + 1, GFP_KERNEL);
3284 if (buf == NULL)
3285 return -ENOMEM;
3286
3287 if (copy_from_user(buf, ubuf, cnt)) {
3288 kfree(buf);
3289 return -EFAULT;
3290 }
3291
3292 /* Cut from the first nil or newline. */
3293 buf[cnt] = '\0';
3294 end = strchr(buf, '\n');
3295 if (end)
3296 *end = '\0';
3297
3298 cnt = mark_printk("%s\n", buf);
3299 kfree(buf);
3300 *fpos += cnt;
3301
3302 return cnt;
3303}
3304
5e2336a0 3305static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
3306 .open = tracing_open_generic,
3307 .read = tracing_max_lat_read,
3308 .write = tracing_max_lat_write,
bc0c38d1
SR
3309};
3310
5e2336a0 3311static const struct file_operations tracing_ctrl_fops = {
4bf39a94
IM
3312 .open = tracing_open_generic,
3313 .read = tracing_ctrl_read,
3314 .write = tracing_ctrl_write,
bc0c38d1
SR
3315};
3316
5e2336a0 3317static const struct file_operations set_tracer_fops = {
4bf39a94
IM
3318 .open = tracing_open_generic,
3319 .read = tracing_set_trace_read,
3320 .write = tracing_set_trace_write,
bc0c38d1
SR
3321};
3322
5e2336a0 3323static const struct file_operations tracing_pipe_fops = {
4bf39a94 3324 .open = tracing_open_pipe,
2a2cc8f7 3325 .poll = tracing_poll_pipe,
4bf39a94 3326 .read = tracing_read_pipe,
3c56819b 3327 .splice_read = tracing_splice_read_pipe,
4bf39a94 3328 .release = tracing_release_pipe,
b3806b43
SR
3329};
3330
5e2336a0 3331static const struct file_operations tracing_entries_fops = {
a98a3c3f
SR
3332 .open = tracing_open_generic,
3333 .read = tracing_entries_read,
3334 .write = tracing_entries_write,
3335};
3336
5e2336a0 3337static const struct file_operations tracing_mark_fops = {
43a15386 3338 .open = tracing_open_generic,
5bf9a1ee
PP
3339 .write = tracing_mark_write,
3340};
3341
2cadf913
SR
3342struct ftrace_buffer_info {
3343 struct trace_array *tr;
3344 void *spare;
3345 int cpu;
3346 unsigned int read;
3347};
3348
3349static int tracing_buffers_open(struct inode *inode, struct file *filp)
3350{
3351 int cpu = (int)(long)inode->i_private;
3352 struct ftrace_buffer_info *info;
3353
3354 if (tracing_disabled)
3355 return -ENODEV;
3356
3357 info = kzalloc(sizeof(*info), GFP_KERNEL);
3358 if (!info)
3359 return -ENOMEM;
3360
3361 info->tr = &global_trace;
3362 info->cpu = cpu;
ddd538f3 3363 info->spare = NULL;
2cadf913
SR
3364 /* Force reading ring buffer for first read */
3365 info->read = (unsigned int)-1;
2cadf913
SR
3366
3367 filp->private_data = info;
3368
d1e7e02f 3369 return nonseekable_open(inode, filp);
2cadf913
SR
3370}
3371
3372static ssize_t
3373tracing_buffers_read(struct file *filp, char __user *ubuf,
3374 size_t count, loff_t *ppos)
3375{
3376 struct ftrace_buffer_info *info = filp->private_data;
3377 unsigned int pos;
3378 ssize_t ret;
3379 size_t size;
3380
2dc5d12b
SR
3381 if (!count)
3382 return 0;
3383
ddd538f3
LJ
3384 if (!info->spare)
3385 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3386 if (!info->spare)
3387 return -ENOMEM;
3388
2cadf913
SR
3389 /* Do we have previous read data to read? */
3390 if (info->read < PAGE_SIZE)
3391 goto read;
3392
3393 info->read = 0;
3394
3395 ret = ring_buffer_read_page(info->tr->buffer,
3396 &info->spare,
3397 count,
3398 info->cpu, 0);
3399 if (ret < 0)
3400 return 0;
3401
3402 pos = ring_buffer_page_len(info->spare);
3403
3404 if (pos < PAGE_SIZE)
3405 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3406
3407read:
3408 size = PAGE_SIZE - info->read;
3409 if (size > count)
3410 size = count;
3411
3412 ret = copy_to_user(ubuf, info->spare + info->read, size);
2dc5d12b 3413 if (ret == size)
2cadf913 3414 return -EFAULT;
2dc5d12b
SR
3415 size -= ret;
3416
2cadf913
SR
3417 *ppos += size;
3418 info->read += size;
3419
3420 return size;
3421}
3422
3423static int tracing_buffers_release(struct inode *inode, struct file *file)
3424{
3425 struct ftrace_buffer_info *info = file->private_data;
3426
ddd538f3
LJ
3427 if (info->spare)
3428 ring_buffer_free_read_page(info->tr->buffer, info->spare);
2cadf913
SR
3429 kfree(info);
3430
3431 return 0;
3432}
3433
3434struct buffer_ref {
3435 struct ring_buffer *buffer;
3436 void *page;
3437 int ref;
3438};
3439
3440static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3441 struct pipe_buffer *buf)
3442{
3443 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3444
3445 if (--ref->ref)
3446 return;
3447
3448 ring_buffer_free_read_page(ref->buffer, ref->page);
3449 kfree(ref);
3450 buf->private = 0;
3451}
3452
3453static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3454 struct pipe_buffer *buf)
3455{
3456 return 1;
3457}
3458
3459static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3460 struct pipe_buffer *buf)
3461{
3462 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3463
3464 ref->ref++;
3465}
3466
3467/* Pipe buffer operations for a buffer. */
3468static struct pipe_buf_operations buffer_pipe_buf_ops = {
3469 .can_merge = 0,
3470 .map = generic_pipe_buf_map,
3471 .unmap = generic_pipe_buf_unmap,
3472 .confirm = generic_pipe_buf_confirm,
3473 .release = buffer_pipe_buf_release,
3474 .steal = buffer_pipe_buf_steal,
3475 .get = buffer_pipe_buf_get,
3476};
3477
3478/*
3479 * Callback from splice_to_pipe(), if we need to release some pages
3480 * at the end of the spd in case we error'ed out in filling the pipe.
3481 */
3482static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3483{
3484 struct buffer_ref *ref =
3485 (struct buffer_ref *)spd->partial[i].private;
3486
3487 if (--ref->ref)
3488 return;
3489
3490 ring_buffer_free_read_page(ref->buffer, ref->page);
3491 kfree(ref);
3492 spd->partial[i].private = 0;
3493}
3494
3495static ssize_t
3496tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3497 struct pipe_inode_info *pipe, size_t len,
3498 unsigned int flags)
3499{
3500 struct ftrace_buffer_info *info = file->private_data;
3501 struct partial_page partial[PIPE_BUFFERS];
3502 struct page *pages[PIPE_BUFFERS];
3503 struct splice_pipe_desc spd = {
3504 .pages = pages,
3505 .partial = partial,
3506 .flags = flags,
3507 .ops = &buffer_pipe_buf_ops,
3508 .spd_release = buffer_spd_release,
3509 };
3510 struct buffer_ref *ref;
93459c6c 3511 int entries, size, i;
2cadf913
SR
3512 size_t ret;
3513
93cfb3c9
LJ
3514 if (*ppos & (PAGE_SIZE - 1)) {
3515 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3516 return -EINVAL;
3517 }
2cadf913 3518
93cfb3c9
LJ
3519 if (len & (PAGE_SIZE - 1)) {
3520 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3521 if (len < PAGE_SIZE)
3522 return -EINVAL;
3523 len &= PAGE_MASK;
3524 }
2cadf913 3525
93459c6c
SR
3526 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3527
3528 for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
3529 struct page *page;
3530 int r;
3531
3532 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3533 if (!ref)
3534 break;
3535
5beae6ef 3536 ref->ref = 1;
2cadf913
SR
3537 ref->buffer = info->tr->buffer;
3538 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3539 if (!ref->page) {
3540 kfree(ref);
3541 break;
3542 }
3543
3544 r = ring_buffer_read_page(ref->buffer, &ref->page,
f2957f1f 3545 len, info->cpu, 1);
2cadf913
SR
3546 if (r < 0) {
3547 ring_buffer_free_read_page(ref->buffer,
3548 ref->page);
3549 kfree(ref);
3550 break;
3551 }
3552
3553 /*
3554 * zero out any left over data, this is going to
3555 * user land.
3556 */
3557 size = ring_buffer_page_len(ref->page);
3558 if (size < PAGE_SIZE)
3559 memset(ref->page + size, 0, PAGE_SIZE - size);
3560
3561 page = virt_to_page(ref->page);
3562
3563 spd.pages[i] = page;
3564 spd.partial[i].len = PAGE_SIZE;
3565 spd.partial[i].offset = 0;
3566 spd.partial[i].private = (unsigned long)ref;
3567 spd.nr_pages++;
93cfb3c9 3568 *ppos += PAGE_SIZE;
93459c6c
SR
3569
3570 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
2cadf913
SR
3571 }
3572
3573 spd.nr_pages = i;
3574
3575 /* did we read anything? */
3576 if (!spd.nr_pages) {
3577 if (flags & SPLICE_F_NONBLOCK)
3578 ret = -EAGAIN;
3579 else
3580 ret = 0;
3581 /* TODO: block */
3582 return ret;
3583 }
3584
3585 ret = splice_to_pipe(pipe, &spd);
3586
3587 return ret;
3588}
3589
3590static const struct file_operations tracing_buffers_fops = {
3591 .open = tracing_buffers_open,
3592 .read = tracing_buffers_read,
3593 .release = tracing_buffers_release,
3594 .splice_read = tracing_buffers_splice_read,
3595 .llseek = no_llseek,
3596};
3597
c8d77183
SR
3598static ssize_t
3599tracing_stats_read(struct file *filp, char __user *ubuf,
3600 size_t count, loff_t *ppos)
3601{
3602 unsigned long cpu = (unsigned long)filp->private_data;
3603 struct trace_array *tr = &global_trace;
3604 struct trace_seq *s;
3605 unsigned long cnt;
3606
3607 s = kmalloc(sizeof(*s), GFP_ATOMIC);
3608 if (!s)
3609 return ENOMEM;
3610
3611 trace_seq_init(s);
3612
3613 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3614 trace_seq_printf(s, "entries: %ld\n", cnt);
3615
3616 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3617 trace_seq_printf(s, "overrun: %ld\n", cnt);
3618
3619 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3620 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3621
3622 cnt = ring_buffer_nmi_dropped_cpu(tr->buffer, cpu);
3623 trace_seq_printf(s, "nmi dropped: %ld\n", cnt);
3624
3625 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3626
3627 kfree(s);
3628
3629 return count;
3630}
3631
3632static const struct file_operations tracing_stats_fops = {
3633 .open = tracing_open_generic,
3634 .read = tracing_stats_read,
3635};
3636
bc0c38d1
SR
3637#ifdef CONFIG_DYNAMIC_FTRACE
3638
b807c3d0
SR
3639int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3640{
3641 return 0;
3642}
3643
bc0c38d1 3644static ssize_t
b807c3d0 3645tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
3646 size_t cnt, loff_t *ppos)
3647{
a26a2a27
SR
3648 static char ftrace_dyn_info_buffer[1024];
3649 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 3650 unsigned long *p = filp->private_data;
b807c3d0 3651 char *buf = ftrace_dyn_info_buffer;
a26a2a27 3652 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
3653 int r;
3654
b807c3d0
SR
3655 mutex_lock(&dyn_info_mutex);
3656 r = sprintf(buf, "%ld ", *p);
4bf39a94 3657
a26a2a27 3658 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
3659 buf[r++] = '\n';
3660
3661 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3662
3663 mutex_unlock(&dyn_info_mutex);
3664
3665 return r;
bc0c38d1
SR
3666}
3667
5e2336a0 3668static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 3669 .open = tracing_open_generic,
b807c3d0 3670 .read = tracing_read_dyn_info,
bc0c38d1
SR
3671};
3672#endif
3673
3674static struct dentry *d_tracer;
3675
3676struct dentry *tracing_init_dentry(void)
3677{
3678 static int once;
3679
3680 if (d_tracer)
3681 return d_tracer;
3682
3e1f60b8
FW
3683 if (!debugfs_initialized())
3684 return NULL;
3685
bc0c38d1
SR
3686 d_tracer = debugfs_create_dir("tracing", NULL);
3687
3688 if (!d_tracer && !once) {
3689 once = 1;
3690 pr_warning("Could not create debugfs directory 'tracing'\n");
3691 return NULL;
3692 }
3693
3694 return d_tracer;
3695}
3696
b04cc6b1
FW
3697static struct dentry *d_percpu;
3698
3699struct dentry *tracing_dentry_percpu(void)
3700{
3701 static int once;
3702 struct dentry *d_tracer;
3703
3704 if (d_percpu)
3705 return d_percpu;
3706
3707 d_tracer = tracing_init_dentry();
3708
3709 if (!d_tracer)
3710 return NULL;
3711
3712 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3713
3714 if (!d_percpu && !once) {
3715 once = 1;
3716 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3717 return NULL;
3718 }
3719
3720 return d_percpu;
3721}
3722
3723static void tracing_init_debugfs_percpu(long cpu)
3724{
3725 struct dentry *d_percpu = tracing_dentry_percpu();
5452af66 3726 struct dentry *d_cpu;
8656e7a2
FW
3727 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3728 char cpu_dir[7];
b04cc6b1
FW
3729
3730 if (cpu > 999 || cpu < 0)
3731 return;
3732
8656e7a2
FW
3733 sprintf(cpu_dir, "cpu%ld", cpu);
3734 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3735 if (!d_cpu) {
3736 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3737 return;
3738 }
b04cc6b1 3739
8656e7a2 3740 /* per cpu trace_pipe */
5452af66
FW
3741 trace_create_file("trace_pipe", 0444, d_cpu,
3742 (void *) cpu, &tracing_pipe_fops);
b04cc6b1
FW
3743
3744 /* per cpu trace */
5452af66
FW
3745 trace_create_file("trace", 0644, d_cpu,
3746 (void *) cpu, &tracing_fops);
7f96f93f 3747
5452af66
FW
3748 trace_create_file("trace_pipe_raw", 0444, d_cpu,
3749 (void *) cpu, &tracing_buffers_fops);
c8d77183
SR
3750
3751 trace_create_file("stats", 0444, d_cpu,
3752 (void *) cpu, &tracing_stats_fops);
b04cc6b1
FW
3753}
3754
60a11774
SR
3755#ifdef CONFIG_FTRACE_SELFTEST
3756/* Let selftest have access to static functions in this file */
3757#include "trace_selftest.c"
3758#endif
3759
577b785f
SR
3760struct trace_option_dentry {
3761 struct tracer_opt *opt;
3762 struct tracer_flags *flags;
3763 struct dentry *entry;
3764};
3765
3766static ssize_t
3767trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3768 loff_t *ppos)
3769{
3770 struct trace_option_dentry *topt = filp->private_data;
3771 char *buf;
3772
3773 if (topt->flags->val & topt->opt->bit)
3774 buf = "1\n";
3775 else
3776 buf = "0\n";
3777
3778 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3779}
3780
3781static ssize_t
3782trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3783 loff_t *ppos)
3784{
3785 struct trace_option_dentry *topt = filp->private_data;
3786 unsigned long val;
3787 char buf[64];
3788 int ret;
3789
3790 if (cnt >= sizeof(buf))
3791 return -EINVAL;
3792
3793 if (copy_from_user(&buf, ubuf, cnt))
3794 return -EFAULT;
3795
3796 buf[cnt] = 0;
3797
3798 ret = strict_strtoul(buf, 10, &val);
3799 if (ret < 0)
3800 return ret;
3801
3802 ret = 0;
3803 switch (val) {
3804 case 0:
3805 /* do nothing if already cleared */
3806 if (!(topt->flags->val & topt->opt->bit))
3807 break;
3808
3809 mutex_lock(&trace_types_lock);
3810 if (current_trace->set_flag)
3811 ret = current_trace->set_flag(topt->flags->val,
3812 topt->opt->bit, 0);
3813 mutex_unlock(&trace_types_lock);
3814 if (ret)
3815 return ret;
3816 topt->flags->val &= ~topt->opt->bit;
3817 break;
3818 case 1:
3819 /* do nothing if already set */
3820 if (topt->flags->val & topt->opt->bit)
3821 break;
3822
3823 mutex_lock(&trace_types_lock);
3824 if (current_trace->set_flag)
3825 ret = current_trace->set_flag(topt->flags->val,
3826 topt->opt->bit, 1);
3827 mutex_unlock(&trace_types_lock);
3828 if (ret)
3829 return ret;
3830 topt->flags->val |= topt->opt->bit;
3831 break;
3832
3833 default:
3834 return -EINVAL;
3835 }
3836
3837 *ppos += cnt;
3838
3839 return cnt;
3840}
3841
3842
3843static const struct file_operations trace_options_fops = {
3844 .open = tracing_open_generic,
3845 .read = trace_options_read,
3846 .write = trace_options_write,
3847};
3848
a8259075
SR
3849static ssize_t
3850trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
3851 loff_t *ppos)
3852{
3853 long index = (long)filp->private_data;
3854 char *buf;
3855
3856 if (trace_flags & (1 << index))
3857 buf = "1\n";
3858 else
3859 buf = "0\n";
3860
3861 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3862}
3863
3864static ssize_t
3865trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3866 loff_t *ppos)
3867{
3868 long index = (long)filp->private_data;
3869 char buf[64];
3870 unsigned long val;
3871 int ret;
3872
3873 if (cnt >= sizeof(buf))
3874 return -EINVAL;
3875
3876 if (copy_from_user(&buf, ubuf, cnt))
3877 return -EFAULT;
3878
3879 buf[cnt] = 0;
3880
3881 ret = strict_strtoul(buf, 10, &val);
3882 if (ret < 0)
3883 return ret;
3884
3885 switch (val) {
3886 case 0:
3887 trace_flags &= ~(1 << index);
3888 break;
3889 case 1:
3890 trace_flags |= 1 << index;
3891 break;
3892
3893 default:
3894 return -EINVAL;
3895 }
3896
3897 *ppos += cnt;
3898
3899 return cnt;
3900}
3901
a8259075
SR
3902static const struct file_operations trace_options_core_fops = {
3903 .open = tracing_open_generic,
3904 .read = trace_options_core_read,
3905 .write = trace_options_core_write,
3906};
3907
5452af66
FW
3908struct dentry *trace_create_file(const char *name,
3909 mode_t mode,
3910 struct dentry *parent,
3911 void *data,
3912 const struct file_operations *fops)
3913{
3914 struct dentry *ret;
3915
3916 ret = debugfs_create_file(name, mode, parent, data, fops);
3917 if (!ret)
3918 pr_warning("Could not create debugfs '%s' entry\n", name);
3919
3920 return ret;
3921}
3922
3923
a8259075
SR
3924static struct dentry *trace_options_init_dentry(void)
3925{
3926 struct dentry *d_tracer;
3927 static struct dentry *t_options;
3928
3929 if (t_options)
3930 return t_options;
3931
3932 d_tracer = tracing_init_dentry();
3933 if (!d_tracer)
3934 return NULL;
3935
3936 t_options = debugfs_create_dir("options", d_tracer);
3937 if (!t_options) {
3938 pr_warning("Could not create debugfs directory 'options'\n");
3939 return NULL;
3940 }
3941
3942 return t_options;
3943}
3944
577b785f
SR
3945static void
3946create_trace_option_file(struct trace_option_dentry *topt,
3947 struct tracer_flags *flags,
3948 struct tracer_opt *opt)
3949{
3950 struct dentry *t_options;
577b785f
SR
3951
3952 t_options = trace_options_init_dentry();
3953 if (!t_options)
3954 return;
3955
3956 topt->flags = flags;
3957 topt->opt = opt;
3958
5452af66 3959 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
3960 &trace_options_fops);
3961
577b785f
SR
3962}
3963
3964static struct trace_option_dentry *
3965create_trace_option_files(struct tracer *tracer)
3966{
3967 struct trace_option_dentry *topts;
3968 struct tracer_flags *flags;
3969 struct tracer_opt *opts;
3970 int cnt;
3971
3972 if (!tracer)
3973 return NULL;
3974
3975 flags = tracer->flags;
3976
3977 if (!flags || !flags->opts)
3978 return NULL;
3979
3980 opts = flags->opts;
3981
3982 for (cnt = 0; opts[cnt].name; cnt++)
3983 ;
3984
0cfe8245 3985 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
3986 if (!topts)
3987 return NULL;
3988
3989 for (cnt = 0; opts[cnt].name; cnt++)
3990 create_trace_option_file(&topts[cnt], flags,
3991 &opts[cnt]);
3992
3993 return topts;
3994}
3995
3996static void
3997destroy_trace_option_files(struct trace_option_dentry *topts)
3998{
3999 int cnt;
4000
4001 if (!topts)
4002 return;
4003
4004 for (cnt = 0; topts[cnt].opt; cnt++) {
4005 if (topts[cnt].entry)
4006 debugfs_remove(topts[cnt].entry);
4007 }
4008
4009 kfree(topts);
4010}
4011
a8259075
SR
4012static struct dentry *
4013create_trace_option_core_file(const char *option, long index)
4014{
4015 struct dentry *t_options;
a8259075
SR
4016
4017 t_options = trace_options_init_dentry();
4018 if (!t_options)
4019 return NULL;
4020
5452af66 4021 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 4022 &trace_options_core_fops);
a8259075
SR
4023}
4024
4025static __init void create_trace_options_dir(void)
4026{
4027 struct dentry *t_options;
a8259075
SR
4028 int i;
4029
4030 t_options = trace_options_init_dentry();
4031 if (!t_options)
4032 return;
4033
5452af66
FW
4034 for (i = 0; trace_options[i]; i++)
4035 create_trace_option_core_file(trace_options[i], i);
a8259075
SR
4036}
4037
b5ad384e 4038static __init int tracer_init_debugfs(void)
bc0c38d1
SR
4039{
4040 struct dentry *d_tracer;
b04cc6b1 4041 int cpu;
bc0c38d1
SR
4042
4043 d_tracer = tracing_init_dentry();
4044
5452af66
FW
4045 trace_create_file("tracing_enabled", 0644, d_tracer,
4046 &global_trace, &tracing_ctrl_fops);
bc0c38d1 4047
5452af66
FW
4048 trace_create_file("trace_options", 0644, d_tracer,
4049 NULL, &tracing_iter_fops);
bc0c38d1 4050
5452af66
FW
4051 trace_create_file("tracing_cpumask", 0644, d_tracer,
4052 NULL, &tracing_cpumask_fops);
4053
4054 trace_create_file("trace", 0644, d_tracer,
4055 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4056
4057 trace_create_file("available_tracers", 0444, d_tracer,
4058 &global_trace, &show_traces_fops);
4059
339ae5d3 4060 trace_create_file("current_tracer", 0644, d_tracer,
5452af66
FW
4061 &global_trace, &set_tracer_fops);
4062
4063 trace_create_file("tracing_max_latency", 0644, d_tracer,
4064 &tracing_max_latency, &tracing_max_lat_fops);
4065
4066 trace_create_file("tracing_thresh", 0644, d_tracer,
4067 &tracing_thresh, &tracing_max_lat_fops);
a8259075 4068
339ae5d3 4069 trace_create_file("README", 0444, d_tracer,
5452af66
FW
4070 NULL, &tracing_readme_fops);
4071
4072 trace_create_file("trace_pipe", 0444, d_tracer,
b04cc6b1 4073 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
5452af66
FW
4074
4075 trace_create_file("buffer_size_kb", 0644, d_tracer,
4076 &global_trace, &tracing_entries_fops);
4077
4078 trace_create_file("trace_marker", 0220, d_tracer,
4079 NULL, &tracing_mark_fops);
5bf9a1ee 4080
69abe6a5
AP
4081 trace_create_file("saved_cmdlines", 0444, d_tracer,
4082 NULL, &tracing_saved_cmdlines_fops);
4083
bc0c38d1 4084#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
4085 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4086 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 4087#endif
d618b3e6
IM
4088#ifdef CONFIG_SYSPROF_TRACER
4089 init_tracer_sysprof_debugfs(d_tracer);
4090#endif
b04cc6b1 4091
5452af66
FW
4092 create_trace_options_dir();
4093
b04cc6b1
FW
4094 for_each_tracing_cpu(cpu)
4095 tracing_init_debugfs_percpu(cpu);
4096
b5ad384e 4097 return 0;
bc0c38d1
SR
4098}
4099
3f5a54e3
SR
4100static int trace_panic_handler(struct notifier_block *this,
4101 unsigned long event, void *unused)
4102{
944ac425
SR
4103 if (ftrace_dump_on_oops)
4104 ftrace_dump();
3f5a54e3
SR
4105 return NOTIFY_OK;
4106}
4107
4108static struct notifier_block trace_panic_notifier = {
4109 .notifier_call = trace_panic_handler,
4110 .next = NULL,
4111 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4112};
4113
4114static int trace_die_handler(struct notifier_block *self,
4115 unsigned long val,
4116 void *data)
4117{
4118 switch (val) {
4119 case DIE_OOPS:
944ac425
SR
4120 if (ftrace_dump_on_oops)
4121 ftrace_dump();
3f5a54e3
SR
4122 break;
4123 default:
4124 break;
4125 }
4126 return NOTIFY_OK;
4127}
4128
4129static struct notifier_block trace_die_notifier = {
4130 .notifier_call = trace_die_handler,
4131 .priority = 200
4132};
4133
4134/*
4135 * printk is set to max of 1024, we really don't need it that big.
4136 * Nothing should be printing 1000 characters anyway.
4137 */
4138#define TRACE_MAX_PRINT 1000
4139
4140/*
4141 * Define here KERN_TRACE so that we have one place to modify
4142 * it if we decide to change what log level the ftrace dump
4143 * should be at.
4144 */
428aee14 4145#define KERN_TRACE KERN_EMERG
3f5a54e3
SR
4146
4147static void
4148trace_printk_seq(struct trace_seq *s)
4149{
4150 /* Probably should print a warning here. */
4151 if (s->len >= 1000)
4152 s->len = 1000;
4153
4154 /* should be zero ended, but we are paranoid. */
4155 s->buffer[s->len] = 0;
4156
4157 printk(KERN_TRACE "%s", s->buffer);
4158
f9520750 4159 trace_seq_init(s);
3f5a54e3
SR
4160}
4161
cf586b61 4162static void __ftrace_dump(bool disable_tracing)
3f5a54e3 4163{
cd891ae0
SR
4164 static raw_spinlock_t ftrace_dump_lock =
4165 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
3f5a54e3
SR
4166 /* use static because iter can be a bit big for the stack */
4167 static struct trace_iterator iter;
cf586b61 4168 unsigned int old_userobj;
3f5a54e3 4169 static int dump_ran;
d769041f
SR
4170 unsigned long flags;
4171 int cnt = 0, cpu;
3f5a54e3
SR
4172
4173 /* only one dump */
cd891ae0
SR
4174 local_irq_save(flags);
4175 __raw_spin_lock(&ftrace_dump_lock);
3f5a54e3
SR
4176 if (dump_ran)
4177 goto out;
4178
4179 dump_ran = 1;
4180
0ee6b6cf 4181 tracing_off();
cf586b61
FW
4182
4183 if (disable_tracing)
4184 ftrace_kill();
3f5a54e3 4185
d769041f
SR
4186 for_each_tracing_cpu(cpu) {
4187 atomic_inc(&global_trace.data[cpu]->disabled);
4188 }
4189
cf586b61
FW
4190 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4191
b54d3de9
TE
4192 /* don't look at user memory in panic mode */
4193 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4194
3f5a54e3
SR
4195 printk(KERN_TRACE "Dumping ftrace buffer:\n");
4196
e543ad76 4197 /* Simulate the iterator */
3f5a54e3
SR
4198 iter.tr = &global_trace;
4199 iter.trace = current_trace;
e543ad76 4200 iter.cpu_file = TRACE_PIPE_ALL_CPU;
3f5a54e3
SR
4201
4202 /*
4203 * We need to stop all tracing on all CPUS to read the
4204 * the next buffer. This is a bit expensive, but is
4205 * not done often. We fill all what we can read,
4206 * and then release the locks again.
4207 */
4208
3f5a54e3
SR
4209 while (!trace_empty(&iter)) {
4210
4211 if (!cnt)
4212 printk(KERN_TRACE "---------------------------------\n");
4213
4214 cnt++;
4215
4216 /* reset all but tr, trace, and overruns */
4217 memset(&iter.seq, 0,
4218 sizeof(struct trace_iterator) -
4219 offsetof(struct trace_iterator, seq));
4220 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4221 iter.pos = -1;
4222
4223 if (find_next_entry_inc(&iter) != NULL) {
4224 print_trace_line(&iter);
4225 trace_consume(&iter);
4226 }
4227
4228 trace_printk_seq(&iter.seq);
4229 }
4230
4231 if (!cnt)
4232 printk(KERN_TRACE " (ftrace buffer empty)\n");
4233 else
4234 printk(KERN_TRACE "---------------------------------\n");
4235
cf586b61
FW
4236 /* Re-enable tracing if requested */
4237 if (!disable_tracing) {
4238 trace_flags |= old_userobj;
4239
4240 for_each_tracing_cpu(cpu) {
4241 atomic_dec(&global_trace.data[cpu]->disabled);
4242 }
4243 tracing_on();
4244 }
4245
3f5a54e3 4246 out:
cd891ae0
SR
4247 __raw_spin_unlock(&ftrace_dump_lock);
4248 local_irq_restore(flags);
3f5a54e3
SR
4249}
4250
cf586b61
FW
4251/* By default: disable tracing after the dump */
4252void ftrace_dump(void)
4253{
4254 __ftrace_dump(true);
4255}
4256
3928a8a2 4257__init static int tracer_alloc_buffers(void)
bc0c38d1 4258{
4c11d7ae 4259 struct trace_array_cpu *data;
73c5162a 4260 int ring_buf_size;
4c11d7ae 4261 int i;
9e01c1b7 4262 int ret = -ENOMEM;
4c11d7ae 4263
9e01c1b7
RR
4264 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4265 goto out;
4266
4267 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4268 goto out_free_buffer_mask;
4c11d7ae 4269
b04cc6b1
FW
4270 if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
4271 goto out_free_tracing_cpumask;
4272
73c5162a
SR
4273 /* To save memory, keep the ring buffer size to its minimum */
4274 if (ring_buffer_expanded)
4275 ring_buf_size = trace_buf_size;
4276 else
4277 ring_buf_size = 1;
4278
9e01c1b7
RR
4279 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4280 cpumask_copy(tracing_cpumask, cpu_all_mask);
b04cc6b1 4281 cpumask_clear(tracing_reader_cpumask);
9e01c1b7
RR
4282
4283 /* TODO: make the number of buffers hot pluggable with CPUS */
73c5162a 4284 global_trace.buffer = ring_buffer_alloc(ring_buf_size,
3928a8a2
SR
4285 TRACE_BUFFER_FLAGS);
4286 if (!global_trace.buffer) {
4287 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4288 WARN_ON(1);
9e01c1b7 4289 goto out_free_cpumask;
4c11d7ae 4290 }
3928a8a2 4291 global_trace.entries = ring_buffer_size(global_trace.buffer);
4c11d7ae 4292
9e01c1b7 4293
4c11d7ae 4294#ifdef CONFIG_TRACER_MAX_TRACE
73c5162a 4295 max_tr.buffer = ring_buffer_alloc(ring_buf_size,
3928a8a2
SR
4296 TRACE_BUFFER_FLAGS);
4297 if (!max_tr.buffer) {
4298 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4299 WARN_ON(1);
4300 ring_buffer_free(global_trace.buffer);
9e01c1b7 4301 goto out_free_cpumask;
4c11d7ae 4302 }
3928a8a2
SR
4303 max_tr.entries = ring_buffer_size(max_tr.buffer);
4304 WARN_ON(max_tr.entries != global_trace.entries);
a98a3c3f 4305#endif
ab46428c 4306
4c11d7ae 4307 /* Allocate the first page for all buffers */
ab46428c 4308 for_each_tracing_cpu(i) {
4c11d7ae 4309 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
bc0c38d1 4310 max_tr.data[i] = &per_cpu(max_data, i);
4c11d7ae 4311 }
bc0c38d1 4312
bc0c38d1
SR
4313 trace_init_cmdlines();
4314
43a15386 4315 register_tracer(&nop_trace);
79fb0768 4316 current_trace = &nop_trace;
b5ad384e
FW
4317#ifdef CONFIG_BOOT_TRACER
4318 register_tracer(&boot_tracer);
b5ad384e 4319#endif
60a11774
SR
4320 /* All seems OK, enable tracing */
4321 tracing_disabled = 0;
3928a8a2 4322
3f5a54e3
SR
4323 atomic_notifier_chain_register(&panic_notifier_list,
4324 &trace_panic_notifier);
4325
4326 register_die_notifier(&trace_die_notifier);
2fc1dfbe
FW
4327
4328 return 0;
3f5a54e3 4329
9e01c1b7 4330out_free_cpumask:
b04cc6b1
FW
4331 free_cpumask_var(tracing_reader_cpumask);
4332out_free_tracing_cpumask:
9e01c1b7
RR
4333 free_cpumask_var(tracing_cpumask);
4334out_free_buffer_mask:
4335 free_cpumask_var(tracing_buffer_mask);
4336out:
4337 return ret;
bc0c38d1 4338}
b2821ae6
SR
4339
4340__init static int clear_boot_tracer(void)
4341{
4342 /*
4343 * The default tracer at boot buffer is an init section.
4344 * This function is called in lateinit. If we did not
4345 * find the boot tracer, then clear it out, to prevent
4346 * later registration from accessing the buffer that is
4347 * about to be freed.
4348 */
4349 if (!default_bootup_tracer)
4350 return 0;
4351
4352 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4353 default_bootup_tracer);
4354 default_bootup_tracer = NULL;
4355
4356 return 0;
4357}
4358
b5ad384e
FW
4359early_initcall(tracer_alloc_buffers);
4360fs_initcall(tracer_init_debugfs);
b2821ae6 4361late_initcall(clear_boot_tracer);