]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
tracing/events: Convert format output to seq_file
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
405f5571 20#include <linux/smp_lock.h>
3f5a54e3 21#include <linux/notifier.h>
2cadf913 22#include <linux/irqflags.h>
bc0c38d1 23#include <linux/debugfs.h>
4c11d7ae 24#include <linux/pagemap.h>
bc0c38d1
SR
25#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
2cadf913 28#include <linux/kprobes.h>
bc0c38d1
SR
29#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
2cadf913 32#include <linux/splice.h>
3f5a54e3 33#include <linux/kdebug.h>
5f0c6c03 34#include <linux/string.h>
7e53bd42 35#include <linux/rwsem.h>
5a0e3ad6 36#include <linux/slab.h>
bc0c38d1
SR
37#include <linux/ctype.h>
38#include <linux/init.h>
2a2cc8f7 39#include <linux/poll.h>
bc0c38d1 40#include <linux/fs.h>
86387f7e 41
bc0c38d1 42#include "trace.h"
f0868d1e 43#include "trace_output.h"
bc0c38d1 44
3928a8a2
SR
45#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
46
73c5162a
SR
47/*
48 * On boot up, the ring buffer is set to the minimum size, so that
49 * we do not waste memory on systems that are not using tracing.
50 */
020e5f85 51int ring_buffer_expanded;
73c5162a 52
8e1b82e0
FW
53/*
54 * We need to change this state when a selftest is running.
ff32504f
FW
55 * A selftest will lurk into the ring-buffer to count the
56 * entries inserted during the selftest although some concurrent
5e1607a0 57 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
58 * at the same time, giving false positive or negative results.
59 */
8e1b82e0 60static bool __read_mostly tracing_selftest_running;
ff32504f 61
b2821ae6
SR
62/*
63 * If a tracer is running, we do not want to run SELFTEST.
64 */
020e5f85 65bool __read_mostly tracing_selftest_disabled;
b2821ae6 66
adf9f195
FW
67/* For tracers that don't implement custom flags */
68static struct tracer_opt dummy_tracer_opt[] = {
69 { }
70};
71
72static struct tracer_flags dummy_tracer_flags = {
73 .val = 0,
74 .opts = dummy_tracer_opt
75};
76
77static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78{
79 return 0;
80}
0f048701
SR
81
82/*
83 * Kill all tracing for good (never come back).
84 * It is initialized to 1 but will turn to zero if the initialization
85 * of the tracer is successful. But that is the only place that sets
86 * this back to zero.
87 */
4fd27358 88static int tracing_disabled = 1;
0f048701 89
9288f99a 90DEFINE_PER_CPU(int, ftrace_cpu_disabled);
d769041f
SR
91
92static inline void ftrace_disable_cpu(void)
93{
94 preempt_disable();
dd17c8f7 95 __this_cpu_inc(ftrace_cpu_disabled);
d769041f
SR
96}
97
98static inline void ftrace_enable_cpu(void)
99{
dd17c8f7 100 __this_cpu_dec(ftrace_cpu_disabled);
d769041f
SR
101 preempt_enable();
102}
103
9e01c1b7 104static cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c
SR
105
106#define for_each_tracing_cpu(cpu) \
9e01c1b7 107 for_each_cpu(cpu, tracing_buffer_mask)
ab46428c 108
944ac425
SR
109/*
110 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 *
112 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113 * is set, then ftrace_dump is called. This will output the contents
114 * of the ftrace buffers to the console. This is very useful for
115 * capturing traces that lead to crashes and outputing it to a
116 * serial console.
117 *
118 * It is default off, but you can enable it with either specifying
119 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
120 * /proc/sys/kernel/ftrace_dump_on_oops
121 * Set 1 if you want to dump buffers of all CPUs
122 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 123 */
cecbca96
FW
124
125enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 126
b2821ae6
SR
127static int tracing_set_tracer(const char *buf);
128
ee6c2c1b
LZ
129#define MAX_TRACER_SIZE 100
130static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 131static char *default_bootup_tracer;
d9e54076 132
1beee96b 133static int __init set_cmdline_ftrace(char *str)
d9e54076 134{
ee6c2c1b 135 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 136 default_bootup_tracer = bootup_tracer_buf;
73c5162a
SR
137 /* We are using ftrace early, expand it */
138 ring_buffer_expanded = 1;
d9e54076
PZ
139 return 1;
140}
1beee96b 141__setup("ftrace=", set_cmdline_ftrace);
d9e54076 142
944ac425
SR
143static int __init set_ftrace_dump_on_oops(char *str)
144{
cecbca96
FW
145 if (*str++ != '=' || !*str) {
146 ftrace_dump_on_oops = DUMP_ALL;
147 return 1;
148 }
149
150 if (!strcmp("orig_cpu", str)) {
151 ftrace_dump_on_oops = DUMP_ORIG;
152 return 1;
153 }
154
155 return 0;
944ac425
SR
156}
157__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 158
cf8e3474 159unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
160{
161 nsec += 500;
162 do_div(nsec, 1000);
163 return nsec;
164}
165
4fcdae83
SR
166/*
167 * The global_trace is the descriptor that holds the tracing
168 * buffers for the live tracing. For each CPU, it contains
169 * a link list of pages that will store trace entries. The
170 * page descriptor of the pages in the memory is used to hold
171 * the link list by linking the lru item in the page descriptor
172 * to each of the pages in the buffer per CPU.
173 *
174 * For each active CPU there is a data field that holds the
175 * pages for the buffer for that CPU. Each CPU has the same number
176 * of pages allocated for its buffer.
177 */
bc0c38d1
SR
178static struct trace_array global_trace;
179
180static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
181
e77405ad
SR
182int filter_current_check_discard(struct ring_buffer *buffer,
183 struct ftrace_event_call *call, void *rec,
eb02ce01
TZ
184 struct ring_buffer_event *event)
185{
e77405ad 186 return filter_check_discard(call, rec, buffer, event);
eb02ce01 187}
17c873ec 188EXPORT_SYMBOL_GPL(filter_current_check_discard);
eb02ce01 189
37886f6a
SR
190cycle_t ftrace_now(int cpu)
191{
192 u64 ts;
193
194 /* Early boot up does not have a buffer yet */
195 if (!global_trace.buffer)
196 return trace_clock_local();
197
198 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
199 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
200
201 return ts;
202}
bc0c38d1 203
4fcdae83
SR
204/*
205 * The max_tr is used to snapshot the global_trace when a maximum
206 * latency is reached. Some tracers will use this to store a maximum
207 * trace while it continues examining live traces.
208 *
209 * The buffers for the max_tr are set up the same as the global_trace.
210 * When a snapshot is taken, the link list of the max_tr is swapped
211 * with the link list of the global_trace and the buffers are reset for
212 * the global_trace so the tracing can continue.
213 */
bc0c38d1
SR
214static struct trace_array max_tr;
215
9705f69e 216static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
bc0c38d1 217
4fcdae83 218/* tracer_enabled is used to toggle activation of a tracer */
26994ead 219static int tracer_enabled = 1;
4fcdae83 220
9036990d
SR
221/**
222 * tracing_is_enabled - return tracer_enabled status
223 *
224 * This function is used by other tracers to know the status
225 * of the tracer_enabled flag. Tracers may use this function
226 * to know if it should enable their features when starting
227 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
228 */
229int tracing_is_enabled(void)
230{
231 return tracer_enabled;
232}
233
4fcdae83 234/*
3928a8a2
SR
235 * trace_buf_size is the size in bytes that is allocated
236 * for a buffer. Note, the number of bytes is always rounded
237 * to page size.
3f5a54e3
SR
238 *
239 * This number is purposely set to a low number of 16384.
240 * If the dump on oops happens, it will be much appreciated
241 * to not have to wait for all that output. Anyway this can be
242 * boot time and run time configurable.
4fcdae83 243 */
3928a8a2 244#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 245
3928a8a2 246static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 247
4fcdae83 248/* trace_types holds a link list of available tracers. */
bc0c38d1 249static struct tracer *trace_types __read_mostly;
4fcdae83
SR
250
251/* current_trace points to the tracer that is currently active */
bc0c38d1 252static struct tracer *current_trace __read_mostly;
4fcdae83 253
4fcdae83
SR
254/*
255 * trace_types_lock is used to protect the trace_types list.
4fcdae83 256 */
bc0c38d1 257static DEFINE_MUTEX(trace_types_lock);
4fcdae83 258
7e53bd42
LJ
259/*
260 * serialize the access of the ring buffer
261 *
262 * ring buffer serializes readers, but it is low level protection.
263 * The validity of the events (which returns by ring_buffer_peek() ..etc)
264 * are not protected by ring buffer.
265 *
266 * The content of events may become garbage if we allow other process consumes
267 * these events concurrently:
268 * A) the page of the consumed events may become a normal page
269 * (not reader page) in ring buffer, and this page will be rewrited
270 * by events producer.
271 * B) The page of the consumed events may become a page for splice_read,
272 * and this page will be returned to system.
273 *
274 * These primitives allow multi process access to different cpu ring buffer
275 * concurrently.
276 *
277 * These primitives don't distinguish read-only and read-consume access.
278 * Multi read-only access are also serialized.
279 */
280
281#ifdef CONFIG_SMP
282static DECLARE_RWSEM(all_cpu_access_lock);
283static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
284
285static inline void trace_access_lock(int cpu)
286{
287 if (cpu == TRACE_PIPE_ALL_CPU) {
288 /* gain it for accessing the whole ring buffer. */
289 down_write(&all_cpu_access_lock);
290 } else {
291 /* gain it for accessing a cpu ring buffer. */
292
293 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
294 down_read(&all_cpu_access_lock);
295
296 /* Secondly block other access to this @cpu ring buffer. */
297 mutex_lock(&per_cpu(cpu_access_lock, cpu));
298 }
299}
300
301static inline void trace_access_unlock(int cpu)
302{
303 if (cpu == TRACE_PIPE_ALL_CPU) {
304 up_write(&all_cpu_access_lock);
305 } else {
306 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
307 up_read(&all_cpu_access_lock);
308 }
309}
310
311static inline void trace_access_lock_init(void)
312{
313 int cpu;
314
315 for_each_possible_cpu(cpu)
316 mutex_init(&per_cpu(cpu_access_lock, cpu));
317}
318
319#else
320
321static DEFINE_MUTEX(access_lock);
322
323static inline void trace_access_lock(int cpu)
324{
325 (void)cpu;
326 mutex_lock(&access_lock);
327}
328
329static inline void trace_access_unlock(int cpu)
330{
331 (void)cpu;
332 mutex_unlock(&access_lock);
333}
334
335static inline void trace_access_lock_init(void)
336{
337}
338
339#endif
340
4fcdae83 341/* trace_wait is a waitqueue for tasks blocked on trace_poll */
4e655519
IM
342static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
343
ee6bce52 344/* trace_flags holds trace_options default values */
12ef7d44 345unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a
SR
346 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
347 TRACE_ITER_GRAPH_TIME;
4e655519 348
b8de7bd1
SR
349static int trace_stop_count;
350static DEFINE_SPINLOCK(tracing_start_lock);
351
4fcdae83
SR
352/**
353 * trace_wake_up - wake up tasks waiting for trace input
354 *
355 * Simply wakes up any task that is blocked on the trace_wait
356 * queue. These is used with trace_poll for tasks polling the trace.
357 */
4e655519
IM
358void trace_wake_up(void)
359{
89f19f04
AM
360 int cpu;
361
362 if (trace_flags & TRACE_ITER_BLOCK)
363 return;
017730c1
IM
364 /*
365 * The runqueue_is_locked() can fail, but this is the best we
366 * have for now:
367 */
89f19f04
AM
368 cpu = get_cpu();
369 if (!runqueue_is_locked(cpu))
4e655519 370 wake_up(&trace_wait);
89f19f04 371 put_cpu();
4e655519 372}
bc0c38d1 373
3928a8a2 374static int __init set_buf_size(char *str)
bc0c38d1 375{
3928a8a2 376 unsigned long buf_size;
c6caeeb1 377
bc0c38d1
SR
378 if (!str)
379 return 0;
9d612bef 380 buf_size = memparse(str, &str);
c6caeeb1 381 /* nr_entries can not be zero */
9d612bef 382 if (buf_size == 0)
c6caeeb1 383 return 0;
3928a8a2 384 trace_buf_size = buf_size;
bc0c38d1
SR
385 return 1;
386}
3928a8a2 387__setup("trace_buf_size=", set_buf_size);
bc0c38d1 388
0e950173
TB
389static int __init set_tracing_thresh(char *str)
390{
391 unsigned long threshhold;
392 int ret;
393
394 if (!str)
395 return 0;
396 ret = strict_strtoul(str, 0, &threshhold);
397 if (ret < 0)
398 return 0;
399 tracing_thresh = threshhold * 1000;
400 return 1;
401}
402__setup("tracing_thresh=", set_tracing_thresh);
403
57f50be1
SR
404unsigned long nsecs_to_usecs(unsigned long nsecs)
405{
406 return nsecs / 1000;
407}
408
4fcdae83 409/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
410static const char *trace_options[] = {
411 "print-parent",
412 "sym-offset",
413 "sym-addr",
414 "verbose",
f9896bf3 415 "raw",
5e3ca0ec 416 "hex",
cb0f12aa 417 "bin",
2a2cc8f7 418 "block",
86387f7e 419 "stacktrace",
5e1607a0 420 "trace_printk",
b2a866f9 421 "ftrace_preempt",
9f029e83 422 "branch",
12ef7d44 423 "annotate",
02b67518 424 "userstacktrace",
b54d3de9 425 "sym-userobj",
66896a85 426 "printk-msg-only",
c4a8e8be 427 "context-info",
c032ef64 428 "latency-format",
be6f164a 429 "sleep-time",
a2a16d6a 430 "graph-time",
bc0c38d1
SR
431 NULL
432};
433
5079f326
Z
434static struct {
435 u64 (*func)(void);
436 const char *name;
437} trace_clocks[] = {
438 { trace_clock_local, "local" },
439 { trace_clock_global, "global" },
440};
441
442int trace_clock_id;
443
b63f39ea 444/*
445 * trace_parser_get_init - gets the buffer for trace parser
446 */
447int trace_parser_get_init(struct trace_parser *parser, int size)
448{
449 memset(parser, 0, sizeof(*parser));
450
451 parser->buffer = kmalloc(size, GFP_KERNEL);
452 if (!parser->buffer)
453 return 1;
454
455 parser->size = size;
456 return 0;
457}
458
459/*
460 * trace_parser_put - frees the buffer for trace parser
461 */
462void trace_parser_put(struct trace_parser *parser)
463{
464 kfree(parser->buffer);
465}
466
467/*
468 * trace_get_user - reads the user input string separated by space
469 * (matched by isspace(ch))
470 *
471 * For each string found the 'struct trace_parser' is updated,
472 * and the function returns.
473 *
474 * Returns number of bytes read.
475 *
476 * See kernel/trace/trace.h for 'struct trace_parser' details.
477 */
478int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
479 size_t cnt, loff_t *ppos)
480{
481 char ch;
482 size_t read = 0;
483 ssize_t ret;
484
485 if (!*ppos)
486 trace_parser_clear(parser);
487
488 ret = get_user(ch, ubuf++);
489 if (ret)
490 goto out;
491
492 read++;
493 cnt--;
494
495 /*
496 * The parser is not finished with the last write,
497 * continue reading the user input without skipping spaces.
498 */
499 if (!parser->cont) {
500 /* skip white space */
501 while (cnt && isspace(ch)) {
502 ret = get_user(ch, ubuf++);
503 if (ret)
504 goto out;
505 read++;
506 cnt--;
507 }
508
509 /* only spaces were written */
510 if (isspace(ch)) {
511 *ppos += read;
512 ret = read;
513 goto out;
514 }
515
516 parser->idx = 0;
517 }
518
519 /* read the non-space input */
520 while (cnt && !isspace(ch)) {
3c235a33 521 if (parser->idx < parser->size - 1)
b63f39ea 522 parser->buffer[parser->idx++] = ch;
523 else {
524 ret = -EINVAL;
525 goto out;
526 }
527 ret = get_user(ch, ubuf++);
528 if (ret)
529 goto out;
530 read++;
531 cnt--;
532 }
533
534 /* We either got finished input or we have to wait for another call. */
535 if (isspace(ch)) {
536 parser->buffer[parser->idx] = 0;
537 parser->cont = false;
538 } else {
539 parser->cont = true;
540 parser->buffer[parser->idx++] = ch;
541 }
542
543 *ppos += read;
544 ret = read;
545
546out:
547 return ret;
548}
549
6c6c2796
PP
550ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
551{
552 int len;
553 int ret;
554
2dc5d12b
SR
555 if (!cnt)
556 return 0;
557
6c6c2796
PP
558 if (s->len <= s->readpos)
559 return -EBUSY;
560
561 len = s->len - s->readpos;
562 if (cnt > len)
563 cnt = len;
564 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
2dc5d12b 565 if (ret == cnt)
6c6c2796
PP
566 return -EFAULT;
567
2dc5d12b
SR
568 cnt -= ret;
569
e74da523 570 s->readpos += cnt;
6c6c2796 571 return cnt;
214023c3
SR
572}
573
b8b94265 574static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
575{
576 int len;
577 void *ret;
578
579 if (s->len <= s->readpos)
580 return -EBUSY;
581
582 len = s->len - s->readpos;
583 if (cnt > len)
584 cnt = len;
585 ret = memcpy(buf, s->buffer + s->readpos, cnt);
586 if (!ret)
587 return -EFAULT;
588
e74da523 589 s->readpos += cnt;
3c56819b
EGM
590 return cnt;
591}
592
5d4a9dba
SR
593/*
594 * ftrace_max_lock is used to protect the swapping of buffers
595 * when taking a max snapshot. The buffers themselves are
596 * protected by per_cpu spinlocks. But the action of the swap
597 * needs its own lock.
598 *
445c8951 599 * This is defined as a arch_spinlock_t in order to help
5d4a9dba
SR
600 * with performance when lockdep debugging is enabled.
601 *
602 * It is also used in other places outside the update_max_tr
603 * so it needs to be defined outside of the
604 * CONFIG_TRACER_MAX_TRACE.
605 */
445c8951 606static arch_spinlock_t ftrace_max_lock =
edc35bd7 607 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5d4a9dba 608
0e950173
TB
609unsigned long __read_mostly tracing_thresh;
610
5d4a9dba
SR
611#ifdef CONFIG_TRACER_MAX_TRACE
612unsigned long __read_mostly tracing_max_latency;
5d4a9dba
SR
613
614/*
615 * Copy the new maximum trace into the separate maximum-trace
616 * structure. (this way the maximum trace is permanently saved,
617 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
618 */
619static void
620__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
621{
622 struct trace_array_cpu *data = tr->data[cpu];
1acaa1b2 623 struct trace_array_cpu *max_data;
5d4a9dba
SR
624
625 max_tr.cpu = cpu;
626 max_tr.time_start = data->preempt_timestamp;
627
8248ac05
SR
628 max_data = max_tr.data[cpu];
629 max_data->saved_latency = tracing_max_latency;
630 max_data->critical_start = data->critical_start;
631 max_data->critical_end = data->critical_end;
5d4a9dba 632
1acaa1b2 633 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05
SR
634 max_data->pid = tsk->pid;
635 max_data->uid = task_uid(tsk);
636 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
637 max_data->policy = tsk->policy;
638 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
639
640 /* record this tasks comm */
641 tracing_record_cmdline(tsk);
642}
643
4fcdae83
SR
644/**
645 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
646 * @tr: tracer
647 * @tsk: the task with the latency
648 * @cpu: The cpu that initiated the trace.
649 *
650 * Flip the buffers between the @tr and the max_tr and record information
651 * about which task was the cause of this latency.
652 */
e309b41d 653void
bc0c38d1
SR
654update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
655{
3928a8a2 656 struct ring_buffer *buf = tr->buffer;
bc0c38d1 657
b8de7bd1
SR
658 if (trace_stop_count)
659 return;
660
4c11d7ae 661 WARN_ON_ONCE(!irqs_disabled());
0199c4e6 662 arch_spin_lock(&ftrace_max_lock);
3928a8a2
SR
663
664 tr->buffer = max_tr.buffer;
665 max_tr.buffer = buf;
666
bc0c38d1 667 __update_max_tr(tr, tsk, cpu);
0199c4e6 668 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
669}
670
671/**
672 * update_max_tr_single - only copy one trace over, and reset the rest
673 * @tr - tracer
674 * @tsk - task with the latency
675 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
676 *
677 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 678 */
e309b41d 679void
bc0c38d1
SR
680update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
681{
3928a8a2 682 int ret;
bc0c38d1 683
b8de7bd1
SR
684 if (trace_stop_count)
685 return;
686
4c11d7ae 687 WARN_ON_ONCE(!irqs_disabled());
0199c4e6 688 arch_spin_lock(&ftrace_max_lock);
bc0c38d1 689
d769041f
SR
690 ftrace_disable_cpu();
691
3928a8a2
SR
692 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
693
e8165dbb
SR
694 if (ret == -EBUSY) {
695 /*
696 * We failed to swap the buffer due to a commit taking
697 * place on this CPU. We fail to record, but we reset
698 * the max trace buffer (no one writes directly to it)
699 * and flag that it failed.
700 */
701 trace_array_printk(&max_tr, _THIS_IP_,
702 "Failed to swap buffers due to commit in progress\n");
703 }
704
d769041f
SR
705 ftrace_enable_cpu();
706
e8165dbb 707 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
708
709 __update_max_tr(tr, tsk, cpu);
0199c4e6 710 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1 711}
5d4a9dba 712#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 713
4fcdae83
SR
714/**
715 * register_tracer - register a tracer with the ftrace system.
716 * @type - the plugin for the tracer
717 *
718 * Register a new plugin tracer.
719 */
bc0c38d1 720int register_tracer(struct tracer *type)
e7669b8e
HE
721__releases(kernel_lock)
722__acquires(kernel_lock)
bc0c38d1
SR
723{
724 struct tracer *t;
bc0c38d1
SR
725 int ret = 0;
726
727 if (!type->name) {
728 pr_info("Tracer must have a name\n");
729 return -1;
730 }
731
ee6c2c1b
LZ
732 if (strlen(type->name) > MAX_TRACER_SIZE) {
733 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
734 return -1;
735 }
736
86fa2f60
IM
737 /*
738 * When this gets called we hold the BKL which means that
739 * preemption is disabled. Various trace selftests however
740 * need to disable and enable preemption for successful tests.
741 * So we drop the BKL here and grab it after the tests again.
742 */
743 unlock_kernel();
bc0c38d1 744 mutex_lock(&trace_types_lock);
86fa2f60 745
8e1b82e0
FW
746 tracing_selftest_running = true;
747
bc0c38d1
SR
748 for (t = trace_types; t; t = t->next) {
749 if (strcmp(type->name, t->name) == 0) {
750 /* already found */
ee6c2c1b 751 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
752 type->name);
753 ret = -1;
754 goto out;
755 }
756 }
757
adf9f195
FW
758 if (!type->set_flag)
759 type->set_flag = &dummy_set_flag;
760 if (!type->flags)
761 type->flags = &dummy_tracer_flags;
762 else
763 if (!type->flags->opts)
764 type->flags->opts = dummy_tracer_opt;
6eaaa5d5
FW
765 if (!type->wait_pipe)
766 type->wait_pipe = default_wait_pipe;
767
adf9f195 768
60a11774 769#ifdef CONFIG_FTRACE_STARTUP_TEST
b2821ae6 770 if (type->selftest && !tracing_selftest_disabled) {
60a11774 771 struct tracer *saved_tracer = current_trace;
60a11774 772 struct trace_array *tr = &global_trace;
ff32504f 773
60a11774
SR
774 /*
775 * Run a selftest on this tracer.
776 * Here we reset the trace buffer, and set the current
777 * tracer to be this tracer. The tracer can then run some
778 * internal tracing to verify that everything is in order.
779 * If we fail, we do not register this tracer.
780 */
76f0d073 781 tracing_reset_online_cpus(tr);
86fa2f60 782
60a11774 783 current_trace = type;
60a11774
SR
784 /* the test is responsible for initializing and enabling */
785 pr_info("Testing tracer %s: ", type->name);
786 ret = type->selftest(type, tr);
787 /* the test is responsible for resetting too */
788 current_trace = saved_tracer;
60a11774
SR
789 if (ret) {
790 printk(KERN_CONT "FAILED!\n");
791 goto out;
792 }
1d4db00a 793 /* Only reset on passing, to avoid touching corrupted buffers */
76f0d073 794 tracing_reset_online_cpus(tr);
86fa2f60 795
60a11774
SR
796 printk(KERN_CONT "PASSED\n");
797 }
798#endif
799
bc0c38d1
SR
800 type->next = trace_types;
801 trace_types = type;
60a11774 802
bc0c38d1 803 out:
8e1b82e0 804 tracing_selftest_running = false;
bc0c38d1
SR
805 mutex_unlock(&trace_types_lock);
806
dac74940
SR
807 if (ret || !default_bootup_tracer)
808 goto out_unlock;
809
ee6c2c1b 810 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
811 goto out_unlock;
812
813 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
814 /* Do we want this tracer to start on bootup? */
815 tracing_set_tracer(type->name);
816 default_bootup_tracer = NULL;
817 /* disable other selftests, since this will break it. */
818 tracing_selftest_disabled = 1;
b2821ae6 819#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
820 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
821 type->name);
b2821ae6 822#endif
b2821ae6 823
dac74940 824 out_unlock:
b2821ae6 825 lock_kernel();
bc0c38d1
SR
826 return ret;
827}
828
829void unregister_tracer(struct tracer *type)
830{
831 struct tracer **t;
bc0c38d1
SR
832
833 mutex_lock(&trace_types_lock);
834 for (t = &trace_types; *t; t = &(*t)->next) {
835 if (*t == type)
836 goto found;
837 }
ee6c2c1b 838 pr_info("Tracer %s not registered\n", type->name);
bc0c38d1
SR
839 goto out;
840
841 found:
842 *t = (*t)->next;
b5db03c4
ACM
843
844 if (type == current_trace && tracer_enabled) {
845 tracer_enabled = 0;
846 tracing_stop();
847 if (current_trace->stop)
848 current_trace->stop(&global_trace);
849 current_trace = &nop_trace;
850 }
ee6c2c1b 851out:
bc0c38d1
SR
852 mutex_unlock(&trace_types_lock);
853}
854
283740c6 855static void __tracing_reset(struct ring_buffer *buffer, int cpu)
bc0c38d1 856{
d769041f 857 ftrace_disable_cpu();
283740c6 858 ring_buffer_reset_cpu(buffer, cpu);
d769041f 859 ftrace_enable_cpu();
bc0c38d1
SR
860}
861
f633903a
SR
862void tracing_reset(struct trace_array *tr, int cpu)
863{
864 struct ring_buffer *buffer = tr->buffer;
865
866 ring_buffer_record_disable(buffer);
867
868 /* Make sure all commits have finished */
869 synchronize_sched();
283740c6 870 __tracing_reset(buffer, cpu);
f633903a
SR
871
872 ring_buffer_record_enable(buffer);
873}
874
213cc060
PE
875void tracing_reset_online_cpus(struct trace_array *tr)
876{
621968cd 877 struct ring_buffer *buffer = tr->buffer;
213cc060
PE
878 int cpu;
879
621968cd
SR
880 ring_buffer_record_disable(buffer);
881
882 /* Make sure all commits have finished */
883 synchronize_sched();
884
213cc060
PE
885 tr->time_start = ftrace_now(tr->cpu);
886
887 for_each_online_cpu(cpu)
283740c6 888 __tracing_reset(buffer, cpu);
621968cd
SR
889
890 ring_buffer_record_enable(buffer);
213cc060
PE
891}
892
9456f0fa
SR
893void tracing_reset_current(int cpu)
894{
895 tracing_reset(&global_trace, cpu);
896}
897
898void tracing_reset_current_online_cpus(void)
899{
900 tracing_reset_online_cpus(&global_trace);
901}
902
bc0c38d1 903#define SAVED_CMDLINES 128
2c7eea4c 904#define NO_CMDLINE_MAP UINT_MAX
bc0c38d1
SR
905static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
906static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
907static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
908static int cmdline_idx;
edc35bd7 909static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
25b0b44a 910
25b0b44a 911/* temporary disable recording */
4fd27358 912static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1
SR
913
914static void trace_init_cmdlines(void)
915{
2c7eea4c
TG
916 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
917 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
bc0c38d1
SR
918 cmdline_idx = 0;
919}
920
b5130b1e
CE
921int is_tracing_stopped(void)
922{
923 return trace_stop_count;
924}
925
69bb54ec
SR
926/**
927 * ftrace_off_permanent - disable all ftrace code permanently
928 *
929 * This should only be called when a serious anomally has
930 * been detected. This will turn off the function tracing,
931 * ring buffers, and other tracing utilites. It takes no
932 * locks and can be called from any context.
933 */
934void ftrace_off_permanent(void)
935{
936 tracing_disabled = 1;
937 ftrace_stop();
938 tracing_off_permanent();
939}
940
0f048701
SR
941/**
942 * tracing_start - quick start of the tracer
943 *
944 * If tracing is enabled but was stopped by tracing_stop,
945 * this will start the tracer back up.
946 */
947void tracing_start(void)
948{
949 struct ring_buffer *buffer;
950 unsigned long flags;
951
952 if (tracing_disabled)
953 return;
954
955 spin_lock_irqsave(&tracing_start_lock, flags);
b06a8301
SR
956 if (--trace_stop_count) {
957 if (trace_stop_count < 0) {
958 /* Someone screwed up their debugging */
959 WARN_ON_ONCE(1);
960 trace_stop_count = 0;
961 }
0f048701
SR
962 goto out;
963 }
964
a2f80714
SR
965 /* Prevent the buffers from switching */
966 arch_spin_lock(&ftrace_max_lock);
0f048701
SR
967
968 buffer = global_trace.buffer;
969 if (buffer)
970 ring_buffer_record_enable(buffer);
971
972 buffer = max_tr.buffer;
973 if (buffer)
974 ring_buffer_record_enable(buffer);
975
a2f80714
SR
976 arch_spin_unlock(&ftrace_max_lock);
977
0f048701
SR
978 ftrace_start();
979 out:
980 spin_unlock_irqrestore(&tracing_start_lock, flags);
981}
982
983/**
984 * tracing_stop - quick stop of the tracer
985 *
986 * Light weight way to stop tracing. Use in conjunction with
987 * tracing_start.
988 */
989void tracing_stop(void)
990{
991 struct ring_buffer *buffer;
992 unsigned long flags;
993
994 ftrace_stop();
995 spin_lock_irqsave(&tracing_start_lock, flags);
996 if (trace_stop_count++)
997 goto out;
998
a2f80714
SR
999 /* Prevent the buffers from switching */
1000 arch_spin_lock(&ftrace_max_lock);
1001
0f048701
SR
1002 buffer = global_trace.buffer;
1003 if (buffer)
1004 ring_buffer_record_disable(buffer);
1005
1006 buffer = max_tr.buffer;
1007 if (buffer)
1008 ring_buffer_record_disable(buffer);
1009
a2f80714
SR
1010 arch_spin_unlock(&ftrace_max_lock);
1011
0f048701
SR
1012 out:
1013 spin_unlock_irqrestore(&tracing_start_lock, flags);
1014}
1015
e309b41d 1016void trace_stop_cmdline_recording(void);
bc0c38d1 1017
e309b41d 1018static void trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1019{
a635cf04 1020 unsigned pid, idx;
bc0c38d1
SR
1021
1022 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1023 return;
1024
1025 /*
1026 * It's not the end of the world if we don't get
1027 * the lock, but we also don't want to spin
1028 * nor do we want to disable interrupts,
1029 * so if we miss here, then better luck next time.
1030 */
0199c4e6 1031 if (!arch_spin_trylock(&trace_cmdline_lock))
bc0c38d1
SR
1032 return;
1033
1034 idx = map_pid_to_cmdline[tsk->pid];
2c7eea4c 1035 if (idx == NO_CMDLINE_MAP) {
bc0c38d1
SR
1036 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1037
a635cf04
CE
1038 /*
1039 * Check whether the cmdline buffer at idx has a pid
1040 * mapped. We are going to overwrite that entry so we
1041 * need to clear the map_pid_to_cmdline. Otherwise we
1042 * would read the new comm for the old pid.
1043 */
1044 pid = map_cmdline_to_pid[idx];
1045 if (pid != NO_CMDLINE_MAP)
1046 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1047
a635cf04 1048 map_cmdline_to_pid[idx] = tsk->pid;
bc0c38d1
SR
1049 map_pid_to_cmdline[tsk->pid] = idx;
1050
1051 cmdline_idx = idx;
1052 }
1053
1054 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1055
0199c4e6 1056 arch_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
1057}
1058
4ca53085 1059void trace_find_cmdline(int pid, char comm[])
bc0c38d1 1060{
bc0c38d1
SR
1061 unsigned map;
1062
4ca53085
SR
1063 if (!pid) {
1064 strcpy(comm, "<idle>");
1065 return;
1066 }
bc0c38d1 1067
74bf4076
SR
1068 if (WARN_ON_ONCE(pid < 0)) {
1069 strcpy(comm, "<XXX>");
1070 return;
1071 }
1072
4ca53085
SR
1073 if (pid > PID_MAX_DEFAULT) {
1074 strcpy(comm, "<...>");
1075 return;
1076 }
bc0c38d1 1077
5b6045a9 1078 preempt_disable();
0199c4e6 1079 arch_spin_lock(&trace_cmdline_lock);
bc0c38d1 1080 map = map_pid_to_cmdline[pid];
50d88758
TG
1081 if (map != NO_CMDLINE_MAP)
1082 strcpy(comm, saved_cmdlines[map]);
1083 else
1084 strcpy(comm, "<...>");
bc0c38d1 1085
0199c4e6 1086 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1087 preempt_enable();
bc0c38d1
SR
1088}
1089
e309b41d 1090void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1091{
18aecd36
TG
1092 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1093 !tracing_is_on())
bc0c38d1
SR
1094 return;
1095
1096 trace_save_cmdline(tsk);
1097}
1098
45dcd8b8 1099void
38697053
SR
1100tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1101 int pc)
bc0c38d1
SR
1102{
1103 struct task_struct *tsk = current;
bc0c38d1 1104
777e208d
SR
1105 entry->preempt_count = pc & 0xff;
1106 entry->pid = (tsk) ? tsk->pid : 0;
637e7e86 1107 entry->lock_depth = (tsk) ? tsk->lock_depth : 0;
777e208d 1108 entry->flags =
9244489a 1109#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1110 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1111#else
1112 TRACE_FLAG_IRQS_NOSUPPORT |
1113#endif
bc0c38d1
SR
1114 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1115 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1116 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1117}
f413cdb8 1118EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1119
e77405ad
SR
1120struct ring_buffer_event *
1121trace_buffer_lock_reserve(struct ring_buffer *buffer,
1122 int type,
1123 unsigned long len,
1124 unsigned long flags, int pc)
51a763dd
ACM
1125{
1126 struct ring_buffer_event *event;
1127
e77405ad 1128 event = ring_buffer_lock_reserve(buffer, len);
51a763dd
ACM
1129 if (event != NULL) {
1130 struct trace_entry *ent = ring_buffer_event_data(event);
1131
1132 tracing_generic_entry_update(ent, flags, pc);
1133 ent->type = type;
1134 }
1135
1136 return event;
1137}
51a763dd 1138
e77405ad
SR
1139static inline void
1140__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1141 struct ring_buffer_event *event,
1142 unsigned long flags, int pc,
1143 int wake)
51a763dd 1144{
e77405ad 1145 ring_buffer_unlock_commit(buffer, event);
51a763dd 1146
e77405ad
SR
1147 ftrace_trace_stack(buffer, flags, 6, pc);
1148 ftrace_trace_userstack(buffer, flags, pc);
07edf712
FW
1149
1150 if (wake)
1151 trace_wake_up();
1152}
1153
e77405ad
SR
1154void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1155 struct ring_buffer_event *event,
1156 unsigned long flags, int pc)
07edf712 1157{
e77405ad 1158 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
51a763dd
ACM
1159}
1160
ef5580d0 1161struct ring_buffer_event *
e77405ad
SR
1162trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1163 int type, unsigned long len,
ef5580d0
SR
1164 unsigned long flags, int pc)
1165{
e77405ad
SR
1166 *current_rb = global_trace.buffer;
1167 return trace_buffer_lock_reserve(*current_rb,
ef5580d0
SR
1168 type, len, flags, pc);
1169}
94487d6d 1170EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0 1171
e77405ad
SR
1172void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1173 struct ring_buffer_event *event,
ef5580d0
SR
1174 unsigned long flags, int pc)
1175{
e77405ad 1176 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
07edf712 1177}
94487d6d 1178EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712 1179
e77405ad
SR
1180void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1181 struct ring_buffer_event *event,
1182 unsigned long flags, int pc)
07edf712 1183{
e77405ad 1184 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
77d9f465 1185}
94487d6d 1186EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
77d9f465 1187
e77405ad
SR
1188void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1189 struct ring_buffer_event *event)
77d9f465 1190{
e77405ad 1191 ring_buffer_discard_commit(buffer, event);
ef5580d0 1192}
12acd473 1193EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 1194
e309b41d 1195void
7be42151 1196trace_function(struct trace_array *tr,
38697053
SR
1197 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1198 int pc)
bc0c38d1 1199{
e1112b4d 1200 struct ftrace_event_call *call = &event_function;
e77405ad 1201 struct ring_buffer *buffer = tr->buffer;
3928a8a2 1202 struct ring_buffer_event *event;
777e208d 1203 struct ftrace_entry *entry;
bc0c38d1 1204
d769041f 1205 /* If we are reading the ring buffer, don't trace */
dd17c8f7 1206 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
d769041f
SR
1207 return;
1208
e77405ad 1209 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
51a763dd 1210 flags, pc);
3928a8a2
SR
1211 if (!event)
1212 return;
1213 entry = ring_buffer_event_data(event);
777e208d
SR
1214 entry->ip = ip;
1215 entry->parent_ip = parent_ip;
e1112b4d 1216
e77405ad
SR
1217 if (!filter_check_discard(call, entry, buffer, event))
1218 ring_buffer_unlock_commit(buffer, event);
bc0c38d1
SR
1219}
1220
e309b41d 1221void
2e0f5761 1222ftrace(struct trace_array *tr, struct trace_array_cpu *data,
38697053
SR
1223 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1224 int pc)
2e0f5761
IM
1225{
1226 if (likely(!atomic_read(&data->disabled)))
7be42151 1227 trace_function(tr, ip, parent_ip, flags, pc);
2e0f5761
IM
1228}
1229
c0a0d0d3 1230#ifdef CONFIG_STACKTRACE
e77405ad 1231static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991
SR
1232 unsigned long flags,
1233 int skip, int pc)
86387f7e 1234{
e1112b4d 1235 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1236 struct ring_buffer_event *event;
777e208d 1237 struct stack_entry *entry;
86387f7e
IM
1238 struct stack_trace trace;
1239
e77405ad 1240 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
51a763dd 1241 sizeof(*entry), flags, pc);
3928a8a2
SR
1242 if (!event)
1243 return;
1244 entry = ring_buffer_event_data(event);
777e208d 1245 memset(&entry->caller, 0, sizeof(entry->caller));
86387f7e
IM
1246
1247 trace.nr_entries = 0;
1248 trace.max_entries = FTRACE_STACK_ENTRIES;
1249 trace.skip = skip;
777e208d 1250 trace.entries = entry->caller;
86387f7e
IM
1251
1252 save_stack_trace(&trace);
e77405ad
SR
1253 if (!filter_check_discard(call, entry, buffer, event))
1254 ring_buffer_unlock_commit(buffer, event);
f0a920d5
IM
1255}
1256
e77405ad
SR
1257void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1258 int skip, int pc)
53614991
SR
1259{
1260 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1261 return;
1262
e77405ad 1263 __ftrace_trace_stack(buffer, flags, skip, pc);
53614991
SR
1264}
1265
c0a0d0d3
FW
1266void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1267 int pc)
38697053 1268{
e77405ad 1269 __ftrace_trace_stack(tr->buffer, flags, skip, pc);
38697053
SR
1270}
1271
03889384
SR
1272/**
1273 * trace_dump_stack - record a stack back trace in the trace buffer
1274 */
1275void trace_dump_stack(void)
1276{
1277 unsigned long flags;
1278
1279 if (tracing_disabled || tracing_selftest_running)
e36c5458 1280 return;
03889384
SR
1281
1282 local_save_flags(flags);
1283
1284 /* skipping 3 traces, seems to get us at the caller of this function */
1285 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1286}
1287
e77405ad
SR
1288void
1289ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 1290{
e1112b4d 1291 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1292 struct ring_buffer_event *event;
02b67518
TE
1293 struct userstack_entry *entry;
1294 struct stack_trace trace;
02b67518
TE
1295
1296 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1297 return;
1298
b6345879
SR
1299 /*
1300 * NMIs can not handle page faults, even with fix ups.
1301 * The save user stack can (and often does) fault.
1302 */
1303 if (unlikely(in_nmi()))
1304 return;
02b67518 1305
e77405ad 1306 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
51a763dd 1307 sizeof(*entry), flags, pc);
02b67518
TE
1308 if (!event)
1309 return;
1310 entry = ring_buffer_event_data(event);
02b67518 1311
48659d31 1312 entry->tgid = current->tgid;
02b67518
TE
1313 memset(&entry->caller, 0, sizeof(entry->caller));
1314
1315 trace.nr_entries = 0;
1316 trace.max_entries = FTRACE_STACK_ENTRIES;
1317 trace.skip = 0;
1318 trace.entries = entry->caller;
1319
1320 save_stack_trace_user(&trace);
e77405ad
SR
1321 if (!filter_check_discard(call, entry, buffer, event))
1322 ring_buffer_unlock_commit(buffer, event);
02b67518
TE
1323}
1324
4fd27358
HE
1325#ifdef UNUSED
1326static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1327{
7be42151 1328 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1329}
4fd27358 1330#endif /* UNUSED */
02b67518 1331
c0a0d0d3
FW
1332#endif /* CONFIG_STACKTRACE */
1333
38697053 1334static void
7be42151 1335ftrace_trace_special(void *__tr,
38697053
SR
1336 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1337 int pc)
a4feb834 1338{
60ba7702 1339 struct ftrace_event_call *call = &event_special;
3928a8a2 1340 struct ring_buffer_event *event;
a4feb834 1341 struct trace_array *tr = __tr;
e77405ad 1342 struct ring_buffer *buffer = tr->buffer;
777e208d 1343 struct special_entry *entry;
a4feb834 1344
e77405ad 1345 event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
51a763dd 1346 sizeof(*entry), 0, pc);
3928a8a2
SR
1347 if (!event)
1348 return;
1349 entry = ring_buffer_event_data(event);
777e208d
SR
1350 entry->arg1 = arg1;
1351 entry->arg2 = arg2;
1352 entry->arg3 = arg3;
60ba7702
SR
1353
1354 if (!filter_check_discard(call, entry, buffer, event))
1355 trace_buffer_unlock_commit(buffer, event, 0, pc);
a4feb834
IM
1356}
1357
38697053
SR
1358void
1359__trace_special(void *__tr, void *__data,
1360 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1361{
7be42151 1362 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
38697053
SR
1363}
1364
4902f884
SR
1365void
1366ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1367{
1368 struct trace_array *tr = &global_trace;
1369 struct trace_array_cpu *data;
5aa1ba6a 1370 unsigned long flags;
4902f884 1371 int cpu;
38697053 1372 int pc;
4902f884 1373
c76f0694 1374 if (tracing_disabled)
4902f884
SR
1375 return;
1376
38697053 1377 pc = preempt_count();
5aa1ba6a 1378 local_irq_save(flags);
4902f884
SR
1379 cpu = raw_smp_processor_id();
1380 data = tr->data[cpu];
4902f884 1381
5aa1ba6a 1382 if (likely(atomic_inc_return(&data->disabled) == 1))
7be42151 1383 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
4902f884 1384
5aa1ba6a
SR
1385 atomic_dec(&data->disabled);
1386 local_irq_restore(flags);
4902f884
SR
1387}
1388
769b0441 1389/**
48ead020 1390 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
1391 *
1392 */
40ce74f1 1393int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 1394{
445c8951 1395 static arch_spinlock_t trace_buf_lock =
edc35bd7 1396 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
769b0441
FW
1397 static u32 trace_buf[TRACE_BUF_SIZE];
1398
e1112b4d 1399 struct ftrace_event_call *call = &event_bprint;
769b0441 1400 struct ring_buffer_event *event;
e77405ad 1401 struct ring_buffer *buffer;
769b0441
FW
1402 struct trace_array *tr = &global_trace;
1403 struct trace_array_cpu *data;
48ead020 1404 struct bprint_entry *entry;
769b0441 1405 unsigned long flags;
3189cdb3 1406 int disable;
769b0441
FW
1407 int resched;
1408 int cpu, len = 0, size, pc;
1409
1410 if (unlikely(tracing_selftest_running || tracing_disabled))
1411 return 0;
1412
1413 /* Don't pollute graph traces with trace_vprintk internals */
1414 pause_graph_tracing();
1415
1416 pc = preempt_count();
1417 resched = ftrace_preempt_disable();
1418 cpu = raw_smp_processor_id();
1419 data = tr->data[cpu];
1420
3189cdb3
SR
1421 disable = atomic_inc_return(&data->disabled);
1422 if (unlikely(disable != 1))
769b0441
FW
1423 goto out;
1424
80370cb7
SR
1425 /* Lockdep uses trace_printk for lock tracing */
1426 local_irq_save(flags);
0199c4e6 1427 arch_spin_lock(&trace_buf_lock);
769b0441
FW
1428 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1429
1430 if (len > TRACE_BUF_SIZE || len < 0)
1431 goto out_unlock;
1432
1433 size = sizeof(*entry) + sizeof(u32) * len;
e77405ad
SR
1434 buffer = tr->buffer;
1435 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1436 flags, pc);
769b0441
FW
1437 if (!event)
1438 goto out_unlock;
1439 entry = ring_buffer_event_data(event);
1440 entry->ip = ip;
769b0441
FW
1441 entry->fmt = fmt;
1442
1443 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
d931369b 1444 if (!filter_check_discard(call, entry, buffer, event)) {
e77405ad 1445 ring_buffer_unlock_commit(buffer, event);
d931369b
SR
1446 ftrace_trace_stack(buffer, flags, 6, pc);
1447 }
769b0441
FW
1448
1449out_unlock:
0199c4e6 1450 arch_spin_unlock(&trace_buf_lock);
80370cb7 1451 local_irq_restore(flags);
769b0441
FW
1452
1453out:
3189cdb3 1454 atomic_dec_return(&data->disabled);
769b0441
FW
1455 ftrace_preempt_enable(resched);
1456 unpause_graph_tracing();
1457
1458 return len;
1459}
48ead020
FW
1460EXPORT_SYMBOL_GPL(trace_vbprintk);
1461
659372d3
SR
1462int trace_array_printk(struct trace_array *tr,
1463 unsigned long ip, const char *fmt, ...)
1464{
1465 int ret;
1466 va_list ap;
1467
1468 if (!(trace_flags & TRACE_ITER_PRINTK))
1469 return 0;
1470
1471 va_start(ap, fmt);
1472 ret = trace_array_vprintk(tr, ip, fmt, ap);
1473 va_end(ap);
1474 return ret;
1475}
1476
1477int trace_array_vprintk(struct trace_array *tr,
1478 unsigned long ip, const char *fmt, va_list args)
48ead020 1479{
edc35bd7 1480 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
48ead020
FW
1481 static char trace_buf[TRACE_BUF_SIZE];
1482
e1112b4d 1483 struct ftrace_event_call *call = &event_print;
48ead020 1484 struct ring_buffer_event *event;
e77405ad 1485 struct ring_buffer *buffer;
48ead020
FW
1486 struct trace_array_cpu *data;
1487 int cpu, len = 0, size, pc;
1488 struct print_entry *entry;
1489 unsigned long irq_flags;
3189cdb3 1490 int disable;
48ead020
FW
1491
1492 if (tracing_disabled || tracing_selftest_running)
1493 return 0;
1494
1495 pc = preempt_count();
1496 preempt_disable_notrace();
1497 cpu = raw_smp_processor_id();
1498 data = tr->data[cpu];
1499
3189cdb3
SR
1500 disable = atomic_inc_return(&data->disabled);
1501 if (unlikely(disable != 1))
48ead020
FW
1502 goto out;
1503
1504 pause_graph_tracing();
1505 raw_local_irq_save(irq_flags);
0199c4e6 1506 arch_spin_lock(&trace_buf_lock);
f2942487 1507 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
48ead020
FW
1508
1509 size = sizeof(*entry) + len + 1;
e77405ad
SR
1510 buffer = tr->buffer;
1511 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1512 irq_flags, pc);
48ead020
FW
1513 if (!event)
1514 goto out_unlock;
1515 entry = ring_buffer_event_data(event);
c13d2f7c 1516 entry->ip = ip;
48ead020
FW
1517
1518 memcpy(&entry->buf, trace_buf, len);
c13d2f7c 1519 entry->buf[len] = '\0';
d931369b 1520 if (!filter_check_discard(call, entry, buffer, event)) {
e77405ad 1521 ring_buffer_unlock_commit(buffer, event);
d931369b
SR
1522 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1523 }
48ead020
FW
1524
1525 out_unlock:
0199c4e6 1526 arch_spin_unlock(&trace_buf_lock);
48ead020
FW
1527 raw_local_irq_restore(irq_flags);
1528 unpause_graph_tracing();
1529 out:
3189cdb3 1530 atomic_dec_return(&data->disabled);
48ead020
FW
1531 preempt_enable_notrace();
1532
1533 return len;
1534}
659372d3
SR
1535
1536int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1537{
a813a159 1538 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 1539}
769b0441
FW
1540EXPORT_SYMBOL_GPL(trace_vprintk);
1541
bc0c38d1
SR
1542enum trace_file_type {
1543 TRACE_FILE_LAT_FMT = 1,
12ef7d44 1544 TRACE_FILE_ANNOTATE = 2,
bc0c38d1
SR
1545};
1546
e2ac8ef5 1547static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 1548{
d769041f
SR
1549 /* Don't allow ftrace to trace into the ring buffers */
1550 ftrace_disable_cpu();
1551
5a90f577 1552 iter->idx++;
d769041f
SR
1553 if (iter->buffer_iter[iter->cpu])
1554 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1555
1556 ftrace_enable_cpu();
5a90f577
SR
1557}
1558
e309b41d 1559static struct trace_entry *
bc21b478
SR
1560peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1561 unsigned long *lost_events)
dd0e545f 1562{
3928a8a2
SR
1563 struct ring_buffer_event *event;
1564 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
dd0e545f 1565
d769041f
SR
1566 /* Don't allow ftrace to trace into the ring buffers */
1567 ftrace_disable_cpu();
1568
1569 if (buf_iter)
1570 event = ring_buffer_iter_peek(buf_iter, ts);
1571 else
bc21b478
SR
1572 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1573 lost_events);
d769041f
SR
1574
1575 ftrace_enable_cpu();
1576
3928a8a2 1577 return event ? ring_buffer_event_data(event) : NULL;
dd0e545f 1578}
d769041f 1579
dd0e545f 1580static struct trace_entry *
bc21b478
SR
1581__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1582 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 1583{
3928a8a2 1584 struct ring_buffer *buffer = iter->tr->buffer;
bc0c38d1 1585 struct trace_entry *ent, *next = NULL;
aa27497c 1586 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 1587 int cpu_file = iter->cpu_file;
3928a8a2 1588 u64 next_ts = 0, ts;
bc0c38d1
SR
1589 int next_cpu = -1;
1590 int cpu;
1591
b04cc6b1
FW
1592 /*
1593 * If we are in a per_cpu trace file, don't bother by iterating over
1594 * all cpu and peek directly.
1595 */
1596 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1597 if (ring_buffer_empty_cpu(buffer, cpu_file))
1598 return NULL;
bc21b478 1599 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
1600 if (ent_cpu)
1601 *ent_cpu = cpu_file;
1602
1603 return ent;
1604 }
1605
ab46428c 1606 for_each_tracing_cpu(cpu) {
dd0e545f 1607
3928a8a2
SR
1608 if (ring_buffer_empty_cpu(buffer, cpu))
1609 continue;
dd0e545f 1610
bc21b478 1611 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 1612
cdd31cd2
IM
1613 /*
1614 * Pick the entry with the smallest timestamp:
1615 */
3928a8a2 1616 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
1617 next = ent;
1618 next_cpu = cpu;
3928a8a2 1619 next_ts = ts;
bc21b478 1620 next_lost = lost_events;
bc0c38d1
SR
1621 }
1622 }
1623
1624 if (ent_cpu)
1625 *ent_cpu = next_cpu;
1626
3928a8a2
SR
1627 if (ent_ts)
1628 *ent_ts = next_ts;
1629
bc21b478
SR
1630 if (missing_events)
1631 *missing_events = next_lost;
1632
bc0c38d1
SR
1633 return next;
1634}
1635
dd0e545f 1636/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
1637struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1638 int *ent_cpu, u64 *ent_ts)
bc0c38d1 1639{
bc21b478 1640 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
1641}
1642
1643/* Find the next real entry, and increment the iterator to the next entry */
1644static void *find_next_entry_inc(struct trace_iterator *iter)
1645{
bc21b478
SR
1646 iter->ent = __find_next_entry(iter, &iter->cpu,
1647 &iter->lost_events, &iter->ts);
dd0e545f 1648
3928a8a2 1649 if (iter->ent)
e2ac8ef5 1650 trace_iterator_increment(iter);
dd0e545f 1651
3928a8a2 1652 return iter->ent ? iter : NULL;
b3806b43 1653}
bc0c38d1 1654
e309b41d 1655static void trace_consume(struct trace_iterator *iter)
b3806b43 1656{
d769041f
SR
1657 /* Don't allow ftrace to trace into the ring buffers */
1658 ftrace_disable_cpu();
bc21b478
SR
1659 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1660 &iter->lost_events);
d769041f 1661 ftrace_enable_cpu();
bc0c38d1
SR
1662}
1663
e309b41d 1664static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
1665{
1666 struct trace_iterator *iter = m->private;
bc0c38d1 1667 int i = (int)*pos;
4e3c3333 1668 void *ent;
bc0c38d1 1669
a63ce5b3
SR
1670 WARN_ON_ONCE(iter->leftover);
1671
bc0c38d1
SR
1672 (*pos)++;
1673
1674 /* can't go backwards */
1675 if (iter->idx > i)
1676 return NULL;
1677
1678 if (iter->idx < 0)
1679 ent = find_next_entry_inc(iter);
1680 else
1681 ent = iter;
1682
1683 while (ent && iter->idx < i)
1684 ent = find_next_entry_inc(iter);
1685
1686 iter->pos = *pos;
1687
bc0c38d1
SR
1688 return ent;
1689}
1690
2f26ebd5
SR
1691static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1692{
1693 struct trace_array *tr = iter->tr;
1694 struct ring_buffer_event *event;
1695 struct ring_buffer_iter *buf_iter;
1696 unsigned long entries = 0;
1697 u64 ts;
1698
1699 tr->data[cpu]->skipped_entries = 0;
1700
1701 if (!iter->buffer_iter[cpu])
1702 return;
1703
1704 buf_iter = iter->buffer_iter[cpu];
1705 ring_buffer_iter_reset(buf_iter);
1706
1707 /*
1708 * We could have the case with the max latency tracers
1709 * that a reset never took place on a cpu. This is evident
1710 * by the timestamp being before the start of the buffer.
1711 */
1712 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1713 if (ts >= iter->tr->time_start)
1714 break;
1715 entries++;
1716 ring_buffer_read(buf_iter, NULL);
1717 }
1718
1719 tr->data[cpu]->skipped_entries = entries;
1720}
1721
d7350c3f 1722/*
d7350c3f
FW
1723 * The current tracer is copied to avoid a global locking
1724 * all around.
1725 */
bc0c38d1
SR
1726static void *s_start(struct seq_file *m, loff_t *pos)
1727{
1728 struct trace_iterator *iter = m->private;
d7350c3f 1729 static struct tracer *old_tracer;
b04cc6b1 1730 int cpu_file = iter->cpu_file;
bc0c38d1
SR
1731 void *p = NULL;
1732 loff_t l = 0;
3928a8a2 1733 int cpu;
bc0c38d1 1734
d7350c3f 1735 /* copy the tracer to avoid using a global lock all around */
bc0c38d1 1736 mutex_lock(&trace_types_lock);
d7350c3f
FW
1737 if (unlikely(old_tracer != current_trace && current_trace)) {
1738 old_tracer = current_trace;
1739 *iter->trace = *current_trace;
d15f57f2 1740 }
d7350c3f 1741 mutex_unlock(&trace_types_lock);
bc0c38d1
SR
1742
1743 atomic_inc(&trace_record_cmdline_disabled);
1744
bc0c38d1
SR
1745 if (*pos != iter->pos) {
1746 iter->ent = NULL;
1747 iter->cpu = 0;
1748 iter->idx = -1;
1749
d769041f
SR
1750 ftrace_disable_cpu();
1751
b04cc6b1
FW
1752 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1753 for_each_tracing_cpu(cpu)
2f26ebd5 1754 tracing_iter_reset(iter, cpu);
b04cc6b1 1755 } else
2f26ebd5 1756 tracing_iter_reset(iter, cpu_file);
bc0c38d1 1757
d769041f
SR
1758 ftrace_enable_cpu();
1759
ac91d854 1760 iter->leftover = 0;
bc0c38d1
SR
1761 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1762 ;
1763
1764 } else {
a63ce5b3
SR
1765 /*
1766 * If we overflowed the seq_file before, then we want
1767 * to just reuse the trace_seq buffer again.
1768 */
1769 if (iter->leftover)
1770 p = iter;
1771 else {
1772 l = *pos - 1;
1773 p = s_next(m, p, &l);
1774 }
bc0c38d1
SR
1775 }
1776
4f535968 1777 trace_event_read_lock();
7e53bd42 1778 trace_access_lock(cpu_file);
bc0c38d1
SR
1779 return p;
1780}
1781
1782static void s_stop(struct seq_file *m, void *p)
1783{
7e53bd42
LJ
1784 struct trace_iterator *iter = m->private;
1785
bc0c38d1 1786 atomic_dec(&trace_record_cmdline_disabled);
7e53bd42 1787 trace_access_unlock(iter->cpu_file);
4f535968 1788 trace_event_read_unlock();
bc0c38d1
SR
1789}
1790
e309b41d 1791static void print_lat_help_header(struct seq_file *m)
bc0c38d1 1792{
a6168353
ME
1793 seq_puts(m, "# _------=> CPU# \n");
1794 seq_puts(m, "# / _-----=> irqs-off \n");
1795 seq_puts(m, "# | / _----=> need-resched \n");
1796 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1797 seq_puts(m, "# ||| / _--=> preempt-depth \n");
637e7e86
SR
1798 seq_puts(m, "# |||| /_--=> lock-depth \n");
1799 seq_puts(m, "# |||||/ delay \n");
1800 seq_puts(m, "# cmd pid |||||| time | caller \n");
1801 seq_puts(m, "# \\ / |||||| \\ | / \n");
bc0c38d1
SR
1802}
1803
e309b41d 1804static void print_func_help_header(struct seq_file *m)
bc0c38d1 1805{
a6168353
ME
1806 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1807 seq_puts(m, "# | | | | |\n");
bc0c38d1
SR
1808}
1809
1810
62b915f1 1811void
bc0c38d1
SR
1812print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1813{
1814 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1815 struct trace_array *tr = iter->tr;
1816 struct trace_array_cpu *data = tr->data[tr->cpu];
1817 struct tracer *type = current_trace;
2f26ebd5
SR
1818 unsigned long entries = 0;
1819 unsigned long total = 0;
1820 unsigned long count;
bc0c38d1 1821 const char *name = "preemption";
2f26ebd5 1822 int cpu;
bc0c38d1
SR
1823
1824 if (type)
1825 name = type->name;
1826
2f26ebd5
SR
1827
1828 for_each_tracing_cpu(cpu) {
1829 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1830 /*
1831 * If this buffer has skipped entries, then we hold all
1832 * entries for the trace and we need to ignore the
1833 * ones before the time stamp.
1834 */
1835 if (tr->data[cpu]->skipped_entries) {
1836 count -= tr->data[cpu]->skipped_entries;
1837 /* total is the same as the entries */
1838 total += count;
1839 } else
1840 total += count +
1841 ring_buffer_overrun_cpu(tr->buffer, cpu);
1842 entries += count;
1843 }
bc0c38d1 1844
888b55dc 1845 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 1846 name, UTS_RELEASE);
888b55dc 1847 seq_puts(m, "# -----------------------------------"
bc0c38d1 1848 "---------------------------------\n");
888b55dc 1849 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 1850 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 1851 nsecs_to_usecs(data->saved_latency),
bc0c38d1 1852 entries,
4c11d7ae 1853 total,
bc0c38d1
SR
1854 tr->cpu,
1855#if defined(CONFIG_PREEMPT_NONE)
1856 "server",
1857#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1858 "desktop",
b5c21b45 1859#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
1860 "preempt",
1861#else
1862 "unknown",
1863#endif
1864 /* These are reserved for later use */
1865 0, 0, 0, 0);
1866#ifdef CONFIG_SMP
1867 seq_printf(m, " #P:%d)\n", num_online_cpus());
1868#else
1869 seq_puts(m, ")\n");
1870#endif
888b55dc
KM
1871 seq_puts(m, "# -----------------\n");
1872 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1
SR
1873 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1874 data->comm, data->pid, data->uid, data->nice,
1875 data->policy, data->rt_priority);
888b55dc 1876 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
1877
1878 if (data->critical_start) {
888b55dc 1879 seq_puts(m, "# => started at: ");
214023c3
SR
1880 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1881 trace_print_seq(m, &iter->seq);
888b55dc 1882 seq_puts(m, "\n# => ended at: ");
214023c3
SR
1883 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1884 trace_print_seq(m, &iter->seq);
8248ac05 1885 seq_puts(m, "\n#\n");
bc0c38d1
SR
1886 }
1887
888b55dc 1888 seq_puts(m, "#\n");
bc0c38d1
SR
1889}
1890
a309720c
SR
1891static void test_cpu_buff_start(struct trace_iterator *iter)
1892{
1893 struct trace_seq *s = &iter->seq;
1894
12ef7d44
SR
1895 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1896 return;
1897
1898 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1899 return;
1900
4462344e 1901 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
1902 return;
1903
2f26ebd5
SR
1904 if (iter->tr->data[iter->cpu]->skipped_entries)
1905 return;
1906
4462344e 1907 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
1908
1909 /* Don't print started cpu buffer for the first entry of the trace */
1910 if (iter->idx > 1)
1911 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1912 iter->cpu);
a309720c
SR
1913}
1914
2c4f035f 1915static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 1916{
214023c3 1917 struct trace_seq *s = &iter->seq;
bc0c38d1 1918 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 1919 struct trace_entry *entry;
f633cef0 1920 struct trace_event *event;
bc0c38d1 1921
4e3c3333 1922 entry = iter->ent;
dd0e545f 1923
a309720c
SR
1924 test_cpu_buff_start(iter);
1925
c4a8e8be 1926 event = ftrace_find_event(entry->type);
bc0c38d1 1927
c4a8e8be 1928 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
27d48be8
SR
1929 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1930 if (!trace_print_lat_context(iter))
1931 goto partial;
1932 } else {
1933 if (!trace_print_context(iter))
1934 goto partial;
1935 }
c4a8e8be 1936 }
bc0c38d1 1937
268ccda0 1938 if (event)
a9a57763 1939 return event->funcs->trace(iter, sym_flags, event);
d9793bd8
ACM
1940
1941 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1942 goto partial;
02b67518 1943
2c4f035f 1944 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
1945partial:
1946 return TRACE_TYPE_PARTIAL_LINE;
bc0c38d1
SR
1947}
1948
2c4f035f 1949static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
1950{
1951 struct trace_seq *s = &iter->seq;
1952 struct trace_entry *entry;
f633cef0 1953 struct trace_event *event;
f9896bf3
IM
1954
1955 entry = iter->ent;
dd0e545f 1956
c4a8e8be 1957 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
d9793bd8
ACM
1958 if (!trace_seq_printf(s, "%d %d %llu ",
1959 entry->pid, iter->cpu, iter->ts))
1960 goto partial;
c4a8e8be 1961 }
f9896bf3 1962
f633cef0 1963 event = ftrace_find_event(entry->type);
268ccda0 1964 if (event)
a9a57763 1965 return event->funcs->raw(iter, 0, event);
d9793bd8
ACM
1966
1967 if (!trace_seq_printf(s, "%d ?\n", entry->type))
1968 goto partial;
777e208d 1969
2c4f035f 1970 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
1971partial:
1972 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3
IM
1973}
1974
2c4f035f 1975static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
1976{
1977 struct trace_seq *s = &iter->seq;
1978 unsigned char newline = '\n';
1979 struct trace_entry *entry;
f633cef0 1980 struct trace_event *event;
5e3ca0ec
IM
1981
1982 entry = iter->ent;
dd0e545f 1983
c4a8e8be
FW
1984 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1985 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1986 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1987 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1988 }
5e3ca0ec 1989
f633cef0 1990 event = ftrace_find_event(entry->type);
268ccda0 1991 if (event) {
a9a57763 1992 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
1993 if (ret != TRACE_TYPE_HANDLED)
1994 return ret;
1995 }
7104f300 1996
5e3ca0ec
IM
1997 SEQ_PUT_FIELD_RET(s, newline);
1998
2c4f035f 1999 return TRACE_TYPE_HANDLED;
5e3ca0ec
IM
2000}
2001
2c4f035f 2002static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
2003{
2004 struct trace_seq *s = &iter->seq;
2005 struct trace_entry *entry;
f633cef0 2006 struct trace_event *event;
cb0f12aa
IM
2007
2008 entry = iter->ent;
dd0e545f 2009
c4a8e8be
FW
2010 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2011 SEQ_PUT_FIELD_RET(s, entry->pid);
1830b52d 2012 SEQ_PUT_FIELD_RET(s, iter->cpu);
c4a8e8be
FW
2013 SEQ_PUT_FIELD_RET(s, iter->ts);
2014 }
cb0f12aa 2015
f633cef0 2016 event = ftrace_find_event(entry->type);
a9a57763
SR
2017 return event ? event->funcs->binary(iter, 0, event) :
2018 TRACE_TYPE_HANDLED;
cb0f12aa
IM
2019}
2020
62b915f1 2021int trace_empty(struct trace_iterator *iter)
bc0c38d1 2022{
bc0c38d1
SR
2023 int cpu;
2024
9aba60fe
SR
2025 /* If we are looking at one CPU buffer, only check that one */
2026 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2027 cpu = iter->cpu_file;
2028 if (iter->buffer_iter[cpu]) {
2029 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2030 return 0;
2031 } else {
2032 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2033 return 0;
2034 }
2035 return 1;
2036 }
2037
ab46428c 2038 for_each_tracing_cpu(cpu) {
d769041f
SR
2039 if (iter->buffer_iter[cpu]) {
2040 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2041 return 0;
2042 } else {
2043 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2044 return 0;
2045 }
bc0c38d1 2046 }
d769041f 2047
797d3712 2048 return 1;
bc0c38d1
SR
2049}
2050
4f535968 2051/* Called with trace_event_read_lock() held. */
2c4f035f 2052static enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 2053{
2c4f035f
FW
2054 enum print_line_t ret;
2055
bc21b478
SR
2056 if (iter->lost_events)
2057 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2058 iter->cpu, iter->lost_events);
2059
2c4f035f
FW
2060 if (iter->trace && iter->trace->print_line) {
2061 ret = iter->trace->print_line(iter);
2062 if (ret != TRACE_TYPE_UNHANDLED)
2063 return ret;
2064 }
72829bc3 2065
48ead020
FW
2066 if (iter->ent->type == TRACE_BPRINT &&
2067 trace_flags & TRACE_ITER_PRINTK &&
2068 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2069 return trace_print_bprintk_msg_only(iter);
48ead020 2070
66896a85
FW
2071 if (iter->ent->type == TRACE_PRINT &&
2072 trace_flags & TRACE_ITER_PRINTK &&
2073 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2074 return trace_print_printk_msg_only(iter);
66896a85 2075
cb0f12aa
IM
2076 if (trace_flags & TRACE_ITER_BIN)
2077 return print_bin_fmt(iter);
2078
5e3ca0ec
IM
2079 if (trace_flags & TRACE_ITER_HEX)
2080 return print_hex_fmt(iter);
2081
f9896bf3
IM
2082 if (trace_flags & TRACE_ITER_RAW)
2083 return print_raw_fmt(iter);
2084
f9896bf3
IM
2085 return print_trace_fmt(iter);
2086}
2087
62b915f1
JO
2088void trace_default_header(struct seq_file *m)
2089{
2090 struct trace_iterator *iter = m->private;
2091
2092 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2093 /* print nothing if the buffers are empty */
2094 if (trace_empty(iter))
2095 return;
2096 print_trace_header(m, iter);
2097 if (!(trace_flags & TRACE_ITER_VERBOSE))
2098 print_lat_help_header(m);
2099 } else {
2100 if (!(trace_flags & TRACE_ITER_VERBOSE))
2101 print_func_help_header(m);
2102 }
2103}
2104
bc0c38d1
SR
2105static int s_show(struct seq_file *m, void *v)
2106{
2107 struct trace_iterator *iter = v;
a63ce5b3 2108 int ret;
bc0c38d1
SR
2109
2110 if (iter->ent == NULL) {
2111 if (iter->tr) {
2112 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2113 seq_puts(m, "#\n");
2114 }
8bba1bf5
MM
2115 if (iter->trace && iter->trace->print_header)
2116 iter->trace->print_header(m);
62b915f1
JO
2117 else
2118 trace_default_header(m);
2119
a63ce5b3
SR
2120 } else if (iter->leftover) {
2121 /*
2122 * If we filled the seq_file buffer earlier, we
2123 * want to just show it now.
2124 */
2125 ret = trace_print_seq(m, &iter->seq);
2126
2127 /* ret should this time be zero, but you never know */
2128 iter->leftover = ret;
2129
bc0c38d1 2130 } else {
f9896bf3 2131 print_trace_line(iter);
a63ce5b3
SR
2132 ret = trace_print_seq(m, &iter->seq);
2133 /*
2134 * If we overflow the seq_file buffer, then it will
2135 * ask us for this data again at start up.
2136 * Use that instead.
2137 * ret is 0 if seq_file write succeeded.
2138 * -1 otherwise.
2139 */
2140 iter->leftover = ret;
bc0c38d1
SR
2141 }
2142
2143 return 0;
2144}
2145
88e9d34c 2146static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
2147 .start = s_start,
2148 .next = s_next,
2149 .stop = s_stop,
2150 .show = s_show,
bc0c38d1
SR
2151};
2152
e309b41d 2153static struct trace_iterator *
85a2f9b4 2154__tracing_open(struct inode *inode, struct file *file)
bc0c38d1 2155{
b04cc6b1 2156 long cpu_file = (long) inode->i_private;
85a2f9b4 2157 void *fail_ret = ERR_PTR(-ENOMEM);
bc0c38d1 2158 struct trace_iterator *iter;
3928a8a2 2159 struct seq_file *m;
85a2f9b4 2160 int cpu, ret;
bc0c38d1 2161
85a2f9b4
SR
2162 if (tracing_disabled)
2163 return ERR_PTR(-ENODEV);
60a11774 2164
bc0c38d1 2165 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
85a2f9b4
SR
2166 if (!iter)
2167 return ERR_PTR(-ENOMEM);
bc0c38d1 2168
d7350c3f
FW
2169 /*
2170 * We make a copy of the current tracer to avoid concurrent
2171 * changes on it while we are reading.
2172 */
bc0c38d1 2173 mutex_lock(&trace_types_lock);
d7350c3f 2174 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 2175 if (!iter->trace)
d7350c3f 2176 goto fail;
85a2f9b4 2177
d7350c3f
FW
2178 if (current_trace)
2179 *iter->trace = *current_trace;
2180
79f55997 2181 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
2182 goto fail;
2183
bc0c38d1
SR
2184 if (current_trace && current_trace->print_max)
2185 iter->tr = &max_tr;
2186 else
b04cc6b1 2187 iter->tr = &global_trace;
bc0c38d1 2188 iter->pos = -1;
d7350c3f 2189 mutex_init(&iter->mutex);
b04cc6b1 2190 iter->cpu_file = cpu_file;
bc0c38d1 2191
8bba1bf5
MM
2192 /* Notify the tracer early; before we stop tracing. */
2193 if (iter->trace && iter->trace->open)
a93751ca 2194 iter->trace->open(iter);
8bba1bf5 2195
12ef7d44
SR
2196 /* Annotate start of buffers if we had overruns */
2197 if (ring_buffer_overruns(iter->tr->buffer))
2198 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2199
2f26ebd5
SR
2200 /* stop the trace while dumping */
2201 tracing_stop();
2202
b04cc6b1
FW
2203 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2204 for_each_tracing_cpu(cpu) {
b04cc6b1 2205 iter->buffer_iter[cpu] =
72c9ddfd
DM
2206 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2207 }
2208 ring_buffer_read_prepare_sync();
2209 for_each_tracing_cpu(cpu) {
2210 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2211 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
2212 }
2213 } else {
2214 cpu = iter->cpu_file;
3928a8a2 2215 iter->buffer_iter[cpu] =
72c9ddfd
DM
2216 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2217 ring_buffer_read_prepare_sync();
2218 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2219 tracing_iter_reset(iter, cpu);
3928a8a2
SR
2220 }
2221
85a2f9b4
SR
2222 ret = seq_open(file, &tracer_seq_ops);
2223 if (ret < 0) {
2224 fail_ret = ERR_PTR(ret);
3928a8a2 2225 goto fail_buffer;
85a2f9b4 2226 }
bc0c38d1 2227
3928a8a2
SR
2228 m = file->private_data;
2229 m->private = iter;
bc0c38d1 2230
bc0c38d1
SR
2231 mutex_unlock(&trace_types_lock);
2232
bc0c38d1 2233 return iter;
3928a8a2
SR
2234
2235 fail_buffer:
2236 for_each_tracing_cpu(cpu) {
2237 if (iter->buffer_iter[cpu])
2238 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2239 }
b0dfa978 2240 free_cpumask_var(iter->started);
2f26ebd5 2241 tracing_start();
d7350c3f 2242 fail:
3928a8a2 2243 mutex_unlock(&trace_types_lock);
d7350c3f 2244 kfree(iter->trace);
0bb943c7 2245 kfree(iter);
3928a8a2 2246
85a2f9b4 2247 return fail_ret;
bc0c38d1
SR
2248}
2249
2250int tracing_open_generic(struct inode *inode, struct file *filp)
2251{
60a11774
SR
2252 if (tracing_disabled)
2253 return -ENODEV;
2254
bc0c38d1
SR
2255 filp->private_data = inode->i_private;
2256 return 0;
2257}
2258
4fd27358 2259static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1
SR
2260{
2261 struct seq_file *m = (struct seq_file *)file->private_data;
4acd4d00 2262 struct trace_iterator *iter;
3928a8a2 2263 int cpu;
bc0c38d1 2264
4acd4d00
SR
2265 if (!(file->f_mode & FMODE_READ))
2266 return 0;
2267
2268 iter = m->private;
2269
bc0c38d1 2270 mutex_lock(&trace_types_lock);
3928a8a2
SR
2271 for_each_tracing_cpu(cpu) {
2272 if (iter->buffer_iter[cpu])
2273 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2274 }
2275
bc0c38d1
SR
2276 if (iter->trace && iter->trace->close)
2277 iter->trace->close(iter);
2278
2279 /* reenable tracing if it was previously enabled */
9036990d 2280 tracing_start();
bc0c38d1
SR
2281 mutex_unlock(&trace_types_lock);
2282
2283 seq_release(inode, file);
d7350c3f 2284 mutex_destroy(&iter->mutex);
b0dfa978 2285 free_cpumask_var(iter->started);
d7350c3f 2286 kfree(iter->trace);
bc0c38d1
SR
2287 kfree(iter);
2288 return 0;
2289}
2290
2291static int tracing_open(struct inode *inode, struct file *file)
2292{
85a2f9b4
SR
2293 struct trace_iterator *iter;
2294 int ret = 0;
bc0c38d1 2295
4acd4d00
SR
2296 /* If this file was open for write, then erase contents */
2297 if ((file->f_mode & FMODE_WRITE) &&
8650ae32 2298 (file->f_flags & O_TRUNC)) {
4acd4d00 2299 long cpu = (long) inode->i_private;
bc0c38d1 2300
4acd4d00
SR
2301 if (cpu == TRACE_PIPE_ALL_CPU)
2302 tracing_reset_online_cpus(&global_trace);
2303 else
2304 tracing_reset(&global_trace, cpu);
2305 }
bc0c38d1 2306
4acd4d00
SR
2307 if (file->f_mode & FMODE_READ) {
2308 iter = __tracing_open(inode, file);
2309 if (IS_ERR(iter))
2310 ret = PTR_ERR(iter);
2311 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2312 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2313 }
bc0c38d1
SR
2314 return ret;
2315}
2316
e309b41d 2317static void *
bc0c38d1
SR
2318t_next(struct seq_file *m, void *v, loff_t *pos)
2319{
f129e965 2320 struct tracer *t = v;
bc0c38d1
SR
2321
2322 (*pos)++;
2323
2324 if (t)
2325 t = t->next;
2326
bc0c38d1
SR
2327 return t;
2328}
2329
2330static void *t_start(struct seq_file *m, loff_t *pos)
2331{
f129e965 2332 struct tracer *t;
bc0c38d1
SR
2333 loff_t l = 0;
2334
2335 mutex_lock(&trace_types_lock);
f129e965 2336 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
bc0c38d1
SR
2337 ;
2338
2339 return t;
2340}
2341
2342static void t_stop(struct seq_file *m, void *p)
2343{
2344 mutex_unlock(&trace_types_lock);
2345}
2346
2347static int t_show(struct seq_file *m, void *v)
2348{
2349 struct tracer *t = v;
2350
2351 if (!t)
2352 return 0;
2353
2354 seq_printf(m, "%s", t->name);
2355 if (t->next)
2356 seq_putc(m, ' ');
2357 else
2358 seq_putc(m, '\n');
2359
2360 return 0;
2361}
2362
88e9d34c 2363static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
2364 .start = t_start,
2365 .next = t_next,
2366 .stop = t_stop,
2367 .show = t_show,
bc0c38d1
SR
2368};
2369
2370static int show_traces_open(struct inode *inode, struct file *file)
2371{
60a11774
SR
2372 if (tracing_disabled)
2373 return -ENODEV;
2374
f129e965 2375 return seq_open(file, &show_traces_seq_ops);
bc0c38d1
SR
2376}
2377
4acd4d00
SR
2378static ssize_t
2379tracing_write_stub(struct file *filp, const char __user *ubuf,
2380 size_t count, loff_t *ppos)
2381{
2382 return count;
2383}
2384
5e2336a0 2385static const struct file_operations tracing_fops = {
4bf39a94
IM
2386 .open = tracing_open,
2387 .read = seq_read,
4acd4d00 2388 .write = tracing_write_stub,
4bf39a94
IM
2389 .llseek = seq_lseek,
2390 .release = tracing_release,
bc0c38d1
SR
2391};
2392
5e2336a0 2393static const struct file_operations show_traces_fops = {
c7078de1
IM
2394 .open = show_traces_open,
2395 .read = seq_read,
2396 .release = seq_release,
2397};
2398
36dfe925
IM
2399/*
2400 * Only trace on a CPU if the bitmask is set:
2401 */
9e01c1b7 2402static cpumask_var_t tracing_cpumask;
36dfe925
IM
2403
2404/*
2405 * The tracer itself will not take this lock, but still we want
2406 * to provide a consistent cpumask to user-space:
2407 */
2408static DEFINE_MUTEX(tracing_cpumask_update_lock);
2409
2410/*
2411 * Temporary storage for the character representation of the
2412 * CPU bitmask (and one more byte for the newline):
2413 */
2414static char mask_str[NR_CPUS + 1];
2415
c7078de1
IM
2416static ssize_t
2417tracing_cpumask_read(struct file *filp, char __user *ubuf,
2418 size_t count, loff_t *ppos)
2419{
36dfe925 2420 int len;
c7078de1
IM
2421
2422 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 2423
9e01c1b7 2424 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
36dfe925
IM
2425 if (count - len < 2) {
2426 count = -EINVAL;
2427 goto out_err;
2428 }
2429 len += sprintf(mask_str + len, "\n");
2430 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2431
2432out_err:
c7078de1
IM
2433 mutex_unlock(&tracing_cpumask_update_lock);
2434
2435 return count;
2436}
2437
2438static ssize_t
2439tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2440 size_t count, loff_t *ppos)
2441{
36dfe925 2442 int err, cpu;
9e01c1b7
RR
2443 cpumask_var_t tracing_cpumask_new;
2444
2445 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2446 return -ENOMEM;
c7078de1 2447
9e01c1b7 2448 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 2449 if (err)
36dfe925
IM
2450 goto err_unlock;
2451
215368e8
LZ
2452 mutex_lock(&tracing_cpumask_update_lock);
2453
a5e25883 2454 local_irq_disable();
0199c4e6 2455 arch_spin_lock(&ftrace_max_lock);
ab46428c 2456 for_each_tracing_cpu(cpu) {
36dfe925
IM
2457 /*
2458 * Increase/decrease the disabled counter if we are
2459 * about to flip a bit in the cpumask:
2460 */
9e01c1b7
RR
2461 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2462 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
36dfe925
IM
2463 atomic_inc(&global_trace.data[cpu]->disabled);
2464 }
9e01c1b7
RR
2465 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2466 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
36dfe925
IM
2467 atomic_dec(&global_trace.data[cpu]->disabled);
2468 }
2469 }
0199c4e6 2470 arch_spin_unlock(&ftrace_max_lock);
a5e25883 2471 local_irq_enable();
36dfe925 2472
9e01c1b7 2473 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
2474
2475 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 2476 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
2477
2478 return count;
36dfe925
IM
2479
2480err_unlock:
215368e8 2481 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
2482
2483 return err;
c7078de1
IM
2484}
2485
5e2336a0 2486static const struct file_operations tracing_cpumask_fops = {
c7078de1
IM
2487 .open = tracing_open_generic,
2488 .read = tracing_cpumask_read,
2489 .write = tracing_cpumask_write,
bc0c38d1
SR
2490};
2491
fdb372ed 2492static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 2493{
d8e83d26
SR
2494 struct tracer_opt *trace_opts;
2495 u32 tracer_flags;
d8e83d26 2496 int i;
adf9f195 2497
d8e83d26
SR
2498 mutex_lock(&trace_types_lock);
2499 tracer_flags = current_trace->flags->val;
2500 trace_opts = current_trace->flags->opts;
2501
bc0c38d1
SR
2502 for (i = 0; trace_options[i]; i++) {
2503 if (trace_flags & (1 << i))
fdb372ed 2504 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 2505 else
fdb372ed 2506 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
2507 }
2508
adf9f195
FW
2509 for (i = 0; trace_opts[i].name; i++) {
2510 if (tracer_flags & trace_opts[i].bit)
fdb372ed 2511 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 2512 else
fdb372ed 2513 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 2514 }
d8e83d26 2515 mutex_unlock(&trace_types_lock);
adf9f195 2516
fdb372ed 2517 return 0;
bc0c38d1 2518}
bc0c38d1 2519
8d18eaaf
LZ
2520static int __set_tracer_option(struct tracer *trace,
2521 struct tracer_flags *tracer_flags,
2522 struct tracer_opt *opts, int neg)
2523{
2524 int ret;
bc0c38d1 2525
8d18eaaf
LZ
2526 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2527 if (ret)
2528 return ret;
2529
2530 if (neg)
2531 tracer_flags->val &= ~opts->bit;
2532 else
2533 tracer_flags->val |= opts->bit;
2534 return 0;
bc0c38d1
SR
2535}
2536
adf9f195
FW
2537/* Try to assign a tracer specific option */
2538static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2539{
7770841e 2540 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 2541 struct tracer_opt *opts = NULL;
8d18eaaf 2542 int i;
adf9f195 2543
7770841e
Z
2544 for (i = 0; tracer_flags->opts[i].name; i++) {
2545 opts = &tracer_flags->opts[i];
adf9f195 2546
8d18eaaf
LZ
2547 if (strcmp(cmp, opts->name) == 0)
2548 return __set_tracer_option(trace, trace->flags,
2549 opts, neg);
adf9f195 2550 }
adf9f195 2551
8d18eaaf 2552 return -EINVAL;
adf9f195
FW
2553}
2554
af4617bd
SR
2555static void set_tracer_flags(unsigned int mask, int enabled)
2556{
2557 /* do nothing if flag is already set */
2558 if (!!(trace_flags & mask) == !!enabled)
2559 return;
2560
2561 if (enabled)
2562 trace_flags |= mask;
2563 else
2564 trace_flags &= ~mask;
af4617bd
SR
2565}
2566
bc0c38d1 2567static ssize_t
ee6bce52 2568tracing_trace_options_write(struct file *filp, const char __user *ubuf,
bc0c38d1
SR
2569 size_t cnt, loff_t *ppos)
2570{
2571 char buf[64];
8d18eaaf 2572 char *cmp;
bc0c38d1 2573 int neg = 0;
adf9f195 2574 int ret;
bc0c38d1
SR
2575 int i;
2576
cffae437
SR
2577 if (cnt >= sizeof(buf))
2578 return -EINVAL;
bc0c38d1
SR
2579
2580 if (copy_from_user(&buf, ubuf, cnt))
2581 return -EFAULT;
2582
2583 buf[cnt] = 0;
8d18eaaf 2584 cmp = strstrip(buf);
bc0c38d1 2585
8d18eaaf 2586 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
2587 neg = 1;
2588 cmp += 2;
2589 }
2590
2591 for (i = 0; trace_options[i]; i++) {
8d18eaaf 2592 if (strcmp(cmp, trace_options[i]) == 0) {
af4617bd 2593 set_tracer_flags(1 << i, !neg);
bc0c38d1
SR
2594 break;
2595 }
2596 }
adf9f195
FW
2597
2598 /* If no option could be set, test the specific tracer options */
2599 if (!trace_options[i]) {
d8e83d26 2600 mutex_lock(&trace_types_lock);
adf9f195 2601 ret = set_tracer_option(current_trace, cmp, neg);
d8e83d26 2602 mutex_unlock(&trace_types_lock);
adf9f195
FW
2603 if (ret)
2604 return ret;
2605 }
bc0c38d1 2606
cf8517cf 2607 *ppos += cnt;
bc0c38d1
SR
2608
2609 return cnt;
2610}
2611
fdb372ed
LZ
2612static int tracing_trace_options_open(struct inode *inode, struct file *file)
2613{
2614 if (tracing_disabled)
2615 return -ENODEV;
2616 return single_open(file, tracing_trace_options_show, NULL);
2617}
2618
5e2336a0 2619static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
2620 .open = tracing_trace_options_open,
2621 .read = seq_read,
2622 .llseek = seq_lseek,
2623 .release = single_release,
ee6bce52 2624 .write = tracing_trace_options_write,
bc0c38d1
SR
2625};
2626
7bd2f24c
IM
2627static const char readme_msg[] =
2628 "tracing mini-HOWTO:\n\n"
156f5a78
GL
2629 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2630 "# cat /sys/kernel/debug/tracing/available_tracers\n"
bc2b6871 2631 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
156f5a78 2632 "# cat /sys/kernel/debug/tracing/current_tracer\n"
bc2b6871 2633 "nop\n"
156f5a78
GL
2634 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2635 "# cat /sys/kernel/debug/tracing/current_tracer\n"
7bd2f24c 2636 "sched_switch\n"
156f5a78 2637 "# cat /sys/kernel/debug/tracing/trace_options\n"
7bd2f24c 2638 "noprint-parent nosym-offset nosym-addr noverbose\n"
156f5a78
GL
2639 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2640 "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2641 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2642 "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
7bd2f24c
IM
2643;
2644
2645static ssize_t
2646tracing_readme_read(struct file *filp, char __user *ubuf,
2647 size_t cnt, loff_t *ppos)
2648{
2649 return simple_read_from_buffer(ubuf, cnt, ppos,
2650 readme_msg, strlen(readme_msg));
2651}
2652
5e2336a0 2653static const struct file_operations tracing_readme_fops = {
c7078de1
IM
2654 .open = tracing_open_generic,
2655 .read = tracing_readme_read,
7bd2f24c
IM
2656};
2657
69abe6a5
AP
2658static ssize_t
2659tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2660 size_t cnt, loff_t *ppos)
2661{
2662 char *buf_comm;
2663 char *file_buf;
2664 char *buf;
2665 int len = 0;
2666 int pid;
2667 int i;
2668
2669 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2670 if (!file_buf)
2671 return -ENOMEM;
2672
2673 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2674 if (!buf_comm) {
2675 kfree(file_buf);
2676 return -ENOMEM;
2677 }
2678
2679 buf = file_buf;
2680
2681 for (i = 0; i < SAVED_CMDLINES; i++) {
2682 int r;
2683
2684 pid = map_cmdline_to_pid[i];
2685 if (pid == -1 || pid == NO_CMDLINE_MAP)
2686 continue;
2687
2688 trace_find_cmdline(pid, buf_comm);
2689 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2690 buf += r;
2691 len += r;
2692 }
2693
2694 len = simple_read_from_buffer(ubuf, cnt, ppos,
2695 file_buf, len);
2696
2697 kfree(file_buf);
2698 kfree(buf_comm);
2699
2700 return len;
2701}
2702
2703static const struct file_operations tracing_saved_cmdlines_fops = {
2704 .open = tracing_open_generic,
2705 .read = tracing_saved_cmdlines_read,
2706};
2707
bc0c38d1
SR
2708static ssize_t
2709tracing_ctrl_read(struct file *filp, char __user *ubuf,
2710 size_t cnt, loff_t *ppos)
2711{
bc0c38d1
SR
2712 char buf[64];
2713 int r;
2714
9036990d 2715 r = sprintf(buf, "%u\n", tracer_enabled);
4e3c3333 2716 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2717}
2718
2719static ssize_t
2720tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2721 size_t cnt, loff_t *ppos)
2722{
2723 struct trace_array *tr = filp->private_data;
bc0c38d1 2724 char buf[64];
5e39841c 2725 unsigned long val;
c6caeeb1 2726 int ret;
bc0c38d1 2727
cffae437
SR
2728 if (cnt >= sizeof(buf))
2729 return -EINVAL;
bc0c38d1
SR
2730
2731 if (copy_from_user(&buf, ubuf, cnt))
2732 return -EFAULT;
2733
2734 buf[cnt] = 0;
2735
c6caeeb1
SR
2736 ret = strict_strtoul(buf, 10, &val);
2737 if (ret < 0)
2738 return ret;
bc0c38d1
SR
2739
2740 val = !!val;
2741
2742 mutex_lock(&trace_types_lock);
9036990d
SR
2743 if (tracer_enabled ^ val) {
2744 if (val) {
bc0c38d1 2745 tracer_enabled = 1;
9036990d
SR
2746 if (current_trace->start)
2747 current_trace->start(tr);
2748 tracing_start();
2749 } else {
bc0c38d1 2750 tracer_enabled = 0;
9036990d
SR
2751 tracing_stop();
2752 if (current_trace->stop)
2753 current_trace->stop(tr);
2754 }
bc0c38d1
SR
2755 }
2756 mutex_unlock(&trace_types_lock);
2757
cf8517cf 2758 *ppos += cnt;
bc0c38d1
SR
2759
2760 return cnt;
2761}
2762
2763static ssize_t
2764tracing_set_trace_read(struct file *filp, char __user *ubuf,
2765 size_t cnt, loff_t *ppos)
2766{
ee6c2c1b 2767 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
2768 int r;
2769
2770 mutex_lock(&trace_types_lock);
2771 if (current_trace)
2772 r = sprintf(buf, "%s\n", current_trace->name);
2773 else
2774 r = sprintf(buf, "\n");
2775 mutex_unlock(&trace_types_lock);
2776
4bf39a94 2777 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2778}
2779
b6f11df2
ACM
2780int tracer_init(struct tracer *t, struct trace_array *tr)
2781{
2782 tracing_reset_online_cpus(tr);
2783 return t->init(tr);
2784}
2785
73c5162a
SR
2786static int tracing_resize_ring_buffer(unsigned long size)
2787{
2788 int ret;
2789
2790 /*
2791 * If kernel or user changes the size of the ring buffer
a123c52b
SR
2792 * we use the size that was given, and we can forget about
2793 * expanding it later.
73c5162a
SR
2794 */
2795 ring_buffer_expanded = 1;
2796
2797 ret = ring_buffer_resize(global_trace.buffer, size);
2798 if (ret < 0)
2799 return ret;
2800
2801 ret = ring_buffer_resize(max_tr.buffer, size);
2802 if (ret < 0) {
2803 int r;
2804
2805 r = ring_buffer_resize(global_trace.buffer,
2806 global_trace.entries);
2807 if (r < 0) {
a123c52b
SR
2808 /*
2809 * AARGH! We are left with different
2810 * size max buffer!!!!
2811 * The max buffer is our "snapshot" buffer.
2812 * When a tracer needs a snapshot (one of the
2813 * latency tracers), it swaps the max buffer
2814 * with the saved snap shot. We succeeded to
2815 * update the size of the main buffer, but failed to
2816 * update the size of the max buffer. But when we tried
2817 * to reset the main buffer to the original size, we
2818 * failed there too. This is very unlikely to
2819 * happen, but if it does, warn and kill all
2820 * tracing.
2821 */
73c5162a
SR
2822 WARN_ON(1);
2823 tracing_disabled = 1;
2824 }
2825 return ret;
2826 }
2827
2828 global_trace.entries = size;
2829
2830 return ret;
2831}
2832
1852fcce
SR
2833/**
2834 * tracing_update_buffers - used by tracing facility to expand ring buffers
2835 *
2836 * To save on memory when the tracing is never used on a system with it
2837 * configured in. The ring buffers are set to a minimum size. But once
2838 * a user starts to use the tracing facility, then they need to grow
2839 * to their default size.
2840 *
2841 * This function is to be called when a tracer is about to be used.
2842 */
2843int tracing_update_buffers(void)
2844{
2845 int ret = 0;
2846
1027fcb2 2847 mutex_lock(&trace_types_lock);
1852fcce
SR
2848 if (!ring_buffer_expanded)
2849 ret = tracing_resize_ring_buffer(trace_buf_size);
1027fcb2 2850 mutex_unlock(&trace_types_lock);
1852fcce
SR
2851
2852 return ret;
2853}
2854
577b785f
SR
2855struct trace_option_dentry;
2856
2857static struct trace_option_dentry *
2858create_trace_option_files(struct tracer *tracer);
2859
2860static void
2861destroy_trace_option_files(struct trace_option_dentry *topts);
2862
b2821ae6 2863static int tracing_set_tracer(const char *buf)
bc0c38d1 2864{
577b785f 2865 static struct trace_option_dentry *topts;
bc0c38d1
SR
2866 struct trace_array *tr = &global_trace;
2867 struct tracer *t;
d9e54076 2868 int ret = 0;
bc0c38d1 2869
1027fcb2
SR
2870 mutex_lock(&trace_types_lock);
2871
73c5162a
SR
2872 if (!ring_buffer_expanded) {
2873 ret = tracing_resize_ring_buffer(trace_buf_size);
2874 if (ret < 0)
59f586db 2875 goto out;
73c5162a
SR
2876 ret = 0;
2877 }
2878
bc0c38d1
SR
2879 for (t = trace_types; t; t = t->next) {
2880 if (strcmp(t->name, buf) == 0)
2881 break;
2882 }
c2931e05
FW
2883 if (!t) {
2884 ret = -EINVAL;
2885 goto out;
2886 }
2887 if (t == current_trace)
bc0c38d1
SR
2888 goto out;
2889
9f029e83 2890 trace_branch_disable();
bc0c38d1
SR
2891 if (current_trace && current_trace->reset)
2892 current_trace->reset(tr);
2893
577b785f
SR
2894 destroy_trace_option_files(topts);
2895
bc0c38d1 2896 current_trace = t;
577b785f
SR
2897
2898 topts = create_trace_option_files(current_trace);
2899
1c80025a 2900 if (t->init) {
b6f11df2 2901 ret = tracer_init(t, tr);
1c80025a
FW
2902 if (ret)
2903 goto out;
2904 }
bc0c38d1 2905
9f029e83 2906 trace_branch_enable(tr);
bc0c38d1
SR
2907 out:
2908 mutex_unlock(&trace_types_lock);
2909
d9e54076
PZ
2910 return ret;
2911}
2912
2913static ssize_t
2914tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2915 size_t cnt, loff_t *ppos)
2916{
ee6c2c1b 2917 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
2918 int i;
2919 size_t ret;
e6e7a65a
FW
2920 int err;
2921
2922 ret = cnt;
d9e54076 2923
ee6c2c1b
LZ
2924 if (cnt > MAX_TRACER_SIZE)
2925 cnt = MAX_TRACER_SIZE;
d9e54076
PZ
2926
2927 if (copy_from_user(&buf, ubuf, cnt))
2928 return -EFAULT;
2929
2930 buf[cnt] = 0;
2931
2932 /* strip ending whitespace. */
2933 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2934 buf[i] = 0;
2935
e6e7a65a
FW
2936 err = tracing_set_tracer(buf);
2937 if (err)
2938 return err;
d9e54076 2939
cf8517cf 2940 *ppos += ret;
bc0c38d1 2941
c2931e05 2942 return ret;
bc0c38d1
SR
2943}
2944
2945static ssize_t
2946tracing_max_lat_read(struct file *filp, char __user *ubuf,
2947 size_t cnt, loff_t *ppos)
2948{
2949 unsigned long *ptr = filp->private_data;
2950 char buf[64];
2951 int r;
2952
cffae437 2953 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 2954 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
2955 if (r > sizeof(buf))
2956 r = sizeof(buf);
4bf39a94 2957 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
2958}
2959
2960static ssize_t
2961tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2962 size_t cnt, loff_t *ppos)
2963{
5e39841c 2964 unsigned long *ptr = filp->private_data;
bc0c38d1 2965 char buf[64];
5e39841c 2966 unsigned long val;
c6caeeb1 2967 int ret;
bc0c38d1 2968
cffae437
SR
2969 if (cnt >= sizeof(buf))
2970 return -EINVAL;
bc0c38d1
SR
2971
2972 if (copy_from_user(&buf, ubuf, cnt))
2973 return -EFAULT;
2974
2975 buf[cnt] = 0;
2976
c6caeeb1
SR
2977 ret = strict_strtoul(buf, 10, &val);
2978 if (ret < 0)
2979 return ret;
bc0c38d1
SR
2980
2981 *ptr = val * 1000;
2982
2983 return cnt;
2984}
2985
b3806b43
SR
2986static int tracing_open_pipe(struct inode *inode, struct file *filp)
2987{
b04cc6b1 2988 long cpu_file = (long) inode->i_private;
b3806b43 2989 struct trace_iterator *iter;
b04cc6b1 2990 int ret = 0;
b3806b43
SR
2991
2992 if (tracing_disabled)
2993 return -ENODEV;
2994
b04cc6b1
FW
2995 mutex_lock(&trace_types_lock);
2996
b3806b43
SR
2997 /* create a buffer to store the information to pass to userspace */
2998 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
2999 if (!iter) {
3000 ret = -ENOMEM;
3001 goto out;
3002 }
b3806b43 3003
d7350c3f
FW
3004 /*
3005 * We make a copy of the current tracer to avoid concurrent
3006 * changes on it while we are reading.
3007 */
3008 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3009 if (!iter->trace) {
3010 ret = -ENOMEM;
3011 goto fail;
3012 }
3013 if (current_trace)
3014 *iter->trace = *current_trace;
3015
4462344e 3016 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 3017 ret = -ENOMEM;
d7350c3f 3018 goto fail;
4462344e
RR
3019 }
3020
a309720c 3021 /* trace pipe does not show start of buffer */
4462344e 3022 cpumask_setall(iter->started);
a309720c 3023
112f38a7
SR
3024 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3025 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3026
b04cc6b1 3027 iter->cpu_file = cpu_file;
b3806b43 3028 iter->tr = &global_trace;
d7350c3f 3029 mutex_init(&iter->mutex);
b3806b43
SR
3030 filp->private_data = iter;
3031
107bad8b
SR
3032 if (iter->trace->pipe_open)
3033 iter->trace->pipe_open(iter);
107bad8b 3034
b04cc6b1
FW
3035out:
3036 mutex_unlock(&trace_types_lock);
3037 return ret;
d7350c3f
FW
3038
3039fail:
3040 kfree(iter->trace);
3041 kfree(iter);
3042 mutex_unlock(&trace_types_lock);
3043 return ret;
b3806b43
SR
3044}
3045
3046static int tracing_release_pipe(struct inode *inode, struct file *file)
3047{
3048 struct trace_iterator *iter = file->private_data;
3049
b04cc6b1
FW
3050 mutex_lock(&trace_types_lock);
3051
29bf4a5e 3052 if (iter->trace->pipe_close)
c521efd1
SR
3053 iter->trace->pipe_close(iter);
3054
b04cc6b1
FW
3055 mutex_unlock(&trace_types_lock);
3056
4462344e 3057 free_cpumask_var(iter->started);
d7350c3f
FW
3058 mutex_destroy(&iter->mutex);
3059 kfree(iter->trace);
b3806b43 3060 kfree(iter);
b3806b43
SR
3061
3062 return 0;
3063}
3064
2a2cc8f7
SSP
3065static unsigned int
3066tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3067{
3068 struct trace_iterator *iter = filp->private_data;
3069
3070 if (trace_flags & TRACE_ITER_BLOCK) {
3071 /*
3072 * Always select as readable when in blocking mode
3073 */
3074 return POLLIN | POLLRDNORM;
afc2abc0 3075 } else {
2a2cc8f7
SSP
3076 if (!trace_empty(iter))
3077 return POLLIN | POLLRDNORM;
3078 poll_wait(filp, &trace_wait, poll_table);
3079 if (!trace_empty(iter))
3080 return POLLIN | POLLRDNORM;
3081
3082 return 0;
3083 }
3084}
3085
6eaaa5d5
FW
3086
3087void default_wait_pipe(struct trace_iterator *iter)
3088{
3089 DEFINE_WAIT(wait);
3090
3091 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3092
3093 if (trace_empty(iter))
3094 schedule();
3095
3096 finish_wait(&trace_wait, &wait);
3097}
3098
3099/*
3100 * This is a make-shift waitqueue.
3101 * A tracer might use this callback on some rare cases:
3102 *
3103 * 1) the current tracer might hold the runqueue lock when it wakes up
3104 * a reader, hence a deadlock (sched, function, and function graph tracers)
3105 * 2) the function tracers, trace all functions, we don't want
3106 * the overhead of calling wake_up and friends
3107 * (and tracing them too)
3108 *
3109 * Anyway, this is really very primitive wakeup.
3110 */
3111void poll_wait_pipe(struct trace_iterator *iter)
3112{
3113 set_current_state(TASK_INTERRUPTIBLE);
3114 /* sleep for 100 msecs, and try again. */
3115 schedule_timeout(HZ / 10);
3116}
3117
ff98781b
EGM
3118/* Must be called with trace_types_lock mutex held. */
3119static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
3120{
3121 struct trace_iterator *iter = filp->private_data;
b3806b43 3122
b3806b43 3123 while (trace_empty(iter)) {
2dc8f095 3124
107bad8b 3125 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 3126 return -EAGAIN;
107bad8b 3127 }
2dc8f095 3128
d7350c3f 3129 mutex_unlock(&iter->mutex);
107bad8b 3130
6eaaa5d5 3131 iter->trace->wait_pipe(iter);
b3806b43 3132
d7350c3f 3133 mutex_lock(&iter->mutex);
107bad8b 3134
6eaaa5d5 3135 if (signal_pending(current))
ff98781b 3136 return -EINTR;
b3806b43
SR
3137
3138 /*
3139 * We block until we read something and tracing is disabled.
3140 * We still block if tracing is disabled, but we have never
3141 * read anything. This allows a user to cat this file, and
3142 * then enable tracing. But after we have read something,
3143 * we give an EOF when tracing is again disabled.
3144 *
3145 * iter->pos will be 0 if we haven't read anything.
3146 */
3147 if (!tracer_enabled && iter->pos)
3148 break;
b3806b43
SR
3149 }
3150
ff98781b
EGM
3151 return 1;
3152}
3153
3154/*
3155 * Consumer reader.
3156 */
3157static ssize_t
3158tracing_read_pipe(struct file *filp, char __user *ubuf,
3159 size_t cnt, loff_t *ppos)
3160{
3161 struct trace_iterator *iter = filp->private_data;
d7350c3f 3162 static struct tracer *old_tracer;
ff98781b
EGM
3163 ssize_t sret;
3164
3165 /* return any leftover data */
3166 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3167 if (sret != -EBUSY)
3168 return sret;
3169
f9520750 3170 trace_seq_init(&iter->seq);
ff98781b 3171
d7350c3f 3172 /* copy the tracer to avoid using a global lock all around */
ff98781b 3173 mutex_lock(&trace_types_lock);
d7350c3f
FW
3174 if (unlikely(old_tracer != current_trace && current_trace)) {
3175 old_tracer = current_trace;
3176 *iter->trace = *current_trace;
3177 }
3178 mutex_unlock(&trace_types_lock);
3179
3180 /*
3181 * Avoid more than one consumer on a single file descriptor
3182 * This is just a matter of traces coherency, the ring buffer itself
3183 * is protected.
3184 */
3185 mutex_lock(&iter->mutex);
ff98781b
EGM
3186 if (iter->trace->read) {
3187 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3188 if (sret)
3189 goto out;
3190 }
3191
3192waitagain:
3193 sret = tracing_wait_pipe(filp);
3194 if (sret <= 0)
3195 goto out;
3196
b3806b43 3197 /* stop when tracing is finished */
ff98781b
EGM
3198 if (trace_empty(iter)) {
3199 sret = 0;
107bad8b 3200 goto out;
ff98781b 3201 }
b3806b43
SR
3202
3203 if (cnt >= PAGE_SIZE)
3204 cnt = PAGE_SIZE - 1;
3205
53d0aa77 3206 /* reset all but tr, trace, and overruns */
53d0aa77
SR
3207 memset(&iter->seq, 0,
3208 sizeof(struct trace_iterator) -
3209 offsetof(struct trace_iterator, seq));
4823ed7e 3210 iter->pos = -1;
b3806b43 3211
4f535968 3212 trace_event_read_lock();
7e53bd42 3213 trace_access_lock(iter->cpu_file);
088b1e42 3214 while (find_next_entry_inc(iter) != NULL) {
2c4f035f 3215 enum print_line_t ret;
088b1e42
SR
3216 int len = iter->seq.len;
3217
f9896bf3 3218 ret = print_trace_line(iter);
2c4f035f 3219 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
3220 /* don't print partial lines */
3221 iter->seq.len = len;
b3806b43 3222 break;
088b1e42 3223 }
b91facc3
FW
3224 if (ret != TRACE_TYPE_NO_CONSUME)
3225 trace_consume(iter);
b3806b43
SR
3226
3227 if (iter->seq.len >= cnt)
3228 break;
b3806b43 3229 }
7e53bd42 3230 trace_access_unlock(iter->cpu_file);
4f535968 3231 trace_event_read_unlock();
b3806b43 3232
b3806b43 3233 /* Now copy what we have to the user */
6c6c2796
PP
3234 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3235 if (iter->seq.readpos >= iter->seq.len)
f9520750 3236 trace_seq_init(&iter->seq);
9ff4b974
PP
3237
3238 /*
3239 * If there was nothing to send to user, inspite of consuming trace
3240 * entries, go back to wait for more entries.
3241 */
6c6c2796 3242 if (sret == -EBUSY)
9ff4b974 3243 goto waitagain;
b3806b43 3244
107bad8b 3245out:
d7350c3f 3246 mutex_unlock(&iter->mutex);
107bad8b 3247
6c6c2796 3248 return sret;
b3806b43
SR
3249}
3250
3c56819b
EGM
3251static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3252 struct pipe_buffer *buf)
3253{
3254 __free_page(buf->page);
3255}
3256
3257static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3258 unsigned int idx)
3259{
3260 __free_page(spd->pages[idx]);
3261}
3262
28dfef8f 3263static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998
SR
3264 .can_merge = 0,
3265 .map = generic_pipe_buf_map,
3266 .unmap = generic_pipe_buf_unmap,
3267 .confirm = generic_pipe_buf_confirm,
3268 .release = tracing_pipe_buf_release,
3269 .steal = generic_pipe_buf_steal,
3270 .get = generic_pipe_buf_get,
3c56819b
EGM
3271};
3272
34cd4998 3273static size_t
fa7c7f6e 3274tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
3275{
3276 size_t count;
3277 int ret;
3278
3279 /* Seq buffer is page-sized, exactly what we need. */
3280 for (;;) {
3281 count = iter->seq.len;
3282 ret = print_trace_line(iter);
3283 count = iter->seq.len - count;
3284 if (rem < count) {
3285 rem = 0;
3286 iter->seq.len -= count;
3287 break;
3288 }
3289 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3290 iter->seq.len -= count;
3291 break;
3292 }
3293
74e7ff8c
LJ
3294 if (ret != TRACE_TYPE_NO_CONSUME)
3295 trace_consume(iter);
34cd4998
SR
3296 rem -= count;
3297 if (!find_next_entry_inc(iter)) {
3298 rem = 0;
3299 iter->ent = NULL;
3300 break;
3301 }
3302 }
3303
3304 return rem;
3305}
3306
3c56819b
EGM
3307static ssize_t tracing_splice_read_pipe(struct file *filp,
3308 loff_t *ppos,
3309 struct pipe_inode_info *pipe,
3310 size_t len,
3311 unsigned int flags)
3312{
35f3d14d
JA
3313 struct page *pages_def[PIPE_DEF_BUFFERS];
3314 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
3315 struct trace_iterator *iter = filp->private_data;
3316 struct splice_pipe_desc spd = {
35f3d14d
JA
3317 .pages = pages_def,
3318 .partial = partial_def,
34cd4998
SR
3319 .nr_pages = 0, /* This gets updated below. */
3320 .flags = flags,
3321 .ops = &tracing_pipe_buf_ops,
3322 .spd_release = tracing_spd_release_pipe,
3c56819b 3323 };
d7350c3f 3324 static struct tracer *old_tracer;
3c56819b 3325 ssize_t ret;
34cd4998 3326 size_t rem;
3c56819b
EGM
3327 unsigned int i;
3328
35f3d14d
JA
3329 if (splice_grow_spd(pipe, &spd))
3330 return -ENOMEM;
3331
d7350c3f 3332 /* copy the tracer to avoid using a global lock all around */
3c56819b 3333 mutex_lock(&trace_types_lock);
d7350c3f
FW
3334 if (unlikely(old_tracer != current_trace && current_trace)) {
3335 old_tracer = current_trace;
3336 *iter->trace = *current_trace;
3337 }
3338 mutex_unlock(&trace_types_lock);
3339
3340 mutex_lock(&iter->mutex);
3c56819b
EGM
3341
3342 if (iter->trace->splice_read) {
3343 ret = iter->trace->splice_read(iter, filp,
3344 ppos, pipe, len, flags);
3345 if (ret)
34cd4998 3346 goto out_err;
3c56819b
EGM
3347 }
3348
3349 ret = tracing_wait_pipe(filp);
3350 if (ret <= 0)
34cd4998 3351 goto out_err;
3c56819b
EGM
3352
3353 if (!iter->ent && !find_next_entry_inc(iter)) {
3354 ret = -EFAULT;
34cd4998 3355 goto out_err;
3c56819b
EGM
3356 }
3357
4f535968 3358 trace_event_read_lock();
7e53bd42 3359 trace_access_lock(iter->cpu_file);
4f535968 3360
3c56819b 3361 /* Fill as many pages as possible. */
35f3d14d
JA
3362 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3363 spd.pages[i] = alloc_page(GFP_KERNEL);
3364 if (!spd.pages[i])
34cd4998 3365 break;
3c56819b 3366
fa7c7f6e 3367 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
3368
3369 /* Copy the data into the page, so we can start over. */
3370 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 3371 page_address(spd.pages[i]),
3c56819b
EGM
3372 iter->seq.len);
3373 if (ret < 0) {
35f3d14d 3374 __free_page(spd.pages[i]);
3c56819b
EGM
3375 break;
3376 }
35f3d14d
JA
3377 spd.partial[i].offset = 0;
3378 spd.partial[i].len = iter->seq.len;
3c56819b 3379
f9520750 3380 trace_seq_init(&iter->seq);
3c56819b
EGM
3381 }
3382
7e53bd42 3383 trace_access_unlock(iter->cpu_file);
4f535968 3384 trace_event_read_unlock();
d7350c3f 3385 mutex_unlock(&iter->mutex);
3c56819b
EGM
3386
3387 spd.nr_pages = i;
3388
35f3d14d
JA
3389 ret = splice_to_pipe(pipe, &spd);
3390out:
3391 splice_shrink_spd(pipe, &spd);
3392 return ret;
3c56819b 3393
34cd4998 3394out_err:
d7350c3f 3395 mutex_unlock(&iter->mutex);
35f3d14d 3396 goto out;
3c56819b
EGM
3397}
3398
a98a3c3f
SR
3399static ssize_t
3400tracing_entries_read(struct file *filp, char __user *ubuf,
3401 size_t cnt, loff_t *ppos)
3402{
3403 struct trace_array *tr = filp->private_data;
db526ca3 3404 char buf[96];
a98a3c3f
SR
3405 int r;
3406
db526ca3
SR
3407 mutex_lock(&trace_types_lock);
3408 if (!ring_buffer_expanded)
3409 r = sprintf(buf, "%lu (expanded: %lu)\n",
3410 tr->entries >> 10,
3411 trace_buf_size >> 10);
3412 else
3413 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3414 mutex_unlock(&trace_types_lock);
3415
a98a3c3f
SR
3416 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3417}
3418
3419static ssize_t
3420tracing_entries_write(struct file *filp, const char __user *ubuf,
3421 size_t cnt, loff_t *ppos)
3422{
3423 unsigned long val;
3424 char buf[64];
bf5e6519 3425 int ret, cpu;
a98a3c3f 3426
cffae437
SR
3427 if (cnt >= sizeof(buf))
3428 return -EINVAL;
a98a3c3f
SR
3429
3430 if (copy_from_user(&buf, ubuf, cnt))
3431 return -EFAULT;
3432
3433 buf[cnt] = 0;
3434
c6caeeb1
SR
3435 ret = strict_strtoul(buf, 10, &val);
3436 if (ret < 0)
3437 return ret;
a98a3c3f
SR
3438
3439 /* must have at least 1 entry */
3440 if (!val)
3441 return -EINVAL;
3442
3443 mutex_lock(&trace_types_lock);
3444
c76f0694 3445 tracing_stop();
a98a3c3f 3446
bf5e6519
SR
3447 /* disable all cpu buffers */
3448 for_each_tracing_cpu(cpu) {
3449 if (global_trace.data[cpu])
3450 atomic_inc(&global_trace.data[cpu]->disabled);
3451 if (max_tr.data[cpu])
3452 atomic_inc(&max_tr.data[cpu]->disabled);
3453 }
3454
1696b2b0
SR
3455 /* value is in KB */
3456 val <<= 10;
3457
3928a8a2 3458 if (val != global_trace.entries) {
73c5162a 3459 ret = tracing_resize_ring_buffer(val);
3928a8a2
SR
3460 if (ret < 0) {
3461 cnt = ret;
3eefae99
SR
3462 goto out;
3463 }
a98a3c3f
SR
3464 }
3465
cf8517cf 3466 *ppos += cnt;
a98a3c3f 3467
19384c03
SR
3468 /* If check pages failed, return ENOMEM */
3469 if (tracing_disabled)
3470 cnt = -ENOMEM;
a98a3c3f 3471 out:
bf5e6519
SR
3472 for_each_tracing_cpu(cpu) {
3473 if (global_trace.data[cpu])
3474 atomic_dec(&global_trace.data[cpu]->disabled);
3475 if (max_tr.data[cpu])
3476 atomic_dec(&max_tr.data[cpu]->disabled);
3477 }
3478
c76f0694 3479 tracing_start();
a98a3c3f
SR
3480 max_tr.entries = global_trace.entries;
3481 mutex_unlock(&trace_types_lock);
3482
3483 return cnt;
3484}
3485
f2942487
CE
3486static int mark_printk(const char *fmt, ...)
3487{
3488 int ret;
3489 va_list args;
3490 va_start(args, fmt);
3491 ret = trace_vprintk(0, fmt, args);
3492 va_end(args);
3493 return ret;
3494}
3495
5bf9a1ee
PP
3496static ssize_t
3497tracing_mark_write(struct file *filp, const char __user *ubuf,
3498 size_t cnt, loff_t *fpos)
3499{
3500 char *buf;
5bf9a1ee 3501
c76f0694 3502 if (tracing_disabled)
5bf9a1ee
PP
3503 return -EINVAL;
3504
3505 if (cnt > TRACE_BUF_SIZE)
3506 cnt = TRACE_BUF_SIZE;
3507
c13d2f7c 3508 buf = kmalloc(cnt + 2, GFP_KERNEL);
5bf9a1ee
PP
3509 if (buf == NULL)
3510 return -ENOMEM;
3511
3512 if (copy_from_user(buf, ubuf, cnt)) {
3513 kfree(buf);
3514 return -EFAULT;
3515 }
c13d2f7c
CE
3516 if (buf[cnt-1] != '\n') {
3517 buf[cnt] = '\n';
3518 buf[cnt+1] = '\0';
3519 } else
3520 buf[cnt] = '\0';
5bf9a1ee 3521
f2942487 3522 cnt = mark_printk("%s", buf);
5bf9a1ee
PP
3523 kfree(buf);
3524 *fpos += cnt;
3525
3526 return cnt;
3527}
3528
13f16d20 3529static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 3530{
5079f326
Z
3531 int i;
3532
3533 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 3534 seq_printf(m,
5079f326
Z
3535 "%s%s%s%s", i ? " " : "",
3536 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3537 i == trace_clock_id ? "]" : "");
13f16d20 3538 seq_putc(m, '\n');
5079f326 3539
13f16d20 3540 return 0;
5079f326
Z
3541}
3542
3543static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3544 size_t cnt, loff_t *fpos)
3545{
3546 char buf[64];
3547 const char *clockstr;
3548 int i;
3549
3550 if (cnt >= sizeof(buf))
3551 return -EINVAL;
3552
3553 if (copy_from_user(&buf, ubuf, cnt))
3554 return -EFAULT;
3555
3556 buf[cnt] = 0;
3557
3558 clockstr = strstrip(buf);
3559
3560 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3561 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3562 break;
3563 }
3564 if (i == ARRAY_SIZE(trace_clocks))
3565 return -EINVAL;
3566
3567 trace_clock_id = i;
3568
3569 mutex_lock(&trace_types_lock);
3570
3571 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3572 if (max_tr.buffer)
3573 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3574
3575 mutex_unlock(&trace_types_lock);
3576
3577 *fpos += cnt;
3578
3579 return cnt;
3580}
3581
13f16d20
LZ
3582static int tracing_clock_open(struct inode *inode, struct file *file)
3583{
3584 if (tracing_disabled)
3585 return -ENODEV;
3586 return single_open(file, tracing_clock_show, NULL);
3587}
3588
5e2336a0 3589static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
3590 .open = tracing_open_generic,
3591 .read = tracing_max_lat_read,
3592 .write = tracing_max_lat_write,
bc0c38d1
SR
3593};
3594
5e2336a0 3595static const struct file_operations tracing_ctrl_fops = {
4bf39a94
IM
3596 .open = tracing_open_generic,
3597 .read = tracing_ctrl_read,
3598 .write = tracing_ctrl_write,
bc0c38d1
SR
3599};
3600
5e2336a0 3601static const struct file_operations set_tracer_fops = {
4bf39a94
IM
3602 .open = tracing_open_generic,
3603 .read = tracing_set_trace_read,
3604 .write = tracing_set_trace_write,
bc0c38d1
SR
3605};
3606
5e2336a0 3607static const struct file_operations tracing_pipe_fops = {
4bf39a94 3608 .open = tracing_open_pipe,
2a2cc8f7 3609 .poll = tracing_poll_pipe,
4bf39a94 3610 .read = tracing_read_pipe,
3c56819b 3611 .splice_read = tracing_splice_read_pipe,
4bf39a94 3612 .release = tracing_release_pipe,
b3806b43
SR
3613};
3614
5e2336a0 3615static const struct file_operations tracing_entries_fops = {
a98a3c3f
SR
3616 .open = tracing_open_generic,
3617 .read = tracing_entries_read,
3618 .write = tracing_entries_write,
3619};
3620
5e2336a0 3621static const struct file_operations tracing_mark_fops = {
43a15386 3622 .open = tracing_open_generic,
5bf9a1ee
PP
3623 .write = tracing_mark_write,
3624};
3625
5079f326 3626static const struct file_operations trace_clock_fops = {
13f16d20
LZ
3627 .open = tracing_clock_open,
3628 .read = seq_read,
3629 .llseek = seq_lseek,
3630 .release = single_release,
5079f326
Z
3631 .write = tracing_clock_write,
3632};
3633
2cadf913
SR
3634struct ftrace_buffer_info {
3635 struct trace_array *tr;
3636 void *spare;
3637 int cpu;
3638 unsigned int read;
3639};
3640
3641static int tracing_buffers_open(struct inode *inode, struct file *filp)
3642{
3643 int cpu = (int)(long)inode->i_private;
3644 struct ftrace_buffer_info *info;
3645
3646 if (tracing_disabled)
3647 return -ENODEV;
3648
3649 info = kzalloc(sizeof(*info), GFP_KERNEL);
3650 if (!info)
3651 return -ENOMEM;
3652
3653 info->tr = &global_trace;
3654 info->cpu = cpu;
ddd538f3 3655 info->spare = NULL;
2cadf913
SR
3656 /* Force reading ring buffer for first read */
3657 info->read = (unsigned int)-1;
2cadf913
SR
3658
3659 filp->private_data = info;
3660
d1e7e02f 3661 return nonseekable_open(inode, filp);
2cadf913
SR
3662}
3663
3664static ssize_t
3665tracing_buffers_read(struct file *filp, char __user *ubuf,
3666 size_t count, loff_t *ppos)
3667{
3668 struct ftrace_buffer_info *info = filp->private_data;
2cadf913
SR
3669 ssize_t ret;
3670 size_t size;
3671
2dc5d12b
SR
3672 if (!count)
3673 return 0;
3674
ddd538f3
LJ
3675 if (!info->spare)
3676 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3677 if (!info->spare)
3678 return -ENOMEM;
3679
2cadf913
SR
3680 /* Do we have previous read data to read? */
3681 if (info->read < PAGE_SIZE)
3682 goto read;
3683
3684 info->read = 0;
3685
7e53bd42 3686 trace_access_lock(info->cpu);
2cadf913
SR
3687 ret = ring_buffer_read_page(info->tr->buffer,
3688 &info->spare,
3689 count,
3690 info->cpu, 0);
7e53bd42 3691 trace_access_unlock(info->cpu);
2cadf913
SR
3692 if (ret < 0)
3693 return 0;
3694
2cadf913
SR
3695read:
3696 size = PAGE_SIZE - info->read;
3697 if (size > count)
3698 size = count;
3699
3700 ret = copy_to_user(ubuf, info->spare + info->read, size);
2dc5d12b 3701 if (ret == size)
2cadf913 3702 return -EFAULT;
2dc5d12b
SR
3703 size -= ret;
3704
2cadf913
SR
3705 *ppos += size;
3706 info->read += size;
3707
3708 return size;
3709}
3710
3711static int tracing_buffers_release(struct inode *inode, struct file *file)
3712{
3713 struct ftrace_buffer_info *info = file->private_data;
3714
ddd538f3
LJ
3715 if (info->spare)
3716 ring_buffer_free_read_page(info->tr->buffer, info->spare);
2cadf913
SR
3717 kfree(info);
3718
3719 return 0;
3720}
3721
3722struct buffer_ref {
3723 struct ring_buffer *buffer;
3724 void *page;
3725 int ref;
3726};
3727
3728static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3729 struct pipe_buffer *buf)
3730{
3731 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3732
3733 if (--ref->ref)
3734 return;
3735
3736 ring_buffer_free_read_page(ref->buffer, ref->page);
3737 kfree(ref);
3738 buf->private = 0;
3739}
3740
3741static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3742 struct pipe_buffer *buf)
3743{
3744 return 1;
3745}
3746
3747static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3748 struct pipe_buffer *buf)
3749{
3750 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3751
3752 ref->ref++;
3753}
3754
3755/* Pipe buffer operations for a buffer. */
28dfef8f 3756static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913
SR
3757 .can_merge = 0,
3758 .map = generic_pipe_buf_map,
3759 .unmap = generic_pipe_buf_unmap,
3760 .confirm = generic_pipe_buf_confirm,
3761 .release = buffer_pipe_buf_release,
3762 .steal = buffer_pipe_buf_steal,
3763 .get = buffer_pipe_buf_get,
3764};
3765
3766/*
3767 * Callback from splice_to_pipe(), if we need to release some pages
3768 * at the end of the spd in case we error'ed out in filling the pipe.
3769 */
3770static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3771{
3772 struct buffer_ref *ref =
3773 (struct buffer_ref *)spd->partial[i].private;
3774
3775 if (--ref->ref)
3776 return;
3777
3778 ring_buffer_free_read_page(ref->buffer, ref->page);
3779 kfree(ref);
3780 spd->partial[i].private = 0;
3781}
3782
3783static ssize_t
3784tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3785 struct pipe_inode_info *pipe, size_t len,
3786 unsigned int flags)
3787{
3788 struct ftrace_buffer_info *info = file->private_data;
35f3d14d
JA
3789 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3790 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 3791 struct splice_pipe_desc spd = {
35f3d14d
JA
3792 .pages = pages_def,
3793 .partial = partial_def,
2cadf913
SR
3794 .flags = flags,
3795 .ops = &buffer_pipe_buf_ops,
3796 .spd_release = buffer_spd_release,
3797 };
3798 struct buffer_ref *ref;
93459c6c 3799 int entries, size, i;
2cadf913
SR
3800 size_t ret;
3801
35f3d14d
JA
3802 if (splice_grow_spd(pipe, &spd))
3803 return -ENOMEM;
3804
93cfb3c9
LJ
3805 if (*ppos & (PAGE_SIZE - 1)) {
3806 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
35f3d14d
JA
3807 ret = -EINVAL;
3808 goto out;
93cfb3c9
LJ
3809 }
3810
3811 if (len & (PAGE_SIZE - 1)) {
3812 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
35f3d14d
JA
3813 if (len < PAGE_SIZE) {
3814 ret = -EINVAL;
3815 goto out;
3816 }
93cfb3c9
LJ
3817 len &= PAGE_MASK;
3818 }
3819
7e53bd42 3820 trace_access_lock(info->cpu);
93459c6c
SR
3821 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3822
35f3d14d 3823 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
3824 struct page *page;
3825 int r;
3826
3827 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3828 if (!ref)
3829 break;
3830
7267fa68 3831 ref->ref = 1;
2cadf913
SR
3832 ref->buffer = info->tr->buffer;
3833 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3834 if (!ref->page) {
3835 kfree(ref);
3836 break;
3837 }
3838
3839 r = ring_buffer_read_page(ref->buffer, &ref->page,
f2957f1f 3840 len, info->cpu, 1);
2cadf913
SR
3841 if (r < 0) {
3842 ring_buffer_free_read_page(ref->buffer,
3843 ref->page);
3844 kfree(ref);
3845 break;
3846 }
3847
3848 /*
3849 * zero out any left over data, this is going to
3850 * user land.
3851 */
3852 size = ring_buffer_page_len(ref->page);
3853 if (size < PAGE_SIZE)
3854 memset(ref->page + size, 0, PAGE_SIZE - size);
3855
3856 page = virt_to_page(ref->page);
3857
3858 spd.pages[i] = page;
3859 spd.partial[i].len = PAGE_SIZE;
3860 spd.partial[i].offset = 0;
3861 spd.partial[i].private = (unsigned long)ref;
3862 spd.nr_pages++;
93cfb3c9 3863 *ppos += PAGE_SIZE;
93459c6c
SR
3864
3865 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
2cadf913
SR
3866 }
3867
7e53bd42 3868 trace_access_unlock(info->cpu);
2cadf913
SR
3869 spd.nr_pages = i;
3870
3871 /* did we read anything? */
3872 if (!spd.nr_pages) {
3873 if (flags & SPLICE_F_NONBLOCK)
3874 ret = -EAGAIN;
3875 else
3876 ret = 0;
3877 /* TODO: block */
35f3d14d 3878 goto out;
2cadf913
SR
3879 }
3880
3881 ret = splice_to_pipe(pipe, &spd);
35f3d14d
JA
3882 splice_shrink_spd(pipe, &spd);
3883out:
2cadf913
SR
3884 return ret;
3885}
3886
3887static const struct file_operations tracing_buffers_fops = {
3888 .open = tracing_buffers_open,
3889 .read = tracing_buffers_read,
3890 .release = tracing_buffers_release,
3891 .splice_read = tracing_buffers_splice_read,
3892 .llseek = no_llseek,
3893};
3894
c8d77183
SR
3895static ssize_t
3896tracing_stats_read(struct file *filp, char __user *ubuf,
3897 size_t count, loff_t *ppos)
3898{
3899 unsigned long cpu = (unsigned long)filp->private_data;
3900 struct trace_array *tr = &global_trace;
3901 struct trace_seq *s;
3902 unsigned long cnt;
3903
e4f2d10f 3904 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 3905 if (!s)
a646365c 3906 return -ENOMEM;
c8d77183
SR
3907
3908 trace_seq_init(s);
3909
3910 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3911 trace_seq_printf(s, "entries: %ld\n", cnt);
3912
3913 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3914 trace_seq_printf(s, "overrun: %ld\n", cnt);
3915
3916 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3917 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3918
c8d77183
SR
3919 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3920
3921 kfree(s);
3922
3923 return count;
3924}
3925
3926static const struct file_operations tracing_stats_fops = {
3927 .open = tracing_open_generic,
3928 .read = tracing_stats_read,
3929};
3930
bc0c38d1
SR
3931#ifdef CONFIG_DYNAMIC_FTRACE
3932
b807c3d0
SR
3933int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3934{
3935 return 0;
3936}
3937
bc0c38d1 3938static ssize_t
b807c3d0 3939tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
3940 size_t cnt, loff_t *ppos)
3941{
a26a2a27
SR
3942 static char ftrace_dyn_info_buffer[1024];
3943 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 3944 unsigned long *p = filp->private_data;
b807c3d0 3945 char *buf = ftrace_dyn_info_buffer;
a26a2a27 3946 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
3947 int r;
3948
b807c3d0
SR
3949 mutex_lock(&dyn_info_mutex);
3950 r = sprintf(buf, "%ld ", *p);
4bf39a94 3951
a26a2a27 3952 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
3953 buf[r++] = '\n';
3954
3955 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3956
3957 mutex_unlock(&dyn_info_mutex);
3958
3959 return r;
bc0c38d1
SR
3960}
3961
5e2336a0 3962static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 3963 .open = tracing_open_generic,
b807c3d0 3964 .read = tracing_read_dyn_info,
bc0c38d1
SR
3965};
3966#endif
3967
3968static struct dentry *d_tracer;
3969
3970struct dentry *tracing_init_dentry(void)
3971{
3972 static int once;
3973
3974 if (d_tracer)
3975 return d_tracer;
3976
3e1f60b8
FW
3977 if (!debugfs_initialized())
3978 return NULL;
3979
bc0c38d1
SR
3980 d_tracer = debugfs_create_dir("tracing", NULL);
3981
3982 if (!d_tracer && !once) {
3983 once = 1;
3984 pr_warning("Could not create debugfs directory 'tracing'\n");
3985 return NULL;
3986 }
3987
3988 return d_tracer;
3989}
3990
b04cc6b1
FW
3991static struct dentry *d_percpu;
3992
3993struct dentry *tracing_dentry_percpu(void)
3994{
3995 static int once;
3996 struct dentry *d_tracer;
3997
3998 if (d_percpu)
3999 return d_percpu;
4000
4001 d_tracer = tracing_init_dentry();
4002
4003 if (!d_tracer)
4004 return NULL;
4005
4006 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4007
4008 if (!d_percpu && !once) {
4009 once = 1;
4010 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4011 return NULL;
4012 }
4013
4014 return d_percpu;
4015}
4016
4017static void tracing_init_debugfs_percpu(long cpu)
4018{
4019 struct dentry *d_percpu = tracing_dentry_percpu();
5452af66 4020 struct dentry *d_cpu;
8656e7a2
FW
4021 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
4022 char cpu_dir[7];
b04cc6b1
FW
4023
4024 if (cpu > 999 || cpu < 0)
4025 return;
4026
8656e7a2
FW
4027 sprintf(cpu_dir, "cpu%ld", cpu);
4028 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4029 if (!d_cpu) {
4030 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4031 return;
4032 }
b04cc6b1 4033
8656e7a2 4034 /* per cpu trace_pipe */
5452af66
FW
4035 trace_create_file("trace_pipe", 0444, d_cpu,
4036 (void *) cpu, &tracing_pipe_fops);
b04cc6b1
FW
4037
4038 /* per cpu trace */
5452af66
FW
4039 trace_create_file("trace", 0644, d_cpu,
4040 (void *) cpu, &tracing_fops);
7f96f93f 4041
5452af66
FW
4042 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4043 (void *) cpu, &tracing_buffers_fops);
7f96f93f 4044
c8d77183
SR
4045 trace_create_file("stats", 0444, d_cpu,
4046 (void *) cpu, &tracing_stats_fops);
b04cc6b1
FW
4047}
4048
60a11774
SR
4049#ifdef CONFIG_FTRACE_SELFTEST
4050/* Let selftest have access to static functions in this file */
4051#include "trace_selftest.c"
4052#endif
4053
577b785f
SR
4054struct trace_option_dentry {
4055 struct tracer_opt *opt;
4056 struct tracer_flags *flags;
4057 struct dentry *entry;
4058};
4059
4060static ssize_t
4061trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4062 loff_t *ppos)
4063{
4064 struct trace_option_dentry *topt = filp->private_data;
4065 char *buf;
4066
4067 if (topt->flags->val & topt->opt->bit)
4068 buf = "1\n";
4069 else
4070 buf = "0\n";
4071
4072 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4073}
4074
4075static ssize_t
4076trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4077 loff_t *ppos)
4078{
4079 struct trace_option_dentry *topt = filp->private_data;
4080 unsigned long val;
4081 char buf[64];
4082 int ret;
4083
4084 if (cnt >= sizeof(buf))
4085 return -EINVAL;
4086
4087 if (copy_from_user(&buf, ubuf, cnt))
4088 return -EFAULT;
4089
4090 buf[cnt] = 0;
4091
4092 ret = strict_strtoul(buf, 10, &val);
4093 if (ret < 0)
4094 return ret;
4095
8d18eaaf
LZ
4096 if (val != 0 && val != 1)
4097 return -EINVAL;
577b785f 4098
8d18eaaf 4099 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 4100 mutex_lock(&trace_types_lock);
8d18eaaf 4101 ret = __set_tracer_option(current_trace, topt->flags,
c757bea9 4102 topt->opt, !val);
577b785f
SR
4103 mutex_unlock(&trace_types_lock);
4104 if (ret)
4105 return ret;
577b785f
SR
4106 }
4107
4108 *ppos += cnt;
4109
4110 return cnt;
4111}
4112
4113
4114static const struct file_operations trace_options_fops = {
4115 .open = tracing_open_generic,
4116 .read = trace_options_read,
4117 .write = trace_options_write,
4118};
4119
a8259075
SR
4120static ssize_t
4121trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4122 loff_t *ppos)
4123{
4124 long index = (long)filp->private_data;
4125 char *buf;
4126
4127 if (trace_flags & (1 << index))
4128 buf = "1\n";
4129 else
4130 buf = "0\n";
4131
4132 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4133}
4134
4135static ssize_t
4136trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4137 loff_t *ppos)
4138{
4139 long index = (long)filp->private_data;
4140 char buf[64];
4141 unsigned long val;
4142 int ret;
4143
4144 if (cnt >= sizeof(buf))
4145 return -EINVAL;
4146
4147 if (copy_from_user(&buf, ubuf, cnt))
4148 return -EFAULT;
4149
4150 buf[cnt] = 0;
4151
4152 ret = strict_strtoul(buf, 10, &val);
4153 if (ret < 0)
4154 return ret;
4155
f2d84b65 4156 if (val != 0 && val != 1)
a8259075 4157 return -EINVAL;
f2d84b65 4158 set_tracer_flags(1 << index, val);
a8259075
SR
4159
4160 *ppos += cnt;
4161
4162 return cnt;
4163}
4164
a8259075
SR
4165static const struct file_operations trace_options_core_fops = {
4166 .open = tracing_open_generic,
4167 .read = trace_options_core_read,
4168 .write = trace_options_core_write,
4169};
4170
5452af66
FW
4171struct dentry *trace_create_file(const char *name,
4172 mode_t mode,
4173 struct dentry *parent,
4174 void *data,
4175 const struct file_operations *fops)
4176{
4177 struct dentry *ret;
4178
4179 ret = debugfs_create_file(name, mode, parent, data, fops);
4180 if (!ret)
4181 pr_warning("Could not create debugfs '%s' entry\n", name);
4182
4183 return ret;
4184}
4185
4186
a8259075
SR
4187static struct dentry *trace_options_init_dentry(void)
4188{
4189 struct dentry *d_tracer;
4190 static struct dentry *t_options;
4191
4192 if (t_options)
4193 return t_options;
4194
4195 d_tracer = tracing_init_dentry();
4196 if (!d_tracer)
4197 return NULL;
4198
4199 t_options = debugfs_create_dir("options", d_tracer);
4200 if (!t_options) {
4201 pr_warning("Could not create debugfs directory 'options'\n");
4202 return NULL;
4203 }
4204
4205 return t_options;
4206}
4207
577b785f
SR
4208static void
4209create_trace_option_file(struct trace_option_dentry *topt,
4210 struct tracer_flags *flags,
4211 struct tracer_opt *opt)
4212{
4213 struct dentry *t_options;
577b785f
SR
4214
4215 t_options = trace_options_init_dentry();
4216 if (!t_options)
4217 return;
4218
4219 topt->flags = flags;
4220 topt->opt = opt;
4221
5452af66 4222 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
4223 &trace_options_fops);
4224
577b785f
SR
4225}
4226
4227static struct trace_option_dentry *
4228create_trace_option_files(struct tracer *tracer)
4229{
4230 struct trace_option_dentry *topts;
4231 struct tracer_flags *flags;
4232 struct tracer_opt *opts;
4233 int cnt;
4234
4235 if (!tracer)
4236 return NULL;
4237
4238 flags = tracer->flags;
4239
4240 if (!flags || !flags->opts)
4241 return NULL;
4242
4243 opts = flags->opts;
4244
4245 for (cnt = 0; opts[cnt].name; cnt++)
4246 ;
4247
0cfe8245 4248 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
4249 if (!topts)
4250 return NULL;
4251
4252 for (cnt = 0; opts[cnt].name; cnt++)
4253 create_trace_option_file(&topts[cnt], flags,
4254 &opts[cnt]);
4255
4256 return topts;
4257}
4258
4259static void
4260destroy_trace_option_files(struct trace_option_dentry *topts)
4261{
4262 int cnt;
4263
4264 if (!topts)
4265 return;
4266
4267 for (cnt = 0; topts[cnt].opt; cnt++) {
4268 if (topts[cnt].entry)
4269 debugfs_remove(topts[cnt].entry);
4270 }
4271
4272 kfree(topts);
4273}
4274
a8259075
SR
4275static struct dentry *
4276create_trace_option_core_file(const char *option, long index)
4277{
4278 struct dentry *t_options;
a8259075
SR
4279
4280 t_options = trace_options_init_dentry();
4281 if (!t_options)
4282 return NULL;
4283
5452af66 4284 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 4285 &trace_options_core_fops);
a8259075
SR
4286}
4287
4288static __init void create_trace_options_dir(void)
4289{
4290 struct dentry *t_options;
a8259075
SR
4291 int i;
4292
4293 t_options = trace_options_init_dentry();
4294 if (!t_options)
4295 return;
4296
5452af66
FW
4297 for (i = 0; trace_options[i]; i++)
4298 create_trace_option_core_file(trace_options[i], i);
a8259075
SR
4299}
4300
b5ad384e 4301static __init int tracer_init_debugfs(void)
bc0c38d1
SR
4302{
4303 struct dentry *d_tracer;
b04cc6b1 4304 int cpu;
bc0c38d1 4305
7e53bd42
LJ
4306 trace_access_lock_init();
4307
bc0c38d1
SR
4308 d_tracer = tracing_init_dentry();
4309
5452af66
FW
4310 trace_create_file("tracing_enabled", 0644, d_tracer,
4311 &global_trace, &tracing_ctrl_fops);
bc0c38d1 4312
5452af66
FW
4313 trace_create_file("trace_options", 0644, d_tracer,
4314 NULL, &tracing_iter_fops);
bc0c38d1 4315
5452af66
FW
4316 trace_create_file("tracing_cpumask", 0644, d_tracer,
4317 NULL, &tracing_cpumask_fops);
4318
4319 trace_create_file("trace", 0644, d_tracer,
4320 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
a8259075 4321
5452af66
FW
4322 trace_create_file("available_tracers", 0444, d_tracer,
4323 &global_trace, &show_traces_fops);
4324
339ae5d3 4325 trace_create_file("current_tracer", 0644, d_tracer,
5452af66
FW
4326 &global_trace, &set_tracer_fops);
4327
5d4a9dba 4328#ifdef CONFIG_TRACER_MAX_TRACE
5452af66
FW
4329 trace_create_file("tracing_max_latency", 0644, d_tracer,
4330 &tracing_max_latency, &tracing_max_lat_fops);
0e950173 4331#endif
5452af66
FW
4332
4333 trace_create_file("tracing_thresh", 0644, d_tracer,
4334 &tracing_thresh, &tracing_max_lat_fops);
a8259075 4335
339ae5d3 4336 trace_create_file("README", 0444, d_tracer,
5452af66
FW
4337 NULL, &tracing_readme_fops);
4338
4339 trace_create_file("trace_pipe", 0444, d_tracer,
b04cc6b1 4340 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
5452af66
FW
4341
4342 trace_create_file("buffer_size_kb", 0644, d_tracer,
4343 &global_trace, &tracing_entries_fops);
4344
4345 trace_create_file("trace_marker", 0220, d_tracer,
4346 NULL, &tracing_mark_fops);
5bf9a1ee 4347
69abe6a5
AP
4348 trace_create_file("saved_cmdlines", 0444, d_tracer,
4349 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 4350
5079f326
Z
4351 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4352 &trace_clock_fops);
4353
bc0c38d1 4354#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
4355 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4356 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 4357#endif
d618b3e6
IM
4358#ifdef CONFIG_SYSPROF_TRACER
4359 init_tracer_sysprof_debugfs(d_tracer);
4360#endif
b04cc6b1 4361
5452af66
FW
4362 create_trace_options_dir();
4363
b04cc6b1
FW
4364 for_each_tracing_cpu(cpu)
4365 tracing_init_debugfs_percpu(cpu);
4366
b5ad384e 4367 return 0;
bc0c38d1
SR
4368}
4369
3f5a54e3
SR
4370static int trace_panic_handler(struct notifier_block *this,
4371 unsigned long event, void *unused)
4372{
944ac425 4373 if (ftrace_dump_on_oops)
cecbca96 4374 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
4375 return NOTIFY_OK;
4376}
4377
4378static struct notifier_block trace_panic_notifier = {
4379 .notifier_call = trace_panic_handler,
4380 .next = NULL,
4381 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4382};
4383
4384static int trace_die_handler(struct notifier_block *self,
4385 unsigned long val,
4386 void *data)
4387{
4388 switch (val) {
4389 case DIE_OOPS:
944ac425 4390 if (ftrace_dump_on_oops)
cecbca96 4391 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
4392 break;
4393 default:
4394 break;
4395 }
4396 return NOTIFY_OK;
4397}
4398
4399static struct notifier_block trace_die_notifier = {
4400 .notifier_call = trace_die_handler,
4401 .priority = 200
4402};
4403
4404/*
4405 * printk is set to max of 1024, we really don't need it that big.
4406 * Nothing should be printing 1000 characters anyway.
4407 */
4408#define TRACE_MAX_PRINT 1000
4409
4410/*
4411 * Define here KERN_TRACE so that we have one place to modify
4412 * it if we decide to change what log level the ftrace dump
4413 * should be at.
4414 */
428aee14 4415#define KERN_TRACE KERN_EMERG
3f5a54e3
SR
4416
4417static void
4418trace_printk_seq(struct trace_seq *s)
4419{
4420 /* Probably should print a warning here. */
4421 if (s->len >= 1000)
4422 s->len = 1000;
4423
4424 /* should be zero ended, but we are paranoid. */
4425 s->buffer[s->len] = 0;
4426
4427 printk(KERN_TRACE "%s", s->buffer);
4428
f9520750 4429 trace_seq_init(s);
3f5a54e3
SR
4430}
4431
cecbca96
FW
4432static void
4433__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 4434{
445c8951 4435 static arch_spinlock_t ftrace_dump_lock =
edc35bd7 4436 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
3f5a54e3
SR
4437 /* use static because iter can be a bit big for the stack */
4438 static struct trace_iterator iter;
cf586b61 4439 unsigned int old_userobj;
3f5a54e3 4440 static int dump_ran;
d769041f
SR
4441 unsigned long flags;
4442 int cnt = 0, cpu;
3f5a54e3
SR
4443
4444 /* only one dump */
cd891ae0 4445 local_irq_save(flags);
0199c4e6 4446 arch_spin_lock(&ftrace_dump_lock);
3f5a54e3
SR
4447 if (dump_ran)
4448 goto out;
4449
4450 dump_ran = 1;
4451
0ee6b6cf 4452 tracing_off();
cf586b61
FW
4453
4454 if (disable_tracing)
4455 ftrace_kill();
3f5a54e3 4456
d769041f
SR
4457 for_each_tracing_cpu(cpu) {
4458 atomic_inc(&global_trace.data[cpu]->disabled);
4459 }
4460
cf586b61
FW
4461 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4462
b54d3de9
TE
4463 /* don't look at user memory in panic mode */
4464 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4465
e543ad76 4466 /* Simulate the iterator */
3f5a54e3
SR
4467 iter.tr = &global_trace;
4468 iter.trace = current_trace;
cecbca96
FW
4469
4470 switch (oops_dump_mode) {
4471 case DUMP_ALL:
4472 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4473 break;
4474 case DUMP_ORIG:
4475 iter.cpu_file = raw_smp_processor_id();
4476 break;
4477 case DUMP_NONE:
4478 goto out_enable;
4479 default:
4480 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4481 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4482 }
4483
4484 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3
SR
4485
4486 /*
4487 * We need to stop all tracing on all CPUS to read the
4488 * the next buffer. This is a bit expensive, but is
4489 * not done often. We fill all what we can read,
4490 * and then release the locks again.
4491 */
4492
3f5a54e3
SR
4493 while (!trace_empty(&iter)) {
4494
4495 if (!cnt)
4496 printk(KERN_TRACE "---------------------------------\n");
4497
4498 cnt++;
4499
4500 /* reset all but tr, trace, and overruns */
4501 memset(&iter.seq, 0,
4502 sizeof(struct trace_iterator) -
4503 offsetof(struct trace_iterator, seq));
4504 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4505 iter.pos = -1;
4506
4507 if (find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
4508 int ret;
4509
4510 ret = print_trace_line(&iter);
4511 if (ret != TRACE_TYPE_NO_CONSUME)
4512 trace_consume(&iter);
3f5a54e3
SR
4513 }
4514
4515 trace_printk_seq(&iter.seq);
4516 }
4517
4518 if (!cnt)
4519 printk(KERN_TRACE " (ftrace buffer empty)\n");
4520 else
4521 printk(KERN_TRACE "---------------------------------\n");
4522
cecbca96 4523 out_enable:
cf586b61
FW
4524 /* Re-enable tracing if requested */
4525 if (!disable_tracing) {
4526 trace_flags |= old_userobj;
4527
4528 for_each_tracing_cpu(cpu) {
4529 atomic_dec(&global_trace.data[cpu]->disabled);
4530 }
4531 tracing_on();
4532 }
4533
3f5a54e3 4534 out:
0199c4e6 4535 arch_spin_unlock(&ftrace_dump_lock);
cd891ae0 4536 local_irq_restore(flags);
3f5a54e3
SR
4537}
4538
cf586b61 4539/* By default: disable tracing after the dump */
cecbca96 4540void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
cf586b61 4541{
cecbca96 4542 __ftrace_dump(true, oops_dump_mode);
cf586b61
FW
4543}
4544
3928a8a2 4545__init static int tracer_alloc_buffers(void)
bc0c38d1 4546{
73c5162a 4547 int ring_buf_size;
4c11d7ae 4548 int i;
9e01c1b7 4549 int ret = -ENOMEM;
4c11d7ae 4550
9e01c1b7
RR
4551 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4552 goto out;
4553
4554 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4555 goto out_free_buffer_mask;
4c11d7ae 4556
73c5162a
SR
4557 /* To save memory, keep the ring buffer size to its minimum */
4558 if (ring_buffer_expanded)
4559 ring_buf_size = trace_buf_size;
4560 else
4561 ring_buf_size = 1;
4562
9e01c1b7
RR
4563 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4564 cpumask_copy(tracing_cpumask, cpu_all_mask);
4565
4566 /* TODO: make the number of buffers hot pluggable with CPUS */
73c5162a 4567 global_trace.buffer = ring_buffer_alloc(ring_buf_size,
3928a8a2
SR
4568 TRACE_BUFFER_FLAGS);
4569 if (!global_trace.buffer) {
4570 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4571 WARN_ON(1);
9e01c1b7 4572 goto out_free_cpumask;
4c11d7ae 4573 }
3928a8a2 4574 global_trace.entries = ring_buffer_size(global_trace.buffer);
4c11d7ae 4575
9e01c1b7 4576
4c11d7ae 4577#ifdef CONFIG_TRACER_MAX_TRACE
73c5162a 4578 max_tr.buffer = ring_buffer_alloc(ring_buf_size,
3928a8a2
SR
4579 TRACE_BUFFER_FLAGS);
4580 if (!max_tr.buffer) {
4581 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4582 WARN_ON(1);
4583 ring_buffer_free(global_trace.buffer);
9e01c1b7 4584 goto out_free_cpumask;
4c11d7ae 4585 }
3928a8a2
SR
4586 max_tr.entries = ring_buffer_size(max_tr.buffer);
4587 WARN_ON(max_tr.entries != global_trace.entries);
a98a3c3f 4588#endif
ab46428c 4589
4c11d7ae 4590 /* Allocate the first page for all buffers */
ab46428c 4591 for_each_tracing_cpu(i) {
566b0aaf 4592 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
9705f69e 4593 max_tr.data[i] = &per_cpu(max_tr_data, i);
4c11d7ae 4594 }
bc0c38d1 4595
bc0c38d1
SR
4596 trace_init_cmdlines();
4597
43a15386 4598 register_tracer(&nop_trace);
79fb0768 4599 current_trace = &nop_trace;
b5ad384e
FW
4600#ifdef CONFIG_BOOT_TRACER
4601 register_tracer(&boot_tracer);
b5ad384e 4602#endif
60a11774
SR
4603 /* All seems OK, enable tracing */
4604 tracing_disabled = 0;
3928a8a2 4605
3f5a54e3
SR
4606 atomic_notifier_chain_register(&panic_notifier_list,
4607 &trace_panic_notifier);
4608
4609 register_die_notifier(&trace_die_notifier);
2fc1dfbe
FW
4610
4611 return 0;
3f5a54e3 4612
9e01c1b7
RR
4613out_free_cpumask:
4614 free_cpumask_var(tracing_cpumask);
4615out_free_buffer_mask:
4616 free_cpumask_var(tracing_buffer_mask);
4617out:
4618 return ret;
bc0c38d1 4619}
b2821ae6
SR
4620
4621__init static int clear_boot_tracer(void)
4622{
4623 /*
4624 * The default tracer at boot buffer is an init section.
4625 * This function is called in lateinit. If we did not
4626 * find the boot tracer, then clear it out, to prevent
4627 * later registration from accessing the buffer that is
4628 * about to be freed.
4629 */
4630 if (!default_bootup_tracer)
4631 return 0;
4632
4633 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4634 default_bootup_tracer);
4635 default_bootup_tracer = NULL;
4636
4637 return 0;
4638}
4639
b5ad384e
FW
4640early_initcall(tracer_alloc_buffers);
4641fs_initcall(tracer_init_debugfs);
b2821ae6 4642late_initcall(clear_boot_tracer);