]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
tracing: Have trace_array keep track if snapshot buffer is allocated
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
4c11d7ae 23#include <linux/pagemap.h>
bc0c38d1
SR
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
2cadf913 27#include <linux/kprobes.h>
bc0c38d1
SR
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
2cadf913 31#include <linux/splice.h>
3f5a54e3 32#include <linux/kdebug.h>
5f0c6c03 33#include <linux/string.h>
7e53bd42 34#include <linux/rwsem.h>
5a0e3ad6 35#include <linux/slab.h>
bc0c38d1
SR
36#include <linux/ctype.h>
37#include <linux/init.h>
2a2cc8f7 38#include <linux/poll.h>
b892e5c8 39#include <linux/nmi.h>
bc0c38d1 40#include <linux/fs.h>
8bd75c77 41#include <linux/sched/rt.h>
86387f7e 42
bc0c38d1 43#include "trace.h"
f0868d1e 44#include "trace_output.h"
bc0c38d1 45
73c5162a
SR
46/*
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
020e5f85 50int ring_buffer_expanded;
73c5162a 51
8e1b82e0
FW
52/*
53 * We need to change this state when a selftest is running.
ff32504f
FW
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
5e1607a0 56 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
57 * at the same time, giving false positive or negative results.
58 */
8e1b82e0 59static bool __read_mostly tracing_selftest_running;
ff32504f 60
b2821ae6
SR
61/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
020e5f85 64bool __read_mostly tracing_selftest_disabled;
b2821ae6 65
adf9f195
FW
66/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
0f048701 80
7ffbd48d
SR
81/*
82 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
84 * occurred.
85 */
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
0f048701
SR
88/*
89 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
92 * this back to zero.
93 */
4fd27358 94static int tracing_disabled = 1;
0f048701 95
9288f99a 96DEFINE_PER_CPU(int, ftrace_cpu_disabled);
d769041f 97
955b61e5 98cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 99
944ac425
SR
100/*
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102 *
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
107 * serial console.
108 *
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 114 */
cecbca96
FW
115
116enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 117
b2821ae6
SR
118static int tracing_set_tracer(const char *buf);
119
ee6c2c1b
LZ
120#define MAX_TRACER_SIZE 100
121static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 122static char *default_bootup_tracer;
d9e54076 123
1beee96b 124static int __init set_cmdline_ftrace(char *str)
d9e54076 125{
ee6c2c1b 126 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 127 default_bootup_tracer = bootup_tracer_buf;
73c5162a
SR
128 /* We are using ftrace early, expand it */
129 ring_buffer_expanded = 1;
d9e54076
PZ
130 return 1;
131}
1beee96b 132__setup("ftrace=", set_cmdline_ftrace);
d9e54076 133
944ac425
SR
134static int __init set_ftrace_dump_on_oops(char *str)
135{
cecbca96
FW
136 if (*str++ != '=' || !*str) {
137 ftrace_dump_on_oops = DUMP_ALL;
138 return 1;
139 }
140
141 if (!strcmp("orig_cpu", str)) {
142 ftrace_dump_on_oops = DUMP_ORIG;
143 return 1;
144 }
145
146 return 0;
944ac425
SR
147}
148__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 149
7bcfaf54
SR
150
151static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
152static char *trace_boot_options __initdata;
153
154static int __init set_trace_boot_options(char *str)
155{
156 strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
157 trace_boot_options = trace_boot_options_buf;
158 return 0;
159}
160__setup("trace_options=", set_trace_boot_options);
161
cf8e3474 162unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
163{
164 nsec += 500;
165 do_div(nsec, 1000);
166 return nsec;
167}
168
4fcdae83
SR
169/*
170 * The global_trace is the descriptor that holds the tracing
171 * buffers for the live tracing. For each CPU, it contains
172 * a link list of pages that will store trace entries. The
173 * page descriptor of the pages in the memory is used to hold
174 * the link list by linking the lru item in the page descriptor
175 * to each of the pages in the buffer per CPU.
176 *
177 * For each active CPU there is a data field that holds the
178 * pages for the buffer for that CPU. Each CPU has the same number
179 * of pages allocated for its buffer.
180 */
bc0c38d1
SR
181static struct trace_array global_trace;
182
ae63b31e
SR
183LIST_HEAD(ftrace_trace_arrays);
184
e77405ad
SR
185int filter_current_check_discard(struct ring_buffer *buffer,
186 struct ftrace_event_call *call, void *rec,
eb02ce01
TZ
187 struct ring_buffer_event *event)
188{
e77405ad 189 return filter_check_discard(call, rec, buffer, event);
eb02ce01 190}
17c873ec 191EXPORT_SYMBOL_GPL(filter_current_check_discard);
eb02ce01 192
37886f6a
SR
193cycle_t ftrace_now(int cpu)
194{
195 u64 ts;
196
197 /* Early boot up does not have a buffer yet */
12883efb 198 if (!global_trace.trace_buffer.buffer)
37886f6a
SR
199 return trace_clock_local();
200
12883efb
SRRH
201 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
202 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
37886f6a
SR
203
204 return ts;
205}
bc0c38d1 206
9036990d
SR
207int tracing_is_enabled(void)
208{
0fb9656d 209 return tracing_is_on();
9036990d
SR
210}
211
4fcdae83 212/*
3928a8a2
SR
213 * trace_buf_size is the size in bytes that is allocated
214 * for a buffer. Note, the number of bytes is always rounded
215 * to page size.
3f5a54e3
SR
216 *
217 * This number is purposely set to a low number of 16384.
218 * If the dump on oops happens, it will be much appreciated
219 * to not have to wait for all that output. Anyway this can be
220 * boot time and run time configurable.
4fcdae83 221 */
3928a8a2 222#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 223
3928a8a2 224static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 225
4fcdae83 226/* trace_types holds a link list of available tracers. */
bc0c38d1 227static struct tracer *trace_types __read_mostly;
4fcdae83 228
4fcdae83
SR
229/*
230 * trace_types_lock is used to protect the trace_types list.
4fcdae83 231 */
bc0c38d1 232static DEFINE_MUTEX(trace_types_lock);
4fcdae83 233
7e53bd42
LJ
234/*
235 * serialize the access of the ring buffer
236 *
237 * ring buffer serializes readers, but it is low level protection.
238 * The validity of the events (which returns by ring_buffer_peek() ..etc)
239 * are not protected by ring buffer.
240 *
241 * The content of events may become garbage if we allow other process consumes
242 * these events concurrently:
243 * A) the page of the consumed events may become a normal page
244 * (not reader page) in ring buffer, and this page will be rewrited
245 * by events producer.
246 * B) The page of the consumed events may become a page for splice_read,
247 * and this page will be returned to system.
248 *
249 * These primitives allow multi process access to different cpu ring buffer
250 * concurrently.
251 *
252 * These primitives don't distinguish read-only and read-consume access.
253 * Multi read-only access are also serialized.
254 */
255
256#ifdef CONFIG_SMP
257static DECLARE_RWSEM(all_cpu_access_lock);
258static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
259
260static inline void trace_access_lock(int cpu)
261{
ae3b5093 262 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
263 /* gain it for accessing the whole ring buffer. */
264 down_write(&all_cpu_access_lock);
265 } else {
266 /* gain it for accessing a cpu ring buffer. */
267
ae3b5093 268 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
269 down_read(&all_cpu_access_lock);
270
271 /* Secondly block other access to this @cpu ring buffer. */
272 mutex_lock(&per_cpu(cpu_access_lock, cpu));
273 }
274}
275
276static inline void trace_access_unlock(int cpu)
277{
ae3b5093 278 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
279 up_write(&all_cpu_access_lock);
280 } else {
281 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
282 up_read(&all_cpu_access_lock);
283 }
284}
285
286static inline void trace_access_lock_init(void)
287{
288 int cpu;
289
290 for_each_possible_cpu(cpu)
291 mutex_init(&per_cpu(cpu_access_lock, cpu));
292}
293
294#else
295
296static DEFINE_MUTEX(access_lock);
297
298static inline void trace_access_lock(int cpu)
299{
300 (void)cpu;
301 mutex_lock(&access_lock);
302}
303
304static inline void trace_access_unlock(int cpu)
305{
306 (void)cpu;
307 mutex_unlock(&access_lock);
308}
309
310static inline void trace_access_lock_init(void)
311{
312}
313
314#endif
315
ee6bce52 316/* trace_flags holds trace_options default values */
12ef7d44 317unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a 318 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
77271ce4 319 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
5224c3a3 320 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
4e655519 321
499e5470
SR
322/**
323 * tracing_on - enable tracing buffers
324 *
325 * This function enables tracing buffers that may have been
326 * disabled with tracing_off.
327 */
328void tracing_on(void)
329{
12883efb
SRRH
330 if (global_trace.trace_buffer.buffer)
331 ring_buffer_record_on(global_trace.trace_buffer.buffer);
499e5470
SR
332 /*
333 * This flag is only looked at when buffers haven't been
334 * allocated yet. We don't really care about the race
335 * between setting this flag and actually turning
336 * on the buffer.
337 */
338 global_trace.buffer_disabled = 0;
339}
340EXPORT_SYMBOL_GPL(tracing_on);
341
342/**
343 * tracing_off - turn off tracing buffers
344 *
345 * This function stops the tracing buffers from recording data.
346 * It does not disable any overhead the tracers themselves may
347 * be causing. This function simply causes all recording to
348 * the ring buffers to fail.
349 */
350void tracing_off(void)
351{
12883efb
SRRH
352 if (global_trace.trace_buffer.buffer)
353 ring_buffer_record_off(global_trace.trace_buffer.buffer);
499e5470
SR
354 /*
355 * This flag is only looked at when buffers haven't been
356 * allocated yet. We don't really care about the race
357 * between setting this flag and actually turning
358 * on the buffer.
359 */
360 global_trace.buffer_disabled = 1;
361}
362EXPORT_SYMBOL_GPL(tracing_off);
363
364/**
365 * tracing_is_on - show state of ring buffers enabled
366 */
367int tracing_is_on(void)
368{
12883efb
SRRH
369 if (global_trace.trace_buffer.buffer)
370 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
499e5470
SR
371 return !global_trace.buffer_disabled;
372}
373EXPORT_SYMBOL_GPL(tracing_is_on);
374
3928a8a2 375static int __init set_buf_size(char *str)
bc0c38d1 376{
3928a8a2 377 unsigned long buf_size;
c6caeeb1 378
bc0c38d1
SR
379 if (!str)
380 return 0;
9d612bef 381 buf_size = memparse(str, &str);
c6caeeb1 382 /* nr_entries can not be zero */
9d612bef 383 if (buf_size == 0)
c6caeeb1 384 return 0;
3928a8a2 385 trace_buf_size = buf_size;
bc0c38d1
SR
386 return 1;
387}
3928a8a2 388__setup("trace_buf_size=", set_buf_size);
bc0c38d1 389
0e950173
TB
390static int __init set_tracing_thresh(char *str)
391{
87abb3b1 392 unsigned long threshold;
0e950173
TB
393 int ret;
394
395 if (!str)
396 return 0;
bcd83ea6 397 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
398 if (ret < 0)
399 return 0;
87abb3b1 400 tracing_thresh = threshold * 1000;
0e950173
TB
401 return 1;
402}
403__setup("tracing_thresh=", set_tracing_thresh);
404
57f50be1
SR
405unsigned long nsecs_to_usecs(unsigned long nsecs)
406{
407 return nsecs / 1000;
408}
409
4fcdae83 410/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
411static const char *trace_options[] = {
412 "print-parent",
413 "sym-offset",
414 "sym-addr",
415 "verbose",
f9896bf3 416 "raw",
5e3ca0ec 417 "hex",
cb0f12aa 418 "bin",
2a2cc8f7 419 "block",
86387f7e 420 "stacktrace",
5e1607a0 421 "trace_printk",
b2a866f9 422 "ftrace_preempt",
9f029e83 423 "branch",
12ef7d44 424 "annotate",
02b67518 425 "userstacktrace",
b54d3de9 426 "sym-userobj",
66896a85 427 "printk-msg-only",
c4a8e8be 428 "context-info",
c032ef64 429 "latency-format",
be6f164a 430 "sleep-time",
a2a16d6a 431 "graph-time",
e870e9a1 432 "record-cmd",
750912fa 433 "overwrite",
cf30cf67 434 "disable_on_free",
77271ce4 435 "irq-info",
5224c3a3 436 "markers",
bc0c38d1
SR
437 NULL
438};
439
5079f326
Z
440static struct {
441 u64 (*func)(void);
442 const char *name;
8be0709f 443 int in_ns; /* is this clock in nanoseconds? */
5079f326 444} trace_clocks[] = {
8be0709f
DS
445 { trace_clock_local, "local", 1 },
446 { trace_clock_global, "global", 1 },
447 { trace_clock_counter, "counter", 0 },
8cbd9cc6 448 ARCH_TRACE_CLOCKS
5079f326
Z
449};
450
451int trace_clock_id;
452
b63f39ea 453/*
454 * trace_parser_get_init - gets the buffer for trace parser
455 */
456int trace_parser_get_init(struct trace_parser *parser, int size)
457{
458 memset(parser, 0, sizeof(*parser));
459
460 parser->buffer = kmalloc(size, GFP_KERNEL);
461 if (!parser->buffer)
462 return 1;
463
464 parser->size = size;
465 return 0;
466}
467
468/*
469 * trace_parser_put - frees the buffer for trace parser
470 */
471void trace_parser_put(struct trace_parser *parser)
472{
473 kfree(parser->buffer);
474}
475
476/*
477 * trace_get_user - reads the user input string separated by space
478 * (matched by isspace(ch))
479 *
480 * For each string found the 'struct trace_parser' is updated,
481 * and the function returns.
482 *
483 * Returns number of bytes read.
484 *
485 * See kernel/trace/trace.h for 'struct trace_parser' details.
486 */
487int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
488 size_t cnt, loff_t *ppos)
489{
490 char ch;
491 size_t read = 0;
492 ssize_t ret;
493
494 if (!*ppos)
495 trace_parser_clear(parser);
496
497 ret = get_user(ch, ubuf++);
498 if (ret)
499 goto out;
500
501 read++;
502 cnt--;
503
504 /*
505 * The parser is not finished with the last write,
506 * continue reading the user input without skipping spaces.
507 */
508 if (!parser->cont) {
509 /* skip white space */
510 while (cnt && isspace(ch)) {
511 ret = get_user(ch, ubuf++);
512 if (ret)
513 goto out;
514 read++;
515 cnt--;
516 }
517
518 /* only spaces were written */
519 if (isspace(ch)) {
520 *ppos += read;
521 ret = read;
522 goto out;
523 }
524
525 parser->idx = 0;
526 }
527
528 /* read the non-space input */
529 while (cnt && !isspace(ch)) {
3c235a33 530 if (parser->idx < parser->size - 1)
b63f39ea 531 parser->buffer[parser->idx++] = ch;
532 else {
533 ret = -EINVAL;
534 goto out;
535 }
536 ret = get_user(ch, ubuf++);
537 if (ret)
538 goto out;
539 read++;
540 cnt--;
541 }
542
543 /* We either got finished input or we have to wait for another call. */
544 if (isspace(ch)) {
545 parser->buffer[parser->idx] = 0;
546 parser->cont = false;
547 } else {
548 parser->cont = true;
549 parser->buffer[parser->idx++] = ch;
550 }
551
552 *ppos += read;
553 ret = read;
554
555out:
556 return ret;
557}
558
6c6c2796
PP
559ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
560{
561 int len;
562 int ret;
563
2dc5d12b
SR
564 if (!cnt)
565 return 0;
566
6c6c2796
PP
567 if (s->len <= s->readpos)
568 return -EBUSY;
569
570 len = s->len - s->readpos;
571 if (cnt > len)
572 cnt = len;
573 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
2dc5d12b 574 if (ret == cnt)
6c6c2796
PP
575 return -EFAULT;
576
2dc5d12b
SR
577 cnt -= ret;
578
e74da523 579 s->readpos += cnt;
6c6c2796 580 return cnt;
214023c3
SR
581}
582
b8b94265 583static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
584{
585 int len;
3c56819b
EGM
586
587 if (s->len <= s->readpos)
588 return -EBUSY;
589
590 len = s->len - s->readpos;
591 if (cnt > len)
592 cnt = len;
5a26c8f0 593 memcpy(buf, s->buffer + s->readpos, cnt);
3c56819b 594
e74da523 595 s->readpos += cnt;
3c56819b
EGM
596 return cnt;
597}
598
5d4a9dba
SR
599/*
600 * ftrace_max_lock is used to protect the swapping of buffers
601 * when taking a max snapshot. The buffers themselves are
602 * protected by per_cpu spinlocks. But the action of the swap
603 * needs its own lock.
604 *
445c8951 605 * This is defined as a arch_spinlock_t in order to help
5d4a9dba
SR
606 * with performance when lockdep debugging is enabled.
607 *
608 * It is also used in other places outside the update_max_tr
609 * so it needs to be defined outside of the
610 * CONFIG_TRACER_MAX_TRACE.
611 */
445c8951 612static arch_spinlock_t ftrace_max_lock =
edc35bd7 613 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5d4a9dba 614
0e950173
TB
615unsigned long __read_mostly tracing_thresh;
616
5d4a9dba
SR
617#ifdef CONFIG_TRACER_MAX_TRACE
618unsigned long __read_mostly tracing_max_latency;
5d4a9dba
SR
619
620/*
621 * Copy the new maximum trace into the separate maximum-trace
622 * structure. (this way the maximum trace is permanently saved,
623 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
624 */
625static void
626__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
627{
12883efb
SRRH
628 struct trace_buffer *trace_buf = &tr->trace_buffer;
629 struct trace_buffer *max_buf = &tr->max_buffer;
630 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
631 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 632
12883efb
SRRH
633 max_buf->cpu = cpu;
634 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 635
8248ac05
SR
636 max_data->saved_latency = tracing_max_latency;
637 max_data->critical_start = data->critical_start;
638 max_data->critical_end = data->critical_end;
5d4a9dba 639
1acaa1b2 640 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05
SR
641 max_data->pid = tsk->pid;
642 max_data->uid = task_uid(tsk);
643 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
644 max_data->policy = tsk->policy;
645 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
646
647 /* record this tasks comm */
648 tracing_record_cmdline(tsk);
649}
650
4fcdae83
SR
651/**
652 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
653 * @tr: tracer
654 * @tsk: the task with the latency
655 * @cpu: The cpu that initiated the trace.
656 *
657 * Flip the buffers between the @tr and the max_tr and record information
658 * about which task was the cause of this latency.
659 */
e309b41d 660void
bc0c38d1
SR
661update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
662{
2721e72d 663 struct ring_buffer *buf;
bc0c38d1 664
2b6080f2 665 if (tr->stop_count)
b8de7bd1
SR
666 return;
667
4c11d7ae 668 WARN_ON_ONCE(!irqs_disabled());
34600f0e 669
45ad21ca 670 if (!tr->allocated_snapshot) {
debdd57f 671 /* Only the nop tracer should hit this when disabling */
2b6080f2 672 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 673 return;
debdd57f 674 }
34600f0e 675
0199c4e6 676 arch_spin_lock(&ftrace_max_lock);
3928a8a2 677
12883efb
SRRH
678 buf = tr->trace_buffer.buffer;
679 tr->trace_buffer.buffer = tr->max_buffer.buffer;
680 tr->max_buffer.buffer = buf;
3928a8a2 681
bc0c38d1 682 __update_max_tr(tr, tsk, cpu);
0199c4e6 683 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
684}
685
686/**
687 * update_max_tr_single - only copy one trace over, and reset the rest
688 * @tr - tracer
689 * @tsk - task with the latency
690 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
691 *
692 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 693 */
e309b41d 694void
bc0c38d1
SR
695update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
696{
3928a8a2 697 int ret;
bc0c38d1 698
2b6080f2 699 if (tr->stop_count)
b8de7bd1
SR
700 return;
701
4c11d7ae 702 WARN_ON_ONCE(!irqs_disabled());
45ad21ca 703 if (WARN_ON_ONCE(!tr->allocated_snapshot))
ef710e10 704 return;
ef710e10 705
0199c4e6 706 arch_spin_lock(&ftrace_max_lock);
bc0c38d1 707
12883efb 708 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 709
e8165dbb
SR
710 if (ret == -EBUSY) {
711 /*
712 * We failed to swap the buffer due to a commit taking
713 * place on this CPU. We fail to record, but we reset
714 * the max trace buffer (no one writes directly to it)
715 * and flag that it failed.
716 */
12883efb 717 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
718 "Failed to swap buffers due to commit in progress\n");
719 }
720
e8165dbb 721 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
722
723 __update_max_tr(tr, tsk, cpu);
0199c4e6 724 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1 725}
5d4a9dba 726#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 727
0d5c6e1c
SR
728static void default_wait_pipe(struct trace_iterator *iter)
729{
15693458
SRRH
730 /* Iterators are static, they should be filled or empty */
731 if (trace_buffer_iter(iter, iter->cpu_file))
732 return;
0d5c6e1c 733
12883efb 734 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
0d5c6e1c
SR
735}
736
4fcdae83
SR
737/**
738 * register_tracer - register a tracer with the ftrace system.
739 * @type - the plugin for the tracer
740 *
741 * Register a new plugin tracer.
742 */
bc0c38d1
SR
743int register_tracer(struct tracer *type)
744{
745 struct tracer *t;
bc0c38d1
SR
746 int ret = 0;
747
748 if (!type->name) {
749 pr_info("Tracer must have a name\n");
750 return -1;
751 }
752
24a461d5 753 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
754 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
755 return -1;
756 }
757
bc0c38d1 758 mutex_lock(&trace_types_lock);
86fa2f60 759
8e1b82e0
FW
760 tracing_selftest_running = true;
761
bc0c38d1
SR
762 for (t = trace_types; t; t = t->next) {
763 if (strcmp(type->name, t->name) == 0) {
764 /* already found */
ee6c2c1b 765 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
766 type->name);
767 ret = -1;
768 goto out;
769 }
770 }
771
adf9f195
FW
772 if (!type->set_flag)
773 type->set_flag = &dummy_set_flag;
774 if (!type->flags)
775 type->flags = &dummy_tracer_flags;
776 else
777 if (!type->flags->opts)
778 type->flags->opts = dummy_tracer_opt;
6eaaa5d5
FW
779 if (!type->wait_pipe)
780 type->wait_pipe = default_wait_pipe;
781
adf9f195 782
60a11774 783#ifdef CONFIG_FTRACE_STARTUP_TEST
b2821ae6 784 if (type->selftest && !tracing_selftest_disabled) {
60a11774 785 struct trace_array *tr = &global_trace;
2b6080f2 786 struct tracer *saved_tracer = tr->current_trace;
ff32504f 787
60a11774
SR
788 /*
789 * Run a selftest on this tracer.
790 * Here we reset the trace buffer, and set the current
791 * tracer to be this tracer. The tracer can then run some
792 * internal tracing to verify that everything is in order.
793 * If we fail, we do not register this tracer.
794 */
12883efb 795 tracing_reset_online_cpus(&tr->trace_buffer);
86fa2f60 796
2b6080f2 797 tr->current_trace = type;
4a0b1665 798
12883efb 799#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
800 if (type->use_max_tr) {
801 /* If we expanded the buffers, make sure the max is expanded too */
802 if (ring_buffer_expanded)
12883efb 803 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
debdd57f 804 RING_BUFFER_ALL_CPUS);
45ad21ca 805 tr->allocated_snapshot = true;
debdd57f 806 }
12883efb 807#endif
4a0b1665 808
60a11774
SR
809 /* the test is responsible for initializing and enabling */
810 pr_info("Testing tracer %s: ", type->name);
811 ret = type->selftest(type, tr);
812 /* the test is responsible for resetting too */
2b6080f2 813 tr->current_trace = saved_tracer;
60a11774
SR
814 if (ret) {
815 printk(KERN_CONT "FAILED!\n");
0be61ebc
SR
816 /* Add the warning after printing 'FAILED' */
817 WARN_ON(1);
60a11774
SR
818 goto out;
819 }
1d4db00a 820 /* Only reset on passing, to avoid touching corrupted buffers */
12883efb 821 tracing_reset_online_cpus(&tr->trace_buffer);
86fa2f60 822
12883efb 823#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f 824 if (type->use_max_tr) {
45ad21ca 825 tr->allocated_snapshot = false;
debdd57f
HT
826
827 /* Shrink the max buffer again */
828 if (ring_buffer_expanded)
12883efb 829 ring_buffer_resize(tr->max_buffer.buffer, 1,
debdd57f
HT
830 RING_BUFFER_ALL_CPUS);
831 }
12883efb 832#endif
4a0b1665 833
60a11774
SR
834 printk(KERN_CONT "PASSED\n");
835 }
836#endif
837
bc0c38d1
SR
838 type->next = trace_types;
839 trace_types = type;
60a11774 840
bc0c38d1 841 out:
8e1b82e0 842 tracing_selftest_running = false;
bc0c38d1
SR
843 mutex_unlock(&trace_types_lock);
844
dac74940
SR
845 if (ret || !default_bootup_tracer)
846 goto out_unlock;
847
ee6c2c1b 848 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
849 goto out_unlock;
850
851 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
852 /* Do we want this tracer to start on bootup? */
853 tracing_set_tracer(type->name);
854 default_bootup_tracer = NULL;
855 /* disable other selftests, since this will break it. */
856 tracing_selftest_disabled = 1;
b2821ae6 857#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
858 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
859 type->name);
b2821ae6 860#endif
b2821ae6 861
dac74940 862 out_unlock:
bc0c38d1
SR
863 return ret;
864}
865
12883efb 866void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 867{
12883efb 868 struct ring_buffer *buffer = buf->buffer;
f633903a 869
a5416411
HT
870 if (!buffer)
871 return;
872
f633903a
SR
873 ring_buffer_record_disable(buffer);
874
875 /* Make sure all commits have finished */
876 synchronize_sched();
68179686 877 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
878
879 ring_buffer_record_enable(buffer);
880}
881
12883efb 882void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 883{
12883efb 884 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
885 int cpu;
886
a5416411
HT
887 if (!buffer)
888 return;
889
621968cd
SR
890 ring_buffer_record_disable(buffer);
891
892 /* Make sure all commits have finished */
893 synchronize_sched();
894
12883efb 895 buf->time_start = ftrace_now(buf->cpu);
213cc060
PE
896
897 for_each_online_cpu(cpu)
68179686 898 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
899
900 ring_buffer_record_enable(buffer);
213cc060
PE
901}
902
9456f0fa
SR
903void tracing_reset_current(int cpu)
904{
12883efb 905 tracing_reset(&global_trace.trace_buffer, cpu);
9456f0fa
SR
906}
907
873c642f 908void tracing_reset_all_online_cpus(void)
9456f0fa 909{
873c642f
SRRH
910 struct trace_array *tr;
911
912 mutex_lock(&trace_types_lock);
913 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
12883efb
SRRH
914 tracing_reset_online_cpus(&tr->trace_buffer);
915#ifdef CONFIG_TRACER_MAX_TRACE
916 tracing_reset_online_cpus(&tr->max_buffer);
917#endif
873c642f
SRRH
918 }
919 mutex_unlock(&trace_types_lock);
9456f0fa
SR
920}
921
bc0c38d1 922#define SAVED_CMDLINES 128
2c7eea4c 923#define NO_CMDLINE_MAP UINT_MAX
bc0c38d1
SR
924static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
925static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
926static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
927static int cmdline_idx;
edc35bd7 928static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
25b0b44a 929
25b0b44a 930/* temporary disable recording */
4fd27358 931static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1
SR
932
933static void trace_init_cmdlines(void)
934{
2c7eea4c
TG
935 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
936 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
bc0c38d1
SR
937 cmdline_idx = 0;
938}
939
b5130b1e
CE
940int is_tracing_stopped(void)
941{
2b6080f2 942 return global_trace.stop_count;
b5130b1e
CE
943}
944
69bb54ec
SR
945/**
946 * ftrace_off_permanent - disable all ftrace code permanently
947 *
948 * This should only be called when a serious anomally has
949 * been detected. This will turn off the function tracing,
950 * ring buffers, and other tracing utilites. It takes no
951 * locks and can be called from any context.
952 */
953void ftrace_off_permanent(void)
954{
955 tracing_disabled = 1;
956 ftrace_stop();
957 tracing_off_permanent();
958}
959
0f048701
SR
960/**
961 * tracing_start - quick start of the tracer
962 *
963 * If tracing is enabled but was stopped by tracing_stop,
964 * this will start the tracer back up.
965 */
966void tracing_start(void)
967{
968 struct ring_buffer *buffer;
969 unsigned long flags;
970
971 if (tracing_disabled)
972 return;
973
2b6080f2
SR
974 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
975 if (--global_trace.stop_count) {
976 if (global_trace.stop_count < 0) {
b06a8301
SR
977 /* Someone screwed up their debugging */
978 WARN_ON_ONCE(1);
2b6080f2 979 global_trace.stop_count = 0;
b06a8301 980 }
0f048701
SR
981 goto out;
982 }
983
a2f80714
SR
984 /* Prevent the buffers from switching */
985 arch_spin_lock(&ftrace_max_lock);
0f048701 986
12883efb 987 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
988 if (buffer)
989 ring_buffer_record_enable(buffer);
990
12883efb
SRRH
991#ifdef CONFIG_TRACER_MAX_TRACE
992 buffer = global_trace.max_buffer.buffer;
0f048701
SR
993 if (buffer)
994 ring_buffer_record_enable(buffer);
12883efb 995#endif
0f048701 996
a2f80714
SR
997 arch_spin_unlock(&ftrace_max_lock);
998
0f048701
SR
999 ftrace_start();
1000 out:
2b6080f2
SR
1001 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1002}
1003
1004static void tracing_start_tr(struct trace_array *tr)
1005{
1006 struct ring_buffer *buffer;
1007 unsigned long flags;
1008
1009 if (tracing_disabled)
1010 return;
1011
1012 /* If global, we need to also start the max tracer */
1013 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1014 return tracing_start();
1015
1016 raw_spin_lock_irqsave(&tr->start_lock, flags);
1017
1018 if (--tr->stop_count) {
1019 if (tr->stop_count < 0) {
1020 /* Someone screwed up their debugging */
1021 WARN_ON_ONCE(1);
1022 tr->stop_count = 0;
1023 }
1024 goto out;
1025 }
1026
12883efb 1027 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1028 if (buffer)
1029 ring_buffer_record_enable(buffer);
1030
1031 out:
1032 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1033}
1034
1035/**
1036 * tracing_stop - quick stop of the tracer
1037 *
1038 * Light weight way to stop tracing. Use in conjunction with
1039 * tracing_start.
1040 */
1041void tracing_stop(void)
1042{
1043 struct ring_buffer *buffer;
1044 unsigned long flags;
1045
1046 ftrace_stop();
2b6080f2
SR
1047 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1048 if (global_trace.stop_count++)
0f048701
SR
1049 goto out;
1050
a2f80714
SR
1051 /* Prevent the buffers from switching */
1052 arch_spin_lock(&ftrace_max_lock);
1053
12883efb 1054 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1055 if (buffer)
1056 ring_buffer_record_disable(buffer);
1057
12883efb
SRRH
1058#ifdef CONFIG_TRACER_MAX_TRACE
1059 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1060 if (buffer)
1061 ring_buffer_record_disable(buffer);
12883efb 1062#endif
0f048701 1063
a2f80714
SR
1064 arch_spin_unlock(&ftrace_max_lock);
1065
0f048701 1066 out:
2b6080f2
SR
1067 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1068}
1069
1070static void tracing_stop_tr(struct trace_array *tr)
1071{
1072 struct ring_buffer *buffer;
1073 unsigned long flags;
1074
1075 /* If global, we need to also stop the max tracer */
1076 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1077 return tracing_stop();
1078
1079 raw_spin_lock_irqsave(&tr->start_lock, flags);
1080 if (tr->stop_count++)
1081 goto out;
1082
12883efb 1083 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1084 if (buffer)
1085 ring_buffer_record_disable(buffer);
1086
1087 out:
1088 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1089}
1090
e309b41d 1091void trace_stop_cmdline_recording(void);
bc0c38d1 1092
e309b41d 1093static void trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1094{
a635cf04 1095 unsigned pid, idx;
bc0c38d1
SR
1096
1097 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1098 return;
1099
1100 /*
1101 * It's not the end of the world if we don't get
1102 * the lock, but we also don't want to spin
1103 * nor do we want to disable interrupts,
1104 * so if we miss here, then better luck next time.
1105 */
0199c4e6 1106 if (!arch_spin_trylock(&trace_cmdline_lock))
bc0c38d1
SR
1107 return;
1108
1109 idx = map_pid_to_cmdline[tsk->pid];
2c7eea4c 1110 if (idx == NO_CMDLINE_MAP) {
bc0c38d1
SR
1111 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1112
a635cf04
CE
1113 /*
1114 * Check whether the cmdline buffer at idx has a pid
1115 * mapped. We are going to overwrite that entry so we
1116 * need to clear the map_pid_to_cmdline. Otherwise we
1117 * would read the new comm for the old pid.
1118 */
1119 pid = map_cmdline_to_pid[idx];
1120 if (pid != NO_CMDLINE_MAP)
1121 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1122
a635cf04 1123 map_cmdline_to_pid[idx] = tsk->pid;
bc0c38d1
SR
1124 map_pid_to_cmdline[tsk->pid] = idx;
1125
1126 cmdline_idx = idx;
1127 }
1128
1129 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1130
0199c4e6 1131 arch_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
1132}
1133
4ca53085 1134void trace_find_cmdline(int pid, char comm[])
bc0c38d1 1135{
bc0c38d1
SR
1136 unsigned map;
1137
4ca53085
SR
1138 if (!pid) {
1139 strcpy(comm, "<idle>");
1140 return;
1141 }
bc0c38d1 1142
74bf4076
SR
1143 if (WARN_ON_ONCE(pid < 0)) {
1144 strcpy(comm, "<XXX>");
1145 return;
1146 }
1147
4ca53085
SR
1148 if (pid > PID_MAX_DEFAULT) {
1149 strcpy(comm, "<...>");
1150 return;
1151 }
bc0c38d1 1152
5b6045a9 1153 preempt_disable();
0199c4e6 1154 arch_spin_lock(&trace_cmdline_lock);
bc0c38d1 1155 map = map_pid_to_cmdline[pid];
50d88758
TG
1156 if (map != NO_CMDLINE_MAP)
1157 strcpy(comm, saved_cmdlines[map]);
1158 else
1159 strcpy(comm, "<...>");
bc0c38d1 1160
0199c4e6 1161 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1162 preempt_enable();
bc0c38d1
SR
1163}
1164
e309b41d 1165void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1166{
0fb9656d 1167 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
bc0c38d1
SR
1168 return;
1169
7ffbd48d
SR
1170 if (!__this_cpu_read(trace_cmdline_save))
1171 return;
1172
1173 __this_cpu_write(trace_cmdline_save, false);
1174
bc0c38d1
SR
1175 trace_save_cmdline(tsk);
1176}
1177
45dcd8b8 1178void
38697053
SR
1179tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1180 int pc)
bc0c38d1
SR
1181{
1182 struct task_struct *tsk = current;
bc0c38d1 1183
777e208d
SR
1184 entry->preempt_count = pc & 0xff;
1185 entry->pid = (tsk) ? tsk->pid : 0;
1186 entry->flags =
9244489a 1187#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1188 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1189#else
1190 TRACE_FLAG_IRQS_NOSUPPORT |
1191#endif
bc0c38d1
SR
1192 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1193 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1194 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1195}
f413cdb8 1196EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1197
e77405ad
SR
1198struct ring_buffer_event *
1199trace_buffer_lock_reserve(struct ring_buffer *buffer,
1200 int type,
1201 unsigned long len,
1202 unsigned long flags, int pc)
51a763dd
ACM
1203{
1204 struct ring_buffer_event *event;
1205
e77405ad 1206 event = ring_buffer_lock_reserve(buffer, len);
51a763dd
ACM
1207 if (event != NULL) {
1208 struct trace_entry *ent = ring_buffer_event_data(event);
1209
1210 tracing_generic_entry_update(ent, flags, pc);
1211 ent->type = type;
1212 }
1213
1214 return event;
1215}
51a763dd 1216
7ffbd48d
SR
1217void
1218__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1219{
1220 __this_cpu_write(trace_cmdline_save, true);
1221 ring_buffer_unlock_commit(buffer, event);
1222}
1223
e77405ad
SR
1224static inline void
1225__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1226 struct ring_buffer_event *event,
0d5c6e1c 1227 unsigned long flags, int pc)
51a763dd 1228{
7ffbd48d 1229 __buffer_unlock_commit(buffer, event);
51a763dd 1230
e77405ad
SR
1231 ftrace_trace_stack(buffer, flags, 6, pc);
1232 ftrace_trace_userstack(buffer, flags, pc);
07edf712
FW
1233}
1234
e77405ad
SR
1235void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1236 struct ring_buffer_event *event,
1237 unsigned long flags, int pc)
07edf712 1238{
0d5c6e1c 1239 __trace_buffer_unlock_commit(buffer, event, flags, pc);
51a763dd 1240}
0d5c6e1c 1241EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
51a763dd 1242
ccb469a1
SR
1243struct ring_buffer_event *
1244trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1245 struct ftrace_event_file *ftrace_file,
1246 int type, unsigned long len,
1247 unsigned long flags, int pc)
1248{
12883efb 1249 *current_rb = ftrace_file->tr->trace_buffer.buffer;
ccb469a1
SR
1250 return trace_buffer_lock_reserve(*current_rb,
1251 type, len, flags, pc);
1252}
1253EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1254
ef5580d0 1255struct ring_buffer_event *
e77405ad
SR
1256trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1257 int type, unsigned long len,
ef5580d0
SR
1258 unsigned long flags, int pc)
1259{
12883efb 1260 *current_rb = global_trace.trace_buffer.buffer;
e77405ad 1261 return trace_buffer_lock_reserve(*current_rb,
ef5580d0
SR
1262 type, len, flags, pc);
1263}
94487d6d 1264EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0 1265
e77405ad
SR
1266void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1267 struct ring_buffer_event *event,
ef5580d0
SR
1268 unsigned long flags, int pc)
1269{
0d5c6e1c 1270 __trace_buffer_unlock_commit(buffer, event, flags, pc);
07edf712 1271}
94487d6d 1272EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712 1273
0d5c6e1c
SR
1274void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1275 struct ring_buffer_event *event,
1276 unsigned long flags, int pc,
1277 struct pt_regs *regs)
1fd8df2c 1278{
7ffbd48d 1279 __buffer_unlock_commit(buffer, event);
1fd8df2c
MH
1280
1281 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1282 ftrace_trace_userstack(buffer, flags, pc);
1283}
0d5c6e1c 1284EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1fd8df2c 1285
e77405ad
SR
1286void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1287 struct ring_buffer_event *event)
77d9f465 1288{
e77405ad 1289 ring_buffer_discard_commit(buffer, event);
ef5580d0 1290}
12acd473 1291EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 1292
e309b41d 1293void
7be42151 1294trace_function(struct trace_array *tr,
38697053
SR
1295 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1296 int pc)
bc0c38d1 1297{
e1112b4d 1298 struct ftrace_event_call *call = &event_function;
12883efb 1299 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 1300 struct ring_buffer_event *event;
777e208d 1301 struct ftrace_entry *entry;
bc0c38d1 1302
d769041f 1303 /* If we are reading the ring buffer, don't trace */
dd17c8f7 1304 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
d769041f
SR
1305 return;
1306
e77405ad 1307 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
51a763dd 1308 flags, pc);
3928a8a2
SR
1309 if (!event)
1310 return;
1311 entry = ring_buffer_event_data(event);
777e208d
SR
1312 entry->ip = ip;
1313 entry->parent_ip = parent_ip;
e1112b4d 1314
e77405ad 1315 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1316 __buffer_unlock_commit(buffer, event);
bc0c38d1
SR
1317}
1318
e309b41d 1319void
2e0f5761 1320ftrace(struct trace_array *tr, struct trace_array_cpu *data,
38697053
SR
1321 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1322 int pc)
2e0f5761
IM
1323{
1324 if (likely(!atomic_read(&data->disabled)))
7be42151 1325 trace_function(tr, ip, parent_ip, flags, pc);
2e0f5761
IM
1326}
1327
c0a0d0d3 1328#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
1329
1330#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1331struct ftrace_stack {
1332 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1333};
1334
1335static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1336static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1337
e77405ad 1338static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 1339 unsigned long flags,
1fd8df2c 1340 int skip, int pc, struct pt_regs *regs)
86387f7e 1341{
e1112b4d 1342 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1343 struct ring_buffer_event *event;
777e208d 1344 struct stack_entry *entry;
86387f7e 1345 struct stack_trace trace;
4a9bd3f1
SR
1346 int use_stack;
1347 int size = FTRACE_STACK_ENTRIES;
1348
1349 trace.nr_entries = 0;
1350 trace.skip = skip;
1351
1352 /*
1353 * Since events can happen in NMIs there's no safe way to
1354 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1355 * or NMI comes in, it will just have to use the default
1356 * FTRACE_STACK_SIZE.
1357 */
1358 preempt_disable_notrace();
1359
82146529 1360 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
1361 /*
1362 * We don't need any atomic variables, just a barrier.
1363 * If an interrupt comes in, we don't care, because it would
1364 * have exited and put the counter back to what we want.
1365 * We just need a barrier to keep gcc from moving things
1366 * around.
1367 */
1368 barrier();
1369 if (use_stack == 1) {
1370 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1371 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1372
1373 if (regs)
1374 save_stack_trace_regs(regs, &trace);
1375 else
1376 save_stack_trace(&trace);
1377
1378 if (trace.nr_entries > size)
1379 size = trace.nr_entries;
1380 } else
1381 /* From now on, use_stack is a boolean */
1382 use_stack = 0;
1383
1384 size *= sizeof(unsigned long);
86387f7e 1385
e77405ad 1386 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
4a9bd3f1 1387 sizeof(*entry) + size, flags, pc);
3928a8a2 1388 if (!event)
4a9bd3f1
SR
1389 goto out;
1390 entry = ring_buffer_event_data(event);
86387f7e 1391
4a9bd3f1
SR
1392 memset(&entry->caller, 0, size);
1393
1394 if (use_stack)
1395 memcpy(&entry->caller, trace.entries,
1396 trace.nr_entries * sizeof(unsigned long));
1397 else {
1398 trace.max_entries = FTRACE_STACK_ENTRIES;
1399 trace.entries = entry->caller;
1400 if (regs)
1401 save_stack_trace_regs(regs, &trace);
1402 else
1403 save_stack_trace(&trace);
1404 }
1405
1406 entry->size = trace.nr_entries;
86387f7e 1407
e77405ad 1408 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1409 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
1410
1411 out:
1412 /* Again, don't let gcc optimize things here */
1413 barrier();
82146529 1414 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
1415 preempt_enable_notrace();
1416
f0a920d5
IM
1417}
1418
1fd8df2c
MH
1419void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1420 int skip, int pc, struct pt_regs *regs)
1421{
1422 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1423 return;
1424
1425 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1426}
1427
e77405ad
SR
1428void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1429 int skip, int pc)
53614991
SR
1430{
1431 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1432 return;
1433
1fd8df2c 1434 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
53614991
SR
1435}
1436
c0a0d0d3
FW
1437void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1438 int pc)
38697053 1439{
12883efb 1440 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
38697053
SR
1441}
1442
03889384
SR
1443/**
1444 * trace_dump_stack - record a stack back trace in the trace buffer
1445 */
1446void trace_dump_stack(void)
1447{
1448 unsigned long flags;
1449
1450 if (tracing_disabled || tracing_selftest_running)
e36c5458 1451 return;
03889384
SR
1452
1453 local_save_flags(flags);
1454
1455 /* skipping 3 traces, seems to get us at the caller of this function */
12883efb
SRRH
1456 __ftrace_trace_stack(global_trace.trace_buffer.buffer, flags, 3,
1457 preempt_count(), NULL);
03889384
SR
1458}
1459
91e86e56
SR
1460static DEFINE_PER_CPU(int, user_stack_count);
1461
e77405ad
SR
1462void
1463ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 1464{
e1112b4d 1465 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1466 struct ring_buffer_event *event;
02b67518
TE
1467 struct userstack_entry *entry;
1468 struct stack_trace trace;
02b67518
TE
1469
1470 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1471 return;
1472
b6345879
SR
1473 /*
1474 * NMIs can not handle page faults, even with fix ups.
1475 * The save user stack can (and often does) fault.
1476 */
1477 if (unlikely(in_nmi()))
1478 return;
02b67518 1479
91e86e56
SR
1480 /*
1481 * prevent recursion, since the user stack tracing may
1482 * trigger other kernel events.
1483 */
1484 preempt_disable();
1485 if (__this_cpu_read(user_stack_count))
1486 goto out;
1487
1488 __this_cpu_inc(user_stack_count);
1489
e77405ad 1490 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
51a763dd 1491 sizeof(*entry), flags, pc);
02b67518 1492 if (!event)
1dbd1951 1493 goto out_drop_count;
02b67518 1494 entry = ring_buffer_event_data(event);
02b67518 1495
48659d31 1496 entry->tgid = current->tgid;
02b67518
TE
1497 memset(&entry->caller, 0, sizeof(entry->caller));
1498
1499 trace.nr_entries = 0;
1500 trace.max_entries = FTRACE_STACK_ENTRIES;
1501 trace.skip = 0;
1502 trace.entries = entry->caller;
1503
1504 save_stack_trace_user(&trace);
e77405ad 1505 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1506 __buffer_unlock_commit(buffer, event);
91e86e56 1507
1dbd1951 1508 out_drop_count:
91e86e56 1509 __this_cpu_dec(user_stack_count);
91e86e56
SR
1510 out:
1511 preempt_enable();
02b67518
TE
1512}
1513
4fd27358
HE
1514#ifdef UNUSED
1515static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1516{
7be42151 1517 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1518}
4fd27358 1519#endif /* UNUSED */
02b67518 1520
c0a0d0d3
FW
1521#endif /* CONFIG_STACKTRACE */
1522
07d777fe
SR
1523/* created for use with alloc_percpu */
1524struct trace_buffer_struct {
1525 char buffer[TRACE_BUF_SIZE];
1526};
1527
1528static struct trace_buffer_struct *trace_percpu_buffer;
1529static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1530static struct trace_buffer_struct *trace_percpu_irq_buffer;
1531static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1532
1533/*
1534 * The buffer used is dependent on the context. There is a per cpu
1535 * buffer for normal context, softirq contex, hard irq context and
1536 * for NMI context. Thise allows for lockless recording.
1537 *
1538 * Note, if the buffers failed to be allocated, then this returns NULL
1539 */
1540static char *get_trace_buf(void)
1541{
1542 struct trace_buffer_struct *percpu_buffer;
07d777fe
SR
1543
1544 /*
1545 * If we have allocated per cpu buffers, then we do not
1546 * need to do any locking.
1547 */
1548 if (in_nmi())
1549 percpu_buffer = trace_percpu_nmi_buffer;
1550 else if (in_irq())
1551 percpu_buffer = trace_percpu_irq_buffer;
1552 else if (in_softirq())
1553 percpu_buffer = trace_percpu_sirq_buffer;
1554 else
1555 percpu_buffer = trace_percpu_buffer;
1556
1557 if (!percpu_buffer)
1558 return NULL;
1559
d8a0349c 1560 return this_cpu_ptr(&percpu_buffer->buffer[0]);
07d777fe
SR
1561}
1562
1563static int alloc_percpu_trace_buffer(void)
1564{
1565 struct trace_buffer_struct *buffers;
1566 struct trace_buffer_struct *sirq_buffers;
1567 struct trace_buffer_struct *irq_buffers;
1568 struct trace_buffer_struct *nmi_buffers;
1569
1570 buffers = alloc_percpu(struct trace_buffer_struct);
1571 if (!buffers)
1572 goto err_warn;
1573
1574 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1575 if (!sirq_buffers)
1576 goto err_sirq;
1577
1578 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1579 if (!irq_buffers)
1580 goto err_irq;
1581
1582 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1583 if (!nmi_buffers)
1584 goto err_nmi;
1585
1586 trace_percpu_buffer = buffers;
1587 trace_percpu_sirq_buffer = sirq_buffers;
1588 trace_percpu_irq_buffer = irq_buffers;
1589 trace_percpu_nmi_buffer = nmi_buffers;
1590
1591 return 0;
1592
1593 err_nmi:
1594 free_percpu(irq_buffers);
1595 err_irq:
1596 free_percpu(sirq_buffers);
1597 err_sirq:
1598 free_percpu(buffers);
1599 err_warn:
1600 WARN(1, "Could not allocate percpu trace_printk buffer");
1601 return -ENOMEM;
1602}
1603
81698831
SR
1604static int buffers_allocated;
1605
07d777fe
SR
1606void trace_printk_init_buffers(void)
1607{
07d777fe
SR
1608 if (buffers_allocated)
1609 return;
1610
1611 if (alloc_percpu_trace_buffer())
1612 return;
1613
1614 pr_info("ftrace: Allocated trace_printk buffers\n");
1615
b382ede6
SR
1616 /* Expand the buffers to set size */
1617 tracing_update_buffers();
1618
07d777fe 1619 buffers_allocated = 1;
81698831
SR
1620
1621 /*
1622 * trace_printk_init_buffers() can be called by modules.
1623 * If that happens, then we need to start cmdline recording
1624 * directly here. If the global_trace.buffer is already
1625 * allocated here, then this was called by module code.
1626 */
12883efb 1627 if (global_trace.trace_buffer.buffer)
81698831
SR
1628 tracing_start_cmdline_record();
1629}
1630
1631void trace_printk_start_comm(void)
1632{
1633 /* Start tracing comms if trace printk is set */
1634 if (!buffers_allocated)
1635 return;
1636 tracing_start_cmdline_record();
1637}
1638
1639static void trace_printk_start_stop_comm(int enabled)
1640{
1641 if (!buffers_allocated)
1642 return;
1643
1644 if (enabled)
1645 tracing_start_cmdline_record();
1646 else
1647 tracing_stop_cmdline_record();
07d777fe
SR
1648}
1649
769b0441 1650/**
48ead020 1651 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
1652 *
1653 */
40ce74f1 1654int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 1655{
e1112b4d 1656 struct ftrace_event_call *call = &event_bprint;
769b0441 1657 struct ring_buffer_event *event;
e77405ad 1658 struct ring_buffer *buffer;
769b0441 1659 struct trace_array *tr = &global_trace;
48ead020 1660 struct bprint_entry *entry;
769b0441 1661 unsigned long flags;
07d777fe
SR
1662 char *tbuffer;
1663 int len = 0, size, pc;
769b0441
FW
1664
1665 if (unlikely(tracing_selftest_running || tracing_disabled))
1666 return 0;
1667
1668 /* Don't pollute graph traces with trace_vprintk internals */
1669 pause_graph_tracing();
1670
1671 pc = preempt_count();
5168ae50 1672 preempt_disable_notrace();
769b0441 1673
07d777fe
SR
1674 tbuffer = get_trace_buf();
1675 if (!tbuffer) {
1676 len = 0;
769b0441 1677 goto out;
07d777fe 1678 }
769b0441 1679
07d777fe 1680 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 1681
07d777fe
SR
1682 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1683 goto out;
769b0441 1684
07d777fe 1685 local_save_flags(flags);
769b0441 1686 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 1687 buffer = tr->trace_buffer.buffer;
e77405ad
SR
1688 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1689 flags, pc);
769b0441 1690 if (!event)
07d777fe 1691 goto out;
769b0441
FW
1692 entry = ring_buffer_event_data(event);
1693 entry->ip = ip;
769b0441
FW
1694 entry->fmt = fmt;
1695
07d777fe 1696 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
d931369b 1697 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 1698 __buffer_unlock_commit(buffer, event);
d931369b
SR
1699 ftrace_trace_stack(buffer, flags, 6, pc);
1700 }
769b0441 1701
769b0441 1702out:
5168ae50 1703 preempt_enable_notrace();
769b0441
FW
1704 unpause_graph_tracing();
1705
1706 return len;
1707}
48ead020
FW
1708EXPORT_SYMBOL_GPL(trace_vbprintk);
1709
12883efb
SRRH
1710static int
1711__trace_array_vprintk(struct ring_buffer *buffer,
1712 unsigned long ip, const char *fmt, va_list args)
48ead020 1713{
e1112b4d 1714 struct ftrace_event_call *call = &event_print;
48ead020 1715 struct ring_buffer_event *event;
07d777fe 1716 int len = 0, size, pc;
48ead020 1717 struct print_entry *entry;
07d777fe
SR
1718 unsigned long flags;
1719 char *tbuffer;
48ead020
FW
1720
1721 if (tracing_disabled || tracing_selftest_running)
1722 return 0;
1723
07d777fe
SR
1724 /* Don't pollute graph traces with trace_vprintk internals */
1725 pause_graph_tracing();
1726
48ead020
FW
1727 pc = preempt_count();
1728 preempt_disable_notrace();
48ead020 1729
07d777fe
SR
1730
1731 tbuffer = get_trace_buf();
1732 if (!tbuffer) {
1733 len = 0;
48ead020 1734 goto out;
07d777fe 1735 }
48ead020 1736
07d777fe
SR
1737 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1738 if (len > TRACE_BUF_SIZE)
1739 goto out;
48ead020 1740
07d777fe 1741 local_save_flags(flags);
48ead020 1742 size = sizeof(*entry) + len + 1;
e77405ad 1743 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
07d777fe 1744 flags, pc);
48ead020 1745 if (!event)
07d777fe 1746 goto out;
48ead020 1747 entry = ring_buffer_event_data(event);
c13d2f7c 1748 entry->ip = ip;
48ead020 1749
07d777fe 1750 memcpy(&entry->buf, tbuffer, len);
c13d2f7c 1751 entry->buf[len] = '\0';
d931369b 1752 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 1753 __buffer_unlock_commit(buffer, event);
07d777fe 1754 ftrace_trace_stack(buffer, flags, 6, pc);
d931369b 1755 }
48ead020
FW
1756 out:
1757 preempt_enable_notrace();
07d777fe 1758 unpause_graph_tracing();
48ead020
FW
1759
1760 return len;
1761}
659372d3 1762
12883efb
SRRH
1763int trace_array_vprintk(struct trace_array *tr,
1764 unsigned long ip, const char *fmt, va_list args)
1765{
1766 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1767}
1768
1769int trace_array_printk(struct trace_array *tr,
1770 unsigned long ip, const char *fmt, ...)
1771{
1772 int ret;
1773 va_list ap;
1774
1775 if (!(trace_flags & TRACE_ITER_PRINTK))
1776 return 0;
1777
1778 va_start(ap, fmt);
1779 ret = trace_array_vprintk(tr, ip, fmt, ap);
1780 va_end(ap);
1781 return ret;
1782}
1783
1784int trace_array_printk_buf(struct ring_buffer *buffer,
1785 unsigned long ip, const char *fmt, ...)
1786{
1787 int ret;
1788 va_list ap;
1789
1790 if (!(trace_flags & TRACE_ITER_PRINTK))
1791 return 0;
1792
1793 va_start(ap, fmt);
1794 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
1795 va_end(ap);
1796 return ret;
1797}
1798
659372d3
SR
1799int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1800{
a813a159 1801 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 1802}
769b0441
FW
1803EXPORT_SYMBOL_GPL(trace_vprintk);
1804
e2ac8ef5 1805static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 1806{
6d158a81
SR
1807 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1808
5a90f577 1809 iter->idx++;
6d158a81
SR
1810 if (buf_iter)
1811 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
1812}
1813
e309b41d 1814static struct trace_entry *
bc21b478
SR
1815peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1816 unsigned long *lost_events)
dd0e545f 1817{
3928a8a2 1818 struct ring_buffer_event *event;
6d158a81 1819 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 1820
d769041f
SR
1821 if (buf_iter)
1822 event = ring_buffer_iter_peek(buf_iter, ts);
1823 else
12883efb 1824 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 1825 lost_events);
d769041f 1826
4a9bd3f1
SR
1827 if (event) {
1828 iter->ent_size = ring_buffer_event_length(event);
1829 return ring_buffer_event_data(event);
1830 }
1831 iter->ent_size = 0;
1832 return NULL;
dd0e545f 1833}
d769041f 1834
dd0e545f 1835static struct trace_entry *
bc21b478
SR
1836__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1837 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 1838{
12883efb 1839 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 1840 struct trace_entry *ent, *next = NULL;
aa27497c 1841 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 1842 int cpu_file = iter->cpu_file;
3928a8a2 1843 u64 next_ts = 0, ts;
bc0c38d1 1844 int next_cpu = -1;
12b5da34 1845 int next_size = 0;
bc0c38d1
SR
1846 int cpu;
1847
b04cc6b1
FW
1848 /*
1849 * If we are in a per_cpu trace file, don't bother by iterating over
1850 * all cpu and peek directly.
1851 */
ae3b5093 1852 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
1853 if (ring_buffer_empty_cpu(buffer, cpu_file))
1854 return NULL;
bc21b478 1855 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
1856 if (ent_cpu)
1857 *ent_cpu = cpu_file;
1858
1859 return ent;
1860 }
1861
ab46428c 1862 for_each_tracing_cpu(cpu) {
dd0e545f 1863
3928a8a2
SR
1864 if (ring_buffer_empty_cpu(buffer, cpu))
1865 continue;
dd0e545f 1866
bc21b478 1867 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 1868
cdd31cd2
IM
1869 /*
1870 * Pick the entry with the smallest timestamp:
1871 */
3928a8a2 1872 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
1873 next = ent;
1874 next_cpu = cpu;
3928a8a2 1875 next_ts = ts;
bc21b478 1876 next_lost = lost_events;
12b5da34 1877 next_size = iter->ent_size;
bc0c38d1
SR
1878 }
1879 }
1880
12b5da34
SR
1881 iter->ent_size = next_size;
1882
bc0c38d1
SR
1883 if (ent_cpu)
1884 *ent_cpu = next_cpu;
1885
3928a8a2
SR
1886 if (ent_ts)
1887 *ent_ts = next_ts;
1888
bc21b478
SR
1889 if (missing_events)
1890 *missing_events = next_lost;
1891
bc0c38d1
SR
1892 return next;
1893}
1894
dd0e545f 1895/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
1896struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1897 int *ent_cpu, u64 *ent_ts)
bc0c38d1 1898{
bc21b478 1899 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
1900}
1901
1902/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 1903void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 1904{
bc21b478
SR
1905 iter->ent = __find_next_entry(iter, &iter->cpu,
1906 &iter->lost_events, &iter->ts);
dd0e545f 1907
3928a8a2 1908 if (iter->ent)
e2ac8ef5 1909 trace_iterator_increment(iter);
dd0e545f 1910
3928a8a2 1911 return iter->ent ? iter : NULL;
b3806b43 1912}
bc0c38d1 1913
e309b41d 1914static void trace_consume(struct trace_iterator *iter)
b3806b43 1915{
12883efb 1916 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 1917 &iter->lost_events);
bc0c38d1
SR
1918}
1919
e309b41d 1920static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
1921{
1922 struct trace_iterator *iter = m->private;
bc0c38d1 1923 int i = (int)*pos;
4e3c3333 1924 void *ent;
bc0c38d1 1925
a63ce5b3
SR
1926 WARN_ON_ONCE(iter->leftover);
1927
bc0c38d1
SR
1928 (*pos)++;
1929
1930 /* can't go backwards */
1931 if (iter->idx > i)
1932 return NULL;
1933
1934 if (iter->idx < 0)
955b61e5 1935 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
1936 else
1937 ent = iter;
1938
1939 while (ent && iter->idx < i)
955b61e5 1940 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
1941
1942 iter->pos = *pos;
1943
bc0c38d1
SR
1944 return ent;
1945}
1946
955b61e5 1947void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 1948{
2f26ebd5
SR
1949 struct ring_buffer_event *event;
1950 struct ring_buffer_iter *buf_iter;
1951 unsigned long entries = 0;
1952 u64 ts;
1953
12883efb 1954 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 1955
6d158a81
SR
1956 buf_iter = trace_buffer_iter(iter, cpu);
1957 if (!buf_iter)
2f26ebd5
SR
1958 return;
1959
2f26ebd5
SR
1960 ring_buffer_iter_reset(buf_iter);
1961
1962 /*
1963 * We could have the case with the max latency tracers
1964 * that a reset never took place on a cpu. This is evident
1965 * by the timestamp being before the start of the buffer.
1966 */
1967 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 1968 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
1969 break;
1970 entries++;
1971 ring_buffer_read(buf_iter, NULL);
1972 }
1973
12883efb 1974 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
1975}
1976
d7350c3f 1977/*
d7350c3f
FW
1978 * The current tracer is copied to avoid a global locking
1979 * all around.
1980 */
bc0c38d1
SR
1981static void *s_start(struct seq_file *m, loff_t *pos)
1982{
1983 struct trace_iterator *iter = m->private;
2b6080f2 1984 struct trace_array *tr = iter->tr;
b04cc6b1 1985 int cpu_file = iter->cpu_file;
bc0c38d1
SR
1986 void *p = NULL;
1987 loff_t l = 0;
3928a8a2 1988 int cpu;
bc0c38d1 1989
2fd196ec
HT
1990 /*
1991 * copy the tracer to avoid using a global lock all around.
1992 * iter->trace is a copy of current_trace, the pointer to the
1993 * name may be used instead of a strcmp(), as iter->trace->name
1994 * will point to the same string as current_trace->name.
1995 */
bc0c38d1 1996 mutex_lock(&trace_types_lock);
2b6080f2
SR
1997 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
1998 *iter->trace = *tr->current_trace;
d7350c3f 1999 mutex_unlock(&trace_types_lock);
bc0c38d1 2000
12883efb 2001#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2002 if (iter->snapshot && iter->trace->use_max_tr)
2003 return ERR_PTR(-EBUSY);
12883efb 2004#endif
debdd57f
HT
2005
2006 if (!iter->snapshot)
2007 atomic_inc(&trace_record_cmdline_disabled);
bc0c38d1 2008
bc0c38d1
SR
2009 if (*pos != iter->pos) {
2010 iter->ent = NULL;
2011 iter->cpu = 0;
2012 iter->idx = -1;
2013
ae3b5093 2014 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2015 for_each_tracing_cpu(cpu)
2f26ebd5 2016 tracing_iter_reset(iter, cpu);
b04cc6b1 2017 } else
2f26ebd5 2018 tracing_iter_reset(iter, cpu_file);
bc0c38d1 2019
ac91d854 2020 iter->leftover = 0;
bc0c38d1
SR
2021 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2022 ;
2023
2024 } else {
a63ce5b3
SR
2025 /*
2026 * If we overflowed the seq_file before, then we want
2027 * to just reuse the trace_seq buffer again.
2028 */
2029 if (iter->leftover)
2030 p = iter;
2031 else {
2032 l = *pos - 1;
2033 p = s_next(m, p, &l);
2034 }
bc0c38d1
SR
2035 }
2036
4f535968 2037 trace_event_read_lock();
7e53bd42 2038 trace_access_lock(cpu_file);
bc0c38d1
SR
2039 return p;
2040}
2041
2042static void s_stop(struct seq_file *m, void *p)
2043{
7e53bd42
LJ
2044 struct trace_iterator *iter = m->private;
2045
12883efb 2046#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2047 if (iter->snapshot && iter->trace->use_max_tr)
2048 return;
12883efb 2049#endif
debdd57f
HT
2050
2051 if (!iter->snapshot)
2052 atomic_dec(&trace_record_cmdline_disabled);
12883efb 2053
7e53bd42 2054 trace_access_unlock(iter->cpu_file);
4f535968 2055 trace_event_read_unlock();
bc0c38d1
SR
2056}
2057
39eaf7ef 2058static void
12883efb
SRRH
2059get_total_entries(struct trace_buffer *buf,
2060 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
2061{
2062 unsigned long count;
2063 int cpu;
2064
2065 *total = 0;
2066 *entries = 0;
2067
2068 for_each_tracing_cpu(cpu) {
12883efb 2069 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
2070 /*
2071 * If this buffer has skipped entries, then we hold all
2072 * entries for the trace and we need to ignore the
2073 * ones before the time stamp.
2074 */
12883efb
SRRH
2075 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2076 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
2077 /* total is the same as the entries */
2078 *total += count;
2079 } else
2080 *total += count +
12883efb 2081 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
2082 *entries += count;
2083 }
2084}
2085
e309b41d 2086static void print_lat_help_header(struct seq_file *m)
bc0c38d1 2087{
a6168353
ME
2088 seq_puts(m, "# _------=> CPU# \n");
2089 seq_puts(m, "# / _-----=> irqs-off \n");
2090 seq_puts(m, "# | / _----=> need-resched \n");
2091 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2092 seq_puts(m, "# ||| / _--=> preempt-depth \n");
e6e1e259
SR
2093 seq_puts(m, "# |||| / delay \n");
2094 seq_puts(m, "# cmd pid ||||| time | caller \n");
2095 seq_puts(m, "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
2096}
2097
12883efb 2098static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 2099{
39eaf7ef
SR
2100 unsigned long total;
2101 unsigned long entries;
2102
12883efb 2103 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
2104 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2105 entries, total, num_online_cpus());
2106 seq_puts(m, "#\n");
2107}
2108
12883efb 2109static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
39eaf7ef 2110{
12883efb 2111 print_event_info(buf, m);
77271ce4 2112 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
a6168353 2113 seq_puts(m, "# | | | | |\n");
bc0c38d1
SR
2114}
2115
12883efb 2116static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
77271ce4 2117{
12883efb 2118 print_event_info(buf, m);
77271ce4
SR
2119 seq_puts(m, "# _-----=> irqs-off\n");
2120 seq_puts(m, "# / _----=> need-resched\n");
2121 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2122 seq_puts(m, "# || / _--=> preempt-depth\n");
2123 seq_puts(m, "# ||| / delay\n");
2124 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2125 seq_puts(m, "# | | | |||| | |\n");
2126}
bc0c38d1 2127
62b915f1 2128void
bc0c38d1
SR
2129print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2130{
2131 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
2132 struct trace_buffer *buf = iter->trace_buffer;
2133 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 2134 struct tracer *type = iter->trace;
39eaf7ef
SR
2135 unsigned long entries;
2136 unsigned long total;
bc0c38d1
SR
2137 const char *name = "preemption";
2138
d840f718 2139 name = type->name;
bc0c38d1 2140
12883efb 2141 get_total_entries(buf, &total, &entries);
bc0c38d1 2142
888b55dc 2143 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 2144 name, UTS_RELEASE);
888b55dc 2145 seq_puts(m, "# -----------------------------------"
bc0c38d1 2146 "---------------------------------\n");
888b55dc 2147 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 2148 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 2149 nsecs_to_usecs(data->saved_latency),
bc0c38d1 2150 entries,
4c11d7ae 2151 total,
12883efb 2152 buf->cpu,
bc0c38d1
SR
2153#if defined(CONFIG_PREEMPT_NONE)
2154 "server",
2155#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2156 "desktop",
b5c21b45 2157#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
2158 "preempt",
2159#else
2160 "unknown",
2161#endif
2162 /* These are reserved for later use */
2163 0, 0, 0, 0);
2164#ifdef CONFIG_SMP
2165 seq_printf(m, " #P:%d)\n", num_online_cpus());
2166#else
2167 seq_puts(m, ")\n");
2168#endif
888b55dc
KM
2169 seq_puts(m, "# -----------------\n");
2170 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 2171 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
2172 data->comm, data->pid,
2173 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 2174 data->policy, data->rt_priority);
888b55dc 2175 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
2176
2177 if (data->critical_start) {
888b55dc 2178 seq_puts(m, "# => started at: ");
214023c3
SR
2179 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2180 trace_print_seq(m, &iter->seq);
888b55dc 2181 seq_puts(m, "\n# => ended at: ");
214023c3
SR
2182 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2183 trace_print_seq(m, &iter->seq);
8248ac05 2184 seq_puts(m, "\n#\n");
bc0c38d1
SR
2185 }
2186
888b55dc 2187 seq_puts(m, "#\n");
bc0c38d1
SR
2188}
2189
a309720c
SR
2190static void test_cpu_buff_start(struct trace_iterator *iter)
2191{
2192 struct trace_seq *s = &iter->seq;
2193
12ef7d44
SR
2194 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2195 return;
2196
2197 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2198 return;
2199
4462344e 2200 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
2201 return;
2202
12883efb 2203 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
2204 return;
2205
4462344e 2206 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
2207
2208 /* Don't print started cpu buffer for the first entry of the trace */
2209 if (iter->idx > 1)
2210 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2211 iter->cpu);
a309720c
SR
2212}
2213
2c4f035f 2214static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 2215{
214023c3 2216 struct trace_seq *s = &iter->seq;
bc0c38d1 2217 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 2218 struct trace_entry *entry;
f633cef0 2219 struct trace_event *event;
bc0c38d1 2220
4e3c3333 2221 entry = iter->ent;
dd0e545f 2222
a309720c
SR
2223 test_cpu_buff_start(iter);
2224
c4a8e8be 2225 event = ftrace_find_event(entry->type);
bc0c38d1 2226
c4a8e8be 2227 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
27d48be8
SR
2228 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2229 if (!trace_print_lat_context(iter))
2230 goto partial;
2231 } else {
2232 if (!trace_print_context(iter))
2233 goto partial;
2234 }
c4a8e8be 2235 }
bc0c38d1 2236
268ccda0 2237 if (event)
a9a57763 2238 return event->funcs->trace(iter, sym_flags, event);
d9793bd8
ACM
2239
2240 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2241 goto partial;
02b67518 2242
2c4f035f 2243 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2244partial:
2245 return TRACE_TYPE_PARTIAL_LINE;
bc0c38d1
SR
2246}
2247
2c4f035f 2248static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
2249{
2250 struct trace_seq *s = &iter->seq;
2251 struct trace_entry *entry;
f633cef0 2252 struct trace_event *event;
f9896bf3
IM
2253
2254 entry = iter->ent;
dd0e545f 2255
c4a8e8be 2256 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
d9793bd8
ACM
2257 if (!trace_seq_printf(s, "%d %d %llu ",
2258 entry->pid, iter->cpu, iter->ts))
2259 goto partial;
c4a8e8be 2260 }
f9896bf3 2261
f633cef0 2262 event = ftrace_find_event(entry->type);
268ccda0 2263 if (event)
a9a57763 2264 return event->funcs->raw(iter, 0, event);
d9793bd8
ACM
2265
2266 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2267 goto partial;
777e208d 2268
2c4f035f 2269 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2270partial:
2271 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3
IM
2272}
2273
2c4f035f 2274static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
2275{
2276 struct trace_seq *s = &iter->seq;
2277 unsigned char newline = '\n';
2278 struct trace_entry *entry;
f633cef0 2279 struct trace_event *event;
5e3ca0ec
IM
2280
2281 entry = iter->ent;
dd0e545f 2282
c4a8e8be
FW
2283 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2284 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2285 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2286 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2287 }
5e3ca0ec 2288
f633cef0 2289 event = ftrace_find_event(entry->type);
268ccda0 2290 if (event) {
a9a57763 2291 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
2292 if (ret != TRACE_TYPE_HANDLED)
2293 return ret;
2294 }
7104f300 2295
5e3ca0ec
IM
2296 SEQ_PUT_FIELD_RET(s, newline);
2297
2c4f035f 2298 return TRACE_TYPE_HANDLED;
5e3ca0ec
IM
2299}
2300
2c4f035f 2301static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
2302{
2303 struct trace_seq *s = &iter->seq;
2304 struct trace_entry *entry;
f633cef0 2305 struct trace_event *event;
cb0f12aa
IM
2306
2307 entry = iter->ent;
dd0e545f 2308
c4a8e8be
FW
2309 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2310 SEQ_PUT_FIELD_RET(s, entry->pid);
1830b52d 2311 SEQ_PUT_FIELD_RET(s, iter->cpu);
c4a8e8be
FW
2312 SEQ_PUT_FIELD_RET(s, iter->ts);
2313 }
cb0f12aa 2314
f633cef0 2315 event = ftrace_find_event(entry->type);
a9a57763
SR
2316 return event ? event->funcs->binary(iter, 0, event) :
2317 TRACE_TYPE_HANDLED;
cb0f12aa
IM
2318}
2319
62b915f1 2320int trace_empty(struct trace_iterator *iter)
bc0c38d1 2321{
6d158a81 2322 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
2323 int cpu;
2324
9aba60fe 2325 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 2326 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 2327 cpu = iter->cpu_file;
6d158a81
SR
2328 buf_iter = trace_buffer_iter(iter, cpu);
2329 if (buf_iter) {
2330 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
2331 return 0;
2332 } else {
12883efb 2333 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
2334 return 0;
2335 }
2336 return 1;
2337 }
2338
ab46428c 2339 for_each_tracing_cpu(cpu) {
6d158a81
SR
2340 buf_iter = trace_buffer_iter(iter, cpu);
2341 if (buf_iter) {
2342 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
2343 return 0;
2344 } else {
12883efb 2345 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
2346 return 0;
2347 }
bc0c38d1 2348 }
d769041f 2349
797d3712 2350 return 1;
bc0c38d1
SR
2351}
2352
4f535968 2353/* Called with trace_event_read_lock() held. */
955b61e5 2354enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 2355{
2c4f035f
FW
2356 enum print_line_t ret;
2357
ee5e51f5
JO
2358 if (iter->lost_events &&
2359 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2360 iter->cpu, iter->lost_events))
2361 return TRACE_TYPE_PARTIAL_LINE;
bc21b478 2362
2c4f035f
FW
2363 if (iter->trace && iter->trace->print_line) {
2364 ret = iter->trace->print_line(iter);
2365 if (ret != TRACE_TYPE_UNHANDLED)
2366 return ret;
2367 }
72829bc3 2368
48ead020
FW
2369 if (iter->ent->type == TRACE_BPRINT &&
2370 trace_flags & TRACE_ITER_PRINTK &&
2371 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2372 return trace_print_bprintk_msg_only(iter);
48ead020 2373
66896a85
FW
2374 if (iter->ent->type == TRACE_PRINT &&
2375 trace_flags & TRACE_ITER_PRINTK &&
2376 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2377 return trace_print_printk_msg_only(iter);
66896a85 2378
cb0f12aa
IM
2379 if (trace_flags & TRACE_ITER_BIN)
2380 return print_bin_fmt(iter);
2381
5e3ca0ec
IM
2382 if (trace_flags & TRACE_ITER_HEX)
2383 return print_hex_fmt(iter);
2384
f9896bf3
IM
2385 if (trace_flags & TRACE_ITER_RAW)
2386 return print_raw_fmt(iter);
2387
f9896bf3
IM
2388 return print_trace_fmt(iter);
2389}
2390
7e9a49ef
JO
2391void trace_latency_header(struct seq_file *m)
2392{
2393 struct trace_iterator *iter = m->private;
2394
2395 /* print nothing if the buffers are empty */
2396 if (trace_empty(iter))
2397 return;
2398
2399 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2400 print_trace_header(m, iter);
2401
2402 if (!(trace_flags & TRACE_ITER_VERBOSE))
2403 print_lat_help_header(m);
2404}
2405
62b915f1
JO
2406void trace_default_header(struct seq_file *m)
2407{
2408 struct trace_iterator *iter = m->private;
2409
f56e7f8e
JO
2410 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2411 return;
2412
62b915f1
JO
2413 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2414 /* print nothing if the buffers are empty */
2415 if (trace_empty(iter))
2416 return;
2417 print_trace_header(m, iter);
2418 if (!(trace_flags & TRACE_ITER_VERBOSE))
2419 print_lat_help_header(m);
2420 } else {
77271ce4
SR
2421 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2422 if (trace_flags & TRACE_ITER_IRQ_INFO)
12883efb 2423 print_func_help_header_irq(iter->trace_buffer, m);
77271ce4 2424 else
12883efb 2425 print_func_help_header(iter->trace_buffer, m);
77271ce4 2426 }
62b915f1
JO
2427 }
2428}
2429
e0a413f6
SR
2430static void test_ftrace_alive(struct seq_file *m)
2431{
2432 if (!ftrace_is_dead())
2433 return;
2434 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2435 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2436}
2437
d8741e2e 2438#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa
SRRH
2439static void show_snapshot_main_help(struct seq_file *m)
2440{
2441 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2442 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2443 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2444 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2445 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2446 seq_printf(m, "# is not a '0' or '1')\n");
2447}
2448
2449static void show_snapshot_percpu_help(struct seq_file *m)
2450{
2451 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2452#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2453 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2454 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2455#else
2456 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2457 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2458#endif
2459 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2460 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2461 seq_printf(m, "# is not a '0' or '1')\n");
2462}
2463
d8741e2e
SRRH
2464static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2465{
45ad21ca 2466 if (iter->tr->allocated_snapshot)
d8741e2e
SRRH
2467 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2468 else
2469 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2470
2471 seq_printf(m, "# Snapshot commands:\n");
f1affcaa
SRRH
2472 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2473 show_snapshot_main_help(m);
2474 else
2475 show_snapshot_percpu_help(m);
d8741e2e
SRRH
2476}
2477#else
2478/* Should never be called */
2479static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2480#endif
2481
bc0c38d1
SR
2482static int s_show(struct seq_file *m, void *v)
2483{
2484 struct trace_iterator *iter = v;
a63ce5b3 2485 int ret;
bc0c38d1
SR
2486
2487 if (iter->ent == NULL) {
2488 if (iter->tr) {
2489 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2490 seq_puts(m, "#\n");
e0a413f6 2491 test_ftrace_alive(m);
bc0c38d1 2492 }
d8741e2e
SRRH
2493 if (iter->snapshot && trace_empty(iter))
2494 print_snapshot_help(m, iter);
2495 else if (iter->trace && iter->trace->print_header)
8bba1bf5 2496 iter->trace->print_header(m);
62b915f1
JO
2497 else
2498 trace_default_header(m);
2499
a63ce5b3
SR
2500 } else if (iter->leftover) {
2501 /*
2502 * If we filled the seq_file buffer earlier, we
2503 * want to just show it now.
2504 */
2505 ret = trace_print_seq(m, &iter->seq);
2506
2507 /* ret should this time be zero, but you never know */
2508 iter->leftover = ret;
2509
bc0c38d1 2510 } else {
f9896bf3 2511 print_trace_line(iter);
a63ce5b3
SR
2512 ret = trace_print_seq(m, &iter->seq);
2513 /*
2514 * If we overflow the seq_file buffer, then it will
2515 * ask us for this data again at start up.
2516 * Use that instead.
2517 * ret is 0 if seq_file write succeeded.
2518 * -1 otherwise.
2519 */
2520 iter->leftover = ret;
bc0c38d1
SR
2521 }
2522
2523 return 0;
2524}
2525
88e9d34c 2526static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
2527 .start = s_start,
2528 .next = s_next,
2529 .stop = s_stop,
2530 .show = s_show,
bc0c38d1
SR
2531};
2532
e309b41d 2533static struct trace_iterator *
debdd57f 2534__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 2535{
2b6080f2
SR
2536 struct trace_cpu *tc = inode->i_private;
2537 struct trace_array *tr = tc->tr;
bc0c38d1 2538 struct trace_iterator *iter;
50e18b94 2539 int cpu;
bc0c38d1 2540
85a2f9b4
SR
2541 if (tracing_disabled)
2542 return ERR_PTR(-ENODEV);
60a11774 2543
50e18b94 2544 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
2545 if (!iter)
2546 return ERR_PTR(-ENOMEM);
bc0c38d1 2547
6d158a81
SR
2548 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2549 GFP_KERNEL);
93574fcc
DC
2550 if (!iter->buffer_iter)
2551 goto release;
2552
d7350c3f
FW
2553 /*
2554 * We make a copy of the current tracer to avoid concurrent
2555 * changes on it while we are reading.
2556 */
bc0c38d1 2557 mutex_lock(&trace_types_lock);
d7350c3f 2558 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 2559 if (!iter->trace)
d7350c3f 2560 goto fail;
85a2f9b4 2561
2b6080f2 2562 *iter->trace = *tr->current_trace;
d7350c3f 2563
79f55997 2564 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
2565 goto fail;
2566
12883efb
SRRH
2567 iter->tr = tr;
2568
2569#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
2570 /* Currently only the top directory has a snapshot */
2571 if (tr->current_trace->print_max || snapshot)
12883efb 2572 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 2573 else
12883efb
SRRH
2574#endif
2575 iter->trace_buffer = &tr->trace_buffer;
debdd57f 2576 iter->snapshot = snapshot;
bc0c38d1 2577 iter->pos = -1;
d7350c3f 2578 mutex_init(&iter->mutex);
2b6080f2 2579 iter->cpu_file = tc->cpu;
bc0c38d1 2580
8bba1bf5
MM
2581 /* Notify the tracer early; before we stop tracing. */
2582 if (iter->trace && iter->trace->open)
a93751ca 2583 iter->trace->open(iter);
8bba1bf5 2584
12ef7d44 2585 /* Annotate start of buffers if we had overruns */
12883efb 2586 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
2587 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2588
8be0709f
DS
2589 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2590 if (trace_clocks[trace_clock_id].in_ns)
2591 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2592
debdd57f
HT
2593 /* stop the trace while dumping if we are not opening "snapshot" */
2594 if (!iter->snapshot)
2b6080f2 2595 tracing_stop_tr(tr);
2f26ebd5 2596
ae3b5093 2597 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2598 for_each_tracing_cpu(cpu) {
b04cc6b1 2599 iter->buffer_iter[cpu] =
12883efb 2600 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2601 }
2602 ring_buffer_read_prepare_sync();
2603 for_each_tracing_cpu(cpu) {
2604 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2605 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
2606 }
2607 } else {
2608 cpu = iter->cpu_file;
3928a8a2 2609 iter->buffer_iter[cpu] =
12883efb 2610 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2611 ring_buffer_read_prepare_sync();
2612 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2613 tracing_iter_reset(iter, cpu);
3928a8a2
SR
2614 }
2615
bc0c38d1
SR
2616 mutex_unlock(&trace_types_lock);
2617
bc0c38d1 2618 return iter;
3928a8a2 2619
d7350c3f 2620 fail:
3928a8a2 2621 mutex_unlock(&trace_types_lock);
d7350c3f 2622 kfree(iter->trace);
6d158a81 2623 kfree(iter->buffer_iter);
93574fcc 2624release:
50e18b94
JO
2625 seq_release_private(inode, file);
2626 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
2627}
2628
2629int tracing_open_generic(struct inode *inode, struct file *filp)
2630{
60a11774
SR
2631 if (tracing_disabled)
2632 return -ENODEV;
2633
bc0c38d1
SR
2634 filp->private_data = inode->i_private;
2635 return 0;
2636}
2637
4fd27358 2638static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 2639{
907f2784 2640 struct seq_file *m = file->private_data;
4acd4d00 2641 struct trace_iterator *iter;
2b6080f2 2642 struct trace_array *tr;
3928a8a2 2643 int cpu;
bc0c38d1 2644
4acd4d00
SR
2645 if (!(file->f_mode & FMODE_READ))
2646 return 0;
2647
2648 iter = m->private;
12883efb 2649 tr = iter->tr;
2b6080f2 2650
bc0c38d1 2651 mutex_lock(&trace_types_lock);
3928a8a2
SR
2652 for_each_tracing_cpu(cpu) {
2653 if (iter->buffer_iter[cpu])
2654 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2655 }
2656
bc0c38d1
SR
2657 if (iter->trace && iter->trace->close)
2658 iter->trace->close(iter);
2659
debdd57f
HT
2660 if (!iter->snapshot)
2661 /* reenable tracing if it was previously enabled */
2b6080f2 2662 tracing_start_tr(tr);
bc0c38d1
SR
2663 mutex_unlock(&trace_types_lock);
2664
d7350c3f 2665 mutex_destroy(&iter->mutex);
b0dfa978 2666 free_cpumask_var(iter->started);
d7350c3f 2667 kfree(iter->trace);
6d158a81 2668 kfree(iter->buffer_iter);
50e18b94 2669 seq_release_private(inode, file);
bc0c38d1
SR
2670 return 0;
2671}
2672
2673static int tracing_open(struct inode *inode, struct file *file)
2674{
85a2f9b4
SR
2675 struct trace_iterator *iter;
2676 int ret = 0;
bc0c38d1 2677
4acd4d00
SR
2678 /* If this file was open for write, then erase contents */
2679 if ((file->f_mode & FMODE_WRITE) &&
8650ae32 2680 (file->f_flags & O_TRUNC)) {
2b6080f2
SR
2681 struct trace_cpu *tc = inode->i_private;
2682 struct trace_array *tr = tc->tr;
bc0c38d1 2683
2b6080f2 2684 if (tc->cpu == RING_BUFFER_ALL_CPUS)
12883efb 2685 tracing_reset_online_cpus(&tr->trace_buffer);
4acd4d00 2686 else
12883efb 2687 tracing_reset(&tr->trace_buffer, tc->cpu);
4acd4d00 2688 }
bc0c38d1 2689
4acd4d00 2690 if (file->f_mode & FMODE_READ) {
debdd57f 2691 iter = __tracing_open(inode, file, false);
4acd4d00
SR
2692 if (IS_ERR(iter))
2693 ret = PTR_ERR(iter);
2694 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2695 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2696 }
bc0c38d1
SR
2697 return ret;
2698}
2699
e309b41d 2700static void *
bc0c38d1
SR
2701t_next(struct seq_file *m, void *v, loff_t *pos)
2702{
f129e965 2703 struct tracer *t = v;
bc0c38d1
SR
2704
2705 (*pos)++;
2706
2707 if (t)
2708 t = t->next;
2709
bc0c38d1
SR
2710 return t;
2711}
2712
2713static void *t_start(struct seq_file *m, loff_t *pos)
2714{
f129e965 2715 struct tracer *t;
bc0c38d1
SR
2716 loff_t l = 0;
2717
2718 mutex_lock(&trace_types_lock);
f129e965 2719 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
bc0c38d1
SR
2720 ;
2721
2722 return t;
2723}
2724
2725static void t_stop(struct seq_file *m, void *p)
2726{
2727 mutex_unlock(&trace_types_lock);
2728}
2729
2730static int t_show(struct seq_file *m, void *v)
2731{
2732 struct tracer *t = v;
2733
2734 if (!t)
2735 return 0;
2736
2737 seq_printf(m, "%s", t->name);
2738 if (t->next)
2739 seq_putc(m, ' ');
2740 else
2741 seq_putc(m, '\n');
2742
2743 return 0;
2744}
2745
88e9d34c 2746static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
2747 .start = t_start,
2748 .next = t_next,
2749 .stop = t_stop,
2750 .show = t_show,
bc0c38d1
SR
2751};
2752
2753static int show_traces_open(struct inode *inode, struct file *file)
2754{
60a11774
SR
2755 if (tracing_disabled)
2756 return -ENODEV;
2757
f129e965 2758 return seq_open(file, &show_traces_seq_ops);
bc0c38d1
SR
2759}
2760
4acd4d00
SR
2761static ssize_t
2762tracing_write_stub(struct file *filp, const char __user *ubuf,
2763 size_t count, loff_t *ppos)
2764{
2765 return count;
2766}
2767
364829b1
SP
2768static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2769{
2770 if (file->f_mode & FMODE_READ)
2771 return seq_lseek(file, offset, origin);
2772 else
2773 return 0;
2774}
2775
5e2336a0 2776static const struct file_operations tracing_fops = {
4bf39a94
IM
2777 .open = tracing_open,
2778 .read = seq_read,
4acd4d00 2779 .write = tracing_write_stub,
364829b1 2780 .llseek = tracing_seek,
4bf39a94 2781 .release = tracing_release,
bc0c38d1
SR
2782};
2783
5e2336a0 2784static const struct file_operations show_traces_fops = {
c7078de1
IM
2785 .open = show_traces_open,
2786 .read = seq_read,
2787 .release = seq_release,
b444786f 2788 .llseek = seq_lseek,
c7078de1
IM
2789};
2790
36dfe925
IM
2791/*
2792 * Only trace on a CPU if the bitmask is set:
2793 */
9e01c1b7 2794static cpumask_var_t tracing_cpumask;
36dfe925
IM
2795
2796/*
2797 * The tracer itself will not take this lock, but still we want
2798 * to provide a consistent cpumask to user-space:
2799 */
2800static DEFINE_MUTEX(tracing_cpumask_update_lock);
2801
2802/*
2803 * Temporary storage for the character representation of the
2804 * CPU bitmask (and one more byte for the newline):
2805 */
2806static char mask_str[NR_CPUS + 1];
2807
c7078de1
IM
2808static ssize_t
2809tracing_cpumask_read(struct file *filp, char __user *ubuf,
2810 size_t count, loff_t *ppos)
2811{
36dfe925 2812 int len;
c7078de1
IM
2813
2814 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 2815
9e01c1b7 2816 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
36dfe925
IM
2817 if (count - len < 2) {
2818 count = -EINVAL;
2819 goto out_err;
2820 }
2821 len += sprintf(mask_str + len, "\n");
2822 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2823
2824out_err:
c7078de1
IM
2825 mutex_unlock(&tracing_cpumask_update_lock);
2826
2827 return count;
2828}
2829
2830static ssize_t
2831tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2832 size_t count, loff_t *ppos)
2833{
2b6080f2 2834 struct trace_array *tr = filp->private_data;
9e01c1b7 2835 cpumask_var_t tracing_cpumask_new;
2b6080f2 2836 int err, cpu;
9e01c1b7
RR
2837
2838 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2839 return -ENOMEM;
c7078de1 2840
9e01c1b7 2841 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 2842 if (err)
36dfe925
IM
2843 goto err_unlock;
2844
215368e8
LZ
2845 mutex_lock(&tracing_cpumask_update_lock);
2846
a5e25883 2847 local_irq_disable();
0199c4e6 2848 arch_spin_lock(&ftrace_max_lock);
ab46428c 2849 for_each_tracing_cpu(cpu) {
36dfe925
IM
2850 /*
2851 * Increase/decrease the disabled counter if we are
2852 * about to flip a bit in the cpumask:
2853 */
9e01c1b7
RR
2854 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2855 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
2856 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
2857 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 2858 }
9e01c1b7
RR
2859 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2860 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
2861 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
2862 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
2863 }
2864 }
0199c4e6 2865 arch_spin_unlock(&ftrace_max_lock);
a5e25883 2866 local_irq_enable();
36dfe925 2867
9e01c1b7 2868 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
2869
2870 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 2871 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
2872
2873 return count;
36dfe925
IM
2874
2875err_unlock:
215368e8 2876 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
2877
2878 return err;
c7078de1
IM
2879}
2880
5e2336a0 2881static const struct file_operations tracing_cpumask_fops = {
c7078de1
IM
2882 .open = tracing_open_generic,
2883 .read = tracing_cpumask_read,
2884 .write = tracing_cpumask_write,
b444786f 2885 .llseek = generic_file_llseek,
bc0c38d1
SR
2886};
2887
fdb372ed 2888static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 2889{
d8e83d26 2890 struct tracer_opt *trace_opts;
2b6080f2 2891 struct trace_array *tr = m->private;
d8e83d26 2892 u32 tracer_flags;
d8e83d26 2893 int i;
adf9f195 2894
d8e83d26 2895 mutex_lock(&trace_types_lock);
2b6080f2
SR
2896 tracer_flags = tr->current_trace->flags->val;
2897 trace_opts = tr->current_trace->flags->opts;
d8e83d26 2898
bc0c38d1
SR
2899 for (i = 0; trace_options[i]; i++) {
2900 if (trace_flags & (1 << i))
fdb372ed 2901 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 2902 else
fdb372ed 2903 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
2904 }
2905
adf9f195
FW
2906 for (i = 0; trace_opts[i].name; i++) {
2907 if (tracer_flags & trace_opts[i].bit)
fdb372ed 2908 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 2909 else
fdb372ed 2910 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 2911 }
d8e83d26 2912 mutex_unlock(&trace_types_lock);
adf9f195 2913
fdb372ed 2914 return 0;
bc0c38d1 2915}
bc0c38d1 2916
8d18eaaf
LZ
2917static int __set_tracer_option(struct tracer *trace,
2918 struct tracer_flags *tracer_flags,
2919 struct tracer_opt *opts, int neg)
2920{
2921 int ret;
bc0c38d1 2922
8d18eaaf
LZ
2923 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2924 if (ret)
2925 return ret;
2926
2927 if (neg)
2928 tracer_flags->val &= ~opts->bit;
2929 else
2930 tracer_flags->val |= opts->bit;
2931 return 0;
bc0c38d1
SR
2932}
2933
adf9f195
FW
2934/* Try to assign a tracer specific option */
2935static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2936{
7770841e 2937 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 2938 struct tracer_opt *opts = NULL;
8d18eaaf 2939 int i;
adf9f195 2940
7770841e
Z
2941 for (i = 0; tracer_flags->opts[i].name; i++) {
2942 opts = &tracer_flags->opts[i];
adf9f195 2943
8d18eaaf
LZ
2944 if (strcmp(cmp, opts->name) == 0)
2945 return __set_tracer_option(trace, trace->flags,
2946 opts, neg);
adf9f195 2947 }
adf9f195 2948
8d18eaaf 2949 return -EINVAL;
adf9f195
FW
2950}
2951
613f04a0
SRRH
2952/* Some tracers require overwrite to stay enabled */
2953int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
2954{
2955 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
2956 return -1;
2957
2958 return 0;
2959}
2960
2b6080f2 2961int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
2962{
2963 /* do nothing if flag is already set */
2964 if (!!(trace_flags & mask) == !!enabled)
613f04a0
SRRH
2965 return 0;
2966
2967 /* Give the tracer a chance to approve the change */
2b6080f2
SR
2968 if (tr->current_trace->flag_changed)
2969 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
613f04a0 2970 return -EINVAL;
af4617bd
SR
2971
2972 if (enabled)
2973 trace_flags |= mask;
2974 else
2975 trace_flags &= ~mask;
e870e9a1
LZ
2976
2977 if (mask == TRACE_ITER_RECORD_CMD)
2978 trace_event_enable_cmd_record(enabled);
750912fa 2979
80902822 2980 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 2981 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 2982#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 2983 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
2984#endif
2985 }
81698831
SR
2986
2987 if (mask == TRACE_ITER_PRINTK)
2988 trace_printk_start_stop_comm(enabled);
613f04a0
SRRH
2989
2990 return 0;
af4617bd
SR
2991}
2992
2b6080f2 2993static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 2994{
8d18eaaf 2995 char *cmp;
bc0c38d1 2996 int neg = 0;
613f04a0 2997 int ret = -ENODEV;
bc0c38d1
SR
2998 int i;
2999
7bcfaf54 3000 cmp = strstrip(option);
bc0c38d1 3001
8d18eaaf 3002 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
3003 neg = 1;
3004 cmp += 2;
3005 }
3006
69d34da2
SRRH
3007 mutex_lock(&trace_types_lock);
3008
bc0c38d1 3009 for (i = 0; trace_options[i]; i++) {
8d18eaaf 3010 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 3011 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
3012 break;
3013 }
3014 }
adf9f195
FW
3015
3016 /* If no option could be set, test the specific tracer options */
69d34da2 3017 if (!trace_options[i])
2b6080f2 3018 ret = set_tracer_option(tr->current_trace, cmp, neg);
69d34da2
SRRH
3019
3020 mutex_unlock(&trace_types_lock);
bc0c38d1 3021
7bcfaf54
SR
3022 return ret;
3023}
3024
3025static ssize_t
3026tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3027 size_t cnt, loff_t *ppos)
3028{
2b6080f2
SR
3029 struct seq_file *m = filp->private_data;
3030 struct trace_array *tr = m->private;
7bcfaf54 3031 char buf[64];
613f04a0 3032 int ret;
7bcfaf54
SR
3033
3034 if (cnt >= sizeof(buf))
3035 return -EINVAL;
3036
3037 if (copy_from_user(&buf, ubuf, cnt))
3038 return -EFAULT;
3039
a8dd2176
SR
3040 buf[cnt] = 0;
3041
2b6080f2 3042 ret = trace_set_options(tr, buf);
613f04a0
SRRH
3043 if (ret < 0)
3044 return ret;
7bcfaf54 3045
cf8517cf 3046 *ppos += cnt;
bc0c38d1
SR
3047
3048 return cnt;
3049}
3050
fdb372ed
LZ
3051static int tracing_trace_options_open(struct inode *inode, struct file *file)
3052{
3053 if (tracing_disabled)
3054 return -ENODEV;
2b6080f2
SR
3055
3056 return single_open(file, tracing_trace_options_show, inode->i_private);
fdb372ed
LZ
3057}
3058
5e2336a0 3059static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
3060 .open = tracing_trace_options_open,
3061 .read = seq_read,
3062 .llseek = seq_lseek,
3063 .release = single_release,
ee6bce52 3064 .write = tracing_trace_options_write,
bc0c38d1
SR
3065};
3066
7bd2f24c
IM
3067static const char readme_msg[] =
3068 "tracing mini-HOWTO:\n\n"
156f5a78
GL
3069 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
3070 "# cat /sys/kernel/debug/tracing/available_tracers\n"
1e42e83f 3071 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
156f5a78 3072 "# cat /sys/kernel/debug/tracing/current_tracer\n"
bc2b6871 3073 "nop\n"
1e42e83f 3074 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
156f5a78 3075 "# cat /sys/kernel/debug/tracing/current_tracer\n"
1e42e83f 3076 "wakeup\n"
156f5a78 3077 "# cat /sys/kernel/debug/tracing/trace_options\n"
7bd2f24c 3078 "noprint-parent nosym-offset nosym-addr noverbose\n"
156f5a78 3079 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
9b5f8b31 3080 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
156f5a78 3081 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
9b5f8b31 3082 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
7bd2f24c
IM
3083;
3084
3085static ssize_t
3086tracing_readme_read(struct file *filp, char __user *ubuf,
3087 size_t cnt, loff_t *ppos)
3088{
3089 return simple_read_from_buffer(ubuf, cnt, ppos,
3090 readme_msg, strlen(readme_msg));
3091}
3092
5e2336a0 3093static const struct file_operations tracing_readme_fops = {
c7078de1
IM
3094 .open = tracing_open_generic,
3095 .read = tracing_readme_read,
b444786f 3096 .llseek = generic_file_llseek,
7bd2f24c
IM
3097};
3098
69abe6a5
AP
3099static ssize_t
3100tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3101 size_t cnt, loff_t *ppos)
3102{
3103 char *buf_comm;
3104 char *file_buf;
3105 char *buf;
3106 int len = 0;
3107 int pid;
3108 int i;
3109
3110 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3111 if (!file_buf)
3112 return -ENOMEM;
3113
3114 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3115 if (!buf_comm) {
3116 kfree(file_buf);
3117 return -ENOMEM;
3118 }
3119
3120 buf = file_buf;
3121
3122 for (i = 0; i < SAVED_CMDLINES; i++) {
3123 int r;
3124
3125 pid = map_cmdline_to_pid[i];
3126 if (pid == -1 || pid == NO_CMDLINE_MAP)
3127 continue;
3128
3129 trace_find_cmdline(pid, buf_comm);
3130 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3131 buf += r;
3132 len += r;
3133 }
3134
3135 len = simple_read_from_buffer(ubuf, cnt, ppos,
3136 file_buf, len);
3137
3138 kfree(file_buf);
3139 kfree(buf_comm);
3140
3141 return len;
3142}
3143
3144static const struct file_operations tracing_saved_cmdlines_fops = {
3145 .open = tracing_open_generic,
3146 .read = tracing_saved_cmdlines_read,
b444786f 3147 .llseek = generic_file_llseek,
69abe6a5
AP
3148};
3149
bc0c38d1
SR
3150static ssize_t
3151tracing_set_trace_read(struct file *filp, char __user *ubuf,
3152 size_t cnt, loff_t *ppos)
3153{
2b6080f2 3154 struct trace_array *tr = filp->private_data;
ee6c2c1b 3155 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
3156 int r;
3157
3158 mutex_lock(&trace_types_lock);
2b6080f2 3159 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
3160 mutex_unlock(&trace_types_lock);
3161
4bf39a94 3162 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3163}
3164
b6f11df2
ACM
3165int tracer_init(struct tracer *t, struct trace_array *tr)
3166{
12883efb 3167 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
3168 return t->init(tr);
3169}
3170
12883efb 3171static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
3172{
3173 int cpu;
3174 for_each_tracing_cpu(cpu)
12883efb 3175 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
3176}
3177
12883efb 3178#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 3179/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
3180static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3181 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
3182{
3183 int cpu, ret = 0;
3184
3185 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3186 for_each_tracing_cpu(cpu) {
12883efb
SRRH
3187 ret = ring_buffer_resize(trace_buf->buffer,
3188 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
3189 if (ret < 0)
3190 break;
12883efb
SRRH
3191 per_cpu_ptr(trace_buf->data, cpu)->entries =
3192 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
3193 }
3194 } else {
12883efb
SRRH
3195 ret = ring_buffer_resize(trace_buf->buffer,
3196 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 3197 if (ret == 0)
12883efb
SRRH
3198 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3199 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
3200 }
3201
3202 return ret;
3203}
12883efb 3204#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 3205
2b6080f2
SR
3206static int __tracing_resize_ring_buffer(struct trace_array *tr,
3207 unsigned long size, int cpu)
73c5162a
SR
3208{
3209 int ret;
3210
3211 /*
3212 * If kernel or user changes the size of the ring buffer
a123c52b
SR
3213 * we use the size that was given, and we can forget about
3214 * expanding it later.
73c5162a
SR
3215 */
3216 ring_buffer_expanded = 1;
3217
b382ede6 3218 /* May be called before buffers are initialized */
12883efb 3219 if (!tr->trace_buffer.buffer)
b382ede6
SR
3220 return 0;
3221
12883efb 3222 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
3223 if (ret < 0)
3224 return ret;
3225
12883efb 3226#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3227 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3228 !tr->current_trace->use_max_tr)
ef710e10
KM
3229 goto out;
3230
12883efb 3231 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 3232 if (ret < 0) {
12883efb
SRRH
3233 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3234 &tr->trace_buffer, cpu);
73c5162a 3235 if (r < 0) {
a123c52b
SR
3236 /*
3237 * AARGH! We are left with different
3238 * size max buffer!!!!
3239 * The max buffer is our "snapshot" buffer.
3240 * When a tracer needs a snapshot (one of the
3241 * latency tracers), it swaps the max buffer
3242 * with the saved snap shot. We succeeded to
3243 * update the size of the main buffer, but failed to
3244 * update the size of the max buffer. But when we tried
3245 * to reset the main buffer to the original size, we
3246 * failed there too. This is very unlikely to
3247 * happen, but if it does, warn and kill all
3248 * tracing.
3249 */
73c5162a
SR
3250 WARN_ON(1);
3251 tracing_disabled = 1;
3252 }
3253 return ret;
3254 }
3255
438ced17 3256 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3257 set_buffer_entries(&tr->max_buffer, size);
438ced17 3258 else
12883efb 3259 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 3260
ef710e10 3261 out:
12883efb
SRRH
3262#endif /* CONFIG_TRACER_MAX_TRACE */
3263
438ced17 3264 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3265 set_buffer_entries(&tr->trace_buffer, size);
438ced17 3266 else
12883efb 3267 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
3268
3269 return ret;
3270}
3271
2b6080f2
SR
3272static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3273 unsigned long size, int cpu_id)
4f271a2a 3274{
83f40318 3275 int ret = size;
4f271a2a
VN
3276
3277 mutex_lock(&trace_types_lock);
3278
438ced17
VN
3279 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3280 /* make sure, this cpu is enabled in the mask */
3281 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3282 ret = -EINVAL;
3283 goto out;
3284 }
3285 }
4f271a2a 3286
2b6080f2 3287 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
3288 if (ret < 0)
3289 ret = -ENOMEM;
3290
438ced17 3291out:
4f271a2a
VN
3292 mutex_unlock(&trace_types_lock);
3293
3294 return ret;
3295}
3296
ef710e10 3297
1852fcce
SR
3298/**
3299 * tracing_update_buffers - used by tracing facility to expand ring buffers
3300 *
3301 * To save on memory when the tracing is never used on a system with it
3302 * configured in. The ring buffers are set to a minimum size. But once
3303 * a user starts to use the tracing facility, then they need to grow
3304 * to their default size.
3305 *
3306 * This function is to be called when a tracer is about to be used.
3307 */
3308int tracing_update_buffers(void)
3309{
3310 int ret = 0;
3311
1027fcb2 3312 mutex_lock(&trace_types_lock);
1852fcce 3313 if (!ring_buffer_expanded)
2b6080f2 3314 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 3315 RING_BUFFER_ALL_CPUS);
1027fcb2 3316 mutex_unlock(&trace_types_lock);
1852fcce
SR
3317
3318 return ret;
3319}
3320
577b785f
SR
3321struct trace_option_dentry;
3322
3323static struct trace_option_dentry *
2b6080f2 3324create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f
SR
3325
3326static void
3327destroy_trace_option_files(struct trace_option_dentry *topts);
3328
b2821ae6 3329static int tracing_set_tracer(const char *buf)
bc0c38d1 3330{
577b785f 3331 static struct trace_option_dentry *topts;
bc0c38d1
SR
3332 struct trace_array *tr = &global_trace;
3333 struct tracer *t;
12883efb 3334#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3335 bool had_max_tr;
12883efb 3336#endif
d9e54076 3337 int ret = 0;
bc0c38d1 3338
1027fcb2
SR
3339 mutex_lock(&trace_types_lock);
3340
73c5162a 3341 if (!ring_buffer_expanded) {
2b6080f2 3342 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 3343 RING_BUFFER_ALL_CPUS);
73c5162a 3344 if (ret < 0)
59f586db 3345 goto out;
73c5162a
SR
3346 ret = 0;
3347 }
3348
bc0c38d1
SR
3349 for (t = trace_types; t; t = t->next) {
3350 if (strcmp(t->name, buf) == 0)
3351 break;
3352 }
c2931e05
FW
3353 if (!t) {
3354 ret = -EINVAL;
3355 goto out;
3356 }
2b6080f2 3357 if (t == tr->current_trace)
bc0c38d1
SR
3358 goto out;
3359
9f029e83 3360 trace_branch_disable();
613f04a0 3361
2b6080f2 3362 tr->current_trace->enabled = false;
613f04a0 3363
2b6080f2
SR
3364 if (tr->current_trace->reset)
3365 tr->current_trace->reset(tr);
34600f0e 3366
12883efb 3367 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 3368 tr->current_trace = &nop_trace;
34600f0e 3369
45ad21ca
SRRH
3370#ifdef CONFIG_TRACER_MAX_TRACE
3371 had_max_tr = tr->allocated_snapshot;
3372
34600f0e
SR
3373 if (had_max_tr && !t->use_max_tr) {
3374 /*
3375 * We need to make sure that the update_max_tr sees that
3376 * current_trace changed to nop_trace to keep it from
3377 * swapping the buffers after we resize it.
3378 * The update_max_tr is called from interrupts disabled
3379 * so a synchronized_sched() is sufficient.
3380 */
3381 synchronize_sched();
ef710e10
KM
3382 /*
3383 * We don't free the ring buffer. instead, resize it because
3384 * The max_tr ring buffer has some state (e.g. ring->clock) and
3385 * we want preserve it.
3386 */
12883efb
SRRH
3387 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
3388 set_buffer_entries(&tr->max_buffer, 1);
3389 tracing_reset_online_cpus(&tr->max_buffer);
45ad21ca 3390 tr->allocated_snapshot = false;
ef710e10 3391 }
12883efb 3392#endif
577b785f
SR
3393 destroy_trace_option_files(topts);
3394
2b6080f2 3395 topts = create_trace_option_files(tr, t);
12883efb
SRRH
3396
3397#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3398 if (t->use_max_tr && !had_max_tr) {
438ced17 3399 /* we need to make per cpu buffer sizes equivalent */
12883efb 3400 ret = resize_buffer_duplicate_size(&tr->max_buffer, &tr->trace_buffer,
d60da506
HT
3401 RING_BUFFER_ALL_CPUS);
3402 if (ret < 0)
3403 goto out;
45ad21ca 3404 tr->allocated_snapshot = true;
ef710e10 3405 }
12883efb 3406#endif
577b785f 3407
1c80025a 3408 if (t->init) {
b6f11df2 3409 ret = tracer_init(t, tr);
1c80025a
FW
3410 if (ret)
3411 goto out;
3412 }
bc0c38d1 3413
2b6080f2
SR
3414 tr->current_trace = t;
3415 tr->current_trace->enabled = true;
9f029e83 3416 trace_branch_enable(tr);
bc0c38d1
SR
3417 out:
3418 mutex_unlock(&trace_types_lock);
3419
d9e54076
PZ
3420 return ret;
3421}
3422
3423static ssize_t
3424tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3425 size_t cnt, loff_t *ppos)
3426{
ee6c2c1b 3427 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
3428 int i;
3429 size_t ret;
e6e7a65a
FW
3430 int err;
3431
3432 ret = cnt;
d9e54076 3433
ee6c2c1b
LZ
3434 if (cnt > MAX_TRACER_SIZE)
3435 cnt = MAX_TRACER_SIZE;
d9e54076
PZ
3436
3437 if (copy_from_user(&buf, ubuf, cnt))
3438 return -EFAULT;
3439
3440 buf[cnt] = 0;
3441
3442 /* strip ending whitespace. */
3443 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3444 buf[i] = 0;
3445
e6e7a65a
FW
3446 err = tracing_set_tracer(buf);
3447 if (err)
3448 return err;
d9e54076 3449
cf8517cf 3450 *ppos += ret;
bc0c38d1 3451
c2931e05 3452 return ret;
bc0c38d1
SR
3453}
3454
3455static ssize_t
3456tracing_max_lat_read(struct file *filp, char __user *ubuf,
3457 size_t cnt, loff_t *ppos)
3458{
3459 unsigned long *ptr = filp->private_data;
3460 char buf[64];
3461 int r;
3462
cffae437 3463 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 3464 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
3465 if (r > sizeof(buf))
3466 r = sizeof(buf);
4bf39a94 3467 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3468}
3469
3470static ssize_t
3471tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3472 size_t cnt, loff_t *ppos)
3473{
5e39841c 3474 unsigned long *ptr = filp->private_data;
5e39841c 3475 unsigned long val;
c6caeeb1 3476 int ret;
bc0c38d1 3477
22fe9b54
PH
3478 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3479 if (ret)
c6caeeb1 3480 return ret;
bc0c38d1
SR
3481
3482 *ptr = val * 1000;
3483
3484 return cnt;
3485}
3486
b3806b43
SR
3487static int tracing_open_pipe(struct inode *inode, struct file *filp)
3488{
2b6080f2
SR
3489 struct trace_cpu *tc = inode->i_private;
3490 struct trace_array *tr = tc->tr;
b3806b43 3491 struct trace_iterator *iter;
b04cc6b1 3492 int ret = 0;
b3806b43
SR
3493
3494 if (tracing_disabled)
3495 return -ENODEV;
3496
b04cc6b1
FW
3497 mutex_lock(&trace_types_lock);
3498
b3806b43
SR
3499 /* create a buffer to store the information to pass to userspace */
3500 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
3501 if (!iter) {
3502 ret = -ENOMEM;
3503 goto out;
3504 }
b3806b43 3505
d7350c3f
FW
3506 /*
3507 * We make a copy of the current tracer to avoid concurrent
3508 * changes on it while we are reading.
3509 */
3510 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3511 if (!iter->trace) {
3512 ret = -ENOMEM;
3513 goto fail;
3514 }
2b6080f2 3515 *iter->trace = *tr->current_trace;
d7350c3f 3516
4462344e 3517 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 3518 ret = -ENOMEM;
d7350c3f 3519 goto fail;
4462344e
RR
3520 }
3521
a309720c 3522 /* trace pipe does not show start of buffer */
4462344e 3523 cpumask_setall(iter->started);
a309720c 3524
112f38a7
SR
3525 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3526 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3527
8be0709f
DS
3528 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3529 if (trace_clocks[trace_clock_id].in_ns)
3530 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3531
2b6080f2
SR
3532 iter->cpu_file = tc->cpu;
3533 iter->tr = tc->tr;
12883efb 3534 iter->trace_buffer = &tc->tr->trace_buffer;
d7350c3f 3535 mutex_init(&iter->mutex);
b3806b43
SR
3536 filp->private_data = iter;
3537
107bad8b
SR
3538 if (iter->trace->pipe_open)
3539 iter->trace->pipe_open(iter);
107bad8b 3540
b444786f 3541 nonseekable_open(inode, filp);
b04cc6b1
FW
3542out:
3543 mutex_unlock(&trace_types_lock);
3544 return ret;
d7350c3f
FW
3545
3546fail:
3547 kfree(iter->trace);
3548 kfree(iter);
3549 mutex_unlock(&trace_types_lock);
3550 return ret;
b3806b43
SR
3551}
3552
3553static int tracing_release_pipe(struct inode *inode, struct file *file)
3554{
3555 struct trace_iterator *iter = file->private_data;
3556
b04cc6b1
FW
3557 mutex_lock(&trace_types_lock);
3558
29bf4a5e 3559 if (iter->trace->pipe_close)
c521efd1
SR
3560 iter->trace->pipe_close(iter);
3561
b04cc6b1
FW
3562 mutex_unlock(&trace_types_lock);
3563
4462344e 3564 free_cpumask_var(iter->started);
d7350c3f
FW
3565 mutex_destroy(&iter->mutex);
3566 kfree(iter->trace);
b3806b43 3567 kfree(iter);
b3806b43
SR
3568
3569 return 0;
3570}
3571
2a2cc8f7 3572static unsigned int
cc60cdc9 3573trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 3574{
15693458
SRRH
3575 /* Iterators are static, they should be filled or empty */
3576 if (trace_buffer_iter(iter, iter->cpu_file))
3577 return POLLIN | POLLRDNORM;
3578
3579 if (trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
3580 /*
3581 * Always select as readable when in blocking mode
3582 */
3583 return POLLIN | POLLRDNORM;
15693458 3584 else
12883efb 3585 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 3586 filp, poll_table);
2a2cc8f7
SSP
3587}
3588
cc60cdc9
SR
3589static unsigned int
3590tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3591{
3592 struct trace_iterator *iter = filp->private_data;
3593
3594 return trace_poll(iter, filp, poll_table);
3595}
3596
6eaaa5d5
FW
3597/*
3598 * This is a make-shift waitqueue.
3599 * A tracer might use this callback on some rare cases:
3600 *
3601 * 1) the current tracer might hold the runqueue lock when it wakes up
3602 * a reader, hence a deadlock (sched, function, and function graph tracers)
3603 * 2) the function tracers, trace all functions, we don't want
3604 * the overhead of calling wake_up and friends
3605 * (and tracing them too)
3606 *
3607 * Anyway, this is really very primitive wakeup.
3608 */
3609void poll_wait_pipe(struct trace_iterator *iter)
3610{
3611 set_current_state(TASK_INTERRUPTIBLE);
3612 /* sleep for 100 msecs, and try again. */
3613 schedule_timeout(HZ / 10);
3614}
3615
ff98781b
EGM
3616/* Must be called with trace_types_lock mutex held. */
3617static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
3618{
3619 struct trace_iterator *iter = filp->private_data;
b3806b43 3620
b3806b43 3621 while (trace_empty(iter)) {
2dc8f095 3622
107bad8b 3623 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 3624 return -EAGAIN;
107bad8b 3625 }
2dc8f095 3626
d7350c3f 3627 mutex_unlock(&iter->mutex);
107bad8b 3628
6eaaa5d5 3629 iter->trace->wait_pipe(iter);
b3806b43 3630
d7350c3f 3631 mutex_lock(&iter->mutex);
107bad8b 3632
6eaaa5d5 3633 if (signal_pending(current))
ff98781b 3634 return -EINTR;
b3806b43
SR
3635
3636 /*
250bfd3d 3637 * We block until we read something and tracing is disabled.
b3806b43
SR
3638 * We still block if tracing is disabled, but we have never
3639 * read anything. This allows a user to cat this file, and
3640 * then enable tracing. But after we have read something,
3641 * we give an EOF when tracing is again disabled.
3642 *
3643 * iter->pos will be 0 if we haven't read anything.
3644 */
250bfd3d 3645 if (!tracing_is_enabled() && iter->pos)
b3806b43 3646 break;
b3806b43
SR
3647 }
3648
ff98781b
EGM
3649 return 1;
3650}
3651
3652/*
3653 * Consumer reader.
3654 */
3655static ssize_t
3656tracing_read_pipe(struct file *filp, char __user *ubuf,
3657 size_t cnt, loff_t *ppos)
3658{
3659 struct trace_iterator *iter = filp->private_data;
2b6080f2 3660 struct trace_array *tr = iter->tr;
ff98781b
EGM
3661 ssize_t sret;
3662
3663 /* return any leftover data */
3664 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3665 if (sret != -EBUSY)
3666 return sret;
3667
f9520750 3668 trace_seq_init(&iter->seq);
ff98781b 3669
d7350c3f 3670 /* copy the tracer to avoid using a global lock all around */
ff98781b 3671 mutex_lock(&trace_types_lock);
2b6080f2
SR
3672 if (unlikely(iter->trace->name != tr->current_trace->name))
3673 *iter->trace = *tr->current_trace;
d7350c3f
FW
3674 mutex_unlock(&trace_types_lock);
3675
3676 /*
3677 * Avoid more than one consumer on a single file descriptor
3678 * This is just a matter of traces coherency, the ring buffer itself
3679 * is protected.
3680 */
3681 mutex_lock(&iter->mutex);
ff98781b
EGM
3682 if (iter->trace->read) {
3683 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3684 if (sret)
3685 goto out;
3686 }
3687
3688waitagain:
3689 sret = tracing_wait_pipe(filp);
3690 if (sret <= 0)
3691 goto out;
3692
b3806b43 3693 /* stop when tracing is finished */
ff98781b
EGM
3694 if (trace_empty(iter)) {
3695 sret = 0;
107bad8b 3696 goto out;
ff98781b 3697 }
b3806b43
SR
3698
3699 if (cnt >= PAGE_SIZE)
3700 cnt = PAGE_SIZE - 1;
3701
53d0aa77 3702 /* reset all but tr, trace, and overruns */
53d0aa77
SR
3703 memset(&iter->seq, 0,
3704 sizeof(struct trace_iterator) -
3705 offsetof(struct trace_iterator, seq));
4823ed7e 3706 iter->pos = -1;
b3806b43 3707
4f535968 3708 trace_event_read_lock();
7e53bd42 3709 trace_access_lock(iter->cpu_file);
955b61e5 3710 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 3711 enum print_line_t ret;
088b1e42
SR
3712 int len = iter->seq.len;
3713
f9896bf3 3714 ret = print_trace_line(iter);
2c4f035f 3715 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
3716 /* don't print partial lines */
3717 iter->seq.len = len;
b3806b43 3718 break;
088b1e42 3719 }
b91facc3
FW
3720 if (ret != TRACE_TYPE_NO_CONSUME)
3721 trace_consume(iter);
b3806b43
SR
3722
3723 if (iter->seq.len >= cnt)
3724 break;
ee5e51f5
JO
3725
3726 /*
3727 * Setting the full flag means we reached the trace_seq buffer
3728 * size and we should leave by partial output condition above.
3729 * One of the trace_seq_* functions is not used properly.
3730 */
3731 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3732 iter->ent->type);
b3806b43 3733 }
7e53bd42 3734 trace_access_unlock(iter->cpu_file);
4f535968 3735 trace_event_read_unlock();
b3806b43 3736
b3806b43 3737 /* Now copy what we have to the user */
6c6c2796
PP
3738 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3739 if (iter->seq.readpos >= iter->seq.len)
f9520750 3740 trace_seq_init(&iter->seq);
9ff4b974
PP
3741
3742 /*
25985edc 3743 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
3744 * entries, go back to wait for more entries.
3745 */
6c6c2796 3746 if (sret == -EBUSY)
9ff4b974 3747 goto waitagain;
b3806b43 3748
107bad8b 3749out:
d7350c3f 3750 mutex_unlock(&iter->mutex);
107bad8b 3751
6c6c2796 3752 return sret;
b3806b43
SR
3753}
3754
3c56819b
EGM
3755static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3756 struct pipe_buffer *buf)
3757{
3758 __free_page(buf->page);
3759}
3760
3761static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3762 unsigned int idx)
3763{
3764 __free_page(spd->pages[idx]);
3765}
3766
28dfef8f 3767static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998
SR
3768 .can_merge = 0,
3769 .map = generic_pipe_buf_map,
3770 .unmap = generic_pipe_buf_unmap,
3771 .confirm = generic_pipe_buf_confirm,
3772 .release = tracing_pipe_buf_release,
3773 .steal = generic_pipe_buf_steal,
3774 .get = generic_pipe_buf_get,
3c56819b
EGM
3775};
3776
34cd4998 3777static size_t
fa7c7f6e 3778tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
3779{
3780 size_t count;
3781 int ret;
3782
3783 /* Seq buffer is page-sized, exactly what we need. */
3784 for (;;) {
3785 count = iter->seq.len;
3786 ret = print_trace_line(iter);
3787 count = iter->seq.len - count;
3788 if (rem < count) {
3789 rem = 0;
3790 iter->seq.len -= count;
3791 break;
3792 }
3793 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3794 iter->seq.len -= count;
3795 break;
3796 }
3797
74e7ff8c
LJ
3798 if (ret != TRACE_TYPE_NO_CONSUME)
3799 trace_consume(iter);
34cd4998 3800 rem -= count;
955b61e5 3801 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
3802 rem = 0;
3803 iter->ent = NULL;
3804 break;
3805 }
3806 }
3807
3808 return rem;
3809}
3810
3c56819b
EGM
3811static ssize_t tracing_splice_read_pipe(struct file *filp,
3812 loff_t *ppos,
3813 struct pipe_inode_info *pipe,
3814 size_t len,
3815 unsigned int flags)
3816{
35f3d14d
JA
3817 struct page *pages_def[PIPE_DEF_BUFFERS];
3818 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
3819 struct trace_iterator *iter = filp->private_data;
3820 struct splice_pipe_desc spd = {
35f3d14d
JA
3821 .pages = pages_def,
3822 .partial = partial_def,
34cd4998 3823 .nr_pages = 0, /* This gets updated below. */
047fe360 3824 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
3825 .flags = flags,
3826 .ops = &tracing_pipe_buf_ops,
3827 .spd_release = tracing_spd_release_pipe,
3c56819b 3828 };
2b6080f2 3829 struct trace_array *tr = iter->tr;
3c56819b 3830 ssize_t ret;
34cd4998 3831 size_t rem;
3c56819b
EGM
3832 unsigned int i;
3833
35f3d14d
JA
3834 if (splice_grow_spd(pipe, &spd))
3835 return -ENOMEM;
3836
d7350c3f 3837 /* copy the tracer to avoid using a global lock all around */
3c56819b 3838 mutex_lock(&trace_types_lock);
2b6080f2
SR
3839 if (unlikely(iter->trace->name != tr->current_trace->name))
3840 *iter->trace = *tr->current_trace;
d7350c3f
FW
3841 mutex_unlock(&trace_types_lock);
3842
3843 mutex_lock(&iter->mutex);
3c56819b
EGM
3844
3845 if (iter->trace->splice_read) {
3846 ret = iter->trace->splice_read(iter, filp,
3847 ppos, pipe, len, flags);
3848 if (ret)
34cd4998 3849 goto out_err;
3c56819b
EGM
3850 }
3851
3852 ret = tracing_wait_pipe(filp);
3853 if (ret <= 0)
34cd4998 3854 goto out_err;
3c56819b 3855
955b61e5 3856 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 3857 ret = -EFAULT;
34cd4998 3858 goto out_err;
3c56819b
EGM
3859 }
3860
4f535968 3861 trace_event_read_lock();
7e53bd42 3862 trace_access_lock(iter->cpu_file);
4f535968 3863
3c56819b 3864 /* Fill as many pages as possible. */
35f3d14d
JA
3865 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3866 spd.pages[i] = alloc_page(GFP_KERNEL);
3867 if (!spd.pages[i])
34cd4998 3868 break;
3c56819b 3869
fa7c7f6e 3870 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
3871
3872 /* Copy the data into the page, so we can start over. */
3873 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 3874 page_address(spd.pages[i]),
3c56819b
EGM
3875 iter->seq.len);
3876 if (ret < 0) {
35f3d14d 3877 __free_page(spd.pages[i]);
3c56819b
EGM
3878 break;
3879 }
35f3d14d
JA
3880 spd.partial[i].offset = 0;
3881 spd.partial[i].len = iter->seq.len;
3c56819b 3882
f9520750 3883 trace_seq_init(&iter->seq);
3c56819b
EGM
3884 }
3885
7e53bd42 3886 trace_access_unlock(iter->cpu_file);
4f535968 3887 trace_event_read_unlock();
d7350c3f 3888 mutex_unlock(&iter->mutex);
3c56819b
EGM
3889
3890 spd.nr_pages = i;
3891
35f3d14d
JA
3892 ret = splice_to_pipe(pipe, &spd);
3893out:
047fe360 3894 splice_shrink_spd(&spd);
35f3d14d 3895 return ret;
3c56819b 3896
34cd4998 3897out_err:
d7350c3f 3898 mutex_unlock(&iter->mutex);
35f3d14d 3899 goto out;
3c56819b
EGM
3900}
3901
a98a3c3f
SR
3902static ssize_t
3903tracing_entries_read(struct file *filp, char __user *ubuf,
3904 size_t cnt, loff_t *ppos)
3905{
2b6080f2
SR
3906 struct trace_cpu *tc = filp->private_data;
3907 struct trace_array *tr = tc->tr;
438ced17
VN
3908 char buf[64];
3909 int r = 0;
3910 ssize_t ret;
a98a3c3f 3911
db526ca3 3912 mutex_lock(&trace_types_lock);
438ced17 3913
2b6080f2 3914 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
3915 int cpu, buf_size_same;
3916 unsigned long size;
3917
3918 size = 0;
3919 buf_size_same = 1;
3920 /* check if all cpu sizes are same */
3921 for_each_tracing_cpu(cpu) {
3922 /* fill in the size from first enabled cpu */
3923 if (size == 0)
12883efb
SRRH
3924 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
3925 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
3926 buf_size_same = 0;
3927 break;
3928 }
3929 }
3930
3931 if (buf_size_same) {
3932 if (!ring_buffer_expanded)
3933 r = sprintf(buf, "%lu (expanded: %lu)\n",
3934 size >> 10,
3935 trace_buf_size >> 10);
3936 else
3937 r = sprintf(buf, "%lu\n", size >> 10);
3938 } else
3939 r = sprintf(buf, "X\n");
3940 } else
12883efb 3941 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
438ced17 3942
db526ca3
SR
3943 mutex_unlock(&trace_types_lock);
3944
438ced17
VN
3945 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3946 return ret;
a98a3c3f
SR
3947}
3948
3949static ssize_t
3950tracing_entries_write(struct file *filp, const char __user *ubuf,
3951 size_t cnt, loff_t *ppos)
3952{
2b6080f2 3953 struct trace_cpu *tc = filp->private_data;
a98a3c3f 3954 unsigned long val;
4f271a2a 3955 int ret;
a98a3c3f 3956
22fe9b54
PH
3957 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3958 if (ret)
c6caeeb1 3959 return ret;
a98a3c3f
SR
3960
3961 /* must have at least 1 entry */
3962 if (!val)
3963 return -EINVAL;
3964
1696b2b0
SR
3965 /* value is in KB */
3966 val <<= 10;
3967
2b6080f2 3968 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4f271a2a
VN
3969 if (ret < 0)
3970 return ret;
a98a3c3f 3971
cf8517cf 3972 *ppos += cnt;
a98a3c3f 3973
4f271a2a
VN
3974 return cnt;
3975}
bf5e6519 3976
f81ab074
VN
3977static ssize_t
3978tracing_total_entries_read(struct file *filp, char __user *ubuf,
3979 size_t cnt, loff_t *ppos)
3980{
3981 struct trace_array *tr = filp->private_data;
3982 char buf[64];
3983 int r, cpu;
3984 unsigned long size = 0, expanded_size = 0;
3985
3986 mutex_lock(&trace_types_lock);
3987 for_each_tracing_cpu(cpu) {
12883efb 3988 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
3989 if (!ring_buffer_expanded)
3990 expanded_size += trace_buf_size >> 10;
3991 }
3992 if (ring_buffer_expanded)
3993 r = sprintf(buf, "%lu\n", size);
3994 else
3995 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3996 mutex_unlock(&trace_types_lock);
3997
3998 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3999}
4000
4f271a2a
VN
4001static ssize_t
4002tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4003 size_t cnt, loff_t *ppos)
4004{
4005 /*
4006 * There is no need to read what the user has written, this function
4007 * is just to make sure that there is no error when "echo" is used
4008 */
4009
4010 *ppos += cnt;
a98a3c3f
SR
4011
4012 return cnt;
4013}
4014
4f271a2a
VN
4015static int
4016tracing_free_buffer_release(struct inode *inode, struct file *filp)
4017{
2b6080f2
SR
4018 struct trace_array *tr = inode->i_private;
4019
cf30cf67
SR
4020 /* disable tracing ? */
4021 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4022 tracing_off();
4f271a2a 4023 /* resize the ring buffer to 0 */
2b6080f2 4024 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a
VN
4025
4026 return 0;
4027}
4028
5bf9a1ee
PP
4029static ssize_t
4030tracing_mark_write(struct file *filp, const char __user *ubuf,
4031 size_t cnt, loff_t *fpos)
4032{
d696b58c
SR
4033 unsigned long addr = (unsigned long)ubuf;
4034 struct ring_buffer_event *event;
4035 struct ring_buffer *buffer;
4036 struct print_entry *entry;
4037 unsigned long irq_flags;
4038 struct page *pages[2];
6edb2a8a 4039 void *map_page[2];
d696b58c
SR
4040 int nr_pages = 1;
4041 ssize_t written;
d696b58c
SR
4042 int offset;
4043 int size;
4044 int len;
4045 int ret;
6edb2a8a 4046 int i;
5bf9a1ee 4047
c76f0694 4048 if (tracing_disabled)
5bf9a1ee
PP
4049 return -EINVAL;
4050
5224c3a3
MSB
4051 if (!(trace_flags & TRACE_ITER_MARKERS))
4052 return -EINVAL;
4053
5bf9a1ee
PP
4054 if (cnt > TRACE_BUF_SIZE)
4055 cnt = TRACE_BUF_SIZE;
4056
d696b58c
SR
4057 /*
4058 * Userspace is injecting traces into the kernel trace buffer.
4059 * We want to be as non intrusive as possible.
4060 * To do so, we do not want to allocate any special buffers
4061 * or take any locks, but instead write the userspace data
4062 * straight into the ring buffer.
4063 *
4064 * First we need to pin the userspace buffer into memory,
4065 * which, most likely it is, because it just referenced it.
4066 * But there's no guarantee that it is. By using get_user_pages_fast()
4067 * and kmap_atomic/kunmap_atomic() we can get access to the
4068 * pages directly. We then write the data directly into the
4069 * ring buffer.
4070 */
4071 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 4072
d696b58c
SR
4073 /* check if we cross pages */
4074 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4075 nr_pages = 2;
4076
4077 offset = addr & (PAGE_SIZE - 1);
4078 addr &= PAGE_MASK;
4079
4080 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4081 if (ret < nr_pages) {
4082 while (--ret >= 0)
4083 put_page(pages[ret]);
4084 written = -EFAULT;
4085 goto out;
5bf9a1ee 4086 }
d696b58c 4087
6edb2a8a
SR
4088 for (i = 0; i < nr_pages; i++)
4089 map_page[i] = kmap_atomic(pages[i]);
d696b58c
SR
4090
4091 local_save_flags(irq_flags);
4092 size = sizeof(*entry) + cnt + 2; /* possible \n added */
12883efb 4093 buffer = global_trace.trace_buffer.buffer;
d696b58c
SR
4094 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4095 irq_flags, preempt_count());
4096 if (!event) {
4097 /* Ring buffer disabled, return as if not open for write */
4098 written = -EBADF;
4099 goto out_unlock;
5bf9a1ee 4100 }
d696b58c
SR
4101
4102 entry = ring_buffer_event_data(event);
4103 entry->ip = _THIS_IP_;
4104
4105 if (nr_pages == 2) {
4106 len = PAGE_SIZE - offset;
6edb2a8a
SR
4107 memcpy(&entry->buf, map_page[0] + offset, len);
4108 memcpy(&entry->buf[len], map_page[1], cnt - len);
c13d2f7c 4109 } else
6edb2a8a 4110 memcpy(&entry->buf, map_page[0] + offset, cnt);
5bf9a1ee 4111
d696b58c
SR
4112 if (entry->buf[cnt - 1] != '\n') {
4113 entry->buf[cnt] = '\n';
4114 entry->buf[cnt + 1] = '\0';
4115 } else
4116 entry->buf[cnt] = '\0';
4117
7ffbd48d 4118 __buffer_unlock_commit(buffer, event);
5bf9a1ee 4119
d696b58c 4120 written = cnt;
5bf9a1ee 4121
d696b58c 4122 *fpos += written;
1aa54bca 4123
d696b58c 4124 out_unlock:
6edb2a8a
SR
4125 for (i = 0; i < nr_pages; i++){
4126 kunmap_atomic(map_page[i]);
4127 put_page(pages[i]);
4128 }
d696b58c 4129 out:
1aa54bca 4130 return written;
5bf9a1ee
PP
4131}
4132
13f16d20 4133static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 4134{
2b6080f2 4135 struct trace_array *tr = m->private;
5079f326
Z
4136 int i;
4137
4138 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 4139 seq_printf(m,
5079f326 4140 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
4141 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4142 i == tr->clock_id ? "]" : "");
13f16d20 4143 seq_putc(m, '\n');
5079f326 4144
13f16d20 4145 return 0;
5079f326
Z
4146}
4147
4148static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4149 size_t cnt, loff_t *fpos)
4150{
2b6080f2
SR
4151 struct seq_file *m = filp->private_data;
4152 struct trace_array *tr = m->private;
5079f326
Z
4153 char buf[64];
4154 const char *clockstr;
4155 int i;
4156
4157 if (cnt >= sizeof(buf))
4158 return -EINVAL;
4159
4160 if (copy_from_user(&buf, ubuf, cnt))
4161 return -EFAULT;
4162
4163 buf[cnt] = 0;
4164
4165 clockstr = strstrip(buf);
4166
4167 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4168 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4169 break;
4170 }
4171 if (i == ARRAY_SIZE(trace_clocks))
4172 return -EINVAL;
4173
5079f326
Z
4174 mutex_lock(&trace_types_lock);
4175
2b6080f2
SR
4176 tr->clock_id = i;
4177
12883efb 4178 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 4179
60303ed3
DS
4180 /*
4181 * New clock may not be consistent with the previous clock.
4182 * Reset the buffer so that it doesn't have incomparable timestamps.
4183 */
12883efb
SRRH
4184 tracing_reset_online_cpus(&global_trace.trace_buffer);
4185
4186#ifdef CONFIG_TRACER_MAX_TRACE
4187 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4188 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4189 tracing_reset_online_cpus(&global_trace.max_buffer);
4190#endif
60303ed3 4191
5079f326
Z
4192 mutex_unlock(&trace_types_lock);
4193
4194 *fpos += cnt;
4195
4196 return cnt;
4197}
4198
13f16d20
LZ
4199static int tracing_clock_open(struct inode *inode, struct file *file)
4200{
4201 if (tracing_disabled)
4202 return -ENODEV;
2b6080f2
SR
4203
4204 return single_open(file, tracing_clock_show, inode->i_private);
13f16d20
LZ
4205}
4206
6de58e62
SRRH
4207struct ftrace_buffer_info {
4208 struct trace_iterator iter;
4209 void *spare;
4210 unsigned int read;
4211};
4212
debdd57f
HT
4213#ifdef CONFIG_TRACER_SNAPSHOT
4214static int tracing_snapshot_open(struct inode *inode, struct file *file)
4215{
2b6080f2 4216 struct trace_cpu *tc = inode->i_private;
debdd57f 4217 struct trace_iterator *iter;
2b6080f2 4218 struct seq_file *m;
debdd57f
HT
4219 int ret = 0;
4220
4221 if (file->f_mode & FMODE_READ) {
4222 iter = __tracing_open(inode, file, true);
4223 if (IS_ERR(iter))
4224 ret = PTR_ERR(iter);
2b6080f2
SR
4225 } else {
4226 /* Writes still need the seq_file to hold the private data */
4227 m = kzalloc(sizeof(*m), GFP_KERNEL);
4228 if (!m)
4229 return -ENOMEM;
4230 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4231 if (!iter) {
4232 kfree(m);
4233 return -ENOMEM;
4234 }
4235 iter->tr = tc->tr;
12883efb 4236 iter->trace_buffer = &tc->tr->max_buffer;
f1affcaa 4237 iter->cpu_file = tc->cpu;
2b6080f2
SR
4238 m->private = iter;
4239 file->private_data = m;
debdd57f 4240 }
2b6080f2 4241
debdd57f
HT
4242 return ret;
4243}
4244
4245static ssize_t
4246tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4247 loff_t *ppos)
4248{
2b6080f2
SR
4249 struct seq_file *m = filp->private_data;
4250 struct trace_iterator *iter = m->private;
4251 struct trace_array *tr = iter->tr;
debdd57f
HT
4252 unsigned long val;
4253 int ret;
4254
4255 ret = tracing_update_buffers();
4256 if (ret < 0)
4257 return ret;
4258
4259 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4260 if (ret)
4261 return ret;
4262
4263 mutex_lock(&trace_types_lock);
4264
2b6080f2 4265 if (tr->current_trace->use_max_tr) {
debdd57f
HT
4266 ret = -EBUSY;
4267 goto out;
4268 }
4269
4270 switch (val) {
4271 case 0:
f1affcaa
SRRH
4272 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4273 ret = -EINVAL;
4274 break;
4275 }
45ad21ca 4276 if (tr->allocated_snapshot) {
debdd57f 4277 /* free spare buffer */
12883efb 4278 ring_buffer_resize(tr->max_buffer.buffer, 1,
debdd57f 4279 RING_BUFFER_ALL_CPUS);
12883efb
SRRH
4280 set_buffer_entries(&tr->max_buffer, 1);
4281 tracing_reset_online_cpus(&tr->max_buffer);
45ad21ca 4282 tr->allocated_snapshot = false;
debdd57f
HT
4283 }
4284 break;
4285 case 1:
f1affcaa
SRRH
4286/* Only allow per-cpu swap if the ring buffer supports it */
4287#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4288 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4289 ret = -EINVAL;
4290 break;
4291 }
4292#endif
45ad21ca 4293 if (!tr->allocated_snapshot) {
debdd57f 4294 /* allocate spare buffer */
12883efb
SRRH
4295 ret = resize_buffer_duplicate_size(&tr->max_buffer,
4296 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
debdd57f
HT
4297 if (ret < 0)
4298 break;
45ad21ca 4299 tr->allocated_snapshot = true;
debdd57f 4300 }
debdd57f
HT
4301 local_irq_disable();
4302 /* Now, we're going to swap */
f1affcaa
SRRH
4303 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4304 update_max_tr(&global_trace, current, smp_processor_id());
4305 else
4306 update_max_tr_single(&global_trace, current, iter->cpu_file);
debdd57f
HT
4307 local_irq_enable();
4308 break;
4309 default:
45ad21ca 4310 if (tr->allocated_snapshot) {
f1affcaa
SRRH
4311 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4312 tracing_reset_online_cpus(&tr->max_buffer);
4313 else
4314 tracing_reset(&tr->max_buffer, iter->cpu_file);
4315 }
debdd57f
HT
4316 break;
4317 }
4318
4319 if (ret >= 0) {
4320 *ppos += cnt;
4321 ret = cnt;
4322 }
4323out:
4324 mutex_unlock(&trace_types_lock);
4325 return ret;
4326}
2b6080f2
SR
4327
4328static int tracing_snapshot_release(struct inode *inode, struct file *file)
4329{
4330 struct seq_file *m = file->private_data;
4331
4332 if (file->f_mode & FMODE_READ)
4333 return tracing_release(inode, file);
4334
4335 /* If write only, the seq_file is just a stub */
4336 if (m)
4337 kfree(m->private);
4338 kfree(m);
4339
4340 return 0;
4341}
4342
6de58e62
SRRH
4343static int tracing_buffers_open(struct inode *inode, struct file *filp);
4344static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4345 size_t count, loff_t *ppos);
4346static int tracing_buffers_release(struct inode *inode, struct file *file);
4347static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4348 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4349
4350static int snapshot_raw_open(struct inode *inode, struct file *filp)
4351{
4352 struct ftrace_buffer_info *info;
4353 int ret;
4354
4355 ret = tracing_buffers_open(inode, filp);
4356 if (ret < 0)
4357 return ret;
4358
4359 info = filp->private_data;
4360
4361 if (info->iter.trace->use_max_tr) {
4362 tracing_buffers_release(inode, filp);
4363 return -EBUSY;
4364 }
4365
4366 info->iter.snapshot = true;
4367 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4368
4369 return ret;
4370}
4371
debdd57f
HT
4372#endif /* CONFIG_TRACER_SNAPSHOT */
4373
4374
5e2336a0 4375static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
4376 .open = tracing_open_generic,
4377 .read = tracing_max_lat_read,
4378 .write = tracing_max_lat_write,
b444786f 4379 .llseek = generic_file_llseek,
bc0c38d1
SR
4380};
4381
5e2336a0 4382static const struct file_operations set_tracer_fops = {
4bf39a94
IM
4383 .open = tracing_open_generic,
4384 .read = tracing_set_trace_read,
4385 .write = tracing_set_trace_write,
b444786f 4386 .llseek = generic_file_llseek,
bc0c38d1
SR
4387};
4388
5e2336a0 4389static const struct file_operations tracing_pipe_fops = {
4bf39a94 4390 .open = tracing_open_pipe,
2a2cc8f7 4391 .poll = tracing_poll_pipe,
4bf39a94 4392 .read = tracing_read_pipe,
3c56819b 4393 .splice_read = tracing_splice_read_pipe,
4bf39a94 4394 .release = tracing_release_pipe,
b444786f 4395 .llseek = no_llseek,
b3806b43
SR
4396};
4397
5e2336a0 4398static const struct file_operations tracing_entries_fops = {
2b6080f2 4399 .open = tracing_open_generic,
a98a3c3f
SR
4400 .read = tracing_entries_read,
4401 .write = tracing_entries_write,
b444786f 4402 .llseek = generic_file_llseek,
a98a3c3f
SR
4403};
4404
f81ab074
VN
4405static const struct file_operations tracing_total_entries_fops = {
4406 .open = tracing_open_generic,
4407 .read = tracing_total_entries_read,
4408 .llseek = generic_file_llseek,
4409};
4410
4f271a2a
VN
4411static const struct file_operations tracing_free_buffer_fops = {
4412 .write = tracing_free_buffer_write,
4413 .release = tracing_free_buffer_release,
4414};
4415
5e2336a0 4416static const struct file_operations tracing_mark_fops = {
43a15386 4417 .open = tracing_open_generic,
5bf9a1ee 4418 .write = tracing_mark_write,
b444786f 4419 .llseek = generic_file_llseek,
5bf9a1ee
PP
4420};
4421
5079f326 4422static const struct file_operations trace_clock_fops = {
13f16d20
LZ
4423 .open = tracing_clock_open,
4424 .read = seq_read,
4425 .llseek = seq_lseek,
4426 .release = single_release,
5079f326
Z
4427 .write = tracing_clock_write,
4428};
4429
debdd57f
HT
4430#ifdef CONFIG_TRACER_SNAPSHOT
4431static const struct file_operations snapshot_fops = {
4432 .open = tracing_snapshot_open,
4433 .read = seq_read,
4434 .write = tracing_snapshot_write,
4435 .llseek = tracing_seek,
2b6080f2 4436 .release = tracing_snapshot_release,
debdd57f 4437};
debdd57f 4438
6de58e62
SRRH
4439static const struct file_operations snapshot_raw_fops = {
4440 .open = snapshot_raw_open,
4441 .read = tracing_buffers_read,
4442 .release = tracing_buffers_release,
4443 .splice_read = tracing_buffers_splice_read,
4444 .llseek = no_llseek,
2cadf913
SR
4445};
4446
6de58e62
SRRH
4447#endif /* CONFIG_TRACER_SNAPSHOT */
4448
2cadf913
SR
4449static int tracing_buffers_open(struct inode *inode, struct file *filp)
4450{
2b6080f2
SR
4451 struct trace_cpu *tc = inode->i_private;
4452 struct trace_array *tr = tc->tr;
2cadf913
SR
4453 struct ftrace_buffer_info *info;
4454
4455 if (tracing_disabled)
4456 return -ENODEV;
4457
4458 info = kzalloc(sizeof(*info), GFP_KERNEL);
4459 if (!info)
4460 return -ENOMEM;
4461
cc60cdc9
SR
4462 info->iter.tr = tr;
4463 info->iter.cpu_file = tc->cpu;
b627344f 4464 info->iter.trace = tr->current_trace;
12883efb 4465 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 4466 info->spare = NULL;
2cadf913 4467 /* Force reading ring buffer for first read */
cc60cdc9 4468 info->read = (unsigned int)-1;
2cadf913
SR
4469
4470 filp->private_data = info;
4471
d1e7e02f 4472 return nonseekable_open(inode, filp);
2cadf913
SR
4473}
4474
cc60cdc9
SR
4475static unsigned int
4476tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4477{
4478 struct ftrace_buffer_info *info = filp->private_data;
4479 struct trace_iterator *iter = &info->iter;
4480
4481 return trace_poll(iter, filp, poll_table);
4482}
4483
2cadf913
SR
4484static ssize_t
4485tracing_buffers_read(struct file *filp, char __user *ubuf,
4486 size_t count, loff_t *ppos)
4487{
4488 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 4489 struct trace_iterator *iter = &info->iter;
2cadf913 4490 ssize_t ret;
6de58e62 4491 ssize_t size;
2cadf913 4492
2dc5d12b
SR
4493 if (!count)
4494 return 0;
4495
6de58e62
SRRH
4496 mutex_lock(&trace_types_lock);
4497
4498#ifdef CONFIG_TRACER_MAX_TRACE
4499 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4500 size = -EBUSY;
4501 goto out_unlock;
4502 }
4503#endif
4504
ddd538f3 4505 if (!info->spare)
12883efb
SRRH
4506 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4507 iter->cpu_file);
6de58e62 4508 size = -ENOMEM;
ddd538f3 4509 if (!info->spare)
6de58e62 4510 goto out_unlock;
ddd538f3 4511
2cadf913
SR
4512 /* Do we have previous read data to read? */
4513 if (info->read < PAGE_SIZE)
4514 goto read;
4515
b627344f 4516 again:
cc60cdc9 4517 trace_access_lock(iter->cpu_file);
12883efb 4518 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
4519 &info->spare,
4520 count,
cc60cdc9
SR
4521 iter->cpu_file, 0);
4522 trace_access_unlock(iter->cpu_file);
b627344f
SR
4523
4524 if (ret < 0) {
4525 if (trace_empty(iter)) {
6de58e62
SRRH
4526 if ((filp->f_flags & O_NONBLOCK)) {
4527 size = -EAGAIN;
4528 goto out_unlock;
4529 }
4530 mutex_unlock(&trace_types_lock);
b627344f 4531 iter->trace->wait_pipe(iter);
6de58e62
SRRH
4532 mutex_lock(&trace_types_lock);
4533 if (signal_pending(current)) {
4534 size = -EINTR;
4535 goto out_unlock;
4536 }
b627344f
SR
4537 goto again;
4538 }
6de58e62
SRRH
4539 size = 0;
4540 goto out_unlock;
b627344f 4541 }
2cadf913 4542
436fc280 4543 info->read = 0;
b627344f 4544 read:
2cadf913
SR
4545 size = PAGE_SIZE - info->read;
4546 if (size > count)
4547 size = count;
4548
4549 ret = copy_to_user(ubuf, info->spare + info->read, size);
6de58e62
SRRH
4550 if (ret == size) {
4551 size = -EFAULT;
4552 goto out_unlock;
4553 }
2dc5d12b
SR
4554 size -= ret;
4555
2cadf913
SR
4556 *ppos += size;
4557 info->read += size;
4558
6de58e62
SRRH
4559 out_unlock:
4560 mutex_unlock(&trace_types_lock);
4561
2cadf913
SR
4562 return size;
4563}
4564
4565static int tracing_buffers_release(struct inode *inode, struct file *file)
4566{
4567 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 4568 struct trace_iterator *iter = &info->iter;
2cadf913 4569
ddd538f3 4570 if (info->spare)
12883efb 4571 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
2cadf913
SR
4572 kfree(info);
4573
4574 return 0;
4575}
4576
4577struct buffer_ref {
4578 struct ring_buffer *buffer;
4579 void *page;
4580 int ref;
4581};
4582
4583static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4584 struct pipe_buffer *buf)
4585{
4586 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4587
4588 if (--ref->ref)
4589 return;
4590
4591 ring_buffer_free_read_page(ref->buffer, ref->page);
4592 kfree(ref);
4593 buf->private = 0;
4594}
4595
2cadf913
SR
4596static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4597 struct pipe_buffer *buf)
4598{
4599 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4600
4601 ref->ref++;
4602}
4603
4604/* Pipe buffer operations for a buffer. */
28dfef8f 4605static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913
SR
4606 .can_merge = 0,
4607 .map = generic_pipe_buf_map,
4608 .unmap = generic_pipe_buf_unmap,
4609 .confirm = generic_pipe_buf_confirm,
4610 .release = buffer_pipe_buf_release,
d55cb6cf 4611 .steal = generic_pipe_buf_steal,
2cadf913
SR
4612 .get = buffer_pipe_buf_get,
4613};
4614
4615/*
4616 * Callback from splice_to_pipe(), if we need to release some pages
4617 * at the end of the spd in case we error'ed out in filling the pipe.
4618 */
4619static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4620{
4621 struct buffer_ref *ref =
4622 (struct buffer_ref *)spd->partial[i].private;
4623
4624 if (--ref->ref)
4625 return;
4626
4627 ring_buffer_free_read_page(ref->buffer, ref->page);
4628 kfree(ref);
4629 spd->partial[i].private = 0;
4630}
4631
4632static ssize_t
4633tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4634 struct pipe_inode_info *pipe, size_t len,
4635 unsigned int flags)
4636{
4637 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 4638 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
4639 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4640 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 4641 struct splice_pipe_desc spd = {
35f3d14d
JA
4642 .pages = pages_def,
4643 .partial = partial_def,
047fe360 4644 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
4645 .flags = flags,
4646 .ops = &buffer_pipe_buf_ops,
4647 .spd_release = buffer_spd_release,
4648 };
4649 struct buffer_ref *ref;
93459c6c 4650 int entries, size, i;
6de58e62 4651 ssize_t ret;
2cadf913 4652
6de58e62
SRRH
4653 mutex_lock(&trace_types_lock);
4654
4655#ifdef CONFIG_TRACER_MAX_TRACE
4656 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4657 ret = -EBUSY;
4658 goto out;
4659 }
4660#endif
4661
4662 if (splice_grow_spd(pipe, &spd)) {
4663 ret = -ENOMEM;
4664 goto out;
4665 }
35f3d14d 4666
93cfb3c9 4667 if (*ppos & (PAGE_SIZE - 1)) {
35f3d14d
JA
4668 ret = -EINVAL;
4669 goto out;
93cfb3c9
LJ
4670 }
4671
4672 if (len & (PAGE_SIZE - 1)) {
35f3d14d
JA
4673 if (len < PAGE_SIZE) {
4674 ret = -EINVAL;
4675 goto out;
4676 }
93cfb3c9
LJ
4677 len &= PAGE_MASK;
4678 }
4679
cc60cdc9
SR
4680 again:
4681 trace_access_lock(iter->cpu_file);
12883efb 4682 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 4683
35f3d14d 4684 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
4685 struct page *page;
4686 int r;
4687
4688 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4689 if (!ref)
4690 break;
4691
7267fa68 4692 ref->ref = 1;
12883efb 4693 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 4694 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
2cadf913
SR
4695 if (!ref->page) {
4696 kfree(ref);
4697 break;
4698 }
4699
4700 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 4701 len, iter->cpu_file, 1);
2cadf913 4702 if (r < 0) {
7ea59064 4703 ring_buffer_free_read_page(ref->buffer, ref->page);
2cadf913
SR
4704 kfree(ref);
4705 break;
4706 }
4707
4708 /*
4709 * zero out any left over data, this is going to
4710 * user land.
4711 */
4712 size = ring_buffer_page_len(ref->page);
4713 if (size < PAGE_SIZE)
4714 memset(ref->page + size, 0, PAGE_SIZE - size);
4715
4716 page = virt_to_page(ref->page);
4717
4718 spd.pages[i] = page;
4719 spd.partial[i].len = PAGE_SIZE;
4720 spd.partial[i].offset = 0;
4721 spd.partial[i].private = (unsigned long)ref;
4722 spd.nr_pages++;
93cfb3c9 4723 *ppos += PAGE_SIZE;
93459c6c 4724
12883efb 4725 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
4726 }
4727
cc60cdc9 4728 trace_access_unlock(iter->cpu_file);
2cadf913
SR
4729 spd.nr_pages = i;
4730
4731 /* did we read anything? */
4732 if (!spd.nr_pages) {
cc60cdc9 4733 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
2cadf913 4734 ret = -EAGAIN;
cc60cdc9
SR
4735 goto out;
4736 }
6de58e62 4737 mutex_unlock(&trace_types_lock);
b627344f 4738 iter->trace->wait_pipe(iter);
6de58e62 4739 mutex_lock(&trace_types_lock);
cc60cdc9
SR
4740 if (signal_pending(current)) {
4741 ret = -EINTR;
4742 goto out;
4743 }
4744 goto again;
2cadf913
SR
4745 }
4746
4747 ret = splice_to_pipe(pipe, &spd);
047fe360 4748 splice_shrink_spd(&spd);
35f3d14d 4749out:
6de58e62
SRRH
4750 mutex_unlock(&trace_types_lock);
4751
2cadf913
SR
4752 return ret;
4753}
4754
4755static const struct file_operations tracing_buffers_fops = {
4756 .open = tracing_buffers_open,
4757 .read = tracing_buffers_read,
cc60cdc9 4758 .poll = tracing_buffers_poll,
2cadf913
SR
4759 .release = tracing_buffers_release,
4760 .splice_read = tracing_buffers_splice_read,
4761 .llseek = no_llseek,
4762};
4763
c8d77183
SR
4764static ssize_t
4765tracing_stats_read(struct file *filp, char __user *ubuf,
4766 size_t count, loff_t *ppos)
4767{
2b6080f2
SR
4768 struct trace_cpu *tc = filp->private_data;
4769 struct trace_array *tr = tc->tr;
12883efb 4770 struct trace_buffer *trace_buf = &tr->trace_buffer;
c8d77183
SR
4771 struct trace_seq *s;
4772 unsigned long cnt;
c64e148a
VN
4773 unsigned long long t;
4774 unsigned long usec_rem;
2b6080f2 4775 int cpu = tc->cpu;
c8d77183 4776
e4f2d10f 4777 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 4778 if (!s)
a646365c 4779 return -ENOMEM;
c8d77183
SR
4780
4781 trace_seq_init(s);
4782
12883efb 4783 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
4784 trace_seq_printf(s, "entries: %ld\n", cnt);
4785
12883efb 4786 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
4787 trace_seq_printf(s, "overrun: %ld\n", cnt);
4788
12883efb 4789 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
4790 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4791
12883efb 4792 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
4793 trace_seq_printf(s, "bytes: %ld\n", cnt);
4794
11043d8b
YY
4795 if (trace_clocks[trace_clock_id].in_ns) {
4796 /* local or global for trace_clock */
12883efb 4797 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
4798 usec_rem = do_div(t, USEC_PER_SEC);
4799 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
4800 t, usec_rem);
4801
12883efb 4802 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
4803 usec_rem = do_div(t, USEC_PER_SEC);
4804 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4805 } else {
4806 /* counter or tsc mode for trace_clock */
4807 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 4808 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 4809
11043d8b 4810 trace_seq_printf(s, "now ts: %llu\n",
12883efb 4811 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 4812 }
c64e148a 4813
12883efb 4814 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
4815 trace_seq_printf(s, "dropped events: %ld\n", cnt);
4816
12883efb 4817 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
4818 trace_seq_printf(s, "read events: %ld\n", cnt);
4819
c8d77183
SR
4820 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4821
4822 kfree(s);
4823
4824 return count;
4825}
4826
4827static const struct file_operations tracing_stats_fops = {
4828 .open = tracing_open_generic,
4829 .read = tracing_stats_read,
b444786f 4830 .llseek = generic_file_llseek,
c8d77183
SR
4831};
4832
bc0c38d1
SR
4833#ifdef CONFIG_DYNAMIC_FTRACE
4834
b807c3d0
SR
4835int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4836{
4837 return 0;
4838}
4839
bc0c38d1 4840static ssize_t
b807c3d0 4841tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
4842 size_t cnt, loff_t *ppos)
4843{
a26a2a27
SR
4844 static char ftrace_dyn_info_buffer[1024];
4845 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 4846 unsigned long *p = filp->private_data;
b807c3d0 4847 char *buf = ftrace_dyn_info_buffer;
a26a2a27 4848 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
4849 int r;
4850
b807c3d0
SR
4851 mutex_lock(&dyn_info_mutex);
4852 r = sprintf(buf, "%ld ", *p);
4bf39a94 4853
a26a2a27 4854 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
4855 buf[r++] = '\n';
4856
4857 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4858
4859 mutex_unlock(&dyn_info_mutex);
4860
4861 return r;
bc0c38d1
SR
4862}
4863
5e2336a0 4864static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 4865 .open = tracing_open_generic,
b807c3d0 4866 .read = tracing_read_dyn_info,
b444786f 4867 .llseek = generic_file_llseek,
bc0c38d1
SR
4868};
4869#endif
4870
2b6080f2 4871struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
bc0c38d1
SR
4872{
4873 static int once;
4874
2b6080f2
SR
4875 if (tr->dir)
4876 return tr->dir;
bc0c38d1 4877
3e1f60b8
FW
4878 if (!debugfs_initialized())
4879 return NULL;
4880
2b6080f2
SR
4881 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
4882 tr->dir = debugfs_create_dir("tracing", NULL);
bc0c38d1 4883
2b6080f2 4884 if (!tr->dir && !once) {
bc0c38d1
SR
4885 once = 1;
4886 pr_warning("Could not create debugfs directory 'tracing'\n");
4887 return NULL;
4888 }
4889
2b6080f2 4890 return tr->dir;
bc0c38d1
SR
4891}
4892
2b6080f2
SR
4893struct dentry *tracing_init_dentry(void)
4894{
4895 return tracing_init_dentry_tr(&global_trace);
4896}
b04cc6b1 4897
2b6080f2 4898static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 4899{
b04cc6b1
FW
4900 struct dentry *d_tracer;
4901
2b6080f2
SR
4902 if (tr->percpu_dir)
4903 return tr->percpu_dir;
b04cc6b1 4904
2b6080f2 4905 d_tracer = tracing_init_dentry_tr(tr);
b04cc6b1
FW
4906 if (!d_tracer)
4907 return NULL;
4908
2b6080f2 4909 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
b04cc6b1 4910
2b6080f2
SR
4911 WARN_ONCE(!tr->percpu_dir,
4912 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 4913
2b6080f2 4914 return tr->percpu_dir;
b04cc6b1
FW
4915}
4916
2b6080f2
SR
4917static void
4918tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 4919{
12883efb 4920 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
2b6080f2 4921 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 4922 struct dentry *d_cpu;
dd49a38c 4923 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 4924
0a3d7ce7
NK
4925 if (!d_percpu)
4926 return;
4927
dd49a38c 4928 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8656e7a2
FW
4929 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4930 if (!d_cpu) {
4931 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4932 return;
4933 }
b04cc6b1 4934
8656e7a2 4935 /* per cpu trace_pipe */
5452af66 4936 trace_create_file("trace_pipe", 0444, d_cpu,
2b6080f2 4937 (void *)&data->trace_cpu, &tracing_pipe_fops);
b04cc6b1
FW
4938
4939 /* per cpu trace */
5452af66 4940 trace_create_file("trace", 0644, d_cpu,
2b6080f2 4941 (void *)&data->trace_cpu, &tracing_fops);
7f96f93f 4942
5452af66 4943 trace_create_file("trace_pipe_raw", 0444, d_cpu,
2b6080f2 4944 (void *)&data->trace_cpu, &tracing_buffers_fops);
7f96f93f 4945
c8d77183 4946 trace_create_file("stats", 0444, d_cpu,
2b6080f2 4947 (void *)&data->trace_cpu, &tracing_stats_fops);
438ced17
VN
4948
4949 trace_create_file("buffer_size_kb", 0444, d_cpu,
2b6080f2 4950 (void *)&data->trace_cpu, &tracing_entries_fops);
f1affcaa
SRRH
4951
4952#ifdef CONFIG_TRACER_SNAPSHOT
4953 trace_create_file("snapshot", 0644, d_cpu,
4954 (void *)&data->trace_cpu, &snapshot_fops);
6de58e62
SRRH
4955
4956 trace_create_file("snapshot_raw", 0444, d_cpu,
4957 (void *)&data->trace_cpu, &snapshot_raw_fops);
f1affcaa 4958#endif
b04cc6b1
FW
4959}
4960
60a11774
SR
4961#ifdef CONFIG_FTRACE_SELFTEST
4962/* Let selftest have access to static functions in this file */
4963#include "trace_selftest.c"
4964#endif
4965
577b785f
SR
4966struct trace_option_dentry {
4967 struct tracer_opt *opt;
4968 struct tracer_flags *flags;
2b6080f2 4969 struct trace_array *tr;
577b785f
SR
4970 struct dentry *entry;
4971};
4972
4973static ssize_t
4974trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4975 loff_t *ppos)
4976{
4977 struct trace_option_dentry *topt = filp->private_data;
4978 char *buf;
4979
4980 if (topt->flags->val & topt->opt->bit)
4981 buf = "1\n";
4982 else
4983 buf = "0\n";
4984
4985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4986}
4987
4988static ssize_t
4989trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4990 loff_t *ppos)
4991{
4992 struct trace_option_dentry *topt = filp->private_data;
4993 unsigned long val;
577b785f
SR
4994 int ret;
4995
22fe9b54
PH
4996 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4997 if (ret)
577b785f
SR
4998 return ret;
4999
8d18eaaf
LZ
5000 if (val != 0 && val != 1)
5001 return -EINVAL;
577b785f 5002
8d18eaaf 5003 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 5004 mutex_lock(&trace_types_lock);
2b6080f2 5005 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
c757bea9 5006 topt->opt, !val);
577b785f
SR
5007 mutex_unlock(&trace_types_lock);
5008 if (ret)
5009 return ret;
577b785f
SR
5010 }
5011
5012 *ppos += cnt;
5013
5014 return cnt;
5015}
5016
5017
5018static const struct file_operations trace_options_fops = {
5019 .open = tracing_open_generic,
5020 .read = trace_options_read,
5021 .write = trace_options_write,
b444786f 5022 .llseek = generic_file_llseek,
577b785f
SR
5023};
5024
a8259075
SR
5025static ssize_t
5026trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5027 loff_t *ppos)
5028{
5029 long index = (long)filp->private_data;
5030 char *buf;
5031
5032 if (trace_flags & (1 << index))
5033 buf = "1\n";
5034 else
5035 buf = "0\n";
5036
5037 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5038}
5039
5040static ssize_t
5041trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5042 loff_t *ppos)
5043{
2b6080f2 5044 struct trace_array *tr = &global_trace;
a8259075 5045 long index = (long)filp->private_data;
a8259075
SR
5046 unsigned long val;
5047 int ret;
5048
22fe9b54
PH
5049 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5050 if (ret)
a8259075
SR
5051 return ret;
5052
f2d84b65 5053 if (val != 0 && val != 1)
a8259075 5054 return -EINVAL;
69d34da2
SRRH
5055
5056 mutex_lock(&trace_types_lock);
2b6080f2 5057 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 5058 mutex_unlock(&trace_types_lock);
a8259075 5059
613f04a0
SRRH
5060 if (ret < 0)
5061 return ret;
5062
a8259075
SR
5063 *ppos += cnt;
5064
5065 return cnt;
5066}
5067
a8259075
SR
5068static const struct file_operations trace_options_core_fops = {
5069 .open = tracing_open_generic,
5070 .read = trace_options_core_read,
5071 .write = trace_options_core_write,
b444786f 5072 .llseek = generic_file_llseek,
a8259075
SR
5073};
5074
5452af66 5075struct dentry *trace_create_file(const char *name,
f4ae40a6 5076 umode_t mode,
5452af66
FW
5077 struct dentry *parent,
5078 void *data,
5079 const struct file_operations *fops)
5080{
5081 struct dentry *ret;
5082
5083 ret = debugfs_create_file(name, mode, parent, data, fops);
5084 if (!ret)
5085 pr_warning("Could not create debugfs '%s' entry\n", name);
5086
5087 return ret;
5088}
5089
5090
2b6080f2 5091static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
5092{
5093 struct dentry *d_tracer;
a8259075 5094
2b6080f2
SR
5095 if (tr->options)
5096 return tr->options;
a8259075 5097
2b6080f2 5098 d_tracer = tracing_init_dentry_tr(tr);
a8259075
SR
5099 if (!d_tracer)
5100 return NULL;
5101
2b6080f2
SR
5102 tr->options = debugfs_create_dir("options", d_tracer);
5103 if (!tr->options) {
a8259075
SR
5104 pr_warning("Could not create debugfs directory 'options'\n");
5105 return NULL;
5106 }
5107
2b6080f2 5108 return tr->options;
a8259075
SR
5109}
5110
577b785f 5111static void
2b6080f2
SR
5112create_trace_option_file(struct trace_array *tr,
5113 struct trace_option_dentry *topt,
577b785f
SR
5114 struct tracer_flags *flags,
5115 struct tracer_opt *opt)
5116{
5117 struct dentry *t_options;
577b785f 5118
2b6080f2 5119 t_options = trace_options_init_dentry(tr);
577b785f
SR
5120 if (!t_options)
5121 return;
5122
5123 topt->flags = flags;
5124 topt->opt = opt;
2b6080f2 5125 topt->tr = tr;
577b785f 5126
5452af66 5127 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
5128 &trace_options_fops);
5129
577b785f
SR
5130}
5131
5132static struct trace_option_dentry *
2b6080f2 5133create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
5134{
5135 struct trace_option_dentry *topts;
5136 struct tracer_flags *flags;
5137 struct tracer_opt *opts;
5138 int cnt;
5139
5140 if (!tracer)
5141 return NULL;
5142
5143 flags = tracer->flags;
5144
5145 if (!flags || !flags->opts)
5146 return NULL;
5147
5148 opts = flags->opts;
5149
5150 for (cnt = 0; opts[cnt].name; cnt++)
5151 ;
5152
0cfe8245 5153 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
5154 if (!topts)
5155 return NULL;
5156
5157 for (cnt = 0; opts[cnt].name; cnt++)
2b6080f2 5158 create_trace_option_file(tr, &topts[cnt], flags,
577b785f
SR
5159 &opts[cnt]);
5160
5161 return topts;
5162}
5163
5164static void
5165destroy_trace_option_files(struct trace_option_dentry *topts)
5166{
5167 int cnt;
5168
5169 if (!topts)
5170 return;
5171
5172 for (cnt = 0; topts[cnt].opt; cnt++) {
5173 if (topts[cnt].entry)
5174 debugfs_remove(topts[cnt].entry);
5175 }
5176
5177 kfree(topts);
5178}
5179
a8259075 5180static struct dentry *
2b6080f2
SR
5181create_trace_option_core_file(struct trace_array *tr,
5182 const char *option, long index)
a8259075
SR
5183{
5184 struct dentry *t_options;
a8259075 5185
2b6080f2 5186 t_options = trace_options_init_dentry(tr);
a8259075
SR
5187 if (!t_options)
5188 return NULL;
5189
5452af66 5190 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 5191 &trace_options_core_fops);
a8259075
SR
5192}
5193
2b6080f2 5194static __init void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
5195{
5196 struct dentry *t_options;
a8259075
SR
5197 int i;
5198
2b6080f2 5199 t_options = trace_options_init_dentry(tr);
a8259075
SR
5200 if (!t_options)
5201 return;
5202
5452af66 5203 for (i = 0; trace_options[i]; i++)
2b6080f2 5204 create_trace_option_core_file(tr, trace_options[i], i);
a8259075
SR
5205}
5206
499e5470
SR
5207static ssize_t
5208rb_simple_read(struct file *filp, char __user *ubuf,
5209 size_t cnt, loff_t *ppos)
5210{
348f0fc2 5211 struct trace_array *tr = filp->private_data;
12883efb 5212 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
5213 char buf[64];
5214 int r;
5215
5216 if (buffer)
5217 r = ring_buffer_record_is_on(buffer);
5218 else
5219 r = 0;
5220
5221 r = sprintf(buf, "%d\n", r);
5222
5223 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5224}
5225
5226static ssize_t
5227rb_simple_write(struct file *filp, const char __user *ubuf,
5228 size_t cnt, loff_t *ppos)
5229{
348f0fc2 5230 struct trace_array *tr = filp->private_data;
12883efb 5231 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
5232 unsigned long val;
5233 int ret;
5234
5235 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5236 if (ret)
5237 return ret;
5238
5239 if (buffer) {
2df8f8a6
SR
5240 mutex_lock(&trace_types_lock);
5241 if (val) {
499e5470 5242 ring_buffer_record_on(buffer);
2b6080f2
SR
5243 if (tr->current_trace->start)
5244 tr->current_trace->start(tr);
2df8f8a6 5245 } else {
499e5470 5246 ring_buffer_record_off(buffer);
2b6080f2
SR
5247 if (tr->current_trace->stop)
5248 tr->current_trace->stop(tr);
2df8f8a6
SR
5249 }
5250 mutex_unlock(&trace_types_lock);
499e5470
SR
5251 }
5252
5253 (*ppos)++;
5254
5255 return cnt;
5256}
5257
5258static const struct file_operations rb_simple_fops = {
5259 .open = tracing_open_generic,
5260 .read = rb_simple_read,
5261 .write = rb_simple_write,
5262 .llseek = default_llseek,
5263};
5264
277ba044
SR
5265struct dentry *trace_instance_dir;
5266
5267static void
5268init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5269
5270static int new_instance_create(const char *name)
5271{
5272 enum ring_buffer_flags rb_flags;
5273 struct trace_array *tr;
5274 int ret;
5275 int i;
5276
5277 mutex_lock(&trace_types_lock);
5278
5279 ret = -EEXIST;
5280 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5281 if (tr->name && strcmp(tr->name, name) == 0)
5282 goto out_unlock;
5283 }
5284
5285 ret = -ENOMEM;
5286 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5287 if (!tr)
5288 goto out_unlock;
5289
5290 tr->name = kstrdup(name, GFP_KERNEL);
5291 if (!tr->name)
5292 goto out_free_tr;
5293
5294 raw_spin_lock_init(&tr->start_lock);
5295
5296 tr->current_trace = &nop_trace;
5297
5298 INIT_LIST_HEAD(&tr->systems);
5299 INIT_LIST_HEAD(&tr->events);
5300
5301 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5302
12883efb
SRRH
5303 tr->trace_buffer.buffer = ring_buffer_alloc(trace_buf_size, rb_flags);
5304 if (!tr->trace_buffer.buffer)
277ba044
SR
5305 goto out_free_tr;
5306
12883efb
SRRH
5307 tr->trace_buffer.data = alloc_percpu(struct trace_array_cpu);
5308 if (!tr->trace_buffer.data)
277ba044
SR
5309 goto out_free_tr;
5310
5311 for_each_tracing_cpu(i) {
12883efb
SRRH
5312 memset(per_cpu_ptr(tr->trace_buffer.data, i), 0, sizeof(struct trace_array_cpu));
5313 per_cpu_ptr(tr->trace_buffer.data, i)->trace_cpu.cpu = i;
5314 per_cpu_ptr(tr->trace_buffer.data, i)->trace_cpu.tr = tr;
277ba044
SR
5315 }
5316
5317 /* Holder for file callbacks */
5318 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5319 tr->trace_cpu.tr = tr;
5320
5321 tr->dir = debugfs_create_dir(name, trace_instance_dir);
5322 if (!tr->dir)
5323 goto out_free_tr;
5324
5325 ret = event_trace_add_tracer(tr->dir, tr);
5326 if (ret)
5327 goto out_free_tr;
5328
5329 init_tracer_debugfs(tr, tr->dir);
5330
5331 list_add(&tr->list, &ftrace_trace_arrays);
5332
5333 mutex_unlock(&trace_types_lock);
5334
5335 return 0;
5336
5337 out_free_tr:
12883efb
SRRH
5338 if (tr->trace_buffer.buffer)
5339 ring_buffer_free(tr->trace_buffer.buffer);
277ba044
SR
5340 kfree(tr->name);
5341 kfree(tr);
5342
5343 out_unlock:
5344 mutex_unlock(&trace_types_lock);
5345
5346 return ret;
5347
5348}
5349
0c8916c3
SR
5350static int instance_delete(const char *name)
5351{
5352 struct trace_array *tr;
5353 int found = 0;
5354 int ret;
5355
5356 mutex_lock(&trace_types_lock);
5357
5358 ret = -ENODEV;
5359 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5360 if (tr->name && strcmp(tr->name, name) == 0) {
5361 found = 1;
5362 break;
5363 }
5364 }
5365 if (!found)
5366 goto out_unlock;
5367
5368 list_del(&tr->list);
5369
5370 event_trace_del_tracer(tr);
5371 debugfs_remove_recursive(tr->dir);
12883efb
SRRH
5372 free_percpu(tr->trace_buffer.data);
5373 ring_buffer_free(tr->trace_buffer.buffer);
0c8916c3
SR
5374
5375 kfree(tr->name);
5376 kfree(tr);
5377
5378 ret = 0;
5379
5380 out_unlock:
5381 mutex_unlock(&trace_types_lock);
5382
5383 return ret;
5384}
5385
277ba044
SR
5386static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5387{
5388 struct dentry *parent;
5389 int ret;
5390
5391 /* Paranoid: Make sure the parent is the "instances" directory */
5392 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5393 if (WARN_ON_ONCE(parent != trace_instance_dir))
5394 return -ENOENT;
5395
5396 /*
5397 * The inode mutex is locked, but debugfs_create_dir() will also
5398 * take the mutex. As the instances directory can not be destroyed
5399 * or changed in any other way, it is safe to unlock it, and
5400 * let the dentry try. If two users try to make the same dir at
5401 * the same time, then the new_instance_create() will determine the
5402 * winner.
5403 */
5404 mutex_unlock(&inode->i_mutex);
5405
5406 ret = new_instance_create(dentry->d_iname);
5407
5408 mutex_lock(&inode->i_mutex);
5409
5410 return ret;
5411}
5412
0c8916c3
SR
5413static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5414{
5415 struct dentry *parent;
5416 int ret;
5417
5418 /* Paranoid: Make sure the parent is the "instances" directory */
5419 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5420 if (WARN_ON_ONCE(parent != trace_instance_dir))
5421 return -ENOENT;
5422
5423 /* The caller did a dget() on dentry */
5424 mutex_unlock(&dentry->d_inode->i_mutex);
5425
5426 /*
5427 * The inode mutex is locked, but debugfs_create_dir() will also
5428 * take the mutex. As the instances directory can not be destroyed
5429 * or changed in any other way, it is safe to unlock it, and
5430 * let the dentry try. If two users try to make the same dir at
5431 * the same time, then the instance_delete() will determine the
5432 * winner.
5433 */
5434 mutex_unlock(&inode->i_mutex);
5435
5436 ret = instance_delete(dentry->d_iname);
5437
5438 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5439 mutex_lock(&dentry->d_inode->i_mutex);
5440
5441 return ret;
5442}
5443
277ba044
SR
5444static const struct inode_operations instance_dir_inode_operations = {
5445 .lookup = simple_lookup,
5446 .mkdir = instance_mkdir,
0c8916c3 5447 .rmdir = instance_rmdir,
277ba044
SR
5448};
5449
5450static __init void create_trace_instances(struct dentry *d_tracer)
5451{
5452 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5453 if (WARN_ON(!trace_instance_dir))
5454 return;
5455
5456 /* Hijack the dir inode operations, to allow mkdir */
5457 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5458}
5459
2b6080f2
SR
5460static void
5461init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5462{
5463
5464 trace_create_file("trace_options", 0644, d_tracer,
5465 tr, &tracing_iter_fops);
5466
5467 trace_create_file("trace", 0644, d_tracer,
5468 (void *)&tr->trace_cpu, &tracing_fops);
5469
5470 trace_create_file("trace_pipe", 0444, d_tracer,
5471 (void *)&tr->trace_cpu, &tracing_pipe_fops);
5472
5473 trace_create_file("buffer_size_kb", 0644, d_tracer,
5474 (void *)&tr->trace_cpu, &tracing_entries_fops);
5475
5476 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5477 tr, &tracing_total_entries_fops);
5478
5479 trace_create_file("free_buffer", 0644, d_tracer,
5480 tr, &tracing_free_buffer_fops);
5481
5482 trace_create_file("trace_marker", 0220, d_tracer,
5483 tr, &tracing_mark_fops);
5484
5485 trace_create_file("trace_clock", 0644, d_tracer, tr,
5486 &trace_clock_fops);
5487
5488 trace_create_file("tracing_on", 0644, d_tracer,
5489 tr, &rb_simple_fops);
5490}
5491
b5ad384e 5492static __init int tracer_init_debugfs(void)
bc0c38d1
SR
5493{
5494 struct dentry *d_tracer;
b04cc6b1 5495 int cpu;
bc0c38d1 5496
7e53bd42
LJ
5497 trace_access_lock_init();
5498
bc0c38d1
SR
5499 d_tracer = tracing_init_dentry();
5500
2b6080f2 5501 init_tracer_debugfs(&global_trace, d_tracer);
bc0c38d1 5502
5452af66 5503 trace_create_file("tracing_cpumask", 0644, d_tracer,
2b6080f2 5504 &global_trace, &tracing_cpumask_fops);
a8259075 5505
5452af66
FW
5506 trace_create_file("available_tracers", 0444, d_tracer,
5507 &global_trace, &show_traces_fops);
5508
339ae5d3 5509 trace_create_file("current_tracer", 0644, d_tracer,
5452af66
FW
5510 &global_trace, &set_tracer_fops);
5511
5d4a9dba 5512#ifdef CONFIG_TRACER_MAX_TRACE
5452af66
FW
5513 trace_create_file("tracing_max_latency", 0644, d_tracer,
5514 &tracing_max_latency, &tracing_max_lat_fops);
0e950173 5515#endif
5452af66
FW
5516
5517 trace_create_file("tracing_thresh", 0644, d_tracer,
5518 &tracing_thresh, &tracing_max_lat_fops);
a8259075 5519
339ae5d3 5520 trace_create_file("README", 0444, d_tracer,
5452af66
FW
5521 NULL, &tracing_readme_fops);
5522
69abe6a5
AP
5523 trace_create_file("saved_cmdlines", 0444, d_tracer,
5524 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 5525
bc0c38d1 5526#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
5527 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5528 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 5529#endif
b04cc6b1 5530
debdd57f
HT
5531#ifdef CONFIG_TRACER_SNAPSHOT
5532 trace_create_file("snapshot", 0644, d_tracer,
2b6080f2 5533 (void *)&global_trace.trace_cpu, &snapshot_fops);
debdd57f
HT
5534#endif
5535
277ba044
SR
5536 create_trace_instances(d_tracer);
5537
2b6080f2 5538 create_trace_options_dir(&global_trace);
5452af66 5539
b04cc6b1 5540 for_each_tracing_cpu(cpu)
2b6080f2 5541 tracing_init_debugfs_percpu(&global_trace, cpu);
b04cc6b1 5542
b5ad384e 5543 return 0;
bc0c38d1
SR
5544}
5545
3f5a54e3
SR
5546static int trace_panic_handler(struct notifier_block *this,
5547 unsigned long event, void *unused)
5548{
944ac425 5549 if (ftrace_dump_on_oops)
cecbca96 5550 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
5551 return NOTIFY_OK;
5552}
5553
5554static struct notifier_block trace_panic_notifier = {
5555 .notifier_call = trace_panic_handler,
5556 .next = NULL,
5557 .priority = 150 /* priority: INT_MAX >= x >= 0 */
5558};
5559
5560static int trace_die_handler(struct notifier_block *self,
5561 unsigned long val,
5562 void *data)
5563{
5564 switch (val) {
5565 case DIE_OOPS:
944ac425 5566 if (ftrace_dump_on_oops)
cecbca96 5567 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
5568 break;
5569 default:
5570 break;
5571 }
5572 return NOTIFY_OK;
5573}
5574
5575static struct notifier_block trace_die_notifier = {
5576 .notifier_call = trace_die_handler,
5577 .priority = 200
5578};
5579
5580/*
5581 * printk is set to max of 1024, we really don't need it that big.
5582 * Nothing should be printing 1000 characters anyway.
5583 */
5584#define TRACE_MAX_PRINT 1000
5585
5586/*
5587 * Define here KERN_TRACE so that we have one place to modify
5588 * it if we decide to change what log level the ftrace dump
5589 * should be at.
5590 */
428aee14 5591#define KERN_TRACE KERN_EMERG
3f5a54e3 5592
955b61e5 5593void
3f5a54e3
SR
5594trace_printk_seq(struct trace_seq *s)
5595{
5596 /* Probably should print a warning here. */
5597 if (s->len >= 1000)
5598 s->len = 1000;
5599
5600 /* should be zero ended, but we are paranoid. */
5601 s->buffer[s->len] = 0;
5602
5603 printk(KERN_TRACE "%s", s->buffer);
5604
f9520750 5605 trace_seq_init(s);
3f5a54e3
SR
5606}
5607
955b61e5
JW
5608void trace_init_global_iter(struct trace_iterator *iter)
5609{
5610 iter->tr = &global_trace;
2b6080f2 5611 iter->trace = iter->tr->current_trace;
ae3b5093 5612 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 5613 iter->trace_buffer = &global_trace.trace_buffer;
955b61e5
JW
5614}
5615
cecbca96
FW
5616static void
5617__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 5618{
445c8951 5619 static arch_spinlock_t ftrace_dump_lock =
edc35bd7 5620 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
3f5a54e3
SR
5621 /* use static because iter can be a bit big for the stack */
5622 static struct trace_iterator iter;
cf586b61 5623 unsigned int old_userobj;
3f5a54e3 5624 static int dump_ran;
d769041f
SR
5625 unsigned long flags;
5626 int cnt = 0, cpu;
3f5a54e3
SR
5627
5628 /* only one dump */
cd891ae0 5629 local_irq_save(flags);
0199c4e6 5630 arch_spin_lock(&ftrace_dump_lock);
3f5a54e3
SR
5631 if (dump_ran)
5632 goto out;
5633
5634 dump_ran = 1;
5635
0ee6b6cf 5636 tracing_off();
cf586b61 5637
e0a413f6
SR
5638 /* Did function tracer already get disabled? */
5639 if (ftrace_is_dead()) {
5640 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
5641 printk("# MAY BE MISSING FUNCTION EVENTS\n");
5642 }
5643
cf586b61
FW
5644 if (disable_tracing)
5645 ftrace_kill();
3f5a54e3 5646
38dbe0b1 5647 /* Simulate the iterator */
955b61e5
JW
5648 trace_init_global_iter(&iter);
5649
d769041f 5650 for_each_tracing_cpu(cpu) {
12883efb 5651 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
d769041f
SR
5652 }
5653
cf586b61
FW
5654 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
5655
b54d3de9
TE
5656 /* don't look at user memory in panic mode */
5657 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
5658
cecbca96
FW
5659 switch (oops_dump_mode) {
5660 case DUMP_ALL:
ae3b5093 5661 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
5662 break;
5663 case DUMP_ORIG:
5664 iter.cpu_file = raw_smp_processor_id();
5665 break;
5666 case DUMP_NONE:
5667 goto out_enable;
5668 default:
5669 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 5670 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
5671 }
5672
5673 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3
SR
5674
5675 /*
5676 * We need to stop all tracing on all CPUS to read the
5677 * the next buffer. This is a bit expensive, but is
5678 * not done often. We fill all what we can read,
5679 * and then release the locks again.
5680 */
5681
3f5a54e3
SR
5682 while (!trace_empty(&iter)) {
5683
5684 if (!cnt)
5685 printk(KERN_TRACE "---------------------------------\n");
5686
5687 cnt++;
5688
5689 /* reset all but tr, trace, and overruns */
5690 memset(&iter.seq, 0,
5691 sizeof(struct trace_iterator) -
5692 offsetof(struct trace_iterator, seq));
5693 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5694 iter.pos = -1;
5695
955b61e5 5696 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
5697 int ret;
5698
5699 ret = print_trace_line(&iter);
5700 if (ret != TRACE_TYPE_NO_CONSUME)
5701 trace_consume(&iter);
3f5a54e3 5702 }
b892e5c8 5703 touch_nmi_watchdog();
3f5a54e3
SR
5704
5705 trace_printk_seq(&iter.seq);
5706 }
5707
5708 if (!cnt)
5709 printk(KERN_TRACE " (ftrace buffer empty)\n");
5710 else
5711 printk(KERN_TRACE "---------------------------------\n");
5712
cecbca96 5713 out_enable:
cf586b61
FW
5714 /* Re-enable tracing if requested */
5715 if (!disable_tracing) {
5716 trace_flags |= old_userobj;
5717
5718 for_each_tracing_cpu(cpu) {
12883efb 5719 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61
FW
5720 }
5721 tracing_on();
5722 }
5723
3f5a54e3 5724 out:
0199c4e6 5725 arch_spin_unlock(&ftrace_dump_lock);
cd891ae0 5726 local_irq_restore(flags);
3f5a54e3
SR
5727}
5728
cf586b61 5729/* By default: disable tracing after the dump */
cecbca96 5730void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
cf586b61 5731{
cecbca96 5732 __ftrace_dump(true, oops_dump_mode);
cf586b61 5733}
a8eecf22 5734EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 5735
3928a8a2 5736__init static int tracer_alloc_buffers(void)
bc0c38d1 5737{
73c5162a 5738 int ring_buf_size;
750912fa 5739 enum ring_buffer_flags rb_flags;
4c11d7ae 5740 int i;
9e01c1b7 5741 int ret = -ENOMEM;
4c11d7ae 5742
750912fa 5743
9e01c1b7
RR
5744 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5745 goto out;
5746
5747 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5748 goto out_free_buffer_mask;
4c11d7ae 5749
07d777fe
SR
5750 /* Only allocate trace_printk buffers if a trace_printk exists */
5751 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 5752 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
5753 trace_printk_init_buffers();
5754
73c5162a
SR
5755 /* To save memory, keep the ring buffer size to its minimum */
5756 if (ring_buffer_expanded)
5757 ring_buf_size = trace_buf_size;
5758 else
5759 ring_buf_size = 1;
5760
750912fa
DS
5761 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5762
9e01c1b7
RR
5763 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5764 cpumask_copy(tracing_cpumask, cpu_all_mask);
5765
2b6080f2
SR
5766 raw_spin_lock_init(&global_trace.start_lock);
5767
9e01c1b7 5768 /* TODO: make the number of buffers hot pluggable with CPUS */
12883efb
SRRH
5769 global_trace.trace_buffer.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5770 if (!global_trace.trace_buffer.buffer) {
3928a8a2
SR
5771 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5772 WARN_ON(1);
9e01c1b7 5773 goto out_free_cpumask;
4c11d7ae 5774 }
a7603ff4 5775
12883efb 5776 global_trace.trace_buffer.data = alloc_percpu(struct trace_array_cpu);
a7603ff4 5777
12883efb 5778 if (!global_trace.trace_buffer.data) {
a7603ff4
SR
5779 printk(KERN_ERR "tracer: failed to allocate percpu memory!\n");
5780 WARN_ON(1);
5781 goto out_free_cpumask;
5782 }
5783
5784 for_each_tracing_cpu(i) {
12883efb
SRRH
5785 memset(per_cpu_ptr(global_trace.trace_buffer.data, i), 0,
5786 sizeof(struct trace_array_cpu));
5787 per_cpu_ptr(global_trace.trace_buffer.data, i)->trace_cpu.cpu = i;
5788 per_cpu_ptr(global_trace.trace_buffer.data, i)->trace_cpu.tr = &global_trace;
a7603ff4
SR
5789 }
5790
499e5470
SR
5791 if (global_trace.buffer_disabled)
5792 tracing_off();
4c11d7ae
SR
5793
5794#ifdef CONFIG_TRACER_MAX_TRACE
12883efb
SRRH
5795 global_trace.max_buffer.data = alloc_percpu(struct trace_array_cpu);
5796 if (!global_trace.max_buffer.data) {
a7603ff4
SR
5797 printk(KERN_ERR "tracer: failed to allocate percpu memory!\n");
5798 WARN_ON(1);
5799 goto out_free_cpumask;
5800 }
12883efb
SRRH
5801 global_trace.max_buffer.buffer = ring_buffer_alloc(1, rb_flags);
5802 if (!global_trace.max_buffer.buffer) {
3928a8a2
SR
5803 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5804 WARN_ON(1);
12883efb 5805 ring_buffer_free(global_trace.trace_buffer.buffer);
9e01c1b7 5806 goto out_free_cpumask;
4c11d7ae 5807 }
ab46428c 5808
ab46428c 5809 for_each_tracing_cpu(i) {
12883efb
SRRH
5810 memset(per_cpu_ptr(global_trace.max_buffer.data, i), 0,
5811 sizeof(struct trace_array_cpu));
5812 per_cpu_ptr(global_trace.max_buffer.data, i)->trace_cpu.cpu = i;
5813 per_cpu_ptr(global_trace.max_buffer.data, i)->trace_cpu.tr = &global_trace;
4c11d7ae 5814 }
a7603ff4 5815#endif
438ced17 5816
a7603ff4 5817 /* Allocate the first page for all buffers */
12883efb
SRRH
5818 set_buffer_entries(&global_trace.trace_buffer,
5819 ring_buffer_size(global_trace.trace_buffer.buffer, 0));
438ced17 5820#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 5821 set_buffer_entries(&global_trace.max_buffer, 1);
438ced17 5822#endif
bc0c38d1 5823
bc0c38d1
SR
5824 trace_init_cmdlines();
5825
43a15386 5826 register_tracer(&nop_trace);
d840f718 5827
2b6080f2
SR
5828 global_trace.current_trace = &nop_trace;
5829
60a11774
SR
5830 /* All seems OK, enable tracing */
5831 tracing_disabled = 0;
3928a8a2 5832
3f5a54e3
SR
5833 atomic_notifier_chain_register(&panic_notifier_list,
5834 &trace_panic_notifier);
5835
5836 register_die_notifier(&trace_die_notifier);
2fc1dfbe 5837
ae63b31e
SR
5838 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
5839
2b6080f2
SR
5840 /* Holder for file callbacks */
5841 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5842 global_trace.trace_cpu.tr = &global_trace;
5843
ae63b31e
SR
5844 INIT_LIST_HEAD(&global_trace.systems);
5845 INIT_LIST_HEAD(&global_trace.events);
5846 list_add(&global_trace.list, &ftrace_trace_arrays);
5847
7bcfaf54
SR
5848 while (trace_boot_options) {
5849 char *option;
5850
5851 option = strsep(&trace_boot_options, ",");
2b6080f2 5852 trace_set_options(&global_trace, option);
7bcfaf54
SR
5853 }
5854
2fc1dfbe 5855 return 0;
3f5a54e3 5856
9e01c1b7 5857out_free_cpumask:
12883efb
SRRH
5858 free_percpu(global_trace.trace_buffer.data);
5859#ifdef CONFIG_TRACER_MAX_TRACE
5860 free_percpu(global_trace.max_buffer.data);
5861#endif
9e01c1b7
RR
5862 free_cpumask_var(tracing_cpumask);
5863out_free_buffer_mask:
5864 free_cpumask_var(tracing_buffer_mask);
5865out:
5866 return ret;
bc0c38d1 5867}
b2821ae6
SR
5868
5869__init static int clear_boot_tracer(void)
5870{
5871 /*
5872 * The default tracer at boot buffer is an init section.
5873 * This function is called in lateinit. If we did not
5874 * find the boot tracer, then clear it out, to prevent
5875 * later registration from accessing the buffer that is
5876 * about to be freed.
5877 */
5878 if (!default_bootup_tracer)
5879 return 0;
5880
5881 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5882 default_bootup_tracer);
5883 default_bootup_tracer = NULL;
5884
5885 return 0;
5886}
5887
b5ad384e
FW
5888early_initcall(tracer_alloc_buffers);
5889fs_initcall(tracer_init_debugfs);
b2821ae6 5890late_initcall(clear_boot_tracer);