]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
ftrace: Do not run selftest if command line parameter is set
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
4c11d7ae 23#include <linux/pagemap.h>
bc0c38d1
SR
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
2cadf913 27#include <linux/kprobes.h>
bc0c38d1
SR
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
2cadf913 31#include <linux/splice.h>
3f5a54e3 32#include <linux/kdebug.h>
5f0c6c03 33#include <linux/string.h>
7e53bd42 34#include <linux/rwsem.h>
5a0e3ad6 35#include <linux/slab.h>
bc0c38d1
SR
36#include <linux/ctype.h>
37#include <linux/init.h>
2a2cc8f7 38#include <linux/poll.h>
b892e5c8 39#include <linux/nmi.h>
bc0c38d1 40#include <linux/fs.h>
8bd75c77 41#include <linux/sched/rt.h>
86387f7e 42
bc0c38d1 43#include "trace.h"
f0868d1e 44#include "trace_output.h"
bc0c38d1 45
73c5162a
SR
46/*
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
55034cd6 50bool ring_buffer_expanded;
73c5162a 51
8e1b82e0
FW
52/*
53 * We need to change this state when a selftest is running.
ff32504f
FW
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
5e1607a0 56 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
57 * at the same time, giving false positive or negative results.
58 */
8e1b82e0 59static bool __read_mostly tracing_selftest_running;
ff32504f 60
b2821ae6
SR
61/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
020e5f85 64bool __read_mostly tracing_selftest_disabled;
b2821ae6 65
adf9f195
FW
66/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
0f048701 80
7ffbd48d
SR
81/*
82 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
84 * occurred.
85 */
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
0f048701
SR
88/*
89 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
92 * this back to zero.
93 */
4fd27358 94static int tracing_disabled = 1;
0f048701 95
9288f99a 96DEFINE_PER_CPU(int, ftrace_cpu_disabled);
d769041f 97
955b61e5 98cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 99
944ac425
SR
100/*
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102 *
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
107 * serial console.
108 *
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 114 */
cecbca96
FW
115
116enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 117
de7edd31
SRRH
118/* When set, tracing will stop when a WARN*() is hit */
119int __disable_trace_on_warning;
120
b2821ae6
SR
121static int tracing_set_tracer(const char *buf);
122
ee6c2c1b
LZ
123#define MAX_TRACER_SIZE 100
124static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 125static char *default_bootup_tracer;
d9e54076 126
55034cd6
SRRH
127static bool allocate_snapshot;
128
1beee96b 129static int __init set_cmdline_ftrace(char *str)
d9e54076 130{
67012ab1 131 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 132 default_bootup_tracer = bootup_tracer_buf;
73c5162a 133 /* We are using ftrace early, expand it */
55034cd6 134 ring_buffer_expanded = true;
d9e54076
PZ
135 return 1;
136}
1beee96b 137__setup("ftrace=", set_cmdline_ftrace);
d9e54076 138
944ac425
SR
139static int __init set_ftrace_dump_on_oops(char *str)
140{
cecbca96
FW
141 if (*str++ != '=' || !*str) {
142 ftrace_dump_on_oops = DUMP_ALL;
143 return 1;
144 }
145
146 if (!strcmp("orig_cpu", str)) {
147 ftrace_dump_on_oops = DUMP_ORIG;
148 return 1;
149 }
150
151 return 0;
944ac425
SR
152}
153__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 154
de7edd31
SRRH
155static int __init stop_trace_on_warning(char *str)
156{
157 __disable_trace_on_warning = 1;
158 return 1;
159}
160__setup("traceoff_on_warning=", stop_trace_on_warning);
161
3209cff4 162static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
163{
164 allocate_snapshot = true;
165 /* We also need the main ring buffer expanded */
166 ring_buffer_expanded = true;
167 return 1;
168}
3209cff4 169__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 170
7bcfaf54
SR
171
172static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
173static char *trace_boot_options __initdata;
174
175static int __init set_trace_boot_options(char *str)
176{
67012ab1 177 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
178 trace_boot_options = trace_boot_options_buf;
179 return 0;
180}
181__setup("trace_options=", set_trace_boot_options);
182
de7edd31 183
cf8e3474 184unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
185{
186 nsec += 500;
187 do_div(nsec, 1000);
188 return nsec;
189}
190
4fcdae83
SR
191/*
192 * The global_trace is the descriptor that holds the tracing
193 * buffers for the live tracing. For each CPU, it contains
194 * a link list of pages that will store trace entries. The
195 * page descriptor of the pages in the memory is used to hold
196 * the link list by linking the lru item in the page descriptor
197 * to each of the pages in the buffer per CPU.
198 *
199 * For each active CPU there is a data field that holds the
200 * pages for the buffer for that CPU. Each CPU has the same number
201 * of pages allocated for its buffer.
202 */
bc0c38d1
SR
203static struct trace_array global_trace;
204
ae63b31e 205LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 206
e77405ad
SR
207int filter_current_check_discard(struct ring_buffer *buffer,
208 struct ftrace_event_call *call, void *rec,
eb02ce01
TZ
209 struct ring_buffer_event *event)
210{
e77405ad 211 return filter_check_discard(call, rec, buffer, event);
eb02ce01 212}
17c873ec 213EXPORT_SYMBOL_GPL(filter_current_check_discard);
eb02ce01 214
37886f6a
SR
215cycle_t ftrace_now(int cpu)
216{
217 u64 ts;
218
219 /* Early boot up does not have a buffer yet */
12883efb 220 if (!global_trace.trace_buffer.buffer)
37886f6a
SR
221 return trace_clock_local();
222
12883efb
SRRH
223 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
224 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
37886f6a
SR
225
226 return ts;
227}
bc0c38d1 228
10246fa3
SRRH
229/**
230 * tracing_is_enabled - Show if global_trace has been disabled
231 *
232 * Shows if the global trace has been enabled or not. It uses the
233 * mirror flag "buffer_disabled" to be used in fast paths such as for
234 * the irqsoff tracer. But it may be inaccurate due to races. If you
235 * need to know the accurate state, use tracing_is_on() which is a little
236 * slower, but accurate.
237 */
9036990d
SR
238int tracing_is_enabled(void)
239{
10246fa3
SRRH
240 /*
241 * For quick access (irqsoff uses this in fast path), just
242 * return the mirror variable of the state of the ring buffer.
243 * It's a little racy, but we don't really care.
244 */
245 smp_rmb();
246 return !global_trace.buffer_disabled;
9036990d
SR
247}
248
4fcdae83 249/*
3928a8a2
SR
250 * trace_buf_size is the size in bytes that is allocated
251 * for a buffer. Note, the number of bytes is always rounded
252 * to page size.
3f5a54e3
SR
253 *
254 * This number is purposely set to a low number of 16384.
255 * If the dump on oops happens, it will be much appreciated
256 * to not have to wait for all that output. Anyway this can be
257 * boot time and run time configurable.
4fcdae83 258 */
3928a8a2 259#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 260
3928a8a2 261static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 262
4fcdae83 263/* trace_types holds a link list of available tracers. */
bc0c38d1 264static struct tracer *trace_types __read_mostly;
4fcdae83 265
4fcdae83
SR
266/*
267 * trace_types_lock is used to protect the trace_types list.
4fcdae83 268 */
bc0c38d1 269static DEFINE_MUTEX(trace_types_lock);
4fcdae83 270
7e53bd42
LJ
271/*
272 * serialize the access of the ring buffer
273 *
274 * ring buffer serializes readers, but it is low level protection.
275 * The validity of the events (which returns by ring_buffer_peek() ..etc)
276 * are not protected by ring buffer.
277 *
278 * The content of events may become garbage if we allow other process consumes
279 * these events concurrently:
280 * A) the page of the consumed events may become a normal page
281 * (not reader page) in ring buffer, and this page will be rewrited
282 * by events producer.
283 * B) The page of the consumed events may become a page for splice_read,
284 * and this page will be returned to system.
285 *
286 * These primitives allow multi process access to different cpu ring buffer
287 * concurrently.
288 *
289 * These primitives don't distinguish read-only and read-consume access.
290 * Multi read-only access are also serialized.
291 */
292
293#ifdef CONFIG_SMP
294static DECLARE_RWSEM(all_cpu_access_lock);
295static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
296
297static inline void trace_access_lock(int cpu)
298{
ae3b5093 299 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
300 /* gain it for accessing the whole ring buffer. */
301 down_write(&all_cpu_access_lock);
302 } else {
303 /* gain it for accessing a cpu ring buffer. */
304
ae3b5093 305 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
306 down_read(&all_cpu_access_lock);
307
308 /* Secondly block other access to this @cpu ring buffer. */
309 mutex_lock(&per_cpu(cpu_access_lock, cpu));
310 }
311}
312
313static inline void trace_access_unlock(int cpu)
314{
ae3b5093 315 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
316 up_write(&all_cpu_access_lock);
317 } else {
318 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
319 up_read(&all_cpu_access_lock);
320 }
321}
322
323static inline void trace_access_lock_init(void)
324{
325 int cpu;
326
327 for_each_possible_cpu(cpu)
328 mutex_init(&per_cpu(cpu_access_lock, cpu));
329}
330
331#else
332
333static DEFINE_MUTEX(access_lock);
334
335static inline void trace_access_lock(int cpu)
336{
337 (void)cpu;
338 mutex_lock(&access_lock);
339}
340
341static inline void trace_access_unlock(int cpu)
342{
343 (void)cpu;
344 mutex_unlock(&access_lock);
345}
346
347static inline void trace_access_lock_init(void)
348{
349}
350
351#endif
352
ee6bce52 353/* trace_flags holds trace_options default values */
12ef7d44 354unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a 355 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
77271ce4 356 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
328df475 357 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
e7e2ee89 358
10246fa3
SRRH
359void tracer_tracing_on(struct trace_array *tr)
360{
361 if (tr->trace_buffer.buffer)
362 ring_buffer_record_on(tr->trace_buffer.buffer);
363 /*
364 * This flag is looked at when buffers haven't been allocated
365 * yet, or by some tracers (like irqsoff), that just want to
366 * know if the ring buffer has been disabled, but it can handle
367 * races of where it gets disabled but we still do a record.
368 * As the check is in the fast path of the tracers, it is more
369 * important to be fast than accurate.
370 */
371 tr->buffer_disabled = 0;
372 /* Make the flag seen by readers */
373 smp_wmb();
374}
375
499e5470
SR
376/**
377 * tracing_on - enable tracing buffers
378 *
379 * This function enables tracing buffers that may have been
380 * disabled with tracing_off.
381 */
382void tracing_on(void)
383{
10246fa3 384 tracer_tracing_on(&global_trace);
499e5470
SR
385}
386EXPORT_SYMBOL_GPL(tracing_on);
387
09ae7234
SRRH
388/**
389 * __trace_puts - write a constant string into the trace buffer.
390 * @ip: The address of the caller
391 * @str: The constant string to write
392 * @size: The size of the string.
393 */
394int __trace_puts(unsigned long ip, const char *str, int size)
395{
396 struct ring_buffer_event *event;
397 struct ring_buffer *buffer;
398 struct print_entry *entry;
399 unsigned long irq_flags;
400 int alloc;
401
402 alloc = sizeof(*entry) + size + 2; /* possible \n added */
403
404 local_save_flags(irq_flags);
405 buffer = global_trace.trace_buffer.buffer;
406 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
407 irq_flags, preempt_count());
408 if (!event)
409 return 0;
410
411 entry = ring_buffer_event_data(event);
412 entry->ip = ip;
413
414 memcpy(&entry->buf, str, size);
415
416 /* Add a newline if necessary */
417 if (entry->buf[size - 1] != '\n') {
418 entry->buf[size] = '\n';
419 entry->buf[size + 1] = '\0';
420 } else
421 entry->buf[size] = '\0';
422
423 __buffer_unlock_commit(buffer, event);
424
425 return size;
426}
427EXPORT_SYMBOL_GPL(__trace_puts);
428
429/**
430 * __trace_bputs - write the pointer to a constant string into trace buffer
431 * @ip: The address of the caller
432 * @str: The constant string to write to the buffer to
433 */
434int __trace_bputs(unsigned long ip, const char *str)
435{
436 struct ring_buffer_event *event;
437 struct ring_buffer *buffer;
438 struct bputs_entry *entry;
439 unsigned long irq_flags;
440 int size = sizeof(struct bputs_entry);
441
442 local_save_flags(irq_flags);
443 buffer = global_trace.trace_buffer.buffer;
444 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
445 irq_flags, preempt_count());
446 if (!event)
447 return 0;
448
449 entry = ring_buffer_event_data(event);
450 entry->ip = ip;
451 entry->str = str;
452
453 __buffer_unlock_commit(buffer, event);
454
455 return 1;
456}
457EXPORT_SYMBOL_GPL(__trace_bputs);
458
ad909e21
SRRH
459#ifdef CONFIG_TRACER_SNAPSHOT
460/**
461 * trace_snapshot - take a snapshot of the current buffer.
462 *
463 * This causes a swap between the snapshot buffer and the current live
464 * tracing buffer. You can use this to take snapshots of the live
465 * trace when some condition is triggered, but continue to trace.
466 *
467 * Note, make sure to allocate the snapshot with either
468 * a tracing_snapshot_alloc(), or by doing it manually
469 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
470 *
471 * If the snapshot buffer is not allocated, it will stop tracing.
472 * Basically making a permanent snapshot.
473 */
474void tracing_snapshot(void)
475{
476 struct trace_array *tr = &global_trace;
477 struct tracer *tracer = tr->current_trace;
478 unsigned long flags;
479
1b22e382
SRRH
480 if (in_nmi()) {
481 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
482 internal_trace_puts("*** snapshot is being ignored ***\n");
483 return;
484 }
485
ad909e21 486 if (!tr->allocated_snapshot) {
ca268da6
SRRH
487 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
488 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
489 tracing_off();
490 return;
491 }
492
493 /* Note, snapshot can not be used when the tracer uses it */
494 if (tracer->use_max_tr) {
ca268da6
SRRH
495 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
496 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
497 return;
498 }
499
500 local_irq_save(flags);
501 update_max_tr(tr, current, smp_processor_id());
502 local_irq_restore(flags);
503}
1b22e382 504EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
505
506static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
507 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
508static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
509
510static int alloc_snapshot(struct trace_array *tr)
511{
512 int ret;
513
514 if (!tr->allocated_snapshot) {
515
516 /* allocate spare buffer */
517 ret = resize_buffer_duplicate_size(&tr->max_buffer,
518 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
519 if (ret < 0)
520 return ret;
521
522 tr->allocated_snapshot = true;
523 }
524
525 return 0;
526}
527
528void free_snapshot(struct trace_array *tr)
529{
530 /*
531 * We don't free the ring buffer. instead, resize it because
532 * The max_tr ring buffer has some state (e.g. ring->clock) and
533 * we want preserve it.
534 */
535 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
536 set_buffer_entries(&tr->max_buffer, 1);
537 tracing_reset_online_cpus(&tr->max_buffer);
538 tr->allocated_snapshot = false;
539}
ad909e21
SRRH
540
541/**
542 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
543 *
544 * This is similar to trace_snapshot(), but it will allocate the
545 * snapshot buffer if it isn't already allocated. Use this only
546 * where it is safe to sleep, as the allocation may sleep.
547 *
548 * This causes a swap between the snapshot buffer and the current live
549 * tracing buffer. You can use this to take snapshots of the live
550 * trace when some condition is triggered, but continue to trace.
551 */
552void tracing_snapshot_alloc(void)
553{
554 struct trace_array *tr = &global_trace;
555 int ret;
556
3209cff4
SRRH
557 ret = alloc_snapshot(tr);
558 if (WARN_ON(ret < 0))
559 return;
ad909e21
SRRH
560
561 tracing_snapshot();
562}
1b22e382 563EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
564#else
565void tracing_snapshot(void)
566{
567 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
568}
1b22e382 569EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
570void tracing_snapshot_alloc(void)
571{
572 /* Give warning */
573 tracing_snapshot();
574}
1b22e382 575EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
576#endif /* CONFIG_TRACER_SNAPSHOT */
577
10246fa3
SRRH
578void tracer_tracing_off(struct trace_array *tr)
579{
580 if (tr->trace_buffer.buffer)
581 ring_buffer_record_off(tr->trace_buffer.buffer);
582 /*
583 * This flag is looked at when buffers haven't been allocated
584 * yet, or by some tracers (like irqsoff), that just want to
585 * know if the ring buffer has been disabled, but it can handle
586 * races of where it gets disabled but we still do a record.
587 * As the check is in the fast path of the tracers, it is more
588 * important to be fast than accurate.
589 */
590 tr->buffer_disabled = 1;
591 /* Make the flag seen by readers */
592 smp_wmb();
593}
594
499e5470
SR
595/**
596 * tracing_off - turn off tracing buffers
597 *
598 * This function stops the tracing buffers from recording data.
599 * It does not disable any overhead the tracers themselves may
600 * be causing. This function simply causes all recording to
601 * the ring buffers to fail.
602 */
603void tracing_off(void)
604{
10246fa3 605 tracer_tracing_off(&global_trace);
499e5470
SR
606}
607EXPORT_SYMBOL_GPL(tracing_off);
608
de7edd31
SRRH
609void disable_trace_on_warning(void)
610{
611 if (__disable_trace_on_warning)
612 tracing_off();
613}
614
10246fa3
SRRH
615/**
616 * tracer_tracing_is_on - show real state of ring buffer enabled
617 * @tr : the trace array to know if ring buffer is enabled
618 *
619 * Shows real state of the ring buffer if it is enabled or not.
620 */
621int tracer_tracing_is_on(struct trace_array *tr)
622{
623 if (tr->trace_buffer.buffer)
624 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
625 return !tr->buffer_disabled;
626}
627
499e5470
SR
628/**
629 * tracing_is_on - show state of ring buffers enabled
630 */
631int tracing_is_on(void)
632{
10246fa3 633 return tracer_tracing_is_on(&global_trace);
499e5470
SR
634}
635EXPORT_SYMBOL_GPL(tracing_is_on);
636
3928a8a2 637static int __init set_buf_size(char *str)
bc0c38d1 638{
3928a8a2 639 unsigned long buf_size;
c6caeeb1 640
bc0c38d1
SR
641 if (!str)
642 return 0;
9d612bef 643 buf_size = memparse(str, &str);
c6caeeb1 644 /* nr_entries can not be zero */
9d612bef 645 if (buf_size == 0)
c6caeeb1 646 return 0;
3928a8a2 647 trace_buf_size = buf_size;
bc0c38d1
SR
648 return 1;
649}
3928a8a2 650__setup("trace_buf_size=", set_buf_size);
bc0c38d1 651
0e950173
TB
652static int __init set_tracing_thresh(char *str)
653{
87abb3b1 654 unsigned long threshold;
0e950173
TB
655 int ret;
656
657 if (!str)
658 return 0;
bcd83ea6 659 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
660 if (ret < 0)
661 return 0;
87abb3b1 662 tracing_thresh = threshold * 1000;
0e950173
TB
663 return 1;
664}
665__setup("tracing_thresh=", set_tracing_thresh);
666
57f50be1
SR
667unsigned long nsecs_to_usecs(unsigned long nsecs)
668{
669 return nsecs / 1000;
670}
671
4fcdae83 672/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
673static const char *trace_options[] = {
674 "print-parent",
675 "sym-offset",
676 "sym-addr",
677 "verbose",
f9896bf3 678 "raw",
5e3ca0ec 679 "hex",
cb0f12aa 680 "bin",
2a2cc8f7 681 "block",
86387f7e 682 "stacktrace",
5e1607a0 683 "trace_printk",
b2a866f9 684 "ftrace_preempt",
9f029e83 685 "branch",
12ef7d44 686 "annotate",
02b67518 687 "userstacktrace",
b54d3de9 688 "sym-userobj",
66896a85 689 "printk-msg-only",
c4a8e8be 690 "context-info",
c032ef64 691 "latency-format",
be6f164a 692 "sleep-time",
a2a16d6a 693 "graph-time",
e870e9a1 694 "record-cmd",
750912fa 695 "overwrite",
cf30cf67 696 "disable_on_free",
77271ce4 697 "irq-info",
5224c3a3 698 "markers",
328df475 699 "function-trace",
bc0c38d1
SR
700 NULL
701};
702
5079f326
Z
703static struct {
704 u64 (*func)(void);
705 const char *name;
8be0709f 706 int in_ns; /* is this clock in nanoseconds? */
5079f326 707} trace_clocks[] = {
8be0709f
DS
708 { trace_clock_local, "local", 1 },
709 { trace_clock_global, "global", 1 },
710 { trace_clock_counter, "counter", 0 },
8aacf017 711 { trace_clock_jiffies, "uptime", 1 },
76f11917 712 { trace_clock, "perf", 1 },
8cbd9cc6 713 ARCH_TRACE_CLOCKS
5079f326
Z
714};
715
716int trace_clock_id;
717
b63f39ea 718/*
719 * trace_parser_get_init - gets the buffer for trace parser
720 */
721int trace_parser_get_init(struct trace_parser *parser, int size)
722{
723 memset(parser, 0, sizeof(*parser));
724
725 parser->buffer = kmalloc(size, GFP_KERNEL);
726 if (!parser->buffer)
727 return 1;
728
729 parser->size = size;
730 return 0;
731}
732
733/*
734 * trace_parser_put - frees the buffer for trace parser
735 */
736void trace_parser_put(struct trace_parser *parser)
737{
738 kfree(parser->buffer);
739}
740
741/*
742 * trace_get_user - reads the user input string separated by space
743 * (matched by isspace(ch))
744 *
745 * For each string found the 'struct trace_parser' is updated,
746 * and the function returns.
747 *
748 * Returns number of bytes read.
749 *
750 * See kernel/trace/trace.h for 'struct trace_parser' details.
751 */
752int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
753 size_t cnt, loff_t *ppos)
754{
755 char ch;
756 size_t read = 0;
757 ssize_t ret;
758
759 if (!*ppos)
760 trace_parser_clear(parser);
761
762 ret = get_user(ch, ubuf++);
763 if (ret)
764 goto out;
765
766 read++;
767 cnt--;
768
769 /*
770 * The parser is not finished with the last write,
771 * continue reading the user input without skipping spaces.
772 */
773 if (!parser->cont) {
774 /* skip white space */
775 while (cnt && isspace(ch)) {
776 ret = get_user(ch, ubuf++);
777 if (ret)
778 goto out;
779 read++;
780 cnt--;
781 }
782
783 /* only spaces were written */
784 if (isspace(ch)) {
785 *ppos += read;
786 ret = read;
787 goto out;
788 }
789
790 parser->idx = 0;
791 }
792
793 /* read the non-space input */
794 while (cnt && !isspace(ch)) {
3c235a33 795 if (parser->idx < parser->size - 1)
b63f39ea 796 parser->buffer[parser->idx++] = ch;
797 else {
798 ret = -EINVAL;
799 goto out;
800 }
801 ret = get_user(ch, ubuf++);
802 if (ret)
803 goto out;
804 read++;
805 cnt--;
806 }
807
808 /* We either got finished input or we have to wait for another call. */
809 if (isspace(ch)) {
810 parser->buffer[parser->idx] = 0;
811 parser->cont = false;
812 } else {
813 parser->cont = true;
814 parser->buffer[parser->idx++] = ch;
815 }
816
817 *ppos += read;
818 ret = read;
819
820out:
821 return ret;
822}
823
6c6c2796
PP
824ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
825{
826 int len;
827 int ret;
828
2dc5d12b
SR
829 if (!cnt)
830 return 0;
831
6c6c2796
PP
832 if (s->len <= s->readpos)
833 return -EBUSY;
834
835 len = s->len - s->readpos;
836 if (cnt > len)
837 cnt = len;
838 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
2dc5d12b 839 if (ret == cnt)
6c6c2796
PP
840 return -EFAULT;
841
2dc5d12b
SR
842 cnt -= ret;
843
e74da523 844 s->readpos += cnt;
6c6c2796 845 return cnt;
214023c3
SR
846}
847
b8b94265 848static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
849{
850 int len;
3c56819b
EGM
851
852 if (s->len <= s->readpos)
853 return -EBUSY;
854
855 len = s->len - s->readpos;
856 if (cnt > len)
857 cnt = len;
5a26c8f0 858 memcpy(buf, s->buffer + s->readpos, cnt);
3c56819b 859
e74da523 860 s->readpos += cnt;
3c56819b
EGM
861 return cnt;
862}
863
5d4a9dba
SR
864/*
865 * ftrace_max_lock is used to protect the swapping of buffers
866 * when taking a max snapshot. The buffers themselves are
867 * protected by per_cpu spinlocks. But the action of the swap
868 * needs its own lock.
869 *
445c8951 870 * This is defined as a arch_spinlock_t in order to help
5d4a9dba
SR
871 * with performance when lockdep debugging is enabled.
872 *
873 * It is also used in other places outside the update_max_tr
874 * so it needs to be defined outside of the
875 * CONFIG_TRACER_MAX_TRACE.
876 */
445c8951 877static arch_spinlock_t ftrace_max_lock =
edc35bd7 878 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5d4a9dba 879
0e950173
TB
880unsigned long __read_mostly tracing_thresh;
881
5d4a9dba
SR
882#ifdef CONFIG_TRACER_MAX_TRACE
883unsigned long __read_mostly tracing_max_latency;
5d4a9dba
SR
884
885/*
886 * Copy the new maximum trace into the separate maximum-trace
887 * structure. (this way the maximum trace is permanently saved,
888 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
889 */
890static void
891__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
892{
12883efb
SRRH
893 struct trace_buffer *trace_buf = &tr->trace_buffer;
894 struct trace_buffer *max_buf = &tr->max_buffer;
895 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
896 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 897
12883efb
SRRH
898 max_buf->cpu = cpu;
899 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 900
8248ac05
SR
901 max_data->saved_latency = tracing_max_latency;
902 max_data->critical_start = data->critical_start;
903 max_data->critical_end = data->critical_end;
5d4a9dba 904
1acaa1b2 905 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05 906 max_data->pid = tsk->pid;
f17a5194
SRRH
907 /*
908 * If tsk == current, then use current_uid(), as that does not use
909 * RCU. The irq tracer can be called out of RCU scope.
910 */
911 if (tsk == current)
912 max_data->uid = current_uid();
913 else
914 max_data->uid = task_uid(tsk);
915
8248ac05
SR
916 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
917 max_data->policy = tsk->policy;
918 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
919
920 /* record this tasks comm */
921 tracing_record_cmdline(tsk);
922}
923
4fcdae83
SR
924/**
925 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
926 * @tr: tracer
927 * @tsk: the task with the latency
928 * @cpu: The cpu that initiated the trace.
929 *
930 * Flip the buffers between the @tr and the max_tr and record information
931 * about which task was the cause of this latency.
932 */
e309b41d 933void
bc0c38d1
SR
934update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
935{
2721e72d 936 struct ring_buffer *buf;
bc0c38d1 937
2b6080f2 938 if (tr->stop_count)
b8de7bd1
SR
939 return;
940
4c11d7ae 941 WARN_ON_ONCE(!irqs_disabled());
34600f0e 942
45ad21ca 943 if (!tr->allocated_snapshot) {
debdd57f 944 /* Only the nop tracer should hit this when disabling */
2b6080f2 945 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 946 return;
debdd57f 947 }
34600f0e 948
0199c4e6 949 arch_spin_lock(&ftrace_max_lock);
3928a8a2 950
12883efb
SRRH
951 buf = tr->trace_buffer.buffer;
952 tr->trace_buffer.buffer = tr->max_buffer.buffer;
953 tr->max_buffer.buffer = buf;
3928a8a2 954
bc0c38d1 955 __update_max_tr(tr, tsk, cpu);
0199c4e6 956 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
957}
958
959/**
960 * update_max_tr_single - only copy one trace over, and reset the rest
961 * @tr - tracer
962 * @tsk - task with the latency
963 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
964 *
965 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 966 */
e309b41d 967void
bc0c38d1
SR
968update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
969{
3928a8a2 970 int ret;
bc0c38d1 971
2b6080f2 972 if (tr->stop_count)
b8de7bd1
SR
973 return;
974
4c11d7ae 975 WARN_ON_ONCE(!irqs_disabled());
6c24499f 976 if (!tr->allocated_snapshot) {
2930e04d 977 /* Only the nop tracer should hit this when disabling */
9e8529af 978 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 979 return;
2930e04d 980 }
ef710e10 981
0199c4e6 982 arch_spin_lock(&ftrace_max_lock);
bc0c38d1 983
12883efb 984 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 985
e8165dbb
SR
986 if (ret == -EBUSY) {
987 /*
988 * We failed to swap the buffer due to a commit taking
989 * place on this CPU. We fail to record, but we reset
990 * the max trace buffer (no one writes directly to it)
991 * and flag that it failed.
992 */
12883efb 993 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
994 "Failed to swap buffers due to commit in progress\n");
995 }
996
e8165dbb 997 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
998
999 __update_max_tr(tr, tsk, cpu);
0199c4e6 1000 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1 1001}
5d4a9dba 1002#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 1003
0d5c6e1c
SR
1004static void default_wait_pipe(struct trace_iterator *iter)
1005{
15693458
SRRH
1006 /* Iterators are static, they should be filled or empty */
1007 if (trace_buffer_iter(iter, iter->cpu_file))
1008 return;
0d5c6e1c 1009
12883efb 1010 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
0d5c6e1c
SR
1011}
1012
f4e781c0
SRRH
1013#ifdef CONFIG_FTRACE_STARTUP_TEST
1014static int run_tracer_selftest(struct tracer *type)
1015{
1016 struct trace_array *tr = &global_trace;
1017 struct tracer *saved_tracer = tr->current_trace;
1018 int ret;
0d5c6e1c 1019
f4e781c0
SRRH
1020 if (!type->selftest || tracing_selftest_disabled)
1021 return 0;
0d5c6e1c
SR
1022
1023 /*
f4e781c0
SRRH
1024 * Run a selftest on this tracer.
1025 * Here we reset the trace buffer, and set the current
1026 * tracer to be this tracer. The tracer can then run some
1027 * internal tracing to verify that everything is in order.
1028 * If we fail, we do not register this tracer.
0d5c6e1c 1029 */
f4e781c0 1030 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 1031
f4e781c0
SRRH
1032 tr->current_trace = type;
1033
1034#ifdef CONFIG_TRACER_MAX_TRACE
1035 if (type->use_max_tr) {
1036 /* If we expanded the buffers, make sure the max is expanded too */
1037 if (ring_buffer_expanded)
1038 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1039 RING_BUFFER_ALL_CPUS);
1040 tr->allocated_snapshot = true;
1041 }
1042#endif
1043
1044 /* the test is responsible for initializing and enabling */
1045 pr_info("Testing tracer %s: ", type->name);
1046 ret = type->selftest(type, tr);
1047 /* the test is responsible for resetting too */
1048 tr->current_trace = saved_tracer;
1049 if (ret) {
1050 printk(KERN_CONT "FAILED!\n");
1051 /* Add the warning after printing 'FAILED' */
1052 WARN_ON(1);
1053 return -1;
1054 }
1055 /* Only reset on passing, to avoid touching corrupted buffers */
1056 tracing_reset_online_cpus(&tr->trace_buffer);
1057
1058#ifdef CONFIG_TRACER_MAX_TRACE
1059 if (type->use_max_tr) {
1060 tr->allocated_snapshot = false;
0d5c6e1c 1061
f4e781c0
SRRH
1062 /* Shrink the max buffer again */
1063 if (ring_buffer_expanded)
1064 ring_buffer_resize(tr->max_buffer.buffer, 1,
1065 RING_BUFFER_ALL_CPUS);
1066 }
1067#endif
1068
1069 printk(KERN_CONT "PASSED\n");
1070 return 0;
1071}
1072#else
1073static inline int run_tracer_selftest(struct tracer *type)
1074{
1075 return 0;
0d5c6e1c 1076}
f4e781c0 1077#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1078
4fcdae83
SR
1079/**
1080 * register_tracer - register a tracer with the ftrace system.
1081 * @type - the plugin for the tracer
1082 *
1083 * Register a new plugin tracer.
1084 */
bc0c38d1
SR
1085int register_tracer(struct tracer *type)
1086{
1087 struct tracer *t;
bc0c38d1
SR
1088 int ret = 0;
1089
1090 if (!type->name) {
1091 pr_info("Tracer must have a name\n");
1092 return -1;
1093 }
1094
24a461d5 1095 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1096 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1097 return -1;
1098 }
1099
bc0c38d1 1100 mutex_lock(&trace_types_lock);
86fa2f60 1101
8e1b82e0
FW
1102 tracing_selftest_running = true;
1103
bc0c38d1
SR
1104 for (t = trace_types; t; t = t->next) {
1105 if (strcmp(type->name, t->name) == 0) {
1106 /* already found */
ee6c2c1b 1107 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1108 type->name);
1109 ret = -1;
1110 goto out;
1111 }
1112 }
1113
adf9f195
FW
1114 if (!type->set_flag)
1115 type->set_flag = &dummy_set_flag;
1116 if (!type->flags)
1117 type->flags = &dummy_tracer_flags;
1118 else
1119 if (!type->flags->opts)
1120 type->flags->opts = dummy_tracer_opt;
6eaaa5d5
FW
1121 if (!type->wait_pipe)
1122 type->wait_pipe = default_wait_pipe;
1123
f4e781c0
SRRH
1124 ret = run_tracer_selftest(type);
1125 if (ret < 0)
1126 goto out;
60a11774 1127
bc0c38d1
SR
1128 type->next = trace_types;
1129 trace_types = type;
60a11774 1130
bc0c38d1 1131 out:
8e1b82e0 1132 tracing_selftest_running = false;
bc0c38d1
SR
1133 mutex_unlock(&trace_types_lock);
1134
dac74940
SR
1135 if (ret || !default_bootup_tracer)
1136 goto out_unlock;
1137
ee6c2c1b 1138 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1139 goto out_unlock;
1140
1141 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1142 /* Do we want this tracer to start on bootup? */
1143 tracing_set_tracer(type->name);
1144 default_bootup_tracer = NULL;
1145 /* disable other selftests, since this will break it. */
55034cd6 1146 tracing_selftest_disabled = true;
b2821ae6 1147#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1148 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1149 type->name);
b2821ae6 1150#endif
b2821ae6 1151
dac74940 1152 out_unlock:
bc0c38d1
SR
1153 return ret;
1154}
1155
12883efb 1156void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1157{
12883efb 1158 struct ring_buffer *buffer = buf->buffer;
f633903a 1159
a5416411
HT
1160 if (!buffer)
1161 return;
1162
f633903a
SR
1163 ring_buffer_record_disable(buffer);
1164
1165 /* Make sure all commits have finished */
1166 synchronize_sched();
68179686 1167 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1168
1169 ring_buffer_record_enable(buffer);
1170}
1171
12883efb 1172void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1173{
12883efb 1174 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1175 int cpu;
1176
a5416411
HT
1177 if (!buffer)
1178 return;
1179
621968cd
SR
1180 ring_buffer_record_disable(buffer);
1181
1182 /* Make sure all commits have finished */
1183 synchronize_sched();
1184
12883efb 1185 buf->time_start = ftrace_now(buf->cpu);
213cc060
PE
1186
1187 for_each_online_cpu(cpu)
68179686 1188 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1189
1190 ring_buffer_record_enable(buffer);
213cc060
PE
1191}
1192
9456f0fa
SR
1193void tracing_reset_current(int cpu)
1194{
12883efb 1195 tracing_reset(&global_trace.trace_buffer, cpu);
9456f0fa
SR
1196}
1197
873c642f 1198void tracing_reset_all_online_cpus(void)
9456f0fa 1199{
873c642f
SRRH
1200 struct trace_array *tr;
1201
1202 mutex_lock(&trace_types_lock);
1203 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
12883efb
SRRH
1204 tracing_reset_online_cpus(&tr->trace_buffer);
1205#ifdef CONFIG_TRACER_MAX_TRACE
1206 tracing_reset_online_cpus(&tr->max_buffer);
1207#endif
873c642f
SRRH
1208 }
1209 mutex_unlock(&trace_types_lock);
9456f0fa
SR
1210}
1211
bc0c38d1 1212#define SAVED_CMDLINES 128
2c7eea4c 1213#define NO_CMDLINE_MAP UINT_MAX
bc0c38d1
SR
1214static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1215static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1216static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1217static int cmdline_idx;
edc35bd7 1218static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
25b0b44a 1219
25b0b44a 1220/* temporary disable recording */
4fd27358 1221static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1
SR
1222
1223static void trace_init_cmdlines(void)
1224{
2c7eea4c
TG
1225 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1226 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
bc0c38d1
SR
1227 cmdline_idx = 0;
1228}
1229
b5130b1e
CE
1230int is_tracing_stopped(void)
1231{
2b6080f2 1232 return global_trace.stop_count;
b5130b1e
CE
1233}
1234
69bb54ec
SR
1235/**
1236 * ftrace_off_permanent - disable all ftrace code permanently
1237 *
1238 * This should only be called when a serious anomally has
1239 * been detected. This will turn off the function tracing,
1240 * ring buffers, and other tracing utilites. It takes no
1241 * locks and can be called from any context.
1242 */
1243void ftrace_off_permanent(void)
1244{
1245 tracing_disabled = 1;
1246 ftrace_stop();
1247 tracing_off_permanent();
1248}
1249
0f048701
SR
1250/**
1251 * tracing_start - quick start of the tracer
1252 *
1253 * If tracing is enabled but was stopped by tracing_stop,
1254 * this will start the tracer back up.
1255 */
1256void tracing_start(void)
1257{
1258 struct ring_buffer *buffer;
1259 unsigned long flags;
1260
1261 if (tracing_disabled)
1262 return;
1263
2b6080f2
SR
1264 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1265 if (--global_trace.stop_count) {
1266 if (global_trace.stop_count < 0) {
b06a8301
SR
1267 /* Someone screwed up their debugging */
1268 WARN_ON_ONCE(1);
2b6080f2 1269 global_trace.stop_count = 0;
b06a8301 1270 }
0f048701
SR
1271 goto out;
1272 }
1273
a2f80714
SR
1274 /* Prevent the buffers from switching */
1275 arch_spin_lock(&ftrace_max_lock);
0f048701 1276
12883efb 1277 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1278 if (buffer)
1279 ring_buffer_record_enable(buffer);
1280
12883efb
SRRH
1281#ifdef CONFIG_TRACER_MAX_TRACE
1282 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1283 if (buffer)
1284 ring_buffer_record_enable(buffer);
12883efb 1285#endif
0f048701 1286
a2f80714
SR
1287 arch_spin_unlock(&ftrace_max_lock);
1288
0f048701
SR
1289 ftrace_start();
1290 out:
2b6080f2
SR
1291 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1292}
1293
1294static void tracing_start_tr(struct trace_array *tr)
1295{
1296 struct ring_buffer *buffer;
1297 unsigned long flags;
1298
1299 if (tracing_disabled)
1300 return;
1301
1302 /* If global, we need to also start the max tracer */
1303 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1304 return tracing_start();
1305
1306 raw_spin_lock_irqsave(&tr->start_lock, flags);
1307
1308 if (--tr->stop_count) {
1309 if (tr->stop_count < 0) {
1310 /* Someone screwed up their debugging */
1311 WARN_ON_ONCE(1);
1312 tr->stop_count = 0;
1313 }
1314 goto out;
1315 }
1316
12883efb 1317 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1318 if (buffer)
1319 ring_buffer_record_enable(buffer);
1320
1321 out:
1322 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1323}
1324
1325/**
1326 * tracing_stop - quick stop of the tracer
1327 *
1328 * Light weight way to stop tracing. Use in conjunction with
1329 * tracing_start.
1330 */
1331void tracing_stop(void)
1332{
1333 struct ring_buffer *buffer;
1334 unsigned long flags;
1335
1336 ftrace_stop();
2b6080f2
SR
1337 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1338 if (global_trace.stop_count++)
0f048701
SR
1339 goto out;
1340
a2f80714
SR
1341 /* Prevent the buffers from switching */
1342 arch_spin_lock(&ftrace_max_lock);
1343
12883efb 1344 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1345 if (buffer)
1346 ring_buffer_record_disable(buffer);
1347
12883efb
SRRH
1348#ifdef CONFIG_TRACER_MAX_TRACE
1349 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1350 if (buffer)
1351 ring_buffer_record_disable(buffer);
12883efb 1352#endif
0f048701 1353
a2f80714
SR
1354 arch_spin_unlock(&ftrace_max_lock);
1355
0f048701 1356 out:
2b6080f2
SR
1357 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1358}
1359
1360static void tracing_stop_tr(struct trace_array *tr)
1361{
1362 struct ring_buffer *buffer;
1363 unsigned long flags;
1364
1365 /* If global, we need to also stop the max tracer */
1366 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1367 return tracing_stop();
1368
1369 raw_spin_lock_irqsave(&tr->start_lock, flags);
1370 if (tr->stop_count++)
1371 goto out;
1372
12883efb 1373 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1374 if (buffer)
1375 ring_buffer_record_disable(buffer);
1376
1377 out:
1378 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1379}
1380
e309b41d 1381void trace_stop_cmdline_recording(void);
bc0c38d1 1382
e309b41d 1383static void trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1384{
a635cf04 1385 unsigned pid, idx;
bc0c38d1
SR
1386
1387 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1388 return;
1389
1390 /*
1391 * It's not the end of the world if we don't get
1392 * the lock, but we also don't want to spin
1393 * nor do we want to disable interrupts,
1394 * so if we miss here, then better luck next time.
1395 */
0199c4e6 1396 if (!arch_spin_trylock(&trace_cmdline_lock))
bc0c38d1
SR
1397 return;
1398
1399 idx = map_pid_to_cmdline[tsk->pid];
2c7eea4c 1400 if (idx == NO_CMDLINE_MAP) {
bc0c38d1
SR
1401 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1402
a635cf04
CE
1403 /*
1404 * Check whether the cmdline buffer at idx has a pid
1405 * mapped. We are going to overwrite that entry so we
1406 * need to clear the map_pid_to_cmdline. Otherwise we
1407 * would read the new comm for the old pid.
1408 */
1409 pid = map_cmdline_to_pid[idx];
1410 if (pid != NO_CMDLINE_MAP)
1411 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1412
a635cf04 1413 map_cmdline_to_pid[idx] = tsk->pid;
bc0c38d1
SR
1414 map_pid_to_cmdline[tsk->pid] = idx;
1415
1416 cmdline_idx = idx;
1417 }
1418
1419 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1420
0199c4e6 1421 arch_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
1422}
1423
4ca53085 1424void trace_find_cmdline(int pid, char comm[])
bc0c38d1 1425{
bc0c38d1
SR
1426 unsigned map;
1427
4ca53085
SR
1428 if (!pid) {
1429 strcpy(comm, "<idle>");
1430 return;
1431 }
bc0c38d1 1432
74bf4076
SR
1433 if (WARN_ON_ONCE(pid < 0)) {
1434 strcpy(comm, "<XXX>");
1435 return;
1436 }
1437
4ca53085
SR
1438 if (pid > PID_MAX_DEFAULT) {
1439 strcpy(comm, "<...>");
1440 return;
1441 }
bc0c38d1 1442
5b6045a9 1443 preempt_disable();
0199c4e6 1444 arch_spin_lock(&trace_cmdline_lock);
bc0c38d1 1445 map = map_pid_to_cmdline[pid];
50d88758
TG
1446 if (map != NO_CMDLINE_MAP)
1447 strcpy(comm, saved_cmdlines[map]);
1448 else
1449 strcpy(comm, "<...>");
bc0c38d1 1450
0199c4e6 1451 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1452 preempt_enable();
bc0c38d1
SR
1453}
1454
e309b41d 1455void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1456{
0fb9656d 1457 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
bc0c38d1
SR
1458 return;
1459
7ffbd48d
SR
1460 if (!__this_cpu_read(trace_cmdline_save))
1461 return;
1462
1463 __this_cpu_write(trace_cmdline_save, false);
1464
bc0c38d1
SR
1465 trace_save_cmdline(tsk);
1466}
1467
45dcd8b8 1468void
38697053
SR
1469tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1470 int pc)
bc0c38d1
SR
1471{
1472 struct task_struct *tsk = current;
bc0c38d1 1473
777e208d
SR
1474 entry->preempt_count = pc & 0xff;
1475 entry->pid = (tsk) ? tsk->pid : 0;
1476 entry->flags =
9244489a 1477#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1478 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1479#else
1480 TRACE_FLAG_IRQS_NOSUPPORT |
1481#endif
bc0c38d1
SR
1482 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1483 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1484 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1485}
f413cdb8 1486EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1487
e77405ad
SR
1488struct ring_buffer_event *
1489trace_buffer_lock_reserve(struct ring_buffer *buffer,
1490 int type,
1491 unsigned long len,
1492 unsigned long flags, int pc)
51a763dd
ACM
1493{
1494 struct ring_buffer_event *event;
1495
e77405ad 1496 event = ring_buffer_lock_reserve(buffer, len);
51a763dd
ACM
1497 if (event != NULL) {
1498 struct trace_entry *ent = ring_buffer_event_data(event);
1499
1500 tracing_generic_entry_update(ent, flags, pc);
1501 ent->type = type;
1502 }
1503
1504 return event;
1505}
51a763dd 1506
7ffbd48d
SR
1507void
1508__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1509{
1510 __this_cpu_write(trace_cmdline_save, true);
1511 ring_buffer_unlock_commit(buffer, event);
1512}
1513
e77405ad
SR
1514static inline void
1515__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1516 struct ring_buffer_event *event,
0d5c6e1c 1517 unsigned long flags, int pc)
51a763dd 1518{
7ffbd48d 1519 __buffer_unlock_commit(buffer, event);
51a763dd 1520
e77405ad
SR
1521 ftrace_trace_stack(buffer, flags, 6, pc);
1522 ftrace_trace_userstack(buffer, flags, pc);
07edf712
FW
1523}
1524
e77405ad
SR
1525void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1526 struct ring_buffer_event *event,
1527 unsigned long flags, int pc)
07edf712 1528{
0d5c6e1c 1529 __trace_buffer_unlock_commit(buffer, event, flags, pc);
51a763dd 1530}
0d5c6e1c 1531EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
51a763dd 1532
ccb469a1
SR
1533struct ring_buffer_event *
1534trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1535 struct ftrace_event_file *ftrace_file,
1536 int type, unsigned long len,
1537 unsigned long flags, int pc)
1538{
12883efb 1539 *current_rb = ftrace_file->tr->trace_buffer.buffer;
ccb469a1
SR
1540 return trace_buffer_lock_reserve(*current_rb,
1541 type, len, flags, pc);
1542}
1543EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1544
ef5580d0 1545struct ring_buffer_event *
e77405ad
SR
1546trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1547 int type, unsigned long len,
ef5580d0
SR
1548 unsigned long flags, int pc)
1549{
12883efb 1550 *current_rb = global_trace.trace_buffer.buffer;
e77405ad 1551 return trace_buffer_lock_reserve(*current_rb,
ef5580d0
SR
1552 type, len, flags, pc);
1553}
94487d6d 1554EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0 1555
e77405ad
SR
1556void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1557 struct ring_buffer_event *event,
ef5580d0
SR
1558 unsigned long flags, int pc)
1559{
0d5c6e1c 1560 __trace_buffer_unlock_commit(buffer, event, flags, pc);
07edf712 1561}
94487d6d 1562EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712 1563
0d5c6e1c
SR
1564void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1565 struct ring_buffer_event *event,
1566 unsigned long flags, int pc,
1567 struct pt_regs *regs)
1fd8df2c 1568{
7ffbd48d 1569 __buffer_unlock_commit(buffer, event);
1fd8df2c
MH
1570
1571 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1572 ftrace_trace_userstack(buffer, flags, pc);
1573}
0d5c6e1c 1574EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1fd8df2c 1575
e77405ad
SR
1576void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1577 struct ring_buffer_event *event)
77d9f465 1578{
e77405ad 1579 ring_buffer_discard_commit(buffer, event);
ef5580d0 1580}
12acd473 1581EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 1582
e309b41d 1583void
7be42151 1584trace_function(struct trace_array *tr,
38697053
SR
1585 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1586 int pc)
bc0c38d1 1587{
e1112b4d 1588 struct ftrace_event_call *call = &event_function;
12883efb 1589 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 1590 struct ring_buffer_event *event;
777e208d 1591 struct ftrace_entry *entry;
bc0c38d1 1592
d769041f 1593 /* If we are reading the ring buffer, don't trace */
dd17c8f7 1594 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
d769041f
SR
1595 return;
1596
e77405ad 1597 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
51a763dd 1598 flags, pc);
3928a8a2
SR
1599 if (!event)
1600 return;
1601 entry = ring_buffer_event_data(event);
777e208d
SR
1602 entry->ip = ip;
1603 entry->parent_ip = parent_ip;
e1112b4d 1604
e77405ad 1605 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1606 __buffer_unlock_commit(buffer, event);
bc0c38d1
SR
1607}
1608
e309b41d 1609void
2e0f5761 1610ftrace(struct trace_array *tr, struct trace_array_cpu *data,
38697053
SR
1611 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1612 int pc)
2e0f5761
IM
1613{
1614 if (likely(!atomic_read(&data->disabled)))
7be42151 1615 trace_function(tr, ip, parent_ip, flags, pc);
2e0f5761
IM
1616}
1617
c0a0d0d3 1618#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
1619
1620#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1621struct ftrace_stack {
1622 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1623};
1624
1625static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1626static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1627
e77405ad 1628static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 1629 unsigned long flags,
1fd8df2c 1630 int skip, int pc, struct pt_regs *regs)
86387f7e 1631{
e1112b4d 1632 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1633 struct ring_buffer_event *event;
777e208d 1634 struct stack_entry *entry;
86387f7e 1635 struct stack_trace trace;
4a9bd3f1
SR
1636 int use_stack;
1637 int size = FTRACE_STACK_ENTRIES;
1638
1639 trace.nr_entries = 0;
1640 trace.skip = skip;
1641
1642 /*
1643 * Since events can happen in NMIs there's no safe way to
1644 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1645 * or NMI comes in, it will just have to use the default
1646 * FTRACE_STACK_SIZE.
1647 */
1648 preempt_disable_notrace();
1649
82146529 1650 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
1651 /*
1652 * We don't need any atomic variables, just a barrier.
1653 * If an interrupt comes in, we don't care, because it would
1654 * have exited and put the counter back to what we want.
1655 * We just need a barrier to keep gcc from moving things
1656 * around.
1657 */
1658 barrier();
1659 if (use_stack == 1) {
1660 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1661 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1662
1663 if (regs)
1664 save_stack_trace_regs(regs, &trace);
1665 else
1666 save_stack_trace(&trace);
1667
1668 if (trace.nr_entries > size)
1669 size = trace.nr_entries;
1670 } else
1671 /* From now on, use_stack is a boolean */
1672 use_stack = 0;
1673
1674 size *= sizeof(unsigned long);
86387f7e 1675
e77405ad 1676 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
4a9bd3f1 1677 sizeof(*entry) + size, flags, pc);
3928a8a2 1678 if (!event)
4a9bd3f1
SR
1679 goto out;
1680 entry = ring_buffer_event_data(event);
86387f7e 1681
4a9bd3f1
SR
1682 memset(&entry->caller, 0, size);
1683
1684 if (use_stack)
1685 memcpy(&entry->caller, trace.entries,
1686 trace.nr_entries * sizeof(unsigned long));
1687 else {
1688 trace.max_entries = FTRACE_STACK_ENTRIES;
1689 trace.entries = entry->caller;
1690 if (regs)
1691 save_stack_trace_regs(regs, &trace);
1692 else
1693 save_stack_trace(&trace);
1694 }
1695
1696 entry->size = trace.nr_entries;
86387f7e 1697
e77405ad 1698 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1699 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
1700
1701 out:
1702 /* Again, don't let gcc optimize things here */
1703 barrier();
82146529 1704 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
1705 preempt_enable_notrace();
1706
f0a920d5
IM
1707}
1708
1fd8df2c
MH
1709void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1710 int skip, int pc, struct pt_regs *regs)
1711{
1712 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1713 return;
1714
1715 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1716}
1717
e77405ad
SR
1718void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1719 int skip, int pc)
53614991
SR
1720{
1721 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1722 return;
1723
1fd8df2c 1724 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
53614991
SR
1725}
1726
c0a0d0d3
FW
1727void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1728 int pc)
38697053 1729{
12883efb 1730 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
38697053
SR
1731}
1732
03889384
SR
1733/**
1734 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 1735 * @skip: Number of functions to skip (helper handlers)
03889384 1736 */
c142be8e 1737void trace_dump_stack(int skip)
03889384
SR
1738{
1739 unsigned long flags;
1740
1741 if (tracing_disabled || tracing_selftest_running)
e36c5458 1742 return;
03889384
SR
1743
1744 local_save_flags(flags);
1745
c142be8e
SRRH
1746 /*
1747 * Skip 3 more, seems to get us at the caller of
1748 * this function.
1749 */
1750 skip += 3;
1751 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1752 flags, skip, preempt_count(), NULL);
03889384
SR
1753}
1754
91e86e56
SR
1755static DEFINE_PER_CPU(int, user_stack_count);
1756
e77405ad
SR
1757void
1758ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 1759{
e1112b4d 1760 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1761 struct ring_buffer_event *event;
02b67518
TE
1762 struct userstack_entry *entry;
1763 struct stack_trace trace;
02b67518
TE
1764
1765 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1766 return;
1767
b6345879
SR
1768 /*
1769 * NMIs can not handle page faults, even with fix ups.
1770 * The save user stack can (and often does) fault.
1771 */
1772 if (unlikely(in_nmi()))
1773 return;
02b67518 1774
91e86e56
SR
1775 /*
1776 * prevent recursion, since the user stack tracing may
1777 * trigger other kernel events.
1778 */
1779 preempt_disable();
1780 if (__this_cpu_read(user_stack_count))
1781 goto out;
1782
1783 __this_cpu_inc(user_stack_count);
1784
e77405ad 1785 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
51a763dd 1786 sizeof(*entry), flags, pc);
02b67518 1787 if (!event)
1dbd1951 1788 goto out_drop_count;
02b67518 1789 entry = ring_buffer_event_data(event);
02b67518 1790
48659d31 1791 entry->tgid = current->tgid;
02b67518
TE
1792 memset(&entry->caller, 0, sizeof(entry->caller));
1793
1794 trace.nr_entries = 0;
1795 trace.max_entries = FTRACE_STACK_ENTRIES;
1796 trace.skip = 0;
1797 trace.entries = entry->caller;
1798
1799 save_stack_trace_user(&trace);
e77405ad 1800 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1801 __buffer_unlock_commit(buffer, event);
91e86e56 1802
1dbd1951 1803 out_drop_count:
91e86e56 1804 __this_cpu_dec(user_stack_count);
91e86e56
SR
1805 out:
1806 preempt_enable();
02b67518
TE
1807}
1808
4fd27358
HE
1809#ifdef UNUSED
1810static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1811{
7be42151 1812 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1813}
4fd27358 1814#endif /* UNUSED */
02b67518 1815
c0a0d0d3
FW
1816#endif /* CONFIG_STACKTRACE */
1817
07d777fe
SR
1818/* created for use with alloc_percpu */
1819struct trace_buffer_struct {
1820 char buffer[TRACE_BUF_SIZE];
1821};
1822
1823static struct trace_buffer_struct *trace_percpu_buffer;
1824static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1825static struct trace_buffer_struct *trace_percpu_irq_buffer;
1826static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1827
1828/*
1829 * The buffer used is dependent on the context. There is a per cpu
1830 * buffer for normal context, softirq contex, hard irq context and
1831 * for NMI context. Thise allows for lockless recording.
1832 *
1833 * Note, if the buffers failed to be allocated, then this returns NULL
1834 */
1835static char *get_trace_buf(void)
1836{
1837 struct trace_buffer_struct *percpu_buffer;
07d777fe
SR
1838
1839 /*
1840 * If we have allocated per cpu buffers, then we do not
1841 * need to do any locking.
1842 */
1843 if (in_nmi())
1844 percpu_buffer = trace_percpu_nmi_buffer;
1845 else if (in_irq())
1846 percpu_buffer = trace_percpu_irq_buffer;
1847 else if (in_softirq())
1848 percpu_buffer = trace_percpu_sirq_buffer;
1849 else
1850 percpu_buffer = trace_percpu_buffer;
1851
1852 if (!percpu_buffer)
1853 return NULL;
1854
d8a0349c 1855 return this_cpu_ptr(&percpu_buffer->buffer[0]);
07d777fe
SR
1856}
1857
1858static int alloc_percpu_trace_buffer(void)
1859{
1860 struct trace_buffer_struct *buffers;
1861 struct trace_buffer_struct *sirq_buffers;
1862 struct trace_buffer_struct *irq_buffers;
1863 struct trace_buffer_struct *nmi_buffers;
1864
1865 buffers = alloc_percpu(struct trace_buffer_struct);
1866 if (!buffers)
1867 goto err_warn;
1868
1869 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1870 if (!sirq_buffers)
1871 goto err_sirq;
1872
1873 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1874 if (!irq_buffers)
1875 goto err_irq;
1876
1877 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1878 if (!nmi_buffers)
1879 goto err_nmi;
1880
1881 trace_percpu_buffer = buffers;
1882 trace_percpu_sirq_buffer = sirq_buffers;
1883 trace_percpu_irq_buffer = irq_buffers;
1884 trace_percpu_nmi_buffer = nmi_buffers;
1885
1886 return 0;
1887
1888 err_nmi:
1889 free_percpu(irq_buffers);
1890 err_irq:
1891 free_percpu(sirq_buffers);
1892 err_sirq:
1893 free_percpu(buffers);
1894 err_warn:
1895 WARN(1, "Could not allocate percpu trace_printk buffer");
1896 return -ENOMEM;
1897}
1898
81698831
SR
1899static int buffers_allocated;
1900
07d777fe
SR
1901void trace_printk_init_buffers(void)
1902{
07d777fe
SR
1903 if (buffers_allocated)
1904 return;
1905
1906 if (alloc_percpu_trace_buffer())
1907 return;
1908
1909 pr_info("ftrace: Allocated trace_printk buffers\n");
1910
b382ede6
SR
1911 /* Expand the buffers to set size */
1912 tracing_update_buffers();
1913
07d777fe 1914 buffers_allocated = 1;
81698831
SR
1915
1916 /*
1917 * trace_printk_init_buffers() can be called by modules.
1918 * If that happens, then we need to start cmdline recording
1919 * directly here. If the global_trace.buffer is already
1920 * allocated here, then this was called by module code.
1921 */
12883efb 1922 if (global_trace.trace_buffer.buffer)
81698831
SR
1923 tracing_start_cmdline_record();
1924}
1925
1926void trace_printk_start_comm(void)
1927{
1928 /* Start tracing comms if trace printk is set */
1929 if (!buffers_allocated)
1930 return;
1931 tracing_start_cmdline_record();
1932}
1933
1934static void trace_printk_start_stop_comm(int enabled)
1935{
1936 if (!buffers_allocated)
1937 return;
1938
1939 if (enabled)
1940 tracing_start_cmdline_record();
1941 else
1942 tracing_stop_cmdline_record();
07d777fe
SR
1943}
1944
769b0441 1945/**
48ead020 1946 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
1947 *
1948 */
40ce74f1 1949int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 1950{
e1112b4d 1951 struct ftrace_event_call *call = &event_bprint;
769b0441 1952 struct ring_buffer_event *event;
e77405ad 1953 struct ring_buffer *buffer;
769b0441 1954 struct trace_array *tr = &global_trace;
48ead020 1955 struct bprint_entry *entry;
769b0441 1956 unsigned long flags;
07d777fe
SR
1957 char *tbuffer;
1958 int len = 0, size, pc;
769b0441
FW
1959
1960 if (unlikely(tracing_selftest_running || tracing_disabled))
1961 return 0;
1962
1963 /* Don't pollute graph traces with trace_vprintk internals */
1964 pause_graph_tracing();
1965
1966 pc = preempt_count();
5168ae50 1967 preempt_disable_notrace();
769b0441 1968
07d777fe
SR
1969 tbuffer = get_trace_buf();
1970 if (!tbuffer) {
1971 len = 0;
769b0441 1972 goto out;
07d777fe 1973 }
769b0441 1974
07d777fe 1975 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 1976
07d777fe
SR
1977 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1978 goto out;
769b0441 1979
07d777fe 1980 local_save_flags(flags);
769b0441 1981 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 1982 buffer = tr->trace_buffer.buffer;
e77405ad
SR
1983 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1984 flags, pc);
769b0441 1985 if (!event)
07d777fe 1986 goto out;
769b0441
FW
1987 entry = ring_buffer_event_data(event);
1988 entry->ip = ip;
769b0441
FW
1989 entry->fmt = fmt;
1990
07d777fe 1991 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
d931369b 1992 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 1993 __buffer_unlock_commit(buffer, event);
d931369b
SR
1994 ftrace_trace_stack(buffer, flags, 6, pc);
1995 }
769b0441 1996
769b0441 1997out:
5168ae50 1998 preempt_enable_notrace();
769b0441
FW
1999 unpause_graph_tracing();
2000
2001 return len;
2002}
48ead020
FW
2003EXPORT_SYMBOL_GPL(trace_vbprintk);
2004
12883efb
SRRH
2005static int
2006__trace_array_vprintk(struct ring_buffer *buffer,
2007 unsigned long ip, const char *fmt, va_list args)
48ead020 2008{
e1112b4d 2009 struct ftrace_event_call *call = &event_print;
48ead020 2010 struct ring_buffer_event *event;
07d777fe 2011 int len = 0, size, pc;
48ead020 2012 struct print_entry *entry;
07d777fe
SR
2013 unsigned long flags;
2014 char *tbuffer;
48ead020
FW
2015
2016 if (tracing_disabled || tracing_selftest_running)
2017 return 0;
2018
07d777fe
SR
2019 /* Don't pollute graph traces with trace_vprintk internals */
2020 pause_graph_tracing();
2021
48ead020
FW
2022 pc = preempt_count();
2023 preempt_disable_notrace();
48ead020 2024
07d777fe
SR
2025
2026 tbuffer = get_trace_buf();
2027 if (!tbuffer) {
2028 len = 0;
48ead020 2029 goto out;
07d777fe 2030 }
48ead020 2031
07d777fe
SR
2032 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2033 if (len > TRACE_BUF_SIZE)
2034 goto out;
48ead020 2035
07d777fe 2036 local_save_flags(flags);
48ead020 2037 size = sizeof(*entry) + len + 1;
e77405ad 2038 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
07d777fe 2039 flags, pc);
48ead020 2040 if (!event)
07d777fe 2041 goto out;
48ead020 2042 entry = ring_buffer_event_data(event);
c13d2f7c 2043 entry->ip = ip;
48ead020 2044
07d777fe 2045 memcpy(&entry->buf, tbuffer, len);
c13d2f7c 2046 entry->buf[len] = '\0';
d931369b 2047 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2048 __buffer_unlock_commit(buffer, event);
07d777fe 2049 ftrace_trace_stack(buffer, flags, 6, pc);
d931369b 2050 }
48ead020
FW
2051 out:
2052 preempt_enable_notrace();
07d777fe 2053 unpause_graph_tracing();
48ead020
FW
2054
2055 return len;
2056}
659372d3 2057
12883efb
SRRH
2058int trace_array_vprintk(struct trace_array *tr,
2059 unsigned long ip, const char *fmt, va_list args)
2060{
2061 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2062}
2063
2064int trace_array_printk(struct trace_array *tr,
2065 unsigned long ip, const char *fmt, ...)
2066{
2067 int ret;
2068 va_list ap;
2069
2070 if (!(trace_flags & TRACE_ITER_PRINTK))
2071 return 0;
2072
2073 va_start(ap, fmt);
2074 ret = trace_array_vprintk(tr, ip, fmt, ap);
2075 va_end(ap);
2076 return ret;
2077}
2078
2079int trace_array_printk_buf(struct ring_buffer *buffer,
2080 unsigned long ip, const char *fmt, ...)
2081{
2082 int ret;
2083 va_list ap;
2084
2085 if (!(trace_flags & TRACE_ITER_PRINTK))
2086 return 0;
2087
2088 va_start(ap, fmt);
2089 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2090 va_end(ap);
2091 return ret;
2092}
2093
659372d3
SR
2094int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2095{
a813a159 2096 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 2097}
769b0441
FW
2098EXPORT_SYMBOL_GPL(trace_vprintk);
2099
e2ac8ef5 2100static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 2101{
6d158a81
SR
2102 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2103
5a90f577 2104 iter->idx++;
6d158a81
SR
2105 if (buf_iter)
2106 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
2107}
2108
e309b41d 2109static struct trace_entry *
bc21b478
SR
2110peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2111 unsigned long *lost_events)
dd0e545f 2112{
3928a8a2 2113 struct ring_buffer_event *event;
6d158a81 2114 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 2115
d769041f
SR
2116 if (buf_iter)
2117 event = ring_buffer_iter_peek(buf_iter, ts);
2118 else
12883efb 2119 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 2120 lost_events);
d769041f 2121
4a9bd3f1
SR
2122 if (event) {
2123 iter->ent_size = ring_buffer_event_length(event);
2124 return ring_buffer_event_data(event);
2125 }
2126 iter->ent_size = 0;
2127 return NULL;
dd0e545f 2128}
d769041f 2129
dd0e545f 2130static struct trace_entry *
bc21b478
SR
2131__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2132 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 2133{
12883efb 2134 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 2135 struct trace_entry *ent, *next = NULL;
aa27497c 2136 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 2137 int cpu_file = iter->cpu_file;
3928a8a2 2138 u64 next_ts = 0, ts;
bc0c38d1 2139 int next_cpu = -1;
12b5da34 2140 int next_size = 0;
bc0c38d1
SR
2141 int cpu;
2142
b04cc6b1
FW
2143 /*
2144 * If we are in a per_cpu trace file, don't bother by iterating over
2145 * all cpu and peek directly.
2146 */
ae3b5093 2147 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
2148 if (ring_buffer_empty_cpu(buffer, cpu_file))
2149 return NULL;
bc21b478 2150 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
2151 if (ent_cpu)
2152 *ent_cpu = cpu_file;
2153
2154 return ent;
2155 }
2156
ab46428c 2157 for_each_tracing_cpu(cpu) {
dd0e545f 2158
3928a8a2
SR
2159 if (ring_buffer_empty_cpu(buffer, cpu))
2160 continue;
dd0e545f 2161
bc21b478 2162 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 2163
cdd31cd2
IM
2164 /*
2165 * Pick the entry with the smallest timestamp:
2166 */
3928a8a2 2167 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
2168 next = ent;
2169 next_cpu = cpu;
3928a8a2 2170 next_ts = ts;
bc21b478 2171 next_lost = lost_events;
12b5da34 2172 next_size = iter->ent_size;
bc0c38d1
SR
2173 }
2174 }
2175
12b5da34
SR
2176 iter->ent_size = next_size;
2177
bc0c38d1
SR
2178 if (ent_cpu)
2179 *ent_cpu = next_cpu;
2180
3928a8a2
SR
2181 if (ent_ts)
2182 *ent_ts = next_ts;
2183
bc21b478
SR
2184 if (missing_events)
2185 *missing_events = next_lost;
2186
bc0c38d1
SR
2187 return next;
2188}
2189
dd0e545f 2190/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
2191struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2192 int *ent_cpu, u64 *ent_ts)
bc0c38d1 2193{
bc21b478 2194 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
2195}
2196
2197/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 2198void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 2199{
bc21b478
SR
2200 iter->ent = __find_next_entry(iter, &iter->cpu,
2201 &iter->lost_events, &iter->ts);
dd0e545f 2202
3928a8a2 2203 if (iter->ent)
e2ac8ef5 2204 trace_iterator_increment(iter);
dd0e545f 2205
3928a8a2 2206 return iter->ent ? iter : NULL;
b3806b43 2207}
bc0c38d1 2208
e309b41d 2209static void trace_consume(struct trace_iterator *iter)
b3806b43 2210{
12883efb 2211 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 2212 &iter->lost_events);
bc0c38d1
SR
2213}
2214
e309b41d 2215static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
2216{
2217 struct trace_iterator *iter = m->private;
bc0c38d1 2218 int i = (int)*pos;
4e3c3333 2219 void *ent;
bc0c38d1 2220
a63ce5b3
SR
2221 WARN_ON_ONCE(iter->leftover);
2222
bc0c38d1
SR
2223 (*pos)++;
2224
2225 /* can't go backwards */
2226 if (iter->idx > i)
2227 return NULL;
2228
2229 if (iter->idx < 0)
955b61e5 2230 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2231 else
2232 ent = iter;
2233
2234 while (ent && iter->idx < i)
955b61e5 2235 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2236
2237 iter->pos = *pos;
2238
bc0c38d1
SR
2239 return ent;
2240}
2241
955b61e5 2242void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 2243{
2f26ebd5
SR
2244 struct ring_buffer_event *event;
2245 struct ring_buffer_iter *buf_iter;
2246 unsigned long entries = 0;
2247 u64 ts;
2248
12883efb 2249 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 2250
6d158a81
SR
2251 buf_iter = trace_buffer_iter(iter, cpu);
2252 if (!buf_iter)
2f26ebd5
SR
2253 return;
2254
2f26ebd5
SR
2255 ring_buffer_iter_reset(buf_iter);
2256
2257 /*
2258 * We could have the case with the max latency tracers
2259 * that a reset never took place on a cpu. This is evident
2260 * by the timestamp being before the start of the buffer.
2261 */
2262 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 2263 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
2264 break;
2265 entries++;
2266 ring_buffer_read(buf_iter, NULL);
2267 }
2268
12883efb 2269 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
2270}
2271
d7350c3f 2272/*
d7350c3f
FW
2273 * The current tracer is copied to avoid a global locking
2274 * all around.
2275 */
bc0c38d1
SR
2276static void *s_start(struct seq_file *m, loff_t *pos)
2277{
2278 struct trace_iterator *iter = m->private;
2b6080f2 2279 struct trace_array *tr = iter->tr;
b04cc6b1 2280 int cpu_file = iter->cpu_file;
bc0c38d1
SR
2281 void *p = NULL;
2282 loff_t l = 0;
3928a8a2 2283 int cpu;
bc0c38d1 2284
2fd196ec
HT
2285 /*
2286 * copy the tracer to avoid using a global lock all around.
2287 * iter->trace is a copy of current_trace, the pointer to the
2288 * name may be used instead of a strcmp(), as iter->trace->name
2289 * will point to the same string as current_trace->name.
2290 */
bc0c38d1 2291 mutex_lock(&trace_types_lock);
2b6080f2
SR
2292 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2293 *iter->trace = *tr->current_trace;
d7350c3f 2294 mutex_unlock(&trace_types_lock);
bc0c38d1 2295
12883efb 2296#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2297 if (iter->snapshot && iter->trace->use_max_tr)
2298 return ERR_PTR(-EBUSY);
12883efb 2299#endif
debdd57f
HT
2300
2301 if (!iter->snapshot)
2302 atomic_inc(&trace_record_cmdline_disabled);
bc0c38d1 2303
bc0c38d1
SR
2304 if (*pos != iter->pos) {
2305 iter->ent = NULL;
2306 iter->cpu = 0;
2307 iter->idx = -1;
2308
ae3b5093 2309 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2310 for_each_tracing_cpu(cpu)
2f26ebd5 2311 tracing_iter_reset(iter, cpu);
b04cc6b1 2312 } else
2f26ebd5 2313 tracing_iter_reset(iter, cpu_file);
bc0c38d1 2314
ac91d854 2315 iter->leftover = 0;
bc0c38d1
SR
2316 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2317 ;
2318
2319 } else {
a63ce5b3
SR
2320 /*
2321 * If we overflowed the seq_file before, then we want
2322 * to just reuse the trace_seq buffer again.
2323 */
2324 if (iter->leftover)
2325 p = iter;
2326 else {
2327 l = *pos - 1;
2328 p = s_next(m, p, &l);
2329 }
bc0c38d1
SR
2330 }
2331
4f535968 2332 trace_event_read_lock();
7e53bd42 2333 trace_access_lock(cpu_file);
bc0c38d1
SR
2334 return p;
2335}
2336
2337static void s_stop(struct seq_file *m, void *p)
2338{
7e53bd42
LJ
2339 struct trace_iterator *iter = m->private;
2340
12883efb 2341#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2342 if (iter->snapshot && iter->trace->use_max_tr)
2343 return;
12883efb 2344#endif
debdd57f
HT
2345
2346 if (!iter->snapshot)
2347 atomic_dec(&trace_record_cmdline_disabled);
12883efb 2348
7e53bd42 2349 trace_access_unlock(iter->cpu_file);
4f535968 2350 trace_event_read_unlock();
bc0c38d1
SR
2351}
2352
39eaf7ef 2353static void
12883efb
SRRH
2354get_total_entries(struct trace_buffer *buf,
2355 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
2356{
2357 unsigned long count;
2358 int cpu;
2359
2360 *total = 0;
2361 *entries = 0;
2362
2363 for_each_tracing_cpu(cpu) {
12883efb 2364 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
2365 /*
2366 * If this buffer has skipped entries, then we hold all
2367 * entries for the trace and we need to ignore the
2368 * ones before the time stamp.
2369 */
12883efb
SRRH
2370 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2371 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
2372 /* total is the same as the entries */
2373 *total += count;
2374 } else
2375 *total += count +
12883efb 2376 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
2377 *entries += count;
2378 }
2379}
2380
e309b41d 2381static void print_lat_help_header(struct seq_file *m)
bc0c38d1 2382{
a6168353
ME
2383 seq_puts(m, "# _------=> CPU# \n");
2384 seq_puts(m, "# / _-----=> irqs-off \n");
2385 seq_puts(m, "# | / _----=> need-resched \n");
2386 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2387 seq_puts(m, "# ||| / _--=> preempt-depth \n");
e6e1e259
SR
2388 seq_puts(m, "# |||| / delay \n");
2389 seq_puts(m, "# cmd pid ||||| time | caller \n");
2390 seq_puts(m, "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
2391}
2392
12883efb 2393static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 2394{
39eaf7ef
SR
2395 unsigned long total;
2396 unsigned long entries;
2397
12883efb 2398 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
2399 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2400 entries, total, num_online_cpus());
2401 seq_puts(m, "#\n");
2402}
2403
12883efb 2404static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
39eaf7ef 2405{
12883efb 2406 print_event_info(buf, m);
77271ce4 2407 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
a6168353 2408 seq_puts(m, "# | | | | |\n");
bc0c38d1
SR
2409}
2410
12883efb 2411static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
77271ce4 2412{
12883efb 2413 print_event_info(buf, m);
77271ce4
SR
2414 seq_puts(m, "# _-----=> irqs-off\n");
2415 seq_puts(m, "# / _----=> need-resched\n");
2416 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2417 seq_puts(m, "# || / _--=> preempt-depth\n");
2418 seq_puts(m, "# ||| / delay\n");
2419 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2420 seq_puts(m, "# | | | |||| | |\n");
2421}
bc0c38d1 2422
62b915f1 2423void
bc0c38d1
SR
2424print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2425{
2426 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
2427 struct trace_buffer *buf = iter->trace_buffer;
2428 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 2429 struct tracer *type = iter->trace;
39eaf7ef
SR
2430 unsigned long entries;
2431 unsigned long total;
bc0c38d1
SR
2432 const char *name = "preemption";
2433
d840f718 2434 name = type->name;
bc0c38d1 2435
12883efb 2436 get_total_entries(buf, &total, &entries);
bc0c38d1 2437
888b55dc 2438 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 2439 name, UTS_RELEASE);
888b55dc 2440 seq_puts(m, "# -----------------------------------"
bc0c38d1 2441 "---------------------------------\n");
888b55dc 2442 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 2443 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 2444 nsecs_to_usecs(data->saved_latency),
bc0c38d1 2445 entries,
4c11d7ae 2446 total,
12883efb 2447 buf->cpu,
bc0c38d1
SR
2448#if defined(CONFIG_PREEMPT_NONE)
2449 "server",
2450#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2451 "desktop",
b5c21b45 2452#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
2453 "preempt",
2454#else
2455 "unknown",
2456#endif
2457 /* These are reserved for later use */
2458 0, 0, 0, 0);
2459#ifdef CONFIG_SMP
2460 seq_printf(m, " #P:%d)\n", num_online_cpus());
2461#else
2462 seq_puts(m, ")\n");
2463#endif
888b55dc
KM
2464 seq_puts(m, "# -----------------\n");
2465 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 2466 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
2467 data->comm, data->pid,
2468 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 2469 data->policy, data->rt_priority);
888b55dc 2470 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
2471
2472 if (data->critical_start) {
888b55dc 2473 seq_puts(m, "# => started at: ");
214023c3
SR
2474 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2475 trace_print_seq(m, &iter->seq);
888b55dc 2476 seq_puts(m, "\n# => ended at: ");
214023c3
SR
2477 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2478 trace_print_seq(m, &iter->seq);
8248ac05 2479 seq_puts(m, "\n#\n");
bc0c38d1
SR
2480 }
2481
888b55dc 2482 seq_puts(m, "#\n");
bc0c38d1
SR
2483}
2484
a309720c
SR
2485static void test_cpu_buff_start(struct trace_iterator *iter)
2486{
2487 struct trace_seq *s = &iter->seq;
2488
12ef7d44
SR
2489 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2490 return;
2491
2492 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2493 return;
2494
4462344e 2495 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
2496 return;
2497
12883efb 2498 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
2499 return;
2500
4462344e 2501 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
2502
2503 /* Don't print started cpu buffer for the first entry of the trace */
2504 if (iter->idx > 1)
2505 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2506 iter->cpu);
a309720c
SR
2507}
2508
2c4f035f 2509static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 2510{
214023c3 2511 struct trace_seq *s = &iter->seq;
bc0c38d1 2512 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 2513 struct trace_entry *entry;
f633cef0 2514 struct trace_event *event;
bc0c38d1 2515
4e3c3333 2516 entry = iter->ent;
dd0e545f 2517
a309720c
SR
2518 test_cpu_buff_start(iter);
2519
c4a8e8be 2520 event = ftrace_find_event(entry->type);
bc0c38d1 2521
c4a8e8be 2522 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
27d48be8
SR
2523 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2524 if (!trace_print_lat_context(iter))
2525 goto partial;
2526 } else {
2527 if (!trace_print_context(iter))
2528 goto partial;
2529 }
c4a8e8be 2530 }
bc0c38d1 2531
268ccda0 2532 if (event)
a9a57763 2533 return event->funcs->trace(iter, sym_flags, event);
d9793bd8
ACM
2534
2535 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2536 goto partial;
02b67518 2537
2c4f035f 2538 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2539partial:
2540 return TRACE_TYPE_PARTIAL_LINE;
bc0c38d1
SR
2541}
2542
2c4f035f 2543static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
2544{
2545 struct trace_seq *s = &iter->seq;
2546 struct trace_entry *entry;
f633cef0 2547 struct trace_event *event;
f9896bf3
IM
2548
2549 entry = iter->ent;
dd0e545f 2550
c4a8e8be 2551 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
d9793bd8
ACM
2552 if (!trace_seq_printf(s, "%d %d %llu ",
2553 entry->pid, iter->cpu, iter->ts))
2554 goto partial;
c4a8e8be 2555 }
f9896bf3 2556
f633cef0 2557 event = ftrace_find_event(entry->type);
268ccda0 2558 if (event)
a9a57763 2559 return event->funcs->raw(iter, 0, event);
d9793bd8
ACM
2560
2561 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2562 goto partial;
777e208d 2563
2c4f035f 2564 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2565partial:
2566 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3
IM
2567}
2568
2c4f035f 2569static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
2570{
2571 struct trace_seq *s = &iter->seq;
2572 unsigned char newline = '\n';
2573 struct trace_entry *entry;
f633cef0 2574 struct trace_event *event;
5e3ca0ec
IM
2575
2576 entry = iter->ent;
dd0e545f 2577
c4a8e8be
FW
2578 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2579 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2580 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2581 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2582 }
5e3ca0ec 2583
f633cef0 2584 event = ftrace_find_event(entry->type);
268ccda0 2585 if (event) {
a9a57763 2586 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
2587 if (ret != TRACE_TYPE_HANDLED)
2588 return ret;
2589 }
7104f300 2590
5e3ca0ec
IM
2591 SEQ_PUT_FIELD_RET(s, newline);
2592
2c4f035f 2593 return TRACE_TYPE_HANDLED;
5e3ca0ec
IM
2594}
2595
2c4f035f 2596static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
2597{
2598 struct trace_seq *s = &iter->seq;
2599 struct trace_entry *entry;
f633cef0 2600 struct trace_event *event;
cb0f12aa
IM
2601
2602 entry = iter->ent;
dd0e545f 2603
c4a8e8be
FW
2604 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2605 SEQ_PUT_FIELD_RET(s, entry->pid);
1830b52d 2606 SEQ_PUT_FIELD_RET(s, iter->cpu);
c4a8e8be
FW
2607 SEQ_PUT_FIELD_RET(s, iter->ts);
2608 }
cb0f12aa 2609
f633cef0 2610 event = ftrace_find_event(entry->type);
a9a57763
SR
2611 return event ? event->funcs->binary(iter, 0, event) :
2612 TRACE_TYPE_HANDLED;
cb0f12aa
IM
2613}
2614
62b915f1 2615int trace_empty(struct trace_iterator *iter)
bc0c38d1 2616{
6d158a81 2617 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
2618 int cpu;
2619
9aba60fe 2620 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 2621 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 2622 cpu = iter->cpu_file;
6d158a81
SR
2623 buf_iter = trace_buffer_iter(iter, cpu);
2624 if (buf_iter) {
2625 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
2626 return 0;
2627 } else {
12883efb 2628 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
2629 return 0;
2630 }
2631 return 1;
2632 }
2633
ab46428c 2634 for_each_tracing_cpu(cpu) {
6d158a81
SR
2635 buf_iter = trace_buffer_iter(iter, cpu);
2636 if (buf_iter) {
2637 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
2638 return 0;
2639 } else {
12883efb 2640 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
2641 return 0;
2642 }
bc0c38d1 2643 }
d769041f 2644
797d3712 2645 return 1;
bc0c38d1
SR
2646}
2647
4f535968 2648/* Called with trace_event_read_lock() held. */
955b61e5 2649enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 2650{
2c4f035f
FW
2651 enum print_line_t ret;
2652
ee5e51f5
JO
2653 if (iter->lost_events &&
2654 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2655 iter->cpu, iter->lost_events))
2656 return TRACE_TYPE_PARTIAL_LINE;
bc21b478 2657
2c4f035f
FW
2658 if (iter->trace && iter->trace->print_line) {
2659 ret = iter->trace->print_line(iter);
2660 if (ret != TRACE_TYPE_UNHANDLED)
2661 return ret;
2662 }
72829bc3 2663
09ae7234
SRRH
2664 if (iter->ent->type == TRACE_BPUTS &&
2665 trace_flags & TRACE_ITER_PRINTK &&
2666 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2667 return trace_print_bputs_msg_only(iter);
2668
48ead020
FW
2669 if (iter->ent->type == TRACE_BPRINT &&
2670 trace_flags & TRACE_ITER_PRINTK &&
2671 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2672 return trace_print_bprintk_msg_only(iter);
48ead020 2673
66896a85
FW
2674 if (iter->ent->type == TRACE_PRINT &&
2675 trace_flags & TRACE_ITER_PRINTK &&
2676 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2677 return trace_print_printk_msg_only(iter);
66896a85 2678
cb0f12aa
IM
2679 if (trace_flags & TRACE_ITER_BIN)
2680 return print_bin_fmt(iter);
2681
5e3ca0ec
IM
2682 if (trace_flags & TRACE_ITER_HEX)
2683 return print_hex_fmt(iter);
2684
f9896bf3
IM
2685 if (trace_flags & TRACE_ITER_RAW)
2686 return print_raw_fmt(iter);
2687
f9896bf3
IM
2688 return print_trace_fmt(iter);
2689}
2690
7e9a49ef
JO
2691void trace_latency_header(struct seq_file *m)
2692{
2693 struct trace_iterator *iter = m->private;
2694
2695 /* print nothing if the buffers are empty */
2696 if (trace_empty(iter))
2697 return;
2698
2699 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2700 print_trace_header(m, iter);
2701
2702 if (!(trace_flags & TRACE_ITER_VERBOSE))
2703 print_lat_help_header(m);
2704}
2705
62b915f1
JO
2706void trace_default_header(struct seq_file *m)
2707{
2708 struct trace_iterator *iter = m->private;
2709
f56e7f8e
JO
2710 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2711 return;
2712
62b915f1
JO
2713 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2714 /* print nothing if the buffers are empty */
2715 if (trace_empty(iter))
2716 return;
2717 print_trace_header(m, iter);
2718 if (!(trace_flags & TRACE_ITER_VERBOSE))
2719 print_lat_help_header(m);
2720 } else {
77271ce4
SR
2721 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2722 if (trace_flags & TRACE_ITER_IRQ_INFO)
12883efb 2723 print_func_help_header_irq(iter->trace_buffer, m);
77271ce4 2724 else
12883efb 2725 print_func_help_header(iter->trace_buffer, m);
77271ce4 2726 }
62b915f1
JO
2727 }
2728}
2729
e0a413f6
SR
2730static void test_ftrace_alive(struct seq_file *m)
2731{
2732 if (!ftrace_is_dead())
2733 return;
2734 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2735 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2736}
2737
d8741e2e 2738#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 2739static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 2740{
d8741e2e
SRRH
2741 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2742 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2743 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2744 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2745 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2746 seq_printf(m, "# is not a '0' or '1')\n");
2747}
f1affcaa
SRRH
2748
2749static void show_snapshot_percpu_help(struct seq_file *m)
2750{
2751 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2752#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2753 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2754 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2755#else
2756 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2757 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2758#endif
2759 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2760 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2761 seq_printf(m, "# is not a '0' or '1')\n");
2762}
2763
d8741e2e
SRRH
2764static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2765{
45ad21ca 2766 if (iter->tr->allocated_snapshot)
d8741e2e
SRRH
2767 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2768 else
2769 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2770
2771 seq_printf(m, "# Snapshot commands:\n");
f1affcaa
SRRH
2772 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2773 show_snapshot_main_help(m);
2774 else
2775 show_snapshot_percpu_help(m);
d8741e2e
SRRH
2776}
2777#else
2778/* Should never be called */
2779static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2780#endif
2781
bc0c38d1
SR
2782static int s_show(struct seq_file *m, void *v)
2783{
2784 struct trace_iterator *iter = v;
a63ce5b3 2785 int ret;
bc0c38d1
SR
2786
2787 if (iter->ent == NULL) {
2788 if (iter->tr) {
2789 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2790 seq_puts(m, "#\n");
e0a413f6 2791 test_ftrace_alive(m);
bc0c38d1 2792 }
d8741e2e
SRRH
2793 if (iter->snapshot && trace_empty(iter))
2794 print_snapshot_help(m, iter);
2795 else if (iter->trace && iter->trace->print_header)
8bba1bf5 2796 iter->trace->print_header(m);
62b915f1
JO
2797 else
2798 trace_default_header(m);
2799
a63ce5b3
SR
2800 } else if (iter->leftover) {
2801 /*
2802 * If we filled the seq_file buffer earlier, we
2803 * want to just show it now.
2804 */
2805 ret = trace_print_seq(m, &iter->seq);
2806
2807 /* ret should this time be zero, but you never know */
2808 iter->leftover = ret;
2809
bc0c38d1 2810 } else {
f9896bf3 2811 print_trace_line(iter);
a63ce5b3
SR
2812 ret = trace_print_seq(m, &iter->seq);
2813 /*
2814 * If we overflow the seq_file buffer, then it will
2815 * ask us for this data again at start up.
2816 * Use that instead.
2817 * ret is 0 if seq_file write succeeded.
2818 * -1 otherwise.
2819 */
2820 iter->leftover = ret;
bc0c38d1
SR
2821 }
2822
2823 return 0;
2824}
2825
88e9d34c 2826static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
2827 .start = s_start,
2828 .next = s_next,
2829 .stop = s_stop,
2830 .show = s_show,
bc0c38d1
SR
2831};
2832
e309b41d 2833static struct trace_iterator *
debdd57f 2834__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 2835{
2b6080f2
SR
2836 struct trace_cpu *tc = inode->i_private;
2837 struct trace_array *tr = tc->tr;
bc0c38d1 2838 struct trace_iterator *iter;
50e18b94 2839 int cpu;
bc0c38d1 2840
85a2f9b4
SR
2841 if (tracing_disabled)
2842 return ERR_PTR(-ENODEV);
60a11774 2843
50e18b94 2844 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
2845 if (!iter)
2846 return ERR_PTR(-ENOMEM);
bc0c38d1 2847
6d158a81
SR
2848 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2849 GFP_KERNEL);
93574fcc
DC
2850 if (!iter->buffer_iter)
2851 goto release;
2852
d7350c3f
FW
2853 /*
2854 * We make a copy of the current tracer to avoid concurrent
2855 * changes on it while we are reading.
2856 */
bc0c38d1 2857 mutex_lock(&trace_types_lock);
d7350c3f 2858 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 2859 if (!iter->trace)
d7350c3f 2860 goto fail;
85a2f9b4 2861
2b6080f2 2862 *iter->trace = *tr->current_trace;
d7350c3f 2863
79f55997 2864 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
2865 goto fail;
2866
12883efb
SRRH
2867 iter->tr = tr;
2868
2869#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
2870 /* Currently only the top directory has a snapshot */
2871 if (tr->current_trace->print_max || snapshot)
12883efb 2872 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 2873 else
12883efb
SRRH
2874#endif
2875 iter->trace_buffer = &tr->trace_buffer;
debdd57f 2876 iter->snapshot = snapshot;
bc0c38d1 2877 iter->pos = -1;
d7350c3f 2878 mutex_init(&iter->mutex);
2b6080f2 2879 iter->cpu_file = tc->cpu;
bc0c38d1 2880
8bba1bf5
MM
2881 /* Notify the tracer early; before we stop tracing. */
2882 if (iter->trace && iter->trace->open)
a93751ca 2883 iter->trace->open(iter);
8bba1bf5 2884
12ef7d44 2885 /* Annotate start of buffers if we had overruns */
12883efb 2886 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
2887 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2888
8be0709f
DS
2889 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2890 if (trace_clocks[trace_clock_id].in_ns)
2891 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2892
debdd57f
HT
2893 /* stop the trace while dumping if we are not opening "snapshot" */
2894 if (!iter->snapshot)
2b6080f2 2895 tracing_stop_tr(tr);
2f26ebd5 2896
ae3b5093 2897 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2898 for_each_tracing_cpu(cpu) {
b04cc6b1 2899 iter->buffer_iter[cpu] =
12883efb 2900 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2901 }
2902 ring_buffer_read_prepare_sync();
2903 for_each_tracing_cpu(cpu) {
2904 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2905 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
2906 }
2907 } else {
2908 cpu = iter->cpu_file;
3928a8a2 2909 iter->buffer_iter[cpu] =
12883efb 2910 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2911 ring_buffer_read_prepare_sync();
2912 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2913 tracing_iter_reset(iter, cpu);
3928a8a2
SR
2914 }
2915
a695cb58
SRRH
2916 tr->ref++;
2917
bc0c38d1
SR
2918 mutex_unlock(&trace_types_lock);
2919
bc0c38d1 2920 return iter;
3928a8a2 2921
d7350c3f 2922 fail:
3928a8a2 2923 mutex_unlock(&trace_types_lock);
d7350c3f 2924 kfree(iter->trace);
6d158a81 2925 kfree(iter->buffer_iter);
93574fcc 2926release:
50e18b94
JO
2927 seq_release_private(inode, file);
2928 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
2929}
2930
2931int tracing_open_generic(struct inode *inode, struct file *filp)
2932{
60a11774
SR
2933 if (tracing_disabled)
2934 return -ENODEV;
2935
bc0c38d1
SR
2936 filp->private_data = inode->i_private;
2937 return 0;
2938}
2939
4fd27358 2940static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 2941{
907f2784 2942 struct seq_file *m = file->private_data;
4acd4d00 2943 struct trace_iterator *iter;
2b6080f2 2944 struct trace_array *tr;
3928a8a2 2945 int cpu;
bc0c38d1 2946
4acd4d00
SR
2947 if (!(file->f_mode & FMODE_READ))
2948 return 0;
2949
2950 iter = m->private;
12883efb 2951 tr = iter->tr;
4acd4d00 2952
bc0c38d1 2953 mutex_lock(&trace_types_lock);
a695cb58
SRRH
2954
2955 WARN_ON(!tr->ref);
2956 tr->ref--;
2957
3928a8a2
SR
2958 for_each_tracing_cpu(cpu) {
2959 if (iter->buffer_iter[cpu])
2960 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2961 }
2962
bc0c38d1
SR
2963 if (iter->trace && iter->trace->close)
2964 iter->trace->close(iter);
2965
debdd57f
HT
2966 if (!iter->snapshot)
2967 /* reenable tracing if it was previously enabled */
2b6080f2 2968 tracing_start_tr(tr);
bc0c38d1
SR
2969 mutex_unlock(&trace_types_lock);
2970
d7350c3f 2971 mutex_destroy(&iter->mutex);
b0dfa978 2972 free_cpumask_var(iter->started);
d7350c3f 2973 kfree(iter->trace);
6d158a81 2974 kfree(iter->buffer_iter);
50e18b94 2975 seq_release_private(inode, file);
bc0c38d1
SR
2976 return 0;
2977}
2978
2979static int tracing_open(struct inode *inode, struct file *file)
2980{
85a2f9b4
SR
2981 struct trace_iterator *iter;
2982 int ret = 0;
bc0c38d1 2983
4acd4d00
SR
2984 /* If this file was open for write, then erase contents */
2985 if ((file->f_mode & FMODE_WRITE) &&
8650ae32 2986 (file->f_flags & O_TRUNC)) {
2b6080f2
SR
2987 struct trace_cpu *tc = inode->i_private;
2988 struct trace_array *tr = tc->tr;
bc0c38d1 2989
2b6080f2 2990 if (tc->cpu == RING_BUFFER_ALL_CPUS)
12883efb 2991 tracing_reset_online_cpus(&tr->trace_buffer);
4acd4d00 2992 else
12883efb 2993 tracing_reset(&tr->trace_buffer, tc->cpu);
4acd4d00 2994 }
bc0c38d1 2995
4acd4d00 2996 if (file->f_mode & FMODE_READ) {
debdd57f 2997 iter = __tracing_open(inode, file, false);
4acd4d00
SR
2998 if (IS_ERR(iter))
2999 ret = PTR_ERR(iter);
3000 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3001 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3002 }
bc0c38d1
SR
3003 return ret;
3004}
3005
e309b41d 3006static void *
bc0c38d1
SR
3007t_next(struct seq_file *m, void *v, loff_t *pos)
3008{
f129e965 3009 struct tracer *t = v;
bc0c38d1
SR
3010
3011 (*pos)++;
3012
3013 if (t)
3014 t = t->next;
3015
bc0c38d1
SR
3016 return t;
3017}
3018
3019static void *t_start(struct seq_file *m, loff_t *pos)
3020{
f129e965 3021 struct tracer *t;
bc0c38d1
SR
3022 loff_t l = 0;
3023
3024 mutex_lock(&trace_types_lock);
f129e965 3025 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
bc0c38d1
SR
3026 ;
3027
3028 return t;
3029}
3030
3031static void t_stop(struct seq_file *m, void *p)
3032{
3033 mutex_unlock(&trace_types_lock);
3034}
3035
3036static int t_show(struct seq_file *m, void *v)
3037{
3038 struct tracer *t = v;
3039
3040 if (!t)
3041 return 0;
3042
3043 seq_printf(m, "%s", t->name);
3044 if (t->next)
3045 seq_putc(m, ' ');
3046 else
3047 seq_putc(m, '\n');
3048
3049 return 0;
3050}
3051
88e9d34c 3052static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
3053 .start = t_start,
3054 .next = t_next,
3055 .stop = t_stop,
3056 .show = t_show,
bc0c38d1
SR
3057};
3058
3059static int show_traces_open(struct inode *inode, struct file *file)
3060{
60a11774
SR
3061 if (tracing_disabled)
3062 return -ENODEV;
3063
f129e965 3064 return seq_open(file, &show_traces_seq_ops);
bc0c38d1
SR
3065}
3066
4acd4d00
SR
3067static ssize_t
3068tracing_write_stub(struct file *filp, const char __user *ubuf,
3069 size_t count, loff_t *ppos)
3070{
3071 return count;
3072}
3073
364829b1
SP
3074static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3075{
3076 if (file->f_mode & FMODE_READ)
3077 return seq_lseek(file, offset, origin);
3078 else
3079 return 0;
3080}
3081
5e2336a0 3082static const struct file_operations tracing_fops = {
4bf39a94
IM
3083 .open = tracing_open,
3084 .read = seq_read,
4acd4d00 3085 .write = tracing_write_stub,
364829b1 3086 .llseek = tracing_seek,
4bf39a94 3087 .release = tracing_release,
bc0c38d1
SR
3088};
3089
5e2336a0 3090static const struct file_operations show_traces_fops = {
c7078de1
IM
3091 .open = show_traces_open,
3092 .read = seq_read,
3093 .release = seq_release,
b444786f 3094 .llseek = seq_lseek,
c7078de1
IM
3095};
3096
36dfe925
IM
3097/*
3098 * Only trace on a CPU if the bitmask is set:
3099 */
9e01c1b7 3100static cpumask_var_t tracing_cpumask;
36dfe925
IM
3101
3102/*
3103 * The tracer itself will not take this lock, but still we want
3104 * to provide a consistent cpumask to user-space:
3105 */
3106static DEFINE_MUTEX(tracing_cpumask_update_lock);
3107
3108/*
3109 * Temporary storage for the character representation of the
3110 * CPU bitmask (and one more byte for the newline):
3111 */
3112static char mask_str[NR_CPUS + 1];
3113
c7078de1
IM
3114static ssize_t
3115tracing_cpumask_read(struct file *filp, char __user *ubuf,
3116 size_t count, loff_t *ppos)
3117{
36dfe925 3118 int len;
c7078de1
IM
3119
3120 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 3121
9e01c1b7 3122 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
36dfe925
IM
3123 if (count - len < 2) {
3124 count = -EINVAL;
3125 goto out_err;
3126 }
3127 len += sprintf(mask_str + len, "\n");
3128 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3129
3130out_err:
c7078de1
IM
3131 mutex_unlock(&tracing_cpumask_update_lock);
3132
3133 return count;
3134}
3135
3136static ssize_t
3137tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3138 size_t count, loff_t *ppos)
3139{
2b6080f2 3140 struct trace_array *tr = filp->private_data;
9e01c1b7 3141 cpumask_var_t tracing_cpumask_new;
2b6080f2 3142 int err, cpu;
9e01c1b7
RR
3143
3144 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3145 return -ENOMEM;
c7078de1 3146
9e01c1b7 3147 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 3148 if (err)
36dfe925
IM
3149 goto err_unlock;
3150
215368e8
LZ
3151 mutex_lock(&tracing_cpumask_update_lock);
3152
a5e25883 3153 local_irq_disable();
0199c4e6 3154 arch_spin_lock(&ftrace_max_lock);
ab46428c 3155 for_each_tracing_cpu(cpu) {
36dfe925
IM
3156 /*
3157 * Increase/decrease the disabled counter if we are
3158 * about to flip a bit in the cpumask:
3159 */
9e01c1b7
RR
3160 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3161 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3162 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3163 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 3164 }
9e01c1b7
RR
3165 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3166 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3167 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3168 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
3169 }
3170 }
0199c4e6 3171 arch_spin_unlock(&ftrace_max_lock);
a5e25883 3172 local_irq_enable();
36dfe925 3173
9e01c1b7 3174 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
3175
3176 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 3177 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
3178
3179 return count;
36dfe925
IM
3180
3181err_unlock:
215368e8 3182 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
3183
3184 return err;
c7078de1
IM
3185}
3186
5e2336a0 3187static const struct file_operations tracing_cpumask_fops = {
c7078de1
IM
3188 .open = tracing_open_generic,
3189 .read = tracing_cpumask_read,
3190 .write = tracing_cpumask_write,
b444786f 3191 .llseek = generic_file_llseek,
bc0c38d1
SR
3192};
3193
fdb372ed 3194static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 3195{
d8e83d26 3196 struct tracer_opt *trace_opts;
2b6080f2 3197 struct trace_array *tr = m->private;
d8e83d26 3198 u32 tracer_flags;
d8e83d26 3199 int i;
adf9f195 3200
d8e83d26 3201 mutex_lock(&trace_types_lock);
2b6080f2
SR
3202 tracer_flags = tr->current_trace->flags->val;
3203 trace_opts = tr->current_trace->flags->opts;
d8e83d26 3204
bc0c38d1
SR
3205 for (i = 0; trace_options[i]; i++) {
3206 if (trace_flags & (1 << i))
fdb372ed 3207 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 3208 else
fdb372ed 3209 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
3210 }
3211
adf9f195
FW
3212 for (i = 0; trace_opts[i].name; i++) {
3213 if (tracer_flags & trace_opts[i].bit)
fdb372ed 3214 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 3215 else
fdb372ed 3216 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 3217 }
d8e83d26 3218 mutex_unlock(&trace_types_lock);
adf9f195 3219
fdb372ed 3220 return 0;
bc0c38d1 3221}
bc0c38d1 3222
8d18eaaf
LZ
3223static int __set_tracer_option(struct tracer *trace,
3224 struct tracer_flags *tracer_flags,
3225 struct tracer_opt *opts, int neg)
3226{
3227 int ret;
bc0c38d1 3228
8d18eaaf
LZ
3229 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3230 if (ret)
3231 return ret;
3232
3233 if (neg)
3234 tracer_flags->val &= ~opts->bit;
3235 else
3236 tracer_flags->val |= opts->bit;
3237 return 0;
bc0c38d1
SR
3238}
3239
adf9f195
FW
3240/* Try to assign a tracer specific option */
3241static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3242{
7770841e 3243 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 3244 struct tracer_opt *opts = NULL;
8d18eaaf 3245 int i;
adf9f195 3246
7770841e
Z
3247 for (i = 0; tracer_flags->opts[i].name; i++) {
3248 opts = &tracer_flags->opts[i];
adf9f195 3249
8d18eaaf
LZ
3250 if (strcmp(cmp, opts->name) == 0)
3251 return __set_tracer_option(trace, trace->flags,
3252 opts, neg);
adf9f195 3253 }
adf9f195 3254
8d18eaaf 3255 return -EINVAL;
adf9f195
FW
3256}
3257
613f04a0
SRRH
3258/* Some tracers require overwrite to stay enabled */
3259int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3260{
3261 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3262 return -1;
3263
3264 return 0;
3265}
3266
2b6080f2 3267int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
3268{
3269 /* do nothing if flag is already set */
3270 if (!!(trace_flags & mask) == !!enabled)
613f04a0
SRRH
3271 return 0;
3272
3273 /* Give the tracer a chance to approve the change */
2b6080f2
SR
3274 if (tr->current_trace->flag_changed)
3275 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
613f04a0 3276 return -EINVAL;
af4617bd
SR
3277
3278 if (enabled)
3279 trace_flags |= mask;
3280 else
3281 trace_flags &= ~mask;
e870e9a1
LZ
3282
3283 if (mask == TRACE_ITER_RECORD_CMD)
3284 trace_event_enable_cmd_record(enabled);
750912fa 3285
80902822 3286 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 3287 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 3288#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 3289 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
3290#endif
3291 }
81698831
SR
3292
3293 if (mask == TRACE_ITER_PRINTK)
3294 trace_printk_start_stop_comm(enabled);
613f04a0
SRRH
3295
3296 return 0;
af4617bd
SR
3297}
3298
2b6080f2 3299static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 3300{
8d18eaaf 3301 char *cmp;
bc0c38d1 3302 int neg = 0;
613f04a0 3303 int ret = -ENODEV;
bc0c38d1
SR
3304 int i;
3305
7bcfaf54 3306 cmp = strstrip(option);
bc0c38d1 3307
8d18eaaf 3308 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
3309 neg = 1;
3310 cmp += 2;
3311 }
3312
69d34da2
SRRH
3313 mutex_lock(&trace_types_lock);
3314
bc0c38d1 3315 for (i = 0; trace_options[i]; i++) {
8d18eaaf 3316 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 3317 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
3318 break;
3319 }
3320 }
adf9f195
FW
3321
3322 /* If no option could be set, test the specific tracer options */
69d34da2 3323 if (!trace_options[i])
2b6080f2 3324 ret = set_tracer_option(tr->current_trace, cmp, neg);
69d34da2
SRRH
3325
3326 mutex_unlock(&trace_types_lock);
bc0c38d1 3327
7bcfaf54
SR
3328 return ret;
3329}
3330
3331static ssize_t
3332tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3333 size_t cnt, loff_t *ppos)
3334{
2b6080f2
SR
3335 struct seq_file *m = filp->private_data;
3336 struct trace_array *tr = m->private;
7bcfaf54 3337 char buf[64];
613f04a0 3338 int ret;
7bcfaf54
SR
3339
3340 if (cnt >= sizeof(buf))
3341 return -EINVAL;
3342
3343 if (copy_from_user(&buf, ubuf, cnt))
3344 return -EFAULT;
3345
a8dd2176
SR
3346 buf[cnt] = 0;
3347
2b6080f2 3348 ret = trace_set_options(tr, buf);
613f04a0
SRRH
3349 if (ret < 0)
3350 return ret;
7bcfaf54 3351
cf8517cf 3352 *ppos += cnt;
bc0c38d1
SR
3353
3354 return cnt;
3355}
3356
fdb372ed
LZ
3357static int tracing_trace_options_open(struct inode *inode, struct file *file)
3358{
3359 if (tracing_disabled)
3360 return -ENODEV;
2b6080f2
SR
3361
3362 return single_open(file, tracing_trace_options_show, inode->i_private);
fdb372ed
LZ
3363}
3364
5e2336a0 3365static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
3366 .open = tracing_trace_options_open,
3367 .read = seq_read,
3368 .llseek = seq_lseek,
3369 .release = single_release,
ee6bce52 3370 .write = tracing_trace_options_write,
bc0c38d1
SR
3371};
3372
7bd2f24c
IM
3373static const char readme_msg[] =
3374 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
3375 "# echo 0 > tracing_on : quick way to disable tracing\n"
3376 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3377 " Important files:\n"
3378 " trace\t\t\t- The static contents of the buffer\n"
3379 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3380 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3381 " current_tracer\t- function and latency tracers\n"
3382 " available_tracers\t- list of configured tracers for current_tracer\n"
3383 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3384 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3385 " trace_clock\t\t-change the clock used to order events\n"
3386 " local: Per cpu clock but may not be synced across CPUs\n"
3387 " global: Synced across CPUs but slows tracing down.\n"
3388 " counter: Not a clock, but just an increment\n"
3389 " uptime: Jiffy counter from time of boot\n"
3390 " perf: Same clock that perf events use\n"
3391#ifdef CONFIG_X86_64
3392 " x86-tsc: TSC cycle counter\n"
3393#endif
3394 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3395 " tracing_cpumask\t- Limit which CPUs to trace\n"
3396 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3397 "\t\t\t Remove sub-buffer with rmdir\n"
3398 " trace_options\t\t- Set format or modify how tracing happens\n"
3399 "\t\t\t Disable an option by adding a suffix 'no' to the option name\n"
3400#ifdef CONFIG_DYNAMIC_FTRACE
3401 "\n available_filter_functions - list of functions that can be filtered on\n"
3402 " set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3403 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3404 " modules: Can select a group via module\n"
3405 " Format: :mod:<module-name>\n"
3406 " example: echo :mod:ext3 > set_ftrace_filter\n"
3407 " triggers: a command to perform when function is hit\n"
3408 " Format: <function>:<trigger>[:count]\n"
3409 " trigger: traceon, traceoff\n"
3410 " enable_event:<system>:<event>\n"
3411 " disable_event:<system>:<event>\n"
3412#ifdef CONFIG_STACKTRACE
3413 " stacktrace\n"
3414#endif
3415#ifdef CONFIG_TRACER_SNAPSHOT
3416 " snapshot\n"
3417#endif
3418 " example: echo do_fault:traceoff > set_ftrace_filter\n"
3419 " echo do_trap:traceoff:3 > set_ftrace_filter\n"
3420 " The first one will disable tracing every time do_fault is hit\n"
3421 " The second will disable tracing at most 3 times when do_trap is hit\n"
3422 " The first time do trap is hit and it disables tracing, the counter\n"
3423 " will decrement to 2. If tracing is already disabled, the counter\n"
3424 " will not decrement. It only decrements when the trigger did work\n"
3425 " To remove trigger without count:\n"
3426 " echo '!<function>:<trigger> > set_ftrace_filter\n"
3427 " To remove trigger with a count:\n"
3428 " echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3429 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3430 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3431 " modules: Can select a group via module command :mod:\n"
3432 " Does not accept triggers\n"
3433#endif /* CONFIG_DYNAMIC_FTRACE */
3434#ifdef CONFIG_FUNCTION_TRACER
3435 " set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3436#endif
3437#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3438 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3439 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3440#endif
3441#ifdef CONFIG_TRACER_SNAPSHOT
3442 "\n snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3443 "\t\t\t Read the contents for more information\n"
3444#endif
3445#ifdef CONFIG_STACKTRACE
3446 " stack_trace\t\t- Shows the max stack trace when active\n"
3447 " stack_max_size\t- Shows current max stack size that was traced\n"
3448 "\t\t\t Write into this file to reset the max size (trigger a new trace)\n"
3449#ifdef CONFIG_DYNAMIC_FTRACE
3450 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3451#endif
3452#endif /* CONFIG_STACKTRACE */
7bd2f24c
IM
3453;
3454
3455static ssize_t
3456tracing_readme_read(struct file *filp, char __user *ubuf,
3457 size_t cnt, loff_t *ppos)
3458{
3459 return simple_read_from_buffer(ubuf, cnt, ppos,
3460 readme_msg, strlen(readme_msg));
3461}
3462
5e2336a0 3463static const struct file_operations tracing_readme_fops = {
c7078de1
IM
3464 .open = tracing_open_generic,
3465 .read = tracing_readme_read,
b444786f 3466 .llseek = generic_file_llseek,
7bd2f24c
IM
3467};
3468
69abe6a5
AP
3469static ssize_t
3470tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3471 size_t cnt, loff_t *ppos)
3472{
3473 char *buf_comm;
3474 char *file_buf;
3475 char *buf;
3476 int len = 0;
3477 int pid;
3478 int i;
3479
3480 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3481 if (!file_buf)
3482 return -ENOMEM;
3483
3484 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3485 if (!buf_comm) {
3486 kfree(file_buf);
3487 return -ENOMEM;
3488 }
3489
3490 buf = file_buf;
3491
3492 for (i = 0; i < SAVED_CMDLINES; i++) {
3493 int r;
3494
3495 pid = map_cmdline_to_pid[i];
3496 if (pid == -1 || pid == NO_CMDLINE_MAP)
3497 continue;
3498
3499 trace_find_cmdline(pid, buf_comm);
3500 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3501 buf += r;
3502 len += r;
3503 }
3504
3505 len = simple_read_from_buffer(ubuf, cnt, ppos,
3506 file_buf, len);
3507
3508 kfree(file_buf);
3509 kfree(buf_comm);
3510
3511 return len;
3512}
3513
3514static const struct file_operations tracing_saved_cmdlines_fops = {
3515 .open = tracing_open_generic,
3516 .read = tracing_saved_cmdlines_read,
b444786f 3517 .llseek = generic_file_llseek,
69abe6a5
AP
3518};
3519
bc0c38d1
SR
3520static ssize_t
3521tracing_set_trace_read(struct file *filp, char __user *ubuf,
3522 size_t cnt, loff_t *ppos)
3523{
2b6080f2 3524 struct trace_array *tr = filp->private_data;
ee6c2c1b 3525 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
3526 int r;
3527
3528 mutex_lock(&trace_types_lock);
2b6080f2 3529 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
3530 mutex_unlock(&trace_types_lock);
3531
4bf39a94 3532 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3533}
3534
b6f11df2
ACM
3535int tracer_init(struct tracer *t, struct trace_array *tr)
3536{
12883efb 3537 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
3538 return t->init(tr);
3539}
3540
12883efb 3541static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
3542{
3543 int cpu;
737223fb 3544
438ced17 3545 for_each_tracing_cpu(cpu)
12883efb 3546 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
3547}
3548
12883efb 3549#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 3550/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
3551static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3552 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
3553{
3554 int cpu, ret = 0;
3555
3556 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3557 for_each_tracing_cpu(cpu) {
12883efb
SRRH
3558 ret = ring_buffer_resize(trace_buf->buffer,
3559 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
3560 if (ret < 0)
3561 break;
12883efb
SRRH
3562 per_cpu_ptr(trace_buf->data, cpu)->entries =
3563 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
3564 }
3565 } else {
12883efb
SRRH
3566 ret = ring_buffer_resize(trace_buf->buffer,
3567 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 3568 if (ret == 0)
12883efb
SRRH
3569 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3570 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
3571 }
3572
3573 return ret;
3574}
12883efb 3575#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 3576
2b6080f2
SR
3577static int __tracing_resize_ring_buffer(struct trace_array *tr,
3578 unsigned long size, int cpu)
73c5162a
SR
3579{
3580 int ret;
3581
3582 /*
3583 * If kernel or user changes the size of the ring buffer
a123c52b
SR
3584 * we use the size that was given, and we can forget about
3585 * expanding it later.
73c5162a 3586 */
55034cd6 3587 ring_buffer_expanded = true;
73c5162a 3588
b382ede6 3589 /* May be called before buffers are initialized */
12883efb 3590 if (!tr->trace_buffer.buffer)
b382ede6
SR
3591 return 0;
3592
12883efb 3593 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
3594 if (ret < 0)
3595 return ret;
3596
12883efb 3597#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3598 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3599 !tr->current_trace->use_max_tr)
ef710e10
KM
3600 goto out;
3601
12883efb 3602 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 3603 if (ret < 0) {
12883efb
SRRH
3604 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3605 &tr->trace_buffer, cpu);
73c5162a 3606 if (r < 0) {
a123c52b
SR
3607 /*
3608 * AARGH! We are left with different
3609 * size max buffer!!!!
3610 * The max buffer is our "snapshot" buffer.
3611 * When a tracer needs a snapshot (one of the
3612 * latency tracers), it swaps the max buffer
3613 * with the saved snap shot. We succeeded to
3614 * update the size of the main buffer, but failed to
3615 * update the size of the max buffer. But when we tried
3616 * to reset the main buffer to the original size, we
3617 * failed there too. This is very unlikely to
3618 * happen, but if it does, warn and kill all
3619 * tracing.
3620 */
73c5162a
SR
3621 WARN_ON(1);
3622 tracing_disabled = 1;
3623 }
3624 return ret;
3625 }
3626
438ced17 3627 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3628 set_buffer_entries(&tr->max_buffer, size);
438ced17 3629 else
12883efb 3630 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 3631
ef710e10 3632 out:
12883efb
SRRH
3633#endif /* CONFIG_TRACER_MAX_TRACE */
3634
438ced17 3635 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3636 set_buffer_entries(&tr->trace_buffer, size);
438ced17 3637 else
12883efb 3638 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
3639
3640 return ret;
3641}
3642
2b6080f2
SR
3643static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3644 unsigned long size, int cpu_id)
4f271a2a 3645{
83f40318 3646 int ret = size;
4f271a2a
VN
3647
3648 mutex_lock(&trace_types_lock);
3649
438ced17
VN
3650 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3651 /* make sure, this cpu is enabled in the mask */
3652 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3653 ret = -EINVAL;
3654 goto out;
3655 }
3656 }
4f271a2a 3657
2b6080f2 3658 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
3659 if (ret < 0)
3660 ret = -ENOMEM;
3661
438ced17 3662out:
4f271a2a
VN
3663 mutex_unlock(&trace_types_lock);
3664
3665 return ret;
3666}
3667
ef710e10 3668
1852fcce
SR
3669/**
3670 * tracing_update_buffers - used by tracing facility to expand ring buffers
3671 *
3672 * To save on memory when the tracing is never used on a system with it
3673 * configured in. The ring buffers are set to a minimum size. But once
3674 * a user starts to use the tracing facility, then they need to grow
3675 * to their default size.
3676 *
3677 * This function is to be called when a tracer is about to be used.
3678 */
3679int tracing_update_buffers(void)
3680{
3681 int ret = 0;
3682
1027fcb2 3683 mutex_lock(&trace_types_lock);
1852fcce 3684 if (!ring_buffer_expanded)
2b6080f2 3685 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 3686 RING_BUFFER_ALL_CPUS);
1027fcb2 3687 mutex_unlock(&trace_types_lock);
1852fcce
SR
3688
3689 return ret;
3690}
3691
577b785f
SR
3692struct trace_option_dentry;
3693
3694static struct trace_option_dentry *
2b6080f2 3695create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f
SR
3696
3697static void
3698destroy_trace_option_files(struct trace_option_dentry *topts);
3699
b2821ae6 3700static int tracing_set_tracer(const char *buf)
bc0c38d1 3701{
577b785f 3702 static struct trace_option_dentry *topts;
bc0c38d1
SR
3703 struct trace_array *tr = &global_trace;
3704 struct tracer *t;
12883efb 3705#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3706 bool had_max_tr;
12883efb 3707#endif
d9e54076 3708 int ret = 0;
bc0c38d1 3709
1027fcb2
SR
3710 mutex_lock(&trace_types_lock);
3711
73c5162a 3712 if (!ring_buffer_expanded) {
2b6080f2 3713 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 3714 RING_BUFFER_ALL_CPUS);
73c5162a 3715 if (ret < 0)
59f586db 3716 goto out;
73c5162a
SR
3717 ret = 0;
3718 }
3719
bc0c38d1
SR
3720 for (t = trace_types; t; t = t->next) {
3721 if (strcmp(t->name, buf) == 0)
3722 break;
3723 }
c2931e05
FW
3724 if (!t) {
3725 ret = -EINVAL;
3726 goto out;
3727 }
2b6080f2 3728 if (t == tr->current_trace)
bc0c38d1
SR
3729 goto out;
3730
9f029e83 3731 trace_branch_disable();
613f04a0 3732
2b6080f2 3733 tr->current_trace->enabled = false;
613f04a0 3734
2b6080f2
SR
3735 if (tr->current_trace->reset)
3736 tr->current_trace->reset(tr);
34600f0e 3737
12883efb 3738 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 3739 tr->current_trace = &nop_trace;
34600f0e 3740
45ad21ca
SRRH
3741#ifdef CONFIG_TRACER_MAX_TRACE
3742 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
3743
3744 if (had_max_tr && !t->use_max_tr) {
3745 /*
3746 * We need to make sure that the update_max_tr sees that
3747 * current_trace changed to nop_trace to keep it from
3748 * swapping the buffers after we resize it.
3749 * The update_max_tr is called from interrupts disabled
3750 * so a synchronized_sched() is sufficient.
3751 */
3752 synchronize_sched();
3209cff4 3753 free_snapshot(tr);
ef710e10 3754 }
12883efb 3755#endif
577b785f
SR
3756 destroy_trace_option_files(topts);
3757
2b6080f2 3758 topts = create_trace_option_files(tr, t);
12883efb
SRRH
3759
3760#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3761 if (t->use_max_tr && !had_max_tr) {
3209cff4 3762 ret = alloc_snapshot(tr);
d60da506
HT
3763 if (ret < 0)
3764 goto out;
ef710e10 3765 }
12883efb 3766#endif
577b785f 3767
1c80025a 3768 if (t->init) {
b6f11df2 3769 ret = tracer_init(t, tr);
1c80025a
FW
3770 if (ret)
3771 goto out;
3772 }
bc0c38d1 3773
2b6080f2
SR
3774 tr->current_trace = t;
3775 tr->current_trace->enabled = true;
9f029e83 3776 trace_branch_enable(tr);
bc0c38d1
SR
3777 out:
3778 mutex_unlock(&trace_types_lock);
3779
d9e54076
PZ
3780 return ret;
3781}
3782
3783static ssize_t
3784tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3785 size_t cnt, loff_t *ppos)
3786{
ee6c2c1b 3787 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
3788 int i;
3789 size_t ret;
e6e7a65a
FW
3790 int err;
3791
3792 ret = cnt;
d9e54076 3793
ee6c2c1b
LZ
3794 if (cnt > MAX_TRACER_SIZE)
3795 cnt = MAX_TRACER_SIZE;
d9e54076
PZ
3796
3797 if (copy_from_user(&buf, ubuf, cnt))
3798 return -EFAULT;
3799
3800 buf[cnt] = 0;
3801
3802 /* strip ending whitespace. */
3803 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3804 buf[i] = 0;
3805
e6e7a65a
FW
3806 err = tracing_set_tracer(buf);
3807 if (err)
3808 return err;
d9e54076 3809
cf8517cf 3810 *ppos += ret;
bc0c38d1 3811
c2931e05 3812 return ret;
bc0c38d1
SR
3813}
3814
3815static ssize_t
3816tracing_max_lat_read(struct file *filp, char __user *ubuf,
3817 size_t cnt, loff_t *ppos)
3818{
3819 unsigned long *ptr = filp->private_data;
3820 char buf[64];
3821 int r;
3822
cffae437 3823 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 3824 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
3825 if (r > sizeof(buf))
3826 r = sizeof(buf);
4bf39a94 3827 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3828}
3829
3830static ssize_t
3831tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3832 size_t cnt, loff_t *ppos)
3833{
5e39841c 3834 unsigned long *ptr = filp->private_data;
5e39841c 3835 unsigned long val;
c6caeeb1 3836 int ret;
bc0c38d1 3837
22fe9b54
PH
3838 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3839 if (ret)
c6caeeb1 3840 return ret;
bc0c38d1
SR
3841
3842 *ptr = val * 1000;
3843
3844 return cnt;
3845}
3846
b3806b43
SR
3847static int tracing_open_pipe(struct inode *inode, struct file *filp)
3848{
2b6080f2
SR
3849 struct trace_cpu *tc = inode->i_private;
3850 struct trace_array *tr = tc->tr;
b3806b43 3851 struct trace_iterator *iter;
b04cc6b1 3852 int ret = 0;
b3806b43
SR
3853
3854 if (tracing_disabled)
3855 return -ENODEV;
3856
b04cc6b1
FW
3857 mutex_lock(&trace_types_lock);
3858
b3806b43
SR
3859 /* create a buffer to store the information to pass to userspace */
3860 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
3861 if (!iter) {
3862 ret = -ENOMEM;
3863 goto out;
3864 }
b3806b43 3865
d7350c3f
FW
3866 /*
3867 * We make a copy of the current tracer to avoid concurrent
3868 * changes on it while we are reading.
3869 */
3870 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3871 if (!iter->trace) {
3872 ret = -ENOMEM;
3873 goto fail;
3874 }
2b6080f2 3875 *iter->trace = *tr->current_trace;
d7350c3f 3876
4462344e 3877 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 3878 ret = -ENOMEM;
d7350c3f 3879 goto fail;
4462344e
RR
3880 }
3881
a309720c 3882 /* trace pipe does not show start of buffer */
4462344e 3883 cpumask_setall(iter->started);
a309720c 3884
112f38a7
SR
3885 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3886 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3887
8be0709f
DS
3888 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3889 if (trace_clocks[trace_clock_id].in_ns)
3890 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3891
2b6080f2
SR
3892 iter->cpu_file = tc->cpu;
3893 iter->tr = tc->tr;
12883efb 3894 iter->trace_buffer = &tc->tr->trace_buffer;
d7350c3f 3895 mutex_init(&iter->mutex);
b3806b43
SR
3896 filp->private_data = iter;
3897
107bad8b
SR
3898 if (iter->trace->pipe_open)
3899 iter->trace->pipe_open(iter);
107bad8b 3900
b444786f 3901 nonseekable_open(inode, filp);
b04cc6b1
FW
3902out:
3903 mutex_unlock(&trace_types_lock);
3904 return ret;
d7350c3f
FW
3905
3906fail:
3907 kfree(iter->trace);
3908 kfree(iter);
3909 mutex_unlock(&trace_types_lock);
3910 return ret;
b3806b43
SR
3911}
3912
3913static int tracing_release_pipe(struct inode *inode, struct file *file)
3914{
3915 struct trace_iterator *iter = file->private_data;
3916
b04cc6b1
FW
3917 mutex_lock(&trace_types_lock);
3918
29bf4a5e 3919 if (iter->trace->pipe_close)
c521efd1
SR
3920 iter->trace->pipe_close(iter);
3921
b04cc6b1
FW
3922 mutex_unlock(&trace_types_lock);
3923
4462344e 3924 free_cpumask_var(iter->started);
d7350c3f
FW
3925 mutex_destroy(&iter->mutex);
3926 kfree(iter->trace);
b3806b43 3927 kfree(iter);
b3806b43
SR
3928
3929 return 0;
3930}
3931
2a2cc8f7 3932static unsigned int
cc60cdc9 3933trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 3934{
15693458
SRRH
3935 /* Iterators are static, they should be filled or empty */
3936 if (trace_buffer_iter(iter, iter->cpu_file))
3937 return POLLIN | POLLRDNORM;
2a2cc8f7 3938
15693458 3939 if (trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
3940 /*
3941 * Always select as readable when in blocking mode
3942 */
3943 return POLLIN | POLLRDNORM;
15693458 3944 else
12883efb 3945 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 3946 filp, poll_table);
2a2cc8f7 3947}
2a2cc8f7 3948
cc60cdc9
SR
3949static unsigned int
3950tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3951{
3952 struct trace_iterator *iter = filp->private_data;
3953
3954 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
3955}
3956
6eaaa5d5
FW
3957/*
3958 * This is a make-shift waitqueue.
3959 * A tracer might use this callback on some rare cases:
3960 *
3961 * 1) the current tracer might hold the runqueue lock when it wakes up
3962 * a reader, hence a deadlock (sched, function, and function graph tracers)
3963 * 2) the function tracers, trace all functions, we don't want
3964 * the overhead of calling wake_up and friends
3965 * (and tracing them too)
3966 *
3967 * Anyway, this is really very primitive wakeup.
3968 */
3969void poll_wait_pipe(struct trace_iterator *iter)
3970{
3971 set_current_state(TASK_INTERRUPTIBLE);
3972 /* sleep for 100 msecs, and try again. */
3973 schedule_timeout(HZ / 10);
3974}
3975
ff98781b
EGM
3976/* Must be called with trace_types_lock mutex held. */
3977static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
3978{
3979 struct trace_iterator *iter = filp->private_data;
b3806b43 3980
b3806b43 3981 while (trace_empty(iter)) {
2dc8f095 3982
107bad8b 3983 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 3984 return -EAGAIN;
107bad8b 3985 }
2dc8f095 3986
d7350c3f 3987 mutex_unlock(&iter->mutex);
107bad8b 3988
6eaaa5d5 3989 iter->trace->wait_pipe(iter);
b3806b43 3990
d7350c3f 3991 mutex_lock(&iter->mutex);
107bad8b 3992
6eaaa5d5 3993 if (signal_pending(current))
ff98781b 3994 return -EINTR;
b3806b43
SR
3995
3996 /*
250bfd3d 3997 * We block until we read something and tracing is disabled.
b3806b43
SR
3998 * We still block if tracing is disabled, but we have never
3999 * read anything. This allows a user to cat this file, and
4000 * then enable tracing. But after we have read something,
4001 * we give an EOF when tracing is again disabled.
4002 *
4003 * iter->pos will be 0 if we haven't read anything.
4004 */
10246fa3 4005 if (!tracing_is_on() && iter->pos)
b3806b43 4006 break;
b3806b43
SR
4007 }
4008
ff98781b
EGM
4009 return 1;
4010}
4011
4012/*
4013 * Consumer reader.
4014 */
4015static ssize_t
4016tracing_read_pipe(struct file *filp, char __user *ubuf,
4017 size_t cnt, loff_t *ppos)
4018{
4019 struct trace_iterator *iter = filp->private_data;
2b6080f2 4020 struct trace_array *tr = iter->tr;
ff98781b
EGM
4021 ssize_t sret;
4022
4023 /* return any leftover data */
4024 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4025 if (sret != -EBUSY)
4026 return sret;
4027
f9520750 4028 trace_seq_init(&iter->seq);
ff98781b 4029
d7350c3f 4030 /* copy the tracer to avoid using a global lock all around */
ff98781b 4031 mutex_lock(&trace_types_lock);
2b6080f2
SR
4032 if (unlikely(iter->trace->name != tr->current_trace->name))
4033 *iter->trace = *tr->current_trace;
d7350c3f
FW
4034 mutex_unlock(&trace_types_lock);
4035
4036 /*
4037 * Avoid more than one consumer on a single file descriptor
4038 * This is just a matter of traces coherency, the ring buffer itself
4039 * is protected.
4040 */
4041 mutex_lock(&iter->mutex);
ff98781b
EGM
4042 if (iter->trace->read) {
4043 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4044 if (sret)
4045 goto out;
4046 }
4047
4048waitagain:
4049 sret = tracing_wait_pipe(filp);
4050 if (sret <= 0)
4051 goto out;
4052
b3806b43 4053 /* stop when tracing is finished */
ff98781b
EGM
4054 if (trace_empty(iter)) {
4055 sret = 0;
107bad8b 4056 goto out;
ff98781b 4057 }
b3806b43
SR
4058
4059 if (cnt >= PAGE_SIZE)
4060 cnt = PAGE_SIZE - 1;
4061
53d0aa77 4062 /* reset all but tr, trace, and overruns */
53d0aa77
SR
4063 memset(&iter->seq, 0,
4064 sizeof(struct trace_iterator) -
4065 offsetof(struct trace_iterator, seq));
4823ed7e 4066 iter->pos = -1;
b3806b43 4067
4f535968 4068 trace_event_read_lock();
7e53bd42 4069 trace_access_lock(iter->cpu_file);
955b61e5 4070 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 4071 enum print_line_t ret;
088b1e42
SR
4072 int len = iter->seq.len;
4073
f9896bf3 4074 ret = print_trace_line(iter);
2c4f035f 4075 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
4076 /* don't print partial lines */
4077 iter->seq.len = len;
b3806b43 4078 break;
088b1e42 4079 }
b91facc3
FW
4080 if (ret != TRACE_TYPE_NO_CONSUME)
4081 trace_consume(iter);
b3806b43
SR
4082
4083 if (iter->seq.len >= cnt)
4084 break;
ee5e51f5
JO
4085
4086 /*
4087 * Setting the full flag means we reached the trace_seq buffer
4088 * size and we should leave by partial output condition above.
4089 * One of the trace_seq_* functions is not used properly.
4090 */
4091 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4092 iter->ent->type);
b3806b43 4093 }
7e53bd42 4094 trace_access_unlock(iter->cpu_file);
4f535968 4095 trace_event_read_unlock();
b3806b43 4096
b3806b43 4097 /* Now copy what we have to the user */
6c6c2796
PP
4098 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4099 if (iter->seq.readpos >= iter->seq.len)
f9520750 4100 trace_seq_init(&iter->seq);
9ff4b974
PP
4101
4102 /*
25985edc 4103 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
4104 * entries, go back to wait for more entries.
4105 */
6c6c2796 4106 if (sret == -EBUSY)
9ff4b974 4107 goto waitagain;
b3806b43 4108
107bad8b 4109out:
d7350c3f 4110 mutex_unlock(&iter->mutex);
107bad8b 4111
6c6c2796 4112 return sret;
b3806b43
SR
4113}
4114
3c56819b
EGM
4115static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4116 struct pipe_buffer *buf)
4117{
4118 __free_page(buf->page);
4119}
4120
4121static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4122 unsigned int idx)
4123{
4124 __free_page(spd->pages[idx]);
4125}
4126
28dfef8f 4127static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998
SR
4128 .can_merge = 0,
4129 .map = generic_pipe_buf_map,
4130 .unmap = generic_pipe_buf_unmap,
4131 .confirm = generic_pipe_buf_confirm,
4132 .release = tracing_pipe_buf_release,
4133 .steal = generic_pipe_buf_steal,
4134 .get = generic_pipe_buf_get,
3c56819b
EGM
4135};
4136
34cd4998 4137static size_t
fa7c7f6e 4138tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
4139{
4140 size_t count;
4141 int ret;
4142
4143 /* Seq buffer is page-sized, exactly what we need. */
4144 for (;;) {
4145 count = iter->seq.len;
4146 ret = print_trace_line(iter);
4147 count = iter->seq.len - count;
4148 if (rem < count) {
4149 rem = 0;
4150 iter->seq.len -= count;
4151 break;
4152 }
4153 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4154 iter->seq.len -= count;
4155 break;
4156 }
4157
74e7ff8c
LJ
4158 if (ret != TRACE_TYPE_NO_CONSUME)
4159 trace_consume(iter);
34cd4998 4160 rem -= count;
955b61e5 4161 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
4162 rem = 0;
4163 iter->ent = NULL;
4164 break;
4165 }
4166 }
4167
4168 return rem;
4169}
4170
3c56819b
EGM
4171static ssize_t tracing_splice_read_pipe(struct file *filp,
4172 loff_t *ppos,
4173 struct pipe_inode_info *pipe,
4174 size_t len,
4175 unsigned int flags)
4176{
35f3d14d
JA
4177 struct page *pages_def[PIPE_DEF_BUFFERS];
4178 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
4179 struct trace_iterator *iter = filp->private_data;
4180 struct splice_pipe_desc spd = {
35f3d14d
JA
4181 .pages = pages_def,
4182 .partial = partial_def,
34cd4998 4183 .nr_pages = 0, /* This gets updated below. */
047fe360 4184 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
4185 .flags = flags,
4186 .ops = &tracing_pipe_buf_ops,
4187 .spd_release = tracing_spd_release_pipe,
3c56819b 4188 };
2b6080f2 4189 struct trace_array *tr = iter->tr;
3c56819b 4190 ssize_t ret;
34cd4998 4191 size_t rem;
3c56819b
EGM
4192 unsigned int i;
4193
35f3d14d
JA
4194 if (splice_grow_spd(pipe, &spd))
4195 return -ENOMEM;
4196
d7350c3f 4197 /* copy the tracer to avoid using a global lock all around */
3c56819b 4198 mutex_lock(&trace_types_lock);
2b6080f2
SR
4199 if (unlikely(iter->trace->name != tr->current_trace->name))
4200 *iter->trace = *tr->current_trace;
d7350c3f
FW
4201 mutex_unlock(&trace_types_lock);
4202
4203 mutex_lock(&iter->mutex);
3c56819b
EGM
4204
4205 if (iter->trace->splice_read) {
4206 ret = iter->trace->splice_read(iter, filp,
4207 ppos, pipe, len, flags);
4208 if (ret)
34cd4998 4209 goto out_err;
3c56819b
EGM
4210 }
4211
4212 ret = tracing_wait_pipe(filp);
4213 if (ret <= 0)
34cd4998 4214 goto out_err;
3c56819b 4215
955b61e5 4216 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 4217 ret = -EFAULT;
34cd4998 4218 goto out_err;
3c56819b
EGM
4219 }
4220
4f535968 4221 trace_event_read_lock();
7e53bd42 4222 trace_access_lock(iter->cpu_file);
4f535968 4223
3c56819b 4224 /* Fill as many pages as possible. */
35f3d14d
JA
4225 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4226 spd.pages[i] = alloc_page(GFP_KERNEL);
4227 if (!spd.pages[i])
34cd4998 4228 break;
3c56819b 4229
fa7c7f6e 4230 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
4231
4232 /* Copy the data into the page, so we can start over. */
4233 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 4234 page_address(spd.pages[i]),
3c56819b
EGM
4235 iter->seq.len);
4236 if (ret < 0) {
35f3d14d 4237 __free_page(spd.pages[i]);
3c56819b
EGM
4238 break;
4239 }
35f3d14d
JA
4240 spd.partial[i].offset = 0;
4241 spd.partial[i].len = iter->seq.len;
3c56819b 4242
f9520750 4243 trace_seq_init(&iter->seq);
3c56819b
EGM
4244 }
4245
7e53bd42 4246 trace_access_unlock(iter->cpu_file);
4f535968 4247 trace_event_read_unlock();
d7350c3f 4248 mutex_unlock(&iter->mutex);
3c56819b
EGM
4249
4250 spd.nr_pages = i;
4251
35f3d14d
JA
4252 ret = splice_to_pipe(pipe, &spd);
4253out:
047fe360 4254 splice_shrink_spd(&spd);
35f3d14d 4255 return ret;
3c56819b 4256
34cd4998 4257out_err:
d7350c3f 4258 mutex_unlock(&iter->mutex);
35f3d14d 4259 goto out;
3c56819b
EGM
4260}
4261
a98a3c3f
SR
4262static ssize_t
4263tracing_entries_read(struct file *filp, char __user *ubuf,
4264 size_t cnt, loff_t *ppos)
4265{
2b6080f2
SR
4266 struct trace_cpu *tc = filp->private_data;
4267 struct trace_array *tr = tc->tr;
438ced17
VN
4268 char buf[64];
4269 int r = 0;
4270 ssize_t ret;
a98a3c3f 4271
db526ca3 4272 mutex_lock(&trace_types_lock);
438ced17 4273
2b6080f2 4274 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
4275 int cpu, buf_size_same;
4276 unsigned long size;
4277
4278 size = 0;
4279 buf_size_same = 1;
4280 /* check if all cpu sizes are same */
4281 for_each_tracing_cpu(cpu) {
4282 /* fill in the size from first enabled cpu */
4283 if (size == 0)
12883efb
SRRH
4284 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4285 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
4286 buf_size_same = 0;
4287 break;
4288 }
4289 }
4290
4291 if (buf_size_same) {
4292 if (!ring_buffer_expanded)
4293 r = sprintf(buf, "%lu (expanded: %lu)\n",
4294 size >> 10,
4295 trace_buf_size >> 10);
4296 else
4297 r = sprintf(buf, "%lu\n", size >> 10);
4298 } else
4299 r = sprintf(buf, "X\n");
4300 } else
12883efb 4301 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
438ced17 4302
db526ca3
SR
4303 mutex_unlock(&trace_types_lock);
4304
438ced17
VN
4305 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4306 return ret;
a98a3c3f
SR
4307}
4308
4309static ssize_t
4310tracing_entries_write(struct file *filp, const char __user *ubuf,
4311 size_t cnt, loff_t *ppos)
4312{
2b6080f2 4313 struct trace_cpu *tc = filp->private_data;
a98a3c3f 4314 unsigned long val;
4f271a2a 4315 int ret;
a98a3c3f 4316
22fe9b54
PH
4317 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4318 if (ret)
c6caeeb1 4319 return ret;
a98a3c3f
SR
4320
4321 /* must have at least 1 entry */
4322 if (!val)
4323 return -EINVAL;
4324
1696b2b0
SR
4325 /* value is in KB */
4326 val <<= 10;
4327
2b6080f2 4328 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4f271a2a
VN
4329 if (ret < 0)
4330 return ret;
a98a3c3f 4331
cf8517cf 4332 *ppos += cnt;
a98a3c3f 4333
4f271a2a
VN
4334 return cnt;
4335}
bf5e6519 4336
f81ab074
VN
4337static ssize_t
4338tracing_total_entries_read(struct file *filp, char __user *ubuf,
4339 size_t cnt, loff_t *ppos)
4340{
4341 struct trace_array *tr = filp->private_data;
4342 char buf[64];
4343 int r, cpu;
4344 unsigned long size = 0, expanded_size = 0;
4345
4346 mutex_lock(&trace_types_lock);
4347 for_each_tracing_cpu(cpu) {
12883efb 4348 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
4349 if (!ring_buffer_expanded)
4350 expanded_size += trace_buf_size >> 10;
4351 }
4352 if (ring_buffer_expanded)
4353 r = sprintf(buf, "%lu\n", size);
4354 else
4355 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4356 mutex_unlock(&trace_types_lock);
4357
4358 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4359}
4360
4f271a2a
VN
4361static ssize_t
4362tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4363 size_t cnt, loff_t *ppos)
4364{
4365 /*
4366 * There is no need to read what the user has written, this function
4367 * is just to make sure that there is no error when "echo" is used
4368 */
4369
4370 *ppos += cnt;
a98a3c3f
SR
4371
4372 return cnt;
4373}
4374
4f271a2a
VN
4375static int
4376tracing_free_buffer_release(struct inode *inode, struct file *filp)
4377{
2b6080f2
SR
4378 struct trace_array *tr = inode->i_private;
4379
cf30cf67
SR
4380 /* disable tracing ? */
4381 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4382 tracing_off();
4f271a2a 4383 /* resize the ring buffer to 0 */
2b6080f2 4384 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a
VN
4385
4386 return 0;
4387}
4388
5bf9a1ee
PP
4389static ssize_t
4390tracing_mark_write(struct file *filp, const char __user *ubuf,
4391 size_t cnt, loff_t *fpos)
4392{
d696b58c
SR
4393 unsigned long addr = (unsigned long)ubuf;
4394 struct ring_buffer_event *event;
4395 struct ring_buffer *buffer;
4396 struct print_entry *entry;
4397 unsigned long irq_flags;
4398 struct page *pages[2];
6edb2a8a 4399 void *map_page[2];
d696b58c
SR
4400 int nr_pages = 1;
4401 ssize_t written;
d696b58c
SR
4402 int offset;
4403 int size;
4404 int len;
4405 int ret;
6edb2a8a 4406 int i;
5bf9a1ee 4407
c76f0694 4408 if (tracing_disabled)
5bf9a1ee
PP
4409 return -EINVAL;
4410
5224c3a3
MSB
4411 if (!(trace_flags & TRACE_ITER_MARKERS))
4412 return -EINVAL;
4413
5bf9a1ee
PP
4414 if (cnt > TRACE_BUF_SIZE)
4415 cnt = TRACE_BUF_SIZE;
4416
d696b58c
SR
4417 /*
4418 * Userspace is injecting traces into the kernel trace buffer.
4419 * We want to be as non intrusive as possible.
4420 * To do so, we do not want to allocate any special buffers
4421 * or take any locks, but instead write the userspace data
4422 * straight into the ring buffer.
4423 *
4424 * First we need to pin the userspace buffer into memory,
4425 * which, most likely it is, because it just referenced it.
4426 * But there's no guarantee that it is. By using get_user_pages_fast()
4427 * and kmap_atomic/kunmap_atomic() we can get access to the
4428 * pages directly. We then write the data directly into the
4429 * ring buffer.
4430 */
4431 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 4432
d696b58c
SR
4433 /* check if we cross pages */
4434 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4435 nr_pages = 2;
4436
4437 offset = addr & (PAGE_SIZE - 1);
4438 addr &= PAGE_MASK;
4439
4440 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4441 if (ret < nr_pages) {
4442 while (--ret >= 0)
4443 put_page(pages[ret]);
4444 written = -EFAULT;
4445 goto out;
5bf9a1ee 4446 }
d696b58c 4447
6edb2a8a
SR
4448 for (i = 0; i < nr_pages; i++)
4449 map_page[i] = kmap_atomic(pages[i]);
d696b58c
SR
4450
4451 local_save_flags(irq_flags);
4452 size = sizeof(*entry) + cnt + 2; /* possible \n added */
12883efb 4453 buffer = global_trace.trace_buffer.buffer;
d696b58c
SR
4454 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4455 irq_flags, preempt_count());
4456 if (!event) {
4457 /* Ring buffer disabled, return as if not open for write */
4458 written = -EBADF;
4459 goto out_unlock;
5bf9a1ee 4460 }
d696b58c
SR
4461
4462 entry = ring_buffer_event_data(event);
4463 entry->ip = _THIS_IP_;
4464
4465 if (nr_pages == 2) {
4466 len = PAGE_SIZE - offset;
6edb2a8a
SR
4467 memcpy(&entry->buf, map_page[0] + offset, len);
4468 memcpy(&entry->buf[len], map_page[1], cnt - len);
c13d2f7c 4469 } else
6edb2a8a 4470 memcpy(&entry->buf, map_page[0] + offset, cnt);
5bf9a1ee 4471
d696b58c
SR
4472 if (entry->buf[cnt - 1] != '\n') {
4473 entry->buf[cnt] = '\n';
4474 entry->buf[cnt + 1] = '\0';
4475 } else
4476 entry->buf[cnt] = '\0';
4477
7ffbd48d 4478 __buffer_unlock_commit(buffer, event);
5bf9a1ee 4479
d696b58c 4480 written = cnt;
5bf9a1ee 4481
d696b58c 4482 *fpos += written;
1aa54bca 4483
d696b58c 4484 out_unlock:
6edb2a8a
SR
4485 for (i = 0; i < nr_pages; i++){
4486 kunmap_atomic(map_page[i]);
4487 put_page(pages[i]);
4488 }
d696b58c 4489 out:
1aa54bca 4490 return written;
5bf9a1ee
PP
4491}
4492
13f16d20 4493static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 4494{
2b6080f2 4495 struct trace_array *tr = m->private;
5079f326
Z
4496 int i;
4497
4498 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 4499 seq_printf(m,
5079f326 4500 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
4501 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4502 i == tr->clock_id ? "]" : "");
13f16d20 4503 seq_putc(m, '\n');
5079f326 4504
13f16d20 4505 return 0;
5079f326
Z
4506}
4507
4508static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4509 size_t cnt, loff_t *fpos)
4510{
2b6080f2
SR
4511 struct seq_file *m = filp->private_data;
4512 struct trace_array *tr = m->private;
5079f326
Z
4513 char buf[64];
4514 const char *clockstr;
4515 int i;
4516
4517 if (cnt >= sizeof(buf))
4518 return -EINVAL;
4519
4520 if (copy_from_user(&buf, ubuf, cnt))
4521 return -EFAULT;
4522
4523 buf[cnt] = 0;
4524
4525 clockstr = strstrip(buf);
4526
4527 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4528 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4529 break;
4530 }
4531 if (i == ARRAY_SIZE(trace_clocks))
4532 return -EINVAL;
4533
5079f326
Z
4534 mutex_lock(&trace_types_lock);
4535
2b6080f2
SR
4536 tr->clock_id = i;
4537
12883efb 4538 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 4539
60303ed3
DS
4540 /*
4541 * New clock may not be consistent with the previous clock.
4542 * Reset the buffer so that it doesn't have incomparable timestamps.
4543 */
12883efb
SRRH
4544 tracing_reset_online_cpus(&global_trace.trace_buffer);
4545
4546#ifdef CONFIG_TRACER_MAX_TRACE
4547 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4548 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4549 tracing_reset_online_cpus(&global_trace.max_buffer);
4550#endif
60303ed3 4551
5079f326
Z
4552 mutex_unlock(&trace_types_lock);
4553
4554 *fpos += cnt;
4555
4556 return cnt;
4557}
4558
13f16d20
LZ
4559static int tracing_clock_open(struct inode *inode, struct file *file)
4560{
4561 if (tracing_disabled)
4562 return -ENODEV;
2b6080f2
SR
4563
4564 return single_open(file, tracing_clock_show, inode->i_private);
13f16d20
LZ
4565}
4566
6de58e62
SRRH
4567struct ftrace_buffer_info {
4568 struct trace_iterator iter;
4569 void *spare;
4570 unsigned int read;
4571};
4572
debdd57f
HT
4573#ifdef CONFIG_TRACER_SNAPSHOT
4574static int tracing_snapshot_open(struct inode *inode, struct file *file)
4575{
2b6080f2 4576 struct trace_cpu *tc = inode->i_private;
debdd57f 4577 struct trace_iterator *iter;
2b6080f2 4578 struct seq_file *m;
debdd57f
HT
4579 int ret = 0;
4580
4581 if (file->f_mode & FMODE_READ) {
4582 iter = __tracing_open(inode, file, true);
4583 if (IS_ERR(iter))
4584 ret = PTR_ERR(iter);
2b6080f2
SR
4585 } else {
4586 /* Writes still need the seq_file to hold the private data */
4587 m = kzalloc(sizeof(*m), GFP_KERNEL);
4588 if (!m)
4589 return -ENOMEM;
4590 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4591 if (!iter) {
4592 kfree(m);
4593 return -ENOMEM;
4594 }
4595 iter->tr = tc->tr;
12883efb 4596 iter->trace_buffer = &tc->tr->max_buffer;
f1affcaa 4597 iter->cpu_file = tc->cpu;
2b6080f2
SR
4598 m->private = iter;
4599 file->private_data = m;
debdd57f 4600 }
2b6080f2 4601
debdd57f
HT
4602 return ret;
4603}
4604
4605static ssize_t
4606tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4607 loff_t *ppos)
4608{
2b6080f2
SR
4609 struct seq_file *m = filp->private_data;
4610 struct trace_iterator *iter = m->private;
4611 struct trace_array *tr = iter->tr;
debdd57f
HT
4612 unsigned long val;
4613 int ret;
4614
4615 ret = tracing_update_buffers();
4616 if (ret < 0)
4617 return ret;
4618
4619 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4620 if (ret)
4621 return ret;
4622
4623 mutex_lock(&trace_types_lock);
4624
2b6080f2 4625 if (tr->current_trace->use_max_tr) {
debdd57f
HT
4626 ret = -EBUSY;
4627 goto out;
4628 }
4629
4630 switch (val) {
4631 case 0:
f1affcaa
SRRH
4632 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4633 ret = -EINVAL;
4634 break;
debdd57f 4635 }
3209cff4
SRRH
4636 if (tr->allocated_snapshot)
4637 free_snapshot(tr);
debdd57f
HT
4638 break;
4639 case 1:
f1affcaa
SRRH
4640/* Only allow per-cpu swap if the ring buffer supports it */
4641#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4642 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4643 ret = -EINVAL;
4644 break;
4645 }
4646#endif
45ad21ca 4647 if (!tr->allocated_snapshot) {
3209cff4 4648 ret = alloc_snapshot(tr);
debdd57f
HT
4649 if (ret < 0)
4650 break;
debdd57f 4651 }
debdd57f
HT
4652 local_irq_disable();
4653 /* Now, we're going to swap */
f1affcaa 4654 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
ce9bae55 4655 update_max_tr(tr, current, smp_processor_id());
f1affcaa 4656 else
ce9bae55 4657 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
4658 local_irq_enable();
4659 break;
4660 default:
45ad21ca 4661 if (tr->allocated_snapshot) {
f1affcaa
SRRH
4662 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4663 tracing_reset_online_cpus(&tr->max_buffer);
4664 else
4665 tracing_reset(&tr->max_buffer, iter->cpu_file);
4666 }
debdd57f
HT
4667 break;
4668 }
4669
4670 if (ret >= 0) {
4671 *ppos += cnt;
4672 ret = cnt;
4673 }
4674out:
4675 mutex_unlock(&trace_types_lock);
4676 return ret;
4677}
2b6080f2
SR
4678
4679static int tracing_snapshot_release(struct inode *inode, struct file *file)
4680{
4681 struct seq_file *m = file->private_data;
4682
4683 if (file->f_mode & FMODE_READ)
4684 return tracing_release(inode, file);
4685
4686 /* If write only, the seq_file is just a stub */
4687 if (m)
4688 kfree(m->private);
4689 kfree(m);
4690
4691 return 0;
4692}
4693
6de58e62
SRRH
4694static int tracing_buffers_open(struct inode *inode, struct file *filp);
4695static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4696 size_t count, loff_t *ppos);
4697static int tracing_buffers_release(struct inode *inode, struct file *file);
4698static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4699 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4700
4701static int snapshot_raw_open(struct inode *inode, struct file *filp)
4702{
4703 struct ftrace_buffer_info *info;
4704 int ret;
4705
4706 ret = tracing_buffers_open(inode, filp);
4707 if (ret < 0)
4708 return ret;
4709
4710 info = filp->private_data;
4711
4712 if (info->iter.trace->use_max_tr) {
4713 tracing_buffers_release(inode, filp);
4714 return -EBUSY;
4715 }
4716
4717 info->iter.snapshot = true;
4718 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4719
4720 return ret;
4721}
4722
debdd57f
HT
4723#endif /* CONFIG_TRACER_SNAPSHOT */
4724
4725
5e2336a0 4726static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
4727 .open = tracing_open_generic,
4728 .read = tracing_max_lat_read,
4729 .write = tracing_max_lat_write,
b444786f 4730 .llseek = generic_file_llseek,
bc0c38d1
SR
4731};
4732
5e2336a0 4733static const struct file_operations set_tracer_fops = {
4bf39a94
IM
4734 .open = tracing_open_generic,
4735 .read = tracing_set_trace_read,
4736 .write = tracing_set_trace_write,
b444786f 4737 .llseek = generic_file_llseek,
bc0c38d1
SR
4738};
4739
5e2336a0 4740static const struct file_operations tracing_pipe_fops = {
4bf39a94 4741 .open = tracing_open_pipe,
2a2cc8f7 4742 .poll = tracing_poll_pipe,
4bf39a94 4743 .read = tracing_read_pipe,
3c56819b 4744 .splice_read = tracing_splice_read_pipe,
4bf39a94 4745 .release = tracing_release_pipe,
b444786f 4746 .llseek = no_llseek,
b3806b43
SR
4747};
4748
5e2336a0 4749static const struct file_operations tracing_entries_fops = {
2b6080f2 4750 .open = tracing_open_generic,
a98a3c3f
SR
4751 .read = tracing_entries_read,
4752 .write = tracing_entries_write,
b444786f 4753 .llseek = generic_file_llseek,
a98a3c3f
SR
4754};
4755
f81ab074
VN
4756static const struct file_operations tracing_total_entries_fops = {
4757 .open = tracing_open_generic,
4758 .read = tracing_total_entries_read,
4759 .llseek = generic_file_llseek,
4760};
4761
4f271a2a
VN
4762static const struct file_operations tracing_free_buffer_fops = {
4763 .write = tracing_free_buffer_write,
4764 .release = tracing_free_buffer_release,
4765};
4766
5e2336a0 4767static const struct file_operations tracing_mark_fops = {
43a15386 4768 .open = tracing_open_generic,
5bf9a1ee 4769 .write = tracing_mark_write,
b444786f 4770 .llseek = generic_file_llseek,
5bf9a1ee
PP
4771};
4772
5079f326 4773static const struct file_operations trace_clock_fops = {
13f16d20
LZ
4774 .open = tracing_clock_open,
4775 .read = seq_read,
4776 .llseek = seq_lseek,
4777 .release = single_release,
5079f326
Z
4778 .write = tracing_clock_write,
4779};
4780
debdd57f
HT
4781#ifdef CONFIG_TRACER_SNAPSHOT
4782static const struct file_operations snapshot_fops = {
4783 .open = tracing_snapshot_open,
4784 .read = seq_read,
4785 .write = tracing_snapshot_write,
4786 .llseek = tracing_seek,
2b6080f2 4787 .release = tracing_snapshot_release,
debdd57f 4788};
debdd57f 4789
6de58e62
SRRH
4790static const struct file_operations snapshot_raw_fops = {
4791 .open = snapshot_raw_open,
4792 .read = tracing_buffers_read,
4793 .release = tracing_buffers_release,
4794 .splice_read = tracing_buffers_splice_read,
4795 .llseek = no_llseek,
2cadf913
SR
4796};
4797
6de58e62
SRRH
4798#endif /* CONFIG_TRACER_SNAPSHOT */
4799
2cadf913
SR
4800static int tracing_buffers_open(struct inode *inode, struct file *filp)
4801{
2b6080f2
SR
4802 struct trace_cpu *tc = inode->i_private;
4803 struct trace_array *tr = tc->tr;
2cadf913
SR
4804 struct ftrace_buffer_info *info;
4805
4806 if (tracing_disabled)
4807 return -ENODEV;
4808
4809 info = kzalloc(sizeof(*info), GFP_KERNEL);
4810 if (!info)
4811 return -ENOMEM;
4812
a695cb58
SRRH
4813 mutex_lock(&trace_types_lock);
4814
4815 tr->ref++;
4816
cc60cdc9
SR
4817 info->iter.tr = tr;
4818 info->iter.cpu_file = tc->cpu;
b627344f 4819 info->iter.trace = tr->current_trace;
12883efb 4820 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 4821 info->spare = NULL;
2cadf913 4822 /* Force reading ring buffer for first read */
cc60cdc9 4823 info->read = (unsigned int)-1;
2cadf913
SR
4824
4825 filp->private_data = info;
4826
a695cb58
SRRH
4827 mutex_unlock(&trace_types_lock);
4828
d1e7e02f 4829 return nonseekable_open(inode, filp);
2cadf913
SR
4830}
4831
cc60cdc9
SR
4832static unsigned int
4833tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4834{
4835 struct ftrace_buffer_info *info = filp->private_data;
4836 struct trace_iterator *iter = &info->iter;
4837
4838 return trace_poll(iter, filp, poll_table);
4839}
4840
2cadf913
SR
4841static ssize_t
4842tracing_buffers_read(struct file *filp, char __user *ubuf,
4843 size_t count, loff_t *ppos)
4844{
4845 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 4846 struct trace_iterator *iter = &info->iter;
2cadf913 4847 ssize_t ret;
6de58e62 4848 ssize_t size;
2cadf913 4849
2dc5d12b
SR
4850 if (!count)
4851 return 0;
4852
6de58e62
SRRH
4853 mutex_lock(&trace_types_lock);
4854
4855#ifdef CONFIG_TRACER_MAX_TRACE
4856 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4857 size = -EBUSY;
4858 goto out_unlock;
4859 }
4860#endif
4861
ddd538f3 4862 if (!info->spare)
12883efb
SRRH
4863 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4864 iter->cpu_file);
6de58e62 4865 size = -ENOMEM;
ddd538f3 4866 if (!info->spare)
6de58e62 4867 goto out_unlock;
ddd538f3 4868
2cadf913
SR
4869 /* Do we have previous read data to read? */
4870 if (info->read < PAGE_SIZE)
4871 goto read;
4872
b627344f 4873 again:
cc60cdc9 4874 trace_access_lock(iter->cpu_file);
12883efb 4875 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
4876 &info->spare,
4877 count,
cc60cdc9
SR
4878 iter->cpu_file, 0);
4879 trace_access_unlock(iter->cpu_file);
2cadf913 4880
b627344f
SR
4881 if (ret < 0) {
4882 if (trace_empty(iter)) {
6de58e62
SRRH
4883 if ((filp->f_flags & O_NONBLOCK)) {
4884 size = -EAGAIN;
4885 goto out_unlock;
4886 }
4887 mutex_unlock(&trace_types_lock);
b627344f 4888 iter->trace->wait_pipe(iter);
6de58e62
SRRH
4889 mutex_lock(&trace_types_lock);
4890 if (signal_pending(current)) {
4891 size = -EINTR;
4892 goto out_unlock;
4893 }
b627344f
SR
4894 goto again;
4895 }
6de58e62
SRRH
4896 size = 0;
4897 goto out_unlock;
b627344f 4898 }
436fc280 4899
436fc280 4900 info->read = 0;
b627344f 4901 read:
2cadf913
SR
4902 size = PAGE_SIZE - info->read;
4903 if (size > count)
4904 size = count;
4905
4906 ret = copy_to_user(ubuf, info->spare + info->read, size);
6de58e62
SRRH
4907 if (ret == size) {
4908 size = -EFAULT;
4909 goto out_unlock;
4910 }
2dc5d12b
SR
4911 size -= ret;
4912
2cadf913
SR
4913 *ppos += size;
4914 info->read += size;
4915
6de58e62
SRRH
4916 out_unlock:
4917 mutex_unlock(&trace_types_lock);
4918
2cadf913
SR
4919 return size;
4920}
4921
4922static int tracing_buffers_release(struct inode *inode, struct file *file)
4923{
4924 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 4925 struct trace_iterator *iter = &info->iter;
2cadf913 4926
a695cb58
SRRH
4927 mutex_lock(&trace_types_lock);
4928
4929 WARN_ON(!iter->tr->ref);
4930 iter->tr->ref--;
2cadf913 4931
ddd538f3 4932 if (info->spare)
12883efb 4933 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
2cadf913
SR
4934 kfree(info);
4935
a695cb58
SRRH
4936 mutex_unlock(&trace_types_lock);
4937
2cadf913
SR
4938 return 0;
4939}
4940
4941struct buffer_ref {
4942 struct ring_buffer *buffer;
4943 void *page;
4944 int ref;
4945};
4946
4947static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4948 struct pipe_buffer *buf)
4949{
4950 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4951
4952 if (--ref->ref)
4953 return;
4954
4955 ring_buffer_free_read_page(ref->buffer, ref->page);
4956 kfree(ref);
4957 buf->private = 0;
4958}
4959
2cadf913
SR
4960static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4961 struct pipe_buffer *buf)
4962{
4963 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4964
4965 ref->ref++;
4966}
4967
4968/* Pipe buffer operations for a buffer. */
28dfef8f 4969static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913
SR
4970 .can_merge = 0,
4971 .map = generic_pipe_buf_map,
4972 .unmap = generic_pipe_buf_unmap,
4973 .confirm = generic_pipe_buf_confirm,
4974 .release = buffer_pipe_buf_release,
d55cb6cf 4975 .steal = generic_pipe_buf_steal,
2cadf913
SR
4976 .get = buffer_pipe_buf_get,
4977};
4978
4979/*
4980 * Callback from splice_to_pipe(), if we need to release some pages
4981 * at the end of the spd in case we error'ed out in filling the pipe.
4982 */
4983static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4984{
4985 struct buffer_ref *ref =
4986 (struct buffer_ref *)spd->partial[i].private;
4987
4988 if (--ref->ref)
4989 return;
4990
4991 ring_buffer_free_read_page(ref->buffer, ref->page);
4992 kfree(ref);
4993 spd->partial[i].private = 0;
4994}
4995
4996static ssize_t
4997tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4998 struct pipe_inode_info *pipe, size_t len,
4999 unsigned int flags)
5000{
5001 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 5002 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
5003 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5004 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 5005 struct splice_pipe_desc spd = {
35f3d14d
JA
5006 .pages = pages_def,
5007 .partial = partial_def,
047fe360 5008 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
5009 .flags = flags,
5010 .ops = &buffer_pipe_buf_ops,
5011 .spd_release = buffer_spd_release,
5012 };
5013 struct buffer_ref *ref;
93459c6c 5014 int entries, size, i;
6de58e62 5015 ssize_t ret;
2cadf913 5016
6de58e62
SRRH
5017 mutex_lock(&trace_types_lock);
5018
5019#ifdef CONFIG_TRACER_MAX_TRACE
5020 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5021 ret = -EBUSY;
5022 goto out;
5023 }
5024#endif
5025
5026 if (splice_grow_spd(pipe, &spd)) {
5027 ret = -ENOMEM;
5028 goto out;
5029 }
35f3d14d 5030
93cfb3c9 5031 if (*ppos & (PAGE_SIZE - 1)) {
35f3d14d
JA
5032 ret = -EINVAL;
5033 goto out;
93cfb3c9
LJ
5034 }
5035
5036 if (len & (PAGE_SIZE - 1)) {
35f3d14d
JA
5037 if (len < PAGE_SIZE) {
5038 ret = -EINVAL;
5039 goto out;
5040 }
93cfb3c9
LJ
5041 len &= PAGE_MASK;
5042 }
5043
cc60cdc9
SR
5044 again:
5045 trace_access_lock(iter->cpu_file);
12883efb 5046 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 5047
35f3d14d 5048 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
5049 struct page *page;
5050 int r;
5051
5052 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5053 if (!ref)
5054 break;
5055
7267fa68 5056 ref->ref = 1;
12883efb 5057 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 5058 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
2cadf913
SR
5059 if (!ref->page) {
5060 kfree(ref);
5061 break;
5062 }
5063
5064 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 5065 len, iter->cpu_file, 1);
2cadf913 5066 if (r < 0) {
7ea59064 5067 ring_buffer_free_read_page(ref->buffer, ref->page);
2cadf913
SR
5068 kfree(ref);
5069 break;
5070 }
5071
5072 /*
5073 * zero out any left over data, this is going to
5074 * user land.
5075 */
5076 size = ring_buffer_page_len(ref->page);
5077 if (size < PAGE_SIZE)
5078 memset(ref->page + size, 0, PAGE_SIZE - size);
5079
5080 page = virt_to_page(ref->page);
5081
5082 spd.pages[i] = page;
5083 spd.partial[i].len = PAGE_SIZE;
5084 spd.partial[i].offset = 0;
5085 spd.partial[i].private = (unsigned long)ref;
5086 spd.nr_pages++;
93cfb3c9 5087 *ppos += PAGE_SIZE;
93459c6c 5088
12883efb 5089 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
5090 }
5091
cc60cdc9 5092 trace_access_unlock(iter->cpu_file);
2cadf913
SR
5093 spd.nr_pages = i;
5094
5095 /* did we read anything? */
5096 if (!spd.nr_pages) {
cc60cdc9 5097 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
2cadf913 5098 ret = -EAGAIN;
cc60cdc9
SR
5099 goto out;
5100 }
6de58e62 5101 mutex_unlock(&trace_types_lock);
b627344f 5102 iter->trace->wait_pipe(iter);
6de58e62 5103 mutex_lock(&trace_types_lock);
cc60cdc9
SR
5104 if (signal_pending(current)) {
5105 ret = -EINTR;
5106 goto out;
5107 }
5108 goto again;
2cadf913
SR
5109 }
5110
5111 ret = splice_to_pipe(pipe, &spd);
047fe360 5112 splice_shrink_spd(&spd);
35f3d14d 5113out:
6de58e62
SRRH
5114 mutex_unlock(&trace_types_lock);
5115
2cadf913
SR
5116 return ret;
5117}
5118
5119static const struct file_operations tracing_buffers_fops = {
5120 .open = tracing_buffers_open,
5121 .read = tracing_buffers_read,
cc60cdc9 5122 .poll = tracing_buffers_poll,
2cadf913
SR
5123 .release = tracing_buffers_release,
5124 .splice_read = tracing_buffers_splice_read,
5125 .llseek = no_llseek,
5126};
5127
c8d77183
SR
5128static ssize_t
5129tracing_stats_read(struct file *filp, char __user *ubuf,
5130 size_t count, loff_t *ppos)
5131{
2b6080f2
SR
5132 struct trace_cpu *tc = filp->private_data;
5133 struct trace_array *tr = tc->tr;
12883efb 5134 struct trace_buffer *trace_buf = &tr->trace_buffer;
c8d77183
SR
5135 struct trace_seq *s;
5136 unsigned long cnt;
c64e148a
VN
5137 unsigned long long t;
5138 unsigned long usec_rem;
2b6080f2 5139 int cpu = tc->cpu;
c8d77183 5140
e4f2d10f 5141 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 5142 if (!s)
a646365c 5143 return -ENOMEM;
c8d77183
SR
5144
5145 trace_seq_init(s);
5146
12883efb 5147 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5148 trace_seq_printf(s, "entries: %ld\n", cnt);
5149
12883efb 5150 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5151 trace_seq_printf(s, "overrun: %ld\n", cnt);
5152
12883efb 5153 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5154 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5155
12883efb 5156 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
5157 trace_seq_printf(s, "bytes: %ld\n", cnt);
5158
11043d8b
YY
5159 if (trace_clocks[trace_clock_id].in_ns) {
5160 /* local or global for trace_clock */
12883efb 5161 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
5162 usec_rem = do_div(t, USEC_PER_SEC);
5163 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5164 t, usec_rem);
5165
12883efb 5166 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
5167 usec_rem = do_div(t, USEC_PER_SEC);
5168 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5169 } else {
5170 /* counter or tsc mode for trace_clock */
5171 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 5172 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 5173
11043d8b 5174 trace_seq_printf(s, "now ts: %llu\n",
12883efb 5175 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 5176 }
c64e148a 5177
12883efb 5178 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
5179 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5180
12883efb 5181 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
5182 trace_seq_printf(s, "read events: %ld\n", cnt);
5183
c8d77183
SR
5184 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5185
5186 kfree(s);
5187
5188 return count;
5189}
5190
5191static const struct file_operations tracing_stats_fops = {
5192 .open = tracing_open_generic,
5193 .read = tracing_stats_read,
b444786f 5194 .llseek = generic_file_llseek,
c8d77183
SR
5195};
5196
bc0c38d1
SR
5197#ifdef CONFIG_DYNAMIC_FTRACE
5198
b807c3d0
SR
5199int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5200{
5201 return 0;
5202}
5203
bc0c38d1 5204static ssize_t
b807c3d0 5205tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
5206 size_t cnt, loff_t *ppos)
5207{
a26a2a27
SR
5208 static char ftrace_dyn_info_buffer[1024];
5209 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 5210 unsigned long *p = filp->private_data;
b807c3d0 5211 char *buf = ftrace_dyn_info_buffer;
a26a2a27 5212 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
5213 int r;
5214
b807c3d0
SR
5215 mutex_lock(&dyn_info_mutex);
5216 r = sprintf(buf, "%ld ", *p);
4bf39a94 5217
a26a2a27 5218 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
5219 buf[r++] = '\n';
5220
5221 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5222
5223 mutex_unlock(&dyn_info_mutex);
5224
5225 return r;
bc0c38d1
SR
5226}
5227
5e2336a0 5228static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 5229 .open = tracing_open_generic,
b807c3d0 5230 .read = tracing_read_dyn_info,
b444786f 5231 .llseek = generic_file_llseek,
bc0c38d1 5232};
77fd5c15 5233#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 5234
77fd5c15
SRRH
5235#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5236static void
5237ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5238{
5239 tracing_snapshot();
5240}
bc0c38d1 5241
77fd5c15
SRRH
5242static void
5243ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
bc0c38d1 5244{
77fd5c15
SRRH
5245 unsigned long *count = (long *)data;
5246
5247 if (!*count)
5248 return;
bc0c38d1 5249
77fd5c15
SRRH
5250 if (*count != -1)
5251 (*count)--;
5252
5253 tracing_snapshot();
5254}
5255
5256static int
5257ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5258 struct ftrace_probe_ops *ops, void *data)
5259{
5260 long count = (long)data;
5261
5262 seq_printf(m, "%ps:", (void *)ip);
5263
5264 seq_printf(m, "snapshot");
5265
5266 if (count == -1)
5267 seq_printf(m, ":unlimited\n");
5268 else
5269 seq_printf(m, ":count=%ld\n", count);
5270
5271 return 0;
5272}
5273
5274static struct ftrace_probe_ops snapshot_probe_ops = {
5275 .func = ftrace_snapshot,
5276 .print = ftrace_snapshot_print,
5277};
5278
5279static struct ftrace_probe_ops snapshot_count_probe_ops = {
5280 .func = ftrace_count_snapshot,
5281 .print = ftrace_snapshot_print,
5282};
5283
5284static int
5285ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5286 char *glob, char *cmd, char *param, int enable)
5287{
5288 struct ftrace_probe_ops *ops;
5289 void *count = (void *)-1;
5290 char *number;
5291 int ret;
5292
5293 /* hash funcs only work with set_ftrace_filter */
5294 if (!enable)
5295 return -EINVAL;
5296
5297 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5298
5299 if (glob[0] == '!') {
5300 unregister_ftrace_function_probe_func(glob+1, ops);
5301 return 0;
5302 }
5303
5304 if (!param)
5305 goto out_reg;
5306
5307 number = strsep(&param, ":");
5308
5309 if (!strlen(number))
5310 goto out_reg;
5311
5312 /*
5313 * We use the callback data field (which is a pointer)
5314 * as our counter.
5315 */
5316 ret = kstrtoul(number, 0, (unsigned long *)&count);
5317 if (ret)
5318 return ret;
5319
5320 out_reg:
5321 ret = register_ftrace_function_probe(glob, ops, count);
5322
5323 if (ret >= 0)
5324 alloc_snapshot(&global_trace);
5325
5326 return ret < 0 ? ret : 0;
5327}
5328
5329static struct ftrace_func_command ftrace_snapshot_cmd = {
5330 .name = "snapshot",
5331 .func = ftrace_trace_snapshot_callback,
5332};
5333
5334static int register_snapshot_cmd(void)
5335{
5336 return register_ftrace_command(&ftrace_snapshot_cmd);
5337}
5338#else
5339static inline int register_snapshot_cmd(void) { return 0; }
5340#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 5341
2b6080f2 5342struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
bc0c38d1 5343{
2b6080f2
SR
5344 if (tr->dir)
5345 return tr->dir;
bc0c38d1 5346
3e1f60b8
FW
5347 if (!debugfs_initialized())
5348 return NULL;
5349
2b6080f2
SR
5350 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5351 tr->dir = debugfs_create_dir("tracing", NULL);
bc0c38d1 5352
687c878a
J
5353 if (!tr->dir)
5354 pr_warn_once("Could not create debugfs directory 'tracing'\n");
bc0c38d1 5355
2b6080f2 5356 return tr->dir;
bc0c38d1
SR
5357}
5358
2b6080f2
SR
5359struct dentry *tracing_init_dentry(void)
5360{
5361 return tracing_init_dentry_tr(&global_trace);
5362}
b04cc6b1 5363
2b6080f2 5364static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 5365{
b04cc6b1
FW
5366 struct dentry *d_tracer;
5367
2b6080f2
SR
5368 if (tr->percpu_dir)
5369 return tr->percpu_dir;
b04cc6b1 5370
2b6080f2 5371 d_tracer = tracing_init_dentry_tr(tr);
b04cc6b1
FW
5372 if (!d_tracer)
5373 return NULL;
5374
2b6080f2 5375 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
b04cc6b1 5376
2b6080f2
SR
5377 WARN_ONCE(!tr->percpu_dir,
5378 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 5379
2b6080f2 5380 return tr->percpu_dir;
b04cc6b1
FW
5381}
5382
2b6080f2
SR
5383static void
5384tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 5385{
12883efb 5386 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
2b6080f2 5387 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 5388 struct dentry *d_cpu;
dd49a38c 5389 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 5390
0a3d7ce7
NK
5391 if (!d_percpu)
5392 return;
5393
dd49a38c 5394 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8656e7a2
FW
5395 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5396 if (!d_cpu) {
5397 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5398 return;
5399 }
b04cc6b1 5400
8656e7a2 5401 /* per cpu trace_pipe */
5452af66 5402 trace_create_file("trace_pipe", 0444, d_cpu,
2b6080f2 5403 (void *)&data->trace_cpu, &tracing_pipe_fops);
b04cc6b1
FW
5404
5405 /* per cpu trace */
5452af66 5406 trace_create_file("trace", 0644, d_cpu,
2b6080f2 5407 (void *)&data->trace_cpu, &tracing_fops);
7f96f93f 5408
5452af66 5409 trace_create_file("trace_pipe_raw", 0444, d_cpu,
2b6080f2 5410 (void *)&data->trace_cpu, &tracing_buffers_fops);
7f96f93f 5411
c8d77183 5412 trace_create_file("stats", 0444, d_cpu,
2b6080f2 5413 (void *)&data->trace_cpu, &tracing_stats_fops);
438ced17
VN
5414
5415 trace_create_file("buffer_size_kb", 0444, d_cpu,
2b6080f2 5416 (void *)&data->trace_cpu, &tracing_entries_fops);
f1affcaa
SRRH
5417
5418#ifdef CONFIG_TRACER_SNAPSHOT
5419 trace_create_file("snapshot", 0644, d_cpu,
5420 (void *)&data->trace_cpu, &snapshot_fops);
6de58e62
SRRH
5421
5422 trace_create_file("snapshot_raw", 0444, d_cpu,
5423 (void *)&data->trace_cpu, &snapshot_raw_fops);
f1affcaa 5424#endif
b04cc6b1
FW
5425}
5426
60a11774
SR
5427#ifdef CONFIG_FTRACE_SELFTEST
5428/* Let selftest have access to static functions in this file */
5429#include "trace_selftest.c"
5430#endif
5431
577b785f
SR
5432struct trace_option_dentry {
5433 struct tracer_opt *opt;
5434 struct tracer_flags *flags;
2b6080f2 5435 struct trace_array *tr;
577b785f
SR
5436 struct dentry *entry;
5437};
5438
5439static ssize_t
5440trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5441 loff_t *ppos)
5442{
5443 struct trace_option_dentry *topt = filp->private_data;
5444 char *buf;
5445
5446 if (topt->flags->val & topt->opt->bit)
5447 buf = "1\n";
5448 else
5449 buf = "0\n";
5450
5451 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5452}
5453
5454static ssize_t
5455trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5456 loff_t *ppos)
5457{
5458 struct trace_option_dentry *topt = filp->private_data;
5459 unsigned long val;
577b785f
SR
5460 int ret;
5461
22fe9b54
PH
5462 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5463 if (ret)
577b785f
SR
5464 return ret;
5465
8d18eaaf
LZ
5466 if (val != 0 && val != 1)
5467 return -EINVAL;
577b785f 5468
8d18eaaf 5469 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 5470 mutex_lock(&trace_types_lock);
2b6080f2 5471 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
c757bea9 5472 topt->opt, !val);
577b785f
SR
5473 mutex_unlock(&trace_types_lock);
5474 if (ret)
5475 return ret;
577b785f
SR
5476 }
5477
5478 *ppos += cnt;
5479
5480 return cnt;
5481}
5482
5483
5484static const struct file_operations trace_options_fops = {
5485 .open = tracing_open_generic,
5486 .read = trace_options_read,
5487 .write = trace_options_write,
b444786f 5488 .llseek = generic_file_llseek,
577b785f
SR
5489};
5490
a8259075
SR
5491static ssize_t
5492trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5493 loff_t *ppos)
5494{
5495 long index = (long)filp->private_data;
5496 char *buf;
5497
5498 if (trace_flags & (1 << index))
5499 buf = "1\n";
5500 else
5501 buf = "0\n";
5502
5503 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5504}
5505
5506static ssize_t
5507trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5508 loff_t *ppos)
5509{
2b6080f2 5510 struct trace_array *tr = &global_trace;
a8259075 5511 long index = (long)filp->private_data;
a8259075
SR
5512 unsigned long val;
5513 int ret;
5514
22fe9b54
PH
5515 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5516 if (ret)
a8259075
SR
5517 return ret;
5518
f2d84b65 5519 if (val != 0 && val != 1)
a8259075 5520 return -EINVAL;
69d34da2
SRRH
5521
5522 mutex_lock(&trace_types_lock);
2b6080f2 5523 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 5524 mutex_unlock(&trace_types_lock);
a8259075 5525
613f04a0
SRRH
5526 if (ret < 0)
5527 return ret;
5528
a8259075
SR
5529 *ppos += cnt;
5530
5531 return cnt;
5532}
5533
a8259075
SR
5534static const struct file_operations trace_options_core_fops = {
5535 .open = tracing_open_generic,
5536 .read = trace_options_core_read,
5537 .write = trace_options_core_write,
b444786f 5538 .llseek = generic_file_llseek,
a8259075
SR
5539};
5540
5452af66 5541struct dentry *trace_create_file(const char *name,
f4ae40a6 5542 umode_t mode,
5452af66
FW
5543 struct dentry *parent,
5544 void *data,
5545 const struct file_operations *fops)
5546{
5547 struct dentry *ret;
5548
5549 ret = debugfs_create_file(name, mode, parent, data, fops);
5550 if (!ret)
5551 pr_warning("Could not create debugfs '%s' entry\n", name);
5552
5553 return ret;
5554}
5555
5556
2b6080f2 5557static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
5558{
5559 struct dentry *d_tracer;
a8259075 5560
2b6080f2
SR
5561 if (tr->options)
5562 return tr->options;
a8259075 5563
2b6080f2 5564 d_tracer = tracing_init_dentry_tr(tr);
a8259075
SR
5565 if (!d_tracer)
5566 return NULL;
5567
2b6080f2
SR
5568 tr->options = debugfs_create_dir("options", d_tracer);
5569 if (!tr->options) {
a8259075
SR
5570 pr_warning("Could not create debugfs directory 'options'\n");
5571 return NULL;
5572 }
5573
2b6080f2 5574 return tr->options;
a8259075
SR
5575}
5576
577b785f 5577static void
2b6080f2
SR
5578create_trace_option_file(struct trace_array *tr,
5579 struct trace_option_dentry *topt,
577b785f
SR
5580 struct tracer_flags *flags,
5581 struct tracer_opt *opt)
5582{
5583 struct dentry *t_options;
577b785f 5584
2b6080f2 5585 t_options = trace_options_init_dentry(tr);
577b785f
SR
5586 if (!t_options)
5587 return;
5588
5589 topt->flags = flags;
5590 topt->opt = opt;
2b6080f2 5591 topt->tr = tr;
577b785f 5592
5452af66 5593 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
5594 &trace_options_fops);
5595
577b785f
SR
5596}
5597
5598static struct trace_option_dentry *
2b6080f2 5599create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
5600{
5601 struct trace_option_dentry *topts;
5602 struct tracer_flags *flags;
5603 struct tracer_opt *opts;
5604 int cnt;
5605
5606 if (!tracer)
5607 return NULL;
5608
5609 flags = tracer->flags;
5610
5611 if (!flags || !flags->opts)
5612 return NULL;
5613
5614 opts = flags->opts;
5615
5616 for (cnt = 0; opts[cnt].name; cnt++)
5617 ;
5618
0cfe8245 5619 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
5620 if (!topts)
5621 return NULL;
5622
5623 for (cnt = 0; opts[cnt].name; cnt++)
2b6080f2 5624 create_trace_option_file(tr, &topts[cnt], flags,
577b785f
SR
5625 &opts[cnt]);
5626
5627 return topts;
5628}
5629
5630static void
5631destroy_trace_option_files(struct trace_option_dentry *topts)
5632{
5633 int cnt;
5634
5635 if (!topts)
5636 return;
5637
5638 for (cnt = 0; topts[cnt].opt; cnt++) {
5639 if (topts[cnt].entry)
5640 debugfs_remove(topts[cnt].entry);
5641 }
5642
5643 kfree(topts);
5644}
5645
a8259075 5646static struct dentry *
2b6080f2
SR
5647create_trace_option_core_file(struct trace_array *tr,
5648 const char *option, long index)
a8259075
SR
5649{
5650 struct dentry *t_options;
a8259075 5651
2b6080f2 5652 t_options = trace_options_init_dentry(tr);
a8259075
SR
5653 if (!t_options)
5654 return NULL;
5655
5452af66 5656 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 5657 &trace_options_core_fops);
a8259075
SR
5658}
5659
2b6080f2 5660static __init void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
5661{
5662 struct dentry *t_options;
a8259075
SR
5663 int i;
5664
2b6080f2 5665 t_options = trace_options_init_dentry(tr);
a8259075
SR
5666 if (!t_options)
5667 return;
5668
5452af66 5669 for (i = 0; trace_options[i]; i++)
2b6080f2 5670 create_trace_option_core_file(tr, trace_options[i], i);
a8259075
SR
5671}
5672
499e5470
SR
5673static ssize_t
5674rb_simple_read(struct file *filp, char __user *ubuf,
5675 size_t cnt, loff_t *ppos)
5676{
348f0fc2 5677 struct trace_array *tr = filp->private_data;
499e5470
SR
5678 char buf[64];
5679 int r;
5680
10246fa3 5681 r = tracer_tracing_is_on(tr);
499e5470
SR
5682 r = sprintf(buf, "%d\n", r);
5683
5684 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5685}
5686
5687static ssize_t
5688rb_simple_write(struct file *filp, const char __user *ubuf,
5689 size_t cnt, loff_t *ppos)
5690{
348f0fc2 5691 struct trace_array *tr = filp->private_data;
12883efb 5692 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
5693 unsigned long val;
5694 int ret;
5695
5696 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5697 if (ret)
5698 return ret;
5699
5700 if (buffer) {
2df8f8a6
SR
5701 mutex_lock(&trace_types_lock);
5702 if (val) {
10246fa3 5703 tracer_tracing_on(tr);
2b6080f2
SR
5704 if (tr->current_trace->start)
5705 tr->current_trace->start(tr);
2df8f8a6 5706 } else {
10246fa3 5707 tracer_tracing_off(tr);
2b6080f2
SR
5708 if (tr->current_trace->stop)
5709 tr->current_trace->stop(tr);
2df8f8a6
SR
5710 }
5711 mutex_unlock(&trace_types_lock);
499e5470
SR
5712 }
5713
5714 (*ppos)++;
5715
5716 return cnt;
5717}
5718
5719static const struct file_operations rb_simple_fops = {
5720 .open = tracing_open_generic,
5721 .read = rb_simple_read,
5722 .write = rb_simple_write,
5723 .llseek = default_llseek,
5724};
5725
277ba044
SR
5726struct dentry *trace_instance_dir;
5727
5728static void
5729init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5730
737223fb
SRRH
5731static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5732{
5733 int cpu;
5734
5735 for_each_tracing_cpu(cpu) {
5736 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5737 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5738 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5739 }
5740}
5741
55034cd6
SRRH
5742static int
5743allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
5744{
5745 enum ring_buffer_flags rb_flags;
737223fb
SRRH
5746
5747 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5748
55034cd6
SRRH
5749 buf->buffer = ring_buffer_alloc(size, rb_flags);
5750 if (!buf->buffer)
5751 return -ENOMEM;
737223fb 5752
55034cd6
SRRH
5753 buf->data = alloc_percpu(struct trace_array_cpu);
5754 if (!buf->data) {
5755 ring_buffer_free(buf->buffer);
5756 return -ENOMEM;
5757 }
737223fb 5758
55034cd6 5759 init_trace_buffers(tr, buf);
737223fb
SRRH
5760
5761 /* Allocate the first page for all buffers */
5762 set_buffer_entries(&tr->trace_buffer,
5763 ring_buffer_size(tr->trace_buffer.buffer, 0));
5764
55034cd6
SRRH
5765 return 0;
5766}
737223fb 5767
55034cd6
SRRH
5768static int allocate_trace_buffers(struct trace_array *tr, int size)
5769{
5770 int ret;
737223fb 5771
55034cd6
SRRH
5772 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5773 if (ret)
5774 return ret;
737223fb 5775
55034cd6
SRRH
5776#ifdef CONFIG_TRACER_MAX_TRACE
5777 ret = allocate_trace_buffer(tr, &tr->max_buffer,
5778 allocate_snapshot ? size : 1);
5779 if (WARN_ON(ret)) {
737223fb 5780 ring_buffer_free(tr->trace_buffer.buffer);
55034cd6
SRRH
5781 free_percpu(tr->trace_buffer.data);
5782 return -ENOMEM;
5783 }
5784 tr->allocated_snapshot = allocate_snapshot;
737223fb 5785
55034cd6
SRRH
5786 /*
5787 * Only the top level trace array gets its snapshot allocated
5788 * from the kernel command line.
5789 */
5790 allocate_snapshot = false;
737223fb 5791#endif
55034cd6 5792 return 0;
737223fb
SRRH
5793}
5794
5795static int new_instance_create(const char *name)
5796{
277ba044
SR
5797 struct trace_array *tr;
5798 int ret;
277ba044
SR
5799
5800 mutex_lock(&trace_types_lock);
5801
5802 ret = -EEXIST;
5803 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5804 if (tr->name && strcmp(tr->name, name) == 0)
5805 goto out_unlock;
5806 }
5807
5808 ret = -ENOMEM;
5809 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5810 if (!tr)
5811 goto out_unlock;
5812
5813 tr->name = kstrdup(name, GFP_KERNEL);
5814 if (!tr->name)
5815 goto out_free_tr;
5816
5817 raw_spin_lock_init(&tr->start_lock);
5818
5819 tr->current_trace = &nop_trace;
5820
5821 INIT_LIST_HEAD(&tr->systems);
5822 INIT_LIST_HEAD(&tr->events);
5823
737223fb 5824 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
5825 goto out_free_tr;
5826
277ba044
SR
5827 /* Holder for file callbacks */
5828 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5829 tr->trace_cpu.tr = tr;
5830
5831 tr->dir = debugfs_create_dir(name, trace_instance_dir);
5832 if (!tr->dir)
5833 goto out_free_tr;
5834
5835 ret = event_trace_add_tracer(tr->dir, tr);
5836 if (ret)
5837 goto out_free_tr;
5838
5839 init_tracer_debugfs(tr, tr->dir);
5840
5841 list_add(&tr->list, &ftrace_trace_arrays);
5842
5843 mutex_unlock(&trace_types_lock);
5844
5845 return 0;
5846
5847 out_free_tr:
12883efb
SRRH
5848 if (tr->trace_buffer.buffer)
5849 ring_buffer_free(tr->trace_buffer.buffer);
277ba044
SR
5850 kfree(tr->name);
5851 kfree(tr);
5852
5853 out_unlock:
5854 mutex_unlock(&trace_types_lock);
5855
5856 return ret;
5857
5858}
5859
0c8916c3
SR
5860static int instance_delete(const char *name)
5861{
5862 struct trace_array *tr;
5863 int found = 0;
5864 int ret;
5865
5866 mutex_lock(&trace_types_lock);
5867
5868 ret = -ENODEV;
5869 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5870 if (tr->name && strcmp(tr->name, name) == 0) {
5871 found = 1;
5872 break;
5873 }
5874 }
5875 if (!found)
5876 goto out_unlock;
5877
a695cb58
SRRH
5878 ret = -EBUSY;
5879 if (tr->ref)
5880 goto out_unlock;
5881
0c8916c3
SR
5882 list_del(&tr->list);
5883
5884 event_trace_del_tracer(tr);
5885 debugfs_remove_recursive(tr->dir);
12883efb
SRRH
5886 free_percpu(tr->trace_buffer.data);
5887 ring_buffer_free(tr->trace_buffer.buffer);
0c8916c3
SR
5888
5889 kfree(tr->name);
5890 kfree(tr);
5891
5892 ret = 0;
5893
5894 out_unlock:
5895 mutex_unlock(&trace_types_lock);
5896
5897 return ret;
5898}
5899
277ba044
SR
5900static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5901{
5902 struct dentry *parent;
5903 int ret;
5904
5905 /* Paranoid: Make sure the parent is the "instances" directory */
5906 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5907 if (WARN_ON_ONCE(parent != trace_instance_dir))
5908 return -ENOENT;
5909
5910 /*
5911 * The inode mutex is locked, but debugfs_create_dir() will also
5912 * take the mutex. As the instances directory can not be destroyed
5913 * or changed in any other way, it is safe to unlock it, and
5914 * let the dentry try. If two users try to make the same dir at
5915 * the same time, then the new_instance_create() will determine the
5916 * winner.
5917 */
5918 mutex_unlock(&inode->i_mutex);
5919
5920 ret = new_instance_create(dentry->d_iname);
5921
5922 mutex_lock(&inode->i_mutex);
5923
5924 return ret;
5925}
5926
0c8916c3
SR
5927static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5928{
5929 struct dentry *parent;
5930 int ret;
5931
5932 /* Paranoid: Make sure the parent is the "instances" directory */
5933 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5934 if (WARN_ON_ONCE(parent != trace_instance_dir))
5935 return -ENOENT;
5936
5937 /* The caller did a dget() on dentry */
5938 mutex_unlock(&dentry->d_inode->i_mutex);
5939
5940 /*
5941 * The inode mutex is locked, but debugfs_create_dir() will also
5942 * take the mutex. As the instances directory can not be destroyed
5943 * or changed in any other way, it is safe to unlock it, and
5944 * let the dentry try. If two users try to make the same dir at
5945 * the same time, then the instance_delete() will determine the
5946 * winner.
5947 */
5948 mutex_unlock(&inode->i_mutex);
5949
5950 ret = instance_delete(dentry->d_iname);
5951
5952 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5953 mutex_lock(&dentry->d_inode->i_mutex);
5954
5955 return ret;
5956}
5957
277ba044
SR
5958static const struct inode_operations instance_dir_inode_operations = {
5959 .lookup = simple_lookup,
5960 .mkdir = instance_mkdir,
0c8916c3 5961 .rmdir = instance_rmdir,
277ba044
SR
5962};
5963
5964static __init void create_trace_instances(struct dentry *d_tracer)
5965{
5966 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5967 if (WARN_ON(!trace_instance_dir))
5968 return;
5969
5970 /* Hijack the dir inode operations, to allow mkdir */
5971 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5972}
5973
2b6080f2
SR
5974static void
5975init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5976{
121aaee7 5977 int cpu;
2b6080f2
SR
5978
5979 trace_create_file("trace_options", 0644, d_tracer,
5980 tr, &tracing_iter_fops);
5981
5982 trace_create_file("trace", 0644, d_tracer,
5983 (void *)&tr->trace_cpu, &tracing_fops);
5984
5985 trace_create_file("trace_pipe", 0444, d_tracer,
5986 (void *)&tr->trace_cpu, &tracing_pipe_fops);
5987
5988 trace_create_file("buffer_size_kb", 0644, d_tracer,
5989 (void *)&tr->trace_cpu, &tracing_entries_fops);
5990
5991 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5992 tr, &tracing_total_entries_fops);
5993
238ae93d 5994 trace_create_file("free_buffer", 0200, d_tracer,
2b6080f2
SR
5995 tr, &tracing_free_buffer_fops);
5996
5997 trace_create_file("trace_marker", 0220, d_tracer,
5998 tr, &tracing_mark_fops);
5999
6000 trace_create_file("trace_clock", 0644, d_tracer, tr,
6001 &trace_clock_fops);
6002
6003 trace_create_file("tracing_on", 0644, d_tracer,
6004 tr, &rb_simple_fops);
ce9bae55
SRRH
6005
6006#ifdef CONFIG_TRACER_SNAPSHOT
6007 trace_create_file("snapshot", 0644, d_tracer,
6008 (void *)&tr->trace_cpu, &snapshot_fops);
6009#endif
121aaee7
SRRH
6010
6011 for_each_tracing_cpu(cpu)
6012 tracing_init_debugfs_percpu(tr, cpu);
6013
2b6080f2
SR
6014}
6015
b5ad384e 6016static __init int tracer_init_debugfs(void)
bc0c38d1
SR
6017{
6018 struct dentry *d_tracer;
bc0c38d1 6019
7e53bd42
LJ
6020 trace_access_lock_init();
6021
bc0c38d1 6022 d_tracer = tracing_init_dentry();
ed6f1c99
NK
6023 if (!d_tracer)
6024 return 0;
bc0c38d1 6025
2b6080f2 6026 init_tracer_debugfs(&global_trace, d_tracer);
bc0c38d1 6027
5452af66 6028 trace_create_file("tracing_cpumask", 0644, d_tracer,
2b6080f2 6029 &global_trace, &tracing_cpumask_fops);
a8259075 6030
5452af66
FW
6031 trace_create_file("available_tracers", 0444, d_tracer,
6032 &global_trace, &show_traces_fops);
6033
339ae5d3 6034 trace_create_file("current_tracer", 0644, d_tracer,
5452af66
FW
6035 &global_trace, &set_tracer_fops);
6036
5d4a9dba 6037#ifdef CONFIG_TRACER_MAX_TRACE
5452af66
FW
6038 trace_create_file("tracing_max_latency", 0644, d_tracer,
6039 &tracing_max_latency, &tracing_max_lat_fops);
0e950173 6040#endif
5452af66
FW
6041
6042 trace_create_file("tracing_thresh", 0644, d_tracer,
6043 &tracing_thresh, &tracing_max_lat_fops);
a8259075 6044
339ae5d3 6045 trace_create_file("README", 0444, d_tracer,
5452af66
FW
6046 NULL, &tracing_readme_fops);
6047
69abe6a5
AP
6048 trace_create_file("saved_cmdlines", 0444, d_tracer,
6049 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 6050
bc0c38d1 6051#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
6052 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6053 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 6054#endif
b04cc6b1 6055
277ba044 6056 create_trace_instances(d_tracer);
5452af66 6057
2b6080f2 6058 create_trace_options_dir(&global_trace);
b04cc6b1 6059
b5ad384e 6060 return 0;
bc0c38d1
SR
6061}
6062
3f5a54e3
SR
6063static int trace_panic_handler(struct notifier_block *this,
6064 unsigned long event, void *unused)
6065{
944ac425 6066 if (ftrace_dump_on_oops)
cecbca96 6067 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6068 return NOTIFY_OK;
6069}
6070
6071static struct notifier_block trace_panic_notifier = {
6072 .notifier_call = trace_panic_handler,
6073 .next = NULL,
6074 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6075};
6076
6077static int trace_die_handler(struct notifier_block *self,
6078 unsigned long val,
6079 void *data)
6080{
6081 switch (val) {
6082 case DIE_OOPS:
944ac425 6083 if (ftrace_dump_on_oops)
cecbca96 6084 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6085 break;
6086 default:
6087 break;
6088 }
6089 return NOTIFY_OK;
6090}
6091
6092static struct notifier_block trace_die_notifier = {
6093 .notifier_call = trace_die_handler,
6094 .priority = 200
6095};
6096
6097/*
6098 * printk is set to max of 1024, we really don't need it that big.
6099 * Nothing should be printing 1000 characters anyway.
6100 */
6101#define TRACE_MAX_PRINT 1000
6102
6103/*
6104 * Define here KERN_TRACE so that we have one place to modify
6105 * it if we decide to change what log level the ftrace dump
6106 * should be at.
6107 */
428aee14 6108#define KERN_TRACE KERN_EMERG
3f5a54e3 6109
955b61e5 6110void
3f5a54e3
SR
6111trace_printk_seq(struct trace_seq *s)
6112{
6113 /* Probably should print a warning here. */
bd6df187
J
6114 if (s->len >= TRACE_MAX_PRINT)
6115 s->len = TRACE_MAX_PRINT;
3f5a54e3
SR
6116
6117 /* should be zero ended, but we are paranoid. */
6118 s->buffer[s->len] = 0;
6119
6120 printk(KERN_TRACE "%s", s->buffer);
6121
f9520750 6122 trace_seq_init(s);
3f5a54e3
SR
6123}
6124
955b61e5
JW
6125void trace_init_global_iter(struct trace_iterator *iter)
6126{
6127 iter->tr = &global_trace;
2b6080f2 6128 iter->trace = iter->tr->current_trace;
ae3b5093 6129 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 6130 iter->trace_buffer = &global_trace.trace_buffer;
955b61e5
JW
6131}
6132
7fe70b57 6133void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 6134{
3f5a54e3
SR
6135 /* use static because iter can be a bit big for the stack */
6136 static struct trace_iterator iter;
7fe70b57 6137 static atomic_t dump_running;
cf586b61 6138 unsigned int old_userobj;
d769041f
SR
6139 unsigned long flags;
6140 int cnt = 0, cpu;
3f5a54e3 6141
7fe70b57
SRRH
6142 /* Only allow one dump user at a time. */
6143 if (atomic_inc_return(&dump_running) != 1) {
6144 atomic_dec(&dump_running);
6145 return;
6146 }
3f5a54e3 6147
7fe70b57
SRRH
6148 /*
6149 * Always turn off tracing when we dump.
6150 * We don't need to show trace output of what happens
6151 * between multiple crashes.
6152 *
6153 * If the user does a sysrq-z, then they can re-enable
6154 * tracing with echo 1 > tracing_on.
6155 */
0ee6b6cf 6156 tracing_off();
cf586b61 6157
7fe70b57 6158 local_irq_save(flags);
3f5a54e3 6159
38dbe0b1 6160 /* Simulate the iterator */
955b61e5
JW
6161 trace_init_global_iter(&iter);
6162
d769041f 6163 for_each_tracing_cpu(cpu) {
12883efb 6164 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
d769041f
SR
6165 }
6166
cf586b61
FW
6167 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6168
b54d3de9
TE
6169 /* don't look at user memory in panic mode */
6170 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6171
cecbca96
FW
6172 switch (oops_dump_mode) {
6173 case DUMP_ALL:
ae3b5093 6174 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6175 break;
6176 case DUMP_ORIG:
6177 iter.cpu_file = raw_smp_processor_id();
6178 break;
6179 case DUMP_NONE:
6180 goto out_enable;
6181 default:
6182 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 6183 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6184 }
6185
6186 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 6187
7fe70b57
SRRH
6188 /* Did function tracer already get disabled? */
6189 if (ftrace_is_dead()) {
6190 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6191 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6192 }
6193
3f5a54e3
SR
6194 /*
6195 * We need to stop all tracing on all CPUS to read the
6196 * the next buffer. This is a bit expensive, but is
6197 * not done often. We fill all what we can read,
6198 * and then release the locks again.
6199 */
6200
3f5a54e3
SR
6201 while (!trace_empty(&iter)) {
6202
6203 if (!cnt)
6204 printk(KERN_TRACE "---------------------------------\n");
6205
6206 cnt++;
6207
6208 /* reset all but tr, trace, and overruns */
6209 memset(&iter.seq, 0,
6210 sizeof(struct trace_iterator) -
6211 offsetof(struct trace_iterator, seq));
6212 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6213 iter.pos = -1;
6214
955b61e5 6215 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
6216 int ret;
6217
6218 ret = print_trace_line(&iter);
6219 if (ret != TRACE_TYPE_NO_CONSUME)
6220 trace_consume(&iter);
3f5a54e3 6221 }
b892e5c8 6222 touch_nmi_watchdog();
3f5a54e3
SR
6223
6224 trace_printk_seq(&iter.seq);
6225 }
6226
6227 if (!cnt)
6228 printk(KERN_TRACE " (ftrace buffer empty)\n");
6229 else
6230 printk(KERN_TRACE "---------------------------------\n");
6231
cecbca96 6232 out_enable:
7fe70b57 6233 trace_flags |= old_userobj;
cf586b61 6234
7fe70b57
SRRH
6235 for_each_tracing_cpu(cpu) {
6236 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 6237 }
7fe70b57 6238 atomic_dec(&dump_running);
cd891ae0 6239 local_irq_restore(flags);
3f5a54e3 6240}
a8eecf22 6241EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 6242
3928a8a2 6243__init static int tracer_alloc_buffers(void)
bc0c38d1 6244{
73c5162a 6245 int ring_buf_size;
9e01c1b7 6246 int ret = -ENOMEM;
4c11d7ae 6247
750912fa 6248
9e01c1b7
RR
6249 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6250 goto out;
6251
6252 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6253 goto out_free_buffer_mask;
4c11d7ae 6254
07d777fe
SR
6255 /* Only allocate trace_printk buffers if a trace_printk exists */
6256 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 6257 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
6258 trace_printk_init_buffers();
6259
73c5162a
SR
6260 /* To save memory, keep the ring buffer size to its minimum */
6261 if (ring_buffer_expanded)
6262 ring_buf_size = trace_buf_size;
6263 else
6264 ring_buf_size = 1;
6265
9e01c1b7
RR
6266 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6267 cpumask_copy(tracing_cpumask, cpu_all_mask);
6268
2b6080f2
SR
6269 raw_spin_lock_init(&global_trace.start_lock);
6270
9e01c1b7 6271 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 6272 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
6273 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6274 WARN_ON(1);
9e01c1b7 6275 goto out_free_cpumask;
4c11d7ae 6276 }
a7603ff4 6277
499e5470
SR
6278 if (global_trace.buffer_disabled)
6279 tracing_off();
4c11d7ae 6280
bc0c38d1
SR
6281 trace_init_cmdlines();
6282
ca164318
SRRH
6283 /*
6284 * register_tracer() might reference current_trace, so it
6285 * needs to be set before we register anything. This is
6286 * just a bootstrap of current_trace anyway.
6287 */
2b6080f2
SR
6288 global_trace.current_trace = &nop_trace;
6289
ca164318
SRRH
6290 register_tracer(&nop_trace);
6291
60a11774
SR
6292 /* All seems OK, enable tracing */
6293 tracing_disabled = 0;
3928a8a2 6294
3f5a54e3
SR
6295 atomic_notifier_chain_register(&panic_notifier_list,
6296 &trace_panic_notifier);
6297
6298 register_die_notifier(&trace_die_notifier);
2fc1dfbe 6299
ae63b31e
SR
6300 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6301
2b6080f2
SR
6302 /* Holder for file callbacks */
6303 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6304 global_trace.trace_cpu.tr = &global_trace;
6305
ae63b31e
SR
6306 INIT_LIST_HEAD(&global_trace.systems);
6307 INIT_LIST_HEAD(&global_trace.events);
6308 list_add(&global_trace.list, &ftrace_trace_arrays);
6309
7bcfaf54
SR
6310 while (trace_boot_options) {
6311 char *option;
6312
6313 option = strsep(&trace_boot_options, ",");
2b6080f2 6314 trace_set_options(&global_trace, option);
7bcfaf54
SR
6315 }
6316
77fd5c15
SRRH
6317 register_snapshot_cmd();
6318
2fc1dfbe 6319 return 0;
3f5a54e3 6320
9e01c1b7 6321out_free_cpumask:
12883efb
SRRH
6322 free_percpu(global_trace.trace_buffer.data);
6323#ifdef CONFIG_TRACER_MAX_TRACE
6324 free_percpu(global_trace.max_buffer.data);
6325#endif
9e01c1b7
RR
6326 free_cpumask_var(tracing_cpumask);
6327out_free_buffer_mask:
6328 free_cpumask_var(tracing_buffer_mask);
6329out:
6330 return ret;
bc0c38d1 6331}
b2821ae6
SR
6332
6333__init static int clear_boot_tracer(void)
6334{
6335 /*
6336 * The default tracer at boot buffer is an init section.
6337 * This function is called in lateinit. If we did not
6338 * find the boot tracer, then clear it out, to prevent
6339 * later registration from accessing the buffer that is
6340 * about to be freed.
6341 */
6342 if (!default_bootup_tracer)
6343 return 0;
6344
6345 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6346 default_bootup_tracer);
6347 default_bootup_tracer = NULL;
6348
6349 return 0;
6350}
6351
b5ad384e
FW
6352early_initcall(tracer_alloc_buffers);
6353fs_initcall(tracer_init_debugfs);
b2821ae6 6354late_initcall(clear_boot_tracer);