]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
tracing: Deletion of an unnecessary check before iput()
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
4c11d7ae 23#include <linux/pagemap.h>
bc0c38d1
SR
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
2cadf913 27#include <linux/kprobes.h>
bc0c38d1
SR
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
2cadf913 31#include <linux/splice.h>
3f5a54e3 32#include <linux/kdebug.h>
5f0c6c03 33#include <linux/string.h>
7e53bd42 34#include <linux/rwsem.h>
5a0e3ad6 35#include <linux/slab.h>
bc0c38d1
SR
36#include <linux/ctype.h>
37#include <linux/init.h>
2a2cc8f7 38#include <linux/poll.h>
b892e5c8 39#include <linux/nmi.h>
bc0c38d1 40#include <linux/fs.h>
8bd75c77 41#include <linux/sched/rt.h>
86387f7e 42
bc0c38d1 43#include "trace.h"
f0868d1e 44#include "trace_output.h"
bc0c38d1 45
73c5162a
SR
46/*
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
55034cd6 50bool ring_buffer_expanded;
73c5162a 51
8e1b82e0
FW
52/*
53 * We need to change this state when a selftest is running.
ff32504f
FW
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
5e1607a0 56 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
57 * at the same time, giving false positive or negative results.
58 */
8e1b82e0 59static bool __read_mostly tracing_selftest_running;
ff32504f 60
b2821ae6
SR
61/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
020e5f85 64bool __read_mostly tracing_selftest_disabled;
b2821ae6 65
adf9f195
FW
66/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
8c1a49ae
SRRH
76static int
77dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
adf9f195
FW
78{
79 return 0;
80}
0f048701 81
7ffbd48d
SR
82/*
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
85 * occurred.
86 */
87static DEFINE_PER_CPU(bool, trace_cmdline_save);
88
0f048701
SR
89/*
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
93 * this back to zero.
94 */
4fd27358 95static int tracing_disabled = 1;
0f048701 96
9288f99a 97DEFINE_PER_CPU(int, ftrace_cpu_disabled);
d769041f 98
955b61e5 99cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 100
944ac425
SR
101/*
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 *
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
108 * serial console.
109 *
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 115 */
cecbca96
FW
116
117enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 118
de7edd31
SRRH
119/* When set, tracing will stop when a WARN*() is hit */
120int __disable_trace_on_warning;
121
607e2ea1 122static int tracing_set_tracer(struct trace_array *tr, const char *buf);
b2821ae6 123
ee6c2c1b
LZ
124#define MAX_TRACER_SIZE 100
125static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 126static char *default_bootup_tracer;
d9e54076 127
55034cd6
SRRH
128static bool allocate_snapshot;
129
1beee96b 130static int __init set_cmdline_ftrace(char *str)
d9e54076 131{
67012ab1 132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 133 default_bootup_tracer = bootup_tracer_buf;
73c5162a 134 /* We are using ftrace early, expand it */
55034cd6 135 ring_buffer_expanded = true;
d9e54076
PZ
136 return 1;
137}
1beee96b 138__setup("ftrace=", set_cmdline_ftrace);
d9e54076 139
944ac425
SR
140static int __init set_ftrace_dump_on_oops(char *str)
141{
cecbca96
FW
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
144 return 1;
145 }
146
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
149 return 1;
150 }
151
152 return 0;
944ac425
SR
153}
154__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 155
de7edd31
SRRH
156static int __init stop_trace_on_warning(char *str)
157{
933ff9f2
LCG
158 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
159 __disable_trace_on_warning = 1;
de7edd31
SRRH
160 return 1;
161}
933ff9f2 162__setup("traceoff_on_warning", stop_trace_on_warning);
de7edd31 163
3209cff4 164static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
165{
166 allocate_snapshot = true;
167 /* We also need the main ring buffer expanded */
168 ring_buffer_expanded = true;
169 return 1;
170}
3209cff4 171__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 172
7bcfaf54
SR
173
174static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
175static char *trace_boot_options __initdata;
176
177static int __init set_trace_boot_options(char *str)
178{
67012ab1 179 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
180 trace_boot_options = trace_boot_options_buf;
181 return 0;
182}
183__setup("trace_options=", set_trace_boot_options);
184
e1e232ca
SR
185static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
186static char *trace_boot_clock __initdata;
187
188static int __init set_trace_boot_clock(char *str)
189{
190 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
191 trace_boot_clock = trace_boot_clock_buf;
192 return 0;
193}
194__setup("trace_clock=", set_trace_boot_clock);
195
de7edd31 196
cf8e3474 197unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
198{
199 nsec += 500;
200 do_div(nsec, 1000);
201 return nsec;
202}
203
4fcdae83
SR
204/*
205 * The global_trace is the descriptor that holds the tracing
206 * buffers for the live tracing. For each CPU, it contains
207 * a link list of pages that will store trace entries. The
208 * page descriptor of the pages in the memory is used to hold
209 * the link list by linking the lru item in the page descriptor
210 * to each of the pages in the buffer per CPU.
211 *
212 * For each active CPU there is a data field that holds the
213 * pages for the buffer for that CPU. Each CPU has the same number
214 * of pages allocated for its buffer.
215 */
bc0c38d1
SR
216static struct trace_array global_trace;
217
ae63b31e 218LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 219
ff451961
SRRH
220int trace_array_get(struct trace_array *this_tr)
221{
222 struct trace_array *tr;
223 int ret = -ENODEV;
224
225 mutex_lock(&trace_types_lock);
226 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
227 if (tr == this_tr) {
228 tr->ref++;
229 ret = 0;
230 break;
231 }
232 }
233 mutex_unlock(&trace_types_lock);
234
235 return ret;
236}
237
238static void __trace_array_put(struct trace_array *this_tr)
239{
240 WARN_ON(!this_tr->ref);
241 this_tr->ref--;
242}
243
244void trace_array_put(struct trace_array *this_tr)
245{
246 mutex_lock(&trace_types_lock);
247 __trace_array_put(this_tr);
248 mutex_unlock(&trace_types_lock);
249}
250
f306cc82
TZ
251int filter_check_discard(struct ftrace_event_file *file, void *rec,
252 struct ring_buffer *buffer,
253 struct ring_buffer_event *event)
eb02ce01 254{
f306cc82
TZ
255 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
256 !filter_match_preds(file->filter, rec)) {
257 ring_buffer_discard_commit(buffer, event);
258 return 1;
259 }
260
261 return 0;
262}
263EXPORT_SYMBOL_GPL(filter_check_discard);
264
265int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
266 struct ring_buffer *buffer,
267 struct ring_buffer_event *event)
268{
269 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
270 !filter_match_preds(call->filter, rec)) {
271 ring_buffer_discard_commit(buffer, event);
272 return 1;
273 }
274
275 return 0;
eb02ce01 276}
f306cc82 277EXPORT_SYMBOL_GPL(call_filter_check_discard);
eb02ce01 278
ad1438a0 279static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
37886f6a
SR
280{
281 u64 ts;
282
283 /* Early boot up does not have a buffer yet */
9457158b 284 if (!buf->buffer)
37886f6a
SR
285 return trace_clock_local();
286
9457158b
AL
287 ts = ring_buffer_time_stamp(buf->buffer, cpu);
288 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
37886f6a
SR
289
290 return ts;
291}
bc0c38d1 292
9457158b
AL
293cycle_t ftrace_now(int cpu)
294{
295 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
296}
297
10246fa3
SRRH
298/**
299 * tracing_is_enabled - Show if global_trace has been disabled
300 *
301 * Shows if the global trace has been enabled or not. It uses the
302 * mirror flag "buffer_disabled" to be used in fast paths such as for
303 * the irqsoff tracer. But it may be inaccurate due to races. If you
304 * need to know the accurate state, use tracing_is_on() which is a little
305 * slower, but accurate.
306 */
9036990d
SR
307int tracing_is_enabled(void)
308{
10246fa3
SRRH
309 /*
310 * For quick access (irqsoff uses this in fast path), just
311 * return the mirror variable of the state of the ring buffer.
312 * It's a little racy, but we don't really care.
313 */
314 smp_rmb();
315 return !global_trace.buffer_disabled;
9036990d
SR
316}
317
4fcdae83 318/*
3928a8a2
SR
319 * trace_buf_size is the size in bytes that is allocated
320 * for a buffer. Note, the number of bytes is always rounded
321 * to page size.
3f5a54e3
SR
322 *
323 * This number is purposely set to a low number of 16384.
324 * If the dump on oops happens, it will be much appreciated
325 * to not have to wait for all that output. Anyway this can be
326 * boot time and run time configurable.
4fcdae83 327 */
3928a8a2 328#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 329
3928a8a2 330static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 331
4fcdae83 332/* trace_types holds a link list of available tracers. */
bc0c38d1 333static struct tracer *trace_types __read_mostly;
4fcdae83 334
4fcdae83
SR
335/*
336 * trace_types_lock is used to protect the trace_types list.
4fcdae83 337 */
a8227415 338DEFINE_MUTEX(trace_types_lock);
4fcdae83 339
7e53bd42
LJ
340/*
341 * serialize the access of the ring buffer
342 *
343 * ring buffer serializes readers, but it is low level protection.
344 * The validity of the events (which returns by ring_buffer_peek() ..etc)
345 * are not protected by ring buffer.
346 *
347 * The content of events may become garbage if we allow other process consumes
348 * these events concurrently:
349 * A) the page of the consumed events may become a normal page
350 * (not reader page) in ring buffer, and this page will be rewrited
351 * by events producer.
352 * B) The page of the consumed events may become a page for splice_read,
353 * and this page will be returned to system.
354 *
355 * These primitives allow multi process access to different cpu ring buffer
356 * concurrently.
357 *
358 * These primitives don't distinguish read-only and read-consume access.
359 * Multi read-only access are also serialized.
360 */
361
362#ifdef CONFIG_SMP
363static DECLARE_RWSEM(all_cpu_access_lock);
364static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365
366static inline void trace_access_lock(int cpu)
367{
ae3b5093 368 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
369 /* gain it for accessing the whole ring buffer. */
370 down_write(&all_cpu_access_lock);
371 } else {
372 /* gain it for accessing a cpu ring buffer. */
373
ae3b5093 374 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
375 down_read(&all_cpu_access_lock);
376
377 /* Secondly block other access to this @cpu ring buffer. */
378 mutex_lock(&per_cpu(cpu_access_lock, cpu));
379 }
380}
381
382static inline void trace_access_unlock(int cpu)
383{
ae3b5093 384 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
385 up_write(&all_cpu_access_lock);
386 } else {
387 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
388 up_read(&all_cpu_access_lock);
389 }
390}
391
392static inline void trace_access_lock_init(void)
393{
394 int cpu;
395
396 for_each_possible_cpu(cpu)
397 mutex_init(&per_cpu(cpu_access_lock, cpu));
398}
399
400#else
401
402static DEFINE_MUTEX(access_lock);
403
404static inline void trace_access_lock(int cpu)
405{
406 (void)cpu;
407 mutex_lock(&access_lock);
408}
409
410static inline void trace_access_unlock(int cpu)
411{
412 (void)cpu;
413 mutex_unlock(&access_lock);
414}
415
416static inline void trace_access_lock_init(void)
417{
418}
419
420#endif
421
ee6bce52 422/* trace_flags holds trace_options default values */
12ef7d44 423unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a 424 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
77271ce4 425 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
328df475 426 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
e7e2ee89 427
5280bcef 428static void tracer_tracing_on(struct trace_array *tr)
10246fa3
SRRH
429{
430 if (tr->trace_buffer.buffer)
431 ring_buffer_record_on(tr->trace_buffer.buffer);
432 /*
433 * This flag is looked at when buffers haven't been allocated
434 * yet, or by some tracers (like irqsoff), that just want to
435 * know if the ring buffer has been disabled, but it can handle
436 * races of where it gets disabled but we still do a record.
437 * As the check is in the fast path of the tracers, it is more
438 * important to be fast than accurate.
439 */
440 tr->buffer_disabled = 0;
441 /* Make the flag seen by readers */
442 smp_wmb();
443}
444
499e5470
SR
445/**
446 * tracing_on - enable tracing buffers
447 *
448 * This function enables tracing buffers that may have been
449 * disabled with tracing_off.
450 */
451void tracing_on(void)
452{
10246fa3 453 tracer_tracing_on(&global_trace);
499e5470
SR
454}
455EXPORT_SYMBOL_GPL(tracing_on);
456
09ae7234
SRRH
457/**
458 * __trace_puts - write a constant string into the trace buffer.
459 * @ip: The address of the caller
460 * @str: The constant string to write
461 * @size: The size of the string.
462 */
463int __trace_puts(unsigned long ip, const char *str, int size)
464{
465 struct ring_buffer_event *event;
466 struct ring_buffer *buffer;
467 struct print_entry *entry;
468 unsigned long irq_flags;
469 int alloc;
8abfb872
J
470 int pc;
471
f0160a5a
J
472 if (!(trace_flags & TRACE_ITER_PRINTK))
473 return 0;
474
8abfb872 475 pc = preempt_count();
09ae7234 476
3132e107
SRRH
477 if (unlikely(tracing_selftest_running || tracing_disabled))
478 return 0;
479
09ae7234
SRRH
480 alloc = sizeof(*entry) + size + 2; /* possible \n added */
481
482 local_save_flags(irq_flags);
483 buffer = global_trace.trace_buffer.buffer;
484 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
8abfb872 485 irq_flags, pc);
09ae7234
SRRH
486 if (!event)
487 return 0;
488
489 entry = ring_buffer_event_data(event);
490 entry->ip = ip;
491
492 memcpy(&entry->buf, str, size);
493
494 /* Add a newline if necessary */
495 if (entry->buf[size - 1] != '\n') {
496 entry->buf[size] = '\n';
497 entry->buf[size + 1] = '\0';
498 } else
499 entry->buf[size] = '\0';
500
501 __buffer_unlock_commit(buffer, event);
8abfb872 502 ftrace_trace_stack(buffer, irq_flags, 4, pc);
09ae7234
SRRH
503
504 return size;
505}
506EXPORT_SYMBOL_GPL(__trace_puts);
507
508/**
509 * __trace_bputs - write the pointer to a constant string into trace buffer
510 * @ip: The address of the caller
511 * @str: The constant string to write to the buffer to
512 */
513int __trace_bputs(unsigned long ip, const char *str)
514{
515 struct ring_buffer_event *event;
516 struct ring_buffer *buffer;
517 struct bputs_entry *entry;
518 unsigned long irq_flags;
519 int size = sizeof(struct bputs_entry);
8abfb872
J
520 int pc;
521
f0160a5a
J
522 if (!(trace_flags & TRACE_ITER_PRINTK))
523 return 0;
524
8abfb872 525 pc = preempt_count();
09ae7234 526
3132e107
SRRH
527 if (unlikely(tracing_selftest_running || tracing_disabled))
528 return 0;
529
09ae7234
SRRH
530 local_save_flags(irq_flags);
531 buffer = global_trace.trace_buffer.buffer;
532 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
8abfb872 533 irq_flags, pc);
09ae7234
SRRH
534 if (!event)
535 return 0;
536
537 entry = ring_buffer_event_data(event);
538 entry->ip = ip;
539 entry->str = str;
540
541 __buffer_unlock_commit(buffer, event);
8abfb872 542 ftrace_trace_stack(buffer, irq_flags, 4, pc);
09ae7234
SRRH
543
544 return 1;
545}
546EXPORT_SYMBOL_GPL(__trace_bputs);
547
ad909e21
SRRH
548#ifdef CONFIG_TRACER_SNAPSHOT
549/**
550 * trace_snapshot - take a snapshot of the current buffer.
551 *
552 * This causes a swap between the snapshot buffer and the current live
553 * tracing buffer. You can use this to take snapshots of the live
554 * trace when some condition is triggered, but continue to trace.
555 *
556 * Note, make sure to allocate the snapshot with either
557 * a tracing_snapshot_alloc(), or by doing it manually
558 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
559 *
560 * If the snapshot buffer is not allocated, it will stop tracing.
561 * Basically making a permanent snapshot.
562 */
563void tracing_snapshot(void)
564{
565 struct trace_array *tr = &global_trace;
566 struct tracer *tracer = tr->current_trace;
567 unsigned long flags;
568
1b22e382
SRRH
569 if (in_nmi()) {
570 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
571 internal_trace_puts("*** snapshot is being ignored ***\n");
572 return;
573 }
574
ad909e21 575 if (!tr->allocated_snapshot) {
ca268da6
SRRH
576 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
577 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
578 tracing_off();
579 return;
580 }
581
582 /* Note, snapshot can not be used when the tracer uses it */
583 if (tracer->use_max_tr) {
ca268da6
SRRH
584 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
585 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
586 return;
587 }
588
589 local_irq_save(flags);
590 update_max_tr(tr, current, smp_processor_id());
591 local_irq_restore(flags);
592}
1b22e382 593EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
594
595static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
596 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
597static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
598
599static int alloc_snapshot(struct trace_array *tr)
600{
601 int ret;
602
603 if (!tr->allocated_snapshot) {
604
605 /* allocate spare buffer */
606 ret = resize_buffer_duplicate_size(&tr->max_buffer,
607 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
608 if (ret < 0)
609 return ret;
610
611 tr->allocated_snapshot = true;
612 }
613
614 return 0;
615}
616
ad1438a0 617static void free_snapshot(struct trace_array *tr)
3209cff4
SRRH
618{
619 /*
620 * We don't free the ring buffer. instead, resize it because
621 * The max_tr ring buffer has some state (e.g. ring->clock) and
622 * we want preserve it.
623 */
624 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
625 set_buffer_entries(&tr->max_buffer, 1);
626 tracing_reset_online_cpus(&tr->max_buffer);
627 tr->allocated_snapshot = false;
628}
ad909e21 629
93e31ffb
TZ
630/**
631 * tracing_alloc_snapshot - allocate snapshot buffer.
632 *
633 * This only allocates the snapshot buffer if it isn't already
634 * allocated - it doesn't also take a snapshot.
635 *
636 * This is meant to be used in cases where the snapshot buffer needs
637 * to be set up for events that can't sleep but need to be able to
638 * trigger a snapshot.
639 */
640int tracing_alloc_snapshot(void)
641{
642 struct trace_array *tr = &global_trace;
643 int ret;
644
645 ret = alloc_snapshot(tr);
646 WARN_ON(ret < 0);
647
648 return ret;
649}
650EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
651
ad909e21
SRRH
652/**
653 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
654 *
655 * This is similar to trace_snapshot(), but it will allocate the
656 * snapshot buffer if it isn't already allocated. Use this only
657 * where it is safe to sleep, as the allocation may sleep.
658 *
659 * This causes a swap between the snapshot buffer and the current live
660 * tracing buffer. You can use this to take snapshots of the live
661 * trace when some condition is triggered, but continue to trace.
662 */
663void tracing_snapshot_alloc(void)
664{
ad909e21
SRRH
665 int ret;
666
93e31ffb
TZ
667 ret = tracing_alloc_snapshot();
668 if (ret < 0)
3209cff4 669 return;
ad909e21
SRRH
670
671 tracing_snapshot();
672}
1b22e382 673EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
674#else
675void tracing_snapshot(void)
676{
677 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
678}
1b22e382 679EXPORT_SYMBOL_GPL(tracing_snapshot);
93e31ffb
TZ
680int tracing_alloc_snapshot(void)
681{
682 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
683 return -ENODEV;
684}
685EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
ad909e21
SRRH
686void tracing_snapshot_alloc(void)
687{
688 /* Give warning */
689 tracing_snapshot();
690}
1b22e382 691EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
692#endif /* CONFIG_TRACER_SNAPSHOT */
693
5280bcef 694static void tracer_tracing_off(struct trace_array *tr)
10246fa3
SRRH
695{
696 if (tr->trace_buffer.buffer)
697 ring_buffer_record_off(tr->trace_buffer.buffer);
698 /*
699 * This flag is looked at when buffers haven't been allocated
700 * yet, or by some tracers (like irqsoff), that just want to
701 * know if the ring buffer has been disabled, but it can handle
702 * races of where it gets disabled but we still do a record.
703 * As the check is in the fast path of the tracers, it is more
704 * important to be fast than accurate.
705 */
706 tr->buffer_disabled = 1;
707 /* Make the flag seen by readers */
708 smp_wmb();
709}
710
499e5470
SR
711/**
712 * tracing_off - turn off tracing buffers
713 *
714 * This function stops the tracing buffers from recording data.
715 * It does not disable any overhead the tracers themselves may
716 * be causing. This function simply causes all recording to
717 * the ring buffers to fail.
718 */
719void tracing_off(void)
720{
10246fa3 721 tracer_tracing_off(&global_trace);
499e5470
SR
722}
723EXPORT_SYMBOL_GPL(tracing_off);
724
de7edd31
SRRH
725void disable_trace_on_warning(void)
726{
727 if (__disable_trace_on_warning)
728 tracing_off();
729}
730
10246fa3
SRRH
731/**
732 * tracer_tracing_is_on - show real state of ring buffer enabled
733 * @tr : the trace array to know if ring buffer is enabled
734 *
735 * Shows real state of the ring buffer if it is enabled or not.
736 */
5280bcef 737static int tracer_tracing_is_on(struct trace_array *tr)
10246fa3
SRRH
738{
739 if (tr->trace_buffer.buffer)
740 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
741 return !tr->buffer_disabled;
742}
743
499e5470
SR
744/**
745 * tracing_is_on - show state of ring buffers enabled
746 */
747int tracing_is_on(void)
748{
10246fa3 749 return tracer_tracing_is_on(&global_trace);
499e5470
SR
750}
751EXPORT_SYMBOL_GPL(tracing_is_on);
752
3928a8a2 753static int __init set_buf_size(char *str)
bc0c38d1 754{
3928a8a2 755 unsigned long buf_size;
c6caeeb1 756
bc0c38d1
SR
757 if (!str)
758 return 0;
9d612bef 759 buf_size = memparse(str, &str);
c6caeeb1 760 /* nr_entries can not be zero */
9d612bef 761 if (buf_size == 0)
c6caeeb1 762 return 0;
3928a8a2 763 trace_buf_size = buf_size;
bc0c38d1
SR
764 return 1;
765}
3928a8a2 766__setup("trace_buf_size=", set_buf_size);
bc0c38d1 767
0e950173
TB
768static int __init set_tracing_thresh(char *str)
769{
87abb3b1 770 unsigned long threshold;
0e950173
TB
771 int ret;
772
773 if (!str)
774 return 0;
bcd83ea6 775 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
776 if (ret < 0)
777 return 0;
87abb3b1 778 tracing_thresh = threshold * 1000;
0e950173
TB
779 return 1;
780}
781__setup("tracing_thresh=", set_tracing_thresh);
782
57f50be1
SR
783unsigned long nsecs_to_usecs(unsigned long nsecs)
784{
785 return nsecs / 1000;
786}
787
4fcdae83 788/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
789static const char *trace_options[] = {
790 "print-parent",
791 "sym-offset",
792 "sym-addr",
793 "verbose",
f9896bf3 794 "raw",
5e3ca0ec 795 "hex",
cb0f12aa 796 "bin",
2a2cc8f7 797 "block",
86387f7e 798 "stacktrace",
5e1607a0 799 "trace_printk",
b2a866f9 800 "ftrace_preempt",
9f029e83 801 "branch",
12ef7d44 802 "annotate",
02b67518 803 "userstacktrace",
b54d3de9 804 "sym-userobj",
66896a85 805 "printk-msg-only",
c4a8e8be 806 "context-info",
c032ef64 807 "latency-format",
be6f164a 808 "sleep-time",
a2a16d6a 809 "graph-time",
e870e9a1 810 "record-cmd",
750912fa 811 "overwrite",
cf30cf67 812 "disable_on_free",
77271ce4 813 "irq-info",
5224c3a3 814 "markers",
328df475 815 "function-trace",
bc0c38d1
SR
816 NULL
817};
818
5079f326
Z
819static struct {
820 u64 (*func)(void);
821 const char *name;
8be0709f 822 int in_ns; /* is this clock in nanoseconds? */
5079f326 823} trace_clocks[] = {
1b3e5c09
TG
824 { trace_clock_local, "local", 1 },
825 { trace_clock_global, "global", 1 },
826 { trace_clock_counter, "counter", 0 },
e7fda6c4 827 { trace_clock_jiffies, "uptime", 0 },
1b3e5c09
TG
828 { trace_clock, "perf", 1 },
829 { ktime_get_mono_fast_ns, "mono", 1 },
8cbd9cc6 830 ARCH_TRACE_CLOCKS
5079f326
Z
831};
832
b63f39ea 833/*
834 * trace_parser_get_init - gets the buffer for trace parser
835 */
836int trace_parser_get_init(struct trace_parser *parser, int size)
837{
838 memset(parser, 0, sizeof(*parser));
839
840 parser->buffer = kmalloc(size, GFP_KERNEL);
841 if (!parser->buffer)
842 return 1;
843
844 parser->size = size;
845 return 0;
846}
847
848/*
849 * trace_parser_put - frees the buffer for trace parser
850 */
851void trace_parser_put(struct trace_parser *parser)
852{
853 kfree(parser->buffer);
854}
855
856/*
857 * trace_get_user - reads the user input string separated by space
858 * (matched by isspace(ch))
859 *
860 * For each string found the 'struct trace_parser' is updated,
861 * and the function returns.
862 *
863 * Returns number of bytes read.
864 *
865 * See kernel/trace/trace.h for 'struct trace_parser' details.
866 */
867int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
868 size_t cnt, loff_t *ppos)
869{
870 char ch;
871 size_t read = 0;
872 ssize_t ret;
873
874 if (!*ppos)
875 trace_parser_clear(parser);
876
877 ret = get_user(ch, ubuf++);
878 if (ret)
879 goto out;
880
881 read++;
882 cnt--;
883
884 /*
885 * The parser is not finished with the last write,
886 * continue reading the user input without skipping spaces.
887 */
888 if (!parser->cont) {
889 /* skip white space */
890 while (cnt && isspace(ch)) {
891 ret = get_user(ch, ubuf++);
892 if (ret)
893 goto out;
894 read++;
895 cnt--;
896 }
897
898 /* only spaces were written */
899 if (isspace(ch)) {
900 *ppos += read;
901 ret = read;
902 goto out;
903 }
904
905 parser->idx = 0;
906 }
907
908 /* read the non-space input */
909 while (cnt && !isspace(ch)) {
3c235a33 910 if (parser->idx < parser->size - 1)
b63f39ea 911 parser->buffer[parser->idx++] = ch;
912 else {
913 ret = -EINVAL;
914 goto out;
915 }
916 ret = get_user(ch, ubuf++);
917 if (ret)
918 goto out;
919 read++;
920 cnt--;
921 }
922
923 /* We either got finished input or we have to wait for another call. */
924 if (isspace(ch)) {
925 parser->buffer[parser->idx] = 0;
926 parser->cont = false;
057db848 927 } else if (parser->idx < parser->size - 1) {
b63f39ea 928 parser->cont = true;
929 parser->buffer[parser->idx++] = ch;
057db848
SR
930 } else {
931 ret = -EINVAL;
932 goto out;
b63f39ea 933 }
934
935 *ppos += read;
936 ret = read;
937
938out:
939 return ret;
940}
941
b8b94265 942static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
943{
944 int len;
3c56819b
EGM
945
946 if (s->len <= s->readpos)
947 return -EBUSY;
948
949 len = s->len - s->readpos;
950 if (cnt > len)
951 cnt = len;
5a26c8f0 952 memcpy(buf, s->buffer + s->readpos, cnt);
3c56819b 953
e74da523 954 s->readpos += cnt;
3c56819b
EGM
955 return cnt;
956}
957
0e950173
TB
958unsigned long __read_mostly tracing_thresh;
959
5d4a9dba 960#ifdef CONFIG_TRACER_MAX_TRACE
5d4a9dba
SR
961/*
962 * Copy the new maximum trace into the separate maximum-trace
963 * structure. (this way the maximum trace is permanently saved,
964 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
965 */
966static void
967__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
968{
12883efb
SRRH
969 struct trace_buffer *trace_buf = &tr->trace_buffer;
970 struct trace_buffer *max_buf = &tr->max_buffer;
971 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
972 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 973
12883efb
SRRH
974 max_buf->cpu = cpu;
975 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 976
6d9b3fa5 977 max_data->saved_latency = tr->max_latency;
8248ac05
SR
978 max_data->critical_start = data->critical_start;
979 max_data->critical_end = data->critical_end;
5d4a9dba 980
1acaa1b2 981 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05 982 max_data->pid = tsk->pid;
f17a5194
SRRH
983 /*
984 * If tsk == current, then use current_uid(), as that does not use
985 * RCU. The irq tracer can be called out of RCU scope.
986 */
987 if (tsk == current)
988 max_data->uid = current_uid();
989 else
990 max_data->uid = task_uid(tsk);
991
8248ac05
SR
992 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
993 max_data->policy = tsk->policy;
994 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
995
996 /* record this tasks comm */
997 tracing_record_cmdline(tsk);
998}
999
4fcdae83
SR
1000/**
1001 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1002 * @tr: tracer
1003 * @tsk: the task with the latency
1004 * @cpu: The cpu that initiated the trace.
1005 *
1006 * Flip the buffers between the @tr and the max_tr and record information
1007 * about which task was the cause of this latency.
1008 */
e309b41d 1009void
bc0c38d1
SR
1010update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1011{
2721e72d 1012 struct ring_buffer *buf;
bc0c38d1 1013
2b6080f2 1014 if (tr->stop_count)
b8de7bd1
SR
1015 return;
1016
4c11d7ae 1017 WARN_ON_ONCE(!irqs_disabled());
34600f0e 1018
45ad21ca 1019 if (!tr->allocated_snapshot) {
debdd57f 1020 /* Only the nop tracer should hit this when disabling */
2b6080f2 1021 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 1022 return;
debdd57f 1023 }
34600f0e 1024
0b9b12c1 1025 arch_spin_lock(&tr->max_lock);
3928a8a2 1026
12883efb
SRRH
1027 buf = tr->trace_buffer.buffer;
1028 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1029 tr->max_buffer.buffer = buf;
3928a8a2 1030
bc0c38d1 1031 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1032 arch_spin_unlock(&tr->max_lock);
bc0c38d1
SR
1033}
1034
1035/**
1036 * update_max_tr_single - only copy one trace over, and reset the rest
1037 * @tr - tracer
1038 * @tsk - task with the latency
1039 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
1040 *
1041 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 1042 */
e309b41d 1043void
bc0c38d1
SR
1044update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1045{
3928a8a2 1046 int ret;
bc0c38d1 1047
2b6080f2 1048 if (tr->stop_count)
b8de7bd1
SR
1049 return;
1050
4c11d7ae 1051 WARN_ON_ONCE(!irqs_disabled());
6c24499f 1052 if (!tr->allocated_snapshot) {
2930e04d 1053 /* Only the nop tracer should hit this when disabling */
9e8529af 1054 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 1055 return;
2930e04d 1056 }
ef710e10 1057
0b9b12c1 1058 arch_spin_lock(&tr->max_lock);
bc0c38d1 1059
12883efb 1060 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 1061
e8165dbb
SR
1062 if (ret == -EBUSY) {
1063 /*
1064 * We failed to swap the buffer due to a commit taking
1065 * place on this CPU. We fail to record, but we reset
1066 * the max trace buffer (no one writes directly to it)
1067 * and flag that it failed.
1068 */
12883efb 1069 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
1070 "Failed to swap buffers due to commit in progress\n");
1071 }
1072
e8165dbb 1073 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
1074
1075 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1076 arch_spin_unlock(&tr->max_lock);
bc0c38d1 1077}
5d4a9dba 1078#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 1079
8b8b3683 1080static int wait_on_pipe(struct trace_iterator *iter)
0d5c6e1c 1081{
15693458
SRRH
1082 /* Iterators are static, they should be filled or empty */
1083 if (trace_buffer_iter(iter, iter->cpu_file))
8b8b3683 1084 return 0;
0d5c6e1c 1085
8b8b3683 1086 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
0d5c6e1c
SR
1087}
1088
f4e781c0
SRRH
1089#ifdef CONFIG_FTRACE_STARTUP_TEST
1090static int run_tracer_selftest(struct tracer *type)
1091{
1092 struct trace_array *tr = &global_trace;
1093 struct tracer *saved_tracer = tr->current_trace;
1094 int ret;
0d5c6e1c 1095
f4e781c0
SRRH
1096 if (!type->selftest || tracing_selftest_disabled)
1097 return 0;
0d5c6e1c
SR
1098
1099 /*
f4e781c0
SRRH
1100 * Run a selftest on this tracer.
1101 * Here we reset the trace buffer, and set the current
1102 * tracer to be this tracer. The tracer can then run some
1103 * internal tracing to verify that everything is in order.
1104 * If we fail, we do not register this tracer.
0d5c6e1c 1105 */
f4e781c0 1106 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 1107
f4e781c0
SRRH
1108 tr->current_trace = type;
1109
1110#ifdef CONFIG_TRACER_MAX_TRACE
1111 if (type->use_max_tr) {
1112 /* If we expanded the buffers, make sure the max is expanded too */
1113 if (ring_buffer_expanded)
1114 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1115 RING_BUFFER_ALL_CPUS);
1116 tr->allocated_snapshot = true;
1117 }
1118#endif
1119
1120 /* the test is responsible for initializing and enabling */
1121 pr_info("Testing tracer %s: ", type->name);
1122 ret = type->selftest(type, tr);
1123 /* the test is responsible for resetting too */
1124 tr->current_trace = saved_tracer;
1125 if (ret) {
1126 printk(KERN_CONT "FAILED!\n");
1127 /* Add the warning after printing 'FAILED' */
1128 WARN_ON(1);
1129 return -1;
1130 }
1131 /* Only reset on passing, to avoid touching corrupted buffers */
1132 tracing_reset_online_cpus(&tr->trace_buffer);
1133
1134#ifdef CONFIG_TRACER_MAX_TRACE
1135 if (type->use_max_tr) {
1136 tr->allocated_snapshot = false;
0d5c6e1c 1137
f4e781c0
SRRH
1138 /* Shrink the max buffer again */
1139 if (ring_buffer_expanded)
1140 ring_buffer_resize(tr->max_buffer.buffer, 1,
1141 RING_BUFFER_ALL_CPUS);
1142 }
1143#endif
1144
1145 printk(KERN_CONT "PASSED\n");
1146 return 0;
1147}
1148#else
1149static inline int run_tracer_selftest(struct tracer *type)
1150{
1151 return 0;
0d5c6e1c 1152}
f4e781c0 1153#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1154
4fcdae83
SR
1155/**
1156 * register_tracer - register a tracer with the ftrace system.
1157 * @type - the plugin for the tracer
1158 *
1159 * Register a new plugin tracer.
1160 */
bc0c38d1
SR
1161int register_tracer(struct tracer *type)
1162{
1163 struct tracer *t;
bc0c38d1
SR
1164 int ret = 0;
1165
1166 if (!type->name) {
1167 pr_info("Tracer must have a name\n");
1168 return -1;
1169 }
1170
24a461d5 1171 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1172 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1173 return -1;
1174 }
1175
bc0c38d1 1176 mutex_lock(&trace_types_lock);
86fa2f60 1177
8e1b82e0
FW
1178 tracing_selftest_running = true;
1179
bc0c38d1
SR
1180 for (t = trace_types; t; t = t->next) {
1181 if (strcmp(type->name, t->name) == 0) {
1182 /* already found */
ee6c2c1b 1183 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1184 type->name);
1185 ret = -1;
1186 goto out;
1187 }
1188 }
1189
adf9f195
FW
1190 if (!type->set_flag)
1191 type->set_flag = &dummy_set_flag;
1192 if (!type->flags)
1193 type->flags = &dummy_tracer_flags;
1194 else
1195 if (!type->flags->opts)
1196 type->flags->opts = dummy_tracer_opt;
6eaaa5d5 1197
f4e781c0
SRRH
1198 ret = run_tracer_selftest(type);
1199 if (ret < 0)
1200 goto out;
60a11774 1201
bc0c38d1
SR
1202 type->next = trace_types;
1203 trace_types = type;
60a11774 1204
bc0c38d1 1205 out:
8e1b82e0 1206 tracing_selftest_running = false;
bc0c38d1
SR
1207 mutex_unlock(&trace_types_lock);
1208
dac74940
SR
1209 if (ret || !default_bootup_tracer)
1210 goto out_unlock;
1211
ee6c2c1b 1212 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1213 goto out_unlock;
1214
1215 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1216 /* Do we want this tracer to start on bootup? */
607e2ea1 1217 tracing_set_tracer(&global_trace, type->name);
dac74940
SR
1218 default_bootup_tracer = NULL;
1219 /* disable other selftests, since this will break it. */
55034cd6 1220 tracing_selftest_disabled = true;
b2821ae6 1221#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1222 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1223 type->name);
b2821ae6 1224#endif
b2821ae6 1225
dac74940 1226 out_unlock:
bc0c38d1
SR
1227 return ret;
1228}
1229
12883efb 1230void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1231{
12883efb 1232 struct ring_buffer *buffer = buf->buffer;
f633903a 1233
a5416411
HT
1234 if (!buffer)
1235 return;
1236
f633903a
SR
1237 ring_buffer_record_disable(buffer);
1238
1239 /* Make sure all commits have finished */
1240 synchronize_sched();
68179686 1241 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1242
1243 ring_buffer_record_enable(buffer);
1244}
1245
12883efb 1246void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1247{
12883efb 1248 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1249 int cpu;
1250
a5416411
HT
1251 if (!buffer)
1252 return;
1253
621968cd
SR
1254 ring_buffer_record_disable(buffer);
1255
1256 /* Make sure all commits have finished */
1257 synchronize_sched();
1258
9457158b 1259 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
213cc060
PE
1260
1261 for_each_online_cpu(cpu)
68179686 1262 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1263
1264 ring_buffer_record_enable(buffer);
213cc060
PE
1265}
1266
09d8091c 1267/* Must have trace_types_lock held */
873c642f 1268void tracing_reset_all_online_cpus(void)
9456f0fa 1269{
873c642f
SRRH
1270 struct trace_array *tr;
1271
873c642f 1272 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
12883efb
SRRH
1273 tracing_reset_online_cpus(&tr->trace_buffer);
1274#ifdef CONFIG_TRACER_MAX_TRACE
1275 tracing_reset_online_cpus(&tr->max_buffer);
1276#endif
873c642f 1277 }
9456f0fa
SR
1278}
1279
939c7a4f 1280#define SAVED_CMDLINES_DEFAULT 128
2c7eea4c 1281#define NO_CMDLINE_MAP UINT_MAX
edc35bd7 1282static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
939c7a4f
YY
1283struct saved_cmdlines_buffer {
1284 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1285 unsigned *map_cmdline_to_pid;
1286 unsigned cmdline_num;
1287 int cmdline_idx;
1288 char *saved_cmdlines;
1289};
1290static struct saved_cmdlines_buffer *savedcmd;
25b0b44a 1291
25b0b44a 1292/* temporary disable recording */
4fd27358 1293static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1 1294
939c7a4f
YY
1295static inline char *get_saved_cmdlines(int idx)
1296{
1297 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1298}
1299
1300static inline void set_cmdline(int idx, const char *cmdline)
bc0c38d1 1301{
939c7a4f
YY
1302 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1303}
1304
1305static int allocate_cmdlines_buffer(unsigned int val,
1306 struct saved_cmdlines_buffer *s)
1307{
1308 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1309 GFP_KERNEL);
1310 if (!s->map_cmdline_to_pid)
1311 return -ENOMEM;
1312
1313 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1314 if (!s->saved_cmdlines) {
1315 kfree(s->map_cmdline_to_pid);
1316 return -ENOMEM;
1317 }
1318
1319 s->cmdline_idx = 0;
1320 s->cmdline_num = val;
1321 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1322 sizeof(s->map_pid_to_cmdline));
1323 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1324 val * sizeof(*s->map_cmdline_to_pid));
1325
1326 return 0;
1327}
1328
1329static int trace_create_savedcmd(void)
1330{
1331 int ret;
1332
a6af8fbf 1333 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
939c7a4f
YY
1334 if (!savedcmd)
1335 return -ENOMEM;
1336
1337 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1338 if (ret < 0) {
1339 kfree(savedcmd);
1340 savedcmd = NULL;
1341 return -ENOMEM;
1342 }
1343
1344 return 0;
bc0c38d1
SR
1345}
1346
b5130b1e
CE
1347int is_tracing_stopped(void)
1348{
2b6080f2 1349 return global_trace.stop_count;
b5130b1e
CE
1350}
1351
0f048701
SR
1352/**
1353 * tracing_start - quick start of the tracer
1354 *
1355 * If tracing is enabled but was stopped by tracing_stop,
1356 * this will start the tracer back up.
1357 */
1358void tracing_start(void)
1359{
1360 struct ring_buffer *buffer;
1361 unsigned long flags;
1362
1363 if (tracing_disabled)
1364 return;
1365
2b6080f2
SR
1366 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1367 if (--global_trace.stop_count) {
1368 if (global_trace.stop_count < 0) {
b06a8301
SR
1369 /* Someone screwed up their debugging */
1370 WARN_ON_ONCE(1);
2b6080f2 1371 global_trace.stop_count = 0;
b06a8301 1372 }
0f048701
SR
1373 goto out;
1374 }
1375
a2f80714 1376 /* Prevent the buffers from switching */
0b9b12c1 1377 arch_spin_lock(&global_trace.max_lock);
0f048701 1378
12883efb 1379 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1380 if (buffer)
1381 ring_buffer_record_enable(buffer);
1382
12883efb
SRRH
1383#ifdef CONFIG_TRACER_MAX_TRACE
1384 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1385 if (buffer)
1386 ring_buffer_record_enable(buffer);
12883efb 1387#endif
0f048701 1388
0b9b12c1 1389 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1390
0f048701 1391 out:
2b6080f2
SR
1392 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1393}
1394
1395static void tracing_start_tr(struct trace_array *tr)
1396{
1397 struct ring_buffer *buffer;
1398 unsigned long flags;
1399
1400 if (tracing_disabled)
1401 return;
1402
1403 /* If global, we need to also start the max tracer */
1404 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1405 return tracing_start();
1406
1407 raw_spin_lock_irqsave(&tr->start_lock, flags);
1408
1409 if (--tr->stop_count) {
1410 if (tr->stop_count < 0) {
1411 /* Someone screwed up their debugging */
1412 WARN_ON_ONCE(1);
1413 tr->stop_count = 0;
1414 }
1415 goto out;
1416 }
1417
12883efb 1418 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1419 if (buffer)
1420 ring_buffer_record_enable(buffer);
1421
1422 out:
1423 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1424}
1425
1426/**
1427 * tracing_stop - quick stop of the tracer
1428 *
1429 * Light weight way to stop tracing. Use in conjunction with
1430 * tracing_start.
1431 */
1432void tracing_stop(void)
1433{
1434 struct ring_buffer *buffer;
1435 unsigned long flags;
1436
2b6080f2
SR
1437 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1438 if (global_trace.stop_count++)
0f048701
SR
1439 goto out;
1440
a2f80714 1441 /* Prevent the buffers from switching */
0b9b12c1 1442 arch_spin_lock(&global_trace.max_lock);
a2f80714 1443
12883efb 1444 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1445 if (buffer)
1446 ring_buffer_record_disable(buffer);
1447
12883efb
SRRH
1448#ifdef CONFIG_TRACER_MAX_TRACE
1449 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1450 if (buffer)
1451 ring_buffer_record_disable(buffer);
12883efb 1452#endif
0f048701 1453
0b9b12c1 1454 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1455
0f048701 1456 out:
2b6080f2
SR
1457 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1458}
1459
1460static void tracing_stop_tr(struct trace_array *tr)
1461{
1462 struct ring_buffer *buffer;
1463 unsigned long flags;
1464
1465 /* If global, we need to also stop the max tracer */
1466 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1467 return tracing_stop();
1468
1469 raw_spin_lock_irqsave(&tr->start_lock, flags);
1470 if (tr->stop_count++)
1471 goto out;
1472
12883efb 1473 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1474 if (buffer)
1475 ring_buffer_record_disable(buffer);
1476
1477 out:
1478 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1479}
1480
e309b41d 1481void trace_stop_cmdline_recording(void);
bc0c38d1 1482
379cfdac 1483static int trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1484{
a635cf04 1485 unsigned pid, idx;
bc0c38d1
SR
1486
1487 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
379cfdac 1488 return 0;
bc0c38d1
SR
1489
1490 /*
1491 * It's not the end of the world if we don't get
1492 * the lock, but we also don't want to spin
1493 * nor do we want to disable interrupts,
1494 * so if we miss here, then better luck next time.
1495 */
0199c4e6 1496 if (!arch_spin_trylock(&trace_cmdline_lock))
379cfdac 1497 return 0;
bc0c38d1 1498
939c7a4f 1499 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2c7eea4c 1500 if (idx == NO_CMDLINE_MAP) {
939c7a4f 1501 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
bc0c38d1 1502
a635cf04
CE
1503 /*
1504 * Check whether the cmdline buffer at idx has a pid
1505 * mapped. We are going to overwrite that entry so we
1506 * need to clear the map_pid_to_cmdline. Otherwise we
1507 * would read the new comm for the old pid.
1508 */
939c7a4f 1509 pid = savedcmd->map_cmdline_to_pid[idx];
a635cf04 1510 if (pid != NO_CMDLINE_MAP)
939c7a4f 1511 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1512
939c7a4f
YY
1513 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1514 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
bc0c38d1 1515
939c7a4f 1516 savedcmd->cmdline_idx = idx;
bc0c38d1
SR
1517 }
1518
939c7a4f 1519 set_cmdline(idx, tsk->comm);
bc0c38d1 1520
0199c4e6 1521 arch_spin_unlock(&trace_cmdline_lock);
379cfdac
SRRH
1522
1523 return 1;
bc0c38d1
SR
1524}
1525
4c27e756 1526static void __trace_find_cmdline(int pid, char comm[])
bc0c38d1 1527{
bc0c38d1
SR
1528 unsigned map;
1529
4ca53085
SR
1530 if (!pid) {
1531 strcpy(comm, "<idle>");
1532 return;
1533 }
bc0c38d1 1534
74bf4076
SR
1535 if (WARN_ON_ONCE(pid < 0)) {
1536 strcpy(comm, "<XXX>");
1537 return;
1538 }
1539
4ca53085
SR
1540 if (pid > PID_MAX_DEFAULT) {
1541 strcpy(comm, "<...>");
1542 return;
1543 }
bc0c38d1 1544
939c7a4f 1545 map = savedcmd->map_pid_to_cmdline[pid];
50d88758 1546 if (map != NO_CMDLINE_MAP)
939c7a4f 1547 strcpy(comm, get_saved_cmdlines(map));
50d88758
TG
1548 else
1549 strcpy(comm, "<...>");
4c27e756
SRRH
1550}
1551
1552void trace_find_cmdline(int pid, char comm[])
1553{
1554 preempt_disable();
1555 arch_spin_lock(&trace_cmdline_lock);
1556
1557 __trace_find_cmdline(pid, comm);
bc0c38d1 1558
0199c4e6 1559 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1560 preempt_enable();
bc0c38d1
SR
1561}
1562
e309b41d 1563void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1564{
0fb9656d 1565 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
bc0c38d1
SR
1566 return;
1567
7ffbd48d
SR
1568 if (!__this_cpu_read(trace_cmdline_save))
1569 return;
1570
379cfdac
SRRH
1571 if (trace_save_cmdline(tsk))
1572 __this_cpu_write(trace_cmdline_save, false);
bc0c38d1
SR
1573}
1574
45dcd8b8 1575void
38697053
SR
1576tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1577 int pc)
bc0c38d1
SR
1578{
1579 struct task_struct *tsk = current;
bc0c38d1 1580
777e208d
SR
1581 entry->preempt_count = pc & 0xff;
1582 entry->pid = (tsk) ? tsk->pid : 0;
1583 entry->flags =
9244489a 1584#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1585 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1586#else
1587 TRACE_FLAG_IRQS_NOSUPPORT |
1588#endif
bc0c38d1
SR
1589 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1590 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
e5137b50
PZ
1591 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1592 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
bc0c38d1 1593}
f413cdb8 1594EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1595
e77405ad
SR
1596struct ring_buffer_event *
1597trace_buffer_lock_reserve(struct ring_buffer *buffer,
1598 int type,
1599 unsigned long len,
1600 unsigned long flags, int pc)
51a763dd
ACM
1601{
1602 struct ring_buffer_event *event;
1603
e77405ad 1604 event = ring_buffer_lock_reserve(buffer, len);
51a763dd
ACM
1605 if (event != NULL) {
1606 struct trace_entry *ent = ring_buffer_event_data(event);
1607
1608 tracing_generic_entry_update(ent, flags, pc);
1609 ent->type = type;
1610 }
1611
1612 return event;
1613}
51a763dd 1614
7ffbd48d
SR
1615void
1616__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1617{
1618 __this_cpu_write(trace_cmdline_save, true);
1619 ring_buffer_unlock_commit(buffer, event);
1620}
1621
e77405ad
SR
1622static inline void
1623__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1624 struct ring_buffer_event *event,
0d5c6e1c 1625 unsigned long flags, int pc)
51a763dd 1626{
7ffbd48d 1627 __buffer_unlock_commit(buffer, event);
51a763dd 1628
e77405ad
SR
1629 ftrace_trace_stack(buffer, flags, 6, pc);
1630 ftrace_trace_userstack(buffer, flags, pc);
07edf712
FW
1631}
1632
e77405ad
SR
1633void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1634 struct ring_buffer_event *event,
1635 unsigned long flags, int pc)
07edf712 1636{
0d5c6e1c 1637 __trace_buffer_unlock_commit(buffer, event, flags, pc);
51a763dd 1638}
0d5c6e1c 1639EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
51a763dd 1640
2c4a33ab
SRRH
1641static struct ring_buffer *temp_buffer;
1642
ccb469a1
SR
1643struct ring_buffer_event *
1644trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1645 struct ftrace_event_file *ftrace_file,
1646 int type, unsigned long len,
1647 unsigned long flags, int pc)
1648{
2c4a33ab
SRRH
1649 struct ring_buffer_event *entry;
1650
12883efb 1651 *current_rb = ftrace_file->tr->trace_buffer.buffer;
2c4a33ab 1652 entry = trace_buffer_lock_reserve(*current_rb,
ccb469a1 1653 type, len, flags, pc);
2c4a33ab
SRRH
1654 /*
1655 * If tracing is off, but we have triggers enabled
1656 * we still need to look at the event data. Use the temp_buffer
1657 * to store the trace event for the tigger to use. It's recusive
1658 * safe and will not be recorded anywhere.
1659 */
1660 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1661 *current_rb = temp_buffer;
1662 entry = trace_buffer_lock_reserve(*current_rb,
1663 type, len, flags, pc);
1664 }
1665 return entry;
ccb469a1
SR
1666}
1667EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1668
ef5580d0 1669struct ring_buffer_event *
e77405ad
SR
1670trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1671 int type, unsigned long len,
ef5580d0
SR
1672 unsigned long flags, int pc)
1673{
12883efb 1674 *current_rb = global_trace.trace_buffer.buffer;
e77405ad 1675 return trace_buffer_lock_reserve(*current_rb,
ef5580d0
SR
1676 type, len, flags, pc);
1677}
94487d6d 1678EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0 1679
e77405ad
SR
1680void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1681 struct ring_buffer_event *event,
ef5580d0
SR
1682 unsigned long flags, int pc)
1683{
0d5c6e1c 1684 __trace_buffer_unlock_commit(buffer, event, flags, pc);
07edf712 1685}
94487d6d 1686EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712 1687
0d5c6e1c
SR
1688void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1689 struct ring_buffer_event *event,
1690 unsigned long flags, int pc,
1691 struct pt_regs *regs)
1fd8df2c 1692{
7ffbd48d 1693 __buffer_unlock_commit(buffer, event);
1fd8df2c
MH
1694
1695 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1696 ftrace_trace_userstack(buffer, flags, pc);
1697}
0d5c6e1c 1698EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1fd8df2c 1699
e77405ad
SR
1700void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1701 struct ring_buffer_event *event)
77d9f465 1702{
e77405ad 1703 ring_buffer_discard_commit(buffer, event);
ef5580d0 1704}
12acd473 1705EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 1706
e309b41d 1707void
7be42151 1708trace_function(struct trace_array *tr,
38697053
SR
1709 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1710 int pc)
bc0c38d1 1711{
e1112b4d 1712 struct ftrace_event_call *call = &event_function;
12883efb 1713 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 1714 struct ring_buffer_event *event;
777e208d 1715 struct ftrace_entry *entry;
bc0c38d1 1716
d769041f 1717 /* If we are reading the ring buffer, don't trace */
dd17c8f7 1718 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
d769041f
SR
1719 return;
1720
e77405ad 1721 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
51a763dd 1722 flags, pc);
3928a8a2
SR
1723 if (!event)
1724 return;
1725 entry = ring_buffer_event_data(event);
777e208d
SR
1726 entry->ip = ip;
1727 entry->parent_ip = parent_ip;
e1112b4d 1728
f306cc82 1729 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 1730 __buffer_unlock_commit(buffer, event);
bc0c38d1
SR
1731}
1732
c0a0d0d3 1733#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
1734
1735#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1736struct ftrace_stack {
1737 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1738};
1739
1740static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1741static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1742
e77405ad 1743static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 1744 unsigned long flags,
1fd8df2c 1745 int skip, int pc, struct pt_regs *regs)
86387f7e 1746{
e1112b4d 1747 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1748 struct ring_buffer_event *event;
777e208d 1749 struct stack_entry *entry;
86387f7e 1750 struct stack_trace trace;
4a9bd3f1
SR
1751 int use_stack;
1752 int size = FTRACE_STACK_ENTRIES;
1753
1754 trace.nr_entries = 0;
1755 trace.skip = skip;
1756
1757 /*
1758 * Since events can happen in NMIs there's no safe way to
1759 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1760 * or NMI comes in, it will just have to use the default
1761 * FTRACE_STACK_SIZE.
1762 */
1763 preempt_disable_notrace();
1764
82146529 1765 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
1766 /*
1767 * We don't need any atomic variables, just a barrier.
1768 * If an interrupt comes in, we don't care, because it would
1769 * have exited and put the counter back to what we want.
1770 * We just need a barrier to keep gcc from moving things
1771 * around.
1772 */
1773 barrier();
1774 if (use_stack == 1) {
bdffd893 1775 trace.entries = this_cpu_ptr(ftrace_stack.calls);
4a9bd3f1
SR
1776 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1777
1778 if (regs)
1779 save_stack_trace_regs(regs, &trace);
1780 else
1781 save_stack_trace(&trace);
1782
1783 if (trace.nr_entries > size)
1784 size = trace.nr_entries;
1785 } else
1786 /* From now on, use_stack is a boolean */
1787 use_stack = 0;
1788
1789 size *= sizeof(unsigned long);
86387f7e 1790
e77405ad 1791 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
4a9bd3f1 1792 sizeof(*entry) + size, flags, pc);
3928a8a2 1793 if (!event)
4a9bd3f1
SR
1794 goto out;
1795 entry = ring_buffer_event_data(event);
86387f7e 1796
4a9bd3f1
SR
1797 memset(&entry->caller, 0, size);
1798
1799 if (use_stack)
1800 memcpy(&entry->caller, trace.entries,
1801 trace.nr_entries * sizeof(unsigned long));
1802 else {
1803 trace.max_entries = FTRACE_STACK_ENTRIES;
1804 trace.entries = entry->caller;
1805 if (regs)
1806 save_stack_trace_regs(regs, &trace);
1807 else
1808 save_stack_trace(&trace);
1809 }
1810
1811 entry->size = trace.nr_entries;
86387f7e 1812
f306cc82 1813 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 1814 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
1815
1816 out:
1817 /* Again, don't let gcc optimize things here */
1818 barrier();
82146529 1819 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
1820 preempt_enable_notrace();
1821
f0a920d5
IM
1822}
1823
1fd8df2c
MH
1824void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1825 int skip, int pc, struct pt_regs *regs)
1826{
1827 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1828 return;
1829
1830 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1831}
1832
e77405ad
SR
1833void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1834 int skip, int pc)
53614991
SR
1835{
1836 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1837 return;
1838
1fd8df2c 1839 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
53614991
SR
1840}
1841
c0a0d0d3
FW
1842void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1843 int pc)
38697053 1844{
12883efb 1845 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
38697053
SR
1846}
1847
03889384
SR
1848/**
1849 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 1850 * @skip: Number of functions to skip (helper handlers)
03889384 1851 */
c142be8e 1852void trace_dump_stack(int skip)
03889384
SR
1853{
1854 unsigned long flags;
1855
1856 if (tracing_disabled || tracing_selftest_running)
e36c5458 1857 return;
03889384
SR
1858
1859 local_save_flags(flags);
1860
c142be8e
SRRH
1861 /*
1862 * Skip 3 more, seems to get us at the caller of
1863 * this function.
1864 */
1865 skip += 3;
1866 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1867 flags, skip, preempt_count(), NULL);
03889384
SR
1868}
1869
91e86e56
SR
1870static DEFINE_PER_CPU(int, user_stack_count);
1871
e77405ad
SR
1872void
1873ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 1874{
e1112b4d 1875 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1876 struct ring_buffer_event *event;
02b67518
TE
1877 struct userstack_entry *entry;
1878 struct stack_trace trace;
02b67518
TE
1879
1880 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1881 return;
1882
b6345879
SR
1883 /*
1884 * NMIs can not handle page faults, even with fix ups.
1885 * The save user stack can (and often does) fault.
1886 */
1887 if (unlikely(in_nmi()))
1888 return;
02b67518 1889
91e86e56
SR
1890 /*
1891 * prevent recursion, since the user stack tracing may
1892 * trigger other kernel events.
1893 */
1894 preempt_disable();
1895 if (__this_cpu_read(user_stack_count))
1896 goto out;
1897
1898 __this_cpu_inc(user_stack_count);
1899
e77405ad 1900 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
51a763dd 1901 sizeof(*entry), flags, pc);
02b67518 1902 if (!event)
1dbd1951 1903 goto out_drop_count;
02b67518 1904 entry = ring_buffer_event_data(event);
02b67518 1905
48659d31 1906 entry->tgid = current->tgid;
02b67518
TE
1907 memset(&entry->caller, 0, sizeof(entry->caller));
1908
1909 trace.nr_entries = 0;
1910 trace.max_entries = FTRACE_STACK_ENTRIES;
1911 trace.skip = 0;
1912 trace.entries = entry->caller;
1913
1914 save_stack_trace_user(&trace);
f306cc82 1915 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 1916 __buffer_unlock_commit(buffer, event);
91e86e56 1917
1dbd1951 1918 out_drop_count:
91e86e56 1919 __this_cpu_dec(user_stack_count);
91e86e56
SR
1920 out:
1921 preempt_enable();
02b67518
TE
1922}
1923
4fd27358
HE
1924#ifdef UNUSED
1925static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1926{
7be42151 1927 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1928}
4fd27358 1929#endif /* UNUSED */
02b67518 1930
c0a0d0d3
FW
1931#endif /* CONFIG_STACKTRACE */
1932
07d777fe
SR
1933/* created for use with alloc_percpu */
1934struct trace_buffer_struct {
1935 char buffer[TRACE_BUF_SIZE];
1936};
1937
1938static struct trace_buffer_struct *trace_percpu_buffer;
1939static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1940static struct trace_buffer_struct *trace_percpu_irq_buffer;
1941static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1942
1943/*
1944 * The buffer used is dependent on the context. There is a per cpu
1945 * buffer for normal context, softirq contex, hard irq context and
1946 * for NMI context. Thise allows for lockless recording.
1947 *
1948 * Note, if the buffers failed to be allocated, then this returns NULL
1949 */
1950static char *get_trace_buf(void)
1951{
1952 struct trace_buffer_struct *percpu_buffer;
07d777fe
SR
1953
1954 /*
1955 * If we have allocated per cpu buffers, then we do not
1956 * need to do any locking.
1957 */
1958 if (in_nmi())
1959 percpu_buffer = trace_percpu_nmi_buffer;
1960 else if (in_irq())
1961 percpu_buffer = trace_percpu_irq_buffer;
1962 else if (in_softirq())
1963 percpu_buffer = trace_percpu_sirq_buffer;
1964 else
1965 percpu_buffer = trace_percpu_buffer;
1966
1967 if (!percpu_buffer)
1968 return NULL;
1969
d8a0349c 1970 return this_cpu_ptr(&percpu_buffer->buffer[0]);
07d777fe
SR
1971}
1972
1973static int alloc_percpu_trace_buffer(void)
1974{
1975 struct trace_buffer_struct *buffers;
1976 struct trace_buffer_struct *sirq_buffers;
1977 struct trace_buffer_struct *irq_buffers;
1978 struct trace_buffer_struct *nmi_buffers;
1979
1980 buffers = alloc_percpu(struct trace_buffer_struct);
1981 if (!buffers)
1982 goto err_warn;
1983
1984 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1985 if (!sirq_buffers)
1986 goto err_sirq;
1987
1988 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1989 if (!irq_buffers)
1990 goto err_irq;
1991
1992 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1993 if (!nmi_buffers)
1994 goto err_nmi;
1995
1996 trace_percpu_buffer = buffers;
1997 trace_percpu_sirq_buffer = sirq_buffers;
1998 trace_percpu_irq_buffer = irq_buffers;
1999 trace_percpu_nmi_buffer = nmi_buffers;
2000
2001 return 0;
2002
2003 err_nmi:
2004 free_percpu(irq_buffers);
2005 err_irq:
2006 free_percpu(sirq_buffers);
2007 err_sirq:
2008 free_percpu(buffers);
2009 err_warn:
2010 WARN(1, "Could not allocate percpu trace_printk buffer");
2011 return -ENOMEM;
2012}
2013
81698831
SR
2014static int buffers_allocated;
2015
07d777fe
SR
2016void trace_printk_init_buffers(void)
2017{
07d777fe
SR
2018 if (buffers_allocated)
2019 return;
2020
2021 if (alloc_percpu_trace_buffer())
2022 return;
2023
2184db46
SR
2024 /* trace_printk() is for debug use only. Don't use it in production. */
2025
2026 pr_warning("\n**********************************************************\n");
2027 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2028 pr_warning("** **\n");
2029 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2030 pr_warning("** **\n");
2031 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2032 pr_warning("** unsafe for produciton use. **\n");
2033 pr_warning("** **\n");
2034 pr_warning("** If you see this message and you are not debugging **\n");
2035 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2036 pr_warning("** **\n");
2037 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2038 pr_warning("**********************************************************\n");
07d777fe 2039
b382ede6
SR
2040 /* Expand the buffers to set size */
2041 tracing_update_buffers();
2042
07d777fe 2043 buffers_allocated = 1;
81698831
SR
2044
2045 /*
2046 * trace_printk_init_buffers() can be called by modules.
2047 * If that happens, then we need to start cmdline recording
2048 * directly here. If the global_trace.buffer is already
2049 * allocated here, then this was called by module code.
2050 */
12883efb 2051 if (global_trace.trace_buffer.buffer)
81698831
SR
2052 tracing_start_cmdline_record();
2053}
2054
2055void trace_printk_start_comm(void)
2056{
2057 /* Start tracing comms if trace printk is set */
2058 if (!buffers_allocated)
2059 return;
2060 tracing_start_cmdline_record();
2061}
2062
2063static void trace_printk_start_stop_comm(int enabled)
2064{
2065 if (!buffers_allocated)
2066 return;
2067
2068 if (enabled)
2069 tracing_start_cmdline_record();
2070 else
2071 tracing_stop_cmdline_record();
07d777fe
SR
2072}
2073
769b0441 2074/**
48ead020 2075 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
2076 *
2077 */
40ce74f1 2078int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 2079{
e1112b4d 2080 struct ftrace_event_call *call = &event_bprint;
769b0441 2081 struct ring_buffer_event *event;
e77405ad 2082 struct ring_buffer *buffer;
769b0441 2083 struct trace_array *tr = &global_trace;
48ead020 2084 struct bprint_entry *entry;
769b0441 2085 unsigned long flags;
07d777fe
SR
2086 char *tbuffer;
2087 int len = 0, size, pc;
769b0441
FW
2088
2089 if (unlikely(tracing_selftest_running || tracing_disabled))
2090 return 0;
2091
2092 /* Don't pollute graph traces with trace_vprintk internals */
2093 pause_graph_tracing();
2094
2095 pc = preempt_count();
5168ae50 2096 preempt_disable_notrace();
769b0441 2097
07d777fe
SR
2098 tbuffer = get_trace_buf();
2099 if (!tbuffer) {
2100 len = 0;
769b0441 2101 goto out;
07d777fe 2102 }
769b0441 2103
07d777fe 2104 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 2105
07d777fe
SR
2106 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2107 goto out;
769b0441 2108
07d777fe 2109 local_save_flags(flags);
769b0441 2110 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 2111 buffer = tr->trace_buffer.buffer;
e77405ad
SR
2112 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2113 flags, pc);
769b0441 2114 if (!event)
07d777fe 2115 goto out;
769b0441
FW
2116 entry = ring_buffer_event_data(event);
2117 entry->ip = ip;
769b0441
FW
2118 entry->fmt = fmt;
2119
07d777fe 2120 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
f306cc82 2121 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2122 __buffer_unlock_commit(buffer, event);
d931369b
SR
2123 ftrace_trace_stack(buffer, flags, 6, pc);
2124 }
769b0441 2125
769b0441 2126out:
5168ae50 2127 preempt_enable_notrace();
769b0441
FW
2128 unpause_graph_tracing();
2129
2130 return len;
2131}
48ead020
FW
2132EXPORT_SYMBOL_GPL(trace_vbprintk);
2133
12883efb
SRRH
2134static int
2135__trace_array_vprintk(struct ring_buffer *buffer,
2136 unsigned long ip, const char *fmt, va_list args)
48ead020 2137{
e1112b4d 2138 struct ftrace_event_call *call = &event_print;
48ead020 2139 struct ring_buffer_event *event;
07d777fe 2140 int len = 0, size, pc;
48ead020 2141 struct print_entry *entry;
07d777fe
SR
2142 unsigned long flags;
2143 char *tbuffer;
48ead020
FW
2144
2145 if (tracing_disabled || tracing_selftest_running)
2146 return 0;
2147
07d777fe
SR
2148 /* Don't pollute graph traces with trace_vprintk internals */
2149 pause_graph_tracing();
2150
48ead020
FW
2151 pc = preempt_count();
2152 preempt_disable_notrace();
48ead020 2153
07d777fe
SR
2154
2155 tbuffer = get_trace_buf();
2156 if (!tbuffer) {
2157 len = 0;
48ead020 2158 goto out;
07d777fe 2159 }
48ead020 2160
07d777fe
SR
2161 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2162 if (len > TRACE_BUF_SIZE)
2163 goto out;
48ead020 2164
07d777fe 2165 local_save_flags(flags);
48ead020 2166 size = sizeof(*entry) + len + 1;
e77405ad 2167 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
07d777fe 2168 flags, pc);
48ead020 2169 if (!event)
07d777fe 2170 goto out;
48ead020 2171 entry = ring_buffer_event_data(event);
c13d2f7c 2172 entry->ip = ip;
48ead020 2173
07d777fe 2174 memcpy(&entry->buf, tbuffer, len);
c13d2f7c 2175 entry->buf[len] = '\0';
f306cc82 2176 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2177 __buffer_unlock_commit(buffer, event);
07d777fe 2178 ftrace_trace_stack(buffer, flags, 6, pc);
d931369b 2179 }
48ead020
FW
2180 out:
2181 preempt_enable_notrace();
07d777fe 2182 unpause_graph_tracing();
48ead020
FW
2183
2184 return len;
2185}
659372d3 2186
12883efb
SRRH
2187int trace_array_vprintk(struct trace_array *tr,
2188 unsigned long ip, const char *fmt, va_list args)
2189{
2190 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2191}
2192
2193int trace_array_printk(struct trace_array *tr,
2194 unsigned long ip, const char *fmt, ...)
2195{
2196 int ret;
2197 va_list ap;
2198
2199 if (!(trace_flags & TRACE_ITER_PRINTK))
2200 return 0;
2201
2202 va_start(ap, fmt);
2203 ret = trace_array_vprintk(tr, ip, fmt, ap);
2204 va_end(ap);
2205 return ret;
2206}
2207
2208int trace_array_printk_buf(struct ring_buffer *buffer,
2209 unsigned long ip, const char *fmt, ...)
2210{
2211 int ret;
2212 va_list ap;
2213
2214 if (!(trace_flags & TRACE_ITER_PRINTK))
2215 return 0;
2216
2217 va_start(ap, fmt);
2218 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2219 va_end(ap);
2220 return ret;
2221}
2222
659372d3
SR
2223int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2224{
a813a159 2225 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 2226}
769b0441
FW
2227EXPORT_SYMBOL_GPL(trace_vprintk);
2228
e2ac8ef5 2229static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 2230{
6d158a81
SR
2231 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2232
5a90f577 2233 iter->idx++;
6d158a81
SR
2234 if (buf_iter)
2235 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
2236}
2237
e309b41d 2238static struct trace_entry *
bc21b478
SR
2239peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2240 unsigned long *lost_events)
dd0e545f 2241{
3928a8a2 2242 struct ring_buffer_event *event;
6d158a81 2243 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 2244
d769041f
SR
2245 if (buf_iter)
2246 event = ring_buffer_iter_peek(buf_iter, ts);
2247 else
12883efb 2248 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 2249 lost_events);
d769041f 2250
4a9bd3f1
SR
2251 if (event) {
2252 iter->ent_size = ring_buffer_event_length(event);
2253 return ring_buffer_event_data(event);
2254 }
2255 iter->ent_size = 0;
2256 return NULL;
dd0e545f 2257}
d769041f 2258
dd0e545f 2259static struct trace_entry *
bc21b478
SR
2260__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2261 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 2262{
12883efb 2263 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 2264 struct trace_entry *ent, *next = NULL;
aa27497c 2265 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 2266 int cpu_file = iter->cpu_file;
3928a8a2 2267 u64 next_ts = 0, ts;
bc0c38d1 2268 int next_cpu = -1;
12b5da34 2269 int next_size = 0;
bc0c38d1
SR
2270 int cpu;
2271
b04cc6b1
FW
2272 /*
2273 * If we are in a per_cpu trace file, don't bother by iterating over
2274 * all cpu and peek directly.
2275 */
ae3b5093 2276 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
2277 if (ring_buffer_empty_cpu(buffer, cpu_file))
2278 return NULL;
bc21b478 2279 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
2280 if (ent_cpu)
2281 *ent_cpu = cpu_file;
2282
2283 return ent;
2284 }
2285
ab46428c 2286 for_each_tracing_cpu(cpu) {
dd0e545f 2287
3928a8a2
SR
2288 if (ring_buffer_empty_cpu(buffer, cpu))
2289 continue;
dd0e545f 2290
bc21b478 2291 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 2292
cdd31cd2
IM
2293 /*
2294 * Pick the entry with the smallest timestamp:
2295 */
3928a8a2 2296 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
2297 next = ent;
2298 next_cpu = cpu;
3928a8a2 2299 next_ts = ts;
bc21b478 2300 next_lost = lost_events;
12b5da34 2301 next_size = iter->ent_size;
bc0c38d1
SR
2302 }
2303 }
2304
12b5da34
SR
2305 iter->ent_size = next_size;
2306
bc0c38d1
SR
2307 if (ent_cpu)
2308 *ent_cpu = next_cpu;
2309
3928a8a2
SR
2310 if (ent_ts)
2311 *ent_ts = next_ts;
2312
bc21b478
SR
2313 if (missing_events)
2314 *missing_events = next_lost;
2315
bc0c38d1
SR
2316 return next;
2317}
2318
dd0e545f 2319/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
2320struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2321 int *ent_cpu, u64 *ent_ts)
bc0c38d1 2322{
bc21b478 2323 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
2324}
2325
2326/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 2327void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 2328{
bc21b478
SR
2329 iter->ent = __find_next_entry(iter, &iter->cpu,
2330 &iter->lost_events, &iter->ts);
dd0e545f 2331
3928a8a2 2332 if (iter->ent)
e2ac8ef5 2333 trace_iterator_increment(iter);
dd0e545f 2334
3928a8a2 2335 return iter->ent ? iter : NULL;
b3806b43 2336}
bc0c38d1 2337
e309b41d 2338static void trace_consume(struct trace_iterator *iter)
b3806b43 2339{
12883efb 2340 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 2341 &iter->lost_events);
bc0c38d1
SR
2342}
2343
e309b41d 2344static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
2345{
2346 struct trace_iterator *iter = m->private;
bc0c38d1 2347 int i = (int)*pos;
4e3c3333 2348 void *ent;
bc0c38d1 2349
a63ce5b3
SR
2350 WARN_ON_ONCE(iter->leftover);
2351
bc0c38d1
SR
2352 (*pos)++;
2353
2354 /* can't go backwards */
2355 if (iter->idx > i)
2356 return NULL;
2357
2358 if (iter->idx < 0)
955b61e5 2359 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2360 else
2361 ent = iter;
2362
2363 while (ent && iter->idx < i)
955b61e5 2364 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2365
2366 iter->pos = *pos;
2367
bc0c38d1
SR
2368 return ent;
2369}
2370
955b61e5 2371void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 2372{
2f26ebd5
SR
2373 struct ring_buffer_event *event;
2374 struct ring_buffer_iter *buf_iter;
2375 unsigned long entries = 0;
2376 u64 ts;
2377
12883efb 2378 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 2379
6d158a81
SR
2380 buf_iter = trace_buffer_iter(iter, cpu);
2381 if (!buf_iter)
2f26ebd5
SR
2382 return;
2383
2f26ebd5
SR
2384 ring_buffer_iter_reset(buf_iter);
2385
2386 /*
2387 * We could have the case with the max latency tracers
2388 * that a reset never took place on a cpu. This is evident
2389 * by the timestamp being before the start of the buffer.
2390 */
2391 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 2392 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
2393 break;
2394 entries++;
2395 ring_buffer_read(buf_iter, NULL);
2396 }
2397
12883efb 2398 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
2399}
2400
d7350c3f 2401/*
d7350c3f
FW
2402 * The current tracer is copied to avoid a global locking
2403 * all around.
2404 */
bc0c38d1
SR
2405static void *s_start(struct seq_file *m, loff_t *pos)
2406{
2407 struct trace_iterator *iter = m->private;
2b6080f2 2408 struct trace_array *tr = iter->tr;
b04cc6b1 2409 int cpu_file = iter->cpu_file;
bc0c38d1
SR
2410 void *p = NULL;
2411 loff_t l = 0;
3928a8a2 2412 int cpu;
bc0c38d1 2413
2fd196ec
HT
2414 /*
2415 * copy the tracer to avoid using a global lock all around.
2416 * iter->trace is a copy of current_trace, the pointer to the
2417 * name may be used instead of a strcmp(), as iter->trace->name
2418 * will point to the same string as current_trace->name.
2419 */
bc0c38d1 2420 mutex_lock(&trace_types_lock);
2b6080f2
SR
2421 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2422 *iter->trace = *tr->current_trace;
d7350c3f 2423 mutex_unlock(&trace_types_lock);
bc0c38d1 2424
12883efb 2425#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2426 if (iter->snapshot && iter->trace->use_max_tr)
2427 return ERR_PTR(-EBUSY);
12883efb 2428#endif
debdd57f
HT
2429
2430 if (!iter->snapshot)
2431 atomic_inc(&trace_record_cmdline_disabled);
bc0c38d1 2432
bc0c38d1
SR
2433 if (*pos != iter->pos) {
2434 iter->ent = NULL;
2435 iter->cpu = 0;
2436 iter->idx = -1;
2437
ae3b5093 2438 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2439 for_each_tracing_cpu(cpu)
2f26ebd5 2440 tracing_iter_reset(iter, cpu);
b04cc6b1 2441 } else
2f26ebd5 2442 tracing_iter_reset(iter, cpu_file);
bc0c38d1 2443
ac91d854 2444 iter->leftover = 0;
bc0c38d1
SR
2445 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2446 ;
2447
2448 } else {
a63ce5b3
SR
2449 /*
2450 * If we overflowed the seq_file before, then we want
2451 * to just reuse the trace_seq buffer again.
2452 */
2453 if (iter->leftover)
2454 p = iter;
2455 else {
2456 l = *pos - 1;
2457 p = s_next(m, p, &l);
2458 }
bc0c38d1
SR
2459 }
2460
4f535968 2461 trace_event_read_lock();
7e53bd42 2462 trace_access_lock(cpu_file);
bc0c38d1
SR
2463 return p;
2464}
2465
2466static void s_stop(struct seq_file *m, void *p)
2467{
7e53bd42
LJ
2468 struct trace_iterator *iter = m->private;
2469
12883efb 2470#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2471 if (iter->snapshot && iter->trace->use_max_tr)
2472 return;
12883efb 2473#endif
debdd57f
HT
2474
2475 if (!iter->snapshot)
2476 atomic_dec(&trace_record_cmdline_disabled);
12883efb 2477
7e53bd42 2478 trace_access_unlock(iter->cpu_file);
4f535968 2479 trace_event_read_unlock();
bc0c38d1
SR
2480}
2481
39eaf7ef 2482static void
12883efb
SRRH
2483get_total_entries(struct trace_buffer *buf,
2484 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
2485{
2486 unsigned long count;
2487 int cpu;
2488
2489 *total = 0;
2490 *entries = 0;
2491
2492 for_each_tracing_cpu(cpu) {
12883efb 2493 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
2494 /*
2495 * If this buffer has skipped entries, then we hold all
2496 * entries for the trace and we need to ignore the
2497 * ones before the time stamp.
2498 */
12883efb
SRRH
2499 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2500 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
2501 /* total is the same as the entries */
2502 *total += count;
2503 } else
2504 *total += count +
12883efb 2505 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
2506 *entries += count;
2507 }
2508}
2509
e309b41d 2510static void print_lat_help_header(struct seq_file *m)
bc0c38d1 2511{
d79ac28f
RV
2512 seq_puts(m, "# _------=> CPU# \n"
2513 "# / _-----=> irqs-off \n"
2514 "# | / _----=> need-resched \n"
2515 "# || / _---=> hardirq/softirq \n"
2516 "# ||| / _--=> preempt-depth \n"
2517 "# |||| / delay \n"
2518 "# cmd pid ||||| time | caller \n"
2519 "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
2520}
2521
12883efb 2522static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 2523{
39eaf7ef
SR
2524 unsigned long total;
2525 unsigned long entries;
2526
12883efb 2527 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
2528 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2529 entries, total, num_online_cpus());
2530 seq_puts(m, "#\n");
2531}
2532
12883efb 2533static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
39eaf7ef 2534{
12883efb 2535 print_event_info(buf, m);
d79ac28f
RV
2536 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2537 "# | | | | |\n");
bc0c38d1
SR
2538}
2539
12883efb 2540static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
77271ce4 2541{
12883efb 2542 print_event_info(buf, m);
d79ac28f
RV
2543 seq_puts(m, "# _-----=> irqs-off\n"
2544 "# / _----=> need-resched\n"
2545 "# | / _---=> hardirq/softirq\n"
2546 "# || / _--=> preempt-depth\n"
2547 "# ||| / delay\n"
2548 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2549 "# | | | |||| | |\n");
77271ce4 2550}
bc0c38d1 2551
62b915f1 2552void
bc0c38d1
SR
2553print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2554{
2555 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
2556 struct trace_buffer *buf = iter->trace_buffer;
2557 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 2558 struct tracer *type = iter->trace;
39eaf7ef
SR
2559 unsigned long entries;
2560 unsigned long total;
bc0c38d1
SR
2561 const char *name = "preemption";
2562
d840f718 2563 name = type->name;
bc0c38d1 2564
12883efb 2565 get_total_entries(buf, &total, &entries);
bc0c38d1 2566
888b55dc 2567 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 2568 name, UTS_RELEASE);
888b55dc 2569 seq_puts(m, "# -----------------------------------"
bc0c38d1 2570 "---------------------------------\n");
888b55dc 2571 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 2572 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 2573 nsecs_to_usecs(data->saved_latency),
bc0c38d1 2574 entries,
4c11d7ae 2575 total,
12883efb 2576 buf->cpu,
bc0c38d1
SR
2577#if defined(CONFIG_PREEMPT_NONE)
2578 "server",
2579#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2580 "desktop",
b5c21b45 2581#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
2582 "preempt",
2583#else
2584 "unknown",
2585#endif
2586 /* These are reserved for later use */
2587 0, 0, 0, 0);
2588#ifdef CONFIG_SMP
2589 seq_printf(m, " #P:%d)\n", num_online_cpus());
2590#else
2591 seq_puts(m, ")\n");
2592#endif
888b55dc
KM
2593 seq_puts(m, "# -----------------\n");
2594 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 2595 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
2596 data->comm, data->pid,
2597 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 2598 data->policy, data->rt_priority);
888b55dc 2599 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
2600
2601 if (data->critical_start) {
888b55dc 2602 seq_puts(m, "# => started at: ");
214023c3
SR
2603 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2604 trace_print_seq(m, &iter->seq);
888b55dc 2605 seq_puts(m, "\n# => ended at: ");
214023c3
SR
2606 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2607 trace_print_seq(m, &iter->seq);
8248ac05 2608 seq_puts(m, "\n#\n");
bc0c38d1
SR
2609 }
2610
888b55dc 2611 seq_puts(m, "#\n");
bc0c38d1
SR
2612}
2613
a309720c
SR
2614static void test_cpu_buff_start(struct trace_iterator *iter)
2615{
2616 struct trace_seq *s = &iter->seq;
2617
12ef7d44
SR
2618 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2619 return;
2620
2621 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2622 return;
2623
4462344e 2624 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
2625 return;
2626
12883efb 2627 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
2628 return;
2629
4462344e 2630 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
2631
2632 /* Don't print started cpu buffer for the first entry of the trace */
2633 if (iter->idx > 1)
2634 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2635 iter->cpu);
a309720c
SR
2636}
2637
2c4f035f 2638static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 2639{
214023c3 2640 struct trace_seq *s = &iter->seq;
bc0c38d1 2641 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 2642 struct trace_entry *entry;
f633cef0 2643 struct trace_event *event;
bc0c38d1 2644
4e3c3333 2645 entry = iter->ent;
dd0e545f 2646
a309720c
SR
2647 test_cpu_buff_start(iter);
2648
c4a8e8be 2649 event = ftrace_find_event(entry->type);
bc0c38d1 2650
c4a8e8be 2651 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
2652 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2653 trace_print_lat_context(iter);
2654 else
2655 trace_print_context(iter);
c4a8e8be 2656 }
bc0c38d1 2657
19a7fe20
SRRH
2658 if (trace_seq_has_overflowed(s))
2659 return TRACE_TYPE_PARTIAL_LINE;
2660
268ccda0 2661 if (event)
a9a57763 2662 return event->funcs->trace(iter, sym_flags, event);
d9793bd8 2663
19a7fe20 2664 trace_seq_printf(s, "Unknown type %d\n", entry->type);
02b67518 2665
19a7fe20 2666 return trace_handle_return(s);
bc0c38d1
SR
2667}
2668
2c4f035f 2669static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
2670{
2671 struct trace_seq *s = &iter->seq;
2672 struct trace_entry *entry;
f633cef0 2673 struct trace_event *event;
f9896bf3
IM
2674
2675 entry = iter->ent;
dd0e545f 2676
19a7fe20
SRRH
2677 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2678 trace_seq_printf(s, "%d %d %llu ",
2679 entry->pid, iter->cpu, iter->ts);
2680
2681 if (trace_seq_has_overflowed(s))
2682 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3 2683
f633cef0 2684 event = ftrace_find_event(entry->type);
268ccda0 2685 if (event)
a9a57763 2686 return event->funcs->raw(iter, 0, event);
d9793bd8 2687
19a7fe20 2688 trace_seq_printf(s, "%d ?\n", entry->type);
777e208d 2689
19a7fe20 2690 return trace_handle_return(s);
f9896bf3
IM
2691}
2692
2c4f035f 2693static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
2694{
2695 struct trace_seq *s = &iter->seq;
2696 unsigned char newline = '\n';
2697 struct trace_entry *entry;
f633cef0 2698 struct trace_event *event;
5e3ca0ec
IM
2699
2700 entry = iter->ent;
dd0e545f 2701
c4a8e8be 2702 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
2703 SEQ_PUT_HEX_FIELD(s, entry->pid);
2704 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2705 SEQ_PUT_HEX_FIELD(s, iter->ts);
2706 if (trace_seq_has_overflowed(s))
2707 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 2708 }
5e3ca0ec 2709
f633cef0 2710 event = ftrace_find_event(entry->type);
268ccda0 2711 if (event) {
a9a57763 2712 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
2713 if (ret != TRACE_TYPE_HANDLED)
2714 return ret;
2715 }
7104f300 2716
19a7fe20 2717 SEQ_PUT_FIELD(s, newline);
5e3ca0ec 2718
19a7fe20 2719 return trace_handle_return(s);
5e3ca0ec
IM
2720}
2721
2c4f035f 2722static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
2723{
2724 struct trace_seq *s = &iter->seq;
2725 struct trace_entry *entry;
f633cef0 2726 struct trace_event *event;
cb0f12aa
IM
2727
2728 entry = iter->ent;
dd0e545f 2729
c4a8e8be 2730 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
2731 SEQ_PUT_FIELD(s, entry->pid);
2732 SEQ_PUT_FIELD(s, iter->cpu);
2733 SEQ_PUT_FIELD(s, iter->ts);
2734 if (trace_seq_has_overflowed(s))
2735 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 2736 }
cb0f12aa 2737
f633cef0 2738 event = ftrace_find_event(entry->type);
a9a57763
SR
2739 return event ? event->funcs->binary(iter, 0, event) :
2740 TRACE_TYPE_HANDLED;
cb0f12aa
IM
2741}
2742
62b915f1 2743int trace_empty(struct trace_iterator *iter)
bc0c38d1 2744{
6d158a81 2745 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
2746 int cpu;
2747
9aba60fe 2748 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 2749 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 2750 cpu = iter->cpu_file;
6d158a81
SR
2751 buf_iter = trace_buffer_iter(iter, cpu);
2752 if (buf_iter) {
2753 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
2754 return 0;
2755 } else {
12883efb 2756 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
2757 return 0;
2758 }
2759 return 1;
2760 }
2761
ab46428c 2762 for_each_tracing_cpu(cpu) {
6d158a81
SR
2763 buf_iter = trace_buffer_iter(iter, cpu);
2764 if (buf_iter) {
2765 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
2766 return 0;
2767 } else {
12883efb 2768 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
2769 return 0;
2770 }
bc0c38d1 2771 }
d769041f 2772
797d3712 2773 return 1;
bc0c38d1
SR
2774}
2775
4f535968 2776/* Called with trace_event_read_lock() held. */
955b61e5 2777enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 2778{
2c4f035f
FW
2779 enum print_line_t ret;
2780
19a7fe20
SRRH
2781 if (iter->lost_events) {
2782 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2783 iter->cpu, iter->lost_events);
2784 if (trace_seq_has_overflowed(&iter->seq))
2785 return TRACE_TYPE_PARTIAL_LINE;
2786 }
bc21b478 2787
2c4f035f
FW
2788 if (iter->trace && iter->trace->print_line) {
2789 ret = iter->trace->print_line(iter);
2790 if (ret != TRACE_TYPE_UNHANDLED)
2791 return ret;
2792 }
72829bc3 2793
09ae7234
SRRH
2794 if (iter->ent->type == TRACE_BPUTS &&
2795 trace_flags & TRACE_ITER_PRINTK &&
2796 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2797 return trace_print_bputs_msg_only(iter);
2798
48ead020
FW
2799 if (iter->ent->type == TRACE_BPRINT &&
2800 trace_flags & TRACE_ITER_PRINTK &&
2801 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2802 return trace_print_bprintk_msg_only(iter);
48ead020 2803
66896a85
FW
2804 if (iter->ent->type == TRACE_PRINT &&
2805 trace_flags & TRACE_ITER_PRINTK &&
2806 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2807 return trace_print_printk_msg_only(iter);
66896a85 2808
cb0f12aa
IM
2809 if (trace_flags & TRACE_ITER_BIN)
2810 return print_bin_fmt(iter);
2811
5e3ca0ec
IM
2812 if (trace_flags & TRACE_ITER_HEX)
2813 return print_hex_fmt(iter);
2814
f9896bf3
IM
2815 if (trace_flags & TRACE_ITER_RAW)
2816 return print_raw_fmt(iter);
2817
f9896bf3
IM
2818 return print_trace_fmt(iter);
2819}
2820
7e9a49ef
JO
2821void trace_latency_header(struct seq_file *m)
2822{
2823 struct trace_iterator *iter = m->private;
2824
2825 /* print nothing if the buffers are empty */
2826 if (trace_empty(iter))
2827 return;
2828
2829 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2830 print_trace_header(m, iter);
2831
2832 if (!(trace_flags & TRACE_ITER_VERBOSE))
2833 print_lat_help_header(m);
2834}
2835
62b915f1
JO
2836void trace_default_header(struct seq_file *m)
2837{
2838 struct trace_iterator *iter = m->private;
2839
f56e7f8e
JO
2840 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2841 return;
2842
62b915f1
JO
2843 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2844 /* print nothing if the buffers are empty */
2845 if (trace_empty(iter))
2846 return;
2847 print_trace_header(m, iter);
2848 if (!(trace_flags & TRACE_ITER_VERBOSE))
2849 print_lat_help_header(m);
2850 } else {
77271ce4
SR
2851 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2852 if (trace_flags & TRACE_ITER_IRQ_INFO)
12883efb 2853 print_func_help_header_irq(iter->trace_buffer, m);
77271ce4 2854 else
12883efb 2855 print_func_help_header(iter->trace_buffer, m);
77271ce4 2856 }
62b915f1
JO
2857 }
2858}
2859
e0a413f6
SR
2860static void test_ftrace_alive(struct seq_file *m)
2861{
2862 if (!ftrace_is_dead())
2863 return;
d79ac28f
RV
2864 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2865 "# MAY BE MISSING FUNCTION EVENTS\n");
e0a413f6
SR
2866}
2867
d8741e2e 2868#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 2869static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 2870{
d79ac28f
RV
2871 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2872 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2873 "# Takes a snapshot of the main buffer.\n"
2874 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2875 "# (Doesn't have to be '2' works with any number that\n"
2876 "# is not a '0' or '1')\n");
d8741e2e 2877}
f1affcaa
SRRH
2878
2879static void show_snapshot_percpu_help(struct seq_file *m)
2880{
fa6f0cc7 2881 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
f1affcaa 2882#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
d79ac28f
RV
2883 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2884 "# Takes a snapshot of the main buffer for this cpu.\n");
f1affcaa 2885#else
d79ac28f
RV
2886 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2887 "# Must use main snapshot file to allocate.\n");
f1affcaa 2888#endif
d79ac28f
RV
2889 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2890 "# (Doesn't have to be '2' works with any number that\n"
2891 "# is not a '0' or '1')\n");
f1affcaa
SRRH
2892}
2893
d8741e2e
SRRH
2894static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2895{
45ad21ca 2896 if (iter->tr->allocated_snapshot)
fa6f0cc7 2897 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
d8741e2e 2898 else
fa6f0cc7 2899 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
d8741e2e 2900
fa6f0cc7 2901 seq_puts(m, "# Snapshot commands:\n");
f1affcaa
SRRH
2902 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2903 show_snapshot_main_help(m);
2904 else
2905 show_snapshot_percpu_help(m);
d8741e2e
SRRH
2906}
2907#else
2908/* Should never be called */
2909static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2910#endif
2911
bc0c38d1
SR
2912static int s_show(struct seq_file *m, void *v)
2913{
2914 struct trace_iterator *iter = v;
a63ce5b3 2915 int ret;
bc0c38d1
SR
2916
2917 if (iter->ent == NULL) {
2918 if (iter->tr) {
2919 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2920 seq_puts(m, "#\n");
e0a413f6 2921 test_ftrace_alive(m);
bc0c38d1 2922 }
d8741e2e
SRRH
2923 if (iter->snapshot && trace_empty(iter))
2924 print_snapshot_help(m, iter);
2925 else if (iter->trace && iter->trace->print_header)
8bba1bf5 2926 iter->trace->print_header(m);
62b915f1
JO
2927 else
2928 trace_default_header(m);
2929
a63ce5b3
SR
2930 } else if (iter->leftover) {
2931 /*
2932 * If we filled the seq_file buffer earlier, we
2933 * want to just show it now.
2934 */
2935 ret = trace_print_seq(m, &iter->seq);
2936
2937 /* ret should this time be zero, but you never know */
2938 iter->leftover = ret;
2939
bc0c38d1 2940 } else {
f9896bf3 2941 print_trace_line(iter);
a63ce5b3
SR
2942 ret = trace_print_seq(m, &iter->seq);
2943 /*
2944 * If we overflow the seq_file buffer, then it will
2945 * ask us for this data again at start up.
2946 * Use that instead.
2947 * ret is 0 if seq_file write succeeded.
2948 * -1 otherwise.
2949 */
2950 iter->leftover = ret;
bc0c38d1
SR
2951 }
2952
2953 return 0;
2954}
2955
649e9c70
ON
2956/*
2957 * Should be used after trace_array_get(), trace_types_lock
2958 * ensures that i_cdev was already initialized.
2959 */
2960static inline int tracing_get_cpu(struct inode *inode)
2961{
2962 if (inode->i_cdev) /* See trace_create_cpu_file() */
2963 return (long)inode->i_cdev - 1;
2964 return RING_BUFFER_ALL_CPUS;
2965}
2966
88e9d34c 2967static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
2968 .start = s_start,
2969 .next = s_next,
2970 .stop = s_stop,
2971 .show = s_show,
bc0c38d1
SR
2972};
2973
e309b41d 2974static struct trace_iterator *
6484c71c 2975__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 2976{
6484c71c 2977 struct trace_array *tr = inode->i_private;
bc0c38d1 2978 struct trace_iterator *iter;
50e18b94 2979 int cpu;
bc0c38d1 2980
85a2f9b4
SR
2981 if (tracing_disabled)
2982 return ERR_PTR(-ENODEV);
60a11774 2983
50e18b94 2984 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
2985 if (!iter)
2986 return ERR_PTR(-ENOMEM);
bc0c38d1 2987
6d158a81
SR
2988 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2989 GFP_KERNEL);
93574fcc
DC
2990 if (!iter->buffer_iter)
2991 goto release;
2992
d7350c3f
FW
2993 /*
2994 * We make a copy of the current tracer to avoid concurrent
2995 * changes on it while we are reading.
2996 */
bc0c38d1 2997 mutex_lock(&trace_types_lock);
d7350c3f 2998 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 2999 if (!iter->trace)
d7350c3f 3000 goto fail;
85a2f9b4 3001
2b6080f2 3002 *iter->trace = *tr->current_trace;
d7350c3f 3003
79f55997 3004 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
3005 goto fail;
3006
12883efb
SRRH
3007 iter->tr = tr;
3008
3009#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3010 /* Currently only the top directory has a snapshot */
3011 if (tr->current_trace->print_max || snapshot)
12883efb 3012 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 3013 else
12883efb
SRRH
3014#endif
3015 iter->trace_buffer = &tr->trace_buffer;
debdd57f 3016 iter->snapshot = snapshot;
bc0c38d1 3017 iter->pos = -1;
6484c71c 3018 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 3019 mutex_init(&iter->mutex);
bc0c38d1 3020
8bba1bf5
MM
3021 /* Notify the tracer early; before we stop tracing. */
3022 if (iter->trace && iter->trace->open)
a93751ca 3023 iter->trace->open(iter);
8bba1bf5 3024
12ef7d44 3025 /* Annotate start of buffers if we had overruns */
12883efb 3026 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
3027 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3028
8be0709f 3029 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 3030 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
3031 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3032
debdd57f
HT
3033 /* stop the trace while dumping if we are not opening "snapshot" */
3034 if (!iter->snapshot)
2b6080f2 3035 tracing_stop_tr(tr);
2f26ebd5 3036
ae3b5093 3037 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3038 for_each_tracing_cpu(cpu) {
b04cc6b1 3039 iter->buffer_iter[cpu] =
12883efb 3040 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3041 }
3042 ring_buffer_read_prepare_sync();
3043 for_each_tracing_cpu(cpu) {
3044 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3045 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
3046 }
3047 } else {
3048 cpu = iter->cpu_file;
3928a8a2 3049 iter->buffer_iter[cpu] =
12883efb 3050 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3051 ring_buffer_read_prepare_sync();
3052 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3053 tracing_iter_reset(iter, cpu);
3928a8a2
SR
3054 }
3055
bc0c38d1
SR
3056 mutex_unlock(&trace_types_lock);
3057
bc0c38d1 3058 return iter;
3928a8a2 3059
d7350c3f 3060 fail:
3928a8a2 3061 mutex_unlock(&trace_types_lock);
d7350c3f 3062 kfree(iter->trace);
6d158a81 3063 kfree(iter->buffer_iter);
93574fcc 3064release:
50e18b94
JO
3065 seq_release_private(inode, file);
3066 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
3067}
3068
3069int tracing_open_generic(struct inode *inode, struct file *filp)
3070{
60a11774
SR
3071 if (tracing_disabled)
3072 return -ENODEV;
3073
bc0c38d1
SR
3074 filp->private_data = inode->i_private;
3075 return 0;
3076}
3077
2e86421d
GB
3078bool tracing_is_disabled(void)
3079{
3080 return (tracing_disabled) ? true: false;
3081}
3082
7b85af63
SRRH
3083/*
3084 * Open and update trace_array ref count.
3085 * Must have the current trace_array passed to it.
3086 */
dcc30223 3087static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
7b85af63
SRRH
3088{
3089 struct trace_array *tr = inode->i_private;
3090
3091 if (tracing_disabled)
3092 return -ENODEV;
3093
3094 if (trace_array_get(tr) < 0)
3095 return -ENODEV;
3096
3097 filp->private_data = inode->i_private;
3098
3099 return 0;
7b85af63
SRRH
3100}
3101
4fd27358 3102static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 3103{
6484c71c 3104 struct trace_array *tr = inode->i_private;
907f2784 3105 struct seq_file *m = file->private_data;
4acd4d00 3106 struct trace_iterator *iter;
3928a8a2 3107 int cpu;
bc0c38d1 3108
ff451961 3109 if (!(file->f_mode & FMODE_READ)) {
6484c71c 3110 trace_array_put(tr);
4acd4d00 3111 return 0;
ff451961 3112 }
4acd4d00 3113
6484c71c 3114 /* Writes do not use seq_file */
4acd4d00 3115 iter = m->private;
bc0c38d1 3116 mutex_lock(&trace_types_lock);
a695cb58 3117
3928a8a2
SR
3118 for_each_tracing_cpu(cpu) {
3119 if (iter->buffer_iter[cpu])
3120 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3121 }
3122
bc0c38d1
SR
3123 if (iter->trace && iter->trace->close)
3124 iter->trace->close(iter);
3125
debdd57f
HT
3126 if (!iter->snapshot)
3127 /* reenable tracing if it was previously enabled */
2b6080f2 3128 tracing_start_tr(tr);
f77d09a3
AL
3129
3130 __trace_array_put(tr);
3131
bc0c38d1
SR
3132 mutex_unlock(&trace_types_lock);
3133
d7350c3f 3134 mutex_destroy(&iter->mutex);
b0dfa978 3135 free_cpumask_var(iter->started);
d7350c3f 3136 kfree(iter->trace);
6d158a81 3137 kfree(iter->buffer_iter);
50e18b94 3138 seq_release_private(inode, file);
ff451961 3139
bc0c38d1
SR
3140 return 0;
3141}
3142
7b85af63
SRRH
3143static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3144{
3145 struct trace_array *tr = inode->i_private;
3146
3147 trace_array_put(tr);
bc0c38d1
SR
3148 return 0;
3149}
3150
7b85af63
SRRH
3151static int tracing_single_release_tr(struct inode *inode, struct file *file)
3152{
3153 struct trace_array *tr = inode->i_private;
3154
3155 trace_array_put(tr);
3156
3157 return single_release(inode, file);
3158}
3159
bc0c38d1
SR
3160static int tracing_open(struct inode *inode, struct file *file)
3161{
6484c71c 3162 struct trace_array *tr = inode->i_private;
85a2f9b4
SR
3163 struct trace_iterator *iter;
3164 int ret = 0;
bc0c38d1 3165
ff451961
SRRH
3166 if (trace_array_get(tr) < 0)
3167 return -ENODEV;
3168
4acd4d00 3169 /* If this file was open for write, then erase contents */
6484c71c
ON
3170 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3171 int cpu = tracing_get_cpu(inode);
3172
3173 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3174 tracing_reset_online_cpus(&tr->trace_buffer);
4acd4d00 3175 else
6484c71c 3176 tracing_reset(&tr->trace_buffer, cpu);
4acd4d00 3177 }
bc0c38d1 3178
4acd4d00 3179 if (file->f_mode & FMODE_READ) {
6484c71c 3180 iter = __tracing_open(inode, file, false);
4acd4d00
SR
3181 if (IS_ERR(iter))
3182 ret = PTR_ERR(iter);
3183 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3184 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3185 }
ff451961
SRRH
3186
3187 if (ret < 0)
3188 trace_array_put(tr);
3189
bc0c38d1
SR
3190 return ret;
3191}
3192
607e2ea1
SRRH
3193/*
3194 * Some tracers are not suitable for instance buffers.
3195 * A tracer is always available for the global array (toplevel)
3196 * or if it explicitly states that it is.
3197 */
3198static bool
3199trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3200{
3201 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3202}
3203
3204/* Find the next tracer that this trace array may use */
3205static struct tracer *
3206get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3207{
3208 while (t && !trace_ok_for_array(t, tr))
3209 t = t->next;
3210
3211 return t;
3212}
3213
e309b41d 3214static void *
bc0c38d1
SR
3215t_next(struct seq_file *m, void *v, loff_t *pos)
3216{
607e2ea1 3217 struct trace_array *tr = m->private;
f129e965 3218 struct tracer *t = v;
bc0c38d1
SR
3219
3220 (*pos)++;
3221
3222 if (t)
607e2ea1 3223 t = get_tracer_for_array(tr, t->next);
bc0c38d1 3224
bc0c38d1
SR
3225 return t;
3226}
3227
3228static void *t_start(struct seq_file *m, loff_t *pos)
3229{
607e2ea1 3230 struct trace_array *tr = m->private;
f129e965 3231 struct tracer *t;
bc0c38d1
SR
3232 loff_t l = 0;
3233
3234 mutex_lock(&trace_types_lock);
607e2ea1
SRRH
3235
3236 t = get_tracer_for_array(tr, trace_types);
3237 for (; t && l < *pos; t = t_next(m, t, &l))
3238 ;
bc0c38d1
SR
3239
3240 return t;
3241}
3242
3243static void t_stop(struct seq_file *m, void *p)
3244{
3245 mutex_unlock(&trace_types_lock);
3246}
3247
3248static int t_show(struct seq_file *m, void *v)
3249{
3250 struct tracer *t = v;
3251
3252 if (!t)
3253 return 0;
3254
fa6f0cc7 3255 seq_puts(m, t->name);
bc0c38d1
SR
3256 if (t->next)
3257 seq_putc(m, ' ');
3258 else
3259 seq_putc(m, '\n');
3260
3261 return 0;
3262}
3263
88e9d34c 3264static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
3265 .start = t_start,
3266 .next = t_next,
3267 .stop = t_stop,
3268 .show = t_show,
bc0c38d1
SR
3269};
3270
3271static int show_traces_open(struct inode *inode, struct file *file)
3272{
607e2ea1
SRRH
3273 struct trace_array *tr = inode->i_private;
3274 struct seq_file *m;
3275 int ret;
3276
60a11774
SR
3277 if (tracing_disabled)
3278 return -ENODEV;
3279
607e2ea1
SRRH
3280 ret = seq_open(file, &show_traces_seq_ops);
3281 if (ret)
3282 return ret;
3283
3284 m = file->private_data;
3285 m->private = tr;
3286
3287 return 0;
bc0c38d1
SR
3288}
3289
4acd4d00
SR
3290static ssize_t
3291tracing_write_stub(struct file *filp, const char __user *ubuf,
3292 size_t count, loff_t *ppos)
3293{
3294 return count;
3295}
3296
098c879e 3297loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
364829b1 3298{
098c879e
SRRH
3299 int ret;
3300
364829b1 3301 if (file->f_mode & FMODE_READ)
098c879e 3302 ret = seq_lseek(file, offset, whence);
364829b1 3303 else
098c879e
SRRH
3304 file->f_pos = ret = 0;
3305
3306 return ret;
364829b1
SP
3307}
3308
5e2336a0 3309static const struct file_operations tracing_fops = {
4bf39a94
IM
3310 .open = tracing_open,
3311 .read = seq_read,
4acd4d00 3312 .write = tracing_write_stub,
098c879e 3313 .llseek = tracing_lseek,
4bf39a94 3314 .release = tracing_release,
bc0c38d1
SR
3315};
3316
5e2336a0 3317static const struct file_operations show_traces_fops = {
c7078de1
IM
3318 .open = show_traces_open,
3319 .read = seq_read,
3320 .release = seq_release,
b444786f 3321 .llseek = seq_lseek,
c7078de1
IM
3322};
3323
36dfe925
IM
3324/*
3325 * The tracer itself will not take this lock, but still we want
3326 * to provide a consistent cpumask to user-space:
3327 */
3328static DEFINE_MUTEX(tracing_cpumask_update_lock);
3329
3330/*
3331 * Temporary storage for the character representation of the
3332 * CPU bitmask (and one more byte for the newline):
3333 */
3334static char mask_str[NR_CPUS + 1];
3335
c7078de1
IM
3336static ssize_t
3337tracing_cpumask_read(struct file *filp, char __user *ubuf,
3338 size_t count, loff_t *ppos)
3339{
ccfe9e42 3340 struct trace_array *tr = file_inode(filp)->i_private;
36dfe925 3341 int len;
c7078de1
IM
3342
3343 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 3344
ccfe9e42 3345 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
36dfe925
IM
3346 if (count - len < 2) {
3347 count = -EINVAL;
3348 goto out_err;
3349 }
3350 len += sprintf(mask_str + len, "\n");
3351 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3352
3353out_err:
c7078de1
IM
3354 mutex_unlock(&tracing_cpumask_update_lock);
3355
3356 return count;
3357}
3358
3359static ssize_t
3360tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3361 size_t count, loff_t *ppos)
3362{
ccfe9e42 3363 struct trace_array *tr = file_inode(filp)->i_private;
9e01c1b7 3364 cpumask_var_t tracing_cpumask_new;
2b6080f2 3365 int err, cpu;
9e01c1b7
RR
3366
3367 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3368 return -ENOMEM;
c7078de1 3369
9e01c1b7 3370 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 3371 if (err)
36dfe925
IM
3372 goto err_unlock;
3373
215368e8
LZ
3374 mutex_lock(&tracing_cpumask_update_lock);
3375
a5e25883 3376 local_irq_disable();
0b9b12c1 3377 arch_spin_lock(&tr->max_lock);
ab46428c 3378 for_each_tracing_cpu(cpu) {
36dfe925
IM
3379 /*
3380 * Increase/decrease the disabled counter if we are
3381 * about to flip a bit in the cpumask:
3382 */
ccfe9e42 3383 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 3384 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3385 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3386 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 3387 }
ccfe9e42 3388 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 3389 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3390 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3391 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
3392 }
3393 }
0b9b12c1 3394 arch_spin_unlock(&tr->max_lock);
a5e25883 3395 local_irq_enable();
36dfe925 3396
ccfe9e42 3397 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
3398
3399 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 3400 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
3401
3402 return count;
36dfe925
IM
3403
3404err_unlock:
215368e8 3405 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
3406
3407 return err;
c7078de1
IM
3408}
3409
5e2336a0 3410static const struct file_operations tracing_cpumask_fops = {
ccfe9e42 3411 .open = tracing_open_generic_tr,
c7078de1
IM
3412 .read = tracing_cpumask_read,
3413 .write = tracing_cpumask_write,
ccfe9e42 3414 .release = tracing_release_generic_tr,
b444786f 3415 .llseek = generic_file_llseek,
bc0c38d1
SR
3416};
3417
fdb372ed 3418static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 3419{
d8e83d26 3420 struct tracer_opt *trace_opts;
2b6080f2 3421 struct trace_array *tr = m->private;
d8e83d26 3422 u32 tracer_flags;
d8e83d26 3423 int i;
adf9f195 3424
d8e83d26 3425 mutex_lock(&trace_types_lock);
2b6080f2
SR
3426 tracer_flags = tr->current_trace->flags->val;
3427 trace_opts = tr->current_trace->flags->opts;
d8e83d26 3428
bc0c38d1
SR
3429 for (i = 0; trace_options[i]; i++) {
3430 if (trace_flags & (1 << i))
fdb372ed 3431 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 3432 else
fdb372ed 3433 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
3434 }
3435
adf9f195
FW
3436 for (i = 0; trace_opts[i].name; i++) {
3437 if (tracer_flags & trace_opts[i].bit)
fdb372ed 3438 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 3439 else
fdb372ed 3440 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 3441 }
d8e83d26 3442 mutex_unlock(&trace_types_lock);
adf9f195 3443
fdb372ed 3444 return 0;
bc0c38d1 3445}
bc0c38d1 3446
8c1a49ae 3447static int __set_tracer_option(struct trace_array *tr,
8d18eaaf
LZ
3448 struct tracer_flags *tracer_flags,
3449 struct tracer_opt *opts, int neg)
3450{
8c1a49ae 3451 struct tracer *trace = tr->current_trace;
8d18eaaf 3452 int ret;
bc0c38d1 3453
8c1a49ae 3454 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
8d18eaaf
LZ
3455 if (ret)
3456 return ret;
3457
3458 if (neg)
3459 tracer_flags->val &= ~opts->bit;
3460 else
3461 tracer_flags->val |= opts->bit;
3462 return 0;
bc0c38d1
SR
3463}
3464
adf9f195 3465/* Try to assign a tracer specific option */
8c1a49ae 3466static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
adf9f195 3467{
8c1a49ae 3468 struct tracer *trace = tr->current_trace;
7770841e 3469 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 3470 struct tracer_opt *opts = NULL;
8d18eaaf 3471 int i;
adf9f195 3472
7770841e
Z
3473 for (i = 0; tracer_flags->opts[i].name; i++) {
3474 opts = &tracer_flags->opts[i];
adf9f195 3475
8d18eaaf 3476 if (strcmp(cmp, opts->name) == 0)
8c1a49ae 3477 return __set_tracer_option(tr, trace->flags, opts, neg);
adf9f195 3478 }
adf9f195 3479
8d18eaaf 3480 return -EINVAL;
adf9f195
FW
3481}
3482
613f04a0
SRRH
3483/* Some tracers require overwrite to stay enabled */
3484int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3485{
3486 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3487 return -1;
3488
3489 return 0;
3490}
3491
2b6080f2 3492int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
3493{
3494 /* do nothing if flag is already set */
3495 if (!!(trace_flags & mask) == !!enabled)
613f04a0
SRRH
3496 return 0;
3497
3498 /* Give the tracer a chance to approve the change */
2b6080f2 3499 if (tr->current_trace->flag_changed)
bf6065b5 3500 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
613f04a0 3501 return -EINVAL;
af4617bd
SR
3502
3503 if (enabled)
3504 trace_flags |= mask;
3505 else
3506 trace_flags &= ~mask;
e870e9a1
LZ
3507
3508 if (mask == TRACE_ITER_RECORD_CMD)
3509 trace_event_enable_cmd_record(enabled);
750912fa 3510
80902822 3511 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 3512 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 3513#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 3514 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
3515#endif
3516 }
81698831
SR
3517
3518 if (mask == TRACE_ITER_PRINTK)
3519 trace_printk_start_stop_comm(enabled);
613f04a0
SRRH
3520
3521 return 0;
af4617bd
SR
3522}
3523
2b6080f2 3524static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 3525{
8d18eaaf 3526 char *cmp;
bc0c38d1 3527 int neg = 0;
613f04a0 3528 int ret = -ENODEV;
bc0c38d1
SR
3529 int i;
3530
7bcfaf54 3531 cmp = strstrip(option);
bc0c38d1 3532
8d18eaaf 3533 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
3534 neg = 1;
3535 cmp += 2;
3536 }
3537
69d34da2
SRRH
3538 mutex_lock(&trace_types_lock);
3539
bc0c38d1 3540 for (i = 0; trace_options[i]; i++) {
8d18eaaf 3541 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 3542 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
3543 break;
3544 }
3545 }
adf9f195
FW
3546
3547 /* If no option could be set, test the specific tracer options */
69d34da2 3548 if (!trace_options[i])
8c1a49ae 3549 ret = set_tracer_option(tr, cmp, neg);
69d34da2
SRRH
3550
3551 mutex_unlock(&trace_types_lock);
bc0c38d1 3552
7bcfaf54
SR
3553 return ret;
3554}
3555
3556static ssize_t
3557tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3558 size_t cnt, loff_t *ppos)
3559{
2b6080f2
SR
3560 struct seq_file *m = filp->private_data;
3561 struct trace_array *tr = m->private;
7bcfaf54 3562 char buf[64];
613f04a0 3563 int ret;
7bcfaf54
SR
3564
3565 if (cnt >= sizeof(buf))
3566 return -EINVAL;
3567
3568 if (copy_from_user(&buf, ubuf, cnt))
3569 return -EFAULT;
3570
a8dd2176
SR
3571 buf[cnt] = 0;
3572
2b6080f2 3573 ret = trace_set_options(tr, buf);
613f04a0
SRRH
3574 if (ret < 0)
3575 return ret;
7bcfaf54 3576
cf8517cf 3577 *ppos += cnt;
bc0c38d1
SR
3578
3579 return cnt;
3580}
3581
fdb372ed
LZ
3582static int tracing_trace_options_open(struct inode *inode, struct file *file)
3583{
7b85af63 3584 struct trace_array *tr = inode->i_private;
f77d09a3 3585 int ret;
7b85af63 3586
fdb372ed
LZ
3587 if (tracing_disabled)
3588 return -ENODEV;
2b6080f2 3589
7b85af63
SRRH
3590 if (trace_array_get(tr) < 0)
3591 return -ENODEV;
3592
f77d09a3
AL
3593 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3594 if (ret < 0)
3595 trace_array_put(tr);
3596
3597 return ret;
fdb372ed
LZ
3598}
3599
5e2336a0 3600static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
3601 .open = tracing_trace_options_open,
3602 .read = seq_read,
3603 .llseek = seq_lseek,
7b85af63 3604 .release = tracing_single_release_tr,
ee6bce52 3605 .write = tracing_trace_options_write,
bc0c38d1
SR
3606};
3607
7bd2f24c
IM
3608static const char readme_msg[] =
3609 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
3610 "# echo 0 > tracing_on : quick way to disable tracing\n"
3611 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3612 " Important files:\n"
3613 " trace\t\t\t- The static contents of the buffer\n"
3614 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3615 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3616 " current_tracer\t- function and latency tracers\n"
3617 " available_tracers\t- list of configured tracers for current_tracer\n"
3618 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3619 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3620 " trace_clock\t\t-change the clock used to order events\n"
3621 " local: Per cpu clock but may not be synced across CPUs\n"
3622 " global: Synced across CPUs but slows tracing down.\n"
3623 " counter: Not a clock, but just an increment\n"
3624 " uptime: Jiffy counter from time of boot\n"
3625 " perf: Same clock that perf events use\n"
3626#ifdef CONFIG_X86_64
3627 " x86-tsc: TSC cycle counter\n"
3628#endif
3629 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3630 " tracing_cpumask\t- Limit which CPUs to trace\n"
3631 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3632 "\t\t\t Remove sub-buffer with rmdir\n"
3633 " trace_options\t\t- Set format or modify how tracing happens\n"
71485c45
SRRH
3634 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3635 "\t\t\t option name\n"
939c7a4f 3636 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
22f45649
SRRH
3637#ifdef CONFIG_DYNAMIC_FTRACE
3638 "\n available_filter_functions - list of functions that can be filtered on\n"
71485c45
SRRH
3639 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3640 "\t\t\t functions\n"
3641 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3642 "\t modules: Can select a group via module\n"
3643 "\t Format: :mod:<module-name>\n"
3644 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3645 "\t triggers: a command to perform when function is hit\n"
3646 "\t Format: <function>:<trigger>[:count]\n"
3647 "\t trigger: traceon, traceoff\n"
3648 "\t\t enable_event:<system>:<event>\n"
3649 "\t\t disable_event:<system>:<event>\n"
22f45649 3650#ifdef CONFIG_STACKTRACE
71485c45 3651 "\t\t stacktrace\n"
22f45649
SRRH
3652#endif
3653#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 3654 "\t\t snapshot\n"
22f45649 3655#endif
17a280ea
SRRH
3656 "\t\t dump\n"
3657 "\t\t cpudump\n"
71485c45
SRRH
3658 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3659 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3660 "\t The first one will disable tracing every time do_fault is hit\n"
3661 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3662 "\t The first time do trap is hit and it disables tracing, the\n"
3663 "\t counter will decrement to 2. If tracing is already disabled,\n"
3664 "\t the counter will not decrement. It only decrements when the\n"
3665 "\t trigger did work\n"
3666 "\t To remove trigger without count:\n"
3667 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3668 "\t To remove trigger with a count:\n"
3669 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
22f45649 3670 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
71485c45
SRRH
3671 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3672 "\t modules: Can select a group via module command :mod:\n"
3673 "\t Does not accept triggers\n"
22f45649
SRRH
3674#endif /* CONFIG_DYNAMIC_FTRACE */
3675#ifdef CONFIG_FUNCTION_TRACER
71485c45
SRRH
3676 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3677 "\t\t (function)\n"
22f45649
SRRH
3678#endif
3679#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3680 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
d048a8c7 3681 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
22f45649
SRRH
3682 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3683#endif
3684#ifdef CONFIG_TRACER_SNAPSHOT
71485c45
SRRH
3685 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3686 "\t\t\t snapshot buffer. Read the contents for more\n"
3687 "\t\t\t information\n"
22f45649 3688#endif
991821c8 3689#ifdef CONFIG_STACK_TRACER
22f45649
SRRH
3690 " stack_trace\t\t- Shows the max stack trace when active\n"
3691 " stack_max_size\t- Shows current max stack size that was traced\n"
71485c45
SRRH
3692 "\t\t\t Write into this file to reset the max size (trigger a\n"
3693 "\t\t\t new trace)\n"
22f45649 3694#ifdef CONFIG_DYNAMIC_FTRACE
71485c45
SRRH
3695 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3696 "\t\t\t traces\n"
22f45649 3697#endif
991821c8 3698#endif /* CONFIG_STACK_TRACER */
26f25564
TZ
3699 " events/\t\t- Directory containing all trace event subsystems:\n"
3700 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3701 " events/<system>/\t- Directory containing all trace events for <system>:\n"
71485c45
SRRH
3702 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3703 "\t\t\t events\n"
26f25564 3704 " filter\t\t- If set, only events passing filter are traced\n"
71485c45
SRRH
3705 " events/<system>/<event>/\t- Directory containing control files for\n"
3706 "\t\t\t <event>:\n"
26f25564
TZ
3707 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3708 " filter\t\t- If set, only events passing filter are traced\n"
3709 " trigger\t\t- If set, a command to perform when event is hit\n"
71485c45
SRRH
3710 "\t Format: <trigger>[:count][if <filter>]\n"
3711 "\t trigger: traceon, traceoff\n"
3712 "\t enable_event:<system>:<event>\n"
3713 "\t disable_event:<system>:<event>\n"
26f25564 3714#ifdef CONFIG_STACKTRACE
71485c45 3715 "\t\t stacktrace\n"
26f25564
TZ
3716#endif
3717#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 3718 "\t\t snapshot\n"
26f25564 3719#endif
71485c45
SRRH
3720 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3721 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3722 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3723 "\t events/block/block_unplug/trigger\n"
3724 "\t The first disables tracing every time block_unplug is hit.\n"
3725 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3726 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3727 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3728 "\t Like function triggers, the counter is only decremented if it\n"
3729 "\t enabled or disabled tracing.\n"
3730 "\t To remove a trigger without a count:\n"
3731 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3732 "\t To remove a trigger with a count:\n"
3733 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3734 "\t Filters can be ignored when removing a trigger.\n"
7bd2f24c
IM
3735;
3736
3737static ssize_t
3738tracing_readme_read(struct file *filp, char __user *ubuf,
3739 size_t cnt, loff_t *ppos)
3740{
3741 return simple_read_from_buffer(ubuf, cnt, ppos,
3742 readme_msg, strlen(readme_msg));
3743}
3744
5e2336a0 3745static const struct file_operations tracing_readme_fops = {
c7078de1
IM
3746 .open = tracing_open_generic,
3747 .read = tracing_readme_read,
b444786f 3748 .llseek = generic_file_llseek,
7bd2f24c
IM
3749};
3750
42584c81
YY
3751static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3752{
3753 unsigned int *ptr = v;
69abe6a5 3754
42584c81
YY
3755 if (*pos || m->count)
3756 ptr++;
69abe6a5 3757
42584c81 3758 (*pos)++;
69abe6a5 3759
939c7a4f
YY
3760 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3761 ptr++) {
42584c81
YY
3762 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3763 continue;
69abe6a5 3764
42584c81
YY
3765 return ptr;
3766 }
69abe6a5 3767
42584c81
YY
3768 return NULL;
3769}
3770
3771static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3772{
3773 void *v;
3774 loff_t l = 0;
69abe6a5 3775
4c27e756
SRRH
3776 preempt_disable();
3777 arch_spin_lock(&trace_cmdline_lock);
3778
939c7a4f 3779 v = &savedcmd->map_cmdline_to_pid[0];
42584c81
YY
3780 while (l <= *pos) {
3781 v = saved_cmdlines_next(m, v, &l);
3782 if (!v)
3783 return NULL;
69abe6a5
AP
3784 }
3785
42584c81
YY
3786 return v;
3787}
3788
3789static void saved_cmdlines_stop(struct seq_file *m, void *v)
3790{
4c27e756
SRRH
3791 arch_spin_unlock(&trace_cmdline_lock);
3792 preempt_enable();
42584c81 3793}
69abe6a5 3794
42584c81
YY
3795static int saved_cmdlines_show(struct seq_file *m, void *v)
3796{
3797 char buf[TASK_COMM_LEN];
3798 unsigned int *pid = v;
69abe6a5 3799
4c27e756 3800 __trace_find_cmdline(*pid, buf);
42584c81
YY
3801 seq_printf(m, "%d %s\n", *pid, buf);
3802 return 0;
3803}
3804
3805static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3806 .start = saved_cmdlines_start,
3807 .next = saved_cmdlines_next,
3808 .stop = saved_cmdlines_stop,
3809 .show = saved_cmdlines_show,
3810};
3811
3812static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3813{
3814 if (tracing_disabled)
3815 return -ENODEV;
3816
3817 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
69abe6a5
AP
3818}
3819
3820static const struct file_operations tracing_saved_cmdlines_fops = {
42584c81
YY
3821 .open = tracing_saved_cmdlines_open,
3822 .read = seq_read,
3823 .llseek = seq_lseek,
3824 .release = seq_release,
69abe6a5
AP
3825};
3826
939c7a4f
YY
3827static ssize_t
3828tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3829 size_t cnt, loff_t *ppos)
3830{
3831 char buf[64];
3832 int r;
3833
3834 arch_spin_lock(&trace_cmdline_lock);
a6af8fbf 3835 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
939c7a4f
YY
3836 arch_spin_unlock(&trace_cmdline_lock);
3837
3838 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3839}
3840
3841static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3842{
3843 kfree(s->saved_cmdlines);
3844 kfree(s->map_cmdline_to_pid);
3845 kfree(s);
3846}
3847
3848static int tracing_resize_saved_cmdlines(unsigned int val)
3849{
3850 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3851
a6af8fbf 3852 s = kmalloc(sizeof(*s), GFP_KERNEL);
939c7a4f
YY
3853 if (!s)
3854 return -ENOMEM;
3855
3856 if (allocate_cmdlines_buffer(val, s) < 0) {
3857 kfree(s);
3858 return -ENOMEM;
3859 }
3860
3861 arch_spin_lock(&trace_cmdline_lock);
3862 savedcmd_temp = savedcmd;
3863 savedcmd = s;
3864 arch_spin_unlock(&trace_cmdline_lock);
3865 free_saved_cmdlines_buffer(savedcmd_temp);
3866
3867 return 0;
3868}
3869
3870static ssize_t
3871tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3872 size_t cnt, loff_t *ppos)
3873{
3874 unsigned long val;
3875 int ret;
3876
3877 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3878 if (ret)
3879 return ret;
3880
3881 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3882 if (!val || val > PID_MAX_DEFAULT)
3883 return -EINVAL;
3884
3885 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3886 if (ret < 0)
3887 return ret;
3888
3889 *ppos += cnt;
3890
3891 return cnt;
3892}
3893
3894static const struct file_operations tracing_saved_cmdlines_size_fops = {
3895 .open = tracing_open_generic,
3896 .read = tracing_saved_cmdlines_size_read,
3897 .write = tracing_saved_cmdlines_size_write,
3898};
3899
bc0c38d1
SR
3900static ssize_t
3901tracing_set_trace_read(struct file *filp, char __user *ubuf,
3902 size_t cnt, loff_t *ppos)
3903{
2b6080f2 3904 struct trace_array *tr = filp->private_data;
ee6c2c1b 3905 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
3906 int r;
3907
3908 mutex_lock(&trace_types_lock);
2b6080f2 3909 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
3910 mutex_unlock(&trace_types_lock);
3911
4bf39a94 3912 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3913}
3914
b6f11df2
ACM
3915int tracer_init(struct tracer *t, struct trace_array *tr)
3916{
12883efb 3917 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
3918 return t->init(tr);
3919}
3920
12883efb 3921static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
3922{
3923 int cpu;
737223fb 3924
438ced17 3925 for_each_tracing_cpu(cpu)
12883efb 3926 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
3927}
3928
12883efb 3929#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 3930/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
3931static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3932 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
3933{
3934 int cpu, ret = 0;
3935
3936 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3937 for_each_tracing_cpu(cpu) {
12883efb
SRRH
3938 ret = ring_buffer_resize(trace_buf->buffer,
3939 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
3940 if (ret < 0)
3941 break;
12883efb
SRRH
3942 per_cpu_ptr(trace_buf->data, cpu)->entries =
3943 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
3944 }
3945 } else {
12883efb
SRRH
3946 ret = ring_buffer_resize(trace_buf->buffer,
3947 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 3948 if (ret == 0)
12883efb
SRRH
3949 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3950 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
3951 }
3952
3953 return ret;
3954}
12883efb 3955#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 3956
2b6080f2
SR
3957static int __tracing_resize_ring_buffer(struct trace_array *tr,
3958 unsigned long size, int cpu)
73c5162a
SR
3959{
3960 int ret;
3961
3962 /*
3963 * If kernel or user changes the size of the ring buffer
a123c52b
SR
3964 * we use the size that was given, and we can forget about
3965 * expanding it later.
73c5162a 3966 */
55034cd6 3967 ring_buffer_expanded = true;
73c5162a 3968
b382ede6 3969 /* May be called before buffers are initialized */
12883efb 3970 if (!tr->trace_buffer.buffer)
b382ede6
SR
3971 return 0;
3972
12883efb 3973 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
3974 if (ret < 0)
3975 return ret;
3976
12883efb 3977#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3978 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3979 !tr->current_trace->use_max_tr)
ef710e10
KM
3980 goto out;
3981
12883efb 3982 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 3983 if (ret < 0) {
12883efb
SRRH
3984 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3985 &tr->trace_buffer, cpu);
73c5162a 3986 if (r < 0) {
a123c52b
SR
3987 /*
3988 * AARGH! We are left with different
3989 * size max buffer!!!!
3990 * The max buffer is our "snapshot" buffer.
3991 * When a tracer needs a snapshot (one of the
3992 * latency tracers), it swaps the max buffer
3993 * with the saved snap shot. We succeeded to
3994 * update the size of the main buffer, but failed to
3995 * update the size of the max buffer. But when we tried
3996 * to reset the main buffer to the original size, we
3997 * failed there too. This is very unlikely to
3998 * happen, but if it does, warn and kill all
3999 * tracing.
4000 */
73c5162a
SR
4001 WARN_ON(1);
4002 tracing_disabled = 1;
4003 }
4004 return ret;
4005 }
4006
438ced17 4007 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 4008 set_buffer_entries(&tr->max_buffer, size);
438ced17 4009 else
12883efb 4010 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 4011
ef710e10 4012 out:
12883efb
SRRH
4013#endif /* CONFIG_TRACER_MAX_TRACE */
4014
438ced17 4015 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 4016 set_buffer_entries(&tr->trace_buffer, size);
438ced17 4017 else
12883efb 4018 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
4019
4020 return ret;
4021}
4022
2b6080f2
SR
4023static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4024 unsigned long size, int cpu_id)
4f271a2a 4025{
83f40318 4026 int ret = size;
4f271a2a
VN
4027
4028 mutex_lock(&trace_types_lock);
4029
438ced17
VN
4030 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4031 /* make sure, this cpu is enabled in the mask */
4032 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4033 ret = -EINVAL;
4034 goto out;
4035 }
4036 }
4f271a2a 4037
2b6080f2 4038 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
4039 if (ret < 0)
4040 ret = -ENOMEM;
4041
438ced17 4042out:
4f271a2a
VN
4043 mutex_unlock(&trace_types_lock);
4044
4045 return ret;
4046}
4047
ef710e10 4048
1852fcce
SR
4049/**
4050 * tracing_update_buffers - used by tracing facility to expand ring buffers
4051 *
4052 * To save on memory when the tracing is never used on a system with it
4053 * configured in. The ring buffers are set to a minimum size. But once
4054 * a user starts to use the tracing facility, then they need to grow
4055 * to their default size.
4056 *
4057 * This function is to be called when a tracer is about to be used.
4058 */
4059int tracing_update_buffers(void)
4060{
4061 int ret = 0;
4062
1027fcb2 4063 mutex_lock(&trace_types_lock);
1852fcce 4064 if (!ring_buffer_expanded)
2b6080f2 4065 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 4066 RING_BUFFER_ALL_CPUS);
1027fcb2 4067 mutex_unlock(&trace_types_lock);
1852fcce
SR
4068
4069 return ret;
4070}
4071
577b785f
SR
4072struct trace_option_dentry;
4073
4074static struct trace_option_dentry *
2b6080f2 4075create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f
SR
4076
4077static void
4078destroy_trace_option_files(struct trace_option_dentry *topts);
4079
6b450d25
SRRH
4080/*
4081 * Used to clear out the tracer before deletion of an instance.
4082 * Must have trace_types_lock held.
4083 */
4084static void tracing_set_nop(struct trace_array *tr)
4085{
4086 if (tr->current_trace == &nop_trace)
4087 return;
4088
50512ab5 4089 tr->current_trace->enabled--;
6b450d25
SRRH
4090
4091 if (tr->current_trace->reset)
4092 tr->current_trace->reset(tr);
4093
4094 tr->current_trace = &nop_trace;
4095}
4096
607e2ea1 4097static int tracing_set_tracer(struct trace_array *tr, const char *buf)
bc0c38d1 4098{
577b785f 4099 static struct trace_option_dentry *topts;
bc0c38d1 4100 struct tracer *t;
12883efb 4101#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 4102 bool had_max_tr;
12883efb 4103#endif
d9e54076 4104 int ret = 0;
bc0c38d1 4105
1027fcb2
SR
4106 mutex_lock(&trace_types_lock);
4107
73c5162a 4108 if (!ring_buffer_expanded) {
2b6080f2 4109 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 4110 RING_BUFFER_ALL_CPUS);
73c5162a 4111 if (ret < 0)
59f586db 4112 goto out;
73c5162a
SR
4113 ret = 0;
4114 }
4115
bc0c38d1
SR
4116 for (t = trace_types; t; t = t->next) {
4117 if (strcmp(t->name, buf) == 0)
4118 break;
4119 }
c2931e05
FW
4120 if (!t) {
4121 ret = -EINVAL;
4122 goto out;
4123 }
2b6080f2 4124 if (t == tr->current_trace)
bc0c38d1
SR
4125 goto out;
4126
607e2ea1
SRRH
4127 /* Some tracers are only allowed for the top level buffer */
4128 if (!trace_ok_for_array(t, tr)) {
4129 ret = -EINVAL;
4130 goto out;
4131 }
4132
9f029e83 4133 trace_branch_disable();
613f04a0 4134
50512ab5 4135 tr->current_trace->enabled--;
613f04a0 4136
2b6080f2
SR
4137 if (tr->current_trace->reset)
4138 tr->current_trace->reset(tr);
34600f0e 4139
12883efb 4140 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 4141 tr->current_trace = &nop_trace;
34600f0e 4142
45ad21ca
SRRH
4143#ifdef CONFIG_TRACER_MAX_TRACE
4144 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
4145
4146 if (had_max_tr && !t->use_max_tr) {
4147 /*
4148 * We need to make sure that the update_max_tr sees that
4149 * current_trace changed to nop_trace to keep it from
4150 * swapping the buffers after we resize it.
4151 * The update_max_tr is called from interrupts disabled
4152 * so a synchronized_sched() is sufficient.
4153 */
4154 synchronize_sched();
3209cff4 4155 free_snapshot(tr);
ef710e10 4156 }
12883efb 4157#endif
f1b21c9a
SRRH
4158 /* Currently, only the top instance has options */
4159 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4160 destroy_trace_option_files(topts);
4161 topts = create_trace_option_files(tr, t);
4162 }
12883efb
SRRH
4163
4164#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 4165 if (t->use_max_tr && !had_max_tr) {
3209cff4 4166 ret = alloc_snapshot(tr);
d60da506
HT
4167 if (ret < 0)
4168 goto out;
ef710e10 4169 }
12883efb 4170#endif
577b785f 4171
1c80025a 4172 if (t->init) {
b6f11df2 4173 ret = tracer_init(t, tr);
1c80025a
FW
4174 if (ret)
4175 goto out;
4176 }
bc0c38d1 4177
2b6080f2 4178 tr->current_trace = t;
50512ab5 4179 tr->current_trace->enabled++;
9f029e83 4180 trace_branch_enable(tr);
bc0c38d1
SR
4181 out:
4182 mutex_unlock(&trace_types_lock);
4183
d9e54076
PZ
4184 return ret;
4185}
4186
4187static ssize_t
4188tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4189 size_t cnt, loff_t *ppos)
4190{
607e2ea1 4191 struct trace_array *tr = filp->private_data;
ee6c2c1b 4192 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
4193 int i;
4194 size_t ret;
e6e7a65a
FW
4195 int err;
4196
4197 ret = cnt;
d9e54076 4198
ee6c2c1b
LZ
4199 if (cnt > MAX_TRACER_SIZE)
4200 cnt = MAX_TRACER_SIZE;
d9e54076
PZ
4201
4202 if (copy_from_user(&buf, ubuf, cnt))
4203 return -EFAULT;
4204
4205 buf[cnt] = 0;
4206
4207 /* strip ending whitespace. */
4208 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4209 buf[i] = 0;
4210
607e2ea1 4211 err = tracing_set_tracer(tr, buf);
e6e7a65a
FW
4212 if (err)
4213 return err;
d9e54076 4214
cf8517cf 4215 *ppos += ret;
bc0c38d1 4216
c2931e05 4217 return ret;
bc0c38d1
SR
4218}
4219
4220static ssize_t
6508fa76
SF
4221tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4222 size_t cnt, loff_t *ppos)
bc0c38d1 4223{
bc0c38d1
SR
4224 char buf[64];
4225 int r;
4226
cffae437 4227 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 4228 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
4229 if (r > sizeof(buf))
4230 r = sizeof(buf);
4bf39a94 4231 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
4232}
4233
4234static ssize_t
6508fa76
SF
4235tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4236 size_t cnt, loff_t *ppos)
bc0c38d1 4237{
5e39841c 4238 unsigned long val;
c6caeeb1 4239 int ret;
bc0c38d1 4240
22fe9b54
PH
4241 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4242 if (ret)
c6caeeb1 4243 return ret;
bc0c38d1
SR
4244
4245 *ptr = val * 1000;
4246
4247 return cnt;
4248}
4249
6508fa76
SF
4250static ssize_t
4251tracing_thresh_read(struct file *filp, char __user *ubuf,
4252 size_t cnt, loff_t *ppos)
4253{
4254 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4255}
4256
4257static ssize_t
4258tracing_thresh_write(struct file *filp, const char __user *ubuf,
4259 size_t cnt, loff_t *ppos)
4260{
4261 struct trace_array *tr = filp->private_data;
4262 int ret;
4263
4264 mutex_lock(&trace_types_lock);
4265 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4266 if (ret < 0)
4267 goto out;
4268
4269 if (tr->current_trace->update_thresh) {
4270 ret = tr->current_trace->update_thresh(tr);
4271 if (ret < 0)
4272 goto out;
4273 }
4274
4275 ret = cnt;
4276out:
4277 mutex_unlock(&trace_types_lock);
4278
4279 return ret;
4280}
4281
4282static ssize_t
4283tracing_max_lat_read(struct file *filp, char __user *ubuf,
4284 size_t cnt, loff_t *ppos)
4285{
4286 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4287}
4288
4289static ssize_t
4290tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4291 size_t cnt, loff_t *ppos)
4292{
4293 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4294}
4295
b3806b43
SR
4296static int tracing_open_pipe(struct inode *inode, struct file *filp)
4297{
15544209 4298 struct trace_array *tr = inode->i_private;
b3806b43 4299 struct trace_iterator *iter;
b04cc6b1 4300 int ret = 0;
b3806b43
SR
4301
4302 if (tracing_disabled)
4303 return -ENODEV;
4304
7b85af63
SRRH
4305 if (trace_array_get(tr) < 0)
4306 return -ENODEV;
4307
b04cc6b1
FW
4308 mutex_lock(&trace_types_lock);
4309
b3806b43
SR
4310 /* create a buffer to store the information to pass to userspace */
4311 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
4312 if (!iter) {
4313 ret = -ENOMEM;
f77d09a3 4314 __trace_array_put(tr);
b04cc6b1
FW
4315 goto out;
4316 }
b3806b43 4317
d7350c3f
FW
4318 /*
4319 * We make a copy of the current tracer to avoid concurrent
4320 * changes on it while we are reading.
4321 */
4322 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4323 if (!iter->trace) {
4324 ret = -ENOMEM;
4325 goto fail;
4326 }
2b6080f2 4327 *iter->trace = *tr->current_trace;
d7350c3f 4328
4462344e 4329 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 4330 ret = -ENOMEM;
d7350c3f 4331 goto fail;
4462344e
RR
4332 }
4333
a309720c 4334 /* trace pipe does not show start of buffer */
4462344e 4335 cpumask_setall(iter->started);
a309720c 4336
112f38a7
SR
4337 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4338 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4339
8be0709f 4340 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 4341 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
4342 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4343
15544209
ON
4344 iter->tr = tr;
4345 iter->trace_buffer = &tr->trace_buffer;
4346 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 4347 mutex_init(&iter->mutex);
b3806b43
SR
4348 filp->private_data = iter;
4349
107bad8b
SR
4350 if (iter->trace->pipe_open)
4351 iter->trace->pipe_open(iter);
107bad8b 4352
b444786f 4353 nonseekable_open(inode, filp);
b04cc6b1
FW
4354out:
4355 mutex_unlock(&trace_types_lock);
4356 return ret;
d7350c3f
FW
4357
4358fail:
4359 kfree(iter->trace);
4360 kfree(iter);
7b85af63 4361 __trace_array_put(tr);
d7350c3f
FW
4362 mutex_unlock(&trace_types_lock);
4363 return ret;
b3806b43
SR
4364}
4365
4366static int tracing_release_pipe(struct inode *inode, struct file *file)
4367{
4368 struct trace_iterator *iter = file->private_data;
15544209 4369 struct trace_array *tr = inode->i_private;
b3806b43 4370
b04cc6b1
FW
4371 mutex_lock(&trace_types_lock);
4372
29bf4a5e 4373 if (iter->trace->pipe_close)
c521efd1
SR
4374 iter->trace->pipe_close(iter);
4375
b04cc6b1
FW
4376 mutex_unlock(&trace_types_lock);
4377
4462344e 4378 free_cpumask_var(iter->started);
d7350c3f
FW
4379 mutex_destroy(&iter->mutex);
4380 kfree(iter->trace);
b3806b43 4381 kfree(iter);
b3806b43 4382
7b85af63
SRRH
4383 trace_array_put(tr);
4384
b3806b43
SR
4385 return 0;
4386}
4387
2a2cc8f7 4388static unsigned int
cc60cdc9 4389trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 4390{
15693458
SRRH
4391 /* Iterators are static, they should be filled or empty */
4392 if (trace_buffer_iter(iter, iter->cpu_file))
4393 return POLLIN | POLLRDNORM;
2a2cc8f7 4394
15693458 4395 if (trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
4396 /*
4397 * Always select as readable when in blocking mode
4398 */
4399 return POLLIN | POLLRDNORM;
15693458 4400 else
12883efb 4401 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 4402 filp, poll_table);
2a2cc8f7 4403}
2a2cc8f7 4404
cc60cdc9
SR
4405static unsigned int
4406tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4407{
4408 struct trace_iterator *iter = filp->private_data;
4409
4410 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
4411}
4412
ff98781b
EGM
4413/* Must be called with trace_types_lock mutex held. */
4414static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
4415{
4416 struct trace_iterator *iter = filp->private_data;
8b8b3683 4417 int ret;
b3806b43 4418
b3806b43 4419 while (trace_empty(iter)) {
2dc8f095 4420
107bad8b 4421 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 4422 return -EAGAIN;
107bad8b 4423 }
2dc8f095 4424
b3806b43 4425 /*
250bfd3d 4426 * We block until we read something and tracing is disabled.
b3806b43
SR
4427 * We still block if tracing is disabled, but we have never
4428 * read anything. This allows a user to cat this file, and
4429 * then enable tracing. But after we have read something,
4430 * we give an EOF when tracing is again disabled.
4431 *
4432 * iter->pos will be 0 if we haven't read anything.
4433 */
10246fa3 4434 if (!tracing_is_on() && iter->pos)
b3806b43 4435 break;
f4874261
SRRH
4436
4437 mutex_unlock(&iter->mutex);
4438
8b8b3683 4439 ret = wait_on_pipe(iter);
f4874261
SRRH
4440
4441 mutex_lock(&iter->mutex);
4442
8b8b3683
SRRH
4443 if (ret)
4444 return ret;
4445
f4874261
SRRH
4446 if (signal_pending(current))
4447 return -EINTR;
b3806b43
SR
4448 }
4449
ff98781b
EGM
4450 return 1;
4451}
4452
4453/*
4454 * Consumer reader.
4455 */
4456static ssize_t
4457tracing_read_pipe(struct file *filp, char __user *ubuf,
4458 size_t cnt, loff_t *ppos)
4459{
4460 struct trace_iterator *iter = filp->private_data;
2b6080f2 4461 struct trace_array *tr = iter->tr;
ff98781b
EGM
4462 ssize_t sret;
4463
4464 /* return any leftover data */
4465 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4466 if (sret != -EBUSY)
4467 return sret;
4468
f9520750 4469 trace_seq_init(&iter->seq);
ff98781b 4470
d7350c3f 4471 /* copy the tracer to avoid using a global lock all around */
ff98781b 4472 mutex_lock(&trace_types_lock);
2b6080f2
SR
4473 if (unlikely(iter->trace->name != tr->current_trace->name))
4474 *iter->trace = *tr->current_trace;
d7350c3f
FW
4475 mutex_unlock(&trace_types_lock);
4476
4477 /*
4478 * Avoid more than one consumer on a single file descriptor
4479 * This is just a matter of traces coherency, the ring buffer itself
4480 * is protected.
4481 */
4482 mutex_lock(&iter->mutex);
ff98781b
EGM
4483 if (iter->trace->read) {
4484 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4485 if (sret)
4486 goto out;
4487 }
4488
4489waitagain:
4490 sret = tracing_wait_pipe(filp);
4491 if (sret <= 0)
4492 goto out;
4493
b3806b43 4494 /* stop when tracing is finished */
ff98781b
EGM
4495 if (trace_empty(iter)) {
4496 sret = 0;
107bad8b 4497 goto out;
ff98781b 4498 }
b3806b43
SR
4499
4500 if (cnt >= PAGE_SIZE)
4501 cnt = PAGE_SIZE - 1;
4502
53d0aa77 4503 /* reset all but tr, trace, and overruns */
53d0aa77
SR
4504 memset(&iter->seq, 0,
4505 sizeof(struct trace_iterator) -
4506 offsetof(struct trace_iterator, seq));
ed5467da 4507 cpumask_clear(iter->started);
4823ed7e 4508 iter->pos = -1;
b3806b43 4509
4f535968 4510 trace_event_read_lock();
7e53bd42 4511 trace_access_lock(iter->cpu_file);
955b61e5 4512 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 4513 enum print_line_t ret;
088b1e42
SR
4514 int len = iter->seq.len;
4515
f9896bf3 4516 ret = print_trace_line(iter);
2c4f035f 4517 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
4518 /* don't print partial lines */
4519 iter->seq.len = len;
b3806b43 4520 break;
088b1e42 4521 }
b91facc3
FW
4522 if (ret != TRACE_TYPE_NO_CONSUME)
4523 trace_consume(iter);
b3806b43
SR
4524
4525 if (iter->seq.len >= cnt)
4526 break;
ee5e51f5
JO
4527
4528 /*
4529 * Setting the full flag means we reached the trace_seq buffer
4530 * size and we should leave by partial output condition above.
4531 * One of the trace_seq_* functions is not used properly.
4532 */
4533 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4534 iter->ent->type);
b3806b43 4535 }
7e53bd42 4536 trace_access_unlock(iter->cpu_file);
4f535968 4537 trace_event_read_unlock();
b3806b43 4538
b3806b43 4539 /* Now copy what we have to the user */
6c6c2796
PP
4540 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4541 if (iter->seq.readpos >= iter->seq.len)
f9520750 4542 trace_seq_init(&iter->seq);
9ff4b974
PP
4543
4544 /*
25985edc 4545 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
4546 * entries, go back to wait for more entries.
4547 */
6c6c2796 4548 if (sret == -EBUSY)
9ff4b974 4549 goto waitagain;
b3806b43 4550
107bad8b 4551out:
d7350c3f 4552 mutex_unlock(&iter->mutex);
107bad8b 4553
6c6c2796 4554 return sret;
b3806b43
SR
4555}
4556
3c56819b
EGM
4557static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4558 unsigned int idx)
4559{
4560 __free_page(spd->pages[idx]);
4561}
4562
28dfef8f 4563static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998 4564 .can_merge = 0,
34cd4998 4565 .confirm = generic_pipe_buf_confirm,
92fdd98c 4566 .release = generic_pipe_buf_release,
34cd4998
SR
4567 .steal = generic_pipe_buf_steal,
4568 .get = generic_pipe_buf_get,
3c56819b
EGM
4569};
4570
34cd4998 4571static size_t
fa7c7f6e 4572tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
4573{
4574 size_t count;
4575 int ret;
4576
4577 /* Seq buffer is page-sized, exactly what we need. */
4578 for (;;) {
4579 count = iter->seq.len;
4580 ret = print_trace_line(iter);
4581 count = iter->seq.len - count;
4582 if (rem < count) {
4583 rem = 0;
4584 iter->seq.len -= count;
4585 break;
4586 }
4587 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4588 iter->seq.len -= count;
4589 break;
4590 }
4591
74e7ff8c
LJ
4592 if (ret != TRACE_TYPE_NO_CONSUME)
4593 trace_consume(iter);
34cd4998 4594 rem -= count;
955b61e5 4595 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
4596 rem = 0;
4597 iter->ent = NULL;
4598 break;
4599 }
4600 }
4601
4602 return rem;
4603}
4604
3c56819b
EGM
4605static ssize_t tracing_splice_read_pipe(struct file *filp,
4606 loff_t *ppos,
4607 struct pipe_inode_info *pipe,
4608 size_t len,
4609 unsigned int flags)
4610{
35f3d14d
JA
4611 struct page *pages_def[PIPE_DEF_BUFFERS];
4612 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
4613 struct trace_iterator *iter = filp->private_data;
4614 struct splice_pipe_desc spd = {
35f3d14d
JA
4615 .pages = pages_def,
4616 .partial = partial_def,
34cd4998 4617 .nr_pages = 0, /* This gets updated below. */
047fe360 4618 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
4619 .flags = flags,
4620 .ops = &tracing_pipe_buf_ops,
4621 .spd_release = tracing_spd_release_pipe,
3c56819b 4622 };
2b6080f2 4623 struct trace_array *tr = iter->tr;
3c56819b 4624 ssize_t ret;
34cd4998 4625 size_t rem;
3c56819b
EGM
4626 unsigned int i;
4627
35f3d14d
JA
4628 if (splice_grow_spd(pipe, &spd))
4629 return -ENOMEM;
4630
d7350c3f 4631 /* copy the tracer to avoid using a global lock all around */
3c56819b 4632 mutex_lock(&trace_types_lock);
2b6080f2
SR
4633 if (unlikely(iter->trace->name != tr->current_trace->name))
4634 *iter->trace = *tr->current_trace;
d7350c3f
FW
4635 mutex_unlock(&trace_types_lock);
4636
4637 mutex_lock(&iter->mutex);
3c56819b
EGM
4638
4639 if (iter->trace->splice_read) {
4640 ret = iter->trace->splice_read(iter, filp,
4641 ppos, pipe, len, flags);
4642 if (ret)
34cd4998 4643 goto out_err;
3c56819b
EGM
4644 }
4645
4646 ret = tracing_wait_pipe(filp);
4647 if (ret <= 0)
34cd4998 4648 goto out_err;
3c56819b 4649
955b61e5 4650 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 4651 ret = -EFAULT;
34cd4998 4652 goto out_err;
3c56819b
EGM
4653 }
4654
4f535968 4655 trace_event_read_lock();
7e53bd42 4656 trace_access_lock(iter->cpu_file);
4f535968 4657
3c56819b 4658 /* Fill as many pages as possible. */
a786c06d 4659 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
35f3d14d
JA
4660 spd.pages[i] = alloc_page(GFP_KERNEL);
4661 if (!spd.pages[i])
34cd4998 4662 break;
3c56819b 4663
fa7c7f6e 4664 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
4665
4666 /* Copy the data into the page, so we can start over. */
4667 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 4668 page_address(spd.pages[i]),
3c56819b
EGM
4669 iter->seq.len);
4670 if (ret < 0) {
35f3d14d 4671 __free_page(spd.pages[i]);
3c56819b
EGM
4672 break;
4673 }
35f3d14d
JA
4674 spd.partial[i].offset = 0;
4675 spd.partial[i].len = iter->seq.len;
3c56819b 4676
f9520750 4677 trace_seq_init(&iter->seq);
3c56819b
EGM
4678 }
4679
7e53bd42 4680 trace_access_unlock(iter->cpu_file);
4f535968 4681 trace_event_read_unlock();
d7350c3f 4682 mutex_unlock(&iter->mutex);
3c56819b
EGM
4683
4684 spd.nr_pages = i;
4685
35f3d14d
JA
4686 ret = splice_to_pipe(pipe, &spd);
4687out:
047fe360 4688 splice_shrink_spd(&spd);
35f3d14d 4689 return ret;
3c56819b 4690
34cd4998 4691out_err:
d7350c3f 4692 mutex_unlock(&iter->mutex);
35f3d14d 4693 goto out;
3c56819b
EGM
4694}
4695
a98a3c3f
SR
4696static ssize_t
4697tracing_entries_read(struct file *filp, char __user *ubuf,
4698 size_t cnt, loff_t *ppos)
4699{
0bc392ee
ON
4700 struct inode *inode = file_inode(filp);
4701 struct trace_array *tr = inode->i_private;
4702 int cpu = tracing_get_cpu(inode);
438ced17
VN
4703 char buf[64];
4704 int r = 0;
4705 ssize_t ret;
a98a3c3f 4706
db526ca3 4707 mutex_lock(&trace_types_lock);
438ced17 4708
0bc392ee 4709 if (cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
4710 int cpu, buf_size_same;
4711 unsigned long size;
4712
4713 size = 0;
4714 buf_size_same = 1;
4715 /* check if all cpu sizes are same */
4716 for_each_tracing_cpu(cpu) {
4717 /* fill in the size from first enabled cpu */
4718 if (size == 0)
12883efb
SRRH
4719 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4720 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
4721 buf_size_same = 0;
4722 break;
4723 }
4724 }
4725
4726 if (buf_size_same) {
4727 if (!ring_buffer_expanded)
4728 r = sprintf(buf, "%lu (expanded: %lu)\n",
4729 size >> 10,
4730 trace_buf_size >> 10);
4731 else
4732 r = sprintf(buf, "%lu\n", size >> 10);
4733 } else
4734 r = sprintf(buf, "X\n");
4735 } else
0bc392ee 4736 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
438ced17 4737
db526ca3
SR
4738 mutex_unlock(&trace_types_lock);
4739
438ced17
VN
4740 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4741 return ret;
a98a3c3f
SR
4742}
4743
4744static ssize_t
4745tracing_entries_write(struct file *filp, const char __user *ubuf,
4746 size_t cnt, loff_t *ppos)
4747{
0bc392ee
ON
4748 struct inode *inode = file_inode(filp);
4749 struct trace_array *tr = inode->i_private;
a98a3c3f 4750 unsigned long val;
4f271a2a 4751 int ret;
a98a3c3f 4752
22fe9b54
PH
4753 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4754 if (ret)
c6caeeb1 4755 return ret;
a98a3c3f
SR
4756
4757 /* must have at least 1 entry */
4758 if (!val)
4759 return -EINVAL;
4760
1696b2b0
SR
4761 /* value is in KB */
4762 val <<= 10;
0bc392ee 4763 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4f271a2a
VN
4764 if (ret < 0)
4765 return ret;
a98a3c3f 4766
cf8517cf 4767 *ppos += cnt;
a98a3c3f 4768
4f271a2a
VN
4769 return cnt;
4770}
bf5e6519 4771
f81ab074
VN
4772static ssize_t
4773tracing_total_entries_read(struct file *filp, char __user *ubuf,
4774 size_t cnt, loff_t *ppos)
4775{
4776 struct trace_array *tr = filp->private_data;
4777 char buf[64];
4778 int r, cpu;
4779 unsigned long size = 0, expanded_size = 0;
4780
4781 mutex_lock(&trace_types_lock);
4782 for_each_tracing_cpu(cpu) {
12883efb 4783 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
4784 if (!ring_buffer_expanded)
4785 expanded_size += trace_buf_size >> 10;
4786 }
4787 if (ring_buffer_expanded)
4788 r = sprintf(buf, "%lu\n", size);
4789 else
4790 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4791 mutex_unlock(&trace_types_lock);
4792
4793 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4794}
4795
4f271a2a
VN
4796static ssize_t
4797tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4798 size_t cnt, loff_t *ppos)
4799{
4800 /*
4801 * There is no need to read what the user has written, this function
4802 * is just to make sure that there is no error when "echo" is used
4803 */
4804
4805 *ppos += cnt;
a98a3c3f
SR
4806
4807 return cnt;
4808}
4809
4f271a2a
VN
4810static int
4811tracing_free_buffer_release(struct inode *inode, struct file *filp)
4812{
2b6080f2
SR
4813 struct trace_array *tr = inode->i_private;
4814
cf30cf67
SR
4815 /* disable tracing ? */
4816 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
711e1243 4817 tracer_tracing_off(tr);
4f271a2a 4818 /* resize the ring buffer to 0 */
2b6080f2 4819 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a 4820
7b85af63
SRRH
4821 trace_array_put(tr);
4822
4f271a2a
VN
4823 return 0;
4824}
4825
5bf9a1ee
PP
4826static ssize_t
4827tracing_mark_write(struct file *filp, const char __user *ubuf,
4828 size_t cnt, loff_t *fpos)
4829{
d696b58c 4830 unsigned long addr = (unsigned long)ubuf;
2d71619c 4831 struct trace_array *tr = filp->private_data;
d696b58c
SR
4832 struct ring_buffer_event *event;
4833 struct ring_buffer *buffer;
4834 struct print_entry *entry;
4835 unsigned long irq_flags;
4836 struct page *pages[2];
6edb2a8a 4837 void *map_page[2];
d696b58c
SR
4838 int nr_pages = 1;
4839 ssize_t written;
d696b58c
SR
4840 int offset;
4841 int size;
4842 int len;
4843 int ret;
6edb2a8a 4844 int i;
5bf9a1ee 4845
c76f0694 4846 if (tracing_disabled)
5bf9a1ee
PP
4847 return -EINVAL;
4848
5224c3a3
MSB
4849 if (!(trace_flags & TRACE_ITER_MARKERS))
4850 return -EINVAL;
4851
5bf9a1ee
PP
4852 if (cnt > TRACE_BUF_SIZE)
4853 cnt = TRACE_BUF_SIZE;
4854
d696b58c
SR
4855 /*
4856 * Userspace is injecting traces into the kernel trace buffer.
4857 * We want to be as non intrusive as possible.
4858 * To do so, we do not want to allocate any special buffers
4859 * or take any locks, but instead write the userspace data
4860 * straight into the ring buffer.
4861 *
4862 * First we need to pin the userspace buffer into memory,
4863 * which, most likely it is, because it just referenced it.
4864 * But there's no guarantee that it is. By using get_user_pages_fast()
4865 * and kmap_atomic/kunmap_atomic() we can get access to the
4866 * pages directly. We then write the data directly into the
4867 * ring buffer.
4868 */
4869 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 4870
d696b58c
SR
4871 /* check if we cross pages */
4872 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4873 nr_pages = 2;
4874
4875 offset = addr & (PAGE_SIZE - 1);
4876 addr &= PAGE_MASK;
4877
4878 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4879 if (ret < nr_pages) {
4880 while (--ret >= 0)
4881 put_page(pages[ret]);
4882 written = -EFAULT;
4883 goto out;
5bf9a1ee 4884 }
d696b58c 4885
6edb2a8a
SR
4886 for (i = 0; i < nr_pages; i++)
4887 map_page[i] = kmap_atomic(pages[i]);
d696b58c
SR
4888
4889 local_save_flags(irq_flags);
4890 size = sizeof(*entry) + cnt + 2; /* possible \n added */
2d71619c 4891 buffer = tr->trace_buffer.buffer;
d696b58c
SR
4892 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4893 irq_flags, preempt_count());
4894 if (!event) {
4895 /* Ring buffer disabled, return as if not open for write */
4896 written = -EBADF;
4897 goto out_unlock;
5bf9a1ee 4898 }
d696b58c
SR
4899
4900 entry = ring_buffer_event_data(event);
4901 entry->ip = _THIS_IP_;
4902
4903 if (nr_pages == 2) {
4904 len = PAGE_SIZE - offset;
6edb2a8a
SR
4905 memcpy(&entry->buf, map_page[0] + offset, len);
4906 memcpy(&entry->buf[len], map_page[1], cnt - len);
c13d2f7c 4907 } else
6edb2a8a 4908 memcpy(&entry->buf, map_page[0] + offset, cnt);
5bf9a1ee 4909
d696b58c
SR
4910 if (entry->buf[cnt - 1] != '\n') {
4911 entry->buf[cnt] = '\n';
4912 entry->buf[cnt + 1] = '\0';
4913 } else
4914 entry->buf[cnt] = '\0';
4915
7ffbd48d 4916 __buffer_unlock_commit(buffer, event);
5bf9a1ee 4917
d696b58c 4918 written = cnt;
5bf9a1ee 4919
d696b58c 4920 *fpos += written;
1aa54bca 4921
d696b58c 4922 out_unlock:
6edb2a8a
SR
4923 for (i = 0; i < nr_pages; i++){
4924 kunmap_atomic(map_page[i]);
4925 put_page(pages[i]);
4926 }
d696b58c 4927 out:
1aa54bca 4928 return written;
5bf9a1ee
PP
4929}
4930
13f16d20 4931static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 4932{
2b6080f2 4933 struct trace_array *tr = m->private;
5079f326
Z
4934 int i;
4935
4936 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 4937 seq_printf(m,
5079f326 4938 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
4939 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4940 i == tr->clock_id ? "]" : "");
13f16d20 4941 seq_putc(m, '\n');
5079f326 4942
13f16d20 4943 return 0;
5079f326
Z
4944}
4945
e1e232ca 4946static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5079f326 4947{
5079f326
Z
4948 int i;
4949
5079f326
Z
4950 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4951 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4952 break;
4953 }
4954 if (i == ARRAY_SIZE(trace_clocks))
4955 return -EINVAL;
4956
5079f326
Z
4957 mutex_lock(&trace_types_lock);
4958
2b6080f2
SR
4959 tr->clock_id = i;
4960
12883efb 4961 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 4962
60303ed3
DS
4963 /*
4964 * New clock may not be consistent with the previous clock.
4965 * Reset the buffer so that it doesn't have incomparable timestamps.
4966 */
9457158b 4967 tracing_reset_online_cpus(&tr->trace_buffer);
12883efb
SRRH
4968
4969#ifdef CONFIG_TRACER_MAX_TRACE
4970 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4971 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
9457158b 4972 tracing_reset_online_cpus(&tr->max_buffer);
12883efb 4973#endif
60303ed3 4974
5079f326
Z
4975 mutex_unlock(&trace_types_lock);
4976
e1e232ca
SR
4977 return 0;
4978}
4979
4980static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4981 size_t cnt, loff_t *fpos)
4982{
4983 struct seq_file *m = filp->private_data;
4984 struct trace_array *tr = m->private;
4985 char buf[64];
4986 const char *clockstr;
4987 int ret;
4988
4989 if (cnt >= sizeof(buf))
4990 return -EINVAL;
4991
4992 if (copy_from_user(&buf, ubuf, cnt))
4993 return -EFAULT;
4994
4995 buf[cnt] = 0;
4996
4997 clockstr = strstrip(buf);
4998
4999 ret = tracing_set_clock(tr, clockstr);
5000 if (ret)
5001 return ret;
5002
5079f326
Z
5003 *fpos += cnt;
5004
5005 return cnt;
5006}
5007
13f16d20
LZ
5008static int tracing_clock_open(struct inode *inode, struct file *file)
5009{
7b85af63
SRRH
5010 struct trace_array *tr = inode->i_private;
5011 int ret;
5012
13f16d20
LZ
5013 if (tracing_disabled)
5014 return -ENODEV;
2b6080f2 5015
7b85af63
SRRH
5016 if (trace_array_get(tr))
5017 return -ENODEV;
5018
5019 ret = single_open(file, tracing_clock_show, inode->i_private);
5020 if (ret < 0)
5021 trace_array_put(tr);
5022
5023 return ret;
13f16d20
LZ
5024}
5025
6de58e62
SRRH
5026struct ftrace_buffer_info {
5027 struct trace_iterator iter;
5028 void *spare;
5029 unsigned int read;
5030};
5031
debdd57f
HT
5032#ifdef CONFIG_TRACER_SNAPSHOT
5033static int tracing_snapshot_open(struct inode *inode, struct file *file)
5034{
6484c71c 5035 struct trace_array *tr = inode->i_private;
debdd57f 5036 struct trace_iterator *iter;
2b6080f2 5037 struct seq_file *m;
debdd57f
HT
5038 int ret = 0;
5039
ff451961
SRRH
5040 if (trace_array_get(tr) < 0)
5041 return -ENODEV;
5042
debdd57f 5043 if (file->f_mode & FMODE_READ) {
6484c71c 5044 iter = __tracing_open(inode, file, true);
debdd57f
HT
5045 if (IS_ERR(iter))
5046 ret = PTR_ERR(iter);
2b6080f2
SR
5047 } else {
5048 /* Writes still need the seq_file to hold the private data */
f77d09a3 5049 ret = -ENOMEM;
2b6080f2
SR
5050 m = kzalloc(sizeof(*m), GFP_KERNEL);
5051 if (!m)
f77d09a3 5052 goto out;
2b6080f2
SR
5053 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5054 if (!iter) {
5055 kfree(m);
f77d09a3 5056 goto out;
2b6080f2 5057 }
f77d09a3
AL
5058 ret = 0;
5059
ff451961 5060 iter->tr = tr;
6484c71c
ON
5061 iter->trace_buffer = &tr->max_buffer;
5062 iter->cpu_file = tracing_get_cpu(inode);
2b6080f2
SR
5063 m->private = iter;
5064 file->private_data = m;
debdd57f 5065 }
f77d09a3 5066out:
ff451961
SRRH
5067 if (ret < 0)
5068 trace_array_put(tr);
5069
debdd57f
HT
5070 return ret;
5071}
5072
5073static ssize_t
5074tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5075 loff_t *ppos)
5076{
2b6080f2
SR
5077 struct seq_file *m = filp->private_data;
5078 struct trace_iterator *iter = m->private;
5079 struct trace_array *tr = iter->tr;
debdd57f
HT
5080 unsigned long val;
5081 int ret;
5082
5083 ret = tracing_update_buffers();
5084 if (ret < 0)
5085 return ret;
5086
5087 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5088 if (ret)
5089 return ret;
5090
5091 mutex_lock(&trace_types_lock);
5092
2b6080f2 5093 if (tr->current_trace->use_max_tr) {
debdd57f
HT
5094 ret = -EBUSY;
5095 goto out;
5096 }
5097
5098 switch (val) {
5099 case 0:
f1affcaa
SRRH
5100 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5101 ret = -EINVAL;
5102 break;
debdd57f 5103 }
3209cff4
SRRH
5104 if (tr->allocated_snapshot)
5105 free_snapshot(tr);
debdd57f
HT
5106 break;
5107 case 1:
f1affcaa
SRRH
5108/* Only allow per-cpu swap if the ring buffer supports it */
5109#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5110 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5111 ret = -EINVAL;
5112 break;
5113 }
5114#endif
45ad21ca 5115 if (!tr->allocated_snapshot) {
3209cff4 5116 ret = alloc_snapshot(tr);
debdd57f
HT
5117 if (ret < 0)
5118 break;
debdd57f 5119 }
debdd57f
HT
5120 local_irq_disable();
5121 /* Now, we're going to swap */
f1affcaa 5122 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
ce9bae55 5123 update_max_tr(tr, current, smp_processor_id());
f1affcaa 5124 else
ce9bae55 5125 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
5126 local_irq_enable();
5127 break;
5128 default:
45ad21ca 5129 if (tr->allocated_snapshot) {
f1affcaa
SRRH
5130 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5131 tracing_reset_online_cpus(&tr->max_buffer);
5132 else
5133 tracing_reset(&tr->max_buffer, iter->cpu_file);
5134 }
debdd57f
HT
5135 break;
5136 }
5137
5138 if (ret >= 0) {
5139 *ppos += cnt;
5140 ret = cnt;
5141 }
5142out:
5143 mutex_unlock(&trace_types_lock);
5144 return ret;
5145}
2b6080f2
SR
5146
5147static int tracing_snapshot_release(struct inode *inode, struct file *file)
5148{
5149 struct seq_file *m = file->private_data;
ff451961
SRRH
5150 int ret;
5151
5152 ret = tracing_release(inode, file);
2b6080f2
SR
5153
5154 if (file->f_mode & FMODE_READ)
ff451961 5155 return ret;
2b6080f2
SR
5156
5157 /* If write only, the seq_file is just a stub */
5158 if (m)
5159 kfree(m->private);
5160 kfree(m);
5161
5162 return 0;
5163}
5164
6de58e62
SRRH
5165static int tracing_buffers_open(struct inode *inode, struct file *filp);
5166static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5167 size_t count, loff_t *ppos);
5168static int tracing_buffers_release(struct inode *inode, struct file *file);
5169static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5170 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5171
5172static int snapshot_raw_open(struct inode *inode, struct file *filp)
5173{
5174 struct ftrace_buffer_info *info;
5175 int ret;
5176
5177 ret = tracing_buffers_open(inode, filp);
5178 if (ret < 0)
5179 return ret;
5180
5181 info = filp->private_data;
5182
5183 if (info->iter.trace->use_max_tr) {
5184 tracing_buffers_release(inode, filp);
5185 return -EBUSY;
5186 }
5187
5188 info->iter.snapshot = true;
5189 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5190
5191 return ret;
5192}
5193
debdd57f
HT
5194#endif /* CONFIG_TRACER_SNAPSHOT */
5195
5196
6508fa76
SF
5197static const struct file_operations tracing_thresh_fops = {
5198 .open = tracing_open_generic,
5199 .read = tracing_thresh_read,
5200 .write = tracing_thresh_write,
5201 .llseek = generic_file_llseek,
5202};
5203
5e2336a0 5204static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
5205 .open = tracing_open_generic,
5206 .read = tracing_max_lat_read,
5207 .write = tracing_max_lat_write,
b444786f 5208 .llseek = generic_file_llseek,
bc0c38d1
SR
5209};
5210
5e2336a0 5211static const struct file_operations set_tracer_fops = {
4bf39a94
IM
5212 .open = tracing_open_generic,
5213 .read = tracing_set_trace_read,
5214 .write = tracing_set_trace_write,
b444786f 5215 .llseek = generic_file_llseek,
bc0c38d1
SR
5216};
5217
5e2336a0 5218static const struct file_operations tracing_pipe_fops = {
4bf39a94 5219 .open = tracing_open_pipe,
2a2cc8f7 5220 .poll = tracing_poll_pipe,
4bf39a94 5221 .read = tracing_read_pipe,
3c56819b 5222 .splice_read = tracing_splice_read_pipe,
4bf39a94 5223 .release = tracing_release_pipe,
b444786f 5224 .llseek = no_llseek,
b3806b43
SR
5225};
5226
5e2336a0 5227static const struct file_operations tracing_entries_fops = {
0bc392ee 5228 .open = tracing_open_generic_tr,
a98a3c3f
SR
5229 .read = tracing_entries_read,
5230 .write = tracing_entries_write,
b444786f 5231 .llseek = generic_file_llseek,
0bc392ee 5232 .release = tracing_release_generic_tr,
a98a3c3f
SR
5233};
5234
f81ab074 5235static const struct file_operations tracing_total_entries_fops = {
7b85af63 5236 .open = tracing_open_generic_tr,
f81ab074
VN
5237 .read = tracing_total_entries_read,
5238 .llseek = generic_file_llseek,
7b85af63 5239 .release = tracing_release_generic_tr,
f81ab074
VN
5240};
5241
4f271a2a 5242static const struct file_operations tracing_free_buffer_fops = {
7b85af63 5243 .open = tracing_open_generic_tr,
4f271a2a
VN
5244 .write = tracing_free_buffer_write,
5245 .release = tracing_free_buffer_release,
5246};
5247
5e2336a0 5248static const struct file_operations tracing_mark_fops = {
7b85af63 5249 .open = tracing_open_generic_tr,
5bf9a1ee 5250 .write = tracing_mark_write,
b444786f 5251 .llseek = generic_file_llseek,
7b85af63 5252 .release = tracing_release_generic_tr,
5bf9a1ee
PP
5253};
5254
5079f326 5255static const struct file_operations trace_clock_fops = {
13f16d20
LZ
5256 .open = tracing_clock_open,
5257 .read = seq_read,
5258 .llseek = seq_lseek,
7b85af63 5259 .release = tracing_single_release_tr,
5079f326
Z
5260 .write = tracing_clock_write,
5261};
5262
debdd57f
HT
5263#ifdef CONFIG_TRACER_SNAPSHOT
5264static const struct file_operations snapshot_fops = {
5265 .open = tracing_snapshot_open,
5266 .read = seq_read,
5267 .write = tracing_snapshot_write,
098c879e 5268 .llseek = tracing_lseek,
2b6080f2 5269 .release = tracing_snapshot_release,
debdd57f 5270};
debdd57f 5271
6de58e62
SRRH
5272static const struct file_operations snapshot_raw_fops = {
5273 .open = snapshot_raw_open,
5274 .read = tracing_buffers_read,
5275 .release = tracing_buffers_release,
5276 .splice_read = tracing_buffers_splice_read,
5277 .llseek = no_llseek,
2cadf913
SR
5278};
5279
6de58e62
SRRH
5280#endif /* CONFIG_TRACER_SNAPSHOT */
5281
2cadf913
SR
5282static int tracing_buffers_open(struct inode *inode, struct file *filp)
5283{
46ef2be0 5284 struct trace_array *tr = inode->i_private;
2cadf913 5285 struct ftrace_buffer_info *info;
7b85af63 5286 int ret;
2cadf913
SR
5287
5288 if (tracing_disabled)
5289 return -ENODEV;
5290
7b85af63
SRRH
5291 if (trace_array_get(tr) < 0)
5292 return -ENODEV;
5293
2cadf913 5294 info = kzalloc(sizeof(*info), GFP_KERNEL);
7b85af63
SRRH
5295 if (!info) {
5296 trace_array_put(tr);
2cadf913 5297 return -ENOMEM;
7b85af63 5298 }
2cadf913 5299
a695cb58
SRRH
5300 mutex_lock(&trace_types_lock);
5301
cc60cdc9 5302 info->iter.tr = tr;
46ef2be0 5303 info->iter.cpu_file = tracing_get_cpu(inode);
b627344f 5304 info->iter.trace = tr->current_trace;
12883efb 5305 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 5306 info->spare = NULL;
2cadf913 5307 /* Force reading ring buffer for first read */
cc60cdc9 5308 info->read = (unsigned int)-1;
2cadf913
SR
5309
5310 filp->private_data = info;
5311
a695cb58
SRRH
5312 mutex_unlock(&trace_types_lock);
5313
7b85af63
SRRH
5314 ret = nonseekable_open(inode, filp);
5315 if (ret < 0)
5316 trace_array_put(tr);
5317
5318 return ret;
2cadf913
SR
5319}
5320
cc60cdc9
SR
5321static unsigned int
5322tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5323{
5324 struct ftrace_buffer_info *info = filp->private_data;
5325 struct trace_iterator *iter = &info->iter;
5326
5327 return trace_poll(iter, filp, poll_table);
5328}
5329
2cadf913
SR
5330static ssize_t
5331tracing_buffers_read(struct file *filp, char __user *ubuf,
5332 size_t count, loff_t *ppos)
5333{
5334 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 5335 struct trace_iterator *iter = &info->iter;
2cadf913 5336 ssize_t ret;
6de58e62 5337 ssize_t size;
2cadf913 5338
2dc5d12b
SR
5339 if (!count)
5340 return 0;
5341
6de58e62
SRRH
5342 mutex_lock(&trace_types_lock);
5343
5344#ifdef CONFIG_TRACER_MAX_TRACE
5345 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5346 size = -EBUSY;
5347 goto out_unlock;
5348 }
5349#endif
5350
ddd538f3 5351 if (!info->spare)
12883efb
SRRH
5352 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5353 iter->cpu_file);
6de58e62 5354 size = -ENOMEM;
ddd538f3 5355 if (!info->spare)
6de58e62 5356 goto out_unlock;
ddd538f3 5357
2cadf913
SR
5358 /* Do we have previous read data to read? */
5359 if (info->read < PAGE_SIZE)
5360 goto read;
5361
b627344f 5362 again:
cc60cdc9 5363 trace_access_lock(iter->cpu_file);
12883efb 5364 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
5365 &info->spare,
5366 count,
cc60cdc9
SR
5367 iter->cpu_file, 0);
5368 trace_access_unlock(iter->cpu_file);
2cadf913 5369
b627344f
SR
5370 if (ret < 0) {
5371 if (trace_empty(iter)) {
6de58e62
SRRH
5372 if ((filp->f_flags & O_NONBLOCK)) {
5373 size = -EAGAIN;
5374 goto out_unlock;
5375 }
5376 mutex_unlock(&trace_types_lock);
8b8b3683 5377 ret = wait_on_pipe(iter);
6de58e62 5378 mutex_lock(&trace_types_lock);
8b8b3683
SRRH
5379 if (ret) {
5380 size = ret;
5381 goto out_unlock;
5382 }
6de58e62
SRRH
5383 if (signal_pending(current)) {
5384 size = -EINTR;
5385 goto out_unlock;
5386 }
b627344f
SR
5387 goto again;
5388 }
6de58e62
SRRH
5389 size = 0;
5390 goto out_unlock;
b627344f 5391 }
436fc280 5392
436fc280 5393 info->read = 0;
b627344f 5394 read:
2cadf913
SR
5395 size = PAGE_SIZE - info->read;
5396 if (size > count)
5397 size = count;
5398
5399 ret = copy_to_user(ubuf, info->spare + info->read, size);
6de58e62
SRRH
5400 if (ret == size) {
5401 size = -EFAULT;
5402 goto out_unlock;
5403 }
2dc5d12b
SR
5404 size -= ret;
5405
2cadf913
SR
5406 *ppos += size;
5407 info->read += size;
5408
6de58e62
SRRH
5409 out_unlock:
5410 mutex_unlock(&trace_types_lock);
5411
2cadf913
SR
5412 return size;
5413}
5414
5415static int tracing_buffers_release(struct inode *inode, struct file *file)
5416{
5417 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 5418 struct trace_iterator *iter = &info->iter;
2cadf913 5419
a695cb58
SRRH
5420 mutex_lock(&trace_types_lock);
5421
ff451961 5422 __trace_array_put(iter->tr);
2cadf913 5423
ddd538f3 5424 if (info->spare)
12883efb 5425 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
2cadf913
SR
5426 kfree(info);
5427
a695cb58
SRRH
5428 mutex_unlock(&trace_types_lock);
5429
2cadf913
SR
5430 return 0;
5431}
5432
5433struct buffer_ref {
5434 struct ring_buffer *buffer;
5435 void *page;
5436 int ref;
5437};
5438
5439static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5440 struct pipe_buffer *buf)
5441{
5442 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5443
5444 if (--ref->ref)
5445 return;
5446
5447 ring_buffer_free_read_page(ref->buffer, ref->page);
5448 kfree(ref);
5449 buf->private = 0;
5450}
5451
2cadf913
SR
5452static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5453 struct pipe_buffer *buf)
5454{
5455 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5456
5457 ref->ref++;
5458}
5459
5460/* Pipe buffer operations for a buffer. */
28dfef8f 5461static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913 5462 .can_merge = 0,
2cadf913
SR
5463 .confirm = generic_pipe_buf_confirm,
5464 .release = buffer_pipe_buf_release,
d55cb6cf 5465 .steal = generic_pipe_buf_steal,
2cadf913
SR
5466 .get = buffer_pipe_buf_get,
5467};
5468
5469/*
5470 * Callback from splice_to_pipe(), if we need to release some pages
5471 * at the end of the spd in case we error'ed out in filling the pipe.
5472 */
5473static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5474{
5475 struct buffer_ref *ref =
5476 (struct buffer_ref *)spd->partial[i].private;
5477
5478 if (--ref->ref)
5479 return;
5480
5481 ring_buffer_free_read_page(ref->buffer, ref->page);
5482 kfree(ref);
5483 spd->partial[i].private = 0;
5484}
5485
5486static ssize_t
5487tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5488 struct pipe_inode_info *pipe, size_t len,
5489 unsigned int flags)
5490{
5491 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 5492 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
5493 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5494 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 5495 struct splice_pipe_desc spd = {
35f3d14d
JA
5496 .pages = pages_def,
5497 .partial = partial_def,
047fe360 5498 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
5499 .flags = flags,
5500 .ops = &buffer_pipe_buf_ops,
5501 .spd_release = buffer_spd_release,
5502 };
5503 struct buffer_ref *ref;
93459c6c 5504 int entries, size, i;
6de58e62 5505 ssize_t ret;
2cadf913 5506
6de58e62
SRRH
5507 mutex_lock(&trace_types_lock);
5508
5509#ifdef CONFIG_TRACER_MAX_TRACE
5510 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5511 ret = -EBUSY;
5512 goto out;
5513 }
5514#endif
5515
5516 if (splice_grow_spd(pipe, &spd)) {
5517 ret = -ENOMEM;
5518 goto out;
5519 }
35f3d14d 5520
93cfb3c9 5521 if (*ppos & (PAGE_SIZE - 1)) {
35f3d14d
JA
5522 ret = -EINVAL;
5523 goto out;
93cfb3c9
LJ
5524 }
5525
5526 if (len & (PAGE_SIZE - 1)) {
35f3d14d
JA
5527 if (len < PAGE_SIZE) {
5528 ret = -EINVAL;
5529 goto out;
5530 }
93cfb3c9
LJ
5531 len &= PAGE_MASK;
5532 }
5533
cc60cdc9
SR
5534 again:
5535 trace_access_lock(iter->cpu_file);
12883efb 5536 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 5537
a786c06d 5538 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
5539 struct page *page;
5540 int r;
5541
5542 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5543 if (!ref)
5544 break;
5545
7267fa68 5546 ref->ref = 1;
12883efb 5547 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 5548 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
2cadf913
SR
5549 if (!ref->page) {
5550 kfree(ref);
5551 break;
5552 }
5553
5554 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 5555 len, iter->cpu_file, 1);
2cadf913 5556 if (r < 0) {
7ea59064 5557 ring_buffer_free_read_page(ref->buffer, ref->page);
2cadf913
SR
5558 kfree(ref);
5559 break;
5560 }
5561
5562 /*
5563 * zero out any left over data, this is going to
5564 * user land.
5565 */
5566 size = ring_buffer_page_len(ref->page);
5567 if (size < PAGE_SIZE)
5568 memset(ref->page + size, 0, PAGE_SIZE - size);
5569
5570 page = virt_to_page(ref->page);
5571
5572 spd.pages[i] = page;
5573 spd.partial[i].len = PAGE_SIZE;
5574 spd.partial[i].offset = 0;
5575 spd.partial[i].private = (unsigned long)ref;
5576 spd.nr_pages++;
93cfb3c9 5577 *ppos += PAGE_SIZE;
93459c6c 5578
12883efb 5579 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
5580 }
5581
cc60cdc9 5582 trace_access_unlock(iter->cpu_file);
2cadf913
SR
5583 spd.nr_pages = i;
5584
5585 /* did we read anything? */
5586 if (!spd.nr_pages) {
cc60cdc9 5587 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
2cadf913 5588 ret = -EAGAIN;
cc60cdc9
SR
5589 goto out;
5590 }
6de58e62 5591 mutex_unlock(&trace_types_lock);
8b8b3683 5592 ret = wait_on_pipe(iter);
6de58e62 5593 mutex_lock(&trace_types_lock);
8b8b3683
SRRH
5594 if (ret)
5595 goto out;
cc60cdc9
SR
5596 if (signal_pending(current)) {
5597 ret = -EINTR;
5598 goto out;
5599 }
5600 goto again;
2cadf913
SR
5601 }
5602
5603 ret = splice_to_pipe(pipe, &spd);
047fe360 5604 splice_shrink_spd(&spd);
35f3d14d 5605out:
6de58e62
SRRH
5606 mutex_unlock(&trace_types_lock);
5607
2cadf913
SR
5608 return ret;
5609}
5610
5611static const struct file_operations tracing_buffers_fops = {
5612 .open = tracing_buffers_open,
5613 .read = tracing_buffers_read,
cc60cdc9 5614 .poll = tracing_buffers_poll,
2cadf913
SR
5615 .release = tracing_buffers_release,
5616 .splice_read = tracing_buffers_splice_read,
5617 .llseek = no_llseek,
5618};
5619
c8d77183
SR
5620static ssize_t
5621tracing_stats_read(struct file *filp, char __user *ubuf,
5622 size_t count, loff_t *ppos)
5623{
4d3435b8
ON
5624 struct inode *inode = file_inode(filp);
5625 struct trace_array *tr = inode->i_private;
12883efb 5626 struct trace_buffer *trace_buf = &tr->trace_buffer;
4d3435b8 5627 int cpu = tracing_get_cpu(inode);
c8d77183
SR
5628 struct trace_seq *s;
5629 unsigned long cnt;
c64e148a
VN
5630 unsigned long long t;
5631 unsigned long usec_rem;
c8d77183 5632
e4f2d10f 5633 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 5634 if (!s)
a646365c 5635 return -ENOMEM;
c8d77183
SR
5636
5637 trace_seq_init(s);
5638
12883efb 5639 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5640 trace_seq_printf(s, "entries: %ld\n", cnt);
5641
12883efb 5642 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5643 trace_seq_printf(s, "overrun: %ld\n", cnt);
5644
12883efb 5645 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5646 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5647
12883efb 5648 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
5649 trace_seq_printf(s, "bytes: %ld\n", cnt);
5650
58e8eedf 5651 if (trace_clocks[tr->clock_id].in_ns) {
11043d8b 5652 /* local or global for trace_clock */
12883efb 5653 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
5654 usec_rem = do_div(t, USEC_PER_SEC);
5655 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5656 t, usec_rem);
5657
12883efb 5658 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
5659 usec_rem = do_div(t, USEC_PER_SEC);
5660 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5661 } else {
5662 /* counter or tsc mode for trace_clock */
5663 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 5664 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 5665
11043d8b 5666 trace_seq_printf(s, "now ts: %llu\n",
12883efb 5667 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 5668 }
c64e148a 5669
12883efb 5670 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
5671 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5672
12883efb 5673 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
5674 trace_seq_printf(s, "read events: %ld\n", cnt);
5675
c8d77183
SR
5676 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5677
5678 kfree(s);
5679
5680 return count;
5681}
5682
5683static const struct file_operations tracing_stats_fops = {
4d3435b8 5684 .open = tracing_open_generic_tr,
c8d77183 5685 .read = tracing_stats_read,
b444786f 5686 .llseek = generic_file_llseek,
4d3435b8 5687 .release = tracing_release_generic_tr,
c8d77183
SR
5688};
5689
bc0c38d1
SR
5690#ifdef CONFIG_DYNAMIC_FTRACE
5691
b807c3d0
SR
5692int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5693{
5694 return 0;
5695}
5696
bc0c38d1 5697static ssize_t
b807c3d0 5698tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
5699 size_t cnt, loff_t *ppos)
5700{
a26a2a27
SR
5701 static char ftrace_dyn_info_buffer[1024];
5702 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 5703 unsigned long *p = filp->private_data;
b807c3d0 5704 char *buf = ftrace_dyn_info_buffer;
a26a2a27 5705 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
5706 int r;
5707
b807c3d0
SR
5708 mutex_lock(&dyn_info_mutex);
5709 r = sprintf(buf, "%ld ", *p);
4bf39a94 5710
a26a2a27 5711 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
5712 buf[r++] = '\n';
5713
5714 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5715
5716 mutex_unlock(&dyn_info_mutex);
5717
5718 return r;
bc0c38d1
SR
5719}
5720
5e2336a0 5721static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 5722 .open = tracing_open_generic,
b807c3d0 5723 .read = tracing_read_dyn_info,
b444786f 5724 .llseek = generic_file_llseek,
bc0c38d1 5725};
77fd5c15 5726#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 5727
77fd5c15
SRRH
5728#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5729static void
5730ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5731{
5732 tracing_snapshot();
5733}
bc0c38d1 5734
77fd5c15
SRRH
5735static void
5736ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
bc0c38d1 5737{
77fd5c15
SRRH
5738 unsigned long *count = (long *)data;
5739
5740 if (!*count)
5741 return;
bc0c38d1 5742
77fd5c15
SRRH
5743 if (*count != -1)
5744 (*count)--;
5745
5746 tracing_snapshot();
5747}
5748
5749static int
5750ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5751 struct ftrace_probe_ops *ops, void *data)
5752{
5753 long count = (long)data;
5754
5755 seq_printf(m, "%ps:", (void *)ip);
5756
fa6f0cc7 5757 seq_puts(m, "snapshot");
77fd5c15
SRRH
5758
5759 if (count == -1)
fa6f0cc7 5760 seq_puts(m, ":unlimited\n");
77fd5c15
SRRH
5761 else
5762 seq_printf(m, ":count=%ld\n", count);
5763
5764 return 0;
5765}
5766
5767static struct ftrace_probe_ops snapshot_probe_ops = {
5768 .func = ftrace_snapshot,
5769 .print = ftrace_snapshot_print,
5770};
5771
5772static struct ftrace_probe_ops snapshot_count_probe_ops = {
5773 .func = ftrace_count_snapshot,
5774 .print = ftrace_snapshot_print,
5775};
5776
5777static int
5778ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5779 char *glob, char *cmd, char *param, int enable)
5780{
5781 struct ftrace_probe_ops *ops;
5782 void *count = (void *)-1;
5783 char *number;
5784 int ret;
5785
5786 /* hash funcs only work with set_ftrace_filter */
5787 if (!enable)
5788 return -EINVAL;
5789
5790 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5791
5792 if (glob[0] == '!') {
5793 unregister_ftrace_function_probe_func(glob+1, ops);
5794 return 0;
5795 }
5796
5797 if (!param)
5798 goto out_reg;
5799
5800 number = strsep(&param, ":");
5801
5802 if (!strlen(number))
5803 goto out_reg;
5804
5805 /*
5806 * We use the callback data field (which is a pointer)
5807 * as our counter.
5808 */
5809 ret = kstrtoul(number, 0, (unsigned long *)&count);
5810 if (ret)
5811 return ret;
5812
5813 out_reg:
5814 ret = register_ftrace_function_probe(glob, ops, count);
5815
5816 if (ret >= 0)
5817 alloc_snapshot(&global_trace);
5818
5819 return ret < 0 ? ret : 0;
5820}
5821
5822static struct ftrace_func_command ftrace_snapshot_cmd = {
5823 .name = "snapshot",
5824 .func = ftrace_trace_snapshot_callback,
5825};
5826
38de93ab 5827static __init int register_snapshot_cmd(void)
77fd5c15
SRRH
5828{
5829 return register_ftrace_command(&ftrace_snapshot_cmd);
5830}
5831#else
38de93ab 5832static inline __init int register_snapshot_cmd(void) { return 0; }
77fd5c15 5833#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 5834
2b6080f2 5835struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
bc0c38d1 5836{
2b6080f2
SR
5837 if (tr->dir)
5838 return tr->dir;
bc0c38d1 5839
3e1f60b8
FW
5840 if (!debugfs_initialized())
5841 return NULL;
5842
2b6080f2
SR
5843 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5844 tr->dir = debugfs_create_dir("tracing", NULL);
bc0c38d1 5845
687c878a
J
5846 if (!tr->dir)
5847 pr_warn_once("Could not create debugfs directory 'tracing'\n");
bc0c38d1 5848
2b6080f2 5849 return tr->dir;
bc0c38d1
SR
5850}
5851
2b6080f2
SR
5852struct dentry *tracing_init_dentry(void)
5853{
5854 return tracing_init_dentry_tr(&global_trace);
5855}
b04cc6b1 5856
2b6080f2 5857static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 5858{
b04cc6b1
FW
5859 struct dentry *d_tracer;
5860
2b6080f2
SR
5861 if (tr->percpu_dir)
5862 return tr->percpu_dir;
b04cc6b1 5863
2b6080f2 5864 d_tracer = tracing_init_dentry_tr(tr);
b04cc6b1
FW
5865 if (!d_tracer)
5866 return NULL;
5867
2b6080f2 5868 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
b04cc6b1 5869
2b6080f2
SR
5870 WARN_ONCE(!tr->percpu_dir,
5871 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 5872
2b6080f2 5873 return tr->percpu_dir;
b04cc6b1
FW
5874}
5875
649e9c70
ON
5876static struct dentry *
5877trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5878 void *data, long cpu, const struct file_operations *fops)
5879{
5880 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5881
5882 if (ret) /* See tracing_get_cpu() */
5883 ret->d_inode->i_cdev = (void *)(cpu + 1);
5884 return ret;
5885}
5886
2b6080f2
SR
5887static void
5888tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 5889{
2b6080f2 5890 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 5891 struct dentry *d_cpu;
dd49a38c 5892 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 5893
0a3d7ce7
NK
5894 if (!d_percpu)
5895 return;
5896
dd49a38c 5897 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8656e7a2
FW
5898 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5899 if (!d_cpu) {
5900 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5901 return;
5902 }
b04cc6b1 5903
8656e7a2 5904 /* per cpu trace_pipe */
649e9c70 5905 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
15544209 5906 tr, cpu, &tracing_pipe_fops);
b04cc6b1
FW
5907
5908 /* per cpu trace */
649e9c70 5909 trace_create_cpu_file("trace", 0644, d_cpu,
6484c71c 5910 tr, cpu, &tracing_fops);
7f96f93f 5911
649e9c70 5912 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
46ef2be0 5913 tr, cpu, &tracing_buffers_fops);
7f96f93f 5914
649e9c70 5915 trace_create_cpu_file("stats", 0444, d_cpu,
4d3435b8 5916 tr, cpu, &tracing_stats_fops);
438ced17 5917
649e9c70 5918 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
0bc392ee 5919 tr, cpu, &tracing_entries_fops);
f1affcaa
SRRH
5920
5921#ifdef CONFIG_TRACER_SNAPSHOT
649e9c70 5922 trace_create_cpu_file("snapshot", 0644, d_cpu,
6484c71c 5923 tr, cpu, &snapshot_fops);
6de58e62 5924
649e9c70 5925 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
46ef2be0 5926 tr, cpu, &snapshot_raw_fops);
f1affcaa 5927#endif
b04cc6b1
FW
5928}
5929
60a11774
SR
5930#ifdef CONFIG_FTRACE_SELFTEST
5931/* Let selftest have access to static functions in this file */
5932#include "trace_selftest.c"
5933#endif
5934
577b785f
SR
5935struct trace_option_dentry {
5936 struct tracer_opt *opt;
5937 struct tracer_flags *flags;
2b6080f2 5938 struct trace_array *tr;
577b785f
SR
5939 struct dentry *entry;
5940};
5941
5942static ssize_t
5943trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5944 loff_t *ppos)
5945{
5946 struct trace_option_dentry *topt = filp->private_data;
5947 char *buf;
5948
5949 if (topt->flags->val & topt->opt->bit)
5950 buf = "1\n";
5951 else
5952 buf = "0\n";
5953
5954 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5955}
5956
5957static ssize_t
5958trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5959 loff_t *ppos)
5960{
5961 struct trace_option_dentry *topt = filp->private_data;
5962 unsigned long val;
577b785f
SR
5963 int ret;
5964
22fe9b54
PH
5965 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5966 if (ret)
577b785f
SR
5967 return ret;
5968
8d18eaaf
LZ
5969 if (val != 0 && val != 1)
5970 return -EINVAL;
577b785f 5971
8d18eaaf 5972 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 5973 mutex_lock(&trace_types_lock);
8c1a49ae 5974 ret = __set_tracer_option(topt->tr, topt->flags,
c757bea9 5975 topt->opt, !val);
577b785f
SR
5976 mutex_unlock(&trace_types_lock);
5977 if (ret)
5978 return ret;
577b785f
SR
5979 }
5980
5981 *ppos += cnt;
5982
5983 return cnt;
5984}
5985
5986
5987static const struct file_operations trace_options_fops = {
5988 .open = tracing_open_generic,
5989 .read = trace_options_read,
5990 .write = trace_options_write,
b444786f 5991 .llseek = generic_file_llseek,
577b785f
SR
5992};
5993
a8259075
SR
5994static ssize_t
5995trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5996 loff_t *ppos)
5997{
5998 long index = (long)filp->private_data;
5999 char *buf;
6000
6001 if (trace_flags & (1 << index))
6002 buf = "1\n";
6003 else
6004 buf = "0\n";
6005
6006 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6007}
6008
6009static ssize_t
6010trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6011 loff_t *ppos)
6012{
2b6080f2 6013 struct trace_array *tr = &global_trace;
a8259075 6014 long index = (long)filp->private_data;
a8259075
SR
6015 unsigned long val;
6016 int ret;
6017
22fe9b54
PH
6018 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6019 if (ret)
a8259075
SR
6020 return ret;
6021
f2d84b65 6022 if (val != 0 && val != 1)
a8259075 6023 return -EINVAL;
69d34da2
SRRH
6024
6025 mutex_lock(&trace_types_lock);
2b6080f2 6026 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 6027 mutex_unlock(&trace_types_lock);
a8259075 6028
613f04a0
SRRH
6029 if (ret < 0)
6030 return ret;
6031
a8259075
SR
6032 *ppos += cnt;
6033
6034 return cnt;
6035}
6036
a8259075
SR
6037static const struct file_operations trace_options_core_fops = {
6038 .open = tracing_open_generic,
6039 .read = trace_options_core_read,
6040 .write = trace_options_core_write,
b444786f 6041 .llseek = generic_file_llseek,
a8259075
SR
6042};
6043
5452af66 6044struct dentry *trace_create_file(const char *name,
f4ae40a6 6045 umode_t mode,
5452af66
FW
6046 struct dentry *parent,
6047 void *data,
6048 const struct file_operations *fops)
6049{
6050 struct dentry *ret;
6051
6052 ret = debugfs_create_file(name, mode, parent, data, fops);
6053 if (!ret)
6054 pr_warning("Could not create debugfs '%s' entry\n", name);
6055
6056 return ret;
6057}
6058
6059
2b6080f2 6060static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
6061{
6062 struct dentry *d_tracer;
a8259075 6063
2b6080f2
SR
6064 if (tr->options)
6065 return tr->options;
a8259075 6066
2b6080f2 6067 d_tracer = tracing_init_dentry_tr(tr);
a8259075
SR
6068 if (!d_tracer)
6069 return NULL;
6070
2b6080f2
SR
6071 tr->options = debugfs_create_dir("options", d_tracer);
6072 if (!tr->options) {
a8259075
SR
6073 pr_warning("Could not create debugfs directory 'options'\n");
6074 return NULL;
6075 }
6076
2b6080f2 6077 return tr->options;
a8259075
SR
6078}
6079
577b785f 6080static void
2b6080f2
SR
6081create_trace_option_file(struct trace_array *tr,
6082 struct trace_option_dentry *topt,
577b785f
SR
6083 struct tracer_flags *flags,
6084 struct tracer_opt *opt)
6085{
6086 struct dentry *t_options;
577b785f 6087
2b6080f2 6088 t_options = trace_options_init_dentry(tr);
577b785f
SR
6089 if (!t_options)
6090 return;
6091
6092 topt->flags = flags;
6093 topt->opt = opt;
2b6080f2 6094 topt->tr = tr;
577b785f 6095
5452af66 6096 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
6097 &trace_options_fops);
6098
577b785f
SR
6099}
6100
6101static struct trace_option_dentry *
2b6080f2 6102create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
6103{
6104 struct trace_option_dentry *topts;
6105 struct tracer_flags *flags;
6106 struct tracer_opt *opts;
6107 int cnt;
6108
6109 if (!tracer)
6110 return NULL;
6111
6112 flags = tracer->flags;
6113
6114 if (!flags || !flags->opts)
6115 return NULL;
6116
6117 opts = flags->opts;
6118
6119 for (cnt = 0; opts[cnt].name; cnt++)
6120 ;
6121
0cfe8245 6122 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
6123 if (!topts)
6124 return NULL;
6125
6126 for (cnt = 0; opts[cnt].name; cnt++)
2b6080f2 6127 create_trace_option_file(tr, &topts[cnt], flags,
577b785f
SR
6128 &opts[cnt]);
6129
6130 return topts;
6131}
6132
6133static void
6134destroy_trace_option_files(struct trace_option_dentry *topts)
6135{
6136 int cnt;
6137
6138 if (!topts)
6139 return;
6140
3f4d8f78
FF
6141 for (cnt = 0; topts[cnt].opt; cnt++)
6142 debugfs_remove(topts[cnt].entry);
577b785f
SR
6143
6144 kfree(topts);
6145}
6146
a8259075 6147static struct dentry *
2b6080f2
SR
6148create_trace_option_core_file(struct trace_array *tr,
6149 const char *option, long index)
a8259075
SR
6150{
6151 struct dentry *t_options;
a8259075 6152
2b6080f2 6153 t_options = trace_options_init_dentry(tr);
a8259075
SR
6154 if (!t_options)
6155 return NULL;
6156
5452af66 6157 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 6158 &trace_options_core_fops);
a8259075
SR
6159}
6160
2b6080f2 6161static __init void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
6162{
6163 struct dentry *t_options;
a8259075
SR
6164 int i;
6165
2b6080f2 6166 t_options = trace_options_init_dentry(tr);
a8259075
SR
6167 if (!t_options)
6168 return;
6169
5452af66 6170 for (i = 0; trace_options[i]; i++)
2b6080f2 6171 create_trace_option_core_file(tr, trace_options[i], i);
a8259075
SR
6172}
6173
499e5470
SR
6174static ssize_t
6175rb_simple_read(struct file *filp, char __user *ubuf,
6176 size_t cnt, loff_t *ppos)
6177{
348f0fc2 6178 struct trace_array *tr = filp->private_data;
499e5470
SR
6179 char buf[64];
6180 int r;
6181
10246fa3 6182 r = tracer_tracing_is_on(tr);
499e5470
SR
6183 r = sprintf(buf, "%d\n", r);
6184
6185 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6186}
6187
6188static ssize_t
6189rb_simple_write(struct file *filp, const char __user *ubuf,
6190 size_t cnt, loff_t *ppos)
6191{
348f0fc2 6192 struct trace_array *tr = filp->private_data;
12883efb 6193 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
6194 unsigned long val;
6195 int ret;
6196
6197 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6198 if (ret)
6199 return ret;
6200
6201 if (buffer) {
2df8f8a6
SR
6202 mutex_lock(&trace_types_lock);
6203 if (val) {
10246fa3 6204 tracer_tracing_on(tr);
2b6080f2
SR
6205 if (tr->current_trace->start)
6206 tr->current_trace->start(tr);
2df8f8a6 6207 } else {
10246fa3 6208 tracer_tracing_off(tr);
2b6080f2
SR
6209 if (tr->current_trace->stop)
6210 tr->current_trace->stop(tr);
2df8f8a6
SR
6211 }
6212 mutex_unlock(&trace_types_lock);
499e5470
SR
6213 }
6214
6215 (*ppos)++;
6216
6217 return cnt;
6218}
6219
6220static const struct file_operations rb_simple_fops = {
7b85af63 6221 .open = tracing_open_generic_tr,
499e5470
SR
6222 .read = rb_simple_read,
6223 .write = rb_simple_write,
7b85af63 6224 .release = tracing_release_generic_tr,
499e5470
SR
6225 .llseek = default_llseek,
6226};
6227
277ba044
SR
6228struct dentry *trace_instance_dir;
6229
6230static void
6231init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6232
55034cd6
SRRH
6233static int
6234allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
6235{
6236 enum ring_buffer_flags rb_flags;
737223fb
SRRH
6237
6238 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6239
dced341b
SRRH
6240 buf->tr = tr;
6241
55034cd6
SRRH
6242 buf->buffer = ring_buffer_alloc(size, rb_flags);
6243 if (!buf->buffer)
6244 return -ENOMEM;
737223fb 6245
55034cd6
SRRH
6246 buf->data = alloc_percpu(struct trace_array_cpu);
6247 if (!buf->data) {
6248 ring_buffer_free(buf->buffer);
6249 return -ENOMEM;
6250 }
737223fb 6251
737223fb
SRRH
6252 /* Allocate the first page for all buffers */
6253 set_buffer_entries(&tr->trace_buffer,
6254 ring_buffer_size(tr->trace_buffer.buffer, 0));
6255
55034cd6
SRRH
6256 return 0;
6257}
737223fb 6258
55034cd6
SRRH
6259static int allocate_trace_buffers(struct trace_array *tr, int size)
6260{
6261 int ret;
737223fb 6262
55034cd6
SRRH
6263 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6264 if (ret)
6265 return ret;
737223fb 6266
55034cd6
SRRH
6267#ifdef CONFIG_TRACER_MAX_TRACE
6268 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6269 allocate_snapshot ? size : 1);
6270 if (WARN_ON(ret)) {
737223fb 6271 ring_buffer_free(tr->trace_buffer.buffer);
55034cd6
SRRH
6272 free_percpu(tr->trace_buffer.data);
6273 return -ENOMEM;
6274 }
6275 tr->allocated_snapshot = allocate_snapshot;
737223fb 6276
55034cd6
SRRH
6277 /*
6278 * Only the top level trace array gets its snapshot allocated
6279 * from the kernel command line.
6280 */
6281 allocate_snapshot = false;
737223fb 6282#endif
55034cd6 6283 return 0;
737223fb
SRRH
6284}
6285
f0b70cc4
SRRH
6286static void free_trace_buffer(struct trace_buffer *buf)
6287{
6288 if (buf->buffer) {
6289 ring_buffer_free(buf->buffer);
6290 buf->buffer = NULL;
6291 free_percpu(buf->data);
6292 buf->data = NULL;
6293 }
6294}
6295
23aaa3c1
SRRH
6296static void free_trace_buffers(struct trace_array *tr)
6297{
6298 if (!tr)
6299 return;
6300
f0b70cc4 6301 free_trace_buffer(&tr->trace_buffer);
23aaa3c1
SRRH
6302
6303#ifdef CONFIG_TRACER_MAX_TRACE
f0b70cc4 6304 free_trace_buffer(&tr->max_buffer);
23aaa3c1
SRRH
6305#endif
6306}
6307
737223fb
SRRH
6308static int new_instance_create(const char *name)
6309{
277ba044
SR
6310 struct trace_array *tr;
6311 int ret;
277ba044
SR
6312
6313 mutex_lock(&trace_types_lock);
6314
6315 ret = -EEXIST;
6316 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6317 if (tr->name && strcmp(tr->name, name) == 0)
6318 goto out_unlock;
6319 }
6320
6321 ret = -ENOMEM;
6322 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6323 if (!tr)
6324 goto out_unlock;
6325
6326 tr->name = kstrdup(name, GFP_KERNEL);
6327 if (!tr->name)
6328 goto out_free_tr;
6329
ccfe9e42
AL
6330 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6331 goto out_free_tr;
6332
6333 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6334
277ba044
SR
6335 raw_spin_lock_init(&tr->start_lock);
6336
0b9b12c1
SRRH
6337 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6338
277ba044
SR
6339 tr->current_trace = &nop_trace;
6340
6341 INIT_LIST_HEAD(&tr->systems);
6342 INIT_LIST_HEAD(&tr->events);
6343
737223fb 6344 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
6345 goto out_free_tr;
6346
277ba044
SR
6347 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6348 if (!tr->dir)
6349 goto out_free_tr;
6350
6351 ret = event_trace_add_tracer(tr->dir, tr);
609e85a7
AL
6352 if (ret) {
6353 debugfs_remove_recursive(tr->dir);
277ba044 6354 goto out_free_tr;
609e85a7 6355 }
277ba044
SR
6356
6357 init_tracer_debugfs(tr, tr->dir);
6358
6359 list_add(&tr->list, &ftrace_trace_arrays);
6360
6361 mutex_unlock(&trace_types_lock);
6362
6363 return 0;
6364
6365 out_free_tr:
23aaa3c1 6366 free_trace_buffers(tr);
ccfe9e42 6367 free_cpumask_var(tr->tracing_cpumask);
277ba044
SR
6368 kfree(tr->name);
6369 kfree(tr);
6370
6371 out_unlock:
6372 mutex_unlock(&trace_types_lock);
6373
6374 return ret;
6375
6376}
6377
0c8916c3
SR
6378static int instance_delete(const char *name)
6379{
6380 struct trace_array *tr;
6381 int found = 0;
6382 int ret;
6383
6384 mutex_lock(&trace_types_lock);
6385
6386 ret = -ENODEV;
6387 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6388 if (tr->name && strcmp(tr->name, name) == 0) {
6389 found = 1;
6390 break;
6391 }
6392 }
6393 if (!found)
6394 goto out_unlock;
6395
a695cb58
SRRH
6396 ret = -EBUSY;
6397 if (tr->ref)
6398 goto out_unlock;
6399
0c8916c3
SR
6400 list_del(&tr->list);
6401
6b450d25 6402 tracing_set_nop(tr);
0c8916c3 6403 event_trace_del_tracer(tr);
591dffda 6404 ftrace_destroy_function_files(tr);
0c8916c3 6405 debugfs_remove_recursive(tr->dir);
a9fcaaac 6406 free_trace_buffers(tr);
0c8916c3
SR
6407
6408 kfree(tr->name);
6409 kfree(tr);
6410
6411 ret = 0;
6412
6413 out_unlock:
6414 mutex_unlock(&trace_types_lock);
6415
6416 return ret;
6417}
6418
277ba044
SR
6419static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6420{
6421 struct dentry *parent;
6422 int ret;
6423
6424 /* Paranoid: Make sure the parent is the "instances" directory */
6425 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6426 if (WARN_ON_ONCE(parent != trace_instance_dir))
6427 return -ENOENT;
6428
6429 /*
6430 * The inode mutex is locked, but debugfs_create_dir() will also
6431 * take the mutex. As the instances directory can not be destroyed
6432 * or changed in any other way, it is safe to unlock it, and
6433 * let the dentry try. If two users try to make the same dir at
6434 * the same time, then the new_instance_create() will determine the
6435 * winner.
6436 */
6437 mutex_unlock(&inode->i_mutex);
6438
6439 ret = new_instance_create(dentry->d_iname);
6440
6441 mutex_lock(&inode->i_mutex);
6442
6443 return ret;
6444}
6445
0c8916c3
SR
6446static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6447{
6448 struct dentry *parent;
6449 int ret;
6450
6451 /* Paranoid: Make sure the parent is the "instances" directory */
6452 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6453 if (WARN_ON_ONCE(parent != trace_instance_dir))
6454 return -ENOENT;
6455
6456 /* The caller did a dget() on dentry */
6457 mutex_unlock(&dentry->d_inode->i_mutex);
6458
6459 /*
6460 * The inode mutex is locked, but debugfs_create_dir() will also
6461 * take the mutex. As the instances directory can not be destroyed
6462 * or changed in any other way, it is safe to unlock it, and
6463 * let the dentry try. If two users try to make the same dir at
6464 * the same time, then the instance_delete() will determine the
6465 * winner.
6466 */
6467 mutex_unlock(&inode->i_mutex);
6468
6469 ret = instance_delete(dentry->d_iname);
6470
6471 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6472 mutex_lock(&dentry->d_inode->i_mutex);
6473
6474 return ret;
6475}
6476
277ba044
SR
6477static const struct inode_operations instance_dir_inode_operations = {
6478 .lookup = simple_lookup,
6479 .mkdir = instance_mkdir,
0c8916c3 6480 .rmdir = instance_rmdir,
277ba044
SR
6481};
6482
6483static __init void create_trace_instances(struct dentry *d_tracer)
6484{
6485 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6486 if (WARN_ON(!trace_instance_dir))
6487 return;
6488
6489 /* Hijack the dir inode operations, to allow mkdir */
6490 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6491}
6492
2b6080f2
SR
6493static void
6494init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6495{
121aaee7 6496 int cpu;
2b6080f2 6497
607e2ea1
SRRH
6498 trace_create_file("available_tracers", 0444, d_tracer,
6499 tr, &show_traces_fops);
6500
6501 trace_create_file("current_tracer", 0644, d_tracer,
6502 tr, &set_tracer_fops);
6503
ccfe9e42
AL
6504 trace_create_file("tracing_cpumask", 0644, d_tracer,
6505 tr, &tracing_cpumask_fops);
6506
2b6080f2
SR
6507 trace_create_file("trace_options", 0644, d_tracer,
6508 tr, &tracing_iter_fops);
6509
6510 trace_create_file("trace", 0644, d_tracer,
6484c71c 6511 tr, &tracing_fops);
2b6080f2
SR
6512
6513 trace_create_file("trace_pipe", 0444, d_tracer,
15544209 6514 tr, &tracing_pipe_fops);
2b6080f2
SR
6515
6516 trace_create_file("buffer_size_kb", 0644, d_tracer,
0bc392ee 6517 tr, &tracing_entries_fops);
2b6080f2
SR
6518
6519 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6520 tr, &tracing_total_entries_fops);
6521
238ae93d 6522 trace_create_file("free_buffer", 0200, d_tracer,
2b6080f2
SR
6523 tr, &tracing_free_buffer_fops);
6524
6525 trace_create_file("trace_marker", 0220, d_tracer,
6526 tr, &tracing_mark_fops);
6527
6528 trace_create_file("trace_clock", 0644, d_tracer, tr,
6529 &trace_clock_fops);
6530
6531 trace_create_file("tracing_on", 0644, d_tracer,
6484c71c 6532 tr, &rb_simple_fops);
ce9bae55 6533
6d9b3fa5
SRRH
6534#ifdef CONFIG_TRACER_MAX_TRACE
6535 trace_create_file("tracing_max_latency", 0644, d_tracer,
6536 &tr->max_latency, &tracing_max_lat_fops);
6537#endif
6538
591dffda
SRRH
6539 if (ftrace_create_function_files(tr, d_tracer))
6540 WARN(1, "Could not allocate function filter files");
6541
ce9bae55
SRRH
6542#ifdef CONFIG_TRACER_SNAPSHOT
6543 trace_create_file("snapshot", 0644, d_tracer,
6484c71c 6544 tr, &snapshot_fops);
ce9bae55 6545#endif
121aaee7
SRRH
6546
6547 for_each_tracing_cpu(cpu)
6548 tracing_init_debugfs_percpu(tr, cpu);
6549
2b6080f2
SR
6550}
6551
b5ad384e 6552static __init int tracer_init_debugfs(void)
bc0c38d1
SR
6553{
6554 struct dentry *d_tracer;
bc0c38d1 6555
7e53bd42
LJ
6556 trace_access_lock_init();
6557
bc0c38d1 6558 d_tracer = tracing_init_dentry();
ed6f1c99
NK
6559 if (!d_tracer)
6560 return 0;
bc0c38d1 6561
2b6080f2 6562 init_tracer_debugfs(&global_trace, d_tracer);
bc0c38d1 6563
5452af66 6564 trace_create_file("tracing_thresh", 0644, d_tracer,
6508fa76 6565 &global_trace, &tracing_thresh_fops);
a8259075 6566
339ae5d3 6567 trace_create_file("README", 0444, d_tracer,
5452af66
FW
6568 NULL, &tracing_readme_fops);
6569
69abe6a5
AP
6570 trace_create_file("saved_cmdlines", 0444, d_tracer,
6571 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 6572
939c7a4f
YY
6573 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6574 NULL, &tracing_saved_cmdlines_size_fops);
6575
bc0c38d1 6576#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
6577 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6578 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 6579#endif
b04cc6b1 6580
277ba044 6581 create_trace_instances(d_tracer);
5452af66 6582
2b6080f2 6583 create_trace_options_dir(&global_trace);
b04cc6b1 6584
b5ad384e 6585 return 0;
bc0c38d1
SR
6586}
6587
3f5a54e3
SR
6588static int trace_panic_handler(struct notifier_block *this,
6589 unsigned long event, void *unused)
6590{
944ac425 6591 if (ftrace_dump_on_oops)
cecbca96 6592 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6593 return NOTIFY_OK;
6594}
6595
6596static struct notifier_block trace_panic_notifier = {
6597 .notifier_call = trace_panic_handler,
6598 .next = NULL,
6599 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6600};
6601
6602static int trace_die_handler(struct notifier_block *self,
6603 unsigned long val,
6604 void *data)
6605{
6606 switch (val) {
6607 case DIE_OOPS:
944ac425 6608 if (ftrace_dump_on_oops)
cecbca96 6609 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6610 break;
6611 default:
6612 break;
6613 }
6614 return NOTIFY_OK;
6615}
6616
6617static struct notifier_block trace_die_notifier = {
6618 .notifier_call = trace_die_handler,
6619 .priority = 200
6620};
6621
6622/*
6623 * printk is set to max of 1024, we really don't need it that big.
6624 * Nothing should be printing 1000 characters anyway.
6625 */
6626#define TRACE_MAX_PRINT 1000
6627
6628/*
6629 * Define here KERN_TRACE so that we have one place to modify
6630 * it if we decide to change what log level the ftrace dump
6631 * should be at.
6632 */
428aee14 6633#define KERN_TRACE KERN_EMERG
3f5a54e3 6634
955b61e5 6635void
3f5a54e3
SR
6636trace_printk_seq(struct trace_seq *s)
6637{
6638 /* Probably should print a warning here. */
bd6df187
J
6639 if (s->len >= TRACE_MAX_PRINT)
6640 s->len = TRACE_MAX_PRINT;
3f5a54e3
SR
6641
6642 /* should be zero ended, but we are paranoid. */
6643 s->buffer[s->len] = 0;
6644
6645 printk(KERN_TRACE "%s", s->buffer);
6646
f9520750 6647 trace_seq_init(s);
3f5a54e3
SR
6648}
6649
955b61e5
JW
6650void trace_init_global_iter(struct trace_iterator *iter)
6651{
6652 iter->tr = &global_trace;
2b6080f2 6653 iter->trace = iter->tr->current_trace;
ae3b5093 6654 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 6655 iter->trace_buffer = &global_trace.trace_buffer;
b2f974d6
CS
6656
6657 if (iter->trace && iter->trace->open)
6658 iter->trace->open(iter);
6659
6660 /* Annotate start of buffers if we had overruns */
6661 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6662 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6663
6664 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6665 if (trace_clocks[iter->tr->clock_id].in_ns)
6666 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
955b61e5
JW
6667}
6668
7fe70b57 6669void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 6670{
3f5a54e3
SR
6671 /* use static because iter can be a bit big for the stack */
6672 static struct trace_iterator iter;
7fe70b57 6673 static atomic_t dump_running;
cf586b61 6674 unsigned int old_userobj;
d769041f
SR
6675 unsigned long flags;
6676 int cnt = 0, cpu;
3f5a54e3 6677
7fe70b57
SRRH
6678 /* Only allow one dump user at a time. */
6679 if (atomic_inc_return(&dump_running) != 1) {
6680 atomic_dec(&dump_running);
6681 return;
6682 }
3f5a54e3 6683
7fe70b57
SRRH
6684 /*
6685 * Always turn off tracing when we dump.
6686 * We don't need to show trace output of what happens
6687 * between multiple crashes.
6688 *
6689 * If the user does a sysrq-z, then they can re-enable
6690 * tracing with echo 1 > tracing_on.
6691 */
0ee6b6cf 6692 tracing_off();
cf586b61 6693
7fe70b57 6694 local_irq_save(flags);
3f5a54e3 6695
38dbe0b1 6696 /* Simulate the iterator */
955b61e5
JW
6697 trace_init_global_iter(&iter);
6698
d769041f 6699 for_each_tracing_cpu(cpu) {
12883efb 6700 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
d769041f
SR
6701 }
6702
cf586b61
FW
6703 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6704
b54d3de9
TE
6705 /* don't look at user memory in panic mode */
6706 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6707
cecbca96
FW
6708 switch (oops_dump_mode) {
6709 case DUMP_ALL:
ae3b5093 6710 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6711 break;
6712 case DUMP_ORIG:
6713 iter.cpu_file = raw_smp_processor_id();
6714 break;
6715 case DUMP_NONE:
6716 goto out_enable;
6717 default:
6718 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 6719 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6720 }
6721
6722 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 6723
7fe70b57
SRRH
6724 /* Did function tracer already get disabled? */
6725 if (ftrace_is_dead()) {
6726 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6727 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6728 }
6729
3f5a54e3
SR
6730 /*
6731 * We need to stop all tracing on all CPUS to read the
6732 * the next buffer. This is a bit expensive, but is
6733 * not done often. We fill all what we can read,
6734 * and then release the locks again.
6735 */
6736
3f5a54e3
SR
6737 while (!trace_empty(&iter)) {
6738
6739 if (!cnt)
6740 printk(KERN_TRACE "---------------------------------\n");
6741
6742 cnt++;
6743
6744 /* reset all but tr, trace, and overruns */
6745 memset(&iter.seq, 0,
6746 sizeof(struct trace_iterator) -
6747 offsetof(struct trace_iterator, seq));
6748 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6749 iter.pos = -1;
6750
955b61e5 6751 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
6752 int ret;
6753
6754 ret = print_trace_line(&iter);
6755 if (ret != TRACE_TYPE_NO_CONSUME)
6756 trace_consume(&iter);
3f5a54e3 6757 }
b892e5c8 6758 touch_nmi_watchdog();
3f5a54e3
SR
6759
6760 trace_printk_seq(&iter.seq);
6761 }
6762
6763 if (!cnt)
6764 printk(KERN_TRACE " (ftrace buffer empty)\n");
6765 else
6766 printk(KERN_TRACE "---------------------------------\n");
6767
cecbca96 6768 out_enable:
7fe70b57 6769 trace_flags |= old_userobj;
cf586b61 6770
7fe70b57
SRRH
6771 for_each_tracing_cpu(cpu) {
6772 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 6773 }
7fe70b57 6774 atomic_dec(&dump_running);
cd891ae0 6775 local_irq_restore(flags);
3f5a54e3 6776}
a8eecf22 6777EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 6778
3928a8a2 6779__init static int tracer_alloc_buffers(void)
bc0c38d1 6780{
73c5162a 6781 int ring_buf_size;
9e01c1b7 6782 int ret = -ENOMEM;
4c11d7ae 6783
750912fa 6784
9e01c1b7
RR
6785 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6786 goto out;
6787
ccfe9e42 6788 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9e01c1b7 6789 goto out_free_buffer_mask;
4c11d7ae 6790
07d777fe
SR
6791 /* Only allocate trace_printk buffers if a trace_printk exists */
6792 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 6793 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
6794 trace_printk_init_buffers();
6795
73c5162a
SR
6796 /* To save memory, keep the ring buffer size to its minimum */
6797 if (ring_buffer_expanded)
6798 ring_buf_size = trace_buf_size;
6799 else
6800 ring_buf_size = 1;
6801
9e01c1b7 6802 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
ccfe9e42 6803 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9e01c1b7 6804
2b6080f2
SR
6805 raw_spin_lock_init(&global_trace.start_lock);
6806
2c4a33ab
SRRH
6807 /* Used for event triggers */
6808 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6809 if (!temp_buffer)
6810 goto out_free_cpumask;
6811
939c7a4f
YY
6812 if (trace_create_savedcmd() < 0)
6813 goto out_free_temp_buffer;
6814
9e01c1b7 6815 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 6816 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
6817 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6818 WARN_ON(1);
939c7a4f 6819 goto out_free_savedcmd;
4c11d7ae 6820 }
a7603ff4 6821
499e5470
SR
6822 if (global_trace.buffer_disabled)
6823 tracing_off();
4c11d7ae 6824
e1e232ca
SR
6825 if (trace_boot_clock) {
6826 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6827 if (ret < 0)
6828 pr_warning("Trace clock %s not defined, going back to default\n",
6829 trace_boot_clock);
6830 }
6831
ca164318
SRRH
6832 /*
6833 * register_tracer() might reference current_trace, so it
6834 * needs to be set before we register anything. This is
6835 * just a bootstrap of current_trace anyway.
6836 */
2b6080f2
SR
6837 global_trace.current_trace = &nop_trace;
6838
0b9b12c1
SRRH
6839 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6840
4104d326
SRRH
6841 ftrace_init_global_array_ops(&global_trace);
6842
ca164318
SRRH
6843 register_tracer(&nop_trace);
6844
60a11774
SR
6845 /* All seems OK, enable tracing */
6846 tracing_disabled = 0;
3928a8a2 6847
3f5a54e3
SR
6848 atomic_notifier_chain_register(&panic_notifier_list,
6849 &trace_panic_notifier);
6850
6851 register_die_notifier(&trace_die_notifier);
2fc1dfbe 6852
ae63b31e
SR
6853 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6854
6855 INIT_LIST_HEAD(&global_trace.systems);
6856 INIT_LIST_HEAD(&global_trace.events);
6857 list_add(&global_trace.list, &ftrace_trace_arrays);
6858
7bcfaf54
SR
6859 while (trace_boot_options) {
6860 char *option;
6861
6862 option = strsep(&trace_boot_options, ",");
2b6080f2 6863 trace_set_options(&global_trace, option);
7bcfaf54
SR
6864 }
6865
77fd5c15
SRRH
6866 register_snapshot_cmd();
6867
2fc1dfbe 6868 return 0;
3f5a54e3 6869
939c7a4f
YY
6870out_free_savedcmd:
6871 free_saved_cmdlines_buffer(savedcmd);
2c4a33ab
SRRH
6872out_free_temp_buffer:
6873 ring_buffer_free(temp_buffer);
9e01c1b7 6874out_free_cpumask:
ccfe9e42 6875 free_cpumask_var(global_trace.tracing_cpumask);
9e01c1b7
RR
6876out_free_buffer_mask:
6877 free_cpumask_var(tracing_buffer_mask);
6878out:
6879 return ret;
bc0c38d1 6880}
b2821ae6
SR
6881
6882__init static int clear_boot_tracer(void)
6883{
6884 /*
6885 * The default tracer at boot buffer is an init section.
6886 * This function is called in lateinit. If we did not
6887 * find the boot tracer, then clear it out, to prevent
6888 * later registration from accessing the buffer that is
6889 * about to be freed.
6890 */
6891 if (!default_bootup_tracer)
6892 return 0;
6893
6894 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6895 default_bootup_tracer);
6896 default_bootup_tracer = NULL;
6897
6898 return 0;
6899}
6900
b5ad384e
FW
6901early_initcall(tracer_alloc_buffers);
6902fs_initcall(tracer_init_debugfs);
b2821ae6 6903late_initcall(clear_boot_tracer);