]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
Merge tag 'localmodconfig-v3.10' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
4c11d7ae 23#include <linux/pagemap.h>
bc0c38d1
SR
24#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
2cadf913 27#include <linux/kprobes.h>
bc0c38d1
SR
28#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
2cadf913 31#include <linux/splice.h>
3f5a54e3 32#include <linux/kdebug.h>
5f0c6c03 33#include <linux/string.h>
7e53bd42 34#include <linux/rwsem.h>
5a0e3ad6 35#include <linux/slab.h>
bc0c38d1
SR
36#include <linux/ctype.h>
37#include <linux/init.h>
2a2cc8f7 38#include <linux/poll.h>
b892e5c8 39#include <linux/nmi.h>
bc0c38d1 40#include <linux/fs.h>
8bd75c77 41#include <linux/sched/rt.h>
86387f7e 42
bc0c38d1 43#include "trace.h"
f0868d1e 44#include "trace_output.h"
bc0c38d1 45
73c5162a
SR
46/*
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
55034cd6 50bool ring_buffer_expanded;
73c5162a 51
8e1b82e0
FW
52/*
53 * We need to change this state when a selftest is running.
ff32504f
FW
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
5e1607a0 56 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
57 * at the same time, giving false positive or negative results.
58 */
8e1b82e0 59static bool __read_mostly tracing_selftest_running;
ff32504f 60
b2821ae6
SR
61/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
020e5f85 64bool __read_mostly tracing_selftest_disabled;
b2821ae6 65
adf9f195
FW
66/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
0f048701 80
7ffbd48d
SR
81/*
82 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
84 * occurred.
85 */
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
0f048701
SR
88/*
89 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
92 * this back to zero.
93 */
4fd27358 94static int tracing_disabled = 1;
0f048701 95
9288f99a 96DEFINE_PER_CPU(int, ftrace_cpu_disabled);
d769041f 97
955b61e5 98cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 99
944ac425
SR
100/*
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102 *
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
107 * serial console.
108 *
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 114 */
cecbca96
FW
115
116enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 117
b2821ae6
SR
118static int tracing_set_tracer(const char *buf);
119
ee6c2c1b
LZ
120#define MAX_TRACER_SIZE 100
121static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 122static char *default_bootup_tracer;
d9e54076 123
55034cd6
SRRH
124static bool allocate_snapshot;
125
1beee96b 126static int __init set_cmdline_ftrace(char *str)
d9e54076 127{
67012ab1 128 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 129 default_bootup_tracer = bootup_tracer_buf;
73c5162a 130 /* We are using ftrace early, expand it */
55034cd6 131 ring_buffer_expanded = true;
d9e54076
PZ
132 return 1;
133}
1beee96b 134__setup("ftrace=", set_cmdline_ftrace);
d9e54076 135
944ac425
SR
136static int __init set_ftrace_dump_on_oops(char *str)
137{
cecbca96
FW
138 if (*str++ != '=' || !*str) {
139 ftrace_dump_on_oops = DUMP_ALL;
140 return 1;
141 }
142
143 if (!strcmp("orig_cpu", str)) {
144 ftrace_dump_on_oops = DUMP_ORIG;
145 return 1;
146 }
147
148 return 0;
944ac425
SR
149}
150__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 151
3209cff4 152static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
153{
154 allocate_snapshot = true;
155 /* We also need the main ring buffer expanded */
156 ring_buffer_expanded = true;
157 return 1;
158}
3209cff4 159__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 160
7bcfaf54
SR
161
162static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163static char *trace_boot_options __initdata;
164
165static int __init set_trace_boot_options(char *str)
166{
67012ab1 167 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
168 trace_boot_options = trace_boot_options_buf;
169 return 0;
170}
171__setup("trace_options=", set_trace_boot_options);
172
cf8e3474 173unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
174{
175 nsec += 500;
176 do_div(nsec, 1000);
177 return nsec;
178}
179
4fcdae83
SR
180/*
181 * The global_trace is the descriptor that holds the tracing
182 * buffers for the live tracing. For each CPU, it contains
183 * a link list of pages that will store trace entries. The
184 * page descriptor of the pages in the memory is used to hold
185 * the link list by linking the lru item in the page descriptor
186 * to each of the pages in the buffer per CPU.
187 *
188 * For each active CPU there is a data field that holds the
189 * pages for the buffer for that CPU. Each CPU has the same number
190 * of pages allocated for its buffer.
191 */
bc0c38d1
SR
192static struct trace_array global_trace;
193
ae63b31e 194LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 195
e77405ad
SR
196int filter_current_check_discard(struct ring_buffer *buffer,
197 struct ftrace_event_call *call, void *rec,
eb02ce01
TZ
198 struct ring_buffer_event *event)
199{
e77405ad 200 return filter_check_discard(call, rec, buffer, event);
eb02ce01 201}
17c873ec 202EXPORT_SYMBOL_GPL(filter_current_check_discard);
eb02ce01 203
37886f6a
SR
204cycle_t ftrace_now(int cpu)
205{
206 u64 ts;
207
208 /* Early boot up does not have a buffer yet */
12883efb 209 if (!global_trace.trace_buffer.buffer)
37886f6a
SR
210 return trace_clock_local();
211
12883efb
SRRH
212 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
37886f6a
SR
214
215 return ts;
216}
bc0c38d1 217
9036990d
SR
218int tracing_is_enabled(void)
219{
0fb9656d 220 return tracing_is_on();
9036990d
SR
221}
222
4fcdae83 223/*
3928a8a2
SR
224 * trace_buf_size is the size in bytes that is allocated
225 * for a buffer. Note, the number of bytes is always rounded
226 * to page size.
3f5a54e3
SR
227 *
228 * This number is purposely set to a low number of 16384.
229 * If the dump on oops happens, it will be much appreciated
230 * to not have to wait for all that output. Anyway this can be
231 * boot time and run time configurable.
4fcdae83 232 */
3928a8a2 233#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 234
3928a8a2 235static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 236
4fcdae83 237/* trace_types holds a link list of available tracers. */
bc0c38d1 238static struct tracer *trace_types __read_mostly;
4fcdae83 239
4fcdae83
SR
240/*
241 * trace_types_lock is used to protect the trace_types list.
4fcdae83 242 */
bc0c38d1 243static DEFINE_MUTEX(trace_types_lock);
4fcdae83 244
7e53bd42
LJ
245/*
246 * serialize the access of the ring buffer
247 *
248 * ring buffer serializes readers, but it is low level protection.
249 * The validity of the events (which returns by ring_buffer_peek() ..etc)
250 * are not protected by ring buffer.
251 *
252 * The content of events may become garbage if we allow other process consumes
253 * these events concurrently:
254 * A) the page of the consumed events may become a normal page
255 * (not reader page) in ring buffer, and this page will be rewrited
256 * by events producer.
257 * B) The page of the consumed events may become a page for splice_read,
258 * and this page will be returned to system.
259 *
260 * These primitives allow multi process access to different cpu ring buffer
261 * concurrently.
262 *
263 * These primitives don't distinguish read-only and read-consume access.
264 * Multi read-only access are also serialized.
265 */
266
267#ifdef CONFIG_SMP
268static DECLARE_RWSEM(all_cpu_access_lock);
269static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271static inline void trace_access_lock(int cpu)
272{
ae3b5093 273 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
274 /* gain it for accessing the whole ring buffer. */
275 down_write(&all_cpu_access_lock);
276 } else {
277 /* gain it for accessing a cpu ring buffer. */
278
ae3b5093 279 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
280 down_read(&all_cpu_access_lock);
281
282 /* Secondly block other access to this @cpu ring buffer. */
283 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284 }
285}
286
287static inline void trace_access_unlock(int cpu)
288{
ae3b5093 289 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
290 up_write(&all_cpu_access_lock);
291 } else {
292 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293 up_read(&all_cpu_access_lock);
294 }
295}
296
297static inline void trace_access_lock_init(void)
298{
299 int cpu;
300
301 for_each_possible_cpu(cpu)
302 mutex_init(&per_cpu(cpu_access_lock, cpu));
303}
304
305#else
306
307static DEFINE_MUTEX(access_lock);
308
309static inline void trace_access_lock(int cpu)
310{
311 (void)cpu;
312 mutex_lock(&access_lock);
313}
314
315static inline void trace_access_unlock(int cpu)
316{
317 (void)cpu;
318 mutex_unlock(&access_lock);
319}
320
321static inline void trace_access_lock_init(void)
322{
323}
324
325#endif
326
ee6bce52 327/* trace_flags holds trace_options default values */
12ef7d44 328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
a2a16d6a 329 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
77271ce4 330 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
328df475 331 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
e7e2ee89 332
499e5470
SR
333/**
334 * tracing_on - enable tracing buffers
335 *
336 * This function enables tracing buffers that may have been
337 * disabled with tracing_off.
338 */
339void tracing_on(void)
340{
12883efb
SRRH
341 if (global_trace.trace_buffer.buffer)
342 ring_buffer_record_on(global_trace.trace_buffer.buffer);
499e5470
SR
343 /*
344 * This flag is only looked at when buffers haven't been
345 * allocated yet. We don't really care about the race
346 * between setting this flag and actually turning
347 * on the buffer.
348 */
349 global_trace.buffer_disabled = 0;
350}
351EXPORT_SYMBOL_GPL(tracing_on);
352
09ae7234
SRRH
353/**
354 * __trace_puts - write a constant string into the trace buffer.
355 * @ip: The address of the caller
356 * @str: The constant string to write
357 * @size: The size of the string.
358 */
359int __trace_puts(unsigned long ip, const char *str, int size)
360{
361 struct ring_buffer_event *event;
362 struct ring_buffer *buffer;
363 struct print_entry *entry;
364 unsigned long irq_flags;
365 int alloc;
366
367 alloc = sizeof(*entry) + size + 2; /* possible \n added */
368
369 local_save_flags(irq_flags);
370 buffer = global_trace.trace_buffer.buffer;
371 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
372 irq_flags, preempt_count());
373 if (!event)
374 return 0;
375
376 entry = ring_buffer_event_data(event);
377 entry->ip = ip;
378
379 memcpy(&entry->buf, str, size);
380
381 /* Add a newline if necessary */
382 if (entry->buf[size - 1] != '\n') {
383 entry->buf[size] = '\n';
384 entry->buf[size + 1] = '\0';
385 } else
386 entry->buf[size] = '\0';
387
388 __buffer_unlock_commit(buffer, event);
389
390 return size;
391}
392EXPORT_SYMBOL_GPL(__trace_puts);
393
394/**
395 * __trace_bputs - write the pointer to a constant string into trace buffer
396 * @ip: The address of the caller
397 * @str: The constant string to write to the buffer to
398 */
399int __trace_bputs(unsigned long ip, const char *str)
400{
401 struct ring_buffer_event *event;
402 struct ring_buffer *buffer;
403 struct bputs_entry *entry;
404 unsigned long irq_flags;
405 int size = sizeof(struct bputs_entry);
406
407 local_save_flags(irq_flags);
408 buffer = global_trace.trace_buffer.buffer;
409 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410 irq_flags, preempt_count());
411 if (!event)
412 return 0;
413
414 entry = ring_buffer_event_data(event);
415 entry->ip = ip;
416 entry->str = str;
417
418 __buffer_unlock_commit(buffer, event);
419
420 return 1;
421}
422EXPORT_SYMBOL_GPL(__trace_bputs);
423
ad909e21
SRRH
424#ifdef CONFIG_TRACER_SNAPSHOT
425/**
426 * trace_snapshot - take a snapshot of the current buffer.
427 *
428 * This causes a swap between the snapshot buffer and the current live
429 * tracing buffer. You can use this to take snapshots of the live
430 * trace when some condition is triggered, but continue to trace.
431 *
432 * Note, make sure to allocate the snapshot with either
433 * a tracing_snapshot_alloc(), or by doing it manually
434 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
435 *
436 * If the snapshot buffer is not allocated, it will stop tracing.
437 * Basically making a permanent snapshot.
438 */
439void tracing_snapshot(void)
440{
441 struct trace_array *tr = &global_trace;
442 struct tracer *tracer = tr->current_trace;
443 unsigned long flags;
444
1b22e382
SRRH
445 if (in_nmi()) {
446 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447 internal_trace_puts("*** snapshot is being ignored ***\n");
448 return;
449 }
450
ad909e21 451 if (!tr->allocated_snapshot) {
ca268da6
SRRH
452 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
454 tracing_off();
455 return;
456 }
457
458 /* Note, snapshot can not be used when the tracer uses it */
459 if (tracer->use_max_tr) {
ca268da6
SRRH
460 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
462 return;
463 }
464
465 local_irq_save(flags);
466 update_max_tr(tr, current, smp_processor_id());
467 local_irq_restore(flags);
468}
1b22e382 469EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
470
471static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
473static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
474
475static int alloc_snapshot(struct trace_array *tr)
476{
477 int ret;
478
479 if (!tr->allocated_snapshot) {
480
481 /* allocate spare buffer */
482 ret = resize_buffer_duplicate_size(&tr->max_buffer,
483 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
484 if (ret < 0)
485 return ret;
486
487 tr->allocated_snapshot = true;
488 }
489
490 return 0;
491}
492
493void free_snapshot(struct trace_array *tr)
494{
495 /*
496 * We don't free the ring buffer. instead, resize it because
497 * The max_tr ring buffer has some state (e.g. ring->clock) and
498 * we want preserve it.
499 */
500 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
501 set_buffer_entries(&tr->max_buffer, 1);
502 tracing_reset_online_cpus(&tr->max_buffer);
503 tr->allocated_snapshot = false;
504}
ad909e21
SRRH
505
506/**
507 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
508 *
509 * This is similar to trace_snapshot(), but it will allocate the
510 * snapshot buffer if it isn't already allocated. Use this only
511 * where it is safe to sleep, as the allocation may sleep.
512 *
513 * This causes a swap between the snapshot buffer and the current live
514 * tracing buffer. You can use this to take snapshots of the live
515 * trace when some condition is triggered, but continue to trace.
516 */
517void tracing_snapshot_alloc(void)
518{
519 struct trace_array *tr = &global_trace;
520 int ret;
521
3209cff4
SRRH
522 ret = alloc_snapshot(tr);
523 if (WARN_ON(ret < 0))
524 return;
ad909e21
SRRH
525
526 tracing_snapshot();
527}
1b22e382 528EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
529#else
530void tracing_snapshot(void)
531{
532 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
533}
1b22e382 534EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
535void tracing_snapshot_alloc(void)
536{
537 /* Give warning */
538 tracing_snapshot();
539}
1b22e382 540EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
541#endif /* CONFIG_TRACER_SNAPSHOT */
542
499e5470
SR
543/**
544 * tracing_off - turn off tracing buffers
545 *
546 * This function stops the tracing buffers from recording data.
547 * It does not disable any overhead the tracers themselves may
548 * be causing. This function simply causes all recording to
549 * the ring buffers to fail.
550 */
551void tracing_off(void)
552{
12883efb
SRRH
553 if (global_trace.trace_buffer.buffer)
554 ring_buffer_record_off(global_trace.trace_buffer.buffer);
499e5470
SR
555 /*
556 * This flag is only looked at when buffers haven't been
557 * allocated yet. We don't really care about the race
558 * between setting this flag and actually turning
559 * on the buffer.
560 */
561 global_trace.buffer_disabled = 1;
562}
563EXPORT_SYMBOL_GPL(tracing_off);
564
565/**
566 * tracing_is_on - show state of ring buffers enabled
567 */
568int tracing_is_on(void)
569{
12883efb
SRRH
570 if (global_trace.trace_buffer.buffer)
571 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
499e5470
SR
572 return !global_trace.buffer_disabled;
573}
574EXPORT_SYMBOL_GPL(tracing_is_on);
575
3928a8a2 576static int __init set_buf_size(char *str)
bc0c38d1 577{
3928a8a2 578 unsigned long buf_size;
c6caeeb1 579
bc0c38d1
SR
580 if (!str)
581 return 0;
9d612bef 582 buf_size = memparse(str, &str);
c6caeeb1 583 /* nr_entries can not be zero */
9d612bef 584 if (buf_size == 0)
c6caeeb1 585 return 0;
3928a8a2 586 trace_buf_size = buf_size;
bc0c38d1
SR
587 return 1;
588}
3928a8a2 589__setup("trace_buf_size=", set_buf_size);
bc0c38d1 590
0e950173
TB
591static int __init set_tracing_thresh(char *str)
592{
87abb3b1 593 unsigned long threshold;
0e950173
TB
594 int ret;
595
596 if (!str)
597 return 0;
bcd83ea6 598 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
599 if (ret < 0)
600 return 0;
87abb3b1 601 tracing_thresh = threshold * 1000;
0e950173
TB
602 return 1;
603}
604__setup("tracing_thresh=", set_tracing_thresh);
605
57f50be1
SR
606unsigned long nsecs_to_usecs(unsigned long nsecs)
607{
608 return nsecs / 1000;
609}
610
4fcdae83 611/* These must match the bit postions in trace_iterator_flags */
bc0c38d1
SR
612static const char *trace_options[] = {
613 "print-parent",
614 "sym-offset",
615 "sym-addr",
616 "verbose",
f9896bf3 617 "raw",
5e3ca0ec 618 "hex",
cb0f12aa 619 "bin",
2a2cc8f7 620 "block",
86387f7e 621 "stacktrace",
5e1607a0 622 "trace_printk",
b2a866f9 623 "ftrace_preempt",
9f029e83 624 "branch",
12ef7d44 625 "annotate",
02b67518 626 "userstacktrace",
b54d3de9 627 "sym-userobj",
66896a85 628 "printk-msg-only",
c4a8e8be 629 "context-info",
c032ef64 630 "latency-format",
be6f164a 631 "sleep-time",
a2a16d6a 632 "graph-time",
e870e9a1 633 "record-cmd",
750912fa 634 "overwrite",
cf30cf67 635 "disable_on_free",
77271ce4 636 "irq-info",
5224c3a3 637 "markers",
328df475 638 "function-trace",
bc0c38d1
SR
639 NULL
640};
641
5079f326
Z
642static struct {
643 u64 (*func)(void);
644 const char *name;
8be0709f 645 int in_ns; /* is this clock in nanoseconds? */
5079f326 646} trace_clocks[] = {
8be0709f
DS
647 { trace_clock_local, "local", 1 },
648 { trace_clock_global, "global", 1 },
649 { trace_clock_counter, "counter", 0 },
8aacf017 650 { trace_clock_jiffies, "uptime", 1 },
76f11917 651 { trace_clock, "perf", 1 },
8cbd9cc6 652 ARCH_TRACE_CLOCKS
5079f326
Z
653};
654
655int trace_clock_id;
656
b63f39ea 657/*
658 * trace_parser_get_init - gets the buffer for trace parser
659 */
660int trace_parser_get_init(struct trace_parser *parser, int size)
661{
662 memset(parser, 0, sizeof(*parser));
663
664 parser->buffer = kmalloc(size, GFP_KERNEL);
665 if (!parser->buffer)
666 return 1;
667
668 parser->size = size;
669 return 0;
670}
671
672/*
673 * trace_parser_put - frees the buffer for trace parser
674 */
675void trace_parser_put(struct trace_parser *parser)
676{
677 kfree(parser->buffer);
678}
679
680/*
681 * trace_get_user - reads the user input string separated by space
682 * (matched by isspace(ch))
683 *
684 * For each string found the 'struct trace_parser' is updated,
685 * and the function returns.
686 *
687 * Returns number of bytes read.
688 *
689 * See kernel/trace/trace.h for 'struct trace_parser' details.
690 */
691int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
692 size_t cnt, loff_t *ppos)
693{
694 char ch;
695 size_t read = 0;
696 ssize_t ret;
697
698 if (!*ppos)
699 trace_parser_clear(parser);
700
701 ret = get_user(ch, ubuf++);
702 if (ret)
703 goto out;
704
705 read++;
706 cnt--;
707
708 /*
709 * The parser is not finished with the last write,
710 * continue reading the user input without skipping spaces.
711 */
712 if (!parser->cont) {
713 /* skip white space */
714 while (cnt && isspace(ch)) {
715 ret = get_user(ch, ubuf++);
716 if (ret)
717 goto out;
718 read++;
719 cnt--;
720 }
721
722 /* only spaces were written */
723 if (isspace(ch)) {
724 *ppos += read;
725 ret = read;
726 goto out;
727 }
728
729 parser->idx = 0;
730 }
731
732 /* read the non-space input */
733 while (cnt && !isspace(ch)) {
3c235a33 734 if (parser->idx < parser->size - 1)
b63f39ea 735 parser->buffer[parser->idx++] = ch;
736 else {
737 ret = -EINVAL;
738 goto out;
739 }
740 ret = get_user(ch, ubuf++);
741 if (ret)
742 goto out;
743 read++;
744 cnt--;
745 }
746
747 /* We either got finished input or we have to wait for another call. */
748 if (isspace(ch)) {
749 parser->buffer[parser->idx] = 0;
750 parser->cont = false;
751 } else {
752 parser->cont = true;
753 parser->buffer[parser->idx++] = ch;
754 }
755
756 *ppos += read;
757 ret = read;
758
759out:
760 return ret;
761}
762
6c6c2796
PP
763ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
764{
765 int len;
766 int ret;
767
2dc5d12b
SR
768 if (!cnt)
769 return 0;
770
6c6c2796
PP
771 if (s->len <= s->readpos)
772 return -EBUSY;
773
774 len = s->len - s->readpos;
775 if (cnt > len)
776 cnt = len;
777 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
2dc5d12b 778 if (ret == cnt)
6c6c2796
PP
779 return -EFAULT;
780
2dc5d12b
SR
781 cnt -= ret;
782
e74da523 783 s->readpos += cnt;
6c6c2796 784 return cnt;
214023c3
SR
785}
786
b8b94265 787static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
788{
789 int len;
3c56819b
EGM
790
791 if (s->len <= s->readpos)
792 return -EBUSY;
793
794 len = s->len - s->readpos;
795 if (cnt > len)
796 cnt = len;
5a26c8f0 797 memcpy(buf, s->buffer + s->readpos, cnt);
3c56819b 798
e74da523 799 s->readpos += cnt;
3c56819b
EGM
800 return cnt;
801}
802
5d4a9dba
SR
803/*
804 * ftrace_max_lock is used to protect the swapping of buffers
805 * when taking a max snapshot. The buffers themselves are
806 * protected by per_cpu spinlocks. But the action of the swap
807 * needs its own lock.
808 *
445c8951 809 * This is defined as a arch_spinlock_t in order to help
5d4a9dba
SR
810 * with performance when lockdep debugging is enabled.
811 *
812 * It is also used in other places outside the update_max_tr
813 * so it needs to be defined outside of the
814 * CONFIG_TRACER_MAX_TRACE.
815 */
445c8951 816static arch_spinlock_t ftrace_max_lock =
edc35bd7 817 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5d4a9dba 818
0e950173
TB
819unsigned long __read_mostly tracing_thresh;
820
5d4a9dba
SR
821#ifdef CONFIG_TRACER_MAX_TRACE
822unsigned long __read_mostly tracing_max_latency;
5d4a9dba
SR
823
824/*
825 * Copy the new maximum trace into the separate maximum-trace
826 * structure. (this way the maximum trace is permanently saved,
827 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
828 */
829static void
830__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
831{
12883efb
SRRH
832 struct trace_buffer *trace_buf = &tr->trace_buffer;
833 struct trace_buffer *max_buf = &tr->max_buffer;
834 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
835 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 836
12883efb
SRRH
837 max_buf->cpu = cpu;
838 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 839
8248ac05
SR
840 max_data->saved_latency = tracing_max_latency;
841 max_data->critical_start = data->critical_start;
842 max_data->critical_end = data->critical_end;
5d4a9dba 843
1acaa1b2 844 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05
SR
845 max_data->pid = tsk->pid;
846 max_data->uid = task_uid(tsk);
847 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
848 max_data->policy = tsk->policy;
849 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
850
851 /* record this tasks comm */
852 tracing_record_cmdline(tsk);
853}
854
4fcdae83
SR
855/**
856 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
857 * @tr: tracer
858 * @tsk: the task with the latency
859 * @cpu: The cpu that initiated the trace.
860 *
861 * Flip the buffers between the @tr and the max_tr and record information
862 * about which task was the cause of this latency.
863 */
e309b41d 864void
bc0c38d1
SR
865update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
866{
2721e72d 867 struct ring_buffer *buf;
bc0c38d1 868
2b6080f2 869 if (tr->stop_count)
b8de7bd1
SR
870 return;
871
4c11d7ae 872 WARN_ON_ONCE(!irqs_disabled());
34600f0e 873
45ad21ca 874 if (!tr->allocated_snapshot) {
debdd57f 875 /* Only the nop tracer should hit this when disabling */
2b6080f2 876 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 877 return;
debdd57f 878 }
34600f0e 879
0199c4e6 880 arch_spin_lock(&ftrace_max_lock);
3928a8a2 881
12883efb
SRRH
882 buf = tr->trace_buffer.buffer;
883 tr->trace_buffer.buffer = tr->max_buffer.buffer;
884 tr->max_buffer.buffer = buf;
3928a8a2 885
bc0c38d1 886 __update_max_tr(tr, tsk, cpu);
0199c4e6 887 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1
SR
888}
889
890/**
891 * update_max_tr_single - only copy one trace over, and reset the rest
892 * @tr - tracer
893 * @tsk - task with the latency
894 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
895 *
896 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 897 */
e309b41d 898void
bc0c38d1
SR
899update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
900{
3928a8a2 901 int ret;
bc0c38d1 902
2b6080f2 903 if (tr->stop_count)
b8de7bd1
SR
904 return;
905
4c11d7ae 906 WARN_ON_ONCE(!irqs_disabled());
9e8529af 907 if (tr->allocated_snapshot) {
2930e04d 908 /* Only the nop tracer should hit this when disabling */
9e8529af 909 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 910 return;
2930e04d 911 }
ef710e10 912
0199c4e6 913 arch_spin_lock(&ftrace_max_lock);
bc0c38d1 914
12883efb 915 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 916
e8165dbb
SR
917 if (ret == -EBUSY) {
918 /*
919 * We failed to swap the buffer due to a commit taking
920 * place on this CPU. We fail to record, but we reset
921 * the max trace buffer (no one writes directly to it)
922 * and flag that it failed.
923 */
12883efb 924 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
925 "Failed to swap buffers due to commit in progress\n");
926 }
927
e8165dbb 928 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
929
930 __update_max_tr(tr, tsk, cpu);
0199c4e6 931 arch_spin_unlock(&ftrace_max_lock);
bc0c38d1 932}
5d4a9dba 933#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 934
0d5c6e1c
SR
935static void default_wait_pipe(struct trace_iterator *iter)
936{
15693458
SRRH
937 /* Iterators are static, they should be filled or empty */
938 if (trace_buffer_iter(iter, iter->cpu_file))
939 return;
0d5c6e1c 940
12883efb 941 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
0d5c6e1c
SR
942}
943
f4e781c0
SRRH
944#ifdef CONFIG_FTRACE_STARTUP_TEST
945static int run_tracer_selftest(struct tracer *type)
946{
947 struct trace_array *tr = &global_trace;
948 struct tracer *saved_tracer = tr->current_trace;
949 int ret;
0d5c6e1c 950
f4e781c0
SRRH
951 if (!type->selftest || tracing_selftest_disabled)
952 return 0;
0d5c6e1c
SR
953
954 /*
f4e781c0
SRRH
955 * Run a selftest on this tracer.
956 * Here we reset the trace buffer, and set the current
957 * tracer to be this tracer. The tracer can then run some
958 * internal tracing to verify that everything is in order.
959 * If we fail, we do not register this tracer.
0d5c6e1c 960 */
f4e781c0 961 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 962
f4e781c0
SRRH
963 tr->current_trace = type;
964
965#ifdef CONFIG_TRACER_MAX_TRACE
966 if (type->use_max_tr) {
967 /* If we expanded the buffers, make sure the max is expanded too */
968 if (ring_buffer_expanded)
969 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
970 RING_BUFFER_ALL_CPUS);
971 tr->allocated_snapshot = true;
972 }
973#endif
974
975 /* the test is responsible for initializing and enabling */
976 pr_info("Testing tracer %s: ", type->name);
977 ret = type->selftest(type, tr);
978 /* the test is responsible for resetting too */
979 tr->current_trace = saved_tracer;
980 if (ret) {
981 printk(KERN_CONT "FAILED!\n");
982 /* Add the warning after printing 'FAILED' */
983 WARN_ON(1);
984 return -1;
985 }
986 /* Only reset on passing, to avoid touching corrupted buffers */
987 tracing_reset_online_cpus(&tr->trace_buffer);
988
989#ifdef CONFIG_TRACER_MAX_TRACE
990 if (type->use_max_tr) {
991 tr->allocated_snapshot = false;
0d5c6e1c 992
f4e781c0
SRRH
993 /* Shrink the max buffer again */
994 if (ring_buffer_expanded)
995 ring_buffer_resize(tr->max_buffer.buffer, 1,
996 RING_BUFFER_ALL_CPUS);
997 }
998#endif
999
1000 printk(KERN_CONT "PASSED\n");
1001 return 0;
1002}
1003#else
1004static inline int run_tracer_selftest(struct tracer *type)
1005{
1006 return 0;
0d5c6e1c 1007}
f4e781c0 1008#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1009
4fcdae83
SR
1010/**
1011 * register_tracer - register a tracer with the ftrace system.
1012 * @type - the plugin for the tracer
1013 *
1014 * Register a new plugin tracer.
1015 */
bc0c38d1
SR
1016int register_tracer(struct tracer *type)
1017{
1018 struct tracer *t;
bc0c38d1
SR
1019 int ret = 0;
1020
1021 if (!type->name) {
1022 pr_info("Tracer must have a name\n");
1023 return -1;
1024 }
1025
24a461d5 1026 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1027 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1028 return -1;
1029 }
1030
bc0c38d1 1031 mutex_lock(&trace_types_lock);
86fa2f60 1032
8e1b82e0
FW
1033 tracing_selftest_running = true;
1034
bc0c38d1
SR
1035 for (t = trace_types; t; t = t->next) {
1036 if (strcmp(type->name, t->name) == 0) {
1037 /* already found */
ee6c2c1b 1038 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1039 type->name);
1040 ret = -1;
1041 goto out;
1042 }
1043 }
1044
adf9f195
FW
1045 if (!type->set_flag)
1046 type->set_flag = &dummy_set_flag;
1047 if (!type->flags)
1048 type->flags = &dummy_tracer_flags;
1049 else
1050 if (!type->flags->opts)
1051 type->flags->opts = dummy_tracer_opt;
6eaaa5d5
FW
1052 if (!type->wait_pipe)
1053 type->wait_pipe = default_wait_pipe;
1054
f4e781c0
SRRH
1055 ret = run_tracer_selftest(type);
1056 if (ret < 0)
1057 goto out;
60a11774 1058
bc0c38d1
SR
1059 type->next = trace_types;
1060 trace_types = type;
60a11774 1061
bc0c38d1 1062 out:
8e1b82e0 1063 tracing_selftest_running = false;
bc0c38d1
SR
1064 mutex_unlock(&trace_types_lock);
1065
dac74940
SR
1066 if (ret || !default_bootup_tracer)
1067 goto out_unlock;
1068
ee6c2c1b 1069 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1070 goto out_unlock;
1071
1072 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1073 /* Do we want this tracer to start on bootup? */
1074 tracing_set_tracer(type->name);
1075 default_bootup_tracer = NULL;
1076 /* disable other selftests, since this will break it. */
55034cd6 1077 tracing_selftest_disabled = true;
b2821ae6 1078#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1079 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1080 type->name);
b2821ae6 1081#endif
b2821ae6 1082
dac74940 1083 out_unlock:
bc0c38d1
SR
1084 return ret;
1085}
1086
12883efb 1087void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1088{
12883efb 1089 struct ring_buffer *buffer = buf->buffer;
f633903a 1090
a5416411
HT
1091 if (!buffer)
1092 return;
1093
f633903a
SR
1094 ring_buffer_record_disable(buffer);
1095
1096 /* Make sure all commits have finished */
1097 synchronize_sched();
68179686 1098 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1099
1100 ring_buffer_record_enable(buffer);
1101}
1102
12883efb 1103void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1104{
12883efb 1105 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1106 int cpu;
1107
a5416411
HT
1108 if (!buffer)
1109 return;
1110
621968cd
SR
1111 ring_buffer_record_disable(buffer);
1112
1113 /* Make sure all commits have finished */
1114 synchronize_sched();
1115
12883efb 1116 buf->time_start = ftrace_now(buf->cpu);
213cc060
PE
1117
1118 for_each_online_cpu(cpu)
68179686 1119 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1120
1121 ring_buffer_record_enable(buffer);
213cc060
PE
1122}
1123
9456f0fa
SR
1124void tracing_reset_current(int cpu)
1125{
12883efb 1126 tracing_reset(&global_trace.trace_buffer, cpu);
9456f0fa
SR
1127}
1128
873c642f 1129void tracing_reset_all_online_cpus(void)
9456f0fa 1130{
873c642f
SRRH
1131 struct trace_array *tr;
1132
1133 mutex_lock(&trace_types_lock);
1134 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
12883efb
SRRH
1135 tracing_reset_online_cpus(&tr->trace_buffer);
1136#ifdef CONFIG_TRACER_MAX_TRACE
1137 tracing_reset_online_cpus(&tr->max_buffer);
1138#endif
873c642f
SRRH
1139 }
1140 mutex_unlock(&trace_types_lock);
9456f0fa
SR
1141}
1142
bc0c38d1 1143#define SAVED_CMDLINES 128
2c7eea4c 1144#define NO_CMDLINE_MAP UINT_MAX
bc0c38d1
SR
1145static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1146static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1147static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1148static int cmdline_idx;
edc35bd7 1149static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
25b0b44a 1150
25b0b44a 1151/* temporary disable recording */
4fd27358 1152static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1
SR
1153
1154static void trace_init_cmdlines(void)
1155{
2c7eea4c
TG
1156 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1157 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
bc0c38d1
SR
1158 cmdline_idx = 0;
1159}
1160
b5130b1e
CE
1161int is_tracing_stopped(void)
1162{
2b6080f2 1163 return global_trace.stop_count;
b5130b1e
CE
1164}
1165
69bb54ec
SR
1166/**
1167 * ftrace_off_permanent - disable all ftrace code permanently
1168 *
1169 * This should only be called when a serious anomally has
1170 * been detected. This will turn off the function tracing,
1171 * ring buffers, and other tracing utilites. It takes no
1172 * locks and can be called from any context.
1173 */
1174void ftrace_off_permanent(void)
1175{
1176 tracing_disabled = 1;
1177 ftrace_stop();
1178 tracing_off_permanent();
1179}
1180
0f048701
SR
1181/**
1182 * tracing_start - quick start of the tracer
1183 *
1184 * If tracing is enabled but was stopped by tracing_stop,
1185 * this will start the tracer back up.
1186 */
1187void tracing_start(void)
1188{
1189 struct ring_buffer *buffer;
1190 unsigned long flags;
1191
1192 if (tracing_disabled)
1193 return;
1194
2b6080f2
SR
1195 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1196 if (--global_trace.stop_count) {
1197 if (global_trace.stop_count < 0) {
b06a8301
SR
1198 /* Someone screwed up their debugging */
1199 WARN_ON_ONCE(1);
2b6080f2 1200 global_trace.stop_count = 0;
b06a8301 1201 }
0f048701
SR
1202 goto out;
1203 }
1204
a2f80714
SR
1205 /* Prevent the buffers from switching */
1206 arch_spin_lock(&ftrace_max_lock);
0f048701 1207
12883efb 1208 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1209 if (buffer)
1210 ring_buffer_record_enable(buffer);
1211
12883efb
SRRH
1212#ifdef CONFIG_TRACER_MAX_TRACE
1213 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1214 if (buffer)
1215 ring_buffer_record_enable(buffer);
12883efb 1216#endif
0f048701 1217
a2f80714
SR
1218 arch_spin_unlock(&ftrace_max_lock);
1219
0f048701
SR
1220 ftrace_start();
1221 out:
2b6080f2
SR
1222 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1223}
1224
1225static void tracing_start_tr(struct trace_array *tr)
1226{
1227 struct ring_buffer *buffer;
1228 unsigned long flags;
1229
1230 if (tracing_disabled)
1231 return;
1232
1233 /* If global, we need to also start the max tracer */
1234 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1235 return tracing_start();
1236
1237 raw_spin_lock_irqsave(&tr->start_lock, flags);
1238
1239 if (--tr->stop_count) {
1240 if (tr->stop_count < 0) {
1241 /* Someone screwed up their debugging */
1242 WARN_ON_ONCE(1);
1243 tr->stop_count = 0;
1244 }
1245 goto out;
1246 }
1247
12883efb 1248 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1249 if (buffer)
1250 ring_buffer_record_enable(buffer);
1251
1252 out:
1253 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1254}
1255
1256/**
1257 * tracing_stop - quick stop of the tracer
1258 *
1259 * Light weight way to stop tracing. Use in conjunction with
1260 * tracing_start.
1261 */
1262void tracing_stop(void)
1263{
1264 struct ring_buffer *buffer;
1265 unsigned long flags;
1266
1267 ftrace_stop();
2b6080f2
SR
1268 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1269 if (global_trace.stop_count++)
0f048701
SR
1270 goto out;
1271
a2f80714
SR
1272 /* Prevent the buffers from switching */
1273 arch_spin_lock(&ftrace_max_lock);
1274
12883efb 1275 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1276 if (buffer)
1277 ring_buffer_record_disable(buffer);
1278
12883efb
SRRH
1279#ifdef CONFIG_TRACER_MAX_TRACE
1280 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1281 if (buffer)
1282 ring_buffer_record_disable(buffer);
12883efb 1283#endif
0f048701 1284
a2f80714
SR
1285 arch_spin_unlock(&ftrace_max_lock);
1286
0f048701 1287 out:
2b6080f2
SR
1288 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1289}
1290
1291static void tracing_stop_tr(struct trace_array *tr)
1292{
1293 struct ring_buffer *buffer;
1294 unsigned long flags;
1295
1296 /* If global, we need to also stop the max tracer */
1297 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1298 return tracing_stop();
1299
1300 raw_spin_lock_irqsave(&tr->start_lock, flags);
1301 if (tr->stop_count++)
1302 goto out;
1303
12883efb 1304 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1305 if (buffer)
1306 ring_buffer_record_disable(buffer);
1307
1308 out:
1309 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1310}
1311
e309b41d 1312void trace_stop_cmdline_recording(void);
bc0c38d1 1313
e309b41d 1314static void trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1315{
a635cf04 1316 unsigned pid, idx;
bc0c38d1
SR
1317
1318 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1319 return;
1320
1321 /*
1322 * It's not the end of the world if we don't get
1323 * the lock, but we also don't want to spin
1324 * nor do we want to disable interrupts,
1325 * so if we miss here, then better luck next time.
1326 */
0199c4e6 1327 if (!arch_spin_trylock(&trace_cmdline_lock))
bc0c38d1
SR
1328 return;
1329
1330 idx = map_pid_to_cmdline[tsk->pid];
2c7eea4c 1331 if (idx == NO_CMDLINE_MAP) {
bc0c38d1
SR
1332 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1333
a635cf04
CE
1334 /*
1335 * Check whether the cmdline buffer at idx has a pid
1336 * mapped. We are going to overwrite that entry so we
1337 * need to clear the map_pid_to_cmdline. Otherwise we
1338 * would read the new comm for the old pid.
1339 */
1340 pid = map_cmdline_to_pid[idx];
1341 if (pid != NO_CMDLINE_MAP)
1342 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1343
a635cf04 1344 map_cmdline_to_pid[idx] = tsk->pid;
bc0c38d1
SR
1345 map_pid_to_cmdline[tsk->pid] = idx;
1346
1347 cmdline_idx = idx;
1348 }
1349
1350 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1351
0199c4e6 1352 arch_spin_unlock(&trace_cmdline_lock);
bc0c38d1
SR
1353}
1354
4ca53085 1355void trace_find_cmdline(int pid, char comm[])
bc0c38d1 1356{
bc0c38d1
SR
1357 unsigned map;
1358
4ca53085
SR
1359 if (!pid) {
1360 strcpy(comm, "<idle>");
1361 return;
1362 }
bc0c38d1 1363
74bf4076
SR
1364 if (WARN_ON_ONCE(pid < 0)) {
1365 strcpy(comm, "<XXX>");
1366 return;
1367 }
1368
4ca53085
SR
1369 if (pid > PID_MAX_DEFAULT) {
1370 strcpy(comm, "<...>");
1371 return;
1372 }
bc0c38d1 1373
5b6045a9 1374 preempt_disable();
0199c4e6 1375 arch_spin_lock(&trace_cmdline_lock);
bc0c38d1 1376 map = map_pid_to_cmdline[pid];
50d88758
TG
1377 if (map != NO_CMDLINE_MAP)
1378 strcpy(comm, saved_cmdlines[map]);
1379 else
1380 strcpy(comm, "<...>");
bc0c38d1 1381
0199c4e6 1382 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1383 preempt_enable();
bc0c38d1
SR
1384}
1385
e309b41d 1386void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1387{
0fb9656d 1388 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
bc0c38d1
SR
1389 return;
1390
7ffbd48d
SR
1391 if (!__this_cpu_read(trace_cmdline_save))
1392 return;
1393
1394 __this_cpu_write(trace_cmdline_save, false);
1395
bc0c38d1
SR
1396 trace_save_cmdline(tsk);
1397}
1398
45dcd8b8 1399void
38697053
SR
1400tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1401 int pc)
bc0c38d1
SR
1402{
1403 struct task_struct *tsk = current;
bc0c38d1 1404
777e208d
SR
1405 entry->preempt_count = pc & 0xff;
1406 entry->pid = (tsk) ? tsk->pid : 0;
1407 entry->flags =
9244489a 1408#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1409 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1410#else
1411 TRACE_FLAG_IRQS_NOSUPPORT |
1412#endif
bc0c38d1
SR
1413 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1414 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1415 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1416}
f413cdb8 1417EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1418
e77405ad
SR
1419struct ring_buffer_event *
1420trace_buffer_lock_reserve(struct ring_buffer *buffer,
1421 int type,
1422 unsigned long len,
1423 unsigned long flags, int pc)
51a763dd
ACM
1424{
1425 struct ring_buffer_event *event;
1426
e77405ad 1427 event = ring_buffer_lock_reserve(buffer, len);
51a763dd
ACM
1428 if (event != NULL) {
1429 struct trace_entry *ent = ring_buffer_event_data(event);
1430
1431 tracing_generic_entry_update(ent, flags, pc);
1432 ent->type = type;
1433 }
1434
1435 return event;
1436}
51a763dd 1437
7ffbd48d
SR
1438void
1439__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1440{
1441 __this_cpu_write(trace_cmdline_save, true);
1442 ring_buffer_unlock_commit(buffer, event);
1443}
1444
e77405ad
SR
1445static inline void
1446__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1447 struct ring_buffer_event *event,
0d5c6e1c 1448 unsigned long flags, int pc)
51a763dd 1449{
7ffbd48d 1450 __buffer_unlock_commit(buffer, event);
51a763dd 1451
e77405ad
SR
1452 ftrace_trace_stack(buffer, flags, 6, pc);
1453 ftrace_trace_userstack(buffer, flags, pc);
07edf712
FW
1454}
1455
e77405ad
SR
1456void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1457 struct ring_buffer_event *event,
1458 unsigned long flags, int pc)
07edf712 1459{
0d5c6e1c 1460 __trace_buffer_unlock_commit(buffer, event, flags, pc);
51a763dd 1461}
0d5c6e1c 1462EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
51a763dd 1463
ccb469a1
SR
1464struct ring_buffer_event *
1465trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1466 struct ftrace_event_file *ftrace_file,
1467 int type, unsigned long len,
1468 unsigned long flags, int pc)
1469{
12883efb 1470 *current_rb = ftrace_file->tr->trace_buffer.buffer;
ccb469a1
SR
1471 return trace_buffer_lock_reserve(*current_rb,
1472 type, len, flags, pc);
1473}
1474EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1475
ef5580d0 1476struct ring_buffer_event *
e77405ad
SR
1477trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1478 int type, unsigned long len,
ef5580d0
SR
1479 unsigned long flags, int pc)
1480{
12883efb 1481 *current_rb = global_trace.trace_buffer.buffer;
e77405ad 1482 return trace_buffer_lock_reserve(*current_rb,
ef5580d0
SR
1483 type, len, flags, pc);
1484}
94487d6d 1485EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
ef5580d0 1486
e77405ad
SR
1487void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1488 struct ring_buffer_event *event,
ef5580d0
SR
1489 unsigned long flags, int pc)
1490{
0d5c6e1c 1491 __trace_buffer_unlock_commit(buffer, event, flags, pc);
07edf712 1492}
94487d6d 1493EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
07edf712 1494
0d5c6e1c
SR
1495void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1496 struct ring_buffer_event *event,
1497 unsigned long flags, int pc,
1498 struct pt_regs *regs)
1fd8df2c 1499{
7ffbd48d 1500 __buffer_unlock_commit(buffer, event);
1fd8df2c
MH
1501
1502 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1503 ftrace_trace_userstack(buffer, flags, pc);
1504}
0d5c6e1c 1505EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1fd8df2c 1506
e77405ad
SR
1507void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1508 struct ring_buffer_event *event)
77d9f465 1509{
e77405ad 1510 ring_buffer_discard_commit(buffer, event);
ef5580d0 1511}
12acd473 1512EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
ef5580d0 1513
e309b41d 1514void
7be42151 1515trace_function(struct trace_array *tr,
38697053
SR
1516 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1517 int pc)
bc0c38d1 1518{
e1112b4d 1519 struct ftrace_event_call *call = &event_function;
12883efb 1520 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 1521 struct ring_buffer_event *event;
777e208d 1522 struct ftrace_entry *entry;
bc0c38d1 1523
d769041f 1524 /* If we are reading the ring buffer, don't trace */
dd17c8f7 1525 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
d769041f
SR
1526 return;
1527
e77405ad 1528 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
51a763dd 1529 flags, pc);
3928a8a2
SR
1530 if (!event)
1531 return;
1532 entry = ring_buffer_event_data(event);
777e208d
SR
1533 entry->ip = ip;
1534 entry->parent_ip = parent_ip;
e1112b4d 1535
e77405ad 1536 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1537 __buffer_unlock_commit(buffer, event);
bc0c38d1
SR
1538}
1539
e309b41d 1540void
2e0f5761 1541ftrace(struct trace_array *tr, struct trace_array_cpu *data,
38697053
SR
1542 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1543 int pc)
2e0f5761
IM
1544{
1545 if (likely(!atomic_read(&data->disabled)))
7be42151 1546 trace_function(tr, ip, parent_ip, flags, pc);
2e0f5761
IM
1547}
1548
c0a0d0d3 1549#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
1550
1551#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1552struct ftrace_stack {
1553 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1554};
1555
1556static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1557static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1558
e77405ad 1559static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 1560 unsigned long flags,
1fd8df2c 1561 int skip, int pc, struct pt_regs *regs)
86387f7e 1562{
e1112b4d 1563 struct ftrace_event_call *call = &event_kernel_stack;
3928a8a2 1564 struct ring_buffer_event *event;
777e208d 1565 struct stack_entry *entry;
86387f7e 1566 struct stack_trace trace;
4a9bd3f1
SR
1567 int use_stack;
1568 int size = FTRACE_STACK_ENTRIES;
1569
1570 trace.nr_entries = 0;
1571 trace.skip = skip;
1572
1573 /*
1574 * Since events can happen in NMIs there's no safe way to
1575 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1576 * or NMI comes in, it will just have to use the default
1577 * FTRACE_STACK_SIZE.
1578 */
1579 preempt_disable_notrace();
1580
82146529 1581 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
1582 /*
1583 * We don't need any atomic variables, just a barrier.
1584 * If an interrupt comes in, we don't care, because it would
1585 * have exited and put the counter back to what we want.
1586 * We just need a barrier to keep gcc from moving things
1587 * around.
1588 */
1589 barrier();
1590 if (use_stack == 1) {
1591 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1592 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1593
1594 if (regs)
1595 save_stack_trace_regs(regs, &trace);
1596 else
1597 save_stack_trace(&trace);
1598
1599 if (trace.nr_entries > size)
1600 size = trace.nr_entries;
1601 } else
1602 /* From now on, use_stack is a boolean */
1603 use_stack = 0;
1604
1605 size *= sizeof(unsigned long);
86387f7e 1606
e77405ad 1607 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
4a9bd3f1 1608 sizeof(*entry) + size, flags, pc);
3928a8a2 1609 if (!event)
4a9bd3f1
SR
1610 goto out;
1611 entry = ring_buffer_event_data(event);
86387f7e 1612
4a9bd3f1
SR
1613 memset(&entry->caller, 0, size);
1614
1615 if (use_stack)
1616 memcpy(&entry->caller, trace.entries,
1617 trace.nr_entries * sizeof(unsigned long));
1618 else {
1619 trace.max_entries = FTRACE_STACK_ENTRIES;
1620 trace.entries = entry->caller;
1621 if (regs)
1622 save_stack_trace_regs(regs, &trace);
1623 else
1624 save_stack_trace(&trace);
1625 }
1626
1627 entry->size = trace.nr_entries;
86387f7e 1628
e77405ad 1629 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1630 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
1631
1632 out:
1633 /* Again, don't let gcc optimize things here */
1634 barrier();
82146529 1635 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
1636 preempt_enable_notrace();
1637
f0a920d5
IM
1638}
1639
1fd8df2c
MH
1640void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1641 int skip, int pc, struct pt_regs *regs)
1642{
1643 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1644 return;
1645
1646 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1647}
1648
e77405ad
SR
1649void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1650 int skip, int pc)
53614991
SR
1651{
1652 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1653 return;
1654
1fd8df2c 1655 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
53614991
SR
1656}
1657
c0a0d0d3
FW
1658void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1659 int pc)
38697053 1660{
12883efb 1661 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
38697053
SR
1662}
1663
03889384
SR
1664/**
1665 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 1666 * @skip: Number of functions to skip (helper handlers)
03889384 1667 */
c142be8e 1668void trace_dump_stack(int skip)
03889384
SR
1669{
1670 unsigned long flags;
1671
1672 if (tracing_disabled || tracing_selftest_running)
e36c5458 1673 return;
03889384
SR
1674
1675 local_save_flags(flags);
1676
c142be8e
SRRH
1677 /*
1678 * Skip 3 more, seems to get us at the caller of
1679 * this function.
1680 */
1681 skip += 3;
1682 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1683 flags, skip, preempt_count(), NULL);
03889384
SR
1684}
1685
91e86e56
SR
1686static DEFINE_PER_CPU(int, user_stack_count);
1687
e77405ad
SR
1688void
1689ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 1690{
e1112b4d 1691 struct ftrace_event_call *call = &event_user_stack;
8d7c6a96 1692 struct ring_buffer_event *event;
02b67518
TE
1693 struct userstack_entry *entry;
1694 struct stack_trace trace;
02b67518
TE
1695
1696 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1697 return;
1698
b6345879
SR
1699 /*
1700 * NMIs can not handle page faults, even with fix ups.
1701 * The save user stack can (and often does) fault.
1702 */
1703 if (unlikely(in_nmi()))
1704 return;
02b67518 1705
91e86e56
SR
1706 /*
1707 * prevent recursion, since the user stack tracing may
1708 * trigger other kernel events.
1709 */
1710 preempt_disable();
1711 if (__this_cpu_read(user_stack_count))
1712 goto out;
1713
1714 __this_cpu_inc(user_stack_count);
1715
e77405ad 1716 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
51a763dd 1717 sizeof(*entry), flags, pc);
02b67518 1718 if (!event)
1dbd1951 1719 goto out_drop_count;
02b67518 1720 entry = ring_buffer_event_data(event);
02b67518 1721
48659d31 1722 entry->tgid = current->tgid;
02b67518
TE
1723 memset(&entry->caller, 0, sizeof(entry->caller));
1724
1725 trace.nr_entries = 0;
1726 trace.max_entries = FTRACE_STACK_ENTRIES;
1727 trace.skip = 0;
1728 trace.entries = entry->caller;
1729
1730 save_stack_trace_user(&trace);
e77405ad 1731 if (!filter_check_discard(call, entry, buffer, event))
7ffbd48d 1732 __buffer_unlock_commit(buffer, event);
91e86e56 1733
1dbd1951 1734 out_drop_count:
91e86e56 1735 __this_cpu_dec(user_stack_count);
91e86e56
SR
1736 out:
1737 preempt_enable();
02b67518
TE
1738}
1739
4fd27358
HE
1740#ifdef UNUSED
1741static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 1742{
7be42151 1743 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 1744}
4fd27358 1745#endif /* UNUSED */
02b67518 1746
c0a0d0d3
FW
1747#endif /* CONFIG_STACKTRACE */
1748
07d777fe
SR
1749/* created for use with alloc_percpu */
1750struct trace_buffer_struct {
1751 char buffer[TRACE_BUF_SIZE];
1752};
1753
1754static struct trace_buffer_struct *trace_percpu_buffer;
1755static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1756static struct trace_buffer_struct *trace_percpu_irq_buffer;
1757static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1758
1759/*
1760 * The buffer used is dependent on the context. There is a per cpu
1761 * buffer for normal context, softirq contex, hard irq context and
1762 * for NMI context. Thise allows for lockless recording.
1763 *
1764 * Note, if the buffers failed to be allocated, then this returns NULL
1765 */
1766static char *get_trace_buf(void)
1767{
1768 struct trace_buffer_struct *percpu_buffer;
07d777fe
SR
1769
1770 /*
1771 * If we have allocated per cpu buffers, then we do not
1772 * need to do any locking.
1773 */
1774 if (in_nmi())
1775 percpu_buffer = trace_percpu_nmi_buffer;
1776 else if (in_irq())
1777 percpu_buffer = trace_percpu_irq_buffer;
1778 else if (in_softirq())
1779 percpu_buffer = trace_percpu_sirq_buffer;
1780 else
1781 percpu_buffer = trace_percpu_buffer;
1782
1783 if (!percpu_buffer)
1784 return NULL;
1785
d8a0349c 1786 return this_cpu_ptr(&percpu_buffer->buffer[0]);
07d777fe
SR
1787}
1788
1789static int alloc_percpu_trace_buffer(void)
1790{
1791 struct trace_buffer_struct *buffers;
1792 struct trace_buffer_struct *sirq_buffers;
1793 struct trace_buffer_struct *irq_buffers;
1794 struct trace_buffer_struct *nmi_buffers;
1795
1796 buffers = alloc_percpu(struct trace_buffer_struct);
1797 if (!buffers)
1798 goto err_warn;
1799
1800 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1801 if (!sirq_buffers)
1802 goto err_sirq;
1803
1804 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1805 if (!irq_buffers)
1806 goto err_irq;
1807
1808 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1809 if (!nmi_buffers)
1810 goto err_nmi;
1811
1812 trace_percpu_buffer = buffers;
1813 trace_percpu_sirq_buffer = sirq_buffers;
1814 trace_percpu_irq_buffer = irq_buffers;
1815 trace_percpu_nmi_buffer = nmi_buffers;
1816
1817 return 0;
1818
1819 err_nmi:
1820 free_percpu(irq_buffers);
1821 err_irq:
1822 free_percpu(sirq_buffers);
1823 err_sirq:
1824 free_percpu(buffers);
1825 err_warn:
1826 WARN(1, "Could not allocate percpu trace_printk buffer");
1827 return -ENOMEM;
1828}
1829
81698831
SR
1830static int buffers_allocated;
1831
07d777fe
SR
1832void trace_printk_init_buffers(void)
1833{
07d777fe
SR
1834 if (buffers_allocated)
1835 return;
1836
1837 if (alloc_percpu_trace_buffer())
1838 return;
1839
1840 pr_info("ftrace: Allocated trace_printk buffers\n");
1841
b382ede6
SR
1842 /* Expand the buffers to set size */
1843 tracing_update_buffers();
1844
07d777fe 1845 buffers_allocated = 1;
81698831
SR
1846
1847 /*
1848 * trace_printk_init_buffers() can be called by modules.
1849 * If that happens, then we need to start cmdline recording
1850 * directly here. If the global_trace.buffer is already
1851 * allocated here, then this was called by module code.
1852 */
12883efb 1853 if (global_trace.trace_buffer.buffer)
81698831
SR
1854 tracing_start_cmdline_record();
1855}
1856
1857void trace_printk_start_comm(void)
1858{
1859 /* Start tracing comms if trace printk is set */
1860 if (!buffers_allocated)
1861 return;
1862 tracing_start_cmdline_record();
1863}
1864
1865static void trace_printk_start_stop_comm(int enabled)
1866{
1867 if (!buffers_allocated)
1868 return;
1869
1870 if (enabled)
1871 tracing_start_cmdline_record();
1872 else
1873 tracing_stop_cmdline_record();
07d777fe
SR
1874}
1875
769b0441 1876/**
48ead020 1877 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
1878 *
1879 */
40ce74f1 1880int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 1881{
e1112b4d 1882 struct ftrace_event_call *call = &event_bprint;
769b0441 1883 struct ring_buffer_event *event;
e77405ad 1884 struct ring_buffer *buffer;
769b0441 1885 struct trace_array *tr = &global_trace;
48ead020 1886 struct bprint_entry *entry;
769b0441 1887 unsigned long flags;
07d777fe
SR
1888 char *tbuffer;
1889 int len = 0, size, pc;
769b0441
FW
1890
1891 if (unlikely(tracing_selftest_running || tracing_disabled))
1892 return 0;
1893
1894 /* Don't pollute graph traces with trace_vprintk internals */
1895 pause_graph_tracing();
1896
1897 pc = preempt_count();
5168ae50 1898 preempt_disable_notrace();
769b0441 1899
07d777fe
SR
1900 tbuffer = get_trace_buf();
1901 if (!tbuffer) {
1902 len = 0;
769b0441 1903 goto out;
07d777fe 1904 }
769b0441 1905
07d777fe 1906 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 1907
07d777fe
SR
1908 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1909 goto out;
769b0441 1910
07d777fe 1911 local_save_flags(flags);
769b0441 1912 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 1913 buffer = tr->trace_buffer.buffer;
e77405ad
SR
1914 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1915 flags, pc);
769b0441 1916 if (!event)
07d777fe 1917 goto out;
769b0441
FW
1918 entry = ring_buffer_event_data(event);
1919 entry->ip = ip;
769b0441
FW
1920 entry->fmt = fmt;
1921
07d777fe 1922 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
d931369b 1923 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 1924 __buffer_unlock_commit(buffer, event);
d931369b
SR
1925 ftrace_trace_stack(buffer, flags, 6, pc);
1926 }
769b0441 1927
769b0441 1928out:
5168ae50 1929 preempt_enable_notrace();
769b0441
FW
1930 unpause_graph_tracing();
1931
1932 return len;
1933}
48ead020
FW
1934EXPORT_SYMBOL_GPL(trace_vbprintk);
1935
12883efb
SRRH
1936static int
1937__trace_array_vprintk(struct ring_buffer *buffer,
1938 unsigned long ip, const char *fmt, va_list args)
48ead020 1939{
e1112b4d 1940 struct ftrace_event_call *call = &event_print;
48ead020 1941 struct ring_buffer_event *event;
07d777fe 1942 int len = 0, size, pc;
48ead020 1943 struct print_entry *entry;
07d777fe
SR
1944 unsigned long flags;
1945 char *tbuffer;
48ead020
FW
1946
1947 if (tracing_disabled || tracing_selftest_running)
1948 return 0;
1949
07d777fe
SR
1950 /* Don't pollute graph traces with trace_vprintk internals */
1951 pause_graph_tracing();
1952
48ead020
FW
1953 pc = preempt_count();
1954 preempt_disable_notrace();
48ead020 1955
07d777fe
SR
1956
1957 tbuffer = get_trace_buf();
1958 if (!tbuffer) {
1959 len = 0;
48ead020 1960 goto out;
07d777fe 1961 }
48ead020 1962
07d777fe
SR
1963 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1964 if (len > TRACE_BUF_SIZE)
1965 goto out;
48ead020 1966
07d777fe 1967 local_save_flags(flags);
48ead020 1968 size = sizeof(*entry) + len + 1;
e77405ad 1969 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
07d777fe 1970 flags, pc);
48ead020 1971 if (!event)
07d777fe 1972 goto out;
48ead020 1973 entry = ring_buffer_event_data(event);
c13d2f7c 1974 entry->ip = ip;
48ead020 1975
07d777fe 1976 memcpy(&entry->buf, tbuffer, len);
c13d2f7c 1977 entry->buf[len] = '\0';
d931369b 1978 if (!filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 1979 __buffer_unlock_commit(buffer, event);
07d777fe 1980 ftrace_trace_stack(buffer, flags, 6, pc);
d931369b 1981 }
48ead020
FW
1982 out:
1983 preempt_enable_notrace();
07d777fe 1984 unpause_graph_tracing();
48ead020
FW
1985
1986 return len;
1987}
659372d3 1988
12883efb
SRRH
1989int trace_array_vprintk(struct trace_array *tr,
1990 unsigned long ip, const char *fmt, va_list args)
1991{
1992 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1993}
1994
1995int trace_array_printk(struct trace_array *tr,
1996 unsigned long ip, const char *fmt, ...)
1997{
1998 int ret;
1999 va_list ap;
2000
2001 if (!(trace_flags & TRACE_ITER_PRINTK))
2002 return 0;
2003
2004 va_start(ap, fmt);
2005 ret = trace_array_vprintk(tr, ip, fmt, ap);
2006 va_end(ap);
2007 return ret;
2008}
2009
2010int trace_array_printk_buf(struct ring_buffer *buffer,
2011 unsigned long ip, const char *fmt, ...)
2012{
2013 int ret;
2014 va_list ap;
2015
2016 if (!(trace_flags & TRACE_ITER_PRINTK))
2017 return 0;
2018
2019 va_start(ap, fmt);
2020 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2021 va_end(ap);
2022 return ret;
2023}
2024
659372d3
SR
2025int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2026{
a813a159 2027 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 2028}
769b0441
FW
2029EXPORT_SYMBOL_GPL(trace_vprintk);
2030
e2ac8ef5 2031static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 2032{
6d158a81
SR
2033 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2034
5a90f577 2035 iter->idx++;
6d158a81
SR
2036 if (buf_iter)
2037 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
2038}
2039
e309b41d 2040static struct trace_entry *
bc21b478
SR
2041peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2042 unsigned long *lost_events)
dd0e545f 2043{
3928a8a2 2044 struct ring_buffer_event *event;
6d158a81 2045 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 2046
d769041f
SR
2047 if (buf_iter)
2048 event = ring_buffer_iter_peek(buf_iter, ts);
2049 else
12883efb 2050 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 2051 lost_events);
d769041f 2052
4a9bd3f1
SR
2053 if (event) {
2054 iter->ent_size = ring_buffer_event_length(event);
2055 return ring_buffer_event_data(event);
2056 }
2057 iter->ent_size = 0;
2058 return NULL;
dd0e545f 2059}
d769041f 2060
dd0e545f 2061static struct trace_entry *
bc21b478
SR
2062__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2063 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 2064{
12883efb 2065 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 2066 struct trace_entry *ent, *next = NULL;
aa27497c 2067 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 2068 int cpu_file = iter->cpu_file;
3928a8a2 2069 u64 next_ts = 0, ts;
bc0c38d1 2070 int next_cpu = -1;
12b5da34 2071 int next_size = 0;
bc0c38d1
SR
2072 int cpu;
2073
b04cc6b1
FW
2074 /*
2075 * If we are in a per_cpu trace file, don't bother by iterating over
2076 * all cpu and peek directly.
2077 */
ae3b5093 2078 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
2079 if (ring_buffer_empty_cpu(buffer, cpu_file))
2080 return NULL;
bc21b478 2081 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
2082 if (ent_cpu)
2083 *ent_cpu = cpu_file;
2084
2085 return ent;
2086 }
2087
ab46428c 2088 for_each_tracing_cpu(cpu) {
dd0e545f 2089
3928a8a2
SR
2090 if (ring_buffer_empty_cpu(buffer, cpu))
2091 continue;
dd0e545f 2092
bc21b478 2093 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 2094
cdd31cd2
IM
2095 /*
2096 * Pick the entry with the smallest timestamp:
2097 */
3928a8a2 2098 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
2099 next = ent;
2100 next_cpu = cpu;
3928a8a2 2101 next_ts = ts;
bc21b478 2102 next_lost = lost_events;
12b5da34 2103 next_size = iter->ent_size;
bc0c38d1
SR
2104 }
2105 }
2106
12b5da34
SR
2107 iter->ent_size = next_size;
2108
bc0c38d1
SR
2109 if (ent_cpu)
2110 *ent_cpu = next_cpu;
2111
3928a8a2
SR
2112 if (ent_ts)
2113 *ent_ts = next_ts;
2114
bc21b478
SR
2115 if (missing_events)
2116 *missing_events = next_lost;
2117
bc0c38d1
SR
2118 return next;
2119}
2120
dd0e545f 2121/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
2122struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2123 int *ent_cpu, u64 *ent_ts)
bc0c38d1 2124{
bc21b478 2125 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
2126}
2127
2128/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 2129void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 2130{
bc21b478
SR
2131 iter->ent = __find_next_entry(iter, &iter->cpu,
2132 &iter->lost_events, &iter->ts);
dd0e545f 2133
3928a8a2 2134 if (iter->ent)
e2ac8ef5 2135 trace_iterator_increment(iter);
dd0e545f 2136
3928a8a2 2137 return iter->ent ? iter : NULL;
b3806b43 2138}
bc0c38d1 2139
e309b41d 2140static void trace_consume(struct trace_iterator *iter)
b3806b43 2141{
12883efb 2142 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 2143 &iter->lost_events);
bc0c38d1
SR
2144}
2145
e309b41d 2146static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
2147{
2148 struct trace_iterator *iter = m->private;
bc0c38d1 2149 int i = (int)*pos;
4e3c3333 2150 void *ent;
bc0c38d1 2151
a63ce5b3
SR
2152 WARN_ON_ONCE(iter->leftover);
2153
bc0c38d1
SR
2154 (*pos)++;
2155
2156 /* can't go backwards */
2157 if (iter->idx > i)
2158 return NULL;
2159
2160 if (iter->idx < 0)
955b61e5 2161 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2162 else
2163 ent = iter;
2164
2165 while (ent && iter->idx < i)
955b61e5 2166 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2167
2168 iter->pos = *pos;
2169
bc0c38d1
SR
2170 return ent;
2171}
2172
955b61e5 2173void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 2174{
2f26ebd5
SR
2175 struct ring_buffer_event *event;
2176 struct ring_buffer_iter *buf_iter;
2177 unsigned long entries = 0;
2178 u64 ts;
2179
12883efb 2180 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 2181
6d158a81
SR
2182 buf_iter = trace_buffer_iter(iter, cpu);
2183 if (!buf_iter)
2f26ebd5
SR
2184 return;
2185
2f26ebd5
SR
2186 ring_buffer_iter_reset(buf_iter);
2187
2188 /*
2189 * We could have the case with the max latency tracers
2190 * that a reset never took place on a cpu. This is evident
2191 * by the timestamp being before the start of the buffer.
2192 */
2193 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 2194 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
2195 break;
2196 entries++;
2197 ring_buffer_read(buf_iter, NULL);
2198 }
2199
12883efb 2200 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
2201}
2202
d7350c3f 2203/*
d7350c3f
FW
2204 * The current tracer is copied to avoid a global locking
2205 * all around.
2206 */
bc0c38d1
SR
2207static void *s_start(struct seq_file *m, loff_t *pos)
2208{
2209 struct trace_iterator *iter = m->private;
2b6080f2 2210 struct trace_array *tr = iter->tr;
b04cc6b1 2211 int cpu_file = iter->cpu_file;
bc0c38d1
SR
2212 void *p = NULL;
2213 loff_t l = 0;
3928a8a2 2214 int cpu;
bc0c38d1 2215
2fd196ec
HT
2216 /*
2217 * copy the tracer to avoid using a global lock all around.
2218 * iter->trace is a copy of current_trace, the pointer to the
2219 * name may be used instead of a strcmp(), as iter->trace->name
2220 * will point to the same string as current_trace->name.
2221 */
bc0c38d1 2222 mutex_lock(&trace_types_lock);
2b6080f2
SR
2223 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2224 *iter->trace = *tr->current_trace;
d7350c3f 2225 mutex_unlock(&trace_types_lock);
bc0c38d1 2226
12883efb 2227#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2228 if (iter->snapshot && iter->trace->use_max_tr)
2229 return ERR_PTR(-EBUSY);
12883efb 2230#endif
debdd57f
HT
2231
2232 if (!iter->snapshot)
2233 atomic_inc(&trace_record_cmdline_disabled);
bc0c38d1 2234
bc0c38d1
SR
2235 if (*pos != iter->pos) {
2236 iter->ent = NULL;
2237 iter->cpu = 0;
2238 iter->idx = -1;
2239
ae3b5093 2240 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2241 for_each_tracing_cpu(cpu)
2f26ebd5 2242 tracing_iter_reset(iter, cpu);
b04cc6b1 2243 } else
2f26ebd5 2244 tracing_iter_reset(iter, cpu_file);
bc0c38d1 2245
ac91d854 2246 iter->leftover = 0;
bc0c38d1
SR
2247 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2248 ;
2249
2250 } else {
a63ce5b3
SR
2251 /*
2252 * If we overflowed the seq_file before, then we want
2253 * to just reuse the trace_seq buffer again.
2254 */
2255 if (iter->leftover)
2256 p = iter;
2257 else {
2258 l = *pos - 1;
2259 p = s_next(m, p, &l);
2260 }
bc0c38d1
SR
2261 }
2262
4f535968 2263 trace_event_read_lock();
7e53bd42 2264 trace_access_lock(cpu_file);
bc0c38d1
SR
2265 return p;
2266}
2267
2268static void s_stop(struct seq_file *m, void *p)
2269{
7e53bd42
LJ
2270 struct trace_iterator *iter = m->private;
2271
12883efb 2272#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
2273 if (iter->snapshot && iter->trace->use_max_tr)
2274 return;
12883efb 2275#endif
debdd57f
HT
2276
2277 if (!iter->snapshot)
2278 atomic_dec(&trace_record_cmdline_disabled);
12883efb 2279
7e53bd42 2280 trace_access_unlock(iter->cpu_file);
4f535968 2281 trace_event_read_unlock();
bc0c38d1
SR
2282}
2283
39eaf7ef 2284static void
12883efb
SRRH
2285get_total_entries(struct trace_buffer *buf,
2286 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
2287{
2288 unsigned long count;
2289 int cpu;
2290
2291 *total = 0;
2292 *entries = 0;
2293
2294 for_each_tracing_cpu(cpu) {
12883efb 2295 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
2296 /*
2297 * If this buffer has skipped entries, then we hold all
2298 * entries for the trace and we need to ignore the
2299 * ones before the time stamp.
2300 */
12883efb
SRRH
2301 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2302 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
2303 /* total is the same as the entries */
2304 *total += count;
2305 } else
2306 *total += count +
12883efb 2307 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
2308 *entries += count;
2309 }
2310}
2311
e309b41d 2312static void print_lat_help_header(struct seq_file *m)
bc0c38d1 2313{
a6168353
ME
2314 seq_puts(m, "# _------=> CPU# \n");
2315 seq_puts(m, "# / _-----=> irqs-off \n");
2316 seq_puts(m, "# | / _----=> need-resched \n");
2317 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2318 seq_puts(m, "# ||| / _--=> preempt-depth \n");
e6e1e259
SR
2319 seq_puts(m, "# |||| / delay \n");
2320 seq_puts(m, "# cmd pid ||||| time | caller \n");
2321 seq_puts(m, "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
2322}
2323
12883efb 2324static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 2325{
39eaf7ef
SR
2326 unsigned long total;
2327 unsigned long entries;
2328
12883efb 2329 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
2330 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2331 entries, total, num_online_cpus());
2332 seq_puts(m, "#\n");
2333}
2334
12883efb 2335static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
39eaf7ef 2336{
12883efb 2337 print_event_info(buf, m);
77271ce4 2338 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
a6168353 2339 seq_puts(m, "# | | | | |\n");
bc0c38d1
SR
2340}
2341
12883efb 2342static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
77271ce4 2343{
12883efb 2344 print_event_info(buf, m);
77271ce4
SR
2345 seq_puts(m, "# _-----=> irqs-off\n");
2346 seq_puts(m, "# / _----=> need-resched\n");
2347 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2348 seq_puts(m, "# || / _--=> preempt-depth\n");
2349 seq_puts(m, "# ||| / delay\n");
2350 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2351 seq_puts(m, "# | | | |||| | |\n");
2352}
bc0c38d1 2353
62b915f1 2354void
bc0c38d1
SR
2355print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2356{
2357 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
2358 struct trace_buffer *buf = iter->trace_buffer;
2359 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 2360 struct tracer *type = iter->trace;
39eaf7ef
SR
2361 unsigned long entries;
2362 unsigned long total;
bc0c38d1
SR
2363 const char *name = "preemption";
2364
d840f718 2365 name = type->name;
bc0c38d1 2366
12883efb 2367 get_total_entries(buf, &total, &entries);
bc0c38d1 2368
888b55dc 2369 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 2370 name, UTS_RELEASE);
888b55dc 2371 seq_puts(m, "# -----------------------------------"
bc0c38d1 2372 "---------------------------------\n");
888b55dc 2373 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 2374 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 2375 nsecs_to_usecs(data->saved_latency),
bc0c38d1 2376 entries,
4c11d7ae 2377 total,
12883efb 2378 buf->cpu,
bc0c38d1
SR
2379#if defined(CONFIG_PREEMPT_NONE)
2380 "server",
2381#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2382 "desktop",
b5c21b45 2383#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
2384 "preempt",
2385#else
2386 "unknown",
2387#endif
2388 /* These are reserved for later use */
2389 0, 0, 0, 0);
2390#ifdef CONFIG_SMP
2391 seq_printf(m, " #P:%d)\n", num_online_cpus());
2392#else
2393 seq_puts(m, ")\n");
2394#endif
888b55dc
KM
2395 seq_puts(m, "# -----------------\n");
2396 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 2397 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
2398 data->comm, data->pid,
2399 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 2400 data->policy, data->rt_priority);
888b55dc 2401 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
2402
2403 if (data->critical_start) {
888b55dc 2404 seq_puts(m, "# => started at: ");
214023c3
SR
2405 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2406 trace_print_seq(m, &iter->seq);
888b55dc 2407 seq_puts(m, "\n# => ended at: ");
214023c3
SR
2408 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2409 trace_print_seq(m, &iter->seq);
8248ac05 2410 seq_puts(m, "\n#\n");
bc0c38d1
SR
2411 }
2412
888b55dc 2413 seq_puts(m, "#\n");
bc0c38d1
SR
2414}
2415
a309720c
SR
2416static void test_cpu_buff_start(struct trace_iterator *iter)
2417{
2418 struct trace_seq *s = &iter->seq;
2419
12ef7d44
SR
2420 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2421 return;
2422
2423 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2424 return;
2425
4462344e 2426 if (cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
2427 return;
2428
12883efb 2429 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
2430 return;
2431
4462344e 2432 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
2433
2434 /* Don't print started cpu buffer for the first entry of the trace */
2435 if (iter->idx > 1)
2436 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2437 iter->cpu);
a309720c
SR
2438}
2439
2c4f035f 2440static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 2441{
214023c3 2442 struct trace_seq *s = &iter->seq;
bc0c38d1 2443 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 2444 struct trace_entry *entry;
f633cef0 2445 struct trace_event *event;
bc0c38d1 2446
4e3c3333 2447 entry = iter->ent;
dd0e545f 2448
a309720c
SR
2449 test_cpu_buff_start(iter);
2450
c4a8e8be 2451 event = ftrace_find_event(entry->type);
bc0c38d1 2452
c4a8e8be 2453 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
27d48be8
SR
2454 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2455 if (!trace_print_lat_context(iter))
2456 goto partial;
2457 } else {
2458 if (!trace_print_context(iter))
2459 goto partial;
2460 }
c4a8e8be 2461 }
bc0c38d1 2462
268ccda0 2463 if (event)
a9a57763 2464 return event->funcs->trace(iter, sym_flags, event);
d9793bd8
ACM
2465
2466 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2467 goto partial;
02b67518 2468
2c4f035f 2469 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2470partial:
2471 return TRACE_TYPE_PARTIAL_LINE;
bc0c38d1
SR
2472}
2473
2c4f035f 2474static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3
IM
2475{
2476 struct trace_seq *s = &iter->seq;
2477 struct trace_entry *entry;
f633cef0 2478 struct trace_event *event;
f9896bf3
IM
2479
2480 entry = iter->ent;
dd0e545f 2481
c4a8e8be 2482 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
d9793bd8
ACM
2483 if (!trace_seq_printf(s, "%d %d %llu ",
2484 entry->pid, iter->cpu, iter->ts))
2485 goto partial;
c4a8e8be 2486 }
f9896bf3 2487
f633cef0 2488 event = ftrace_find_event(entry->type);
268ccda0 2489 if (event)
a9a57763 2490 return event->funcs->raw(iter, 0, event);
d9793bd8
ACM
2491
2492 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2493 goto partial;
777e208d 2494
2c4f035f 2495 return TRACE_TYPE_HANDLED;
d9793bd8
ACM
2496partial:
2497 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3
IM
2498}
2499
2c4f035f 2500static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec
IM
2501{
2502 struct trace_seq *s = &iter->seq;
2503 unsigned char newline = '\n';
2504 struct trace_entry *entry;
f633cef0 2505 struct trace_event *event;
5e3ca0ec
IM
2506
2507 entry = iter->ent;
dd0e545f 2508
c4a8e8be
FW
2509 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2510 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2511 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2512 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2513 }
5e3ca0ec 2514
f633cef0 2515 event = ftrace_find_event(entry->type);
268ccda0 2516 if (event) {
a9a57763 2517 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
2518 if (ret != TRACE_TYPE_HANDLED)
2519 return ret;
2520 }
7104f300 2521
5e3ca0ec
IM
2522 SEQ_PUT_FIELD_RET(s, newline);
2523
2c4f035f 2524 return TRACE_TYPE_HANDLED;
5e3ca0ec
IM
2525}
2526
2c4f035f 2527static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa
IM
2528{
2529 struct trace_seq *s = &iter->seq;
2530 struct trace_entry *entry;
f633cef0 2531 struct trace_event *event;
cb0f12aa
IM
2532
2533 entry = iter->ent;
dd0e545f 2534
c4a8e8be
FW
2535 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2536 SEQ_PUT_FIELD_RET(s, entry->pid);
1830b52d 2537 SEQ_PUT_FIELD_RET(s, iter->cpu);
c4a8e8be
FW
2538 SEQ_PUT_FIELD_RET(s, iter->ts);
2539 }
cb0f12aa 2540
f633cef0 2541 event = ftrace_find_event(entry->type);
a9a57763
SR
2542 return event ? event->funcs->binary(iter, 0, event) :
2543 TRACE_TYPE_HANDLED;
cb0f12aa
IM
2544}
2545
62b915f1 2546int trace_empty(struct trace_iterator *iter)
bc0c38d1 2547{
6d158a81 2548 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
2549 int cpu;
2550
9aba60fe 2551 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 2552 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 2553 cpu = iter->cpu_file;
6d158a81
SR
2554 buf_iter = trace_buffer_iter(iter, cpu);
2555 if (buf_iter) {
2556 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
2557 return 0;
2558 } else {
12883efb 2559 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
2560 return 0;
2561 }
2562 return 1;
2563 }
2564
ab46428c 2565 for_each_tracing_cpu(cpu) {
6d158a81
SR
2566 buf_iter = trace_buffer_iter(iter, cpu);
2567 if (buf_iter) {
2568 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
2569 return 0;
2570 } else {
12883efb 2571 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
2572 return 0;
2573 }
bc0c38d1 2574 }
d769041f 2575
797d3712 2576 return 1;
bc0c38d1
SR
2577}
2578
4f535968 2579/* Called with trace_event_read_lock() held. */
955b61e5 2580enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 2581{
2c4f035f
FW
2582 enum print_line_t ret;
2583
ee5e51f5
JO
2584 if (iter->lost_events &&
2585 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2586 iter->cpu, iter->lost_events))
2587 return TRACE_TYPE_PARTIAL_LINE;
bc21b478 2588
2c4f035f
FW
2589 if (iter->trace && iter->trace->print_line) {
2590 ret = iter->trace->print_line(iter);
2591 if (ret != TRACE_TYPE_UNHANDLED)
2592 return ret;
2593 }
72829bc3 2594
09ae7234
SRRH
2595 if (iter->ent->type == TRACE_BPUTS &&
2596 trace_flags & TRACE_ITER_PRINTK &&
2597 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2598 return trace_print_bputs_msg_only(iter);
2599
48ead020
FW
2600 if (iter->ent->type == TRACE_BPRINT &&
2601 trace_flags & TRACE_ITER_PRINTK &&
2602 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2603 return trace_print_bprintk_msg_only(iter);
48ead020 2604
66896a85
FW
2605 if (iter->ent->type == TRACE_PRINT &&
2606 trace_flags & TRACE_ITER_PRINTK &&
2607 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 2608 return trace_print_printk_msg_only(iter);
66896a85 2609
cb0f12aa
IM
2610 if (trace_flags & TRACE_ITER_BIN)
2611 return print_bin_fmt(iter);
2612
5e3ca0ec
IM
2613 if (trace_flags & TRACE_ITER_HEX)
2614 return print_hex_fmt(iter);
2615
f9896bf3
IM
2616 if (trace_flags & TRACE_ITER_RAW)
2617 return print_raw_fmt(iter);
2618
f9896bf3
IM
2619 return print_trace_fmt(iter);
2620}
2621
7e9a49ef
JO
2622void trace_latency_header(struct seq_file *m)
2623{
2624 struct trace_iterator *iter = m->private;
2625
2626 /* print nothing if the buffers are empty */
2627 if (trace_empty(iter))
2628 return;
2629
2630 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2631 print_trace_header(m, iter);
2632
2633 if (!(trace_flags & TRACE_ITER_VERBOSE))
2634 print_lat_help_header(m);
2635}
2636
62b915f1
JO
2637void trace_default_header(struct seq_file *m)
2638{
2639 struct trace_iterator *iter = m->private;
2640
f56e7f8e
JO
2641 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2642 return;
2643
62b915f1
JO
2644 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2645 /* print nothing if the buffers are empty */
2646 if (trace_empty(iter))
2647 return;
2648 print_trace_header(m, iter);
2649 if (!(trace_flags & TRACE_ITER_VERBOSE))
2650 print_lat_help_header(m);
2651 } else {
77271ce4
SR
2652 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2653 if (trace_flags & TRACE_ITER_IRQ_INFO)
12883efb 2654 print_func_help_header_irq(iter->trace_buffer, m);
77271ce4 2655 else
12883efb 2656 print_func_help_header(iter->trace_buffer, m);
77271ce4 2657 }
62b915f1
JO
2658 }
2659}
2660
e0a413f6
SR
2661static void test_ftrace_alive(struct seq_file *m)
2662{
2663 if (!ftrace_is_dead())
2664 return;
2665 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2666 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2667}
2668
d8741e2e 2669#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 2670static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 2671{
d8741e2e
SRRH
2672 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2673 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2674 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2675 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2676 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2677 seq_printf(m, "# is not a '0' or '1')\n");
2678}
f1affcaa
SRRH
2679
2680static void show_snapshot_percpu_help(struct seq_file *m)
2681{
2682 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2683#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2684 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2685 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2686#else
2687 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2688 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2689#endif
2690 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2691 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2692 seq_printf(m, "# is not a '0' or '1')\n");
2693}
2694
d8741e2e
SRRH
2695static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2696{
45ad21ca 2697 if (iter->tr->allocated_snapshot)
d8741e2e
SRRH
2698 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2699 else
2700 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2701
2702 seq_printf(m, "# Snapshot commands:\n");
f1affcaa
SRRH
2703 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2704 show_snapshot_main_help(m);
2705 else
2706 show_snapshot_percpu_help(m);
d8741e2e
SRRH
2707}
2708#else
2709/* Should never be called */
2710static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2711#endif
2712
bc0c38d1
SR
2713static int s_show(struct seq_file *m, void *v)
2714{
2715 struct trace_iterator *iter = v;
a63ce5b3 2716 int ret;
bc0c38d1
SR
2717
2718 if (iter->ent == NULL) {
2719 if (iter->tr) {
2720 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2721 seq_puts(m, "#\n");
e0a413f6 2722 test_ftrace_alive(m);
bc0c38d1 2723 }
d8741e2e
SRRH
2724 if (iter->snapshot && trace_empty(iter))
2725 print_snapshot_help(m, iter);
2726 else if (iter->trace && iter->trace->print_header)
8bba1bf5 2727 iter->trace->print_header(m);
62b915f1
JO
2728 else
2729 trace_default_header(m);
2730
a63ce5b3
SR
2731 } else if (iter->leftover) {
2732 /*
2733 * If we filled the seq_file buffer earlier, we
2734 * want to just show it now.
2735 */
2736 ret = trace_print_seq(m, &iter->seq);
2737
2738 /* ret should this time be zero, but you never know */
2739 iter->leftover = ret;
2740
bc0c38d1 2741 } else {
f9896bf3 2742 print_trace_line(iter);
a63ce5b3
SR
2743 ret = trace_print_seq(m, &iter->seq);
2744 /*
2745 * If we overflow the seq_file buffer, then it will
2746 * ask us for this data again at start up.
2747 * Use that instead.
2748 * ret is 0 if seq_file write succeeded.
2749 * -1 otherwise.
2750 */
2751 iter->leftover = ret;
bc0c38d1
SR
2752 }
2753
2754 return 0;
2755}
2756
88e9d34c 2757static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
2758 .start = s_start,
2759 .next = s_next,
2760 .stop = s_stop,
2761 .show = s_show,
bc0c38d1
SR
2762};
2763
e309b41d 2764static struct trace_iterator *
debdd57f 2765__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 2766{
2b6080f2
SR
2767 struct trace_cpu *tc = inode->i_private;
2768 struct trace_array *tr = tc->tr;
bc0c38d1 2769 struct trace_iterator *iter;
50e18b94 2770 int cpu;
bc0c38d1 2771
85a2f9b4
SR
2772 if (tracing_disabled)
2773 return ERR_PTR(-ENODEV);
60a11774 2774
50e18b94 2775 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
2776 if (!iter)
2777 return ERR_PTR(-ENOMEM);
bc0c38d1 2778
6d158a81
SR
2779 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2780 GFP_KERNEL);
93574fcc
DC
2781 if (!iter->buffer_iter)
2782 goto release;
2783
d7350c3f
FW
2784 /*
2785 * We make a copy of the current tracer to avoid concurrent
2786 * changes on it while we are reading.
2787 */
bc0c38d1 2788 mutex_lock(&trace_types_lock);
d7350c3f 2789 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 2790 if (!iter->trace)
d7350c3f 2791 goto fail;
85a2f9b4 2792
2b6080f2 2793 *iter->trace = *tr->current_trace;
d7350c3f 2794
79f55997 2795 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
2796 goto fail;
2797
12883efb
SRRH
2798 iter->tr = tr;
2799
2800#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
2801 /* Currently only the top directory has a snapshot */
2802 if (tr->current_trace->print_max || snapshot)
12883efb 2803 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 2804 else
12883efb
SRRH
2805#endif
2806 iter->trace_buffer = &tr->trace_buffer;
debdd57f 2807 iter->snapshot = snapshot;
bc0c38d1 2808 iter->pos = -1;
d7350c3f 2809 mutex_init(&iter->mutex);
2b6080f2 2810 iter->cpu_file = tc->cpu;
bc0c38d1 2811
8bba1bf5
MM
2812 /* Notify the tracer early; before we stop tracing. */
2813 if (iter->trace && iter->trace->open)
a93751ca 2814 iter->trace->open(iter);
8bba1bf5 2815
12ef7d44 2816 /* Annotate start of buffers if we had overruns */
12883efb 2817 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
2818 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2819
8be0709f
DS
2820 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2821 if (trace_clocks[trace_clock_id].in_ns)
2822 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2823
debdd57f
HT
2824 /* stop the trace while dumping if we are not opening "snapshot" */
2825 if (!iter->snapshot)
2b6080f2 2826 tracing_stop_tr(tr);
2f26ebd5 2827
ae3b5093 2828 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 2829 for_each_tracing_cpu(cpu) {
b04cc6b1 2830 iter->buffer_iter[cpu] =
12883efb 2831 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2832 }
2833 ring_buffer_read_prepare_sync();
2834 for_each_tracing_cpu(cpu) {
2835 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2836 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
2837 }
2838 } else {
2839 cpu = iter->cpu_file;
3928a8a2 2840 iter->buffer_iter[cpu] =
12883efb 2841 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
2842 ring_buffer_read_prepare_sync();
2843 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 2844 tracing_iter_reset(iter, cpu);
3928a8a2
SR
2845 }
2846
a695cb58
SRRH
2847 tr->ref++;
2848
bc0c38d1
SR
2849 mutex_unlock(&trace_types_lock);
2850
bc0c38d1 2851 return iter;
3928a8a2 2852
d7350c3f 2853 fail:
3928a8a2 2854 mutex_unlock(&trace_types_lock);
d7350c3f 2855 kfree(iter->trace);
6d158a81 2856 kfree(iter->buffer_iter);
93574fcc 2857release:
50e18b94
JO
2858 seq_release_private(inode, file);
2859 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
2860}
2861
2862int tracing_open_generic(struct inode *inode, struct file *filp)
2863{
60a11774
SR
2864 if (tracing_disabled)
2865 return -ENODEV;
2866
bc0c38d1
SR
2867 filp->private_data = inode->i_private;
2868 return 0;
2869}
2870
4fd27358 2871static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 2872{
907f2784 2873 struct seq_file *m = file->private_data;
4acd4d00 2874 struct trace_iterator *iter;
2b6080f2 2875 struct trace_array *tr;
3928a8a2 2876 int cpu;
bc0c38d1 2877
4acd4d00
SR
2878 if (!(file->f_mode & FMODE_READ))
2879 return 0;
2880
2881 iter = m->private;
12883efb 2882 tr = iter->tr;
4acd4d00 2883
bc0c38d1 2884 mutex_lock(&trace_types_lock);
a695cb58
SRRH
2885
2886 WARN_ON(!tr->ref);
2887 tr->ref--;
2888
3928a8a2
SR
2889 for_each_tracing_cpu(cpu) {
2890 if (iter->buffer_iter[cpu])
2891 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2892 }
2893
bc0c38d1
SR
2894 if (iter->trace && iter->trace->close)
2895 iter->trace->close(iter);
2896
debdd57f
HT
2897 if (!iter->snapshot)
2898 /* reenable tracing if it was previously enabled */
2b6080f2 2899 tracing_start_tr(tr);
bc0c38d1
SR
2900 mutex_unlock(&trace_types_lock);
2901
d7350c3f 2902 mutex_destroy(&iter->mutex);
b0dfa978 2903 free_cpumask_var(iter->started);
d7350c3f 2904 kfree(iter->trace);
6d158a81 2905 kfree(iter->buffer_iter);
50e18b94 2906 seq_release_private(inode, file);
bc0c38d1
SR
2907 return 0;
2908}
2909
2910static int tracing_open(struct inode *inode, struct file *file)
2911{
85a2f9b4
SR
2912 struct trace_iterator *iter;
2913 int ret = 0;
bc0c38d1 2914
4acd4d00
SR
2915 /* If this file was open for write, then erase contents */
2916 if ((file->f_mode & FMODE_WRITE) &&
8650ae32 2917 (file->f_flags & O_TRUNC)) {
2b6080f2
SR
2918 struct trace_cpu *tc = inode->i_private;
2919 struct trace_array *tr = tc->tr;
bc0c38d1 2920
2b6080f2 2921 if (tc->cpu == RING_BUFFER_ALL_CPUS)
12883efb 2922 tracing_reset_online_cpus(&tr->trace_buffer);
4acd4d00 2923 else
12883efb 2924 tracing_reset(&tr->trace_buffer, tc->cpu);
4acd4d00 2925 }
bc0c38d1 2926
4acd4d00 2927 if (file->f_mode & FMODE_READ) {
debdd57f 2928 iter = __tracing_open(inode, file, false);
4acd4d00
SR
2929 if (IS_ERR(iter))
2930 ret = PTR_ERR(iter);
2931 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2932 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2933 }
bc0c38d1
SR
2934 return ret;
2935}
2936
e309b41d 2937static void *
bc0c38d1
SR
2938t_next(struct seq_file *m, void *v, loff_t *pos)
2939{
f129e965 2940 struct tracer *t = v;
bc0c38d1
SR
2941
2942 (*pos)++;
2943
2944 if (t)
2945 t = t->next;
2946
bc0c38d1
SR
2947 return t;
2948}
2949
2950static void *t_start(struct seq_file *m, loff_t *pos)
2951{
f129e965 2952 struct tracer *t;
bc0c38d1
SR
2953 loff_t l = 0;
2954
2955 mutex_lock(&trace_types_lock);
f129e965 2956 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
bc0c38d1
SR
2957 ;
2958
2959 return t;
2960}
2961
2962static void t_stop(struct seq_file *m, void *p)
2963{
2964 mutex_unlock(&trace_types_lock);
2965}
2966
2967static int t_show(struct seq_file *m, void *v)
2968{
2969 struct tracer *t = v;
2970
2971 if (!t)
2972 return 0;
2973
2974 seq_printf(m, "%s", t->name);
2975 if (t->next)
2976 seq_putc(m, ' ');
2977 else
2978 seq_putc(m, '\n');
2979
2980 return 0;
2981}
2982
88e9d34c 2983static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
2984 .start = t_start,
2985 .next = t_next,
2986 .stop = t_stop,
2987 .show = t_show,
bc0c38d1
SR
2988};
2989
2990static int show_traces_open(struct inode *inode, struct file *file)
2991{
60a11774
SR
2992 if (tracing_disabled)
2993 return -ENODEV;
2994
f129e965 2995 return seq_open(file, &show_traces_seq_ops);
bc0c38d1
SR
2996}
2997
4acd4d00
SR
2998static ssize_t
2999tracing_write_stub(struct file *filp, const char __user *ubuf,
3000 size_t count, loff_t *ppos)
3001{
3002 return count;
3003}
3004
364829b1
SP
3005static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3006{
3007 if (file->f_mode & FMODE_READ)
3008 return seq_lseek(file, offset, origin);
3009 else
3010 return 0;
3011}
3012
5e2336a0 3013static const struct file_operations tracing_fops = {
4bf39a94
IM
3014 .open = tracing_open,
3015 .read = seq_read,
4acd4d00 3016 .write = tracing_write_stub,
364829b1 3017 .llseek = tracing_seek,
4bf39a94 3018 .release = tracing_release,
bc0c38d1
SR
3019};
3020
5e2336a0 3021static const struct file_operations show_traces_fops = {
c7078de1
IM
3022 .open = show_traces_open,
3023 .read = seq_read,
3024 .release = seq_release,
b444786f 3025 .llseek = seq_lseek,
c7078de1
IM
3026};
3027
36dfe925
IM
3028/*
3029 * Only trace on a CPU if the bitmask is set:
3030 */
9e01c1b7 3031static cpumask_var_t tracing_cpumask;
36dfe925
IM
3032
3033/*
3034 * The tracer itself will not take this lock, but still we want
3035 * to provide a consistent cpumask to user-space:
3036 */
3037static DEFINE_MUTEX(tracing_cpumask_update_lock);
3038
3039/*
3040 * Temporary storage for the character representation of the
3041 * CPU bitmask (and one more byte for the newline):
3042 */
3043static char mask_str[NR_CPUS + 1];
3044
c7078de1
IM
3045static ssize_t
3046tracing_cpumask_read(struct file *filp, char __user *ubuf,
3047 size_t count, loff_t *ppos)
3048{
36dfe925 3049 int len;
c7078de1
IM
3050
3051 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 3052
9e01c1b7 3053 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
36dfe925
IM
3054 if (count - len < 2) {
3055 count = -EINVAL;
3056 goto out_err;
3057 }
3058 len += sprintf(mask_str + len, "\n");
3059 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3060
3061out_err:
c7078de1
IM
3062 mutex_unlock(&tracing_cpumask_update_lock);
3063
3064 return count;
3065}
3066
3067static ssize_t
3068tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3069 size_t count, loff_t *ppos)
3070{
2b6080f2 3071 struct trace_array *tr = filp->private_data;
9e01c1b7 3072 cpumask_var_t tracing_cpumask_new;
2b6080f2 3073 int err, cpu;
9e01c1b7
RR
3074
3075 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3076 return -ENOMEM;
c7078de1 3077
9e01c1b7 3078 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 3079 if (err)
36dfe925
IM
3080 goto err_unlock;
3081
215368e8
LZ
3082 mutex_lock(&tracing_cpumask_update_lock);
3083
a5e25883 3084 local_irq_disable();
0199c4e6 3085 arch_spin_lock(&ftrace_max_lock);
ab46428c 3086 for_each_tracing_cpu(cpu) {
36dfe925
IM
3087 /*
3088 * Increase/decrease the disabled counter if we are
3089 * about to flip a bit in the cpumask:
3090 */
9e01c1b7
RR
3091 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3092 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3093 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3094 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 3095 }
9e01c1b7
RR
3096 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3097 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
3098 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3099 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
3100 }
3101 }
0199c4e6 3102 arch_spin_unlock(&ftrace_max_lock);
a5e25883 3103 local_irq_enable();
36dfe925 3104
9e01c1b7 3105 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
3106
3107 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 3108 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
3109
3110 return count;
36dfe925
IM
3111
3112err_unlock:
215368e8 3113 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
3114
3115 return err;
c7078de1
IM
3116}
3117
5e2336a0 3118static const struct file_operations tracing_cpumask_fops = {
c7078de1
IM
3119 .open = tracing_open_generic,
3120 .read = tracing_cpumask_read,
3121 .write = tracing_cpumask_write,
b444786f 3122 .llseek = generic_file_llseek,
bc0c38d1
SR
3123};
3124
fdb372ed 3125static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 3126{
d8e83d26 3127 struct tracer_opt *trace_opts;
2b6080f2 3128 struct trace_array *tr = m->private;
d8e83d26 3129 u32 tracer_flags;
d8e83d26 3130 int i;
adf9f195 3131
d8e83d26 3132 mutex_lock(&trace_types_lock);
2b6080f2
SR
3133 tracer_flags = tr->current_trace->flags->val;
3134 trace_opts = tr->current_trace->flags->opts;
d8e83d26 3135
bc0c38d1
SR
3136 for (i = 0; trace_options[i]; i++) {
3137 if (trace_flags & (1 << i))
fdb372ed 3138 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 3139 else
fdb372ed 3140 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
3141 }
3142
adf9f195
FW
3143 for (i = 0; trace_opts[i].name; i++) {
3144 if (tracer_flags & trace_opts[i].bit)
fdb372ed 3145 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 3146 else
fdb372ed 3147 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 3148 }
d8e83d26 3149 mutex_unlock(&trace_types_lock);
adf9f195 3150
fdb372ed 3151 return 0;
bc0c38d1 3152}
bc0c38d1 3153
8d18eaaf
LZ
3154static int __set_tracer_option(struct tracer *trace,
3155 struct tracer_flags *tracer_flags,
3156 struct tracer_opt *opts, int neg)
3157{
3158 int ret;
bc0c38d1 3159
8d18eaaf
LZ
3160 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3161 if (ret)
3162 return ret;
3163
3164 if (neg)
3165 tracer_flags->val &= ~opts->bit;
3166 else
3167 tracer_flags->val |= opts->bit;
3168 return 0;
bc0c38d1
SR
3169}
3170
adf9f195
FW
3171/* Try to assign a tracer specific option */
3172static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3173{
7770841e 3174 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 3175 struct tracer_opt *opts = NULL;
8d18eaaf 3176 int i;
adf9f195 3177
7770841e
Z
3178 for (i = 0; tracer_flags->opts[i].name; i++) {
3179 opts = &tracer_flags->opts[i];
adf9f195 3180
8d18eaaf
LZ
3181 if (strcmp(cmp, opts->name) == 0)
3182 return __set_tracer_option(trace, trace->flags,
3183 opts, neg);
adf9f195 3184 }
adf9f195 3185
8d18eaaf 3186 return -EINVAL;
adf9f195
FW
3187}
3188
613f04a0
SRRH
3189/* Some tracers require overwrite to stay enabled */
3190int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3191{
3192 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3193 return -1;
3194
3195 return 0;
3196}
3197
2b6080f2 3198int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
3199{
3200 /* do nothing if flag is already set */
3201 if (!!(trace_flags & mask) == !!enabled)
613f04a0
SRRH
3202 return 0;
3203
3204 /* Give the tracer a chance to approve the change */
2b6080f2
SR
3205 if (tr->current_trace->flag_changed)
3206 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
613f04a0 3207 return -EINVAL;
af4617bd
SR
3208
3209 if (enabled)
3210 trace_flags |= mask;
3211 else
3212 trace_flags &= ~mask;
e870e9a1
LZ
3213
3214 if (mask == TRACE_ITER_RECORD_CMD)
3215 trace_event_enable_cmd_record(enabled);
750912fa 3216
80902822 3217 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 3218 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 3219#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 3220 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
3221#endif
3222 }
81698831
SR
3223
3224 if (mask == TRACE_ITER_PRINTK)
3225 trace_printk_start_stop_comm(enabled);
613f04a0
SRRH
3226
3227 return 0;
af4617bd
SR
3228}
3229
2b6080f2 3230static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 3231{
8d18eaaf 3232 char *cmp;
bc0c38d1 3233 int neg = 0;
613f04a0 3234 int ret = -ENODEV;
bc0c38d1
SR
3235 int i;
3236
7bcfaf54 3237 cmp = strstrip(option);
bc0c38d1 3238
8d18eaaf 3239 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
3240 neg = 1;
3241 cmp += 2;
3242 }
3243
69d34da2
SRRH
3244 mutex_lock(&trace_types_lock);
3245
bc0c38d1 3246 for (i = 0; trace_options[i]; i++) {
8d18eaaf 3247 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 3248 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
3249 break;
3250 }
3251 }
adf9f195
FW
3252
3253 /* If no option could be set, test the specific tracer options */
69d34da2 3254 if (!trace_options[i])
2b6080f2 3255 ret = set_tracer_option(tr->current_trace, cmp, neg);
69d34da2
SRRH
3256
3257 mutex_unlock(&trace_types_lock);
bc0c38d1 3258
7bcfaf54
SR
3259 return ret;
3260}
3261
3262static ssize_t
3263tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3264 size_t cnt, loff_t *ppos)
3265{
2b6080f2
SR
3266 struct seq_file *m = filp->private_data;
3267 struct trace_array *tr = m->private;
7bcfaf54 3268 char buf[64];
613f04a0 3269 int ret;
7bcfaf54
SR
3270
3271 if (cnt >= sizeof(buf))
3272 return -EINVAL;
3273
3274 if (copy_from_user(&buf, ubuf, cnt))
3275 return -EFAULT;
3276
a8dd2176
SR
3277 buf[cnt] = 0;
3278
2b6080f2 3279 ret = trace_set_options(tr, buf);
613f04a0
SRRH
3280 if (ret < 0)
3281 return ret;
7bcfaf54 3282
cf8517cf 3283 *ppos += cnt;
bc0c38d1
SR
3284
3285 return cnt;
3286}
3287
fdb372ed
LZ
3288static int tracing_trace_options_open(struct inode *inode, struct file *file)
3289{
3290 if (tracing_disabled)
3291 return -ENODEV;
2b6080f2
SR
3292
3293 return single_open(file, tracing_trace_options_show, inode->i_private);
fdb372ed
LZ
3294}
3295
5e2336a0 3296static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
3297 .open = tracing_trace_options_open,
3298 .read = seq_read,
3299 .llseek = seq_lseek,
3300 .release = single_release,
ee6bce52 3301 .write = tracing_trace_options_write,
bc0c38d1
SR
3302};
3303
7bd2f24c
IM
3304static const char readme_msg[] =
3305 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
3306 "# echo 0 > tracing_on : quick way to disable tracing\n"
3307 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3308 " Important files:\n"
3309 " trace\t\t\t- The static contents of the buffer\n"
3310 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3311 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3312 " current_tracer\t- function and latency tracers\n"
3313 " available_tracers\t- list of configured tracers for current_tracer\n"
3314 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3315 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3316 " trace_clock\t\t-change the clock used to order events\n"
3317 " local: Per cpu clock but may not be synced across CPUs\n"
3318 " global: Synced across CPUs but slows tracing down.\n"
3319 " counter: Not a clock, but just an increment\n"
3320 " uptime: Jiffy counter from time of boot\n"
3321 " perf: Same clock that perf events use\n"
3322#ifdef CONFIG_X86_64
3323 " x86-tsc: TSC cycle counter\n"
3324#endif
3325 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3326 " tracing_cpumask\t- Limit which CPUs to trace\n"
3327 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3328 "\t\t\t Remove sub-buffer with rmdir\n"
3329 " trace_options\t\t- Set format or modify how tracing happens\n"
3330 "\t\t\t Disable an option by adding a suffix 'no' to the option name\n"
3331#ifdef CONFIG_DYNAMIC_FTRACE
3332 "\n available_filter_functions - list of functions that can be filtered on\n"
3333 " set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3334 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3335 " modules: Can select a group via module\n"
3336 " Format: :mod:<module-name>\n"
3337 " example: echo :mod:ext3 > set_ftrace_filter\n"
3338 " triggers: a command to perform when function is hit\n"
3339 " Format: <function>:<trigger>[:count]\n"
3340 " trigger: traceon, traceoff\n"
3341 " enable_event:<system>:<event>\n"
3342 " disable_event:<system>:<event>\n"
3343#ifdef CONFIG_STACKTRACE
3344 " stacktrace\n"
3345#endif
3346#ifdef CONFIG_TRACER_SNAPSHOT
3347 " snapshot\n"
3348#endif
3349 " example: echo do_fault:traceoff > set_ftrace_filter\n"
3350 " echo do_trap:traceoff:3 > set_ftrace_filter\n"
3351 " The first one will disable tracing every time do_fault is hit\n"
3352 " The second will disable tracing at most 3 times when do_trap is hit\n"
3353 " The first time do trap is hit and it disables tracing, the counter\n"
3354 " will decrement to 2. If tracing is already disabled, the counter\n"
3355 " will not decrement. It only decrements when the trigger did work\n"
3356 " To remove trigger without count:\n"
3357 " echo '!<function>:<trigger> > set_ftrace_filter\n"
3358 " To remove trigger with a count:\n"
3359 " echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3360 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3361 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3362 " modules: Can select a group via module command :mod:\n"
3363 " Does not accept triggers\n"
3364#endif /* CONFIG_DYNAMIC_FTRACE */
3365#ifdef CONFIG_FUNCTION_TRACER
3366 " set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3367#endif
3368#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3369 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3370 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3371#endif
3372#ifdef CONFIG_TRACER_SNAPSHOT
3373 "\n snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3374 "\t\t\t Read the contents for more information\n"
3375#endif
3376#ifdef CONFIG_STACKTRACE
3377 " stack_trace\t\t- Shows the max stack trace when active\n"
3378 " stack_max_size\t- Shows current max stack size that was traced\n"
3379 "\t\t\t Write into this file to reset the max size (trigger a new trace)\n"
3380#ifdef CONFIG_DYNAMIC_FTRACE
3381 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3382#endif
3383#endif /* CONFIG_STACKTRACE */
7bd2f24c
IM
3384;
3385
3386static ssize_t
3387tracing_readme_read(struct file *filp, char __user *ubuf,
3388 size_t cnt, loff_t *ppos)
3389{
3390 return simple_read_from_buffer(ubuf, cnt, ppos,
3391 readme_msg, strlen(readme_msg));
3392}
3393
5e2336a0 3394static const struct file_operations tracing_readme_fops = {
c7078de1
IM
3395 .open = tracing_open_generic,
3396 .read = tracing_readme_read,
b444786f 3397 .llseek = generic_file_llseek,
7bd2f24c
IM
3398};
3399
69abe6a5
AP
3400static ssize_t
3401tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3402 size_t cnt, loff_t *ppos)
3403{
3404 char *buf_comm;
3405 char *file_buf;
3406 char *buf;
3407 int len = 0;
3408 int pid;
3409 int i;
3410
3411 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3412 if (!file_buf)
3413 return -ENOMEM;
3414
3415 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3416 if (!buf_comm) {
3417 kfree(file_buf);
3418 return -ENOMEM;
3419 }
3420
3421 buf = file_buf;
3422
3423 for (i = 0; i < SAVED_CMDLINES; i++) {
3424 int r;
3425
3426 pid = map_cmdline_to_pid[i];
3427 if (pid == -1 || pid == NO_CMDLINE_MAP)
3428 continue;
3429
3430 trace_find_cmdline(pid, buf_comm);
3431 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3432 buf += r;
3433 len += r;
3434 }
3435
3436 len = simple_read_from_buffer(ubuf, cnt, ppos,
3437 file_buf, len);
3438
3439 kfree(file_buf);
3440 kfree(buf_comm);
3441
3442 return len;
3443}
3444
3445static const struct file_operations tracing_saved_cmdlines_fops = {
3446 .open = tracing_open_generic,
3447 .read = tracing_saved_cmdlines_read,
b444786f 3448 .llseek = generic_file_llseek,
69abe6a5
AP
3449};
3450
bc0c38d1
SR
3451static ssize_t
3452tracing_set_trace_read(struct file *filp, char __user *ubuf,
3453 size_t cnt, loff_t *ppos)
3454{
2b6080f2 3455 struct trace_array *tr = filp->private_data;
ee6c2c1b 3456 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
3457 int r;
3458
3459 mutex_lock(&trace_types_lock);
2b6080f2 3460 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
3461 mutex_unlock(&trace_types_lock);
3462
4bf39a94 3463 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3464}
3465
b6f11df2
ACM
3466int tracer_init(struct tracer *t, struct trace_array *tr)
3467{
12883efb 3468 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
3469 return t->init(tr);
3470}
3471
12883efb 3472static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
3473{
3474 int cpu;
737223fb 3475
438ced17 3476 for_each_tracing_cpu(cpu)
12883efb 3477 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
3478}
3479
12883efb 3480#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 3481/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
3482static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3483 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
3484{
3485 int cpu, ret = 0;
3486
3487 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3488 for_each_tracing_cpu(cpu) {
12883efb
SRRH
3489 ret = ring_buffer_resize(trace_buf->buffer,
3490 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
3491 if (ret < 0)
3492 break;
12883efb
SRRH
3493 per_cpu_ptr(trace_buf->data, cpu)->entries =
3494 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
3495 }
3496 } else {
12883efb
SRRH
3497 ret = ring_buffer_resize(trace_buf->buffer,
3498 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 3499 if (ret == 0)
12883efb
SRRH
3500 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3501 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
3502 }
3503
3504 return ret;
3505}
12883efb 3506#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 3507
2b6080f2
SR
3508static int __tracing_resize_ring_buffer(struct trace_array *tr,
3509 unsigned long size, int cpu)
73c5162a
SR
3510{
3511 int ret;
3512
3513 /*
3514 * If kernel or user changes the size of the ring buffer
a123c52b
SR
3515 * we use the size that was given, and we can forget about
3516 * expanding it later.
73c5162a 3517 */
55034cd6 3518 ring_buffer_expanded = true;
73c5162a 3519
b382ede6 3520 /* May be called before buffers are initialized */
12883efb 3521 if (!tr->trace_buffer.buffer)
b382ede6
SR
3522 return 0;
3523
12883efb 3524 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
3525 if (ret < 0)
3526 return ret;
3527
12883efb 3528#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3529 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3530 !tr->current_trace->use_max_tr)
ef710e10
KM
3531 goto out;
3532
12883efb 3533 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 3534 if (ret < 0) {
12883efb
SRRH
3535 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3536 &tr->trace_buffer, cpu);
73c5162a 3537 if (r < 0) {
a123c52b
SR
3538 /*
3539 * AARGH! We are left with different
3540 * size max buffer!!!!
3541 * The max buffer is our "snapshot" buffer.
3542 * When a tracer needs a snapshot (one of the
3543 * latency tracers), it swaps the max buffer
3544 * with the saved snap shot. We succeeded to
3545 * update the size of the main buffer, but failed to
3546 * update the size of the max buffer. But when we tried
3547 * to reset the main buffer to the original size, we
3548 * failed there too. This is very unlikely to
3549 * happen, but if it does, warn and kill all
3550 * tracing.
3551 */
73c5162a
SR
3552 WARN_ON(1);
3553 tracing_disabled = 1;
3554 }
3555 return ret;
3556 }
3557
438ced17 3558 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3559 set_buffer_entries(&tr->max_buffer, size);
438ced17 3560 else
12883efb 3561 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 3562
ef710e10 3563 out:
12883efb
SRRH
3564#endif /* CONFIG_TRACER_MAX_TRACE */
3565
438ced17 3566 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3567 set_buffer_entries(&tr->trace_buffer, size);
438ced17 3568 else
12883efb 3569 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
3570
3571 return ret;
3572}
3573
2b6080f2
SR
3574static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3575 unsigned long size, int cpu_id)
4f271a2a 3576{
83f40318 3577 int ret = size;
4f271a2a
VN
3578
3579 mutex_lock(&trace_types_lock);
3580
438ced17
VN
3581 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3582 /* make sure, this cpu is enabled in the mask */
3583 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3584 ret = -EINVAL;
3585 goto out;
3586 }
3587 }
4f271a2a 3588
2b6080f2 3589 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
3590 if (ret < 0)
3591 ret = -ENOMEM;
3592
438ced17 3593out:
4f271a2a
VN
3594 mutex_unlock(&trace_types_lock);
3595
3596 return ret;
3597}
3598
ef710e10 3599
1852fcce
SR
3600/**
3601 * tracing_update_buffers - used by tracing facility to expand ring buffers
3602 *
3603 * To save on memory when the tracing is never used on a system with it
3604 * configured in. The ring buffers are set to a minimum size. But once
3605 * a user starts to use the tracing facility, then they need to grow
3606 * to their default size.
3607 *
3608 * This function is to be called when a tracer is about to be used.
3609 */
3610int tracing_update_buffers(void)
3611{
3612 int ret = 0;
3613
1027fcb2 3614 mutex_lock(&trace_types_lock);
1852fcce 3615 if (!ring_buffer_expanded)
2b6080f2 3616 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 3617 RING_BUFFER_ALL_CPUS);
1027fcb2 3618 mutex_unlock(&trace_types_lock);
1852fcce
SR
3619
3620 return ret;
3621}
3622
577b785f
SR
3623struct trace_option_dentry;
3624
3625static struct trace_option_dentry *
2b6080f2 3626create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f
SR
3627
3628static void
3629destroy_trace_option_files(struct trace_option_dentry *topts);
3630
b2821ae6 3631static int tracing_set_tracer(const char *buf)
bc0c38d1 3632{
577b785f 3633 static struct trace_option_dentry *topts;
bc0c38d1
SR
3634 struct trace_array *tr = &global_trace;
3635 struct tracer *t;
12883efb 3636#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3637 bool had_max_tr;
12883efb 3638#endif
d9e54076 3639 int ret = 0;
bc0c38d1 3640
1027fcb2
SR
3641 mutex_lock(&trace_types_lock);
3642
73c5162a 3643 if (!ring_buffer_expanded) {
2b6080f2 3644 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 3645 RING_BUFFER_ALL_CPUS);
73c5162a 3646 if (ret < 0)
59f586db 3647 goto out;
73c5162a
SR
3648 ret = 0;
3649 }
3650
bc0c38d1
SR
3651 for (t = trace_types; t; t = t->next) {
3652 if (strcmp(t->name, buf) == 0)
3653 break;
3654 }
c2931e05
FW
3655 if (!t) {
3656 ret = -EINVAL;
3657 goto out;
3658 }
2b6080f2 3659 if (t == tr->current_trace)
bc0c38d1
SR
3660 goto out;
3661
9f029e83 3662 trace_branch_disable();
613f04a0 3663
2b6080f2 3664 tr->current_trace->enabled = false;
613f04a0 3665
2b6080f2
SR
3666 if (tr->current_trace->reset)
3667 tr->current_trace->reset(tr);
34600f0e 3668
12883efb 3669 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 3670 tr->current_trace = &nop_trace;
34600f0e 3671
45ad21ca
SRRH
3672#ifdef CONFIG_TRACER_MAX_TRACE
3673 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
3674
3675 if (had_max_tr && !t->use_max_tr) {
3676 /*
3677 * We need to make sure that the update_max_tr sees that
3678 * current_trace changed to nop_trace to keep it from
3679 * swapping the buffers after we resize it.
3680 * The update_max_tr is called from interrupts disabled
3681 * so a synchronized_sched() is sufficient.
3682 */
3683 synchronize_sched();
3209cff4 3684 free_snapshot(tr);
ef710e10 3685 }
12883efb 3686#endif
577b785f
SR
3687 destroy_trace_option_files(topts);
3688
2b6080f2 3689 topts = create_trace_option_files(tr, t);
12883efb
SRRH
3690
3691#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 3692 if (t->use_max_tr && !had_max_tr) {
3209cff4 3693 ret = alloc_snapshot(tr);
d60da506
HT
3694 if (ret < 0)
3695 goto out;
ef710e10 3696 }
12883efb 3697#endif
577b785f 3698
1c80025a 3699 if (t->init) {
b6f11df2 3700 ret = tracer_init(t, tr);
1c80025a
FW
3701 if (ret)
3702 goto out;
3703 }
bc0c38d1 3704
2b6080f2
SR
3705 tr->current_trace = t;
3706 tr->current_trace->enabled = true;
9f029e83 3707 trace_branch_enable(tr);
bc0c38d1
SR
3708 out:
3709 mutex_unlock(&trace_types_lock);
3710
d9e54076
PZ
3711 return ret;
3712}
3713
3714static ssize_t
3715tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3716 size_t cnt, loff_t *ppos)
3717{
ee6c2c1b 3718 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
3719 int i;
3720 size_t ret;
e6e7a65a
FW
3721 int err;
3722
3723 ret = cnt;
d9e54076 3724
ee6c2c1b
LZ
3725 if (cnt > MAX_TRACER_SIZE)
3726 cnt = MAX_TRACER_SIZE;
d9e54076
PZ
3727
3728 if (copy_from_user(&buf, ubuf, cnt))
3729 return -EFAULT;
3730
3731 buf[cnt] = 0;
3732
3733 /* strip ending whitespace. */
3734 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3735 buf[i] = 0;
3736
e6e7a65a
FW
3737 err = tracing_set_tracer(buf);
3738 if (err)
3739 return err;
d9e54076 3740
cf8517cf 3741 *ppos += ret;
bc0c38d1 3742
c2931e05 3743 return ret;
bc0c38d1
SR
3744}
3745
3746static ssize_t
3747tracing_max_lat_read(struct file *filp, char __user *ubuf,
3748 size_t cnt, loff_t *ppos)
3749{
3750 unsigned long *ptr = filp->private_data;
3751 char buf[64];
3752 int r;
3753
cffae437 3754 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 3755 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
3756 if (r > sizeof(buf))
3757 r = sizeof(buf);
4bf39a94 3758 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
3759}
3760
3761static ssize_t
3762tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3763 size_t cnt, loff_t *ppos)
3764{
5e39841c 3765 unsigned long *ptr = filp->private_data;
5e39841c 3766 unsigned long val;
c6caeeb1 3767 int ret;
bc0c38d1 3768
22fe9b54
PH
3769 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3770 if (ret)
c6caeeb1 3771 return ret;
bc0c38d1
SR
3772
3773 *ptr = val * 1000;
3774
3775 return cnt;
3776}
3777
b3806b43
SR
3778static int tracing_open_pipe(struct inode *inode, struct file *filp)
3779{
2b6080f2
SR
3780 struct trace_cpu *tc = inode->i_private;
3781 struct trace_array *tr = tc->tr;
b3806b43 3782 struct trace_iterator *iter;
b04cc6b1 3783 int ret = 0;
b3806b43
SR
3784
3785 if (tracing_disabled)
3786 return -ENODEV;
3787
b04cc6b1
FW
3788 mutex_lock(&trace_types_lock);
3789
b3806b43
SR
3790 /* create a buffer to store the information to pass to userspace */
3791 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
3792 if (!iter) {
3793 ret = -ENOMEM;
3794 goto out;
3795 }
b3806b43 3796
d7350c3f
FW
3797 /*
3798 * We make a copy of the current tracer to avoid concurrent
3799 * changes on it while we are reading.
3800 */
3801 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3802 if (!iter->trace) {
3803 ret = -ENOMEM;
3804 goto fail;
3805 }
2b6080f2 3806 *iter->trace = *tr->current_trace;
d7350c3f 3807
4462344e 3808 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 3809 ret = -ENOMEM;
d7350c3f 3810 goto fail;
4462344e
RR
3811 }
3812
a309720c 3813 /* trace pipe does not show start of buffer */
4462344e 3814 cpumask_setall(iter->started);
a309720c 3815
112f38a7
SR
3816 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3817 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3818
8be0709f
DS
3819 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3820 if (trace_clocks[trace_clock_id].in_ns)
3821 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3822
2b6080f2
SR
3823 iter->cpu_file = tc->cpu;
3824 iter->tr = tc->tr;
12883efb 3825 iter->trace_buffer = &tc->tr->trace_buffer;
d7350c3f 3826 mutex_init(&iter->mutex);
b3806b43
SR
3827 filp->private_data = iter;
3828
107bad8b
SR
3829 if (iter->trace->pipe_open)
3830 iter->trace->pipe_open(iter);
107bad8b 3831
b444786f 3832 nonseekable_open(inode, filp);
b04cc6b1
FW
3833out:
3834 mutex_unlock(&trace_types_lock);
3835 return ret;
d7350c3f
FW
3836
3837fail:
3838 kfree(iter->trace);
3839 kfree(iter);
3840 mutex_unlock(&trace_types_lock);
3841 return ret;
b3806b43
SR
3842}
3843
3844static int tracing_release_pipe(struct inode *inode, struct file *file)
3845{
3846 struct trace_iterator *iter = file->private_data;
3847
b04cc6b1
FW
3848 mutex_lock(&trace_types_lock);
3849
29bf4a5e 3850 if (iter->trace->pipe_close)
c521efd1
SR
3851 iter->trace->pipe_close(iter);
3852
b04cc6b1
FW
3853 mutex_unlock(&trace_types_lock);
3854
4462344e 3855 free_cpumask_var(iter->started);
d7350c3f
FW
3856 mutex_destroy(&iter->mutex);
3857 kfree(iter->trace);
b3806b43 3858 kfree(iter);
b3806b43
SR
3859
3860 return 0;
3861}
3862
2a2cc8f7 3863static unsigned int
cc60cdc9 3864trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 3865{
15693458
SRRH
3866 /* Iterators are static, they should be filled or empty */
3867 if (trace_buffer_iter(iter, iter->cpu_file))
3868 return POLLIN | POLLRDNORM;
2a2cc8f7 3869
15693458 3870 if (trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
3871 /*
3872 * Always select as readable when in blocking mode
3873 */
3874 return POLLIN | POLLRDNORM;
15693458 3875 else
12883efb 3876 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 3877 filp, poll_table);
2a2cc8f7 3878}
2a2cc8f7 3879
cc60cdc9
SR
3880static unsigned int
3881tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3882{
3883 struct trace_iterator *iter = filp->private_data;
3884
3885 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
3886}
3887
6eaaa5d5
FW
3888/*
3889 * This is a make-shift waitqueue.
3890 * A tracer might use this callback on some rare cases:
3891 *
3892 * 1) the current tracer might hold the runqueue lock when it wakes up
3893 * a reader, hence a deadlock (sched, function, and function graph tracers)
3894 * 2) the function tracers, trace all functions, we don't want
3895 * the overhead of calling wake_up and friends
3896 * (and tracing them too)
3897 *
3898 * Anyway, this is really very primitive wakeup.
3899 */
3900void poll_wait_pipe(struct trace_iterator *iter)
3901{
3902 set_current_state(TASK_INTERRUPTIBLE);
3903 /* sleep for 100 msecs, and try again. */
3904 schedule_timeout(HZ / 10);
3905}
3906
ff98781b
EGM
3907/* Must be called with trace_types_lock mutex held. */
3908static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
3909{
3910 struct trace_iterator *iter = filp->private_data;
b3806b43 3911
b3806b43 3912 while (trace_empty(iter)) {
2dc8f095 3913
107bad8b 3914 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 3915 return -EAGAIN;
107bad8b 3916 }
2dc8f095 3917
d7350c3f 3918 mutex_unlock(&iter->mutex);
107bad8b 3919
6eaaa5d5 3920 iter->trace->wait_pipe(iter);
b3806b43 3921
d7350c3f 3922 mutex_lock(&iter->mutex);
107bad8b 3923
6eaaa5d5 3924 if (signal_pending(current))
ff98781b 3925 return -EINTR;
b3806b43
SR
3926
3927 /*
250bfd3d 3928 * We block until we read something and tracing is disabled.
b3806b43
SR
3929 * We still block if tracing is disabled, but we have never
3930 * read anything. This allows a user to cat this file, and
3931 * then enable tracing. But after we have read something,
3932 * we give an EOF when tracing is again disabled.
3933 *
3934 * iter->pos will be 0 if we haven't read anything.
3935 */
250bfd3d 3936 if (!tracing_is_enabled() && iter->pos)
b3806b43 3937 break;
b3806b43
SR
3938 }
3939
ff98781b
EGM
3940 return 1;
3941}
3942
3943/*
3944 * Consumer reader.
3945 */
3946static ssize_t
3947tracing_read_pipe(struct file *filp, char __user *ubuf,
3948 size_t cnt, loff_t *ppos)
3949{
3950 struct trace_iterator *iter = filp->private_data;
2b6080f2 3951 struct trace_array *tr = iter->tr;
ff98781b
EGM
3952 ssize_t sret;
3953
3954 /* return any leftover data */
3955 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3956 if (sret != -EBUSY)
3957 return sret;
3958
f9520750 3959 trace_seq_init(&iter->seq);
ff98781b 3960
d7350c3f 3961 /* copy the tracer to avoid using a global lock all around */
ff98781b 3962 mutex_lock(&trace_types_lock);
2b6080f2
SR
3963 if (unlikely(iter->trace->name != tr->current_trace->name))
3964 *iter->trace = *tr->current_trace;
d7350c3f
FW
3965 mutex_unlock(&trace_types_lock);
3966
3967 /*
3968 * Avoid more than one consumer on a single file descriptor
3969 * This is just a matter of traces coherency, the ring buffer itself
3970 * is protected.
3971 */
3972 mutex_lock(&iter->mutex);
ff98781b
EGM
3973 if (iter->trace->read) {
3974 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3975 if (sret)
3976 goto out;
3977 }
3978
3979waitagain:
3980 sret = tracing_wait_pipe(filp);
3981 if (sret <= 0)
3982 goto out;
3983
b3806b43 3984 /* stop when tracing is finished */
ff98781b
EGM
3985 if (trace_empty(iter)) {
3986 sret = 0;
107bad8b 3987 goto out;
ff98781b 3988 }
b3806b43
SR
3989
3990 if (cnt >= PAGE_SIZE)
3991 cnt = PAGE_SIZE - 1;
3992
53d0aa77 3993 /* reset all but tr, trace, and overruns */
53d0aa77
SR
3994 memset(&iter->seq, 0,
3995 sizeof(struct trace_iterator) -
3996 offsetof(struct trace_iterator, seq));
4823ed7e 3997 iter->pos = -1;
b3806b43 3998
4f535968 3999 trace_event_read_lock();
7e53bd42 4000 trace_access_lock(iter->cpu_file);
955b61e5 4001 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 4002 enum print_line_t ret;
088b1e42
SR
4003 int len = iter->seq.len;
4004
f9896bf3 4005 ret = print_trace_line(iter);
2c4f035f 4006 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42
SR
4007 /* don't print partial lines */
4008 iter->seq.len = len;
b3806b43 4009 break;
088b1e42 4010 }
b91facc3
FW
4011 if (ret != TRACE_TYPE_NO_CONSUME)
4012 trace_consume(iter);
b3806b43
SR
4013
4014 if (iter->seq.len >= cnt)
4015 break;
ee5e51f5
JO
4016
4017 /*
4018 * Setting the full flag means we reached the trace_seq buffer
4019 * size and we should leave by partial output condition above.
4020 * One of the trace_seq_* functions is not used properly.
4021 */
4022 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4023 iter->ent->type);
b3806b43 4024 }
7e53bd42 4025 trace_access_unlock(iter->cpu_file);
4f535968 4026 trace_event_read_unlock();
b3806b43 4027
b3806b43 4028 /* Now copy what we have to the user */
6c6c2796
PP
4029 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4030 if (iter->seq.readpos >= iter->seq.len)
f9520750 4031 trace_seq_init(&iter->seq);
9ff4b974
PP
4032
4033 /*
25985edc 4034 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
4035 * entries, go back to wait for more entries.
4036 */
6c6c2796 4037 if (sret == -EBUSY)
9ff4b974 4038 goto waitagain;
b3806b43 4039
107bad8b 4040out:
d7350c3f 4041 mutex_unlock(&iter->mutex);
107bad8b 4042
6c6c2796 4043 return sret;
b3806b43
SR
4044}
4045
3c56819b
EGM
4046static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4047 struct pipe_buffer *buf)
4048{
4049 __free_page(buf->page);
4050}
4051
4052static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4053 unsigned int idx)
4054{
4055 __free_page(spd->pages[idx]);
4056}
4057
28dfef8f 4058static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998
SR
4059 .can_merge = 0,
4060 .map = generic_pipe_buf_map,
4061 .unmap = generic_pipe_buf_unmap,
4062 .confirm = generic_pipe_buf_confirm,
4063 .release = tracing_pipe_buf_release,
4064 .steal = generic_pipe_buf_steal,
4065 .get = generic_pipe_buf_get,
3c56819b
EGM
4066};
4067
34cd4998 4068static size_t
fa7c7f6e 4069tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
4070{
4071 size_t count;
4072 int ret;
4073
4074 /* Seq buffer is page-sized, exactly what we need. */
4075 for (;;) {
4076 count = iter->seq.len;
4077 ret = print_trace_line(iter);
4078 count = iter->seq.len - count;
4079 if (rem < count) {
4080 rem = 0;
4081 iter->seq.len -= count;
4082 break;
4083 }
4084 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4085 iter->seq.len -= count;
4086 break;
4087 }
4088
74e7ff8c
LJ
4089 if (ret != TRACE_TYPE_NO_CONSUME)
4090 trace_consume(iter);
34cd4998 4091 rem -= count;
955b61e5 4092 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
4093 rem = 0;
4094 iter->ent = NULL;
4095 break;
4096 }
4097 }
4098
4099 return rem;
4100}
4101
3c56819b
EGM
4102static ssize_t tracing_splice_read_pipe(struct file *filp,
4103 loff_t *ppos,
4104 struct pipe_inode_info *pipe,
4105 size_t len,
4106 unsigned int flags)
4107{
35f3d14d
JA
4108 struct page *pages_def[PIPE_DEF_BUFFERS];
4109 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
4110 struct trace_iterator *iter = filp->private_data;
4111 struct splice_pipe_desc spd = {
35f3d14d
JA
4112 .pages = pages_def,
4113 .partial = partial_def,
34cd4998 4114 .nr_pages = 0, /* This gets updated below. */
047fe360 4115 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
4116 .flags = flags,
4117 .ops = &tracing_pipe_buf_ops,
4118 .spd_release = tracing_spd_release_pipe,
3c56819b 4119 };
2b6080f2 4120 struct trace_array *tr = iter->tr;
3c56819b 4121 ssize_t ret;
34cd4998 4122 size_t rem;
3c56819b
EGM
4123 unsigned int i;
4124
35f3d14d
JA
4125 if (splice_grow_spd(pipe, &spd))
4126 return -ENOMEM;
4127
d7350c3f 4128 /* copy the tracer to avoid using a global lock all around */
3c56819b 4129 mutex_lock(&trace_types_lock);
2b6080f2
SR
4130 if (unlikely(iter->trace->name != tr->current_trace->name))
4131 *iter->trace = *tr->current_trace;
d7350c3f
FW
4132 mutex_unlock(&trace_types_lock);
4133
4134 mutex_lock(&iter->mutex);
3c56819b
EGM
4135
4136 if (iter->trace->splice_read) {
4137 ret = iter->trace->splice_read(iter, filp,
4138 ppos, pipe, len, flags);
4139 if (ret)
34cd4998 4140 goto out_err;
3c56819b
EGM
4141 }
4142
4143 ret = tracing_wait_pipe(filp);
4144 if (ret <= 0)
34cd4998 4145 goto out_err;
3c56819b 4146
955b61e5 4147 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 4148 ret = -EFAULT;
34cd4998 4149 goto out_err;
3c56819b
EGM
4150 }
4151
4f535968 4152 trace_event_read_lock();
7e53bd42 4153 trace_access_lock(iter->cpu_file);
4f535968 4154
3c56819b 4155 /* Fill as many pages as possible. */
35f3d14d
JA
4156 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4157 spd.pages[i] = alloc_page(GFP_KERNEL);
4158 if (!spd.pages[i])
34cd4998 4159 break;
3c56819b 4160
fa7c7f6e 4161 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
4162
4163 /* Copy the data into the page, so we can start over. */
4164 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 4165 page_address(spd.pages[i]),
3c56819b
EGM
4166 iter->seq.len);
4167 if (ret < 0) {
35f3d14d 4168 __free_page(spd.pages[i]);
3c56819b
EGM
4169 break;
4170 }
35f3d14d
JA
4171 spd.partial[i].offset = 0;
4172 spd.partial[i].len = iter->seq.len;
3c56819b 4173
f9520750 4174 trace_seq_init(&iter->seq);
3c56819b
EGM
4175 }
4176
7e53bd42 4177 trace_access_unlock(iter->cpu_file);
4f535968 4178 trace_event_read_unlock();
d7350c3f 4179 mutex_unlock(&iter->mutex);
3c56819b
EGM
4180
4181 spd.nr_pages = i;
4182
35f3d14d
JA
4183 ret = splice_to_pipe(pipe, &spd);
4184out:
047fe360 4185 splice_shrink_spd(&spd);
35f3d14d 4186 return ret;
3c56819b 4187
34cd4998 4188out_err:
d7350c3f 4189 mutex_unlock(&iter->mutex);
35f3d14d 4190 goto out;
3c56819b
EGM
4191}
4192
a98a3c3f
SR
4193static ssize_t
4194tracing_entries_read(struct file *filp, char __user *ubuf,
4195 size_t cnt, loff_t *ppos)
4196{
2b6080f2
SR
4197 struct trace_cpu *tc = filp->private_data;
4198 struct trace_array *tr = tc->tr;
438ced17
VN
4199 char buf[64];
4200 int r = 0;
4201 ssize_t ret;
a98a3c3f 4202
db526ca3 4203 mutex_lock(&trace_types_lock);
438ced17 4204
2b6080f2 4205 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
4206 int cpu, buf_size_same;
4207 unsigned long size;
4208
4209 size = 0;
4210 buf_size_same = 1;
4211 /* check if all cpu sizes are same */
4212 for_each_tracing_cpu(cpu) {
4213 /* fill in the size from first enabled cpu */
4214 if (size == 0)
12883efb
SRRH
4215 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4216 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
4217 buf_size_same = 0;
4218 break;
4219 }
4220 }
4221
4222 if (buf_size_same) {
4223 if (!ring_buffer_expanded)
4224 r = sprintf(buf, "%lu (expanded: %lu)\n",
4225 size >> 10,
4226 trace_buf_size >> 10);
4227 else
4228 r = sprintf(buf, "%lu\n", size >> 10);
4229 } else
4230 r = sprintf(buf, "X\n");
4231 } else
12883efb 4232 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
438ced17 4233
db526ca3
SR
4234 mutex_unlock(&trace_types_lock);
4235
438ced17
VN
4236 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4237 return ret;
a98a3c3f
SR
4238}
4239
4240static ssize_t
4241tracing_entries_write(struct file *filp, const char __user *ubuf,
4242 size_t cnt, loff_t *ppos)
4243{
2b6080f2 4244 struct trace_cpu *tc = filp->private_data;
a98a3c3f 4245 unsigned long val;
4f271a2a 4246 int ret;
a98a3c3f 4247
22fe9b54
PH
4248 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4249 if (ret)
c6caeeb1 4250 return ret;
a98a3c3f
SR
4251
4252 /* must have at least 1 entry */
4253 if (!val)
4254 return -EINVAL;
4255
1696b2b0
SR
4256 /* value is in KB */
4257 val <<= 10;
4258
2b6080f2 4259 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4f271a2a
VN
4260 if (ret < 0)
4261 return ret;
a98a3c3f 4262
cf8517cf 4263 *ppos += cnt;
a98a3c3f 4264
4f271a2a
VN
4265 return cnt;
4266}
bf5e6519 4267
f81ab074
VN
4268static ssize_t
4269tracing_total_entries_read(struct file *filp, char __user *ubuf,
4270 size_t cnt, loff_t *ppos)
4271{
4272 struct trace_array *tr = filp->private_data;
4273 char buf[64];
4274 int r, cpu;
4275 unsigned long size = 0, expanded_size = 0;
4276
4277 mutex_lock(&trace_types_lock);
4278 for_each_tracing_cpu(cpu) {
12883efb 4279 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
4280 if (!ring_buffer_expanded)
4281 expanded_size += trace_buf_size >> 10;
4282 }
4283 if (ring_buffer_expanded)
4284 r = sprintf(buf, "%lu\n", size);
4285 else
4286 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4287 mutex_unlock(&trace_types_lock);
4288
4289 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4290}
4291
4f271a2a
VN
4292static ssize_t
4293tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4294 size_t cnt, loff_t *ppos)
4295{
4296 /*
4297 * There is no need to read what the user has written, this function
4298 * is just to make sure that there is no error when "echo" is used
4299 */
4300
4301 *ppos += cnt;
a98a3c3f
SR
4302
4303 return cnt;
4304}
4305
4f271a2a
VN
4306static int
4307tracing_free_buffer_release(struct inode *inode, struct file *filp)
4308{
2b6080f2
SR
4309 struct trace_array *tr = inode->i_private;
4310
cf30cf67
SR
4311 /* disable tracing ? */
4312 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4313 tracing_off();
4f271a2a 4314 /* resize the ring buffer to 0 */
2b6080f2 4315 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a
VN
4316
4317 return 0;
4318}
4319
5bf9a1ee
PP
4320static ssize_t
4321tracing_mark_write(struct file *filp, const char __user *ubuf,
4322 size_t cnt, loff_t *fpos)
4323{
d696b58c
SR
4324 unsigned long addr = (unsigned long)ubuf;
4325 struct ring_buffer_event *event;
4326 struct ring_buffer *buffer;
4327 struct print_entry *entry;
4328 unsigned long irq_flags;
4329 struct page *pages[2];
6edb2a8a 4330 void *map_page[2];
d696b58c
SR
4331 int nr_pages = 1;
4332 ssize_t written;
d696b58c
SR
4333 int offset;
4334 int size;
4335 int len;
4336 int ret;
6edb2a8a 4337 int i;
5bf9a1ee 4338
c76f0694 4339 if (tracing_disabled)
5bf9a1ee
PP
4340 return -EINVAL;
4341
5224c3a3
MSB
4342 if (!(trace_flags & TRACE_ITER_MARKERS))
4343 return -EINVAL;
4344
5bf9a1ee
PP
4345 if (cnt > TRACE_BUF_SIZE)
4346 cnt = TRACE_BUF_SIZE;
4347
d696b58c
SR
4348 /*
4349 * Userspace is injecting traces into the kernel trace buffer.
4350 * We want to be as non intrusive as possible.
4351 * To do so, we do not want to allocate any special buffers
4352 * or take any locks, but instead write the userspace data
4353 * straight into the ring buffer.
4354 *
4355 * First we need to pin the userspace buffer into memory,
4356 * which, most likely it is, because it just referenced it.
4357 * But there's no guarantee that it is. By using get_user_pages_fast()
4358 * and kmap_atomic/kunmap_atomic() we can get access to the
4359 * pages directly. We then write the data directly into the
4360 * ring buffer.
4361 */
4362 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 4363
d696b58c
SR
4364 /* check if we cross pages */
4365 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4366 nr_pages = 2;
4367
4368 offset = addr & (PAGE_SIZE - 1);
4369 addr &= PAGE_MASK;
4370
4371 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4372 if (ret < nr_pages) {
4373 while (--ret >= 0)
4374 put_page(pages[ret]);
4375 written = -EFAULT;
4376 goto out;
5bf9a1ee 4377 }
d696b58c 4378
6edb2a8a
SR
4379 for (i = 0; i < nr_pages; i++)
4380 map_page[i] = kmap_atomic(pages[i]);
d696b58c
SR
4381
4382 local_save_flags(irq_flags);
4383 size = sizeof(*entry) + cnt + 2; /* possible \n added */
12883efb 4384 buffer = global_trace.trace_buffer.buffer;
d696b58c
SR
4385 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4386 irq_flags, preempt_count());
4387 if (!event) {
4388 /* Ring buffer disabled, return as if not open for write */
4389 written = -EBADF;
4390 goto out_unlock;
5bf9a1ee 4391 }
d696b58c
SR
4392
4393 entry = ring_buffer_event_data(event);
4394 entry->ip = _THIS_IP_;
4395
4396 if (nr_pages == 2) {
4397 len = PAGE_SIZE - offset;
6edb2a8a
SR
4398 memcpy(&entry->buf, map_page[0] + offset, len);
4399 memcpy(&entry->buf[len], map_page[1], cnt - len);
c13d2f7c 4400 } else
6edb2a8a 4401 memcpy(&entry->buf, map_page[0] + offset, cnt);
5bf9a1ee 4402
d696b58c
SR
4403 if (entry->buf[cnt - 1] != '\n') {
4404 entry->buf[cnt] = '\n';
4405 entry->buf[cnt + 1] = '\0';
4406 } else
4407 entry->buf[cnt] = '\0';
4408
7ffbd48d 4409 __buffer_unlock_commit(buffer, event);
5bf9a1ee 4410
d696b58c 4411 written = cnt;
5bf9a1ee 4412
d696b58c 4413 *fpos += written;
1aa54bca 4414
d696b58c 4415 out_unlock:
6edb2a8a
SR
4416 for (i = 0; i < nr_pages; i++){
4417 kunmap_atomic(map_page[i]);
4418 put_page(pages[i]);
4419 }
d696b58c 4420 out:
1aa54bca 4421 return written;
5bf9a1ee
PP
4422}
4423
13f16d20 4424static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 4425{
2b6080f2 4426 struct trace_array *tr = m->private;
5079f326
Z
4427 int i;
4428
4429 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 4430 seq_printf(m,
5079f326 4431 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
4432 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4433 i == tr->clock_id ? "]" : "");
13f16d20 4434 seq_putc(m, '\n');
5079f326 4435
13f16d20 4436 return 0;
5079f326
Z
4437}
4438
4439static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4440 size_t cnt, loff_t *fpos)
4441{
2b6080f2
SR
4442 struct seq_file *m = filp->private_data;
4443 struct trace_array *tr = m->private;
5079f326
Z
4444 char buf[64];
4445 const char *clockstr;
4446 int i;
4447
4448 if (cnt >= sizeof(buf))
4449 return -EINVAL;
4450
4451 if (copy_from_user(&buf, ubuf, cnt))
4452 return -EFAULT;
4453
4454 buf[cnt] = 0;
4455
4456 clockstr = strstrip(buf);
4457
4458 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4459 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4460 break;
4461 }
4462 if (i == ARRAY_SIZE(trace_clocks))
4463 return -EINVAL;
4464
5079f326
Z
4465 mutex_lock(&trace_types_lock);
4466
2b6080f2
SR
4467 tr->clock_id = i;
4468
12883efb 4469 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 4470
60303ed3
DS
4471 /*
4472 * New clock may not be consistent with the previous clock.
4473 * Reset the buffer so that it doesn't have incomparable timestamps.
4474 */
12883efb
SRRH
4475 tracing_reset_online_cpus(&global_trace.trace_buffer);
4476
4477#ifdef CONFIG_TRACER_MAX_TRACE
4478 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4479 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4480 tracing_reset_online_cpus(&global_trace.max_buffer);
4481#endif
60303ed3 4482
5079f326
Z
4483 mutex_unlock(&trace_types_lock);
4484
4485 *fpos += cnt;
4486
4487 return cnt;
4488}
4489
13f16d20
LZ
4490static int tracing_clock_open(struct inode *inode, struct file *file)
4491{
4492 if (tracing_disabled)
4493 return -ENODEV;
2b6080f2
SR
4494
4495 return single_open(file, tracing_clock_show, inode->i_private);
13f16d20
LZ
4496}
4497
6de58e62
SRRH
4498struct ftrace_buffer_info {
4499 struct trace_iterator iter;
4500 void *spare;
4501 unsigned int read;
4502};
4503
debdd57f
HT
4504#ifdef CONFIG_TRACER_SNAPSHOT
4505static int tracing_snapshot_open(struct inode *inode, struct file *file)
4506{
2b6080f2 4507 struct trace_cpu *tc = inode->i_private;
debdd57f 4508 struct trace_iterator *iter;
2b6080f2 4509 struct seq_file *m;
debdd57f
HT
4510 int ret = 0;
4511
4512 if (file->f_mode & FMODE_READ) {
4513 iter = __tracing_open(inode, file, true);
4514 if (IS_ERR(iter))
4515 ret = PTR_ERR(iter);
2b6080f2
SR
4516 } else {
4517 /* Writes still need the seq_file to hold the private data */
4518 m = kzalloc(sizeof(*m), GFP_KERNEL);
4519 if (!m)
4520 return -ENOMEM;
4521 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4522 if (!iter) {
4523 kfree(m);
4524 return -ENOMEM;
4525 }
4526 iter->tr = tc->tr;
12883efb 4527 iter->trace_buffer = &tc->tr->max_buffer;
f1affcaa 4528 iter->cpu_file = tc->cpu;
2b6080f2
SR
4529 m->private = iter;
4530 file->private_data = m;
debdd57f 4531 }
2b6080f2 4532
debdd57f
HT
4533 return ret;
4534}
4535
4536static ssize_t
4537tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4538 loff_t *ppos)
4539{
2b6080f2
SR
4540 struct seq_file *m = filp->private_data;
4541 struct trace_iterator *iter = m->private;
4542 struct trace_array *tr = iter->tr;
debdd57f
HT
4543 unsigned long val;
4544 int ret;
4545
4546 ret = tracing_update_buffers();
4547 if (ret < 0)
4548 return ret;
4549
4550 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4551 if (ret)
4552 return ret;
4553
4554 mutex_lock(&trace_types_lock);
4555
2b6080f2 4556 if (tr->current_trace->use_max_tr) {
debdd57f
HT
4557 ret = -EBUSY;
4558 goto out;
4559 }
4560
4561 switch (val) {
4562 case 0:
f1affcaa
SRRH
4563 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4564 ret = -EINVAL;
4565 break;
debdd57f 4566 }
3209cff4
SRRH
4567 if (tr->allocated_snapshot)
4568 free_snapshot(tr);
debdd57f
HT
4569 break;
4570 case 1:
f1affcaa
SRRH
4571/* Only allow per-cpu swap if the ring buffer supports it */
4572#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4573 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4574 ret = -EINVAL;
4575 break;
4576 }
4577#endif
45ad21ca 4578 if (!tr->allocated_snapshot) {
3209cff4 4579 ret = alloc_snapshot(tr);
debdd57f
HT
4580 if (ret < 0)
4581 break;
debdd57f 4582 }
debdd57f
HT
4583 local_irq_disable();
4584 /* Now, we're going to swap */
f1affcaa 4585 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
ce9bae55 4586 update_max_tr(tr, current, smp_processor_id());
f1affcaa 4587 else
ce9bae55 4588 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
4589 local_irq_enable();
4590 break;
4591 default:
45ad21ca 4592 if (tr->allocated_snapshot) {
f1affcaa
SRRH
4593 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4594 tracing_reset_online_cpus(&tr->max_buffer);
4595 else
4596 tracing_reset(&tr->max_buffer, iter->cpu_file);
4597 }
debdd57f
HT
4598 break;
4599 }
4600
4601 if (ret >= 0) {
4602 *ppos += cnt;
4603 ret = cnt;
4604 }
4605out:
4606 mutex_unlock(&trace_types_lock);
4607 return ret;
4608}
2b6080f2
SR
4609
4610static int tracing_snapshot_release(struct inode *inode, struct file *file)
4611{
4612 struct seq_file *m = file->private_data;
4613
4614 if (file->f_mode & FMODE_READ)
4615 return tracing_release(inode, file);
4616
4617 /* If write only, the seq_file is just a stub */
4618 if (m)
4619 kfree(m->private);
4620 kfree(m);
4621
4622 return 0;
4623}
4624
6de58e62
SRRH
4625static int tracing_buffers_open(struct inode *inode, struct file *filp);
4626static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4627 size_t count, loff_t *ppos);
4628static int tracing_buffers_release(struct inode *inode, struct file *file);
4629static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4630 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4631
4632static int snapshot_raw_open(struct inode *inode, struct file *filp)
4633{
4634 struct ftrace_buffer_info *info;
4635 int ret;
4636
4637 ret = tracing_buffers_open(inode, filp);
4638 if (ret < 0)
4639 return ret;
4640
4641 info = filp->private_data;
4642
4643 if (info->iter.trace->use_max_tr) {
4644 tracing_buffers_release(inode, filp);
4645 return -EBUSY;
4646 }
4647
4648 info->iter.snapshot = true;
4649 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4650
4651 return ret;
4652}
4653
debdd57f
HT
4654#endif /* CONFIG_TRACER_SNAPSHOT */
4655
4656
5e2336a0 4657static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
4658 .open = tracing_open_generic,
4659 .read = tracing_max_lat_read,
4660 .write = tracing_max_lat_write,
b444786f 4661 .llseek = generic_file_llseek,
bc0c38d1
SR
4662};
4663
5e2336a0 4664static const struct file_operations set_tracer_fops = {
4bf39a94
IM
4665 .open = tracing_open_generic,
4666 .read = tracing_set_trace_read,
4667 .write = tracing_set_trace_write,
b444786f 4668 .llseek = generic_file_llseek,
bc0c38d1
SR
4669};
4670
5e2336a0 4671static const struct file_operations tracing_pipe_fops = {
4bf39a94 4672 .open = tracing_open_pipe,
2a2cc8f7 4673 .poll = tracing_poll_pipe,
4bf39a94 4674 .read = tracing_read_pipe,
3c56819b 4675 .splice_read = tracing_splice_read_pipe,
4bf39a94 4676 .release = tracing_release_pipe,
b444786f 4677 .llseek = no_llseek,
b3806b43
SR
4678};
4679
5e2336a0 4680static const struct file_operations tracing_entries_fops = {
2b6080f2 4681 .open = tracing_open_generic,
a98a3c3f
SR
4682 .read = tracing_entries_read,
4683 .write = tracing_entries_write,
b444786f 4684 .llseek = generic_file_llseek,
a98a3c3f
SR
4685};
4686
f81ab074
VN
4687static const struct file_operations tracing_total_entries_fops = {
4688 .open = tracing_open_generic,
4689 .read = tracing_total_entries_read,
4690 .llseek = generic_file_llseek,
4691};
4692
4f271a2a
VN
4693static const struct file_operations tracing_free_buffer_fops = {
4694 .write = tracing_free_buffer_write,
4695 .release = tracing_free_buffer_release,
4696};
4697
5e2336a0 4698static const struct file_operations tracing_mark_fops = {
43a15386 4699 .open = tracing_open_generic,
5bf9a1ee 4700 .write = tracing_mark_write,
b444786f 4701 .llseek = generic_file_llseek,
5bf9a1ee
PP
4702};
4703
5079f326 4704static const struct file_operations trace_clock_fops = {
13f16d20
LZ
4705 .open = tracing_clock_open,
4706 .read = seq_read,
4707 .llseek = seq_lseek,
4708 .release = single_release,
5079f326
Z
4709 .write = tracing_clock_write,
4710};
4711
debdd57f
HT
4712#ifdef CONFIG_TRACER_SNAPSHOT
4713static const struct file_operations snapshot_fops = {
4714 .open = tracing_snapshot_open,
4715 .read = seq_read,
4716 .write = tracing_snapshot_write,
4717 .llseek = tracing_seek,
2b6080f2 4718 .release = tracing_snapshot_release,
debdd57f 4719};
debdd57f 4720
6de58e62
SRRH
4721static const struct file_operations snapshot_raw_fops = {
4722 .open = snapshot_raw_open,
4723 .read = tracing_buffers_read,
4724 .release = tracing_buffers_release,
4725 .splice_read = tracing_buffers_splice_read,
4726 .llseek = no_llseek,
2cadf913
SR
4727};
4728
6de58e62
SRRH
4729#endif /* CONFIG_TRACER_SNAPSHOT */
4730
2cadf913
SR
4731static int tracing_buffers_open(struct inode *inode, struct file *filp)
4732{
2b6080f2
SR
4733 struct trace_cpu *tc = inode->i_private;
4734 struct trace_array *tr = tc->tr;
2cadf913
SR
4735 struct ftrace_buffer_info *info;
4736
4737 if (tracing_disabled)
4738 return -ENODEV;
4739
4740 info = kzalloc(sizeof(*info), GFP_KERNEL);
4741 if (!info)
4742 return -ENOMEM;
4743
a695cb58
SRRH
4744 mutex_lock(&trace_types_lock);
4745
4746 tr->ref++;
4747
cc60cdc9
SR
4748 info->iter.tr = tr;
4749 info->iter.cpu_file = tc->cpu;
b627344f 4750 info->iter.trace = tr->current_trace;
12883efb 4751 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 4752 info->spare = NULL;
2cadf913 4753 /* Force reading ring buffer for first read */
cc60cdc9 4754 info->read = (unsigned int)-1;
2cadf913
SR
4755
4756 filp->private_data = info;
4757
a695cb58
SRRH
4758 mutex_unlock(&trace_types_lock);
4759
d1e7e02f 4760 return nonseekable_open(inode, filp);
2cadf913
SR
4761}
4762
cc60cdc9
SR
4763static unsigned int
4764tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4765{
4766 struct ftrace_buffer_info *info = filp->private_data;
4767 struct trace_iterator *iter = &info->iter;
4768
4769 return trace_poll(iter, filp, poll_table);
4770}
4771
2cadf913
SR
4772static ssize_t
4773tracing_buffers_read(struct file *filp, char __user *ubuf,
4774 size_t count, loff_t *ppos)
4775{
4776 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 4777 struct trace_iterator *iter = &info->iter;
2cadf913 4778 ssize_t ret;
6de58e62 4779 ssize_t size;
2cadf913 4780
2dc5d12b
SR
4781 if (!count)
4782 return 0;
4783
6de58e62
SRRH
4784 mutex_lock(&trace_types_lock);
4785
4786#ifdef CONFIG_TRACER_MAX_TRACE
4787 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4788 size = -EBUSY;
4789 goto out_unlock;
4790 }
4791#endif
4792
ddd538f3 4793 if (!info->spare)
12883efb
SRRH
4794 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4795 iter->cpu_file);
6de58e62 4796 size = -ENOMEM;
ddd538f3 4797 if (!info->spare)
6de58e62 4798 goto out_unlock;
ddd538f3 4799
2cadf913
SR
4800 /* Do we have previous read data to read? */
4801 if (info->read < PAGE_SIZE)
4802 goto read;
4803
b627344f 4804 again:
cc60cdc9 4805 trace_access_lock(iter->cpu_file);
12883efb 4806 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
4807 &info->spare,
4808 count,
cc60cdc9
SR
4809 iter->cpu_file, 0);
4810 trace_access_unlock(iter->cpu_file);
2cadf913 4811
b627344f
SR
4812 if (ret < 0) {
4813 if (trace_empty(iter)) {
6de58e62
SRRH
4814 if ((filp->f_flags & O_NONBLOCK)) {
4815 size = -EAGAIN;
4816 goto out_unlock;
4817 }
4818 mutex_unlock(&trace_types_lock);
b627344f 4819 iter->trace->wait_pipe(iter);
6de58e62
SRRH
4820 mutex_lock(&trace_types_lock);
4821 if (signal_pending(current)) {
4822 size = -EINTR;
4823 goto out_unlock;
4824 }
b627344f
SR
4825 goto again;
4826 }
6de58e62
SRRH
4827 size = 0;
4828 goto out_unlock;
b627344f 4829 }
436fc280 4830
436fc280 4831 info->read = 0;
b627344f 4832 read:
2cadf913
SR
4833 size = PAGE_SIZE - info->read;
4834 if (size > count)
4835 size = count;
4836
4837 ret = copy_to_user(ubuf, info->spare + info->read, size);
6de58e62
SRRH
4838 if (ret == size) {
4839 size = -EFAULT;
4840 goto out_unlock;
4841 }
2dc5d12b
SR
4842 size -= ret;
4843
2cadf913
SR
4844 *ppos += size;
4845 info->read += size;
4846
6de58e62
SRRH
4847 out_unlock:
4848 mutex_unlock(&trace_types_lock);
4849
2cadf913
SR
4850 return size;
4851}
4852
4853static int tracing_buffers_release(struct inode *inode, struct file *file)
4854{
4855 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 4856 struct trace_iterator *iter = &info->iter;
2cadf913 4857
a695cb58
SRRH
4858 mutex_lock(&trace_types_lock);
4859
4860 WARN_ON(!iter->tr->ref);
4861 iter->tr->ref--;
2cadf913 4862
ddd538f3 4863 if (info->spare)
12883efb 4864 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
2cadf913
SR
4865 kfree(info);
4866
a695cb58
SRRH
4867 mutex_unlock(&trace_types_lock);
4868
2cadf913
SR
4869 return 0;
4870}
4871
4872struct buffer_ref {
4873 struct ring_buffer *buffer;
4874 void *page;
4875 int ref;
4876};
4877
4878static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4879 struct pipe_buffer *buf)
4880{
4881 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4882
4883 if (--ref->ref)
4884 return;
4885
4886 ring_buffer_free_read_page(ref->buffer, ref->page);
4887 kfree(ref);
4888 buf->private = 0;
4889}
4890
2cadf913
SR
4891static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4892 struct pipe_buffer *buf)
4893{
4894 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4895
4896 ref->ref++;
4897}
4898
4899/* Pipe buffer operations for a buffer. */
28dfef8f 4900static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913
SR
4901 .can_merge = 0,
4902 .map = generic_pipe_buf_map,
4903 .unmap = generic_pipe_buf_unmap,
4904 .confirm = generic_pipe_buf_confirm,
4905 .release = buffer_pipe_buf_release,
d55cb6cf 4906 .steal = generic_pipe_buf_steal,
2cadf913
SR
4907 .get = buffer_pipe_buf_get,
4908};
4909
4910/*
4911 * Callback from splice_to_pipe(), if we need to release some pages
4912 * at the end of the spd in case we error'ed out in filling the pipe.
4913 */
4914static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4915{
4916 struct buffer_ref *ref =
4917 (struct buffer_ref *)spd->partial[i].private;
4918
4919 if (--ref->ref)
4920 return;
4921
4922 ring_buffer_free_read_page(ref->buffer, ref->page);
4923 kfree(ref);
4924 spd->partial[i].private = 0;
4925}
4926
4927static ssize_t
4928tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4929 struct pipe_inode_info *pipe, size_t len,
4930 unsigned int flags)
4931{
4932 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 4933 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
4934 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4935 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 4936 struct splice_pipe_desc spd = {
35f3d14d
JA
4937 .pages = pages_def,
4938 .partial = partial_def,
047fe360 4939 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
4940 .flags = flags,
4941 .ops = &buffer_pipe_buf_ops,
4942 .spd_release = buffer_spd_release,
4943 };
4944 struct buffer_ref *ref;
93459c6c 4945 int entries, size, i;
6de58e62 4946 ssize_t ret;
2cadf913 4947
6de58e62
SRRH
4948 mutex_lock(&trace_types_lock);
4949
4950#ifdef CONFIG_TRACER_MAX_TRACE
4951 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4952 ret = -EBUSY;
4953 goto out;
4954 }
4955#endif
4956
4957 if (splice_grow_spd(pipe, &spd)) {
4958 ret = -ENOMEM;
4959 goto out;
4960 }
35f3d14d 4961
93cfb3c9 4962 if (*ppos & (PAGE_SIZE - 1)) {
35f3d14d
JA
4963 ret = -EINVAL;
4964 goto out;
93cfb3c9
LJ
4965 }
4966
4967 if (len & (PAGE_SIZE - 1)) {
35f3d14d
JA
4968 if (len < PAGE_SIZE) {
4969 ret = -EINVAL;
4970 goto out;
4971 }
93cfb3c9
LJ
4972 len &= PAGE_MASK;
4973 }
4974
cc60cdc9
SR
4975 again:
4976 trace_access_lock(iter->cpu_file);
12883efb 4977 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 4978
35f3d14d 4979 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
4980 struct page *page;
4981 int r;
4982
4983 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4984 if (!ref)
4985 break;
4986
7267fa68 4987 ref->ref = 1;
12883efb 4988 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 4989 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
2cadf913
SR
4990 if (!ref->page) {
4991 kfree(ref);
4992 break;
4993 }
4994
4995 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 4996 len, iter->cpu_file, 1);
2cadf913 4997 if (r < 0) {
7ea59064 4998 ring_buffer_free_read_page(ref->buffer, ref->page);
2cadf913
SR
4999 kfree(ref);
5000 break;
5001 }
5002
5003 /*
5004 * zero out any left over data, this is going to
5005 * user land.
5006 */
5007 size = ring_buffer_page_len(ref->page);
5008 if (size < PAGE_SIZE)
5009 memset(ref->page + size, 0, PAGE_SIZE - size);
5010
5011 page = virt_to_page(ref->page);
5012
5013 spd.pages[i] = page;
5014 spd.partial[i].len = PAGE_SIZE;
5015 spd.partial[i].offset = 0;
5016 spd.partial[i].private = (unsigned long)ref;
5017 spd.nr_pages++;
93cfb3c9 5018 *ppos += PAGE_SIZE;
93459c6c 5019
12883efb 5020 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
5021 }
5022
cc60cdc9 5023 trace_access_unlock(iter->cpu_file);
2cadf913
SR
5024 spd.nr_pages = i;
5025
5026 /* did we read anything? */
5027 if (!spd.nr_pages) {
cc60cdc9 5028 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
2cadf913 5029 ret = -EAGAIN;
cc60cdc9
SR
5030 goto out;
5031 }
6de58e62 5032 mutex_unlock(&trace_types_lock);
b627344f 5033 iter->trace->wait_pipe(iter);
6de58e62 5034 mutex_lock(&trace_types_lock);
cc60cdc9
SR
5035 if (signal_pending(current)) {
5036 ret = -EINTR;
5037 goto out;
5038 }
5039 goto again;
2cadf913
SR
5040 }
5041
5042 ret = splice_to_pipe(pipe, &spd);
047fe360 5043 splice_shrink_spd(&spd);
35f3d14d 5044out:
6de58e62
SRRH
5045 mutex_unlock(&trace_types_lock);
5046
2cadf913
SR
5047 return ret;
5048}
5049
5050static const struct file_operations tracing_buffers_fops = {
5051 .open = tracing_buffers_open,
5052 .read = tracing_buffers_read,
cc60cdc9 5053 .poll = tracing_buffers_poll,
2cadf913
SR
5054 .release = tracing_buffers_release,
5055 .splice_read = tracing_buffers_splice_read,
5056 .llseek = no_llseek,
5057};
5058
c8d77183
SR
5059static ssize_t
5060tracing_stats_read(struct file *filp, char __user *ubuf,
5061 size_t count, loff_t *ppos)
5062{
2b6080f2
SR
5063 struct trace_cpu *tc = filp->private_data;
5064 struct trace_array *tr = tc->tr;
12883efb 5065 struct trace_buffer *trace_buf = &tr->trace_buffer;
c8d77183
SR
5066 struct trace_seq *s;
5067 unsigned long cnt;
c64e148a
VN
5068 unsigned long long t;
5069 unsigned long usec_rem;
2b6080f2 5070 int cpu = tc->cpu;
c8d77183 5071
e4f2d10f 5072 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 5073 if (!s)
a646365c 5074 return -ENOMEM;
c8d77183
SR
5075
5076 trace_seq_init(s);
5077
12883efb 5078 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5079 trace_seq_printf(s, "entries: %ld\n", cnt);
5080
12883efb 5081 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5082 trace_seq_printf(s, "overrun: %ld\n", cnt);
5083
12883efb 5084 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
5085 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5086
12883efb 5087 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
5088 trace_seq_printf(s, "bytes: %ld\n", cnt);
5089
11043d8b
YY
5090 if (trace_clocks[trace_clock_id].in_ns) {
5091 /* local or global for trace_clock */
12883efb 5092 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
5093 usec_rem = do_div(t, USEC_PER_SEC);
5094 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5095 t, usec_rem);
5096
12883efb 5097 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
5098 usec_rem = do_div(t, USEC_PER_SEC);
5099 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5100 } else {
5101 /* counter or tsc mode for trace_clock */
5102 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 5103 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 5104
11043d8b 5105 trace_seq_printf(s, "now ts: %llu\n",
12883efb 5106 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 5107 }
c64e148a 5108
12883efb 5109 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
5110 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5111
12883efb 5112 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
5113 trace_seq_printf(s, "read events: %ld\n", cnt);
5114
c8d77183
SR
5115 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5116
5117 kfree(s);
5118
5119 return count;
5120}
5121
5122static const struct file_operations tracing_stats_fops = {
5123 .open = tracing_open_generic,
5124 .read = tracing_stats_read,
b444786f 5125 .llseek = generic_file_llseek,
c8d77183
SR
5126};
5127
bc0c38d1
SR
5128#ifdef CONFIG_DYNAMIC_FTRACE
5129
b807c3d0
SR
5130int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5131{
5132 return 0;
5133}
5134
bc0c38d1 5135static ssize_t
b807c3d0 5136tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
5137 size_t cnt, loff_t *ppos)
5138{
a26a2a27
SR
5139 static char ftrace_dyn_info_buffer[1024];
5140 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 5141 unsigned long *p = filp->private_data;
b807c3d0 5142 char *buf = ftrace_dyn_info_buffer;
a26a2a27 5143 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
5144 int r;
5145
b807c3d0
SR
5146 mutex_lock(&dyn_info_mutex);
5147 r = sprintf(buf, "%ld ", *p);
4bf39a94 5148
a26a2a27 5149 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
5150 buf[r++] = '\n';
5151
5152 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5153
5154 mutex_unlock(&dyn_info_mutex);
5155
5156 return r;
bc0c38d1
SR
5157}
5158
5e2336a0 5159static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 5160 .open = tracing_open_generic,
b807c3d0 5161 .read = tracing_read_dyn_info,
b444786f 5162 .llseek = generic_file_llseek,
bc0c38d1 5163};
77fd5c15 5164#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 5165
77fd5c15
SRRH
5166#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5167static void
5168ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5169{
5170 tracing_snapshot();
5171}
bc0c38d1 5172
77fd5c15
SRRH
5173static void
5174ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
bc0c38d1 5175{
77fd5c15
SRRH
5176 unsigned long *count = (long *)data;
5177
5178 if (!*count)
5179 return;
bc0c38d1 5180
77fd5c15
SRRH
5181 if (*count != -1)
5182 (*count)--;
5183
5184 tracing_snapshot();
5185}
5186
5187static int
5188ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5189 struct ftrace_probe_ops *ops, void *data)
5190{
5191 long count = (long)data;
5192
5193 seq_printf(m, "%ps:", (void *)ip);
5194
5195 seq_printf(m, "snapshot");
5196
5197 if (count == -1)
5198 seq_printf(m, ":unlimited\n");
5199 else
5200 seq_printf(m, ":count=%ld\n", count);
5201
5202 return 0;
5203}
5204
5205static struct ftrace_probe_ops snapshot_probe_ops = {
5206 .func = ftrace_snapshot,
5207 .print = ftrace_snapshot_print,
5208};
5209
5210static struct ftrace_probe_ops snapshot_count_probe_ops = {
5211 .func = ftrace_count_snapshot,
5212 .print = ftrace_snapshot_print,
5213};
5214
5215static int
5216ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5217 char *glob, char *cmd, char *param, int enable)
5218{
5219 struct ftrace_probe_ops *ops;
5220 void *count = (void *)-1;
5221 char *number;
5222 int ret;
5223
5224 /* hash funcs only work with set_ftrace_filter */
5225 if (!enable)
5226 return -EINVAL;
5227
5228 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5229
5230 if (glob[0] == '!') {
5231 unregister_ftrace_function_probe_func(glob+1, ops);
5232 return 0;
5233 }
5234
5235 if (!param)
5236 goto out_reg;
5237
5238 number = strsep(&param, ":");
5239
5240 if (!strlen(number))
5241 goto out_reg;
5242
5243 /*
5244 * We use the callback data field (which is a pointer)
5245 * as our counter.
5246 */
5247 ret = kstrtoul(number, 0, (unsigned long *)&count);
5248 if (ret)
5249 return ret;
5250
5251 out_reg:
5252 ret = register_ftrace_function_probe(glob, ops, count);
5253
5254 if (ret >= 0)
5255 alloc_snapshot(&global_trace);
5256
5257 return ret < 0 ? ret : 0;
5258}
5259
5260static struct ftrace_func_command ftrace_snapshot_cmd = {
5261 .name = "snapshot",
5262 .func = ftrace_trace_snapshot_callback,
5263};
5264
5265static int register_snapshot_cmd(void)
5266{
5267 return register_ftrace_command(&ftrace_snapshot_cmd);
5268}
5269#else
5270static inline int register_snapshot_cmd(void) { return 0; }
5271#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 5272
2b6080f2 5273struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
bc0c38d1 5274{
2b6080f2
SR
5275 if (tr->dir)
5276 return tr->dir;
bc0c38d1 5277
3e1f60b8
FW
5278 if (!debugfs_initialized())
5279 return NULL;
5280
2b6080f2
SR
5281 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5282 tr->dir = debugfs_create_dir("tracing", NULL);
bc0c38d1 5283
687c878a
J
5284 if (!tr->dir)
5285 pr_warn_once("Could not create debugfs directory 'tracing'\n");
bc0c38d1 5286
2b6080f2 5287 return tr->dir;
bc0c38d1
SR
5288}
5289
2b6080f2
SR
5290struct dentry *tracing_init_dentry(void)
5291{
5292 return tracing_init_dentry_tr(&global_trace);
5293}
b04cc6b1 5294
2b6080f2 5295static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 5296{
b04cc6b1
FW
5297 struct dentry *d_tracer;
5298
2b6080f2
SR
5299 if (tr->percpu_dir)
5300 return tr->percpu_dir;
b04cc6b1 5301
2b6080f2 5302 d_tracer = tracing_init_dentry_tr(tr);
b04cc6b1
FW
5303 if (!d_tracer)
5304 return NULL;
5305
2b6080f2 5306 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
b04cc6b1 5307
2b6080f2
SR
5308 WARN_ONCE(!tr->percpu_dir,
5309 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 5310
2b6080f2 5311 return tr->percpu_dir;
b04cc6b1
FW
5312}
5313
2b6080f2
SR
5314static void
5315tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 5316{
12883efb 5317 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
2b6080f2 5318 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 5319 struct dentry *d_cpu;
dd49a38c 5320 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 5321
0a3d7ce7
NK
5322 if (!d_percpu)
5323 return;
5324
dd49a38c 5325 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8656e7a2
FW
5326 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5327 if (!d_cpu) {
5328 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5329 return;
5330 }
b04cc6b1 5331
8656e7a2 5332 /* per cpu trace_pipe */
5452af66 5333 trace_create_file("trace_pipe", 0444, d_cpu,
2b6080f2 5334 (void *)&data->trace_cpu, &tracing_pipe_fops);
b04cc6b1
FW
5335
5336 /* per cpu trace */
5452af66 5337 trace_create_file("trace", 0644, d_cpu,
2b6080f2 5338 (void *)&data->trace_cpu, &tracing_fops);
7f96f93f 5339
5452af66 5340 trace_create_file("trace_pipe_raw", 0444, d_cpu,
2b6080f2 5341 (void *)&data->trace_cpu, &tracing_buffers_fops);
7f96f93f 5342
c8d77183 5343 trace_create_file("stats", 0444, d_cpu,
2b6080f2 5344 (void *)&data->trace_cpu, &tracing_stats_fops);
438ced17
VN
5345
5346 trace_create_file("buffer_size_kb", 0444, d_cpu,
2b6080f2 5347 (void *)&data->trace_cpu, &tracing_entries_fops);
f1affcaa
SRRH
5348
5349#ifdef CONFIG_TRACER_SNAPSHOT
5350 trace_create_file("snapshot", 0644, d_cpu,
5351 (void *)&data->trace_cpu, &snapshot_fops);
6de58e62
SRRH
5352
5353 trace_create_file("snapshot_raw", 0444, d_cpu,
5354 (void *)&data->trace_cpu, &snapshot_raw_fops);
f1affcaa 5355#endif
b04cc6b1
FW
5356}
5357
60a11774
SR
5358#ifdef CONFIG_FTRACE_SELFTEST
5359/* Let selftest have access to static functions in this file */
5360#include "trace_selftest.c"
5361#endif
5362
577b785f
SR
5363struct trace_option_dentry {
5364 struct tracer_opt *opt;
5365 struct tracer_flags *flags;
2b6080f2 5366 struct trace_array *tr;
577b785f
SR
5367 struct dentry *entry;
5368};
5369
5370static ssize_t
5371trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5372 loff_t *ppos)
5373{
5374 struct trace_option_dentry *topt = filp->private_data;
5375 char *buf;
5376
5377 if (topt->flags->val & topt->opt->bit)
5378 buf = "1\n";
5379 else
5380 buf = "0\n";
5381
5382 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5383}
5384
5385static ssize_t
5386trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5387 loff_t *ppos)
5388{
5389 struct trace_option_dentry *topt = filp->private_data;
5390 unsigned long val;
577b785f
SR
5391 int ret;
5392
22fe9b54
PH
5393 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5394 if (ret)
577b785f
SR
5395 return ret;
5396
8d18eaaf
LZ
5397 if (val != 0 && val != 1)
5398 return -EINVAL;
577b785f 5399
8d18eaaf 5400 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 5401 mutex_lock(&trace_types_lock);
2b6080f2 5402 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
c757bea9 5403 topt->opt, !val);
577b785f
SR
5404 mutex_unlock(&trace_types_lock);
5405 if (ret)
5406 return ret;
577b785f
SR
5407 }
5408
5409 *ppos += cnt;
5410
5411 return cnt;
5412}
5413
5414
5415static const struct file_operations trace_options_fops = {
5416 .open = tracing_open_generic,
5417 .read = trace_options_read,
5418 .write = trace_options_write,
b444786f 5419 .llseek = generic_file_llseek,
577b785f
SR
5420};
5421
a8259075
SR
5422static ssize_t
5423trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5424 loff_t *ppos)
5425{
5426 long index = (long)filp->private_data;
5427 char *buf;
5428
5429 if (trace_flags & (1 << index))
5430 buf = "1\n";
5431 else
5432 buf = "0\n";
5433
5434 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5435}
5436
5437static ssize_t
5438trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5439 loff_t *ppos)
5440{
2b6080f2 5441 struct trace_array *tr = &global_trace;
a8259075 5442 long index = (long)filp->private_data;
a8259075
SR
5443 unsigned long val;
5444 int ret;
5445
22fe9b54
PH
5446 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5447 if (ret)
a8259075
SR
5448 return ret;
5449
f2d84b65 5450 if (val != 0 && val != 1)
a8259075 5451 return -EINVAL;
69d34da2
SRRH
5452
5453 mutex_lock(&trace_types_lock);
2b6080f2 5454 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 5455 mutex_unlock(&trace_types_lock);
a8259075 5456
613f04a0
SRRH
5457 if (ret < 0)
5458 return ret;
5459
a8259075
SR
5460 *ppos += cnt;
5461
5462 return cnt;
5463}
5464
a8259075
SR
5465static const struct file_operations trace_options_core_fops = {
5466 .open = tracing_open_generic,
5467 .read = trace_options_core_read,
5468 .write = trace_options_core_write,
b444786f 5469 .llseek = generic_file_llseek,
a8259075
SR
5470};
5471
5452af66 5472struct dentry *trace_create_file(const char *name,
f4ae40a6 5473 umode_t mode,
5452af66
FW
5474 struct dentry *parent,
5475 void *data,
5476 const struct file_operations *fops)
5477{
5478 struct dentry *ret;
5479
5480 ret = debugfs_create_file(name, mode, parent, data, fops);
5481 if (!ret)
5482 pr_warning("Could not create debugfs '%s' entry\n", name);
5483
5484 return ret;
5485}
5486
5487
2b6080f2 5488static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
5489{
5490 struct dentry *d_tracer;
a8259075 5491
2b6080f2
SR
5492 if (tr->options)
5493 return tr->options;
a8259075 5494
2b6080f2 5495 d_tracer = tracing_init_dentry_tr(tr);
a8259075
SR
5496 if (!d_tracer)
5497 return NULL;
5498
2b6080f2
SR
5499 tr->options = debugfs_create_dir("options", d_tracer);
5500 if (!tr->options) {
a8259075
SR
5501 pr_warning("Could not create debugfs directory 'options'\n");
5502 return NULL;
5503 }
5504
2b6080f2 5505 return tr->options;
a8259075
SR
5506}
5507
577b785f 5508static void
2b6080f2
SR
5509create_trace_option_file(struct trace_array *tr,
5510 struct trace_option_dentry *topt,
577b785f
SR
5511 struct tracer_flags *flags,
5512 struct tracer_opt *opt)
5513{
5514 struct dentry *t_options;
577b785f 5515
2b6080f2 5516 t_options = trace_options_init_dentry(tr);
577b785f
SR
5517 if (!t_options)
5518 return;
5519
5520 topt->flags = flags;
5521 topt->opt = opt;
2b6080f2 5522 topt->tr = tr;
577b785f 5523
5452af66 5524 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
5525 &trace_options_fops);
5526
577b785f
SR
5527}
5528
5529static struct trace_option_dentry *
2b6080f2 5530create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
5531{
5532 struct trace_option_dentry *topts;
5533 struct tracer_flags *flags;
5534 struct tracer_opt *opts;
5535 int cnt;
5536
5537 if (!tracer)
5538 return NULL;
5539
5540 flags = tracer->flags;
5541
5542 if (!flags || !flags->opts)
5543 return NULL;
5544
5545 opts = flags->opts;
5546
5547 for (cnt = 0; opts[cnt].name; cnt++)
5548 ;
5549
0cfe8245 5550 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f
SR
5551 if (!topts)
5552 return NULL;
5553
5554 for (cnt = 0; opts[cnt].name; cnt++)
2b6080f2 5555 create_trace_option_file(tr, &topts[cnt], flags,
577b785f
SR
5556 &opts[cnt]);
5557
5558 return topts;
5559}
5560
5561static void
5562destroy_trace_option_files(struct trace_option_dentry *topts)
5563{
5564 int cnt;
5565
5566 if (!topts)
5567 return;
5568
5569 for (cnt = 0; topts[cnt].opt; cnt++) {
5570 if (topts[cnt].entry)
5571 debugfs_remove(topts[cnt].entry);
5572 }
5573
5574 kfree(topts);
5575}
5576
a8259075 5577static struct dentry *
2b6080f2
SR
5578create_trace_option_core_file(struct trace_array *tr,
5579 const char *option, long index)
a8259075
SR
5580{
5581 struct dentry *t_options;
a8259075 5582
2b6080f2 5583 t_options = trace_options_init_dentry(tr);
a8259075
SR
5584 if (!t_options)
5585 return NULL;
5586
5452af66 5587 return trace_create_file(option, 0644, t_options, (void *)index,
a8259075 5588 &trace_options_core_fops);
a8259075
SR
5589}
5590
2b6080f2 5591static __init void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
5592{
5593 struct dentry *t_options;
a8259075
SR
5594 int i;
5595
2b6080f2 5596 t_options = trace_options_init_dentry(tr);
a8259075
SR
5597 if (!t_options)
5598 return;
5599
5452af66 5600 for (i = 0; trace_options[i]; i++)
2b6080f2 5601 create_trace_option_core_file(tr, trace_options[i], i);
a8259075
SR
5602}
5603
499e5470
SR
5604static ssize_t
5605rb_simple_read(struct file *filp, char __user *ubuf,
5606 size_t cnt, loff_t *ppos)
5607{
348f0fc2 5608 struct trace_array *tr = filp->private_data;
12883efb 5609 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
5610 char buf[64];
5611 int r;
5612
5613 if (buffer)
5614 r = ring_buffer_record_is_on(buffer);
5615 else
5616 r = 0;
5617
5618 r = sprintf(buf, "%d\n", r);
5619
5620 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5621}
5622
5623static ssize_t
5624rb_simple_write(struct file *filp, const char __user *ubuf,
5625 size_t cnt, loff_t *ppos)
5626{
348f0fc2 5627 struct trace_array *tr = filp->private_data;
12883efb 5628 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
5629 unsigned long val;
5630 int ret;
5631
5632 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5633 if (ret)
5634 return ret;
5635
5636 if (buffer) {
2df8f8a6
SR
5637 mutex_lock(&trace_types_lock);
5638 if (val) {
499e5470 5639 ring_buffer_record_on(buffer);
2b6080f2
SR
5640 if (tr->current_trace->start)
5641 tr->current_trace->start(tr);
2df8f8a6 5642 } else {
499e5470 5643 ring_buffer_record_off(buffer);
2b6080f2
SR
5644 if (tr->current_trace->stop)
5645 tr->current_trace->stop(tr);
2df8f8a6
SR
5646 }
5647 mutex_unlock(&trace_types_lock);
499e5470
SR
5648 }
5649
5650 (*ppos)++;
5651
5652 return cnt;
5653}
5654
5655static const struct file_operations rb_simple_fops = {
5656 .open = tracing_open_generic,
5657 .read = rb_simple_read,
5658 .write = rb_simple_write,
5659 .llseek = default_llseek,
5660};
5661
277ba044
SR
5662struct dentry *trace_instance_dir;
5663
5664static void
5665init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5666
737223fb
SRRH
5667static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5668{
5669 int cpu;
5670
5671 for_each_tracing_cpu(cpu) {
5672 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5673 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5674 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5675 }
5676}
5677
55034cd6
SRRH
5678static int
5679allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
5680{
5681 enum ring_buffer_flags rb_flags;
737223fb
SRRH
5682
5683 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5684
55034cd6
SRRH
5685 buf->buffer = ring_buffer_alloc(size, rb_flags);
5686 if (!buf->buffer)
5687 return -ENOMEM;
737223fb 5688
55034cd6
SRRH
5689 buf->data = alloc_percpu(struct trace_array_cpu);
5690 if (!buf->data) {
5691 ring_buffer_free(buf->buffer);
5692 return -ENOMEM;
5693 }
737223fb 5694
55034cd6 5695 init_trace_buffers(tr, buf);
737223fb
SRRH
5696
5697 /* Allocate the first page for all buffers */
5698 set_buffer_entries(&tr->trace_buffer,
5699 ring_buffer_size(tr->trace_buffer.buffer, 0));
5700
55034cd6
SRRH
5701 return 0;
5702}
737223fb 5703
55034cd6
SRRH
5704static int allocate_trace_buffers(struct trace_array *tr, int size)
5705{
5706 int ret;
737223fb 5707
55034cd6
SRRH
5708 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5709 if (ret)
5710 return ret;
737223fb 5711
55034cd6
SRRH
5712#ifdef CONFIG_TRACER_MAX_TRACE
5713 ret = allocate_trace_buffer(tr, &tr->max_buffer,
5714 allocate_snapshot ? size : 1);
5715 if (WARN_ON(ret)) {
737223fb 5716 ring_buffer_free(tr->trace_buffer.buffer);
55034cd6
SRRH
5717 free_percpu(tr->trace_buffer.data);
5718 return -ENOMEM;
5719 }
5720 tr->allocated_snapshot = allocate_snapshot;
737223fb 5721
55034cd6
SRRH
5722 /*
5723 * Only the top level trace array gets its snapshot allocated
5724 * from the kernel command line.
5725 */
5726 allocate_snapshot = false;
737223fb 5727#endif
55034cd6 5728 return 0;
737223fb
SRRH
5729}
5730
5731static int new_instance_create(const char *name)
5732{
277ba044
SR
5733 struct trace_array *tr;
5734 int ret;
277ba044
SR
5735
5736 mutex_lock(&trace_types_lock);
5737
5738 ret = -EEXIST;
5739 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5740 if (tr->name && strcmp(tr->name, name) == 0)
5741 goto out_unlock;
5742 }
5743
5744 ret = -ENOMEM;
5745 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5746 if (!tr)
5747 goto out_unlock;
5748
5749 tr->name = kstrdup(name, GFP_KERNEL);
5750 if (!tr->name)
5751 goto out_free_tr;
5752
5753 raw_spin_lock_init(&tr->start_lock);
5754
5755 tr->current_trace = &nop_trace;
5756
5757 INIT_LIST_HEAD(&tr->systems);
5758 INIT_LIST_HEAD(&tr->events);
5759
737223fb 5760 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
5761 goto out_free_tr;
5762
277ba044
SR
5763 /* Holder for file callbacks */
5764 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5765 tr->trace_cpu.tr = tr;
5766
5767 tr->dir = debugfs_create_dir(name, trace_instance_dir);
5768 if (!tr->dir)
5769 goto out_free_tr;
5770
5771 ret = event_trace_add_tracer(tr->dir, tr);
5772 if (ret)
5773 goto out_free_tr;
5774
5775 init_tracer_debugfs(tr, tr->dir);
5776
5777 list_add(&tr->list, &ftrace_trace_arrays);
5778
5779 mutex_unlock(&trace_types_lock);
5780
5781 return 0;
5782
5783 out_free_tr:
12883efb
SRRH
5784 if (tr->trace_buffer.buffer)
5785 ring_buffer_free(tr->trace_buffer.buffer);
277ba044
SR
5786 kfree(tr->name);
5787 kfree(tr);
5788
5789 out_unlock:
5790 mutex_unlock(&trace_types_lock);
5791
5792 return ret;
5793
5794}
5795
0c8916c3
SR
5796static int instance_delete(const char *name)
5797{
5798 struct trace_array *tr;
5799 int found = 0;
5800 int ret;
5801
5802 mutex_lock(&trace_types_lock);
5803
5804 ret = -ENODEV;
5805 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5806 if (tr->name && strcmp(tr->name, name) == 0) {
5807 found = 1;
5808 break;
5809 }
5810 }
5811 if (!found)
5812 goto out_unlock;
5813
a695cb58
SRRH
5814 ret = -EBUSY;
5815 if (tr->ref)
5816 goto out_unlock;
5817
0c8916c3
SR
5818 list_del(&tr->list);
5819
5820 event_trace_del_tracer(tr);
5821 debugfs_remove_recursive(tr->dir);
12883efb
SRRH
5822 free_percpu(tr->trace_buffer.data);
5823 ring_buffer_free(tr->trace_buffer.buffer);
0c8916c3
SR
5824
5825 kfree(tr->name);
5826 kfree(tr);
5827
5828 ret = 0;
5829
5830 out_unlock:
5831 mutex_unlock(&trace_types_lock);
5832
5833 return ret;
5834}
5835
277ba044
SR
5836static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5837{
5838 struct dentry *parent;
5839 int ret;
5840
5841 /* Paranoid: Make sure the parent is the "instances" directory */
5842 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5843 if (WARN_ON_ONCE(parent != trace_instance_dir))
5844 return -ENOENT;
5845
5846 /*
5847 * The inode mutex is locked, but debugfs_create_dir() will also
5848 * take the mutex. As the instances directory can not be destroyed
5849 * or changed in any other way, it is safe to unlock it, and
5850 * let the dentry try. If two users try to make the same dir at
5851 * the same time, then the new_instance_create() will determine the
5852 * winner.
5853 */
5854 mutex_unlock(&inode->i_mutex);
5855
5856 ret = new_instance_create(dentry->d_iname);
5857
5858 mutex_lock(&inode->i_mutex);
5859
5860 return ret;
5861}
5862
0c8916c3
SR
5863static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5864{
5865 struct dentry *parent;
5866 int ret;
5867
5868 /* Paranoid: Make sure the parent is the "instances" directory */
5869 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5870 if (WARN_ON_ONCE(parent != trace_instance_dir))
5871 return -ENOENT;
5872
5873 /* The caller did a dget() on dentry */
5874 mutex_unlock(&dentry->d_inode->i_mutex);
5875
5876 /*
5877 * The inode mutex is locked, but debugfs_create_dir() will also
5878 * take the mutex. As the instances directory can not be destroyed
5879 * or changed in any other way, it is safe to unlock it, and
5880 * let the dentry try. If two users try to make the same dir at
5881 * the same time, then the instance_delete() will determine the
5882 * winner.
5883 */
5884 mutex_unlock(&inode->i_mutex);
5885
5886 ret = instance_delete(dentry->d_iname);
5887
5888 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5889 mutex_lock(&dentry->d_inode->i_mutex);
5890
5891 return ret;
5892}
5893
277ba044
SR
5894static const struct inode_operations instance_dir_inode_operations = {
5895 .lookup = simple_lookup,
5896 .mkdir = instance_mkdir,
0c8916c3 5897 .rmdir = instance_rmdir,
277ba044
SR
5898};
5899
5900static __init void create_trace_instances(struct dentry *d_tracer)
5901{
5902 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5903 if (WARN_ON(!trace_instance_dir))
5904 return;
5905
5906 /* Hijack the dir inode operations, to allow mkdir */
5907 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5908}
5909
2b6080f2
SR
5910static void
5911init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5912{
121aaee7 5913 int cpu;
2b6080f2
SR
5914
5915 trace_create_file("trace_options", 0644, d_tracer,
5916 tr, &tracing_iter_fops);
5917
5918 trace_create_file("trace", 0644, d_tracer,
5919 (void *)&tr->trace_cpu, &tracing_fops);
5920
5921 trace_create_file("trace_pipe", 0444, d_tracer,
5922 (void *)&tr->trace_cpu, &tracing_pipe_fops);
5923
5924 trace_create_file("buffer_size_kb", 0644, d_tracer,
5925 (void *)&tr->trace_cpu, &tracing_entries_fops);
5926
5927 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5928 tr, &tracing_total_entries_fops);
5929
5930 trace_create_file("free_buffer", 0644, d_tracer,
5931 tr, &tracing_free_buffer_fops);
5932
5933 trace_create_file("trace_marker", 0220, d_tracer,
5934 tr, &tracing_mark_fops);
5935
5936 trace_create_file("trace_clock", 0644, d_tracer, tr,
5937 &trace_clock_fops);
5938
5939 trace_create_file("tracing_on", 0644, d_tracer,
5940 tr, &rb_simple_fops);
ce9bae55
SRRH
5941
5942#ifdef CONFIG_TRACER_SNAPSHOT
5943 trace_create_file("snapshot", 0644, d_tracer,
5944 (void *)&tr->trace_cpu, &snapshot_fops);
5945#endif
121aaee7
SRRH
5946
5947 for_each_tracing_cpu(cpu)
5948 tracing_init_debugfs_percpu(tr, cpu);
5949
2b6080f2
SR
5950}
5951
b5ad384e 5952static __init int tracer_init_debugfs(void)
bc0c38d1
SR
5953{
5954 struct dentry *d_tracer;
bc0c38d1 5955
7e53bd42
LJ
5956 trace_access_lock_init();
5957
bc0c38d1 5958 d_tracer = tracing_init_dentry();
ed6f1c99
NK
5959 if (!d_tracer)
5960 return 0;
bc0c38d1 5961
2b6080f2 5962 init_tracer_debugfs(&global_trace, d_tracer);
bc0c38d1 5963
5452af66 5964 trace_create_file("tracing_cpumask", 0644, d_tracer,
2b6080f2 5965 &global_trace, &tracing_cpumask_fops);
a8259075 5966
5452af66
FW
5967 trace_create_file("available_tracers", 0444, d_tracer,
5968 &global_trace, &show_traces_fops);
5969
339ae5d3 5970 trace_create_file("current_tracer", 0644, d_tracer,
5452af66
FW
5971 &global_trace, &set_tracer_fops);
5972
5d4a9dba 5973#ifdef CONFIG_TRACER_MAX_TRACE
5452af66
FW
5974 trace_create_file("tracing_max_latency", 0644, d_tracer,
5975 &tracing_max_latency, &tracing_max_lat_fops);
0e950173 5976#endif
5452af66
FW
5977
5978 trace_create_file("tracing_thresh", 0644, d_tracer,
5979 &tracing_thresh, &tracing_max_lat_fops);
a8259075 5980
339ae5d3 5981 trace_create_file("README", 0444, d_tracer,
5452af66
FW
5982 NULL, &tracing_readme_fops);
5983
69abe6a5
AP
5984 trace_create_file("saved_cmdlines", 0444, d_tracer,
5985 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 5986
bc0c38d1 5987#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
5988 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5989 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 5990#endif
b04cc6b1 5991
277ba044 5992 create_trace_instances(d_tracer);
5452af66 5993
2b6080f2 5994 create_trace_options_dir(&global_trace);
b04cc6b1 5995
b5ad384e 5996 return 0;
bc0c38d1
SR
5997}
5998
3f5a54e3
SR
5999static int trace_panic_handler(struct notifier_block *this,
6000 unsigned long event, void *unused)
6001{
944ac425 6002 if (ftrace_dump_on_oops)
cecbca96 6003 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6004 return NOTIFY_OK;
6005}
6006
6007static struct notifier_block trace_panic_notifier = {
6008 .notifier_call = trace_panic_handler,
6009 .next = NULL,
6010 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6011};
6012
6013static int trace_die_handler(struct notifier_block *self,
6014 unsigned long val,
6015 void *data)
6016{
6017 switch (val) {
6018 case DIE_OOPS:
944ac425 6019 if (ftrace_dump_on_oops)
cecbca96 6020 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
6021 break;
6022 default:
6023 break;
6024 }
6025 return NOTIFY_OK;
6026}
6027
6028static struct notifier_block trace_die_notifier = {
6029 .notifier_call = trace_die_handler,
6030 .priority = 200
6031};
6032
6033/*
6034 * printk is set to max of 1024, we really don't need it that big.
6035 * Nothing should be printing 1000 characters anyway.
6036 */
6037#define TRACE_MAX_PRINT 1000
6038
6039/*
6040 * Define here KERN_TRACE so that we have one place to modify
6041 * it if we decide to change what log level the ftrace dump
6042 * should be at.
6043 */
428aee14 6044#define KERN_TRACE KERN_EMERG
3f5a54e3 6045
955b61e5 6046void
3f5a54e3
SR
6047trace_printk_seq(struct trace_seq *s)
6048{
6049 /* Probably should print a warning here. */
bd6df187
J
6050 if (s->len >= TRACE_MAX_PRINT)
6051 s->len = TRACE_MAX_PRINT;
3f5a54e3
SR
6052
6053 /* should be zero ended, but we are paranoid. */
6054 s->buffer[s->len] = 0;
6055
6056 printk(KERN_TRACE "%s", s->buffer);
6057
f9520750 6058 trace_seq_init(s);
3f5a54e3
SR
6059}
6060
955b61e5
JW
6061void trace_init_global_iter(struct trace_iterator *iter)
6062{
6063 iter->tr = &global_trace;
2b6080f2 6064 iter->trace = iter->tr->current_trace;
ae3b5093 6065 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 6066 iter->trace_buffer = &global_trace.trace_buffer;
955b61e5
JW
6067}
6068
7fe70b57 6069void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 6070{
3f5a54e3
SR
6071 /* use static because iter can be a bit big for the stack */
6072 static struct trace_iterator iter;
7fe70b57 6073 static atomic_t dump_running;
cf586b61 6074 unsigned int old_userobj;
d769041f
SR
6075 unsigned long flags;
6076 int cnt = 0, cpu;
3f5a54e3 6077
7fe70b57
SRRH
6078 /* Only allow one dump user at a time. */
6079 if (atomic_inc_return(&dump_running) != 1) {
6080 atomic_dec(&dump_running);
6081 return;
6082 }
3f5a54e3 6083
7fe70b57
SRRH
6084 /*
6085 * Always turn off tracing when we dump.
6086 * We don't need to show trace output of what happens
6087 * between multiple crashes.
6088 *
6089 * If the user does a sysrq-z, then they can re-enable
6090 * tracing with echo 1 > tracing_on.
6091 */
0ee6b6cf 6092 tracing_off();
cf586b61 6093
7fe70b57 6094 local_irq_save(flags);
3f5a54e3 6095
38dbe0b1 6096 /* Simulate the iterator */
955b61e5
JW
6097 trace_init_global_iter(&iter);
6098
d769041f 6099 for_each_tracing_cpu(cpu) {
12883efb 6100 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
d769041f
SR
6101 }
6102
cf586b61
FW
6103 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6104
b54d3de9
TE
6105 /* don't look at user memory in panic mode */
6106 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6107
cecbca96
FW
6108 switch (oops_dump_mode) {
6109 case DUMP_ALL:
ae3b5093 6110 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6111 break;
6112 case DUMP_ORIG:
6113 iter.cpu_file = raw_smp_processor_id();
6114 break;
6115 case DUMP_NONE:
6116 goto out_enable;
6117 default:
6118 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 6119 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
6120 }
6121
6122 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 6123
7fe70b57
SRRH
6124 /* Did function tracer already get disabled? */
6125 if (ftrace_is_dead()) {
6126 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6127 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6128 }
6129
3f5a54e3
SR
6130 /*
6131 * We need to stop all tracing on all CPUS to read the
6132 * the next buffer. This is a bit expensive, but is
6133 * not done often. We fill all what we can read,
6134 * and then release the locks again.
6135 */
6136
3f5a54e3
SR
6137 while (!trace_empty(&iter)) {
6138
6139 if (!cnt)
6140 printk(KERN_TRACE "---------------------------------\n");
6141
6142 cnt++;
6143
6144 /* reset all but tr, trace, and overruns */
6145 memset(&iter.seq, 0,
6146 sizeof(struct trace_iterator) -
6147 offsetof(struct trace_iterator, seq));
6148 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6149 iter.pos = -1;
6150
955b61e5 6151 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
6152 int ret;
6153
6154 ret = print_trace_line(&iter);
6155 if (ret != TRACE_TYPE_NO_CONSUME)
6156 trace_consume(&iter);
3f5a54e3 6157 }
b892e5c8 6158 touch_nmi_watchdog();
3f5a54e3
SR
6159
6160 trace_printk_seq(&iter.seq);
6161 }
6162
6163 if (!cnt)
6164 printk(KERN_TRACE " (ftrace buffer empty)\n");
6165 else
6166 printk(KERN_TRACE "---------------------------------\n");
6167
cecbca96 6168 out_enable:
7fe70b57 6169 trace_flags |= old_userobj;
cf586b61 6170
7fe70b57
SRRH
6171 for_each_tracing_cpu(cpu) {
6172 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 6173 }
7fe70b57 6174 atomic_dec(&dump_running);
cd891ae0 6175 local_irq_restore(flags);
3f5a54e3 6176}
a8eecf22 6177EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 6178
3928a8a2 6179__init static int tracer_alloc_buffers(void)
bc0c38d1 6180{
73c5162a 6181 int ring_buf_size;
9e01c1b7 6182 int ret = -ENOMEM;
4c11d7ae 6183
750912fa 6184
9e01c1b7
RR
6185 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6186 goto out;
6187
6188 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6189 goto out_free_buffer_mask;
4c11d7ae 6190
07d777fe
SR
6191 /* Only allocate trace_printk buffers if a trace_printk exists */
6192 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 6193 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
6194 trace_printk_init_buffers();
6195
73c5162a
SR
6196 /* To save memory, keep the ring buffer size to its minimum */
6197 if (ring_buffer_expanded)
6198 ring_buf_size = trace_buf_size;
6199 else
6200 ring_buf_size = 1;
6201
9e01c1b7
RR
6202 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6203 cpumask_copy(tracing_cpumask, cpu_all_mask);
6204
2b6080f2
SR
6205 raw_spin_lock_init(&global_trace.start_lock);
6206
9e01c1b7 6207 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 6208 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
6209 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6210 WARN_ON(1);
9e01c1b7 6211 goto out_free_cpumask;
4c11d7ae 6212 }
a7603ff4 6213
499e5470
SR
6214 if (global_trace.buffer_disabled)
6215 tracing_off();
4c11d7ae 6216
bc0c38d1
SR
6217 trace_init_cmdlines();
6218
43a15386 6219 register_tracer(&nop_trace);
d840f718 6220
2b6080f2
SR
6221 global_trace.current_trace = &nop_trace;
6222
60a11774
SR
6223 /* All seems OK, enable tracing */
6224 tracing_disabled = 0;
3928a8a2 6225
3f5a54e3
SR
6226 atomic_notifier_chain_register(&panic_notifier_list,
6227 &trace_panic_notifier);
6228
6229 register_die_notifier(&trace_die_notifier);
2fc1dfbe 6230
ae63b31e
SR
6231 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6232
2b6080f2
SR
6233 /* Holder for file callbacks */
6234 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6235 global_trace.trace_cpu.tr = &global_trace;
6236
ae63b31e
SR
6237 INIT_LIST_HEAD(&global_trace.systems);
6238 INIT_LIST_HEAD(&global_trace.events);
6239 list_add(&global_trace.list, &ftrace_trace_arrays);
6240
7bcfaf54
SR
6241 while (trace_boot_options) {
6242 char *option;
6243
6244 option = strsep(&trace_boot_options, ",");
2b6080f2 6245 trace_set_options(&global_trace, option);
7bcfaf54
SR
6246 }
6247
77fd5c15
SRRH
6248 register_snapshot_cmd();
6249
2fc1dfbe 6250 return 0;
3f5a54e3 6251
9e01c1b7 6252out_free_cpumask:
12883efb
SRRH
6253 free_percpu(global_trace.trace_buffer.data);
6254#ifdef CONFIG_TRACER_MAX_TRACE
6255 free_percpu(global_trace.max_buffer.data);
6256#endif
9e01c1b7
RR
6257 free_cpumask_var(tracing_cpumask);
6258out_free_buffer_mask:
6259 free_cpumask_var(tracing_buffer_mask);
6260out:
6261 return ret;
bc0c38d1 6262}
b2821ae6
SR
6263
6264__init static int clear_boot_tracer(void)
6265{
6266 /*
6267 * The default tracer at boot buffer is an init section.
6268 * This function is called in lateinit. If we did not
6269 * find the boot tracer, then clear it out, to prevent
6270 * later registration from accessing the buffer that is
6271 * about to be freed.
6272 */
6273 if (!default_bootup_tracer)
6274 return 0;
6275
6276 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6277 default_bootup_tracer);
6278 default_bootup_tracer = NULL;
6279
6280 return 0;
6281}
6282
b5ad384e
FW
6283early_initcall(tracer_alloc_buffers);
6284fs_initcall(tracer_init_debugfs);
b2821ae6 6285late_initcall(clear_boot_tracer);