]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
ftrace/x86_32: Set ftrace_stub to weak to prevent gcc from using short jumps to it
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
8434dc93 23#include <linux/tracefs.h>
4c11d7ae 24#include <linux/pagemap.h>
bc0c38d1
SR
25#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
76c813e2 28#include <linux/vmalloc.h>
bc0c38d1
SR
29#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
2cadf913 32#include <linux/splice.h>
3f5a54e3 33#include <linux/kdebug.h>
5f0c6c03 34#include <linux/string.h>
f76180bc 35#include <linux/mount.h>
7e53bd42 36#include <linux/rwsem.h>
5a0e3ad6 37#include <linux/slab.h>
bc0c38d1
SR
38#include <linux/ctype.h>
39#include <linux/init.h>
2a2cc8f7 40#include <linux/poll.h>
b892e5c8 41#include <linux/nmi.h>
bc0c38d1 42#include <linux/fs.h>
478409dd 43#include <linux/trace.h>
8bd75c77 44#include <linux/sched/rt.h>
86387f7e 45
bc0c38d1 46#include "trace.h"
f0868d1e 47#include "trace_output.h"
bc0c38d1 48
73c5162a
SR
49/*
50 * On boot up, the ring buffer is set to the minimum size, so that
51 * we do not waste memory on systems that are not using tracing.
52 */
55034cd6 53bool ring_buffer_expanded;
73c5162a 54
8e1b82e0
FW
55/*
56 * We need to change this state when a selftest is running.
ff32504f
FW
57 * A selftest will lurk into the ring-buffer to count the
58 * entries inserted during the selftest although some concurrent
5e1607a0 59 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
60 * at the same time, giving false positive or negative results.
61 */
8e1b82e0 62static bool __read_mostly tracing_selftest_running;
ff32504f 63
b2821ae6
SR
64/*
65 * If a tracer is running, we do not want to run SELFTEST.
66 */
020e5f85 67bool __read_mostly tracing_selftest_disabled;
b2821ae6 68
0daa2302
SRRH
69/* Pipe tracepoints to printk */
70struct trace_iterator *tracepoint_print_iter;
71int tracepoint_printk;
42391745 72static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
0daa2302 73
adf9f195
FW
74/* For tracers that don't implement custom flags */
75static struct tracer_opt dummy_tracer_opt[] = {
76 { }
77};
78
8c1a49ae
SRRH
79static int
80dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
adf9f195
FW
81{
82 return 0;
83}
0f048701 84
7ffbd48d
SR
85/*
86 * To prevent the comm cache from being overwritten when no
87 * tracing is active, only save the comm when a trace event
88 * occurred.
89 */
90static DEFINE_PER_CPU(bool, trace_cmdline_save);
91
0f048701
SR
92/*
93 * Kill all tracing for good (never come back).
94 * It is initialized to 1 but will turn to zero if the initialization
95 * of the tracer is successful. But that is the only place that sets
96 * this back to zero.
97 */
4fd27358 98static int tracing_disabled = 1;
0f048701 99
955b61e5 100cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 101
944ac425
SR
102/*
103 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 *
105 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
106 * is set, then ftrace_dump is called. This will output the contents
107 * of the ftrace buffers to the console. This is very useful for
108 * capturing traces that lead to crashes and outputing it to a
109 * serial console.
110 *
111 * It is default off, but you can enable it with either specifying
112 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
113 * /proc/sys/kernel/ftrace_dump_on_oops
114 * Set 1 if you want to dump buffers of all CPUs
115 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 116 */
cecbca96
FW
117
118enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 119
de7edd31
SRRH
120/* When set, tracing will stop when a WARN*() is hit */
121int __disable_trace_on_warning;
122
9828413d
SRRH
123#ifdef CONFIG_TRACE_ENUM_MAP_FILE
124/* Map of enums to their values, for "enum_map" file */
125struct trace_enum_map_head {
126 struct module *mod;
127 unsigned long length;
128};
129
130union trace_enum_map_item;
131
132struct trace_enum_map_tail {
133 /*
134 * "end" is first and points to NULL as it must be different
135 * than "mod" or "enum_string"
136 */
137 union trace_enum_map_item *next;
138 const char *end; /* points to NULL */
139};
140
141static DEFINE_MUTEX(trace_enum_mutex);
142
143/*
144 * The trace_enum_maps are saved in an array with two extra elements,
145 * one at the beginning, and one at the end. The beginning item contains
146 * the count of the saved maps (head.length), and the module they
147 * belong to if not built in (head.mod). The ending item contains a
148 * pointer to the next array of saved enum_map items.
149 */
150union trace_enum_map_item {
151 struct trace_enum_map map;
152 struct trace_enum_map_head head;
153 struct trace_enum_map_tail tail;
154};
155
156static union trace_enum_map_item *trace_enum_maps;
157#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
158
607e2ea1 159static int tracing_set_tracer(struct trace_array *tr, const char *buf);
b2821ae6 160
ee6c2c1b
LZ
161#define MAX_TRACER_SIZE 100
162static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 163static char *default_bootup_tracer;
d9e54076 164
55034cd6
SRRH
165static bool allocate_snapshot;
166
1beee96b 167static int __init set_cmdline_ftrace(char *str)
d9e54076 168{
67012ab1 169 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 170 default_bootup_tracer = bootup_tracer_buf;
73c5162a 171 /* We are using ftrace early, expand it */
55034cd6 172 ring_buffer_expanded = true;
d9e54076
PZ
173 return 1;
174}
1beee96b 175__setup("ftrace=", set_cmdline_ftrace);
d9e54076 176
944ac425
SR
177static int __init set_ftrace_dump_on_oops(char *str)
178{
cecbca96
FW
179 if (*str++ != '=' || !*str) {
180 ftrace_dump_on_oops = DUMP_ALL;
181 return 1;
182 }
183
184 if (!strcmp("orig_cpu", str)) {
185 ftrace_dump_on_oops = DUMP_ORIG;
186 return 1;
187 }
188
189 return 0;
944ac425
SR
190}
191__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 192
de7edd31
SRRH
193static int __init stop_trace_on_warning(char *str)
194{
933ff9f2
LCG
195 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
196 __disable_trace_on_warning = 1;
de7edd31
SRRH
197 return 1;
198}
933ff9f2 199__setup("traceoff_on_warning", stop_trace_on_warning);
de7edd31 200
3209cff4 201static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
202{
203 allocate_snapshot = true;
204 /* We also need the main ring buffer expanded */
205 ring_buffer_expanded = true;
206 return 1;
207}
3209cff4 208__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 209
7bcfaf54
SR
210
211static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
7bcfaf54
SR
212
213static int __init set_trace_boot_options(char *str)
214{
67012ab1 215 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
216 return 0;
217}
218__setup("trace_options=", set_trace_boot_options);
219
e1e232ca
SR
220static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
221static char *trace_boot_clock __initdata;
222
223static int __init set_trace_boot_clock(char *str)
224{
225 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
226 trace_boot_clock = trace_boot_clock_buf;
227 return 0;
228}
229__setup("trace_clock=", set_trace_boot_clock);
230
0daa2302
SRRH
231static int __init set_tracepoint_printk(char *str)
232{
233 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
234 tracepoint_printk = 1;
235 return 1;
236}
237__setup("tp_printk", set_tracepoint_printk);
de7edd31 238
cf8e3474 239unsigned long long ns2usecs(cycle_t nsec)
bc0c38d1
SR
240{
241 nsec += 500;
242 do_div(nsec, 1000);
243 return nsec;
244}
245
983f938a
SRRH
246/* trace_flags holds trace_options default values */
247#define TRACE_DEFAULT_FLAGS \
248 (FUNCTION_DEFAULT_FLAGS | \
249 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
250 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
251 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
252 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
253
16270145
SRRH
254/* trace_options that are only supported by global_trace */
255#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
256 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
257
20550622
SRRH
258/* trace_flags that are default zero for instances */
259#define ZEROED_TRACE_FLAGS \
260 TRACE_ITER_EVENT_FORK
16270145 261
4fcdae83
SR
262/*
263 * The global_trace is the descriptor that holds the tracing
264 * buffers for the live tracing. For each CPU, it contains
265 * a link list of pages that will store trace entries. The
266 * page descriptor of the pages in the memory is used to hold
267 * the link list by linking the lru item in the page descriptor
268 * to each of the pages in the buffer per CPU.
269 *
270 * For each active CPU there is a data field that holds the
271 * pages for the buffer for that CPU. Each CPU has the same number
272 * of pages allocated for its buffer.
273 */
983f938a
SRRH
274static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
276};
bc0c38d1 277
ae63b31e 278LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 279
ff451961
SRRH
280int trace_array_get(struct trace_array *this_tr)
281{
282 struct trace_array *tr;
283 int ret = -ENODEV;
284
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
287 if (tr == this_tr) {
288 tr->ref++;
289 ret = 0;
290 break;
291 }
292 }
293 mutex_unlock(&trace_types_lock);
294
295 return ret;
296}
297
298static void __trace_array_put(struct trace_array *this_tr)
299{
300 WARN_ON(!this_tr->ref);
301 this_tr->ref--;
302}
303
304void trace_array_put(struct trace_array *this_tr)
305{
306 mutex_lock(&trace_types_lock);
307 __trace_array_put(this_tr);
308 mutex_unlock(&trace_types_lock);
309}
310
2425bcb9 311int call_filter_check_discard(struct trace_event_call *call, void *rec,
f306cc82
TZ
312 struct ring_buffer *buffer,
313 struct ring_buffer_event *event)
314{
315 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
316 !filter_match_preds(call->filter, rec)) {
0fc1b09f 317 __trace_event_discard_commit(buffer, event);
f306cc82
TZ
318 return 1;
319 }
320
321 return 0;
eb02ce01
TZ
322}
323
76c813e2
SRRH
324void trace_free_pid_list(struct trace_pid_list *pid_list)
325{
326 vfree(pid_list->pids);
327 kfree(pid_list);
328}
329
d8275c45
SR
330/**
331 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
332 * @filtered_pids: The list of pids to check
333 * @search_pid: The PID to find in @filtered_pids
334 *
335 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
336 */
337bool
338trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
339{
340 /*
341 * If pid_max changed after filtered_pids was created, we
342 * by default ignore all pids greater than the previous pid_max.
343 */
344 if (search_pid >= filtered_pids->pid_max)
345 return false;
346
347 return test_bit(search_pid, filtered_pids->pids);
348}
349
350/**
351 * trace_ignore_this_task - should a task be ignored for tracing
352 * @filtered_pids: The list of pids to check
353 * @task: The task that should be ignored if not filtered
354 *
355 * Checks if @task should be traced or not from @filtered_pids.
356 * Returns true if @task should *NOT* be traced.
357 * Returns false if @task should be traced.
358 */
359bool
360trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
361{
362 /*
363 * Return false, because if filtered_pids does not exist,
364 * all pids are good to trace.
365 */
366 if (!filtered_pids)
367 return false;
368
369 return !trace_find_filtered_pid(filtered_pids, task->pid);
370}
371
372/**
373 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
374 * @pid_list: The list to modify
375 * @self: The current task for fork or NULL for exit
376 * @task: The task to add or remove
377 *
378 * If adding a task, if @self is defined, the task is only added if @self
379 * is also included in @pid_list. This happens on fork and tasks should
380 * only be added when the parent is listed. If @self is NULL, then the
381 * @task pid will be removed from the list, which would happen on exit
382 * of a task.
383 */
384void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
385 struct task_struct *self,
386 struct task_struct *task)
387{
388 if (!pid_list)
389 return;
390
391 /* For forks, we only add if the forking task is listed */
392 if (self) {
393 if (!trace_find_filtered_pid(pid_list, self->pid))
394 return;
395 }
396
397 /* Sorry, but we don't support pid_max changing after setting */
398 if (task->pid >= pid_list->pid_max)
399 return;
400
401 /* "self" is set for forks, and NULL for exits */
402 if (self)
403 set_bit(task->pid, pid_list->pids);
404 else
405 clear_bit(task->pid, pid_list->pids);
406}
407
5cc8976b
SRRH
408/**
409 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
410 * @pid_list: The pid list to show
411 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
412 * @pos: The position of the file
413 *
414 * This is used by the seq_file "next" operation to iterate the pids
415 * listed in a trace_pid_list structure.
416 *
417 * Returns the pid+1 as we want to display pid of zero, but NULL would
418 * stop the iteration.
419 */
420void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
421{
422 unsigned long pid = (unsigned long)v;
423
424 (*pos)++;
425
426 /* pid already is +1 of the actual prevous bit */
427 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
428
429 /* Return pid + 1 to allow zero to be represented */
430 if (pid < pid_list->pid_max)
431 return (void *)(pid + 1);
432
433 return NULL;
434}
435
436/**
437 * trace_pid_start - Used for seq_file to start reading pid lists
438 * @pid_list: The pid list to show
439 * @pos: The position of the file
440 *
441 * This is used by seq_file "start" operation to start the iteration
442 * of listing pids.
443 *
444 * Returns the pid+1 as we want to display pid of zero, but NULL would
445 * stop the iteration.
446 */
447void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
448{
449 unsigned long pid;
450 loff_t l = 0;
451
452 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
453 if (pid >= pid_list->pid_max)
454 return NULL;
455
456 /* Return pid + 1 so that zero can be the exit value */
457 for (pid++; pid && l < *pos;
458 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
459 ;
460 return (void *)pid;
461}
462
463/**
464 * trace_pid_show - show the current pid in seq_file processing
465 * @m: The seq_file structure to write into
466 * @v: A void pointer of the pid (+1) value to display
467 *
468 * Can be directly used by seq_file operations to display the current
469 * pid value.
470 */
471int trace_pid_show(struct seq_file *m, void *v)
472{
473 unsigned long pid = (unsigned long)v - 1;
474
475 seq_printf(m, "%lu\n", pid);
476 return 0;
477}
478
76c813e2
SRRH
479/* 128 should be much more than enough */
480#define PID_BUF_SIZE 127
481
482int trace_pid_write(struct trace_pid_list *filtered_pids,
483 struct trace_pid_list **new_pid_list,
484 const char __user *ubuf, size_t cnt)
485{
486 struct trace_pid_list *pid_list;
487 struct trace_parser parser;
488 unsigned long val;
489 int nr_pids = 0;
490 ssize_t read = 0;
491 ssize_t ret = 0;
492 loff_t pos;
493 pid_t pid;
494
495 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
496 return -ENOMEM;
497
498 /*
499 * Always recreate a new array. The write is an all or nothing
500 * operation. Always create a new array when adding new pids by
501 * the user. If the operation fails, then the current list is
502 * not modified.
503 */
504 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
505 if (!pid_list)
506 return -ENOMEM;
507
508 pid_list->pid_max = READ_ONCE(pid_max);
509
510 /* Only truncating will shrink pid_max */
511 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
512 pid_list->pid_max = filtered_pids->pid_max;
513
514 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
515 if (!pid_list->pids) {
516 kfree(pid_list);
517 return -ENOMEM;
518 }
519
520 if (filtered_pids) {
521 /* copy the current bits to the new max */
67f20b08
WY
522 for_each_set_bit(pid, filtered_pids->pids,
523 filtered_pids->pid_max) {
76c813e2 524 set_bit(pid, pid_list->pids);
76c813e2
SRRH
525 nr_pids++;
526 }
527 }
528
529 while (cnt > 0) {
530
531 pos = 0;
532
533 ret = trace_get_user(&parser, ubuf, cnt, &pos);
534 if (ret < 0 || !trace_parser_loaded(&parser))
535 break;
536
537 read += ret;
538 ubuf += ret;
539 cnt -= ret;
540
541 parser.buffer[parser.idx] = 0;
542
543 ret = -EINVAL;
544 if (kstrtoul(parser.buffer, 0, &val))
545 break;
546 if (val >= pid_list->pid_max)
547 break;
548
549 pid = (pid_t)val;
550
551 set_bit(pid, pid_list->pids);
552 nr_pids++;
553
554 trace_parser_clear(&parser);
555 ret = 0;
556 }
557 trace_parser_put(&parser);
558
559 if (ret < 0) {
560 trace_free_pid_list(pid_list);
561 return ret;
562 }
563
564 if (!nr_pids) {
565 /* Cleared the list of pids */
566 trace_free_pid_list(pid_list);
567 read = ret;
568 pid_list = NULL;
569 }
570
571 *new_pid_list = pid_list;
572
573 return read;
574}
575
ad1438a0 576static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
37886f6a
SR
577{
578 u64 ts;
579
580 /* Early boot up does not have a buffer yet */
9457158b 581 if (!buf->buffer)
37886f6a
SR
582 return trace_clock_local();
583
9457158b
AL
584 ts = ring_buffer_time_stamp(buf->buffer, cpu);
585 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
37886f6a
SR
586
587 return ts;
588}
bc0c38d1 589
9457158b
AL
590cycle_t ftrace_now(int cpu)
591{
592 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
593}
594
10246fa3
SRRH
595/**
596 * tracing_is_enabled - Show if global_trace has been disabled
597 *
598 * Shows if the global trace has been enabled or not. It uses the
599 * mirror flag "buffer_disabled" to be used in fast paths such as for
600 * the irqsoff tracer. But it may be inaccurate due to races. If you
601 * need to know the accurate state, use tracing_is_on() which is a little
602 * slower, but accurate.
603 */
9036990d
SR
604int tracing_is_enabled(void)
605{
10246fa3
SRRH
606 /*
607 * For quick access (irqsoff uses this in fast path), just
608 * return the mirror variable of the state of the ring buffer.
609 * It's a little racy, but we don't really care.
610 */
611 smp_rmb();
612 return !global_trace.buffer_disabled;
9036990d
SR
613}
614
4fcdae83 615/*
3928a8a2
SR
616 * trace_buf_size is the size in bytes that is allocated
617 * for a buffer. Note, the number of bytes is always rounded
618 * to page size.
3f5a54e3
SR
619 *
620 * This number is purposely set to a low number of 16384.
621 * If the dump on oops happens, it will be much appreciated
622 * to not have to wait for all that output. Anyway this can be
623 * boot time and run time configurable.
4fcdae83 624 */
3928a8a2 625#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 626
3928a8a2 627static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 628
4fcdae83 629/* trace_types holds a link list of available tracers. */
bc0c38d1 630static struct tracer *trace_types __read_mostly;
4fcdae83 631
4fcdae83
SR
632/*
633 * trace_types_lock is used to protect the trace_types list.
4fcdae83 634 */
a8227415 635DEFINE_MUTEX(trace_types_lock);
4fcdae83 636
7e53bd42
LJ
637/*
638 * serialize the access of the ring buffer
639 *
640 * ring buffer serializes readers, but it is low level protection.
641 * The validity of the events (which returns by ring_buffer_peek() ..etc)
642 * are not protected by ring buffer.
643 *
644 * The content of events may become garbage if we allow other process consumes
645 * these events concurrently:
646 * A) the page of the consumed events may become a normal page
647 * (not reader page) in ring buffer, and this page will be rewrited
648 * by events producer.
649 * B) The page of the consumed events may become a page for splice_read,
650 * and this page will be returned to system.
651 *
652 * These primitives allow multi process access to different cpu ring buffer
653 * concurrently.
654 *
655 * These primitives don't distinguish read-only and read-consume access.
656 * Multi read-only access are also serialized.
657 */
658
659#ifdef CONFIG_SMP
660static DECLARE_RWSEM(all_cpu_access_lock);
661static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
662
663static inline void trace_access_lock(int cpu)
664{
ae3b5093 665 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
666 /* gain it for accessing the whole ring buffer. */
667 down_write(&all_cpu_access_lock);
668 } else {
669 /* gain it for accessing a cpu ring buffer. */
670
ae3b5093 671 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
672 down_read(&all_cpu_access_lock);
673
674 /* Secondly block other access to this @cpu ring buffer. */
675 mutex_lock(&per_cpu(cpu_access_lock, cpu));
676 }
677}
678
679static inline void trace_access_unlock(int cpu)
680{
ae3b5093 681 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
682 up_write(&all_cpu_access_lock);
683 } else {
684 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
685 up_read(&all_cpu_access_lock);
686 }
687}
688
689static inline void trace_access_lock_init(void)
690{
691 int cpu;
692
693 for_each_possible_cpu(cpu)
694 mutex_init(&per_cpu(cpu_access_lock, cpu));
695}
696
697#else
698
699static DEFINE_MUTEX(access_lock);
700
701static inline void trace_access_lock(int cpu)
702{
703 (void)cpu;
704 mutex_lock(&access_lock);
705}
706
707static inline void trace_access_unlock(int cpu)
708{
709 (void)cpu;
710 mutex_unlock(&access_lock);
711}
712
713static inline void trace_access_lock_init(void)
714{
715}
716
717#endif
718
d78a4614
SRRH
719#ifdef CONFIG_STACKTRACE
720static void __ftrace_trace_stack(struct ring_buffer *buffer,
721 unsigned long flags,
722 int skip, int pc, struct pt_regs *regs);
2d34f489
SRRH
723static inline void ftrace_trace_stack(struct trace_array *tr,
724 struct ring_buffer *buffer,
73dddbb5
SRRH
725 unsigned long flags,
726 int skip, int pc, struct pt_regs *regs);
ca475e83 727
d78a4614
SRRH
728#else
729static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
730 unsigned long flags,
731 int skip, int pc, struct pt_regs *regs)
732{
733}
2d34f489
SRRH
734static inline void ftrace_trace_stack(struct trace_array *tr,
735 struct ring_buffer *buffer,
73dddbb5
SRRH
736 unsigned long flags,
737 int skip, int pc, struct pt_regs *regs)
ca475e83
SRRH
738{
739}
740
d78a4614
SRRH
741#endif
742
3e9a8aad
SRRH
743static __always_inline void
744trace_event_setup(struct ring_buffer_event *event,
745 int type, unsigned long flags, int pc)
746{
747 struct trace_entry *ent = ring_buffer_event_data(event);
748
749 tracing_generic_entry_update(ent, flags, pc);
750 ent->type = type;
751}
752
753static __always_inline struct ring_buffer_event *
754__trace_buffer_lock_reserve(struct ring_buffer *buffer,
755 int type,
756 unsigned long len,
757 unsigned long flags, int pc)
758{
759 struct ring_buffer_event *event;
760
761 event = ring_buffer_lock_reserve(buffer, len);
762 if (event != NULL)
763 trace_event_setup(event, type, flags, pc);
764
765 return event;
766}
767
5280bcef 768static void tracer_tracing_on(struct trace_array *tr)
10246fa3
SRRH
769{
770 if (tr->trace_buffer.buffer)
771 ring_buffer_record_on(tr->trace_buffer.buffer);
772 /*
773 * This flag is looked at when buffers haven't been allocated
774 * yet, or by some tracers (like irqsoff), that just want to
775 * know if the ring buffer has been disabled, but it can handle
776 * races of where it gets disabled but we still do a record.
777 * As the check is in the fast path of the tracers, it is more
778 * important to be fast than accurate.
779 */
780 tr->buffer_disabled = 0;
781 /* Make the flag seen by readers */
782 smp_wmb();
783}
784
499e5470
SR
785/**
786 * tracing_on - enable tracing buffers
787 *
788 * This function enables tracing buffers that may have been
789 * disabled with tracing_off.
790 */
791void tracing_on(void)
792{
10246fa3 793 tracer_tracing_on(&global_trace);
499e5470
SR
794}
795EXPORT_SYMBOL_GPL(tracing_on);
796
52ffabe3
SRRH
797
798static __always_inline void
799__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
800{
801 __this_cpu_write(trace_cmdline_save, true);
802
803 /* If this is the temp buffer, we need to commit fully */
804 if (this_cpu_read(trace_buffered_event) == event) {
805 /* Length is in event->array[0] */
806 ring_buffer_write(buffer, event->array[0], &event->array[1]);
807 /* Release the temp buffer */
808 this_cpu_dec(trace_buffered_event_cnt);
809 } else
810 ring_buffer_unlock_commit(buffer, event);
811}
812
09ae7234
SRRH
813/**
814 * __trace_puts - write a constant string into the trace buffer.
815 * @ip: The address of the caller
816 * @str: The constant string to write
817 * @size: The size of the string.
818 */
819int __trace_puts(unsigned long ip, const char *str, int size)
820{
821 struct ring_buffer_event *event;
822 struct ring_buffer *buffer;
823 struct print_entry *entry;
824 unsigned long irq_flags;
825 int alloc;
8abfb872
J
826 int pc;
827
983f938a 828 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
829 return 0;
830
8abfb872 831 pc = preempt_count();
09ae7234 832
3132e107
SRRH
833 if (unlikely(tracing_selftest_running || tracing_disabled))
834 return 0;
835
09ae7234
SRRH
836 alloc = sizeof(*entry) + size + 2; /* possible \n added */
837
838 local_save_flags(irq_flags);
839 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
840 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
841 irq_flags, pc);
09ae7234
SRRH
842 if (!event)
843 return 0;
844
845 entry = ring_buffer_event_data(event);
846 entry->ip = ip;
847
848 memcpy(&entry->buf, str, size);
849
850 /* Add a newline if necessary */
851 if (entry->buf[size - 1] != '\n') {
852 entry->buf[size] = '\n';
853 entry->buf[size + 1] = '\0';
854 } else
855 entry->buf[size] = '\0';
856
857 __buffer_unlock_commit(buffer, event);
2d34f489 858 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
859
860 return size;
861}
862EXPORT_SYMBOL_GPL(__trace_puts);
863
864/**
865 * __trace_bputs - write the pointer to a constant string into trace buffer
866 * @ip: The address of the caller
867 * @str: The constant string to write to the buffer to
868 */
869int __trace_bputs(unsigned long ip, const char *str)
870{
871 struct ring_buffer_event *event;
872 struct ring_buffer *buffer;
873 struct bputs_entry *entry;
874 unsigned long irq_flags;
875 int size = sizeof(struct bputs_entry);
8abfb872
J
876 int pc;
877
983f938a 878 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
879 return 0;
880
8abfb872 881 pc = preempt_count();
09ae7234 882
3132e107
SRRH
883 if (unlikely(tracing_selftest_running || tracing_disabled))
884 return 0;
885
09ae7234
SRRH
886 local_save_flags(irq_flags);
887 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
888 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
889 irq_flags, pc);
09ae7234
SRRH
890 if (!event)
891 return 0;
892
893 entry = ring_buffer_event_data(event);
894 entry->ip = ip;
895 entry->str = str;
896
897 __buffer_unlock_commit(buffer, event);
2d34f489 898 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
899
900 return 1;
901}
902EXPORT_SYMBOL_GPL(__trace_bputs);
903
ad909e21
SRRH
904#ifdef CONFIG_TRACER_SNAPSHOT
905/**
906 * trace_snapshot - take a snapshot of the current buffer.
907 *
908 * This causes a swap between the snapshot buffer and the current live
909 * tracing buffer. You can use this to take snapshots of the live
910 * trace when some condition is triggered, but continue to trace.
911 *
912 * Note, make sure to allocate the snapshot with either
913 * a tracing_snapshot_alloc(), or by doing it manually
914 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
915 *
916 * If the snapshot buffer is not allocated, it will stop tracing.
917 * Basically making a permanent snapshot.
918 */
919void tracing_snapshot(void)
920{
921 struct trace_array *tr = &global_trace;
922 struct tracer *tracer = tr->current_trace;
923 unsigned long flags;
924
1b22e382
SRRH
925 if (in_nmi()) {
926 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
927 internal_trace_puts("*** snapshot is being ignored ***\n");
928 return;
929 }
930
ad909e21 931 if (!tr->allocated_snapshot) {
ca268da6
SRRH
932 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
933 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
934 tracing_off();
935 return;
936 }
937
938 /* Note, snapshot can not be used when the tracer uses it */
939 if (tracer->use_max_tr) {
ca268da6
SRRH
940 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
941 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
942 return;
943 }
944
945 local_irq_save(flags);
946 update_max_tr(tr, current, smp_processor_id());
947 local_irq_restore(flags);
948}
1b22e382 949EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
950
951static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
952 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
953static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
954
955static int alloc_snapshot(struct trace_array *tr)
956{
957 int ret;
958
959 if (!tr->allocated_snapshot) {
960
961 /* allocate spare buffer */
962 ret = resize_buffer_duplicate_size(&tr->max_buffer,
963 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
964 if (ret < 0)
965 return ret;
966
967 tr->allocated_snapshot = true;
968 }
969
970 return 0;
971}
972
ad1438a0 973static void free_snapshot(struct trace_array *tr)
3209cff4
SRRH
974{
975 /*
976 * We don't free the ring buffer. instead, resize it because
977 * The max_tr ring buffer has some state (e.g. ring->clock) and
978 * we want preserve it.
979 */
980 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
981 set_buffer_entries(&tr->max_buffer, 1);
982 tracing_reset_online_cpus(&tr->max_buffer);
983 tr->allocated_snapshot = false;
984}
ad909e21 985
93e31ffb
TZ
986/**
987 * tracing_alloc_snapshot - allocate snapshot buffer.
988 *
989 * This only allocates the snapshot buffer if it isn't already
990 * allocated - it doesn't also take a snapshot.
991 *
992 * This is meant to be used in cases where the snapshot buffer needs
993 * to be set up for events that can't sleep but need to be able to
994 * trigger a snapshot.
995 */
996int tracing_alloc_snapshot(void)
997{
998 struct trace_array *tr = &global_trace;
999 int ret;
1000
1001 ret = alloc_snapshot(tr);
1002 WARN_ON(ret < 0);
1003
1004 return ret;
1005}
1006EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1007
ad909e21
SRRH
1008/**
1009 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
1010 *
1011 * This is similar to trace_snapshot(), but it will allocate the
1012 * snapshot buffer if it isn't already allocated. Use this only
1013 * where it is safe to sleep, as the allocation may sleep.
1014 *
1015 * This causes a swap between the snapshot buffer and the current live
1016 * tracing buffer. You can use this to take snapshots of the live
1017 * trace when some condition is triggered, but continue to trace.
1018 */
1019void tracing_snapshot_alloc(void)
1020{
ad909e21
SRRH
1021 int ret;
1022
93e31ffb
TZ
1023 ret = tracing_alloc_snapshot();
1024 if (ret < 0)
3209cff4 1025 return;
ad909e21
SRRH
1026
1027 tracing_snapshot();
1028}
1b22e382 1029EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
1030#else
1031void tracing_snapshot(void)
1032{
1033 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1034}
1b22e382 1035EXPORT_SYMBOL_GPL(tracing_snapshot);
93e31ffb
TZ
1036int tracing_alloc_snapshot(void)
1037{
1038 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1039 return -ENODEV;
1040}
1041EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
ad909e21
SRRH
1042void tracing_snapshot_alloc(void)
1043{
1044 /* Give warning */
1045 tracing_snapshot();
1046}
1b22e382 1047EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
1048#endif /* CONFIG_TRACER_SNAPSHOT */
1049
5280bcef 1050static void tracer_tracing_off(struct trace_array *tr)
10246fa3
SRRH
1051{
1052 if (tr->trace_buffer.buffer)
1053 ring_buffer_record_off(tr->trace_buffer.buffer);
1054 /*
1055 * This flag is looked at when buffers haven't been allocated
1056 * yet, or by some tracers (like irqsoff), that just want to
1057 * know if the ring buffer has been disabled, but it can handle
1058 * races of where it gets disabled but we still do a record.
1059 * As the check is in the fast path of the tracers, it is more
1060 * important to be fast than accurate.
1061 */
1062 tr->buffer_disabled = 1;
1063 /* Make the flag seen by readers */
1064 smp_wmb();
1065}
1066
499e5470
SR
1067/**
1068 * tracing_off - turn off tracing buffers
1069 *
1070 * This function stops the tracing buffers from recording data.
1071 * It does not disable any overhead the tracers themselves may
1072 * be causing. This function simply causes all recording to
1073 * the ring buffers to fail.
1074 */
1075void tracing_off(void)
1076{
10246fa3 1077 tracer_tracing_off(&global_trace);
499e5470
SR
1078}
1079EXPORT_SYMBOL_GPL(tracing_off);
1080
de7edd31
SRRH
1081void disable_trace_on_warning(void)
1082{
1083 if (__disable_trace_on_warning)
1084 tracing_off();
1085}
1086
10246fa3
SRRH
1087/**
1088 * tracer_tracing_is_on - show real state of ring buffer enabled
1089 * @tr : the trace array to know if ring buffer is enabled
1090 *
1091 * Shows real state of the ring buffer if it is enabled or not.
1092 */
e7c15cd8 1093int tracer_tracing_is_on(struct trace_array *tr)
10246fa3
SRRH
1094{
1095 if (tr->trace_buffer.buffer)
1096 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1097 return !tr->buffer_disabled;
1098}
1099
499e5470
SR
1100/**
1101 * tracing_is_on - show state of ring buffers enabled
1102 */
1103int tracing_is_on(void)
1104{
10246fa3 1105 return tracer_tracing_is_on(&global_trace);
499e5470
SR
1106}
1107EXPORT_SYMBOL_GPL(tracing_is_on);
1108
3928a8a2 1109static int __init set_buf_size(char *str)
bc0c38d1 1110{
3928a8a2 1111 unsigned long buf_size;
c6caeeb1 1112
bc0c38d1
SR
1113 if (!str)
1114 return 0;
9d612bef 1115 buf_size = memparse(str, &str);
c6caeeb1 1116 /* nr_entries can not be zero */
9d612bef 1117 if (buf_size == 0)
c6caeeb1 1118 return 0;
3928a8a2 1119 trace_buf_size = buf_size;
bc0c38d1
SR
1120 return 1;
1121}
3928a8a2 1122__setup("trace_buf_size=", set_buf_size);
bc0c38d1 1123
0e950173
TB
1124static int __init set_tracing_thresh(char *str)
1125{
87abb3b1 1126 unsigned long threshold;
0e950173
TB
1127 int ret;
1128
1129 if (!str)
1130 return 0;
bcd83ea6 1131 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
1132 if (ret < 0)
1133 return 0;
87abb3b1 1134 tracing_thresh = threshold * 1000;
0e950173
TB
1135 return 1;
1136}
1137__setup("tracing_thresh=", set_tracing_thresh);
1138
57f50be1
SR
1139unsigned long nsecs_to_usecs(unsigned long nsecs)
1140{
1141 return nsecs / 1000;
1142}
1143
a3418a36
SRRH
1144/*
1145 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1146 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1147 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1148 * of strings in the order that the enums were defined.
1149 */
1150#undef C
1151#define C(a, b) b
1152
4fcdae83 1153/* These must match the bit postions in trace_iterator_flags */
bc0c38d1 1154static const char *trace_options[] = {
a3418a36 1155 TRACE_FLAGS
bc0c38d1
SR
1156 NULL
1157};
1158
5079f326
Z
1159static struct {
1160 u64 (*func)(void);
1161 const char *name;
8be0709f 1162 int in_ns; /* is this clock in nanoseconds? */
5079f326 1163} trace_clocks[] = {
1b3e5c09
TG
1164 { trace_clock_local, "local", 1 },
1165 { trace_clock_global, "global", 1 },
1166 { trace_clock_counter, "counter", 0 },
e7fda6c4 1167 { trace_clock_jiffies, "uptime", 0 },
1b3e5c09
TG
1168 { trace_clock, "perf", 1 },
1169 { ktime_get_mono_fast_ns, "mono", 1 },
aabfa5f2 1170 { ktime_get_raw_fast_ns, "mono_raw", 1 },
8cbd9cc6 1171 ARCH_TRACE_CLOCKS
5079f326
Z
1172};
1173
b63f39ea 1174/*
1175 * trace_parser_get_init - gets the buffer for trace parser
1176 */
1177int trace_parser_get_init(struct trace_parser *parser, int size)
1178{
1179 memset(parser, 0, sizeof(*parser));
1180
1181 parser->buffer = kmalloc(size, GFP_KERNEL);
1182 if (!parser->buffer)
1183 return 1;
1184
1185 parser->size = size;
1186 return 0;
1187}
1188
1189/*
1190 * trace_parser_put - frees the buffer for trace parser
1191 */
1192void trace_parser_put(struct trace_parser *parser)
1193{
1194 kfree(parser->buffer);
1195}
1196
1197/*
1198 * trace_get_user - reads the user input string separated by space
1199 * (matched by isspace(ch))
1200 *
1201 * For each string found the 'struct trace_parser' is updated,
1202 * and the function returns.
1203 *
1204 * Returns number of bytes read.
1205 *
1206 * See kernel/trace/trace.h for 'struct trace_parser' details.
1207 */
1208int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1209 size_t cnt, loff_t *ppos)
1210{
1211 char ch;
1212 size_t read = 0;
1213 ssize_t ret;
1214
1215 if (!*ppos)
1216 trace_parser_clear(parser);
1217
1218 ret = get_user(ch, ubuf++);
1219 if (ret)
1220 goto out;
1221
1222 read++;
1223 cnt--;
1224
1225 /*
1226 * The parser is not finished with the last write,
1227 * continue reading the user input without skipping spaces.
1228 */
1229 if (!parser->cont) {
1230 /* skip white space */
1231 while (cnt && isspace(ch)) {
1232 ret = get_user(ch, ubuf++);
1233 if (ret)
1234 goto out;
1235 read++;
1236 cnt--;
1237 }
1238
1239 /* only spaces were written */
1240 if (isspace(ch)) {
1241 *ppos += read;
1242 ret = read;
1243 goto out;
1244 }
1245
1246 parser->idx = 0;
1247 }
1248
1249 /* read the non-space input */
1250 while (cnt && !isspace(ch)) {
3c235a33 1251 if (parser->idx < parser->size - 1)
b63f39ea 1252 parser->buffer[parser->idx++] = ch;
1253 else {
1254 ret = -EINVAL;
1255 goto out;
1256 }
1257 ret = get_user(ch, ubuf++);
1258 if (ret)
1259 goto out;
1260 read++;
1261 cnt--;
1262 }
1263
1264 /* We either got finished input or we have to wait for another call. */
1265 if (isspace(ch)) {
1266 parser->buffer[parser->idx] = 0;
1267 parser->cont = false;
057db848 1268 } else if (parser->idx < parser->size - 1) {
b63f39ea 1269 parser->cont = true;
1270 parser->buffer[parser->idx++] = ch;
057db848
SR
1271 } else {
1272 ret = -EINVAL;
1273 goto out;
b63f39ea 1274 }
1275
1276 *ppos += read;
1277 ret = read;
1278
1279out:
1280 return ret;
1281}
1282
3a161d99 1283/* TODO add a seq_buf_to_buffer() */
b8b94265 1284static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
1285{
1286 int len;
3c56819b 1287
5ac48378 1288 if (trace_seq_used(s) <= s->seq.readpos)
3c56819b
EGM
1289 return -EBUSY;
1290
5ac48378 1291 len = trace_seq_used(s) - s->seq.readpos;
3c56819b
EGM
1292 if (cnt > len)
1293 cnt = len;
3a161d99 1294 memcpy(buf, s->buffer + s->seq.readpos, cnt);
3c56819b 1295
3a161d99 1296 s->seq.readpos += cnt;
3c56819b
EGM
1297 return cnt;
1298}
1299
0e950173
TB
1300unsigned long __read_mostly tracing_thresh;
1301
5d4a9dba 1302#ifdef CONFIG_TRACER_MAX_TRACE
5d4a9dba
SR
1303/*
1304 * Copy the new maximum trace into the separate maximum-trace
1305 * structure. (this way the maximum trace is permanently saved,
1306 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1307 */
1308static void
1309__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1310{
12883efb
SRRH
1311 struct trace_buffer *trace_buf = &tr->trace_buffer;
1312 struct trace_buffer *max_buf = &tr->max_buffer;
1313 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1314 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 1315
12883efb
SRRH
1316 max_buf->cpu = cpu;
1317 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 1318
6d9b3fa5 1319 max_data->saved_latency = tr->max_latency;
8248ac05
SR
1320 max_data->critical_start = data->critical_start;
1321 max_data->critical_end = data->critical_end;
5d4a9dba 1322
1acaa1b2 1323 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05 1324 max_data->pid = tsk->pid;
f17a5194
SRRH
1325 /*
1326 * If tsk == current, then use current_uid(), as that does not use
1327 * RCU. The irq tracer can be called out of RCU scope.
1328 */
1329 if (tsk == current)
1330 max_data->uid = current_uid();
1331 else
1332 max_data->uid = task_uid(tsk);
1333
8248ac05
SR
1334 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1335 max_data->policy = tsk->policy;
1336 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
1337
1338 /* record this tasks comm */
1339 tracing_record_cmdline(tsk);
1340}
1341
4fcdae83
SR
1342/**
1343 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1344 * @tr: tracer
1345 * @tsk: the task with the latency
1346 * @cpu: The cpu that initiated the trace.
1347 *
1348 * Flip the buffers between the @tr and the max_tr and record information
1349 * about which task was the cause of this latency.
1350 */
e309b41d 1351void
bc0c38d1
SR
1352update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1353{
2721e72d 1354 struct ring_buffer *buf;
bc0c38d1 1355
2b6080f2 1356 if (tr->stop_count)
b8de7bd1
SR
1357 return;
1358
4c11d7ae 1359 WARN_ON_ONCE(!irqs_disabled());
34600f0e 1360
45ad21ca 1361 if (!tr->allocated_snapshot) {
debdd57f 1362 /* Only the nop tracer should hit this when disabling */
2b6080f2 1363 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 1364 return;
debdd57f 1365 }
34600f0e 1366
0b9b12c1 1367 arch_spin_lock(&tr->max_lock);
3928a8a2 1368
12883efb
SRRH
1369 buf = tr->trace_buffer.buffer;
1370 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1371 tr->max_buffer.buffer = buf;
3928a8a2 1372
bc0c38d1 1373 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1374 arch_spin_unlock(&tr->max_lock);
bc0c38d1
SR
1375}
1376
1377/**
1378 * update_max_tr_single - only copy one trace over, and reset the rest
1379 * @tr - tracer
1380 * @tsk - task with the latency
1381 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
1382 *
1383 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 1384 */
e309b41d 1385void
bc0c38d1
SR
1386update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1387{
3928a8a2 1388 int ret;
bc0c38d1 1389
2b6080f2 1390 if (tr->stop_count)
b8de7bd1
SR
1391 return;
1392
4c11d7ae 1393 WARN_ON_ONCE(!irqs_disabled());
6c24499f 1394 if (!tr->allocated_snapshot) {
2930e04d 1395 /* Only the nop tracer should hit this when disabling */
9e8529af 1396 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 1397 return;
2930e04d 1398 }
ef710e10 1399
0b9b12c1 1400 arch_spin_lock(&tr->max_lock);
bc0c38d1 1401
12883efb 1402 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 1403
e8165dbb
SR
1404 if (ret == -EBUSY) {
1405 /*
1406 * We failed to swap the buffer due to a commit taking
1407 * place on this CPU. We fail to record, but we reset
1408 * the max trace buffer (no one writes directly to it)
1409 * and flag that it failed.
1410 */
12883efb 1411 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
1412 "Failed to swap buffers due to commit in progress\n");
1413 }
1414
e8165dbb 1415 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
1416
1417 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1418 arch_spin_unlock(&tr->max_lock);
bc0c38d1 1419}
5d4a9dba 1420#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 1421
e30f53aa 1422static int wait_on_pipe(struct trace_iterator *iter, bool full)
0d5c6e1c 1423{
15693458
SRRH
1424 /* Iterators are static, they should be filled or empty */
1425 if (trace_buffer_iter(iter, iter->cpu_file))
8b8b3683 1426 return 0;
0d5c6e1c 1427
e30f53aa
RV
1428 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1429 full);
0d5c6e1c
SR
1430}
1431
f4e781c0
SRRH
1432#ifdef CONFIG_FTRACE_STARTUP_TEST
1433static int run_tracer_selftest(struct tracer *type)
1434{
1435 struct trace_array *tr = &global_trace;
1436 struct tracer *saved_tracer = tr->current_trace;
1437 int ret;
0d5c6e1c 1438
f4e781c0
SRRH
1439 if (!type->selftest || tracing_selftest_disabled)
1440 return 0;
0d5c6e1c
SR
1441
1442 /*
f4e781c0
SRRH
1443 * Run a selftest on this tracer.
1444 * Here we reset the trace buffer, and set the current
1445 * tracer to be this tracer. The tracer can then run some
1446 * internal tracing to verify that everything is in order.
1447 * If we fail, we do not register this tracer.
0d5c6e1c 1448 */
f4e781c0 1449 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 1450
f4e781c0
SRRH
1451 tr->current_trace = type;
1452
1453#ifdef CONFIG_TRACER_MAX_TRACE
1454 if (type->use_max_tr) {
1455 /* If we expanded the buffers, make sure the max is expanded too */
1456 if (ring_buffer_expanded)
1457 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1458 RING_BUFFER_ALL_CPUS);
1459 tr->allocated_snapshot = true;
1460 }
1461#endif
1462
1463 /* the test is responsible for initializing and enabling */
1464 pr_info("Testing tracer %s: ", type->name);
1465 ret = type->selftest(type, tr);
1466 /* the test is responsible for resetting too */
1467 tr->current_trace = saved_tracer;
1468 if (ret) {
1469 printk(KERN_CONT "FAILED!\n");
1470 /* Add the warning after printing 'FAILED' */
1471 WARN_ON(1);
1472 return -1;
1473 }
1474 /* Only reset on passing, to avoid touching corrupted buffers */
1475 tracing_reset_online_cpus(&tr->trace_buffer);
1476
1477#ifdef CONFIG_TRACER_MAX_TRACE
1478 if (type->use_max_tr) {
1479 tr->allocated_snapshot = false;
0d5c6e1c 1480
f4e781c0
SRRH
1481 /* Shrink the max buffer again */
1482 if (ring_buffer_expanded)
1483 ring_buffer_resize(tr->max_buffer.buffer, 1,
1484 RING_BUFFER_ALL_CPUS);
1485 }
1486#endif
1487
1488 printk(KERN_CONT "PASSED\n");
1489 return 0;
1490}
1491#else
1492static inline int run_tracer_selftest(struct tracer *type)
1493{
1494 return 0;
0d5c6e1c 1495}
f4e781c0 1496#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1497
41d9c0be
SRRH
1498static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1499
a4d1e688
JW
1500static void __init apply_trace_boot_options(void);
1501
4fcdae83
SR
1502/**
1503 * register_tracer - register a tracer with the ftrace system.
1504 * @type - the plugin for the tracer
1505 *
1506 * Register a new plugin tracer.
1507 */
a4d1e688 1508int __init register_tracer(struct tracer *type)
bc0c38d1
SR
1509{
1510 struct tracer *t;
bc0c38d1
SR
1511 int ret = 0;
1512
1513 if (!type->name) {
1514 pr_info("Tracer must have a name\n");
1515 return -1;
1516 }
1517
24a461d5 1518 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1519 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1520 return -1;
1521 }
1522
bc0c38d1 1523 mutex_lock(&trace_types_lock);
86fa2f60 1524
8e1b82e0
FW
1525 tracing_selftest_running = true;
1526
bc0c38d1
SR
1527 for (t = trace_types; t; t = t->next) {
1528 if (strcmp(type->name, t->name) == 0) {
1529 /* already found */
ee6c2c1b 1530 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1531 type->name);
1532 ret = -1;
1533 goto out;
1534 }
1535 }
1536
adf9f195
FW
1537 if (!type->set_flag)
1538 type->set_flag = &dummy_set_flag;
d39cdd20
CH
1539 if (!type->flags) {
1540 /*allocate a dummy tracer_flags*/
1541 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
c8ca003b
CH
1542 if (!type->flags) {
1543 ret = -ENOMEM;
1544 goto out;
1545 }
d39cdd20
CH
1546 type->flags->val = 0;
1547 type->flags->opts = dummy_tracer_opt;
1548 } else
adf9f195
FW
1549 if (!type->flags->opts)
1550 type->flags->opts = dummy_tracer_opt;
6eaaa5d5 1551
d39cdd20
CH
1552 /* store the tracer for __set_tracer_option */
1553 type->flags->trace = type;
1554
f4e781c0
SRRH
1555 ret = run_tracer_selftest(type);
1556 if (ret < 0)
1557 goto out;
60a11774 1558
bc0c38d1
SR
1559 type->next = trace_types;
1560 trace_types = type;
41d9c0be 1561 add_tracer_options(&global_trace, type);
60a11774 1562
bc0c38d1 1563 out:
8e1b82e0 1564 tracing_selftest_running = false;
bc0c38d1
SR
1565 mutex_unlock(&trace_types_lock);
1566
dac74940
SR
1567 if (ret || !default_bootup_tracer)
1568 goto out_unlock;
1569
ee6c2c1b 1570 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1571 goto out_unlock;
1572
1573 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1574 /* Do we want this tracer to start on bootup? */
607e2ea1 1575 tracing_set_tracer(&global_trace, type->name);
dac74940 1576 default_bootup_tracer = NULL;
a4d1e688
JW
1577
1578 apply_trace_boot_options();
1579
dac74940 1580 /* disable other selftests, since this will break it. */
55034cd6 1581 tracing_selftest_disabled = true;
b2821ae6 1582#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1583 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1584 type->name);
b2821ae6 1585#endif
b2821ae6 1586
dac74940 1587 out_unlock:
bc0c38d1
SR
1588 return ret;
1589}
1590
12883efb 1591void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1592{
12883efb 1593 struct ring_buffer *buffer = buf->buffer;
f633903a 1594
a5416411
HT
1595 if (!buffer)
1596 return;
1597
f633903a
SR
1598 ring_buffer_record_disable(buffer);
1599
1600 /* Make sure all commits have finished */
1601 synchronize_sched();
68179686 1602 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1603
1604 ring_buffer_record_enable(buffer);
1605}
1606
12883efb 1607void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1608{
12883efb 1609 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1610 int cpu;
1611
a5416411
HT
1612 if (!buffer)
1613 return;
1614
621968cd
SR
1615 ring_buffer_record_disable(buffer);
1616
1617 /* Make sure all commits have finished */
1618 synchronize_sched();
1619
9457158b 1620 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
213cc060
PE
1621
1622 for_each_online_cpu(cpu)
68179686 1623 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1624
1625 ring_buffer_record_enable(buffer);
213cc060
PE
1626}
1627
09d8091c 1628/* Must have trace_types_lock held */
873c642f 1629void tracing_reset_all_online_cpus(void)
9456f0fa 1630{
873c642f
SRRH
1631 struct trace_array *tr;
1632
873c642f 1633 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
12883efb
SRRH
1634 tracing_reset_online_cpus(&tr->trace_buffer);
1635#ifdef CONFIG_TRACER_MAX_TRACE
1636 tracing_reset_online_cpus(&tr->max_buffer);
1637#endif
873c642f 1638 }
9456f0fa
SR
1639}
1640
939c7a4f 1641#define SAVED_CMDLINES_DEFAULT 128
2c7eea4c 1642#define NO_CMDLINE_MAP UINT_MAX
edc35bd7 1643static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
939c7a4f
YY
1644struct saved_cmdlines_buffer {
1645 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1646 unsigned *map_cmdline_to_pid;
1647 unsigned cmdline_num;
1648 int cmdline_idx;
1649 char *saved_cmdlines;
1650};
1651static struct saved_cmdlines_buffer *savedcmd;
25b0b44a 1652
25b0b44a 1653/* temporary disable recording */
4fd27358 1654static atomic_t trace_record_cmdline_disabled __read_mostly;
bc0c38d1 1655
939c7a4f
YY
1656static inline char *get_saved_cmdlines(int idx)
1657{
1658 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1659}
1660
1661static inline void set_cmdline(int idx, const char *cmdline)
bc0c38d1 1662{
939c7a4f
YY
1663 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1664}
1665
1666static int allocate_cmdlines_buffer(unsigned int val,
1667 struct saved_cmdlines_buffer *s)
1668{
1669 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1670 GFP_KERNEL);
1671 if (!s->map_cmdline_to_pid)
1672 return -ENOMEM;
1673
1674 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1675 if (!s->saved_cmdlines) {
1676 kfree(s->map_cmdline_to_pid);
1677 return -ENOMEM;
1678 }
1679
1680 s->cmdline_idx = 0;
1681 s->cmdline_num = val;
1682 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1683 sizeof(s->map_pid_to_cmdline));
1684 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1685 val * sizeof(*s->map_cmdline_to_pid));
1686
1687 return 0;
1688}
1689
1690static int trace_create_savedcmd(void)
1691{
1692 int ret;
1693
a6af8fbf 1694 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
939c7a4f
YY
1695 if (!savedcmd)
1696 return -ENOMEM;
1697
1698 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1699 if (ret < 0) {
1700 kfree(savedcmd);
1701 savedcmd = NULL;
1702 return -ENOMEM;
1703 }
1704
1705 return 0;
bc0c38d1
SR
1706}
1707
b5130b1e
CE
1708int is_tracing_stopped(void)
1709{
2b6080f2 1710 return global_trace.stop_count;
b5130b1e
CE
1711}
1712
0f048701
SR
1713/**
1714 * tracing_start - quick start of the tracer
1715 *
1716 * If tracing is enabled but was stopped by tracing_stop,
1717 * this will start the tracer back up.
1718 */
1719void tracing_start(void)
1720{
1721 struct ring_buffer *buffer;
1722 unsigned long flags;
1723
1724 if (tracing_disabled)
1725 return;
1726
2b6080f2
SR
1727 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1728 if (--global_trace.stop_count) {
1729 if (global_trace.stop_count < 0) {
b06a8301
SR
1730 /* Someone screwed up their debugging */
1731 WARN_ON_ONCE(1);
2b6080f2 1732 global_trace.stop_count = 0;
b06a8301 1733 }
0f048701
SR
1734 goto out;
1735 }
1736
a2f80714 1737 /* Prevent the buffers from switching */
0b9b12c1 1738 arch_spin_lock(&global_trace.max_lock);
0f048701 1739
12883efb 1740 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1741 if (buffer)
1742 ring_buffer_record_enable(buffer);
1743
12883efb
SRRH
1744#ifdef CONFIG_TRACER_MAX_TRACE
1745 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1746 if (buffer)
1747 ring_buffer_record_enable(buffer);
12883efb 1748#endif
0f048701 1749
0b9b12c1 1750 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1751
0f048701 1752 out:
2b6080f2
SR
1753 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1754}
1755
1756static void tracing_start_tr(struct trace_array *tr)
1757{
1758 struct ring_buffer *buffer;
1759 unsigned long flags;
1760
1761 if (tracing_disabled)
1762 return;
1763
1764 /* If global, we need to also start the max tracer */
1765 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1766 return tracing_start();
1767
1768 raw_spin_lock_irqsave(&tr->start_lock, flags);
1769
1770 if (--tr->stop_count) {
1771 if (tr->stop_count < 0) {
1772 /* Someone screwed up their debugging */
1773 WARN_ON_ONCE(1);
1774 tr->stop_count = 0;
1775 }
1776 goto out;
1777 }
1778
12883efb 1779 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1780 if (buffer)
1781 ring_buffer_record_enable(buffer);
1782
1783 out:
1784 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1785}
1786
1787/**
1788 * tracing_stop - quick stop of the tracer
1789 *
1790 * Light weight way to stop tracing. Use in conjunction with
1791 * tracing_start.
1792 */
1793void tracing_stop(void)
1794{
1795 struct ring_buffer *buffer;
1796 unsigned long flags;
1797
2b6080f2
SR
1798 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1799 if (global_trace.stop_count++)
0f048701
SR
1800 goto out;
1801
a2f80714 1802 /* Prevent the buffers from switching */
0b9b12c1 1803 arch_spin_lock(&global_trace.max_lock);
a2f80714 1804
12883efb 1805 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1806 if (buffer)
1807 ring_buffer_record_disable(buffer);
1808
12883efb
SRRH
1809#ifdef CONFIG_TRACER_MAX_TRACE
1810 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1811 if (buffer)
1812 ring_buffer_record_disable(buffer);
12883efb 1813#endif
0f048701 1814
0b9b12c1 1815 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1816
0f048701 1817 out:
2b6080f2
SR
1818 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1819}
1820
1821static void tracing_stop_tr(struct trace_array *tr)
1822{
1823 struct ring_buffer *buffer;
1824 unsigned long flags;
1825
1826 /* If global, we need to also stop the max tracer */
1827 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1828 return tracing_stop();
1829
1830 raw_spin_lock_irqsave(&tr->start_lock, flags);
1831 if (tr->stop_count++)
1832 goto out;
1833
12883efb 1834 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1835 if (buffer)
1836 ring_buffer_record_disable(buffer);
1837
1838 out:
1839 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1840}
1841
e309b41d 1842void trace_stop_cmdline_recording(void);
bc0c38d1 1843
379cfdac 1844static int trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1845{
a635cf04 1846 unsigned pid, idx;
bc0c38d1
SR
1847
1848 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
379cfdac 1849 return 0;
bc0c38d1
SR
1850
1851 /*
1852 * It's not the end of the world if we don't get
1853 * the lock, but we also don't want to spin
1854 * nor do we want to disable interrupts,
1855 * so if we miss here, then better luck next time.
1856 */
0199c4e6 1857 if (!arch_spin_trylock(&trace_cmdline_lock))
379cfdac 1858 return 0;
bc0c38d1 1859
939c7a4f 1860 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2c7eea4c 1861 if (idx == NO_CMDLINE_MAP) {
939c7a4f 1862 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
bc0c38d1 1863
a635cf04
CE
1864 /*
1865 * Check whether the cmdline buffer at idx has a pid
1866 * mapped. We are going to overwrite that entry so we
1867 * need to clear the map_pid_to_cmdline. Otherwise we
1868 * would read the new comm for the old pid.
1869 */
939c7a4f 1870 pid = savedcmd->map_cmdline_to_pid[idx];
a635cf04 1871 if (pid != NO_CMDLINE_MAP)
939c7a4f 1872 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1873
939c7a4f
YY
1874 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1875 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
bc0c38d1 1876
939c7a4f 1877 savedcmd->cmdline_idx = idx;
bc0c38d1
SR
1878 }
1879
939c7a4f 1880 set_cmdline(idx, tsk->comm);
bc0c38d1 1881
0199c4e6 1882 arch_spin_unlock(&trace_cmdline_lock);
379cfdac
SRRH
1883
1884 return 1;
bc0c38d1
SR
1885}
1886
4c27e756 1887static void __trace_find_cmdline(int pid, char comm[])
bc0c38d1 1888{
bc0c38d1
SR
1889 unsigned map;
1890
4ca53085
SR
1891 if (!pid) {
1892 strcpy(comm, "<idle>");
1893 return;
1894 }
bc0c38d1 1895
74bf4076
SR
1896 if (WARN_ON_ONCE(pid < 0)) {
1897 strcpy(comm, "<XXX>");
1898 return;
1899 }
1900
4ca53085
SR
1901 if (pid > PID_MAX_DEFAULT) {
1902 strcpy(comm, "<...>");
1903 return;
1904 }
bc0c38d1 1905
939c7a4f 1906 map = savedcmd->map_pid_to_cmdline[pid];
50d88758 1907 if (map != NO_CMDLINE_MAP)
939c7a4f 1908 strcpy(comm, get_saved_cmdlines(map));
50d88758
TG
1909 else
1910 strcpy(comm, "<...>");
4c27e756
SRRH
1911}
1912
1913void trace_find_cmdline(int pid, char comm[])
1914{
1915 preempt_disable();
1916 arch_spin_lock(&trace_cmdline_lock);
1917
1918 __trace_find_cmdline(pid, comm);
bc0c38d1 1919
0199c4e6 1920 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1921 preempt_enable();
bc0c38d1
SR
1922}
1923
e309b41d 1924void tracing_record_cmdline(struct task_struct *tsk)
bc0c38d1 1925{
0fb9656d 1926 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
bc0c38d1
SR
1927 return;
1928
7ffbd48d
SR
1929 if (!__this_cpu_read(trace_cmdline_save))
1930 return;
1931
379cfdac
SRRH
1932 if (trace_save_cmdline(tsk))
1933 __this_cpu_write(trace_cmdline_save, false);
bc0c38d1
SR
1934}
1935
45dcd8b8 1936void
38697053
SR
1937tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1938 int pc)
bc0c38d1
SR
1939{
1940 struct task_struct *tsk = current;
bc0c38d1 1941
777e208d
SR
1942 entry->preempt_count = pc & 0xff;
1943 entry->pid = (tsk) ? tsk->pid : 0;
1944 entry->flags =
9244489a 1945#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 1946 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
1947#else
1948 TRACE_FLAG_IRQS_NOSUPPORT |
1949#endif
7e6867bf 1950 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
bc0c38d1
SR
1951 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1952 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
e5137b50
PZ
1953 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1954 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
bc0c38d1 1955}
f413cdb8 1956EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 1957
e77405ad
SR
1958struct ring_buffer_event *
1959trace_buffer_lock_reserve(struct ring_buffer *buffer,
1960 int type,
1961 unsigned long len,
1962 unsigned long flags, int pc)
51a763dd 1963{
3e9a8aad 1964 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
0fc1b09f
SRRH
1965}
1966
1967DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1968DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1969static int trace_buffered_event_ref;
1970
1971/**
1972 * trace_buffered_event_enable - enable buffering events
1973 *
1974 * When events are being filtered, it is quicker to use a temporary
1975 * buffer to write the event data into if there's a likely chance
1976 * that it will not be committed. The discard of the ring buffer
1977 * is not as fast as committing, and is much slower than copying
1978 * a commit.
1979 *
1980 * When an event is to be filtered, allocate per cpu buffers to
1981 * write the event data into, and if the event is filtered and discarded
1982 * it is simply dropped, otherwise, the entire data is to be committed
1983 * in one shot.
1984 */
1985void trace_buffered_event_enable(void)
1986{
1987 struct ring_buffer_event *event;
1988 struct page *page;
1989 int cpu;
51a763dd 1990
0fc1b09f
SRRH
1991 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1992
1993 if (trace_buffered_event_ref++)
1994 return;
1995
1996 for_each_tracing_cpu(cpu) {
1997 page = alloc_pages_node(cpu_to_node(cpu),
1998 GFP_KERNEL | __GFP_NORETRY, 0);
1999 if (!page)
2000 goto failed;
2001
2002 event = page_address(page);
2003 memset(event, 0, sizeof(*event));
2004
2005 per_cpu(trace_buffered_event, cpu) = event;
2006
2007 preempt_disable();
2008 if (cpu == smp_processor_id() &&
2009 this_cpu_read(trace_buffered_event) !=
2010 per_cpu(trace_buffered_event, cpu))
2011 WARN_ON_ONCE(1);
2012 preempt_enable();
51a763dd
ACM
2013 }
2014
0fc1b09f
SRRH
2015 return;
2016 failed:
2017 trace_buffered_event_disable();
2018}
2019
2020static void enable_trace_buffered_event(void *data)
2021{
2022 /* Probably not needed, but do it anyway */
2023 smp_rmb();
2024 this_cpu_dec(trace_buffered_event_cnt);
2025}
2026
2027static void disable_trace_buffered_event(void *data)
2028{
2029 this_cpu_inc(trace_buffered_event_cnt);
2030}
2031
2032/**
2033 * trace_buffered_event_disable - disable buffering events
2034 *
2035 * When a filter is removed, it is faster to not use the buffered
2036 * events, and to commit directly into the ring buffer. Free up
2037 * the temp buffers when there are no more users. This requires
2038 * special synchronization with current events.
2039 */
2040void trace_buffered_event_disable(void)
2041{
2042 int cpu;
2043
2044 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2045
2046 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2047 return;
2048
2049 if (--trace_buffered_event_ref)
2050 return;
2051
2052 preempt_disable();
2053 /* For each CPU, set the buffer as used. */
2054 smp_call_function_many(tracing_buffer_mask,
2055 disable_trace_buffered_event, NULL, 1);
2056 preempt_enable();
2057
2058 /* Wait for all current users to finish */
2059 synchronize_sched();
2060
2061 for_each_tracing_cpu(cpu) {
2062 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2063 per_cpu(trace_buffered_event, cpu) = NULL;
2064 }
2065 /*
2066 * Make sure trace_buffered_event is NULL before clearing
2067 * trace_buffered_event_cnt.
2068 */
2069 smp_wmb();
2070
2071 preempt_disable();
2072 /* Do the work on each cpu */
2073 smp_call_function_many(tracing_buffer_mask,
2074 enable_trace_buffered_event, NULL, 1);
2075 preempt_enable();
51a763dd 2076}
51a763dd 2077
2c4a33ab
SRRH
2078static struct ring_buffer *temp_buffer;
2079
ccb469a1
SR
2080struct ring_buffer_event *
2081trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
7f1d2f82 2082 struct trace_event_file *trace_file,
ccb469a1
SR
2083 int type, unsigned long len,
2084 unsigned long flags, int pc)
2085{
2c4a33ab 2086 struct ring_buffer_event *entry;
0fc1b09f 2087 int val;
2c4a33ab 2088
7f1d2f82 2089 *current_rb = trace_file->tr->trace_buffer.buffer;
0fc1b09f
SRRH
2090
2091 if ((trace_file->flags &
2092 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2093 (entry = this_cpu_read(trace_buffered_event))) {
2094 /* Try to use the per cpu buffer first */
2095 val = this_cpu_inc_return(trace_buffered_event_cnt);
2096 if (val == 1) {
2097 trace_event_setup(entry, type, flags, pc);
2098 entry->array[0] = len;
2099 return entry;
2100 }
2101 this_cpu_dec(trace_buffered_event_cnt);
2102 }
2103
3e9a8aad
SRRH
2104 entry = __trace_buffer_lock_reserve(*current_rb,
2105 type, len, flags, pc);
2c4a33ab
SRRH
2106 /*
2107 * If tracing is off, but we have triggers enabled
2108 * we still need to look at the event data. Use the temp_buffer
2109 * to store the trace event for the tigger to use. It's recusive
2110 * safe and will not be recorded anywhere.
2111 */
5d6ad960 2112 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2c4a33ab 2113 *current_rb = temp_buffer;
3e9a8aad
SRRH
2114 entry = __trace_buffer_lock_reserve(*current_rb,
2115 type, len, flags, pc);
2c4a33ab
SRRH
2116 }
2117 return entry;
ccb469a1
SR
2118}
2119EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2120
42391745
SRRH
2121static DEFINE_SPINLOCK(tracepoint_iter_lock);
2122static DEFINE_MUTEX(tracepoint_printk_mutex);
2123
2124static void output_printk(struct trace_event_buffer *fbuffer)
2125{
2126 struct trace_event_call *event_call;
2127 struct trace_event *event;
2128 unsigned long flags;
2129 struct trace_iterator *iter = tracepoint_print_iter;
2130
2131 /* We should never get here if iter is NULL */
2132 if (WARN_ON_ONCE(!iter))
2133 return;
2134
2135 event_call = fbuffer->trace_file->event_call;
2136 if (!event_call || !event_call->event.funcs ||
2137 !event_call->event.funcs->trace)
2138 return;
2139
2140 event = &fbuffer->trace_file->event_call->event;
2141
2142 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2143 trace_seq_init(&iter->seq);
2144 iter->ent = fbuffer->entry;
2145 event_call->event.funcs->trace(iter, 0, event);
2146 trace_seq_putc(&iter->seq, 0);
2147 printk("%s", iter->seq.buffer);
2148
2149 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2150}
2151
2152int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2153 void __user *buffer, size_t *lenp,
2154 loff_t *ppos)
2155{
2156 int save_tracepoint_printk;
2157 int ret;
2158
2159 mutex_lock(&tracepoint_printk_mutex);
2160 save_tracepoint_printk = tracepoint_printk;
2161
2162 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2163
2164 /*
2165 * This will force exiting early, as tracepoint_printk
2166 * is always zero when tracepoint_printk_iter is not allocated
2167 */
2168 if (!tracepoint_print_iter)
2169 tracepoint_printk = 0;
2170
2171 if (save_tracepoint_printk == tracepoint_printk)
2172 goto out;
2173
2174 if (tracepoint_printk)
2175 static_key_enable(&tracepoint_printk_key.key);
2176 else
2177 static_key_disable(&tracepoint_printk_key.key);
2178
2179 out:
2180 mutex_unlock(&tracepoint_printk_mutex);
2181
2182 return ret;
2183}
2184
2185void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2186{
2187 if (static_key_false(&tracepoint_printk_key.key))
2188 output_printk(fbuffer);
2189
2190 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2191 fbuffer->event, fbuffer->entry,
2192 fbuffer->flags, fbuffer->pc);
2193}
2194EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2195
b7f0c959
SRRH
2196void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2197 struct ring_buffer *buffer,
0d5c6e1c
SR
2198 struct ring_buffer_event *event,
2199 unsigned long flags, int pc,
2200 struct pt_regs *regs)
1fd8df2c 2201{
7ffbd48d 2202 __buffer_unlock_commit(buffer, event);
1fd8df2c 2203
be54f69c
SRRH
2204 /*
2205 * If regs is not set, then skip the following callers:
2206 * trace_buffer_unlock_commit_regs
2207 * event_trigger_unlock_commit
2208 * trace_event_buffer_commit
2209 * trace_event_raw_event_sched_switch
2210 * Note, we can still get here via blktrace, wakeup tracer
2211 * and mmiotrace, but that's ok if they lose a function or
2212 * two. They are that meaningful.
2213 */
2214 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
1fd8df2c
MH
2215 ftrace_trace_userstack(buffer, flags, pc);
2216}
1fd8df2c 2217
52ffabe3
SRRH
2218/*
2219 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2220 */
2221void
2222trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2223 struct ring_buffer_event *event)
2224{
2225 __buffer_unlock_commit(buffer, event);
2226}
2227
478409dd
CZ
2228static void
2229trace_process_export(struct trace_export *export,
2230 struct ring_buffer_event *event)
2231{
2232 struct trace_entry *entry;
2233 unsigned int size = 0;
2234
2235 entry = ring_buffer_event_data(event);
2236 size = ring_buffer_event_length(event);
2237 export->write(entry, size);
2238}
2239
2240static DEFINE_MUTEX(ftrace_export_lock);
2241
2242static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2243
2244static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2245
2246static inline void ftrace_exports_enable(void)
2247{
2248 static_branch_enable(&ftrace_exports_enabled);
2249}
2250
2251static inline void ftrace_exports_disable(void)
2252{
2253 static_branch_disable(&ftrace_exports_enabled);
2254}
2255
2256void ftrace_exports(struct ring_buffer_event *event)
2257{
2258 struct trace_export *export;
2259
2260 preempt_disable_notrace();
2261
2262 export = rcu_dereference_raw_notrace(ftrace_exports_list);
2263 while (export) {
2264 trace_process_export(export, event);
2265 export = rcu_dereference_raw_notrace(export->next);
2266 }
2267
2268 preempt_enable_notrace();
2269}
2270
2271static inline void
2272add_trace_export(struct trace_export **list, struct trace_export *export)
2273{
2274 rcu_assign_pointer(export->next, *list);
2275 /*
2276 * We are entering export into the list but another
2277 * CPU might be walking that list. We need to make sure
2278 * the export->next pointer is valid before another CPU sees
2279 * the export pointer included into the list.
2280 */
2281 rcu_assign_pointer(*list, export);
2282}
2283
2284static inline int
2285rm_trace_export(struct trace_export **list, struct trace_export *export)
2286{
2287 struct trace_export **p;
2288
2289 for (p = list; *p != NULL; p = &(*p)->next)
2290 if (*p == export)
2291 break;
2292
2293 if (*p != export)
2294 return -1;
2295
2296 rcu_assign_pointer(*p, (*p)->next);
2297
2298 return 0;
2299}
2300
2301static inline void
2302add_ftrace_export(struct trace_export **list, struct trace_export *export)
2303{
2304 if (*list == NULL)
2305 ftrace_exports_enable();
2306
2307 add_trace_export(list, export);
2308}
2309
2310static inline int
2311rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2312{
2313 int ret;
2314
2315 ret = rm_trace_export(list, export);
2316 if (*list == NULL)
2317 ftrace_exports_disable();
2318
2319 return ret;
2320}
2321
2322int register_ftrace_export(struct trace_export *export)
2323{
2324 if (WARN_ON_ONCE(!export->write))
2325 return -1;
2326
2327 mutex_lock(&ftrace_export_lock);
2328
2329 add_ftrace_export(&ftrace_exports_list, export);
2330
2331 mutex_unlock(&ftrace_export_lock);
2332
2333 return 0;
2334}
2335EXPORT_SYMBOL_GPL(register_ftrace_export);
2336
2337int unregister_ftrace_export(struct trace_export *export)
2338{
2339 int ret;
2340
2341 mutex_lock(&ftrace_export_lock);
2342
2343 ret = rm_ftrace_export(&ftrace_exports_list, export);
2344
2345 mutex_unlock(&ftrace_export_lock);
2346
2347 return ret;
2348}
2349EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2350
e309b41d 2351void
7be42151 2352trace_function(struct trace_array *tr,
38697053
SR
2353 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2354 int pc)
bc0c38d1 2355{
2425bcb9 2356 struct trace_event_call *call = &event_function;
12883efb 2357 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 2358 struct ring_buffer_event *event;
777e208d 2359 struct ftrace_entry *entry;
bc0c38d1 2360
3e9a8aad
SRRH
2361 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2362 flags, pc);
3928a8a2
SR
2363 if (!event)
2364 return;
2365 entry = ring_buffer_event_data(event);
777e208d
SR
2366 entry->ip = ip;
2367 entry->parent_ip = parent_ip;
e1112b4d 2368
478409dd
CZ
2369 if (!call_filter_check_discard(call, entry, buffer, event)) {
2370 if (static_branch_unlikely(&ftrace_exports_enabled))
2371 ftrace_exports(event);
7ffbd48d 2372 __buffer_unlock_commit(buffer, event);
478409dd 2373 }
bc0c38d1
SR
2374}
2375
c0a0d0d3 2376#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
2377
2378#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2379struct ftrace_stack {
2380 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2381};
2382
2383static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2384static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2385
e77405ad 2386static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 2387 unsigned long flags,
1fd8df2c 2388 int skip, int pc, struct pt_regs *regs)
86387f7e 2389{
2425bcb9 2390 struct trace_event_call *call = &event_kernel_stack;
3928a8a2 2391 struct ring_buffer_event *event;
777e208d 2392 struct stack_entry *entry;
86387f7e 2393 struct stack_trace trace;
4a9bd3f1
SR
2394 int use_stack;
2395 int size = FTRACE_STACK_ENTRIES;
2396
2397 trace.nr_entries = 0;
2398 trace.skip = skip;
2399
be54f69c
SRRH
2400 /*
2401 * Add two, for this function and the call to save_stack_trace()
2402 * If regs is set, then these functions will not be in the way.
2403 */
2404 if (!regs)
2405 trace.skip += 2;
2406
4a9bd3f1
SR
2407 /*
2408 * Since events can happen in NMIs there's no safe way to
2409 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2410 * or NMI comes in, it will just have to use the default
2411 * FTRACE_STACK_SIZE.
2412 */
2413 preempt_disable_notrace();
2414
82146529 2415 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
2416 /*
2417 * We don't need any atomic variables, just a barrier.
2418 * If an interrupt comes in, we don't care, because it would
2419 * have exited and put the counter back to what we want.
2420 * We just need a barrier to keep gcc from moving things
2421 * around.
2422 */
2423 barrier();
2424 if (use_stack == 1) {
bdffd893 2425 trace.entries = this_cpu_ptr(ftrace_stack.calls);
4a9bd3f1
SR
2426 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2427
2428 if (regs)
2429 save_stack_trace_regs(regs, &trace);
2430 else
2431 save_stack_trace(&trace);
2432
2433 if (trace.nr_entries > size)
2434 size = trace.nr_entries;
2435 } else
2436 /* From now on, use_stack is a boolean */
2437 use_stack = 0;
2438
2439 size *= sizeof(unsigned long);
86387f7e 2440
3e9a8aad
SRRH
2441 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2442 sizeof(*entry) + size, flags, pc);
3928a8a2 2443 if (!event)
4a9bd3f1
SR
2444 goto out;
2445 entry = ring_buffer_event_data(event);
86387f7e 2446
4a9bd3f1
SR
2447 memset(&entry->caller, 0, size);
2448
2449 if (use_stack)
2450 memcpy(&entry->caller, trace.entries,
2451 trace.nr_entries * sizeof(unsigned long));
2452 else {
2453 trace.max_entries = FTRACE_STACK_ENTRIES;
2454 trace.entries = entry->caller;
2455 if (regs)
2456 save_stack_trace_regs(regs, &trace);
2457 else
2458 save_stack_trace(&trace);
2459 }
2460
2461 entry->size = trace.nr_entries;
86387f7e 2462
f306cc82 2463 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2464 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
2465
2466 out:
2467 /* Again, don't let gcc optimize things here */
2468 barrier();
82146529 2469 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
2470 preempt_enable_notrace();
2471
f0a920d5
IM
2472}
2473
2d34f489
SRRH
2474static inline void ftrace_trace_stack(struct trace_array *tr,
2475 struct ring_buffer *buffer,
73dddbb5
SRRH
2476 unsigned long flags,
2477 int skip, int pc, struct pt_regs *regs)
53614991 2478{
2d34f489 2479 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
53614991
SR
2480 return;
2481
73dddbb5 2482 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
53614991
SR
2483}
2484
c0a0d0d3
FW
2485void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2486 int pc)
38697053 2487{
12883efb 2488 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
38697053
SR
2489}
2490
03889384
SR
2491/**
2492 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 2493 * @skip: Number of functions to skip (helper handlers)
03889384 2494 */
c142be8e 2495void trace_dump_stack(int skip)
03889384
SR
2496{
2497 unsigned long flags;
2498
2499 if (tracing_disabled || tracing_selftest_running)
e36c5458 2500 return;
03889384
SR
2501
2502 local_save_flags(flags);
2503
c142be8e
SRRH
2504 /*
2505 * Skip 3 more, seems to get us at the caller of
2506 * this function.
2507 */
2508 skip += 3;
2509 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2510 flags, skip, preempt_count(), NULL);
03889384
SR
2511}
2512
91e86e56
SR
2513static DEFINE_PER_CPU(int, user_stack_count);
2514
e77405ad
SR
2515void
2516ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 2517{
2425bcb9 2518 struct trace_event_call *call = &event_user_stack;
8d7c6a96 2519 struct ring_buffer_event *event;
02b67518
TE
2520 struct userstack_entry *entry;
2521 struct stack_trace trace;
02b67518 2522
983f938a 2523 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
02b67518
TE
2524 return;
2525
b6345879
SR
2526 /*
2527 * NMIs can not handle page faults, even with fix ups.
2528 * The save user stack can (and often does) fault.
2529 */
2530 if (unlikely(in_nmi()))
2531 return;
02b67518 2532
91e86e56
SR
2533 /*
2534 * prevent recursion, since the user stack tracing may
2535 * trigger other kernel events.
2536 */
2537 preempt_disable();
2538 if (__this_cpu_read(user_stack_count))
2539 goto out;
2540
2541 __this_cpu_inc(user_stack_count);
2542
3e9a8aad
SRRH
2543 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2544 sizeof(*entry), flags, pc);
02b67518 2545 if (!event)
1dbd1951 2546 goto out_drop_count;
02b67518 2547 entry = ring_buffer_event_data(event);
02b67518 2548
48659d31 2549 entry->tgid = current->tgid;
02b67518
TE
2550 memset(&entry->caller, 0, sizeof(entry->caller));
2551
2552 trace.nr_entries = 0;
2553 trace.max_entries = FTRACE_STACK_ENTRIES;
2554 trace.skip = 0;
2555 trace.entries = entry->caller;
2556
2557 save_stack_trace_user(&trace);
f306cc82 2558 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2559 __buffer_unlock_commit(buffer, event);
91e86e56 2560
1dbd1951 2561 out_drop_count:
91e86e56 2562 __this_cpu_dec(user_stack_count);
91e86e56
SR
2563 out:
2564 preempt_enable();
02b67518
TE
2565}
2566
4fd27358
HE
2567#ifdef UNUSED
2568static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 2569{
7be42151 2570 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 2571}
4fd27358 2572#endif /* UNUSED */
02b67518 2573
c0a0d0d3
FW
2574#endif /* CONFIG_STACKTRACE */
2575
07d777fe
SR
2576/* created for use with alloc_percpu */
2577struct trace_buffer_struct {
e2ace001
AL
2578 int nesting;
2579 char buffer[4][TRACE_BUF_SIZE];
07d777fe
SR
2580};
2581
2582static struct trace_buffer_struct *trace_percpu_buffer;
07d777fe
SR
2583
2584/*
e2ace001
AL
2585 * Thise allows for lockless recording. If we're nested too deeply, then
2586 * this returns NULL.
07d777fe
SR
2587 */
2588static char *get_trace_buf(void)
2589{
e2ace001 2590 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
07d777fe 2591
e2ace001 2592 if (!buffer || buffer->nesting >= 4)
07d777fe
SR
2593 return NULL;
2594
e2ace001
AL
2595 return &buffer->buffer[buffer->nesting++][0];
2596}
2597
2598static void put_trace_buf(void)
2599{
2600 this_cpu_dec(trace_percpu_buffer->nesting);
07d777fe
SR
2601}
2602
2603static int alloc_percpu_trace_buffer(void)
2604{
2605 struct trace_buffer_struct *buffers;
07d777fe
SR
2606
2607 buffers = alloc_percpu(struct trace_buffer_struct);
e2ace001
AL
2608 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2609 return -ENOMEM;
07d777fe
SR
2610
2611 trace_percpu_buffer = buffers;
07d777fe 2612 return 0;
07d777fe
SR
2613}
2614
81698831
SR
2615static int buffers_allocated;
2616
07d777fe
SR
2617void trace_printk_init_buffers(void)
2618{
07d777fe
SR
2619 if (buffers_allocated)
2620 return;
2621
2622 if (alloc_percpu_trace_buffer())
2623 return;
2624
2184db46
SR
2625 /* trace_printk() is for debug use only. Don't use it in production. */
2626
a395d6a7
JP
2627 pr_warn("\n");
2628 pr_warn("**********************************************************\n");
2629 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2630 pr_warn("** **\n");
2631 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2632 pr_warn("** **\n");
2633 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2634 pr_warn("** unsafe for production use. **\n");
2635 pr_warn("** **\n");
2636 pr_warn("** If you see this message and you are not debugging **\n");
2637 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2638 pr_warn("** **\n");
2639 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2640 pr_warn("**********************************************************\n");
07d777fe 2641
b382ede6
SR
2642 /* Expand the buffers to set size */
2643 tracing_update_buffers();
2644
07d777fe 2645 buffers_allocated = 1;
81698831
SR
2646
2647 /*
2648 * trace_printk_init_buffers() can be called by modules.
2649 * If that happens, then we need to start cmdline recording
2650 * directly here. If the global_trace.buffer is already
2651 * allocated here, then this was called by module code.
2652 */
12883efb 2653 if (global_trace.trace_buffer.buffer)
81698831
SR
2654 tracing_start_cmdline_record();
2655}
2656
2657void trace_printk_start_comm(void)
2658{
2659 /* Start tracing comms if trace printk is set */
2660 if (!buffers_allocated)
2661 return;
2662 tracing_start_cmdline_record();
2663}
2664
2665static void trace_printk_start_stop_comm(int enabled)
2666{
2667 if (!buffers_allocated)
2668 return;
2669
2670 if (enabled)
2671 tracing_start_cmdline_record();
2672 else
2673 tracing_stop_cmdline_record();
07d777fe
SR
2674}
2675
769b0441 2676/**
48ead020 2677 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
2678 *
2679 */
40ce74f1 2680int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 2681{
2425bcb9 2682 struct trace_event_call *call = &event_bprint;
769b0441 2683 struct ring_buffer_event *event;
e77405ad 2684 struct ring_buffer *buffer;
769b0441 2685 struct trace_array *tr = &global_trace;
48ead020 2686 struct bprint_entry *entry;
769b0441 2687 unsigned long flags;
07d777fe
SR
2688 char *tbuffer;
2689 int len = 0, size, pc;
769b0441
FW
2690
2691 if (unlikely(tracing_selftest_running || tracing_disabled))
2692 return 0;
2693
2694 /* Don't pollute graph traces with trace_vprintk internals */
2695 pause_graph_tracing();
2696
2697 pc = preempt_count();
5168ae50 2698 preempt_disable_notrace();
769b0441 2699
07d777fe
SR
2700 tbuffer = get_trace_buf();
2701 if (!tbuffer) {
2702 len = 0;
e2ace001 2703 goto out_nobuffer;
07d777fe 2704 }
769b0441 2705
07d777fe 2706 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 2707
07d777fe
SR
2708 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2709 goto out;
769b0441 2710
07d777fe 2711 local_save_flags(flags);
769b0441 2712 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 2713 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
2714 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2715 flags, pc);
769b0441 2716 if (!event)
07d777fe 2717 goto out;
769b0441
FW
2718 entry = ring_buffer_event_data(event);
2719 entry->ip = ip;
769b0441
FW
2720 entry->fmt = fmt;
2721
07d777fe 2722 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
f306cc82 2723 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2724 __buffer_unlock_commit(buffer, event);
2d34f489 2725 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
d931369b 2726 }
769b0441 2727
769b0441 2728out:
e2ace001
AL
2729 put_trace_buf();
2730
2731out_nobuffer:
5168ae50 2732 preempt_enable_notrace();
769b0441
FW
2733 unpause_graph_tracing();
2734
2735 return len;
2736}
48ead020
FW
2737EXPORT_SYMBOL_GPL(trace_vbprintk);
2738
12883efb
SRRH
2739static int
2740__trace_array_vprintk(struct ring_buffer *buffer,
2741 unsigned long ip, const char *fmt, va_list args)
48ead020 2742{
2425bcb9 2743 struct trace_event_call *call = &event_print;
48ead020 2744 struct ring_buffer_event *event;
07d777fe 2745 int len = 0, size, pc;
48ead020 2746 struct print_entry *entry;
07d777fe
SR
2747 unsigned long flags;
2748 char *tbuffer;
48ead020
FW
2749
2750 if (tracing_disabled || tracing_selftest_running)
2751 return 0;
2752
07d777fe
SR
2753 /* Don't pollute graph traces with trace_vprintk internals */
2754 pause_graph_tracing();
2755
48ead020
FW
2756 pc = preempt_count();
2757 preempt_disable_notrace();
48ead020 2758
07d777fe
SR
2759
2760 tbuffer = get_trace_buf();
2761 if (!tbuffer) {
2762 len = 0;
e2ace001 2763 goto out_nobuffer;
07d777fe 2764 }
48ead020 2765
3558a5ac 2766 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
48ead020 2767
07d777fe 2768 local_save_flags(flags);
48ead020 2769 size = sizeof(*entry) + len + 1;
3e9a8aad
SRRH
2770 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2771 flags, pc);
48ead020 2772 if (!event)
07d777fe 2773 goto out;
48ead020 2774 entry = ring_buffer_event_data(event);
c13d2f7c 2775 entry->ip = ip;
48ead020 2776
3558a5ac 2777 memcpy(&entry->buf, tbuffer, len + 1);
f306cc82 2778 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2779 __buffer_unlock_commit(buffer, event);
2d34f489 2780 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
d931369b 2781 }
e2ace001
AL
2782
2783out:
2784 put_trace_buf();
2785
2786out_nobuffer:
48ead020 2787 preempt_enable_notrace();
07d777fe 2788 unpause_graph_tracing();
48ead020
FW
2789
2790 return len;
2791}
659372d3 2792
12883efb
SRRH
2793int trace_array_vprintk(struct trace_array *tr,
2794 unsigned long ip, const char *fmt, va_list args)
2795{
2796 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2797}
2798
2799int trace_array_printk(struct trace_array *tr,
2800 unsigned long ip, const char *fmt, ...)
2801{
2802 int ret;
2803 va_list ap;
2804
983f938a 2805 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
2806 return 0;
2807
2808 va_start(ap, fmt);
2809 ret = trace_array_vprintk(tr, ip, fmt, ap);
2810 va_end(ap);
2811 return ret;
2812}
2813
2814int trace_array_printk_buf(struct ring_buffer *buffer,
2815 unsigned long ip, const char *fmt, ...)
2816{
2817 int ret;
2818 va_list ap;
2819
983f938a 2820 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
2821 return 0;
2822
2823 va_start(ap, fmt);
2824 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2825 va_end(ap);
2826 return ret;
2827}
2828
659372d3
SR
2829int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2830{
a813a159 2831 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 2832}
769b0441
FW
2833EXPORT_SYMBOL_GPL(trace_vprintk);
2834
e2ac8ef5 2835static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 2836{
6d158a81
SR
2837 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2838
5a90f577 2839 iter->idx++;
6d158a81
SR
2840 if (buf_iter)
2841 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
2842}
2843
e309b41d 2844static struct trace_entry *
bc21b478
SR
2845peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2846 unsigned long *lost_events)
dd0e545f 2847{
3928a8a2 2848 struct ring_buffer_event *event;
6d158a81 2849 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 2850
d769041f
SR
2851 if (buf_iter)
2852 event = ring_buffer_iter_peek(buf_iter, ts);
2853 else
12883efb 2854 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 2855 lost_events);
d769041f 2856
4a9bd3f1
SR
2857 if (event) {
2858 iter->ent_size = ring_buffer_event_length(event);
2859 return ring_buffer_event_data(event);
2860 }
2861 iter->ent_size = 0;
2862 return NULL;
dd0e545f 2863}
d769041f 2864
dd0e545f 2865static struct trace_entry *
bc21b478
SR
2866__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2867 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 2868{
12883efb 2869 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 2870 struct trace_entry *ent, *next = NULL;
aa27497c 2871 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 2872 int cpu_file = iter->cpu_file;
3928a8a2 2873 u64 next_ts = 0, ts;
bc0c38d1 2874 int next_cpu = -1;
12b5da34 2875 int next_size = 0;
bc0c38d1
SR
2876 int cpu;
2877
b04cc6b1
FW
2878 /*
2879 * If we are in a per_cpu trace file, don't bother by iterating over
2880 * all cpu and peek directly.
2881 */
ae3b5093 2882 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
2883 if (ring_buffer_empty_cpu(buffer, cpu_file))
2884 return NULL;
bc21b478 2885 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
2886 if (ent_cpu)
2887 *ent_cpu = cpu_file;
2888
2889 return ent;
2890 }
2891
ab46428c 2892 for_each_tracing_cpu(cpu) {
dd0e545f 2893
3928a8a2
SR
2894 if (ring_buffer_empty_cpu(buffer, cpu))
2895 continue;
dd0e545f 2896
bc21b478 2897 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 2898
cdd31cd2
IM
2899 /*
2900 * Pick the entry with the smallest timestamp:
2901 */
3928a8a2 2902 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
2903 next = ent;
2904 next_cpu = cpu;
3928a8a2 2905 next_ts = ts;
bc21b478 2906 next_lost = lost_events;
12b5da34 2907 next_size = iter->ent_size;
bc0c38d1
SR
2908 }
2909 }
2910
12b5da34
SR
2911 iter->ent_size = next_size;
2912
bc0c38d1
SR
2913 if (ent_cpu)
2914 *ent_cpu = next_cpu;
2915
3928a8a2
SR
2916 if (ent_ts)
2917 *ent_ts = next_ts;
2918
bc21b478
SR
2919 if (missing_events)
2920 *missing_events = next_lost;
2921
bc0c38d1
SR
2922 return next;
2923}
2924
dd0e545f 2925/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
2926struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2927 int *ent_cpu, u64 *ent_ts)
bc0c38d1 2928{
bc21b478 2929 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
2930}
2931
2932/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 2933void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 2934{
bc21b478
SR
2935 iter->ent = __find_next_entry(iter, &iter->cpu,
2936 &iter->lost_events, &iter->ts);
dd0e545f 2937
3928a8a2 2938 if (iter->ent)
e2ac8ef5 2939 trace_iterator_increment(iter);
dd0e545f 2940
3928a8a2 2941 return iter->ent ? iter : NULL;
b3806b43 2942}
bc0c38d1 2943
e309b41d 2944static void trace_consume(struct trace_iterator *iter)
b3806b43 2945{
12883efb 2946 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 2947 &iter->lost_events);
bc0c38d1
SR
2948}
2949
e309b41d 2950static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
2951{
2952 struct trace_iterator *iter = m->private;
bc0c38d1 2953 int i = (int)*pos;
4e3c3333 2954 void *ent;
bc0c38d1 2955
a63ce5b3
SR
2956 WARN_ON_ONCE(iter->leftover);
2957
bc0c38d1
SR
2958 (*pos)++;
2959
2960 /* can't go backwards */
2961 if (iter->idx > i)
2962 return NULL;
2963
2964 if (iter->idx < 0)
955b61e5 2965 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2966 else
2967 ent = iter;
2968
2969 while (ent && iter->idx < i)
955b61e5 2970 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
2971
2972 iter->pos = *pos;
2973
bc0c38d1
SR
2974 return ent;
2975}
2976
955b61e5 2977void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 2978{
2f26ebd5
SR
2979 struct ring_buffer_event *event;
2980 struct ring_buffer_iter *buf_iter;
2981 unsigned long entries = 0;
2982 u64 ts;
2983
12883efb 2984 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 2985
6d158a81
SR
2986 buf_iter = trace_buffer_iter(iter, cpu);
2987 if (!buf_iter)
2f26ebd5
SR
2988 return;
2989
2f26ebd5
SR
2990 ring_buffer_iter_reset(buf_iter);
2991
2992 /*
2993 * We could have the case with the max latency tracers
2994 * that a reset never took place on a cpu. This is evident
2995 * by the timestamp being before the start of the buffer.
2996 */
2997 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 2998 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
2999 break;
3000 entries++;
3001 ring_buffer_read(buf_iter, NULL);
3002 }
3003
12883efb 3004 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
3005}
3006
d7350c3f 3007/*
d7350c3f
FW
3008 * The current tracer is copied to avoid a global locking
3009 * all around.
3010 */
bc0c38d1
SR
3011static void *s_start(struct seq_file *m, loff_t *pos)
3012{
3013 struct trace_iterator *iter = m->private;
2b6080f2 3014 struct trace_array *tr = iter->tr;
b04cc6b1 3015 int cpu_file = iter->cpu_file;
bc0c38d1
SR
3016 void *p = NULL;
3017 loff_t l = 0;
3928a8a2 3018 int cpu;
bc0c38d1 3019
2fd196ec
HT
3020 /*
3021 * copy the tracer to avoid using a global lock all around.
3022 * iter->trace is a copy of current_trace, the pointer to the
3023 * name may be used instead of a strcmp(), as iter->trace->name
3024 * will point to the same string as current_trace->name.
3025 */
bc0c38d1 3026 mutex_lock(&trace_types_lock);
2b6080f2
SR
3027 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3028 *iter->trace = *tr->current_trace;
d7350c3f 3029 mutex_unlock(&trace_types_lock);
bc0c38d1 3030
12883efb 3031#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3032 if (iter->snapshot && iter->trace->use_max_tr)
3033 return ERR_PTR(-EBUSY);
12883efb 3034#endif
debdd57f
HT
3035
3036 if (!iter->snapshot)
3037 atomic_inc(&trace_record_cmdline_disabled);
bc0c38d1 3038
bc0c38d1
SR
3039 if (*pos != iter->pos) {
3040 iter->ent = NULL;
3041 iter->cpu = 0;
3042 iter->idx = -1;
3043
ae3b5093 3044 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3045 for_each_tracing_cpu(cpu)
2f26ebd5 3046 tracing_iter_reset(iter, cpu);
b04cc6b1 3047 } else
2f26ebd5 3048 tracing_iter_reset(iter, cpu_file);
bc0c38d1 3049
ac91d854 3050 iter->leftover = 0;
bc0c38d1
SR
3051 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3052 ;
3053
3054 } else {
a63ce5b3
SR
3055 /*
3056 * If we overflowed the seq_file before, then we want
3057 * to just reuse the trace_seq buffer again.
3058 */
3059 if (iter->leftover)
3060 p = iter;
3061 else {
3062 l = *pos - 1;
3063 p = s_next(m, p, &l);
3064 }
bc0c38d1
SR
3065 }
3066
4f535968 3067 trace_event_read_lock();
7e53bd42 3068 trace_access_lock(cpu_file);
bc0c38d1
SR
3069 return p;
3070}
3071
3072static void s_stop(struct seq_file *m, void *p)
3073{
7e53bd42
LJ
3074 struct trace_iterator *iter = m->private;
3075
12883efb 3076#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3077 if (iter->snapshot && iter->trace->use_max_tr)
3078 return;
12883efb 3079#endif
debdd57f
HT
3080
3081 if (!iter->snapshot)
3082 atomic_dec(&trace_record_cmdline_disabled);
12883efb 3083
7e53bd42 3084 trace_access_unlock(iter->cpu_file);
4f535968 3085 trace_event_read_unlock();
bc0c38d1
SR
3086}
3087
39eaf7ef 3088static void
12883efb
SRRH
3089get_total_entries(struct trace_buffer *buf,
3090 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
3091{
3092 unsigned long count;
3093 int cpu;
3094
3095 *total = 0;
3096 *entries = 0;
3097
3098 for_each_tracing_cpu(cpu) {
12883efb 3099 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
3100 /*
3101 * If this buffer has skipped entries, then we hold all
3102 * entries for the trace and we need to ignore the
3103 * ones before the time stamp.
3104 */
12883efb
SRRH
3105 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3106 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
3107 /* total is the same as the entries */
3108 *total += count;
3109 } else
3110 *total += count +
12883efb 3111 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
3112 *entries += count;
3113 }
3114}
3115
e309b41d 3116static void print_lat_help_header(struct seq_file *m)
bc0c38d1 3117{
d79ac28f
RV
3118 seq_puts(m, "# _------=> CPU# \n"
3119 "# / _-----=> irqs-off \n"
3120 "# | / _----=> need-resched \n"
3121 "# || / _---=> hardirq/softirq \n"
3122 "# ||| / _--=> preempt-depth \n"
3123 "# |||| / delay \n"
3124 "# cmd pid ||||| time | caller \n"
3125 "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
3126}
3127
12883efb 3128static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 3129{
39eaf7ef
SR
3130 unsigned long total;
3131 unsigned long entries;
3132
12883efb 3133 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
3134 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3135 entries, total, num_online_cpus());
3136 seq_puts(m, "#\n");
3137}
3138
12883efb 3139static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
39eaf7ef 3140{
12883efb 3141 print_event_info(buf, m);
d79ac28f
RV
3142 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
3143 "# | | | | |\n");
bc0c38d1
SR
3144}
3145
12883efb 3146static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
77271ce4 3147{
12883efb 3148 print_event_info(buf, m);
d79ac28f
RV
3149 seq_puts(m, "# _-----=> irqs-off\n"
3150 "# / _----=> need-resched\n"
3151 "# | / _---=> hardirq/softirq\n"
3152 "# || / _--=> preempt-depth\n"
3153 "# ||| / delay\n"
3154 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
3155 "# | | | |||| | |\n");
77271ce4 3156}
bc0c38d1 3157
62b915f1 3158void
bc0c38d1
SR
3159print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3160{
983f938a 3161 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
3162 struct trace_buffer *buf = iter->trace_buffer;
3163 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 3164 struct tracer *type = iter->trace;
39eaf7ef
SR
3165 unsigned long entries;
3166 unsigned long total;
bc0c38d1
SR
3167 const char *name = "preemption";
3168
d840f718 3169 name = type->name;
bc0c38d1 3170
12883efb 3171 get_total_entries(buf, &total, &entries);
bc0c38d1 3172
888b55dc 3173 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 3174 name, UTS_RELEASE);
888b55dc 3175 seq_puts(m, "# -----------------------------------"
bc0c38d1 3176 "---------------------------------\n");
888b55dc 3177 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 3178 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 3179 nsecs_to_usecs(data->saved_latency),
bc0c38d1 3180 entries,
4c11d7ae 3181 total,
12883efb 3182 buf->cpu,
bc0c38d1
SR
3183#if defined(CONFIG_PREEMPT_NONE)
3184 "server",
3185#elif defined(CONFIG_PREEMPT_VOLUNTARY)
3186 "desktop",
b5c21b45 3187#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
3188 "preempt",
3189#else
3190 "unknown",
3191#endif
3192 /* These are reserved for later use */
3193 0, 0, 0, 0);
3194#ifdef CONFIG_SMP
3195 seq_printf(m, " #P:%d)\n", num_online_cpus());
3196#else
3197 seq_puts(m, ")\n");
3198#endif
888b55dc
KM
3199 seq_puts(m, "# -----------------\n");
3200 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 3201 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
3202 data->comm, data->pid,
3203 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 3204 data->policy, data->rt_priority);
888b55dc 3205 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
3206
3207 if (data->critical_start) {
888b55dc 3208 seq_puts(m, "# => started at: ");
214023c3
SR
3209 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3210 trace_print_seq(m, &iter->seq);
888b55dc 3211 seq_puts(m, "\n# => ended at: ");
214023c3
SR
3212 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3213 trace_print_seq(m, &iter->seq);
8248ac05 3214 seq_puts(m, "\n#\n");
bc0c38d1
SR
3215 }
3216
888b55dc 3217 seq_puts(m, "#\n");
bc0c38d1
SR
3218}
3219
a309720c
SR
3220static void test_cpu_buff_start(struct trace_iterator *iter)
3221{
3222 struct trace_seq *s = &iter->seq;
983f938a 3223 struct trace_array *tr = iter->tr;
a309720c 3224
983f938a 3225 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
12ef7d44
SR
3226 return;
3227
3228 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3229 return;
3230
919cd979 3231 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
3232 return;
3233
12883efb 3234 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
3235 return;
3236
919cd979
SL
3237 if (iter->started)
3238 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
3239
3240 /* Don't print started cpu buffer for the first entry of the trace */
3241 if (iter->idx > 1)
3242 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3243 iter->cpu);
a309720c
SR
3244}
3245
2c4f035f 3246static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 3247{
983f938a 3248 struct trace_array *tr = iter->tr;
214023c3 3249 struct trace_seq *s = &iter->seq;
983f938a 3250 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 3251 struct trace_entry *entry;
f633cef0 3252 struct trace_event *event;
bc0c38d1 3253
4e3c3333 3254 entry = iter->ent;
dd0e545f 3255
a309720c
SR
3256 test_cpu_buff_start(iter);
3257
c4a8e8be 3258 event = ftrace_find_event(entry->type);
bc0c38d1 3259
983f938a 3260 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3261 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3262 trace_print_lat_context(iter);
3263 else
3264 trace_print_context(iter);
c4a8e8be 3265 }
bc0c38d1 3266
19a7fe20
SRRH
3267 if (trace_seq_has_overflowed(s))
3268 return TRACE_TYPE_PARTIAL_LINE;
3269
268ccda0 3270 if (event)
a9a57763 3271 return event->funcs->trace(iter, sym_flags, event);
d9793bd8 3272
19a7fe20 3273 trace_seq_printf(s, "Unknown type %d\n", entry->type);
02b67518 3274
19a7fe20 3275 return trace_handle_return(s);
bc0c38d1
SR
3276}
3277
2c4f035f 3278static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3 3279{
983f938a 3280 struct trace_array *tr = iter->tr;
f9896bf3
IM
3281 struct trace_seq *s = &iter->seq;
3282 struct trace_entry *entry;
f633cef0 3283 struct trace_event *event;
f9896bf3
IM
3284
3285 entry = iter->ent;
dd0e545f 3286
983f938a 3287 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
19a7fe20
SRRH
3288 trace_seq_printf(s, "%d %d %llu ",
3289 entry->pid, iter->cpu, iter->ts);
3290
3291 if (trace_seq_has_overflowed(s))
3292 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3 3293
f633cef0 3294 event = ftrace_find_event(entry->type);
268ccda0 3295 if (event)
a9a57763 3296 return event->funcs->raw(iter, 0, event);
d9793bd8 3297
19a7fe20 3298 trace_seq_printf(s, "%d ?\n", entry->type);
777e208d 3299
19a7fe20 3300 return trace_handle_return(s);
f9896bf3
IM
3301}
3302
2c4f035f 3303static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec 3304{
983f938a 3305 struct trace_array *tr = iter->tr;
5e3ca0ec
IM
3306 struct trace_seq *s = &iter->seq;
3307 unsigned char newline = '\n';
3308 struct trace_entry *entry;
f633cef0 3309 struct trace_event *event;
5e3ca0ec
IM
3310
3311 entry = iter->ent;
dd0e545f 3312
983f938a 3313 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3314 SEQ_PUT_HEX_FIELD(s, entry->pid);
3315 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3316 SEQ_PUT_HEX_FIELD(s, iter->ts);
3317 if (trace_seq_has_overflowed(s))
3318 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3319 }
5e3ca0ec 3320
f633cef0 3321 event = ftrace_find_event(entry->type);
268ccda0 3322 if (event) {
a9a57763 3323 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
3324 if (ret != TRACE_TYPE_HANDLED)
3325 return ret;
3326 }
7104f300 3327
19a7fe20 3328 SEQ_PUT_FIELD(s, newline);
5e3ca0ec 3329
19a7fe20 3330 return trace_handle_return(s);
5e3ca0ec
IM
3331}
3332
2c4f035f 3333static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa 3334{
983f938a 3335 struct trace_array *tr = iter->tr;
cb0f12aa
IM
3336 struct trace_seq *s = &iter->seq;
3337 struct trace_entry *entry;
f633cef0 3338 struct trace_event *event;
cb0f12aa
IM
3339
3340 entry = iter->ent;
dd0e545f 3341
983f938a 3342 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3343 SEQ_PUT_FIELD(s, entry->pid);
3344 SEQ_PUT_FIELD(s, iter->cpu);
3345 SEQ_PUT_FIELD(s, iter->ts);
3346 if (trace_seq_has_overflowed(s))
3347 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3348 }
cb0f12aa 3349
f633cef0 3350 event = ftrace_find_event(entry->type);
a9a57763
SR
3351 return event ? event->funcs->binary(iter, 0, event) :
3352 TRACE_TYPE_HANDLED;
cb0f12aa
IM
3353}
3354
62b915f1 3355int trace_empty(struct trace_iterator *iter)
bc0c38d1 3356{
6d158a81 3357 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
3358 int cpu;
3359
9aba60fe 3360 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 3361 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 3362 cpu = iter->cpu_file;
6d158a81
SR
3363 buf_iter = trace_buffer_iter(iter, cpu);
3364 if (buf_iter) {
3365 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
3366 return 0;
3367 } else {
12883efb 3368 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
3369 return 0;
3370 }
3371 return 1;
3372 }
3373
ab46428c 3374 for_each_tracing_cpu(cpu) {
6d158a81
SR
3375 buf_iter = trace_buffer_iter(iter, cpu);
3376 if (buf_iter) {
3377 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
3378 return 0;
3379 } else {
12883efb 3380 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
3381 return 0;
3382 }
bc0c38d1 3383 }
d769041f 3384
797d3712 3385 return 1;
bc0c38d1
SR
3386}
3387
4f535968 3388/* Called with trace_event_read_lock() held. */
955b61e5 3389enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 3390{
983f938a
SRRH
3391 struct trace_array *tr = iter->tr;
3392 unsigned long trace_flags = tr->trace_flags;
2c4f035f
FW
3393 enum print_line_t ret;
3394
19a7fe20
SRRH
3395 if (iter->lost_events) {
3396 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3397 iter->cpu, iter->lost_events);
3398 if (trace_seq_has_overflowed(&iter->seq))
3399 return TRACE_TYPE_PARTIAL_LINE;
3400 }
bc21b478 3401
2c4f035f
FW
3402 if (iter->trace && iter->trace->print_line) {
3403 ret = iter->trace->print_line(iter);
3404 if (ret != TRACE_TYPE_UNHANDLED)
3405 return ret;
3406 }
72829bc3 3407
09ae7234
SRRH
3408 if (iter->ent->type == TRACE_BPUTS &&
3409 trace_flags & TRACE_ITER_PRINTK &&
3410 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3411 return trace_print_bputs_msg_only(iter);
3412
48ead020
FW
3413 if (iter->ent->type == TRACE_BPRINT &&
3414 trace_flags & TRACE_ITER_PRINTK &&
3415 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3416 return trace_print_bprintk_msg_only(iter);
48ead020 3417
66896a85
FW
3418 if (iter->ent->type == TRACE_PRINT &&
3419 trace_flags & TRACE_ITER_PRINTK &&
3420 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3421 return trace_print_printk_msg_only(iter);
66896a85 3422
cb0f12aa
IM
3423 if (trace_flags & TRACE_ITER_BIN)
3424 return print_bin_fmt(iter);
3425
5e3ca0ec
IM
3426 if (trace_flags & TRACE_ITER_HEX)
3427 return print_hex_fmt(iter);
3428
f9896bf3
IM
3429 if (trace_flags & TRACE_ITER_RAW)
3430 return print_raw_fmt(iter);
3431
f9896bf3
IM
3432 return print_trace_fmt(iter);
3433}
3434
7e9a49ef
JO
3435void trace_latency_header(struct seq_file *m)
3436{
3437 struct trace_iterator *iter = m->private;
983f938a 3438 struct trace_array *tr = iter->tr;
7e9a49ef
JO
3439
3440 /* print nothing if the buffers are empty */
3441 if (trace_empty(iter))
3442 return;
3443
3444 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3445 print_trace_header(m, iter);
3446
983f938a 3447 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
7e9a49ef
JO
3448 print_lat_help_header(m);
3449}
3450
62b915f1
JO
3451void trace_default_header(struct seq_file *m)
3452{
3453 struct trace_iterator *iter = m->private;
983f938a
SRRH
3454 struct trace_array *tr = iter->tr;
3455 unsigned long trace_flags = tr->trace_flags;
62b915f1 3456
f56e7f8e
JO
3457 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3458 return;
3459
62b915f1
JO
3460 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3461 /* print nothing if the buffers are empty */
3462 if (trace_empty(iter))
3463 return;
3464 print_trace_header(m, iter);
3465 if (!(trace_flags & TRACE_ITER_VERBOSE))
3466 print_lat_help_header(m);
3467 } else {
77271ce4
SR
3468 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3469 if (trace_flags & TRACE_ITER_IRQ_INFO)
12883efb 3470 print_func_help_header_irq(iter->trace_buffer, m);
77271ce4 3471 else
12883efb 3472 print_func_help_header(iter->trace_buffer, m);
77271ce4 3473 }
62b915f1
JO
3474 }
3475}
3476
e0a413f6
SR
3477static void test_ftrace_alive(struct seq_file *m)
3478{
3479 if (!ftrace_is_dead())
3480 return;
d79ac28f
RV
3481 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3482 "# MAY BE MISSING FUNCTION EVENTS\n");
e0a413f6
SR
3483}
3484
d8741e2e 3485#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 3486static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 3487{
d79ac28f
RV
3488 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3489 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3490 "# Takes a snapshot of the main buffer.\n"
3491 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3492 "# (Doesn't have to be '2' works with any number that\n"
3493 "# is not a '0' or '1')\n");
d8741e2e 3494}
f1affcaa
SRRH
3495
3496static void show_snapshot_percpu_help(struct seq_file *m)
3497{
fa6f0cc7 3498 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
f1affcaa 3499#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
d79ac28f
RV
3500 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3501 "# Takes a snapshot of the main buffer for this cpu.\n");
f1affcaa 3502#else
d79ac28f
RV
3503 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3504 "# Must use main snapshot file to allocate.\n");
f1affcaa 3505#endif
d79ac28f
RV
3506 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3507 "# (Doesn't have to be '2' works with any number that\n"
3508 "# is not a '0' or '1')\n");
f1affcaa
SRRH
3509}
3510
d8741e2e
SRRH
3511static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3512{
45ad21ca 3513 if (iter->tr->allocated_snapshot)
fa6f0cc7 3514 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
d8741e2e 3515 else
fa6f0cc7 3516 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
d8741e2e 3517
fa6f0cc7 3518 seq_puts(m, "# Snapshot commands:\n");
f1affcaa
SRRH
3519 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3520 show_snapshot_main_help(m);
3521 else
3522 show_snapshot_percpu_help(m);
d8741e2e
SRRH
3523}
3524#else
3525/* Should never be called */
3526static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3527#endif
3528
bc0c38d1
SR
3529static int s_show(struct seq_file *m, void *v)
3530{
3531 struct trace_iterator *iter = v;
a63ce5b3 3532 int ret;
bc0c38d1
SR
3533
3534 if (iter->ent == NULL) {
3535 if (iter->tr) {
3536 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3537 seq_puts(m, "#\n");
e0a413f6 3538 test_ftrace_alive(m);
bc0c38d1 3539 }
d8741e2e
SRRH
3540 if (iter->snapshot && trace_empty(iter))
3541 print_snapshot_help(m, iter);
3542 else if (iter->trace && iter->trace->print_header)
8bba1bf5 3543 iter->trace->print_header(m);
62b915f1
JO
3544 else
3545 trace_default_header(m);
3546
a63ce5b3
SR
3547 } else if (iter->leftover) {
3548 /*
3549 * If we filled the seq_file buffer earlier, we
3550 * want to just show it now.
3551 */
3552 ret = trace_print_seq(m, &iter->seq);
3553
3554 /* ret should this time be zero, but you never know */
3555 iter->leftover = ret;
3556
bc0c38d1 3557 } else {
f9896bf3 3558 print_trace_line(iter);
a63ce5b3
SR
3559 ret = trace_print_seq(m, &iter->seq);
3560 /*
3561 * If we overflow the seq_file buffer, then it will
3562 * ask us for this data again at start up.
3563 * Use that instead.
3564 * ret is 0 if seq_file write succeeded.
3565 * -1 otherwise.
3566 */
3567 iter->leftover = ret;
bc0c38d1
SR
3568 }
3569
3570 return 0;
3571}
3572
649e9c70
ON
3573/*
3574 * Should be used after trace_array_get(), trace_types_lock
3575 * ensures that i_cdev was already initialized.
3576 */
3577static inline int tracing_get_cpu(struct inode *inode)
3578{
3579 if (inode->i_cdev) /* See trace_create_cpu_file() */
3580 return (long)inode->i_cdev - 1;
3581 return RING_BUFFER_ALL_CPUS;
3582}
3583
88e9d34c 3584static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
3585 .start = s_start,
3586 .next = s_next,
3587 .stop = s_stop,
3588 .show = s_show,
bc0c38d1
SR
3589};
3590
e309b41d 3591static struct trace_iterator *
6484c71c 3592__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 3593{
6484c71c 3594 struct trace_array *tr = inode->i_private;
bc0c38d1 3595 struct trace_iterator *iter;
50e18b94 3596 int cpu;
bc0c38d1 3597
85a2f9b4
SR
3598 if (tracing_disabled)
3599 return ERR_PTR(-ENODEV);
60a11774 3600
50e18b94 3601 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
3602 if (!iter)
3603 return ERR_PTR(-ENOMEM);
bc0c38d1 3604
72917235 3605 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
6d158a81 3606 GFP_KERNEL);
93574fcc
DC
3607 if (!iter->buffer_iter)
3608 goto release;
3609
d7350c3f
FW
3610 /*
3611 * We make a copy of the current tracer to avoid concurrent
3612 * changes on it while we are reading.
3613 */
bc0c38d1 3614 mutex_lock(&trace_types_lock);
d7350c3f 3615 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 3616 if (!iter->trace)
d7350c3f 3617 goto fail;
85a2f9b4 3618
2b6080f2 3619 *iter->trace = *tr->current_trace;
d7350c3f 3620
79f55997 3621 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
3622 goto fail;
3623
12883efb
SRRH
3624 iter->tr = tr;
3625
3626#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3627 /* Currently only the top directory has a snapshot */
3628 if (tr->current_trace->print_max || snapshot)
12883efb 3629 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 3630 else
12883efb
SRRH
3631#endif
3632 iter->trace_buffer = &tr->trace_buffer;
debdd57f 3633 iter->snapshot = snapshot;
bc0c38d1 3634 iter->pos = -1;
6484c71c 3635 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 3636 mutex_init(&iter->mutex);
bc0c38d1 3637
8bba1bf5
MM
3638 /* Notify the tracer early; before we stop tracing. */
3639 if (iter->trace && iter->trace->open)
a93751ca 3640 iter->trace->open(iter);
8bba1bf5 3641
12ef7d44 3642 /* Annotate start of buffers if we had overruns */
12883efb 3643 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
3644 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3645
8be0709f 3646 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 3647 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
3648 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3649
debdd57f
HT
3650 /* stop the trace while dumping if we are not opening "snapshot" */
3651 if (!iter->snapshot)
2b6080f2 3652 tracing_stop_tr(tr);
2f26ebd5 3653
ae3b5093 3654 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3655 for_each_tracing_cpu(cpu) {
b04cc6b1 3656 iter->buffer_iter[cpu] =
12883efb 3657 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3658 }
3659 ring_buffer_read_prepare_sync();
3660 for_each_tracing_cpu(cpu) {
3661 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3662 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
3663 }
3664 } else {
3665 cpu = iter->cpu_file;
3928a8a2 3666 iter->buffer_iter[cpu] =
12883efb 3667 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3668 ring_buffer_read_prepare_sync();
3669 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3670 tracing_iter_reset(iter, cpu);
3928a8a2
SR
3671 }
3672
bc0c38d1
SR
3673 mutex_unlock(&trace_types_lock);
3674
bc0c38d1 3675 return iter;
3928a8a2 3676
d7350c3f 3677 fail:
3928a8a2 3678 mutex_unlock(&trace_types_lock);
d7350c3f 3679 kfree(iter->trace);
6d158a81 3680 kfree(iter->buffer_iter);
93574fcc 3681release:
50e18b94
JO
3682 seq_release_private(inode, file);
3683 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
3684}
3685
3686int tracing_open_generic(struct inode *inode, struct file *filp)
3687{
60a11774
SR
3688 if (tracing_disabled)
3689 return -ENODEV;
3690
bc0c38d1
SR
3691 filp->private_data = inode->i_private;
3692 return 0;
3693}
3694
2e86421d
GB
3695bool tracing_is_disabled(void)
3696{
3697 return (tracing_disabled) ? true: false;
3698}
3699
7b85af63
SRRH
3700/*
3701 * Open and update trace_array ref count.
3702 * Must have the current trace_array passed to it.
3703 */
dcc30223 3704static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
7b85af63
SRRH
3705{
3706 struct trace_array *tr = inode->i_private;
3707
3708 if (tracing_disabled)
3709 return -ENODEV;
3710
3711 if (trace_array_get(tr) < 0)
3712 return -ENODEV;
3713
3714 filp->private_data = inode->i_private;
3715
3716 return 0;
7b85af63
SRRH
3717}
3718
4fd27358 3719static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 3720{
6484c71c 3721 struct trace_array *tr = inode->i_private;
907f2784 3722 struct seq_file *m = file->private_data;
4acd4d00 3723 struct trace_iterator *iter;
3928a8a2 3724 int cpu;
bc0c38d1 3725
ff451961 3726 if (!(file->f_mode & FMODE_READ)) {
6484c71c 3727 trace_array_put(tr);
4acd4d00 3728 return 0;
ff451961 3729 }
4acd4d00 3730
6484c71c 3731 /* Writes do not use seq_file */
4acd4d00 3732 iter = m->private;
bc0c38d1 3733 mutex_lock(&trace_types_lock);
a695cb58 3734
3928a8a2
SR
3735 for_each_tracing_cpu(cpu) {
3736 if (iter->buffer_iter[cpu])
3737 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3738 }
3739
bc0c38d1
SR
3740 if (iter->trace && iter->trace->close)
3741 iter->trace->close(iter);
3742
debdd57f
HT
3743 if (!iter->snapshot)
3744 /* reenable tracing if it was previously enabled */
2b6080f2 3745 tracing_start_tr(tr);
f77d09a3
AL
3746
3747 __trace_array_put(tr);
3748
bc0c38d1
SR
3749 mutex_unlock(&trace_types_lock);
3750
d7350c3f 3751 mutex_destroy(&iter->mutex);
b0dfa978 3752 free_cpumask_var(iter->started);
d7350c3f 3753 kfree(iter->trace);
6d158a81 3754 kfree(iter->buffer_iter);
50e18b94 3755 seq_release_private(inode, file);
ff451961 3756
bc0c38d1
SR
3757 return 0;
3758}
3759
7b85af63
SRRH
3760static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3761{
3762 struct trace_array *tr = inode->i_private;
3763
3764 trace_array_put(tr);
bc0c38d1
SR
3765 return 0;
3766}
3767
7b85af63
SRRH
3768static int tracing_single_release_tr(struct inode *inode, struct file *file)
3769{
3770 struct trace_array *tr = inode->i_private;
3771
3772 trace_array_put(tr);
3773
3774 return single_release(inode, file);
3775}
3776
bc0c38d1
SR
3777static int tracing_open(struct inode *inode, struct file *file)
3778{
6484c71c 3779 struct trace_array *tr = inode->i_private;
85a2f9b4
SR
3780 struct trace_iterator *iter;
3781 int ret = 0;
bc0c38d1 3782
ff451961
SRRH
3783 if (trace_array_get(tr) < 0)
3784 return -ENODEV;
3785
4acd4d00 3786 /* If this file was open for write, then erase contents */
6484c71c
ON
3787 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3788 int cpu = tracing_get_cpu(inode);
3789
3790 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 3791 tracing_reset_online_cpus(&tr->trace_buffer);
4acd4d00 3792 else
6484c71c 3793 tracing_reset(&tr->trace_buffer, cpu);
4acd4d00 3794 }
bc0c38d1 3795
4acd4d00 3796 if (file->f_mode & FMODE_READ) {
6484c71c 3797 iter = __tracing_open(inode, file, false);
4acd4d00
SR
3798 if (IS_ERR(iter))
3799 ret = PTR_ERR(iter);
983f938a 3800 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4acd4d00
SR
3801 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3802 }
ff451961
SRRH
3803
3804 if (ret < 0)
3805 trace_array_put(tr);
3806
bc0c38d1
SR
3807 return ret;
3808}
3809
607e2ea1
SRRH
3810/*
3811 * Some tracers are not suitable for instance buffers.
3812 * A tracer is always available for the global array (toplevel)
3813 * or if it explicitly states that it is.
3814 */
3815static bool
3816trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3817{
3818 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3819}
3820
3821/* Find the next tracer that this trace array may use */
3822static struct tracer *
3823get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3824{
3825 while (t && !trace_ok_for_array(t, tr))
3826 t = t->next;
3827
3828 return t;
3829}
3830
e309b41d 3831static void *
bc0c38d1
SR
3832t_next(struct seq_file *m, void *v, loff_t *pos)
3833{
607e2ea1 3834 struct trace_array *tr = m->private;
f129e965 3835 struct tracer *t = v;
bc0c38d1
SR
3836
3837 (*pos)++;
3838
3839 if (t)
607e2ea1 3840 t = get_tracer_for_array(tr, t->next);
bc0c38d1 3841
bc0c38d1
SR
3842 return t;
3843}
3844
3845static void *t_start(struct seq_file *m, loff_t *pos)
3846{
607e2ea1 3847 struct trace_array *tr = m->private;
f129e965 3848 struct tracer *t;
bc0c38d1
SR
3849 loff_t l = 0;
3850
3851 mutex_lock(&trace_types_lock);
607e2ea1
SRRH
3852
3853 t = get_tracer_for_array(tr, trace_types);
3854 for (; t && l < *pos; t = t_next(m, t, &l))
3855 ;
bc0c38d1
SR
3856
3857 return t;
3858}
3859
3860static void t_stop(struct seq_file *m, void *p)
3861{
3862 mutex_unlock(&trace_types_lock);
3863}
3864
3865static int t_show(struct seq_file *m, void *v)
3866{
3867 struct tracer *t = v;
3868
3869 if (!t)
3870 return 0;
3871
fa6f0cc7 3872 seq_puts(m, t->name);
bc0c38d1
SR
3873 if (t->next)
3874 seq_putc(m, ' ');
3875 else
3876 seq_putc(m, '\n');
3877
3878 return 0;
3879}
3880
88e9d34c 3881static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
3882 .start = t_start,
3883 .next = t_next,
3884 .stop = t_stop,
3885 .show = t_show,
bc0c38d1
SR
3886};
3887
3888static int show_traces_open(struct inode *inode, struct file *file)
3889{
607e2ea1
SRRH
3890 struct trace_array *tr = inode->i_private;
3891 struct seq_file *m;
3892 int ret;
3893
60a11774
SR
3894 if (tracing_disabled)
3895 return -ENODEV;
3896
607e2ea1
SRRH
3897 ret = seq_open(file, &show_traces_seq_ops);
3898 if (ret)
3899 return ret;
3900
3901 m = file->private_data;
3902 m->private = tr;
3903
3904 return 0;
bc0c38d1
SR
3905}
3906
4acd4d00
SR
3907static ssize_t
3908tracing_write_stub(struct file *filp, const char __user *ubuf,
3909 size_t count, loff_t *ppos)
3910{
3911 return count;
3912}
3913
098c879e 3914loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
364829b1 3915{
098c879e
SRRH
3916 int ret;
3917
364829b1 3918 if (file->f_mode & FMODE_READ)
098c879e 3919 ret = seq_lseek(file, offset, whence);
364829b1 3920 else
098c879e
SRRH
3921 file->f_pos = ret = 0;
3922
3923 return ret;
364829b1
SP
3924}
3925
5e2336a0 3926static const struct file_operations tracing_fops = {
4bf39a94
IM
3927 .open = tracing_open,
3928 .read = seq_read,
4acd4d00 3929 .write = tracing_write_stub,
098c879e 3930 .llseek = tracing_lseek,
4bf39a94 3931 .release = tracing_release,
bc0c38d1
SR
3932};
3933
5e2336a0 3934static const struct file_operations show_traces_fops = {
c7078de1
IM
3935 .open = show_traces_open,
3936 .read = seq_read,
3937 .release = seq_release,
b444786f 3938 .llseek = seq_lseek,
c7078de1
IM
3939};
3940
36dfe925
IM
3941/*
3942 * The tracer itself will not take this lock, but still we want
3943 * to provide a consistent cpumask to user-space:
3944 */
3945static DEFINE_MUTEX(tracing_cpumask_update_lock);
3946
3947/*
3948 * Temporary storage for the character representation of the
3949 * CPU bitmask (and one more byte for the newline):
3950 */
3951static char mask_str[NR_CPUS + 1];
3952
c7078de1
IM
3953static ssize_t
3954tracing_cpumask_read(struct file *filp, char __user *ubuf,
3955 size_t count, loff_t *ppos)
3956{
ccfe9e42 3957 struct trace_array *tr = file_inode(filp)->i_private;
36dfe925 3958 int len;
c7078de1
IM
3959
3960 mutex_lock(&tracing_cpumask_update_lock);
36dfe925 3961
1a40243b
TH
3962 len = snprintf(mask_str, count, "%*pb\n",
3963 cpumask_pr_args(tr->tracing_cpumask));
3964 if (len >= count) {
36dfe925
IM
3965 count = -EINVAL;
3966 goto out_err;
3967 }
36dfe925
IM
3968 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3969
3970out_err:
c7078de1
IM
3971 mutex_unlock(&tracing_cpumask_update_lock);
3972
3973 return count;
3974}
3975
3976static ssize_t
3977tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3978 size_t count, loff_t *ppos)
3979{
ccfe9e42 3980 struct trace_array *tr = file_inode(filp)->i_private;
9e01c1b7 3981 cpumask_var_t tracing_cpumask_new;
2b6080f2 3982 int err, cpu;
9e01c1b7
RR
3983
3984 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3985 return -ENOMEM;
c7078de1 3986
9e01c1b7 3987 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 3988 if (err)
36dfe925
IM
3989 goto err_unlock;
3990
215368e8
LZ
3991 mutex_lock(&tracing_cpumask_update_lock);
3992
a5e25883 3993 local_irq_disable();
0b9b12c1 3994 arch_spin_lock(&tr->max_lock);
ab46428c 3995 for_each_tracing_cpu(cpu) {
36dfe925
IM
3996 /*
3997 * Increase/decrease the disabled counter if we are
3998 * about to flip a bit in the cpumask:
3999 */
ccfe9e42 4000 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4001 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4002 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4003 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 4004 }
ccfe9e42 4005 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4006 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4007 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4008 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
4009 }
4010 }
0b9b12c1 4011 arch_spin_unlock(&tr->max_lock);
a5e25883 4012 local_irq_enable();
36dfe925 4013
ccfe9e42 4014 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
36dfe925
IM
4015
4016 mutex_unlock(&tracing_cpumask_update_lock);
9e01c1b7 4017 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
4018
4019 return count;
36dfe925
IM
4020
4021err_unlock:
215368e8 4022 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
4023
4024 return err;
c7078de1
IM
4025}
4026
5e2336a0 4027static const struct file_operations tracing_cpumask_fops = {
ccfe9e42 4028 .open = tracing_open_generic_tr,
c7078de1
IM
4029 .read = tracing_cpumask_read,
4030 .write = tracing_cpumask_write,
ccfe9e42 4031 .release = tracing_release_generic_tr,
b444786f 4032 .llseek = generic_file_llseek,
bc0c38d1
SR
4033};
4034
fdb372ed 4035static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 4036{
d8e83d26 4037 struct tracer_opt *trace_opts;
2b6080f2 4038 struct trace_array *tr = m->private;
d8e83d26 4039 u32 tracer_flags;
d8e83d26 4040 int i;
adf9f195 4041
d8e83d26 4042 mutex_lock(&trace_types_lock);
2b6080f2
SR
4043 tracer_flags = tr->current_trace->flags->val;
4044 trace_opts = tr->current_trace->flags->opts;
d8e83d26 4045
bc0c38d1 4046 for (i = 0; trace_options[i]; i++) {
983f938a 4047 if (tr->trace_flags & (1 << i))
fdb372ed 4048 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 4049 else
fdb372ed 4050 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
4051 }
4052
adf9f195
FW
4053 for (i = 0; trace_opts[i].name; i++) {
4054 if (tracer_flags & trace_opts[i].bit)
fdb372ed 4055 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 4056 else
fdb372ed 4057 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 4058 }
d8e83d26 4059 mutex_unlock(&trace_types_lock);
adf9f195 4060
fdb372ed 4061 return 0;
bc0c38d1 4062}
bc0c38d1 4063
8c1a49ae 4064static int __set_tracer_option(struct trace_array *tr,
8d18eaaf
LZ
4065 struct tracer_flags *tracer_flags,
4066 struct tracer_opt *opts, int neg)
4067{
d39cdd20 4068 struct tracer *trace = tracer_flags->trace;
8d18eaaf 4069 int ret;
bc0c38d1 4070
8c1a49ae 4071 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
8d18eaaf
LZ
4072 if (ret)
4073 return ret;
4074
4075 if (neg)
4076 tracer_flags->val &= ~opts->bit;
4077 else
4078 tracer_flags->val |= opts->bit;
4079 return 0;
bc0c38d1
SR
4080}
4081
adf9f195 4082/* Try to assign a tracer specific option */
8c1a49ae 4083static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
adf9f195 4084{
8c1a49ae 4085 struct tracer *trace = tr->current_trace;
7770841e 4086 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 4087 struct tracer_opt *opts = NULL;
8d18eaaf 4088 int i;
adf9f195 4089
7770841e
Z
4090 for (i = 0; tracer_flags->opts[i].name; i++) {
4091 opts = &tracer_flags->opts[i];
adf9f195 4092
8d18eaaf 4093 if (strcmp(cmp, opts->name) == 0)
8c1a49ae 4094 return __set_tracer_option(tr, trace->flags, opts, neg);
adf9f195 4095 }
adf9f195 4096
8d18eaaf 4097 return -EINVAL;
adf9f195
FW
4098}
4099
613f04a0
SRRH
4100/* Some tracers require overwrite to stay enabled */
4101int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4102{
4103 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4104 return -1;
4105
4106 return 0;
4107}
4108
2b6080f2 4109int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
4110{
4111 /* do nothing if flag is already set */
983f938a 4112 if (!!(tr->trace_flags & mask) == !!enabled)
613f04a0
SRRH
4113 return 0;
4114
4115 /* Give the tracer a chance to approve the change */
2b6080f2 4116 if (tr->current_trace->flag_changed)
bf6065b5 4117 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
613f04a0 4118 return -EINVAL;
af4617bd
SR
4119
4120 if (enabled)
983f938a 4121 tr->trace_flags |= mask;
af4617bd 4122 else
983f938a 4123 tr->trace_flags &= ~mask;
e870e9a1
LZ
4124
4125 if (mask == TRACE_ITER_RECORD_CMD)
4126 trace_event_enable_cmd_record(enabled);
750912fa 4127
c37775d5
SR
4128 if (mask == TRACE_ITER_EVENT_FORK)
4129 trace_event_follow_fork(tr, enabled);
4130
80902822 4131 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 4132 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 4133#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 4134 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
4135#endif
4136 }
81698831 4137
b9f9108c 4138 if (mask == TRACE_ITER_PRINTK) {
81698831 4139 trace_printk_start_stop_comm(enabled);
b9f9108c
SRRH
4140 trace_printk_control(enabled);
4141 }
613f04a0
SRRH
4142
4143 return 0;
af4617bd
SR
4144}
4145
2b6080f2 4146static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 4147{
8d18eaaf 4148 char *cmp;
bc0c38d1 4149 int neg = 0;
613f04a0 4150 int ret = -ENODEV;
bc0c38d1 4151 int i;
a4d1e688 4152 size_t orig_len = strlen(option);
bc0c38d1 4153
7bcfaf54 4154 cmp = strstrip(option);
bc0c38d1 4155
8d18eaaf 4156 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
4157 neg = 1;
4158 cmp += 2;
4159 }
4160
69d34da2
SRRH
4161 mutex_lock(&trace_types_lock);
4162
bc0c38d1 4163 for (i = 0; trace_options[i]; i++) {
8d18eaaf 4164 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 4165 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
4166 break;
4167 }
4168 }
adf9f195
FW
4169
4170 /* If no option could be set, test the specific tracer options */
69d34da2 4171 if (!trace_options[i])
8c1a49ae 4172 ret = set_tracer_option(tr, cmp, neg);
69d34da2
SRRH
4173
4174 mutex_unlock(&trace_types_lock);
bc0c38d1 4175
a4d1e688
JW
4176 /*
4177 * If the first trailing whitespace is replaced with '\0' by strstrip,
4178 * turn it back into a space.
4179 */
4180 if (orig_len > strlen(option))
4181 option[strlen(option)] = ' ';
4182
7bcfaf54
SR
4183 return ret;
4184}
4185
a4d1e688
JW
4186static void __init apply_trace_boot_options(void)
4187{
4188 char *buf = trace_boot_options_buf;
4189 char *option;
4190
4191 while (true) {
4192 option = strsep(&buf, ",");
4193
4194 if (!option)
4195 break;
a4d1e688 4196
43ed3843
SRRH
4197 if (*option)
4198 trace_set_options(&global_trace, option);
a4d1e688
JW
4199
4200 /* Put back the comma to allow this to be called again */
4201 if (buf)
4202 *(buf - 1) = ',';
4203 }
4204}
4205
7bcfaf54
SR
4206static ssize_t
4207tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4208 size_t cnt, loff_t *ppos)
4209{
2b6080f2
SR
4210 struct seq_file *m = filp->private_data;
4211 struct trace_array *tr = m->private;
7bcfaf54 4212 char buf[64];
613f04a0 4213 int ret;
7bcfaf54
SR
4214
4215 if (cnt >= sizeof(buf))
4216 return -EINVAL;
4217
4afe6495 4218 if (copy_from_user(buf, ubuf, cnt))
7bcfaf54
SR
4219 return -EFAULT;
4220
a8dd2176
SR
4221 buf[cnt] = 0;
4222
2b6080f2 4223 ret = trace_set_options(tr, buf);
613f04a0
SRRH
4224 if (ret < 0)
4225 return ret;
7bcfaf54 4226
cf8517cf 4227 *ppos += cnt;
bc0c38d1
SR
4228
4229 return cnt;
4230}
4231
fdb372ed
LZ
4232static int tracing_trace_options_open(struct inode *inode, struct file *file)
4233{
7b85af63 4234 struct trace_array *tr = inode->i_private;
f77d09a3 4235 int ret;
7b85af63 4236
fdb372ed
LZ
4237 if (tracing_disabled)
4238 return -ENODEV;
2b6080f2 4239
7b85af63
SRRH
4240 if (trace_array_get(tr) < 0)
4241 return -ENODEV;
4242
f77d09a3
AL
4243 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4244 if (ret < 0)
4245 trace_array_put(tr);
4246
4247 return ret;
fdb372ed
LZ
4248}
4249
5e2336a0 4250static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
4251 .open = tracing_trace_options_open,
4252 .read = seq_read,
4253 .llseek = seq_lseek,
7b85af63 4254 .release = tracing_single_release_tr,
ee6bce52 4255 .write = tracing_trace_options_write,
bc0c38d1
SR
4256};
4257
7bd2f24c
IM
4258static const char readme_msg[] =
4259 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
4260 "# echo 0 > tracing_on : quick way to disable tracing\n"
4261 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4262 " Important files:\n"
4263 " trace\t\t\t- The static contents of the buffer\n"
4264 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4265 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4266 " current_tracer\t- function and latency tracers\n"
4267 " available_tracers\t- list of configured tracers for current_tracer\n"
4268 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4269 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4270 " trace_clock\t\t-change the clock used to order events\n"
4271 " local: Per cpu clock but may not be synced across CPUs\n"
4272 " global: Synced across CPUs but slows tracing down.\n"
4273 " counter: Not a clock, but just an increment\n"
4274 " uptime: Jiffy counter from time of boot\n"
4275 " perf: Same clock that perf events use\n"
4276#ifdef CONFIG_X86_64
4277 " x86-tsc: TSC cycle counter\n"
4278#endif
4279 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
fa32e855 4280 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
22f45649
SRRH
4281 " tracing_cpumask\t- Limit which CPUs to trace\n"
4282 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4283 "\t\t\t Remove sub-buffer with rmdir\n"
4284 " trace_options\t\t- Set format or modify how tracing happens\n"
71485c45
SRRH
4285 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4286 "\t\t\t option name\n"
939c7a4f 4287 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
22f45649
SRRH
4288#ifdef CONFIG_DYNAMIC_FTRACE
4289 "\n available_filter_functions - list of functions that can be filtered on\n"
71485c45
SRRH
4290 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4291 "\t\t\t functions\n"
60f1d5e3 4292 "\t accepts: func_full_name or glob-matching-pattern\n"
71485c45
SRRH
4293 "\t modules: Can select a group via module\n"
4294 "\t Format: :mod:<module-name>\n"
4295 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4296 "\t triggers: a command to perform when function is hit\n"
4297 "\t Format: <function>:<trigger>[:count]\n"
4298 "\t trigger: traceon, traceoff\n"
4299 "\t\t enable_event:<system>:<event>\n"
4300 "\t\t disable_event:<system>:<event>\n"
22f45649 4301#ifdef CONFIG_STACKTRACE
71485c45 4302 "\t\t stacktrace\n"
22f45649
SRRH
4303#endif
4304#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4305 "\t\t snapshot\n"
22f45649 4306#endif
17a280ea
SRRH
4307 "\t\t dump\n"
4308 "\t\t cpudump\n"
71485c45
SRRH
4309 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4310 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4311 "\t The first one will disable tracing every time do_fault is hit\n"
4312 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4313 "\t The first time do trap is hit and it disables tracing, the\n"
4314 "\t counter will decrement to 2. If tracing is already disabled,\n"
4315 "\t the counter will not decrement. It only decrements when the\n"
4316 "\t trigger did work\n"
4317 "\t To remove trigger without count:\n"
4318 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4319 "\t To remove trigger with a count:\n"
4320 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
22f45649 4321 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
71485c45
SRRH
4322 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4323 "\t modules: Can select a group via module command :mod:\n"
4324 "\t Does not accept triggers\n"
22f45649
SRRH
4325#endif /* CONFIG_DYNAMIC_FTRACE */
4326#ifdef CONFIG_FUNCTION_TRACER
71485c45
SRRH
4327 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4328 "\t\t (function)\n"
22f45649
SRRH
4329#endif
4330#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4331 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
d048a8c7 4332 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
22f45649
SRRH
4333 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4334#endif
4335#ifdef CONFIG_TRACER_SNAPSHOT
71485c45
SRRH
4336 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4337 "\t\t\t snapshot buffer. Read the contents for more\n"
4338 "\t\t\t information\n"
22f45649 4339#endif
991821c8 4340#ifdef CONFIG_STACK_TRACER
22f45649
SRRH
4341 " stack_trace\t\t- Shows the max stack trace when active\n"
4342 " stack_max_size\t- Shows current max stack size that was traced\n"
71485c45
SRRH
4343 "\t\t\t Write into this file to reset the max size (trigger a\n"
4344 "\t\t\t new trace)\n"
22f45649 4345#ifdef CONFIG_DYNAMIC_FTRACE
71485c45
SRRH
4346 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4347 "\t\t\t traces\n"
22f45649 4348#endif
991821c8 4349#endif /* CONFIG_STACK_TRACER */
86425625
MH
4350#ifdef CONFIG_KPROBE_EVENT
4351 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4352 "\t\t\t Write into this file to define/undefine new trace events.\n"
4353#endif
4354#ifdef CONFIG_UPROBE_EVENT
4355 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4356 "\t\t\t Write into this file to define/undefine new trace events.\n"
4357#endif
4358#if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
4359 "\t accepts: event-definitions (one definition per line)\n"
4360 "\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
4361 "\t -:[<group>/]<event>\n"
4362#ifdef CONFIG_KPROBE_EVENT
4363 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4364#endif
4365#ifdef CONFIG_UPROBE_EVENT
4366 "\t place: <path>:<offset>\n"
4367#endif
4368 "\t args: <name>=fetcharg[:type]\n"
4369 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4370 "\t $stack<index>, $stack, $retval, $comm\n"
4371 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4372 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4373#endif
26f25564
TZ
4374 " events/\t\t- Directory containing all trace event subsystems:\n"
4375 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4376 " events/<system>/\t- Directory containing all trace events for <system>:\n"
71485c45
SRRH
4377 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4378 "\t\t\t events\n"
26f25564 4379 " filter\t\t- If set, only events passing filter are traced\n"
71485c45
SRRH
4380 " events/<system>/<event>/\t- Directory containing control files for\n"
4381 "\t\t\t <event>:\n"
26f25564
TZ
4382 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4383 " filter\t\t- If set, only events passing filter are traced\n"
4384 " trigger\t\t- If set, a command to perform when event is hit\n"
71485c45
SRRH
4385 "\t Format: <trigger>[:count][if <filter>]\n"
4386 "\t trigger: traceon, traceoff\n"
4387 "\t enable_event:<system>:<event>\n"
4388 "\t disable_event:<system>:<event>\n"
d0bad49b
TZ
4389#ifdef CONFIG_HIST_TRIGGERS
4390 "\t enable_hist:<system>:<event>\n"
4391 "\t disable_hist:<system>:<event>\n"
4392#endif
26f25564 4393#ifdef CONFIG_STACKTRACE
71485c45 4394 "\t\t stacktrace\n"
26f25564
TZ
4395#endif
4396#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4397 "\t\t snapshot\n"
7ef224d1
TZ
4398#endif
4399#ifdef CONFIG_HIST_TRIGGERS
4400 "\t\t hist (see below)\n"
26f25564 4401#endif
71485c45
SRRH
4402 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4403 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4404 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4405 "\t events/block/block_unplug/trigger\n"
4406 "\t The first disables tracing every time block_unplug is hit.\n"
4407 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4408 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4409 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4410 "\t Like function triggers, the counter is only decremented if it\n"
4411 "\t enabled or disabled tracing.\n"
4412 "\t To remove a trigger without a count:\n"
4413 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4414 "\t To remove a trigger with a count:\n"
4415 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4416 "\t Filters can be ignored when removing a trigger.\n"
7ef224d1
TZ
4417#ifdef CONFIG_HIST_TRIGGERS
4418 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
76a3b0c8 4419 "\t Format: hist:keys=<field1[,field2,...]>\n"
f2606835 4420 "\t [:values=<field1[,field2,...]>]\n"
e62347d2 4421 "\t [:sort=<field1[,field2,...]>]\n"
7ef224d1 4422 "\t [:size=#entries]\n"
e86ae9ba 4423 "\t [:pause][:continue][:clear]\n"
5463bfda 4424 "\t [:name=histname1]\n"
7ef224d1
TZ
4425 "\t [if <filter>]\n\n"
4426 "\t When a matching event is hit, an entry is added to a hash\n"
f2606835
TZ
4427 "\t table using the key(s) and value(s) named, and the value of a\n"
4428 "\t sum called 'hitcount' is incremented. Keys and values\n"
4429 "\t correspond to fields in the event's format description. Keys\n"
69a0200c
TZ
4430 "\t can be any field, or the special string 'stacktrace'.\n"
4431 "\t Compound keys consisting of up to two fields can be specified\n"
4432 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4433 "\t fields. Sort keys consisting of up to two fields can be\n"
4434 "\t specified using the 'sort' keyword. The sort direction can\n"
4435 "\t be modified by appending '.descending' or '.ascending' to a\n"
4436 "\t sort field. The 'size' parameter can be used to specify more\n"
5463bfda
TZ
4437 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4438 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4439 "\t its histogram data will be shared with other triggers of the\n"
4440 "\t same name, and trigger hits will update this common data.\n\n"
7ef224d1 4441 "\t Reading the 'hist' file for the event will dump the hash\n"
52a7f16d
TZ
4442 "\t table in its entirety to stdout. If there are multiple hist\n"
4443 "\t triggers attached to an event, there will be a table for each\n"
5463bfda
TZ
4444 "\t trigger in the output. The table displayed for a named\n"
4445 "\t trigger will be the same as any other instance having the\n"
4446 "\t same name. The default format used to display a given field\n"
4447 "\t can be modified by appending any of the following modifiers\n"
4448 "\t to the field name, as applicable:\n\n"
c6afad49
TZ
4449 "\t .hex display a number as a hex value\n"
4450 "\t .sym display an address as a symbol\n"
6b4827ad 4451 "\t .sym-offset display an address as a symbol and offset\n"
31696198
TZ
4452 "\t .execname display a common_pid as a program name\n"
4453 "\t .syscall display a syscall id as a syscall name\n\n"
4b94f5b7 4454 "\t .log2 display log2 value rather than raw number\n\n"
83e99914
TZ
4455 "\t The 'pause' parameter can be used to pause an existing hist\n"
4456 "\t trigger or to start a hist trigger but not log any events\n"
4457 "\t until told to do so. 'continue' can be used to start or\n"
4458 "\t restart a paused hist trigger.\n\n"
e86ae9ba
TZ
4459 "\t The 'clear' parameter will clear the contents of a running\n"
4460 "\t hist trigger and leave its current paused/active state\n"
4461 "\t unchanged.\n\n"
d0bad49b
TZ
4462 "\t The enable_hist and disable_hist triggers can be used to\n"
4463 "\t have one event conditionally start and stop another event's\n"
4464 "\t already-attached hist trigger. The syntax is analagous to\n"
4465 "\t the enable_event and disable_event triggers.\n"
7ef224d1 4466#endif
7bd2f24c
IM
4467;
4468
4469static ssize_t
4470tracing_readme_read(struct file *filp, char __user *ubuf,
4471 size_t cnt, loff_t *ppos)
4472{
4473 return simple_read_from_buffer(ubuf, cnt, ppos,
4474 readme_msg, strlen(readme_msg));
4475}
4476
5e2336a0 4477static const struct file_operations tracing_readme_fops = {
c7078de1
IM
4478 .open = tracing_open_generic,
4479 .read = tracing_readme_read,
b444786f 4480 .llseek = generic_file_llseek,
7bd2f24c
IM
4481};
4482
42584c81
YY
4483static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4484{
4485 unsigned int *ptr = v;
69abe6a5 4486
42584c81
YY
4487 if (*pos || m->count)
4488 ptr++;
69abe6a5 4489
42584c81 4490 (*pos)++;
69abe6a5 4491
939c7a4f
YY
4492 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4493 ptr++) {
42584c81
YY
4494 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4495 continue;
69abe6a5 4496
42584c81
YY
4497 return ptr;
4498 }
69abe6a5 4499
42584c81
YY
4500 return NULL;
4501}
4502
4503static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4504{
4505 void *v;
4506 loff_t l = 0;
69abe6a5 4507
4c27e756
SRRH
4508 preempt_disable();
4509 arch_spin_lock(&trace_cmdline_lock);
4510
939c7a4f 4511 v = &savedcmd->map_cmdline_to_pid[0];
42584c81
YY
4512 while (l <= *pos) {
4513 v = saved_cmdlines_next(m, v, &l);
4514 if (!v)
4515 return NULL;
69abe6a5
AP
4516 }
4517
42584c81
YY
4518 return v;
4519}
4520
4521static void saved_cmdlines_stop(struct seq_file *m, void *v)
4522{
4c27e756
SRRH
4523 arch_spin_unlock(&trace_cmdline_lock);
4524 preempt_enable();
42584c81 4525}
69abe6a5 4526
42584c81
YY
4527static int saved_cmdlines_show(struct seq_file *m, void *v)
4528{
4529 char buf[TASK_COMM_LEN];
4530 unsigned int *pid = v;
69abe6a5 4531
4c27e756 4532 __trace_find_cmdline(*pid, buf);
42584c81
YY
4533 seq_printf(m, "%d %s\n", *pid, buf);
4534 return 0;
4535}
4536
4537static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4538 .start = saved_cmdlines_start,
4539 .next = saved_cmdlines_next,
4540 .stop = saved_cmdlines_stop,
4541 .show = saved_cmdlines_show,
4542};
4543
4544static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4545{
4546 if (tracing_disabled)
4547 return -ENODEV;
4548
4549 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
69abe6a5
AP
4550}
4551
4552static const struct file_operations tracing_saved_cmdlines_fops = {
42584c81
YY
4553 .open = tracing_saved_cmdlines_open,
4554 .read = seq_read,
4555 .llseek = seq_lseek,
4556 .release = seq_release,
69abe6a5
AP
4557};
4558
939c7a4f
YY
4559static ssize_t
4560tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4561 size_t cnt, loff_t *ppos)
4562{
4563 char buf[64];
4564 int r;
4565
4566 arch_spin_lock(&trace_cmdline_lock);
a6af8fbf 4567 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
939c7a4f
YY
4568 arch_spin_unlock(&trace_cmdline_lock);
4569
4570 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4571}
4572
4573static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4574{
4575 kfree(s->saved_cmdlines);
4576 kfree(s->map_cmdline_to_pid);
4577 kfree(s);
4578}
4579
4580static int tracing_resize_saved_cmdlines(unsigned int val)
4581{
4582 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4583
a6af8fbf 4584 s = kmalloc(sizeof(*s), GFP_KERNEL);
939c7a4f
YY
4585 if (!s)
4586 return -ENOMEM;
4587
4588 if (allocate_cmdlines_buffer(val, s) < 0) {
4589 kfree(s);
4590 return -ENOMEM;
4591 }
4592
4593 arch_spin_lock(&trace_cmdline_lock);
4594 savedcmd_temp = savedcmd;
4595 savedcmd = s;
4596 arch_spin_unlock(&trace_cmdline_lock);
4597 free_saved_cmdlines_buffer(savedcmd_temp);
4598
4599 return 0;
4600}
4601
4602static ssize_t
4603tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4604 size_t cnt, loff_t *ppos)
4605{
4606 unsigned long val;
4607 int ret;
4608
4609 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4610 if (ret)
4611 return ret;
4612
4613 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4614 if (!val || val > PID_MAX_DEFAULT)
4615 return -EINVAL;
4616
4617 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4618 if (ret < 0)
4619 return ret;
4620
4621 *ppos += cnt;
4622
4623 return cnt;
4624}
4625
4626static const struct file_operations tracing_saved_cmdlines_size_fops = {
4627 .open = tracing_open_generic,
4628 .read = tracing_saved_cmdlines_size_read,
4629 .write = tracing_saved_cmdlines_size_write,
4630};
4631
9828413d
SRRH
4632#ifdef CONFIG_TRACE_ENUM_MAP_FILE
4633static union trace_enum_map_item *
4634update_enum_map(union trace_enum_map_item *ptr)
4635{
4636 if (!ptr->map.enum_string) {
4637 if (ptr->tail.next) {
4638 ptr = ptr->tail.next;
4639 /* Set ptr to the next real item (skip head) */
4640 ptr++;
4641 } else
4642 return NULL;
4643 }
4644 return ptr;
4645}
4646
4647static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4648{
4649 union trace_enum_map_item *ptr = v;
4650
4651 /*
4652 * Paranoid! If ptr points to end, we don't want to increment past it.
4653 * This really should never happen.
4654 */
4655 ptr = update_enum_map(ptr);
4656 if (WARN_ON_ONCE(!ptr))
4657 return NULL;
4658
4659 ptr++;
4660
4661 (*pos)++;
4662
4663 ptr = update_enum_map(ptr);
4664
4665 return ptr;
4666}
4667
4668static void *enum_map_start(struct seq_file *m, loff_t *pos)
4669{
4670 union trace_enum_map_item *v;
4671 loff_t l = 0;
4672
4673 mutex_lock(&trace_enum_mutex);
4674
4675 v = trace_enum_maps;
4676 if (v)
4677 v++;
4678
4679 while (v && l < *pos) {
4680 v = enum_map_next(m, v, &l);
4681 }
4682
4683 return v;
4684}
4685
4686static void enum_map_stop(struct seq_file *m, void *v)
4687{
4688 mutex_unlock(&trace_enum_mutex);
4689}
4690
4691static int enum_map_show(struct seq_file *m, void *v)
4692{
4693 union trace_enum_map_item *ptr = v;
4694
4695 seq_printf(m, "%s %ld (%s)\n",
4696 ptr->map.enum_string, ptr->map.enum_value,
4697 ptr->map.system);
4698
4699 return 0;
4700}
4701
4702static const struct seq_operations tracing_enum_map_seq_ops = {
4703 .start = enum_map_start,
4704 .next = enum_map_next,
4705 .stop = enum_map_stop,
4706 .show = enum_map_show,
4707};
4708
4709static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4710{
4711 if (tracing_disabled)
4712 return -ENODEV;
4713
4714 return seq_open(filp, &tracing_enum_map_seq_ops);
4715}
4716
4717static const struct file_operations tracing_enum_map_fops = {
4718 .open = tracing_enum_map_open,
4719 .read = seq_read,
4720 .llseek = seq_lseek,
4721 .release = seq_release,
4722};
4723
4724static inline union trace_enum_map_item *
4725trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4726{
4727 /* Return tail of array given the head */
4728 return ptr + ptr->head.length + 1;
4729}
4730
4731static void
4732trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4733 int len)
4734{
4735 struct trace_enum_map **stop;
4736 struct trace_enum_map **map;
4737 union trace_enum_map_item *map_array;
4738 union trace_enum_map_item *ptr;
4739
4740 stop = start + len;
4741
4742 /*
4743 * The trace_enum_maps contains the map plus a head and tail item,
4744 * where the head holds the module and length of array, and the
4745 * tail holds a pointer to the next list.
4746 */
4747 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4748 if (!map_array) {
a395d6a7 4749 pr_warn("Unable to allocate trace enum mapping\n");
9828413d
SRRH
4750 return;
4751 }
4752
4753 mutex_lock(&trace_enum_mutex);
4754
4755 if (!trace_enum_maps)
4756 trace_enum_maps = map_array;
4757 else {
4758 ptr = trace_enum_maps;
4759 for (;;) {
4760 ptr = trace_enum_jmp_to_tail(ptr);
4761 if (!ptr->tail.next)
4762 break;
4763 ptr = ptr->tail.next;
4764
4765 }
4766 ptr->tail.next = map_array;
4767 }
4768 map_array->head.mod = mod;
4769 map_array->head.length = len;
4770 map_array++;
4771
4772 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4773 map_array->map = **map;
4774 map_array++;
4775 }
4776 memset(map_array, 0, sizeof(*map_array));
4777
4778 mutex_unlock(&trace_enum_mutex);
4779}
4780
4781static void trace_create_enum_file(struct dentry *d_tracer)
4782{
4783 trace_create_file("enum_map", 0444, d_tracer,
4784 NULL, &tracing_enum_map_fops);
4785}
4786
4787#else /* CONFIG_TRACE_ENUM_MAP_FILE */
4788static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4789static inline void trace_insert_enum_map_file(struct module *mod,
4790 struct trace_enum_map **start, int len) { }
4791#endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4792
4793static void trace_insert_enum_map(struct module *mod,
4794 struct trace_enum_map **start, int len)
0c564a53
SRRH
4795{
4796 struct trace_enum_map **map;
0c564a53
SRRH
4797
4798 if (len <= 0)
4799 return;
4800
4801 map = start;
4802
4803 trace_event_enum_update(map, len);
9828413d
SRRH
4804
4805 trace_insert_enum_map_file(mod, start, len);
0c564a53
SRRH
4806}
4807
bc0c38d1
SR
4808static ssize_t
4809tracing_set_trace_read(struct file *filp, char __user *ubuf,
4810 size_t cnt, loff_t *ppos)
4811{
2b6080f2 4812 struct trace_array *tr = filp->private_data;
ee6c2c1b 4813 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
4814 int r;
4815
4816 mutex_lock(&trace_types_lock);
2b6080f2 4817 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
4818 mutex_unlock(&trace_types_lock);
4819
4bf39a94 4820 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
4821}
4822
b6f11df2
ACM
4823int tracer_init(struct tracer *t, struct trace_array *tr)
4824{
12883efb 4825 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
4826 return t->init(tr);
4827}
4828
12883efb 4829static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
4830{
4831 int cpu;
737223fb 4832
438ced17 4833 for_each_tracing_cpu(cpu)
12883efb 4834 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
4835}
4836
12883efb 4837#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 4838/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
4839static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4840 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
4841{
4842 int cpu, ret = 0;
4843
4844 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4845 for_each_tracing_cpu(cpu) {
12883efb
SRRH
4846 ret = ring_buffer_resize(trace_buf->buffer,
4847 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
4848 if (ret < 0)
4849 break;
12883efb
SRRH
4850 per_cpu_ptr(trace_buf->data, cpu)->entries =
4851 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
4852 }
4853 } else {
12883efb
SRRH
4854 ret = ring_buffer_resize(trace_buf->buffer,
4855 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 4856 if (ret == 0)
12883efb
SRRH
4857 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4858 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
4859 }
4860
4861 return ret;
4862}
12883efb 4863#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 4864
2b6080f2
SR
4865static int __tracing_resize_ring_buffer(struct trace_array *tr,
4866 unsigned long size, int cpu)
73c5162a
SR
4867{
4868 int ret;
4869
4870 /*
4871 * If kernel or user changes the size of the ring buffer
a123c52b
SR
4872 * we use the size that was given, and we can forget about
4873 * expanding it later.
73c5162a 4874 */
55034cd6 4875 ring_buffer_expanded = true;
73c5162a 4876
b382ede6 4877 /* May be called before buffers are initialized */
12883efb 4878 if (!tr->trace_buffer.buffer)
b382ede6
SR
4879 return 0;
4880
12883efb 4881 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
4882 if (ret < 0)
4883 return ret;
4884
12883efb 4885#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
4886 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4887 !tr->current_trace->use_max_tr)
ef710e10
KM
4888 goto out;
4889
12883efb 4890 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 4891 if (ret < 0) {
12883efb
SRRH
4892 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4893 &tr->trace_buffer, cpu);
73c5162a 4894 if (r < 0) {
a123c52b
SR
4895 /*
4896 * AARGH! We are left with different
4897 * size max buffer!!!!
4898 * The max buffer is our "snapshot" buffer.
4899 * When a tracer needs a snapshot (one of the
4900 * latency tracers), it swaps the max buffer
4901 * with the saved snap shot. We succeeded to
4902 * update the size of the main buffer, but failed to
4903 * update the size of the max buffer. But when we tried
4904 * to reset the main buffer to the original size, we
4905 * failed there too. This is very unlikely to
4906 * happen, but if it does, warn and kill all
4907 * tracing.
4908 */
73c5162a
SR
4909 WARN_ON(1);
4910 tracing_disabled = 1;
4911 }
4912 return ret;
4913 }
4914
438ced17 4915 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 4916 set_buffer_entries(&tr->max_buffer, size);
438ced17 4917 else
12883efb 4918 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 4919
ef710e10 4920 out:
12883efb
SRRH
4921#endif /* CONFIG_TRACER_MAX_TRACE */
4922
438ced17 4923 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 4924 set_buffer_entries(&tr->trace_buffer, size);
438ced17 4925 else
12883efb 4926 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
4927
4928 return ret;
4929}
4930
2b6080f2
SR
4931static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4932 unsigned long size, int cpu_id)
4f271a2a 4933{
83f40318 4934 int ret = size;
4f271a2a
VN
4935
4936 mutex_lock(&trace_types_lock);
4937
438ced17
VN
4938 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4939 /* make sure, this cpu is enabled in the mask */
4940 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4941 ret = -EINVAL;
4942 goto out;
4943 }
4944 }
4f271a2a 4945
2b6080f2 4946 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
4947 if (ret < 0)
4948 ret = -ENOMEM;
4949
438ced17 4950out:
4f271a2a
VN
4951 mutex_unlock(&trace_types_lock);
4952
4953 return ret;
4954}
4955
ef710e10 4956
1852fcce
SR
4957/**
4958 * tracing_update_buffers - used by tracing facility to expand ring buffers
4959 *
4960 * To save on memory when the tracing is never used on a system with it
4961 * configured in. The ring buffers are set to a minimum size. But once
4962 * a user starts to use the tracing facility, then they need to grow
4963 * to their default size.
4964 *
4965 * This function is to be called when a tracer is about to be used.
4966 */
4967int tracing_update_buffers(void)
4968{
4969 int ret = 0;
4970
1027fcb2 4971 mutex_lock(&trace_types_lock);
1852fcce 4972 if (!ring_buffer_expanded)
2b6080f2 4973 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 4974 RING_BUFFER_ALL_CPUS);
1027fcb2 4975 mutex_unlock(&trace_types_lock);
1852fcce
SR
4976
4977 return ret;
4978}
4979
577b785f
SR
4980struct trace_option_dentry;
4981
37aea98b 4982static void
2b6080f2 4983create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f 4984
6b450d25
SRRH
4985/*
4986 * Used to clear out the tracer before deletion of an instance.
4987 * Must have trace_types_lock held.
4988 */
4989static void tracing_set_nop(struct trace_array *tr)
4990{
4991 if (tr->current_trace == &nop_trace)
4992 return;
4993
50512ab5 4994 tr->current_trace->enabled--;
6b450d25
SRRH
4995
4996 if (tr->current_trace->reset)
4997 tr->current_trace->reset(tr);
4998
4999 tr->current_trace = &nop_trace;
5000}
5001
41d9c0be 5002static void add_tracer_options(struct trace_array *tr, struct tracer *t)
bc0c38d1 5003{
09d23a1d
SRRH
5004 /* Only enable if the directory has been created already. */
5005 if (!tr->dir)
5006 return;
5007
37aea98b 5008 create_trace_option_files(tr, t);
09d23a1d
SRRH
5009}
5010
5011static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5012{
bc0c38d1 5013 struct tracer *t;
12883efb 5014#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5015 bool had_max_tr;
12883efb 5016#endif
d9e54076 5017 int ret = 0;
bc0c38d1 5018
1027fcb2
SR
5019 mutex_lock(&trace_types_lock);
5020
73c5162a 5021 if (!ring_buffer_expanded) {
2b6080f2 5022 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 5023 RING_BUFFER_ALL_CPUS);
73c5162a 5024 if (ret < 0)
59f586db 5025 goto out;
73c5162a
SR
5026 ret = 0;
5027 }
5028
bc0c38d1
SR
5029 for (t = trace_types; t; t = t->next) {
5030 if (strcmp(t->name, buf) == 0)
5031 break;
5032 }
c2931e05
FW
5033 if (!t) {
5034 ret = -EINVAL;
5035 goto out;
5036 }
2b6080f2 5037 if (t == tr->current_trace)
bc0c38d1
SR
5038 goto out;
5039
607e2ea1
SRRH
5040 /* Some tracers are only allowed for the top level buffer */
5041 if (!trace_ok_for_array(t, tr)) {
5042 ret = -EINVAL;
5043 goto out;
5044 }
5045
cf6ab6d9
SRRH
5046 /* If trace pipe files are being read, we can't change the tracer */
5047 if (tr->current_trace->ref) {
5048 ret = -EBUSY;
5049 goto out;
5050 }
5051
9f029e83 5052 trace_branch_disable();
613f04a0 5053
50512ab5 5054 tr->current_trace->enabled--;
613f04a0 5055
2b6080f2
SR
5056 if (tr->current_trace->reset)
5057 tr->current_trace->reset(tr);
34600f0e 5058
12883efb 5059 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 5060 tr->current_trace = &nop_trace;
34600f0e 5061
45ad21ca
SRRH
5062#ifdef CONFIG_TRACER_MAX_TRACE
5063 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
5064
5065 if (had_max_tr && !t->use_max_tr) {
5066 /*
5067 * We need to make sure that the update_max_tr sees that
5068 * current_trace changed to nop_trace to keep it from
5069 * swapping the buffers after we resize it.
5070 * The update_max_tr is called from interrupts disabled
5071 * so a synchronized_sched() is sufficient.
5072 */
5073 synchronize_sched();
3209cff4 5074 free_snapshot(tr);
ef710e10 5075 }
12883efb 5076#endif
12883efb
SRRH
5077
5078#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5079 if (t->use_max_tr && !had_max_tr) {
3209cff4 5080 ret = alloc_snapshot(tr);
d60da506
HT
5081 if (ret < 0)
5082 goto out;
ef710e10 5083 }
12883efb 5084#endif
577b785f 5085
1c80025a 5086 if (t->init) {
b6f11df2 5087 ret = tracer_init(t, tr);
1c80025a
FW
5088 if (ret)
5089 goto out;
5090 }
bc0c38d1 5091
2b6080f2 5092 tr->current_trace = t;
50512ab5 5093 tr->current_trace->enabled++;
9f029e83 5094 trace_branch_enable(tr);
bc0c38d1
SR
5095 out:
5096 mutex_unlock(&trace_types_lock);
5097
d9e54076
PZ
5098 return ret;
5099}
5100
5101static ssize_t
5102tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5103 size_t cnt, loff_t *ppos)
5104{
607e2ea1 5105 struct trace_array *tr = filp->private_data;
ee6c2c1b 5106 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
5107 int i;
5108 size_t ret;
e6e7a65a
FW
5109 int err;
5110
5111 ret = cnt;
d9e54076 5112
ee6c2c1b
LZ
5113 if (cnt > MAX_TRACER_SIZE)
5114 cnt = MAX_TRACER_SIZE;
d9e54076 5115
4afe6495 5116 if (copy_from_user(buf, ubuf, cnt))
d9e54076
PZ
5117 return -EFAULT;
5118
5119 buf[cnt] = 0;
5120
5121 /* strip ending whitespace. */
5122 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5123 buf[i] = 0;
5124
607e2ea1 5125 err = tracing_set_tracer(tr, buf);
e6e7a65a
FW
5126 if (err)
5127 return err;
d9e54076 5128
cf8517cf 5129 *ppos += ret;
bc0c38d1 5130
c2931e05 5131 return ret;
bc0c38d1
SR
5132}
5133
5134static ssize_t
6508fa76
SF
5135tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5136 size_t cnt, loff_t *ppos)
bc0c38d1 5137{
bc0c38d1
SR
5138 char buf[64];
5139 int r;
5140
cffae437 5141 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 5142 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
5143 if (r > sizeof(buf))
5144 r = sizeof(buf);
4bf39a94 5145 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
5146}
5147
5148static ssize_t
6508fa76
SF
5149tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5150 size_t cnt, loff_t *ppos)
bc0c38d1 5151{
5e39841c 5152 unsigned long val;
c6caeeb1 5153 int ret;
bc0c38d1 5154
22fe9b54
PH
5155 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5156 if (ret)
c6caeeb1 5157 return ret;
bc0c38d1
SR
5158
5159 *ptr = val * 1000;
5160
5161 return cnt;
5162}
5163
6508fa76
SF
5164static ssize_t
5165tracing_thresh_read(struct file *filp, char __user *ubuf,
5166 size_t cnt, loff_t *ppos)
5167{
5168 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5169}
5170
5171static ssize_t
5172tracing_thresh_write(struct file *filp, const char __user *ubuf,
5173 size_t cnt, loff_t *ppos)
5174{
5175 struct trace_array *tr = filp->private_data;
5176 int ret;
5177
5178 mutex_lock(&trace_types_lock);
5179 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5180 if (ret < 0)
5181 goto out;
5182
5183 if (tr->current_trace->update_thresh) {
5184 ret = tr->current_trace->update_thresh(tr);
5185 if (ret < 0)
5186 goto out;
5187 }
5188
5189 ret = cnt;
5190out:
5191 mutex_unlock(&trace_types_lock);
5192
5193 return ret;
5194}
5195
f971cc9a 5196#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
e428abbb 5197
6508fa76
SF
5198static ssize_t
5199tracing_max_lat_read(struct file *filp, char __user *ubuf,
5200 size_t cnt, loff_t *ppos)
5201{
5202 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5203}
5204
5205static ssize_t
5206tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5207 size_t cnt, loff_t *ppos)
5208{
5209 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5210}
5211
e428abbb
CG
5212#endif
5213
b3806b43
SR
5214static int tracing_open_pipe(struct inode *inode, struct file *filp)
5215{
15544209 5216 struct trace_array *tr = inode->i_private;
b3806b43 5217 struct trace_iterator *iter;
b04cc6b1 5218 int ret = 0;
b3806b43
SR
5219
5220 if (tracing_disabled)
5221 return -ENODEV;
5222
7b85af63
SRRH
5223 if (trace_array_get(tr) < 0)
5224 return -ENODEV;
5225
b04cc6b1
FW
5226 mutex_lock(&trace_types_lock);
5227
b3806b43
SR
5228 /* create a buffer to store the information to pass to userspace */
5229 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
5230 if (!iter) {
5231 ret = -ENOMEM;
f77d09a3 5232 __trace_array_put(tr);
b04cc6b1
FW
5233 goto out;
5234 }
b3806b43 5235
3a161d99 5236 trace_seq_init(&iter->seq);
d716ff71 5237 iter->trace = tr->current_trace;
d7350c3f 5238
4462344e 5239 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 5240 ret = -ENOMEM;
d7350c3f 5241 goto fail;
4462344e
RR
5242 }
5243
a309720c 5244 /* trace pipe does not show start of buffer */
4462344e 5245 cpumask_setall(iter->started);
a309720c 5246
983f938a 5247 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
112f38a7
SR
5248 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5249
8be0709f 5250 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 5251 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
5252 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5253
15544209
ON
5254 iter->tr = tr;
5255 iter->trace_buffer = &tr->trace_buffer;
5256 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 5257 mutex_init(&iter->mutex);
b3806b43
SR
5258 filp->private_data = iter;
5259
107bad8b
SR
5260 if (iter->trace->pipe_open)
5261 iter->trace->pipe_open(iter);
107bad8b 5262
b444786f 5263 nonseekable_open(inode, filp);
cf6ab6d9
SRRH
5264
5265 tr->current_trace->ref++;
b04cc6b1
FW
5266out:
5267 mutex_unlock(&trace_types_lock);
5268 return ret;
d7350c3f
FW
5269
5270fail:
5271 kfree(iter->trace);
5272 kfree(iter);
7b85af63 5273 __trace_array_put(tr);
d7350c3f
FW
5274 mutex_unlock(&trace_types_lock);
5275 return ret;
b3806b43
SR
5276}
5277
5278static int tracing_release_pipe(struct inode *inode, struct file *file)
5279{
5280 struct trace_iterator *iter = file->private_data;
15544209 5281 struct trace_array *tr = inode->i_private;
b3806b43 5282
b04cc6b1
FW
5283 mutex_lock(&trace_types_lock);
5284
cf6ab6d9
SRRH
5285 tr->current_trace->ref--;
5286
29bf4a5e 5287 if (iter->trace->pipe_close)
c521efd1
SR
5288 iter->trace->pipe_close(iter);
5289
b04cc6b1
FW
5290 mutex_unlock(&trace_types_lock);
5291
4462344e 5292 free_cpumask_var(iter->started);
d7350c3f 5293 mutex_destroy(&iter->mutex);
b3806b43 5294 kfree(iter);
b3806b43 5295
7b85af63
SRRH
5296 trace_array_put(tr);
5297
b3806b43
SR
5298 return 0;
5299}
5300
2a2cc8f7 5301static unsigned int
cc60cdc9 5302trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 5303{
983f938a
SRRH
5304 struct trace_array *tr = iter->tr;
5305
15693458
SRRH
5306 /* Iterators are static, they should be filled or empty */
5307 if (trace_buffer_iter(iter, iter->cpu_file))
5308 return POLLIN | POLLRDNORM;
2a2cc8f7 5309
983f938a 5310 if (tr->trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
5311 /*
5312 * Always select as readable when in blocking mode
5313 */
5314 return POLLIN | POLLRDNORM;
15693458 5315 else
12883efb 5316 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 5317 filp, poll_table);
2a2cc8f7 5318}
2a2cc8f7 5319
cc60cdc9
SR
5320static unsigned int
5321tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5322{
5323 struct trace_iterator *iter = filp->private_data;
5324
5325 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
5326}
5327
d716ff71 5328/* Must be called with iter->mutex held. */
ff98781b 5329static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
5330{
5331 struct trace_iterator *iter = filp->private_data;
8b8b3683 5332 int ret;
b3806b43 5333
b3806b43 5334 while (trace_empty(iter)) {
2dc8f095 5335
107bad8b 5336 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 5337 return -EAGAIN;
107bad8b 5338 }
2dc8f095 5339
b3806b43 5340 /*
250bfd3d 5341 * We block until we read something and tracing is disabled.
b3806b43
SR
5342 * We still block if tracing is disabled, but we have never
5343 * read anything. This allows a user to cat this file, and
5344 * then enable tracing. But after we have read something,
5345 * we give an EOF when tracing is again disabled.
5346 *
5347 * iter->pos will be 0 if we haven't read anything.
5348 */
10246fa3 5349 if (!tracing_is_on() && iter->pos)
b3806b43 5350 break;
f4874261
SRRH
5351
5352 mutex_unlock(&iter->mutex);
5353
e30f53aa 5354 ret = wait_on_pipe(iter, false);
f4874261
SRRH
5355
5356 mutex_lock(&iter->mutex);
5357
8b8b3683
SRRH
5358 if (ret)
5359 return ret;
b3806b43
SR
5360 }
5361
ff98781b
EGM
5362 return 1;
5363}
5364
5365/*
5366 * Consumer reader.
5367 */
5368static ssize_t
5369tracing_read_pipe(struct file *filp, char __user *ubuf,
5370 size_t cnt, loff_t *ppos)
5371{
5372 struct trace_iterator *iter = filp->private_data;
5373 ssize_t sret;
5374
d7350c3f
FW
5375 /*
5376 * Avoid more than one consumer on a single file descriptor
5377 * This is just a matter of traces coherency, the ring buffer itself
5378 * is protected.
5379 */
5380 mutex_lock(&iter->mutex);
1245800c
SRRH
5381
5382 /* return any leftover data */
5383 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5384 if (sret != -EBUSY)
5385 goto out;
5386
5387 trace_seq_init(&iter->seq);
5388
ff98781b
EGM
5389 if (iter->trace->read) {
5390 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5391 if (sret)
5392 goto out;
5393 }
5394
5395waitagain:
5396 sret = tracing_wait_pipe(filp);
5397 if (sret <= 0)
5398 goto out;
5399
b3806b43 5400 /* stop when tracing is finished */
ff98781b
EGM
5401 if (trace_empty(iter)) {
5402 sret = 0;
107bad8b 5403 goto out;
ff98781b 5404 }
b3806b43
SR
5405
5406 if (cnt >= PAGE_SIZE)
5407 cnt = PAGE_SIZE - 1;
5408
53d0aa77 5409 /* reset all but tr, trace, and overruns */
53d0aa77
SR
5410 memset(&iter->seq, 0,
5411 sizeof(struct trace_iterator) -
5412 offsetof(struct trace_iterator, seq));
ed5467da 5413 cpumask_clear(iter->started);
4823ed7e 5414 iter->pos = -1;
b3806b43 5415
4f535968 5416 trace_event_read_lock();
7e53bd42 5417 trace_access_lock(iter->cpu_file);
955b61e5 5418 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 5419 enum print_line_t ret;
5ac48378 5420 int save_len = iter->seq.seq.len;
088b1e42 5421
f9896bf3 5422 ret = print_trace_line(iter);
2c4f035f 5423 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42 5424 /* don't print partial lines */
5ac48378 5425 iter->seq.seq.len = save_len;
b3806b43 5426 break;
088b1e42 5427 }
b91facc3
FW
5428 if (ret != TRACE_TYPE_NO_CONSUME)
5429 trace_consume(iter);
b3806b43 5430
5ac48378 5431 if (trace_seq_used(&iter->seq) >= cnt)
b3806b43 5432 break;
ee5e51f5
JO
5433
5434 /*
5435 * Setting the full flag means we reached the trace_seq buffer
5436 * size and we should leave by partial output condition above.
5437 * One of the trace_seq_* functions is not used properly.
5438 */
5439 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5440 iter->ent->type);
b3806b43 5441 }
7e53bd42 5442 trace_access_unlock(iter->cpu_file);
4f535968 5443 trace_event_read_unlock();
b3806b43 5444
b3806b43 5445 /* Now copy what we have to the user */
6c6c2796 5446 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5ac48378 5447 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
f9520750 5448 trace_seq_init(&iter->seq);
9ff4b974
PP
5449
5450 /*
25985edc 5451 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
5452 * entries, go back to wait for more entries.
5453 */
6c6c2796 5454 if (sret == -EBUSY)
9ff4b974 5455 goto waitagain;
b3806b43 5456
107bad8b 5457out:
d7350c3f 5458 mutex_unlock(&iter->mutex);
107bad8b 5459
6c6c2796 5460 return sret;
b3806b43
SR
5461}
5462
3c56819b
EGM
5463static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5464 unsigned int idx)
5465{
5466 __free_page(spd->pages[idx]);
5467}
5468
28dfef8f 5469static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998 5470 .can_merge = 0,
34cd4998 5471 .confirm = generic_pipe_buf_confirm,
92fdd98c 5472 .release = generic_pipe_buf_release,
34cd4998
SR
5473 .steal = generic_pipe_buf_steal,
5474 .get = generic_pipe_buf_get,
3c56819b
EGM
5475};
5476
34cd4998 5477static size_t
fa7c7f6e 5478tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
5479{
5480 size_t count;
74f06bb7 5481 int save_len;
34cd4998
SR
5482 int ret;
5483
5484 /* Seq buffer is page-sized, exactly what we need. */
5485 for (;;) {
74f06bb7 5486 save_len = iter->seq.seq.len;
34cd4998 5487 ret = print_trace_line(iter);
74f06bb7
SRRH
5488
5489 if (trace_seq_has_overflowed(&iter->seq)) {
5490 iter->seq.seq.len = save_len;
34cd4998
SR
5491 break;
5492 }
74f06bb7
SRRH
5493
5494 /*
5495 * This should not be hit, because it should only
5496 * be set if the iter->seq overflowed. But check it
5497 * anyway to be safe.
5498 */
34cd4998 5499 if (ret == TRACE_TYPE_PARTIAL_LINE) {
74f06bb7
SRRH
5500 iter->seq.seq.len = save_len;
5501 break;
5502 }
5503
5ac48378 5504 count = trace_seq_used(&iter->seq) - save_len;
74f06bb7
SRRH
5505 if (rem < count) {
5506 rem = 0;
5507 iter->seq.seq.len = save_len;
34cd4998
SR
5508 break;
5509 }
5510
74e7ff8c
LJ
5511 if (ret != TRACE_TYPE_NO_CONSUME)
5512 trace_consume(iter);
34cd4998 5513 rem -= count;
955b61e5 5514 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
5515 rem = 0;
5516 iter->ent = NULL;
5517 break;
5518 }
5519 }
5520
5521 return rem;
5522}
5523
3c56819b
EGM
5524static ssize_t tracing_splice_read_pipe(struct file *filp,
5525 loff_t *ppos,
5526 struct pipe_inode_info *pipe,
5527 size_t len,
5528 unsigned int flags)
5529{
35f3d14d
JA
5530 struct page *pages_def[PIPE_DEF_BUFFERS];
5531 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
5532 struct trace_iterator *iter = filp->private_data;
5533 struct splice_pipe_desc spd = {
35f3d14d
JA
5534 .pages = pages_def,
5535 .partial = partial_def,
34cd4998 5536 .nr_pages = 0, /* This gets updated below. */
047fe360 5537 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
5538 .flags = flags,
5539 .ops = &tracing_pipe_buf_ops,
5540 .spd_release = tracing_spd_release_pipe,
3c56819b
EGM
5541 };
5542 ssize_t ret;
34cd4998 5543 size_t rem;
3c56819b
EGM
5544 unsigned int i;
5545
35f3d14d
JA
5546 if (splice_grow_spd(pipe, &spd))
5547 return -ENOMEM;
5548
d7350c3f 5549 mutex_lock(&iter->mutex);
3c56819b
EGM
5550
5551 if (iter->trace->splice_read) {
5552 ret = iter->trace->splice_read(iter, filp,
5553 ppos, pipe, len, flags);
5554 if (ret)
34cd4998 5555 goto out_err;
3c56819b
EGM
5556 }
5557
5558 ret = tracing_wait_pipe(filp);
5559 if (ret <= 0)
34cd4998 5560 goto out_err;
3c56819b 5561
955b61e5 5562 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 5563 ret = -EFAULT;
34cd4998 5564 goto out_err;
3c56819b
EGM
5565 }
5566
4f535968 5567 trace_event_read_lock();
7e53bd42 5568 trace_access_lock(iter->cpu_file);
4f535968 5569
3c56819b 5570 /* Fill as many pages as possible. */
a786c06d 5571 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
35f3d14d
JA
5572 spd.pages[i] = alloc_page(GFP_KERNEL);
5573 if (!spd.pages[i])
34cd4998 5574 break;
3c56819b 5575
fa7c7f6e 5576 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
5577
5578 /* Copy the data into the page, so we can start over. */
5579 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 5580 page_address(spd.pages[i]),
5ac48378 5581 trace_seq_used(&iter->seq));
3c56819b 5582 if (ret < 0) {
35f3d14d 5583 __free_page(spd.pages[i]);
3c56819b
EGM
5584 break;
5585 }
35f3d14d 5586 spd.partial[i].offset = 0;
5ac48378 5587 spd.partial[i].len = trace_seq_used(&iter->seq);
3c56819b 5588
f9520750 5589 trace_seq_init(&iter->seq);
3c56819b
EGM
5590 }
5591
7e53bd42 5592 trace_access_unlock(iter->cpu_file);
4f535968 5593 trace_event_read_unlock();
d7350c3f 5594 mutex_unlock(&iter->mutex);
3c56819b
EGM
5595
5596 spd.nr_pages = i;
5597
a29054d9
SRRH
5598 if (i)
5599 ret = splice_to_pipe(pipe, &spd);
5600 else
5601 ret = 0;
35f3d14d 5602out:
047fe360 5603 splice_shrink_spd(&spd);
35f3d14d 5604 return ret;
3c56819b 5605
34cd4998 5606out_err:
d7350c3f 5607 mutex_unlock(&iter->mutex);
35f3d14d 5608 goto out;
3c56819b
EGM
5609}
5610
a98a3c3f
SR
5611static ssize_t
5612tracing_entries_read(struct file *filp, char __user *ubuf,
5613 size_t cnt, loff_t *ppos)
5614{
0bc392ee
ON
5615 struct inode *inode = file_inode(filp);
5616 struct trace_array *tr = inode->i_private;
5617 int cpu = tracing_get_cpu(inode);
438ced17
VN
5618 char buf[64];
5619 int r = 0;
5620 ssize_t ret;
a98a3c3f 5621
db526ca3 5622 mutex_lock(&trace_types_lock);
438ced17 5623
0bc392ee 5624 if (cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
5625 int cpu, buf_size_same;
5626 unsigned long size;
5627
5628 size = 0;
5629 buf_size_same = 1;
5630 /* check if all cpu sizes are same */
5631 for_each_tracing_cpu(cpu) {
5632 /* fill in the size from first enabled cpu */
5633 if (size == 0)
12883efb
SRRH
5634 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5635 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
5636 buf_size_same = 0;
5637 break;
5638 }
5639 }
5640
5641 if (buf_size_same) {
5642 if (!ring_buffer_expanded)
5643 r = sprintf(buf, "%lu (expanded: %lu)\n",
5644 size >> 10,
5645 trace_buf_size >> 10);
5646 else
5647 r = sprintf(buf, "%lu\n", size >> 10);
5648 } else
5649 r = sprintf(buf, "X\n");
5650 } else
0bc392ee 5651 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
438ced17 5652
db526ca3
SR
5653 mutex_unlock(&trace_types_lock);
5654
438ced17
VN
5655 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5656 return ret;
a98a3c3f
SR
5657}
5658
5659static ssize_t
5660tracing_entries_write(struct file *filp, const char __user *ubuf,
5661 size_t cnt, loff_t *ppos)
5662{
0bc392ee
ON
5663 struct inode *inode = file_inode(filp);
5664 struct trace_array *tr = inode->i_private;
a98a3c3f 5665 unsigned long val;
4f271a2a 5666 int ret;
a98a3c3f 5667
22fe9b54
PH
5668 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5669 if (ret)
c6caeeb1 5670 return ret;
a98a3c3f
SR
5671
5672 /* must have at least 1 entry */
5673 if (!val)
5674 return -EINVAL;
5675
1696b2b0
SR
5676 /* value is in KB */
5677 val <<= 10;
0bc392ee 5678 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4f271a2a
VN
5679 if (ret < 0)
5680 return ret;
a98a3c3f 5681
cf8517cf 5682 *ppos += cnt;
a98a3c3f 5683
4f271a2a
VN
5684 return cnt;
5685}
bf5e6519 5686
f81ab074
VN
5687static ssize_t
5688tracing_total_entries_read(struct file *filp, char __user *ubuf,
5689 size_t cnt, loff_t *ppos)
5690{
5691 struct trace_array *tr = filp->private_data;
5692 char buf[64];
5693 int r, cpu;
5694 unsigned long size = 0, expanded_size = 0;
5695
5696 mutex_lock(&trace_types_lock);
5697 for_each_tracing_cpu(cpu) {
12883efb 5698 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
5699 if (!ring_buffer_expanded)
5700 expanded_size += trace_buf_size >> 10;
5701 }
5702 if (ring_buffer_expanded)
5703 r = sprintf(buf, "%lu\n", size);
5704 else
5705 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5706 mutex_unlock(&trace_types_lock);
5707
5708 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5709}
5710
4f271a2a
VN
5711static ssize_t
5712tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5713 size_t cnt, loff_t *ppos)
5714{
5715 /*
5716 * There is no need to read what the user has written, this function
5717 * is just to make sure that there is no error when "echo" is used
5718 */
5719
5720 *ppos += cnt;
a98a3c3f
SR
5721
5722 return cnt;
5723}
5724
4f271a2a
VN
5725static int
5726tracing_free_buffer_release(struct inode *inode, struct file *filp)
5727{
2b6080f2
SR
5728 struct trace_array *tr = inode->i_private;
5729
cf30cf67 5730 /* disable tracing ? */
983f938a 5731 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
711e1243 5732 tracer_tracing_off(tr);
4f271a2a 5733 /* resize the ring buffer to 0 */
2b6080f2 5734 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a 5735
7b85af63
SRRH
5736 trace_array_put(tr);
5737
4f271a2a
VN
5738 return 0;
5739}
5740
fa32e855
SR
5741static inline int lock_user_pages(const char __user *ubuf, size_t cnt,
5742 struct page **pages, void **map_page,
5743 int *offset)
5bf9a1ee 5744{
d696b58c 5745 unsigned long addr = (unsigned long)ubuf;
d696b58c 5746 int nr_pages = 1;
d696b58c 5747 int ret;
6edb2a8a 5748 int i;
5bf9a1ee 5749
d696b58c
SR
5750 /*
5751 * Userspace is injecting traces into the kernel trace buffer.
5752 * We want to be as non intrusive as possible.
5753 * To do so, we do not want to allocate any special buffers
5754 * or take any locks, but instead write the userspace data
5755 * straight into the ring buffer.
5756 *
5757 * First we need to pin the userspace buffer into memory,
5758 * which, most likely it is, because it just referenced it.
5759 * But there's no guarantee that it is. By using get_user_pages_fast()
5760 * and kmap_atomic/kunmap_atomic() we can get access to the
5761 * pages directly. We then write the data directly into the
5762 * ring buffer.
5763 */
5bf9a1ee 5764
d696b58c
SR
5765 /* check if we cross pages */
5766 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5767 nr_pages = 2;
5768
fa32e855 5769 *offset = addr & (PAGE_SIZE - 1);
d696b58c
SR
5770 addr &= PAGE_MASK;
5771
5772 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5773 if (ret < nr_pages) {
5774 while (--ret >= 0)
5775 put_page(pages[ret]);
fa32e855 5776 return -EFAULT;
5bf9a1ee 5777 }
d696b58c 5778
6edb2a8a
SR
5779 for (i = 0; i < nr_pages; i++)
5780 map_page[i] = kmap_atomic(pages[i]);
d696b58c 5781
fa32e855
SR
5782 return nr_pages;
5783}
5784
5785static inline void unlock_user_pages(struct page **pages,
5786 void **map_page, int nr_pages)
5787{
5788 int i;
5789
5790 for (i = nr_pages - 1; i >= 0; i--) {
5791 kunmap_atomic(map_page[i]);
5792 put_page(pages[i]);
5793 }
5794}
5795
5796static ssize_t
5797tracing_mark_write(struct file *filp, const char __user *ubuf,
5798 size_t cnt, loff_t *fpos)
5799{
5800 struct trace_array *tr = filp->private_data;
5801 struct ring_buffer_event *event;
5802 struct ring_buffer *buffer;
5803 struct print_entry *entry;
5804 unsigned long irq_flags;
5805 struct page *pages[2];
5806 void *map_page[2];
5807 int nr_pages = 1;
5808 ssize_t written;
5809 int offset;
5810 int size;
5811 int len;
5812
5813 if (tracing_disabled)
5814 return -EINVAL;
5815
5816 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5817 return -EINVAL;
5818
5819 if (cnt > TRACE_BUF_SIZE)
5820 cnt = TRACE_BUF_SIZE;
5821
5822 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5823
5824 nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset);
5825 if (nr_pages < 0)
5826 return nr_pages;
5827
d696b58c
SR
5828 local_save_flags(irq_flags);
5829 size = sizeof(*entry) + cnt + 2; /* possible \n added */
2d71619c 5830 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
5831 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5832 irq_flags, preempt_count());
d696b58c
SR
5833 if (!event) {
5834 /* Ring buffer disabled, return as if not open for write */
5835 written = -EBADF;
5836 goto out_unlock;
5bf9a1ee 5837 }
d696b58c
SR
5838
5839 entry = ring_buffer_event_data(event);
5840 entry->ip = _THIS_IP_;
5841
5842 if (nr_pages == 2) {
5843 len = PAGE_SIZE - offset;
6edb2a8a
SR
5844 memcpy(&entry->buf, map_page[0] + offset, len);
5845 memcpy(&entry->buf[len], map_page[1], cnt - len);
c13d2f7c 5846 } else
6edb2a8a 5847 memcpy(&entry->buf, map_page[0] + offset, cnt);
5bf9a1ee 5848
d696b58c
SR
5849 if (entry->buf[cnt - 1] != '\n') {
5850 entry->buf[cnt] = '\n';
5851 entry->buf[cnt + 1] = '\0';
5852 } else
5853 entry->buf[cnt] = '\0';
5854
7ffbd48d 5855 __buffer_unlock_commit(buffer, event);
5bf9a1ee 5856
d696b58c 5857 written = cnt;
5bf9a1ee 5858
d696b58c 5859 *fpos += written;
1aa54bca 5860
d696b58c 5861 out_unlock:
fa32e855
SR
5862 unlock_user_pages(pages, map_page, nr_pages);
5863
5864 return written;
5865}
5866
5867/* Limit it for now to 3K (including tag) */
5868#define RAW_DATA_MAX_SIZE (1024*3)
5869
5870static ssize_t
5871tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
5872 size_t cnt, loff_t *fpos)
5873{
5874 struct trace_array *tr = filp->private_data;
5875 struct ring_buffer_event *event;
5876 struct ring_buffer *buffer;
5877 struct raw_data_entry *entry;
5878 unsigned long irq_flags;
5879 struct page *pages[2];
5880 void *map_page[2];
5881 int nr_pages = 1;
5882 ssize_t written;
5883 int offset;
5884 int size;
5885 int len;
5886
5887 if (tracing_disabled)
5888 return -EINVAL;
5889
5890 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5891 return -EINVAL;
5892
5893 /* The marker must at least have a tag id */
5894 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
5895 return -EINVAL;
5896
5897 if (cnt > TRACE_BUF_SIZE)
5898 cnt = TRACE_BUF_SIZE;
5899
5900 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5901
5902 nr_pages = lock_user_pages(ubuf, cnt, pages, map_page, &offset);
5903 if (nr_pages < 0)
5904 return nr_pages;
5905
5906 local_save_flags(irq_flags);
5907 size = sizeof(*entry) + cnt;
5908 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
5909 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
5910 irq_flags, preempt_count());
fa32e855
SR
5911 if (!event) {
5912 /* Ring buffer disabled, return as if not open for write */
5913 written = -EBADF;
5914 goto out_unlock;
6edb2a8a 5915 }
fa32e855
SR
5916
5917 entry = ring_buffer_event_data(event);
5918
5919 if (nr_pages == 2) {
5920 len = PAGE_SIZE - offset;
5921 memcpy(&entry->id, map_page[0] + offset, len);
5922 memcpy(((char *)&entry->id) + len, map_page[1], cnt - len);
5923 } else
5924 memcpy(&entry->id, map_page[0] + offset, cnt);
5925
5926 __buffer_unlock_commit(buffer, event);
5927
5928 written = cnt;
5929
5930 *fpos += written;
5931
5932 out_unlock:
5933 unlock_user_pages(pages, map_page, nr_pages);
5934
1aa54bca 5935 return written;
5bf9a1ee
PP
5936}
5937
13f16d20 5938static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 5939{
2b6080f2 5940 struct trace_array *tr = m->private;
5079f326
Z
5941 int i;
5942
5943 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 5944 seq_printf(m,
5079f326 5945 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
5946 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5947 i == tr->clock_id ? "]" : "");
13f16d20 5948 seq_putc(m, '\n');
5079f326 5949
13f16d20 5950 return 0;
5079f326
Z
5951}
5952
e1e232ca 5953static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5079f326 5954{
5079f326
Z
5955 int i;
5956
5079f326
Z
5957 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5958 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5959 break;
5960 }
5961 if (i == ARRAY_SIZE(trace_clocks))
5962 return -EINVAL;
5963
5079f326
Z
5964 mutex_lock(&trace_types_lock);
5965
2b6080f2
SR
5966 tr->clock_id = i;
5967
12883efb 5968 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 5969
60303ed3
DS
5970 /*
5971 * New clock may not be consistent with the previous clock.
5972 * Reset the buffer so that it doesn't have incomparable timestamps.
5973 */
9457158b 5974 tracing_reset_online_cpus(&tr->trace_buffer);
12883efb
SRRH
5975
5976#ifdef CONFIG_TRACER_MAX_TRACE
5977 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5978 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
9457158b 5979 tracing_reset_online_cpus(&tr->max_buffer);
12883efb 5980#endif
60303ed3 5981
5079f326
Z
5982 mutex_unlock(&trace_types_lock);
5983
e1e232ca
SR
5984 return 0;
5985}
5986
5987static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5988 size_t cnt, loff_t *fpos)
5989{
5990 struct seq_file *m = filp->private_data;
5991 struct trace_array *tr = m->private;
5992 char buf[64];
5993 const char *clockstr;
5994 int ret;
5995
5996 if (cnt >= sizeof(buf))
5997 return -EINVAL;
5998
4afe6495 5999 if (copy_from_user(buf, ubuf, cnt))
e1e232ca
SR
6000 return -EFAULT;
6001
6002 buf[cnt] = 0;
6003
6004 clockstr = strstrip(buf);
6005
6006 ret = tracing_set_clock(tr, clockstr);
6007 if (ret)
6008 return ret;
6009
5079f326
Z
6010 *fpos += cnt;
6011
6012 return cnt;
6013}
6014
13f16d20
LZ
6015static int tracing_clock_open(struct inode *inode, struct file *file)
6016{
7b85af63
SRRH
6017 struct trace_array *tr = inode->i_private;
6018 int ret;
6019
13f16d20
LZ
6020 if (tracing_disabled)
6021 return -ENODEV;
2b6080f2 6022
7b85af63
SRRH
6023 if (trace_array_get(tr))
6024 return -ENODEV;
6025
6026 ret = single_open(file, tracing_clock_show, inode->i_private);
6027 if (ret < 0)
6028 trace_array_put(tr);
6029
6030 return ret;
13f16d20
LZ
6031}
6032
6de58e62
SRRH
6033struct ftrace_buffer_info {
6034 struct trace_iterator iter;
6035 void *spare;
6036 unsigned int read;
6037};
6038
debdd57f
HT
6039#ifdef CONFIG_TRACER_SNAPSHOT
6040static int tracing_snapshot_open(struct inode *inode, struct file *file)
6041{
6484c71c 6042 struct trace_array *tr = inode->i_private;
debdd57f 6043 struct trace_iterator *iter;
2b6080f2 6044 struct seq_file *m;
debdd57f
HT
6045 int ret = 0;
6046
ff451961
SRRH
6047 if (trace_array_get(tr) < 0)
6048 return -ENODEV;
6049
debdd57f 6050 if (file->f_mode & FMODE_READ) {
6484c71c 6051 iter = __tracing_open(inode, file, true);
debdd57f
HT
6052 if (IS_ERR(iter))
6053 ret = PTR_ERR(iter);
2b6080f2
SR
6054 } else {
6055 /* Writes still need the seq_file to hold the private data */
f77d09a3 6056 ret = -ENOMEM;
2b6080f2
SR
6057 m = kzalloc(sizeof(*m), GFP_KERNEL);
6058 if (!m)
f77d09a3 6059 goto out;
2b6080f2
SR
6060 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6061 if (!iter) {
6062 kfree(m);
f77d09a3 6063 goto out;
2b6080f2 6064 }
f77d09a3
AL
6065 ret = 0;
6066
ff451961 6067 iter->tr = tr;
6484c71c
ON
6068 iter->trace_buffer = &tr->max_buffer;
6069 iter->cpu_file = tracing_get_cpu(inode);
2b6080f2
SR
6070 m->private = iter;
6071 file->private_data = m;
debdd57f 6072 }
f77d09a3 6073out:
ff451961
SRRH
6074 if (ret < 0)
6075 trace_array_put(tr);
6076
debdd57f
HT
6077 return ret;
6078}
6079
6080static ssize_t
6081tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6082 loff_t *ppos)
6083{
2b6080f2
SR
6084 struct seq_file *m = filp->private_data;
6085 struct trace_iterator *iter = m->private;
6086 struct trace_array *tr = iter->tr;
debdd57f
HT
6087 unsigned long val;
6088 int ret;
6089
6090 ret = tracing_update_buffers();
6091 if (ret < 0)
6092 return ret;
6093
6094 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6095 if (ret)
6096 return ret;
6097
6098 mutex_lock(&trace_types_lock);
6099
2b6080f2 6100 if (tr->current_trace->use_max_tr) {
debdd57f
HT
6101 ret = -EBUSY;
6102 goto out;
6103 }
6104
6105 switch (val) {
6106 case 0:
f1affcaa
SRRH
6107 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6108 ret = -EINVAL;
6109 break;
debdd57f 6110 }
3209cff4
SRRH
6111 if (tr->allocated_snapshot)
6112 free_snapshot(tr);
debdd57f
HT
6113 break;
6114 case 1:
f1affcaa
SRRH
6115/* Only allow per-cpu swap if the ring buffer supports it */
6116#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6117 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6118 ret = -EINVAL;
6119 break;
6120 }
6121#endif
45ad21ca 6122 if (!tr->allocated_snapshot) {
3209cff4 6123 ret = alloc_snapshot(tr);
debdd57f
HT
6124 if (ret < 0)
6125 break;
debdd57f 6126 }
debdd57f
HT
6127 local_irq_disable();
6128 /* Now, we're going to swap */
f1affcaa 6129 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
ce9bae55 6130 update_max_tr(tr, current, smp_processor_id());
f1affcaa 6131 else
ce9bae55 6132 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
6133 local_irq_enable();
6134 break;
6135 default:
45ad21ca 6136 if (tr->allocated_snapshot) {
f1affcaa
SRRH
6137 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6138 tracing_reset_online_cpus(&tr->max_buffer);
6139 else
6140 tracing_reset(&tr->max_buffer, iter->cpu_file);
6141 }
debdd57f
HT
6142 break;
6143 }
6144
6145 if (ret >= 0) {
6146 *ppos += cnt;
6147 ret = cnt;
6148 }
6149out:
6150 mutex_unlock(&trace_types_lock);
6151 return ret;
6152}
2b6080f2
SR
6153
6154static int tracing_snapshot_release(struct inode *inode, struct file *file)
6155{
6156 struct seq_file *m = file->private_data;
ff451961
SRRH
6157 int ret;
6158
6159 ret = tracing_release(inode, file);
2b6080f2
SR
6160
6161 if (file->f_mode & FMODE_READ)
ff451961 6162 return ret;
2b6080f2
SR
6163
6164 /* If write only, the seq_file is just a stub */
6165 if (m)
6166 kfree(m->private);
6167 kfree(m);
6168
6169 return 0;
6170}
6171
6de58e62
SRRH
6172static int tracing_buffers_open(struct inode *inode, struct file *filp);
6173static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6174 size_t count, loff_t *ppos);
6175static int tracing_buffers_release(struct inode *inode, struct file *file);
6176static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6177 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6178
6179static int snapshot_raw_open(struct inode *inode, struct file *filp)
6180{
6181 struct ftrace_buffer_info *info;
6182 int ret;
6183
6184 ret = tracing_buffers_open(inode, filp);
6185 if (ret < 0)
6186 return ret;
6187
6188 info = filp->private_data;
6189
6190 if (info->iter.trace->use_max_tr) {
6191 tracing_buffers_release(inode, filp);
6192 return -EBUSY;
6193 }
6194
6195 info->iter.snapshot = true;
6196 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6197
6198 return ret;
6199}
6200
debdd57f
HT
6201#endif /* CONFIG_TRACER_SNAPSHOT */
6202
6203
6508fa76
SF
6204static const struct file_operations tracing_thresh_fops = {
6205 .open = tracing_open_generic,
6206 .read = tracing_thresh_read,
6207 .write = tracing_thresh_write,
6208 .llseek = generic_file_llseek,
6209};
6210
f971cc9a 6211#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5e2336a0 6212static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
6213 .open = tracing_open_generic,
6214 .read = tracing_max_lat_read,
6215 .write = tracing_max_lat_write,
b444786f 6216 .llseek = generic_file_llseek,
bc0c38d1 6217};
e428abbb 6218#endif
bc0c38d1 6219
5e2336a0 6220static const struct file_operations set_tracer_fops = {
4bf39a94
IM
6221 .open = tracing_open_generic,
6222 .read = tracing_set_trace_read,
6223 .write = tracing_set_trace_write,
b444786f 6224 .llseek = generic_file_llseek,
bc0c38d1
SR
6225};
6226
5e2336a0 6227static const struct file_operations tracing_pipe_fops = {
4bf39a94 6228 .open = tracing_open_pipe,
2a2cc8f7 6229 .poll = tracing_poll_pipe,
4bf39a94 6230 .read = tracing_read_pipe,
3c56819b 6231 .splice_read = tracing_splice_read_pipe,
4bf39a94 6232 .release = tracing_release_pipe,
b444786f 6233 .llseek = no_llseek,
b3806b43
SR
6234};
6235
5e2336a0 6236static const struct file_operations tracing_entries_fops = {
0bc392ee 6237 .open = tracing_open_generic_tr,
a98a3c3f
SR
6238 .read = tracing_entries_read,
6239 .write = tracing_entries_write,
b444786f 6240 .llseek = generic_file_llseek,
0bc392ee 6241 .release = tracing_release_generic_tr,
a98a3c3f
SR
6242};
6243
f81ab074 6244static const struct file_operations tracing_total_entries_fops = {
7b85af63 6245 .open = tracing_open_generic_tr,
f81ab074
VN
6246 .read = tracing_total_entries_read,
6247 .llseek = generic_file_llseek,
7b85af63 6248 .release = tracing_release_generic_tr,
f81ab074
VN
6249};
6250
4f271a2a 6251static const struct file_operations tracing_free_buffer_fops = {
7b85af63 6252 .open = tracing_open_generic_tr,
4f271a2a
VN
6253 .write = tracing_free_buffer_write,
6254 .release = tracing_free_buffer_release,
6255};
6256
5e2336a0 6257static const struct file_operations tracing_mark_fops = {
7b85af63 6258 .open = tracing_open_generic_tr,
5bf9a1ee 6259 .write = tracing_mark_write,
b444786f 6260 .llseek = generic_file_llseek,
7b85af63 6261 .release = tracing_release_generic_tr,
5bf9a1ee
PP
6262};
6263
fa32e855
SR
6264static const struct file_operations tracing_mark_raw_fops = {
6265 .open = tracing_open_generic_tr,
6266 .write = tracing_mark_raw_write,
6267 .llseek = generic_file_llseek,
6268 .release = tracing_release_generic_tr,
6269};
6270
5079f326 6271static const struct file_operations trace_clock_fops = {
13f16d20
LZ
6272 .open = tracing_clock_open,
6273 .read = seq_read,
6274 .llseek = seq_lseek,
7b85af63 6275 .release = tracing_single_release_tr,
5079f326
Z
6276 .write = tracing_clock_write,
6277};
6278
debdd57f
HT
6279#ifdef CONFIG_TRACER_SNAPSHOT
6280static const struct file_operations snapshot_fops = {
6281 .open = tracing_snapshot_open,
6282 .read = seq_read,
6283 .write = tracing_snapshot_write,
098c879e 6284 .llseek = tracing_lseek,
2b6080f2 6285 .release = tracing_snapshot_release,
debdd57f 6286};
debdd57f 6287
6de58e62
SRRH
6288static const struct file_operations snapshot_raw_fops = {
6289 .open = snapshot_raw_open,
6290 .read = tracing_buffers_read,
6291 .release = tracing_buffers_release,
6292 .splice_read = tracing_buffers_splice_read,
6293 .llseek = no_llseek,
2cadf913
SR
6294};
6295
6de58e62
SRRH
6296#endif /* CONFIG_TRACER_SNAPSHOT */
6297
2cadf913
SR
6298static int tracing_buffers_open(struct inode *inode, struct file *filp)
6299{
46ef2be0 6300 struct trace_array *tr = inode->i_private;
2cadf913 6301 struct ftrace_buffer_info *info;
7b85af63 6302 int ret;
2cadf913
SR
6303
6304 if (tracing_disabled)
6305 return -ENODEV;
6306
7b85af63
SRRH
6307 if (trace_array_get(tr) < 0)
6308 return -ENODEV;
6309
2cadf913 6310 info = kzalloc(sizeof(*info), GFP_KERNEL);
7b85af63
SRRH
6311 if (!info) {
6312 trace_array_put(tr);
2cadf913 6313 return -ENOMEM;
7b85af63 6314 }
2cadf913 6315
a695cb58
SRRH
6316 mutex_lock(&trace_types_lock);
6317
cc60cdc9 6318 info->iter.tr = tr;
46ef2be0 6319 info->iter.cpu_file = tracing_get_cpu(inode);
b627344f 6320 info->iter.trace = tr->current_trace;
12883efb 6321 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 6322 info->spare = NULL;
2cadf913 6323 /* Force reading ring buffer for first read */
cc60cdc9 6324 info->read = (unsigned int)-1;
2cadf913
SR
6325
6326 filp->private_data = info;
6327
cf6ab6d9
SRRH
6328 tr->current_trace->ref++;
6329
a695cb58
SRRH
6330 mutex_unlock(&trace_types_lock);
6331
7b85af63
SRRH
6332 ret = nonseekable_open(inode, filp);
6333 if (ret < 0)
6334 trace_array_put(tr);
6335
6336 return ret;
2cadf913
SR
6337}
6338
cc60cdc9
SR
6339static unsigned int
6340tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6341{
6342 struct ftrace_buffer_info *info = filp->private_data;
6343 struct trace_iterator *iter = &info->iter;
6344
6345 return trace_poll(iter, filp, poll_table);
6346}
6347
2cadf913
SR
6348static ssize_t
6349tracing_buffers_read(struct file *filp, char __user *ubuf,
6350 size_t count, loff_t *ppos)
6351{
6352 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 6353 struct trace_iterator *iter = &info->iter;
2cadf913 6354 ssize_t ret;
6de58e62 6355 ssize_t size;
2cadf913 6356
2dc5d12b
SR
6357 if (!count)
6358 return 0;
6359
6de58e62 6360#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
6361 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6362 return -EBUSY;
6de58e62
SRRH
6363#endif
6364
ddd538f3 6365 if (!info->spare)
12883efb
SRRH
6366 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6367 iter->cpu_file);
ddd538f3 6368 if (!info->spare)
d716ff71 6369 return -ENOMEM;
ddd538f3 6370
2cadf913
SR
6371 /* Do we have previous read data to read? */
6372 if (info->read < PAGE_SIZE)
6373 goto read;
6374
b627344f 6375 again:
cc60cdc9 6376 trace_access_lock(iter->cpu_file);
12883efb 6377 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
6378 &info->spare,
6379 count,
cc60cdc9
SR
6380 iter->cpu_file, 0);
6381 trace_access_unlock(iter->cpu_file);
2cadf913 6382
b627344f
SR
6383 if (ret < 0) {
6384 if (trace_empty(iter)) {
d716ff71
SRRH
6385 if ((filp->f_flags & O_NONBLOCK))
6386 return -EAGAIN;
6387
e30f53aa 6388 ret = wait_on_pipe(iter, false);
d716ff71
SRRH
6389 if (ret)
6390 return ret;
6391
b627344f
SR
6392 goto again;
6393 }
d716ff71 6394 return 0;
b627344f 6395 }
436fc280 6396
436fc280 6397 info->read = 0;
b627344f 6398 read:
2cadf913
SR
6399 size = PAGE_SIZE - info->read;
6400 if (size > count)
6401 size = count;
6402
6403 ret = copy_to_user(ubuf, info->spare + info->read, size);
d716ff71
SRRH
6404 if (ret == size)
6405 return -EFAULT;
6406
2dc5d12b
SR
6407 size -= ret;
6408
2cadf913
SR
6409 *ppos += size;
6410 info->read += size;
6411
6412 return size;
6413}
6414
6415static int tracing_buffers_release(struct inode *inode, struct file *file)
6416{
6417 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 6418 struct trace_iterator *iter = &info->iter;
2cadf913 6419
a695cb58
SRRH
6420 mutex_lock(&trace_types_lock);
6421
cf6ab6d9
SRRH
6422 iter->tr->current_trace->ref--;
6423
ff451961 6424 __trace_array_put(iter->tr);
2cadf913 6425
ddd538f3 6426 if (info->spare)
12883efb 6427 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
2cadf913
SR
6428 kfree(info);
6429
a695cb58
SRRH
6430 mutex_unlock(&trace_types_lock);
6431
2cadf913
SR
6432 return 0;
6433}
6434
6435struct buffer_ref {
6436 struct ring_buffer *buffer;
6437 void *page;
6438 int ref;
6439};
6440
6441static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6442 struct pipe_buffer *buf)
6443{
6444 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6445
6446 if (--ref->ref)
6447 return;
6448
6449 ring_buffer_free_read_page(ref->buffer, ref->page);
6450 kfree(ref);
6451 buf->private = 0;
6452}
6453
2cadf913
SR
6454static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6455 struct pipe_buffer *buf)
6456{
6457 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6458
6459 ref->ref++;
6460}
6461
6462/* Pipe buffer operations for a buffer. */
28dfef8f 6463static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913 6464 .can_merge = 0,
2cadf913
SR
6465 .confirm = generic_pipe_buf_confirm,
6466 .release = buffer_pipe_buf_release,
d55cb6cf 6467 .steal = generic_pipe_buf_steal,
2cadf913
SR
6468 .get = buffer_pipe_buf_get,
6469};
6470
6471/*
6472 * Callback from splice_to_pipe(), if we need to release some pages
6473 * at the end of the spd in case we error'ed out in filling the pipe.
6474 */
6475static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6476{
6477 struct buffer_ref *ref =
6478 (struct buffer_ref *)spd->partial[i].private;
6479
6480 if (--ref->ref)
6481 return;
6482
6483 ring_buffer_free_read_page(ref->buffer, ref->page);
6484 kfree(ref);
6485 spd->partial[i].private = 0;
6486}
6487
6488static ssize_t
6489tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6490 struct pipe_inode_info *pipe, size_t len,
6491 unsigned int flags)
6492{
6493 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 6494 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
6495 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6496 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 6497 struct splice_pipe_desc spd = {
35f3d14d
JA
6498 .pages = pages_def,
6499 .partial = partial_def,
047fe360 6500 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
6501 .flags = flags,
6502 .ops = &buffer_pipe_buf_ops,
6503 .spd_release = buffer_spd_release,
6504 };
6505 struct buffer_ref *ref;
93459c6c 6506 int entries, size, i;
07906da7 6507 ssize_t ret = 0;
2cadf913 6508
6de58e62 6509#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
6510 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6511 return -EBUSY;
6de58e62
SRRH
6512#endif
6513
d716ff71
SRRH
6514 if (*ppos & (PAGE_SIZE - 1))
6515 return -EINVAL;
93cfb3c9
LJ
6516
6517 if (len & (PAGE_SIZE - 1)) {
d716ff71
SRRH
6518 if (len < PAGE_SIZE)
6519 return -EINVAL;
93cfb3c9
LJ
6520 len &= PAGE_MASK;
6521 }
6522
1ae2293d
AV
6523 if (splice_grow_spd(pipe, &spd))
6524 return -ENOMEM;
6525
cc60cdc9
SR
6526 again:
6527 trace_access_lock(iter->cpu_file);
12883efb 6528 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 6529
a786c06d 6530 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
6531 struct page *page;
6532 int r;
6533
6534 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
07906da7
RV
6535 if (!ref) {
6536 ret = -ENOMEM;
2cadf913 6537 break;
07906da7 6538 }
2cadf913 6539
7267fa68 6540 ref->ref = 1;
12883efb 6541 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 6542 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
2cadf913 6543 if (!ref->page) {
07906da7 6544 ret = -ENOMEM;
2cadf913
SR
6545 kfree(ref);
6546 break;
6547 }
6548
6549 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 6550 len, iter->cpu_file, 1);
2cadf913 6551 if (r < 0) {
7ea59064 6552 ring_buffer_free_read_page(ref->buffer, ref->page);
2cadf913
SR
6553 kfree(ref);
6554 break;
6555 }
6556
6557 /*
6558 * zero out any left over data, this is going to
6559 * user land.
6560 */
6561 size = ring_buffer_page_len(ref->page);
6562 if (size < PAGE_SIZE)
6563 memset(ref->page + size, 0, PAGE_SIZE - size);
6564
6565 page = virt_to_page(ref->page);
6566
6567 spd.pages[i] = page;
6568 spd.partial[i].len = PAGE_SIZE;
6569 spd.partial[i].offset = 0;
6570 spd.partial[i].private = (unsigned long)ref;
6571 spd.nr_pages++;
93cfb3c9 6572 *ppos += PAGE_SIZE;
93459c6c 6573
12883efb 6574 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
6575 }
6576
cc60cdc9 6577 trace_access_unlock(iter->cpu_file);
2cadf913
SR
6578 spd.nr_pages = i;
6579
6580 /* did we read anything? */
6581 if (!spd.nr_pages) {
07906da7 6582 if (ret)
1ae2293d 6583 goto out;
d716ff71 6584
1ae2293d 6585 ret = -EAGAIN;
d716ff71 6586 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
1ae2293d 6587 goto out;
07906da7 6588
e30f53aa 6589 ret = wait_on_pipe(iter, true);
8b8b3683 6590 if (ret)
1ae2293d 6591 goto out;
e30f53aa 6592
cc60cdc9 6593 goto again;
2cadf913
SR
6594 }
6595
6596 ret = splice_to_pipe(pipe, &spd);
1ae2293d 6597out:
047fe360 6598 splice_shrink_spd(&spd);
6de58e62 6599
2cadf913
SR
6600 return ret;
6601}
6602
6603static const struct file_operations tracing_buffers_fops = {
6604 .open = tracing_buffers_open,
6605 .read = tracing_buffers_read,
cc60cdc9 6606 .poll = tracing_buffers_poll,
2cadf913
SR
6607 .release = tracing_buffers_release,
6608 .splice_read = tracing_buffers_splice_read,
6609 .llseek = no_llseek,
6610};
6611
c8d77183
SR
6612static ssize_t
6613tracing_stats_read(struct file *filp, char __user *ubuf,
6614 size_t count, loff_t *ppos)
6615{
4d3435b8
ON
6616 struct inode *inode = file_inode(filp);
6617 struct trace_array *tr = inode->i_private;
12883efb 6618 struct trace_buffer *trace_buf = &tr->trace_buffer;
4d3435b8 6619 int cpu = tracing_get_cpu(inode);
c8d77183
SR
6620 struct trace_seq *s;
6621 unsigned long cnt;
c64e148a
VN
6622 unsigned long long t;
6623 unsigned long usec_rem;
c8d77183 6624
e4f2d10f 6625 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 6626 if (!s)
a646365c 6627 return -ENOMEM;
c8d77183
SR
6628
6629 trace_seq_init(s);
6630
12883efb 6631 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6632 trace_seq_printf(s, "entries: %ld\n", cnt);
6633
12883efb 6634 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6635 trace_seq_printf(s, "overrun: %ld\n", cnt);
6636
12883efb 6637 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6638 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6639
12883efb 6640 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
6641 trace_seq_printf(s, "bytes: %ld\n", cnt);
6642
58e8eedf 6643 if (trace_clocks[tr->clock_id].in_ns) {
11043d8b 6644 /* local or global for trace_clock */
12883efb 6645 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
6646 usec_rem = do_div(t, USEC_PER_SEC);
6647 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6648 t, usec_rem);
6649
12883efb 6650 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
6651 usec_rem = do_div(t, USEC_PER_SEC);
6652 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6653 } else {
6654 /* counter or tsc mode for trace_clock */
6655 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 6656 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 6657
11043d8b 6658 trace_seq_printf(s, "now ts: %llu\n",
12883efb 6659 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 6660 }
c64e148a 6661
12883efb 6662 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
6663 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6664
12883efb 6665 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
6666 trace_seq_printf(s, "read events: %ld\n", cnt);
6667
5ac48378
SRRH
6668 count = simple_read_from_buffer(ubuf, count, ppos,
6669 s->buffer, trace_seq_used(s));
c8d77183
SR
6670
6671 kfree(s);
6672
6673 return count;
6674}
6675
6676static const struct file_operations tracing_stats_fops = {
4d3435b8 6677 .open = tracing_open_generic_tr,
c8d77183 6678 .read = tracing_stats_read,
b444786f 6679 .llseek = generic_file_llseek,
4d3435b8 6680 .release = tracing_release_generic_tr,
c8d77183
SR
6681};
6682
bc0c38d1
SR
6683#ifdef CONFIG_DYNAMIC_FTRACE
6684
b807c3d0
SR
6685int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6686{
6687 return 0;
6688}
6689
bc0c38d1 6690static ssize_t
b807c3d0 6691tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
6692 size_t cnt, loff_t *ppos)
6693{
a26a2a27
SR
6694 static char ftrace_dyn_info_buffer[1024];
6695 static DEFINE_MUTEX(dyn_info_mutex);
bc0c38d1 6696 unsigned long *p = filp->private_data;
b807c3d0 6697 char *buf = ftrace_dyn_info_buffer;
a26a2a27 6698 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
bc0c38d1
SR
6699 int r;
6700
b807c3d0
SR
6701 mutex_lock(&dyn_info_mutex);
6702 r = sprintf(buf, "%ld ", *p);
4bf39a94 6703
a26a2a27 6704 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
b807c3d0
SR
6705 buf[r++] = '\n';
6706
6707 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6708
6709 mutex_unlock(&dyn_info_mutex);
6710
6711 return r;
bc0c38d1
SR
6712}
6713
5e2336a0 6714static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 6715 .open = tracing_open_generic,
b807c3d0 6716 .read = tracing_read_dyn_info,
b444786f 6717 .llseek = generic_file_llseek,
bc0c38d1 6718};
77fd5c15 6719#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 6720
77fd5c15
SRRH
6721#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6722static void
6723ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6724{
6725 tracing_snapshot();
6726}
bc0c38d1 6727
77fd5c15
SRRH
6728static void
6729ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
bc0c38d1 6730{
77fd5c15
SRRH
6731 unsigned long *count = (long *)data;
6732
6733 if (!*count)
6734 return;
bc0c38d1 6735
77fd5c15
SRRH
6736 if (*count != -1)
6737 (*count)--;
6738
6739 tracing_snapshot();
6740}
6741
6742static int
6743ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6744 struct ftrace_probe_ops *ops, void *data)
6745{
6746 long count = (long)data;
6747
6748 seq_printf(m, "%ps:", (void *)ip);
6749
fa6f0cc7 6750 seq_puts(m, "snapshot");
77fd5c15
SRRH
6751
6752 if (count == -1)
fa6f0cc7 6753 seq_puts(m, ":unlimited\n");
77fd5c15
SRRH
6754 else
6755 seq_printf(m, ":count=%ld\n", count);
6756
6757 return 0;
6758}
6759
6760static struct ftrace_probe_ops snapshot_probe_ops = {
6761 .func = ftrace_snapshot,
6762 .print = ftrace_snapshot_print,
6763};
6764
6765static struct ftrace_probe_ops snapshot_count_probe_ops = {
6766 .func = ftrace_count_snapshot,
6767 .print = ftrace_snapshot_print,
6768};
6769
6770static int
6771ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6772 char *glob, char *cmd, char *param, int enable)
6773{
6774 struct ftrace_probe_ops *ops;
6775 void *count = (void *)-1;
6776 char *number;
6777 int ret;
6778
6779 /* hash funcs only work with set_ftrace_filter */
6780 if (!enable)
6781 return -EINVAL;
6782
6783 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6784
6785 if (glob[0] == '!') {
6786 unregister_ftrace_function_probe_func(glob+1, ops);
6787 return 0;
6788 }
6789
6790 if (!param)
6791 goto out_reg;
6792
6793 number = strsep(&param, ":");
6794
6795 if (!strlen(number))
6796 goto out_reg;
6797
6798 /*
6799 * We use the callback data field (which is a pointer)
6800 * as our counter.
6801 */
6802 ret = kstrtoul(number, 0, (unsigned long *)&count);
6803 if (ret)
6804 return ret;
6805
6806 out_reg:
6807 ret = register_ftrace_function_probe(glob, ops, count);
6808
6809 if (ret >= 0)
6810 alloc_snapshot(&global_trace);
6811
6812 return ret < 0 ? ret : 0;
6813}
6814
6815static struct ftrace_func_command ftrace_snapshot_cmd = {
6816 .name = "snapshot",
6817 .func = ftrace_trace_snapshot_callback,
6818};
6819
38de93ab 6820static __init int register_snapshot_cmd(void)
77fd5c15
SRRH
6821{
6822 return register_ftrace_command(&ftrace_snapshot_cmd);
6823}
6824#else
38de93ab 6825static inline __init int register_snapshot_cmd(void) { return 0; }
77fd5c15 6826#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 6827
7eeafbca 6828static struct dentry *tracing_get_dentry(struct trace_array *tr)
bc0c38d1 6829{
8434dc93
SRRH
6830 if (WARN_ON(!tr->dir))
6831 return ERR_PTR(-ENODEV);
6832
6833 /* Top directory uses NULL as the parent */
6834 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6835 return NULL;
6836
6837 /* All sub buffers have a descriptor */
2b6080f2 6838 return tr->dir;
bc0c38d1
SR
6839}
6840
2b6080f2 6841static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 6842{
b04cc6b1
FW
6843 struct dentry *d_tracer;
6844
2b6080f2
SR
6845 if (tr->percpu_dir)
6846 return tr->percpu_dir;
b04cc6b1 6847
7eeafbca 6848 d_tracer = tracing_get_dentry(tr);
14a5ae40 6849 if (IS_ERR(d_tracer))
b04cc6b1
FW
6850 return NULL;
6851
8434dc93 6852 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
b04cc6b1 6853
2b6080f2 6854 WARN_ONCE(!tr->percpu_dir,
8434dc93 6855 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 6856
2b6080f2 6857 return tr->percpu_dir;
b04cc6b1
FW
6858}
6859
649e9c70
ON
6860static struct dentry *
6861trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6862 void *data, long cpu, const struct file_operations *fops)
6863{
6864 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6865
6866 if (ret) /* See tracing_get_cpu() */
7682c918 6867 d_inode(ret)->i_cdev = (void *)(cpu + 1);
649e9c70
ON
6868 return ret;
6869}
6870
2b6080f2 6871static void
8434dc93 6872tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 6873{
2b6080f2 6874 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 6875 struct dentry *d_cpu;
dd49a38c 6876 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 6877
0a3d7ce7
NK
6878 if (!d_percpu)
6879 return;
6880
dd49a38c 6881 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8434dc93 6882 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8656e7a2 6883 if (!d_cpu) {
a395d6a7 6884 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8656e7a2
FW
6885 return;
6886 }
b04cc6b1 6887
8656e7a2 6888 /* per cpu trace_pipe */
649e9c70 6889 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
15544209 6890 tr, cpu, &tracing_pipe_fops);
b04cc6b1
FW
6891
6892 /* per cpu trace */
649e9c70 6893 trace_create_cpu_file("trace", 0644, d_cpu,
6484c71c 6894 tr, cpu, &tracing_fops);
7f96f93f 6895
649e9c70 6896 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
46ef2be0 6897 tr, cpu, &tracing_buffers_fops);
7f96f93f 6898
649e9c70 6899 trace_create_cpu_file("stats", 0444, d_cpu,
4d3435b8 6900 tr, cpu, &tracing_stats_fops);
438ced17 6901
649e9c70 6902 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
0bc392ee 6903 tr, cpu, &tracing_entries_fops);
f1affcaa
SRRH
6904
6905#ifdef CONFIG_TRACER_SNAPSHOT
649e9c70 6906 trace_create_cpu_file("snapshot", 0644, d_cpu,
6484c71c 6907 tr, cpu, &snapshot_fops);
6de58e62 6908
649e9c70 6909 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
46ef2be0 6910 tr, cpu, &snapshot_raw_fops);
f1affcaa 6911#endif
b04cc6b1
FW
6912}
6913
60a11774
SR
6914#ifdef CONFIG_FTRACE_SELFTEST
6915/* Let selftest have access to static functions in this file */
6916#include "trace_selftest.c"
6917#endif
6918
577b785f
SR
6919static ssize_t
6920trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6921 loff_t *ppos)
6922{
6923 struct trace_option_dentry *topt = filp->private_data;
6924 char *buf;
6925
6926 if (topt->flags->val & topt->opt->bit)
6927 buf = "1\n";
6928 else
6929 buf = "0\n";
6930
6931 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6932}
6933
6934static ssize_t
6935trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6936 loff_t *ppos)
6937{
6938 struct trace_option_dentry *topt = filp->private_data;
6939 unsigned long val;
577b785f
SR
6940 int ret;
6941
22fe9b54
PH
6942 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6943 if (ret)
577b785f
SR
6944 return ret;
6945
8d18eaaf
LZ
6946 if (val != 0 && val != 1)
6947 return -EINVAL;
577b785f 6948
8d18eaaf 6949 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 6950 mutex_lock(&trace_types_lock);
8c1a49ae 6951 ret = __set_tracer_option(topt->tr, topt->flags,
c757bea9 6952 topt->opt, !val);
577b785f
SR
6953 mutex_unlock(&trace_types_lock);
6954 if (ret)
6955 return ret;
577b785f
SR
6956 }
6957
6958 *ppos += cnt;
6959
6960 return cnt;
6961}
6962
6963
6964static const struct file_operations trace_options_fops = {
6965 .open = tracing_open_generic,
6966 .read = trace_options_read,
6967 .write = trace_options_write,
b444786f 6968 .llseek = generic_file_llseek,
577b785f
SR
6969};
6970
9a38a885
SRRH
6971/*
6972 * In order to pass in both the trace_array descriptor as well as the index
6973 * to the flag that the trace option file represents, the trace_array
6974 * has a character array of trace_flags_index[], which holds the index
6975 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6976 * The address of this character array is passed to the flag option file
6977 * read/write callbacks.
6978 *
6979 * In order to extract both the index and the trace_array descriptor,
6980 * get_tr_index() uses the following algorithm.
6981 *
6982 * idx = *ptr;
6983 *
6984 * As the pointer itself contains the address of the index (remember
6985 * index[1] == 1).
6986 *
6987 * Then to get the trace_array descriptor, by subtracting that index
6988 * from the ptr, we get to the start of the index itself.
6989 *
6990 * ptr - idx == &index[0]
6991 *
6992 * Then a simple container_of() from that pointer gets us to the
6993 * trace_array descriptor.
6994 */
6995static void get_tr_index(void *data, struct trace_array **ptr,
6996 unsigned int *pindex)
6997{
6998 *pindex = *(unsigned char *)data;
6999
7000 *ptr = container_of(data - *pindex, struct trace_array,
7001 trace_flags_index);
7002}
7003
a8259075
SR
7004static ssize_t
7005trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7006 loff_t *ppos)
7007{
9a38a885
SRRH
7008 void *tr_index = filp->private_data;
7009 struct trace_array *tr;
7010 unsigned int index;
a8259075
SR
7011 char *buf;
7012
9a38a885
SRRH
7013 get_tr_index(tr_index, &tr, &index);
7014
7015 if (tr->trace_flags & (1 << index))
a8259075
SR
7016 buf = "1\n";
7017 else
7018 buf = "0\n";
7019
7020 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7021}
7022
7023static ssize_t
7024trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7025 loff_t *ppos)
7026{
9a38a885
SRRH
7027 void *tr_index = filp->private_data;
7028 struct trace_array *tr;
7029 unsigned int index;
a8259075
SR
7030 unsigned long val;
7031 int ret;
7032
9a38a885
SRRH
7033 get_tr_index(tr_index, &tr, &index);
7034
22fe9b54
PH
7035 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7036 if (ret)
a8259075
SR
7037 return ret;
7038
f2d84b65 7039 if (val != 0 && val != 1)
a8259075 7040 return -EINVAL;
69d34da2
SRRH
7041
7042 mutex_lock(&trace_types_lock);
2b6080f2 7043 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 7044 mutex_unlock(&trace_types_lock);
a8259075 7045
613f04a0
SRRH
7046 if (ret < 0)
7047 return ret;
7048
a8259075
SR
7049 *ppos += cnt;
7050
7051 return cnt;
7052}
7053
a8259075
SR
7054static const struct file_operations trace_options_core_fops = {
7055 .open = tracing_open_generic,
7056 .read = trace_options_core_read,
7057 .write = trace_options_core_write,
b444786f 7058 .llseek = generic_file_llseek,
a8259075
SR
7059};
7060
5452af66 7061struct dentry *trace_create_file(const char *name,
f4ae40a6 7062 umode_t mode,
5452af66
FW
7063 struct dentry *parent,
7064 void *data,
7065 const struct file_operations *fops)
7066{
7067 struct dentry *ret;
7068
8434dc93 7069 ret = tracefs_create_file(name, mode, parent, data, fops);
5452af66 7070 if (!ret)
a395d6a7 7071 pr_warn("Could not create tracefs '%s' entry\n", name);
5452af66
FW
7072
7073 return ret;
7074}
7075
7076
2b6080f2 7077static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
7078{
7079 struct dentry *d_tracer;
a8259075 7080
2b6080f2
SR
7081 if (tr->options)
7082 return tr->options;
a8259075 7083
7eeafbca 7084 d_tracer = tracing_get_dentry(tr);
14a5ae40 7085 if (IS_ERR(d_tracer))
a8259075
SR
7086 return NULL;
7087
8434dc93 7088 tr->options = tracefs_create_dir("options", d_tracer);
2b6080f2 7089 if (!tr->options) {
a395d6a7 7090 pr_warn("Could not create tracefs directory 'options'\n");
a8259075
SR
7091 return NULL;
7092 }
7093
2b6080f2 7094 return tr->options;
a8259075
SR
7095}
7096
577b785f 7097static void
2b6080f2
SR
7098create_trace_option_file(struct trace_array *tr,
7099 struct trace_option_dentry *topt,
577b785f
SR
7100 struct tracer_flags *flags,
7101 struct tracer_opt *opt)
7102{
7103 struct dentry *t_options;
577b785f 7104
2b6080f2 7105 t_options = trace_options_init_dentry(tr);
577b785f
SR
7106 if (!t_options)
7107 return;
7108
7109 topt->flags = flags;
7110 topt->opt = opt;
2b6080f2 7111 topt->tr = tr;
577b785f 7112
5452af66 7113 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
7114 &trace_options_fops);
7115
577b785f
SR
7116}
7117
37aea98b 7118static void
2b6080f2 7119create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
7120{
7121 struct trace_option_dentry *topts;
37aea98b 7122 struct trace_options *tr_topts;
577b785f
SR
7123 struct tracer_flags *flags;
7124 struct tracer_opt *opts;
7125 int cnt;
37aea98b 7126 int i;
577b785f
SR
7127
7128 if (!tracer)
37aea98b 7129 return;
577b785f
SR
7130
7131 flags = tracer->flags;
7132
7133 if (!flags || !flags->opts)
37aea98b
SRRH
7134 return;
7135
7136 /*
7137 * If this is an instance, only create flags for tracers
7138 * the instance may have.
7139 */
7140 if (!trace_ok_for_array(tracer, tr))
7141 return;
7142
7143 for (i = 0; i < tr->nr_topts; i++) {
d39cdd20
CH
7144 /* Make sure there's no duplicate flags. */
7145 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
37aea98b
SRRH
7146 return;
7147 }
577b785f
SR
7148
7149 opts = flags->opts;
7150
7151 for (cnt = 0; opts[cnt].name; cnt++)
7152 ;
7153
0cfe8245 7154 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f 7155 if (!topts)
37aea98b
SRRH
7156 return;
7157
7158 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
7159 GFP_KERNEL);
7160 if (!tr_topts) {
7161 kfree(topts);
7162 return;
7163 }
7164
7165 tr->topts = tr_topts;
7166 tr->topts[tr->nr_topts].tracer = tracer;
7167 tr->topts[tr->nr_topts].topts = topts;
7168 tr->nr_topts++;
577b785f 7169
41d9c0be 7170 for (cnt = 0; opts[cnt].name; cnt++) {
2b6080f2 7171 create_trace_option_file(tr, &topts[cnt], flags,
577b785f 7172 &opts[cnt]);
41d9c0be
SRRH
7173 WARN_ONCE(topts[cnt].entry == NULL,
7174 "Failed to create trace option: %s",
7175 opts[cnt].name);
7176 }
577b785f
SR
7177}
7178
a8259075 7179static struct dentry *
2b6080f2
SR
7180create_trace_option_core_file(struct trace_array *tr,
7181 const char *option, long index)
a8259075
SR
7182{
7183 struct dentry *t_options;
a8259075 7184
2b6080f2 7185 t_options = trace_options_init_dentry(tr);
a8259075
SR
7186 if (!t_options)
7187 return NULL;
7188
9a38a885
SRRH
7189 return trace_create_file(option, 0644, t_options,
7190 (void *)&tr->trace_flags_index[index],
7191 &trace_options_core_fops);
a8259075
SR
7192}
7193
16270145 7194static void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
7195{
7196 struct dentry *t_options;
16270145 7197 bool top_level = tr == &global_trace;
a8259075
SR
7198 int i;
7199
2b6080f2 7200 t_options = trace_options_init_dentry(tr);
a8259075
SR
7201 if (!t_options)
7202 return;
7203
16270145
SRRH
7204 for (i = 0; trace_options[i]; i++) {
7205 if (top_level ||
7206 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
7207 create_trace_option_core_file(tr, trace_options[i], i);
7208 }
a8259075
SR
7209}
7210
499e5470
SR
7211static ssize_t
7212rb_simple_read(struct file *filp, char __user *ubuf,
7213 size_t cnt, loff_t *ppos)
7214{
348f0fc2 7215 struct trace_array *tr = filp->private_data;
499e5470
SR
7216 char buf[64];
7217 int r;
7218
10246fa3 7219 r = tracer_tracing_is_on(tr);
499e5470
SR
7220 r = sprintf(buf, "%d\n", r);
7221
7222 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7223}
7224
7225static ssize_t
7226rb_simple_write(struct file *filp, const char __user *ubuf,
7227 size_t cnt, loff_t *ppos)
7228{
348f0fc2 7229 struct trace_array *tr = filp->private_data;
12883efb 7230 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
7231 unsigned long val;
7232 int ret;
7233
7234 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7235 if (ret)
7236 return ret;
7237
7238 if (buffer) {
2df8f8a6
SR
7239 mutex_lock(&trace_types_lock);
7240 if (val) {
10246fa3 7241 tracer_tracing_on(tr);
2b6080f2
SR
7242 if (tr->current_trace->start)
7243 tr->current_trace->start(tr);
2df8f8a6 7244 } else {
10246fa3 7245 tracer_tracing_off(tr);
2b6080f2
SR
7246 if (tr->current_trace->stop)
7247 tr->current_trace->stop(tr);
2df8f8a6
SR
7248 }
7249 mutex_unlock(&trace_types_lock);
499e5470
SR
7250 }
7251
7252 (*ppos)++;
7253
7254 return cnt;
7255}
7256
7257static const struct file_operations rb_simple_fops = {
7b85af63 7258 .open = tracing_open_generic_tr,
499e5470
SR
7259 .read = rb_simple_read,
7260 .write = rb_simple_write,
7b85af63 7261 .release = tracing_release_generic_tr,
499e5470
SR
7262 .llseek = default_llseek,
7263};
7264
277ba044
SR
7265struct dentry *trace_instance_dir;
7266
7267static void
8434dc93 7268init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
277ba044 7269
55034cd6
SRRH
7270static int
7271allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
7272{
7273 enum ring_buffer_flags rb_flags;
737223fb 7274
983f938a 7275 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
737223fb 7276
dced341b
SRRH
7277 buf->tr = tr;
7278
55034cd6
SRRH
7279 buf->buffer = ring_buffer_alloc(size, rb_flags);
7280 if (!buf->buffer)
7281 return -ENOMEM;
737223fb 7282
55034cd6
SRRH
7283 buf->data = alloc_percpu(struct trace_array_cpu);
7284 if (!buf->data) {
7285 ring_buffer_free(buf->buffer);
7286 return -ENOMEM;
7287 }
737223fb 7288
737223fb
SRRH
7289 /* Allocate the first page for all buffers */
7290 set_buffer_entries(&tr->trace_buffer,
7291 ring_buffer_size(tr->trace_buffer.buffer, 0));
7292
55034cd6
SRRH
7293 return 0;
7294}
737223fb 7295
55034cd6
SRRH
7296static int allocate_trace_buffers(struct trace_array *tr, int size)
7297{
7298 int ret;
737223fb 7299
55034cd6
SRRH
7300 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7301 if (ret)
7302 return ret;
737223fb 7303
55034cd6
SRRH
7304#ifdef CONFIG_TRACER_MAX_TRACE
7305 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7306 allocate_snapshot ? size : 1);
7307 if (WARN_ON(ret)) {
737223fb 7308 ring_buffer_free(tr->trace_buffer.buffer);
55034cd6
SRRH
7309 free_percpu(tr->trace_buffer.data);
7310 return -ENOMEM;
7311 }
7312 tr->allocated_snapshot = allocate_snapshot;
737223fb 7313
55034cd6
SRRH
7314 /*
7315 * Only the top level trace array gets its snapshot allocated
7316 * from the kernel command line.
7317 */
7318 allocate_snapshot = false;
737223fb 7319#endif
55034cd6 7320 return 0;
737223fb
SRRH
7321}
7322
f0b70cc4
SRRH
7323static void free_trace_buffer(struct trace_buffer *buf)
7324{
7325 if (buf->buffer) {
7326 ring_buffer_free(buf->buffer);
7327 buf->buffer = NULL;
7328 free_percpu(buf->data);
7329 buf->data = NULL;
7330 }
7331}
7332
23aaa3c1
SRRH
7333static void free_trace_buffers(struct trace_array *tr)
7334{
7335 if (!tr)
7336 return;
7337
f0b70cc4 7338 free_trace_buffer(&tr->trace_buffer);
23aaa3c1
SRRH
7339
7340#ifdef CONFIG_TRACER_MAX_TRACE
f0b70cc4 7341 free_trace_buffer(&tr->max_buffer);
23aaa3c1
SRRH
7342#endif
7343}
7344
9a38a885
SRRH
7345static void init_trace_flags_index(struct trace_array *tr)
7346{
7347 int i;
7348
7349 /* Used by the trace options files */
7350 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7351 tr->trace_flags_index[i] = i;
7352}
7353
37aea98b
SRRH
7354static void __update_tracer_options(struct trace_array *tr)
7355{
7356 struct tracer *t;
7357
7358 for (t = trace_types; t; t = t->next)
7359 add_tracer_options(tr, t);
7360}
7361
7362static void update_tracer_options(struct trace_array *tr)
7363{
7364 mutex_lock(&trace_types_lock);
7365 __update_tracer_options(tr);
7366 mutex_unlock(&trace_types_lock);
7367}
7368
eae47358 7369static int instance_mkdir(const char *name)
737223fb 7370{
277ba044
SR
7371 struct trace_array *tr;
7372 int ret;
277ba044
SR
7373
7374 mutex_lock(&trace_types_lock);
7375
7376 ret = -EEXIST;
7377 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7378 if (tr->name && strcmp(tr->name, name) == 0)
7379 goto out_unlock;
7380 }
7381
7382 ret = -ENOMEM;
7383 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7384 if (!tr)
7385 goto out_unlock;
7386
7387 tr->name = kstrdup(name, GFP_KERNEL);
7388 if (!tr->name)
7389 goto out_free_tr;
7390
ccfe9e42
AL
7391 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7392 goto out_free_tr;
7393
20550622 7394 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
983f938a 7395
ccfe9e42
AL
7396 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7397
277ba044
SR
7398 raw_spin_lock_init(&tr->start_lock);
7399
0b9b12c1
SRRH
7400 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7401
277ba044
SR
7402 tr->current_trace = &nop_trace;
7403
7404 INIT_LIST_HEAD(&tr->systems);
7405 INIT_LIST_HEAD(&tr->events);
7406
737223fb 7407 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
7408 goto out_free_tr;
7409
8434dc93 7410 tr->dir = tracefs_create_dir(name, trace_instance_dir);
277ba044
SR
7411 if (!tr->dir)
7412 goto out_free_tr;
7413
7414 ret = event_trace_add_tracer(tr->dir, tr);
609e85a7 7415 if (ret) {
8434dc93 7416 tracefs_remove_recursive(tr->dir);
277ba044 7417 goto out_free_tr;
609e85a7 7418 }
277ba044 7419
8434dc93 7420 init_tracer_tracefs(tr, tr->dir);
9a38a885 7421 init_trace_flags_index(tr);
37aea98b 7422 __update_tracer_options(tr);
277ba044
SR
7423
7424 list_add(&tr->list, &ftrace_trace_arrays);
7425
7426 mutex_unlock(&trace_types_lock);
7427
7428 return 0;
7429
7430 out_free_tr:
23aaa3c1 7431 free_trace_buffers(tr);
ccfe9e42 7432 free_cpumask_var(tr->tracing_cpumask);
277ba044
SR
7433 kfree(tr->name);
7434 kfree(tr);
7435
7436 out_unlock:
7437 mutex_unlock(&trace_types_lock);
7438
7439 return ret;
7440
7441}
7442
eae47358 7443static int instance_rmdir(const char *name)
0c8916c3
SR
7444{
7445 struct trace_array *tr;
7446 int found = 0;
7447 int ret;
37aea98b 7448 int i;
0c8916c3
SR
7449
7450 mutex_lock(&trace_types_lock);
7451
7452 ret = -ENODEV;
7453 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7454 if (tr->name && strcmp(tr->name, name) == 0) {
7455 found = 1;
7456 break;
7457 }
7458 }
7459 if (!found)
7460 goto out_unlock;
7461
a695cb58 7462 ret = -EBUSY;
cf6ab6d9 7463 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
a695cb58
SRRH
7464 goto out_unlock;
7465
0c8916c3
SR
7466 list_del(&tr->list);
7467
20550622
SRRH
7468 /* Disable all the flags that were enabled coming in */
7469 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7470 if ((1 << i) & ZEROED_TRACE_FLAGS)
7471 set_tracer_flag(tr, 1 << i, 0);
7472 }
7473
6b450d25 7474 tracing_set_nop(tr);
0c8916c3 7475 event_trace_del_tracer(tr);
591dffda 7476 ftrace_destroy_function_files(tr);
681a4a2f 7477 tracefs_remove_recursive(tr->dir);
a9fcaaac 7478 free_trace_buffers(tr);
0c8916c3 7479
37aea98b
SRRH
7480 for (i = 0; i < tr->nr_topts; i++) {
7481 kfree(tr->topts[i].topts);
7482 }
7483 kfree(tr->topts);
7484
0c8916c3
SR
7485 kfree(tr->name);
7486 kfree(tr);
7487
7488 ret = 0;
7489
7490 out_unlock:
7491 mutex_unlock(&trace_types_lock);
7492
7493 return ret;
7494}
7495
277ba044
SR
7496static __init void create_trace_instances(struct dentry *d_tracer)
7497{
eae47358
SRRH
7498 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7499 instance_mkdir,
7500 instance_rmdir);
277ba044
SR
7501 if (WARN_ON(!trace_instance_dir))
7502 return;
277ba044
SR
7503}
7504
2b6080f2 7505static void
8434dc93 7506init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
2b6080f2 7507{
121aaee7 7508 int cpu;
2b6080f2 7509
607e2ea1
SRRH
7510 trace_create_file("available_tracers", 0444, d_tracer,
7511 tr, &show_traces_fops);
7512
7513 trace_create_file("current_tracer", 0644, d_tracer,
7514 tr, &set_tracer_fops);
7515
ccfe9e42
AL
7516 trace_create_file("tracing_cpumask", 0644, d_tracer,
7517 tr, &tracing_cpumask_fops);
7518
2b6080f2
SR
7519 trace_create_file("trace_options", 0644, d_tracer,
7520 tr, &tracing_iter_fops);
7521
7522 trace_create_file("trace", 0644, d_tracer,
6484c71c 7523 tr, &tracing_fops);
2b6080f2
SR
7524
7525 trace_create_file("trace_pipe", 0444, d_tracer,
15544209 7526 tr, &tracing_pipe_fops);
2b6080f2
SR
7527
7528 trace_create_file("buffer_size_kb", 0644, d_tracer,
0bc392ee 7529 tr, &tracing_entries_fops);
2b6080f2
SR
7530
7531 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7532 tr, &tracing_total_entries_fops);
7533
238ae93d 7534 trace_create_file("free_buffer", 0200, d_tracer,
2b6080f2
SR
7535 tr, &tracing_free_buffer_fops);
7536
7537 trace_create_file("trace_marker", 0220, d_tracer,
7538 tr, &tracing_mark_fops);
7539
fa32e855
SR
7540 trace_create_file("trace_marker_raw", 0220, d_tracer,
7541 tr, &tracing_mark_raw_fops);
7542
2b6080f2
SR
7543 trace_create_file("trace_clock", 0644, d_tracer, tr,
7544 &trace_clock_fops);
7545
7546 trace_create_file("tracing_on", 0644, d_tracer,
6484c71c 7547 tr, &rb_simple_fops);
ce9bae55 7548
16270145
SRRH
7549 create_trace_options_dir(tr);
7550
f971cc9a 7551#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6d9b3fa5
SRRH
7552 trace_create_file("tracing_max_latency", 0644, d_tracer,
7553 &tr->max_latency, &tracing_max_lat_fops);
7554#endif
7555
591dffda
SRRH
7556 if (ftrace_create_function_files(tr, d_tracer))
7557 WARN(1, "Could not allocate function filter files");
7558
ce9bae55
SRRH
7559#ifdef CONFIG_TRACER_SNAPSHOT
7560 trace_create_file("snapshot", 0644, d_tracer,
6484c71c 7561 tr, &snapshot_fops);
ce9bae55 7562#endif
121aaee7
SRRH
7563
7564 for_each_tracing_cpu(cpu)
8434dc93 7565 tracing_init_tracefs_percpu(tr, cpu);
121aaee7 7566
345ddcc8 7567 ftrace_init_tracefs(tr, d_tracer);
2b6080f2
SR
7568}
7569
f76180bc
SRRH
7570static struct vfsmount *trace_automount(void *ingore)
7571{
7572 struct vfsmount *mnt;
7573 struct file_system_type *type;
7574
7575 /*
7576 * To maintain backward compatibility for tools that mount
7577 * debugfs to get to the tracing facility, tracefs is automatically
7578 * mounted to the debugfs/tracing directory.
7579 */
7580 type = get_fs_type("tracefs");
7581 if (!type)
7582 return NULL;
7583 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
7584 put_filesystem(type);
7585 if (IS_ERR(mnt))
7586 return NULL;
7587 mntget(mnt);
7588
7589 return mnt;
7590}
7591
7eeafbca
SRRH
7592/**
7593 * tracing_init_dentry - initialize top level trace array
7594 *
7595 * This is called when creating files or directories in the tracing
7596 * directory. It is called via fs_initcall() by any of the boot up code
7597 * and expects to return the dentry of the top level tracing directory.
7598 */
7599struct dentry *tracing_init_dentry(void)
7600{
7601 struct trace_array *tr = &global_trace;
7602
f76180bc 7603 /* The top level trace array uses NULL as parent */
7eeafbca 7604 if (tr->dir)
f76180bc 7605 return NULL;
7eeafbca 7606
8b129199
JW
7607 if (WARN_ON(!tracefs_initialized()) ||
7608 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7609 WARN_ON(!debugfs_initialized())))
7eeafbca
SRRH
7610 return ERR_PTR(-ENODEV);
7611
f76180bc
SRRH
7612 /*
7613 * As there may still be users that expect the tracing
7614 * files to exist in debugfs/tracing, we must automount
7615 * the tracefs file system there, so older tools still
7616 * work with the newer kerenl.
7617 */
7618 tr->dir = debugfs_create_automount("tracing", NULL,
7619 trace_automount, NULL);
7eeafbca
SRRH
7620 if (!tr->dir) {
7621 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7622 return ERR_PTR(-ENOMEM);
7623 }
7624
8434dc93 7625 return NULL;
7eeafbca
SRRH
7626}
7627
0c564a53
SRRH
7628extern struct trace_enum_map *__start_ftrace_enum_maps[];
7629extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7630
7631static void __init trace_enum_init(void)
7632{
3673b8e4
SRRH
7633 int len;
7634
7635 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
9828413d 7636 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
3673b8e4
SRRH
7637}
7638
7639#ifdef CONFIG_MODULES
7640static void trace_module_add_enums(struct module *mod)
7641{
7642 if (!mod->num_trace_enums)
7643 return;
7644
7645 /*
7646 * Modules with bad taint do not have events created, do
7647 * not bother with enums either.
7648 */
7649 if (trace_module_has_bad_taint(mod))
7650 return;
7651
9828413d 7652 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
3673b8e4
SRRH
7653}
7654
9828413d
SRRH
7655#ifdef CONFIG_TRACE_ENUM_MAP_FILE
7656static void trace_module_remove_enums(struct module *mod)
7657{
7658 union trace_enum_map_item *map;
7659 union trace_enum_map_item **last = &trace_enum_maps;
7660
7661 if (!mod->num_trace_enums)
7662 return;
7663
7664 mutex_lock(&trace_enum_mutex);
7665
7666 map = trace_enum_maps;
7667
7668 while (map) {
7669 if (map->head.mod == mod)
7670 break;
7671 map = trace_enum_jmp_to_tail(map);
7672 last = &map->tail.next;
7673 map = map->tail.next;
7674 }
7675 if (!map)
7676 goto out;
7677
7678 *last = trace_enum_jmp_to_tail(map)->tail.next;
7679 kfree(map);
7680 out:
7681 mutex_unlock(&trace_enum_mutex);
7682}
7683#else
7684static inline void trace_module_remove_enums(struct module *mod) { }
7685#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7686
3673b8e4
SRRH
7687static int trace_module_notify(struct notifier_block *self,
7688 unsigned long val, void *data)
7689{
7690 struct module *mod = data;
7691
7692 switch (val) {
7693 case MODULE_STATE_COMING:
7694 trace_module_add_enums(mod);
7695 break;
9828413d
SRRH
7696 case MODULE_STATE_GOING:
7697 trace_module_remove_enums(mod);
7698 break;
3673b8e4
SRRH
7699 }
7700
7701 return 0;
0c564a53
SRRH
7702}
7703
3673b8e4
SRRH
7704static struct notifier_block trace_module_nb = {
7705 .notifier_call = trace_module_notify,
7706 .priority = 0,
7707};
9828413d 7708#endif /* CONFIG_MODULES */
3673b8e4 7709
8434dc93 7710static __init int tracer_init_tracefs(void)
bc0c38d1
SR
7711{
7712 struct dentry *d_tracer;
bc0c38d1 7713
7e53bd42
LJ
7714 trace_access_lock_init();
7715
bc0c38d1 7716 d_tracer = tracing_init_dentry();
14a5ae40 7717 if (IS_ERR(d_tracer))
ed6f1c99 7718 return 0;
bc0c38d1 7719
8434dc93 7720 init_tracer_tracefs(&global_trace, d_tracer);
501c2375 7721 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
bc0c38d1 7722
5452af66 7723 trace_create_file("tracing_thresh", 0644, d_tracer,
6508fa76 7724 &global_trace, &tracing_thresh_fops);
a8259075 7725
339ae5d3 7726 trace_create_file("README", 0444, d_tracer,
5452af66
FW
7727 NULL, &tracing_readme_fops);
7728
69abe6a5
AP
7729 trace_create_file("saved_cmdlines", 0444, d_tracer,
7730 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 7731
939c7a4f
YY
7732 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7733 NULL, &tracing_saved_cmdlines_size_fops);
7734
0c564a53
SRRH
7735 trace_enum_init();
7736
9828413d
SRRH
7737 trace_create_enum_file(d_tracer);
7738
3673b8e4
SRRH
7739#ifdef CONFIG_MODULES
7740 register_module_notifier(&trace_module_nb);
7741#endif
7742
bc0c38d1 7743#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
7744 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7745 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 7746#endif
b04cc6b1 7747
277ba044 7748 create_trace_instances(d_tracer);
5452af66 7749
37aea98b 7750 update_tracer_options(&global_trace);
09d23a1d 7751
b5ad384e 7752 return 0;
bc0c38d1
SR
7753}
7754
3f5a54e3
SR
7755static int trace_panic_handler(struct notifier_block *this,
7756 unsigned long event, void *unused)
7757{
944ac425 7758 if (ftrace_dump_on_oops)
cecbca96 7759 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
7760 return NOTIFY_OK;
7761}
7762
7763static struct notifier_block trace_panic_notifier = {
7764 .notifier_call = trace_panic_handler,
7765 .next = NULL,
7766 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7767};
7768
7769static int trace_die_handler(struct notifier_block *self,
7770 unsigned long val,
7771 void *data)
7772{
7773 switch (val) {
7774 case DIE_OOPS:
944ac425 7775 if (ftrace_dump_on_oops)
cecbca96 7776 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
7777 break;
7778 default:
7779 break;
7780 }
7781 return NOTIFY_OK;
7782}
7783
7784static struct notifier_block trace_die_notifier = {
7785 .notifier_call = trace_die_handler,
7786 .priority = 200
7787};
7788
7789/*
7790 * printk is set to max of 1024, we really don't need it that big.
7791 * Nothing should be printing 1000 characters anyway.
7792 */
7793#define TRACE_MAX_PRINT 1000
7794
7795/*
7796 * Define here KERN_TRACE so that we have one place to modify
7797 * it if we decide to change what log level the ftrace dump
7798 * should be at.
7799 */
428aee14 7800#define KERN_TRACE KERN_EMERG
3f5a54e3 7801
955b61e5 7802void
3f5a54e3
SR
7803trace_printk_seq(struct trace_seq *s)
7804{
7805 /* Probably should print a warning here. */
3a161d99
SRRH
7806 if (s->seq.len >= TRACE_MAX_PRINT)
7807 s->seq.len = TRACE_MAX_PRINT;
3f5a54e3 7808
820b75f6
SRRH
7809 /*
7810 * More paranoid code. Although the buffer size is set to
7811 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7812 * an extra layer of protection.
7813 */
7814 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7815 s->seq.len = s->seq.size - 1;
3f5a54e3
SR
7816
7817 /* should be zero ended, but we are paranoid. */
3a161d99 7818 s->buffer[s->seq.len] = 0;
3f5a54e3
SR
7819
7820 printk(KERN_TRACE "%s", s->buffer);
7821
f9520750 7822 trace_seq_init(s);
3f5a54e3
SR
7823}
7824
955b61e5
JW
7825void trace_init_global_iter(struct trace_iterator *iter)
7826{
7827 iter->tr = &global_trace;
2b6080f2 7828 iter->trace = iter->tr->current_trace;
ae3b5093 7829 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 7830 iter->trace_buffer = &global_trace.trace_buffer;
b2f974d6
CS
7831
7832 if (iter->trace && iter->trace->open)
7833 iter->trace->open(iter);
7834
7835 /* Annotate start of buffers if we had overruns */
7836 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7837 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7838
7839 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7840 if (trace_clocks[iter->tr->clock_id].in_ns)
7841 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
955b61e5
JW
7842}
7843
7fe70b57 7844void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 7845{
3f5a54e3
SR
7846 /* use static because iter can be a bit big for the stack */
7847 static struct trace_iterator iter;
7fe70b57 7848 static atomic_t dump_running;
983f938a 7849 struct trace_array *tr = &global_trace;
cf586b61 7850 unsigned int old_userobj;
d769041f
SR
7851 unsigned long flags;
7852 int cnt = 0, cpu;
3f5a54e3 7853
7fe70b57
SRRH
7854 /* Only allow one dump user at a time. */
7855 if (atomic_inc_return(&dump_running) != 1) {
7856 atomic_dec(&dump_running);
7857 return;
7858 }
3f5a54e3 7859
7fe70b57
SRRH
7860 /*
7861 * Always turn off tracing when we dump.
7862 * We don't need to show trace output of what happens
7863 * between multiple crashes.
7864 *
7865 * If the user does a sysrq-z, then they can re-enable
7866 * tracing with echo 1 > tracing_on.
7867 */
0ee6b6cf 7868 tracing_off();
cf586b61 7869
7fe70b57 7870 local_irq_save(flags);
3f5a54e3 7871
38dbe0b1 7872 /* Simulate the iterator */
955b61e5
JW
7873 trace_init_global_iter(&iter);
7874
d769041f 7875 for_each_tracing_cpu(cpu) {
5e2d5ef8 7876 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
d769041f
SR
7877 }
7878
983f938a 7879 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
cf586b61 7880
b54d3de9 7881 /* don't look at user memory in panic mode */
983f938a 7882 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
b54d3de9 7883
cecbca96
FW
7884 switch (oops_dump_mode) {
7885 case DUMP_ALL:
ae3b5093 7886 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
7887 break;
7888 case DUMP_ORIG:
7889 iter.cpu_file = raw_smp_processor_id();
7890 break;
7891 case DUMP_NONE:
7892 goto out_enable;
7893 default:
7894 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 7895 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
7896 }
7897
7898 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 7899
7fe70b57
SRRH
7900 /* Did function tracer already get disabled? */
7901 if (ftrace_is_dead()) {
7902 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7903 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7904 }
7905
3f5a54e3
SR
7906 /*
7907 * We need to stop all tracing on all CPUS to read the
7908 * the next buffer. This is a bit expensive, but is
7909 * not done often. We fill all what we can read,
7910 * and then release the locks again.
7911 */
7912
3f5a54e3
SR
7913 while (!trace_empty(&iter)) {
7914
7915 if (!cnt)
7916 printk(KERN_TRACE "---------------------------------\n");
7917
7918 cnt++;
7919
7920 /* reset all but tr, trace, and overruns */
7921 memset(&iter.seq, 0,
7922 sizeof(struct trace_iterator) -
7923 offsetof(struct trace_iterator, seq));
7924 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7925 iter.pos = -1;
7926
955b61e5 7927 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
7928 int ret;
7929
7930 ret = print_trace_line(&iter);
7931 if (ret != TRACE_TYPE_NO_CONSUME)
7932 trace_consume(&iter);
3f5a54e3 7933 }
b892e5c8 7934 touch_nmi_watchdog();
3f5a54e3
SR
7935
7936 trace_printk_seq(&iter.seq);
7937 }
7938
7939 if (!cnt)
7940 printk(KERN_TRACE " (ftrace buffer empty)\n");
7941 else
7942 printk(KERN_TRACE "---------------------------------\n");
7943
cecbca96 7944 out_enable:
983f938a 7945 tr->trace_flags |= old_userobj;
cf586b61 7946
7fe70b57
SRRH
7947 for_each_tracing_cpu(cpu) {
7948 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 7949 }
7fe70b57 7950 atomic_dec(&dump_running);
cd891ae0 7951 local_irq_restore(flags);
3f5a54e3 7952}
a8eecf22 7953EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 7954
3928a8a2 7955__init static int tracer_alloc_buffers(void)
bc0c38d1 7956{
73c5162a 7957 int ring_buf_size;
9e01c1b7 7958 int ret = -ENOMEM;
4c11d7ae 7959
b5e87c05
SRRH
7960 /*
7961 * Make sure we don't accidently add more trace options
7962 * than we have bits for.
7963 */
9a38a885 7964 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
b5e87c05 7965
9e01c1b7
RR
7966 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7967 goto out;
7968
ccfe9e42 7969 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9e01c1b7 7970 goto out_free_buffer_mask;
4c11d7ae 7971
07d777fe
SR
7972 /* Only allocate trace_printk buffers if a trace_printk exists */
7973 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 7974 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
7975 trace_printk_init_buffers();
7976
73c5162a
SR
7977 /* To save memory, keep the ring buffer size to its minimum */
7978 if (ring_buffer_expanded)
7979 ring_buf_size = trace_buf_size;
7980 else
7981 ring_buf_size = 1;
7982
9e01c1b7 7983 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
ccfe9e42 7984 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9e01c1b7 7985
2b6080f2
SR
7986 raw_spin_lock_init(&global_trace.start_lock);
7987
2c4a33ab
SRRH
7988 /* Used for event triggers */
7989 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7990 if (!temp_buffer)
7991 goto out_free_cpumask;
7992
939c7a4f
YY
7993 if (trace_create_savedcmd() < 0)
7994 goto out_free_temp_buffer;
7995
9e01c1b7 7996 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 7997 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
7998 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7999 WARN_ON(1);
939c7a4f 8000 goto out_free_savedcmd;
4c11d7ae 8001 }
a7603ff4 8002
499e5470
SR
8003 if (global_trace.buffer_disabled)
8004 tracing_off();
4c11d7ae 8005
e1e232ca
SR
8006 if (trace_boot_clock) {
8007 ret = tracing_set_clock(&global_trace, trace_boot_clock);
8008 if (ret < 0)
a395d6a7
JP
8009 pr_warn("Trace clock %s not defined, going back to default\n",
8010 trace_boot_clock);
e1e232ca
SR
8011 }
8012
ca164318
SRRH
8013 /*
8014 * register_tracer() might reference current_trace, so it
8015 * needs to be set before we register anything. This is
8016 * just a bootstrap of current_trace anyway.
8017 */
2b6080f2
SR
8018 global_trace.current_trace = &nop_trace;
8019
0b9b12c1
SRRH
8020 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8021
4104d326
SRRH
8022 ftrace_init_global_array_ops(&global_trace);
8023
9a38a885
SRRH
8024 init_trace_flags_index(&global_trace);
8025
ca164318
SRRH
8026 register_tracer(&nop_trace);
8027
60a11774
SR
8028 /* All seems OK, enable tracing */
8029 tracing_disabled = 0;
3928a8a2 8030
3f5a54e3
SR
8031 atomic_notifier_chain_register(&panic_notifier_list,
8032 &trace_panic_notifier);
8033
8034 register_die_notifier(&trace_die_notifier);
2fc1dfbe 8035
ae63b31e
SR
8036 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
8037
8038 INIT_LIST_HEAD(&global_trace.systems);
8039 INIT_LIST_HEAD(&global_trace.events);
8040 list_add(&global_trace.list, &ftrace_trace_arrays);
8041
a4d1e688 8042 apply_trace_boot_options();
7bcfaf54 8043
77fd5c15
SRRH
8044 register_snapshot_cmd();
8045
2fc1dfbe 8046 return 0;
3f5a54e3 8047
939c7a4f
YY
8048out_free_savedcmd:
8049 free_saved_cmdlines_buffer(savedcmd);
2c4a33ab
SRRH
8050out_free_temp_buffer:
8051 ring_buffer_free(temp_buffer);
9e01c1b7 8052out_free_cpumask:
ccfe9e42 8053 free_cpumask_var(global_trace.tracing_cpumask);
9e01c1b7
RR
8054out_free_buffer_mask:
8055 free_cpumask_var(tracing_buffer_mask);
8056out:
8057 return ret;
bc0c38d1 8058}
b2821ae6 8059
5f893b26
SRRH
8060void __init trace_init(void)
8061{
0daa2302
SRRH
8062 if (tracepoint_printk) {
8063 tracepoint_print_iter =
8064 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8065 if (WARN_ON(!tracepoint_print_iter))
8066 tracepoint_printk = 0;
42391745
SRRH
8067 else
8068 static_key_enable(&tracepoint_printk_key.key);
0daa2302 8069 }
5f893b26 8070 tracer_alloc_buffers();
0c564a53 8071 trace_event_init();
5f893b26
SRRH
8072}
8073
b2821ae6
SR
8074__init static int clear_boot_tracer(void)
8075{
8076 /*
8077 * The default tracer at boot buffer is an init section.
8078 * This function is called in lateinit. If we did not
8079 * find the boot tracer, then clear it out, to prevent
8080 * later registration from accessing the buffer that is
8081 * about to be freed.
8082 */
8083 if (!default_bootup_tracer)
8084 return 0;
8085
8086 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
8087 default_bootup_tracer);
8088 default_bootup_tracer = NULL;
8089
8090 return 0;
8091}
8092
8434dc93 8093fs_initcall(tracer_init_tracefs);
b2821ae6 8094late_initcall(clear_boot_tracer);