]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
tracing/dynevent: Pass extra arguments to match operation
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bcea3f96 1// SPDX-License-Identifier: GPL-2.0
bc0c38d1
SR
2/*
3 * ring buffer based function tracer
4 *
2b6080f2 5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 *
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 *
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 13 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 14 */
2cadf913 15#include <linux/ring_buffer.h>
273b281f 16#include <generated/utsrelease.h>
2cadf913
SR
17#include <linux/stacktrace.h>
18#include <linux/writeback.h>
bc0c38d1
SR
19#include <linux/kallsyms.h>
20#include <linux/seq_file.h>
3f5a54e3 21#include <linux/notifier.h>
2cadf913 22#include <linux/irqflags.h>
bc0c38d1 23#include <linux/debugfs.h>
8434dc93 24#include <linux/tracefs.h>
4c11d7ae 25#include <linux/pagemap.h>
bc0c38d1
SR
26#include <linux/hardirq.h>
27#include <linux/linkage.h>
28#include <linux/uaccess.h>
76c813e2 29#include <linux/vmalloc.h>
bc0c38d1
SR
30#include <linux/ftrace.h>
31#include <linux/module.h>
32#include <linux/percpu.h>
2cadf913 33#include <linux/splice.h>
3f5a54e3 34#include <linux/kdebug.h>
5f0c6c03 35#include <linux/string.h>
f76180bc 36#include <linux/mount.h>
7e53bd42 37#include <linux/rwsem.h>
5a0e3ad6 38#include <linux/slab.h>
bc0c38d1
SR
39#include <linux/ctype.h>
40#include <linux/init.h>
2a2cc8f7 41#include <linux/poll.h>
b892e5c8 42#include <linux/nmi.h>
bc0c38d1 43#include <linux/fs.h>
478409dd 44#include <linux/trace.h>
3fd49c9e 45#include <linux/sched/clock.h>
8bd75c77 46#include <linux/sched/rt.h>
86387f7e 47
bc0c38d1 48#include "trace.h"
f0868d1e 49#include "trace_output.h"
bc0c38d1 50
73c5162a
SR
51/*
52 * On boot up, the ring buffer is set to the minimum size, so that
53 * we do not waste memory on systems that are not using tracing.
54 */
55034cd6 55bool ring_buffer_expanded;
73c5162a 56
8e1b82e0
FW
57/*
58 * We need to change this state when a selftest is running.
ff32504f
FW
59 * A selftest will lurk into the ring-buffer to count the
60 * entries inserted during the selftest although some concurrent
5e1607a0 61 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
62 * at the same time, giving false positive or negative results.
63 */
8e1b82e0 64static bool __read_mostly tracing_selftest_running;
ff32504f 65
b2821ae6
SR
66/*
67 * If a tracer is running, we do not want to run SELFTEST.
68 */
020e5f85 69bool __read_mostly tracing_selftest_disabled;
b2821ae6 70
0daa2302
SRRH
71/* Pipe tracepoints to printk */
72struct trace_iterator *tracepoint_print_iter;
73int tracepoint_printk;
42391745 74static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
0daa2302 75
adf9f195
FW
76/* For tracers that don't implement custom flags */
77static struct tracer_opt dummy_tracer_opt[] = {
78 { }
79};
80
8c1a49ae
SRRH
81static int
82dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
adf9f195
FW
83{
84 return 0;
85}
0f048701 86
7ffbd48d
SR
87/*
88 * To prevent the comm cache from being overwritten when no
89 * tracing is active, only save the comm when a trace event
90 * occurred.
91 */
d914ba37 92static DEFINE_PER_CPU(bool, trace_taskinfo_save);
7ffbd48d 93
0f048701
SR
94/*
95 * Kill all tracing for good (never come back).
96 * It is initialized to 1 but will turn to zero if the initialization
97 * of the tracer is successful. But that is the only place that sets
98 * this back to zero.
99 */
4fd27358 100static int tracing_disabled = 1;
0f048701 101
955b61e5 102cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 103
944ac425
SR
104/*
105 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
106 *
107 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
108 * is set, then ftrace_dump is called. This will output the contents
109 * of the ftrace buffers to the console. This is very useful for
110 * capturing traces that lead to crashes and outputing it to a
111 * serial console.
112 *
113 * It is default off, but you can enable it with either specifying
114 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
115 * /proc/sys/kernel/ftrace_dump_on_oops
116 * Set 1 if you want to dump buffers of all CPUs
117 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 118 */
cecbca96
FW
119
120enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 121
de7edd31
SRRH
122/* When set, tracing will stop when a WARN*() is hit */
123int __disable_trace_on_warning;
124
681bec03
JL
125#ifdef CONFIG_TRACE_EVAL_MAP_FILE
126/* Map of enums to their values, for "eval_map" file */
23bf8cb8 127struct trace_eval_map_head {
9828413d
SRRH
128 struct module *mod;
129 unsigned long length;
130};
131
23bf8cb8 132union trace_eval_map_item;
9828413d 133
23bf8cb8 134struct trace_eval_map_tail {
9828413d
SRRH
135 /*
136 * "end" is first and points to NULL as it must be different
00f4b652 137 * than "mod" or "eval_string"
9828413d 138 */
23bf8cb8 139 union trace_eval_map_item *next;
9828413d
SRRH
140 const char *end; /* points to NULL */
141};
142
1793ed93 143static DEFINE_MUTEX(trace_eval_mutex);
9828413d
SRRH
144
145/*
23bf8cb8 146 * The trace_eval_maps are saved in an array with two extra elements,
9828413d
SRRH
147 * one at the beginning, and one at the end. The beginning item contains
148 * the count of the saved maps (head.length), and the module they
149 * belong to if not built in (head.mod). The ending item contains a
681bec03 150 * pointer to the next array of saved eval_map items.
9828413d 151 */
23bf8cb8 152union trace_eval_map_item {
00f4b652 153 struct trace_eval_map map;
23bf8cb8
JL
154 struct trace_eval_map_head head;
155 struct trace_eval_map_tail tail;
9828413d
SRRH
156};
157
23bf8cb8 158static union trace_eval_map_item *trace_eval_maps;
681bec03 159#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 160
607e2ea1 161static int tracing_set_tracer(struct trace_array *tr, const char *buf);
c438f140
TG
162static void ftrace_trace_userstack(struct ring_buffer *buffer,
163 unsigned long flags, int pc);
b2821ae6 164
ee6c2c1b
LZ
165#define MAX_TRACER_SIZE 100
166static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 167static char *default_bootup_tracer;
d9e54076 168
55034cd6
SRRH
169static bool allocate_snapshot;
170
1beee96b 171static int __init set_cmdline_ftrace(char *str)
d9e54076 172{
67012ab1 173 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 174 default_bootup_tracer = bootup_tracer_buf;
73c5162a 175 /* We are using ftrace early, expand it */
55034cd6 176 ring_buffer_expanded = true;
d9e54076
PZ
177 return 1;
178}
1beee96b 179__setup("ftrace=", set_cmdline_ftrace);
d9e54076 180
944ac425
SR
181static int __init set_ftrace_dump_on_oops(char *str)
182{
cecbca96
FW
183 if (*str++ != '=' || !*str) {
184 ftrace_dump_on_oops = DUMP_ALL;
185 return 1;
186 }
187
188 if (!strcmp("orig_cpu", str)) {
189 ftrace_dump_on_oops = DUMP_ORIG;
190 return 1;
191 }
192
193 return 0;
944ac425
SR
194}
195__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 196
de7edd31
SRRH
197static int __init stop_trace_on_warning(char *str)
198{
933ff9f2
LCG
199 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
200 __disable_trace_on_warning = 1;
de7edd31
SRRH
201 return 1;
202}
933ff9f2 203__setup("traceoff_on_warning", stop_trace_on_warning);
de7edd31 204
3209cff4 205static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
206{
207 allocate_snapshot = true;
208 /* We also need the main ring buffer expanded */
209 ring_buffer_expanded = true;
210 return 1;
211}
3209cff4 212__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 213
7bcfaf54
SR
214
215static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
7bcfaf54
SR
216
217static int __init set_trace_boot_options(char *str)
218{
67012ab1 219 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
220 return 0;
221}
222__setup("trace_options=", set_trace_boot_options);
223
e1e232ca
SR
224static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
225static char *trace_boot_clock __initdata;
226
227static int __init set_trace_boot_clock(char *str)
228{
229 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
230 trace_boot_clock = trace_boot_clock_buf;
231 return 0;
232}
233__setup("trace_clock=", set_trace_boot_clock);
234
0daa2302
SRRH
235static int __init set_tracepoint_printk(char *str)
236{
237 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
238 tracepoint_printk = 1;
239 return 1;
240}
241__setup("tp_printk", set_tracepoint_printk);
de7edd31 242
a5a1d1c2 243unsigned long long ns2usecs(u64 nsec)
bc0c38d1
SR
244{
245 nsec += 500;
246 do_div(nsec, 1000);
247 return nsec;
248}
249
983f938a
SRRH
250/* trace_flags holds trace_options default values */
251#define TRACE_DEFAULT_FLAGS \
252 (FUNCTION_DEFAULT_FLAGS | \
253 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
254 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
255 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
256 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
257
16270145
SRRH
258/* trace_options that are only supported by global_trace */
259#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
260 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
261
20550622
SRRH
262/* trace_flags that are default zero for instances */
263#define ZEROED_TRACE_FLAGS \
1e10486f 264 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
16270145 265
4fcdae83 266/*
67d04bb2
JF
267 * The global_trace is the descriptor that holds the top-level tracing
268 * buffers for the live tracing.
4fcdae83 269 */
983f938a
SRRH
270static struct trace_array global_trace = {
271 .trace_flags = TRACE_DEFAULT_FLAGS,
272};
bc0c38d1 273
ae63b31e 274LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 275
ff451961
SRRH
276int trace_array_get(struct trace_array *this_tr)
277{
278 struct trace_array *tr;
279 int ret = -ENODEV;
280
281 mutex_lock(&trace_types_lock);
282 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
283 if (tr == this_tr) {
284 tr->ref++;
285 ret = 0;
286 break;
287 }
288 }
289 mutex_unlock(&trace_types_lock);
290
291 return ret;
292}
293
294static void __trace_array_put(struct trace_array *this_tr)
295{
296 WARN_ON(!this_tr->ref);
297 this_tr->ref--;
298}
299
300void trace_array_put(struct trace_array *this_tr)
301{
302 mutex_lock(&trace_types_lock);
303 __trace_array_put(this_tr);
304 mutex_unlock(&trace_types_lock);
305}
306
2425bcb9 307int call_filter_check_discard(struct trace_event_call *call, void *rec,
f306cc82
TZ
308 struct ring_buffer *buffer,
309 struct ring_buffer_event *event)
310{
311 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
312 !filter_match_preds(call->filter, rec)) {
0fc1b09f 313 __trace_event_discard_commit(buffer, event);
f306cc82
TZ
314 return 1;
315 }
316
317 return 0;
eb02ce01
TZ
318}
319
76c813e2
SRRH
320void trace_free_pid_list(struct trace_pid_list *pid_list)
321{
322 vfree(pid_list->pids);
323 kfree(pid_list);
324}
325
d8275c45
SR
326/**
327 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
328 * @filtered_pids: The list of pids to check
329 * @search_pid: The PID to find in @filtered_pids
330 *
331 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
332 */
333bool
334trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
335{
336 /*
337 * If pid_max changed after filtered_pids was created, we
338 * by default ignore all pids greater than the previous pid_max.
339 */
340 if (search_pid >= filtered_pids->pid_max)
341 return false;
342
343 return test_bit(search_pid, filtered_pids->pids);
344}
345
346/**
347 * trace_ignore_this_task - should a task be ignored for tracing
348 * @filtered_pids: The list of pids to check
349 * @task: The task that should be ignored if not filtered
350 *
351 * Checks if @task should be traced or not from @filtered_pids.
352 * Returns true if @task should *NOT* be traced.
353 * Returns false if @task should be traced.
354 */
355bool
356trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
357{
358 /*
359 * Return false, because if filtered_pids does not exist,
360 * all pids are good to trace.
361 */
362 if (!filtered_pids)
363 return false;
364
365 return !trace_find_filtered_pid(filtered_pids, task->pid);
366}
367
368/**
f08367b3 369 * trace_filter_add_remove_task - Add or remove a task from a pid_list
d8275c45
SR
370 * @pid_list: The list to modify
371 * @self: The current task for fork or NULL for exit
372 * @task: The task to add or remove
373 *
374 * If adding a task, if @self is defined, the task is only added if @self
375 * is also included in @pid_list. This happens on fork and tasks should
376 * only be added when the parent is listed. If @self is NULL, then the
377 * @task pid will be removed from the list, which would happen on exit
378 * of a task.
379 */
380void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
381 struct task_struct *self,
382 struct task_struct *task)
383{
384 if (!pid_list)
385 return;
386
387 /* For forks, we only add if the forking task is listed */
388 if (self) {
389 if (!trace_find_filtered_pid(pid_list, self->pid))
390 return;
391 }
392
393 /* Sorry, but we don't support pid_max changing after setting */
394 if (task->pid >= pid_list->pid_max)
395 return;
396
397 /* "self" is set for forks, and NULL for exits */
398 if (self)
399 set_bit(task->pid, pid_list->pids);
400 else
401 clear_bit(task->pid, pid_list->pids);
402}
403
5cc8976b
SRRH
404/**
405 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
406 * @pid_list: The pid list to show
407 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
408 * @pos: The position of the file
409 *
410 * This is used by the seq_file "next" operation to iterate the pids
411 * listed in a trace_pid_list structure.
412 *
413 * Returns the pid+1 as we want to display pid of zero, but NULL would
414 * stop the iteration.
415 */
416void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
417{
418 unsigned long pid = (unsigned long)v;
419
420 (*pos)++;
421
422 /* pid already is +1 of the actual prevous bit */
423 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
424
425 /* Return pid + 1 to allow zero to be represented */
426 if (pid < pid_list->pid_max)
427 return (void *)(pid + 1);
428
429 return NULL;
430}
431
432/**
433 * trace_pid_start - Used for seq_file to start reading pid lists
434 * @pid_list: The pid list to show
435 * @pos: The position of the file
436 *
437 * This is used by seq_file "start" operation to start the iteration
438 * of listing pids.
439 *
440 * Returns the pid+1 as we want to display pid of zero, but NULL would
441 * stop the iteration.
442 */
443void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
444{
445 unsigned long pid;
446 loff_t l = 0;
447
448 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
449 if (pid >= pid_list->pid_max)
450 return NULL;
451
452 /* Return pid + 1 so that zero can be the exit value */
453 for (pid++; pid && l < *pos;
454 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
455 ;
456 return (void *)pid;
457}
458
459/**
460 * trace_pid_show - show the current pid in seq_file processing
461 * @m: The seq_file structure to write into
462 * @v: A void pointer of the pid (+1) value to display
463 *
464 * Can be directly used by seq_file operations to display the current
465 * pid value.
466 */
467int trace_pid_show(struct seq_file *m, void *v)
468{
469 unsigned long pid = (unsigned long)v - 1;
470
471 seq_printf(m, "%lu\n", pid);
472 return 0;
473}
474
76c813e2
SRRH
475/* 128 should be much more than enough */
476#define PID_BUF_SIZE 127
477
478int trace_pid_write(struct trace_pid_list *filtered_pids,
479 struct trace_pid_list **new_pid_list,
480 const char __user *ubuf, size_t cnt)
481{
482 struct trace_pid_list *pid_list;
483 struct trace_parser parser;
484 unsigned long val;
485 int nr_pids = 0;
486 ssize_t read = 0;
487 ssize_t ret = 0;
488 loff_t pos;
489 pid_t pid;
490
491 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
492 return -ENOMEM;
493
494 /*
495 * Always recreate a new array. The write is an all or nothing
496 * operation. Always create a new array when adding new pids by
497 * the user. If the operation fails, then the current list is
498 * not modified.
499 */
500 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
91862cc7
WW
501 if (!pid_list) {
502 trace_parser_put(&parser);
76c813e2 503 return -ENOMEM;
91862cc7 504 }
76c813e2
SRRH
505
506 pid_list->pid_max = READ_ONCE(pid_max);
507
508 /* Only truncating will shrink pid_max */
509 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
510 pid_list->pid_max = filtered_pids->pid_max;
511
512 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
513 if (!pid_list->pids) {
91862cc7 514 trace_parser_put(&parser);
76c813e2
SRRH
515 kfree(pid_list);
516 return -ENOMEM;
517 }
518
519 if (filtered_pids) {
520 /* copy the current bits to the new max */
67f20b08
WY
521 for_each_set_bit(pid, filtered_pids->pids,
522 filtered_pids->pid_max) {
76c813e2 523 set_bit(pid, pid_list->pids);
76c813e2
SRRH
524 nr_pids++;
525 }
526 }
527
528 while (cnt > 0) {
529
530 pos = 0;
531
532 ret = trace_get_user(&parser, ubuf, cnt, &pos);
533 if (ret < 0 || !trace_parser_loaded(&parser))
534 break;
535
536 read += ret;
537 ubuf += ret;
538 cnt -= ret;
539
76c813e2
SRRH
540 ret = -EINVAL;
541 if (kstrtoul(parser.buffer, 0, &val))
542 break;
543 if (val >= pid_list->pid_max)
544 break;
545
546 pid = (pid_t)val;
547
548 set_bit(pid, pid_list->pids);
549 nr_pids++;
550
551 trace_parser_clear(&parser);
552 ret = 0;
553 }
554 trace_parser_put(&parser);
555
556 if (ret < 0) {
557 trace_free_pid_list(pid_list);
558 return ret;
559 }
560
561 if (!nr_pids) {
562 /* Cleared the list of pids */
563 trace_free_pid_list(pid_list);
564 read = ret;
565 pid_list = NULL;
566 }
567
568 *new_pid_list = pid_list;
569
570 return read;
571}
572
a5a1d1c2 573static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
37886f6a
SR
574{
575 u64 ts;
576
577 /* Early boot up does not have a buffer yet */
9457158b 578 if (!buf->buffer)
37886f6a
SR
579 return trace_clock_local();
580
9457158b
AL
581 ts = ring_buffer_time_stamp(buf->buffer, cpu);
582 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
37886f6a
SR
583
584 return ts;
585}
bc0c38d1 586
a5a1d1c2 587u64 ftrace_now(int cpu)
9457158b
AL
588{
589 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
590}
591
10246fa3
SRRH
592/**
593 * tracing_is_enabled - Show if global_trace has been disabled
594 *
595 * Shows if the global trace has been enabled or not. It uses the
596 * mirror flag "buffer_disabled" to be used in fast paths such as for
597 * the irqsoff tracer. But it may be inaccurate due to races. If you
598 * need to know the accurate state, use tracing_is_on() which is a little
599 * slower, but accurate.
600 */
9036990d
SR
601int tracing_is_enabled(void)
602{
10246fa3
SRRH
603 /*
604 * For quick access (irqsoff uses this in fast path), just
605 * return the mirror variable of the state of the ring buffer.
606 * It's a little racy, but we don't really care.
607 */
608 smp_rmb();
609 return !global_trace.buffer_disabled;
9036990d
SR
610}
611
4fcdae83 612/*
3928a8a2
SR
613 * trace_buf_size is the size in bytes that is allocated
614 * for a buffer. Note, the number of bytes is always rounded
615 * to page size.
3f5a54e3
SR
616 *
617 * This number is purposely set to a low number of 16384.
618 * If the dump on oops happens, it will be much appreciated
619 * to not have to wait for all that output. Anyway this can be
620 * boot time and run time configurable.
4fcdae83 621 */
3928a8a2 622#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 623
3928a8a2 624static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 625
4fcdae83 626/* trace_types holds a link list of available tracers. */
bc0c38d1 627static struct tracer *trace_types __read_mostly;
4fcdae83 628
4fcdae83
SR
629/*
630 * trace_types_lock is used to protect the trace_types list.
4fcdae83 631 */
a8227415 632DEFINE_MUTEX(trace_types_lock);
4fcdae83 633
7e53bd42
LJ
634/*
635 * serialize the access of the ring buffer
636 *
637 * ring buffer serializes readers, but it is low level protection.
638 * The validity of the events (which returns by ring_buffer_peek() ..etc)
639 * are not protected by ring buffer.
640 *
641 * The content of events may become garbage if we allow other process consumes
642 * these events concurrently:
643 * A) the page of the consumed events may become a normal page
644 * (not reader page) in ring buffer, and this page will be rewrited
645 * by events producer.
646 * B) The page of the consumed events may become a page for splice_read,
647 * and this page will be returned to system.
648 *
649 * These primitives allow multi process access to different cpu ring buffer
650 * concurrently.
651 *
652 * These primitives don't distinguish read-only and read-consume access.
653 * Multi read-only access are also serialized.
654 */
655
656#ifdef CONFIG_SMP
657static DECLARE_RWSEM(all_cpu_access_lock);
658static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
659
660static inline void trace_access_lock(int cpu)
661{
ae3b5093 662 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
663 /* gain it for accessing the whole ring buffer. */
664 down_write(&all_cpu_access_lock);
665 } else {
666 /* gain it for accessing a cpu ring buffer. */
667
ae3b5093 668 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
669 down_read(&all_cpu_access_lock);
670
671 /* Secondly block other access to this @cpu ring buffer. */
672 mutex_lock(&per_cpu(cpu_access_lock, cpu));
673 }
674}
675
676static inline void trace_access_unlock(int cpu)
677{
ae3b5093 678 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
679 up_write(&all_cpu_access_lock);
680 } else {
681 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
682 up_read(&all_cpu_access_lock);
683 }
684}
685
686static inline void trace_access_lock_init(void)
687{
688 int cpu;
689
690 for_each_possible_cpu(cpu)
691 mutex_init(&per_cpu(cpu_access_lock, cpu));
692}
693
694#else
695
696static DEFINE_MUTEX(access_lock);
697
698static inline void trace_access_lock(int cpu)
699{
700 (void)cpu;
701 mutex_lock(&access_lock);
702}
703
704static inline void trace_access_unlock(int cpu)
705{
706 (void)cpu;
707 mutex_unlock(&access_lock);
708}
709
710static inline void trace_access_lock_init(void)
711{
712}
713
714#endif
715
d78a4614
SRRH
716#ifdef CONFIG_STACKTRACE
717static void __ftrace_trace_stack(struct ring_buffer *buffer,
718 unsigned long flags,
719 int skip, int pc, struct pt_regs *regs);
2d34f489
SRRH
720static inline void ftrace_trace_stack(struct trace_array *tr,
721 struct ring_buffer *buffer,
73dddbb5
SRRH
722 unsigned long flags,
723 int skip, int pc, struct pt_regs *regs);
ca475e83 724
d78a4614
SRRH
725#else
726static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
727 unsigned long flags,
728 int skip, int pc, struct pt_regs *regs)
729{
730}
2d34f489
SRRH
731static inline void ftrace_trace_stack(struct trace_array *tr,
732 struct ring_buffer *buffer,
73dddbb5
SRRH
733 unsigned long flags,
734 int skip, int pc, struct pt_regs *regs)
ca475e83
SRRH
735{
736}
737
d78a4614
SRRH
738#endif
739
3e9a8aad
SRRH
740static __always_inline void
741trace_event_setup(struct ring_buffer_event *event,
742 int type, unsigned long flags, int pc)
743{
744 struct trace_entry *ent = ring_buffer_event_data(event);
745
46710f3a 746 tracing_generic_entry_update(ent, type, flags, pc);
3e9a8aad
SRRH
747}
748
749static __always_inline struct ring_buffer_event *
750__trace_buffer_lock_reserve(struct ring_buffer *buffer,
751 int type,
752 unsigned long len,
753 unsigned long flags, int pc)
754{
755 struct ring_buffer_event *event;
756
757 event = ring_buffer_lock_reserve(buffer, len);
758 if (event != NULL)
759 trace_event_setup(event, type, flags, pc);
760
761 return event;
762}
763
2290f2c5 764void tracer_tracing_on(struct trace_array *tr)
10246fa3
SRRH
765{
766 if (tr->trace_buffer.buffer)
767 ring_buffer_record_on(tr->trace_buffer.buffer);
768 /*
769 * This flag is looked at when buffers haven't been allocated
770 * yet, or by some tracers (like irqsoff), that just want to
771 * know if the ring buffer has been disabled, but it can handle
772 * races of where it gets disabled but we still do a record.
773 * As the check is in the fast path of the tracers, it is more
774 * important to be fast than accurate.
775 */
776 tr->buffer_disabled = 0;
777 /* Make the flag seen by readers */
778 smp_wmb();
779}
780
499e5470
SR
781/**
782 * tracing_on - enable tracing buffers
783 *
784 * This function enables tracing buffers that may have been
785 * disabled with tracing_off.
786 */
787void tracing_on(void)
788{
10246fa3 789 tracer_tracing_on(&global_trace);
499e5470
SR
790}
791EXPORT_SYMBOL_GPL(tracing_on);
792
52ffabe3
SRRH
793
794static __always_inline void
795__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
796{
d914ba37 797 __this_cpu_write(trace_taskinfo_save, true);
52ffabe3
SRRH
798
799 /* If this is the temp buffer, we need to commit fully */
800 if (this_cpu_read(trace_buffered_event) == event) {
801 /* Length is in event->array[0] */
802 ring_buffer_write(buffer, event->array[0], &event->array[1]);
803 /* Release the temp buffer */
804 this_cpu_dec(trace_buffered_event_cnt);
805 } else
806 ring_buffer_unlock_commit(buffer, event);
807}
808
09ae7234
SRRH
809/**
810 * __trace_puts - write a constant string into the trace buffer.
811 * @ip: The address of the caller
812 * @str: The constant string to write
813 * @size: The size of the string.
814 */
815int __trace_puts(unsigned long ip, const char *str, int size)
816{
817 struct ring_buffer_event *event;
818 struct ring_buffer *buffer;
819 struct print_entry *entry;
820 unsigned long irq_flags;
821 int alloc;
8abfb872
J
822 int pc;
823
983f938a 824 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
825 return 0;
826
8abfb872 827 pc = preempt_count();
09ae7234 828
3132e107
SRRH
829 if (unlikely(tracing_selftest_running || tracing_disabled))
830 return 0;
831
09ae7234
SRRH
832 alloc = sizeof(*entry) + size + 2; /* possible \n added */
833
834 local_save_flags(irq_flags);
835 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
836 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
837 irq_flags, pc);
09ae7234
SRRH
838 if (!event)
839 return 0;
840
841 entry = ring_buffer_event_data(event);
842 entry->ip = ip;
843
844 memcpy(&entry->buf, str, size);
845
846 /* Add a newline if necessary */
847 if (entry->buf[size - 1] != '\n') {
848 entry->buf[size] = '\n';
849 entry->buf[size + 1] = '\0';
850 } else
851 entry->buf[size] = '\0';
852
853 __buffer_unlock_commit(buffer, event);
2d34f489 854 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
855
856 return size;
857}
858EXPORT_SYMBOL_GPL(__trace_puts);
859
860/**
861 * __trace_bputs - write the pointer to a constant string into trace buffer
862 * @ip: The address of the caller
863 * @str: The constant string to write to the buffer to
864 */
865int __trace_bputs(unsigned long ip, const char *str)
866{
867 struct ring_buffer_event *event;
868 struct ring_buffer *buffer;
869 struct bputs_entry *entry;
870 unsigned long irq_flags;
871 int size = sizeof(struct bputs_entry);
8abfb872
J
872 int pc;
873
983f938a 874 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
875 return 0;
876
8abfb872 877 pc = preempt_count();
09ae7234 878
3132e107
SRRH
879 if (unlikely(tracing_selftest_running || tracing_disabled))
880 return 0;
881
09ae7234
SRRH
882 local_save_flags(irq_flags);
883 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
884 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
885 irq_flags, pc);
09ae7234
SRRH
886 if (!event)
887 return 0;
888
889 entry = ring_buffer_event_data(event);
890 entry->ip = ip;
891 entry->str = str;
892
893 __buffer_unlock_commit(buffer, event);
2d34f489 894 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
895
896 return 1;
897}
898EXPORT_SYMBOL_GPL(__trace_bputs);
899
ad909e21 900#ifdef CONFIG_TRACER_SNAPSHOT
a35873a0 901void tracing_snapshot_instance_cond(struct trace_array *tr, void *cond_data)
ad909e21 902{
ad909e21
SRRH
903 struct tracer *tracer = tr->current_trace;
904 unsigned long flags;
905
1b22e382
SRRH
906 if (in_nmi()) {
907 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
908 internal_trace_puts("*** snapshot is being ignored ***\n");
909 return;
910 }
911
ad909e21 912 if (!tr->allocated_snapshot) {
ca268da6
SRRH
913 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
914 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
915 tracing_off();
916 return;
917 }
918
919 /* Note, snapshot can not be used when the tracer uses it */
920 if (tracer->use_max_tr) {
ca268da6
SRRH
921 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
922 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
923 return;
924 }
925
926 local_irq_save(flags);
a35873a0 927 update_max_tr(tr, current, smp_processor_id(), cond_data);
ad909e21
SRRH
928 local_irq_restore(flags);
929}
cab50379 930
a35873a0
TZ
931void tracing_snapshot_instance(struct trace_array *tr)
932{
933 tracing_snapshot_instance_cond(tr, NULL);
934}
935
cab50379 936/**
5a93bae2 937 * tracing_snapshot - take a snapshot of the current buffer.
cab50379
SRV
938 *
939 * This causes a swap between the snapshot buffer and the current live
940 * tracing buffer. You can use this to take snapshots of the live
941 * trace when some condition is triggered, but continue to trace.
942 *
943 * Note, make sure to allocate the snapshot with either
944 * a tracing_snapshot_alloc(), or by doing it manually
945 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
946 *
947 * If the snapshot buffer is not allocated, it will stop tracing.
948 * Basically making a permanent snapshot.
949 */
950void tracing_snapshot(void)
951{
952 struct trace_array *tr = &global_trace;
953
954 tracing_snapshot_instance(tr);
955}
1b22e382 956EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21 957
a35873a0
TZ
958/**
959 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
960 * @tr: The tracing instance to snapshot
961 * @cond_data: The data to be tested conditionally, and possibly saved
962 *
963 * This is the same as tracing_snapshot() except that the snapshot is
964 * conditional - the snapshot will only happen if the
965 * cond_snapshot.update() implementation receiving the cond_data
966 * returns true, which means that the trace array's cond_snapshot
967 * update() operation used the cond_data to determine whether the
968 * snapshot should be taken, and if it was, presumably saved it along
969 * with the snapshot.
970 */
971void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
972{
973 tracing_snapshot_instance_cond(tr, cond_data);
974}
975EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
976
977/**
978 * tracing_snapshot_cond_data - get the user data associated with a snapshot
979 * @tr: The tracing instance
980 *
981 * When the user enables a conditional snapshot using
982 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
983 * with the snapshot. This accessor is used to retrieve it.
984 *
985 * Should not be called from cond_snapshot.update(), since it takes
986 * the tr->max_lock lock, which the code calling
987 * cond_snapshot.update() has already done.
988 *
989 * Returns the cond_data associated with the trace array's snapshot.
990 */
991void *tracing_cond_snapshot_data(struct trace_array *tr)
992{
993 void *cond_data = NULL;
994
995 arch_spin_lock(&tr->max_lock);
996
997 if (tr->cond_snapshot)
998 cond_data = tr->cond_snapshot->cond_data;
999
1000 arch_spin_unlock(&tr->max_lock);
1001
1002 return cond_data;
1003}
1004EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1005
ad909e21
SRRH
1006static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
1007 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
1008static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
1009
2824f503 1010int tracing_alloc_snapshot_instance(struct trace_array *tr)
3209cff4
SRRH
1011{
1012 int ret;
1013
1014 if (!tr->allocated_snapshot) {
1015
1016 /* allocate spare buffer */
1017 ret = resize_buffer_duplicate_size(&tr->max_buffer,
1018 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
1019 if (ret < 0)
1020 return ret;
1021
1022 tr->allocated_snapshot = true;
1023 }
1024
1025 return 0;
1026}
1027
ad1438a0 1028static void free_snapshot(struct trace_array *tr)
3209cff4
SRRH
1029{
1030 /*
1031 * We don't free the ring buffer. instead, resize it because
1032 * The max_tr ring buffer has some state (e.g. ring->clock) and
1033 * we want preserve it.
1034 */
1035 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
1036 set_buffer_entries(&tr->max_buffer, 1);
1037 tracing_reset_online_cpus(&tr->max_buffer);
1038 tr->allocated_snapshot = false;
1039}
ad909e21 1040
93e31ffb
TZ
1041/**
1042 * tracing_alloc_snapshot - allocate snapshot buffer.
1043 *
1044 * This only allocates the snapshot buffer if it isn't already
1045 * allocated - it doesn't also take a snapshot.
1046 *
1047 * This is meant to be used in cases where the snapshot buffer needs
1048 * to be set up for events that can't sleep but need to be able to
1049 * trigger a snapshot.
1050 */
1051int tracing_alloc_snapshot(void)
1052{
1053 struct trace_array *tr = &global_trace;
1054 int ret;
1055
2824f503 1056 ret = tracing_alloc_snapshot_instance(tr);
93e31ffb
TZ
1057 WARN_ON(ret < 0);
1058
1059 return ret;
1060}
1061EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1062
ad909e21 1063/**
5a93bae2 1064 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
ad909e21 1065 *
5a93bae2 1066 * This is similar to tracing_snapshot(), but it will allocate the
ad909e21
SRRH
1067 * snapshot buffer if it isn't already allocated. Use this only
1068 * where it is safe to sleep, as the allocation may sleep.
1069 *
1070 * This causes a swap between the snapshot buffer and the current live
1071 * tracing buffer. You can use this to take snapshots of the live
1072 * trace when some condition is triggered, but continue to trace.
1073 */
1074void tracing_snapshot_alloc(void)
1075{
ad909e21
SRRH
1076 int ret;
1077
93e31ffb
TZ
1078 ret = tracing_alloc_snapshot();
1079 if (ret < 0)
3209cff4 1080 return;
ad909e21
SRRH
1081
1082 tracing_snapshot();
1083}
1b22e382 1084EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
a35873a0
TZ
1085
1086/**
1087 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1088 * @tr: The tracing instance
1089 * @cond_data: User data to associate with the snapshot
1090 * @update: Implementation of the cond_snapshot update function
1091 *
1092 * Check whether the conditional snapshot for the given instance has
1093 * already been enabled, or if the current tracer is already using a
1094 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1095 * save the cond_data and update function inside.
1096 *
1097 * Returns 0 if successful, error otherwise.
1098 */
1099int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1100 cond_update_fn_t update)
1101{
1102 struct cond_snapshot *cond_snapshot;
1103 int ret = 0;
1104
1105 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
1106 if (!cond_snapshot)
1107 return -ENOMEM;
1108
1109 cond_snapshot->cond_data = cond_data;
1110 cond_snapshot->update = update;
1111
1112 mutex_lock(&trace_types_lock);
1113
1114 ret = tracing_alloc_snapshot_instance(tr);
1115 if (ret)
1116 goto fail_unlock;
1117
1118 if (tr->current_trace->use_max_tr) {
1119 ret = -EBUSY;
1120 goto fail_unlock;
1121 }
1122
1c347a94
SRV
1123 /*
1124 * The cond_snapshot can only change to NULL without the
1125 * trace_types_lock. We don't care if we race with it going
1126 * to NULL, but we want to make sure that it's not set to
1127 * something other than NULL when we get here, which we can
1128 * do safely with only holding the trace_types_lock and not
1129 * having to take the max_lock.
1130 */
a35873a0
TZ
1131 if (tr->cond_snapshot) {
1132 ret = -EBUSY;
1133 goto fail_unlock;
1134 }
1135
1136 arch_spin_lock(&tr->max_lock);
1137 tr->cond_snapshot = cond_snapshot;
1138 arch_spin_unlock(&tr->max_lock);
1139
1140 mutex_unlock(&trace_types_lock);
1141
1142 return ret;
1143
1144 fail_unlock:
1145 mutex_unlock(&trace_types_lock);
1146 kfree(cond_snapshot);
1147 return ret;
1148}
1149EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1150
1151/**
1152 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1153 * @tr: The tracing instance
1154 *
1155 * Check whether the conditional snapshot for the given instance is
1156 * enabled; if so, free the cond_snapshot associated with it,
1157 * otherwise return -EINVAL.
1158 *
1159 * Returns 0 if successful, error otherwise.
1160 */
1161int tracing_snapshot_cond_disable(struct trace_array *tr)
1162{
1163 int ret = 0;
1164
1165 arch_spin_lock(&tr->max_lock);
1166
1167 if (!tr->cond_snapshot)
1168 ret = -EINVAL;
1169 else {
1170 kfree(tr->cond_snapshot);
1171 tr->cond_snapshot = NULL;
1172 }
1173
1174 arch_spin_unlock(&tr->max_lock);
1175
1176 return ret;
1177}
1178EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
ad909e21
SRRH
1179#else
1180void tracing_snapshot(void)
1181{
1182 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1183}
1b22e382 1184EXPORT_SYMBOL_GPL(tracing_snapshot);
a35873a0
TZ
1185void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1186{
1187 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1188}
1189EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
93e31ffb
TZ
1190int tracing_alloc_snapshot(void)
1191{
1192 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1193 return -ENODEV;
1194}
1195EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
ad909e21
SRRH
1196void tracing_snapshot_alloc(void)
1197{
1198 /* Give warning */
1199 tracing_snapshot();
1200}
1b22e382 1201EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
a35873a0
TZ
1202void *tracing_cond_snapshot_data(struct trace_array *tr)
1203{
1204 return NULL;
1205}
1206EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1207int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1208{
1209 return -ENODEV;
1210}
1211EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1212int tracing_snapshot_cond_disable(struct trace_array *tr)
1213{
1214 return false;
1215}
1216EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
ad909e21
SRRH
1217#endif /* CONFIG_TRACER_SNAPSHOT */
1218
2290f2c5 1219void tracer_tracing_off(struct trace_array *tr)
10246fa3
SRRH
1220{
1221 if (tr->trace_buffer.buffer)
1222 ring_buffer_record_off(tr->trace_buffer.buffer);
1223 /*
1224 * This flag is looked at when buffers haven't been allocated
1225 * yet, or by some tracers (like irqsoff), that just want to
1226 * know if the ring buffer has been disabled, but it can handle
1227 * races of where it gets disabled but we still do a record.
1228 * As the check is in the fast path of the tracers, it is more
1229 * important to be fast than accurate.
1230 */
1231 tr->buffer_disabled = 1;
1232 /* Make the flag seen by readers */
1233 smp_wmb();
1234}
1235
499e5470
SR
1236/**
1237 * tracing_off - turn off tracing buffers
1238 *
1239 * This function stops the tracing buffers from recording data.
1240 * It does not disable any overhead the tracers themselves may
1241 * be causing. This function simply causes all recording to
1242 * the ring buffers to fail.
1243 */
1244void tracing_off(void)
1245{
10246fa3 1246 tracer_tracing_off(&global_trace);
499e5470
SR
1247}
1248EXPORT_SYMBOL_GPL(tracing_off);
1249
de7edd31
SRRH
1250void disable_trace_on_warning(void)
1251{
1252 if (__disable_trace_on_warning)
1253 tracing_off();
1254}
1255
10246fa3
SRRH
1256/**
1257 * tracer_tracing_is_on - show real state of ring buffer enabled
1258 * @tr : the trace array to know if ring buffer is enabled
1259 *
1260 * Shows real state of the ring buffer if it is enabled or not.
1261 */
ec573508 1262bool tracer_tracing_is_on(struct trace_array *tr)
10246fa3
SRRH
1263{
1264 if (tr->trace_buffer.buffer)
1265 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1266 return !tr->buffer_disabled;
1267}
1268
499e5470
SR
1269/**
1270 * tracing_is_on - show state of ring buffers enabled
1271 */
1272int tracing_is_on(void)
1273{
10246fa3 1274 return tracer_tracing_is_on(&global_trace);
499e5470
SR
1275}
1276EXPORT_SYMBOL_GPL(tracing_is_on);
1277
3928a8a2 1278static int __init set_buf_size(char *str)
bc0c38d1 1279{
3928a8a2 1280 unsigned long buf_size;
c6caeeb1 1281
bc0c38d1
SR
1282 if (!str)
1283 return 0;
9d612bef 1284 buf_size = memparse(str, &str);
c6caeeb1 1285 /* nr_entries can not be zero */
9d612bef 1286 if (buf_size == 0)
c6caeeb1 1287 return 0;
3928a8a2 1288 trace_buf_size = buf_size;
bc0c38d1
SR
1289 return 1;
1290}
3928a8a2 1291__setup("trace_buf_size=", set_buf_size);
bc0c38d1 1292
0e950173
TB
1293static int __init set_tracing_thresh(char *str)
1294{
87abb3b1 1295 unsigned long threshold;
0e950173
TB
1296 int ret;
1297
1298 if (!str)
1299 return 0;
bcd83ea6 1300 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
1301 if (ret < 0)
1302 return 0;
87abb3b1 1303 tracing_thresh = threshold * 1000;
0e950173
TB
1304 return 1;
1305}
1306__setup("tracing_thresh=", set_tracing_thresh);
1307
57f50be1
SR
1308unsigned long nsecs_to_usecs(unsigned long nsecs)
1309{
1310 return nsecs / 1000;
1311}
1312
a3418a36
SRRH
1313/*
1314 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
f57a4143 1315 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
a3418a36 1316 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
f57a4143 1317 * of strings in the order that the evals (enum) were defined.
a3418a36
SRRH
1318 */
1319#undef C
1320#define C(a, b) b
1321
4fcdae83 1322/* These must match the bit postions in trace_iterator_flags */
bc0c38d1 1323static const char *trace_options[] = {
a3418a36 1324 TRACE_FLAGS
bc0c38d1
SR
1325 NULL
1326};
1327
5079f326
Z
1328static struct {
1329 u64 (*func)(void);
1330 const char *name;
8be0709f 1331 int in_ns; /* is this clock in nanoseconds? */
5079f326 1332} trace_clocks[] = {
1b3e5c09
TG
1333 { trace_clock_local, "local", 1 },
1334 { trace_clock_global, "global", 1 },
1335 { trace_clock_counter, "counter", 0 },
e7fda6c4 1336 { trace_clock_jiffies, "uptime", 0 },
1b3e5c09
TG
1337 { trace_clock, "perf", 1 },
1338 { ktime_get_mono_fast_ns, "mono", 1 },
aabfa5f2 1339 { ktime_get_raw_fast_ns, "mono_raw", 1 },
a3ed0e43 1340 { ktime_get_boot_fast_ns, "boot", 1 },
8cbd9cc6 1341 ARCH_TRACE_CLOCKS
5079f326
Z
1342};
1343
860f9f6b
TZ
1344bool trace_clock_in_ns(struct trace_array *tr)
1345{
1346 if (trace_clocks[tr->clock_id].in_ns)
1347 return true;
1348
1349 return false;
1350}
1351
b63f39ea 1352/*
1353 * trace_parser_get_init - gets the buffer for trace parser
1354 */
1355int trace_parser_get_init(struct trace_parser *parser, int size)
1356{
1357 memset(parser, 0, sizeof(*parser));
1358
1359 parser->buffer = kmalloc(size, GFP_KERNEL);
1360 if (!parser->buffer)
1361 return 1;
1362
1363 parser->size = size;
1364 return 0;
1365}
1366
1367/*
1368 * trace_parser_put - frees the buffer for trace parser
1369 */
1370void trace_parser_put(struct trace_parser *parser)
1371{
1372 kfree(parser->buffer);
0e684b65 1373 parser->buffer = NULL;
b63f39ea 1374}
1375
1376/*
1377 * trace_get_user - reads the user input string separated by space
1378 * (matched by isspace(ch))
1379 *
1380 * For each string found the 'struct trace_parser' is updated,
1381 * and the function returns.
1382 *
1383 * Returns number of bytes read.
1384 *
1385 * See kernel/trace/trace.h for 'struct trace_parser' details.
1386 */
1387int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1388 size_t cnt, loff_t *ppos)
1389{
1390 char ch;
1391 size_t read = 0;
1392 ssize_t ret;
1393
1394 if (!*ppos)
1395 trace_parser_clear(parser);
1396
1397 ret = get_user(ch, ubuf++);
1398 if (ret)
1399 goto out;
1400
1401 read++;
1402 cnt--;
1403
1404 /*
1405 * The parser is not finished with the last write,
1406 * continue reading the user input without skipping spaces.
1407 */
1408 if (!parser->cont) {
1409 /* skip white space */
1410 while (cnt && isspace(ch)) {
1411 ret = get_user(ch, ubuf++);
1412 if (ret)
1413 goto out;
1414 read++;
1415 cnt--;
1416 }
1417
76638d96
CD
1418 parser->idx = 0;
1419
b63f39ea 1420 /* only spaces were written */
921a7acd 1421 if (isspace(ch) || !ch) {
b63f39ea 1422 *ppos += read;
1423 ret = read;
1424 goto out;
1425 }
b63f39ea 1426 }
1427
1428 /* read the non-space input */
921a7acd 1429 while (cnt && !isspace(ch) && ch) {
3c235a33 1430 if (parser->idx < parser->size - 1)
b63f39ea 1431 parser->buffer[parser->idx++] = ch;
1432 else {
1433 ret = -EINVAL;
1434 goto out;
1435 }
1436 ret = get_user(ch, ubuf++);
1437 if (ret)
1438 goto out;
1439 read++;
1440 cnt--;
1441 }
1442
1443 /* We either got finished input or we have to wait for another call. */
921a7acd 1444 if (isspace(ch) || !ch) {
b63f39ea 1445 parser->buffer[parser->idx] = 0;
1446 parser->cont = false;
057db848 1447 } else if (parser->idx < parser->size - 1) {
b63f39ea 1448 parser->cont = true;
1449 parser->buffer[parser->idx++] = ch;
f4d0706c
CD
1450 /* Make sure the parsed string always terminates with '\0'. */
1451 parser->buffer[parser->idx] = 0;
057db848
SR
1452 } else {
1453 ret = -EINVAL;
1454 goto out;
b63f39ea 1455 }
1456
1457 *ppos += read;
1458 ret = read;
1459
1460out:
1461 return ret;
1462}
1463
3a161d99 1464/* TODO add a seq_buf_to_buffer() */
b8b94265 1465static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
1466{
1467 int len;
3c56819b 1468
5ac48378 1469 if (trace_seq_used(s) <= s->seq.readpos)
3c56819b
EGM
1470 return -EBUSY;
1471
5ac48378 1472 len = trace_seq_used(s) - s->seq.readpos;
3c56819b
EGM
1473 if (cnt > len)
1474 cnt = len;
3a161d99 1475 memcpy(buf, s->buffer + s->seq.readpos, cnt);
3c56819b 1476
3a161d99 1477 s->seq.readpos += cnt;
3c56819b
EGM
1478 return cnt;
1479}
1480
0e950173
TB
1481unsigned long __read_mostly tracing_thresh;
1482
5d4a9dba 1483#ifdef CONFIG_TRACER_MAX_TRACE
5d4a9dba
SR
1484/*
1485 * Copy the new maximum trace into the separate maximum-trace
1486 * structure. (this way the maximum trace is permanently saved,
5a93bae2 1487 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
5d4a9dba
SR
1488 */
1489static void
1490__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1491{
12883efb
SRRH
1492 struct trace_buffer *trace_buf = &tr->trace_buffer;
1493 struct trace_buffer *max_buf = &tr->max_buffer;
1494 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1495 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 1496
12883efb
SRRH
1497 max_buf->cpu = cpu;
1498 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 1499
6d9b3fa5 1500 max_data->saved_latency = tr->max_latency;
8248ac05
SR
1501 max_data->critical_start = data->critical_start;
1502 max_data->critical_end = data->critical_end;
5d4a9dba 1503
85f726a3 1504 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05 1505 max_data->pid = tsk->pid;
f17a5194
SRRH
1506 /*
1507 * If tsk == current, then use current_uid(), as that does not use
1508 * RCU. The irq tracer can be called out of RCU scope.
1509 */
1510 if (tsk == current)
1511 max_data->uid = current_uid();
1512 else
1513 max_data->uid = task_uid(tsk);
1514
8248ac05
SR
1515 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1516 max_data->policy = tsk->policy;
1517 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
1518
1519 /* record this tasks comm */
1520 tracing_record_cmdline(tsk);
1521}
1522
4fcdae83
SR
1523/**
1524 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1525 * @tr: tracer
1526 * @tsk: the task with the latency
1527 * @cpu: The cpu that initiated the trace.
a35873a0 1528 * @cond_data: User data associated with a conditional snapshot
4fcdae83
SR
1529 *
1530 * Flip the buffers between the @tr and the max_tr and record information
1531 * about which task was the cause of this latency.
1532 */
e309b41d 1533void
a35873a0
TZ
1534update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1535 void *cond_data)
bc0c38d1 1536{
2b6080f2 1537 if (tr->stop_count)
b8de7bd1
SR
1538 return;
1539
4c11d7ae 1540 WARN_ON_ONCE(!irqs_disabled());
34600f0e 1541
45ad21ca 1542 if (!tr->allocated_snapshot) {
debdd57f 1543 /* Only the nop tracer should hit this when disabling */
2b6080f2 1544 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 1545 return;
debdd57f 1546 }
34600f0e 1547
0b9b12c1 1548 arch_spin_lock(&tr->max_lock);
3928a8a2 1549
73c8d894
MH
1550 /* Inherit the recordable setting from trace_buffer */
1551 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1552 ring_buffer_record_on(tr->max_buffer.buffer);
1553 else
1554 ring_buffer_record_off(tr->max_buffer.buffer);
1555
a35873a0
TZ
1556#ifdef CONFIG_TRACER_SNAPSHOT
1557 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
1558 goto out_unlock;
1559#endif
08ae88f8 1560 swap(tr->trace_buffer.buffer, tr->max_buffer.buffer);
3928a8a2 1561
bc0c38d1 1562 __update_max_tr(tr, tsk, cpu);
a35873a0
TZ
1563
1564 out_unlock:
0b9b12c1 1565 arch_spin_unlock(&tr->max_lock);
bc0c38d1
SR
1566}
1567
1568/**
1569 * update_max_tr_single - only copy one trace over, and reset the rest
c68c9ec1
JK
1570 * @tr: tracer
1571 * @tsk: task with the latency
1572 * @cpu: the cpu of the buffer to copy.
4fcdae83
SR
1573 *
1574 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 1575 */
e309b41d 1576void
bc0c38d1
SR
1577update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1578{
3928a8a2 1579 int ret;
bc0c38d1 1580
2b6080f2 1581 if (tr->stop_count)
b8de7bd1
SR
1582 return;
1583
4c11d7ae 1584 WARN_ON_ONCE(!irqs_disabled());
6c24499f 1585 if (!tr->allocated_snapshot) {
2930e04d 1586 /* Only the nop tracer should hit this when disabling */
9e8529af 1587 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 1588 return;
2930e04d 1589 }
ef710e10 1590
0b9b12c1 1591 arch_spin_lock(&tr->max_lock);
bc0c38d1 1592
12883efb 1593 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 1594
e8165dbb
SR
1595 if (ret == -EBUSY) {
1596 /*
1597 * We failed to swap the buffer due to a commit taking
1598 * place on this CPU. We fail to record, but we reset
1599 * the max trace buffer (no one writes directly to it)
1600 * and flag that it failed.
1601 */
12883efb 1602 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
1603 "Failed to swap buffers due to commit in progress\n");
1604 }
1605
e8165dbb 1606 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
1607
1608 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1609 arch_spin_unlock(&tr->max_lock);
bc0c38d1 1610}
5d4a9dba 1611#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 1612
2c2b0a78 1613static int wait_on_pipe(struct trace_iterator *iter, int full)
0d5c6e1c 1614{
15693458
SRRH
1615 /* Iterators are static, they should be filled or empty */
1616 if (trace_buffer_iter(iter, iter->cpu_file))
8b8b3683 1617 return 0;
0d5c6e1c 1618
e30f53aa
RV
1619 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1620 full);
0d5c6e1c
SR
1621}
1622
f4e781c0 1623#ifdef CONFIG_FTRACE_STARTUP_TEST
9afecfbb
SRV
1624static bool selftests_can_run;
1625
1626struct trace_selftests {
1627 struct list_head list;
1628 struct tracer *type;
1629};
1630
1631static LIST_HEAD(postponed_selftests);
1632
1633static int save_selftest(struct tracer *type)
1634{
1635 struct trace_selftests *selftest;
1636
1637 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1638 if (!selftest)
1639 return -ENOMEM;
1640
1641 selftest->type = type;
1642 list_add(&selftest->list, &postponed_selftests);
1643 return 0;
1644}
1645
f4e781c0
SRRH
1646static int run_tracer_selftest(struct tracer *type)
1647{
1648 struct trace_array *tr = &global_trace;
1649 struct tracer *saved_tracer = tr->current_trace;
1650 int ret;
0d5c6e1c 1651
f4e781c0
SRRH
1652 if (!type->selftest || tracing_selftest_disabled)
1653 return 0;
0d5c6e1c 1654
9afecfbb
SRV
1655 /*
1656 * If a tracer registers early in boot up (before scheduling is
1657 * initialized and such), then do not run its selftests yet.
1658 * Instead, run it a little later in the boot process.
1659 */
1660 if (!selftests_can_run)
1661 return save_selftest(type);
1662
0d5c6e1c 1663 /*
f4e781c0
SRRH
1664 * Run a selftest on this tracer.
1665 * Here we reset the trace buffer, and set the current
1666 * tracer to be this tracer. The tracer can then run some
1667 * internal tracing to verify that everything is in order.
1668 * If we fail, we do not register this tracer.
0d5c6e1c 1669 */
f4e781c0 1670 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 1671
f4e781c0
SRRH
1672 tr->current_trace = type;
1673
1674#ifdef CONFIG_TRACER_MAX_TRACE
1675 if (type->use_max_tr) {
1676 /* If we expanded the buffers, make sure the max is expanded too */
1677 if (ring_buffer_expanded)
1678 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1679 RING_BUFFER_ALL_CPUS);
1680 tr->allocated_snapshot = true;
1681 }
1682#endif
1683
1684 /* the test is responsible for initializing and enabling */
1685 pr_info("Testing tracer %s: ", type->name);
1686 ret = type->selftest(type, tr);
1687 /* the test is responsible for resetting too */
1688 tr->current_trace = saved_tracer;
1689 if (ret) {
1690 printk(KERN_CONT "FAILED!\n");
1691 /* Add the warning after printing 'FAILED' */
1692 WARN_ON(1);
1693 return -1;
1694 }
1695 /* Only reset on passing, to avoid touching corrupted buffers */
1696 tracing_reset_online_cpus(&tr->trace_buffer);
1697
1698#ifdef CONFIG_TRACER_MAX_TRACE
1699 if (type->use_max_tr) {
1700 tr->allocated_snapshot = false;
0d5c6e1c 1701
f4e781c0
SRRH
1702 /* Shrink the max buffer again */
1703 if (ring_buffer_expanded)
1704 ring_buffer_resize(tr->max_buffer.buffer, 1,
1705 RING_BUFFER_ALL_CPUS);
1706 }
1707#endif
1708
1709 printk(KERN_CONT "PASSED\n");
1710 return 0;
1711}
9afecfbb
SRV
1712
1713static __init int init_trace_selftests(void)
1714{
1715 struct trace_selftests *p, *n;
1716 struct tracer *t, **last;
1717 int ret;
1718
1719 selftests_can_run = true;
1720
1721 mutex_lock(&trace_types_lock);
1722
1723 if (list_empty(&postponed_selftests))
1724 goto out;
1725
1726 pr_info("Running postponed tracer tests:\n");
1727
1728 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
6fc2171c
AR
1729 /* This loop can take minutes when sanitizers are enabled, so
1730 * lets make sure we allow RCU processing.
1731 */
1732 cond_resched();
9afecfbb
SRV
1733 ret = run_tracer_selftest(p->type);
1734 /* If the test fails, then warn and remove from available_tracers */
1735 if (ret < 0) {
1736 WARN(1, "tracer: %s failed selftest, disabling\n",
1737 p->type->name);
1738 last = &trace_types;
1739 for (t = trace_types; t; t = t->next) {
1740 if (t == p->type) {
1741 *last = t->next;
1742 break;
1743 }
1744 last = &t->next;
1745 }
1746 }
1747 list_del(&p->list);
1748 kfree(p);
1749 }
1750
1751 out:
1752 mutex_unlock(&trace_types_lock);
1753
1754 return 0;
1755}
b9ef0326 1756core_initcall(init_trace_selftests);
f4e781c0
SRRH
1757#else
1758static inline int run_tracer_selftest(struct tracer *type)
1759{
1760 return 0;
0d5c6e1c 1761}
f4e781c0 1762#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1763
41d9c0be
SRRH
1764static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1765
a4d1e688
JW
1766static void __init apply_trace_boot_options(void);
1767
4fcdae83
SR
1768/**
1769 * register_tracer - register a tracer with the ftrace system.
c68c9ec1 1770 * @type: the plugin for the tracer
4fcdae83
SR
1771 *
1772 * Register a new plugin tracer.
1773 */
a4d1e688 1774int __init register_tracer(struct tracer *type)
bc0c38d1
SR
1775{
1776 struct tracer *t;
bc0c38d1
SR
1777 int ret = 0;
1778
1779 if (!type->name) {
1780 pr_info("Tracer must have a name\n");
1781 return -1;
1782 }
1783
24a461d5 1784 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1785 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1786 return -1;
1787 }
1788
bc0c38d1 1789 mutex_lock(&trace_types_lock);
86fa2f60 1790
8e1b82e0
FW
1791 tracing_selftest_running = true;
1792
bc0c38d1
SR
1793 for (t = trace_types; t; t = t->next) {
1794 if (strcmp(type->name, t->name) == 0) {
1795 /* already found */
ee6c2c1b 1796 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1797 type->name);
1798 ret = -1;
1799 goto out;
1800 }
1801 }
1802
adf9f195
FW
1803 if (!type->set_flag)
1804 type->set_flag = &dummy_set_flag;
d39cdd20
CH
1805 if (!type->flags) {
1806 /*allocate a dummy tracer_flags*/
1807 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
c8ca003b
CH
1808 if (!type->flags) {
1809 ret = -ENOMEM;
1810 goto out;
1811 }
d39cdd20
CH
1812 type->flags->val = 0;
1813 type->flags->opts = dummy_tracer_opt;
1814 } else
adf9f195
FW
1815 if (!type->flags->opts)
1816 type->flags->opts = dummy_tracer_opt;
6eaaa5d5 1817
d39cdd20
CH
1818 /* store the tracer for __set_tracer_option */
1819 type->flags->trace = type;
1820
f4e781c0
SRRH
1821 ret = run_tracer_selftest(type);
1822 if (ret < 0)
1823 goto out;
60a11774 1824
bc0c38d1
SR
1825 type->next = trace_types;
1826 trace_types = type;
41d9c0be 1827 add_tracer_options(&global_trace, type);
60a11774 1828
bc0c38d1 1829 out:
8e1b82e0 1830 tracing_selftest_running = false;
bc0c38d1
SR
1831 mutex_unlock(&trace_types_lock);
1832
dac74940
SR
1833 if (ret || !default_bootup_tracer)
1834 goto out_unlock;
1835
ee6c2c1b 1836 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1837 goto out_unlock;
1838
1839 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1840 /* Do we want this tracer to start on bootup? */
607e2ea1 1841 tracing_set_tracer(&global_trace, type->name);
dac74940 1842 default_bootup_tracer = NULL;
a4d1e688
JW
1843
1844 apply_trace_boot_options();
1845
dac74940 1846 /* disable other selftests, since this will break it. */
55034cd6 1847 tracing_selftest_disabled = true;
b2821ae6 1848#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1849 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1850 type->name);
b2821ae6 1851#endif
b2821ae6 1852
dac74940 1853 out_unlock:
bc0c38d1
SR
1854 return ret;
1855}
1856
12883efb 1857void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1858{
12883efb 1859 struct ring_buffer *buffer = buf->buffer;
f633903a 1860
a5416411
HT
1861 if (!buffer)
1862 return;
1863
f633903a
SR
1864 ring_buffer_record_disable(buffer);
1865
1866 /* Make sure all commits have finished */
74401729 1867 synchronize_rcu();
68179686 1868 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1869
1870 ring_buffer_record_enable(buffer);
1871}
1872
12883efb 1873void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1874{
12883efb 1875 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1876 int cpu;
1877
a5416411
HT
1878 if (!buffer)
1879 return;
1880
621968cd
SR
1881 ring_buffer_record_disable(buffer);
1882
1883 /* Make sure all commits have finished */
74401729 1884 synchronize_rcu();
621968cd 1885
9457158b 1886 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
213cc060
PE
1887
1888 for_each_online_cpu(cpu)
68179686 1889 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1890
1891 ring_buffer_record_enable(buffer);
213cc060
PE
1892}
1893
09d8091c 1894/* Must have trace_types_lock held */
873c642f 1895void tracing_reset_all_online_cpus(void)
9456f0fa 1896{
873c642f
SRRH
1897 struct trace_array *tr;
1898
873c642f 1899 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
065e63f9
SRV
1900 if (!tr->clear_trace)
1901 continue;
1902 tr->clear_trace = false;
12883efb
SRRH
1903 tracing_reset_online_cpus(&tr->trace_buffer);
1904#ifdef CONFIG_TRACER_MAX_TRACE
1905 tracing_reset_online_cpus(&tr->max_buffer);
1906#endif
873c642f 1907 }
9456f0fa
SR
1908}
1909
d914ba37
JF
1910static int *tgid_map;
1911
939c7a4f 1912#define SAVED_CMDLINES_DEFAULT 128
2c7eea4c 1913#define NO_CMDLINE_MAP UINT_MAX
edc35bd7 1914static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
939c7a4f
YY
1915struct saved_cmdlines_buffer {
1916 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1917 unsigned *map_cmdline_to_pid;
1918 unsigned cmdline_num;
1919 int cmdline_idx;
1920 char *saved_cmdlines;
1921};
1922static struct saved_cmdlines_buffer *savedcmd;
25b0b44a 1923
25b0b44a 1924/* temporary disable recording */
d914ba37 1925static atomic_t trace_record_taskinfo_disabled __read_mostly;
bc0c38d1 1926
939c7a4f
YY
1927static inline char *get_saved_cmdlines(int idx)
1928{
1929 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1930}
1931
1932static inline void set_cmdline(int idx, const char *cmdline)
bc0c38d1 1933{
85f726a3 1934 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
939c7a4f
YY
1935}
1936
1937static int allocate_cmdlines_buffer(unsigned int val,
1938 struct saved_cmdlines_buffer *s)
1939{
6da2ec56
KC
1940 s->map_cmdline_to_pid = kmalloc_array(val,
1941 sizeof(*s->map_cmdline_to_pid),
1942 GFP_KERNEL);
939c7a4f
YY
1943 if (!s->map_cmdline_to_pid)
1944 return -ENOMEM;
1945
6da2ec56 1946 s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
939c7a4f
YY
1947 if (!s->saved_cmdlines) {
1948 kfree(s->map_cmdline_to_pid);
1949 return -ENOMEM;
1950 }
1951
1952 s->cmdline_idx = 0;
1953 s->cmdline_num = val;
1954 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1955 sizeof(s->map_pid_to_cmdline));
1956 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1957 val * sizeof(*s->map_cmdline_to_pid));
1958
1959 return 0;
1960}
1961
1962static int trace_create_savedcmd(void)
1963{
1964 int ret;
1965
a6af8fbf 1966 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
939c7a4f
YY
1967 if (!savedcmd)
1968 return -ENOMEM;
1969
1970 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1971 if (ret < 0) {
1972 kfree(savedcmd);
1973 savedcmd = NULL;
1974 return -ENOMEM;
1975 }
1976
1977 return 0;
bc0c38d1
SR
1978}
1979
b5130b1e
CE
1980int is_tracing_stopped(void)
1981{
2b6080f2 1982 return global_trace.stop_count;
b5130b1e
CE
1983}
1984
0f048701
SR
1985/**
1986 * tracing_start - quick start of the tracer
1987 *
1988 * If tracing is enabled but was stopped by tracing_stop,
1989 * this will start the tracer back up.
1990 */
1991void tracing_start(void)
1992{
1993 struct ring_buffer *buffer;
1994 unsigned long flags;
1995
1996 if (tracing_disabled)
1997 return;
1998
2b6080f2
SR
1999 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2000 if (--global_trace.stop_count) {
2001 if (global_trace.stop_count < 0) {
b06a8301
SR
2002 /* Someone screwed up their debugging */
2003 WARN_ON_ONCE(1);
2b6080f2 2004 global_trace.stop_count = 0;
b06a8301 2005 }
0f048701
SR
2006 goto out;
2007 }
2008
a2f80714 2009 /* Prevent the buffers from switching */
0b9b12c1 2010 arch_spin_lock(&global_trace.max_lock);
0f048701 2011
12883efb 2012 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
2013 if (buffer)
2014 ring_buffer_record_enable(buffer);
2015
12883efb
SRRH
2016#ifdef CONFIG_TRACER_MAX_TRACE
2017 buffer = global_trace.max_buffer.buffer;
0f048701
SR
2018 if (buffer)
2019 ring_buffer_record_enable(buffer);
12883efb 2020#endif
0f048701 2021
0b9b12c1 2022 arch_spin_unlock(&global_trace.max_lock);
a2f80714 2023
0f048701 2024 out:
2b6080f2
SR
2025 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2026}
2027
2028static void tracing_start_tr(struct trace_array *tr)
2029{
2030 struct ring_buffer *buffer;
2031 unsigned long flags;
2032
2033 if (tracing_disabled)
2034 return;
2035
2036 /* If global, we need to also start the max tracer */
2037 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2038 return tracing_start();
2039
2040 raw_spin_lock_irqsave(&tr->start_lock, flags);
2041
2042 if (--tr->stop_count) {
2043 if (tr->stop_count < 0) {
2044 /* Someone screwed up their debugging */
2045 WARN_ON_ONCE(1);
2046 tr->stop_count = 0;
2047 }
2048 goto out;
2049 }
2050
12883efb 2051 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
2052 if (buffer)
2053 ring_buffer_record_enable(buffer);
2054
2055 out:
2056 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
2057}
2058
2059/**
2060 * tracing_stop - quick stop of the tracer
2061 *
2062 * Light weight way to stop tracing. Use in conjunction with
2063 * tracing_start.
2064 */
2065void tracing_stop(void)
2066{
2067 struct ring_buffer *buffer;
2068 unsigned long flags;
2069
2b6080f2
SR
2070 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
2071 if (global_trace.stop_count++)
0f048701
SR
2072 goto out;
2073
a2f80714 2074 /* Prevent the buffers from switching */
0b9b12c1 2075 arch_spin_lock(&global_trace.max_lock);
a2f80714 2076
12883efb 2077 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
2078 if (buffer)
2079 ring_buffer_record_disable(buffer);
2080
12883efb
SRRH
2081#ifdef CONFIG_TRACER_MAX_TRACE
2082 buffer = global_trace.max_buffer.buffer;
0f048701
SR
2083 if (buffer)
2084 ring_buffer_record_disable(buffer);
12883efb 2085#endif
0f048701 2086
0b9b12c1 2087 arch_spin_unlock(&global_trace.max_lock);
a2f80714 2088
0f048701 2089 out:
2b6080f2
SR
2090 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
2091}
2092
2093static void tracing_stop_tr(struct trace_array *tr)
2094{
2095 struct ring_buffer *buffer;
2096 unsigned long flags;
2097
2098 /* If global, we need to also stop the max tracer */
2099 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
2100 return tracing_stop();
2101
2102 raw_spin_lock_irqsave(&tr->start_lock, flags);
2103 if (tr->stop_count++)
2104 goto out;
2105
12883efb 2106 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
2107 if (buffer)
2108 ring_buffer_record_disable(buffer);
2109
2110 out:
2111 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
2112}
2113
379cfdac 2114static int trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 2115{
a635cf04 2116 unsigned pid, idx;
bc0c38d1 2117
eaf260ac
JF
2118 /* treat recording of idle task as a success */
2119 if (!tsk->pid)
2120 return 1;
2121
2122 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
379cfdac 2123 return 0;
bc0c38d1
SR
2124
2125 /*
2126 * It's not the end of the world if we don't get
2127 * the lock, but we also don't want to spin
2128 * nor do we want to disable interrupts,
2129 * so if we miss here, then better luck next time.
2130 */
0199c4e6 2131 if (!arch_spin_trylock(&trace_cmdline_lock))
379cfdac 2132 return 0;
bc0c38d1 2133
939c7a4f 2134 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2c7eea4c 2135 if (idx == NO_CMDLINE_MAP) {
939c7a4f 2136 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
bc0c38d1 2137
a635cf04
CE
2138 /*
2139 * Check whether the cmdline buffer at idx has a pid
2140 * mapped. We are going to overwrite that entry so we
2141 * need to clear the map_pid_to_cmdline. Otherwise we
2142 * would read the new comm for the old pid.
2143 */
939c7a4f 2144 pid = savedcmd->map_cmdline_to_pid[idx];
a635cf04 2145 if (pid != NO_CMDLINE_MAP)
939c7a4f 2146 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 2147
939c7a4f
YY
2148 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
2149 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
bc0c38d1 2150
939c7a4f 2151 savedcmd->cmdline_idx = idx;
bc0c38d1
SR
2152 }
2153
939c7a4f 2154 set_cmdline(idx, tsk->comm);
bc0c38d1 2155
0199c4e6 2156 arch_spin_unlock(&trace_cmdline_lock);
379cfdac
SRRH
2157
2158 return 1;
bc0c38d1
SR
2159}
2160
4c27e756 2161static void __trace_find_cmdline(int pid, char comm[])
bc0c38d1 2162{
bc0c38d1
SR
2163 unsigned map;
2164
4ca53085
SR
2165 if (!pid) {
2166 strcpy(comm, "<idle>");
2167 return;
2168 }
bc0c38d1 2169
74bf4076
SR
2170 if (WARN_ON_ONCE(pid < 0)) {
2171 strcpy(comm, "<XXX>");
2172 return;
2173 }
2174
4ca53085
SR
2175 if (pid > PID_MAX_DEFAULT) {
2176 strcpy(comm, "<...>");
2177 return;
2178 }
bc0c38d1 2179
939c7a4f 2180 map = savedcmd->map_pid_to_cmdline[pid];
50d88758 2181 if (map != NO_CMDLINE_MAP)
e09e2867 2182 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
50d88758
TG
2183 else
2184 strcpy(comm, "<...>");
4c27e756
SRRH
2185}
2186
2187void trace_find_cmdline(int pid, char comm[])
2188{
2189 preempt_disable();
2190 arch_spin_lock(&trace_cmdline_lock);
2191
2192 __trace_find_cmdline(pid, comm);
bc0c38d1 2193
0199c4e6 2194 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 2195 preempt_enable();
bc0c38d1
SR
2196}
2197
d914ba37
JF
2198int trace_find_tgid(int pid)
2199{
2200 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2201 return 0;
2202
2203 return tgid_map[pid];
2204}
2205
2206static int trace_save_tgid(struct task_struct *tsk)
2207{
bd45d34d
JF
2208 /* treat recording of idle task as a success */
2209 if (!tsk->pid)
2210 return 1;
2211
2212 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
d914ba37
JF
2213 return 0;
2214
2215 tgid_map[tsk->pid] = tsk->tgid;
2216 return 1;
2217}
2218
2219static bool tracing_record_taskinfo_skip(int flags)
2220{
2221 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2222 return true;
2223 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2224 return true;
2225 if (!__this_cpu_read(trace_taskinfo_save))
2226 return true;
2227 return false;
2228}
2229
2230/**
2231 * tracing_record_taskinfo - record the task info of a task
2232 *
c68c9ec1
JK
2233 * @task: task to record
2234 * @flags: TRACE_RECORD_CMDLINE for recording comm
2235 * TRACE_RECORD_TGID for recording tgid
d914ba37
JF
2236 */
2237void tracing_record_taskinfo(struct task_struct *task, int flags)
2238{
29b1a8ad
JF
2239 bool done;
2240
d914ba37
JF
2241 if (tracing_record_taskinfo_skip(flags))
2242 return;
29b1a8ad
JF
2243
2244 /*
2245 * Record as much task information as possible. If some fail, continue
2246 * to try to record the others.
2247 */
2248 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2249 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2250
2251 /* If recording any information failed, retry again soon. */
2252 if (!done)
d914ba37
JF
2253 return;
2254
2255 __this_cpu_write(trace_taskinfo_save, false);
2256}
2257
2258/**
2259 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2260 *
c68c9ec1
JK
2261 * @prev: previous task during sched_switch
2262 * @next: next task during sched_switch
2263 * @flags: TRACE_RECORD_CMDLINE for recording comm
2264 * TRACE_RECORD_TGID for recording tgid
d914ba37
JF
2265 */
2266void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2267 struct task_struct *next, int flags)
bc0c38d1 2268{
29b1a8ad
JF
2269 bool done;
2270
d914ba37
JF
2271 if (tracing_record_taskinfo_skip(flags))
2272 return;
2273
29b1a8ad
JF
2274 /*
2275 * Record as much task information as possible. If some fail, continue
2276 * to try to record the others.
2277 */
2278 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2279 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2280 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2281 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
bc0c38d1 2282
29b1a8ad
JF
2283 /* If recording any information failed, retry again soon. */
2284 if (!done)
7ffbd48d
SR
2285 return;
2286
d914ba37
JF
2287 __this_cpu_write(trace_taskinfo_save, false);
2288}
2289
2290/* Helpers to record a specific task information */
2291void tracing_record_cmdline(struct task_struct *task)
2292{
2293 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2294}
2295
2296void tracing_record_tgid(struct task_struct *task)
2297{
2298 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
bc0c38d1
SR
2299}
2300
af0009fc
SRV
2301/*
2302 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2303 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2304 * simplifies those functions and keeps them in sync.
2305 */
2306enum print_line_t trace_handle_return(struct trace_seq *s)
2307{
2308 return trace_seq_has_overflowed(s) ?
2309 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2310}
2311EXPORT_SYMBOL_GPL(trace_handle_return);
2312
45dcd8b8 2313void
46710f3a
CW
2314tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
2315 unsigned long flags, int pc)
bc0c38d1
SR
2316{
2317 struct task_struct *tsk = current;
bc0c38d1 2318
777e208d
SR
2319 entry->preempt_count = pc & 0xff;
2320 entry->pid = (tsk) ? tsk->pid : 0;
46710f3a 2321 entry->type = type;
777e208d 2322 entry->flags =
9244489a 2323#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 2324 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
2325#else
2326 TRACE_FLAG_IRQS_NOSUPPORT |
2327#endif
7e6867bf 2328 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
bc0c38d1 2329 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
c59f29cb 2330 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
e5137b50
PZ
2331 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2332 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
bc0c38d1 2333}
f413cdb8 2334EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 2335
e77405ad
SR
2336struct ring_buffer_event *
2337trace_buffer_lock_reserve(struct ring_buffer *buffer,
2338 int type,
2339 unsigned long len,
2340 unsigned long flags, int pc)
51a763dd 2341{
3e9a8aad 2342 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
0fc1b09f
SRRH
2343}
2344
2345DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2346DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2347static int trace_buffered_event_ref;
2348
2349/**
2350 * trace_buffered_event_enable - enable buffering events
2351 *
2352 * When events are being filtered, it is quicker to use a temporary
2353 * buffer to write the event data into if there's a likely chance
2354 * that it will not be committed. The discard of the ring buffer
2355 * is not as fast as committing, and is much slower than copying
2356 * a commit.
2357 *
2358 * When an event is to be filtered, allocate per cpu buffers to
2359 * write the event data into, and if the event is filtered and discarded
2360 * it is simply dropped, otherwise, the entire data is to be committed
2361 * in one shot.
2362 */
2363void trace_buffered_event_enable(void)
2364{
2365 struct ring_buffer_event *event;
2366 struct page *page;
2367 int cpu;
51a763dd 2368
0fc1b09f
SRRH
2369 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2370
2371 if (trace_buffered_event_ref++)
2372 return;
2373
2374 for_each_tracing_cpu(cpu) {
2375 page = alloc_pages_node(cpu_to_node(cpu),
2376 GFP_KERNEL | __GFP_NORETRY, 0);
2377 if (!page)
2378 goto failed;
2379
2380 event = page_address(page);
2381 memset(event, 0, sizeof(*event));
2382
2383 per_cpu(trace_buffered_event, cpu) = event;
2384
2385 preempt_disable();
2386 if (cpu == smp_processor_id() &&
2387 this_cpu_read(trace_buffered_event) !=
2388 per_cpu(trace_buffered_event, cpu))
2389 WARN_ON_ONCE(1);
2390 preempt_enable();
51a763dd
ACM
2391 }
2392
0fc1b09f
SRRH
2393 return;
2394 failed:
2395 trace_buffered_event_disable();
2396}
2397
2398static void enable_trace_buffered_event(void *data)
2399{
2400 /* Probably not needed, but do it anyway */
2401 smp_rmb();
2402 this_cpu_dec(trace_buffered_event_cnt);
2403}
2404
2405static void disable_trace_buffered_event(void *data)
2406{
2407 this_cpu_inc(trace_buffered_event_cnt);
2408}
2409
2410/**
2411 * trace_buffered_event_disable - disable buffering events
2412 *
2413 * When a filter is removed, it is faster to not use the buffered
2414 * events, and to commit directly into the ring buffer. Free up
2415 * the temp buffers when there are no more users. This requires
2416 * special synchronization with current events.
2417 */
2418void trace_buffered_event_disable(void)
2419{
2420 int cpu;
2421
2422 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2423
2424 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2425 return;
2426
2427 if (--trace_buffered_event_ref)
2428 return;
2429
2430 preempt_disable();
2431 /* For each CPU, set the buffer as used. */
2432 smp_call_function_many(tracing_buffer_mask,
2433 disable_trace_buffered_event, NULL, 1);
2434 preempt_enable();
2435
2436 /* Wait for all current users to finish */
74401729 2437 synchronize_rcu();
0fc1b09f
SRRH
2438
2439 for_each_tracing_cpu(cpu) {
2440 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2441 per_cpu(trace_buffered_event, cpu) = NULL;
2442 }
2443 /*
2444 * Make sure trace_buffered_event is NULL before clearing
2445 * trace_buffered_event_cnt.
2446 */
2447 smp_wmb();
2448
2449 preempt_disable();
2450 /* Do the work on each cpu */
2451 smp_call_function_many(tracing_buffer_mask,
2452 enable_trace_buffered_event, NULL, 1);
2453 preempt_enable();
51a763dd 2454}
51a763dd 2455
2c4a33ab
SRRH
2456static struct ring_buffer *temp_buffer;
2457
ccb469a1
SR
2458struct ring_buffer_event *
2459trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
7f1d2f82 2460 struct trace_event_file *trace_file,
ccb469a1
SR
2461 int type, unsigned long len,
2462 unsigned long flags, int pc)
2463{
2c4a33ab 2464 struct ring_buffer_event *entry;
0fc1b09f 2465 int val;
2c4a33ab 2466
7f1d2f82 2467 *current_rb = trace_file->tr->trace_buffer.buffer;
0fc1b09f 2468
00b41452 2469 if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
0fc1b09f
SRRH
2470 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2471 (entry = this_cpu_read(trace_buffered_event))) {
2472 /* Try to use the per cpu buffer first */
2473 val = this_cpu_inc_return(trace_buffered_event_cnt);
2474 if (val == 1) {
2475 trace_event_setup(entry, type, flags, pc);
2476 entry->array[0] = len;
2477 return entry;
2478 }
2479 this_cpu_dec(trace_buffered_event_cnt);
2480 }
2481
3e9a8aad
SRRH
2482 entry = __trace_buffer_lock_reserve(*current_rb,
2483 type, len, flags, pc);
2c4a33ab
SRRH
2484 /*
2485 * If tracing is off, but we have triggers enabled
2486 * we still need to look at the event data. Use the temp_buffer
2487 * to store the trace event for the tigger to use. It's recusive
2488 * safe and will not be recorded anywhere.
2489 */
5d6ad960 2490 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2c4a33ab 2491 *current_rb = temp_buffer;
3e9a8aad
SRRH
2492 entry = __trace_buffer_lock_reserve(*current_rb,
2493 type, len, flags, pc);
2c4a33ab
SRRH
2494 }
2495 return entry;
ccb469a1
SR
2496}
2497EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2498
42391745
SRRH
2499static DEFINE_SPINLOCK(tracepoint_iter_lock);
2500static DEFINE_MUTEX(tracepoint_printk_mutex);
2501
2502static void output_printk(struct trace_event_buffer *fbuffer)
2503{
2504 struct trace_event_call *event_call;
2505 struct trace_event *event;
2506 unsigned long flags;
2507 struct trace_iterator *iter = tracepoint_print_iter;
2508
2509 /* We should never get here if iter is NULL */
2510 if (WARN_ON_ONCE(!iter))
2511 return;
2512
2513 event_call = fbuffer->trace_file->event_call;
2514 if (!event_call || !event_call->event.funcs ||
2515 !event_call->event.funcs->trace)
2516 return;
2517
2518 event = &fbuffer->trace_file->event_call->event;
2519
2520 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2521 trace_seq_init(&iter->seq);
2522 iter->ent = fbuffer->entry;
2523 event_call->event.funcs->trace(iter, 0, event);
2524 trace_seq_putc(&iter->seq, 0);
2525 printk("%s", iter->seq.buffer);
2526
2527 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2528}
2529
2530int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2531 void __user *buffer, size_t *lenp,
2532 loff_t *ppos)
2533{
2534 int save_tracepoint_printk;
2535 int ret;
2536
2537 mutex_lock(&tracepoint_printk_mutex);
2538 save_tracepoint_printk = tracepoint_printk;
2539
2540 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2541
2542 /*
2543 * This will force exiting early, as tracepoint_printk
2544 * is always zero when tracepoint_printk_iter is not allocated
2545 */
2546 if (!tracepoint_print_iter)
2547 tracepoint_printk = 0;
2548
2549 if (save_tracepoint_printk == tracepoint_printk)
2550 goto out;
2551
2552 if (tracepoint_printk)
2553 static_key_enable(&tracepoint_printk_key.key);
2554 else
2555 static_key_disable(&tracepoint_printk_key.key);
2556
2557 out:
2558 mutex_unlock(&tracepoint_printk_mutex);
2559
2560 return ret;
2561}
2562
2563void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2564{
2565 if (static_key_false(&tracepoint_printk_key.key))
2566 output_printk(fbuffer);
2567
2568 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2569 fbuffer->event, fbuffer->entry,
2570 fbuffer->flags, fbuffer->pc);
2571}
2572EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2573
2ee5b92a
SRV
2574/*
2575 * Skip 3:
2576 *
2577 * trace_buffer_unlock_commit_regs()
2578 * trace_event_buffer_commit()
2579 * trace_event_raw_event_xxx()
13cf912b 2580 */
2ee5b92a
SRV
2581# define STACK_SKIP 3
2582
b7f0c959
SRRH
2583void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2584 struct ring_buffer *buffer,
0d5c6e1c
SR
2585 struct ring_buffer_event *event,
2586 unsigned long flags, int pc,
2587 struct pt_regs *regs)
1fd8df2c 2588{
7ffbd48d 2589 __buffer_unlock_commit(buffer, event);
1fd8df2c 2590
be54f69c 2591 /*
2ee5b92a 2592 * If regs is not set, then skip the necessary functions.
be54f69c
SRRH
2593 * Note, we can still get here via blktrace, wakeup tracer
2594 * and mmiotrace, but that's ok if they lose a function or
2ee5b92a 2595 * two. They are not that meaningful.
be54f69c 2596 */
2ee5b92a 2597 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
1fd8df2c
MH
2598 ftrace_trace_userstack(buffer, flags, pc);
2599}
1fd8df2c 2600
52ffabe3
SRRH
2601/*
2602 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2603 */
2604void
2605trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2606 struct ring_buffer_event *event)
2607{
2608 __buffer_unlock_commit(buffer, event);
2609}
2610
478409dd
CZ
2611static void
2612trace_process_export(struct trace_export *export,
2613 struct ring_buffer_event *event)
2614{
2615 struct trace_entry *entry;
2616 unsigned int size = 0;
2617
2618 entry = ring_buffer_event_data(event);
2619 size = ring_buffer_event_length(event);
a773d419 2620 export->write(export, entry, size);
478409dd
CZ
2621}
2622
2623static DEFINE_MUTEX(ftrace_export_lock);
2624
2625static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2626
2627static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2628
2629static inline void ftrace_exports_enable(void)
2630{
2631 static_branch_enable(&ftrace_exports_enabled);
2632}
2633
2634static inline void ftrace_exports_disable(void)
2635{
2636 static_branch_disable(&ftrace_exports_enabled);
2637}
2638
1cce377d 2639static void ftrace_exports(struct ring_buffer_event *event)
478409dd
CZ
2640{
2641 struct trace_export *export;
2642
2643 preempt_disable_notrace();
2644
2645 export = rcu_dereference_raw_notrace(ftrace_exports_list);
2646 while (export) {
2647 trace_process_export(export, event);
2648 export = rcu_dereference_raw_notrace(export->next);
2649 }
2650
2651 preempt_enable_notrace();
2652}
2653
2654static inline void
2655add_trace_export(struct trace_export **list, struct trace_export *export)
2656{
2657 rcu_assign_pointer(export->next, *list);
2658 /*
2659 * We are entering export into the list but another
2660 * CPU might be walking that list. We need to make sure
2661 * the export->next pointer is valid before another CPU sees
2662 * the export pointer included into the list.
2663 */
2664 rcu_assign_pointer(*list, export);
2665}
2666
2667static inline int
2668rm_trace_export(struct trace_export **list, struct trace_export *export)
2669{
2670 struct trace_export **p;
2671
2672 for (p = list; *p != NULL; p = &(*p)->next)
2673 if (*p == export)
2674 break;
2675
2676 if (*p != export)
2677 return -1;
2678
2679 rcu_assign_pointer(*p, (*p)->next);
2680
2681 return 0;
2682}
2683
2684static inline void
2685add_ftrace_export(struct trace_export **list, struct trace_export *export)
2686{
2687 if (*list == NULL)
2688 ftrace_exports_enable();
2689
2690 add_trace_export(list, export);
2691}
2692
2693static inline int
2694rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2695{
2696 int ret;
2697
2698 ret = rm_trace_export(list, export);
2699 if (*list == NULL)
2700 ftrace_exports_disable();
2701
2702 return ret;
2703}
2704
2705int register_ftrace_export(struct trace_export *export)
2706{
2707 if (WARN_ON_ONCE(!export->write))
2708 return -1;
2709
2710 mutex_lock(&ftrace_export_lock);
2711
2712 add_ftrace_export(&ftrace_exports_list, export);
2713
2714 mutex_unlock(&ftrace_export_lock);
2715
2716 return 0;
2717}
2718EXPORT_SYMBOL_GPL(register_ftrace_export);
2719
2720int unregister_ftrace_export(struct trace_export *export)
2721{
2722 int ret;
2723
2724 mutex_lock(&ftrace_export_lock);
2725
2726 ret = rm_ftrace_export(&ftrace_exports_list, export);
2727
2728 mutex_unlock(&ftrace_export_lock);
2729
2730 return ret;
2731}
2732EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2733
e309b41d 2734void
7be42151 2735trace_function(struct trace_array *tr,
38697053
SR
2736 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2737 int pc)
bc0c38d1 2738{
2425bcb9 2739 struct trace_event_call *call = &event_function;
12883efb 2740 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 2741 struct ring_buffer_event *event;
777e208d 2742 struct ftrace_entry *entry;
bc0c38d1 2743
3e9a8aad
SRRH
2744 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2745 flags, pc);
3928a8a2
SR
2746 if (!event)
2747 return;
2748 entry = ring_buffer_event_data(event);
777e208d
SR
2749 entry->ip = ip;
2750 entry->parent_ip = parent_ip;
e1112b4d 2751
478409dd
CZ
2752 if (!call_filter_check_discard(call, entry, buffer, event)) {
2753 if (static_branch_unlikely(&ftrace_exports_enabled))
2754 ftrace_exports(event);
7ffbd48d 2755 __buffer_unlock_commit(buffer, event);
478409dd 2756 }
bc0c38d1
SR
2757}
2758
c0a0d0d3 2759#ifdef CONFIG_STACKTRACE
4a9bd3f1 2760
2a820bf7
TG
2761/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2762#define FTRACE_KSTACK_NESTING 4
2763
2764#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2765
4a9bd3f1 2766struct ftrace_stack {
2a820bf7 2767 unsigned long calls[FTRACE_KSTACK_ENTRIES];
4a9bd3f1
SR
2768};
2769
2a820bf7
TG
2770
2771struct ftrace_stacks {
2772 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
4a9bd3f1
SR
2773};
2774
2a820bf7 2775static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
4a9bd3f1
SR
2776static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2777
e77405ad 2778static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 2779 unsigned long flags,
1fd8df2c 2780 int skip, int pc, struct pt_regs *regs)
86387f7e 2781{
2425bcb9 2782 struct trace_event_call *call = &event_kernel_stack;
3928a8a2 2783 struct ring_buffer_event *event;
ee6dd0db 2784 unsigned int size, nr_entries;
2a820bf7 2785 struct ftrace_stack *fstack;
777e208d 2786 struct stack_entry *entry;
2a820bf7 2787 int stackidx;
4a9bd3f1 2788
be54f69c 2789 /*
2ee5b92a 2790 * Add one, for this function and the call to save_stack_trace()
be54f69c
SRRH
2791 * If regs is set, then these functions will not be in the way.
2792 */
2ee5b92a 2793#ifndef CONFIG_UNWINDER_ORC
be54f69c 2794 if (!regs)
ee6dd0db 2795 skip++;
2ee5b92a 2796#endif
be54f69c 2797
4a9bd3f1
SR
2798 /*
2799 * Since events can happen in NMIs there's no safe way to
2800 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2801 * or NMI comes in, it will just have to use the default
2802 * FTRACE_STACK_SIZE.
2803 */
2804 preempt_disable_notrace();
2805
2a820bf7
TG
2806 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2807
2808 /* This should never happen. If it does, yell once and skip */
2809 if (WARN_ON_ONCE(stackidx > FTRACE_KSTACK_NESTING))
2810 goto out;
2811
4a9bd3f1 2812 /*
2a820bf7
TG
2813 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2814 * interrupt will either see the value pre increment or post
2815 * increment. If the interrupt happens pre increment it will have
2816 * restored the counter when it returns. We just need a barrier to
2817 * keep gcc from moving things around.
4a9bd3f1
SR
2818 */
2819 barrier();
4a9bd3f1 2820
2a820bf7 2821 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
ee6dd0db 2822 size = ARRAY_SIZE(fstack->calls);
4a9bd3f1 2823
ee6dd0db
TG
2824 if (regs) {
2825 nr_entries = stack_trace_save_regs(regs, fstack->calls,
2826 size, skip);
2827 } else {
2828 nr_entries = stack_trace_save(fstack->calls, size, skip);
2829 }
86387f7e 2830
ee6dd0db 2831 size = nr_entries * sizeof(unsigned long);
3e9a8aad
SRRH
2832 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2833 sizeof(*entry) + size, flags, pc);
3928a8a2 2834 if (!event)
4a9bd3f1
SR
2835 goto out;
2836 entry = ring_buffer_event_data(event);
86387f7e 2837
ee6dd0db
TG
2838 memcpy(&entry->caller, fstack->calls, size);
2839 entry->size = nr_entries;
86387f7e 2840
f306cc82 2841 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2842 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
2843
2844 out:
2845 /* Again, don't let gcc optimize things here */
2846 barrier();
82146529 2847 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
2848 preempt_enable_notrace();
2849
f0a920d5
IM
2850}
2851
2d34f489
SRRH
2852static inline void ftrace_trace_stack(struct trace_array *tr,
2853 struct ring_buffer *buffer,
73dddbb5
SRRH
2854 unsigned long flags,
2855 int skip, int pc, struct pt_regs *regs)
53614991 2856{
2d34f489 2857 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
53614991
SR
2858 return;
2859
73dddbb5 2860 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
53614991
SR
2861}
2862
c0a0d0d3
FW
2863void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2864 int pc)
38697053 2865{
a33d7d94
SRV
2866 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2867
2868 if (rcu_is_watching()) {
2869 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2870 return;
2871 }
2872
2873 /*
2874 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2875 * but if the above rcu_is_watching() failed, then the NMI
2876 * triggered someplace critical, and rcu_irq_enter() should
2877 * not be called from NMI.
2878 */
2879 if (unlikely(in_nmi()))
2880 return;
2881
a33d7d94
SRV
2882 rcu_irq_enter_irqson();
2883 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2884 rcu_irq_exit_irqson();
38697053
SR
2885}
2886
03889384
SR
2887/**
2888 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 2889 * @skip: Number of functions to skip (helper handlers)
03889384 2890 */
c142be8e 2891void trace_dump_stack(int skip)
03889384
SR
2892{
2893 unsigned long flags;
2894
2895 if (tracing_disabled || tracing_selftest_running)
e36c5458 2896 return;
03889384
SR
2897
2898 local_save_flags(flags);
2899
2ee5b92a
SRV
2900#ifndef CONFIG_UNWINDER_ORC
2901 /* Skip 1 to skip this function. */
2902 skip++;
2903#endif
c142be8e
SRRH
2904 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2905 flags, skip, preempt_count(), NULL);
03889384 2906}
da387e5c 2907EXPORT_SYMBOL_GPL(trace_dump_stack);
03889384 2908
c438f140 2909#ifdef CONFIG_USER_STACKTRACE_SUPPORT
91e86e56
SR
2910static DEFINE_PER_CPU(int, user_stack_count);
2911
c438f140 2912static void
e77405ad 2913ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 2914{
2425bcb9 2915 struct trace_event_call *call = &event_user_stack;
8d7c6a96 2916 struct ring_buffer_event *event;
02b67518 2917 struct userstack_entry *entry;
02b67518 2918
983f938a 2919 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
02b67518
TE
2920 return;
2921
b6345879
SR
2922 /*
2923 * NMIs can not handle page faults, even with fix ups.
2924 * The save user stack can (and often does) fault.
2925 */
2926 if (unlikely(in_nmi()))
2927 return;
02b67518 2928
91e86e56
SR
2929 /*
2930 * prevent recursion, since the user stack tracing may
2931 * trigger other kernel events.
2932 */
2933 preempt_disable();
2934 if (__this_cpu_read(user_stack_count))
2935 goto out;
2936
2937 __this_cpu_inc(user_stack_count);
2938
3e9a8aad
SRRH
2939 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2940 sizeof(*entry), flags, pc);
02b67518 2941 if (!event)
1dbd1951 2942 goto out_drop_count;
02b67518 2943 entry = ring_buffer_event_data(event);
02b67518 2944
48659d31 2945 entry->tgid = current->tgid;
02b67518
TE
2946 memset(&entry->caller, 0, sizeof(entry->caller));
2947
ee6dd0db 2948 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
f306cc82 2949 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2950 __buffer_unlock_commit(buffer, event);
91e86e56 2951
1dbd1951 2952 out_drop_count:
91e86e56 2953 __this_cpu_dec(user_stack_count);
91e86e56
SR
2954 out:
2955 preempt_enable();
02b67518 2956}
c438f140
TG
2957#else /* CONFIG_USER_STACKTRACE_SUPPORT */
2958static void ftrace_trace_userstack(struct ring_buffer *buffer,
2959 unsigned long flags, int pc)
02b67518 2960{
02b67518 2961}
c438f140 2962#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
02b67518 2963
c0a0d0d3
FW
2964#endif /* CONFIG_STACKTRACE */
2965
07d777fe
SR
2966/* created for use with alloc_percpu */
2967struct trace_buffer_struct {
e2ace001
AL
2968 int nesting;
2969 char buffer[4][TRACE_BUF_SIZE];
07d777fe
SR
2970};
2971
2972static struct trace_buffer_struct *trace_percpu_buffer;
07d777fe
SR
2973
2974/*
e2ace001
AL
2975 * Thise allows for lockless recording. If we're nested too deeply, then
2976 * this returns NULL.
07d777fe
SR
2977 */
2978static char *get_trace_buf(void)
2979{
e2ace001 2980 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
07d777fe 2981
e2ace001 2982 if (!buffer || buffer->nesting >= 4)
07d777fe
SR
2983 return NULL;
2984
3d9622c1
SRV
2985 buffer->nesting++;
2986
2987 /* Interrupts must see nesting incremented before we use the buffer */
2988 barrier();
2989 return &buffer->buffer[buffer->nesting][0];
e2ace001
AL
2990}
2991
2992static void put_trace_buf(void)
2993{
3d9622c1
SRV
2994 /* Don't let the decrement of nesting leak before this */
2995 barrier();
e2ace001 2996 this_cpu_dec(trace_percpu_buffer->nesting);
07d777fe
SR
2997}
2998
2999static int alloc_percpu_trace_buffer(void)
3000{
3001 struct trace_buffer_struct *buffers;
07d777fe
SR
3002
3003 buffers = alloc_percpu(struct trace_buffer_struct);
e2ace001
AL
3004 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
3005 return -ENOMEM;
07d777fe
SR
3006
3007 trace_percpu_buffer = buffers;
07d777fe 3008 return 0;
07d777fe
SR
3009}
3010
81698831
SR
3011static int buffers_allocated;
3012
07d777fe
SR
3013void trace_printk_init_buffers(void)
3014{
07d777fe
SR
3015 if (buffers_allocated)
3016 return;
3017
3018 if (alloc_percpu_trace_buffer())
3019 return;
3020
2184db46
SR
3021 /* trace_printk() is for debug use only. Don't use it in production. */
3022
a395d6a7
JP
3023 pr_warn("\n");
3024 pr_warn("**********************************************************\n");
3025 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3026 pr_warn("** **\n");
3027 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3028 pr_warn("** **\n");
3029 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3030 pr_warn("** unsafe for production use. **\n");
3031 pr_warn("** **\n");
3032 pr_warn("** If you see this message and you are not debugging **\n");
3033 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3034 pr_warn("** **\n");
3035 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3036 pr_warn("**********************************************************\n");
07d777fe 3037
b382ede6
SR
3038 /* Expand the buffers to set size */
3039 tracing_update_buffers();
3040
07d777fe 3041 buffers_allocated = 1;
81698831
SR
3042
3043 /*
3044 * trace_printk_init_buffers() can be called by modules.
3045 * If that happens, then we need to start cmdline recording
3046 * directly here. If the global_trace.buffer is already
3047 * allocated here, then this was called by module code.
3048 */
12883efb 3049 if (global_trace.trace_buffer.buffer)
81698831
SR
3050 tracing_start_cmdline_record();
3051}
f45d1225 3052EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
81698831
SR
3053
3054void trace_printk_start_comm(void)
3055{
3056 /* Start tracing comms if trace printk is set */
3057 if (!buffers_allocated)
3058 return;
3059 tracing_start_cmdline_record();
3060}
3061
3062static void trace_printk_start_stop_comm(int enabled)
3063{
3064 if (!buffers_allocated)
3065 return;
3066
3067 if (enabled)
3068 tracing_start_cmdline_record();
3069 else
3070 tracing_stop_cmdline_record();
07d777fe
SR
3071}
3072
769b0441 3073/**
48ead020 3074 * trace_vbprintk - write binary msg to tracing buffer
c68c9ec1
JK
3075 * @ip: The address of the caller
3076 * @fmt: The string format to write to the buffer
3077 * @args: Arguments for @fmt
769b0441 3078 */
40ce74f1 3079int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 3080{
2425bcb9 3081 struct trace_event_call *call = &event_bprint;
769b0441 3082 struct ring_buffer_event *event;
e77405ad 3083 struct ring_buffer *buffer;
769b0441 3084 struct trace_array *tr = &global_trace;
48ead020 3085 struct bprint_entry *entry;
769b0441 3086 unsigned long flags;
07d777fe
SR
3087 char *tbuffer;
3088 int len = 0, size, pc;
769b0441
FW
3089
3090 if (unlikely(tracing_selftest_running || tracing_disabled))
3091 return 0;
3092
3093 /* Don't pollute graph traces with trace_vprintk internals */
3094 pause_graph_tracing();
3095
3096 pc = preempt_count();
5168ae50 3097 preempt_disable_notrace();
769b0441 3098
07d777fe
SR
3099 tbuffer = get_trace_buf();
3100 if (!tbuffer) {
3101 len = 0;
e2ace001 3102 goto out_nobuffer;
07d777fe 3103 }
769b0441 3104
07d777fe 3105 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 3106
07d777fe
SR
3107 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3108 goto out;
769b0441 3109
07d777fe 3110 local_save_flags(flags);
769b0441 3111 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 3112 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
3113 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
3114 flags, pc);
769b0441 3115 if (!event)
07d777fe 3116 goto out;
769b0441
FW
3117 entry = ring_buffer_event_data(event);
3118 entry->ip = ip;
769b0441
FW
3119 entry->fmt = fmt;
3120
07d777fe 3121 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
f306cc82 3122 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 3123 __buffer_unlock_commit(buffer, event);
2d34f489 3124 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
d931369b 3125 }
769b0441 3126
769b0441 3127out:
e2ace001
AL
3128 put_trace_buf();
3129
3130out_nobuffer:
5168ae50 3131 preempt_enable_notrace();
769b0441
FW
3132 unpause_graph_tracing();
3133
3134 return len;
3135}
48ead020
FW
3136EXPORT_SYMBOL_GPL(trace_vbprintk);
3137
26b68dd2 3138__printf(3, 0)
12883efb
SRRH
3139static int
3140__trace_array_vprintk(struct ring_buffer *buffer,
3141 unsigned long ip, const char *fmt, va_list args)
48ead020 3142{
2425bcb9 3143 struct trace_event_call *call = &event_print;
48ead020 3144 struct ring_buffer_event *event;
07d777fe 3145 int len = 0, size, pc;
48ead020 3146 struct print_entry *entry;
07d777fe
SR
3147 unsigned long flags;
3148 char *tbuffer;
48ead020
FW
3149
3150 if (tracing_disabled || tracing_selftest_running)
3151 return 0;
3152
07d777fe
SR
3153 /* Don't pollute graph traces with trace_vprintk internals */
3154 pause_graph_tracing();
3155
48ead020
FW
3156 pc = preempt_count();
3157 preempt_disable_notrace();
48ead020 3158
07d777fe
SR
3159
3160 tbuffer = get_trace_buf();
3161 if (!tbuffer) {
3162 len = 0;
e2ace001 3163 goto out_nobuffer;
07d777fe 3164 }
48ead020 3165
3558a5ac 3166 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
48ead020 3167
07d777fe 3168 local_save_flags(flags);
48ead020 3169 size = sizeof(*entry) + len + 1;
3e9a8aad
SRRH
3170 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3171 flags, pc);
48ead020 3172 if (!event)
07d777fe 3173 goto out;
48ead020 3174 entry = ring_buffer_event_data(event);
c13d2f7c 3175 entry->ip = ip;
48ead020 3176
3558a5ac 3177 memcpy(&entry->buf, tbuffer, len + 1);
f306cc82 3178 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 3179 __buffer_unlock_commit(buffer, event);
2d34f489 3180 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
d931369b 3181 }
e2ace001
AL
3182
3183out:
3184 put_trace_buf();
3185
3186out_nobuffer:
48ead020 3187 preempt_enable_notrace();
07d777fe 3188 unpause_graph_tracing();
48ead020
FW
3189
3190 return len;
3191}
659372d3 3192
26b68dd2 3193__printf(3, 0)
12883efb
SRRH
3194int trace_array_vprintk(struct trace_array *tr,
3195 unsigned long ip, const char *fmt, va_list args)
3196{
3197 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3198}
3199
26b68dd2 3200__printf(3, 0)
12883efb
SRRH
3201int trace_array_printk(struct trace_array *tr,
3202 unsigned long ip, const char *fmt, ...)
3203{
3204 int ret;
3205 va_list ap;
3206
983f938a 3207 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
3208 return 0;
3209
3210 va_start(ap, fmt);
3211 ret = trace_array_vprintk(tr, ip, fmt, ap);
3212 va_end(ap);
3213 return ret;
3214}
f45d1225 3215EXPORT_SYMBOL_GPL(trace_array_printk);
12883efb 3216
26b68dd2 3217__printf(3, 4)
12883efb
SRRH
3218int trace_array_printk_buf(struct ring_buffer *buffer,
3219 unsigned long ip, const char *fmt, ...)
3220{
3221 int ret;
3222 va_list ap;
3223
983f938a 3224 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
3225 return 0;
3226
3227 va_start(ap, fmt);
3228 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3229 va_end(ap);
3230 return ret;
3231}
3232
26b68dd2 3233__printf(2, 0)
659372d3
SR
3234int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3235{
a813a159 3236 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 3237}
769b0441
FW
3238EXPORT_SYMBOL_GPL(trace_vprintk);
3239
e2ac8ef5 3240static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 3241{
6d158a81
SR
3242 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3243
5a90f577 3244 iter->idx++;
6d158a81
SR
3245 if (buf_iter)
3246 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
3247}
3248
e309b41d 3249static struct trace_entry *
bc21b478
SR
3250peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3251 unsigned long *lost_events)
dd0e545f 3252{
3928a8a2 3253 struct ring_buffer_event *event;
6d158a81 3254 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 3255
d769041f
SR
3256 if (buf_iter)
3257 event = ring_buffer_iter_peek(buf_iter, ts);
3258 else
12883efb 3259 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 3260 lost_events);
d769041f 3261
4a9bd3f1
SR
3262 if (event) {
3263 iter->ent_size = ring_buffer_event_length(event);
3264 return ring_buffer_event_data(event);
3265 }
3266 iter->ent_size = 0;
3267 return NULL;
dd0e545f 3268}
d769041f 3269
dd0e545f 3270static struct trace_entry *
bc21b478
SR
3271__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3272 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 3273{
12883efb 3274 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 3275 struct trace_entry *ent, *next = NULL;
aa27497c 3276 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 3277 int cpu_file = iter->cpu_file;
3928a8a2 3278 u64 next_ts = 0, ts;
bc0c38d1 3279 int next_cpu = -1;
12b5da34 3280 int next_size = 0;
bc0c38d1
SR
3281 int cpu;
3282
b04cc6b1
FW
3283 /*
3284 * If we are in a per_cpu trace file, don't bother by iterating over
3285 * all cpu and peek directly.
3286 */
ae3b5093 3287 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
3288 if (ring_buffer_empty_cpu(buffer, cpu_file))
3289 return NULL;
bc21b478 3290 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
3291 if (ent_cpu)
3292 *ent_cpu = cpu_file;
3293
3294 return ent;
3295 }
3296
ab46428c 3297 for_each_tracing_cpu(cpu) {
dd0e545f 3298
3928a8a2
SR
3299 if (ring_buffer_empty_cpu(buffer, cpu))
3300 continue;
dd0e545f 3301
bc21b478 3302 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 3303
cdd31cd2
IM
3304 /*
3305 * Pick the entry with the smallest timestamp:
3306 */
3928a8a2 3307 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
3308 next = ent;
3309 next_cpu = cpu;
3928a8a2 3310 next_ts = ts;
bc21b478 3311 next_lost = lost_events;
12b5da34 3312 next_size = iter->ent_size;
bc0c38d1
SR
3313 }
3314 }
3315
12b5da34
SR
3316 iter->ent_size = next_size;
3317
bc0c38d1
SR
3318 if (ent_cpu)
3319 *ent_cpu = next_cpu;
3320
3928a8a2
SR
3321 if (ent_ts)
3322 *ent_ts = next_ts;
3323
bc21b478
SR
3324 if (missing_events)
3325 *missing_events = next_lost;
3326
bc0c38d1
SR
3327 return next;
3328}
3329
dd0e545f 3330/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
3331struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3332 int *ent_cpu, u64 *ent_ts)
bc0c38d1 3333{
bc21b478 3334 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
3335}
3336
3337/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 3338void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 3339{
bc21b478
SR
3340 iter->ent = __find_next_entry(iter, &iter->cpu,
3341 &iter->lost_events, &iter->ts);
dd0e545f 3342
3928a8a2 3343 if (iter->ent)
e2ac8ef5 3344 trace_iterator_increment(iter);
dd0e545f 3345
3928a8a2 3346 return iter->ent ? iter : NULL;
b3806b43 3347}
bc0c38d1 3348
e309b41d 3349static void trace_consume(struct trace_iterator *iter)
b3806b43 3350{
12883efb 3351 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 3352 &iter->lost_events);
bc0c38d1
SR
3353}
3354
e309b41d 3355static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
3356{
3357 struct trace_iterator *iter = m->private;
bc0c38d1 3358 int i = (int)*pos;
4e3c3333 3359 void *ent;
bc0c38d1 3360
a63ce5b3
SR
3361 WARN_ON_ONCE(iter->leftover);
3362
bc0c38d1
SR
3363 (*pos)++;
3364
3365 /* can't go backwards */
3366 if (iter->idx > i)
3367 return NULL;
3368
3369 if (iter->idx < 0)
955b61e5 3370 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
3371 else
3372 ent = iter;
3373
3374 while (ent && iter->idx < i)
955b61e5 3375 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
3376
3377 iter->pos = *pos;
3378
bc0c38d1
SR
3379 return ent;
3380}
3381
955b61e5 3382void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 3383{
2f26ebd5
SR
3384 struct ring_buffer_event *event;
3385 struct ring_buffer_iter *buf_iter;
3386 unsigned long entries = 0;
3387 u64 ts;
3388
12883efb 3389 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 3390
6d158a81
SR
3391 buf_iter = trace_buffer_iter(iter, cpu);
3392 if (!buf_iter)
2f26ebd5
SR
3393 return;
3394
2f26ebd5
SR
3395 ring_buffer_iter_reset(buf_iter);
3396
3397 /*
3398 * We could have the case with the max latency tracers
3399 * that a reset never took place on a cpu. This is evident
3400 * by the timestamp being before the start of the buffer.
3401 */
3402 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 3403 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
3404 break;
3405 entries++;
3406 ring_buffer_read(buf_iter, NULL);
3407 }
3408
12883efb 3409 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
3410}
3411
d7350c3f 3412/*
d7350c3f
FW
3413 * The current tracer is copied to avoid a global locking
3414 * all around.
3415 */
bc0c38d1
SR
3416static void *s_start(struct seq_file *m, loff_t *pos)
3417{
3418 struct trace_iterator *iter = m->private;
2b6080f2 3419 struct trace_array *tr = iter->tr;
b04cc6b1 3420 int cpu_file = iter->cpu_file;
bc0c38d1
SR
3421 void *p = NULL;
3422 loff_t l = 0;
3928a8a2 3423 int cpu;
bc0c38d1 3424
2fd196ec
HT
3425 /*
3426 * copy the tracer to avoid using a global lock all around.
3427 * iter->trace is a copy of current_trace, the pointer to the
3428 * name may be used instead of a strcmp(), as iter->trace->name
3429 * will point to the same string as current_trace->name.
3430 */
bc0c38d1 3431 mutex_lock(&trace_types_lock);
2b6080f2
SR
3432 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3433 *iter->trace = *tr->current_trace;
d7350c3f 3434 mutex_unlock(&trace_types_lock);
bc0c38d1 3435
12883efb 3436#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3437 if (iter->snapshot && iter->trace->use_max_tr)
3438 return ERR_PTR(-EBUSY);
12883efb 3439#endif
debdd57f
HT
3440
3441 if (!iter->snapshot)
d914ba37 3442 atomic_inc(&trace_record_taskinfo_disabled);
bc0c38d1 3443
bc0c38d1
SR
3444 if (*pos != iter->pos) {
3445 iter->ent = NULL;
3446 iter->cpu = 0;
3447 iter->idx = -1;
3448
ae3b5093 3449 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3450 for_each_tracing_cpu(cpu)
2f26ebd5 3451 tracing_iter_reset(iter, cpu);
b04cc6b1 3452 } else
2f26ebd5 3453 tracing_iter_reset(iter, cpu_file);
bc0c38d1 3454
ac91d854 3455 iter->leftover = 0;
bc0c38d1
SR
3456 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3457 ;
3458
3459 } else {
a63ce5b3
SR
3460 /*
3461 * If we overflowed the seq_file before, then we want
3462 * to just reuse the trace_seq buffer again.
3463 */
3464 if (iter->leftover)
3465 p = iter;
3466 else {
3467 l = *pos - 1;
3468 p = s_next(m, p, &l);
3469 }
bc0c38d1
SR
3470 }
3471
4f535968 3472 trace_event_read_lock();
7e53bd42 3473 trace_access_lock(cpu_file);
bc0c38d1
SR
3474 return p;
3475}
3476
3477static void s_stop(struct seq_file *m, void *p)
3478{
7e53bd42
LJ
3479 struct trace_iterator *iter = m->private;
3480
12883efb 3481#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3482 if (iter->snapshot && iter->trace->use_max_tr)
3483 return;
12883efb 3484#endif
debdd57f
HT
3485
3486 if (!iter->snapshot)
d914ba37 3487 atomic_dec(&trace_record_taskinfo_disabled);
12883efb 3488
7e53bd42 3489 trace_access_unlock(iter->cpu_file);
4f535968 3490 trace_event_read_unlock();
bc0c38d1
SR
3491}
3492
ecffc8a8
DA
3493static void
3494get_total_entries_cpu(struct trace_buffer *buf, unsigned long *total,
3495 unsigned long *entries, int cpu)
3496{
3497 unsigned long count;
3498
3499 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3500 /*
3501 * If this buffer has skipped entries, then we hold all
3502 * entries for the trace and we need to ignore the
3503 * ones before the time stamp.
3504 */
3505 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3506 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3507 /* total is the same as the entries */
3508 *total = count;
3509 } else
3510 *total = count +
3511 ring_buffer_overrun_cpu(buf->buffer, cpu);
3512 *entries = count;
3513}
3514
39eaf7ef 3515static void
12883efb
SRRH
3516get_total_entries(struct trace_buffer *buf,
3517 unsigned long *total, unsigned long *entries)
39eaf7ef 3518{
ecffc8a8 3519 unsigned long t, e;
39eaf7ef
SR
3520 int cpu;
3521
3522 *total = 0;
3523 *entries = 0;
3524
3525 for_each_tracing_cpu(cpu) {
ecffc8a8
DA
3526 get_total_entries_cpu(buf, &t, &e, cpu);
3527 *total += t;
3528 *entries += e;
39eaf7ef
SR
3529 }
3530}
3531
ecffc8a8
DA
3532unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
3533{
3534 unsigned long total, entries;
3535
3536 if (!tr)
3537 tr = &global_trace;
3538
3539 get_total_entries_cpu(&tr->trace_buffer, &total, &entries, cpu);
3540
3541 return entries;
3542}
3543
3544unsigned long trace_total_entries(struct trace_array *tr)
3545{
3546 unsigned long total, entries;
3547
3548 if (!tr)
3549 tr = &global_trace;
3550
3551 get_total_entries(&tr->trace_buffer, &total, &entries);
3552
3553 return entries;
3554}
3555
e309b41d 3556static void print_lat_help_header(struct seq_file *m)
bc0c38d1 3557{
d79ac28f
RV
3558 seq_puts(m, "# _------=> CPU# \n"
3559 "# / _-----=> irqs-off \n"
3560 "# | / _----=> need-resched \n"
3561 "# || / _---=> hardirq/softirq \n"
3562 "# ||| / _--=> preempt-depth \n"
3563 "# |||| / delay \n"
3564 "# cmd pid ||||| time | caller \n"
3565 "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
3566}
3567
12883efb 3568static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 3569{
39eaf7ef
SR
3570 unsigned long total;
3571 unsigned long entries;
3572
12883efb 3573 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
3574 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3575 entries, total, num_online_cpus());
3576 seq_puts(m, "#\n");
3577}
3578
441dae8f
JF
3579static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3580 unsigned int flags)
39eaf7ef 3581{
441dae8f
JF
3582 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3583
12883efb 3584 print_event_info(buf, m);
441dae8f 3585
f8494fa3
JFG
3586 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3587 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
bc0c38d1
SR
3588}
3589
441dae8f
JF
3590static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3591 unsigned int flags)
77271ce4 3592{
441dae8f 3593 bool tgid = flags & TRACE_ITER_RECORD_TGID;
0f5e5a3a
RV
3594 const char *space = " ";
3595 int prec = tgid ? 10 : 2;
b11fb737 3596
9e738215
QP
3597 print_event_info(buf, m);
3598
0f5e5a3a
RV
3599 seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
3600 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
3601 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
3602 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
3603 seq_printf(m, "# %.*s||| / delay\n", prec, space);
3604 seq_printf(m, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
3605 seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
77271ce4 3606}
bc0c38d1 3607
62b915f1 3608void
bc0c38d1
SR
3609print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3610{
983f938a 3611 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
3612 struct trace_buffer *buf = iter->trace_buffer;
3613 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 3614 struct tracer *type = iter->trace;
39eaf7ef
SR
3615 unsigned long entries;
3616 unsigned long total;
bc0c38d1
SR
3617 const char *name = "preemption";
3618
d840f718 3619 name = type->name;
bc0c38d1 3620
12883efb 3621 get_total_entries(buf, &total, &entries);
bc0c38d1 3622
888b55dc 3623 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 3624 name, UTS_RELEASE);
888b55dc 3625 seq_puts(m, "# -----------------------------------"
bc0c38d1 3626 "---------------------------------\n");
888b55dc 3627 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 3628 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 3629 nsecs_to_usecs(data->saved_latency),
bc0c38d1 3630 entries,
4c11d7ae 3631 total,
12883efb 3632 buf->cpu,
bc0c38d1
SR
3633#if defined(CONFIG_PREEMPT_NONE)
3634 "server",
3635#elif defined(CONFIG_PREEMPT_VOLUNTARY)
3636 "desktop",
b5c21b45 3637#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
3638 "preempt",
3639#else
3640 "unknown",
3641#endif
3642 /* These are reserved for later use */
3643 0, 0, 0, 0);
3644#ifdef CONFIG_SMP
3645 seq_printf(m, " #P:%d)\n", num_online_cpus());
3646#else
3647 seq_puts(m, ")\n");
3648#endif
888b55dc
KM
3649 seq_puts(m, "# -----------------\n");
3650 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 3651 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
3652 data->comm, data->pid,
3653 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 3654 data->policy, data->rt_priority);
888b55dc 3655 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
3656
3657 if (data->critical_start) {
888b55dc 3658 seq_puts(m, "# => started at: ");
214023c3
SR
3659 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3660 trace_print_seq(m, &iter->seq);
888b55dc 3661 seq_puts(m, "\n# => ended at: ");
214023c3
SR
3662 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3663 trace_print_seq(m, &iter->seq);
8248ac05 3664 seq_puts(m, "\n#\n");
bc0c38d1
SR
3665 }
3666
888b55dc 3667 seq_puts(m, "#\n");
bc0c38d1
SR
3668}
3669
a309720c
SR
3670static void test_cpu_buff_start(struct trace_iterator *iter)
3671{
3672 struct trace_seq *s = &iter->seq;
983f938a 3673 struct trace_array *tr = iter->tr;
a309720c 3674
983f938a 3675 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
12ef7d44
SR
3676 return;
3677
3678 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3679 return;
3680
4dbbe2d8
MK
3681 if (cpumask_available(iter->started) &&
3682 cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
3683 return;
3684
12883efb 3685 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
3686 return;
3687
4dbbe2d8 3688 if (cpumask_available(iter->started))
919cd979 3689 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
3690
3691 /* Don't print started cpu buffer for the first entry of the trace */
3692 if (iter->idx > 1)
3693 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3694 iter->cpu);
a309720c
SR
3695}
3696
2c4f035f 3697static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 3698{
983f938a 3699 struct trace_array *tr = iter->tr;
214023c3 3700 struct trace_seq *s = &iter->seq;
983f938a 3701 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 3702 struct trace_entry *entry;
f633cef0 3703 struct trace_event *event;
bc0c38d1 3704
4e3c3333 3705 entry = iter->ent;
dd0e545f 3706
a309720c
SR
3707 test_cpu_buff_start(iter);
3708
c4a8e8be 3709 event = ftrace_find_event(entry->type);
bc0c38d1 3710
983f938a 3711 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3712 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3713 trace_print_lat_context(iter);
3714 else
3715 trace_print_context(iter);
c4a8e8be 3716 }
bc0c38d1 3717
19a7fe20
SRRH
3718 if (trace_seq_has_overflowed(s))
3719 return TRACE_TYPE_PARTIAL_LINE;
3720
268ccda0 3721 if (event)
a9a57763 3722 return event->funcs->trace(iter, sym_flags, event);
d9793bd8 3723
19a7fe20 3724 trace_seq_printf(s, "Unknown type %d\n", entry->type);
02b67518 3725
19a7fe20 3726 return trace_handle_return(s);
bc0c38d1
SR
3727}
3728
2c4f035f 3729static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3 3730{
983f938a 3731 struct trace_array *tr = iter->tr;
f9896bf3
IM
3732 struct trace_seq *s = &iter->seq;
3733 struct trace_entry *entry;
f633cef0 3734 struct trace_event *event;
f9896bf3
IM
3735
3736 entry = iter->ent;
dd0e545f 3737
983f938a 3738 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
19a7fe20
SRRH
3739 trace_seq_printf(s, "%d %d %llu ",
3740 entry->pid, iter->cpu, iter->ts);
3741
3742 if (trace_seq_has_overflowed(s))
3743 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3 3744
f633cef0 3745 event = ftrace_find_event(entry->type);
268ccda0 3746 if (event)
a9a57763 3747 return event->funcs->raw(iter, 0, event);
d9793bd8 3748
19a7fe20 3749 trace_seq_printf(s, "%d ?\n", entry->type);
777e208d 3750
19a7fe20 3751 return trace_handle_return(s);
f9896bf3
IM
3752}
3753
2c4f035f 3754static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec 3755{
983f938a 3756 struct trace_array *tr = iter->tr;
5e3ca0ec
IM
3757 struct trace_seq *s = &iter->seq;
3758 unsigned char newline = '\n';
3759 struct trace_entry *entry;
f633cef0 3760 struct trace_event *event;
5e3ca0ec
IM
3761
3762 entry = iter->ent;
dd0e545f 3763
983f938a 3764 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3765 SEQ_PUT_HEX_FIELD(s, entry->pid);
3766 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3767 SEQ_PUT_HEX_FIELD(s, iter->ts);
3768 if (trace_seq_has_overflowed(s))
3769 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3770 }
5e3ca0ec 3771
f633cef0 3772 event = ftrace_find_event(entry->type);
268ccda0 3773 if (event) {
a9a57763 3774 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
3775 if (ret != TRACE_TYPE_HANDLED)
3776 return ret;
3777 }
7104f300 3778
19a7fe20 3779 SEQ_PUT_FIELD(s, newline);
5e3ca0ec 3780
19a7fe20 3781 return trace_handle_return(s);
5e3ca0ec
IM
3782}
3783
2c4f035f 3784static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa 3785{
983f938a 3786 struct trace_array *tr = iter->tr;
cb0f12aa
IM
3787 struct trace_seq *s = &iter->seq;
3788 struct trace_entry *entry;
f633cef0 3789 struct trace_event *event;
cb0f12aa
IM
3790
3791 entry = iter->ent;
dd0e545f 3792
983f938a 3793 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3794 SEQ_PUT_FIELD(s, entry->pid);
3795 SEQ_PUT_FIELD(s, iter->cpu);
3796 SEQ_PUT_FIELD(s, iter->ts);
3797 if (trace_seq_has_overflowed(s))
3798 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3799 }
cb0f12aa 3800
f633cef0 3801 event = ftrace_find_event(entry->type);
a9a57763
SR
3802 return event ? event->funcs->binary(iter, 0, event) :
3803 TRACE_TYPE_HANDLED;
cb0f12aa
IM
3804}
3805
62b915f1 3806int trace_empty(struct trace_iterator *iter)
bc0c38d1 3807{
6d158a81 3808 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
3809 int cpu;
3810
9aba60fe 3811 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 3812 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 3813 cpu = iter->cpu_file;
6d158a81
SR
3814 buf_iter = trace_buffer_iter(iter, cpu);
3815 if (buf_iter) {
3816 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
3817 return 0;
3818 } else {
12883efb 3819 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
3820 return 0;
3821 }
3822 return 1;
3823 }
3824
ab46428c 3825 for_each_tracing_cpu(cpu) {
6d158a81
SR
3826 buf_iter = trace_buffer_iter(iter, cpu);
3827 if (buf_iter) {
3828 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
3829 return 0;
3830 } else {
12883efb 3831 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
3832 return 0;
3833 }
bc0c38d1 3834 }
d769041f 3835
797d3712 3836 return 1;
bc0c38d1
SR
3837}
3838
4f535968 3839/* Called with trace_event_read_lock() held. */
955b61e5 3840enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 3841{
983f938a
SRRH
3842 struct trace_array *tr = iter->tr;
3843 unsigned long trace_flags = tr->trace_flags;
2c4f035f
FW
3844 enum print_line_t ret;
3845
19a7fe20
SRRH
3846 if (iter->lost_events) {
3847 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3848 iter->cpu, iter->lost_events);
3849 if (trace_seq_has_overflowed(&iter->seq))
3850 return TRACE_TYPE_PARTIAL_LINE;
3851 }
bc21b478 3852
2c4f035f
FW
3853 if (iter->trace && iter->trace->print_line) {
3854 ret = iter->trace->print_line(iter);
3855 if (ret != TRACE_TYPE_UNHANDLED)
3856 return ret;
3857 }
72829bc3 3858
09ae7234
SRRH
3859 if (iter->ent->type == TRACE_BPUTS &&
3860 trace_flags & TRACE_ITER_PRINTK &&
3861 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3862 return trace_print_bputs_msg_only(iter);
3863
48ead020
FW
3864 if (iter->ent->type == TRACE_BPRINT &&
3865 trace_flags & TRACE_ITER_PRINTK &&
3866 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3867 return trace_print_bprintk_msg_only(iter);
48ead020 3868
66896a85
FW
3869 if (iter->ent->type == TRACE_PRINT &&
3870 trace_flags & TRACE_ITER_PRINTK &&
3871 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3872 return trace_print_printk_msg_only(iter);
66896a85 3873
cb0f12aa
IM
3874 if (trace_flags & TRACE_ITER_BIN)
3875 return print_bin_fmt(iter);
3876
5e3ca0ec
IM
3877 if (trace_flags & TRACE_ITER_HEX)
3878 return print_hex_fmt(iter);
3879
f9896bf3
IM
3880 if (trace_flags & TRACE_ITER_RAW)
3881 return print_raw_fmt(iter);
3882
f9896bf3
IM
3883 return print_trace_fmt(iter);
3884}
3885
7e9a49ef
JO
3886void trace_latency_header(struct seq_file *m)
3887{
3888 struct trace_iterator *iter = m->private;
983f938a 3889 struct trace_array *tr = iter->tr;
7e9a49ef
JO
3890
3891 /* print nothing if the buffers are empty */
3892 if (trace_empty(iter))
3893 return;
3894
3895 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3896 print_trace_header(m, iter);
3897
983f938a 3898 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
7e9a49ef
JO
3899 print_lat_help_header(m);
3900}
3901
62b915f1
JO
3902void trace_default_header(struct seq_file *m)
3903{
3904 struct trace_iterator *iter = m->private;
983f938a
SRRH
3905 struct trace_array *tr = iter->tr;
3906 unsigned long trace_flags = tr->trace_flags;
62b915f1 3907
f56e7f8e
JO
3908 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3909 return;
3910
62b915f1
JO
3911 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3912 /* print nothing if the buffers are empty */
3913 if (trace_empty(iter))
3914 return;
3915 print_trace_header(m, iter);
3916 if (!(trace_flags & TRACE_ITER_VERBOSE))
3917 print_lat_help_header(m);
3918 } else {
77271ce4
SR
3919 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3920 if (trace_flags & TRACE_ITER_IRQ_INFO)
441dae8f
JF
3921 print_func_help_header_irq(iter->trace_buffer,
3922 m, trace_flags);
77271ce4 3923 else
441dae8f
JF
3924 print_func_help_header(iter->trace_buffer, m,
3925 trace_flags);
77271ce4 3926 }
62b915f1
JO
3927 }
3928}
3929
e0a413f6
SR
3930static void test_ftrace_alive(struct seq_file *m)
3931{
3932 if (!ftrace_is_dead())
3933 return;
d79ac28f
RV
3934 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3935 "# MAY BE MISSING FUNCTION EVENTS\n");
e0a413f6
SR
3936}
3937
d8741e2e 3938#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 3939static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 3940{
d79ac28f
RV
3941 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3942 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3943 "# Takes a snapshot of the main buffer.\n"
3944 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3945 "# (Doesn't have to be '2' works with any number that\n"
3946 "# is not a '0' or '1')\n");
d8741e2e 3947}
f1affcaa
SRRH
3948
3949static void show_snapshot_percpu_help(struct seq_file *m)
3950{
fa6f0cc7 3951 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
f1affcaa 3952#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
d79ac28f
RV
3953 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3954 "# Takes a snapshot of the main buffer for this cpu.\n");
f1affcaa 3955#else
d79ac28f
RV
3956 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3957 "# Must use main snapshot file to allocate.\n");
f1affcaa 3958#endif
d79ac28f
RV
3959 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3960 "# (Doesn't have to be '2' works with any number that\n"
3961 "# is not a '0' or '1')\n");
f1affcaa
SRRH
3962}
3963
d8741e2e
SRRH
3964static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3965{
45ad21ca 3966 if (iter->tr->allocated_snapshot)
fa6f0cc7 3967 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
d8741e2e 3968 else
fa6f0cc7 3969 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
d8741e2e 3970
fa6f0cc7 3971 seq_puts(m, "# Snapshot commands:\n");
f1affcaa
SRRH
3972 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3973 show_snapshot_main_help(m);
3974 else
3975 show_snapshot_percpu_help(m);
d8741e2e
SRRH
3976}
3977#else
3978/* Should never be called */
3979static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3980#endif
3981
bc0c38d1
SR
3982static int s_show(struct seq_file *m, void *v)
3983{
3984 struct trace_iterator *iter = v;
a63ce5b3 3985 int ret;
bc0c38d1
SR
3986
3987 if (iter->ent == NULL) {
3988 if (iter->tr) {
3989 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3990 seq_puts(m, "#\n");
e0a413f6 3991 test_ftrace_alive(m);
bc0c38d1 3992 }
d8741e2e
SRRH
3993 if (iter->snapshot && trace_empty(iter))
3994 print_snapshot_help(m, iter);
3995 else if (iter->trace && iter->trace->print_header)
8bba1bf5 3996 iter->trace->print_header(m);
62b915f1
JO
3997 else
3998 trace_default_header(m);
3999
a63ce5b3
SR
4000 } else if (iter->leftover) {
4001 /*
4002 * If we filled the seq_file buffer earlier, we
4003 * want to just show it now.
4004 */
4005 ret = trace_print_seq(m, &iter->seq);
4006
4007 /* ret should this time be zero, but you never know */
4008 iter->leftover = ret;
4009
bc0c38d1 4010 } else {
f9896bf3 4011 print_trace_line(iter);
a63ce5b3
SR
4012 ret = trace_print_seq(m, &iter->seq);
4013 /*
4014 * If we overflow the seq_file buffer, then it will
4015 * ask us for this data again at start up.
4016 * Use that instead.
4017 * ret is 0 if seq_file write succeeded.
4018 * -1 otherwise.
4019 */
4020 iter->leftover = ret;
bc0c38d1
SR
4021 }
4022
4023 return 0;
4024}
4025
649e9c70
ON
4026/*
4027 * Should be used after trace_array_get(), trace_types_lock
4028 * ensures that i_cdev was already initialized.
4029 */
4030static inline int tracing_get_cpu(struct inode *inode)
4031{
4032 if (inode->i_cdev) /* See trace_create_cpu_file() */
4033 return (long)inode->i_cdev - 1;
4034 return RING_BUFFER_ALL_CPUS;
4035}
4036
88e9d34c 4037static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
4038 .start = s_start,
4039 .next = s_next,
4040 .stop = s_stop,
4041 .show = s_show,
bc0c38d1
SR
4042};
4043
e309b41d 4044static struct trace_iterator *
6484c71c 4045__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 4046{
6484c71c 4047 struct trace_array *tr = inode->i_private;
bc0c38d1 4048 struct trace_iterator *iter;
50e18b94 4049 int cpu;
bc0c38d1 4050
85a2f9b4
SR
4051 if (tracing_disabled)
4052 return ERR_PTR(-ENODEV);
60a11774 4053
50e18b94 4054 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
4055 if (!iter)
4056 return ERR_PTR(-ENOMEM);
bc0c38d1 4057
72917235 4058 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
6d158a81 4059 GFP_KERNEL);
93574fcc
DC
4060 if (!iter->buffer_iter)
4061 goto release;
4062
d7350c3f
FW
4063 /*
4064 * We make a copy of the current tracer to avoid concurrent
4065 * changes on it while we are reading.
4066 */
bc0c38d1 4067 mutex_lock(&trace_types_lock);
d7350c3f 4068 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 4069 if (!iter->trace)
d7350c3f 4070 goto fail;
85a2f9b4 4071
2b6080f2 4072 *iter->trace = *tr->current_trace;
d7350c3f 4073
79f55997 4074 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
4075 goto fail;
4076
12883efb
SRRH
4077 iter->tr = tr;
4078
4079#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
4080 /* Currently only the top directory has a snapshot */
4081 if (tr->current_trace->print_max || snapshot)
12883efb 4082 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 4083 else
12883efb
SRRH
4084#endif
4085 iter->trace_buffer = &tr->trace_buffer;
debdd57f 4086 iter->snapshot = snapshot;
bc0c38d1 4087 iter->pos = -1;
6484c71c 4088 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 4089 mutex_init(&iter->mutex);
bc0c38d1 4090
8bba1bf5
MM
4091 /* Notify the tracer early; before we stop tracing. */
4092 if (iter->trace && iter->trace->open)
a93751ca 4093 iter->trace->open(iter);
8bba1bf5 4094
12ef7d44 4095 /* Annotate start of buffers if we had overruns */
12883efb 4096 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
4097 iter->iter_flags |= TRACE_FILE_ANNOTATE;
4098
8be0709f 4099 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 4100 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
4101 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4102
debdd57f
HT
4103 /* stop the trace while dumping if we are not opening "snapshot" */
4104 if (!iter->snapshot)
2b6080f2 4105 tracing_stop_tr(tr);
2f26ebd5 4106
ae3b5093 4107 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 4108 for_each_tracing_cpu(cpu) {
b04cc6b1 4109 iter->buffer_iter[cpu] =
31b265b3
DA
4110 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4111 cpu, GFP_KERNEL);
72c9ddfd
DM
4112 }
4113 ring_buffer_read_prepare_sync();
4114 for_each_tracing_cpu(cpu) {
4115 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 4116 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
4117 }
4118 } else {
4119 cpu = iter->cpu_file;
3928a8a2 4120 iter->buffer_iter[cpu] =
31b265b3
DA
4121 ring_buffer_read_prepare(iter->trace_buffer->buffer,
4122 cpu, GFP_KERNEL);
72c9ddfd
DM
4123 ring_buffer_read_prepare_sync();
4124 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 4125 tracing_iter_reset(iter, cpu);
3928a8a2
SR
4126 }
4127
bc0c38d1
SR
4128 mutex_unlock(&trace_types_lock);
4129
bc0c38d1 4130 return iter;
3928a8a2 4131
d7350c3f 4132 fail:
3928a8a2 4133 mutex_unlock(&trace_types_lock);
d7350c3f 4134 kfree(iter->trace);
6d158a81 4135 kfree(iter->buffer_iter);
93574fcc 4136release:
50e18b94
JO
4137 seq_release_private(inode, file);
4138 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
4139}
4140
4141int tracing_open_generic(struct inode *inode, struct file *filp)
4142{
60a11774
SR
4143 if (tracing_disabled)
4144 return -ENODEV;
4145
bc0c38d1
SR
4146 filp->private_data = inode->i_private;
4147 return 0;
4148}
4149
2e86421d
GB
4150bool tracing_is_disabled(void)
4151{
4152 return (tracing_disabled) ? true: false;
4153}
4154
7b85af63
SRRH
4155/*
4156 * Open and update trace_array ref count.
4157 * Must have the current trace_array passed to it.
4158 */
dcc30223 4159static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
7b85af63
SRRH
4160{
4161 struct trace_array *tr = inode->i_private;
4162
4163 if (tracing_disabled)
4164 return -ENODEV;
4165
4166 if (trace_array_get(tr) < 0)
4167 return -ENODEV;
4168
4169 filp->private_data = inode->i_private;
4170
4171 return 0;
7b85af63
SRRH
4172}
4173
4fd27358 4174static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 4175{
6484c71c 4176 struct trace_array *tr = inode->i_private;
907f2784 4177 struct seq_file *m = file->private_data;
4acd4d00 4178 struct trace_iterator *iter;
3928a8a2 4179 int cpu;
bc0c38d1 4180
ff451961 4181 if (!(file->f_mode & FMODE_READ)) {
6484c71c 4182 trace_array_put(tr);
4acd4d00 4183 return 0;
ff451961 4184 }
4acd4d00 4185
6484c71c 4186 /* Writes do not use seq_file */
4acd4d00 4187 iter = m->private;
bc0c38d1 4188 mutex_lock(&trace_types_lock);
a695cb58 4189
3928a8a2
SR
4190 for_each_tracing_cpu(cpu) {
4191 if (iter->buffer_iter[cpu])
4192 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4193 }
4194
bc0c38d1
SR
4195 if (iter->trace && iter->trace->close)
4196 iter->trace->close(iter);
4197
debdd57f
HT
4198 if (!iter->snapshot)
4199 /* reenable tracing if it was previously enabled */
2b6080f2 4200 tracing_start_tr(tr);
f77d09a3
AL
4201
4202 __trace_array_put(tr);
4203
bc0c38d1
SR
4204 mutex_unlock(&trace_types_lock);
4205
d7350c3f 4206 mutex_destroy(&iter->mutex);
b0dfa978 4207 free_cpumask_var(iter->started);
d7350c3f 4208 kfree(iter->trace);
6d158a81 4209 kfree(iter->buffer_iter);
50e18b94 4210 seq_release_private(inode, file);
ff451961 4211
bc0c38d1
SR
4212 return 0;
4213}
4214
7b85af63
SRRH
4215static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4216{
4217 struct trace_array *tr = inode->i_private;
4218
4219 trace_array_put(tr);
bc0c38d1
SR
4220 return 0;
4221}
4222
7b85af63
SRRH
4223static int tracing_single_release_tr(struct inode *inode, struct file *file)
4224{
4225 struct trace_array *tr = inode->i_private;
4226
4227 trace_array_put(tr);
4228
4229 return single_release(inode, file);
4230}
4231
bc0c38d1
SR
4232static int tracing_open(struct inode *inode, struct file *file)
4233{
6484c71c 4234 struct trace_array *tr = inode->i_private;
85a2f9b4
SR
4235 struct trace_iterator *iter;
4236 int ret = 0;
bc0c38d1 4237
ff451961
SRRH
4238 if (trace_array_get(tr) < 0)
4239 return -ENODEV;
4240
4acd4d00 4241 /* If this file was open for write, then erase contents */
6484c71c
ON
4242 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4243 int cpu = tracing_get_cpu(inode);
8dd33bcb
BY
4244 struct trace_buffer *trace_buf = &tr->trace_buffer;
4245
4246#ifdef CONFIG_TRACER_MAX_TRACE
4247 if (tr->current_trace->print_max)
4248 trace_buf = &tr->max_buffer;
4249#endif
6484c71c
ON
4250
4251 if (cpu == RING_BUFFER_ALL_CPUS)
8dd33bcb 4252 tracing_reset_online_cpus(trace_buf);
4acd4d00 4253 else
8dd33bcb 4254 tracing_reset(trace_buf, cpu);
4acd4d00 4255 }
bc0c38d1 4256
4acd4d00 4257 if (file->f_mode & FMODE_READ) {
6484c71c 4258 iter = __tracing_open(inode, file, false);
4acd4d00
SR
4259 if (IS_ERR(iter))
4260 ret = PTR_ERR(iter);
983f938a 4261 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4acd4d00
SR
4262 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4263 }
ff451961
SRRH
4264
4265 if (ret < 0)
4266 trace_array_put(tr);
4267
bc0c38d1
SR
4268 return ret;
4269}
4270
607e2ea1
SRRH
4271/*
4272 * Some tracers are not suitable for instance buffers.
4273 * A tracer is always available for the global array (toplevel)
4274 * or if it explicitly states that it is.
4275 */
4276static bool
4277trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4278{
4279 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4280}
4281
4282/* Find the next tracer that this trace array may use */
4283static struct tracer *
4284get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4285{
4286 while (t && !trace_ok_for_array(t, tr))
4287 t = t->next;
4288
4289 return t;
4290}
4291
e309b41d 4292static void *
bc0c38d1
SR
4293t_next(struct seq_file *m, void *v, loff_t *pos)
4294{
607e2ea1 4295 struct trace_array *tr = m->private;
f129e965 4296 struct tracer *t = v;
bc0c38d1
SR
4297
4298 (*pos)++;
4299
4300 if (t)
607e2ea1 4301 t = get_tracer_for_array(tr, t->next);
bc0c38d1 4302
bc0c38d1
SR
4303 return t;
4304}
4305
4306static void *t_start(struct seq_file *m, loff_t *pos)
4307{
607e2ea1 4308 struct trace_array *tr = m->private;
f129e965 4309 struct tracer *t;
bc0c38d1
SR
4310 loff_t l = 0;
4311
4312 mutex_lock(&trace_types_lock);
607e2ea1
SRRH
4313
4314 t = get_tracer_for_array(tr, trace_types);
4315 for (; t && l < *pos; t = t_next(m, t, &l))
4316 ;
bc0c38d1
SR
4317
4318 return t;
4319}
4320
4321static void t_stop(struct seq_file *m, void *p)
4322{
4323 mutex_unlock(&trace_types_lock);
4324}
4325
4326static int t_show(struct seq_file *m, void *v)
4327{
4328 struct tracer *t = v;
4329
4330 if (!t)
4331 return 0;
4332
fa6f0cc7 4333 seq_puts(m, t->name);
bc0c38d1
SR
4334 if (t->next)
4335 seq_putc(m, ' ');
4336 else
4337 seq_putc(m, '\n');
4338
4339 return 0;
4340}
4341
88e9d34c 4342static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
4343 .start = t_start,
4344 .next = t_next,
4345 .stop = t_stop,
4346 .show = t_show,
bc0c38d1
SR
4347};
4348
4349static int show_traces_open(struct inode *inode, struct file *file)
4350{
607e2ea1
SRRH
4351 struct trace_array *tr = inode->i_private;
4352 struct seq_file *m;
4353 int ret;
4354
60a11774
SR
4355 if (tracing_disabled)
4356 return -ENODEV;
4357
607e2ea1
SRRH
4358 ret = seq_open(file, &show_traces_seq_ops);
4359 if (ret)
4360 return ret;
4361
4362 m = file->private_data;
4363 m->private = tr;
4364
4365 return 0;
bc0c38d1
SR
4366}
4367
4acd4d00
SR
4368static ssize_t
4369tracing_write_stub(struct file *filp, const char __user *ubuf,
4370 size_t count, loff_t *ppos)
4371{
4372 return count;
4373}
4374
098c879e 4375loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
364829b1 4376{
098c879e
SRRH
4377 int ret;
4378
364829b1 4379 if (file->f_mode & FMODE_READ)
098c879e 4380 ret = seq_lseek(file, offset, whence);
364829b1 4381 else
098c879e
SRRH
4382 file->f_pos = ret = 0;
4383
4384 return ret;
364829b1
SP
4385}
4386
5e2336a0 4387static const struct file_operations tracing_fops = {
4bf39a94
IM
4388 .open = tracing_open,
4389 .read = seq_read,
4acd4d00 4390 .write = tracing_write_stub,
098c879e 4391 .llseek = tracing_lseek,
4bf39a94 4392 .release = tracing_release,
bc0c38d1
SR
4393};
4394
5e2336a0 4395static const struct file_operations show_traces_fops = {
c7078de1
IM
4396 .open = show_traces_open,
4397 .read = seq_read,
4398 .release = seq_release,
b444786f 4399 .llseek = seq_lseek,
c7078de1
IM
4400};
4401
4402static ssize_t
4403tracing_cpumask_read(struct file *filp, char __user *ubuf,
4404 size_t count, loff_t *ppos)
4405{
ccfe9e42 4406 struct trace_array *tr = file_inode(filp)->i_private;
90e406f9 4407 char *mask_str;
36dfe925 4408 int len;
c7078de1 4409
90e406f9
CD
4410 len = snprintf(NULL, 0, "%*pb\n",
4411 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4412 mask_str = kmalloc(len, GFP_KERNEL);
4413 if (!mask_str)
4414 return -ENOMEM;
36dfe925 4415
90e406f9 4416 len = snprintf(mask_str, len, "%*pb\n",
1a40243b
TH
4417 cpumask_pr_args(tr->tracing_cpumask));
4418 if (len >= count) {
36dfe925
IM
4419 count = -EINVAL;
4420 goto out_err;
4421 }
90e406f9 4422 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
36dfe925
IM
4423
4424out_err:
90e406f9 4425 kfree(mask_str);
c7078de1
IM
4426
4427 return count;
4428}
4429
4430static ssize_t
4431tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4432 size_t count, loff_t *ppos)
4433{
ccfe9e42 4434 struct trace_array *tr = file_inode(filp)->i_private;
9e01c1b7 4435 cpumask_var_t tracing_cpumask_new;
2b6080f2 4436 int err, cpu;
9e01c1b7
RR
4437
4438 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4439 return -ENOMEM;
c7078de1 4440
9e01c1b7 4441 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 4442 if (err)
36dfe925
IM
4443 goto err_unlock;
4444
a5e25883 4445 local_irq_disable();
0b9b12c1 4446 arch_spin_lock(&tr->max_lock);
ab46428c 4447 for_each_tracing_cpu(cpu) {
36dfe925
IM
4448 /*
4449 * Increase/decrease the disabled counter if we are
4450 * about to flip a bit in the cpumask:
4451 */
ccfe9e42 4452 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4453 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4454 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4455 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 4456 }
ccfe9e42 4457 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4458 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4459 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4460 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
4461 }
4462 }
0b9b12c1 4463 arch_spin_unlock(&tr->max_lock);
a5e25883 4464 local_irq_enable();
36dfe925 4465
ccfe9e42 4466 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
9e01c1b7 4467 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
4468
4469 return count;
36dfe925
IM
4470
4471err_unlock:
215368e8 4472 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
4473
4474 return err;
c7078de1
IM
4475}
4476
5e2336a0 4477static const struct file_operations tracing_cpumask_fops = {
ccfe9e42 4478 .open = tracing_open_generic_tr,
c7078de1
IM
4479 .read = tracing_cpumask_read,
4480 .write = tracing_cpumask_write,
ccfe9e42 4481 .release = tracing_release_generic_tr,
b444786f 4482 .llseek = generic_file_llseek,
bc0c38d1
SR
4483};
4484
fdb372ed 4485static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 4486{
d8e83d26 4487 struct tracer_opt *trace_opts;
2b6080f2 4488 struct trace_array *tr = m->private;
d8e83d26 4489 u32 tracer_flags;
d8e83d26 4490 int i;
adf9f195 4491
d8e83d26 4492 mutex_lock(&trace_types_lock);
2b6080f2
SR
4493 tracer_flags = tr->current_trace->flags->val;
4494 trace_opts = tr->current_trace->flags->opts;
d8e83d26 4495
bc0c38d1 4496 for (i = 0; trace_options[i]; i++) {
983f938a 4497 if (tr->trace_flags & (1 << i))
fdb372ed 4498 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 4499 else
fdb372ed 4500 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
4501 }
4502
adf9f195
FW
4503 for (i = 0; trace_opts[i].name; i++) {
4504 if (tracer_flags & trace_opts[i].bit)
fdb372ed 4505 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 4506 else
fdb372ed 4507 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 4508 }
d8e83d26 4509 mutex_unlock(&trace_types_lock);
adf9f195 4510
fdb372ed 4511 return 0;
bc0c38d1 4512}
bc0c38d1 4513
8c1a49ae 4514static int __set_tracer_option(struct trace_array *tr,
8d18eaaf
LZ
4515 struct tracer_flags *tracer_flags,
4516 struct tracer_opt *opts, int neg)
4517{
d39cdd20 4518 struct tracer *trace = tracer_flags->trace;
8d18eaaf 4519 int ret;
bc0c38d1 4520
8c1a49ae 4521 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
8d18eaaf
LZ
4522 if (ret)
4523 return ret;
4524
4525 if (neg)
4526 tracer_flags->val &= ~opts->bit;
4527 else
4528 tracer_flags->val |= opts->bit;
4529 return 0;
bc0c38d1
SR
4530}
4531
adf9f195 4532/* Try to assign a tracer specific option */
8c1a49ae 4533static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
adf9f195 4534{
8c1a49ae 4535 struct tracer *trace = tr->current_trace;
7770841e 4536 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 4537 struct tracer_opt *opts = NULL;
8d18eaaf 4538 int i;
adf9f195 4539
7770841e
Z
4540 for (i = 0; tracer_flags->opts[i].name; i++) {
4541 opts = &tracer_flags->opts[i];
adf9f195 4542
8d18eaaf 4543 if (strcmp(cmp, opts->name) == 0)
8c1a49ae 4544 return __set_tracer_option(tr, trace->flags, opts, neg);
adf9f195 4545 }
adf9f195 4546
8d18eaaf 4547 return -EINVAL;
adf9f195
FW
4548}
4549
613f04a0
SRRH
4550/* Some tracers require overwrite to stay enabled */
4551int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4552{
4553 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4554 return -1;
4555
4556 return 0;
4557}
4558
2b6080f2 4559int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
4560{
4561 /* do nothing if flag is already set */
983f938a 4562 if (!!(tr->trace_flags & mask) == !!enabled)
613f04a0
SRRH
4563 return 0;
4564
4565 /* Give the tracer a chance to approve the change */
2b6080f2 4566 if (tr->current_trace->flag_changed)
bf6065b5 4567 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
613f04a0 4568 return -EINVAL;
af4617bd
SR
4569
4570 if (enabled)
983f938a 4571 tr->trace_flags |= mask;
af4617bd 4572 else
983f938a 4573 tr->trace_flags &= ~mask;
e870e9a1
LZ
4574
4575 if (mask == TRACE_ITER_RECORD_CMD)
4576 trace_event_enable_cmd_record(enabled);
750912fa 4577
d914ba37
JF
4578 if (mask == TRACE_ITER_RECORD_TGID) {
4579 if (!tgid_map)
6396bb22
KC
4580 tgid_map = kcalloc(PID_MAX_DEFAULT + 1,
4581 sizeof(*tgid_map),
d914ba37
JF
4582 GFP_KERNEL);
4583 if (!tgid_map) {
4584 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4585 return -ENOMEM;
4586 }
4587
4588 trace_event_enable_tgid_record(enabled);
4589 }
4590
c37775d5
SR
4591 if (mask == TRACE_ITER_EVENT_FORK)
4592 trace_event_follow_fork(tr, enabled);
4593
1e10486f
NK
4594 if (mask == TRACE_ITER_FUNC_FORK)
4595 ftrace_pid_follow_fork(tr, enabled);
4596
80902822 4597 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 4598 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 4599#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 4600 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
4601#endif
4602 }
81698831 4603
b9f9108c 4604 if (mask == TRACE_ITER_PRINTK) {
81698831 4605 trace_printk_start_stop_comm(enabled);
b9f9108c
SRRH
4606 trace_printk_control(enabled);
4607 }
613f04a0
SRRH
4608
4609 return 0;
af4617bd
SR
4610}
4611
2b6080f2 4612static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 4613{
8d18eaaf 4614 char *cmp;
bc0c38d1 4615 int neg = 0;
591a033d 4616 int ret;
a4d1e688 4617 size_t orig_len = strlen(option);
3d739c1f 4618 int len;
bc0c38d1 4619
7bcfaf54 4620 cmp = strstrip(option);
bc0c38d1 4621
3d739c1f
SRV
4622 len = str_has_prefix(cmp, "no");
4623 if (len)
bc0c38d1 4624 neg = 1;
3d739c1f
SRV
4625
4626 cmp += len;
bc0c38d1 4627
69d34da2
SRRH
4628 mutex_lock(&trace_types_lock);
4629
591a033d 4630 ret = match_string(trace_options, -1, cmp);
adf9f195 4631 /* If no option could be set, test the specific tracer options */
591a033d 4632 if (ret < 0)
8c1a49ae 4633 ret = set_tracer_option(tr, cmp, neg);
591a033d
YX
4634 else
4635 ret = set_tracer_flag(tr, 1 << ret, !neg);
69d34da2
SRRH
4636
4637 mutex_unlock(&trace_types_lock);
bc0c38d1 4638
a4d1e688
JW
4639 /*
4640 * If the first trailing whitespace is replaced with '\0' by strstrip,
4641 * turn it back into a space.
4642 */
4643 if (orig_len > strlen(option))
4644 option[strlen(option)] = ' ';
4645
7bcfaf54
SR
4646 return ret;
4647}
4648
a4d1e688
JW
4649static void __init apply_trace_boot_options(void)
4650{
4651 char *buf = trace_boot_options_buf;
4652 char *option;
4653
4654 while (true) {
4655 option = strsep(&buf, ",");
4656
4657 if (!option)
4658 break;
a4d1e688 4659
43ed3843
SRRH
4660 if (*option)
4661 trace_set_options(&global_trace, option);
a4d1e688
JW
4662
4663 /* Put back the comma to allow this to be called again */
4664 if (buf)
4665 *(buf - 1) = ',';
4666 }
4667}
4668
7bcfaf54
SR
4669static ssize_t
4670tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4671 size_t cnt, loff_t *ppos)
4672{
2b6080f2
SR
4673 struct seq_file *m = filp->private_data;
4674 struct trace_array *tr = m->private;
7bcfaf54 4675 char buf[64];
613f04a0 4676 int ret;
7bcfaf54
SR
4677
4678 if (cnt >= sizeof(buf))
4679 return -EINVAL;
4680
4afe6495 4681 if (copy_from_user(buf, ubuf, cnt))
7bcfaf54
SR
4682 return -EFAULT;
4683
a8dd2176
SR
4684 buf[cnt] = 0;
4685
2b6080f2 4686 ret = trace_set_options(tr, buf);
613f04a0
SRRH
4687 if (ret < 0)
4688 return ret;
7bcfaf54 4689
cf8517cf 4690 *ppos += cnt;
bc0c38d1
SR
4691
4692 return cnt;
4693}
4694
fdb372ed
LZ
4695static int tracing_trace_options_open(struct inode *inode, struct file *file)
4696{
7b85af63 4697 struct trace_array *tr = inode->i_private;
f77d09a3 4698 int ret;
7b85af63 4699
fdb372ed
LZ
4700 if (tracing_disabled)
4701 return -ENODEV;
2b6080f2 4702
7b85af63
SRRH
4703 if (trace_array_get(tr) < 0)
4704 return -ENODEV;
4705
f77d09a3
AL
4706 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4707 if (ret < 0)
4708 trace_array_put(tr);
4709
4710 return ret;
fdb372ed
LZ
4711}
4712
5e2336a0 4713static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
4714 .open = tracing_trace_options_open,
4715 .read = seq_read,
4716 .llseek = seq_lseek,
7b85af63 4717 .release = tracing_single_release_tr,
ee6bce52 4718 .write = tracing_trace_options_write,
bc0c38d1
SR
4719};
4720
7bd2f24c
IM
4721static const char readme_msg[] =
4722 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
4723 "# echo 0 > tracing_on : quick way to disable tracing\n"
4724 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4725 " Important files:\n"
4726 " trace\t\t\t- The static contents of the buffer\n"
4727 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4728 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4729 " current_tracer\t- function and latency tracers\n"
4730 " available_tracers\t- list of configured tracers for current_tracer\n"
a8d65579 4731 " error_log\t- error log for failed commands (that support it)\n"
22f45649
SRRH
4732 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4733 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4734 " trace_clock\t\t-change the clock used to order events\n"
4735 " local: Per cpu clock but may not be synced across CPUs\n"
4736 " global: Synced across CPUs but slows tracing down.\n"
4737 " counter: Not a clock, but just an increment\n"
4738 " uptime: Jiffy counter from time of boot\n"
4739 " perf: Same clock that perf events use\n"
4740#ifdef CONFIG_X86_64
4741 " x86-tsc: TSC cycle counter\n"
4742#endif
2c1ea60b
TZ
4743 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4744 " delta: Delta difference against a buffer-wide timestamp\n"
4745 " absolute: Absolute (standalone) timestamp\n"
22f45649 4746 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
fa32e855 4747 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
22f45649
SRRH
4748 " tracing_cpumask\t- Limit which CPUs to trace\n"
4749 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4750 "\t\t\t Remove sub-buffer with rmdir\n"
4751 " trace_options\t\t- Set format or modify how tracing happens\n"
b9416997 4752 "\t\t\t Disable an option by prefixing 'no' to the\n"
71485c45 4753 "\t\t\t option name\n"
939c7a4f 4754 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
22f45649
SRRH
4755#ifdef CONFIG_DYNAMIC_FTRACE
4756 "\n available_filter_functions - list of functions that can be filtered on\n"
71485c45
SRRH
4757 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4758 "\t\t\t functions\n"
60f1d5e3 4759 "\t accepts: func_full_name or glob-matching-pattern\n"
71485c45
SRRH
4760 "\t modules: Can select a group via module\n"
4761 "\t Format: :mod:<module-name>\n"
4762 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4763 "\t triggers: a command to perform when function is hit\n"
4764 "\t Format: <function>:<trigger>[:count]\n"
4765 "\t trigger: traceon, traceoff\n"
4766 "\t\t enable_event:<system>:<event>\n"
4767 "\t\t disable_event:<system>:<event>\n"
22f45649 4768#ifdef CONFIG_STACKTRACE
71485c45 4769 "\t\t stacktrace\n"
22f45649
SRRH
4770#endif
4771#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4772 "\t\t snapshot\n"
22f45649 4773#endif
17a280ea
SRRH
4774 "\t\t dump\n"
4775 "\t\t cpudump\n"
71485c45
SRRH
4776 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4777 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4778 "\t The first one will disable tracing every time do_fault is hit\n"
4779 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4780 "\t The first time do trap is hit and it disables tracing, the\n"
4781 "\t counter will decrement to 2. If tracing is already disabled,\n"
4782 "\t the counter will not decrement. It only decrements when the\n"
4783 "\t trigger did work\n"
4784 "\t To remove trigger without count:\n"
4785 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4786 "\t To remove trigger with a count:\n"
4787 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
22f45649 4788 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
71485c45
SRRH
4789 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4790 "\t modules: Can select a group via module command :mod:\n"
4791 "\t Does not accept triggers\n"
22f45649
SRRH
4792#endif /* CONFIG_DYNAMIC_FTRACE */
4793#ifdef CONFIG_FUNCTION_TRACER
71485c45
SRRH
4794 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4795 "\t\t (function)\n"
22f45649
SRRH
4796#endif
4797#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4798 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
d048a8c7 4799 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
22f45649
SRRH
4800 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4801#endif
4802#ifdef CONFIG_TRACER_SNAPSHOT
71485c45
SRRH
4803 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4804 "\t\t\t snapshot buffer. Read the contents for more\n"
4805 "\t\t\t information\n"
22f45649 4806#endif
991821c8 4807#ifdef CONFIG_STACK_TRACER
22f45649
SRRH
4808 " stack_trace\t\t- Shows the max stack trace when active\n"
4809 " stack_max_size\t- Shows current max stack size that was traced\n"
71485c45
SRRH
4810 "\t\t\t Write into this file to reset the max size (trigger a\n"
4811 "\t\t\t new trace)\n"
22f45649 4812#ifdef CONFIG_DYNAMIC_FTRACE
71485c45
SRRH
4813 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4814 "\t\t\t traces\n"
22f45649 4815#endif
991821c8 4816#endif /* CONFIG_STACK_TRACER */
5448d44c
MH
4817#ifdef CONFIG_DYNAMIC_EVENTS
4818 " dynamic_events\t\t- Add/remove/show the generic dynamic events\n"
4819 "\t\t\t Write into this file to define/undefine new trace events.\n"
4820#endif
6b0b7551 4821#ifdef CONFIG_KPROBE_EVENTS
86425625
MH
4822 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4823 "\t\t\t Write into this file to define/undefine new trace events.\n"
4824#endif
6b0b7551 4825#ifdef CONFIG_UPROBE_EVENTS
86425625
MH
4826 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4827 "\t\t\t Write into this file to define/undefine new trace events.\n"
4828#endif
6b0b7551 4829#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
86425625 4830 "\t accepts: event-definitions (one definition per line)\n"
c3ca46ef
MH
4831 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4832 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
7bbab38d
MH
4833#ifdef CONFIG_HIST_TRIGGERS
4834 "\t s:[synthetic/]<event> <field> [<field>]\n"
4835#endif
86425625 4836 "\t -:[<group>/]<event>\n"
6b0b7551 4837#ifdef CONFIG_KPROBE_EVENTS
86425625 4838 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
35b6f55a 4839 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
86425625 4840#endif
6b0b7551 4841#ifdef CONFIG_UPROBE_EVENTS
1cc33161 4842 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
86425625
MH
4843#endif
4844 "\t args: <name>=fetcharg[:type]\n"
4845 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
a1303af5 4846#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
e65f7ae7 4847 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
a1303af5 4848#else
e65f7ae7 4849 "\t $stack<index>, $stack, $retval, $comm,\n"
a1303af5 4850#endif
e65f7ae7 4851 "\t +|-[u]<offset>(<fetcharg>)\n"
60c2e0ce 4852 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
88903c46 4853 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
40b53b77 4854 "\t <type>\\[<array-size>\\]\n"
7bbab38d
MH
4855#ifdef CONFIG_HIST_TRIGGERS
4856 "\t field: <stype> <name>;\n"
4857 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4858 "\t [unsigned] char/int/long\n"
4859#endif
86425625 4860#endif
26f25564
TZ
4861 " events/\t\t- Directory containing all trace event subsystems:\n"
4862 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4863 " events/<system>/\t- Directory containing all trace events for <system>:\n"
71485c45
SRRH
4864 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4865 "\t\t\t events\n"
26f25564 4866 " filter\t\t- If set, only events passing filter are traced\n"
71485c45
SRRH
4867 " events/<system>/<event>/\t- Directory containing control files for\n"
4868 "\t\t\t <event>:\n"
26f25564
TZ
4869 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4870 " filter\t\t- If set, only events passing filter are traced\n"
4871 " trigger\t\t- If set, a command to perform when event is hit\n"
71485c45
SRRH
4872 "\t Format: <trigger>[:count][if <filter>]\n"
4873 "\t trigger: traceon, traceoff\n"
4874 "\t enable_event:<system>:<event>\n"
4875 "\t disable_event:<system>:<event>\n"
d0bad49b
TZ
4876#ifdef CONFIG_HIST_TRIGGERS
4877 "\t enable_hist:<system>:<event>\n"
4878 "\t disable_hist:<system>:<event>\n"
4879#endif
26f25564 4880#ifdef CONFIG_STACKTRACE
71485c45 4881 "\t\t stacktrace\n"
26f25564
TZ
4882#endif
4883#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4884 "\t\t snapshot\n"
7ef224d1
TZ
4885#endif
4886#ifdef CONFIG_HIST_TRIGGERS
4887 "\t\t hist (see below)\n"
26f25564 4888#endif
71485c45
SRRH
4889 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4890 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4891 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4892 "\t events/block/block_unplug/trigger\n"
4893 "\t The first disables tracing every time block_unplug is hit.\n"
4894 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4895 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4896 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4897 "\t Like function triggers, the counter is only decremented if it\n"
4898 "\t enabled or disabled tracing.\n"
4899 "\t To remove a trigger without a count:\n"
4900 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4901 "\t To remove a trigger with a count:\n"
4902 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4903 "\t Filters can be ignored when removing a trigger.\n"
7ef224d1
TZ
4904#ifdef CONFIG_HIST_TRIGGERS
4905 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
76a3b0c8 4906 "\t Format: hist:keys=<field1[,field2,...]>\n"
f2606835 4907 "\t [:values=<field1[,field2,...]>]\n"
e62347d2 4908 "\t [:sort=<field1[,field2,...]>]\n"
7ef224d1 4909 "\t [:size=#entries]\n"
e86ae9ba 4910 "\t [:pause][:continue][:clear]\n"
5463bfda 4911 "\t [:name=histname1]\n"
c3e49506 4912 "\t [:<handler>.<action>]\n"
7ef224d1
TZ
4913 "\t [if <filter>]\n\n"
4914 "\t When a matching event is hit, an entry is added to a hash\n"
f2606835
TZ
4915 "\t table using the key(s) and value(s) named, and the value of a\n"
4916 "\t sum called 'hitcount' is incremented. Keys and values\n"
4917 "\t correspond to fields in the event's format description. Keys\n"
69a0200c
TZ
4918 "\t can be any field, or the special string 'stacktrace'.\n"
4919 "\t Compound keys consisting of up to two fields can be specified\n"
4920 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4921 "\t fields. Sort keys consisting of up to two fields can be\n"
4922 "\t specified using the 'sort' keyword. The sort direction can\n"
4923 "\t be modified by appending '.descending' or '.ascending' to a\n"
4924 "\t sort field. The 'size' parameter can be used to specify more\n"
5463bfda
TZ
4925 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4926 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4927 "\t its histogram data will be shared with other triggers of the\n"
4928 "\t same name, and trigger hits will update this common data.\n\n"
7ef224d1 4929 "\t Reading the 'hist' file for the event will dump the hash\n"
52a7f16d
TZ
4930 "\t table in its entirety to stdout. If there are multiple hist\n"
4931 "\t triggers attached to an event, there will be a table for each\n"
5463bfda
TZ
4932 "\t trigger in the output. The table displayed for a named\n"
4933 "\t trigger will be the same as any other instance having the\n"
4934 "\t same name. The default format used to display a given field\n"
4935 "\t can be modified by appending any of the following modifiers\n"
4936 "\t to the field name, as applicable:\n\n"
c6afad49
TZ
4937 "\t .hex display a number as a hex value\n"
4938 "\t .sym display an address as a symbol\n"
6b4827ad 4939 "\t .sym-offset display an address as a symbol and offset\n"
31696198 4940 "\t .execname display a common_pid as a program name\n"
860f9f6b
TZ
4941 "\t .syscall display a syscall id as a syscall name\n"
4942 "\t .log2 display log2 value rather than raw number\n"
4943 "\t .usecs display a common_timestamp in microseconds\n\n"
83e99914
TZ
4944 "\t The 'pause' parameter can be used to pause an existing hist\n"
4945 "\t trigger or to start a hist trigger but not log any events\n"
4946 "\t until told to do so. 'continue' can be used to start or\n"
4947 "\t restart a paused hist trigger.\n\n"
e86ae9ba
TZ
4948 "\t The 'clear' parameter will clear the contents of a running\n"
4949 "\t hist trigger and leave its current paused/active state\n"
4950 "\t unchanged.\n\n"
d0bad49b
TZ
4951 "\t The enable_hist and disable_hist triggers can be used to\n"
4952 "\t have one event conditionally start and stop another event's\n"
9e5a36a3 4953 "\t already-attached hist trigger. The syntax is analogous to\n"
c3e49506
TZ
4954 "\t the enable_event and disable_event triggers.\n\n"
4955 "\t Hist trigger handlers and actions are executed whenever a\n"
4956 "\t a histogram entry is added or updated. They take the form:\n\n"
4957 "\t <handler>.<action>\n\n"
4958 "\t The available handlers are:\n\n"
4959 "\t onmatch(matching.event) - invoke on addition or update\n"
dff81f55
TZ
4960 "\t onmax(var) - invoke if var exceeds current max\n"
4961 "\t onchange(var) - invoke action if var changes\n\n"
c3e49506 4962 "\t The available actions are:\n\n"
e91eefd7 4963 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
c3e49506 4964 "\t save(field,...) - save current event fields\n"
a3785b7e
TZ
4965#ifdef CONFIG_TRACER_SNAPSHOT
4966 "\t snapshot() - snapshot the trace buffer\n"
4967#endif
7ef224d1 4968#endif
7bd2f24c
IM
4969;
4970
4971static ssize_t
4972tracing_readme_read(struct file *filp, char __user *ubuf,
4973 size_t cnt, loff_t *ppos)
4974{
4975 return simple_read_from_buffer(ubuf, cnt, ppos,
4976 readme_msg, strlen(readme_msg));
4977}
4978
5e2336a0 4979static const struct file_operations tracing_readme_fops = {
c7078de1
IM
4980 .open = tracing_open_generic,
4981 .read = tracing_readme_read,
b444786f 4982 .llseek = generic_file_llseek,
7bd2f24c
IM
4983};
4984
99c621d7
MS
4985static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
4986{
4987 int *ptr = v;
4988
4989 if (*pos || m->count)
4990 ptr++;
4991
4992 (*pos)++;
4993
4994 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
4995 if (trace_find_tgid(*ptr))
4996 return ptr;
4997 }
4998
4999 return NULL;
5000}
5001
5002static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
5003{
5004 void *v;
5005 loff_t l = 0;
5006
5007 if (!tgid_map)
5008 return NULL;
5009
5010 v = &tgid_map[0];
5011 while (l <= *pos) {
5012 v = saved_tgids_next(m, v, &l);
5013 if (!v)
5014 return NULL;
5015 }
5016
5017 return v;
5018}
5019
5020static void saved_tgids_stop(struct seq_file *m, void *v)
5021{
5022}
5023
5024static int saved_tgids_show(struct seq_file *m, void *v)
5025{
5026 int pid = (int *)v - tgid_map;
5027
5028 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
5029 return 0;
5030}
5031
5032static const struct seq_operations tracing_saved_tgids_seq_ops = {
5033 .start = saved_tgids_start,
5034 .stop = saved_tgids_stop,
5035 .next = saved_tgids_next,
5036 .show = saved_tgids_show,
5037};
5038
5039static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
5040{
5041 if (tracing_disabled)
5042 return -ENODEV;
5043
5044 return seq_open(filp, &tracing_saved_tgids_seq_ops);
5045}
5046
5047
5048static const struct file_operations tracing_saved_tgids_fops = {
5049 .open = tracing_saved_tgids_open,
5050 .read = seq_read,
5051 .llseek = seq_lseek,
5052 .release = seq_release,
5053};
5054
42584c81
YY
5055static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
5056{
5057 unsigned int *ptr = v;
69abe6a5 5058
42584c81
YY
5059 if (*pos || m->count)
5060 ptr++;
69abe6a5 5061
42584c81 5062 (*pos)++;
69abe6a5 5063
939c7a4f
YY
5064 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
5065 ptr++) {
42584c81
YY
5066 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
5067 continue;
69abe6a5 5068
42584c81
YY
5069 return ptr;
5070 }
69abe6a5 5071
42584c81
YY
5072 return NULL;
5073}
5074
5075static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
5076{
5077 void *v;
5078 loff_t l = 0;
69abe6a5 5079
4c27e756
SRRH
5080 preempt_disable();
5081 arch_spin_lock(&trace_cmdline_lock);
5082
939c7a4f 5083 v = &savedcmd->map_cmdline_to_pid[0];
42584c81
YY
5084 while (l <= *pos) {
5085 v = saved_cmdlines_next(m, v, &l);
5086 if (!v)
5087 return NULL;
69abe6a5
AP
5088 }
5089
42584c81
YY
5090 return v;
5091}
5092
5093static void saved_cmdlines_stop(struct seq_file *m, void *v)
5094{
4c27e756
SRRH
5095 arch_spin_unlock(&trace_cmdline_lock);
5096 preempt_enable();
42584c81 5097}
69abe6a5 5098
42584c81
YY
5099static int saved_cmdlines_show(struct seq_file *m, void *v)
5100{
5101 char buf[TASK_COMM_LEN];
5102 unsigned int *pid = v;
69abe6a5 5103
4c27e756 5104 __trace_find_cmdline(*pid, buf);
42584c81
YY
5105 seq_printf(m, "%d %s\n", *pid, buf);
5106 return 0;
5107}
5108
5109static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
5110 .start = saved_cmdlines_start,
5111 .next = saved_cmdlines_next,
5112 .stop = saved_cmdlines_stop,
5113 .show = saved_cmdlines_show,
5114};
5115
5116static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
5117{
5118 if (tracing_disabled)
5119 return -ENODEV;
5120
5121 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
69abe6a5
AP
5122}
5123
5124static const struct file_operations tracing_saved_cmdlines_fops = {
42584c81
YY
5125 .open = tracing_saved_cmdlines_open,
5126 .read = seq_read,
5127 .llseek = seq_lseek,
5128 .release = seq_release,
69abe6a5
AP
5129};
5130
939c7a4f
YY
5131static ssize_t
5132tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
5133 size_t cnt, loff_t *ppos)
5134{
5135 char buf[64];
5136 int r;
5137
5138 arch_spin_lock(&trace_cmdline_lock);
a6af8fbf 5139 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
939c7a4f
YY
5140 arch_spin_unlock(&trace_cmdline_lock);
5141
5142 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5143}
5144
5145static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
5146{
5147 kfree(s->saved_cmdlines);
5148 kfree(s->map_cmdline_to_pid);
5149 kfree(s);
5150}
5151
5152static int tracing_resize_saved_cmdlines(unsigned int val)
5153{
5154 struct saved_cmdlines_buffer *s, *savedcmd_temp;
5155
a6af8fbf 5156 s = kmalloc(sizeof(*s), GFP_KERNEL);
939c7a4f
YY
5157 if (!s)
5158 return -ENOMEM;
5159
5160 if (allocate_cmdlines_buffer(val, s) < 0) {
5161 kfree(s);
5162 return -ENOMEM;
5163 }
5164
5165 arch_spin_lock(&trace_cmdline_lock);
5166 savedcmd_temp = savedcmd;
5167 savedcmd = s;
5168 arch_spin_unlock(&trace_cmdline_lock);
5169 free_saved_cmdlines_buffer(savedcmd_temp);
5170
5171 return 0;
5172}
5173
5174static ssize_t
5175tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
5176 size_t cnt, loff_t *ppos)
5177{
5178 unsigned long val;
5179 int ret;
5180
5181 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5182 if (ret)
5183 return ret;
5184
5185 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5186 if (!val || val > PID_MAX_DEFAULT)
5187 return -EINVAL;
5188
5189 ret = tracing_resize_saved_cmdlines((unsigned int)val);
5190 if (ret < 0)
5191 return ret;
5192
5193 *ppos += cnt;
5194
5195 return cnt;
5196}
5197
5198static const struct file_operations tracing_saved_cmdlines_size_fops = {
5199 .open = tracing_open_generic,
5200 .read = tracing_saved_cmdlines_size_read,
5201 .write = tracing_saved_cmdlines_size_write,
5202};
5203
681bec03 5204#ifdef CONFIG_TRACE_EVAL_MAP_FILE
23bf8cb8 5205static union trace_eval_map_item *
f57a4143 5206update_eval_map(union trace_eval_map_item *ptr)
9828413d 5207{
00f4b652 5208 if (!ptr->map.eval_string) {
9828413d
SRRH
5209 if (ptr->tail.next) {
5210 ptr = ptr->tail.next;
5211 /* Set ptr to the next real item (skip head) */
5212 ptr++;
5213 } else
5214 return NULL;
5215 }
5216 return ptr;
5217}
5218
f57a4143 5219static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
9828413d 5220{
23bf8cb8 5221 union trace_eval_map_item *ptr = v;
9828413d
SRRH
5222
5223 /*
5224 * Paranoid! If ptr points to end, we don't want to increment past it.
5225 * This really should never happen.
5226 */
f57a4143 5227 ptr = update_eval_map(ptr);
9828413d
SRRH
5228 if (WARN_ON_ONCE(!ptr))
5229 return NULL;
5230
5231 ptr++;
5232
5233 (*pos)++;
5234
f57a4143 5235 ptr = update_eval_map(ptr);
9828413d
SRRH
5236
5237 return ptr;
5238}
5239
f57a4143 5240static void *eval_map_start(struct seq_file *m, loff_t *pos)
9828413d 5241{
23bf8cb8 5242 union trace_eval_map_item *v;
9828413d
SRRH
5243 loff_t l = 0;
5244
1793ed93 5245 mutex_lock(&trace_eval_mutex);
9828413d 5246
23bf8cb8 5247 v = trace_eval_maps;
9828413d
SRRH
5248 if (v)
5249 v++;
5250
5251 while (v && l < *pos) {
f57a4143 5252 v = eval_map_next(m, v, &l);
9828413d
SRRH
5253 }
5254
5255 return v;
5256}
5257
f57a4143 5258static void eval_map_stop(struct seq_file *m, void *v)
9828413d 5259{
1793ed93 5260 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
5261}
5262
f57a4143 5263static int eval_map_show(struct seq_file *m, void *v)
9828413d 5264{
23bf8cb8 5265 union trace_eval_map_item *ptr = v;
9828413d
SRRH
5266
5267 seq_printf(m, "%s %ld (%s)\n",
00f4b652 5268 ptr->map.eval_string, ptr->map.eval_value,
9828413d
SRRH
5269 ptr->map.system);
5270
5271 return 0;
5272}
5273
f57a4143
JL
5274static const struct seq_operations tracing_eval_map_seq_ops = {
5275 .start = eval_map_start,
5276 .next = eval_map_next,
5277 .stop = eval_map_stop,
5278 .show = eval_map_show,
9828413d
SRRH
5279};
5280
f57a4143 5281static int tracing_eval_map_open(struct inode *inode, struct file *filp)
9828413d
SRRH
5282{
5283 if (tracing_disabled)
5284 return -ENODEV;
5285
f57a4143 5286 return seq_open(filp, &tracing_eval_map_seq_ops);
9828413d
SRRH
5287}
5288
f57a4143
JL
5289static const struct file_operations tracing_eval_map_fops = {
5290 .open = tracing_eval_map_open,
9828413d
SRRH
5291 .read = seq_read,
5292 .llseek = seq_lseek,
5293 .release = seq_release,
5294};
5295
23bf8cb8 5296static inline union trace_eval_map_item *
5f60b351 5297trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
9828413d
SRRH
5298{
5299 /* Return tail of array given the head */
5300 return ptr + ptr->head.length + 1;
5301}
5302
5303static void
f57a4143 5304trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
9828413d
SRRH
5305 int len)
5306{
00f4b652
JL
5307 struct trace_eval_map **stop;
5308 struct trace_eval_map **map;
23bf8cb8
JL
5309 union trace_eval_map_item *map_array;
5310 union trace_eval_map_item *ptr;
9828413d
SRRH
5311
5312 stop = start + len;
5313
5314 /*
23bf8cb8 5315 * The trace_eval_maps contains the map plus a head and tail item,
9828413d
SRRH
5316 * where the head holds the module and length of array, and the
5317 * tail holds a pointer to the next list.
5318 */
6da2ec56 5319 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
9828413d 5320 if (!map_array) {
f57a4143 5321 pr_warn("Unable to allocate trace eval mapping\n");
9828413d
SRRH
5322 return;
5323 }
5324
1793ed93 5325 mutex_lock(&trace_eval_mutex);
9828413d 5326
23bf8cb8
JL
5327 if (!trace_eval_maps)
5328 trace_eval_maps = map_array;
9828413d 5329 else {
23bf8cb8 5330 ptr = trace_eval_maps;
9828413d 5331 for (;;) {
5f60b351 5332 ptr = trace_eval_jmp_to_tail(ptr);
9828413d
SRRH
5333 if (!ptr->tail.next)
5334 break;
5335 ptr = ptr->tail.next;
5336
5337 }
5338 ptr->tail.next = map_array;
5339 }
5340 map_array->head.mod = mod;
5341 map_array->head.length = len;
5342 map_array++;
5343
5344 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5345 map_array->map = **map;
5346 map_array++;
5347 }
5348 memset(map_array, 0, sizeof(*map_array));
5349
1793ed93 5350 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
5351}
5352
f57a4143 5353static void trace_create_eval_file(struct dentry *d_tracer)
9828413d 5354{
681bec03 5355 trace_create_file("eval_map", 0444, d_tracer,
f57a4143 5356 NULL, &tracing_eval_map_fops);
9828413d
SRRH
5357}
5358
681bec03 5359#else /* CONFIG_TRACE_EVAL_MAP_FILE */
f57a4143
JL
5360static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5361static inline void trace_insert_eval_map_file(struct module *mod,
00f4b652 5362 struct trace_eval_map **start, int len) { }
681bec03 5363#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 5364
f57a4143 5365static void trace_insert_eval_map(struct module *mod,
00f4b652 5366 struct trace_eval_map **start, int len)
0c564a53 5367{
00f4b652 5368 struct trace_eval_map **map;
0c564a53
SRRH
5369
5370 if (len <= 0)
5371 return;
5372
5373 map = start;
5374
f57a4143 5375 trace_event_eval_update(map, len);
9828413d 5376
f57a4143 5377 trace_insert_eval_map_file(mod, start, len);
0c564a53
SRRH
5378}
5379
bc0c38d1
SR
5380static ssize_t
5381tracing_set_trace_read(struct file *filp, char __user *ubuf,
5382 size_t cnt, loff_t *ppos)
5383{
2b6080f2 5384 struct trace_array *tr = filp->private_data;
ee6c2c1b 5385 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
5386 int r;
5387
5388 mutex_lock(&trace_types_lock);
2b6080f2 5389 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
5390 mutex_unlock(&trace_types_lock);
5391
4bf39a94 5392 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
5393}
5394
b6f11df2
ACM
5395int tracer_init(struct tracer *t, struct trace_array *tr)
5396{
12883efb 5397 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
5398 return t->init(tr);
5399}
5400
12883efb 5401static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
5402{
5403 int cpu;
737223fb 5404
438ced17 5405 for_each_tracing_cpu(cpu)
12883efb 5406 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
5407}
5408
12883efb 5409#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 5410/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
5411static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5412 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
5413{
5414 int cpu, ret = 0;
5415
5416 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5417 for_each_tracing_cpu(cpu) {
12883efb
SRRH
5418 ret = ring_buffer_resize(trace_buf->buffer,
5419 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
5420 if (ret < 0)
5421 break;
12883efb
SRRH
5422 per_cpu_ptr(trace_buf->data, cpu)->entries =
5423 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
5424 }
5425 } else {
12883efb
SRRH
5426 ret = ring_buffer_resize(trace_buf->buffer,
5427 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 5428 if (ret == 0)
12883efb
SRRH
5429 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5430 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
5431 }
5432
5433 return ret;
5434}
12883efb 5435#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 5436
2b6080f2
SR
5437static int __tracing_resize_ring_buffer(struct trace_array *tr,
5438 unsigned long size, int cpu)
73c5162a
SR
5439{
5440 int ret;
5441
5442 /*
5443 * If kernel or user changes the size of the ring buffer
a123c52b
SR
5444 * we use the size that was given, and we can forget about
5445 * expanding it later.
73c5162a 5446 */
55034cd6 5447 ring_buffer_expanded = true;
73c5162a 5448
b382ede6 5449 /* May be called before buffers are initialized */
12883efb 5450 if (!tr->trace_buffer.buffer)
b382ede6
SR
5451 return 0;
5452
12883efb 5453 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
5454 if (ret < 0)
5455 return ret;
5456
12883efb 5457#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
5458 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5459 !tr->current_trace->use_max_tr)
ef710e10
KM
5460 goto out;
5461
12883efb 5462 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 5463 if (ret < 0) {
12883efb
SRRH
5464 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5465 &tr->trace_buffer, cpu);
73c5162a 5466 if (r < 0) {
a123c52b
SR
5467 /*
5468 * AARGH! We are left with different
5469 * size max buffer!!!!
5470 * The max buffer is our "snapshot" buffer.
5471 * When a tracer needs a snapshot (one of the
5472 * latency tracers), it swaps the max buffer
5473 * with the saved snap shot. We succeeded to
5474 * update the size of the main buffer, but failed to
5475 * update the size of the max buffer. But when we tried
5476 * to reset the main buffer to the original size, we
5477 * failed there too. This is very unlikely to
5478 * happen, but if it does, warn and kill all
5479 * tracing.
5480 */
73c5162a
SR
5481 WARN_ON(1);
5482 tracing_disabled = 1;
5483 }
5484 return ret;
5485 }
5486
438ced17 5487 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 5488 set_buffer_entries(&tr->max_buffer, size);
438ced17 5489 else
12883efb 5490 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 5491
ef710e10 5492 out:
12883efb
SRRH
5493#endif /* CONFIG_TRACER_MAX_TRACE */
5494
438ced17 5495 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 5496 set_buffer_entries(&tr->trace_buffer, size);
438ced17 5497 else
12883efb 5498 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
5499
5500 return ret;
5501}
5502
2b6080f2
SR
5503static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5504 unsigned long size, int cpu_id)
4f271a2a 5505{
83f40318 5506 int ret = size;
4f271a2a
VN
5507
5508 mutex_lock(&trace_types_lock);
5509
438ced17
VN
5510 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5511 /* make sure, this cpu is enabled in the mask */
5512 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5513 ret = -EINVAL;
5514 goto out;
5515 }
5516 }
4f271a2a 5517
2b6080f2 5518 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
5519 if (ret < 0)
5520 ret = -ENOMEM;
5521
438ced17 5522out:
4f271a2a
VN
5523 mutex_unlock(&trace_types_lock);
5524
5525 return ret;
5526}
5527
ef710e10 5528
1852fcce
SR
5529/**
5530 * tracing_update_buffers - used by tracing facility to expand ring buffers
5531 *
5532 * To save on memory when the tracing is never used on a system with it
5533 * configured in. The ring buffers are set to a minimum size. But once
5534 * a user starts to use the tracing facility, then they need to grow
5535 * to their default size.
5536 *
5537 * This function is to be called when a tracer is about to be used.
5538 */
5539int tracing_update_buffers(void)
5540{
5541 int ret = 0;
5542
1027fcb2 5543 mutex_lock(&trace_types_lock);
1852fcce 5544 if (!ring_buffer_expanded)
2b6080f2 5545 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 5546 RING_BUFFER_ALL_CPUS);
1027fcb2 5547 mutex_unlock(&trace_types_lock);
1852fcce
SR
5548
5549 return ret;
5550}
5551
577b785f
SR
5552struct trace_option_dentry;
5553
37aea98b 5554static void
2b6080f2 5555create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f 5556
6b450d25
SRRH
5557/*
5558 * Used to clear out the tracer before deletion of an instance.
5559 * Must have trace_types_lock held.
5560 */
5561static void tracing_set_nop(struct trace_array *tr)
5562{
5563 if (tr->current_trace == &nop_trace)
5564 return;
5565
50512ab5 5566 tr->current_trace->enabled--;
6b450d25
SRRH
5567
5568 if (tr->current_trace->reset)
5569 tr->current_trace->reset(tr);
5570
5571 tr->current_trace = &nop_trace;
5572}
5573
41d9c0be 5574static void add_tracer_options(struct trace_array *tr, struct tracer *t)
bc0c38d1 5575{
09d23a1d
SRRH
5576 /* Only enable if the directory has been created already. */
5577 if (!tr->dir)
5578 return;
5579
37aea98b 5580 create_trace_option_files(tr, t);
09d23a1d
SRRH
5581}
5582
5583static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5584{
bc0c38d1 5585 struct tracer *t;
12883efb 5586#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5587 bool had_max_tr;
12883efb 5588#endif
d9e54076 5589 int ret = 0;
bc0c38d1 5590
1027fcb2
SR
5591 mutex_lock(&trace_types_lock);
5592
73c5162a 5593 if (!ring_buffer_expanded) {
2b6080f2 5594 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 5595 RING_BUFFER_ALL_CPUS);
73c5162a 5596 if (ret < 0)
59f586db 5597 goto out;
73c5162a
SR
5598 ret = 0;
5599 }
5600
bc0c38d1
SR
5601 for (t = trace_types; t; t = t->next) {
5602 if (strcmp(t->name, buf) == 0)
5603 break;
5604 }
c2931e05
FW
5605 if (!t) {
5606 ret = -EINVAL;
5607 goto out;
5608 }
2b6080f2 5609 if (t == tr->current_trace)
bc0c38d1
SR
5610 goto out;
5611
a35873a0
TZ
5612#ifdef CONFIG_TRACER_SNAPSHOT
5613 if (t->use_max_tr) {
5614 arch_spin_lock(&tr->max_lock);
5615 if (tr->cond_snapshot)
5616 ret = -EBUSY;
5617 arch_spin_unlock(&tr->max_lock);
5618 if (ret)
5619 goto out;
5620 }
5621#endif
c7b3ae0b
ZSZ
5622 /* Some tracers won't work on kernel command line */
5623 if (system_state < SYSTEM_RUNNING && t->noboot) {
5624 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5625 t->name);
5626 goto out;
5627 }
5628
607e2ea1
SRRH
5629 /* Some tracers are only allowed for the top level buffer */
5630 if (!trace_ok_for_array(t, tr)) {
5631 ret = -EINVAL;
5632 goto out;
5633 }
5634
cf6ab6d9
SRRH
5635 /* If trace pipe files are being read, we can't change the tracer */
5636 if (tr->current_trace->ref) {
5637 ret = -EBUSY;
5638 goto out;
5639 }
5640
9f029e83 5641 trace_branch_disable();
613f04a0 5642
50512ab5 5643 tr->current_trace->enabled--;
613f04a0 5644
2b6080f2
SR
5645 if (tr->current_trace->reset)
5646 tr->current_trace->reset(tr);
34600f0e 5647
74401729 5648 /* Current trace needs to be nop_trace before synchronize_rcu */
2b6080f2 5649 tr->current_trace = &nop_trace;
34600f0e 5650
45ad21ca
SRRH
5651#ifdef CONFIG_TRACER_MAX_TRACE
5652 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
5653
5654 if (had_max_tr && !t->use_max_tr) {
5655 /*
5656 * We need to make sure that the update_max_tr sees that
5657 * current_trace changed to nop_trace to keep it from
5658 * swapping the buffers after we resize it.
5659 * The update_max_tr is called from interrupts disabled
5660 * so a synchronized_sched() is sufficient.
5661 */
74401729 5662 synchronize_rcu();
3209cff4 5663 free_snapshot(tr);
ef710e10 5664 }
12883efb 5665#endif
12883efb
SRRH
5666
5667#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5668 if (t->use_max_tr && !had_max_tr) {
2824f503 5669 ret = tracing_alloc_snapshot_instance(tr);
d60da506
HT
5670 if (ret < 0)
5671 goto out;
ef710e10 5672 }
12883efb 5673#endif
577b785f 5674
1c80025a 5675 if (t->init) {
b6f11df2 5676 ret = tracer_init(t, tr);
1c80025a
FW
5677 if (ret)
5678 goto out;
5679 }
bc0c38d1 5680
2b6080f2 5681 tr->current_trace = t;
50512ab5 5682 tr->current_trace->enabled++;
9f029e83 5683 trace_branch_enable(tr);
bc0c38d1
SR
5684 out:
5685 mutex_unlock(&trace_types_lock);
5686
d9e54076
PZ
5687 return ret;
5688}
5689
5690static ssize_t
5691tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5692 size_t cnt, loff_t *ppos)
5693{
607e2ea1 5694 struct trace_array *tr = filp->private_data;
ee6c2c1b 5695 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
5696 int i;
5697 size_t ret;
e6e7a65a
FW
5698 int err;
5699
5700 ret = cnt;
d9e54076 5701
ee6c2c1b
LZ
5702 if (cnt > MAX_TRACER_SIZE)
5703 cnt = MAX_TRACER_SIZE;
d9e54076 5704
4afe6495 5705 if (copy_from_user(buf, ubuf, cnt))
d9e54076
PZ
5706 return -EFAULT;
5707
5708 buf[cnt] = 0;
5709
5710 /* strip ending whitespace. */
5711 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5712 buf[i] = 0;
5713
607e2ea1 5714 err = tracing_set_tracer(tr, buf);
e6e7a65a
FW
5715 if (err)
5716 return err;
d9e54076 5717
cf8517cf 5718 *ppos += ret;
bc0c38d1 5719
c2931e05 5720 return ret;
bc0c38d1
SR
5721}
5722
5723static ssize_t
6508fa76
SF
5724tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5725 size_t cnt, loff_t *ppos)
bc0c38d1 5726{
bc0c38d1
SR
5727 char buf[64];
5728 int r;
5729
cffae437 5730 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 5731 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
5732 if (r > sizeof(buf))
5733 r = sizeof(buf);
4bf39a94 5734 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
5735}
5736
5737static ssize_t
6508fa76
SF
5738tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5739 size_t cnt, loff_t *ppos)
bc0c38d1 5740{
5e39841c 5741 unsigned long val;
c6caeeb1 5742 int ret;
bc0c38d1 5743
22fe9b54
PH
5744 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5745 if (ret)
c6caeeb1 5746 return ret;
bc0c38d1
SR
5747
5748 *ptr = val * 1000;
5749
5750 return cnt;
5751}
5752
6508fa76
SF
5753static ssize_t
5754tracing_thresh_read(struct file *filp, char __user *ubuf,
5755 size_t cnt, loff_t *ppos)
5756{
5757 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5758}
5759
5760static ssize_t
5761tracing_thresh_write(struct file *filp, const char __user *ubuf,
5762 size_t cnt, loff_t *ppos)
5763{
5764 struct trace_array *tr = filp->private_data;
5765 int ret;
5766
5767 mutex_lock(&trace_types_lock);
5768 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5769 if (ret < 0)
5770 goto out;
5771
5772 if (tr->current_trace->update_thresh) {
5773 ret = tr->current_trace->update_thresh(tr);
5774 if (ret < 0)
5775 goto out;
5776 }
5777
5778 ret = cnt;
5779out:
5780 mutex_unlock(&trace_types_lock);
5781
5782 return ret;
5783}
5784
f971cc9a 5785#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
e428abbb 5786
6508fa76
SF
5787static ssize_t
5788tracing_max_lat_read(struct file *filp, char __user *ubuf,
5789 size_t cnt, loff_t *ppos)
5790{
5791 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5792}
5793
5794static ssize_t
5795tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5796 size_t cnt, loff_t *ppos)
5797{
5798 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5799}
5800
e428abbb
CG
5801#endif
5802
b3806b43
SR
5803static int tracing_open_pipe(struct inode *inode, struct file *filp)
5804{
15544209 5805 struct trace_array *tr = inode->i_private;
b3806b43 5806 struct trace_iterator *iter;
b04cc6b1 5807 int ret = 0;
b3806b43
SR
5808
5809 if (tracing_disabled)
5810 return -ENODEV;
5811
7b85af63
SRRH
5812 if (trace_array_get(tr) < 0)
5813 return -ENODEV;
5814
b04cc6b1
FW
5815 mutex_lock(&trace_types_lock);
5816
b3806b43
SR
5817 /* create a buffer to store the information to pass to userspace */
5818 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
5819 if (!iter) {
5820 ret = -ENOMEM;
f77d09a3 5821 __trace_array_put(tr);
b04cc6b1
FW
5822 goto out;
5823 }
b3806b43 5824
3a161d99 5825 trace_seq_init(&iter->seq);
d716ff71 5826 iter->trace = tr->current_trace;
d7350c3f 5827
4462344e 5828 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 5829 ret = -ENOMEM;
d7350c3f 5830 goto fail;
4462344e
RR
5831 }
5832
a309720c 5833 /* trace pipe does not show start of buffer */
4462344e 5834 cpumask_setall(iter->started);
a309720c 5835
983f938a 5836 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
112f38a7
SR
5837 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5838
8be0709f 5839 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 5840 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
5841 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5842
15544209
ON
5843 iter->tr = tr;
5844 iter->trace_buffer = &tr->trace_buffer;
5845 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 5846 mutex_init(&iter->mutex);
b3806b43
SR
5847 filp->private_data = iter;
5848
107bad8b
SR
5849 if (iter->trace->pipe_open)
5850 iter->trace->pipe_open(iter);
107bad8b 5851
b444786f 5852 nonseekable_open(inode, filp);
cf6ab6d9
SRRH
5853
5854 tr->current_trace->ref++;
b04cc6b1
FW
5855out:
5856 mutex_unlock(&trace_types_lock);
5857 return ret;
d7350c3f
FW
5858
5859fail:
d7350c3f 5860 kfree(iter);
7b85af63 5861 __trace_array_put(tr);
d7350c3f
FW
5862 mutex_unlock(&trace_types_lock);
5863 return ret;
b3806b43
SR
5864}
5865
5866static int tracing_release_pipe(struct inode *inode, struct file *file)
5867{
5868 struct trace_iterator *iter = file->private_data;
15544209 5869 struct trace_array *tr = inode->i_private;
b3806b43 5870
b04cc6b1
FW
5871 mutex_lock(&trace_types_lock);
5872
cf6ab6d9
SRRH
5873 tr->current_trace->ref--;
5874
29bf4a5e 5875 if (iter->trace->pipe_close)
c521efd1
SR
5876 iter->trace->pipe_close(iter);
5877
b04cc6b1
FW
5878 mutex_unlock(&trace_types_lock);
5879
4462344e 5880 free_cpumask_var(iter->started);
d7350c3f 5881 mutex_destroy(&iter->mutex);
b3806b43 5882 kfree(iter);
b3806b43 5883
7b85af63
SRRH
5884 trace_array_put(tr);
5885
b3806b43
SR
5886 return 0;
5887}
5888
9dd95748 5889static __poll_t
cc60cdc9 5890trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 5891{
983f938a
SRRH
5892 struct trace_array *tr = iter->tr;
5893
15693458
SRRH
5894 /* Iterators are static, they should be filled or empty */
5895 if (trace_buffer_iter(iter, iter->cpu_file))
a9a08845 5896 return EPOLLIN | EPOLLRDNORM;
2a2cc8f7 5897
983f938a 5898 if (tr->trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
5899 /*
5900 * Always select as readable when in blocking mode
5901 */
a9a08845 5902 return EPOLLIN | EPOLLRDNORM;
15693458 5903 else
12883efb 5904 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 5905 filp, poll_table);
2a2cc8f7 5906}
2a2cc8f7 5907
9dd95748 5908static __poll_t
cc60cdc9
SR
5909tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5910{
5911 struct trace_iterator *iter = filp->private_data;
5912
5913 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
5914}
5915
d716ff71 5916/* Must be called with iter->mutex held. */
ff98781b 5917static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
5918{
5919 struct trace_iterator *iter = filp->private_data;
8b8b3683 5920 int ret;
b3806b43 5921
b3806b43 5922 while (trace_empty(iter)) {
2dc8f095 5923
107bad8b 5924 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 5925 return -EAGAIN;
107bad8b 5926 }
2dc8f095 5927
b3806b43 5928 /*
250bfd3d 5929 * We block until we read something and tracing is disabled.
b3806b43
SR
5930 * We still block if tracing is disabled, but we have never
5931 * read anything. This allows a user to cat this file, and
5932 * then enable tracing. But after we have read something,
5933 * we give an EOF when tracing is again disabled.
5934 *
5935 * iter->pos will be 0 if we haven't read anything.
5936 */
75df6e68 5937 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
b3806b43 5938 break;
f4874261
SRRH
5939
5940 mutex_unlock(&iter->mutex);
5941
2c2b0a78 5942 ret = wait_on_pipe(iter, 0);
f4874261
SRRH
5943
5944 mutex_lock(&iter->mutex);
5945
8b8b3683
SRRH
5946 if (ret)
5947 return ret;
b3806b43
SR
5948 }
5949
ff98781b
EGM
5950 return 1;
5951}
5952
5953/*
5954 * Consumer reader.
5955 */
5956static ssize_t
5957tracing_read_pipe(struct file *filp, char __user *ubuf,
5958 size_t cnt, loff_t *ppos)
5959{
5960 struct trace_iterator *iter = filp->private_data;
5961 ssize_t sret;
5962
d7350c3f
FW
5963 /*
5964 * Avoid more than one consumer on a single file descriptor
5965 * This is just a matter of traces coherency, the ring buffer itself
5966 * is protected.
5967 */
5968 mutex_lock(&iter->mutex);
1245800c
SRRH
5969
5970 /* return any leftover data */
5971 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5972 if (sret != -EBUSY)
5973 goto out;
5974
5975 trace_seq_init(&iter->seq);
5976
ff98781b
EGM
5977 if (iter->trace->read) {
5978 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5979 if (sret)
5980 goto out;
5981 }
5982
5983waitagain:
5984 sret = tracing_wait_pipe(filp);
5985 if (sret <= 0)
5986 goto out;
5987
b3806b43 5988 /* stop when tracing is finished */
ff98781b
EGM
5989 if (trace_empty(iter)) {
5990 sret = 0;
107bad8b 5991 goto out;
ff98781b 5992 }
b3806b43
SR
5993
5994 if (cnt >= PAGE_SIZE)
5995 cnt = PAGE_SIZE - 1;
5996
53d0aa77 5997 /* reset all but tr, trace, and overruns */
53d0aa77
SR
5998 memset(&iter->seq, 0,
5999 sizeof(struct trace_iterator) -
6000 offsetof(struct trace_iterator, seq));
ed5467da 6001 cpumask_clear(iter->started);
4823ed7e 6002 iter->pos = -1;
b3806b43 6003
4f535968 6004 trace_event_read_lock();
7e53bd42 6005 trace_access_lock(iter->cpu_file);
955b61e5 6006 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 6007 enum print_line_t ret;
5ac48378 6008 int save_len = iter->seq.seq.len;
088b1e42 6009
f9896bf3 6010 ret = print_trace_line(iter);
2c4f035f 6011 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42 6012 /* don't print partial lines */
5ac48378 6013 iter->seq.seq.len = save_len;
b3806b43 6014 break;
088b1e42 6015 }
b91facc3
FW
6016 if (ret != TRACE_TYPE_NO_CONSUME)
6017 trace_consume(iter);
b3806b43 6018
5ac48378 6019 if (trace_seq_used(&iter->seq) >= cnt)
b3806b43 6020 break;
ee5e51f5
JO
6021
6022 /*
6023 * Setting the full flag means we reached the trace_seq buffer
6024 * size and we should leave by partial output condition above.
6025 * One of the trace_seq_* functions is not used properly.
6026 */
6027 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6028 iter->ent->type);
b3806b43 6029 }
7e53bd42 6030 trace_access_unlock(iter->cpu_file);
4f535968 6031 trace_event_read_unlock();
b3806b43 6032
b3806b43 6033 /* Now copy what we have to the user */
6c6c2796 6034 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5ac48378 6035 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
f9520750 6036 trace_seq_init(&iter->seq);
9ff4b974
PP
6037
6038 /*
25985edc 6039 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
6040 * entries, go back to wait for more entries.
6041 */
6c6c2796 6042 if (sret == -EBUSY)
9ff4b974 6043 goto waitagain;
b3806b43 6044
107bad8b 6045out:
d7350c3f 6046 mutex_unlock(&iter->mutex);
107bad8b 6047
6c6c2796 6048 return sret;
b3806b43
SR
6049}
6050
3c56819b
EGM
6051static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6052 unsigned int idx)
6053{
6054 __free_page(spd->pages[idx]);
6055}
6056
28dfef8f 6057static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998 6058 .confirm = generic_pipe_buf_confirm,
92fdd98c 6059 .release = generic_pipe_buf_release,
34cd4998
SR
6060 .steal = generic_pipe_buf_steal,
6061 .get = generic_pipe_buf_get,
3c56819b
EGM
6062};
6063
34cd4998 6064static size_t
fa7c7f6e 6065tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
6066{
6067 size_t count;
74f06bb7 6068 int save_len;
34cd4998
SR
6069 int ret;
6070
6071 /* Seq buffer is page-sized, exactly what we need. */
6072 for (;;) {
74f06bb7 6073 save_len = iter->seq.seq.len;
34cd4998 6074 ret = print_trace_line(iter);
74f06bb7
SRRH
6075
6076 if (trace_seq_has_overflowed(&iter->seq)) {
6077 iter->seq.seq.len = save_len;
34cd4998
SR
6078 break;
6079 }
74f06bb7
SRRH
6080
6081 /*
6082 * This should not be hit, because it should only
6083 * be set if the iter->seq overflowed. But check it
6084 * anyway to be safe.
6085 */
34cd4998 6086 if (ret == TRACE_TYPE_PARTIAL_LINE) {
74f06bb7
SRRH
6087 iter->seq.seq.len = save_len;
6088 break;
6089 }
6090
5ac48378 6091 count = trace_seq_used(&iter->seq) - save_len;
74f06bb7
SRRH
6092 if (rem < count) {
6093 rem = 0;
6094 iter->seq.seq.len = save_len;
34cd4998
SR
6095 break;
6096 }
6097
74e7ff8c
LJ
6098 if (ret != TRACE_TYPE_NO_CONSUME)
6099 trace_consume(iter);
34cd4998 6100 rem -= count;
955b61e5 6101 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
6102 rem = 0;
6103 iter->ent = NULL;
6104 break;
6105 }
6106 }
6107
6108 return rem;
6109}
6110
3c56819b
EGM
6111static ssize_t tracing_splice_read_pipe(struct file *filp,
6112 loff_t *ppos,
6113 struct pipe_inode_info *pipe,
6114 size_t len,
6115 unsigned int flags)
6116{
35f3d14d
JA
6117 struct page *pages_def[PIPE_DEF_BUFFERS];
6118 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
6119 struct trace_iterator *iter = filp->private_data;
6120 struct splice_pipe_desc spd = {
35f3d14d
JA
6121 .pages = pages_def,
6122 .partial = partial_def,
34cd4998 6123 .nr_pages = 0, /* This gets updated below. */
047fe360 6124 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
6125 .ops = &tracing_pipe_buf_ops,
6126 .spd_release = tracing_spd_release_pipe,
3c56819b
EGM
6127 };
6128 ssize_t ret;
34cd4998 6129 size_t rem;
3c56819b
EGM
6130 unsigned int i;
6131
35f3d14d
JA
6132 if (splice_grow_spd(pipe, &spd))
6133 return -ENOMEM;
6134
d7350c3f 6135 mutex_lock(&iter->mutex);
3c56819b
EGM
6136
6137 if (iter->trace->splice_read) {
6138 ret = iter->trace->splice_read(iter, filp,
6139 ppos, pipe, len, flags);
6140 if (ret)
34cd4998 6141 goto out_err;
3c56819b
EGM
6142 }
6143
6144 ret = tracing_wait_pipe(filp);
6145 if (ret <= 0)
34cd4998 6146 goto out_err;
3c56819b 6147
955b61e5 6148 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 6149 ret = -EFAULT;
34cd4998 6150 goto out_err;
3c56819b
EGM
6151 }
6152
4f535968 6153 trace_event_read_lock();
7e53bd42 6154 trace_access_lock(iter->cpu_file);
4f535968 6155
3c56819b 6156 /* Fill as many pages as possible. */
a786c06d 6157 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
35f3d14d
JA
6158 spd.pages[i] = alloc_page(GFP_KERNEL);
6159 if (!spd.pages[i])
34cd4998 6160 break;
3c56819b 6161
fa7c7f6e 6162 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
6163
6164 /* Copy the data into the page, so we can start over. */
6165 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 6166 page_address(spd.pages[i]),
5ac48378 6167 trace_seq_used(&iter->seq));
3c56819b 6168 if (ret < 0) {
35f3d14d 6169 __free_page(spd.pages[i]);
3c56819b
EGM
6170 break;
6171 }
35f3d14d 6172 spd.partial[i].offset = 0;
5ac48378 6173 spd.partial[i].len = trace_seq_used(&iter->seq);
3c56819b 6174
f9520750 6175 trace_seq_init(&iter->seq);
3c56819b
EGM
6176 }
6177
7e53bd42 6178 trace_access_unlock(iter->cpu_file);
4f535968 6179 trace_event_read_unlock();
d7350c3f 6180 mutex_unlock(&iter->mutex);
3c56819b
EGM
6181
6182 spd.nr_pages = i;
6183
a29054d9
SRRH
6184 if (i)
6185 ret = splice_to_pipe(pipe, &spd);
6186 else
6187 ret = 0;
35f3d14d 6188out:
047fe360 6189 splice_shrink_spd(&spd);
35f3d14d 6190 return ret;
3c56819b 6191
34cd4998 6192out_err:
d7350c3f 6193 mutex_unlock(&iter->mutex);
35f3d14d 6194 goto out;
3c56819b
EGM
6195}
6196
a98a3c3f
SR
6197static ssize_t
6198tracing_entries_read(struct file *filp, char __user *ubuf,
6199 size_t cnt, loff_t *ppos)
6200{
0bc392ee
ON
6201 struct inode *inode = file_inode(filp);
6202 struct trace_array *tr = inode->i_private;
6203 int cpu = tracing_get_cpu(inode);
438ced17
VN
6204 char buf[64];
6205 int r = 0;
6206 ssize_t ret;
a98a3c3f 6207
db526ca3 6208 mutex_lock(&trace_types_lock);
438ced17 6209
0bc392ee 6210 if (cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
6211 int cpu, buf_size_same;
6212 unsigned long size;
6213
6214 size = 0;
6215 buf_size_same = 1;
6216 /* check if all cpu sizes are same */
6217 for_each_tracing_cpu(cpu) {
6218 /* fill in the size from first enabled cpu */
6219 if (size == 0)
12883efb
SRRH
6220 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
6221 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
6222 buf_size_same = 0;
6223 break;
6224 }
6225 }
6226
6227 if (buf_size_same) {
6228 if (!ring_buffer_expanded)
6229 r = sprintf(buf, "%lu (expanded: %lu)\n",
6230 size >> 10,
6231 trace_buf_size >> 10);
6232 else
6233 r = sprintf(buf, "%lu\n", size >> 10);
6234 } else
6235 r = sprintf(buf, "X\n");
6236 } else
0bc392ee 6237 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
438ced17 6238
db526ca3
SR
6239 mutex_unlock(&trace_types_lock);
6240
438ced17
VN
6241 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6242 return ret;
a98a3c3f
SR
6243}
6244
6245static ssize_t
6246tracing_entries_write(struct file *filp, const char __user *ubuf,
6247 size_t cnt, loff_t *ppos)
6248{
0bc392ee
ON
6249 struct inode *inode = file_inode(filp);
6250 struct trace_array *tr = inode->i_private;
a98a3c3f 6251 unsigned long val;
4f271a2a 6252 int ret;
a98a3c3f 6253
22fe9b54
PH
6254 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6255 if (ret)
c6caeeb1 6256 return ret;
a98a3c3f
SR
6257
6258 /* must have at least 1 entry */
6259 if (!val)
6260 return -EINVAL;
6261
1696b2b0
SR
6262 /* value is in KB */
6263 val <<= 10;
0bc392ee 6264 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4f271a2a
VN
6265 if (ret < 0)
6266 return ret;
a98a3c3f 6267
cf8517cf 6268 *ppos += cnt;
a98a3c3f 6269
4f271a2a
VN
6270 return cnt;
6271}
bf5e6519 6272
f81ab074
VN
6273static ssize_t
6274tracing_total_entries_read(struct file *filp, char __user *ubuf,
6275 size_t cnt, loff_t *ppos)
6276{
6277 struct trace_array *tr = filp->private_data;
6278 char buf[64];
6279 int r, cpu;
6280 unsigned long size = 0, expanded_size = 0;
6281
6282 mutex_lock(&trace_types_lock);
6283 for_each_tracing_cpu(cpu) {
12883efb 6284 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
6285 if (!ring_buffer_expanded)
6286 expanded_size += trace_buf_size >> 10;
6287 }
6288 if (ring_buffer_expanded)
6289 r = sprintf(buf, "%lu\n", size);
6290 else
6291 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6292 mutex_unlock(&trace_types_lock);
6293
6294 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6295}
6296
4f271a2a
VN
6297static ssize_t
6298tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6299 size_t cnt, loff_t *ppos)
6300{
6301 /*
6302 * There is no need to read what the user has written, this function
6303 * is just to make sure that there is no error when "echo" is used
6304 */
6305
6306 *ppos += cnt;
a98a3c3f
SR
6307
6308 return cnt;
6309}
6310
4f271a2a
VN
6311static int
6312tracing_free_buffer_release(struct inode *inode, struct file *filp)
6313{
2b6080f2
SR
6314 struct trace_array *tr = inode->i_private;
6315
cf30cf67 6316 /* disable tracing ? */
983f938a 6317 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
711e1243 6318 tracer_tracing_off(tr);
4f271a2a 6319 /* resize the ring buffer to 0 */
2b6080f2 6320 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a 6321
7b85af63
SRRH
6322 trace_array_put(tr);
6323
4f271a2a
VN
6324 return 0;
6325}
6326
5bf9a1ee
PP
6327static ssize_t
6328tracing_mark_write(struct file *filp, const char __user *ubuf,
6329 size_t cnt, loff_t *fpos)
6330{
2d71619c 6331 struct trace_array *tr = filp->private_data;
d696b58c 6332 struct ring_buffer_event *event;
3dd80953 6333 enum event_trigger_type tt = ETT_NONE;
d696b58c
SR
6334 struct ring_buffer *buffer;
6335 struct print_entry *entry;
6336 unsigned long irq_flags;
d696b58c 6337 ssize_t written;
d696b58c
SR
6338 int size;
6339 int len;
fa32e855 6340
656c7f0d 6341/* Used in tracing_mark_raw_write() as well */
0f5e5a3a
RV
6342#define FAULTED_STR "<faulted>"
6343#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
5bf9a1ee 6344
c76f0694 6345 if (tracing_disabled)
5bf9a1ee
PP
6346 return -EINVAL;
6347
983f938a 6348 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5224c3a3
MSB
6349 return -EINVAL;
6350
5bf9a1ee
PP
6351 if (cnt > TRACE_BUF_SIZE)
6352 cnt = TRACE_BUF_SIZE;
6353
d696b58c 6354 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 6355
d696b58c 6356 local_save_flags(irq_flags);
656c7f0d 6357 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
d696b58c 6358
656c7f0d
SRRH
6359 /* If less than "<faulted>", then make sure we can still add that */
6360 if (cnt < FAULTED_SIZE)
6361 size += FAULTED_SIZE - cnt;
d696b58c 6362
2d71619c 6363 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
6364 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6365 irq_flags, preempt_count());
656c7f0d 6366 if (unlikely(!event))
d696b58c 6367 /* Ring buffer disabled, return as if not open for write */
656c7f0d 6368 return -EBADF;
d696b58c
SR
6369
6370 entry = ring_buffer_event_data(event);
6371 entry->ip = _THIS_IP_;
6372
656c7f0d
SRRH
6373 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6374 if (len) {
0f5e5a3a 6375 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
656c7f0d
SRRH
6376 cnt = FAULTED_SIZE;
6377 written = -EFAULT;
c13d2f7c 6378 } else
656c7f0d
SRRH
6379 written = cnt;
6380 len = cnt;
5bf9a1ee 6381
3dd80953
SRV
6382 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
6383 /* do not add \n before testing triggers, but add \0 */
6384 entry->buf[cnt] = '\0';
6385 tt = event_triggers_call(tr->trace_marker_file, entry, event);
6386 }
6387
d696b58c
SR
6388 if (entry->buf[cnt - 1] != '\n') {
6389 entry->buf[cnt] = '\n';
6390 entry->buf[cnt + 1] = '\0';
6391 } else
6392 entry->buf[cnt] = '\0';
6393
7ffbd48d 6394 __buffer_unlock_commit(buffer, event);
5bf9a1ee 6395
3dd80953
SRV
6396 if (tt)
6397 event_triggers_post_call(tr->trace_marker_file, tt);
6398
656c7f0d
SRRH
6399 if (written > 0)
6400 *fpos += written;
5bf9a1ee 6401
fa32e855
SR
6402 return written;
6403}
6404
6405/* Limit it for now to 3K (including tag) */
6406#define RAW_DATA_MAX_SIZE (1024*3)
6407
6408static ssize_t
6409tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6410 size_t cnt, loff_t *fpos)
6411{
6412 struct trace_array *tr = filp->private_data;
6413 struct ring_buffer_event *event;
6414 struct ring_buffer *buffer;
6415 struct raw_data_entry *entry;
6416 unsigned long irq_flags;
fa32e855 6417 ssize_t written;
fa32e855
SR
6418 int size;
6419 int len;
6420
656c7f0d
SRRH
6421#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6422
fa32e855
SR
6423 if (tracing_disabled)
6424 return -EINVAL;
6425
6426 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6427 return -EINVAL;
6428
6429 /* The marker must at least have a tag id */
6430 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6431 return -EINVAL;
6432
6433 if (cnt > TRACE_BUF_SIZE)
6434 cnt = TRACE_BUF_SIZE;
6435
6436 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6437
fa32e855
SR
6438 local_save_flags(irq_flags);
6439 size = sizeof(*entry) + cnt;
656c7f0d
SRRH
6440 if (cnt < FAULT_SIZE_ID)
6441 size += FAULT_SIZE_ID - cnt;
6442
fa32e855 6443 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
6444 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6445 irq_flags, preempt_count());
656c7f0d 6446 if (!event)
fa32e855 6447 /* Ring buffer disabled, return as if not open for write */
656c7f0d 6448 return -EBADF;
fa32e855
SR
6449
6450 entry = ring_buffer_event_data(event);
6451
656c7f0d
SRRH
6452 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6453 if (len) {
6454 entry->id = -1;
0f5e5a3a 6455 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
656c7f0d 6456 written = -EFAULT;
fa32e855 6457 } else
656c7f0d 6458 written = cnt;
fa32e855
SR
6459
6460 __buffer_unlock_commit(buffer, event);
6461
656c7f0d
SRRH
6462 if (written > 0)
6463 *fpos += written;
1aa54bca
MS
6464
6465 return written;
5bf9a1ee
PP
6466}
6467
13f16d20 6468static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 6469{
2b6080f2 6470 struct trace_array *tr = m->private;
5079f326
Z
6471 int i;
6472
6473 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 6474 seq_printf(m,
5079f326 6475 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
6476 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6477 i == tr->clock_id ? "]" : "");
13f16d20 6478 seq_putc(m, '\n');
5079f326 6479
13f16d20 6480 return 0;
5079f326
Z
6481}
6482
d71bd34d 6483int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5079f326 6484{
5079f326
Z
6485 int i;
6486
5079f326
Z
6487 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6488 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6489 break;
6490 }
6491 if (i == ARRAY_SIZE(trace_clocks))
6492 return -EINVAL;
6493
5079f326
Z
6494 mutex_lock(&trace_types_lock);
6495
2b6080f2
SR
6496 tr->clock_id = i;
6497
12883efb 6498 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 6499
60303ed3
DS
6500 /*
6501 * New clock may not be consistent with the previous clock.
6502 * Reset the buffer so that it doesn't have incomparable timestamps.
6503 */
9457158b 6504 tracing_reset_online_cpus(&tr->trace_buffer);
12883efb
SRRH
6505
6506#ifdef CONFIG_TRACER_MAX_TRACE
170b3b10 6507 if (tr->max_buffer.buffer)
12883efb 6508 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
9457158b 6509 tracing_reset_online_cpus(&tr->max_buffer);
12883efb 6510#endif
60303ed3 6511
5079f326
Z
6512 mutex_unlock(&trace_types_lock);
6513
e1e232ca
SR
6514 return 0;
6515}
6516
6517static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6518 size_t cnt, loff_t *fpos)
6519{
6520 struct seq_file *m = filp->private_data;
6521 struct trace_array *tr = m->private;
6522 char buf[64];
6523 const char *clockstr;
6524 int ret;
6525
6526 if (cnt >= sizeof(buf))
6527 return -EINVAL;
6528
4afe6495 6529 if (copy_from_user(buf, ubuf, cnt))
e1e232ca
SR
6530 return -EFAULT;
6531
6532 buf[cnt] = 0;
6533
6534 clockstr = strstrip(buf);
6535
6536 ret = tracing_set_clock(tr, clockstr);
6537 if (ret)
6538 return ret;
6539
5079f326
Z
6540 *fpos += cnt;
6541
6542 return cnt;
6543}
6544
13f16d20
LZ
6545static int tracing_clock_open(struct inode *inode, struct file *file)
6546{
7b85af63
SRRH
6547 struct trace_array *tr = inode->i_private;
6548 int ret;
6549
13f16d20
LZ
6550 if (tracing_disabled)
6551 return -ENODEV;
2b6080f2 6552
7b85af63
SRRH
6553 if (trace_array_get(tr))
6554 return -ENODEV;
6555
6556 ret = single_open(file, tracing_clock_show, inode->i_private);
6557 if (ret < 0)
6558 trace_array_put(tr);
6559
6560 return ret;
13f16d20
LZ
6561}
6562
2c1ea60b
TZ
6563static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
6564{
6565 struct trace_array *tr = m->private;
6566
6567 mutex_lock(&trace_types_lock);
6568
6569 if (ring_buffer_time_stamp_abs(tr->trace_buffer.buffer))
6570 seq_puts(m, "delta [absolute]\n");
6571 else
6572 seq_puts(m, "[delta] absolute\n");
6573
6574 mutex_unlock(&trace_types_lock);
6575
6576 return 0;
6577}
6578
6579static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
6580{
6581 struct trace_array *tr = inode->i_private;
6582 int ret;
6583
6584 if (tracing_disabled)
6585 return -ENODEV;
6586
6587 if (trace_array_get(tr))
6588 return -ENODEV;
6589
6590 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
6591 if (ret < 0)
6592 trace_array_put(tr);
6593
6594 return ret;
6595}
6596
00b41452
TZ
6597int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
6598{
6599 int ret = 0;
6600
6601 mutex_lock(&trace_types_lock);
6602
6603 if (abs && tr->time_stamp_abs_ref++)
6604 goto out;
6605
6606 if (!abs) {
6607 if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
6608 ret = -EINVAL;
6609 goto out;
6610 }
6611
6612 if (--tr->time_stamp_abs_ref)
6613 goto out;
6614 }
6615
6616 ring_buffer_set_time_stamp_abs(tr->trace_buffer.buffer, abs);
6617
6618#ifdef CONFIG_TRACER_MAX_TRACE
6619 if (tr->max_buffer.buffer)
6620 ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
6621#endif
6622 out:
6623 mutex_unlock(&trace_types_lock);
6624
6625 return ret;
6626}
6627
6de58e62
SRRH
6628struct ftrace_buffer_info {
6629 struct trace_iterator iter;
6630 void *spare;
73a757e6 6631 unsigned int spare_cpu;
6de58e62
SRRH
6632 unsigned int read;
6633};
6634
debdd57f
HT
6635#ifdef CONFIG_TRACER_SNAPSHOT
6636static int tracing_snapshot_open(struct inode *inode, struct file *file)
6637{
6484c71c 6638 struct trace_array *tr = inode->i_private;
debdd57f 6639 struct trace_iterator *iter;
2b6080f2 6640 struct seq_file *m;
debdd57f
HT
6641 int ret = 0;
6642
ff451961
SRRH
6643 if (trace_array_get(tr) < 0)
6644 return -ENODEV;
6645
debdd57f 6646 if (file->f_mode & FMODE_READ) {
6484c71c 6647 iter = __tracing_open(inode, file, true);
debdd57f
HT
6648 if (IS_ERR(iter))
6649 ret = PTR_ERR(iter);
2b6080f2
SR
6650 } else {
6651 /* Writes still need the seq_file to hold the private data */
f77d09a3 6652 ret = -ENOMEM;
2b6080f2
SR
6653 m = kzalloc(sizeof(*m), GFP_KERNEL);
6654 if (!m)
f77d09a3 6655 goto out;
2b6080f2
SR
6656 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6657 if (!iter) {
6658 kfree(m);
f77d09a3 6659 goto out;
2b6080f2 6660 }
f77d09a3
AL
6661 ret = 0;
6662
ff451961 6663 iter->tr = tr;
6484c71c
ON
6664 iter->trace_buffer = &tr->max_buffer;
6665 iter->cpu_file = tracing_get_cpu(inode);
2b6080f2
SR
6666 m->private = iter;
6667 file->private_data = m;
debdd57f 6668 }
f77d09a3 6669out:
ff451961
SRRH
6670 if (ret < 0)
6671 trace_array_put(tr);
6672
debdd57f
HT
6673 return ret;
6674}
6675
6676static ssize_t
6677tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6678 loff_t *ppos)
6679{
2b6080f2
SR
6680 struct seq_file *m = filp->private_data;
6681 struct trace_iterator *iter = m->private;
6682 struct trace_array *tr = iter->tr;
debdd57f
HT
6683 unsigned long val;
6684 int ret;
6685
6686 ret = tracing_update_buffers();
6687 if (ret < 0)
6688 return ret;
6689
6690 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6691 if (ret)
6692 return ret;
6693
6694 mutex_lock(&trace_types_lock);
6695
2b6080f2 6696 if (tr->current_trace->use_max_tr) {
debdd57f
HT
6697 ret = -EBUSY;
6698 goto out;
6699 }
6700
a35873a0
TZ
6701 arch_spin_lock(&tr->max_lock);
6702 if (tr->cond_snapshot)
6703 ret = -EBUSY;
6704 arch_spin_unlock(&tr->max_lock);
6705 if (ret)
6706 goto out;
6707
debdd57f
HT
6708 switch (val) {
6709 case 0:
f1affcaa
SRRH
6710 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6711 ret = -EINVAL;
6712 break;
debdd57f 6713 }
3209cff4
SRRH
6714 if (tr->allocated_snapshot)
6715 free_snapshot(tr);
debdd57f
HT
6716 break;
6717 case 1:
f1affcaa
SRRH
6718/* Only allow per-cpu swap if the ring buffer supports it */
6719#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6720 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6721 ret = -EINVAL;
6722 break;
6723 }
6724#endif
46cc0b44
ET
6725 if (tr->allocated_snapshot)
6726 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6727 &tr->trace_buffer, iter->cpu_file);
6728 else
2824f503 6729 ret = tracing_alloc_snapshot_instance(tr);
46cc0b44
ET
6730 if (ret < 0)
6731 break;
debdd57f
HT
6732 local_irq_disable();
6733 /* Now, we're going to swap */
f1affcaa 6734 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
a35873a0 6735 update_max_tr(tr, current, smp_processor_id(), NULL);
f1affcaa 6736 else
ce9bae55 6737 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
6738 local_irq_enable();
6739 break;
6740 default:
45ad21ca 6741 if (tr->allocated_snapshot) {
f1affcaa
SRRH
6742 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6743 tracing_reset_online_cpus(&tr->max_buffer);
6744 else
6745 tracing_reset(&tr->max_buffer, iter->cpu_file);
6746 }
debdd57f
HT
6747 break;
6748 }
6749
6750 if (ret >= 0) {
6751 *ppos += cnt;
6752 ret = cnt;
6753 }
6754out:
6755 mutex_unlock(&trace_types_lock);
6756 return ret;
6757}
2b6080f2
SR
6758
6759static int tracing_snapshot_release(struct inode *inode, struct file *file)
6760{
6761 struct seq_file *m = file->private_data;
ff451961
SRRH
6762 int ret;
6763
6764 ret = tracing_release(inode, file);
2b6080f2
SR
6765
6766 if (file->f_mode & FMODE_READ)
ff451961 6767 return ret;
2b6080f2
SR
6768
6769 /* If write only, the seq_file is just a stub */
6770 if (m)
6771 kfree(m->private);
6772 kfree(m);
6773
6774 return 0;
6775}
6776
6de58e62
SRRH
6777static int tracing_buffers_open(struct inode *inode, struct file *filp);
6778static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6779 size_t count, loff_t *ppos);
6780static int tracing_buffers_release(struct inode *inode, struct file *file);
6781static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6782 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6783
6784static int snapshot_raw_open(struct inode *inode, struct file *filp)
6785{
6786 struct ftrace_buffer_info *info;
6787 int ret;
6788
6789 ret = tracing_buffers_open(inode, filp);
6790 if (ret < 0)
6791 return ret;
6792
6793 info = filp->private_data;
6794
6795 if (info->iter.trace->use_max_tr) {
6796 tracing_buffers_release(inode, filp);
6797 return -EBUSY;
6798 }
6799
6800 info->iter.snapshot = true;
6801 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6802
6803 return ret;
6804}
6805
debdd57f
HT
6806#endif /* CONFIG_TRACER_SNAPSHOT */
6807
6808
6508fa76
SF
6809static const struct file_operations tracing_thresh_fops = {
6810 .open = tracing_open_generic,
6811 .read = tracing_thresh_read,
6812 .write = tracing_thresh_write,
6813 .llseek = generic_file_llseek,
6814};
6815
f971cc9a 6816#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5e2336a0 6817static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
6818 .open = tracing_open_generic,
6819 .read = tracing_max_lat_read,
6820 .write = tracing_max_lat_write,
b444786f 6821 .llseek = generic_file_llseek,
bc0c38d1 6822};
e428abbb 6823#endif
bc0c38d1 6824
5e2336a0 6825static const struct file_operations set_tracer_fops = {
4bf39a94
IM
6826 .open = tracing_open_generic,
6827 .read = tracing_set_trace_read,
6828 .write = tracing_set_trace_write,
b444786f 6829 .llseek = generic_file_llseek,
bc0c38d1
SR
6830};
6831
5e2336a0 6832static const struct file_operations tracing_pipe_fops = {
4bf39a94 6833 .open = tracing_open_pipe,
2a2cc8f7 6834 .poll = tracing_poll_pipe,
4bf39a94 6835 .read = tracing_read_pipe,
3c56819b 6836 .splice_read = tracing_splice_read_pipe,
4bf39a94 6837 .release = tracing_release_pipe,
b444786f 6838 .llseek = no_llseek,
b3806b43
SR
6839};
6840
5e2336a0 6841static const struct file_operations tracing_entries_fops = {
0bc392ee 6842 .open = tracing_open_generic_tr,
a98a3c3f
SR
6843 .read = tracing_entries_read,
6844 .write = tracing_entries_write,
b444786f 6845 .llseek = generic_file_llseek,
0bc392ee 6846 .release = tracing_release_generic_tr,
a98a3c3f
SR
6847};
6848
f81ab074 6849static const struct file_operations tracing_total_entries_fops = {
7b85af63 6850 .open = tracing_open_generic_tr,
f81ab074
VN
6851 .read = tracing_total_entries_read,
6852 .llseek = generic_file_llseek,
7b85af63 6853 .release = tracing_release_generic_tr,
f81ab074
VN
6854};
6855
4f271a2a 6856static const struct file_operations tracing_free_buffer_fops = {
7b85af63 6857 .open = tracing_open_generic_tr,
4f271a2a
VN
6858 .write = tracing_free_buffer_write,
6859 .release = tracing_free_buffer_release,
6860};
6861
5e2336a0 6862static const struct file_operations tracing_mark_fops = {
7b85af63 6863 .open = tracing_open_generic_tr,
5bf9a1ee 6864 .write = tracing_mark_write,
b444786f 6865 .llseek = generic_file_llseek,
7b85af63 6866 .release = tracing_release_generic_tr,
5bf9a1ee
PP
6867};
6868
fa32e855
SR
6869static const struct file_operations tracing_mark_raw_fops = {
6870 .open = tracing_open_generic_tr,
6871 .write = tracing_mark_raw_write,
6872 .llseek = generic_file_llseek,
6873 .release = tracing_release_generic_tr,
6874};
6875
5079f326 6876static const struct file_operations trace_clock_fops = {
13f16d20
LZ
6877 .open = tracing_clock_open,
6878 .read = seq_read,
6879 .llseek = seq_lseek,
7b85af63 6880 .release = tracing_single_release_tr,
5079f326
Z
6881 .write = tracing_clock_write,
6882};
6883
2c1ea60b
TZ
6884static const struct file_operations trace_time_stamp_mode_fops = {
6885 .open = tracing_time_stamp_mode_open,
6886 .read = seq_read,
6887 .llseek = seq_lseek,
6888 .release = tracing_single_release_tr,
6889};
6890
debdd57f
HT
6891#ifdef CONFIG_TRACER_SNAPSHOT
6892static const struct file_operations snapshot_fops = {
6893 .open = tracing_snapshot_open,
6894 .read = seq_read,
6895 .write = tracing_snapshot_write,
098c879e 6896 .llseek = tracing_lseek,
2b6080f2 6897 .release = tracing_snapshot_release,
debdd57f 6898};
debdd57f 6899
6de58e62
SRRH
6900static const struct file_operations snapshot_raw_fops = {
6901 .open = snapshot_raw_open,
6902 .read = tracing_buffers_read,
6903 .release = tracing_buffers_release,
6904 .splice_read = tracing_buffers_splice_read,
6905 .llseek = no_llseek,
2cadf913
SR
6906};
6907
6de58e62
SRRH
6908#endif /* CONFIG_TRACER_SNAPSHOT */
6909
8a062902
TZ
6910#define TRACING_LOG_ERRS_MAX 8
6911#define TRACING_LOG_LOC_MAX 128
6912
6913#define CMD_PREFIX " Command: "
6914
6915struct err_info {
6916 const char **errs; /* ptr to loc-specific array of err strings */
6917 u8 type; /* index into errs -> specific err string */
6918 u8 pos; /* MAX_FILTER_STR_VAL = 256 */
6919 u64 ts;
6920};
6921
6922struct tracing_log_err {
6923 struct list_head list;
6924 struct err_info info;
6925 char loc[TRACING_LOG_LOC_MAX]; /* err location */
6926 char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
6927};
6928
8a062902
TZ
6929static DEFINE_MUTEX(tracing_err_log_lock);
6930
ff585c5b 6931static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
8a062902
TZ
6932{
6933 struct tracing_log_err *err;
6934
2f754e77 6935 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
8a062902
TZ
6936 err = kzalloc(sizeof(*err), GFP_KERNEL);
6937 if (!err)
6938 err = ERR_PTR(-ENOMEM);
2f754e77 6939 tr->n_err_log_entries++;
8a062902
TZ
6940
6941 return err;
6942 }
6943
2f754e77 6944 err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
8a062902
TZ
6945 list_del(&err->list);
6946
6947 return err;
6948}
6949
6950/**
6951 * err_pos - find the position of a string within a command for error careting
6952 * @cmd: The tracing command that caused the error
6953 * @str: The string to position the caret at within @cmd
6954 *
6955 * Finds the position of the first occurence of @str within @cmd. The
6956 * return value can be passed to tracing_log_err() for caret placement
6957 * within @cmd.
6958 *
6959 * Returns the index within @cmd of the first occurence of @str or 0
6960 * if @str was not found.
6961 */
6962unsigned int err_pos(char *cmd, const char *str)
6963{
6964 char *found;
6965
6966 if (WARN_ON(!strlen(cmd)))
6967 return 0;
6968
6969 found = strstr(cmd, str);
6970 if (found)
6971 return found - cmd;
6972
6973 return 0;
6974}
6975
6976/**
6977 * tracing_log_err - write an error to the tracing error log
2f754e77 6978 * @tr: The associated trace array for the error (NULL for top level array)
8a062902
TZ
6979 * @loc: A string describing where the error occurred
6980 * @cmd: The tracing command that caused the error
6981 * @errs: The array of loc-specific static error strings
6982 * @type: The index into errs[], which produces the specific static err string
6983 * @pos: The position the caret should be placed in the cmd
6984 *
6985 * Writes an error into tracing/error_log of the form:
6986 *
6987 * <loc>: error: <text>
6988 * Command: <cmd>
6989 * ^
6990 *
6991 * tracing/error_log is a small log file containing the last
6992 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
6993 * unless there has been a tracing error, and the error log can be
6994 * cleared and have its memory freed by writing the empty string in
6995 * truncation mode to it i.e. echo > tracing/error_log.
6996 *
6997 * NOTE: the @errs array along with the @type param are used to
6998 * produce a static error string - this string is not copied and saved
6999 * when the error is logged - only a pointer to it is saved. See
7000 * existing callers for examples of how static strings are typically
7001 * defined for use with tracing_log_err().
7002 */
2f754e77
SRV
7003void tracing_log_err(struct trace_array *tr,
7004 const char *loc, const char *cmd,
8a062902
TZ
7005 const char **errs, u8 type, u8 pos)
7006{
7007 struct tracing_log_err *err;
7008
2f754e77
SRV
7009 if (!tr)
7010 tr = &global_trace;
7011
8a062902 7012 mutex_lock(&tracing_err_log_lock);
2f754e77 7013 err = get_tracing_log_err(tr);
8a062902
TZ
7014 if (PTR_ERR(err) == -ENOMEM) {
7015 mutex_unlock(&tracing_err_log_lock);
7016 return;
7017 }
7018
7019 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
7020 snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
7021
7022 err->info.errs = errs;
7023 err->info.type = type;
7024 err->info.pos = pos;
7025 err->info.ts = local_clock();
7026
2f754e77 7027 list_add_tail(&err->list, &tr->err_log);
8a062902
TZ
7028 mutex_unlock(&tracing_err_log_lock);
7029}
7030
2f754e77 7031static void clear_tracing_err_log(struct trace_array *tr)
8a062902
TZ
7032{
7033 struct tracing_log_err *err, *next;
7034
7035 mutex_lock(&tracing_err_log_lock);
2f754e77 7036 list_for_each_entry_safe(err, next, &tr->err_log, list) {
8a062902
TZ
7037 list_del(&err->list);
7038 kfree(err);
7039 }
7040
2f754e77 7041 tr->n_err_log_entries = 0;
8a062902
TZ
7042 mutex_unlock(&tracing_err_log_lock);
7043}
7044
7045static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7046{
2f754e77
SRV
7047 struct trace_array *tr = m->private;
7048
8a062902
TZ
7049 mutex_lock(&tracing_err_log_lock);
7050
2f754e77 7051 return seq_list_start(&tr->err_log, *pos);
8a062902
TZ
7052}
7053
7054static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7055{
2f754e77
SRV
7056 struct trace_array *tr = m->private;
7057
7058 return seq_list_next(v, &tr->err_log, pos);
8a062902
TZ
7059}
7060
7061static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7062{
7063 mutex_unlock(&tracing_err_log_lock);
7064}
7065
7066static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
7067{
7068 u8 i;
7069
7070 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7071 seq_putc(m, ' ');
7072 for (i = 0; i < pos; i++)
7073 seq_putc(m, ' ');
7074 seq_puts(m, "^\n");
7075}
7076
7077static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7078{
7079 struct tracing_log_err *err = v;
7080
7081 if (err) {
7082 const char *err_text = err->info.errs[err->info.type];
7083 u64 sec = err->info.ts;
7084 u32 nsec;
7085
7086 nsec = do_div(sec, NSEC_PER_SEC);
7087 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
7088 err->loc, err_text);
7089 seq_printf(m, "%s", err->cmd);
7090 tracing_err_log_show_pos(m, err->info.pos);
7091 }
7092
7093 return 0;
7094}
7095
7096static const struct seq_operations tracing_err_log_seq_ops = {
7097 .start = tracing_err_log_seq_start,
7098 .next = tracing_err_log_seq_next,
7099 .stop = tracing_err_log_seq_stop,
7100 .show = tracing_err_log_seq_show
7101};
7102
7103static int tracing_err_log_open(struct inode *inode, struct file *file)
7104{
2f754e77 7105 struct trace_array *tr = inode->i_private;
8a062902
TZ
7106 int ret = 0;
7107
2f754e77
SRV
7108 if (trace_array_get(tr) < 0)
7109 return -ENODEV;
7110
8a062902
TZ
7111 /* If this file was opened for write, then erase contents */
7112 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
2f754e77 7113 clear_tracing_err_log(tr);
8a062902 7114
2f754e77 7115 if (file->f_mode & FMODE_READ) {
8a062902 7116 ret = seq_open(file, &tracing_err_log_seq_ops);
2f754e77
SRV
7117 if (!ret) {
7118 struct seq_file *m = file->private_data;
7119 m->private = tr;
7120 } else {
7121 trace_array_put(tr);
7122 }
7123 }
8a062902
TZ
7124 return ret;
7125}
7126
7127static ssize_t tracing_err_log_write(struct file *file,
7128 const char __user *buffer,
7129 size_t count, loff_t *ppos)
7130{
7131 return count;
7132}
7133
d122ed62
TM
7134static int tracing_err_log_release(struct inode *inode, struct file *file)
7135{
7136 struct trace_array *tr = inode->i_private;
7137
7138 trace_array_put(tr);
7139
7140 if (file->f_mode & FMODE_READ)
7141 seq_release(inode, file);
7142
7143 return 0;
7144}
7145
8a062902
TZ
7146static const struct file_operations tracing_err_log_fops = {
7147 .open = tracing_err_log_open,
7148 .write = tracing_err_log_write,
7149 .read = seq_read,
7150 .llseek = seq_lseek,
d122ed62 7151 .release = tracing_err_log_release,
8a062902
TZ
7152};
7153
2cadf913
SR
7154static int tracing_buffers_open(struct inode *inode, struct file *filp)
7155{
46ef2be0 7156 struct trace_array *tr = inode->i_private;
2cadf913 7157 struct ftrace_buffer_info *info;
7b85af63 7158 int ret;
2cadf913
SR
7159
7160 if (tracing_disabled)
7161 return -ENODEV;
7162
7b85af63
SRRH
7163 if (trace_array_get(tr) < 0)
7164 return -ENODEV;
7165
2cadf913 7166 info = kzalloc(sizeof(*info), GFP_KERNEL);
7b85af63
SRRH
7167 if (!info) {
7168 trace_array_put(tr);
2cadf913 7169 return -ENOMEM;
7b85af63 7170 }
2cadf913 7171
a695cb58
SRRH
7172 mutex_lock(&trace_types_lock);
7173
cc60cdc9 7174 info->iter.tr = tr;
46ef2be0 7175 info->iter.cpu_file = tracing_get_cpu(inode);
b627344f 7176 info->iter.trace = tr->current_trace;
12883efb 7177 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 7178 info->spare = NULL;
2cadf913 7179 /* Force reading ring buffer for first read */
cc60cdc9 7180 info->read = (unsigned int)-1;
2cadf913
SR
7181
7182 filp->private_data = info;
7183
cf6ab6d9
SRRH
7184 tr->current_trace->ref++;
7185
a695cb58
SRRH
7186 mutex_unlock(&trace_types_lock);
7187
7b85af63
SRRH
7188 ret = nonseekable_open(inode, filp);
7189 if (ret < 0)
7190 trace_array_put(tr);
7191
7192 return ret;
2cadf913
SR
7193}
7194
9dd95748 7195static __poll_t
cc60cdc9
SR
7196tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7197{
7198 struct ftrace_buffer_info *info = filp->private_data;
7199 struct trace_iterator *iter = &info->iter;
7200
7201 return trace_poll(iter, filp, poll_table);
7202}
7203
2cadf913
SR
7204static ssize_t
7205tracing_buffers_read(struct file *filp, char __user *ubuf,
7206 size_t count, loff_t *ppos)
7207{
7208 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 7209 struct trace_iterator *iter = &info->iter;
a7e52ad7 7210 ssize_t ret = 0;
6de58e62 7211 ssize_t size;
2cadf913 7212
2dc5d12b
SR
7213 if (!count)
7214 return 0;
7215
6de58e62 7216#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
7217 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7218 return -EBUSY;
6de58e62
SRRH
7219#endif
7220
73a757e6 7221 if (!info->spare) {
12883efb
SRRH
7222 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
7223 iter->cpu_file);
a7e52ad7
SRV
7224 if (IS_ERR(info->spare)) {
7225 ret = PTR_ERR(info->spare);
7226 info->spare = NULL;
7227 } else {
7228 info->spare_cpu = iter->cpu_file;
7229 }
73a757e6 7230 }
ddd538f3 7231 if (!info->spare)
a7e52ad7 7232 return ret;
ddd538f3 7233
2cadf913
SR
7234 /* Do we have previous read data to read? */
7235 if (info->read < PAGE_SIZE)
7236 goto read;
7237
b627344f 7238 again:
cc60cdc9 7239 trace_access_lock(iter->cpu_file);
12883efb 7240 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
7241 &info->spare,
7242 count,
cc60cdc9
SR
7243 iter->cpu_file, 0);
7244 trace_access_unlock(iter->cpu_file);
2cadf913 7245
b627344f
SR
7246 if (ret < 0) {
7247 if (trace_empty(iter)) {
d716ff71
SRRH
7248 if ((filp->f_flags & O_NONBLOCK))
7249 return -EAGAIN;
7250
2c2b0a78 7251 ret = wait_on_pipe(iter, 0);
d716ff71
SRRH
7252 if (ret)
7253 return ret;
7254
b627344f
SR
7255 goto again;
7256 }
d716ff71 7257 return 0;
b627344f 7258 }
436fc280 7259
436fc280 7260 info->read = 0;
b627344f 7261 read:
2cadf913
SR
7262 size = PAGE_SIZE - info->read;
7263 if (size > count)
7264 size = count;
7265
7266 ret = copy_to_user(ubuf, info->spare + info->read, size);
d716ff71
SRRH
7267 if (ret == size)
7268 return -EFAULT;
7269
2dc5d12b
SR
7270 size -= ret;
7271
2cadf913
SR
7272 *ppos += size;
7273 info->read += size;
7274
7275 return size;
7276}
7277
7278static int tracing_buffers_release(struct inode *inode, struct file *file)
7279{
7280 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 7281 struct trace_iterator *iter = &info->iter;
2cadf913 7282
a695cb58
SRRH
7283 mutex_lock(&trace_types_lock);
7284
cf6ab6d9
SRRH
7285 iter->tr->current_trace->ref--;
7286
ff451961 7287 __trace_array_put(iter->tr);
2cadf913 7288
ddd538f3 7289 if (info->spare)
73a757e6
SRV
7290 ring_buffer_free_read_page(iter->trace_buffer->buffer,
7291 info->spare_cpu, info->spare);
2cadf913
SR
7292 kfree(info);
7293
a695cb58
SRRH
7294 mutex_unlock(&trace_types_lock);
7295
2cadf913
SR
7296 return 0;
7297}
7298
7299struct buffer_ref {
7300 struct ring_buffer *buffer;
7301 void *page;
73a757e6 7302 int cpu;
b9872226 7303 refcount_t refcount;
2cadf913
SR
7304};
7305
b9872226
JH
7306static void buffer_ref_release(struct buffer_ref *ref)
7307{
7308 if (!refcount_dec_and_test(&ref->refcount))
7309 return;
7310 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
7311 kfree(ref);
7312}
7313
2cadf913
SR
7314static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
7315 struct pipe_buffer *buf)
7316{
7317 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7318
b9872226 7319 buffer_ref_release(ref);
2cadf913
SR
7320 buf->private = 0;
7321}
7322
15fab63e 7323static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
2cadf913
SR
7324 struct pipe_buffer *buf)
7325{
7326 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
7327
e9e1a2e7 7328 if (refcount_read(&ref->refcount) > INT_MAX/2)
15fab63e
MW
7329 return false;
7330
b9872226 7331 refcount_inc(&ref->refcount);
15fab63e 7332 return true;
2cadf913
SR
7333}
7334
7335/* Pipe buffer operations for a buffer. */
28dfef8f 7336static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913
SR
7337 .confirm = generic_pipe_buf_confirm,
7338 .release = buffer_pipe_buf_release,
b9872226 7339 .steal = generic_pipe_buf_nosteal,
2cadf913
SR
7340 .get = buffer_pipe_buf_get,
7341};
7342
7343/*
7344 * Callback from splice_to_pipe(), if we need to release some pages
7345 * at the end of the spd in case we error'ed out in filling the pipe.
7346 */
7347static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
7348{
7349 struct buffer_ref *ref =
7350 (struct buffer_ref *)spd->partial[i].private;
7351
b9872226 7352 buffer_ref_release(ref);
2cadf913
SR
7353 spd->partial[i].private = 0;
7354}
7355
7356static ssize_t
7357tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7358 struct pipe_inode_info *pipe, size_t len,
7359 unsigned int flags)
7360{
7361 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 7362 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
7363 struct partial_page partial_def[PIPE_DEF_BUFFERS];
7364 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 7365 struct splice_pipe_desc spd = {
35f3d14d
JA
7366 .pages = pages_def,
7367 .partial = partial_def,
047fe360 7368 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
7369 .ops = &buffer_pipe_buf_ops,
7370 .spd_release = buffer_spd_release,
7371 };
7372 struct buffer_ref *ref;
6b7e633f 7373 int entries, i;
07906da7 7374 ssize_t ret = 0;
2cadf913 7375
6de58e62 7376#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
7377 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7378 return -EBUSY;
6de58e62
SRRH
7379#endif
7380
d716ff71
SRRH
7381 if (*ppos & (PAGE_SIZE - 1))
7382 return -EINVAL;
93cfb3c9
LJ
7383
7384 if (len & (PAGE_SIZE - 1)) {
d716ff71
SRRH
7385 if (len < PAGE_SIZE)
7386 return -EINVAL;
93cfb3c9
LJ
7387 len &= PAGE_MASK;
7388 }
7389
1ae2293d
AV
7390 if (splice_grow_spd(pipe, &spd))
7391 return -ENOMEM;
7392
cc60cdc9
SR
7393 again:
7394 trace_access_lock(iter->cpu_file);
12883efb 7395 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 7396
a786c06d 7397 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
7398 struct page *page;
7399 int r;
7400
7401 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
07906da7
RV
7402 if (!ref) {
7403 ret = -ENOMEM;
2cadf913 7404 break;
07906da7 7405 }
2cadf913 7406
b9872226 7407 refcount_set(&ref->refcount, 1);
12883efb 7408 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 7409 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
a7e52ad7
SRV
7410 if (IS_ERR(ref->page)) {
7411 ret = PTR_ERR(ref->page);
7412 ref->page = NULL;
2cadf913
SR
7413 kfree(ref);
7414 break;
7415 }
73a757e6 7416 ref->cpu = iter->cpu_file;
2cadf913
SR
7417
7418 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 7419 len, iter->cpu_file, 1);
2cadf913 7420 if (r < 0) {
73a757e6
SRV
7421 ring_buffer_free_read_page(ref->buffer, ref->cpu,
7422 ref->page);
2cadf913
SR
7423 kfree(ref);
7424 break;
7425 }
7426
2cadf913
SR
7427 page = virt_to_page(ref->page);
7428
7429 spd.pages[i] = page;
7430 spd.partial[i].len = PAGE_SIZE;
7431 spd.partial[i].offset = 0;
7432 spd.partial[i].private = (unsigned long)ref;
7433 spd.nr_pages++;
93cfb3c9 7434 *ppos += PAGE_SIZE;
93459c6c 7435
12883efb 7436 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
7437 }
7438
cc60cdc9 7439 trace_access_unlock(iter->cpu_file);
2cadf913
SR
7440 spd.nr_pages = i;
7441
7442 /* did we read anything? */
7443 if (!spd.nr_pages) {
07906da7 7444 if (ret)
1ae2293d 7445 goto out;
d716ff71 7446
1ae2293d 7447 ret = -EAGAIN;
d716ff71 7448 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
1ae2293d 7449 goto out;
07906da7 7450
03329f99 7451 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
8b8b3683 7452 if (ret)
1ae2293d 7453 goto out;
e30f53aa 7454
cc60cdc9 7455 goto again;
2cadf913
SR
7456 }
7457
7458 ret = splice_to_pipe(pipe, &spd);
1ae2293d 7459out:
047fe360 7460 splice_shrink_spd(&spd);
6de58e62 7461
2cadf913
SR
7462 return ret;
7463}
7464
7465static const struct file_operations tracing_buffers_fops = {
7466 .open = tracing_buffers_open,
7467 .read = tracing_buffers_read,
cc60cdc9 7468 .poll = tracing_buffers_poll,
2cadf913
SR
7469 .release = tracing_buffers_release,
7470 .splice_read = tracing_buffers_splice_read,
7471 .llseek = no_llseek,
7472};
7473
c8d77183
SR
7474static ssize_t
7475tracing_stats_read(struct file *filp, char __user *ubuf,
7476 size_t count, loff_t *ppos)
7477{
4d3435b8
ON
7478 struct inode *inode = file_inode(filp);
7479 struct trace_array *tr = inode->i_private;
12883efb 7480 struct trace_buffer *trace_buf = &tr->trace_buffer;
4d3435b8 7481 int cpu = tracing_get_cpu(inode);
c8d77183
SR
7482 struct trace_seq *s;
7483 unsigned long cnt;
c64e148a
VN
7484 unsigned long long t;
7485 unsigned long usec_rem;
c8d77183 7486
e4f2d10f 7487 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 7488 if (!s)
a646365c 7489 return -ENOMEM;
c8d77183
SR
7490
7491 trace_seq_init(s);
7492
12883efb 7493 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
7494 trace_seq_printf(s, "entries: %ld\n", cnt);
7495
12883efb 7496 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
7497 trace_seq_printf(s, "overrun: %ld\n", cnt);
7498
12883efb 7499 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
7500 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
7501
12883efb 7502 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
7503 trace_seq_printf(s, "bytes: %ld\n", cnt);
7504
58e8eedf 7505 if (trace_clocks[tr->clock_id].in_ns) {
11043d8b 7506 /* local or global for trace_clock */
12883efb 7507 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
7508 usec_rem = do_div(t, USEC_PER_SEC);
7509 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
7510 t, usec_rem);
7511
12883efb 7512 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
7513 usec_rem = do_div(t, USEC_PER_SEC);
7514 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
7515 } else {
7516 /* counter or tsc mode for trace_clock */
7517 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 7518 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 7519
11043d8b 7520 trace_seq_printf(s, "now ts: %llu\n",
12883efb 7521 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 7522 }
c64e148a 7523
12883efb 7524 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
7525 trace_seq_printf(s, "dropped events: %ld\n", cnt);
7526
12883efb 7527 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
7528 trace_seq_printf(s, "read events: %ld\n", cnt);
7529
5ac48378
SRRH
7530 count = simple_read_from_buffer(ubuf, count, ppos,
7531 s->buffer, trace_seq_used(s));
c8d77183
SR
7532
7533 kfree(s);
7534
7535 return count;
7536}
7537
7538static const struct file_operations tracing_stats_fops = {
4d3435b8 7539 .open = tracing_open_generic_tr,
c8d77183 7540 .read = tracing_stats_read,
b444786f 7541 .llseek = generic_file_llseek,
4d3435b8 7542 .release = tracing_release_generic_tr,
c8d77183
SR
7543};
7544
bc0c38d1
SR
7545#ifdef CONFIG_DYNAMIC_FTRACE
7546
7547static ssize_t
b807c3d0 7548tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
7549 size_t cnt, loff_t *ppos)
7550{
7551 unsigned long *p = filp->private_data;
6a9c981b 7552 char buf[64]; /* Not too big for a shallow stack */
bc0c38d1
SR
7553 int r;
7554
6a9c981b 7555 r = scnprintf(buf, 63, "%ld", *p);
b807c3d0
SR
7556 buf[r++] = '\n';
7557
6a9c981b 7558 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
7559}
7560
5e2336a0 7561static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 7562 .open = tracing_open_generic,
b807c3d0 7563 .read = tracing_read_dyn_info,
b444786f 7564 .llseek = generic_file_llseek,
bc0c38d1 7565};
77fd5c15 7566#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 7567
77fd5c15
SRRH
7568#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7569static void
bca6c8d0 7570ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
b5f081b5 7571 struct trace_array *tr, struct ftrace_probe_ops *ops,
6e444319 7572 void *data)
77fd5c15 7573{
cab50379 7574 tracing_snapshot_instance(tr);
77fd5c15 7575}
bc0c38d1 7576
77fd5c15 7577static void
bca6c8d0 7578ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
b5f081b5 7579 struct trace_array *tr, struct ftrace_probe_ops *ops,
6e444319 7580 void *data)
bc0c38d1 7581{
6e444319 7582 struct ftrace_func_mapper *mapper = data;
1a93f8bd 7583 long *count = NULL;
77fd5c15 7584
1a93f8bd
SRV
7585 if (mapper)
7586 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7587
7588 if (count) {
7589
7590 if (*count <= 0)
7591 return;
bc0c38d1 7592
77fd5c15 7593 (*count)--;
1a93f8bd 7594 }
77fd5c15 7595
cab50379 7596 tracing_snapshot_instance(tr);
77fd5c15
SRRH
7597}
7598
7599static int
7600ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7601 struct ftrace_probe_ops *ops, void *data)
7602{
6e444319 7603 struct ftrace_func_mapper *mapper = data;
1a93f8bd 7604 long *count = NULL;
77fd5c15
SRRH
7605
7606 seq_printf(m, "%ps:", (void *)ip);
7607
fa6f0cc7 7608 seq_puts(m, "snapshot");
77fd5c15 7609
1a93f8bd
SRV
7610 if (mapper)
7611 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7612
7613 if (count)
7614 seq_printf(m, ":count=%ld\n", *count);
77fd5c15 7615 else
1a93f8bd 7616 seq_puts(m, ":unlimited\n");
77fd5c15
SRRH
7617
7618 return 0;
7619}
7620
1a93f8bd 7621static int
b5f081b5 7622ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
6e444319 7623 unsigned long ip, void *init_data, void **data)
1a93f8bd 7624{
6e444319
SRV
7625 struct ftrace_func_mapper *mapper = *data;
7626
7627 if (!mapper) {
7628 mapper = allocate_ftrace_func_mapper();
7629 if (!mapper)
7630 return -ENOMEM;
7631 *data = mapper;
7632 }
1a93f8bd 7633
6e444319 7634 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
1a93f8bd
SRV
7635}
7636
7637static void
b5f081b5 7638ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
6e444319 7639 unsigned long ip, void *data)
1a93f8bd 7640{
6e444319
SRV
7641 struct ftrace_func_mapper *mapper = data;
7642
7643 if (!ip) {
7644 if (!mapper)
7645 return;
7646 free_ftrace_func_mapper(mapper, NULL);
7647 return;
7648 }
1a93f8bd
SRV
7649
7650 ftrace_func_mapper_remove_ip(mapper, ip);
7651}
7652
77fd5c15
SRRH
7653static struct ftrace_probe_ops snapshot_probe_ops = {
7654 .func = ftrace_snapshot,
7655 .print = ftrace_snapshot_print,
7656};
7657
7658static struct ftrace_probe_ops snapshot_count_probe_ops = {
7659 .func = ftrace_count_snapshot,
7660 .print = ftrace_snapshot_print,
1a93f8bd
SRV
7661 .init = ftrace_snapshot_init,
7662 .free = ftrace_snapshot_free,
77fd5c15
SRRH
7663};
7664
7665static int
04ec7bb6 7666ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
77fd5c15
SRRH
7667 char *glob, char *cmd, char *param, int enable)
7668{
7669 struct ftrace_probe_ops *ops;
7670 void *count = (void *)-1;
7671 char *number;
7672 int ret;
7673
0f179765
SRV
7674 if (!tr)
7675 return -ENODEV;
7676
77fd5c15
SRRH
7677 /* hash funcs only work with set_ftrace_filter */
7678 if (!enable)
7679 return -EINVAL;
7680
7681 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7682
d3d532d7 7683 if (glob[0] == '!')
7b60f3d8 7684 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
77fd5c15
SRRH
7685
7686 if (!param)
7687 goto out_reg;
7688
7689 number = strsep(&param, ":");
7690
7691 if (!strlen(number))
7692 goto out_reg;
7693
7694 /*
7695 * We use the callback data field (which is a pointer)
7696 * as our counter.
7697 */
7698 ret = kstrtoul(number, 0, (unsigned long *)&count);
7699 if (ret)
7700 return ret;
7701
7702 out_reg:
2824f503 7703 ret = tracing_alloc_snapshot_instance(tr);
df62db5b
SRV
7704 if (ret < 0)
7705 goto out;
77fd5c15 7706
4c174688 7707 ret = register_ftrace_function_probe(glob, tr, ops, count);
77fd5c15 7708
df62db5b 7709 out:
77fd5c15
SRRH
7710 return ret < 0 ? ret : 0;
7711}
7712
7713static struct ftrace_func_command ftrace_snapshot_cmd = {
7714 .name = "snapshot",
7715 .func = ftrace_trace_snapshot_callback,
7716};
7717
38de93ab 7718static __init int register_snapshot_cmd(void)
77fd5c15
SRRH
7719{
7720 return register_ftrace_command(&ftrace_snapshot_cmd);
7721}
7722#else
38de93ab 7723static inline __init int register_snapshot_cmd(void) { return 0; }
77fd5c15 7724#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 7725
7eeafbca 7726static struct dentry *tracing_get_dentry(struct trace_array *tr)
bc0c38d1 7727{
8434dc93
SRRH
7728 if (WARN_ON(!tr->dir))
7729 return ERR_PTR(-ENODEV);
7730
7731 /* Top directory uses NULL as the parent */
7732 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7733 return NULL;
7734
7735 /* All sub buffers have a descriptor */
2b6080f2 7736 return tr->dir;
bc0c38d1
SR
7737}
7738
2b6080f2 7739static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 7740{
b04cc6b1
FW
7741 struct dentry *d_tracer;
7742
2b6080f2
SR
7743 if (tr->percpu_dir)
7744 return tr->percpu_dir;
b04cc6b1 7745
7eeafbca 7746 d_tracer = tracing_get_dentry(tr);
14a5ae40 7747 if (IS_ERR(d_tracer))
b04cc6b1
FW
7748 return NULL;
7749
8434dc93 7750 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
b04cc6b1 7751
2b6080f2 7752 WARN_ONCE(!tr->percpu_dir,
8434dc93 7753 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 7754
2b6080f2 7755 return tr->percpu_dir;
b04cc6b1
FW
7756}
7757
649e9c70
ON
7758static struct dentry *
7759trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7760 void *data, long cpu, const struct file_operations *fops)
7761{
7762 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7763
7764 if (ret) /* See tracing_get_cpu() */
7682c918 7765 d_inode(ret)->i_cdev = (void *)(cpu + 1);
649e9c70
ON
7766 return ret;
7767}
7768
2b6080f2 7769static void
8434dc93 7770tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 7771{
2b6080f2 7772 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 7773 struct dentry *d_cpu;
dd49a38c 7774 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 7775
0a3d7ce7
NK
7776 if (!d_percpu)
7777 return;
7778
dd49a38c 7779 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8434dc93 7780 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8656e7a2 7781 if (!d_cpu) {
a395d6a7 7782 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8656e7a2
FW
7783 return;
7784 }
b04cc6b1 7785
8656e7a2 7786 /* per cpu trace_pipe */
649e9c70 7787 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
15544209 7788 tr, cpu, &tracing_pipe_fops);
b04cc6b1
FW
7789
7790 /* per cpu trace */
649e9c70 7791 trace_create_cpu_file("trace", 0644, d_cpu,
6484c71c 7792 tr, cpu, &tracing_fops);
7f96f93f 7793
649e9c70 7794 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
46ef2be0 7795 tr, cpu, &tracing_buffers_fops);
7f96f93f 7796
649e9c70 7797 trace_create_cpu_file("stats", 0444, d_cpu,
4d3435b8 7798 tr, cpu, &tracing_stats_fops);
438ced17 7799
649e9c70 7800 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
0bc392ee 7801 tr, cpu, &tracing_entries_fops);
f1affcaa
SRRH
7802
7803#ifdef CONFIG_TRACER_SNAPSHOT
649e9c70 7804 trace_create_cpu_file("snapshot", 0644, d_cpu,
6484c71c 7805 tr, cpu, &snapshot_fops);
6de58e62 7806
649e9c70 7807 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
46ef2be0 7808 tr, cpu, &snapshot_raw_fops);
f1affcaa 7809#endif
b04cc6b1
FW
7810}
7811
60a11774
SR
7812#ifdef CONFIG_FTRACE_SELFTEST
7813/* Let selftest have access to static functions in this file */
7814#include "trace_selftest.c"
7815#endif
7816
577b785f
SR
7817static ssize_t
7818trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7819 loff_t *ppos)
7820{
7821 struct trace_option_dentry *topt = filp->private_data;
7822 char *buf;
7823
7824 if (topt->flags->val & topt->opt->bit)
7825 buf = "1\n";
7826 else
7827 buf = "0\n";
7828
7829 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7830}
7831
7832static ssize_t
7833trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7834 loff_t *ppos)
7835{
7836 struct trace_option_dentry *topt = filp->private_data;
7837 unsigned long val;
577b785f
SR
7838 int ret;
7839
22fe9b54
PH
7840 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7841 if (ret)
577b785f
SR
7842 return ret;
7843
8d18eaaf
LZ
7844 if (val != 0 && val != 1)
7845 return -EINVAL;
577b785f 7846
8d18eaaf 7847 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 7848 mutex_lock(&trace_types_lock);
8c1a49ae 7849 ret = __set_tracer_option(topt->tr, topt->flags,
c757bea9 7850 topt->opt, !val);
577b785f
SR
7851 mutex_unlock(&trace_types_lock);
7852 if (ret)
7853 return ret;
577b785f
SR
7854 }
7855
7856 *ppos += cnt;
7857
7858 return cnt;
7859}
7860
7861
7862static const struct file_operations trace_options_fops = {
7863 .open = tracing_open_generic,
7864 .read = trace_options_read,
7865 .write = trace_options_write,
b444786f 7866 .llseek = generic_file_llseek,
577b785f
SR
7867};
7868
9a38a885
SRRH
7869/*
7870 * In order to pass in both the trace_array descriptor as well as the index
7871 * to the flag that the trace option file represents, the trace_array
7872 * has a character array of trace_flags_index[], which holds the index
7873 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7874 * The address of this character array is passed to the flag option file
7875 * read/write callbacks.
7876 *
7877 * In order to extract both the index and the trace_array descriptor,
7878 * get_tr_index() uses the following algorithm.
7879 *
7880 * idx = *ptr;
7881 *
7882 * As the pointer itself contains the address of the index (remember
7883 * index[1] == 1).
7884 *
7885 * Then to get the trace_array descriptor, by subtracting that index
7886 * from the ptr, we get to the start of the index itself.
7887 *
7888 * ptr - idx == &index[0]
7889 *
7890 * Then a simple container_of() from that pointer gets us to the
7891 * trace_array descriptor.
7892 */
7893static void get_tr_index(void *data, struct trace_array **ptr,
7894 unsigned int *pindex)
7895{
7896 *pindex = *(unsigned char *)data;
7897
7898 *ptr = container_of(data - *pindex, struct trace_array,
7899 trace_flags_index);
7900}
7901
a8259075
SR
7902static ssize_t
7903trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7904 loff_t *ppos)
7905{
9a38a885
SRRH
7906 void *tr_index = filp->private_data;
7907 struct trace_array *tr;
7908 unsigned int index;
a8259075
SR
7909 char *buf;
7910
9a38a885
SRRH
7911 get_tr_index(tr_index, &tr, &index);
7912
7913 if (tr->trace_flags & (1 << index))
a8259075
SR
7914 buf = "1\n";
7915 else
7916 buf = "0\n";
7917
7918 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7919}
7920
7921static ssize_t
7922trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7923 loff_t *ppos)
7924{
9a38a885
SRRH
7925 void *tr_index = filp->private_data;
7926 struct trace_array *tr;
7927 unsigned int index;
a8259075
SR
7928 unsigned long val;
7929 int ret;
7930
9a38a885
SRRH
7931 get_tr_index(tr_index, &tr, &index);
7932
22fe9b54
PH
7933 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7934 if (ret)
a8259075
SR
7935 return ret;
7936
f2d84b65 7937 if (val != 0 && val != 1)
a8259075 7938 return -EINVAL;
69d34da2
SRRH
7939
7940 mutex_lock(&trace_types_lock);
2b6080f2 7941 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 7942 mutex_unlock(&trace_types_lock);
a8259075 7943
613f04a0
SRRH
7944 if (ret < 0)
7945 return ret;
7946
a8259075
SR
7947 *ppos += cnt;
7948
7949 return cnt;
7950}
7951
a8259075
SR
7952static const struct file_operations trace_options_core_fops = {
7953 .open = tracing_open_generic,
7954 .read = trace_options_core_read,
7955 .write = trace_options_core_write,
b444786f 7956 .llseek = generic_file_llseek,
a8259075
SR
7957};
7958
5452af66 7959struct dentry *trace_create_file(const char *name,
f4ae40a6 7960 umode_t mode,
5452af66
FW
7961 struct dentry *parent,
7962 void *data,
7963 const struct file_operations *fops)
7964{
7965 struct dentry *ret;
7966
8434dc93 7967 ret = tracefs_create_file(name, mode, parent, data, fops);
5452af66 7968 if (!ret)
a395d6a7 7969 pr_warn("Could not create tracefs '%s' entry\n", name);
5452af66
FW
7970
7971 return ret;
7972}
7973
7974
2b6080f2 7975static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
7976{
7977 struct dentry *d_tracer;
a8259075 7978
2b6080f2
SR
7979 if (tr->options)
7980 return tr->options;
a8259075 7981
7eeafbca 7982 d_tracer = tracing_get_dentry(tr);
14a5ae40 7983 if (IS_ERR(d_tracer))
a8259075
SR
7984 return NULL;
7985
8434dc93 7986 tr->options = tracefs_create_dir("options", d_tracer);
2b6080f2 7987 if (!tr->options) {
a395d6a7 7988 pr_warn("Could not create tracefs directory 'options'\n");
a8259075
SR
7989 return NULL;
7990 }
7991
2b6080f2 7992 return tr->options;
a8259075
SR
7993}
7994
577b785f 7995static void
2b6080f2
SR
7996create_trace_option_file(struct trace_array *tr,
7997 struct trace_option_dentry *topt,
577b785f
SR
7998 struct tracer_flags *flags,
7999 struct tracer_opt *opt)
8000{
8001 struct dentry *t_options;
577b785f 8002
2b6080f2 8003 t_options = trace_options_init_dentry(tr);
577b785f
SR
8004 if (!t_options)
8005 return;
8006
8007 topt->flags = flags;
8008 topt->opt = opt;
2b6080f2 8009 topt->tr = tr;
577b785f 8010
5452af66 8011 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
8012 &trace_options_fops);
8013
577b785f
SR
8014}
8015
37aea98b 8016static void
2b6080f2 8017create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
8018{
8019 struct trace_option_dentry *topts;
37aea98b 8020 struct trace_options *tr_topts;
577b785f
SR
8021 struct tracer_flags *flags;
8022 struct tracer_opt *opts;
8023 int cnt;
37aea98b 8024 int i;
577b785f
SR
8025
8026 if (!tracer)
37aea98b 8027 return;
577b785f
SR
8028
8029 flags = tracer->flags;
8030
8031 if (!flags || !flags->opts)
37aea98b
SRRH
8032 return;
8033
8034 /*
8035 * If this is an instance, only create flags for tracers
8036 * the instance may have.
8037 */
8038 if (!trace_ok_for_array(tracer, tr))
8039 return;
8040
8041 for (i = 0; i < tr->nr_topts; i++) {
d39cdd20
CH
8042 /* Make sure there's no duplicate flags. */
8043 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
37aea98b
SRRH
8044 return;
8045 }
577b785f
SR
8046
8047 opts = flags->opts;
8048
8049 for (cnt = 0; opts[cnt].name; cnt++)
8050 ;
8051
0cfe8245 8052 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f 8053 if (!topts)
37aea98b
SRRH
8054 return;
8055
8056 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
8057 GFP_KERNEL);
8058 if (!tr_topts) {
8059 kfree(topts);
8060 return;
8061 }
8062
8063 tr->topts = tr_topts;
8064 tr->topts[tr->nr_topts].tracer = tracer;
8065 tr->topts[tr->nr_topts].topts = topts;
8066 tr->nr_topts++;
577b785f 8067
41d9c0be 8068 for (cnt = 0; opts[cnt].name; cnt++) {
2b6080f2 8069 create_trace_option_file(tr, &topts[cnt], flags,
577b785f 8070 &opts[cnt]);
41d9c0be
SRRH
8071 WARN_ONCE(topts[cnt].entry == NULL,
8072 "Failed to create trace option: %s",
8073 opts[cnt].name);
8074 }
577b785f
SR
8075}
8076
a8259075 8077static struct dentry *
2b6080f2
SR
8078create_trace_option_core_file(struct trace_array *tr,
8079 const char *option, long index)
a8259075
SR
8080{
8081 struct dentry *t_options;
a8259075 8082
2b6080f2 8083 t_options = trace_options_init_dentry(tr);
a8259075
SR
8084 if (!t_options)
8085 return NULL;
8086
9a38a885
SRRH
8087 return trace_create_file(option, 0644, t_options,
8088 (void *)&tr->trace_flags_index[index],
8089 &trace_options_core_fops);
a8259075
SR
8090}
8091
16270145 8092static void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
8093{
8094 struct dentry *t_options;
16270145 8095 bool top_level = tr == &global_trace;
a8259075
SR
8096 int i;
8097
2b6080f2 8098 t_options = trace_options_init_dentry(tr);
a8259075
SR
8099 if (!t_options)
8100 return;
8101
16270145
SRRH
8102 for (i = 0; trace_options[i]; i++) {
8103 if (top_level ||
8104 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8105 create_trace_option_core_file(tr, trace_options[i], i);
8106 }
a8259075
SR
8107}
8108
499e5470
SR
8109static ssize_t
8110rb_simple_read(struct file *filp, char __user *ubuf,
8111 size_t cnt, loff_t *ppos)
8112{
348f0fc2 8113 struct trace_array *tr = filp->private_data;
499e5470
SR
8114 char buf[64];
8115 int r;
8116
10246fa3 8117 r = tracer_tracing_is_on(tr);
499e5470
SR
8118 r = sprintf(buf, "%d\n", r);
8119
8120 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8121}
8122
8123static ssize_t
8124rb_simple_write(struct file *filp, const char __user *ubuf,
8125 size_t cnt, loff_t *ppos)
8126{
348f0fc2 8127 struct trace_array *tr = filp->private_data;
12883efb 8128 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
8129 unsigned long val;
8130 int ret;
8131
8132 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8133 if (ret)
8134 return ret;
8135
8136 if (buffer) {
2df8f8a6 8137 mutex_lock(&trace_types_lock);
f143641b
SRV
8138 if (!!val == tracer_tracing_is_on(tr)) {
8139 val = 0; /* do nothing */
8140 } else if (val) {
10246fa3 8141 tracer_tracing_on(tr);
2b6080f2
SR
8142 if (tr->current_trace->start)
8143 tr->current_trace->start(tr);
2df8f8a6 8144 } else {
10246fa3 8145 tracer_tracing_off(tr);
2b6080f2
SR
8146 if (tr->current_trace->stop)
8147 tr->current_trace->stop(tr);
2df8f8a6
SR
8148 }
8149 mutex_unlock(&trace_types_lock);
499e5470
SR
8150 }
8151
8152 (*ppos)++;
8153
8154 return cnt;
8155}
8156
8157static const struct file_operations rb_simple_fops = {
7b85af63 8158 .open = tracing_open_generic_tr,
499e5470
SR
8159 .read = rb_simple_read,
8160 .write = rb_simple_write,
7b85af63 8161 .release = tracing_release_generic_tr,
499e5470
SR
8162 .llseek = default_llseek,
8163};
8164
03329f99
SRV
8165static ssize_t
8166buffer_percent_read(struct file *filp, char __user *ubuf,
8167 size_t cnt, loff_t *ppos)
8168{
8169 struct trace_array *tr = filp->private_data;
8170 char buf[64];
8171 int r;
8172
8173 r = tr->buffer_percent;
8174 r = sprintf(buf, "%d\n", r);
8175
8176 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
8177}
8178
8179static ssize_t
8180buffer_percent_write(struct file *filp, const char __user *ubuf,
8181 size_t cnt, loff_t *ppos)
8182{
8183 struct trace_array *tr = filp->private_data;
8184 unsigned long val;
8185 int ret;
8186
8187 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
8188 if (ret)
8189 return ret;
8190
8191 if (val > 100)
8192 return -EINVAL;
8193
8194 if (!val)
8195 val = 1;
8196
8197 tr->buffer_percent = val;
8198
8199 (*ppos)++;
8200
8201 return cnt;
8202}
8203
8204static const struct file_operations buffer_percent_fops = {
8205 .open = tracing_open_generic_tr,
8206 .read = buffer_percent_read,
8207 .write = buffer_percent_write,
8208 .release = tracing_release_generic_tr,
8209 .llseek = default_llseek,
8210};
8211
ff585c5b 8212static struct dentry *trace_instance_dir;
277ba044
SR
8213
8214static void
8434dc93 8215init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
277ba044 8216
55034cd6
SRRH
8217static int
8218allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
8219{
8220 enum ring_buffer_flags rb_flags;
737223fb 8221
983f938a 8222 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
737223fb 8223
dced341b
SRRH
8224 buf->tr = tr;
8225
55034cd6
SRRH
8226 buf->buffer = ring_buffer_alloc(size, rb_flags);
8227 if (!buf->buffer)
8228 return -ENOMEM;
737223fb 8229
55034cd6
SRRH
8230 buf->data = alloc_percpu(struct trace_array_cpu);
8231 if (!buf->data) {
8232 ring_buffer_free(buf->buffer);
4397f045 8233 buf->buffer = NULL;
55034cd6
SRRH
8234 return -ENOMEM;
8235 }
737223fb 8236
737223fb
SRRH
8237 /* Allocate the first page for all buffers */
8238 set_buffer_entries(&tr->trace_buffer,
8239 ring_buffer_size(tr->trace_buffer.buffer, 0));
8240
55034cd6
SRRH
8241 return 0;
8242}
737223fb 8243
55034cd6
SRRH
8244static int allocate_trace_buffers(struct trace_array *tr, int size)
8245{
8246 int ret;
737223fb 8247
55034cd6
SRRH
8248 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
8249 if (ret)
8250 return ret;
737223fb 8251
55034cd6
SRRH
8252#ifdef CONFIG_TRACER_MAX_TRACE
8253 ret = allocate_trace_buffer(tr, &tr->max_buffer,
8254 allocate_snapshot ? size : 1);
8255 if (WARN_ON(ret)) {
737223fb 8256 ring_buffer_free(tr->trace_buffer.buffer);
24f2aaf9 8257 tr->trace_buffer.buffer = NULL;
55034cd6 8258 free_percpu(tr->trace_buffer.data);
24f2aaf9 8259 tr->trace_buffer.data = NULL;
55034cd6
SRRH
8260 return -ENOMEM;
8261 }
8262 tr->allocated_snapshot = allocate_snapshot;
737223fb 8263
55034cd6
SRRH
8264 /*
8265 * Only the top level trace array gets its snapshot allocated
8266 * from the kernel command line.
8267 */
8268 allocate_snapshot = false;
737223fb 8269#endif
55034cd6 8270 return 0;
737223fb
SRRH
8271}
8272
f0b70cc4
SRRH
8273static void free_trace_buffer(struct trace_buffer *buf)
8274{
8275 if (buf->buffer) {
8276 ring_buffer_free(buf->buffer);
8277 buf->buffer = NULL;
8278 free_percpu(buf->data);
8279 buf->data = NULL;
8280 }
8281}
8282
23aaa3c1
SRRH
8283static void free_trace_buffers(struct trace_array *tr)
8284{
8285 if (!tr)
8286 return;
8287
f0b70cc4 8288 free_trace_buffer(&tr->trace_buffer);
23aaa3c1
SRRH
8289
8290#ifdef CONFIG_TRACER_MAX_TRACE
f0b70cc4 8291 free_trace_buffer(&tr->max_buffer);
23aaa3c1
SRRH
8292#endif
8293}
8294
9a38a885
SRRH
8295static void init_trace_flags_index(struct trace_array *tr)
8296{
8297 int i;
8298
8299 /* Used by the trace options files */
8300 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
8301 tr->trace_flags_index[i] = i;
8302}
8303
37aea98b
SRRH
8304static void __update_tracer_options(struct trace_array *tr)
8305{
8306 struct tracer *t;
8307
8308 for (t = trace_types; t; t = t->next)
8309 add_tracer_options(tr, t);
8310}
8311
8312static void update_tracer_options(struct trace_array *tr)
8313{
8314 mutex_lock(&trace_types_lock);
8315 __update_tracer_options(tr);
8316 mutex_unlock(&trace_types_lock);
8317}
8318
f45d1225 8319struct trace_array *trace_array_create(const char *name)
737223fb 8320{
277ba044
SR
8321 struct trace_array *tr;
8322 int ret;
277ba044 8323
12ecef0c 8324 mutex_lock(&event_mutex);
277ba044
SR
8325 mutex_lock(&trace_types_lock);
8326
8327 ret = -EEXIST;
8328 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8329 if (tr->name && strcmp(tr->name, name) == 0)
8330 goto out_unlock;
8331 }
8332
8333 ret = -ENOMEM;
8334 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
8335 if (!tr)
8336 goto out_unlock;
8337
8338 tr->name = kstrdup(name, GFP_KERNEL);
8339 if (!tr->name)
8340 goto out_free_tr;
8341
ccfe9e42
AL
8342 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
8343 goto out_free_tr;
8344
20550622 8345 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
983f938a 8346
ccfe9e42
AL
8347 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
8348
277ba044
SR
8349 raw_spin_lock_init(&tr->start_lock);
8350
0b9b12c1
SRRH
8351 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8352
277ba044
SR
8353 tr->current_trace = &nop_trace;
8354
8355 INIT_LIST_HEAD(&tr->systems);
8356 INIT_LIST_HEAD(&tr->events);
067fe038 8357 INIT_LIST_HEAD(&tr->hist_vars);
2f754e77 8358 INIT_LIST_HEAD(&tr->err_log);
277ba044 8359
737223fb 8360 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
8361 goto out_free_tr;
8362
8434dc93 8363 tr->dir = tracefs_create_dir(name, trace_instance_dir);
277ba044
SR
8364 if (!tr->dir)
8365 goto out_free_tr;
8366
8367 ret = event_trace_add_tracer(tr->dir, tr);
609e85a7 8368 if (ret) {
8434dc93 8369 tracefs_remove_recursive(tr->dir);
277ba044 8370 goto out_free_tr;
609e85a7 8371 }
277ba044 8372
04ec7bb6
SRV
8373 ftrace_init_trace_array(tr);
8374
8434dc93 8375 init_tracer_tracefs(tr, tr->dir);
9a38a885 8376 init_trace_flags_index(tr);
37aea98b 8377 __update_tracer_options(tr);
277ba044
SR
8378
8379 list_add(&tr->list, &ftrace_trace_arrays);
8380
8381 mutex_unlock(&trace_types_lock);
12ecef0c 8382 mutex_unlock(&event_mutex);
277ba044 8383
f45d1225 8384 return tr;
277ba044
SR
8385
8386 out_free_tr:
23aaa3c1 8387 free_trace_buffers(tr);
ccfe9e42 8388 free_cpumask_var(tr->tracing_cpumask);
277ba044
SR
8389 kfree(tr->name);
8390 kfree(tr);
8391
8392 out_unlock:
8393 mutex_unlock(&trace_types_lock);
12ecef0c 8394 mutex_unlock(&event_mutex);
277ba044 8395
f45d1225
DI
8396 return ERR_PTR(ret);
8397}
8398EXPORT_SYMBOL_GPL(trace_array_create);
277ba044 8399
f45d1225
DI
8400static int instance_mkdir(const char *name)
8401{
8402 return PTR_ERR_OR_ZERO(trace_array_create(name));
277ba044
SR
8403}
8404
f45d1225 8405static int __remove_instance(struct trace_array *tr)
0c8916c3 8406{
37aea98b 8407 int i;
0c8916c3 8408
cf6ab6d9 8409 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
f45d1225 8410 return -EBUSY;
a695cb58 8411
0c8916c3
SR
8412 list_del(&tr->list);
8413
20550622
SRRH
8414 /* Disable all the flags that were enabled coming in */
8415 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
8416 if ((1 << i) & ZEROED_TRACE_FLAGS)
8417 set_tracer_flag(tr, 1 << i, 0);
8418 }
8419
6b450d25 8420 tracing_set_nop(tr);
a0e6369e 8421 clear_ftrace_function_probes(tr);
0c8916c3 8422 event_trace_del_tracer(tr);
d879d0b8 8423 ftrace_clear_pids(tr);
591dffda 8424 ftrace_destroy_function_files(tr);
681a4a2f 8425 tracefs_remove_recursive(tr->dir);
a9fcaaac 8426 free_trace_buffers(tr);
0c8916c3 8427
37aea98b
SRRH
8428 for (i = 0; i < tr->nr_topts; i++) {
8429 kfree(tr->topts[i].topts);
8430 }
8431 kfree(tr->topts);
8432
db9108e0 8433 free_cpumask_var(tr->tracing_cpumask);
0c8916c3
SR
8434 kfree(tr->name);
8435 kfree(tr);
f45d1225 8436 tr = NULL;
0c8916c3 8437
f45d1225
DI
8438 return 0;
8439}
8440
8441int trace_array_destroy(struct trace_array *tr)
8442{
8443 int ret;
8444
8445 if (!tr)
8446 return -EINVAL;
8447
8448 mutex_lock(&event_mutex);
8449 mutex_lock(&trace_types_lock);
8450
8451 ret = __remove_instance(tr);
8452
8453 mutex_unlock(&trace_types_lock);
8454 mutex_unlock(&event_mutex);
8455
8456 return ret;
8457}
8458EXPORT_SYMBOL_GPL(trace_array_destroy);
8459
8460static int instance_rmdir(const char *name)
8461{
8462 struct trace_array *tr;
8463 int ret;
8464
8465 mutex_lock(&event_mutex);
8466 mutex_lock(&trace_types_lock);
8467
8468 ret = -ENODEV;
8469 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
8470 if (tr->name && strcmp(tr->name, name) == 0) {
8471 ret = __remove_instance(tr);
8472 break;
8473 }
8474 }
0c8916c3 8475
0c8916c3 8476 mutex_unlock(&trace_types_lock);
12ecef0c 8477 mutex_unlock(&event_mutex);
0c8916c3
SR
8478
8479 return ret;
8480}
8481
277ba044
SR
8482static __init void create_trace_instances(struct dentry *d_tracer)
8483{
eae47358
SRRH
8484 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
8485 instance_mkdir,
8486 instance_rmdir);
277ba044
SR
8487 if (WARN_ON(!trace_instance_dir))
8488 return;
277ba044
SR
8489}
8490
2b6080f2 8491static void
8434dc93 8492init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
2b6080f2 8493{
3dd80953 8494 struct trace_event_file *file;
121aaee7 8495 int cpu;
2b6080f2 8496
607e2ea1
SRRH
8497 trace_create_file("available_tracers", 0444, d_tracer,
8498 tr, &show_traces_fops);
8499
8500 trace_create_file("current_tracer", 0644, d_tracer,
8501 tr, &set_tracer_fops);
8502
ccfe9e42
AL
8503 trace_create_file("tracing_cpumask", 0644, d_tracer,
8504 tr, &tracing_cpumask_fops);
8505
2b6080f2
SR
8506 trace_create_file("trace_options", 0644, d_tracer,
8507 tr, &tracing_iter_fops);
8508
8509 trace_create_file("trace", 0644, d_tracer,
6484c71c 8510 tr, &tracing_fops);
2b6080f2
SR
8511
8512 trace_create_file("trace_pipe", 0444, d_tracer,
15544209 8513 tr, &tracing_pipe_fops);
2b6080f2
SR
8514
8515 trace_create_file("buffer_size_kb", 0644, d_tracer,
0bc392ee 8516 tr, &tracing_entries_fops);
2b6080f2
SR
8517
8518 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
8519 tr, &tracing_total_entries_fops);
8520
238ae93d 8521 trace_create_file("free_buffer", 0200, d_tracer,
2b6080f2
SR
8522 tr, &tracing_free_buffer_fops);
8523
8524 trace_create_file("trace_marker", 0220, d_tracer,
8525 tr, &tracing_mark_fops);
8526
3dd80953
SRV
8527 file = __find_event_file(tr, "ftrace", "print");
8528 if (file && file->dir)
8529 trace_create_file("trigger", 0644, file->dir, file,
8530 &event_trigger_fops);
8531 tr->trace_marker_file = file;
8532
fa32e855
SR
8533 trace_create_file("trace_marker_raw", 0220, d_tracer,
8534 tr, &tracing_mark_raw_fops);
8535
2b6080f2
SR
8536 trace_create_file("trace_clock", 0644, d_tracer, tr,
8537 &trace_clock_fops);
8538
8539 trace_create_file("tracing_on", 0644, d_tracer,
6484c71c 8540 tr, &rb_simple_fops);
ce9bae55 8541
2c1ea60b
TZ
8542 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
8543 &trace_time_stamp_mode_fops);
8544
a7b1d74e 8545 tr->buffer_percent = 50;
03329f99
SRV
8546
8547 trace_create_file("buffer_percent", 0444, d_tracer,
8548 tr, &buffer_percent_fops);
8549
16270145
SRRH
8550 create_trace_options_dir(tr);
8551
f971cc9a 8552#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6d9b3fa5
SRRH
8553 trace_create_file("tracing_max_latency", 0644, d_tracer,
8554 &tr->max_latency, &tracing_max_lat_fops);
8555#endif
8556
591dffda
SRRH
8557 if (ftrace_create_function_files(tr, d_tracer))
8558 WARN(1, "Could not allocate function filter files");
8559
ce9bae55
SRRH
8560#ifdef CONFIG_TRACER_SNAPSHOT
8561 trace_create_file("snapshot", 0644, d_tracer,
6484c71c 8562 tr, &snapshot_fops);
ce9bae55 8563#endif
121aaee7 8564
8a062902
TZ
8565 trace_create_file("error_log", 0644, d_tracer,
8566 tr, &tracing_err_log_fops);
8567
121aaee7 8568 for_each_tracing_cpu(cpu)
8434dc93 8569 tracing_init_tracefs_percpu(tr, cpu);
121aaee7 8570
345ddcc8 8571 ftrace_init_tracefs(tr, d_tracer);
2b6080f2
SR
8572}
8573
93faccbb 8574static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
f76180bc
SRRH
8575{
8576 struct vfsmount *mnt;
8577 struct file_system_type *type;
8578
8579 /*
8580 * To maintain backward compatibility for tools that mount
8581 * debugfs to get to the tracing facility, tracefs is automatically
8582 * mounted to the debugfs/tracing directory.
8583 */
8584 type = get_fs_type("tracefs");
8585 if (!type)
8586 return NULL;
93faccbb 8587 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
f76180bc
SRRH
8588 put_filesystem(type);
8589 if (IS_ERR(mnt))
8590 return NULL;
8591 mntget(mnt);
8592
8593 return mnt;
8594}
8595
7eeafbca
SRRH
8596/**
8597 * tracing_init_dentry - initialize top level trace array
8598 *
8599 * This is called when creating files or directories in the tracing
8600 * directory. It is called via fs_initcall() by any of the boot up code
8601 * and expects to return the dentry of the top level tracing directory.
8602 */
8603struct dentry *tracing_init_dentry(void)
8604{
8605 struct trace_array *tr = &global_trace;
8606
f76180bc 8607 /* The top level trace array uses NULL as parent */
7eeafbca 8608 if (tr->dir)
f76180bc 8609 return NULL;
7eeafbca 8610
8b129199
JW
8611 if (WARN_ON(!tracefs_initialized()) ||
8612 (IS_ENABLED(CONFIG_DEBUG_FS) &&
8613 WARN_ON(!debugfs_initialized())))
7eeafbca
SRRH
8614 return ERR_PTR(-ENODEV);
8615
f76180bc
SRRH
8616 /*
8617 * As there may still be users that expect the tracing
8618 * files to exist in debugfs/tracing, we must automount
8619 * the tracefs file system there, so older tools still
8620 * work with the newer kerenl.
8621 */
8622 tr->dir = debugfs_create_automount("tracing", NULL,
8623 trace_automount, NULL);
7eeafbca 8624
8434dc93 8625 return NULL;
7eeafbca
SRRH
8626}
8627
00f4b652
JL
8628extern struct trace_eval_map *__start_ftrace_eval_maps[];
8629extern struct trace_eval_map *__stop_ftrace_eval_maps[];
0c564a53 8630
5f60b351 8631static void __init trace_eval_init(void)
0c564a53 8632{
3673b8e4
SRRH
8633 int len;
8634
02fd7f68 8635 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
f57a4143 8636 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
3673b8e4
SRRH
8637}
8638
8639#ifdef CONFIG_MODULES
f57a4143 8640static void trace_module_add_evals(struct module *mod)
3673b8e4 8641{
99be647c 8642 if (!mod->num_trace_evals)
3673b8e4
SRRH
8643 return;
8644
8645 /*
8646 * Modules with bad taint do not have events created, do
8647 * not bother with enums either.
8648 */
8649 if (trace_module_has_bad_taint(mod))
8650 return;
8651
f57a4143 8652 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
3673b8e4
SRRH
8653}
8654
681bec03 8655#ifdef CONFIG_TRACE_EVAL_MAP_FILE
f57a4143 8656static void trace_module_remove_evals(struct module *mod)
9828413d 8657{
23bf8cb8
JL
8658 union trace_eval_map_item *map;
8659 union trace_eval_map_item **last = &trace_eval_maps;
9828413d 8660
99be647c 8661 if (!mod->num_trace_evals)
9828413d
SRRH
8662 return;
8663
1793ed93 8664 mutex_lock(&trace_eval_mutex);
9828413d 8665
23bf8cb8 8666 map = trace_eval_maps;
9828413d
SRRH
8667
8668 while (map) {
8669 if (map->head.mod == mod)
8670 break;
5f60b351 8671 map = trace_eval_jmp_to_tail(map);
9828413d
SRRH
8672 last = &map->tail.next;
8673 map = map->tail.next;
8674 }
8675 if (!map)
8676 goto out;
8677
5f60b351 8678 *last = trace_eval_jmp_to_tail(map)->tail.next;
9828413d
SRRH
8679 kfree(map);
8680 out:
1793ed93 8681 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
8682}
8683#else
f57a4143 8684static inline void trace_module_remove_evals(struct module *mod) { }
681bec03 8685#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 8686
3673b8e4
SRRH
8687static int trace_module_notify(struct notifier_block *self,
8688 unsigned long val, void *data)
8689{
8690 struct module *mod = data;
8691
8692 switch (val) {
8693 case MODULE_STATE_COMING:
f57a4143 8694 trace_module_add_evals(mod);
3673b8e4 8695 break;
9828413d 8696 case MODULE_STATE_GOING:
f57a4143 8697 trace_module_remove_evals(mod);
9828413d 8698 break;
3673b8e4
SRRH
8699 }
8700
8701 return 0;
0c564a53
SRRH
8702}
8703
3673b8e4
SRRH
8704static struct notifier_block trace_module_nb = {
8705 .notifier_call = trace_module_notify,
8706 .priority = 0,
8707};
9828413d 8708#endif /* CONFIG_MODULES */
3673b8e4 8709
8434dc93 8710static __init int tracer_init_tracefs(void)
bc0c38d1
SR
8711{
8712 struct dentry *d_tracer;
bc0c38d1 8713
7e53bd42
LJ
8714 trace_access_lock_init();
8715
bc0c38d1 8716 d_tracer = tracing_init_dentry();
14a5ae40 8717 if (IS_ERR(d_tracer))
ed6f1c99 8718 return 0;
bc0c38d1 8719
58b92547
SRV
8720 event_trace_init();
8721
8434dc93 8722 init_tracer_tracefs(&global_trace, d_tracer);
501c2375 8723 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
bc0c38d1 8724
5452af66 8725 trace_create_file("tracing_thresh", 0644, d_tracer,
6508fa76 8726 &global_trace, &tracing_thresh_fops);
a8259075 8727
339ae5d3 8728 trace_create_file("README", 0444, d_tracer,
5452af66
FW
8729 NULL, &tracing_readme_fops);
8730
69abe6a5
AP
8731 trace_create_file("saved_cmdlines", 0444, d_tracer,
8732 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 8733
939c7a4f
YY
8734 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8735 NULL, &tracing_saved_cmdlines_size_fops);
8736
99c621d7
MS
8737 trace_create_file("saved_tgids", 0444, d_tracer,
8738 NULL, &tracing_saved_tgids_fops);
8739
5f60b351 8740 trace_eval_init();
0c564a53 8741
f57a4143 8742 trace_create_eval_file(d_tracer);
9828413d 8743
3673b8e4
SRRH
8744#ifdef CONFIG_MODULES
8745 register_module_notifier(&trace_module_nb);
8746#endif
8747
bc0c38d1 8748#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
8749 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8750 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 8751#endif
b04cc6b1 8752
277ba044 8753 create_trace_instances(d_tracer);
5452af66 8754
37aea98b 8755 update_tracer_options(&global_trace);
09d23a1d 8756
b5ad384e 8757 return 0;
bc0c38d1
SR
8758}
8759
3f5a54e3
SR
8760static int trace_panic_handler(struct notifier_block *this,
8761 unsigned long event, void *unused)
8762{
944ac425 8763 if (ftrace_dump_on_oops)
cecbca96 8764 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
8765 return NOTIFY_OK;
8766}
8767
8768static struct notifier_block trace_panic_notifier = {
8769 .notifier_call = trace_panic_handler,
8770 .next = NULL,
8771 .priority = 150 /* priority: INT_MAX >= x >= 0 */
8772};
8773
8774static int trace_die_handler(struct notifier_block *self,
8775 unsigned long val,
8776 void *data)
8777{
8778 switch (val) {
8779 case DIE_OOPS:
944ac425 8780 if (ftrace_dump_on_oops)
cecbca96 8781 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
8782 break;
8783 default:
8784 break;
8785 }
8786 return NOTIFY_OK;
8787}
8788
8789static struct notifier_block trace_die_notifier = {
8790 .notifier_call = trace_die_handler,
8791 .priority = 200
8792};
8793
8794/*
8795 * printk is set to max of 1024, we really don't need it that big.
8796 * Nothing should be printing 1000 characters anyway.
8797 */
8798#define TRACE_MAX_PRINT 1000
8799
8800/*
8801 * Define here KERN_TRACE so that we have one place to modify
8802 * it if we decide to change what log level the ftrace dump
8803 * should be at.
8804 */
428aee14 8805#define KERN_TRACE KERN_EMERG
3f5a54e3 8806
955b61e5 8807void
3f5a54e3
SR
8808trace_printk_seq(struct trace_seq *s)
8809{
8810 /* Probably should print a warning here. */
3a161d99
SRRH
8811 if (s->seq.len >= TRACE_MAX_PRINT)
8812 s->seq.len = TRACE_MAX_PRINT;
3f5a54e3 8813
820b75f6
SRRH
8814 /*
8815 * More paranoid code. Although the buffer size is set to
8816 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8817 * an extra layer of protection.
8818 */
8819 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8820 s->seq.len = s->seq.size - 1;
3f5a54e3
SR
8821
8822 /* should be zero ended, but we are paranoid. */
3a161d99 8823 s->buffer[s->seq.len] = 0;
3f5a54e3
SR
8824
8825 printk(KERN_TRACE "%s", s->buffer);
8826
f9520750 8827 trace_seq_init(s);
3f5a54e3
SR
8828}
8829
955b61e5
JW
8830void trace_init_global_iter(struct trace_iterator *iter)
8831{
8832 iter->tr = &global_trace;
2b6080f2 8833 iter->trace = iter->tr->current_trace;
ae3b5093 8834 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 8835 iter->trace_buffer = &global_trace.trace_buffer;
b2f974d6
CS
8836
8837 if (iter->trace && iter->trace->open)
8838 iter->trace->open(iter);
8839
8840 /* Annotate start of buffers if we had overruns */
8841 if (ring_buffer_overruns(iter->trace_buffer->buffer))
8842 iter->iter_flags |= TRACE_FILE_ANNOTATE;
8843
8844 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8845 if (trace_clocks[iter->tr->clock_id].in_ns)
8846 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
955b61e5
JW
8847}
8848
7fe70b57 8849void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 8850{
3f5a54e3
SR
8851 /* use static because iter can be a bit big for the stack */
8852 static struct trace_iterator iter;
7fe70b57 8853 static atomic_t dump_running;
983f938a 8854 struct trace_array *tr = &global_trace;
cf586b61 8855 unsigned int old_userobj;
d769041f
SR
8856 unsigned long flags;
8857 int cnt = 0, cpu;
3f5a54e3 8858
7fe70b57
SRRH
8859 /* Only allow one dump user at a time. */
8860 if (atomic_inc_return(&dump_running) != 1) {
8861 atomic_dec(&dump_running);
8862 return;
8863 }
3f5a54e3 8864
7fe70b57
SRRH
8865 /*
8866 * Always turn off tracing when we dump.
8867 * We don't need to show trace output of what happens
8868 * between multiple crashes.
8869 *
8870 * If the user does a sysrq-z, then they can re-enable
8871 * tracing with echo 1 > tracing_on.
8872 */
0ee6b6cf 8873 tracing_off();
cf586b61 8874
7fe70b57 8875 local_irq_save(flags);
03fc7f9c 8876 printk_nmi_direct_enter();
3f5a54e3 8877
38dbe0b1 8878 /* Simulate the iterator */
955b61e5
JW
8879 trace_init_global_iter(&iter);
8880
d769041f 8881 for_each_tracing_cpu(cpu) {
5e2d5ef8 8882 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
d769041f
SR
8883 }
8884
983f938a 8885 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
cf586b61 8886
b54d3de9 8887 /* don't look at user memory in panic mode */
983f938a 8888 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
b54d3de9 8889
cecbca96
FW
8890 switch (oops_dump_mode) {
8891 case DUMP_ALL:
ae3b5093 8892 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
8893 break;
8894 case DUMP_ORIG:
8895 iter.cpu_file = raw_smp_processor_id();
8896 break;
8897 case DUMP_NONE:
8898 goto out_enable;
8899 default:
8900 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 8901 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
8902 }
8903
8904 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 8905
7fe70b57
SRRH
8906 /* Did function tracer already get disabled? */
8907 if (ftrace_is_dead()) {
8908 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8909 printk("# MAY BE MISSING FUNCTION EVENTS\n");
8910 }
8911
3f5a54e3
SR
8912 /*
8913 * We need to stop all tracing on all CPUS to read the
8914 * the next buffer. This is a bit expensive, but is
8915 * not done often. We fill all what we can read,
8916 * and then release the locks again.
8917 */
8918
3f5a54e3
SR
8919 while (!trace_empty(&iter)) {
8920
8921 if (!cnt)
8922 printk(KERN_TRACE "---------------------------------\n");
8923
8924 cnt++;
8925
0c97bf86 8926 trace_iterator_reset(&iter);
3f5a54e3 8927 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3f5a54e3 8928
955b61e5 8929 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
8930 int ret;
8931
8932 ret = print_trace_line(&iter);
8933 if (ret != TRACE_TYPE_NO_CONSUME)
8934 trace_consume(&iter);
3f5a54e3 8935 }
b892e5c8 8936 touch_nmi_watchdog();
3f5a54e3
SR
8937
8938 trace_printk_seq(&iter.seq);
8939 }
8940
8941 if (!cnt)
8942 printk(KERN_TRACE " (ftrace buffer empty)\n");
8943 else
8944 printk(KERN_TRACE "---------------------------------\n");
8945
cecbca96 8946 out_enable:
983f938a 8947 tr->trace_flags |= old_userobj;
cf586b61 8948
7fe70b57
SRRH
8949 for_each_tracing_cpu(cpu) {
8950 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 8951 }
03fc7f9c
PM
8952 atomic_dec(&dump_running);
8953 printk_nmi_direct_exit();
cd891ae0 8954 local_irq_restore(flags);
3f5a54e3 8955}
a8eecf22 8956EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 8957
7e465baa
TZ
8958int trace_run_command(const char *buf, int (*createfn)(int, char **))
8959{
8960 char **argv;
8961 int argc, ret;
8962
8963 argc = 0;
8964 ret = 0;
8965 argv = argv_split(GFP_KERNEL, buf, &argc);
8966 if (!argv)
8967 return -ENOMEM;
8968
8969 if (argc)
8970 ret = createfn(argc, argv);
8971
8972 argv_free(argv);
8973
8974 return ret;
8975}
8976
8977#define WRITE_BUFSIZE 4096
8978
8979ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
8980 size_t count, loff_t *ppos,
8981 int (*createfn)(int, char **))
8982{
8983 char *kbuf, *buf, *tmp;
8984 int ret = 0;
8985 size_t done = 0;
8986 size_t size;
8987
8988 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
8989 if (!kbuf)
8990 return -ENOMEM;
8991
8992 while (done < count) {
8993 size = count - done;
8994
8995 if (size >= WRITE_BUFSIZE)
8996 size = WRITE_BUFSIZE - 1;
8997
8998 if (copy_from_user(kbuf, buffer + done, size)) {
8999 ret = -EFAULT;
9000 goto out;
9001 }
9002 kbuf[size] = '\0';
9003 buf = kbuf;
9004 do {
9005 tmp = strchr(buf, '\n');
9006 if (tmp) {
9007 *tmp = '\0';
9008 size = tmp - buf + 1;
9009 } else {
9010 size = strlen(buf);
9011 if (done + size < count) {
9012 if (buf != kbuf)
9013 break;
9014 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9015 pr_warn("Line length is too long: Should be less than %d\n",
9016 WRITE_BUFSIZE - 2);
9017 ret = -EINVAL;
9018 goto out;
9019 }
9020 }
9021 done += size;
9022
9023 /* Remove comments */
9024 tmp = strchr(buf, '#');
9025
9026 if (tmp)
9027 *tmp = '\0';
9028
9029 ret = trace_run_command(buf, createfn);
9030 if (ret)
9031 goto out;
9032 buf += size;
9033
9034 } while (done < count);
9035 }
9036 ret = done;
9037
9038out:
9039 kfree(kbuf);
9040
9041 return ret;
9042}
9043
3928a8a2 9044__init static int tracer_alloc_buffers(void)
bc0c38d1 9045{
73c5162a 9046 int ring_buf_size;
9e01c1b7 9047 int ret = -ENOMEM;
4c11d7ae 9048
b5e87c05
SRRH
9049 /*
9050 * Make sure we don't accidently add more trace options
9051 * than we have bits for.
9052 */
9a38a885 9053 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
b5e87c05 9054
9e01c1b7
RR
9055 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
9056 goto out;
9057
ccfe9e42 9058 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9e01c1b7 9059 goto out_free_buffer_mask;
4c11d7ae 9060
07d777fe
SR
9061 /* Only allocate trace_printk buffers if a trace_printk exists */
9062 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 9063 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
9064 trace_printk_init_buffers();
9065
73c5162a
SR
9066 /* To save memory, keep the ring buffer size to its minimum */
9067 if (ring_buffer_expanded)
9068 ring_buf_size = trace_buf_size;
9069 else
9070 ring_buf_size = 1;
9071
9e01c1b7 9072 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
ccfe9e42 9073 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9e01c1b7 9074
2b6080f2
SR
9075 raw_spin_lock_init(&global_trace.start_lock);
9076
b32614c0
SAS
9077 /*
9078 * The prepare callbacks allocates some memory for the ring buffer. We
9079 * don't free the buffer if the if the CPU goes down. If we were to free
9080 * the buffer, then the user would lose any trace that was in the
9081 * buffer. The memory will be removed once the "instance" is removed.
9082 */
9083 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
9084 "trace/RB:preapre", trace_rb_cpu_prepare,
9085 NULL);
9086 if (ret < 0)
9087 goto out_free_cpumask;
2c4a33ab 9088 /* Used for event triggers */
147d88e0 9089 ret = -ENOMEM;
2c4a33ab
SRRH
9090 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
9091 if (!temp_buffer)
b32614c0 9092 goto out_rm_hp_state;
2c4a33ab 9093
939c7a4f
YY
9094 if (trace_create_savedcmd() < 0)
9095 goto out_free_temp_buffer;
9096
9e01c1b7 9097 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 9098 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
9099 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
9100 WARN_ON(1);
939c7a4f 9101 goto out_free_savedcmd;
4c11d7ae 9102 }
a7603ff4 9103
499e5470
SR
9104 if (global_trace.buffer_disabled)
9105 tracing_off();
4c11d7ae 9106
e1e232ca
SR
9107 if (trace_boot_clock) {
9108 ret = tracing_set_clock(&global_trace, trace_boot_clock);
9109 if (ret < 0)
a395d6a7
JP
9110 pr_warn("Trace clock %s not defined, going back to default\n",
9111 trace_boot_clock);
e1e232ca
SR
9112 }
9113
ca164318
SRRH
9114 /*
9115 * register_tracer() might reference current_trace, so it
9116 * needs to be set before we register anything. This is
9117 * just a bootstrap of current_trace anyway.
9118 */
2b6080f2
SR
9119 global_trace.current_trace = &nop_trace;
9120
0b9b12c1
SRRH
9121 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9122
4104d326
SRRH
9123 ftrace_init_global_array_ops(&global_trace);
9124
9a38a885
SRRH
9125 init_trace_flags_index(&global_trace);
9126
ca164318
SRRH
9127 register_tracer(&nop_trace);
9128
dbeafd0d
SRV
9129 /* Function tracing may start here (via kernel command line) */
9130 init_function_trace();
9131
60a11774
SR
9132 /* All seems OK, enable tracing */
9133 tracing_disabled = 0;
3928a8a2 9134
3f5a54e3
SR
9135 atomic_notifier_chain_register(&panic_notifier_list,
9136 &trace_panic_notifier);
9137
9138 register_die_notifier(&trace_die_notifier);
2fc1dfbe 9139
ae63b31e
SR
9140 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
9141
9142 INIT_LIST_HEAD(&global_trace.systems);
9143 INIT_LIST_HEAD(&global_trace.events);
067fe038 9144 INIT_LIST_HEAD(&global_trace.hist_vars);
2f754e77 9145 INIT_LIST_HEAD(&global_trace.err_log);
ae63b31e
SR
9146 list_add(&global_trace.list, &ftrace_trace_arrays);
9147
a4d1e688 9148 apply_trace_boot_options();
7bcfaf54 9149
77fd5c15
SRRH
9150 register_snapshot_cmd();
9151
2fc1dfbe 9152 return 0;
3f5a54e3 9153
939c7a4f
YY
9154out_free_savedcmd:
9155 free_saved_cmdlines_buffer(savedcmd);
2c4a33ab
SRRH
9156out_free_temp_buffer:
9157 ring_buffer_free(temp_buffer);
b32614c0
SAS
9158out_rm_hp_state:
9159 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9e01c1b7 9160out_free_cpumask:
ccfe9e42 9161 free_cpumask_var(global_trace.tracing_cpumask);
9e01c1b7
RR
9162out_free_buffer_mask:
9163 free_cpumask_var(tracing_buffer_mask);
9164out:
9165 return ret;
bc0c38d1 9166}
b2821ae6 9167
e725c731 9168void __init early_trace_init(void)
5f893b26 9169{
0daa2302
SRRH
9170 if (tracepoint_printk) {
9171 tracepoint_print_iter =
9172 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
9173 if (WARN_ON(!tracepoint_print_iter))
9174 tracepoint_printk = 0;
42391745
SRRH
9175 else
9176 static_key_enable(&tracepoint_printk_key.key);
0daa2302 9177 }
5f893b26 9178 tracer_alloc_buffers();
e725c731
SRV
9179}
9180
9181void __init trace_init(void)
9182{
0c564a53 9183 trace_event_init();
5f893b26
SRRH
9184}
9185
b2821ae6
SR
9186__init static int clear_boot_tracer(void)
9187{
9188 /*
9189 * The default tracer at boot buffer is an init section.
9190 * This function is called in lateinit. If we did not
9191 * find the boot tracer, then clear it out, to prevent
9192 * later registration from accessing the buffer that is
9193 * about to be freed.
9194 */
9195 if (!default_bootup_tracer)
9196 return 0;
9197
9198 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
9199 default_bootup_tracer);
9200 default_bootup_tracer = NULL;
9201
9202 return 0;
9203}
9204
8434dc93 9205fs_initcall(tracer_init_tracefs);
4bb0f0e7 9206late_initcall_sync(clear_boot_tracer);
3fd49c9e
CW
9207
9208#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9209__init static int tracing_set_default_clock(void)
9210{
9211 /* sched_clock_stable() is determined in late_initcall */
5125eee4 9212 if (!trace_boot_clock && !sched_clock_stable()) {
3fd49c9e
CW
9213 printk(KERN_WARNING
9214 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9215 "If you want to keep using the local clock, then add:\n"
9216 " \"trace_clock=local\"\n"
9217 "on the kernel command line\n");
9218 tracing_set_clock(&global_trace, "global");
9219 }
9220
9221 return 0;
9222}
9223late_initcall_sync(tracing_set_default_clock);
9224#endif