]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/trace/trace.c
ftrace, orc, x86: Handle ftrace dynamically allocated trampolines
[mirror_ubuntu-jammy-kernel.git] / kernel / trace / trace.c
CommitLineData
bc0c38d1
SR
1/*
2 * ring buffer based function tracer
3 *
2b6080f2 4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
bc0c38d1
SR
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
6d49e352 12 * Copyright (C) 2004 Nadia Yvette Chambers
bc0c38d1 13 */
2cadf913 14#include <linux/ring_buffer.h>
273b281f 15#include <generated/utsrelease.h>
2cadf913
SR
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
bc0c38d1
SR
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
3f5a54e3 20#include <linux/notifier.h>
2cadf913 21#include <linux/irqflags.h>
bc0c38d1 22#include <linux/debugfs.h>
8434dc93 23#include <linux/tracefs.h>
4c11d7ae 24#include <linux/pagemap.h>
bc0c38d1
SR
25#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
76c813e2 28#include <linux/vmalloc.h>
bc0c38d1
SR
29#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
2cadf913 32#include <linux/splice.h>
3f5a54e3 33#include <linux/kdebug.h>
5f0c6c03 34#include <linux/string.h>
f76180bc 35#include <linux/mount.h>
7e53bd42 36#include <linux/rwsem.h>
5a0e3ad6 37#include <linux/slab.h>
bc0c38d1
SR
38#include <linux/ctype.h>
39#include <linux/init.h>
2a2cc8f7 40#include <linux/poll.h>
b892e5c8 41#include <linux/nmi.h>
bc0c38d1 42#include <linux/fs.h>
478409dd 43#include <linux/trace.h>
8bd75c77 44#include <linux/sched/rt.h>
86387f7e 45
bc0c38d1 46#include "trace.h"
f0868d1e 47#include "trace_output.h"
bc0c38d1 48
73c5162a
SR
49/*
50 * On boot up, the ring buffer is set to the minimum size, so that
51 * we do not waste memory on systems that are not using tracing.
52 */
55034cd6 53bool ring_buffer_expanded;
73c5162a 54
8e1b82e0
FW
55/*
56 * We need to change this state when a selftest is running.
ff32504f
FW
57 * A selftest will lurk into the ring-buffer to count the
58 * entries inserted during the selftest although some concurrent
5e1607a0 59 * insertions into the ring-buffer such as trace_printk could occurred
ff32504f
FW
60 * at the same time, giving false positive or negative results.
61 */
8e1b82e0 62static bool __read_mostly tracing_selftest_running;
ff32504f 63
b2821ae6
SR
64/*
65 * If a tracer is running, we do not want to run SELFTEST.
66 */
020e5f85 67bool __read_mostly tracing_selftest_disabled;
b2821ae6 68
0daa2302
SRRH
69/* Pipe tracepoints to printk */
70struct trace_iterator *tracepoint_print_iter;
71int tracepoint_printk;
42391745 72static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
0daa2302 73
adf9f195
FW
74/* For tracers that don't implement custom flags */
75static struct tracer_opt dummy_tracer_opt[] = {
76 { }
77};
78
8c1a49ae
SRRH
79static int
80dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
adf9f195
FW
81{
82 return 0;
83}
0f048701 84
7ffbd48d
SR
85/*
86 * To prevent the comm cache from being overwritten when no
87 * tracing is active, only save the comm when a trace event
88 * occurred.
89 */
d914ba37 90static DEFINE_PER_CPU(bool, trace_taskinfo_save);
7ffbd48d 91
0f048701
SR
92/*
93 * Kill all tracing for good (never come back).
94 * It is initialized to 1 but will turn to zero if the initialization
95 * of the tracer is successful. But that is the only place that sets
96 * this back to zero.
97 */
4fd27358 98static int tracing_disabled = 1;
0f048701 99
955b61e5 100cpumask_var_t __read_mostly tracing_buffer_mask;
ab46428c 101
944ac425
SR
102/*
103 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 *
105 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
106 * is set, then ftrace_dump is called. This will output the contents
107 * of the ftrace buffers to the console. This is very useful for
108 * capturing traces that lead to crashes and outputing it to a
109 * serial console.
110 *
111 * It is default off, but you can enable it with either specifying
112 * "ftrace_dump_on_oops" in the kernel command line, or setting
cecbca96
FW
113 * /proc/sys/kernel/ftrace_dump_on_oops
114 * Set 1 if you want to dump buffers of all CPUs
115 * Set 2 if you want to dump the buffer of the CPU that triggered oops
944ac425 116 */
cecbca96
FW
117
118enum ftrace_dump_mode ftrace_dump_on_oops;
944ac425 119
de7edd31
SRRH
120/* When set, tracing will stop when a WARN*() is hit */
121int __disable_trace_on_warning;
122
681bec03
JL
123#ifdef CONFIG_TRACE_EVAL_MAP_FILE
124/* Map of enums to their values, for "eval_map" file */
23bf8cb8 125struct trace_eval_map_head {
9828413d
SRRH
126 struct module *mod;
127 unsigned long length;
128};
129
23bf8cb8 130union trace_eval_map_item;
9828413d 131
23bf8cb8 132struct trace_eval_map_tail {
9828413d
SRRH
133 /*
134 * "end" is first and points to NULL as it must be different
00f4b652 135 * than "mod" or "eval_string"
9828413d 136 */
23bf8cb8 137 union trace_eval_map_item *next;
9828413d
SRRH
138 const char *end; /* points to NULL */
139};
140
1793ed93 141static DEFINE_MUTEX(trace_eval_mutex);
9828413d
SRRH
142
143/*
23bf8cb8 144 * The trace_eval_maps are saved in an array with two extra elements,
9828413d
SRRH
145 * one at the beginning, and one at the end. The beginning item contains
146 * the count of the saved maps (head.length), and the module they
147 * belong to if not built in (head.mod). The ending item contains a
681bec03 148 * pointer to the next array of saved eval_map items.
9828413d 149 */
23bf8cb8 150union trace_eval_map_item {
00f4b652 151 struct trace_eval_map map;
23bf8cb8
JL
152 struct trace_eval_map_head head;
153 struct trace_eval_map_tail tail;
9828413d
SRRH
154};
155
23bf8cb8 156static union trace_eval_map_item *trace_eval_maps;
681bec03 157#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 158
607e2ea1 159static int tracing_set_tracer(struct trace_array *tr, const char *buf);
b2821ae6 160
ee6c2c1b
LZ
161#define MAX_TRACER_SIZE 100
162static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
b2821ae6 163static char *default_bootup_tracer;
d9e54076 164
55034cd6
SRRH
165static bool allocate_snapshot;
166
1beee96b 167static int __init set_cmdline_ftrace(char *str)
d9e54076 168{
67012ab1 169 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
b2821ae6 170 default_bootup_tracer = bootup_tracer_buf;
73c5162a 171 /* We are using ftrace early, expand it */
55034cd6 172 ring_buffer_expanded = true;
d9e54076
PZ
173 return 1;
174}
1beee96b 175__setup("ftrace=", set_cmdline_ftrace);
d9e54076 176
944ac425
SR
177static int __init set_ftrace_dump_on_oops(char *str)
178{
cecbca96
FW
179 if (*str++ != '=' || !*str) {
180 ftrace_dump_on_oops = DUMP_ALL;
181 return 1;
182 }
183
184 if (!strcmp("orig_cpu", str)) {
185 ftrace_dump_on_oops = DUMP_ORIG;
186 return 1;
187 }
188
189 return 0;
944ac425
SR
190}
191__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
60a11774 192
de7edd31
SRRH
193static int __init stop_trace_on_warning(char *str)
194{
933ff9f2
LCG
195 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
196 __disable_trace_on_warning = 1;
de7edd31
SRRH
197 return 1;
198}
933ff9f2 199__setup("traceoff_on_warning", stop_trace_on_warning);
de7edd31 200
3209cff4 201static int __init boot_alloc_snapshot(char *str)
55034cd6
SRRH
202{
203 allocate_snapshot = true;
204 /* We also need the main ring buffer expanded */
205 ring_buffer_expanded = true;
206 return 1;
207}
3209cff4 208__setup("alloc_snapshot", boot_alloc_snapshot);
55034cd6 209
7bcfaf54
SR
210
211static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
7bcfaf54
SR
212
213static int __init set_trace_boot_options(char *str)
214{
67012ab1 215 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
7bcfaf54
SR
216 return 0;
217}
218__setup("trace_options=", set_trace_boot_options);
219
e1e232ca
SR
220static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
221static char *trace_boot_clock __initdata;
222
223static int __init set_trace_boot_clock(char *str)
224{
225 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
226 trace_boot_clock = trace_boot_clock_buf;
227 return 0;
228}
229__setup("trace_clock=", set_trace_boot_clock);
230
0daa2302
SRRH
231static int __init set_tracepoint_printk(char *str)
232{
233 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
234 tracepoint_printk = 1;
235 return 1;
236}
237__setup("tp_printk", set_tracepoint_printk);
de7edd31 238
a5a1d1c2 239unsigned long long ns2usecs(u64 nsec)
bc0c38d1
SR
240{
241 nsec += 500;
242 do_div(nsec, 1000);
243 return nsec;
244}
245
983f938a
SRRH
246/* trace_flags holds trace_options default values */
247#define TRACE_DEFAULT_FLAGS \
248 (FUNCTION_DEFAULT_FLAGS | \
249 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
250 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
251 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
252 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
253
16270145
SRRH
254/* trace_options that are only supported by global_trace */
255#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
256 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
257
20550622
SRRH
258/* trace_flags that are default zero for instances */
259#define ZEROED_TRACE_FLAGS \
1e10486f 260 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
16270145 261
4fcdae83 262/*
67d04bb2
JF
263 * The global_trace is the descriptor that holds the top-level tracing
264 * buffers for the live tracing.
4fcdae83 265 */
983f938a
SRRH
266static struct trace_array global_trace = {
267 .trace_flags = TRACE_DEFAULT_FLAGS,
268};
bc0c38d1 269
ae63b31e 270LIST_HEAD(ftrace_trace_arrays);
bc0c38d1 271
ff451961
SRRH
272int trace_array_get(struct trace_array *this_tr)
273{
274 struct trace_array *tr;
275 int ret = -ENODEV;
276
277 mutex_lock(&trace_types_lock);
278 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
279 if (tr == this_tr) {
280 tr->ref++;
281 ret = 0;
282 break;
283 }
284 }
285 mutex_unlock(&trace_types_lock);
286
287 return ret;
288}
289
290static void __trace_array_put(struct trace_array *this_tr)
291{
292 WARN_ON(!this_tr->ref);
293 this_tr->ref--;
294}
295
296void trace_array_put(struct trace_array *this_tr)
297{
298 mutex_lock(&trace_types_lock);
299 __trace_array_put(this_tr);
300 mutex_unlock(&trace_types_lock);
301}
302
2425bcb9 303int call_filter_check_discard(struct trace_event_call *call, void *rec,
f306cc82
TZ
304 struct ring_buffer *buffer,
305 struct ring_buffer_event *event)
306{
307 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
308 !filter_match_preds(call->filter, rec)) {
0fc1b09f 309 __trace_event_discard_commit(buffer, event);
f306cc82
TZ
310 return 1;
311 }
312
313 return 0;
eb02ce01
TZ
314}
315
76c813e2
SRRH
316void trace_free_pid_list(struct trace_pid_list *pid_list)
317{
318 vfree(pid_list->pids);
319 kfree(pid_list);
320}
321
d8275c45
SR
322/**
323 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
324 * @filtered_pids: The list of pids to check
325 * @search_pid: The PID to find in @filtered_pids
326 *
327 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
328 */
329bool
330trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
331{
332 /*
333 * If pid_max changed after filtered_pids was created, we
334 * by default ignore all pids greater than the previous pid_max.
335 */
336 if (search_pid >= filtered_pids->pid_max)
337 return false;
338
339 return test_bit(search_pid, filtered_pids->pids);
340}
341
342/**
343 * trace_ignore_this_task - should a task be ignored for tracing
344 * @filtered_pids: The list of pids to check
345 * @task: The task that should be ignored if not filtered
346 *
347 * Checks if @task should be traced or not from @filtered_pids.
348 * Returns true if @task should *NOT* be traced.
349 * Returns false if @task should be traced.
350 */
351bool
352trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
353{
354 /*
355 * Return false, because if filtered_pids does not exist,
356 * all pids are good to trace.
357 */
358 if (!filtered_pids)
359 return false;
360
361 return !trace_find_filtered_pid(filtered_pids, task->pid);
362}
363
364/**
5a93bae2 365 * trace_pid_filter_add_remove_task - Add or remove a task from a pid_list
d8275c45
SR
366 * @pid_list: The list to modify
367 * @self: The current task for fork or NULL for exit
368 * @task: The task to add or remove
369 *
370 * If adding a task, if @self is defined, the task is only added if @self
371 * is also included in @pid_list. This happens on fork and tasks should
372 * only be added when the parent is listed. If @self is NULL, then the
373 * @task pid will be removed from the list, which would happen on exit
374 * of a task.
375 */
376void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
377 struct task_struct *self,
378 struct task_struct *task)
379{
380 if (!pid_list)
381 return;
382
383 /* For forks, we only add if the forking task is listed */
384 if (self) {
385 if (!trace_find_filtered_pid(pid_list, self->pid))
386 return;
387 }
388
389 /* Sorry, but we don't support pid_max changing after setting */
390 if (task->pid >= pid_list->pid_max)
391 return;
392
393 /* "self" is set for forks, and NULL for exits */
394 if (self)
395 set_bit(task->pid, pid_list->pids);
396 else
397 clear_bit(task->pid, pid_list->pids);
398}
399
5cc8976b
SRRH
400/**
401 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
402 * @pid_list: The pid list to show
403 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
404 * @pos: The position of the file
405 *
406 * This is used by the seq_file "next" operation to iterate the pids
407 * listed in a trace_pid_list structure.
408 *
409 * Returns the pid+1 as we want to display pid of zero, but NULL would
410 * stop the iteration.
411 */
412void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
413{
414 unsigned long pid = (unsigned long)v;
415
416 (*pos)++;
417
418 /* pid already is +1 of the actual prevous bit */
419 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
420
421 /* Return pid + 1 to allow zero to be represented */
422 if (pid < pid_list->pid_max)
423 return (void *)(pid + 1);
424
425 return NULL;
426}
427
428/**
429 * trace_pid_start - Used for seq_file to start reading pid lists
430 * @pid_list: The pid list to show
431 * @pos: The position of the file
432 *
433 * This is used by seq_file "start" operation to start the iteration
434 * of listing pids.
435 *
436 * Returns the pid+1 as we want to display pid of zero, but NULL would
437 * stop the iteration.
438 */
439void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
440{
441 unsigned long pid;
442 loff_t l = 0;
443
444 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
445 if (pid >= pid_list->pid_max)
446 return NULL;
447
448 /* Return pid + 1 so that zero can be the exit value */
449 for (pid++; pid && l < *pos;
450 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
451 ;
452 return (void *)pid;
453}
454
455/**
456 * trace_pid_show - show the current pid in seq_file processing
457 * @m: The seq_file structure to write into
458 * @v: A void pointer of the pid (+1) value to display
459 *
460 * Can be directly used by seq_file operations to display the current
461 * pid value.
462 */
463int trace_pid_show(struct seq_file *m, void *v)
464{
465 unsigned long pid = (unsigned long)v - 1;
466
467 seq_printf(m, "%lu\n", pid);
468 return 0;
469}
470
76c813e2
SRRH
471/* 128 should be much more than enough */
472#define PID_BUF_SIZE 127
473
474int trace_pid_write(struct trace_pid_list *filtered_pids,
475 struct trace_pid_list **new_pid_list,
476 const char __user *ubuf, size_t cnt)
477{
478 struct trace_pid_list *pid_list;
479 struct trace_parser parser;
480 unsigned long val;
481 int nr_pids = 0;
482 ssize_t read = 0;
483 ssize_t ret = 0;
484 loff_t pos;
485 pid_t pid;
486
487 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
488 return -ENOMEM;
489
490 /*
491 * Always recreate a new array. The write is an all or nothing
492 * operation. Always create a new array when adding new pids by
493 * the user. If the operation fails, then the current list is
494 * not modified.
495 */
496 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
497 if (!pid_list)
498 return -ENOMEM;
499
500 pid_list->pid_max = READ_ONCE(pid_max);
501
502 /* Only truncating will shrink pid_max */
503 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
504 pid_list->pid_max = filtered_pids->pid_max;
505
506 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
507 if (!pid_list->pids) {
508 kfree(pid_list);
509 return -ENOMEM;
510 }
511
512 if (filtered_pids) {
513 /* copy the current bits to the new max */
67f20b08
WY
514 for_each_set_bit(pid, filtered_pids->pids,
515 filtered_pids->pid_max) {
76c813e2 516 set_bit(pid, pid_list->pids);
76c813e2
SRRH
517 nr_pids++;
518 }
519 }
520
521 while (cnt > 0) {
522
523 pos = 0;
524
525 ret = trace_get_user(&parser, ubuf, cnt, &pos);
526 if (ret < 0 || !trace_parser_loaded(&parser))
527 break;
528
529 read += ret;
530 ubuf += ret;
531 cnt -= ret;
532
533 parser.buffer[parser.idx] = 0;
534
535 ret = -EINVAL;
536 if (kstrtoul(parser.buffer, 0, &val))
537 break;
538 if (val >= pid_list->pid_max)
539 break;
540
541 pid = (pid_t)val;
542
543 set_bit(pid, pid_list->pids);
544 nr_pids++;
545
546 trace_parser_clear(&parser);
547 ret = 0;
548 }
549 trace_parser_put(&parser);
550
551 if (ret < 0) {
552 trace_free_pid_list(pid_list);
553 return ret;
554 }
555
556 if (!nr_pids) {
557 /* Cleared the list of pids */
558 trace_free_pid_list(pid_list);
559 read = ret;
560 pid_list = NULL;
561 }
562
563 *new_pid_list = pid_list;
564
565 return read;
566}
567
a5a1d1c2 568static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
37886f6a
SR
569{
570 u64 ts;
571
572 /* Early boot up does not have a buffer yet */
9457158b 573 if (!buf->buffer)
37886f6a
SR
574 return trace_clock_local();
575
9457158b
AL
576 ts = ring_buffer_time_stamp(buf->buffer, cpu);
577 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
37886f6a
SR
578
579 return ts;
580}
bc0c38d1 581
a5a1d1c2 582u64 ftrace_now(int cpu)
9457158b
AL
583{
584 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
585}
586
10246fa3
SRRH
587/**
588 * tracing_is_enabled - Show if global_trace has been disabled
589 *
590 * Shows if the global trace has been enabled or not. It uses the
591 * mirror flag "buffer_disabled" to be used in fast paths such as for
592 * the irqsoff tracer. But it may be inaccurate due to races. If you
593 * need to know the accurate state, use tracing_is_on() which is a little
594 * slower, but accurate.
595 */
9036990d
SR
596int tracing_is_enabled(void)
597{
10246fa3
SRRH
598 /*
599 * For quick access (irqsoff uses this in fast path), just
600 * return the mirror variable of the state of the ring buffer.
601 * It's a little racy, but we don't really care.
602 */
603 smp_rmb();
604 return !global_trace.buffer_disabled;
9036990d
SR
605}
606
4fcdae83 607/*
3928a8a2
SR
608 * trace_buf_size is the size in bytes that is allocated
609 * for a buffer. Note, the number of bytes is always rounded
610 * to page size.
3f5a54e3
SR
611 *
612 * This number is purposely set to a low number of 16384.
613 * If the dump on oops happens, it will be much appreciated
614 * to not have to wait for all that output. Anyway this can be
615 * boot time and run time configurable.
4fcdae83 616 */
3928a8a2 617#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
3f5a54e3 618
3928a8a2 619static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
bc0c38d1 620
4fcdae83 621/* trace_types holds a link list of available tracers. */
bc0c38d1 622static struct tracer *trace_types __read_mostly;
4fcdae83 623
4fcdae83
SR
624/*
625 * trace_types_lock is used to protect the trace_types list.
4fcdae83 626 */
a8227415 627DEFINE_MUTEX(trace_types_lock);
4fcdae83 628
7e53bd42
LJ
629/*
630 * serialize the access of the ring buffer
631 *
632 * ring buffer serializes readers, but it is low level protection.
633 * The validity of the events (which returns by ring_buffer_peek() ..etc)
634 * are not protected by ring buffer.
635 *
636 * The content of events may become garbage if we allow other process consumes
637 * these events concurrently:
638 * A) the page of the consumed events may become a normal page
639 * (not reader page) in ring buffer, and this page will be rewrited
640 * by events producer.
641 * B) The page of the consumed events may become a page for splice_read,
642 * and this page will be returned to system.
643 *
644 * These primitives allow multi process access to different cpu ring buffer
645 * concurrently.
646 *
647 * These primitives don't distinguish read-only and read-consume access.
648 * Multi read-only access are also serialized.
649 */
650
651#ifdef CONFIG_SMP
652static DECLARE_RWSEM(all_cpu_access_lock);
653static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
654
655static inline void trace_access_lock(int cpu)
656{
ae3b5093 657 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
658 /* gain it for accessing the whole ring buffer. */
659 down_write(&all_cpu_access_lock);
660 } else {
661 /* gain it for accessing a cpu ring buffer. */
662
ae3b5093 663 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
7e53bd42
LJ
664 down_read(&all_cpu_access_lock);
665
666 /* Secondly block other access to this @cpu ring buffer. */
667 mutex_lock(&per_cpu(cpu_access_lock, cpu));
668 }
669}
670
671static inline void trace_access_unlock(int cpu)
672{
ae3b5093 673 if (cpu == RING_BUFFER_ALL_CPUS) {
7e53bd42
LJ
674 up_write(&all_cpu_access_lock);
675 } else {
676 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
677 up_read(&all_cpu_access_lock);
678 }
679}
680
681static inline void trace_access_lock_init(void)
682{
683 int cpu;
684
685 for_each_possible_cpu(cpu)
686 mutex_init(&per_cpu(cpu_access_lock, cpu));
687}
688
689#else
690
691static DEFINE_MUTEX(access_lock);
692
693static inline void trace_access_lock(int cpu)
694{
695 (void)cpu;
696 mutex_lock(&access_lock);
697}
698
699static inline void trace_access_unlock(int cpu)
700{
701 (void)cpu;
702 mutex_unlock(&access_lock);
703}
704
705static inline void trace_access_lock_init(void)
706{
707}
708
709#endif
710
d78a4614
SRRH
711#ifdef CONFIG_STACKTRACE
712static void __ftrace_trace_stack(struct ring_buffer *buffer,
713 unsigned long flags,
714 int skip, int pc, struct pt_regs *regs);
2d34f489
SRRH
715static inline void ftrace_trace_stack(struct trace_array *tr,
716 struct ring_buffer *buffer,
73dddbb5
SRRH
717 unsigned long flags,
718 int skip, int pc, struct pt_regs *regs);
ca475e83 719
d78a4614
SRRH
720#else
721static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
722 unsigned long flags,
723 int skip, int pc, struct pt_regs *regs)
724{
725}
2d34f489
SRRH
726static inline void ftrace_trace_stack(struct trace_array *tr,
727 struct ring_buffer *buffer,
73dddbb5
SRRH
728 unsigned long flags,
729 int skip, int pc, struct pt_regs *regs)
ca475e83
SRRH
730{
731}
732
d78a4614
SRRH
733#endif
734
3e9a8aad
SRRH
735static __always_inline void
736trace_event_setup(struct ring_buffer_event *event,
737 int type, unsigned long flags, int pc)
738{
739 struct trace_entry *ent = ring_buffer_event_data(event);
740
741 tracing_generic_entry_update(ent, flags, pc);
742 ent->type = type;
743}
744
745static __always_inline struct ring_buffer_event *
746__trace_buffer_lock_reserve(struct ring_buffer *buffer,
747 int type,
748 unsigned long len,
749 unsigned long flags, int pc)
750{
751 struct ring_buffer_event *event;
752
753 event = ring_buffer_lock_reserve(buffer, len);
754 if (event != NULL)
755 trace_event_setup(event, type, flags, pc);
756
757 return event;
758}
759
2290f2c5 760void tracer_tracing_on(struct trace_array *tr)
10246fa3
SRRH
761{
762 if (tr->trace_buffer.buffer)
763 ring_buffer_record_on(tr->trace_buffer.buffer);
764 /*
765 * This flag is looked at when buffers haven't been allocated
766 * yet, or by some tracers (like irqsoff), that just want to
767 * know if the ring buffer has been disabled, but it can handle
768 * races of where it gets disabled but we still do a record.
769 * As the check is in the fast path of the tracers, it is more
770 * important to be fast than accurate.
771 */
772 tr->buffer_disabled = 0;
773 /* Make the flag seen by readers */
774 smp_wmb();
775}
776
499e5470
SR
777/**
778 * tracing_on - enable tracing buffers
779 *
780 * This function enables tracing buffers that may have been
781 * disabled with tracing_off.
782 */
783void tracing_on(void)
784{
10246fa3 785 tracer_tracing_on(&global_trace);
499e5470
SR
786}
787EXPORT_SYMBOL_GPL(tracing_on);
788
52ffabe3
SRRH
789
790static __always_inline void
791__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
792{
d914ba37 793 __this_cpu_write(trace_taskinfo_save, true);
52ffabe3
SRRH
794
795 /* If this is the temp buffer, we need to commit fully */
796 if (this_cpu_read(trace_buffered_event) == event) {
797 /* Length is in event->array[0] */
798 ring_buffer_write(buffer, event->array[0], &event->array[1]);
799 /* Release the temp buffer */
800 this_cpu_dec(trace_buffered_event_cnt);
801 } else
802 ring_buffer_unlock_commit(buffer, event);
803}
804
09ae7234
SRRH
805/**
806 * __trace_puts - write a constant string into the trace buffer.
807 * @ip: The address of the caller
808 * @str: The constant string to write
809 * @size: The size of the string.
810 */
811int __trace_puts(unsigned long ip, const char *str, int size)
812{
813 struct ring_buffer_event *event;
814 struct ring_buffer *buffer;
815 struct print_entry *entry;
816 unsigned long irq_flags;
817 int alloc;
8abfb872
J
818 int pc;
819
983f938a 820 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
821 return 0;
822
8abfb872 823 pc = preempt_count();
09ae7234 824
3132e107
SRRH
825 if (unlikely(tracing_selftest_running || tracing_disabled))
826 return 0;
827
09ae7234
SRRH
828 alloc = sizeof(*entry) + size + 2; /* possible \n added */
829
830 local_save_flags(irq_flags);
831 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
832 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
833 irq_flags, pc);
09ae7234
SRRH
834 if (!event)
835 return 0;
836
837 entry = ring_buffer_event_data(event);
838 entry->ip = ip;
839
840 memcpy(&entry->buf, str, size);
841
842 /* Add a newline if necessary */
843 if (entry->buf[size - 1] != '\n') {
844 entry->buf[size] = '\n';
845 entry->buf[size + 1] = '\0';
846 } else
847 entry->buf[size] = '\0';
848
849 __buffer_unlock_commit(buffer, event);
2d34f489 850 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
851
852 return size;
853}
854EXPORT_SYMBOL_GPL(__trace_puts);
855
856/**
857 * __trace_bputs - write the pointer to a constant string into trace buffer
858 * @ip: The address of the caller
859 * @str: The constant string to write to the buffer to
860 */
861int __trace_bputs(unsigned long ip, const char *str)
862{
863 struct ring_buffer_event *event;
864 struct ring_buffer *buffer;
865 struct bputs_entry *entry;
866 unsigned long irq_flags;
867 int size = sizeof(struct bputs_entry);
8abfb872
J
868 int pc;
869
983f938a 870 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
f0160a5a
J
871 return 0;
872
8abfb872 873 pc = preempt_count();
09ae7234 874
3132e107
SRRH
875 if (unlikely(tracing_selftest_running || tracing_disabled))
876 return 0;
877
09ae7234
SRRH
878 local_save_flags(irq_flags);
879 buffer = global_trace.trace_buffer.buffer;
3e9a8aad
SRRH
880 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
881 irq_flags, pc);
09ae7234
SRRH
882 if (!event)
883 return 0;
884
885 entry = ring_buffer_event_data(event);
886 entry->ip = ip;
887 entry->str = str;
888
889 __buffer_unlock_commit(buffer, event);
2d34f489 890 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
09ae7234
SRRH
891
892 return 1;
893}
894EXPORT_SYMBOL_GPL(__trace_bputs);
895
ad909e21 896#ifdef CONFIG_TRACER_SNAPSHOT
cab50379 897static void tracing_snapshot_instance(struct trace_array *tr)
ad909e21 898{
ad909e21
SRRH
899 struct tracer *tracer = tr->current_trace;
900 unsigned long flags;
901
1b22e382
SRRH
902 if (in_nmi()) {
903 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
904 internal_trace_puts("*** snapshot is being ignored ***\n");
905 return;
906 }
907
ad909e21 908 if (!tr->allocated_snapshot) {
ca268da6
SRRH
909 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
910 internal_trace_puts("*** stopping trace here! ***\n");
ad909e21
SRRH
911 tracing_off();
912 return;
913 }
914
915 /* Note, snapshot can not be used when the tracer uses it */
916 if (tracer->use_max_tr) {
ca268da6
SRRH
917 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
918 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
ad909e21
SRRH
919 return;
920 }
921
922 local_irq_save(flags);
923 update_max_tr(tr, current, smp_processor_id());
924 local_irq_restore(flags);
925}
cab50379
SRV
926
927/**
5a93bae2 928 * tracing_snapshot - take a snapshot of the current buffer.
cab50379
SRV
929 *
930 * This causes a swap between the snapshot buffer and the current live
931 * tracing buffer. You can use this to take snapshots of the live
932 * trace when some condition is triggered, but continue to trace.
933 *
934 * Note, make sure to allocate the snapshot with either
935 * a tracing_snapshot_alloc(), or by doing it manually
936 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
937 *
938 * If the snapshot buffer is not allocated, it will stop tracing.
939 * Basically making a permanent snapshot.
940 */
941void tracing_snapshot(void)
942{
943 struct trace_array *tr = &global_trace;
944
945 tracing_snapshot_instance(tr);
946}
1b22e382 947EXPORT_SYMBOL_GPL(tracing_snapshot);
ad909e21
SRRH
948
949static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
950 struct trace_buffer *size_buf, int cpu_id);
3209cff4
SRRH
951static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
952
953static int alloc_snapshot(struct trace_array *tr)
954{
955 int ret;
956
957 if (!tr->allocated_snapshot) {
958
959 /* allocate spare buffer */
960 ret = resize_buffer_duplicate_size(&tr->max_buffer,
961 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
962 if (ret < 0)
963 return ret;
964
965 tr->allocated_snapshot = true;
966 }
967
968 return 0;
969}
970
ad1438a0 971static void free_snapshot(struct trace_array *tr)
3209cff4
SRRH
972{
973 /*
974 * We don't free the ring buffer. instead, resize it because
975 * The max_tr ring buffer has some state (e.g. ring->clock) and
976 * we want preserve it.
977 */
978 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
979 set_buffer_entries(&tr->max_buffer, 1);
980 tracing_reset_online_cpus(&tr->max_buffer);
981 tr->allocated_snapshot = false;
982}
ad909e21 983
93e31ffb
TZ
984/**
985 * tracing_alloc_snapshot - allocate snapshot buffer.
986 *
987 * This only allocates the snapshot buffer if it isn't already
988 * allocated - it doesn't also take a snapshot.
989 *
990 * This is meant to be used in cases where the snapshot buffer needs
991 * to be set up for events that can't sleep but need to be able to
992 * trigger a snapshot.
993 */
994int tracing_alloc_snapshot(void)
995{
996 struct trace_array *tr = &global_trace;
997 int ret;
998
999 ret = alloc_snapshot(tr);
1000 WARN_ON(ret < 0);
1001
1002 return ret;
1003}
1004EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1005
ad909e21 1006/**
5a93bae2 1007 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
ad909e21 1008 *
5a93bae2 1009 * This is similar to tracing_snapshot(), but it will allocate the
ad909e21
SRRH
1010 * snapshot buffer if it isn't already allocated. Use this only
1011 * where it is safe to sleep, as the allocation may sleep.
1012 *
1013 * This causes a swap between the snapshot buffer and the current live
1014 * tracing buffer. You can use this to take snapshots of the live
1015 * trace when some condition is triggered, but continue to trace.
1016 */
1017void tracing_snapshot_alloc(void)
1018{
ad909e21
SRRH
1019 int ret;
1020
93e31ffb
TZ
1021 ret = tracing_alloc_snapshot();
1022 if (ret < 0)
3209cff4 1023 return;
ad909e21
SRRH
1024
1025 tracing_snapshot();
1026}
1b22e382 1027EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
1028#else
1029void tracing_snapshot(void)
1030{
1031 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1032}
1b22e382 1033EXPORT_SYMBOL_GPL(tracing_snapshot);
93e31ffb
TZ
1034int tracing_alloc_snapshot(void)
1035{
1036 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1037 return -ENODEV;
1038}
1039EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
ad909e21
SRRH
1040void tracing_snapshot_alloc(void)
1041{
1042 /* Give warning */
1043 tracing_snapshot();
1044}
1b22e382 1045EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
ad909e21
SRRH
1046#endif /* CONFIG_TRACER_SNAPSHOT */
1047
2290f2c5 1048void tracer_tracing_off(struct trace_array *tr)
10246fa3
SRRH
1049{
1050 if (tr->trace_buffer.buffer)
1051 ring_buffer_record_off(tr->trace_buffer.buffer);
1052 /*
1053 * This flag is looked at when buffers haven't been allocated
1054 * yet, or by some tracers (like irqsoff), that just want to
1055 * know if the ring buffer has been disabled, but it can handle
1056 * races of where it gets disabled but we still do a record.
1057 * As the check is in the fast path of the tracers, it is more
1058 * important to be fast than accurate.
1059 */
1060 tr->buffer_disabled = 1;
1061 /* Make the flag seen by readers */
1062 smp_wmb();
1063}
1064
499e5470
SR
1065/**
1066 * tracing_off - turn off tracing buffers
1067 *
1068 * This function stops the tracing buffers from recording data.
1069 * It does not disable any overhead the tracers themselves may
1070 * be causing. This function simply causes all recording to
1071 * the ring buffers to fail.
1072 */
1073void tracing_off(void)
1074{
10246fa3 1075 tracer_tracing_off(&global_trace);
499e5470
SR
1076}
1077EXPORT_SYMBOL_GPL(tracing_off);
1078
de7edd31
SRRH
1079void disable_trace_on_warning(void)
1080{
1081 if (__disable_trace_on_warning)
1082 tracing_off();
1083}
1084
10246fa3
SRRH
1085/**
1086 * tracer_tracing_is_on - show real state of ring buffer enabled
1087 * @tr : the trace array to know if ring buffer is enabled
1088 *
1089 * Shows real state of the ring buffer if it is enabled or not.
1090 */
e7c15cd8 1091int tracer_tracing_is_on(struct trace_array *tr)
10246fa3
SRRH
1092{
1093 if (tr->trace_buffer.buffer)
1094 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1095 return !tr->buffer_disabled;
1096}
1097
499e5470
SR
1098/**
1099 * tracing_is_on - show state of ring buffers enabled
1100 */
1101int tracing_is_on(void)
1102{
10246fa3 1103 return tracer_tracing_is_on(&global_trace);
499e5470
SR
1104}
1105EXPORT_SYMBOL_GPL(tracing_is_on);
1106
3928a8a2 1107static int __init set_buf_size(char *str)
bc0c38d1 1108{
3928a8a2 1109 unsigned long buf_size;
c6caeeb1 1110
bc0c38d1
SR
1111 if (!str)
1112 return 0;
9d612bef 1113 buf_size = memparse(str, &str);
c6caeeb1 1114 /* nr_entries can not be zero */
9d612bef 1115 if (buf_size == 0)
c6caeeb1 1116 return 0;
3928a8a2 1117 trace_buf_size = buf_size;
bc0c38d1
SR
1118 return 1;
1119}
3928a8a2 1120__setup("trace_buf_size=", set_buf_size);
bc0c38d1 1121
0e950173
TB
1122static int __init set_tracing_thresh(char *str)
1123{
87abb3b1 1124 unsigned long threshold;
0e950173
TB
1125 int ret;
1126
1127 if (!str)
1128 return 0;
bcd83ea6 1129 ret = kstrtoul(str, 0, &threshold);
0e950173
TB
1130 if (ret < 0)
1131 return 0;
87abb3b1 1132 tracing_thresh = threshold * 1000;
0e950173
TB
1133 return 1;
1134}
1135__setup("tracing_thresh=", set_tracing_thresh);
1136
57f50be1
SR
1137unsigned long nsecs_to_usecs(unsigned long nsecs)
1138{
1139 return nsecs / 1000;
1140}
1141
a3418a36
SRRH
1142/*
1143 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
f57a4143 1144 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
a3418a36 1145 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
f57a4143 1146 * of strings in the order that the evals (enum) were defined.
a3418a36
SRRH
1147 */
1148#undef C
1149#define C(a, b) b
1150
4fcdae83 1151/* These must match the bit postions in trace_iterator_flags */
bc0c38d1 1152static const char *trace_options[] = {
a3418a36 1153 TRACE_FLAGS
bc0c38d1
SR
1154 NULL
1155};
1156
5079f326
Z
1157static struct {
1158 u64 (*func)(void);
1159 const char *name;
8be0709f 1160 int in_ns; /* is this clock in nanoseconds? */
5079f326 1161} trace_clocks[] = {
1b3e5c09
TG
1162 { trace_clock_local, "local", 1 },
1163 { trace_clock_global, "global", 1 },
1164 { trace_clock_counter, "counter", 0 },
e7fda6c4 1165 { trace_clock_jiffies, "uptime", 0 },
1b3e5c09
TG
1166 { trace_clock, "perf", 1 },
1167 { ktime_get_mono_fast_ns, "mono", 1 },
aabfa5f2 1168 { ktime_get_raw_fast_ns, "mono_raw", 1 },
80ec3552 1169 { ktime_get_boot_fast_ns, "boot", 1 },
8cbd9cc6 1170 ARCH_TRACE_CLOCKS
5079f326
Z
1171};
1172
b63f39ea 1173/*
1174 * trace_parser_get_init - gets the buffer for trace parser
1175 */
1176int trace_parser_get_init(struct trace_parser *parser, int size)
1177{
1178 memset(parser, 0, sizeof(*parser));
1179
1180 parser->buffer = kmalloc(size, GFP_KERNEL);
1181 if (!parser->buffer)
1182 return 1;
1183
1184 parser->size = size;
1185 return 0;
1186}
1187
1188/*
1189 * trace_parser_put - frees the buffer for trace parser
1190 */
1191void trace_parser_put(struct trace_parser *parser)
1192{
1193 kfree(parser->buffer);
0e684b65 1194 parser->buffer = NULL;
b63f39ea 1195}
1196
1197/*
1198 * trace_get_user - reads the user input string separated by space
1199 * (matched by isspace(ch))
1200 *
1201 * For each string found the 'struct trace_parser' is updated,
1202 * and the function returns.
1203 *
1204 * Returns number of bytes read.
1205 *
1206 * See kernel/trace/trace.h for 'struct trace_parser' details.
1207 */
1208int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1209 size_t cnt, loff_t *ppos)
1210{
1211 char ch;
1212 size_t read = 0;
1213 ssize_t ret;
1214
1215 if (!*ppos)
1216 trace_parser_clear(parser);
1217
1218 ret = get_user(ch, ubuf++);
1219 if (ret)
1220 goto out;
1221
1222 read++;
1223 cnt--;
1224
1225 /*
1226 * The parser is not finished with the last write,
1227 * continue reading the user input without skipping spaces.
1228 */
1229 if (!parser->cont) {
1230 /* skip white space */
1231 while (cnt && isspace(ch)) {
1232 ret = get_user(ch, ubuf++);
1233 if (ret)
1234 goto out;
1235 read++;
1236 cnt--;
1237 }
1238
1239 /* only spaces were written */
1240 if (isspace(ch)) {
1241 *ppos += read;
1242 ret = read;
1243 goto out;
1244 }
1245
1246 parser->idx = 0;
1247 }
1248
1249 /* read the non-space input */
1250 while (cnt && !isspace(ch)) {
3c235a33 1251 if (parser->idx < parser->size - 1)
b63f39ea 1252 parser->buffer[parser->idx++] = ch;
1253 else {
1254 ret = -EINVAL;
1255 goto out;
1256 }
1257 ret = get_user(ch, ubuf++);
1258 if (ret)
1259 goto out;
1260 read++;
1261 cnt--;
1262 }
1263
1264 /* We either got finished input or we have to wait for another call. */
1265 if (isspace(ch)) {
1266 parser->buffer[parser->idx] = 0;
1267 parser->cont = false;
057db848 1268 } else if (parser->idx < parser->size - 1) {
b63f39ea 1269 parser->cont = true;
1270 parser->buffer[parser->idx++] = ch;
057db848
SR
1271 } else {
1272 ret = -EINVAL;
1273 goto out;
b63f39ea 1274 }
1275
1276 *ppos += read;
1277 ret = read;
1278
1279out:
1280 return ret;
1281}
1282
3a161d99 1283/* TODO add a seq_buf_to_buffer() */
b8b94265 1284static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
3c56819b
EGM
1285{
1286 int len;
3c56819b 1287
5ac48378 1288 if (trace_seq_used(s) <= s->seq.readpos)
3c56819b
EGM
1289 return -EBUSY;
1290
5ac48378 1291 len = trace_seq_used(s) - s->seq.readpos;
3c56819b
EGM
1292 if (cnt > len)
1293 cnt = len;
3a161d99 1294 memcpy(buf, s->buffer + s->seq.readpos, cnt);
3c56819b 1295
3a161d99 1296 s->seq.readpos += cnt;
3c56819b
EGM
1297 return cnt;
1298}
1299
0e950173
TB
1300unsigned long __read_mostly tracing_thresh;
1301
5d4a9dba 1302#ifdef CONFIG_TRACER_MAX_TRACE
5d4a9dba
SR
1303/*
1304 * Copy the new maximum trace into the separate maximum-trace
1305 * structure. (this way the maximum trace is permanently saved,
5a93bae2 1306 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
5d4a9dba
SR
1307 */
1308static void
1309__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1310{
12883efb
SRRH
1311 struct trace_buffer *trace_buf = &tr->trace_buffer;
1312 struct trace_buffer *max_buf = &tr->max_buffer;
1313 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1314 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
5d4a9dba 1315
12883efb
SRRH
1316 max_buf->cpu = cpu;
1317 max_buf->time_start = data->preempt_timestamp;
5d4a9dba 1318
6d9b3fa5 1319 max_data->saved_latency = tr->max_latency;
8248ac05
SR
1320 max_data->critical_start = data->critical_start;
1321 max_data->critical_end = data->critical_end;
5d4a9dba 1322
1acaa1b2 1323 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
8248ac05 1324 max_data->pid = tsk->pid;
f17a5194
SRRH
1325 /*
1326 * If tsk == current, then use current_uid(), as that does not use
1327 * RCU. The irq tracer can be called out of RCU scope.
1328 */
1329 if (tsk == current)
1330 max_data->uid = current_uid();
1331 else
1332 max_data->uid = task_uid(tsk);
1333
8248ac05
SR
1334 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1335 max_data->policy = tsk->policy;
1336 max_data->rt_priority = tsk->rt_priority;
5d4a9dba
SR
1337
1338 /* record this tasks comm */
1339 tracing_record_cmdline(tsk);
1340}
1341
4fcdae83
SR
1342/**
1343 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1344 * @tr: tracer
1345 * @tsk: the task with the latency
1346 * @cpu: The cpu that initiated the trace.
1347 *
1348 * Flip the buffers between the @tr and the max_tr and record information
1349 * about which task was the cause of this latency.
1350 */
e309b41d 1351void
bc0c38d1
SR
1352update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1353{
2721e72d 1354 struct ring_buffer *buf;
bc0c38d1 1355
2b6080f2 1356 if (tr->stop_count)
b8de7bd1
SR
1357 return;
1358
4c11d7ae 1359 WARN_ON_ONCE(!irqs_disabled());
34600f0e 1360
45ad21ca 1361 if (!tr->allocated_snapshot) {
debdd57f 1362 /* Only the nop tracer should hit this when disabling */
2b6080f2 1363 WARN_ON_ONCE(tr->current_trace != &nop_trace);
34600f0e 1364 return;
debdd57f 1365 }
34600f0e 1366
0b9b12c1 1367 arch_spin_lock(&tr->max_lock);
3928a8a2 1368
12883efb
SRRH
1369 buf = tr->trace_buffer.buffer;
1370 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1371 tr->max_buffer.buffer = buf;
3928a8a2 1372
bc0c38d1 1373 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1374 arch_spin_unlock(&tr->max_lock);
bc0c38d1
SR
1375}
1376
1377/**
1378 * update_max_tr_single - only copy one trace over, and reset the rest
1379 * @tr - tracer
1380 * @tsk - task with the latency
1381 * @cpu - the cpu of the buffer to copy.
4fcdae83
SR
1382 *
1383 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
bc0c38d1 1384 */
e309b41d 1385void
bc0c38d1
SR
1386update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1387{
3928a8a2 1388 int ret;
bc0c38d1 1389
2b6080f2 1390 if (tr->stop_count)
b8de7bd1
SR
1391 return;
1392
4c11d7ae 1393 WARN_ON_ONCE(!irqs_disabled());
6c24499f 1394 if (!tr->allocated_snapshot) {
2930e04d 1395 /* Only the nop tracer should hit this when disabling */
9e8529af 1396 WARN_ON_ONCE(tr->current_trace != &nop_trace);
ef710e10 1397 return;
2930e04d 1398 }
ef710e10 1399
0b9b12c1 1400 arch_spin_lock(&tr->max_lock);
bc0c38d1 1401
12883efb 1402 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
3928a8a2 1403
e8165dbb
SR
1404 if (ret == -EBUSY) {
1405 /*
1406 * We failed to swap the buffer due to a commit taking
1407 * place on this CPU. We fail to record, but we reset
1408 * the max trace buffer (no one writes directly to it)
1409 * and flag that it failed.
1410 */
12883efb 1411 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
e8165dbb
SR
1412 "Failed to swap buffers due to commit in progress\n");
1413 }
1414
e8165dbb 1415 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
bc0c38d1
SR
1416
1417 __update_max_tr(tr, tsk, cpu);
0b9b12c1 1418 arch_spin_unlock(&tr->max_lock);
bc0c38d1 1419}
5d4a9dba 1420#endif /* CONFIG_TRACER_MAX_TRACE */
bc0c38d1 1421
e30f53aa 1422static int wait_on_pipe(struct trace_iterator *iter, bool full)
0d5c6e1c 1423{
15693458
SRRH
1424 /* Iterators are static, they should be filled or empty */
1425 if (trace_buffer_iter(iter, iter->cpu_file))
8b8b3683 1426 return 0;
0d5c6e1c 1427
e30f53aa
RV
1428 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1429 full);
0d5c6e1c
SR
1430}
1431
f4e781c0 1432#ifdef CONFIG_FTRACE_STARTUP_TEST
9afecfbb
SRV
1433static bool selftests_can_run;
1434
1435struct trace_selftests {
1436 struct list_head list;
1437 struct tracer *type;
1438};
1439
1440static LIST_HEAD(postponed_selftests);
1441
1442static int save_selftest(struct tracer *type)
1443{
1444 struct trace_selftests *selftest;
1445
1446 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1447 if (!selftest)
1448 return -ENOMEM;
1449
1450 selftest->type = type;
1451 list_add(&selftest->list, &postponed_selftests);
1452 return 0;
1453}
1454
f4e781c0
SRRH
1455static int run_tracer_selftest(struct tracer *type)
1456{
1457 struct trace_array *tr = &global_trace;
1458 struct tracer *saved_tracer = tr->current_trace;
1459 int ret;
0d5c6e1c 1460
f4e781c0
SRRH
1461 if (!type->selftest || tracing_selftest_disabled)
1462 return 0;
0d5c6e1c 1463
9afecfbb
SRV
1464 /*
1465 * If a tracer registers early in boot up (before scheduling is
1466 * initialized and such), then do not run its selftests yet.
1467 * Instead, run it a little later in the boot process.
1468 */
1469 if (!selftests_can_run)
1470 return save_selftest(type);
1471
0d5c6e1c 1472 /*
f4e781c0
SRRH
1473 * Run a selftest on this tracer.
1474 * Here we reset the trace buffer, and set the current
1475 * tracer to be this tracer. The tracer can then run some
1476 * internal tracing to verify that everything is in order.
1477 * If we fail, we do not register this tracer.
0d5c6e1c 1478 */
f4e781c0 1479 tracing_reset_online_cpus(&tr->trace_buffer);
0d5c6e1c 1480
f4e781c0
SRRH
1481 tr->current_trace = type;
1482
1483#ifdef CONFIG_TRACER_MAX_TRACE
1484 if (type->use_max_tr) {
1485 /* If we expanded the buffers, make sure the max is expanded too */
1486 if (ring_buffer_expanded)
1487 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1488 RING_BUFFER_ALL_CPUS);
1489 tr->allocated_snapshot = true;
1490 }
1491#endif
1492
1493 /* the test is responsible for initializing and enabling */
1494 pr_info("Testing tracer %s: ", type->name);
1495 ret = type->selftest(type, tr);
1496 /* the test is responsible for resetting too */
1497 tr->current_trace = saved_tracer;
1498 if (ret) {
1499 printk(KERN_CONT "FAILED!\n");
1500 /* Add the warning after printing 'FAILED' */
1501 WARN_ON(1);
1502 return -1;
1503 }
1504 /* Only reset on passing, to avoid touching corrupted buffers */
1505 tracing_reset_online_cpus(&tr->trace_buffer);
1506
1507#ifdef CONFIG_TRACER_MAX_TRACE
1508 if (type->use_max_tr) {
1509 tr->allocated_snapshot = false;
0d5c6e1c 1510
f4e781c0
SRRH
1511 /* Shrink the max buffer again */
1512 if (ring_buffer_expanded)
1513 ring_buffer_resize(tr->max_buffer.buffer, 1,
1514 RING_BUFFER_ALL_CPUS);
1515 }
1516#endif
1517
1518 printk(KERN_CONT "PASSED\n");
1519 return 0;
1520}
9afecfbb
SRV
1521
1522static __init int init_trace_selftests(void)
1523{
1524 struct trace_selftests *p, *n;
1525 struct tracer *t, **last;
1526 int ret;
1527
1528 selftests_can_run = true;
1529
1530 mutex_lock(&trace_types_lock);
1531
1532 if (list_empty(&postponed_selftests))
1533 goto out;
1534
1535 pr_info("Running postponed tracer tests:\n");
1536
1537 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1538 ret = run_tracer_selftest(p->type);
1539 /* If the test fails, then warn and remove from available_tracers */
1540 if (ret < 0) {
1541 WARN(1, "tracer: %s failed selftest, disabling\n",
1542 p->type->name);
1543 last = &trace_types;
1544 for (t = trace_types; t; t = t->next) {
1545 if (t == p->type) {
1546 *last = t->next;
1547 break;
1548 }
1549 last = &t->next;
1550 }
1551 }
1552 list_del(&p->list);
1553 kfree(p);
1554 }
1555
1556 out:
1557 mutex_unlock(&trace_types_lock);
1558
1559 return 0;
1560}
b9ef0326 1561core_initcall(init_trace_selftests);
f4e781c0
SRRH
1562#else
1563static inline int run_tracer_selftest(struct tracer *type)
1564{
1565 return 0;
0d5c6e1c 1566}
f4e781c0 1567#endif /* CONFIG_FTRACE_STARTUP_TEST */
0d5c6e1c 1568
41d9c0be
SRRH
1569static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1570
a4d1e688
JW
1571static void __init apply_trace_boot_options(void);
1572
4fcdae83
SR
1573/**
1574 * register_tracer - register a tracer with the ftrace system.
1575 * @type - the plugin for the tracer
1576 *
1577 * Register a new plugin tracer.
1578 */
a4d1e688 1579int __init register_tracer(struct tracer *type)
bc0c38d1
SR
1580{
1581 struct tracer *t;
bc0c38d1
SR
1582 int ret = 0;
1583
1584 if (!type->name) {
1585 pr_info("Tracer must have a name\n");
1586 return -1;
1587 }
1588
24a461d5 1589 if (strlen(type->name) >= MAX_TRACER_SIZE) {
ee6c2c1b
LZ
1590 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1591 return -1;
1592 }
1593
bc0c38d1 1594 mutex_lock(&trace_types_lock);
86fa2f60 1595
8e1b82e0
FW
1596 tracing_selftest_running = true;
1597
bc0c38d1
SR
1598 for (t = trace_types; t; t = t->next) {
1599 if (strcmp(type->name, t->name) == 0) {
1600 /* already found */
ee6c2c1b 1601 pr_info("Tracer %s already registered\n",
bc0c38d1
SR
1602 type->name);
1603 ret = -1;
1604 goto out;
1605 }
1606 }
1607
adf9f195
FW
1608 if (!type->set_flag)
1609 type->set_flag = &dummy_set_flag;
d39cdd20
CH
1610 if (!type->flags) {
1611 /*allocate a dummy tracer_flags*/
1612 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
c8ca003b
CH
1613 if (!type->flags) {
1614 ret = -ENOMEM;
1615 goto out;
1616 }
d39cdd20
CH
1617 type->flags->val = 0;
1618 type->flags->opts = dummy_tracer_opt;
1619 } else
adf9f195
FW
1620 if (!type->flags->opts)
1621 type->flags->opts = dummy_tracer_opt;
6eaaa5d5 1622
d39cdd20
CH
1623 /* store the tracer for __set_tracer_option */
1624 type->flags->trace = type;
1625
f4e781c0
SRRH
1626 ret = run_tracer_selftest(type);
1627 if (ret < 0)
1628 goto out;
60a11774 1629
bc0c38d1
SR
1630 type->next = trace_types;
1631 trace_types = type;
41d9c0be 1632 add_tracer_options(&global_trace, type);
60a11774 1633
bc0c38d1 1634 out:
8e1b82e0 1635 tracing_selftest_running = false;
bc0c38d1
SR
1636 mutex_unlock(&trace_types_lock);
1637
dac74940
SR
1638 if (ret || !default_bootup_tracer)
1639 goto out_unlock;
1640
ee6c2c1b 1641 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
dac74940
SR
1642 goto out_unlock;
1643
1644 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1645 /* Do we want this tracer to start on bootup? */
607e2ea1 1646 tracing_set_tracer(&global_trace, type->name);
dac74940 1647 default_bootup_tracer = NULL;
a4d1e688
JW
1648
1649 apply_trace_boot_options();
1650
dac74940 1651 /* disable other selftests, since this will break it. */
55034cd6 1652 tracing_selftest_disabled = true;
b2821ae6 1653#ifdef CONFIG_FTRACE_STARTUP_TEST
dac74940
SR
1654 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1655 type->name);
b2821ae6 1656#endif
b2821ae6 1657
dac74940 1658 out_unlock:
bc0c38d1
SR
1659 return ret;
1660}
1661
12883efb 1662void tracing_reset(struct trace_buffer *buf, int cpu)
f633903a 1663{
12883efb 1664 struct ring_buffer *buffer = buf->buffer;
f633903a 1665
a5416411
HT
1666 if (!buffer)
1667 return;
1668
f633903a
SR
1669 ring_buffer_record_disable(buffer);
1670
1671 /* Make sure all commits have finished */
1672 synchronize_sched();
68179686 1673 ring_buffer_reset_cpu(buffer, cpu);
f633903a
SR
1674
1675 ring_buffer_record_enable(buffer);
1676}
1677
12883efb 1678void tracing_reset_online_cpus(struct trace_buffer *buf)
213cc060 1679{
12883efb 1680 struct ring_buffer *buffer = buf->buffer;
213cc060
PE
1681 int cpu;
1682
a5416411
HT
1683 if (!buffer)
1684 return;
1685
621968cd
SR
1686 ring_buffer_record_disable(buffer);
1687
1688 /* Make sure all commits have finished */
1689 synchronize_sched();
1690
9457158b 1691 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
213cc060
PE
1692
1693 for_each_online_cpu(cpu)
68179686 1694 ring_buffer_reset_cpu(buffer, cpu);
621968cd
SR
1695
1696 ring_buffer_record_enable(buffer);
213cc060
PE
1697}
1698
09d8091c 1699/* Must have trace_types_lock held */
873c642f 1700void tracing_reset_all_online_cpus(void)
9456f0fa 1701{
873c642f
SRRH
1702 struct trace_array *tr;
1703
873c642f 1704 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
065e63f9
SRV
1705 if (!tr->clear_trace)
1706 continue;
1707 tr->clear_trace = false;
12883efb
SRRH
1708 tracing_reset_online_cpus(&tr->trace_buffer);
1709#ifdef CONFIG_TRACER_MAX_TRACE
1710 tracing_reset_online_cpus(&tr->max_buffer);
1711#endif
873c642f 1712 }
9456f0fa
SR
1713}
1714
d914ba37
JF
1715static int *tgid_map;
1716
939c7a4f 1717#define SAVED_CMDLINES_DEFAULT 128
2c7eea4c 1718#define NO_CMDLINE_MAP UINT_MAX
edc35bd7 1719static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
939c7a4f
YY
1720struct saved_cmdlines_buffer {
1721 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1722 unsigned *map_cmdline_to_pid;
1723 unsigned cmdline_num;
1724 int cmdline_idx;
1725 char *saved_cmdlines;
1726};
1727static struct saved_cmdlines_buffer *savedcmd;
25b0b44a 1728
25b0b44a 1729/* temporary disable recording */
d914ba37 1730static atomic_t trace_record_taskinfo_disabled __read_mostly;
bc0c38d1 1731
939c7a4f
YY
1732static inline char *get_saved_cmdlines(int idx)
1733{
1734 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1735}
1736
1737static inline void set_cmdline(int idx, const char *cmdline)
bc0c38d1 1738{
939c7a4f
YY
1739 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1740}
1741
1742static int allocate_cmdlines_buffer(unsigned int val,
1743 struct saved_cmdlines_buffer *s)
1744{
1745 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1746 GFP_KERNEL);
1747 if (!s->map_cmdline_to_pid)
1748 return -ENOMEM;
1749
1750 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1751 if (!s->saved_cmdlines) {
1752 kfree(s->map_cmdline_to_pid);
1753 return -ENOMEM;
1754 }
1755
1756 s->cmdline_idx = 0;
1757 s->cmdline_num = val;
1758 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1759 sizeof(s->map_pid_to_cmdline));
1760 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1761 val * sizeof(*s->map_cmdline_to_pid));
1762
1763 return 0;
1764}
1765
1766static int trace_create_savedcmd(void)
1767{
1768 int ret;
1769
a6af8fbf 1770 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
939c7a4f
YY
1771 if (!savedcmd)
1772 return -ENOMEM;
1773
1774 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1775 if (ret < 0) {
1776 kfree(savedcmd);
1777 savedcmd = NULL;
1778 return -ENOMEM;
1779 }
1780
1781 return 0;
bc0c38d1
SR
1782}
1783
b5130b1e
CE
1784int is_tracing_stopped(void)
1785{
2b6080f2 1786 return global_trace.stop_count;
b5130b1e
CE
1787}
1788
0f048701
SR
1789/**
1790 * tracing_start - quick start of the tracer
1791 *
1792 * If tracing is enabled but was stopped by tracing_stop,
1793 * this will start the tracer back up.
1794 */
1795void tracing_start(void)
1796{
1797 struct ring_buffer *buffer;
1798 unsigned long flags;
1799
1800 if (tracing_disabled)
1801 return;
1802
2b6080f2
SR
1803 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1804 if (--global_trace.stop_count) {
1805 if (global_trace.stop_count < 0) {
b06a8301
SR
1806 /* Someone screwed up their debugging */
1807 WARN_ON_ONCE(1);
2b6080f2 1808 global_trace.stop_count = 0;
b06a8301 1809 }
0f048701
SR
1810 goto out;
1811 }
1812
a2f80714 1813 /* Prevent the buffers from switching */
0b9b12c1 1814 arch_spin_lock(&global_trace.max_lock);
0f048701 1815
12883efb 1816 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1817 if (buffer)
1818 ring_buffer_record_enable(buffer);
1819
12883efb
SRRH
1820#ifdef CONFIG_TRACER_MAX_TRACE
1821 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1822 if (buffer)
1823 ring_buffer_record_enable(buffer);
12883efb 1824#endif
0f048701 1825
0b9b12c1 1826 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1827
0f048701 1828 out:
2b6080f2
SR
1829 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1830}
1831
1832static void tracing_start_tr(struct trace_array *tr)
1833{
1834 struct ring_buffer *buffer;
1835 unsigned long flags;
1836
1837 if (tracing_disabled)
1838 return;
1839
1840 /* If global, we need to also start the max tracer */
1841 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1842 return tracing_start();
1843
1844 raw_spin_lock_irqsave(&tr->start_lock, flags);
1845
1846 if (--tr->stop_count) {
1847 if (tr->stop_count < 0) {
1848 /* Someone screwed up their debugging */
1849 WARN_ON_ONCE(1);
1850 tr->stop_count = 0;
1851 }
1852 goto out;
1853 }
1854
12883efb 1855 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1856 if (buffer)
1857 ring_buffer_record_enable(buffer);
1858
1859 out:
1860 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1861}
1862
1863/**
1864 * tracing_stop - quick stop of the tracer
1865 *
1866 * Light weight way to stop tracing. Use in conjunction with
1867 * tracing_start.
1868 */
1869void tracing_stop(void)
1870{
1871 struct ring_buffer *buffer;
1872 unsigned long flags;
1873
2b6080f2
SR
1874 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1875 if (global_trace.stop_count++)
0f048701
SR
1876 goto out;
1877
a2f80714 1878 /* Prevent the buffers from switching */
0b9b12c1 1879 arch_spin_lock(&global_trace.max_lock);
a2f80714 1880
12883efb 1881 buffer = global_trace.trace_buffer.buffer;
0f048701
SR
1882 if (buffer)
1883 ring_buffer_record_disable(buffer);
1884
12883efb
SRRH
1885#ifdef CONFIG_TRACER_MAX_TRACE
1886 buffer = global_trace.max_buffer.buffer;
0f048701
SR
1887 if (buffer)
1888 ring_buffer_record_disable(buffer);
12883efb 1889#endif
0f048701 1890
0b9b12c1 1891 arch_spin_unlock(&global_trace.max_lock);
a2f80714 1892
0f048701 1893 out:
2b6080f2
SR
1894 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1895}
1896
1897static void tracing_stop_tr(struct trace_array *tr)
1898{
1899 struct ring_buffer *buffer;
1900 unsigned long flags;
1901
1902 /* If global, we need to also stop the max tracer */
1903 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1904 return tracing_stop();
1905
1906 raw_spin_lock_irqsave(&tr->start_lock, flags);
1907 if (tr->stop_count++)
1908 goto out;
1909
12883efb 1910 buffer = tr->trace_buffer.buffer;
2b6080f2
SR
1911 if (buffer)
1912 ring_buffer_record_disable(buffer);
1913
1914 out:
1915 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
0f048701
SR
1916}
1917
379cfdac 1918static int trace_save_cmdline(struct task_struct *tsk)
bc0c38d1 1919{
a635cf04 1920 unsigned pid, idx;
bc0c38d1 1921
eaf260ac
JF
1922 /* treat recording of idle task as a success */
1923 if (!tsk->pid)
1924 return 1;
1925
1926 if (unlikely(tsk->pid > PID_MAX_DEFAULT))
379cfdac 1927 return 0;
bc0c38d1
SR
1928
1929 /*
1930 * It's not the end of the world if we don't get
1931 * the lock, but we also don't want to spin
1932 * nor do we want to disable interrupts,
1933 * so if we miss here, then better luck next time.
1934 */
0199c4e6 1935 if (!arch_spin_trylock(&trace_cmdline_lock))
379cfdac 1936 return 0;
bc0c38d1 1937
939c7a4f 1938 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
2c7eea4c 1939 if (idx == NO_CMDLINE_MAP) {
939c7a4f 1940 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
bc0c38d1 1941
a635cf04
CE
1942 /*
1943 * Check whether the cmdline buffer at idx has a pid
1944 * mapped. We are going to overwrite that entry so we
1945 * need to clear the map_pid_to_cmdline. Otherwise we
1946 * would read the new comm for the old pid.
1947 */
939c7a4f 1948 pid = savedcmd->map_cmdline_to_pid[idx];
a635cf04 1949 if (pid != NO_CMDLINE_MAP)
939c7a4f 1950 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
bc0c38d1 1951
939c7a4f
YY
1952 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1953 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
bc0c38d1 1954
939c7a4f 1955 savedcmd->cmdline_idx = idx;
bc0c38d1
SR
1956 }
1957
939c7a4f 1958 set_cmdline(idx, tsk->comm);
bc0c38d1 1959
0199c4e6 1960 arch_spin_unlock(&trace_cmdline_lock);
379cfdac
SRRH
1961
1962 return 1;
bc0c38d1
SR
1963}
1964
4c27e756 1965static void __trace_find_cmdline(int pid, char comm[])
bc0c38d1 1966{
bc0c38d1
SR
1967 unsigned map;
1968
4ca53085
SR
1969 if (!pid) {
1970 strcpy(comm, "<idle>");
1971 return;
1972 }
bc0c38d1 1973
74bf4076
SR
1974 if (WARN_ON_ONCE(pid < 0)) {
1975 strcpy(comm, "<XXX>");
1976 return;
1977 }
1978
4ca53085
SR
1979 if (pid > PID_MAX_DEFAULT) {
1980 strcpy(comm, "<...>");
1981 return;
1982 }
bc0c38d1 1983
939c7a4f 1984 map = savedcmd->map_pid_to_cmdline[pid];
50d88758 1985 if (map != NO_CMDLINE_MAP)
e09e2867 1986 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
50d88758
TG
1987 else
1988 strcpy(comm, "<...>");
4c27e756
SRRH
1989}
1990
1991void trace_find_cmdline(int pid, char comm[])
1992{
1993 preempt_disable();
1994 arch_spin_lock(&trace_cmdline_lock);
1995
1996 __trace_find_cmdline(pid, comm);
bc0c38d1 1997
0199c4e6 1998 arch_spin_unlock(&trace_cmdline_lock);
5b6045a9 1999 preempt_enable();
bc0c38d1
SR
2000}
2001
d914ba37
JF
2002int trace_find_tgid(int pid)
2003{
2004 if (unlikely(!tgid_map || !pid || pid > PID_MAX_DEFAULT))
2005 return 0;
2006
2007 return tgid_map[pid];
2008}
2009
2010static int trace_save_tgid(struct task_struct *tsk)
2011{
bd45d34d
JF
2012 /* treat recording of idle task as a success */
2013 if (!tsk->pid)
2014 return 1;
2015
2016 if (unlikely(!tgid_map || tsk->pid > PID_MAX_DEFAULT))
d914ba37
JF
2017 return 0;
2018
2019 tgid_map[tsk->pid] = tsk->tgid;
2020 return 1;
2021}
2022
2023static bool tracing_record_taskinfo_skip(int flags)
2024{
2025 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2026 return true;
2027 if (atomic_read(&trace_record_taskinfo_disabled) || !tracing_is_on())
2028 return true;
2029 if (!__this_cpu_read(trace_taskinfo_save))
2030 return true;
2031 return false;
2032}
2033
2034/**
2035 * tracing_record_taskinfo - record the task info of a task
2036 *
2037 * @task - task to record
2038 * @flags - TRACE_RECORD_CMDLINE for recording comm
2039 * - TRACE_RECORD_TGID for recording tgid
2040 */
2041void tracing_record_taskinfo(struct task_struct *task, int flags)
2042{
29b1a8ad
JF
2043 bool done;
2044
d914ba37
JF
2045 if (tracing_record_taskinfo_skip(flags))
2046 return;
29b1a8ad
JF
2047
2048 /*
2049 * Record as much task information as possible. If some fail, continue
2050 * to try to record the others.
2051 */
2052 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2053 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2054
2055 /* If recording any information failed, retry again soon. */
2056 if (!done)
d914ba37
JF
2057 return;
2058
2059 __this_cpu_write(trace_taskinfo_save, false);
2060}
2061
2062/**
2063 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2064 *
2065 * @prev - previous task during sched_switch
2066 * @next - next task during sched_switch
2067 * @flags - TRACE_RECORD_CMDLINE for recording comm
2068 * TRACE_RECORD_TGID for recording tgid
2069 */
2070void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2071 struct task_struct *next, int flags)
bc0c38d1 2072{
29b1a8ad
JF
2073 bool done;
2074
d914ba37
JF
2075 if (tracing_record_taskinfo_skip(flags))
2076 return;
2077
29b1a8ad
JF
2078 /*
2079 * Record as much task information as possible. If some fail, continue
2080 * to try to record the others.
2081 */
2082 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2083 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2084 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2085 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
bc0c38d1 2086
29b1a8ad
JF
2087 /* If recording any information failed, retry again soon. */
2088 if (!done)
7ffbd48d
SR
2089 return;
2090
d914ba37
JF
2091 __this_cpu_write(trace_taskinfo_save, false);
2092}
2093
2094/* Helpers to record a specific task information */
2095void tracing_record_cmdline(struct task_struct *task)
2096{
2097 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2098}
2099
2100void tracing_record_tgid(struct task_struct *task)
2101{
2102 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
bc0c38d1
SR
2103}
2104
af0009fc
SRV
2105/*
2106 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2107 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2108 * simplifies those functions and keeps them in sync.
2109 */
2110enum print_line_t trace_handle_return(struct trace_seq *s)
2111{
2112 return trace_seq_has_overflowed(s) ?
2113 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2114}
2115EXPORT_SYMBOL_GPL(trace_handle_return);
2116
45dcd8b8 2117void
38697053
SR
2118tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
2119 int pc)
bc0c38d1
SR
2120{
2121 struct task_struct *tsk = current;
bc0c38d1 2122
777e208d
SR
2123 entry->preempt_count = pc & 0xff;
2124 entry->pid = (tsk) ? tsk->pid : 0;
2125 entry->flags =
9244489a 2126#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2e2ca155 2127 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
9244489a
SR
2128#else
2129 TRACE_FLAG_IRQS_NOSUPPORT |
2130#endif
7e6867bf 2131 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
bc0c38d1 2132 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
c59f29cb 2133 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
e5137b50
PZ
2134 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2135 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
bc0c38d1 2136}
f413cdb8 2137EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
bc0c38d1 2138
e77405ad
SR
2139struct ring_buffer_event *
2140trace_buffer_lock_reserve(struct ring_buffer *buffer,
2141 int type,
2142 unsigned long len,
2143 unsigned long flags, int pc)
51a763dd 2144{
3e9a8aad 2145 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
0fc1b09f
SRRH
2146}
2147
2148DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2149DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2150static int trace_buffered_event_ref;
2151
2152/**
2153 * trace_buffered_event_enable - enable buffering events
2154 *
2155 * When events are being filtered, it is quicker to use a temporary
2156 * buffer to write the event data into if there's a likely chance
2157 * that it will not be committed. The discard of the ring buffer
2158 * is not as fast as committing, and is much slower than copying
2159 * a commit.
2160 *
2161 * When an event is to be filtered, allocate per cpu buffers to
2162 * write the event data into, and if the event is filtered and discarded
2163 * it is simply dropped, otherwise, the entire data is to be committed
2164 * in one shot.
2165 */
2166void trace_buffered_event_enable(void)
2167{
2168 struct ring_buffer_event *event;
2169 struct page *page;
2170 int cpu;
51a763dd 2171
0fc1b09f
SRRH
2172 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2173
2174 if (trace_buffered_event_ref++)
2175 return;
2176
2177 for_each_tracing_cpu(cpu) {
2178 page = alloc_pages_node(cpu_to_node(cpu),
2179 GFP_KERNEL | __GFP_NORETRY, 0);
2180 if (!page)
2181 goto failed;
2182
2183 event = page_address(page);
2184 memset(event, 0, sizeof(*event));
2185
2186 per_cpu(trace_buffered_event, cpu) = event;
2187
2188 preempt_disable();
2189 if (cpu == smp_processor_id() &&
2190 this_cpu_read(trace_buffered_event) !=
2191 per_cpu(trace_buffered_event, cpu))
2192 WARN_ON_ONCE(1);
2193 preempt_enable();
51a763dd
ACM
2194 }
2195
0fc1b09f
SRRH
2196 return;
2197 failed:
2198 trace_buffered_event_disable();
2199}
2200
2201static void enable_trace_buffered_event(void *data)
2202{
2203 /* Probably not needed, but do it anyway */
2204 smp_rmb();
2205 this_cpu_dec(trace_buffered_event_cnt);
2206}
2207
2208static void disable_trace_buffered_event(void *data)
2209{
2210 this_cpu_inc(trace_buffered_event_cnt);
2211}
2212
2213/**
2214 * trace_buffered_event_disable - disable buffering events
2215 *
2216 * When a filter is removed, it is faster to not use the buffered
2217 * events, and to commit directly into the ring buffer. Free up
2218 * the temp buffers when there are no more users. This requires
2219 * special synchronization with current events.
2220 */
2221void trace_buffered_event_disable(void)
2222{
2223 int cpu;
2224
2225 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2226
2227 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2228 return;
2229
2230 if (--trace_buffered_event_ref)
2231 return;
2232
2233 preempt_disable();
2234 /* For each CPU, set the buffer as used. */
2235 smp_call_function_many(tracing_buffer_mask,
2236 disable_trace_buffered_event, NULL, 1);
2237 preempt_enable();
2238
2239 /* Wait for all current users to finish */
2240 synchronize_sched();
2241
2242 for_each_tracing_cpu(cpu) {
2243 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2244 per_cpu(trace_buffered_event, cpu) = NULL;
2245 }
2246 /*
2247 * Make sure trace_buffered_event is NULL before clearing
2248 * trace_buffered_event_cnt.
2249 */
2250 smp_wmb();
2251
2252 preempt_disable();
2253 /* Do the work on each cpu */
2254 smp_call_function_many(tracing_buffer_mask,
2255 enable_trace_buffered_event, NULL, 1);
2256 preempt_enable();
51a763dd 2257}
51a763dd 2258
2c4a33ab
SRRH
2259static struct ring_buffer *temp_buffer;
2260
ccb469a1
SR
2261struct ring_buffer_event *
2262trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
7f1d2f82 2263 struct trace_event_file *trace_file,
ccb469a1
SR
2264 int type, unsigned long len,
2265 unsigned long flags, int pc)
2266{
2c4a33ab 2267 struct ring_buffer_event *entry;
0fc1b09f 2268 int val;
2c4a33ab 2269
7f1d2f82 2270 *current_rb = trace_file->tr->trace_buffer.buffer;
0fc1b09f
SRRH
2271
2272 if ((trace_file->flags &
2273 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2274 (entry = this_cpu_read(trace_buffered_event))) {
2275 /* Try to use the per cpu buffer first */
2276 val = this_cpu_inc_return(trace_buffered_event_cnt);
2277 if (val == 1) {
2278 trace_event_setup(entry, type, flags, pc);
2279 entry->array[0] = len;
2280 return entry;
2281 }
2282 this_cpu_dec(trace_buffered_event_cnt);
2283 }
2284
3e9a8aad
SRRH
2285 entry = __trace_buffer_lock_reserve(*current_rb,
2286 type, len, flags, pc);
2c4a33ab
SRRH
2287 /*
2288 * If tracing is off, but we have triggers enabled
2289 * we still need to look at the event data. Use the temp_buffer
2290 * to store the trace event for the tigger to use. It's recusive
2291 * safe and will not be recorded anywhere.
2292 */
5d6ad960 2293 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2c4a33ab 2294 *current_rb = temp_buffer;
3e9a8aad
SRRH
2295 entry = __trace_buffer_lock_reserve(*current_rb,
2296 type, len, flags, pc);
2c4a33ab
SRRH
2297 }
2298 return entry;
ccb469a1
SR
2299}
2300EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2301
42391745
SRRH
2302static DEFINE_SPINLOCK(tracepoint_iter_lock);
2303static DEFINE_MUTEX(tracepoint_printk_mutex);
2304
2305static void output_printk(struct trace_event_buffer *fbuffer)
2306{
2307 struct trace_event_call *event_call;
2308 struct trace_event *event;
2309 unsigned long flags;
2310 struct trace_iterator *iter = tracepoint_print_iter;
2311
2312 /* We should never get here if iter is NULL */
2313 if (WARN_ON_ONCE(!iter))
2314 return;
2315
2316 event_call = fbuffer->trace_file->event_call;
2317 if (!event_call || !event_call->event.funcs ||
2318 !event_call->event.funcs->trace)
2319 return;
2320
2321 event = &fbuffer->trace_file->event_call->event;
2322
2323 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2324 trace_seq_init(&iter->seq);
2325 iter->ent = fbuffer->entry;
2326 event_call->event.funcs->trace(iter, 0, event);
2327 trace_seq_putc(&iter->seq, 0);
2328 printk("%s", iter->seq.buffer);
2329
2330 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2331}
2332
2333int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2334 void __user *buffer, size_t *lenp,
2335 loff_t *ppos)
2336{
2337 int save_tracepoint_printk;
2338 int ret;
2339
2340 mutex_lock(&tracepoint_printk_mutex);
2341 save_tracepoint_printk = tracepoint_printk;
2342
2343 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2344
2345 /*
2346 * This will force exiting early, as tracepoint_printk
2347 * is always zero when tracepoint_printk_iter is not allocated
2348 */
2349 if (!tracepoint_print_iter)
2350 tracepoint_printk = 0;
2351
2352 if (save_tracepoint_printk == tracepoint_printk)
2353 goto out;
2354
2355 if (tracepoint_printk)
2356 static_key_enable(&tracepoint_printk_key.key);
2357 else
2358 static_key_disable(&tracepoint_printk_key.key);
2359
2360 out:
2361 mutex_unlock(&tracepoint_printk_mutex);
2362
2363 return ret;
2364}
2365
2366void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2367{
2368 if (static_key_false(&tracepoint_printk_key.key))
2369 output_printk(fbuffer);
2370
2371 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2372 fbuffer->event, fbuffer->entry,
2373 fbuffer->flags, fbuffer->pc);
2374}
2375EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2376
b7f0c959
SRRH
2377void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2378 struct ring_buffer *buffer,
0d5c6e1c
SR
2379 struct ring_buffer_event *event,
2380 unsigned long flags, int pc,
2381 struct pt_regs *regs)
1fd8df2c 2382{
7ffbd48d 2383 __buffer_unlock_commit(buffer, event);
1fd8df2c 2384
be54f69c
SRRH
2385 /*
2386 * If regs is not set, then skip the following callers:
2387 * trace_buffer_unlock_commit_regs
2388 * event_trigger_unlock_commit
2389 * trace_event_buffer_commit
2390 * trace_event_raw_event_sched_switch
2391 * Note, we can still get here via blktrace, wakeup tracer
2392 * and mmiotrace, but that's ok if they lose a function or
2393 * two. They are that meaningful.
2394 */
2395 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
1fd8df2c
MH
2396 ftrace_trace_userstack(buffer, flags, pc);
2397}
1fd8df2c 2398
52ffabe3
SRRH
2399/*
2400 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2401 */
2402void
2403trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2404 struct ring_buffer_event *event)
2405{
2406 __buffer_unlock_commit(buffer, event);
2407}
2408
478409dd
CZ
2409static void
2410trace_process_export(struct trace_export *export,
2411 struct ring_buffer_event *event)
2412{
2413 struct trace_entry *entry;
2414 unsigned int size = 0;
2415
2416 entry = ring_buffer_event_data(event);
2417 size = ring_buffer_event_length(event);
a773d419 2418 export->write(export, entry, size);
478409dd
CZ
2419}
2420
2421static DEFINE_MUTEX(ftrace_export_lock);
2422
2423static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2424
2425static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2426
2427static inline void ftrace_exports_enable(void)
2428{
2429 static_branch_enable(&ftrace_exports_enabled);
2430}
2431
2432static inline void ftrace_exports_disable(void)
2433{
2434 static_branch_disable(&ftrace_exports_enabled);
2435}
2436
2437void ftrace_exports(struct ring_buffer_event *event)
2438{
2439 struct trace_export *export;
2440
2441 preempt_disable_notrace();
2442
2443 export = rcu_dereference_raw_notrace(ftrace_exports_list);
2444 while (export) {
2445 trace_process_export(export, event);
2446 export = rcu_dereference_raw_notrace(export->next);
2447 }
2448
2449 preempt_enable_notrace();
2450}
2451
2452static inline void
2453add_trace_export(struct trace_export **list, struct trace_export *export)
2454{
2455 rcu_assign_pointer(export->next, *list);
2456 /*
2457 * We are entering export into the list but another
2458 * CPU might be walking that list. We need to make sure
2459 * the export->next pointer is valid before another CPU sees
2460 * the export pointer included into the list.
2461 */
2462 rcu_assign_pointer(*list, export);
2463}
2464
2465static inline int
2466rm_trace_export(struct trace_export **list, struct trace_export *export)
2467{
2468 struct trace_export **p;
2469
2470 for (p = list; *p != NULL; p = &(*p)->next)
2471 if (*p == export)
2472 break;
2473
2474 if (*p != export)
2475 return -1;
2476
2477 rcu_assign_pointer(*p, (*p)->next);
2478
2479 return 0;
2480}
2481
2482static inline void
2483add_ftrace_export(struct trace_export **list, struct trace_export *export)
2484{
2485 if (*list == NULL)
2486 ftrace_exports_enable();
2487
2488 add_trace_export(list, export);
2489}
2490
2491static inline int
2492rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2493{
2494 int ret;
2495
2496 ret = rm_trace_export(list, export);
2497 if (*list == NULL)
2498 ftrace_exports_disable();
2499
2500 return ret;
2501}
2502
2503int register_ftrace_export(struct trace_export *export)
2504{
2505 if (WARN_ON_ONCE(!export->write))
2506 return -1;
2507
2508 mutex_lock(&ftrace_export_lock);
2509
2510 add_ftrace_export(&ftrace_exports_list, export);
2511
2512 mutex_unlock(&ftrace_export_lock);
2513
2514 return 0;
2515}
2516EXPORT_SYMBOL_GPL(register_ftrace_export);
2517
2518int unregister_ftrace_export(struct trace_export *export)
2519{
2520 int ret;
2521
2522 mutex_lock(&ftrace_export_lock);
2523
2524 ret = rm_ftrace_export(&ftrace_exports_list, export);
2525
2526 mutex_unlock(&ftrace_export_lock);
2527
2528 return ret;
2529}
2530EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2531
e309b41d 2532void
7be42151 2533trace_function(struct trace_array *tr,
38697053
SR
2534 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2535 int pc)
bc0c38d1 2536{
2425bcb9 2537 struct trace_event_call *call = &event_function;
12883efb 2538 struct ring_buffer *buffer = tr->trace_buffer.buffer;
3928a8a2 2539 struct ring_buffer_event *event;
777e208d 2540 struct ftrace_entry *entry;
bc0c38d1 2541
3e9a8aad
SRRH
2542 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2543 flags, pc);
3928a8a2
SR
2544 if (!event)
2545 return;
2546 entry = ring_buffer_event_data(event);
777e208d
SR
2547 entry->ip = ip;
2548 entry->parent_ip = parent_ip;
e1112b4d 2549
478409dd
CZ
2550 if (!call_filter_check_discard(call, entry, buffer, event)) {
2551 if (static_branch_unlikely(&ftrace_exports_enabled))
2552 ftrace_exports(event);
7ffbd48d 2553 __buffer_unlock_commit(buffer, event);
478409dd 2554 }
bc0c38d1
SR
2555}
2556
c0a0d0d3 2557#ifdef CONFIG_STACKTRACE
4a9bd3f1
SR
2558
2559#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2560struct ftrace_stack {
2561 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2562};
2563
2564static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2565static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2566
e77405ad 2567static void __ftrace_trace_stack(struct ring_buffer *buffer,
53614991 2568 unsigned long flags,
1fd8df2c 2569 int skip, int pc, struct pt_regs *regs)
86387f7e 2570{
2425bcb9 2571 struct trace_event_call *call = &event_kernel_stack;
3928a8a2 2572 struct ring_buffer_event *event;
777e208d 2573 struct stack_entry *entry;
86387f7e 2574 struct stack_trace trace;
4a9bd3f1
SR
2575 int use_stack;
2576 int size = FTRACE_STACK_ENTRIES;
2577
2578 trace.nr_entries = 0;
2579 trace.skip = skip;
2580
be54f69c
SRRH
2581 /*
2582 * Add two, for this function and the call to save_stack_trace()
2583 * If regs is set, then these functions will not be in the way.
2584 */
2585 if (!regs)
2586 trace.skip += 2;
2587
4a9bd3f1
SR
2588 /*
2589 * Since events can happen in NMIs there's no safe way to
2590 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2591 * or NMI comes in, it will just have to use the default
2592 * FTRACE_STACK_SIZE.
2593 */
2594 preempt_disable_notrace();
2595
82146529 2596 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
4a9bd3f1
SR
2597 /*
2598 * We don't need any atomic variables, just a barrier.
2599 * If an interrupt comes in, we don't care, because it would
2600 * have exited and put the counter back to what we want.
2601 * We just need a barrier to keep gcc from moving things
2602 * around.
2603 */
2604 barrier();
2605 if (use_stack == 1) {
bdffd893 2606 trace.entries = this_cpu_ptr(ftrace_stack.calls);
4a9bd3f1
SR
2607 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2608
2609 if (regs)
2610 save_stack_trace_regs(regs, &trace);
2611 else
2612 save_stack_trace(&trace);
2613
2614 if (trace.nr_entries > size)
2615 size = trace.nr_entries;
2616 } else
2617 /* From now on, use_stack is a boolean */
2618 use_stack = 0;
2619
2620 size *= sizeof(unsigned long);
86387f7e 2621
3e9a8aad
SRRH
2622 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2623 sizeof(*entry) + size, flags, pc);
3928a8a2 2624 if (!event)
4a9bd3f1
SR
2625 goto out;
2626 entry = ring_buffer_event_data(event);
86387f7e 2627
4a9bd3f1
SR
2628 memset(&entry->caller, 0, size);
2629
2630 if (use_stack)
2631 memcpy(&entry->caller, trace.entries,
2632 trace.nr_entries * sizeof(unsigned long));
2633 else {
2634 trace.max_entries = FTRACE_STACK_ENTRIES;
2635 trace.entries = entry->caller;
2636 if (regs)
2637 save_stack_trace_regs(regs, &trace);
2638 else
2639 save_stack_trace(&trace);
2640 }
2641
2642 entry->size = trace.nr_entries;
86387f7e 2643
f306cc82 2644 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2645 __buffer_unlock_commit(buffer, event);
4a9bd3f1
SR
2646
2647 out:
2648 /* Again, don't let gcc optimize things here */
2649 barrier();
82146529 2650 __this_cpu_dec(ftrace_stack_reserve);
4a9bd3f1
SR
2651 preempt_enable_notrace();
2652
f0a920d5
IM
2653}
2654
2d34f489
SRRH
2655static inline void ftrace_trace_stack(struct trace_array *tr,
2656 struct ring_buffer *buffer,
73dddbb5
SRRH
2657 unsigned long flags,
2658 int skip, int pc, struct pt_regs *regs)
53614991 2659{
2d34f489 2660 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
53614991
SR
2661 return;
2662
73dddbb5 2663 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
53614991
SR
2664}
2665
c0a0d0d3
FW
2666void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2667 int pc)
38697053 2668{
a33d7d94
SRV
2669 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2670
2671 if (rcu_is_watching()) {
2672 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2673 return;
2674 }
2675
2676 /*
2677 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2678 * but if the above rcu_is_watching() failed, then the NMI
2679 * triggered someplace critical, and rcu_irq_enter() should
2680 * not be called from NMI.
2681 */
2682 if (unlikely(in_nmi()))
2683 return;
2684
2685 /*
2686 * It is possible that a function is being traced in a
2687 * location that RCU is not watching. A call to
2688 * rcu_irq_enter() will make sure that it is, but there's
2689 * a few internal rcu functions that could be traced
2690 * where that wont work either. In those cases, we just
2691 * do nothing.
2692 */
2693 if (unlikely(rcu_irq_enter_disabled()))
2694 return;
2695
2696 rcu_irq_enter_irqson();
2697 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2698 rcu_irq_exit_irqson();
38697053
SR
2699}
2700
03889384
SR
2701/**
2702 * trace_dump_stack - record a stack back trace in the trace buffer
c142be8e 2703 * @skip: Number of functions to skip (helper handlers)
03889384 2704 */
c142be8e 2705void trace_dump_stack(int skip)
03889384
SR
2706{
2707 unsigned long flags;
2708
2709 if (tracing_disabled || tracing_selftest_running)
e36c5458 2710 return;
03889384
SR
2711
2712 local_save_flags(flags);
2713
c142be8e
SRRH
2714 /*
2715 * Skip 3 more, seems to get us at the caller of
2716 * this function.
2717 */
2718 skip += 3;
2719 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2720 flags, skip, preempt_count(), NULL);
03889384
SR
2721}
2722
91e86e56
SR
2723static DEFINE_PER_CPU(int, user_stack_count);
2724
e77405ad
SR
2725void
2726ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
02b67518 2727{
2425bcb9 2728 struct trace_event_call *call = &event_user_stack;
8d7c6a96 2729 struct ring_buffer_event *event;
02b67518
TE
2730 struct userstack_entry *entry;
2731 struct stack_trace trace;
02b67518 2732
983f938a 2733 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
02b67518
TE
2734 return;
2735
b6345879
SR
2736 /*
2737 * NMIs can not handle page faults, even with fix ups.
2738 * The save user stack can (and often does) fault.
2739 */
2740 if (unlikely(in_nmi()))
2741 return;
02b67518 2742
91e86e56
SR
2743 /*
2744 * prevent recursion, since the user stack tracing may
2745 * trigger other kernel events.
2746 */
2747 preempt_disable();
2748 if (__this_cpu_read(user_stack_count))
2749 goto out;
2750
2751 __this_cpu_inc(user_stack_count);
2752
3e9a8aad
SRRH
2753 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2754 sizeof(*entry), flags, pc);
02b67518 2755 if (!event)
1dbd1951 2756 goto out_drop_count;
02b67518 2757 entry = ring_buffer_event_data(event);
02b67518 2758
48659d31 2759 entry->tgid = current->tgid;
02b67518
TE
2760 memset(&entry->caller, 0, sizeof(entry->caller));
2761
2762 trace.nr_entries = 0;
2763 trace.max_entries = FTRACE_STACK_ENTRIES;
2764 trace.skip = 0;
2765 trace.entries = entry->caller;
2766
2767 save_stack_trace_user(&trace);
f306cc82 2768 if (!call_filter_check_discard(call, entry, buffer, event))
7ffbd48d 2769 __buffer_unlock_commit(buffer, event);
91e86e56 2770
1dbd1951 2771 out_drop_count:
91e86e56 2772 __this_cpu_dec(user_stack_count);
91e86e56
SR
2773 out:
2774 preempt_enable();
02b67518
TE
2775}
2776
4fd27358
HE
2777#ifdef UNUSED
2778static void __trace_userstack(struct trace_array *tr, unsigned long flags)
02b67518 2779{
7be42151 2780 ftrace_trace_userstack(tr, flags, preempt_count());
02b67518 2781}
4fd27358 2782#endif /* UNUSED */
02b67518 2783
c0a0d0d3
FW
2784#endif /* CONFIG_STACKTRACE */
2785
07d777fe
SR
2786/* created for use with alloc_percpu */
2787struct trace_buffer_struct {
e2ace001
AL
2788 int nesting;
2789 char buffer[4][TRACE_BUF_SIZE];
07d777fe
SR
2790};
2791
2792static struct trace_buffer_struct *trace_percpu_buffer;
07d777fe
SR
2793
2794/*
e2ace001
AL
2795 * Thise allows for lockless recording. If we're nested too deeply, then
2796 * this returns NULL.
07d777fe
SR
2797 */
2798static char *get_trace_buf(void)
2799{
e2ace001 2800 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
07d777fe 2801
e2ace001 2802 if (!buffer || buffer->nesting >= 4)
07d777fe
SR
2803 return NULL;
2804
3d9622c1
SRV
2805 buffer->nesting++;
2806
2807 /* Interrupts must see nesting incremented before we use the buffer */
2808 barrier();
2809 return &buffer->buffer[buffer->nesting][0];
e2ace001
AL
2810}
2811
2812static void put_trace_buf(void)
2813{
3d9622c1
SRV
2814 /* Don't let the decrement of nesting leak before this */
2815 barrier();
e2ace001 2816 this_cpu_dec(trace_percpu_buffer->nesting);
07d777fe
SR
2817}
2818
2819static int alloc_percpu_trace_buffer(void)
2820{
2821 struct trace_buffer_struct *buffers;
07d777fe
SR
2822
2823 buffers = alloc_percpu(struct trace_buffer_struct);
e2ace001
AL
2824 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2825 return -ENOMEM;
07d777fe
SR
2826
2827 trace_percpu_buffer = buffers;
07d777fe 2828 return 0;
07d777fe
SR
2829}
2830
81698831
SR
2831static int buffers_allocated;
2832
07d777fe
SR
2833void trace_printk_init_buffers(void)
2834{
07d777fe
SR
2835 if (buffers_allocated)
2836 return;
2837
2838 if (alloc_percpu_trace_buffer())
2839 return;
2840
2184db46
SR
2841 /* trace_printk() is for debug use only. Don't use it in production. */
2842
a395d6a7
JP
2843 pr_warn("\n");
2844 pr_warn("**********************************************************\n");
2845 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2846 pr_warn("** **\n");
2847 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2848 pr_warn("** **\n");
2849 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2850 pr_warn("** unsafe for production use. **\n");
2851 pr_warn("** **\n");
2852 pr_warn("** If you see this message and you are not debugging **\n");
2853 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2854 pr_warn("** **\n");
2855 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2856 pr_warn("**********************************************************\n");
07d777fe 2857
b382ede6
SR
2858 /* Expand the buffers to set size */
2859 tracing_update_buffers();
2860
07d777fe 2861 buffers_allocated = 1;
81698831
SR
2862
2863 /*
2864 * trace_printk_init_buffers() can be called by modules.
2865 * If that happens, then we need to start cmdline recording
2866 * directly here. If the global_trace.buffer is already
2867 * allocated here, then this was called by module code.
2868 */
12883efb 2869 if (global_trace.trace_buffer.buffer)
81698831
SR
2870 tracing_start_cmdline_record();
2871}
2872
2873void trace_printk_start_comm(void)
2874{
2875 /* Start tracing comms if trace printk is set */
2876 if (!buffers_allocated)
2877 return;
2878 tracing_start_cmdline_record();
2879}
2880
2881static void trace_printk_start_stop_comm(int enabled)
2882{
2883 if (!buffers_allocated)
2884 return;
2885
2886 if (enabled)
2887 tracing_start_cmdline_record();
2888 else
2889 tracing_stop_cmdline_record();
07d777fe
SR
2890}
2891
769b0441 2892/**
48ead020 2893 * trace_vbprintk - write binary msg to tracing buffer
769b0441
FW
2894 *
2895 */
40ce74f1 2896int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
769b0441 2897{
2425bcb9 2898 struct trace_event_call *call = &event_bprint;
769b0441 2899 struct ring_buffer_event *event;
e77405ad 2900 struct ring_buffer *buffer;
769b0441 2901 struct trace_array *tr = &global_trace;
48ead020 2902 struct bprint_entry *entry;
769b0441 2903 unsigned long flags;
07d777fe
SR
2904 char *tbuffer;
2905 int len = 0, size, pc;
769b0441
FW
2906
2907 if (unlikely(tracing_selftest_running || tracing_disabled))
2908 return 0;
2909
2910 /* Don't pollute graph traces with trace_vprintk internals */
2911 pause_graph_tracing();
2912
2913 pc = preempt_count();
5168ae50 2914 preempt_disable_notrace();
769b0441 2915
07d777fe
SR
2916 tbuffer = get_trace_buf();
2917 if (!tbuffer) {
2918 len = 0;
e2ace001 2919 goto out_nobuffer;
07d777fe 2920 }
769b0441 2921
07d777fe 2922 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
769b0441 2923
07d777fe
SR
2924 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2925 goto out;
769b0441 2926
07d777fe 2927 local_save_flags(flags);
769b0441 2928 size = sizeof(*entry) + sizeof(u32) * len;
12883efb 2929 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
2930 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2931 flags, pc);
769b0441 2932 if (!event)
07d777fe 2933 goto out;
769b0441
FW
2934 entry = ring_buffer_event_data(event);
2935 entry->ip = ip;
769b0441
FW
2936 entry->fmt = fmt;
2937
07d777fe 2938 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
f306cc82 2939 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2940 __buffer_unlock_commit(buffer, event);
2d34f489 2941 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
d931369b 2942 }
769b0441 2943
769b0441 2944out:
e2ace001
AL
2945 put_trace_buf();
2946
2947out_nobuffer:
5168ae50 2948 preempt_enable_notrace();
769b0441
FW
2949 unpause_graph_tracing();
2950
2951 return len;
2952}
48ead020
FW
2953EXPORT_SYMBOL_GPL(trace_vbprintk);
2954
12883efb
SRRH
2955static int
2956__trace_array_vprintk(struct ring_buffer *buffer,
2957 unsigned long ip, const char *fmt, va_list args)
48ead020 2958{
2425bcb9 2959 struct trace_event_call *call = &event_print;
48ead020 2960 struct ring_buffer_event *event;
07d777fe 2961 int len = 0, size, pc;
48ead020 2962 struct print_entry *entry;
07d777fe
SR
2963 unsigned long flags;
2964 char *tbuffer;
48ead020
FW
2965
2966 if (tracing_disabled || tracing_selftest_running)
2967 return 0;
2968
07d777fe
SR
2969 /* Don't pollute graph traces with trace_vprintk internals */
2970 pause_graph_tracing();
2971
48ead020
FW
2972 pc = preempt_count();
2973 preempt_disable_notrace();
48ead020 2974
07d777fe
SR
2975
2976 tbuffer = get_trace_buf();
2977 if (!tbuffer) {
2978 len = 0;
e2ace001 2979 goto out_nobuffer;
07d777fe 2980 }
48ead020 2981
3558a5ac 2982 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
48ead020 2983
07d777fe 2984 local_save_flags(flags);
48ead020 2985 size = sizeof(*entry) + len + 1;
3e9a8aad
SRRH
2986 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2987 flags, pc);
48ead020 2988 if (!event)
07d777fe 2989 goto out;
48ead020 2990 entry = ring_buffer_event_data(event);
c13d2f7c 2991 entry->ip = ip;
48ead020 2992
3558a5ac 2993 memcpy(&entry->buf, tbuffer, len + 1);
f306cc82 2994 if (!call_filter_check_discard(call, entry, buffer, event)) {
7ffbd48d 2995 __buffer_unlock_commit(buffer, event);
2d34f489 2996 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
d931369b 2997 }
e2ace001
AL
2998
2999out:
3000 put_trace_buf();
3001
3002out_nobuffer:
48ead020 3003 preempt_enable_notrace();
07d777fe 3004 unpause_graph_tracing();
48ead020
FW
3005
3006 return len;
3007}
659372d3 3008
12883efb
SRRH
3009int trace_array_vprintk(struct trace_array *tr,
3010 unsigned long ip, const char *fmt, va_list args)
3011{
3012 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3013}
3014
3015int trace_array_printk(struct trace_array *tr,
3016 unsigned long ip, const char *fmt, ...)
3017{
3018 int ret;
3019 va_list ap;
3020
983f938a 3021 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
3022 return 0;
3023
3024 va_start(ap, fmt);
3025 ret = trace_array_vprintk(tr, ip, fmt, ap);
3026 va_end(ap);
3027 return ret;
3028}
3029
3030int trace_array_printk_buf(struct ring_buffer *buffer,
3031 unsigned long ip, const char *fmt, ...)
3032{
3033 int ret;
3034 va_list ap;
3035
983f938a 3036 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
12883efb
SRRH
3037 return 0;
3038
3039 va_start(ap, fmt);
3040 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3041 va_end(ap);
3042 return ret;
3043}
3044
659372d3
SR
3045int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3046{
a813a159 3047 return trace_array_vprintk(&global_trace, ip, fmt, args);
659372d3 3048}
769b0441
FW
3049EXPORT_SYMBOL_GPL(trace_vprintk);
3050
e2ac8ef5 3051static void trace_iterator_increment(struct trace_iterator *iter)
5a90f577 3052{
6d158a81
SR
3053 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3054
5a90f577 3055 iter->idx++;
6d158a81
SR
3056 if (buf_iter)
3057 ring_buffer_read(buf_iter, NULL);
5a90f577
SR
3058}
3059
e309b41d 3060static struct trace_entry *
bc21b478
SR
3061peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3062 unsigned long *lost_events)
dd0e545f 3063{
3928a8a2 3064 struct ring_buffer_event *event;
6d158a81 3065 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
dd0e545f 3066
d769041f
SR
3067 if (buf_iter)
3068 event = ring_buffer_iter_peek(buf_iter, ts);
3069 else
12883efb 3070 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
bc21b478 3071 lost_events);
d769041f 3072
4a9bd3f1
SR
3073 if (event) {
3074 iter->ent_size = ring_buffer_event_length(event);
3075 return ring_buffer_event_data(event);
3076 }
3077 iter->ent_size = 0;
3078 return NULL;
dd0e545f 3079}
d769041f 3080
dd0e545f 3081static struct trace_entry *
bc21b478
SR
3082__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3083 unsigned long *missing_events, u64 *ent_ts)
bc0c38d1 3084{
12883efb 3085 struct ring_buffer *buffer = iter->trace_buffer->buffer;
bc0c38d1 3086 struct trace_entry *ent, *next = NULL;
aa27497c 3087 unsigned long lost_events = 0, next_lost = 0;
b04cc6b1 3088 int cpu_file = iter->cpu_file;
3928a8a2 3089 u64 next_ts = 0, ts;
bc0c38d1 3090 int next_cpu = -1;
12b5da34 3091 int next_size = 0;
bc0c38d1
SR
3092 int cpu;
3093
b04cc6b1
FW
3094 /*
3095 * If we are in a per_cpu trace file, don't bother by iterating over
3096 * all cpu and peek directly.
3097 */
ae3b5093 3098 if (cpu_file > RING_BUFFER_ALL_CPUS) {
b04cc6b1
FW
3099 if (ring_buffer_empty_cpu(buffer, cpu_file))
3100 return NULL;
bc21b478 3101 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
b04cc6b1
FW
3102 if (ent_cpu)
3103 *ent_cpu = cpu_file;
3104
3105 return ent;
3106 }
3107
ab46428c 3108 for_each_tracing_cpu(cpu) {
dd0e545f 3109
3928a8a2
SR
3110 if (ring_buffer_empty_cpu(buffer, cpu))
3111 continue;
dd0e545f 3112
bc21b478 3113 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
dd0e545f 3114
cdd31cd2
IM
3115 /*
3116 * Pick the entry with the smallest timestamp:
3117 */
3928a8a2 3118 if (ent && (!next || ts < next_ts)) {
bc0c38d1
SR
3119 next = ent;
3120 next_cpu = cpu;
3928a8a2 3121 next_ts = ts;
bc21b478 3122 next_lost = lost_events;
12b5da34 3123 next_size = iter->ent_size;
bc0c38d1
SR
3124 }
3125 }
3126
12b5da34
SR
3127 iter->ent_size = next_size;
3128
bc0c38d1
SR
3129 if (ent_cpu)
3130 *ent_cpu = next_cpu;
3131
3928a8a2
SR
3132 if (ent_ts)
3133 *ent_ts = next_ts;
3134
bc21b478
SR
3135 if (missing_events)
3136 *missing_events = next_lost;
3137
bc0c38d1
SR
3138 return next;
3139}
3140
dd0e545f 3141/* Find the next real entry, without updating the iterator itself */
c4a8e8be
FW
3142struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3143 int *ent_cpu, u64 *ent_ts)
bc0c38d1 3144{
bc21b478 3145 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
dd0e545f
SR
3146}
3147
3148/* Find the next real entry, and increment the iterator to the next entry */
955b61e5 3149void *trace_find_next_entry_inc(struct trace_iterator *iter)
dd0e545f 3150{
bc21b478
SR
3151 iter->ent = __find_next_entry(iter, &iter->cpu,
3152 &iter->lost_events, &iter->ts);
dd0e545f 3153
3928a8a2 3154 if (iter->ent)
e2ac8ef5 3155 trace_iterator_increment(iter);
dd0e545f 3156
3928a8a2 3157 return iter->ent ? iter : NULL;
b3806b43 3158}
bc0c38d1 3159
e309b41d 3160static void trace_consume(struct trace_iterator *iter)
b3806b43 3161{
12883efb 3162 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
bc21b478 3163 &iter->lost_events);
bc0c38d1
SR
3164}
3165
e309b41d 3166static void *s_next(struct seq_file *m, void *v, loff_t *pos)
bc0c38d1
SR
3167{
3168 struct trace_iterator *iter = m->private;
bc0c38d1 3169 int i = (int)*pos;
4e3c3333 3170 void *ent;
bc0c38d1 3171
a63ce5b3
SR
3172 WARN_ON_ONCE(iter->leftover);
3173
bc0c38d1
SR
3174 (*pos)++;
3175
3176 /* can't go backwards */
3177 if (iter->idx > i)
3178 return NULL;
3179
3180 if (iter->idx < 0)
955b61e5 3181 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
3182 else
3183 ent = iter;
3184
3185 while (ent && iter->idx < i)
955b61e5 3186 ent = trace_find_next_entry_inc(iter);
bc0c38d1
SR
3187
3188 iter->pos = *pos;
3189
bc0c38d1
SR
3190 return ent;
3191}
3192
955b61e5 3193void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2f26ebd5 3194{
2f26ebd5
SR
3195 struct ring_buffer_event *event;
3196 struct ring_buffer_iter *buf_iter;
3197 unsigned long entries = 0;
3198 u64 ts;
3199
12883efb 3200 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2f26ebd5 3201
6d158a81
SR
3202 buf_iter = trace_buffer_iter(iter, cpu);
3203 if (!buf_iter)
2f26ebd5
SR
3204 return;
3205
2f26ebd5
SR
3206 ring_buffer_iter_reset(buf_iter);
3207
3208 /*
3209 * We could have the case with the max latency tracers
3210 * that a reset never took place on a cpu. This is evident
3211 * by the timestamp being before the start of the buffer.
3212 */
3213 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
12883efb 3214 if (ts >= iter->trace_buffer->time_start)
2f26ebd5
SR
3215 break;
3216 entries++;
3217 ring_buffer_read(buf_iter, NULL);
3218 }
3219
12883efb 3220 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2f26ebd5
SR
3221}
3222
d7350c3f 3223/*
d7350c3f
FW
3224 * The current tracer is copied to avoid a global locking
3225 * all around.
3226 */
bc0c38d1
SR
3227static void *s_start(struct seq_file *m, loff_t *pos)
3228{
3229 struct trace_iterator *iter = m->private;
2b6080f2 3230 struct trace_array *tr = iter->tr;
b04cc6b1 3231 int cpu_file = iter->cpu_file;
bc0c38d1
SR
3232 void *p = NULL;
3233 loff_t l = 0;
3928a8a2 3234 int cpu;
bc0c38d1 3235
2fd196ec
HT
3236 /*
3237 * copy the tracer to avoid using a global lock all around.
3238 * iter->trace is a copy of current_trace, the pointer to the
3239 * name may be used instead of a strcmp(), as iter->trace->name
3240 * will point to the same string as current_trace->name.
3241 */
bc0c38d1 3242 mutex_lock(&trace_types_lock);
2b6080f2
SR
3243 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3244 *iter->trace = *tr->current_trace;
d7350c3f 3245 mutex_unlock(&trace_types_lock);
bc0c38d1 3246
12883efb 3247#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3248 if (iter->snapshot && iter->trace->use_max_tr)
3249 return ERR_PTR(-EBUSY);
12883efb 3250#endif
debdd57f
HT
3251
3252 if (!iter->snapshot)
d914ba37 3253 atomic_inc(&trace_record_taskinfo_disabled);
bc0c38d1 3254
bc0c38d1
SR
3255 if (*pos != iter->pos) {
3256 iter->ent = NULL;
3257 iter->cpu = 0;
3258 iter->idx = -1;
3259
ae3b5093 3260 if (cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3261 for_each_tracing_cpu(cpu)
2f26ebd5 3262 tracing_iter_reset(iter, cpu);
b04cc6b1 3263 } else
2f26ebd5 3264 tracing_iter_reset(iter, cpu_file);
bc0c38d1 3265
ac91d854 3266 iter->leftover = 0;
bc0c38d1
SR
3267 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3268 ;
3269
3270 } else {
a63ce5b3
SR
3271 /*
3272 * If we overflowed the seq_file before, then we want
3273 * to just reuse the trace_seq buffer again.
3274 */
3275 if (iter->leftover)
3276 p = iter;
3277 else {
3278 l = *pos - 1;
3279 p = s_next(m, p, &l);
3280 }
bc0c38d1
SR
3281 }
3282
4f535968 3283 trace_event_read_lock();
7e53bd42 3284 trace_access_lock(cpu_file);
bc0c38d1
SR
3285 return p;
3286}
3287
3288static void s_stop(struct seq_file *m, void *p)
3289{
7e53bd42
LJ
3290 struct trace_iterator *iter = m->private;
3291
12883efb 3292#ifdef CONFIG_TRACER_MAX_TRACE
debdd57f
HT
3293 if (iter->snapshot && iter->trace->use_max_tr)
3294 return;
12883efb 3295#endif
debdd57f
HT
3296
3297 if (!iter->snapshot)
d914ba37 3298 atomic_dec(&trace_record_taskinfo_disabled);
12883efb 3299
7e53bd42 3300 trace_access_unlock(iter->cpu_file);
4f535968 3301 trace_event_read_unlock();
bc0c38d1
SR
3302}
3303
39eaf7ef 3304static void
12883efb
SRRH
3305get_total_entries(struct trace_buffer *buf,
3306 unsigned long *total, unsigned long *entries)
39eaf7ef
SR
3307{
3308 unsigned long count;
3309 int cpu;
3310
3311 *total = 0;
3312 *entries = 0;
3313
3314 for_each_tracing_cpu(cpu) {
12883efb 3315 count = ring_buffer_entries_cpu(buf->buffer, cpu);
39eaf7ef
SR
3316 /*
3317 * If this buffer has skipped entries, then we hold all
3318 * entries for the trace and we need to ignore the
3319 * ones before the time stamp.
3320 */
12883efb
SRRH
3321 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3322 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
39eaf7ef
SR
3323 /* total is the same as the entries */
3324 *total += count;
3325 } else
3326 *total += count +
12883efb 3327 ring_buffer_overrun_cpu(buf->buffer, cpu);
39eaf7ef
SR
3328 *entries += count;
3329 }
3330}
3331
e309b41d 3332static void print_lat_help_header(struct seq_file *m)
bc0c38d1 3333{
d79ac28f
RV
3334 seq_puts(m, "# _------=> CPU# \n"
3335 "# / _-----=> irqs-off \n"
3336 "# | / _----=> need-resched \n"
3337 "# || / _---=> hardirq/softirq \n"
3338 "# ||| / _--=> preempt-depth \n"
3339 "# |||| / delay \n"
3340 "# cmd pid ||||| time | caller \n"
3341 "# \\ / ||||| \\ | / \n");
bc0c38d1
SR
3342}
3343
12883efb 3344static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
bc0c38d1 3345{
39eaf7ef
SR
3346 unsigned long total;
3347 unsigned long entries;
3348
12883efb 3349 get_total_entries(buf, &total, &entries);
39eaf7ef
SR
3350 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3351 entries, total, num_online_cpus());
3352 seq_puts(m, "#\n");
3353}
3354
441dae8f
JF
3355static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3356 unsigned int flags)
39eaf7ef 3357{
441dae8f
JF
3358 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3359
12883efb 3360 print_event_info(buf, m);
441dae8f
JF
3361
3362 seq_printf(m, "# TASK-PID CPU# %s TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3363 seq_printf(m, "# | | | %s | |\n", tgid ? " | " : "");
bc0c38d1
SR
3364}
3365
441dae8f
JF
3366static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3367 unsigned int flags)
77271ce4 3368{
441dae8f 3369 bool tgid = flags & TRACE_ITER_RECORD_TGID;
b11fb737
SRV
3370 const char tgid_space[] = " ";
3371 const char space[] = " ";
3372
3373 seq_printf(m, "# %s _-----=> irqs-off\n",
3374 tgid ? tgid_space : space);
3375 seq_printf(m, "# %s / _----=> need-resched\n",
3376 tgid ? tgid_space : space);
3377 seq_printf(m, "# %s| / _---=> hardirq/softirq\n",
3378 tgid ? tgid_space : space);
3379 seq_printf(m, "# %s|| / _--=> preempt-depth\n",
3380 tgid ? tgid_space : space);
3381 seq_printf(m, "# %s||| / delay\n",
3382 tgid ? tgid_space : space);
3383 seq_printf(m, "# TASK-PID CPU#%s|||| TIMESTAMP FUNCTION\n",
3384 tgid ? " TGID " : space);
3385 seq_printf(m, "# | | | %s|||| | |\n",
3386 tgid ? " | " : space);
77271ce4 3387}
bc0c38d1 3388
62b915f1 3389void
bc0c38d1
SR
3390print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3391{
983f938a 3392 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
12883efb
SRRH
3393 struct trace_buffer *buf = iter->trace_buffer;
3394 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2b6080f2 3395 struct tracer *type = iter->trace;
39eaf7ef
SR
3396 unsigned long entries;
3397 unsigned long total;
bc0c38d1
SR
3398 const char *name = "preemption";
3399
d840f718 3400 name = type->name;
bc0c38d1 3401
12883efb 3402 get_total_entries(buf, &total, &entries);
bc0c38d1 3403
888b55dc 3404 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
bc0c38d1 3405 name, UTS_RELEASE);
888b55dc 3406 seq_puts(m, "# -----------------------------------"
bc0c38d1 3407 "---------------------------------\n");
888b55dc 3408 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
bc0c38d1 3409 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
57f50be1 3410 nsecs_to_usecs(data->saved_latency),
bc0c38d1 3411 entries,
4c11d7ae 3412 total,
12883efb 3413 buf->cpu,
bc0c38d1
SR
3414#if defined(CONFIG_PREEMPT_NONE)
3415 "server",
3416#elif defined(CONFIG_PREEMPT_VOLUNTARY)
3417 "desktop",
b5c21b45 3418#elif defined(CONFIG_PREEMPT)
bc0c38d1
SR
3419 "preempt",
3420#else
3421 "unknown",
3422#endif
3423 /* These are reserved for later use */
3424 0, 0, 0, 0);
3425#ifdef CONFIG_SMP
3426 seq_printf(m, " #P:%d)\n", num_online_cpus());
3427#else
3428 seq_puts(m, ")\n");
3429#endif
888b55dc
KM
3430 seq_puts(m, "# -----------------\n");
3431 seq_printf(m, "# | task: %.16s-%d "
bc0c38d1 3432 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
d20b92ab
EB
3433 data->comm, data->pid,
3434 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
bc0c38d1 3435 data->policy, data->rt_priority);
888b55dc 3436 seq_puts(m, "# -----------------\n");
bc0c38d1
SR
3437
3438 if (data->critical_start) {
888b55dc 3439 seq_puts(m, "# => started at: ");
214023c3
SR
3440 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3441 trace_print_seq(m, &iter->seq);
888b55dc 3442 seq_puts(m, "\n# => ended at: ");
214023c3
SR
3443 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3444 trace_print_seq(m, &iter->seq);
8248ac05 3445 seq_puts(m, "\n#\n");
bc0c38d1
SR
3446 }
3447
888b55dc 3448 seq_puts(m, "#\n");
bc0c38d1
SR
3449}
3450
a309720c
SR
3451static void test_cpu_buff_start(struct trace_iterator *iter)
3452{
3453 struct trace_seq *s = &iter->seq;
983f938a 3454 struct trace_array *tr = iter->tr;
a309720c 3455
983f938a 3456 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
12ef7d44
SR
3457 return;
3458
3459 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3460 return;
3461
4dbbe2d8
MK
3462 if (cpumask_available(iter->started) &&
3463 cpumask_test_cpu(iter->cpu, iter->started))
a309720c
SR
3464 return;
3465
12883efb 3466 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2f26ebd5
SR
3467 return;
3468
4dbbe2d8 3469 if (cpumask_available(iter->started))
919cd979 3470 cpumask_set_cpu(iter->cpu, iter->started);
b0dfa978
FW
3471
3472 /* Don't print started cpu buffer for the first entry of the trace */
3473 if (iter->idx > 1)
3474 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3475 iter->cpu);
a309720c
SR
3476}
3477
2c4f035f 3478static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
bc0c38d1 3479{
983f938a 3480 struct trace_array *tr = iter->tr;
214023c3 3481 struct trace_seq *s = &iter->seq;
983f938a 3482 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4e3c3333 3483 struct trace_entry *entry;
f633cef0 3484 struct trace_event *event;
bc0c38d1 3485
4e3c3333 3486 entry = iter->ent;
dd0e545f 3487
a309720c
SR
3488 test_cpu_buff_start(iter);
3489
c4a8e8be 3490 event = ftrace_find_event(entry->type);
bc0c38d1 3491
983f938a 3492 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3493 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3494 trace_print_lat_context(iter);
3495 else
3496 trace_print_context(iter);
c4a8e8be 3497 }
bc0c38d1 3498
19a7fe20
SRRH
3499 if (trace_seq_has_overflowed(s))
3500 return TRACE_TYPE_PARTIAL_LINE;
3501
268ccda0 3502 if (event)
a9a57763 3503 return event->funcs->trace(iter, sym_flags, event);
d9793bd8 3504
19a7fe20 3505 trace_seq_printf(s, "Unknown type %d\n", entry->type);
02b67518 3506
19a7fe20 3507 return trace_handle_return(s);
bc0c38d1
SR
3508}
3509
2c4f035f 3510static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
f9896bf3 3511{
983f938a 3512 struct trace_array *tr = iter->tr;
f9896bf3
IM
3513 struct trace_seq *s = &iter->seq;
3514 struct trace_entry *entry;
f633cef0 3515 struct trace_event *event;
f9896bf3
IM
3516
3517 entry = iter->ent;
dd0e545f 3518
983f938a 3519 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
19a7fe20
SRRH
3520 trace_seq_printf(s, "%d %d %llu ",
3521 entry->pid, iter->cpu, iter->ts);
3522
3523 if (trace_seq_has_overflowed(s))
3524 return TRACE_TYPE_PARTIAL_LINE;
f9896bf3 3525
f633cef0 3526 event = ftrace_find_event(entry->type);
268ccda0 3527 if (event)
a9a57763 3528 return event->funcs->raw(iter, 0, event);
d9793bd8 3529
19a7fe20 3530 trace_seq_printf(s, "%d ?\n", entry->type);
777e208d 3531
19a7fe20 3532 return trace_handle_return(s);
f9896bf3
IM
3533}
3534
2c4f035f 3535static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
5e3ca0ec 3536{
983f938a 3537 struct trace_array *tr = iter->tr;
5e3ca0ec
IM
3538 struct trace_seq *s = &iter->seq;
3539 unsigned char newline = '\n';
3540 struct trace_entry *entry;
f633cef0 3541 struct trace_event *event;
5e3ca0ec
IM
3542
3543 entry = iter->ent;
dd0e545f 3544
983f938a 3545 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3546 SEQ_PUT_HEX_FIELD(s, entry->pid);
3547 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3548 SEQ_PUT_HEX_FIELD(s, iter->ts);
3549 if (trace_seq_has_overflowed(s))
3550 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3551 }
5e3ca0ec 3552
f633cef0 3553 event = ftrace_find_event(entry->type);
268ccda0 3554 if (event) {
a9a57763 3555 enum print_line_t ret = event->funcs->hex(iter, 0, event);
d9793bd8
ACM
3556 if (ret != TRACE_TYPE_HANDLED)
3557 return ret;
3558 }
7104f300 3559
19a7fe20 3560 SEQ_PUT_FIELD(s, newline);
5e3ca0ec 3561
19a7fe20 3562 return trace_handle_return(s);
5e3ca0ec
IM
3563}
3564
2c4f035f 3565static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
cb0f12aa 3566{
983f938a 3567 struct trace_array *tr = iter->tr;
cb0f12aa
IM
3568 struct trace_seq *s = &iter->seq;
3569 struct trace_entry *entry;
f633cef0 3570 struct trace_event *event;
cb0f12aa
IM
3571
3572 entry = iter->ent;
dd0e545f 3573
983f938a 3574 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
19a7fe20
SRRH
3575 SEQ_PUT_FIELD(s, entry->pid);
3576 SEQ_PUT_FIELD(s, iter->cpu);
3577 SEQ_PUT_FIELD(s, iter->ts);
3578 if (trace_seq_has_overflowed(s))
3579 return TRACE_TYPE_PARTIAL_LINE;
c4a8e8be 3580 }
cb0f12aa 3581
f633cef0 3582 event = ftrace_find_event(entry->type);
a9a57763
SR
3583 return event ? event->funcs->binary(iter, 0, event) :
3584 TRACE_TYPE_HANDLED;
cb0f12aa
IM
3585}
3586
62b915f1 3587int trace_empty(struct trace_iterator *iter)
bc0c38d1 3588{
6d158a81 3589 struct ring_buffer_iter *buf_iter;
bc0c38d1
SR
3590 int cpu;
3591
9aba60fe 3592 /* If we are looking at one CPU buffer, only check that one */
ae3b5093 3593 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
9aba60fe 3594 cpu = iter->cpu_file;
6d158a81
SR
3595 buf_iter = trace_buffer_iter(iter, cpu);
3596 if (buf_iter) {
3597 if (!ring_buffer_iter_empty(buf_iter))
9aba60fe
SR
3598 return 0;
3599 } else {
12883efb 3600 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
9aba60fe
SR
3601 return 0;
3602 }
3603 return 1;
3604 }
3605
ab46428c 3606 for_each_tracing_cpu(cpu) {
6d158a81
SR
3607 buf_iter = trace_buffer_iter(iter, cpu);
3608 if (buf_iter) {
3609 if (!ring_buffer_iter_empty(buf_iter))
d769041f
SR
3610 return 0;
3611 } else {
12883efb 3612 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
d769041f
SR
3613 return 0;
3614 }
bc0c38d1 3615 }
d769041f 3616
797d3712 3617 return 1;
bc0c38d1
SR
3618}
3619
4f535968 3620/* Called with trace_event_read_lock() held. */
955b61e5 3621enum print_line_t print_trace_line(struct trace_iterator *iter)
f9896bf3 3622{
983f938a
SRRH
3623 struct trace_array *tr = iter->tr;
3624 unsigned long trace_flags = tr->trace_flags;
2c4f035f
FW
3625 enum print_line_t ret;
3626
19a7fe20
SRRH
3627 if (iter->lost_events) {
3628 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3629 iter->cpu, iter->lost_events);
3630 if (trace_seq_has_overflowed(&iter->seq))
3631 return TRACE_TYPE_PARTIAL_LINE;
3632 }
bc21b478 3633
2c4f035f
FW
3634 if (iter->trace && iter->trace->print_line) {
3635 ret = iter->trace->print_line(iter);
3636 if (ret != TRACE_TYPE_UNHANDLED)
3637 return ret;
3638 }
72829bc3 3639
09ae7234
SRRH
3640 if (iter->ent->type == TRACE_BPUTS &&
3641 trace_flags & TRACE_ITER_PRINTK &&
3642 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3643 return trace_print_bputs_msg_only(iter);
3644
48ead020
FW
3645 if (iter->ent->type == TRACE_BPRINT &&
3646 trace_flags & TRACE_ITER_PRINTK &&
3647 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3648 return trace_print_bprintk_msg_only(iter);
48ead020 3649
66896a85
FW
3650 if (iter->ent->type == TRACE_PRINT &&
3651 trace_flags & TRACE_ITER_PRINTK &&
3652 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
5ef841f6 3653 return trace_print_printk_msg_only(iter);
66896a85 3654
cb0f12aa
IM
3655 if (trace_flags & TRACE_ITER_BIN)
3656 return print_bin_fmt(iter);
3657
5e3ca0ec
IM
3658 if (trace_flags & TRACE_ITER_HEX)
3659 return print_hex_fmt(iter);
3660
f9896bf3
IM
3661 if (trace_flags & TRACE_ITER_RAW)
3662 return print_raw_fmt(iter);
3663
f9896bf3
IM
3664 return print_trace_fmt(iter);
3665}
3666
7e9a49ef
JO
3667void trace_latency_header(struct seq_file *m)
3668{
3669 struct trace_iterator *iter = m->private;
983f938a 3670 struct trace_array *tr = iter->tr;
7e9a49ef
JO
3671
3672 /* print nothing if the buffers are empty */
3673 if (trace_empty(iter))
3674 return;
3675
3676 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3677 print_trace_header(m, iter);
3678
983f938a 3679 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
7e9a49ef
JO
3680 print_lat_help_header(m);
3681}
3682
62b915f1
JO
3683void trace_default_header(struct seq_file *m)
3684{
3685 struct trace_iterator *iter = m->private;
983f938a
SRRH
3686 struct trace_array *tr = iter->tr;
3687 unsigned long trace_flags = tr->trace_flags;
62b915f1 3688
f56e7f8e
JO
3689 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3690 return;
3691
62b915f1
JO
3692 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3693 /* print nothing if the buffers are empty */
3694 if (trace_empty(iter))
3695 return;
3696 print_trace_header(m, iter);
3697 if (!(trace_flags & TRACE_ITER_VERBOSE))
3698 print_lat_help_header(m);
3699 } else {
77271ce4
SR
3700 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3701 if (trace_flags & TRACE_ITER_IRQ_INFO)
441dae8f
JF
3702 print_func_help_header_irq(iter->trace_buffer,
3703 m, trace_flags);
77271ce4 3704 else
441dae8f
JF
3705 print_func_help_header(iter->trace_buffer, m,
3706 trace_flags);
77271ce4 3707 }
62b915f1
JO
3708 }
3709}
3710
e0a413f6
SR
3711static void test_ftrace_alive(struct seq_file *m)
3712{
3713 if (!ftrace_is_dead())
3714 return;
d79ac28f
RV
3715 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3716 "# MAY BE MISSING FUNCTION EVENTS\n");
e0a413f6
SR
3717}
3718
d8741e2e 3719#ifdef CONFIG_TRACER_MAX_TRACE
f1affcaa 3720static void show_snapshot_main_help(struct seq_file *m)
d8741e2e 3721{
d79ac28f
RV
3722 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3723 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3724 "# Takes a snapshot of the main buffer.\n"
3725 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3726 "# (Doesn't have to be '2' works with any number that\n"
3727 "# is not a '0' or '1')\n");
d8741e2e 3728}
f1affcaa
SRRH
3729
3730static void show_snapshot_percpu_help(struct seq_file *m)
3731{
fa6f0cc7 3732 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
f1affcaa 3733#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
d79ac28f
RV
3734 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3735 "# Takes a snapshot of the main buffer for this cpu.\n");
f1affcaa 3736#else
d79ac28f
RV
3737 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3738 "# Must use main snapshot file to allocate.\n");
f1affcaa 3739#endif
d79ac28f
RV
3740 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3741 "# (Doesn't have to be '2' works with any number that\n"
3742 "# is not a '0' or '1')\n");
f1affcaa
SRRH
3743}
3744
d8741e2e
SRRH
3745static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3746{
45ad21ca 3747 if (iter->tr->allocated_snapshot)
fa6f0cc7 3748 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
d8741e2e 3749 else
fa6f0cc7 3750 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
d8741e2e 3751
fa6f0cc7 3752 seq_puts(m, "# Snapshot commands:\n");
f1affcaa
SRRH
3753 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3754 show_snapshot_main_help(m);
3755 else
3756 show_snapshot_percpu_help(m);
d8741e2e
SRRH
3757}
3758#else
3759/* Should never be called */
3760static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3761#endif
3762
bc0c38d1
SR
3763static int s_show(struct seq_file *m, void *v)
3764{
3765 struct trace_iterator *iter = v;
a63ce5b3 3766 int ret;
bc0c38d1
SR
3767
3768 if (iter->ent == NULL) {
3769 if (iter->tr) {
3770 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3771 seq_puts(m, "#\n");
e0a413f6 3772 test_ftrace_alive(m);
bc0c38d1 3773 }
d8741e2e
SRRH
3774 if (iter->snapshot && trace_empty(iter))
3775 print_snapshot_help(m, iter);
3776 else if (iter->trace && iter->trace->print_header)
8bba1bf5 3777 iter->trace->print_header(m);
62b915f1
JO
3778 else
3779 trace_default_header(m);
3780
a63ce5b3
SR
3781 } else if (iter->leftover) {
3782 /*
3783 * If we filled the seq_file buffer earlier, we
3784 * want to just show it now.
3785 */
3786 ret = trace_print_seq(m, &iter->seq);
3787
3788 /* ret should this time be zero, but you never know */
3789 iter->leftover = ret;
3790
bc0c38d1 3791 } else {
f9896bf3 3792 print_trace_line(iter);
a63ce5b3
SR
3793 ret = trace_print_seq(m, &iter->seq);
3794 /*
3795 * If we overflow the seq_file buffer, then it will
3796 * ask us for this data again at start up.
3797 * Use that instead.
3798 * ret is 0 if seq_file write succeeded.
3799 * -1 otherwise.
3800 */
3801 iter->leftover = ret;
bc0c38d1
SR
3802 }
3803
3804 return 0;
3805}
3806
649e9c70
ON
3807/*
3808 * Should be used after trace_array_get(), trace_types_lock
3809 * ensures that i_cdev was already initialized.
3810 */
3811static inline int tracing_get_cpu(struct inode *inode)
3812{
3813 if (inode->i_cdev) /* See trace_create_cpu_file() */
3814 return (long)inode->i_cdev - 1;
3815 return RING_BUFFER_ALL_CPUS;
3816}
3817
88e9d34c 3818static const struct seq_operations tracer_seq_ops = {
4bf39a94
IM
3819 .start = s_start,
3820 .next = s_next,
3821 .stop = s_stop,
3822 .show = s_show,
bc0c38d1
SR
3823};
3824
e309b41d 3825static struct trace_iterator *
6484c71c 3826__tracing_open(struct inode *inode, struct file *file, bool snapshot)
bc0c38d1 3827{
6484c71c 3828 struct trace_array *tr = inode->i_private;
bc0c38d1 3829 struct trace_iterator *iter;
50e18b94 3830 int cpu;
bc0c38d1 3831
85a2f9b4
SR
3832 if (tracing_disabled)
3833 return ERR_PTR(-ENODEV);
60a11774 3834
50e18b94 3835 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
85a2f9b4
SR
3836 if (!iter)
3837 return ERR_PTR(-ENOMEM);
bc0c38d1 3838
72917235 3839 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
6d158a81 3840 GFP_KERNEL);
93574fcc
DC
3841 if (!iter->buffer_iter)
3842 goto release;
3843
d7350c3f
FW
3844 /*
3845 * We make a copy of the current tracer to avoid concurrent
3846 * changes on it while we are reading.
3847 */
bc0c38d1 3848 mutex_lock(&trace_types_lock);
d7350c3f 3849 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
85a2f9b4 3850 if (!iter->trace)
d7350c3f 3851 goto fail;
85a2f9b4 3852
2b6080f2 3853 *iter->trace = *tr->current_trace;
d7350c3f 3854
79f55997 3855 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
b0dfa978
FW
3856 goto fail;
3857
12883efb
SRRH
3858 iter->tr = tr;
3859
3860#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
3861 /* Currently only the top directory has a snapshot */
3862 if (tr->current_trace->print_max || snapshot)
12883efb 3863 iter->trace_buffer = &tr->max_buffer;
bc0c38d1 3864 else
12883efb
SRRH
3865#endif
3866 iter->trace_buffer = &tr->trace_buffer;
debdd57f 3867 iter->snapshot = snapshot;
bc0c38d1 3868 iter->pos = -1;
6484c71c 3869 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 3870 mutex_init(&iter->mutex);
bc0c38d1 3871
8bba1bf5
MM
3872 /* Notify the tracer early; before we stop tracing. */
3873 if (iter->trace && iter->trace->open)
a93751ca 3874 iter->trace->open(iter);
8bba1bf5 3875
12ef7d44 3876 /* Annotate start of buffers if we had overruns */
12883efb 3877 if (ring_buffer_overruns(iter->trace_buffer->buffer))
12ef7d44
SR
3878 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3879
8be0709f 3880 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 3881 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
3882 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3883
debdd57f
HT
3884 /* stop the trace while dumping if we are not opening "snapshot" */
3885 if (!iter->snapshot)
2b6080f2 3886 tracing_stop_tr(tr);
2f26ebd5 3887
ae3b5093 3888 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
b04cc6b1 3889 for_each_tracing_cpu(cpu) {
b04cc6b1 3890 iter->buffer_iter[cpu] =
12883efb 3891 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3892 }
3893 ring_buffer_read_prepare_sync();
3894 for_each_tracing_cpu(cpu) {
3895 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3896 tracing_iter_reset(iter, cpu);
b04cc6b1
FW
3897 }
3898 } else {
3899 cpu = iter->cpu_file;
3928a8a2 3900 iter->buffer_iter[cpu] =
12883efb 3901 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
72c9ddfd
DM
3902 ring_buffer_read_prepare_sync();
3903 ring_buffer_read_start(iter->buffer_iter[cpu]);
2f26ebd5 3904 tracing_iter_reset(iter, cpu);
3928a8a2
SR
3905 }
3906
bc0c38d1
SR
3907 mutex_unlock(&trace_types_lock);
3908
bc0c38d1 3909 return iter;
3928a8a2 3910
d7350c3f 3911 fail:
3928a8a2 3912 mutex_unlock(&trace_types_lock);
d7350c3f 3913 kfree(iter->trace);
6d158a81 3914 kfree(iter->buffer_iter);
93574fcc 3915release:
50e18b94
JO
3916 seq_release_private(inode, file);
3917 return ERR_PTR(-ENOMEM);
bc0c38d1
SR
3918}
3919
3920int tracing_open_generic(struct inode *inode, struct file *filp)
3921{
60a11774
SR
3922 if (tracing_disabled)
3923 return -ENODEV;
3924
bc0c38d1
SR
3925 filp->private_data = inode->i_private;
3926 return 0;
3927}
3928
2e86421d
GB
3929bool tracing_is_disabled(void)
3930{
3931 return (tracing_disabled) ? true: false;
3932}
3933
7b85af63
SRRH
3934/*
3935 * Open and update trace_array ref count.
3936 * Must have the current trace_array passed to it.
3937 */
dcc30223 3938static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
7b85af63
SRRH
3939{
3940 struct trace_array *tr = inode->i_private;
3941
3942 if (tracing_disabled)
3943 return -ENODEV;
3944
3945 if (trace_array_get(tr) < 0)
3946 return -ENODEV;
3947
3948 filp->private_data = inode->i_private;
3949
3950 return 0;
7b85af63
SRRH
3951}
3952
4fd27358 3953static int tracing_release(struct inode *inode, struct file *file)
bc0c38d1 3954{
6484c71c 3955 struct trace_array *tr = inode->i_private;
907f2784 3956 struct seq_file *m = file->private_data;
4acd4d00 3957 struct trace_iterator *iter;
3928a8a2 3958 int cpu;
bc0c38d1 3959
ff451961 3960 if (!(file->f_mode & FMODE_READ)) {
6484c71c 3961 trace_array_put(tr);
4acd4d00 3962 return 0;
ff451961 3963 }
4acd4d00 3964
6484c71c 3965 /* Writes do not use seq_file */
4acd4d00 3966 iter = m->private;
bc0c38d1 3967 mutex_lock(&trace_types_lock);
a695cb58 3968
3928a8a2
SR
3969 for_each_tracing_cpu(cpu) {
3970 if (iter->buffer_iter[cpu])
3971 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3972 }
3973
bc0c38d1
SR
3974 if (iter->trace && iter->trace->close)
3975 iter->trace->close(iter);
3976
debdd57f
HT
3977 if (!iter->snapshot)
3978 /* reenable tracing if it was previously enabled */
2b6080f2 3979 tracing_start_tr(tr);
f77d09a3
AL
3980
3981 __trace_array_put(tr);
3982
bc0c38d1
SR
3983 mutex_unlock(&trace_types_lock);
3984
d7350c3f 3985 mutex_destroy(&iter->mutex);
b0dfa978 3986 free_cpumask_var(iter->started);
d7350c3f 3987 kfree(iter->trace);
6d158a81 3988 kfree(iter->buffer_iter);
50e18b94 3989 seq_release_private(inode, file);
ff451961 3990
bc0c38d1
SR
3991 return 0;
3992}
3993
7b85af63
SRRH
3994static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3995{
3996 struct trace_array *tr = inode->i_private;
3997
3998 trace_array_put(tr);
bc0c38d1
SR
3999 return 0;
4000}
4001
7b85af63
SRRH
4002static int tracing_single_release_tr(struct inode *inode, struct file *file)
4003{
4004 struct trace_array *tr = inode->i_private;
4005
4006 trace_array_put(tr);
4007
4008 return single_release(inode, file);
4009}
4010
bc0c38d1
SR
4011static int tracing_open(struct inode *inode, struct file *file)
4012{
6484c71c 4013 struct trace_array *tr = inode->i_private;
85a2f9b4
SR
4014 struct trace_iterator *iter;
4015 int ret = 0;
bc0c38d1 4016
ff451961
SRRH
4017 if (trace_array_get(tr) < 0)
4018 return -ENODEV;
4019
4acd4d00 4020 /* If this file was open for write, then erase contents */
6484c71c
ON
4021 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4022 int cpu = tracing_get_cpu(inode);
8dd33bcb
BY
4023 struct trace_buffer *trace_buf = &tr->trace_buffer;
4024
4025#ifdef CONFIG_TRACER_MAX_TRACE
4026 if (tr->current_trace->print_max)
4027 trace_buf = &tr->max_buffer;
4028#endif
6484c71c
ON
4029
4030 if (cpu == RING_BUFFER_ALL_CPUS)
8dd33bcb 4031 tracing_reset_online_cpus(trace_buf);
4acd4d00 4032 else
8dd33bcb 4033 tracing_reset(trace_buf, cpu);
4acd4d00 4034 }
bc0c38d1 4035
4acd4d00 4036 if (file->f_mode & FMODE_READ) {
6484c71c 4037 iter = __tracing_open(inode, file, false);
4acd4d00
SR
4038 if (IS_ERR(iter))
4039 ret = PTR_ERR(iter);
983f938a 4040 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4acd4d00
SR
4041 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4042 }
ff451961
SRRH
4043
4044 if (ret < 0)
4045 trace_array_put(tr);
4046
bc0c38d1
SR
4047 return ret;
4048}
4049
607e2ea1
SRRH
4050/*
4051 * Some tracers are not suitable for instance buffers.
4052 * A tracer is always available for the global array (toplevel)
4053 * or if it explicitly states that it is.
4054 */
4055static bool
4056trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4057{
4058 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4059}
4060
4061/* Find the next tracer that this trace array may use */
4062static struct tracer *
4063get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4064{
4065 while (t && !trace_ok_for_array(t, tr))
4066 t = t->next;
4067
4068 return t;
4069}
4070
e309b41d 4071static void *
bc0c38d1
SR
4072t_next(struct seq_file *m, void *v, loff_t *pos)
4073{
607e2ea1 4074 struct trace_array *tr = m->private;
f129e965 4075 struct tracer *t = v;
bc0c38d1
SR
4076
4077 (*pos)++;
4078
4079 if (t)
607e2ea1 4080 t = get_tracer_for_array(tr, t->next);
bc0c38d1 4081
bc0c38d1
SR
4082 return t;
4083}
4084
4085static void *t_start(struct seq_file *m, loff_t *pos)
4086{
607e2ea1 4087 struct trace_array *tr = m->private;
f129e965 4088 struct tracer *t;
bc0c38d1
SR
4089 loff_t l = 0;
4090
4091 mutex_lock(&trace_types_lock);
607e2ea1
SRRH
4092
4093 t = get_tracer_for_array(tr, trace_types);
4094 for (; t && l < *pos; t = t_next(m, t, &l))
4095 ;
bc0c38d1
SR
4096
4097 return t;
4098}
4099
4100static void t_stop(struct seq_file *m, void *p)
4101{
4102 mutex_unlock(&trace_types_lock);
4103}
4104
4105static int t_show(struct seq_file *m, void *v)
4106{
4107 struct tracer *t = v;
4108
4109 if (!t)
4110 return 0;
4111
fa6f0cc7 4112 seq_puts(m, t->name);
bc0c38d1
SR
4113 if (t->next)
4114 seq_putc(m, ' ');
4115 else
4116 seq_putc(m, '\n');
4117
4118 return 0;
4119}
4120
88e9d34c 4121static const struct seq_operations show_traces_seq_ops = {
4bf39a94
IM
4122 .start = t_start,
4123 .next = t_next,
4124 .stop = t_stop,
4125 .show = t_show,
bc0c38d1
SR
4126};
4127
4128static int show_traces_open(struct inode *inode, struct file *file)
4129{
607e2ea1
SRRH
4130 struct trace_array *tr = inode->i_private;
4131 struct seq_file *m;
4132 int ret;
4133
60a11774
SR
4134 if (tracing_disabled)
4135 return -ENODEV;
4136
607e2ea1
SRRH
4137 ret = seq_open(file, &show_traces_seq_ops);
4138 if (ret)
4139 return ret;
4140
4141 m = file->private_data;
4142 m->private = tr;
4143
4144 return 0;
bc0c38d1
SR
4145}
4146
4acd4d00
SR
4147static ssize_t
4148tracing_write_stub(struct file *filp, const char __user *ubuf,
4149 size_t count, loff_t *ppos)
4150{
4151 return count;
4152}
4153
098c879e 4154loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
364829b1 4155{
098c879e
SRRH
4156 int ret;
4157
364829b1 4158 if (file->f_mode & FMODE_READ)
098c879e 4159 ret = seq_lseek(file, offset, whence);
364829b1 4160 else
098c879e
SRRH
4161 file->f_pos = ret = 0;
4162
4163 return ret;
364829b1
SP
4164}
4165
5e2336a0 4166static const struct file_operations tracing_fops = {
4bf39a94
IM
4167 .open = tracing_open,
4168 .read = seq_read,
4acd4d00 4169 .write = tracing_write_stub,
098c879e 4170 .llseek = tracing_lseek,
4bf39a94 4171 .release = tracing_release,
bc0c38d1
SR
4172};
4173
5e2336a0 4174static const struct file_operations show_traces_fops = {
c7078de1
IM
4175 .open = show_traces_open,
4176 .read = seq_read,
4177 .release = seq_release,
b444786f 4178 .llseek = seq_lseek,
c7078de1
IM
4179};
4180
4181static ssize_t
4182tracing_cpumask_read(struct file *filp, char __user *ubuf,
4183 size_t count, loff_t *ppos)
4184{
ccfe9e42 4185 struct trace_array *tr = file_inode(filp)->i_private;
90e406f9 4186 char *mask_str;
36dfe925 4187 int len;
c7078de1 4188
90e406f9
CD
4189 len = snprintf(NULL, 0, "%*pb\n",
4190 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4191 mask_str = kmalloc(len, GFP_KERNEL);
4192 if (!mask_str)
4193 return -ENOMEM;
36dfe925 4194
90e406f9 4195 len = snprintf(mask_str, len, "%*pb\n",
1a40243b
TH
4196 cpumask_pr_args(tr->tracing_cpumask));
4197 if (len >= count) {
36dfe925
IM
4198 count = -EINVAL;
4199 goto out_err;
4200 }
90e406f9 4201 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
36dfe925
IM
4202
4203out_err:
90e406f9 4204 kfree(mask_str);
c7078de1
IM
4205
4206 return count;
4207}
4208
4209static ssize_t
4210tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4211 size_t count, loff_t *ppos)
4212{
ccfe9e42 4213 struct trace_array *tr = file_inode(filp)->i_private;
9e01c1b7 4214 cpumask_var_t tracing_cpumask_new;
2b6080f2 4215 int err, cpu;
9e01c1b7
RR
4216
4217 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4218 return -ENOMEM;
c7078de1 4219
9e01c1b7 4220 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
c7078de1 4221 if (err)
36dfe925
IM
4222 goto err_unlock;
4223
a5e25883 4224 local_irq_disable();
0b9b12c1 4225 arch_spin_lock(&tr->max_lock);
ab46428c 4226 for_each_tracing_cpu(cpu) {
36dfe925
IM
4227 /*
4228 * Increase/decrease the disabled counter if we are
4229 * about to flip a bit in the cpumask:
4230 */
ccfe9e42 4231 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4232 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4233 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4234 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925 4235 }
ccfe9e42 4236 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
9e01c1b7 4237 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
12883efb
SRRH
4238 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4239 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
36dfe925
IM
4240 }
4241 }
0b9b12c1 4242 arch_spin_unlock(&tr->max_lock);
a5e25883 4243 local_irq_enable();
36dfe925 4244
ccfe9e42 4245 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
9e01c1b7 4246 free_cpumask_var(tracing_cpumask_new);
c7078de1
IM
4247
4248 return count;
36dfe925
IM
4249
4250err_unlock:
215368e8 4251 free_cpumask_var(tracing_cpumask_new);
36dfe925
IM
4252
4253 return err;
c7078de1
IM
4254}
4255
5e2336a0 4256static const struct file_operations tracing_cpumask_fops = {
ccfe9e42 4257 .open = tracing_open_generic_tr,
c7078de1
IM
4258 .read = tracing_cpumask_read,
4259 .write = tracing_cpumask_write,
ccfe9e42 4260 .release = tracing_release_generic_tr,
b444786f 4261 .llseek = generic_file_llseek,
bc0c38d1
SR
4262};
4263
fdb372ed 4264static int tracing_trace_options_show(struct seq_file *m, void *v)
bc0c38d1 4265{
d8e83d26 4266 struct tracer_opt *trace_opts;
2b6080f2 4267 struct trace_array *tr = m->private;
d8e83d26 4268 u32 tracer_flags;
d8e83d26 4269 int i;
adf9f195 4270
d8e83d26 4271 mutex_lock(&trace_types_lock);
2b6080f2
SR
4272 tracer_flags = tr->current_trace->flags->val;
4273 trace_opts = tr->current_trace->flags->opts;
d8e83d26 4274
bc0c38d1 4275 for (i = 0; trace_options[i]; i++) {
983f938a 4276 if (tr->trace_flags & (1 << i))
fdb372ed 4277 seq_printf(m, "%s\n", trace_options[i]);
bc0c38d1 4278 else
fdb372ed 4279 seq_printf(m, "no%s\n", trace_options[i]);
bc0c38d1
SR
4280 }
4281
adf9f195
FW
4282 for (i = 0; trace_opts[i].name; i++) {
4283 if (tracer_flags & trace_opts[i].bit)
fdb372ed 4284 seq_printf(m, "%s\n", trace_opts[i].name);
adf9f195 4285 else
fdb372ed 4286 seq_printf(m, "no%s\n", trace_opts[i].name);
adf9f195 4287 }
d8e83d26 4288 mutex_unlock(&trace_types_lock);
adf9f195 4289
fdb372ed 4290 return 0;
bc0c38d1 4291}
bc0c38d1 4292
8c1a49ae 4293static int __set_tracer_option(struct trace_array *tr,
8d18eaaf
LZ
4294 struct tracer_flags *tracer_flags,
4295 struct tracer_opt *opts, int neg)
4296{
d39cdd20 4297 struct tracer *trace = tracer_flags->trace;
8d18eaaf 4298 int ret;
bc0c38d1 4299
8c1a49ae 4300 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
8d18eaaf
LZ
4301 if (ret)
4302 return ret;
4303
4304 if (neg)
4305 tracer_flags->val &= ~opts->bit;
4306 else
4307 tracer_flags->val |= opts->bit;
4308 return 0;
bc0c38d1
SR
4309}
4310
adf9f195 4311/* Try to assign a tracer specific option */
8c1a49ae 4312static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
adf9f195 4313{
8c1a49ae 4314 struct tracer *trace = tr->current_trace;
7770841e 4315 struct tracer_flags *tracer_flags = trace->flags;
adf9f195 4316 struct tracer_opt *opts = NULL;
8d18eaaf 4317 int i;
adf9f195 4318
7770841e
Z
4319 for (i = 0; tracer_flags->opts[i].name; i++) {
4320 opts = &tracer_flags->opts[i];
adf9f195 4321
8d18eaaf 4322 if (strcmp(cmp, opts->name) == 0)
8c1a49ae 4323 return __set_tracer_option(tr, trace->flags, opts, neg);
adf9f195 4324 }
adf9f195 4325
8d18eaaf 4326 return -EINVAL;
adf9f195
FW
4327}
4328
613f04a0
SRRH
4329/* Some tracers require overwrite to stay enabled */
4330int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4331{
4332 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4333 return -1;
4334
4335 return 0;
4336}
4337
2b6080f2 4338int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
af4617bd
SR
4339{
4340 /* do nothing if flag is already set */
983f938a 4341 if (!!(tr->trace_flags & mask) == !!enabled)
613f04a0
SRRH
4342 return 0;
4343
4344 /* Give the tracer a chance to approve the change */
2b6080f2 4345 if (tr->current_trace->flag_changed)
bf6065b5 4346 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
613f04a0 4347 return -EINVAL;
af4617bd
SR
4348
4349 if (enabled)
983f938a 4350 tr->trace_flags |= mask;
af4617bd 4351 else
983f938a 4352 tr->trace_flags &= ~mask;
e870e9a1
LZ
4353
4354 if (mask == TRACE_ITER_RECORD_CMD)
4355 trace_event_enable_cmd_record(enabled);
750912fa 4356
d914ba37
JF
4357 if (mask == TRACE_ITER_RECORD_TGID) {
4358 if (!tgid_map)
4359 tgid_map = kzalloc((PID_MAX_DEFAULT + 1) * sizeof(*tgid_map),
4360 GFP_KERNEL);
4361 if (!tgid_map) {
4362 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4363 return -ENOMEM;
4364 }
4365
4366 trace_event_enable_tgid_record(enabled);
4367 }
4368
c37775d5
SR
4369 if (mask == TRACE_ITER_EVENT_FORK)
4370 trace_event_follow_fork(tr, enabled);
4371
1e10486f
NK
4372 if (mask == TRACE_ITER_FUNC_FORK)
4373 ftrace_pid_follow_fork(tr, enabled);
4374
80902822 4375 if (mask == TRACE_ITER_OVERWRITE) {
12883efb 4376 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
80902822 4377#ifdef CONFIG_TRACER_MAX_TRACE
12883efb 4378 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
80902822
SRRH
4379#endif
4380 }
81698831 4381
b9f9108c 4382 if (mask == TRACE_ITER_PRINTK) {
81698831 4383 trace_printk_start_stop_comm(enabled);
b9f9108c
SRRH
4384 trace_printk_control(enabled);
4385 }
613f04a0
SRRH
4386
4387 return 0;
af4617bd
SR
4388}
4389
2b6080f2 4390static int trace_set_options(struct trace_array *tr, char *option)
bc0c38d1 4391{
8d18eaaf 4392 char *cmp;
bc0c38d1 4393 int neg = 0;
613f04a0 4394 int ret = -ENODEV;
bc0c38d1 4395 int i;
a4d1e688 4396 size_t orig_len = strlen(option);
bc0c38d1 4397
7bcfaf54 4398 cmp = strstrip(option);
bc0c38d1 4399
8d18eaaf 4400 if (strncmp(cmp, "no", 2) == 0) {
bc0c38d1
SR
4401 neg = 1;
4402 cmp += 2;
4403 }
4404
69d34da2
SRRH
4405 mutex_lock(&trace_types_lock);
4406
bc0c38d1 4407 for (i = 0; trace_options[i]; i++) {
8d18eaaf 4408 if (strcmp(cmp, trace_options[i]) == 0) {
2b6080f2 4409 ret = set_tracer_flag(tr, 1 << i, !neg);
bc0c38d1
SR
4410 break;
4411 }
4412 }
adf9f195
FW
4413
4414 /* If no option could be set, test the specific tracer options */
69d34da2 4415 if (!trace_options[i])
8c1a49ae 4416 ret = set_tracer_option(tr, cmp, neg);
69d34da2
SRRH
4417
4418 mutex_unlock(&trace_types_lock);
bc0c38d1 4419
a4d1e688
JW
4420 /*
4421 * If the first trailing whitespace is replaced with '\0' by strstrip,
4422 * turn it back into a space.
4423 */
4424 if (orig_len > strlen(option))
4425 option[strlen(option)] = ' ';
4426
7bcfaf54
SR
4427 return ret;
4428}
4429
a4d1e688
JW
4430static void __init apply_trace_boot_options(void)
4431{
4432 char *buf = trace_boot_options_buf;
4433 char *option;
4434
4435 while (true) {
4436 option = strsep(&buf, ",");
4437
4438 if (!option)
4439 break;
a4d1e688 4440
43ed3843
SRRH
4441 if (*option)
4442 trace_set_options(&global_trace, option);
a4d1e688
JW
4443
4444 /* Put back the comma to allow this to be called again */
4445 if (buf)
4446 *(buf - 1) = ',';
4447 }
4448}
4449
7bcfaf54
SR
4450static ssize_t
4451tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4452 size_t cnt, loff_t *ppos)
4453{
2b6080f2
SR
4454 struct seq_file *m = filp->private_data;
4455 struct trace_array *tr = m->private;
7bcfaf54 4456 char buf[64];
613f04a0 4457 int ret;
7bcfaf54
SR
4458
4459 if (cnt >= sizeof(buf))
4460 return -EINVAL;
4461
4afe6495 4462 if (copy_from_user(buf, ubuf, cnt))
7bcfaf54
SR
4463 return -EFAULT;
4464
a8dd2176
SR
4465 buf[cnt] = 0;
4466
2b6080f2 4467 ret = trace_set_options(tr, buf);
613f04a0
SRRH
4468 if (ret < 0)
4469 return ret;
7bcfaf54 4470
cf8517cf 4471 *ppos += cnt;
bc0c38d1
SR
4472
4473 return cnt;
4474}
4475
fdb372ed
LZ
4476static int tracing_trace_options_open(struct inode *inode, struct file *file)
4477{
7b85af63 4478 struct trace_array *tr = inode->i_private;
f77d09a3 4479 int ret;
7b85af63 4480
fdb372ed
LZ
4481 if (tracing_disabled)
4482 return -ENODEV;
2b6080f2 4483
7b85af63
SRRH
4484 if (trace_array_get(tr) < 0)
4485 return -ENODEV;
4486
f77d09a3
AL
4487 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4488 if (ret < 0)
4489 trace_array_put(tr);
4490
4491 return ret;
fdb372ed
LZ
4492}
4493
5e2336a0 4494static const struct file_operations tracing_iter_fops = {
fdb372ed
LZ
4495 .open = tracing_trace_options_open,
4496 .read = seq_read,
4497 .llseek = seq_lseek,
7b85af63 4498 .release = tracing_single_release_tr,
ee6bce52 4499 .write = tracing_trace_options_write,
bc0c38d1
SR
4500};
4501
7bd2f24c
IM
4502static const char readme_msg[] =
4503 "tracing mini-HOWTO:\n\n"
22f45649
SRRH
4504 "# echo 0 > tracing_on : quick way to disable tracing\n"
4505 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4506 " Important files:\n"
4507 " trace\t\t\t- The static contents of the buffer\n"
4508 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4509 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4510 " current_tracer\t- function and latency tracers\n"
4511 " available_tracers\t- list of configured tracers for current_tracer\n"
4512 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4513 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4514 " trace_clock\t\t-change the clock used to order events\n"
4515 " local: Per cpu clock but may not be synced across CPUs\n"
4516 " global: Synced across CPUs but slows tracing down.\n"
4517 " counter: Not a clock, but just an increment\n"
4518 " uptime: Jiffy counter from time of boot\n"
4519 " perf: Same clock that perf events use\n"
4520#ifdef CONFIG_X86_64
4521 " x86-tsc: TSC cycle counter\n"
4522#endif
4523 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
fa32e855 4524 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
22f45649
SRRH
4525 " tracing_cpumask\t- Limit which CPUs to trace\n"
4526 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4527 "\t\t\t Remove sub-buffer with rmdir\n"
4528 " trace_options\t\t- Set format or modify how tracing happens\n"
71485c45
SRRH
4529 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4530 "\t\t\t option name\n"
939c7a4f 4531 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
22f45649
SRRH
4532#ifdef CONFIG_DYNAMIC_FTRACE
4533 "\n available_filter_functions - list of functions that can be filtered on\n"
71485c45
SRRH
4534 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4535 "\t\t\t functions\n"
60f1d5e3 4536 "\t accepts: func_full_name or glob-matching-pattern\n"
71485c45
SRRH
4537 "\t modules: Can select a group via module\n"
4538 "\t Format: :mod:<module-name>\n"
4539 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4540 "\t triggers: a command to perform when function is hit\n"
4541 "\t Format: <function>:<trigger>[:count]\n"
4542 "\t trigger: traceon, traceoff\n"
4543 "\t\t enable_event:<system>:<event>\n"
4544 "\t\t disable_event:<system>:<event>\n"
22f45649 4545#ifdef CONFIG_STACKTRACE
71485c45 4546 "\t\t stacktrace\n"
22f45649
SRRH
4547#endif
4548#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4549 "\t\t snapshot\n"
22f45649 4550#endif
17a280ea
SRRH
4551 "\t\t dump\n"
4552 "\t\t cpudump\n"
71485c45
SRRH
4553 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4554 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4555 "\t The first one will disable tracing every time do_fault is hit\n"
4556 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4557 "\t The first time do trap is hit and it disables tracing, the\n"
4558 "\t counter will decrement to 2. If tracing is already disabled,\n"
4559 "\t the counter will not decrement. It only decrements when the\n"
4560 "\t trigger did work\n"
4561 "\t To remove trigger without count:\n"
4562 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4563 "\t To remove trigger with a count:\n"
4564 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
22f45649 4565 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
71485c45
SRRH
4566 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4567 "\t modules: Can select a group via module command :mod:\n"
4568 "\t Does not accept triggers\n"
22f45649
SRRH
4569#endif /* CONFIG_DYNAMIC_FTRACE */
4570#ifdef CONFIG_FUNCTION_TRACER
71485c45
SRRH
4571 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4572 "\t\t (function)\n"
22f45649
SRRH
4573#endif
4574#ifdef CONFIG_FUNCTION_GRAPH_TRACER
4575 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
d048a8c7 4576 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
22f45649
SRRH
4577 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4578#endif
4579#ifdef CONFIG_TRACER_SNAPSHOT
71485c45
SRRH
4580 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4581 "\t\t\t snapshot buffer. Read the contents for more\n"
4582 "\t\t\t information\n"
22f45649 4583#endif
991821c8 4584#ifdef CONFIG_STACK_TRACER
22f45649
SRRH
4585 " stack_trace\t\t- Shows the max stack trace when active\n"
4586 " stack_max_size\t- Shows current max stack size that was traced\n"
71485c45
SRRH
4587 "\t\t\t Write into this file to reset the max size (trigger a\n"
4588 "\t\t\t new trace)\n"
22f45649 4589#ifdef CONFIG_DYNAMIC_FTRACE
71485c45
SRRH
4590 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4591 "\t\t\t traces\n"
22f45649 4592#endif
991821c8 4593#endif /* CONFIG_STACK_TRACER */
6b0b7551 4594#ifdef CONFIG_KPROBE_EVENTS
86425625
MH
4595 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4596 "\t\t\t Write into this file to define/undefine new trace events.\n"
4597#endif
6b0b7551 4598#ifdef CONFIG_UPROBE_EVENTS
86425625
MH
4599 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4600 "\t\t\t Write into this file to define/undefine new trace events.\n"
4601#endif
6b0b7551 4602#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
86425625 4603 "\t accepts: event-definitions (one definition per line)\n"
c3ca46ef
MH
4604 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4605 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
86425625 4606 "\t -:[<group>/]<event>\n"
6b0b7551 4607#ifdef CONFIG_KPROBE_EVENTS
86425625 4608 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
35b6f55a 4609 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
86425625 4610#endif
6b0b7551 4611#ifdef CONFIG_UPROBE_EVENTS
86425625
MH
4612 "\t place: <path>:<offset>\n"
4613#endif
4614 "\t args: <name>=fetcharg[:type]\n"
4615 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4616 "\t $stack<index>, $stack, $retval, $comm\n"
4617 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4618 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4619#endif
26f25564
TZ
4620 " events/\t\t- Directory containing all trace event subsystems:\n"
4621 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4622 " events/<system>/\t- Directory containing all trace events for <system>:\n"
71485c45
SRRH
4623 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4624 "\t\t\t events\n"
26f25564 4625 " filter\t\t- If set, only events passing filter are traced\n"
71485c45
SRRH
4626 " events/<system>/<event>/\t- Directory containing control files for\n"
4627 "\t\t\t <event>:\n"
26f25564
TZ
4628 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4629 " filter\t\t- If set, only events passing filter are traced\n"
4630 " trigger\t\t- If set, a command to perform when event is hit\n"
71485c45
SRRH
4631 "\t Format: <trigger>[:count][if <filter>]\n"
4632 "\t trigger: traceon, traceoff\n"
4633 "\t enable_event:<system>:<event>\n"
4634 "\t disable_event:<system>:<event>\n"
d0bad49b
TZ
4635#ifdef CONFIG_HIST_TRIGGERS
4636 "\t enable_hist:<system>:<event>\n"
4637 "\t disable_hist:<system>:<event>\n"
4638#endif
26f25564 4639#ifdef CONFIG_STACKTRACE
71485c45 4640 "\t\t stacktrace\n"
26f25564
TZ
4641#endif
4642#ifdef CONFIG_TRACER_SNAPSHOT
71485c45 4643 "\t\t snapshot\n"
7ef224d1
TZ
4644#endif
4645#ifdef CONFIG_HIST_TRIGGERS
4646 "\t\t hist (see below)\n"
26f25564 4647#endif
71485c45
SRRH
4648 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4649 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4650 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4651 "\t events/block/block_unplug/trigger\n"
4652 "\t The first disables tracing every time block_unplug is hit.\n"
4653 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4654 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4655 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4656 "\t Like function triggers, the counter is only decremented if it\n"
4657 "\t enabled or disabled tracing.\n"
4658 "\t To remove a trigger without a count:\n"
4659 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4660 "\t To remove a trigger with a count:\n"
4661 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4662 "\t Filters can be ignored when removing a trigger.\n"
7ef224d1
TZ
4663#ifdef CONFIG_HIST_TRIGGERS
4664 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
76a3b0c8 4665 "\t Format: hist:keys=<field1[,field2,...]>\n"
f2606835 4666 "\t [:values=<field1[,field2,...]>]\n"
e62347d2 4667 "\t [:sort=<field1[,field2,...]>]\n"
7ef224d1 4668 "\t [:size=#entries]\n"
e86ae9ba 4669 "\t [:pause][:continue][:clear]\n"
5463bfda 4670 "\t [:name=histname1]\n"
7ef224d1
TZ
4671 "\t [if <filter>]\n\n"
4672 "\t When a matching event is hit, an entry is added to a hash\n"
f2606835
TZ
4673 "\t table using the key(s) and value(s) named, and the value of a\n"
4674 "\t sum called 'hitcount' is incremented. Keys and values\n"
4675 "\t correspond to fields in the event's format description. Keys\n"
69a0200c
TZ
4676 "\t can be any field, or the special string 'stacktrace'.\n"
4677 "\t Compound keys consisting of up to two fields can be specified\n"
4678 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4679 "\t fields. Sort keys consisting of up to two fields can be\n"
4680 "\t specified using the 'sort' keyword. The sort direction can\n"
4681 "\t be modified by appending '.descending' or '.ascending' to a\n"
4682 "\t sort field. The 'size' parameter can be used to specify more\n"
5463bfda
TZ
4683 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4684 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4685 "\t its histogram data will be shared with other triggers of the\n"
4686 "\t same name, and trigger hits will update this common data.\n\n"
7ef224d1 4687 "\t Reading the 'hist' file for the event will dump the hash\n"
52a7f16d
TZ
4688 "\t table in its entirety to stdout. If there are multiple hist\n"
4689 "\t triggers attached to an event, there will be a table for each\n"
5463bfda
TZ
4690 "\t trigger in the output. The table displayed for a named\n"
4691 "\t trigger will be the same as any other instance having the\n"
4692 "\t same name. The default format used to display a given field\n"
4693 "\t can be modified by appending any of the following modifiers\n"
4694 "\t to the field name, as applicable:\n\n"
c6afad49
TZ
4695 "\t .hex display a number as a hex value\n"
4696 "\t .sym display an address as a symbol\n"
6b4827ad 4697 "\t .sym-offset display an address as a symbol and offset\n"
31696198
TZ
4698 "\t .execname display a common_pid as a program name\n"
4699 "\t .syscall display a syscall id as a syscall name\n\n"
4b94f5b7 4700 "\t .log2 display log2 value rather than raw number\n\n"
83e99914
TZ
4701 "\t The 'pause' parameter can be used to pause an existing hist\n"
4702 "\t trigger or to start a hist trigger but not log any events\n"
4703 "\t until told to do so. 'continue' can be used to start or\n"
4704 "\t restart a paused hist trigger.\n\n"
e86ae9ba
TZ
4705 "\t The 'clear' parameter will clear the contents of a running\n"
4706 "\t hist trigger and leave its current paused/active state\n"
4707 "\t unchanged.\n\n"
d0bad49b
TZ
4708 "\t The enable_hist and disable_hist triggers can be used to\n"
4709 "\t have one event conditionally start and stop another event's\n"
4710 "\t already-attached hist trigger. The syntax is analagous to\n"
4711 "\t the enable_event and disable_event triggers.\n"
7ef224d1 4712#endif
7bd2f24c
IM
4713;
4714
4715static ssize_t
4716tracing_readme_read(struct file *filp, char __user *ubuf,
4717 size_t cnt, loff_t *ppos)
4718{
4719 return simple_read_from_buffer(ubuf, cnt, ppos,
4720 readme_msg, strlen(readme_msg));
4721}
4722
5e2336a0 4723static const struct file_operations tracing_readme_fops = {
c7078de1
IM
4724 .open = tracing_open_generic,
4725 .read = tracing_readme_read,
b444786f 4726 .llseek = generic_file_llseek,
7bd2f24c
IM
4727};
4728
99c621d7
MS
4729static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
4730{
4731 int *ptr = v;
4732
4733 if (*pos || m->count)
4734 ptr++;
4735
4736 (*pos)++;
4737
4738 for (; ptr <= &tgid_map[PID_MAX_DEFAULT]; ptr++) {
4739 if (trace_find_tgid(*ptr))
4740 return ptr;
4741 }
4742
4743 return NULL;
4744}
4745
4746static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
4747{
4748 void *v;
4749 loff_t l = 0;
4750
4751 if (!tgid_map)
4752 return NULL;
4753
4754 v = &tgid_map[0];
4755 while (l <= *pos) {
4756 v = saved_tgids_next(m, v, &l);
4757 if (!v)
4758 return NULL;
4759 }
4760
4761 return v;
4762}
4763
4764static void saved_tgids_stop(struct seq_file *m, void *v)
4765{
4766}
4767
4768static int saved_tgids_show(struct seq_file *m, void *v)
4769{
4770 int pid = (int *)v - tgid_map;
4771
4772 seq_printf(m, "%d %d\n", pid, trace_find_tgid(pid));
4773 return 0;
4774}
4775
4776static const struct seq_operations tracing_saved_tgids_seq_ops = {
4777 .start = saved_tgids_start,
4778 .stop = saved_tgids_stop,
4779 .next = saved_tgids_next,
4780 .show = saved_tgids_show,
4781};
4782
4783static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
4784{
4785 if (tracing_disabled)
4786 return -ENODEV;
4787
4788 return seq_open(filp, &tracing_saved_tgids_seq_ops);
4789}
4790
4791
4792static const struct file_operations tracing_saved_tgids_fops = {
4793 .open = tracing_saved_tgids_open,
4794 .read = seq_read,
4795 .llseek = seq_lseek,
4796 .release = seq_release,
4797};
4798
42584c81
YY
4799static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4800{
4801 unsigned int *ptr = v;
69abe6a5 4802
42584c81
YY
4803 if (*pos || m->count)
4804 ptr++;
69abe6a5 4805
42584c81 4806 (*pos)++;
69abe6a5 4807
939c7a4f
YY
4808 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4809 ptr++) {
42584c81
YY
4810 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4811 continue;
69abe6a5 4812
42584c81
YY
4813 return ptr;
4814 }
69abe6a5 4815
42584c81
YY
4816 return NULL;
4817}
4818
4819static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4820{
4821 void *v;
4822 loff_t l = 0;
69abe6a5 4823
4c27e756
SRRH
4824 preempt_disable();
4825 arch_spin_lock(&trace_cmdline_lock);
4826
939c7a4f 4827 v = &savedcmd->map_cmdline_to_pid[0];
42584c81
YY
4828 while (l <= *pos) {
4829 v = saved_cmdlines_next(m, v, &l);
4830 if (!v)
4831 return NULL;
69abe6a5
AP
4832 }
4833
42584c81
YY
4834 return v;
4835}
4836
4837static void saved_cmdlines_stop(struct seq_file *m, void *v)
4838{
4c27e756
SRRH
4839 arch_spin_unlock(&trace_cmdline_lock);
4840 preempt_enable();
42584c81 4841}
69abe6a5 4842
42584c81
YY
4843static int saved_cmdlines_show(struct seq_file *m, void *v)
4844{
4845 char buf[TASK_COMM_LEN];
4846 unsigned int *pid = v;
69abe6a5 4847
4c27e756 4848 __trace_find_cmdline(*pid, buf);
42584c81
YY
4849 seq_printf(m, "%d %s\n", *pid, buf);
4850 return 0;
4851}
4852
4853static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4854 .start = saved_cmdlines_start,
4855 .next = saved_cmdlines_next,
4856 .stop = saved_cmdlines_stop,
4857 .show = saved_cmdlines_show,
4858};
4859
4860static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4861{
4862 if (tracing_disabled)
4863 return -ENODEV;
4864
4865 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
69abe6a5
AP
4866}
4867
4868static const struct file_operations tracing_saved_cmdlines_fops = {
42584c81
YY
4869 .open = tracing_saved_cmdlines_open,
4870 .read = seq_read,
4871 .llseek = seq_lseek,
4872 .release = seq_release,
69abe6a5
AP
4873};
4874
939c7a4f
YY
4875static ssize_t
4876tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4877 size_t cnt, loff_t *ppos)
4878{
4879 char buf[64];
4880 int r;
4881
4882 arch_spin_lock(&trace_cmdline_lock);
a6af8fbf 4883 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
939c7a4f
YY
4884 arch_spin_unlock(&trace_cmdline_lock);
4885
4886 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4887}
4888
4889static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4890{
4891 kfree(s->saved_cmdlines);
4892 kfree(s->map_cmdline_to_pid);
4893 kfree(s);
4894}
4895
4896static int tracing_resize_saved_cmdlines(unsigned int val)
4897{
4898 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4899
a6af8fbf 4900 s = kmalloc(sizeof(*s), GFP_KERNEL);
939c7a4f
YY
4901 if (!s)
4902 return -ENOMEM;
4903
4904 if (allocate_cmdlines_buffer(val, s) < 0) {
4905 kfree(s);
4906 return -ENOMEM;
4907 }
4908
4909 arch_spin_lock(&trace_cmdline_lock);
4910 savedcmd_temp = savedcmd;
4911 savedcmd = s;
4912 arch_spin_unlock(&trace_cmdline_lock);
4913 free_saved_cmdlines_buffer(savedcmd_temp);
4914
4915 return 0;
4916}
4917
4918static ssize_t
4919tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4920 size_t cnt, loff_t *ppos)
4921{
4922 unsigned long val;
4923 int ret;
4924
4925 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4926 if (ret)
4927 return ret;
4928
4929 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4930 if (!val || val > PID_MAX_DEFAULT)
4931 return -EINVAL;
4932
4933 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4934 if (ret < 0)
4935 return ret;
4936
4937 *ppos += cnt;
4938
4939 return cnt;
4940}
4941
4942static const struct file_operations tracing_saved_cmdlines_size_fops = {
4943 .open = tracing_open_generic,
4944 .read = tracing_saved_cmdlines_size_read,
4945 .write = tracing_saved_cmdlines_size_write,
4946};
4947
681bec03 4948#ifdef CONFIG_TRACE_EVAL_MAP_FILE
23bf8cb8 4949static union trace_eval_map_item *
f57a4143 4950update_eval_map(union trace_eval_map_item *ptr)
9828413d 4951{
00f4b652 4952 if (!ptr->map.eval_string) {
9828413d
SRRH
4953 if (ptr->tail.next) {
4954 ptr = ptr->tail.next;
4955 /* Set ptr to the next real item (skip head) */
4956 ptr++;
4957 } else
4958 return NULL;
4959 }
4960 return ptr;
4961}
4962
f57a4143 4963static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
9828413d 4964{
23bf8cb8 4965 union trace_eval_map_item *ptr = v;
9828413d
SRRH
4966
4967 /*
4968 * Paranoid! If ptr points to end, we don't want to increment past it.
4969 * This really should never happen.
4970 */
f57a4143 4971 ptr = update_eval_map(ptr);
9828413d
SRRH
4972 if (WARN_ON_ONCE(!ptr))
4973 return NULL;
4974
4975 ptr++;
4976
4977 (*pos)++;
4978
f57a4143 4979 ptr = update_eval_map(ptr);
9828413d
SRRH
4980
4981 return ptr;
4982}
4983
f57a4143 4984static void *eval_map_start(struct seq_file *m, loff_t *pos)
9828413d 4985{
23bf8cb8 4986 union trace_eval_map_item *v;
9828413d
SRRH
4987 loff_t l = 0;
4988
1793ed93 4989 mutex_lock(&trace_eval_mutex);
9828413d 4990
23bf8cb8 4991 v = trace_eval_maps;
9828413d
SRRH
4992 if (v)
4993 v++;
4994
4995 while (v && l < *pos) {
f57a4143 4996 v = eval_map_next(m, v, &l);
9828413d
SRRH
4997 }
4998
4999 return v;
5000}
5001
f57a4143 5002static void eval_map_stop(struct seq_file *m, void *v)
9828413d 5003{
1793ed93 5004 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
5005}
5006
f57a4143 5007static int eval_map_show(struct seq_file *m, void *v)
9828413d 5008{
23bf8cb8 5009 union trace_eval_map_item *ptr = v;
9828413d
SRRH
5010
5011 seq_printf(m, "%s %ld (%s)\n",
00f4b652 5012 ptr->map.eval_string, ptr->map.eval_value,
9828413d
SRRH
5013 ptr->map.system);
5014
5015 return 0;
5016}
5017
f57a4143
JL
5018static const struct seq_operations tracing_eval_map_seq_ops = {
5019 .start = eval_map_start,
5020 .next = eval_map_next,
5021 .stop = eval_map_stop,
5022 .show = eval_map_show,
9828413d
SRRH
5023};
5024
f57a4143 5025static int tracing_eval_map_open(struct inode *inode, struct file *filp)
9828413d
SRRH
5026{
5027 if (tracing_disabled)
5028 return -ENODEV;
5029
f57a4143 5030 return seq_open(filp, &tracing_eval_map_seq_ops);
9828413d
SRRH
5031}
5032
f57a4143
JL
5033static const struct file_operations tracing_eval_map_fops = {
5034 .open = tracing_eval_map_open,
9828413d
SRRH
5035 .read = seq_read,
5036 .llseek = seq_lseek,
5037 .release = seq_release,
5038};
5039
23bf8cb8 5040static inline union trace_eval_map_item *
5f60b351 5041trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
9828413d
SRRH
5042{
5043 /* Return tail of array given the head */
5044 return ptr + ptr->head.length + 1;
5045}
5046
5047static void
f57a4143 5048trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
9828413d
SRRH
5049 int len)
5050{
00f4b652
JL
5051 struct trace_eval_map **stop;
5052 struct trace_eval_map **map;
23bf8cb8
JL
5053 union trace_eval_map_item *map_array;
5054 union trace_eval_map_item *ptr;
9828413d
SRRH
5055
5056 stop = start + len;
5057
5058 /*
23bf8cb8 5059 * The trace_eval_maps contains the map plus a head and tail item,
9828413d
SRRH
5060 * where the head holds the module and length of array, and the
5061 * tail holds a pointer to the next list.
5062 */
5063 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
5064 if (!map_array) {
f57a4143 5065 pr_warn("Unable to allocate trace eval mapping\n");
9828413d
SRRH
5066 return;
5067 }
5068
1793ed93 5069 mutex_lock(&trace_eval_mutex);
9828413d 5070
23bf8cb8
JL
5071 if (!trace_eval_maps)
5072 trace_eval_maps = map_array;
9828413d 5073 else {
23bf8cb8 5074 ptr = trace_eval_maps;
9828413d 5075 for (;;) {
5f60b351 5076 ptr = trace_eval_jmp_to_tail(ptr);
9828413d
SRRH
5077 if (!ptr->tail.next)
5078 break;
5079 ptr = ptr->tail.next;
5080
5081 }
5082 ptr->tail.next = map_array;
5083 }
5084 map_array->head.mod = mod;
5085 map_array->head.length = len;
5086 map_array++;
5087
5088 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5089 map_array->map = **map;
5090 map_array++;
5091 }
5092 memset(map_array, 0, sizeof(*map_array));
5093
1793ed93 5094 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
5095}
5096
f57a4143 5097static void trace_create_eval_file(struct dentry *d_tracer)
9828413d 5098{
681bec03 5099 trace_create_file("eval_map", 0444, d_tracer,
f57a4143 5100 NULL, &tracing_eval_map_fops);
9828413d
SRRH
5101}
5102
681bec03 5103#else /* CONFIG_TRACE_EVAL_MAP_FILE */
f57a4143
JL
5104static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5105static inline void trace_insert_eval_map_file(struct module *mod,
00f4b652 5106 struct trace_eval_map **start, int len) { }
681bec03 5107#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 5108
f57a4143 5109static void trace_insert_eval_map(struct module *mod,
00f4b652 5110 struct trace_eval_map **start, int len)
0c564a53 5111{
00f4b652 5112 struct trace_eval_map **map;
0c564a53
SRRH
5113
5114 if (len <= 0)
5115 return;
5116
5117 map = start;
5118
f57a4143 5119 trace_event_eval_update(map, len);
9828413d 5120
f57a4143 5121 trace_insert_eval_map_file(mod, start, len);
0c564a53
SRRH
5122}
5123
bc0c38d1
SR
5124static ssize_t
5125tracing_set_trace_read(struct file *filp, char __user *ubuf,
5126 size_t cnt, loff_t *ppos)
5127{
2b6080f2 5128 struct trace_array *tr = filp->private_data;
ee6c2c1b 5129 char buf[MAX_TRACER_SIZE+2];
bc0c38d1
SR
5130 int r;
5131
5132 mutex_lock(&trace_types_lock);
2b6080f2 5133 r = sprintf(buf, "%s\n", tr->current_trace->name);
bc0c38d1
SR
5134 mutex_unlock(&trace_types_lock);
5135
4bf39a94 5136 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
5137}
5138
b6f11df2
ACM
5139int tracer_init(struct tracer *t, struct trace_array *tr)
5140{
12883efb 5141 tracing_reset_online_cpus(&tr->trace_buffer);
b6f11df2
ACM
5142 return t->init(tr);
5143}
5144
12883efb 5145static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
438ced17
VN
5146{
5147 int cpu;
737223fb 5148
438ced17 5149 for_each_tracing_cpu(cpu)
12883efb 5150 per_cpu_ptr(buf->data, cpu)->entries = val;
438ced17
VN
5151}
5152
12883efb 5153#ifdef CONFIG_TRACER_MAX_TRACE
d60da506 5154/* resize @tr's buffer to the size of @size_tr's entries */
12883efb
SRRH
5155static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5156 struct trace_buffer *size_buf, int cpu_id)
d60da506
HT
5157{
5158 int cpu, ret = 0;
5159
5160 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5161 for_each_tracing_cpu(cpu) {
12883efb
SRRH
5162 ret = ring_buffer_resize(trace_buf->buffer,
5163 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
d60da506
HT
5164 if (ret < 0)
5165 break;
12883efb
SRRH
5166 per_cpu_ptr(trace_buf->data, cpu)->entries =
5167 per_cpu_ptr(size_buf->data, cpu)->entries;
d60da506
HT
5168 }
5169 } else {
12883efb
SRRH
5170 ret = ring_buffer_resize(trace_buf->buffer,
5171 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
d60da506 5172 if (ret == 0)
12883efb
SRRH
5173 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5174 per_cpu_ptr(size_buf->data, cpu_id)->entries;
d60da506
HT
5175 }
5176
5177 return ret;
5178}
12883efb 5179#endif /* CONFIG_TRACER_MAX_TRACE */
d60da506 5180
2b6080f2
SR
5181static int __tracing_resize_ring_buffer(struct trace_array *tr,
5182 unsigned long size, int cpu)
73c5162a
SR
5183{
5184 int ret;
5185
5186 /*
5187 * If kernel or user changes the size of the ring buffer
a123c52b
SR
5188 * we use the size that was given, and we can forget about
5189 * expanding it later.
73c5162a 5190 */
55034cd6 5191 ring_buffer_expanded = true;
73c5162a 5192
b382ede6 5193 /* May be called before buffers are initialized */
12883efb 5194 if (!tr->trace_buffer.buffer)
b382ede6
SR
5195 return 0;
5196
12883efb 5197 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
73c5162a
SR
5198 if (ret < 0)
5199 return ret;
5200
12883efb 5201#ifdef CONFIG_TRACER_MAX_TRACE
2b6080f2
SR
5202 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5203 !tr->current_trace->use_max_tr)
ef710e10
KM
5204 goto out;
5205
12883efb 5206 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
73c5162a 5207 if (ret < 0) {
12883efb
SRRH
5208 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5209 &tr->trace_buffer, cpu);
73c5162a 5210 if (r < 0) {
a123c52b
SR
5211 /*
5212 * AARGH! We are left with different
5213 * size max buffer!!!!
5214 * The max buffer is our "snapshot" buffer.
5215 * When a tracer needs a snapshot (one of the
5216 * latency tracers), it swaps the max buffer
5217 * with the saved snap shot. We succeeded to
5218 * update the size of the main buffer, but failed to
5219 * update the size of the max buffer. But when we tried
5220 * to reset the main buffer to the original size, we
5221 * failed there too. This is very unlikely to
5222 * happen, but if it does, warn and kill all
5223 * tracing.
5224 */
73c5162a
SR
5225 WARN_ON(1);
5226 tracing_disabled = 1;
5227 }
5228 return ret;
5229 }
5230
438ced17 5231 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 5232 set_buffer_entries(&tr->max_buffer, size);
438ced17 5233 else
12883efb 5234 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
438ced17 5235
ef710e10 5236 out:
12883efb
SRRH
5237#endif /* CONFIG_TRACER_MAX_TRACE */
5238
438ced17 5239 if (cpu == RING_BUFFER_ALL_CPUS)
12883efb 5240 set_buffer_entries(&tr->trace_buffer, size);
438ced17 5241 else
12883efb 5242 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
73c5162a
SR
5243
5244 return ret;
5245}
5246
2b6080f2
SR
5247static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5248 unsigned long size, int cpu_id)
4f271a2a 5249{
83f40318 5250 int ret = size;
4f271a2a
VN
5251
5252 mutex_lock(&trace_types_lock);
5253
438ced17
VN
5254 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5255 /* make sure, this cpu is enabled in the mask */
5256 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5257 ret = -EINVAL;
5258 goto out;
5259 }
5260 }
4f271a2a 5261
2b6080f2 5262 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4f271a2a
VN
5263 if (ret < 0)
5264 ret = -ENOMEM;
5265
438ced17 5266out:
4f271a2a
VN
5267 mutex_unlock(&trace_types_lock);
5268
5269 return ret;
5270}
5271
ef710e10 5272
1852fcce
SR
5273/**
5274 * tracing_update_buffers - used by tracing facility to expand ring buffers
5275 *
5276 * To save on memory when the tracing is never used on a system with it
5277 * configured in. The ring buffers are set to a minimum size. But once
5278 * a user starts to use the tracing facility, then they need to grow
5279 * to their default size.
5280 *
5281 * This function is to be called when a tracer is about to be used.
5282 */
5283int tracing_update_buffers(void)
5284{
5285 int ret = 0;
5286
1027fcb2 5287 mutex_lock(&trace_types_lock);
1852fcce 5288 if (!ring_buffer_expanded)
2b6080f2 5289 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
438ced17 5290 RING_BUFFER_ALL_CPUS);
1027fcb2 5291 mutex_unlock(&trace_types_lock);
1852fcce
SR
5292
5293 return ret;
5294}
5295
577b785f
SR
5296struct trace_option_dentry;
5297
37aea98b 5298static void
2b6080f2 5299create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
577b785f 5300
6b450d25
SRRH
5301/*
5302 * Used to clear out the tracer before deletion of an instance.
5303 * Must have trace_types_lock held.
5304 */
5305static void tracing_set_nop(struct trace_array *tr)
5306{
5307 if (tr->current_trace == &nop_trace)
5308 return;
5309
50512ab5 5310 tr->current_trace->enabled--;
6b450d25
SRRH
5311
5312 if (tr->current_trace->reset)
5313 tr->current_trace->reset(tr);
5314
5315 tr->current_trace = &nop_trace;
5316}
5317
41d9c0be 5318static void add_tracer_options(struct trace_array *tr, struct tracer *t)
bc0c38d1 5319{
09d23a1d
SRRH
5320 /* Only enable if the directory has been created already. */
5321 if (!tr->dir)
5322 return;
5323
37aea98b 5324 create_trace_option_files(tr, t);
09d23a1d
SRRH
5325}
5326
5327static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5328{
bc0c38d1 5329 struct tracer *t;
12883efb 5330#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5331 bool had_max_tr;
12883efb 5332#endif
d9e54076 5333 int ret = 0;
bc0c38d1 5334
1027fcb2
SR
5335 mutex_lock(&trace_types_lock);
5336
73c5162a 5337 if (!ring_buffer_expanded) {
2b6080f2 5338 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
438ced17 5339 RING_BUFFER_ALL_CPUS);
73c5162a 5340 if (ret < 0)
59f586db 5341 goto out;
73c5162a
SR
5342 ret = 0;
5343 }
5344
bc0c38d1
SR
5345 for (t = trace_types; t; t = t->next) {
5346 if (strcmp(t->name, buf) == 0)
5347 break;
5348 }
c2931e05
FW
5349 if (!t) {
5350 ret = -EINVAL;
5351 goto out;
5352 }
2b6080f2 5353 if (t == tr->current_trace)
bc0c38d1
SR
5354 goto out;
5355
c7b3ae0b
ZSZ
5356 /* Some tracers won't work on kernel command line */
5357 if (system_state < SYSTEM_RUNNING && t->noboot) {
5358 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5359 t->name);
5360 goto out;
5361 }
5362
607e2ea1
SRRH
5363 /* Some tracers are only allowed for the top level buffer */
5364 if (!trace_ok_for_array(t, tr)) {
5365 ret = -EINVAL;
5366 goto out;
5367 }
5368
cf6ab6d9
SRRH
5369 /* If trace pipe files are being read, we can't change the tracer */
5370 if (tr->current_trace->ref) {
5371 ret = -EBUSY;
5372 goto out;
5373 }
5374
9f029e83 5375 trace_branch_disable();
613f04a0 5376
50512ab5 5377 tr->current_trace->enabled--;
613f04a0 5378
2b6080f2
SR
5379 if (tr->current_trace->reset)
5380 tr->current_trace->reset(tr);
34600f0e 5381
12883efb 5382 /* Current trace needs to be nop_trace before synchronize_sched */
2b6080f2 5383 tr->current_trace = &nop_trace;
34600f0e 5384
45ad21ca
SRRH
5385#ifdef CONFIG_TRACER_MAX_TRACE
5386 had_max_tr = tr->allocated_snapshot;
34600f0e
SR
5387
5388 if (had_max_tr && !t->use_max_tr) {
5389 /*
5390 * We need to make sure that the update_max_tr sees that
5391 * current_trace changed to nop_trace to keep it from
5392 * swapping the buffers after we resize it.
5393 * The update_max_tr is called from interrupts disabled
5394 * so a synchronized_sched() is sufficient.
5395 */
5396 synchronize_sched();
3209cff4 5397 free_snapshot(tr);
ef710e10 5398 }
12883efb 5399#endif
12883efb
SRRH
5400
5401#ifdef CONFIG_TRACER_MAX_TRACE
34600f0e 5402 if (t->use_max_tr && !had_max_tr) {
3209cff4 5403 ret = alloc_snapshot(tr);
d60da506
HT
5404 if (ret < 0)
5405 goto out;
ef710e10 5406 }
12883efb 5407#endif
577b785f 5408
1c80025a 5409 if (t->init) {
b6f11df2 5410 ret = tracer_init(t, tr);
1c80025a
FW
5411 if (ret)
5412 goto out;
5413 }
bc0c38d1 5414
2b6080f2 5415 tr->current_trace = t;
50512ab5 5416 tr->current_trace->enabled++;
9f029e83 5417 trace_branch_enable(tr);
bc0c38d1
SR
5418 out:
5419 mutex_unlock(&trace_types_lock);
5420
d9e54076
PZ
5421 return ret;
5422}
5423
5424static ssize_t
5425tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5426 size_t cnt, loff_t *ppos)
5427{
607e2ea1 5428 struct trace_array *tr = filp->private_data;
ee6c2c1b 5429 char buf[MAX_TRACER_SIZE+1];
d9e54076
PZ
5430 int i;
5431 size_t ret;
e6e7a65a
FW
5432 int err;
5433
5434 ret = cnt;
d9e54076 5435
ee6c2c1b
LZ
5436 if (cnt > MAX_TRACER_SIZE)
5437 cnt = MAX_TRACER_SIZE;
d9e54076 5438
4afe6495 5439 if (copy_from_user(buf, ubuf, cnt))
d9e54076
PZ
5440 return -EFAULT;
5441
5442 buf[cnt] = 0;
5443
5444 /* strip ending whitespace. */
5445 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5446 buf[i] = 0;
5447
607e2ea1 5448 err = tracing_set_tracer(tr, buf);
e6e7a65a
FW
5449 if (err)
5450 return err;
d9e54076 5451
cf8517cf 5452 *ppos += ret;
bc0c38d1 5453
c2931e05 5454 return ret;
bc0c38d1
SR
5455}
5456
5457static ssize_t
6508fa76
SF
5458tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5459 size_t cnt, loff_t *ppos)
bc0c38d1 5460{
bc0c38d1
SR
5461 char buf[64];
5462 int r;
5463
cffae437 5464 r = snprintf(buf, sizeof(buf), "%ld\n",
bc0c38d1 5465 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
cffae437
SR
5466 if (r > sizeof(buf))
5467 r = sizeof(buf);
4bf39a94 5468 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
5469}
5470
5471static ssize_t
6508fa76
SF
5472tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5473 size_t cnt, loff_t *ppos)
bc0c38d1 5474{
5e39841c 5475 unsigned long val;
c6caeeb1 5476 int ret;
bc0c38d1 5477
22fe9b54
PH
5478 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5479 if (ret)
c6caeeb1 5480 return ret;
bc0c38d1
SR
5481
5482 *ptr = val * 1000;
5483
5484 return cnt;
5485}
5486
6508fa76
SF
5487static ssize_t
5488tracing_thresh_read(struct file *filp, char __user *ubuf,
5489 size_t cnt, loff_t *ppos)
5490{
5491 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5492}
5493
5494static ssize_t
5495tracing_thresh_write(struct file *filp, const char __user *ubuf,
5496 size_t cnt, loff_t *ppos)
5497{
5498 struct trace_array *tr = filp->private_data;
5499 int ret;
5500
5501 mutex_lock(&trace_types_lock);
5502 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5503 if (ret < 0)
5504 goto out;
5505
5506 if (tr->current_trace->update_thresh) {
5507 ret = tr->current_trace->update_thresh(tr);
5508 if (ret < 0)
5509 goto out;
5510 }
5511
5512 ret = cnt;
5513out:
5514 mutex_unlock(&trace_types_lock);
5515
5516 return ret;
5517}
5518
f971cc9a 5519#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
e428abbb 5520
6508fa76
SF
5521static ssize_t
5522tracing_max_lat_read(struct file *filp, char __user *ubuf,
5523 size_t cnt, loff_t *ppos)
5524{
5525 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5526}
5527
5528static ssize_t
5529tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5530 size_t cnt, loff_t *ppos)
5531{
5532 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5533}
5534
e428abbb
CG
5535#endif
5536
b3806b43
SR
5537static int tracing_open_pipe(struct inode *inode, struct file *filp)
5538{
15544209 5539 struct trace_array *tr = inode->i_private;
b3806b43 5540 struct trace_iterator *iter;
b04cc6b1 5541 int ret = 0;
b3806b43
SR
5542
5543 if (tracing_disabled)
5544 return -ENODEV;
5545
7b85af63
SRRH
5546 if (trace_array_get(tr) < 0)
5547 return -ENODEV;
5548
b04cc6b1
FW
5549 mutex_lock(&trace_types_lock);
5550
b3806b43
SR
5551 /* create a buffer to store the information to pass to userspace */
5552 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
b04cc6b1
FW
5553 if (!iter) {
5554 ret = -ENOMEM;
f77d09a3 5555 __trace_array_put(tr);
b04cc6b1
FW
5556 goto out;
5557 }
b3806b43 5558
3a161d99 5559 trace_seq_init(&iter->seq);
d716ff71 5560 iter->trace = tr->current_trace;
d7350c3f 5561
4462344e 5562 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
b04cc6b1 5563 ret = -ENOMEM;
d7350c3f 5564 goto fail;
4462344e
RR
5565 }
5566
a309720c 5567 /* trace pipe does not show start of buffer */
4462344e 5568 cpumask_setall(iter->started);
a309720c 5569
983f938a 5570 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
112f38a7
SR
5571 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5572
8be0709f 5573 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
58e8eedf 5574 if (trace_clocks[tr->clock_id].in_ns)
8be0709f
DS
5575 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5576
15544209
ON
5577 iter->tr = tr;
5578 iter->trace_buffer = &tr->trace_buffer;
5579 iter->cpu_file = tracing_get_cpu(inode);
d7350c3f 5580 mutex_init(&iter->mutex);
b3806b43
SR
5581 filp->private_data = iter;
5582
107bad8b
SR
5583 if (iter->trace->pipe_open)
5584 iter->trace->pipe_open(iter);
107bad8b 5585
b444786f 5586 nonseekable_open(inode, filp);
cf6ab6d9
SRRH
5587
5588 tr->current_trace->ref++;
b04cc6b1
FW
5589out:
5590 mutex_unlock(&trace_types_lock);
5591 return ret;
d7350c3f
FW
5592
5593fail:
5594 kfree(iter->trace);
5595 kfree(iter);
7b85af63 5596 __trace_array_put(tr);
d7350c3f
FW
5597 mutex_unlock(&trace_types_lock);
5598 return ret;
b3806b43
SR
5599}
5600
5601static int tracing_release_pipe(struct inode *inode, struct file *file)
5602{
5603 struct trace_iterator *iter = file->private_data;
15544209 5604 struct trace_array *tr = inode->i_private;
b3806b43 5605
b04cc6b1
FW
5606 mutex_lock(&trace_types_lock);
5607
cf6ab6d9
SRRH
5608 tr->current_trace->ref--;
5609
29bf4a5e 5610 if (iter->trace->pipe_close)
c521efd1
SR
5611 iter->trace->pipe_close(iter);
5612
b04cc6b1
FW
5613 mutex_unlock(&trace_types_lock);
5614
4462344e 5615 free_cpumask_var(iter->started);
d7350c3f 5616 mutex_destroy(&iter->mutex);
b3806b43 5617 kfree(iter);
b3806b43 5618
7b85af63
SRRH
5619 trace_array_put(tr);
5620
b3806b43
SR
5621 return 0;
5622}
5623
2a2cc8f7 5624static unsigned int
cc60cdc9 5625trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
2a2cc8f7 5626{
983f938a
SRRH
5627 struct trace_array *tr = iter->tr;
5628
15693458
SRRH
5629 /* Iterators are static, they should be filled or empty */
5630 if (trace_buffer_iter(iter, iter->cpu_file))
5631 return POLLIN | POLLRDNORM;
2a2cc8f7 5632
983f938a 5633 if (tr->trace_flags & TRACE_ITER_BLOCK)
2a2cc8f7
SSP
5634 /*
5635 * Always select as readable when in blocking mode
5636 */
5637 return POLLIN | POLLRDNORM;
15693458 5638 else
12883efb 5639 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
15693458 5640 filp, poll_table);
2a2cc8f7 5641}
2a2cc8f7 5642
cc60cdc9
SR
5643static unsigned int
5644tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5645{
5646 struct trace_iterator *iter = filp->private_data;
5647
5648 return trace_poll(iter, filp, poll_table);
2a2cc8f7
SSP
5649}
5650
d716ff71 5651/* Must be called with iter->mutex held. */
ff98781b 5652static int tracing_wait_pipe(struct file *filp)
b3806b43
SR
5653{
5654 struct trace_iterator *iter = filp->private_data;
8b8b3683 5655 int ret;
b3806b43 5656
b3806b43 5657 while (trace_empty(iter)) {
2dc8f095 5658
107bad8b 5659 if ((filp->f_flags & O_NONBLOCK)) {
ff98781b 5660 return -EAGAIN;
107bad8b 5661 }
2dc8f095 5662
b3806b43 5663 /*
250bfd3d 5664 * We block until we read something and tracing is disabled.
b3806b43
SR
5665 * We still block if tracing is disabled, but we have never
5666 * read anything. This allows a user to cat this file, and
5667 * then enable tracing. But after we have read something,
5668 * we give an EOF when tracing is again disabled.
5669 *
5670 * iter->pos will be 0 if we haven't read anything.
5671 */
75df6e68 5672 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
b3806b43 5673 break;
f4874261
SRRH
5674
5675 mutex_unlock(&iter->mutex);
5676
e30f53aa 5677 ret = wait_on_pipe(iter, false);
f4874261
SRRH
5678
5679 mutex_lock(&iter->mutex);
5680
8b8b3683
SRRH
5681 if (ret)
5682 return ret;
b3806b43
SR
5683 }
5684
ff98781b
EGM
5685 return 1;
5686}
5687
5688/*
5689 * Consumer reader.
5690 */
5691static ssize_t
5692tracing_read_pipe(struct file *filp, char __user *ubuf,
5693 size_t cnt, loff_t *ppos)
5694{
5695 struct trace_iterator *iter = filp->private_data;
5696 ssize_t sret;
5697
d7350c3f
FW
5698 /*
5699 * Avoid more than one consumer on a single file descriptor
5700 * This is just a matter of traces coherency, the ring buffer itself
5701 * is protected.
5702 */
5703 mutex_lock(&iter->mutex);
1245800c
SRRH
5704
5705 /* return any leftover data */
5706 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5707 if (sret != -EBUSY)
5708 goto out;
5709
5710 trace_seq_init(&iter->seq);
5711
ff98781b
EGM
5712 if (iter->trace->read) {
5713 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5714 if (sret)
5715 goto out;
5716 }
5717
5718waitagain:
5719 sret = tracing_wait_pipe(filp);
5720 if (sret <= 0)
5721 goto out;
5722
b3806b43 5723 /* stop when tracing is finished */
ff98781b
EGM
5724 if (trace_empty(iter)) {
5725 sret = 0;
107bad8b 5726 goto out;
ff98781b 5727 }
b3806b43
SR
5728
5729 if (cnt >= PAGE_SIZE)
5730 cnt = PAGE_SIZE - 1;
5731
53d0aa77 5732 /* reset all but tr, trace, and overruns */
53d0aa77
SR
5733 memset(&iter->seq, 0,
5734 sizeof(struct trace_iterator) -
5735 offsetof(struct trace_iterator, seq));
ed5467da 5736 cpumask_clear(iter->started);
4823ed7e 5737 iter->pos = -1;
b3806b43 5738
4f535968 5739 trace_event_read_lock();
7e53bd42 5740 trace_access_lock(iter->cpu_file);
955b61e5 5741 while (trace_find_next_entry_inc(iter) != NULL) {
2c4f035f 5742 enum print_line_t ret;
5ac48378 5743 int save_len = iter->seq.seq.len;
088b1e42 5744
f9896bf3 5745 ret = print_trace_line(iter);
2c4f035f 5746 if (ret == TRACE_TYPE_PARTIAL_LINE) {
088b1e42 5747 /* don't print partial lines */
5ac48378 5748 iter->seq.seq.len = save_len;
b3806b43 5749 break;
088b1e42 5750 }
b91facc3
FW
5751 if (ret != TRACE_TYPE_NO_CONSUME)
5752 trace_consume(iter);
b3806b43 5753
5ac48378 5754 if (trace_seq_used(&iter->seq) >= cnt)
b3806b43 5755 break;
ee5e51f5
JO
5756
5757 /*
5758 * Setting the full flag means we reached the trace_seq buffer
5759 * size and we should leave by partial output condition above.
5760 * One of the trace_seq_* functions is not used properly.
5761 */
5762 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5763 iter->ent->type);
b3806b43 5764 }
7e53bd42 5765 trace_access_unlock(iter->cpu_file);
4f535968 5766 trace_event_read_unlock();
b3806b43 5767
b3806b43 5768 /* Now copy what we have to the user */
6c6c2796 5769 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5ac48378 5770 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
f9520750 5771 trace_seq_init(&iter->seq);
9ff4b974
PP
5772
5773 /*
25985edc 5774 * If there was nothing to send to user, in spite of consuming trace
9ff4b974
PP
5775 * entries, go back to wait for more entries.
5776 */
6c6c2796 5777 if (sret == -EBUSY)
9ff4b974 5778 goto waitagain;
b3806b43 5779
107bad8b 5780out:
d7350c3f 5781 mutex_unlock(&iter->mutex);
107bad8b 5782
6c6c2796 5783 return sret;
b3806b43
SR
5784}
5785
3c56819b
EGM
5786static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5787 unsigned int idx)
5788{
5789 __free_page(spd->pages[idx]);
5790}
5791
28dfef8f 5792static const struct pipe_buf_operations tracing_pipe_buf_ops = {
34cd4998 5793 .can_merge = 0,
34cd4998 5794 .confirm = generic_pipe_buf_confirm,
92fdd98c 5795 .release = generic_pipe_buf_release,
34cd4998
SR
5796 .steal = generic_pipe_buf_steal,
5797 .get = generic_pipe_buf_get,
3c56819b
EGM
5798};
5799
34cd4998 5800static size_t
fa7c7f6e 5801tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
34cd4998
SR
5802{
5803 size_t count;
74f06bb7 5804 int save_len;
34cd4998
SR
5805 int ret;
5806
5807 /* Seq buffer is page-sized, exactly what we need. */
5808 for (;;) {
74f06bb7 5809 save_len = iter->seq.seq.len;
34cd4998 5810 ret = print_trace_line(iter);
74f06bb7
SRRH
5811
5812 if (trace_seq_has_overflowed(&iter->seq)) {
5813 iter->seq.seq.len = save_len;
34cd4998
SR
5814 break;
5815 }
74f06bb7
SRRH
5816
5817 /*
5818 * This should not be hit, because it should only
5819 * be set if the iter->seq overflowed. But check it
5820 * anyway to be safe.
5821 */
34cd4998 5822 if (ret == TRACE_TYPE_PARTIAL_LINE) {
74f06bb7
SRRH
5823 iter->seq.seq.len = save_len;
5824 break;
5825 }
5826
5ac48378 5827 count = trace_seq_used(&iter->seq) - save_len;
74f06bb7
SRRH
5828 if (rem < count) {
5829 rem = 0;
5830 iter->seq.seq.len = save_len;
34cd4998
SR
5831 break;
5832 }
5833
74e7ff8c
LJ
5834 if (ret != TRACE_TYPE_NO_CONSUME)
5835 trace_consume(iter);
34cd4998 5836 rem -= count;
955b61e5 5837 if (!trace_find_next_entry_inc(iter)) {
34cd4998
SR
5838 rem = 0;
5839 iter->ent = NULL;
5840 break;
5841 }
5842 }
5843
5844 return rem;
5845}
5846
3c56819b
EGM
5847static ssize_t tracing_splice_read_pipe(struct file *filp,
5848 loff_t *ppos,
5849 struct pipe_inode_info *pipe,
5850 size_t len,
5851 unsigned int flags)
5852{
35f3d14d
JA
5853 struct page *pages_def[PIPE_DEF_BUFFERS];
5854 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3c56819b
EGM
5855 struct trace_iterator *iter = filp->private_data;
5856 struct splice_pipe_desc spd = {
35f3d14d
JA
5857 .pages = pages_def,
5858 .partial = partial_def,
34cd4998 5859 .nr_pages = 0, /* This gets updated below. */
047fe360 5860 .nr_pages_max = PIPE_DEF_BUFFERS,
34cd4998
SR
5861 .ops = &tracing_pipe_buf_ops,
5862 .spd_release = tracing_spd_release_pipe,
3c56819b
EGM
5863 };
5864 ssize_t ret;
34cd4998 5865 size_t rem;
3c56819b
EGM
5866 unsigned int i;
5867
35f3d14d
JA
5868 if (splice_grow_spd(pipe, &spd))
5869 return -ENOMEM;
5870
d7350c3f 5871 mutex_lock(&iter->mutex);
3c56819b
EGM
5872
5873 if (iter->trace->splice_read) {
5874 ret = iter->trace->splice_read(iter, filp,
5875 ppos, pipe, len, flags);
5876 if (ret)
34cd4998 5877 goto out_err;
3c56819b
EGM
5878 }
5879
5880 ret = tracing_wait_pipe(filp);
5881 if (ret <= 0)
34cd4998 5882 goto out_err;
3c56819b 5883
955b61e5 5884 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3c56819b 5885 ret = -EFAULT;
34cd4998 5886 goto out_err;
3c56819b
EGM
5887 }
5888
4f535968 5889 trace_event_read_lock();
7e53bd42 5890 trace_access_lock(iter->cpu_file);
4f535968 5891
3c56819b 5892 /* Fill as many pages as possible. */
a786c06d 5893 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
35f3d14d
JA
5894 spd.pages[i] = alloc_page(GFP_KERNEL);
5895 if (!spd.pages[i])
34cd4998 5896 break;
3c56819b 5897
fa7c7f6e 5898 rem = tracing_fill_pipe_page(rem, iter);
3c56819b
EGM
5899
5900 /* Copy the data into the page, so we can start over. */
5901 ret = trace_seq_to_buffer(&iter->seq,
35f3d14d 5902 page_address(spd.pages[i]),
5ac48378 5903 trace_seq_used(&iter->seq));
3c56819b 5904 if (ret < 0) {
35f3d14d 5905 __free_page(spd.pages[i]);
3c56819b
EGM
5906 break;
5907 }
35f3d14d 5908 spd.partial[i].offset = 0;
5ac48378 5909 spd.partial[i].len = trace_seq_used(&iter->seq);
3c56819b 5910
f9520750 5911 trace_seq_init(&iter->seq);
3c56819b
EGM
5912 }
5913
7e53bd42 5914 trace_access_unlock(iter->cpu_file);
4f535968 5915 trace_event_read_unlock();
d7350c3f 5916 mutex_unlock(&iter->mutex);
3c56819b
EGM
5917
5918 spd.nr_pages = i;
5919
a29054d9
SRRH
5920 if (i)
5921 ret = splice_to_pipe(pipe, &spd);
5922 else
5923 ret = 0;
35f3d14d 5924out:
047fe360 5925 splice_shrink_spd(&spd);
35f3d14d 5926 return ret;
3c56819b 5927
34cd4998 5928out_err:
d7350c3f 5929 mutex_unlock(&iter->mutex);
35f3d14d 5930 goto out;
3c56819b
EGM
5931}
5932
a98a3c3f
SR
5933static ssize_t
5934tracing_entries_read(struct file *filp, char __user *ubuf,
5935 size_t cnt, loff_t *ppos)
5936{
0bc392ee
ON
5937 struct inode *inode = file_inode(filp);
5938 struct trace_array *tr = inode->i_private;
5939 int cpu = tracing_get_cpu(inode);
438ced17
VN
5940 char buf[64];
5941 int r = 0;
5942 ssize_t ret;
a98a3c3f 5943
db526ca3 5944 mutex_lock(&trace_types_lock);
438ced17 5945
0bc392ee 5946 if (cpu == RING_BUFFER_ALL_CPUS) {
438ced17
VN
5947 int cpu, buf_size_same;
5948 unsigned long size;
5949
5950 size = 0;
5951 buf_size_same = 1;
5952 /* check if all cpu sizes are same */
5953 for_each_tracing_cpu(cpu) {
5954 /* fill in the size from first enabled cpu */
5955 if (size == 0)
12883efb
SRRH
5956 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5957 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
438ced17
VN
5958 buf_size_same = 0;
5959 break;
5960 }
5961 }
5962
5963 if (buf_size_same) {
5964 if (!ring_buffer_expanded)
5965 r = sprintf(buf, "%lu (expanded: %lu)\n",
5966 size >> 10,
5967 trace_buf_size >> 10);
5968 else
5969 r = sprintf(buf, "%lu\n", size >> 10);
5970 } else
5971 r = sprintf(buf, "X\n");
5972 } else
0bc392ee 5973 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
438ced17 5974
db526ca3
SR
5975 mutex_unlock(&trace_types_lock);
5976
438ced17
VN
5977 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5978 return ret;
a98a3c3f
SR
5979}
5980
5981static ssize_t
5982tracing_entries_write(struct file *filp, const char __user *ubuf,
5983 size_t cnt, loff_t *ppos)
5984{
0bc392ee
ON
5985 struct inode *inode = file_inode(filp);
5986 struct trace_array *tr = inode->i_private;
a98a3c3f 5987 unsigned long val;
4f271a2a 5988 int ret;
a98a3c3f 5989
22fe9b54
PH
5990 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5991 if (ret)
c6caeeb1 5992 return ret;
a98a3c3f
SR
5993
5994 /* must have at least 1 entry */
5995 if (!val)
5996 return -EINVAL;
5997
1696b2b0
SR
5998 /* value is in KB */
5999 val <<= 10;
0bc392ee 6000 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4f271a2a
VN
6001 if (ret < 0)
6002 return ret;
a98a3c3f 6003
cf8517cf 6004 *ppos += cnt;
a98a3c3f 6005
4f271a2a
VN
6006 return cnt;
6007}
bf5e6519 6008
f81ab074
VN
6009static ssize_t
6010tracing_total_entries_read(struct file *filp, char __user *ubuf,
6011 size_t cnt, loff_t *ppos)
6012{
6013 struct trace_array *tr = filp->private_data;
6014 char buf[64];
6015 int r, cpu;
6016 unsigned long size = 0, expanded_size = 0;
6017
6018 mutex_lock(&trace_types_lock);
6019 for_each_tracing_cpu(cpu) {
12883efb 6020 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
f81ab074
VN
6021 if (!ring_buffer_expanded)
6022 expanded_size += trace_buf_size >> 10;
6023 }
6024 if (ring_buffer_expanded)
6025 r = sprintf(buf, "%lu\n", size);
6026 else
6027 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6028 mutex_unlock(&trace_types_lock);
6029
6030 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6031}
6032
4f271a2a
VN
6033static ssize_t
6034tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6035 size_t cnt, loff_t *ppos)
6036{
6037 /*
6038 * There is no need to read what the user has written, this function
6039 * is just to make sure that there is no error when "echo" is used
6040 */
6041
6042 *ppos += cnt;
a98a3c3f
SR
6043
6044 return cnt;
6045}
6046
4f271a2a
VN
6047static int
6048tracing_free_buffer_release(struct inode *inode, struct file *filp)
6049{
2b6080f2
SR
6050 struct trace_array *tr = inode->i_private;
6051
cf30cf67 6052 /* disable tracing ? */
983f938a 6053 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
711e1243 6054 tracer_tracing_off(tr);
4f271a2a 6055 /* resize the ring buffer to 0 */
2b6080f2 6056 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4f271a2a 6057
7b85af63
SRRH
6058 trace_array_put(tr);
6059
4f271a2a
VN
6060 return 0;
6061}
6062
5bf9a1ee
PP
6063static ssize_t
6064tracing_mark_write(struct file *filp, const char __user *ubuf,
6065 size_t cnt, loff_t *fpos)
6066{
2d71619c 6067 struct trace_array *tr = filp->private_data;
d696b58c
SR
6068 struct ring_buffer_event *event;
6069 struct ring_buffer *buffer;
6070 struct print_entry *entry;
6071 unsigned long irq_flags;
656c7f0d 6072 const char faulted[] = "<faulted>";
d696b58c 6073 ssize_t written;
d696b58c
SR
6074 int size;
6075 int len;
fa32e855 6076
656c7f0d
SRRH
6077/* Used in tracing_mark_raw_write() as well */
6078#define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
5bf9a1ee 6079
c76f0694 6080 if (tracing_disabled)
5bf9a1ee
PP
6081 return -EINVAL;
6082
983f938a 6083 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5224c3a3
MSB
6084 return -EINVAL;
6085
5bf9a1ee
PP
6086 if (cnt > TRACE_BUF_SIZE)
6087 cnt = TRACE_BUF_SIZE;
6088
d696b58c 6089 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5bf9a1ee 6090
d696b58c 6091 local_save_flags(irq_flags);
656c7f0d 6092 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
d696b58c 6093
656c7f0d
SRRH
6094 /* If less than "<faulted>", then make sure we can still add that */
6095 if (cnt < FAULTED_SIZE)
6096 size += FAULTED_SIZE - cnt;
d696b58c 6097
2d71619c 6098 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
6099 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6100 irq_flags, preempt_count());
656c7f0d 6101 if (unlikely(!event))
d696b58c 6102 /* Ring buffer disabled, return as if not open for write */
656c7f0d 6103 return -EBADF;
d696b58c
SR
6104
6105 entry = ring_buffer_event_data(event);
6106 entry->ip = _THIS_IP_;
6107
656c7f0d
SRRH
6108 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6109 if (len) {
6110 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6111 cnt = FAULTED_SIZE;
6112 written = -EFAULT;
c13d2f7c 6113 } else
656c7f0d
SRRH
6114 written = cnt;
6115 len = cnt;
5bf9a1ee 6116
d696b58c
SR
6117 if (entry->buf[cnt - 1] != '\n') {
6118 entry->buf[cnt] = '\n';
6119 entry->buf[cnt + 1] = '\0';
6120 } else
6121 entry->buf[cnt] = '\0';
6122
7ffbd48d 6123 __buffer_unlock_commit(buffer, event);
5bf9a1ee 6124
656c7f0d
SRRH
6125 if (written > 0)
6126 *fpos += written;
5bf9a1ee 6127
fa32e855
SR
6128 return written;
6129}
6130
6131/* Limit it for now to 3K (including tag) */
6132#define RAW_DATA_MAX_SIZE (1024*3)
6133
6134static ssize_t
6135tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6136 size_t cnt, loff_t *fpos)
6137{
6138 struct trace_array *tr = filp->private_data;
6139 struct ring_buffer_event *event;
6140 struct ring_buffer *buffer;
6141 struct raw_data_entry *entry;
656c7f0d 6142 const char faulted[] = "<faulted>";
fa32e855 6143 unsigned long irq_flags;
fa32e855 6144 ssize_t written;
fa32e855
SR
6145 int size;
6146 int len;
6147
656c7f0d
SRRH
6148#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6149
fa32e855
SR
6150 if (tracing_disabled)
6151 return -EINVAL;
6152
6153 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6154 return -EINVAL;
6155
6156 /* The marker must at least have a tag id */
6157 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6158 return -EINVAL;
6159
6160 if (cnt > TRACE_BUF_SIZE)
6161 cnt = TRACE_BUF_SIZE;
6162
6163 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6164
fa32e855
SR
6165 local_save_flags(irq_flags);
6166 size = sizeof(*entry) + cnt;
656c7f0d
SRRH
6167 if (cnt < FAULT_SIZE_ID)
6168 size += FAULT_SIZE_ID - cnt;
6169
fa32e855 6170 buffer = tr->trace_buffer.buffer;
3e9a8aad
SRRH
6171 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6172 irq_flags, preempt_count());
656c7f0d 6173 if (!event)
fa32e855 6174 /* Ring buffer disabled, return as if not open for write */
656c7f0d 6175 return -EBADF;
fa32e855
SR
6176
6177 entry = ring_buffer_event_data(event);
6178
656c7f0d
SRRH
6179 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6180 if (len) {
6181 entry->id = -1;
6182 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6183 written = -EFAULT;
fa32e855 6184 } else
656c7f0d 6185 written = cnt;
fa32e855
SR
6186
6187 __buffer_unlock_commit(buffer, event);
6188
656c7f0d
SRRH
6189 if (written > 0)
6190 *fpos += written;
1aa54bca
MS
6191
6192 return written;
5bf9a1ee
PP
6193}
6194
13f16d20 6195static int tracing_clock_show(struct seq_file *m, void *v)
5079f326 6196{
2b6080f2 6197 struct trace_array *tr = m->private;
5079f326
Z
6198 int i;
6199
6200 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
13f16d20 6201 seq_printf(m,
5079f326 6202 "%s%s%s%s", i ? " " : "",
2b6080f2
SR
6203 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6204 i == tr->clock_id ? "]" : "");
13f16d20 6205 seq_putc(m, '\n');
5079f326 6206
13f16d20 6207 return 0;
5079f326
Z
6208}
6209
e1e232ca 6210static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5079f326 6211{
5079f326
Z
6212 int i;
6213
5079f326
Z
6214 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6215 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6216 break;
6217 }
6218 if (i == ARRAY_SIZE(trace_clocks))
6219 return -EINVAL;
6220
5079f326
Z
6221 mutex_lock(&trace_types_lock);
6222
2b6080f2
SR
6223 tr->clock_id = i;
6224
12883efb 6225 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5079f326 6226
60303ed3
DS
6227 /*
6228 * New clock may not be consistent with the previous clock.
6229 * Reset the buffer so that it doesn't have incomparable timestamps.
6230 */
9457158b 6231 tracing_reset_online_cpus(&tr->trace_buffer);
12883efb
SRRH
6232
6233#ifdef CONFIG_TRACER_MAX_TRACE
170b3b10 6234 if (tr->max_buffer.buffer)
12883efb 6235 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
9457158b 6236 tracing_reset_online_cpus(&tr->max_buffer);
12883efb 6237#endif
60303ed3 6238
5079f326
Z
6239 mutex_unlock(&trace_types_lock);
6240
e1e232ca
SR
6241 return 0;
6242}
6243
6244static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6245 size_t cnt, loff_t *fpos)
6246{
6247 struct seq_file *m = filp->private_data;
6248 struct trace_array *tr = m->private;
6249 char buf[64];
6250 const char *clockstr;
6251 int ret;
6252
6253 if (cnt >= sizeof(buf))
6254 return -EINVAL;
6255
4afe6495 6256 if (copy_from_user(buf, ubuf, cnt))
e1e232ca
SR
6257 return -EFAULT;
6258
6259 buf[cnt] = 0;
6260
6261 clockstr = strstrip(buf);
6262
6263 ret = tracing_set_clock(tr, clockstr);
6264 if (ret)
6265 return ret;
6266
5079f326
Z
6267 *fpos += cnt;
6268
6269 return cnt;
6270}
6271
13f16d20
LZ
6272static int tracing_clock_open(struct inode *inode, struct file *file)
6273{
7b85af63
SRRH
6274 struct trace_array *tr = inode->i_private;
6275 int ret;
6276
13f16d20
LZ
6277 if (tracing_disabled)
6278 return -ENODEV;
2b6080f2 6279
7b85af63
SRRH
6280 if (trace_array_get(tr))
6281 return -ENODEV;
6282
6283 ret = single_open(file, tracing_clock_show, inode->i_private);
6284 if (ret < 0)
6285 trace_array_put(tr);
6286
6287 return ret;
13f16d20
LZ
6288}
6289
6de58e62
SRRH
6290struct ftrace_buffer_info {
6291 struct trace_iterator iter;
6292 void *spare;
73a757e6 6293 unsigned int spare_cpu;
6de58e62
SRRH
6294 unsigned int read;
6295};
6296
debdd57f
HT
6297#ifdef CONFIG_TRACER_SNAPSHOT
6298static int tracing_snapshot_open(struct inode *inode, struct file *file)
6299{
6484c71c 6300 struct trace_array *tr = inode->i_private;
debdd57f 6301 struct trace_iterator *iter;
2b6080f2 6302 struct seq_file *m;
debdd57f
HT
6303 int ret = 0;
6304
ff451961
SRRH
6305 if (trace_array_get(tr) < 0)
6306 return -ENODEV;
6307
debdd57f 6308 if (file->f_mode & FMODE_READ) {
6484c71c 6309 iter = __tracing_open(inode, file, true);
debdd57f
HT
6310 if (IS_ERR(iter))
6311 ret = PTR_ERR(iter);
2b6080f2
SR
6312 } else {
6313 /* Writes still need the seq_file to hold the private data */
f77d09a3 6314 ret = -ENOMEM;
2b6080f2
SR
6315 m = kzalloc(sizeof(*m), GFP_KERNEL);
6316 if (!m)
f77d09a3 6317 goto out;
2b6080f2
SR
6318 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6319 if (!iter) {
6320 kfree(m);
f77d09a3 6321 goto out;
2b6080f2 6322 }
f77d09a3
AL
6323 ret = 0;
6324
ff451961 6325 iter->tr = tr;
6484c71c
ON
6326 iter->trace_buffer = &tr->max_buffer;
6327 iter->cpu_file = tracing_get_cpu(inode);
2b6080f2
SR
6328 m->private = iter;
6329 file->private_data = m;
debdd57f 6330 }
f77d09a3 6331out:
ff451961
SRRH
6332 if (ret < 0)
6333 trace_array_put(tr);
6334
debdd57f
HT
6335 return ret;
6336}
6337
6338static ssize_t
6339tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6340 loff_t *ppos)
6341{
2b6080f2
SR
6342 struct seq_file *m = filp->private_data;
6343 struct trace_iterator *iter = m->private;
6344 struct trace_array *tr = iter->tr;
debdd57f
HT
6345 unsigned long val;
6346 int ret;
6347
6348 ret = tracing_update_buffers();
6349 if (ret < 0)
6350 return ret;
6351
6352 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6353 if (ret)
6354 return ret;
6355
6356 mutex_lock(&trace_types_lock);
6357
2b6080f2 6358 if (tr->current_trace->use_max_tr) {
debdd57f
HT
6359 ret = -EBUSY;
6360 goto out;
6361 }
6362
6363 switch (val) {
6364 case 0:
f1affcaa
SRRH
6365 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6366 ret = -EINVAL;
6367 break;
debdd57f 6368 }
3209cff4
SRRH
6369 if (tr->allocated_snapshot)
6370 free_snapshot(tr);
debdd57f
HT
6371 break;
6372 case 1:
f1affcaa
SRRH
6373/* Only allow per-cpu swap if the ring buffer supports it */
6374#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6375 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6376 ret = -EINVAL;
6377 break;
6378 }
6379#endif
45ad21ca 6380 if (!tr->allocated_snapshot) {
3209cff4 6381 ret = alloc_snapshot(tr);
debdd57f
HT
6382 if (ret < 0)
6383 break;
debdd57f 6384 }
debdd57f
HT
6385 local_irq_disable();
6386 /* Now, we're going to swap */
f1affcaa 6387 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
ce9bae55 6388 update_max_tr(tr, current, smp_processor_id());
f1affcaa 6389 else
ce9bae55 6390 update_max_tr_single(tr, current, iter->cpu_file);
debdd57f
HT
6391 local_irq_enable();
6392 break;
6393 default:
45ad21ca 6394 if (tr->allocated_snapshot) {
f1affcaa
SRRH
6395 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6396 tracing_reset_online_cpus(&tr->max_buffer);
6397 else
6398 tracing_reset(&tr->max_buffer, iter->cpu_file);
6399 }
debdd57f
HT
6400 break;
6401 }
6402
6403 if (ret >= 0) {
6404 *ppos += cnt;
6405 ret = cnt;
6406 }
6407out:
6408 mutex_unlock(&trace_types_lock);
6409 return ret;
6410}
2b6080f2
SR
6411
6412static int tracing_snapshot_release(struct inode *inode, struct file *file)
6413{
6414 struct seq_file *m = file->private_data;
ff451961
SRRH
6415 int ret;
6416
6417 ret = tracing_release(inode, file);
2b6080f2
SR
6418
6419 if (file->f_mode & FMODE_READ)
ff451961 6420 return ret;
2b6080f2
SR
6421
6422 /* If write only, the seq_file is just a stub */
6423 if (m)
6424 kfree(m->private);
6425 kfree(m);
6426
6427 return 0;
6428}
6429
6de58e62
SRRH
6430static int tracing_buffers_open(struct inode *inode, struct file *filp);
6431static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6432 size_t count, loff_t *ppos);
6433static int tracing_buffers_release(struct inode *inode, struct file *file);
6434static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6435 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6436
6437static int snapshot_raw_open(struct inode *inode, struct file *filp)
6438{
6439 struct ftrace_buffer_info *info;
6440 int ret;
6441
6442 ret = tracing_buffers_open(inode, filp);
6443 if (ret < 0)
6444 return ret;
6445
6446 info = filp->private_data;
6447
6448 if (info->iter.trace->use_max_tr) {
6449 tracing_buffers_release(inode, filp);
6450 return -EBUSY;
6451 }
6452
6453 info->iter.snapshot = true;
6454 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6455
6456 return ret;
6457}
6458
debdd57f
HT
6459#endif /* CONFIG_TRACER_SNAPSHOT */
6460
6461
6508fa76
SF
6462static const struct file_operations tracing_thresh_fops = {
6463 .open = tracing_open_generic,
6464 .read = tracing_thresh_read,
6465 .write = tracing_thresh_write,
6466 .llseek = generic_file_llseek,
6467};
6468
f971cc9a 6469#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5e2336a0 6470static const struct file_operations tracing_max_lat_fops = {
4bf39a94
IM
6471 .open = tracing_open_generic,
6472 .read = tracing_max_lat_read,
6473 .write = tracing_max_lat_write,
b444786f 6474 .llseek = generic_file_llseek,
bc0c38d1 6475};
e428abbb 6476#endif
bc0c38d1 6477
5e2336a0 6478static const struct file_operations set_tracer_fops = {
4bf39a94
IM
6479 .open = tracing_open_generic,
6480 .read = tracing_set_trace_read,
6481 .write = tracing_set_trace_write,
b444786f 6482 .llseek = generic_file_llseek,
bc0c38d1
SR
6483};
6484
5e2336a0 6485static const struct file_operations tracing_pipe_fops = {
4bf39a94 6486 .open = tracing_open_pipe,
2a2cc8f7 6487 .poll = tracing_poll_pipe,
4bf39a94 6488 .read = tracing_read_pipe,
3c56819b 6489 .splice_read = tracing_splice_read_pipe,
4bf39a94 6490 .release = tracing_release_pipe,
b444786f 6491 .llseek = no_llseek,
b3806b43
SR
6492};
6493
5e2336a0 6494static const struct file_operations tracing_entries_fops = {
0bc392ee 6495 .open = tracing_open_generic_tr,
a98a3c3f
SR
6496 .read = tracing_entries_read,
6497 .write = tracing_entries_write,
b444786f 6498 .llseek = generic_file_llseek,
0bc392ee 6499 .release = tracing_release_generic_tr,
a98a3c3f
SR
6500};
6501
f81ab074 6502static const struct file_operations tracing_total_entries_fops = {
7b85af63 6503 .open = tracing_open_generic_tr,
f81ab074
VN
6504 .read = tracing_total_entries_read,
6505 .llseek = generic_file_llseek,
7b85af63 6506 .release = tracing_release_generic_tr,
f81ab074
VN
6507};
6508
4f271a2a 6509static const struct file_operations tracing_free_buffer_fops = {
7b85af63 6510 .open = tracing_open_generic_tr,
4f271a2a
VN
6511 .write = tracing_free_buffer_write,
6512 .release = tracing_free_buffer_release,
6513};
6514
5e2336a0 6515static const struct file_operations tracing_mark_fops = {
7b85af63 6516 .open = tracing_open_generic_tr,
5bf9a1ee 6517 .write = tracing_mark_write,
b444786f 6518 .llseek = generic_file_llseek,
7b85af63 6519 .release = tracing_release_generic_tr,
5bf9a1ee
PP
6520};
6521
fa32e855
SR
6522static const struct file_operations tracing_mark_raw_fops = {
6523 .open = tracing_open_generic_tr,
6524 .write = tracing_mark_raw_write,
6525 .llseek = generic_file_llseek,
6526 .release = tracing_release_generic_tr,
6527};
6528
5079f326 6529static const struct file_operations trace_clock_fops = {
13f16d20
LZ
6530 .open = tracing_clock_open,
6531 .read = seq_read,
6532 .llseek = seq_lseek,
7b85af63 6533 .release = tracing_single_release_tr,
5079f326
Z
6534 .write = tracing_clock_write,
6535};
6536
debdd57f
HT
6537#ifdef CONFIG_TRACER_SNAPSHOT
6538static const struct file_operations snapshot_fops = {
6539 .open = tracing_snapshot_open,
6540 .read = seq_read,
6541 .write = tracing_snapshot_write,
098c879e 6542 .llseek = tracing_lseek,
2b6080f2 6543 .release = tracing_snapshot_release,
debdd57f 6544};
debdd57f 6545
6de58e62
SRRH
6546static const struct file_operations snapshot_raw_fops = {
6547 .open = snapshot_raw_open,
6548 .read = tracing_buffers_read,
6549 .release = tracing_buffers_release,
6550 .splice_read = tracing_buffers_splice_read,
6551 .llseek = no_llseek,
2cadf913
SR
6552};
6553
6de58e62
SRRH
6554#endif /* CONFIG_TRACER_SNAPSHOT */
6555
2cadf913
SR
6556static int tracing_buffers_open(struct inode *inode, struct file *filp)
6557{
46ef2be0 6558 struct trace_array *tr = inode->i_private;
2cadf913 6559 struct ftrace_buffer_info *info;
7b85af63 6560 int ret;
2cadf913
SR
6561
6562 if (tracing_disabled)
6563 return -ENODEV;
6564
7b85af63
SRRH
6565 if (trace_array_get(tr) < 0)
6566 return -ENODEV;
6567
2cadf913 6568 info = kzalloc(sizeof(*info), GFP_KERNEL);
7b85af63
SRRH
6569 if (!info) {
6570 trace_array_put(tr);
2cadf913 6571 return -ENOMEM;
7b85af63 6572 }
2cadf913 6573
a695cb58
SRRH
6574 mutex_lock(&trace_types_lock);
6575
cc60cdc9 6576 info->iter.tr = tr;
46ef2be0 6577 info->iter.cpu_file = tracing_get_cpu(inode);
b627344f 6578 info->iter.trace = tr->current_trace;
12883efb 6579 info->iter.trace_buffer = &tr->trace_buffer;
cc60cdc9 6580 info->spare = NULL;
2cadf913 6581 /* Force reading ring buffer for first read */
cc60cdc9 6582 info->read = (unsigned int)-1;
2cadf913
SR
6583
6584 filp->private_data = info;
6585
cf6ab6d9
SRRH
6586 tr->current_trace->ref++;
6587
a695cb58
SRRH
6588 mutex_unlock(&trace_types_lock);
6589
7b85af63
SRRH
6590 ret = nonseekable_open(inode, filp);
6591 if (ret < 0)
6592 trace_array_put(tr);
6593
6594 return ret;
2cadf913
SR
6595}
6596
cc60cdc9
SR
6597static unsigned int
6598tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6599{
6600 struct ftrace_buffer_info *info = filp->private_data;
6601 struct trace_iterator *iter = &info->iter;
6602
6603 return trace_poll(iter, filp, poll_table);
6604}
6605
2cadf913
SR
6606static ssize_t
6607tracing_buffers_read(struct file *filp, char __user *ubuf,
6608 size_t count, loff_t *ppos)
6609{
6610 struct ftrace_buffer_info *info = filp->private_data;
cc60cdc9 6611 struct trace_iterator *iter = &info->iter;
a7e52ad7 6612 ssize_t ret = 0;
6de58e62 6613 ssize_t size;
2cadf913 6614
2dc5d12b
SR
6615 if (!count)
6616 return 0;
6617
6de58e62 6618#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
6619 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6620 return -EBUSY;
6de58e62
SRRH
6621#endif
6622
73a757e6 6623 if (!info->spare) {
12883efb
SRRH
6624 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6625 iter->cpu_file);
a7e52ad7
SRV
6626 if (IS_ERR(info->spare)) {
6627 ret = PTR_ERR(info->spare);
6628 info->spare = NULL;
6629 } else {
6630 info->spare_cpu = iter->cpu_file;
6631 }
73a757e6 6632 }
ddd538f3 6633 if (!info->spare)
a7e52ad7 6634 return ret;
ddd538f3 6635
2cadf913
SR
6636 /* Do we have previous read data to read? */
6637 if (info->read < PAGE_SIZE)
6638 goto read;
6639
b627344f 6640 again:
cc60cdc9 6641 trace_access_lock(iter->cpu_file);
12883efb 6642 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
2cadf913
SR
6643 &info->spare,
6644 count,
cc60cdc9
SR
6645 iter->cpu_file, 0);
6646 trace_access_unlock(iter->cpu_file);
2cadf913 6647
b627344f
SR
6648 if (ret < 0) {
6649 if (trace_empty(iter)) {
d716ff71
SRRH
6650 if ((filp->f_flags & O_NONBLOCK))
6651 return -EAGAIN;
6652
e30f53aa 6653 ret = wait_on_pipe(iter, false);
d716ff71
SRRH
6654 if (ret)
6655 return ret;
6656
b627344f
SR
6657 goto again;
6658 }
d716ff71 6659 return 0;
b627344f 6660 }
436fc280 6661
436fc280 6662 info->read = 0;
b627344f 6663 read:
2cadf913
SR
6664 size = PAGE_SIZE - info->read;
6665 if (size > count)
6666 size = count;
6667
6668 ret = copy_to_user(ubuf, info->spare + info->read, size);
d716ff71
SRRH
6669 if (ret == size)
6670 return -EFAULT;
6671
2dc5d12b
SR
6672 size -= ret;
6673
2cadf913
SR
6674 *ppos += size;
6675 info->read += size;
6676
6677 return size;
6678}
6679
6680static int tracing_buffers_release(struct inode *inode, struct file *file)
6681{
6682 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 6683 struct trace_iterator *iter = &info->iter;
2cadf913 6684
a695cb58
SRRH
6685 mutex_lock(&trace_types_lock);
6686
cf6ab6d9
SRRH
6687 iter->tr->current_trace->ref--;
6688
ff451961 6689 __trace_array_put(iter->tr);
2cadf913 6690
ddd538f3 6691 if (info->spare)
73a757e6
SRV
6692 ring_buffer_free_read_page(iter->trace_buffer->buffer,
6693 info->spare_cpu, info->spare);
2cadf913
SR
6694 kfree(info);
6695
a695cb58
SRRH
6696 mutex_unlock(&trace_types_lock);
6697
2cadf913
SR
6698 return 0;
6699}
6700
6701struct buffer_ref {
6702 struct ring_buffer *buffer;
6703 void *page;
73a757e6 6704 int cpu;
2cadf913
SR
6705 int ref;
6706};
6707
6708static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6709 struct pipe_buffer *buf)
6710{
6711 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6712
6713 if (--ref->ref)
6714 return;
6715
73a757e6 6716 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
2cadf913
SR
6717 kfree(ref);
6718 buf->private = 0;
6719}
6720
2cadf913
SR
6721static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6722 struct pipe_buffer *buf)
6723{
6724 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6725
6726 ref->ref++;
6727}
6728
6729/* Pipe buffer operations for a buffer. */
28dfef8f 6730static const struct pipe_buf_operations buffer_pipe_buf_ops = {
2cadf913 6731 .can_merge = 0,
2cadf913
SR
6732 .confirm = generic_pipe_buf_confirm,
6733 .release = buffer_pipe_buf_release,
d55cb6cf 6734 .steal = generic_pipe_buf_steal,
2cadf913
SR
6735 .get = buffer_pipe_buf_get,
6736};
6737
6738/*
6739 * Callback from splice_to_pipe(), if we need to release some pages
6740 * at the end of the spd in case we error'ed out in filling the pipe.
6741 */
6742static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6743{
6744 struct buffer_ref *ref =
6745 (struct buffer_ref *)spd->partial[i].private;
6746
6747 if (--ref->ref)
6748 return;
6749
73a757e6 6750 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
2cadf913
SR
6751 kfree(ref);
6752 spd->partial[i].private = 0;
6753}
6754
6755static ssize_t
6756tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6757 struct pipe_inode_info *pipe, size_t len,
6758 unsigned int flags)
6759{
6760 struct ftrace_buffer_info *info = file->private_data;
cc60cdc9 6761 struct trace_iterator *iter = &info->iter;
35f3d14d
JA
6762 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6763 struct page *pages_def[PIPE_DEF_BUFFERS];
2cadf913 6764 struct splice_pipe_desc spd = {
35f3d14d
JA
6765 .pages = pages_def,
6766 .partial = partial_def,
047fe360 6767 .nr_pages_max = PIPE_DEF_BUFFERS,
2cadf913
SR
6768 .ops = &buffer_pipe_buf_ops,
6769 .spd_release = buffer_spd_release,
6770 };
6771 struct buffer_ref *ref;
6b7e633f 6772 int entries, i;
07906da7 6773 ssize_t ret = 0;
2cadf913 6774
6de58e62 6775#ifdef CONFIG_TRACER_MAX_TRACE
d716ff71
SRRH
6776 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6777 return -EBUSY;
6de58e62
SRRH
6778#endif
6779
d716ff71
SRRH
6780 if (*ppos & (PAGE_SIZE - 1))
6781 return -EINVAL;
93cfb3c9
LJ
6782
6783 if (len & (PAGE_SIZE - 1)) {
d716ff71
SRRH
6784 if (len < PAGE_SIZE)
6785 return -EINVAL;
93cfb3c9
LJ
6786 len &= PAGE_MASK;
6787 }
6788
1ae2293d
AV
6789 if (splice_grow_spd(pipe, &spd))
6790 return -ENOMEM;
6791
cc60cdc9
SR
6792 again:
6793 trace_access_lock(iter->cpu_file);
12883efb 6794 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
93459c6c 6795
a786c06d 6796 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
2cadf913
SR
6797 struct page *page;
6798 int r;
6799
6800 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
07906da7
RV
6801 if (!ref) {
6802 ret = -ENOMEM;
2cadf913 6803 break;
07906da7 6804 }
2cadf913 6805
7267fa68 6806 ref->ref = 1;
12883efb 6807 ref->buffer = iter->trace_buffer->buffer;
cc60cdc9 6808 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
a7e52ad7
SRV
6809 if (IS_ERR(ref->page)) {
6810 ret = PTR_ERR(ref->page);
6811 ref->page = NULL;
2cadf913
SR
6812 kfree(ref);
6813 break;
6814 }
73a757e6 6815 ref->cpu = iter->cpu_file;
2cadf913
SR
6816
6817 r = ring_buffer_read_page(ref->buffer, &ref->page,
cc60cdc9 6818 len, iter->cpu_file, 1);
2cadf913 6819 if (r < 0) {
73a757e6
SRV
6820 ring_buffer_free_read_page(ref->buffer, ref->cpu,
6821 ref->page);
2cadf913
SR
6822 kfree(ref);
6823 break;
6824 }
6825
2cadf913
SR
6826 page = virt_to_page(ref->page);
6827
6828 spd.pages[i] = page;
6829 spd.partial[i].len = PAGE_SIZE;
6830 spd.partial[i].offset = 0;
6831 spd.partial[i].private = (unsigned long)ref;
6832 spd.nr_pages++;
93cfb3c9 6833 *ppos += PAGE_SIZE;
93459c6c 6834
12883efb 6835 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
2cadf913
SR
6836 }
6837
cc60cdc9 6838 trace_access_unlock(iter->cpu_file);
2cadf913
SR
6839 spd.nr_pages = i;
6840
6841 /* did we read anything? */
6842 if (!spd.nr_pages) {
07906da7 6843 if (ret)
1ae2293d 6844 goto out;
d716ff71 6845
1ae2293d 6846 ret = -EAGAIN;
d716ff71 6847 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
1ae2293d 6848 goto out;
07906da7 6849
e30f53aa 6850 ret = wait_on_pipe(iter, true);
8b8b3683 6851 if (ret)
1ae2293d 6852 goto out;
e30f53aa 6853
cc60cdc9 6854 goto again;
2cadf913
SR
6855 }
6856
6857 ret = splice_to_pipe(pipe, &spd);
1ae2293d 6858out:
047fe360 6859 splice_shrink_spd(&spd);
6de58e62 6860
2cadf913
SR
6861 return ret;
6862}
6863
6864static const struct file_operations tracing_buffers_fops = {
6865 .open = tracing_buffers_open,
6866 .read = tracing_buffers_read,
cc60cdc9 6867 .poll = tracing_buffers_poll,
2cadf913
SR
6868 .release = tracing_buffers_release,
6869 .splice_read = tracing_buffers_splice_read,
6870 .llseek = no_llseek,
6871};
6872
c8d77183
SR
6873static ssize_t
6874tracing_stats_read(struct file *filp, char __user *ubuf,
6875 size_t count, loff_t *ppos)
6876{
4d3435b8
ON
6877 struct inode *inode = file_inode(filp);
6878 struct trace_array *tr = inode->i_private;
12883efb 6879 struct trace_buffer *trace_buf = &tr->trace_buffer;
4d3435b8 6880 int cpu = tracing_get_cpu(inode);
c8d77183
SR
6881 struct trace_seq *s;
6882 unsigned long cnt;
c64e148a
VN
6883 unsigned long long t;
6884 unsigned long usec_rem;
c8d77183 6885
e4f2d10f 6886 s = kmalloc(sizeof(*s), GFP_KERNEL);
c8d77183 6887 if (!s)
a646365c 6888 return -ENOMEM;
c8d77183
SR
6889
6890 trace_seq_init(s);
6891
12883efb 6892 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6893 trace_seq_printf(s, "entries: %ld\n", cnt);
6894
12883efb 6895 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6896 trace_seq_printf(s, "overrun: %ld\n", cnt);
6897
12883efb 6898 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
c8d77183
SR
6899 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6900
12883efb 6901 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
c64e148a
VN
6902 trace_seq_printf(s, "bytes: %ld\n", cnt);
6903
58e8eedf 6904 if (trace_clocks[tr->clock_id].in_ns) {
11043d8b 6905 /* local or global for trace_clock */
12883efb 6906 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
11043d8b
YY
6907 usec_rem = do_div(t, USEC_PER_SEC);
6908 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6909 t, usec_rem);
6910
12883efb 6911 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b
YY
6912 usec_rem = do_div(t, USEC_PER_SEC);
6913 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6914 } else {
6915 /* counter or tsc mode for trace_clock */
6916 trace_seq_printf(s, "oldest event ts: %llu\n",
12883efb 6917 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
c64e148a 6918
11043d8b 6919 trace_seq_printf(s, "now ts: %llu\n",
12883efb 6920 ring_buffer_time_stamp(trace_buf->buffer, cpu));
11043d8b 6921 }
c64e148a 6922
12883efb 6923 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
884bfe89
SP
6924 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6925
12883efb 6926 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
ad964704
SRRH
6927 trace_seq_printf(s, "read events: %ld\n", cnt);
6928
5ac48378
SRRH
6929 count = simple_read_from_buffer(ubuf, count, ppos,
6930 s->buffer, trace_seq_used(s));
c8d77183
SR
6931
6932 kfree(s);
6933
6934 return count;
6935}
6936
6937static const struct file_operations tracing_stats_fops = {
4d3435b8 6938 .open = tracing_open_generic_tr,
c8d77183 6939 .read = tracing_stats_read,
b444786f 6940 .llseek = generic_file_llseek,
4d3435b8 6941 .release = tracing_release_generic_tr,
c8d77183
SR
6942};
6943
bc0c38d1
SR
6944#ifdef CONFIG_DYNAMIC_FTRACE
6945
6946static ssize_t
b807c3d0 6947tracing_read_dyn_info(struct file *filp, char __user *ubuf,
bc0c38d1
SR
6948 size_t cnt, loff_t *ppos)
6949{
6950 unsigned long *p = filp->private_data;
6a9c981b 6951 char buf[64]; /* Not too big for a shallow stack */
bc0c38d1
SR
6952 int r;
6953
6a9c981b 6954 r = scnprintf(buf, 63, "%ld", *p);
b807c3d0
SR
6955 buf[r++] = '\n';
6956
6a9c981b 6957 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
bc0c38d1
SR
6958}
6959
5e2336a0 6960static const struct file_operations tracing_dyn_info_fops = {
4bf39a94 6961 .open = tracing_open_generic,
b807c3d0 6962 .read = tracing_read_dyn_info,
b444786f 6963 .llseek = generic_file_llseek,
bc0c38d1 6964};
77fd5c15 6965#endif /* CONFIG_DYNAMIC_FTRACE */
bc0c38d1 6966
77fd5c15
SRRH
6967#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6968static void
bca6c8d0 6969ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
b5f081b5 6970 struct trace_array *tr, struct ftrace_probe_ops *ops,
6e444319 6971 void *data)
77fd5c15 6972{
cab50379 6973 tracing_snapshot_instance(tr);
77fd5c15 6974}
bc0c38d1 6975
77fd5c15 6976static void
bca6c8d0 6977ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
b5f081b5 6978 struct trace_array *tr, struct ftrace_probe_ops *ops,
6e444319 6979 void *data)
bc0c38d1 6980{
6e444319 6981 struct ftrace_func_mapper *mapper = data;
1a93f8bd 6982 long *count = NULL;
77fd5c15 6983
1a93f8bd
SRV
6984 if (mapper)
6985 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
6986
6987 if (count) {
6988
6989 if (*count <= 0)
6990 return;
bc0c38d1 6991
77fd5c15 6992 (*count)--;
1a93f8bd 6993 }
77fd5c15 6994
cab50379 6995 tracing_snapshot_instance(tr);
77fd5c15
SRRH
6996}
6997
6998static int
6999ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7000 struct ftrace_probe_ops *ops, void *data)
7001{
6e444319 7002 struct ftrace_func_mapper *mapper = data;
1a93f8bd 7003 long *count = NULL;
77fd5c15
SRRH
7004
7005 seq_printf(m, "%ps:", (void *)ip);
7006
fa6f0cc7 7007 seq_puts(m, "snapshot");
77fd5c15 7008
1a93f8bd
SRV
7009 if (mapper)
7010 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7011
7012 if (count)
7013 seq_printf(m, ":count=%ld\n", *count);
77fd5c15 7014 else
1a93f8bd 7015 seq_puts(m, ":unlimited\n");
77fd5c15
SRRH
7016
7017 return 0;
7018}
7019
1a93f8bd 7020static int
b5f081b5 7021ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
6e444319 7022 unsigned long ip, void *init_data, void **data)
1a93f8bd 7023{
6e444319
SRV
7024 struct ftrace_func_mapper *mapper = *data;
7025
7026 if (!mapper) {
7027 mapper = allocate_ftrace_func_mapper();
7028 if (!mapper)
7029 return -ENOMEM;
7030 *data = mapper;
7031 }
1a93f8bd 7032
6e444319 7033 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
1a93f8bd
SRV
7034}
7035
7036static void
b5f081b5 7037ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
6e444319 7038 unsigned long ip, void *data)
1a93f8bd 7039{
6e444319
SRV
7040 struct ftrace_func_mapper *mapper = data;
7041
7042 if (!ip) {
7043 if (!mapper)
7044 return;
7045 free_ftrace_func_mapper(mapper, NULL);
7046 return;
7047 }
1a93f8bd
SRV
7048
7049 ftrace_func_mapper_remove_ip(mapper, ip);
7050}
7051
77fd5c15
SRRH
7052static struct ftrace_probe_ops snapshot_probe_ops = {
7053 .func = ftrace_snapshot,
7054 .print = ftrace_snapshot_print,
7055};
7056
7057static struct ftrace_probe_ops snapshot_count_probe_ops = {
7058 .func = ftrace_count_snapshot,
7059 .print = ftrace_snapshot_print,
1a93f8bd
SRV
7060 .init = ftrace_snapshot_init,
7061 .free = ftrace_snapshot_free,
77fd5c15
SRRH
7062};
7063
7064static int
04ec7bb6 7065ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
77fd5c15
SRRH
7066 char *glob, char *cmd, char *param, int enable)
7067{
7068 struct ftrace_probe_ops *ops;
7069 void *count = (void *)-1;
7070 char *number;
7071 int ret;
7072
0f179765
SRV
7073 if (!tr)
7074 return -ENODEV;
7075
77fd5c15
SRRH
7076 /* hash funcs only work with set_ftrace_filter */
7077 if (!enable)
7078 return -EINVAL;
7079
7080 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7081
d3d532d7 7082 if (glob[0] == '!')
7b60f3d8 7083 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
77fd5c15
SRRH
7084
7085 if (!param)
7086 goto out_reg;
7087
7088 number = strsep(&param, ":");
7089
7090 if (!strlen(number))
7091 goto out_reg;
7092
7093 /*
7094 * We use the callback data field (which is a pointer)
7095 * as our counter.
7096 */
7097 ret = kstrtoul(number, 0, (unsigned long *)&count);
7098 if (ret)
7099 return ret;
7100
7101 out_reg:
4c174688 7102 ret = alloc_snapshot(tr);
df62db5b
SRV
7103 if (ret < 0)
7104 goto out;
77fd5c15 7105
4c174688 7106 ret = register_ftrace_function_probe(glob, tr, ops, count);
77fd5c15 7107
df62db5b 7108 out:
77fd5c15
SRRH
7109 return ret < 0 ? ret : 0;
7110}
7111
7112static struct ftrace_func_command ftrace_snapshot_cmd = {
7113 .name = "snapshot",
7114 .func = ftrace_trace_snapshot_callback,
7115};
7116
38de93ab 7117static __init int register_snapshot_cmd(void)
77fd5c15
SRRH
7118{
7119 return register_ftrace_command(&ftrace_snapshot_cmd);
7120}
7121#else
38de93ab 7122static inline __init int register_snapshot_cmd(void) { return 0; }
77fd5c15 7123#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
bc0c38d1 7124
7eeafbca 7125static struct dentry *tracing_get_dentry(struct trace_array *tr)
bc0c38d1 7126{
8434dc93
SRRH
7127 if (WARN_ON(!tr->dir))
7128 return ERR_PTR(-ENODEV);
7129
7130 /* Top directory uses NULL as the parent */
7131 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7132 return NULL;
7133
7134 /* All sub buffers have a descriptor */
2b6080f2 7135 return tr->dir;
bc0c38d1
SR
7136}
7137
2b6080f2 7138static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
b04cc6b1 7139{
b04cc6b1
FW
7140 struct dentry *d_tracer;
7141
2b6080f2
SR
7142 if (tr->percpu_dir)
7143 return tr->percpu_dir;
b04cc6b1 7144
7eeafbca 7145 d_tracer = tracing_get_dentry(tr);
14a5ae40 7146 if (IS_ERR(d_tracer))
b04cc6b1
FW
7147 return NULL;
7148
8434dc93 7149 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
b04cc6b1 7150
2b6080f2 7151 WARN_ONCE(!tr->percpu_dir,
8434dc93 7152 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
b04cc6b1 7153
2b6080f2 7154 return tr->percpu_dir;
b04cc6b1
FW
7155}
7156
649e9c70
ON
7157static struct dentry *
7158trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7159 void *data, long cpu, const struct file_operations *fops)
7160{
7161 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7162
7163 if (ret) /* See tracing_get_cpu() */
7682c918 7164 d_inode(ret)->i_cdev = (void *)(cpu + 1);
649e9c70
ON
7165 return ret;
7166}
7167
2b6080f2 7168static void
8434dc93 7169tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
b04cc6b1 7170{
2b6080f2 7171 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5452af66 7172 struct dentry *d_cpu;
dd49a38c 7173 char cpu_dir[30]; /* 30 characters should be more than enough */
b04cc6b1 7174
0a3d7ce7
NK
7175 if (!d_percpu)
7176 return;
7177
dd49a38c 7178 snprintf(cpu_dir, 30, "cpu%ld", cpu);
8434dc93 7179 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
8656e7a2 7180 if (!d_cpu) {
a395d6a7 7181 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8656e7a2
FW
7182 return;
7183 }
b04cc6b1 7184
8656e7a2 7185 /* per cpu trace_pipe */
649e9c70 7186 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
15544209 7187 tr, cpu, &tracing_pipe_fops);
b04cc6b1
FW
7188
7189 /* per cpu trace */
649e9c70 7190 trace_create_cpu_file("trace", 0644, d_cpu,
6484c71c 7191 tr, cpu, &tracing_fops);
7f96f93f 7192
649e9c70 7193 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
46ef2be0 7194 tr, cpu, &tracing_buffers_fops);
7f96f93f 7195
649e9c70 7196 trace_create_cpu_file("stats", 0444, d_cpu,
4d3435b8 7197 tr, cpu, &tracing_stats_fops);
438ced17 7198
649e9c70 7199 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
0bc392ee 7200 tr, cpu, &tracing_entries_fops);
f1affcaa
SRRH
7201
7202#ifdef CONFIG_TRACER_SNAPSHOT
649e9c70 7203 trace_create_cpu_file("snapshot", 0644, d_cpu,
6484c71c 7204 tr, cpu, &snapshot_fops);
6de58e62 7205
649e9c70 7206 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
46ef2be0 7207 tr, cpu, &snapshot_raw_fops);
f1affcaa 7208#endif
b04cc6b1
FW
7209}
7210
60a11774
SR
7211#ifdef CONFIG_FTRACE_SELFTEST
7212/* Let selftest have access to static functions in this file */
7213#include "trace_selftest.c"
7214#endif
7215
577b785f
SR
7216static ssize_t
7217trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7218 loff_t *ppos)
7219{
7220 struct trace_option_dentry *topt = filp->private_data;
7221 char *buf;
7222
7223 if (topt->flags->val & topt->opt->bit)
7224 buf = "1\n";
7225 else
7226 buf = "0\n";
7227
7228 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7229}
7230
7231static ssize_t
7232trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7233 loff_t *ppos)
7234{
7235 struct trace_option_dentry *topt = filp->private_data;
7236 unsigned long val;
577b785f
SR
7237 int ret;
7238
22fe9b54
PH
7239 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7240 if (ret)
577b785f
SR
7241 return ret;
7242
8d18eaaf
LZ
7243 if (val != 0 && val != 1)
7244 return -EINVAL;
577b785f 7245
8d18eaaf 7246 if (!!(topt->flags->val & topt->opt->bit) != val) {
577b785f 7247 mutex_lock(&trace_types_lock);
8c1a49ae 7248 ret = __set_tracer_option(topt->tr, topt->flags,
c757bea9 7249 topt->opt, !val);
577b785f
SR
7250 mutex_unlock(&trace_types_lock);
7251 if (ret)
7252 return ret;
577b785f
SR
7253 }
7254
7255 *ppos += cnt;
7256
7257 return cnt;
7258}
7259
7260
7261static const struct file_operations trace_options_fops = {
7262 .open = tracing_open_generic,
7263 .read = trace_options_read,
7264 .write = trace_options_write,
b444786f 7265 .llseek = generic_file_llseek,
577b785f
SR
7266};
7267
9a38a885
SRRH
7268/*
7269 * In order to pass in both the trace_array descriptor as well as the index
7270 * to the flag that the trace option file represents, the trace_array
7271 * has a character array of trace_flags_index[], which holds the index
7272 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7273 * The address of this character array is passed to the flag option file
7274 * read/write callbacks.
7275 *
7276 * In order to extract both the index and the trace_array descriptor,
7277 * get_tr_index() uses the following algorithm.
7278 *
7279 * idx = *ptr;
7280 *
7281 * As the pointer itself contains the address of the index (remember
7282 * index[1] == 1).
7283 *
7284 * Then to get the trace_array descriptor, by subtracting that index
7285 * from the ptr, we get to the start of the index itself.
7286 *
7287 * ptr - idx == &index[0]
7288 *
7289 * Then a simple container_of() from that pointer gets us to the
7290 * trace_array descriptor.
7291 */
7292static void get_tr_index(void *data, struct trace_array **ptr,
7293 unsigned int *pindex)
7294{
7295 *pindex = *(unsigned char *)data;
7296
7297 *ptr = container_of(data - *pindex, struct trace_array,
7298 trace_flags_index);
7299}
7300
a8259075
SR
7301static ssize_t
7302trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7303 loff_t *ppos)
7304{
9a38a885
SRRH
7305 void *tr_index = filp->private_data;
7306 struct trace_array *tr;
7307 unsigned int index;
a8259075
SR
7308 char *buf;
7309
9a38a885
SRRH
7310 get_tr_index(tr_index, &tr, &index);
7311
7312 if (tr->trace_flags & (1 << index))
a8259075
SR
7313 buf = "1\n";
7314 else
7315 buf = "0\n";
7316
7317 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7318}
7319
7320static ssize_t
7321trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7322 loff_t *ppos)
7323{
9a38a885
SRRH
7324 void *tr_index = filp->private_data;
7325 struct trace_array *tr;
7326 unsigned int index;
a8259075
SR
7327 unsigned long val;
7328 int ret;
7329
9a38a885
SRRH
7330 get_tr_index(tr_index, &tr, &index);
7331
22fe9b54
PH
7332 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7333 if (ret)
a8259075
SR
7334 return ret;
7335
f2d84b65 7336 if (val != 0 && val != 1)
a8259075 7337 return -EINVAL;
69d34da2
SRRH
7338
7339 mutex_lock(&trace_types_lock);
2b6080f2 7340 ret = set_tracer_flag(tr, 1 << index, val);
69d34da2 7341 mutex_unlock(&trace_types_lock);
a8259075 7342
613f04a0
SRRH
7343 if (ret < 0)
7344 return ret;
7345
a8259075
SR
7346 *ppos += cnt;
7347
7348 return cnt;
7349}
7350
a8259075
SR
7351static const struct file_operations trace_options_core_fops = {
7352 .open = tracing_open_generic,
7353 .read = trace_options_core_read,
7354 .write = trace_options_core_write,
b444786f 7355 .llseek = generic_file_llseek,
a8259075
SR
7356};
7357
5452af66 7358struct dentry *trace_create_file(const char *name,
f4ae40a6 7359 umode_t mode,
5452af66
FW
7360 struct dentry *parent,
7361 void *data,
7362 const struct file_operations *fops)
7363{
7364 struct dentry *ret;
7365
8434dc93 7366 ret = tracefs_create_file(name, mode, parent, data, fops);
5452af66 7367 if (!ret)
a395d6a7 7368 pr_warn("Could not create tracefs '%s' entry\n", name);
5452af66
FW
7369
7370 return ret;
7371}
7372
7373
2b6080f2 7374static struct dentry *trace_options_init_dentry(struct trace_array *tr)
a8259075
SR
7375{
7376 struct dentry *d_tracer;
a8259075 7377
2b6080f2
SR
7378 if (tr->options)
7379 return tr->options;
a8259075 7380
7eeafbca 7381 d_tracer = tracing_get_dentry(tr);
14a5ae40 7382 if (IS_ERR(d_tracer))
a8259075
SR
7383 return NULL;
7384
8434dc93 7385 tr->options = tracefs_create_dir("options", d_tracer);
2b6080f2 7386 if (!tr->options) {
a395d6a7 7387 pr_warn("Could not create tracefs directory 'options'\n");
a8259075
SR
7388 return NULL;
7389 }
7390
2b6080f2 7391 return tr->options;
a8259075
SR
7392}
7393
577b785f 7394static void
2b6080f2
SR
7395create_trace_option_file(struct trace_array *tr,
7396 struct trace_option_dentry *topt,
577b785f
SR
7397 struct tracer_flags *flags,
7398 struct tracer_opt *opt)
7399{
7400 struct dentry *t_options;
577b785f 7401
2b6080f2 7402 t_options = trace_options_init_dentry(tr);
577b785f
SR
7403 if (!t_options)
7404 return;
7405
7406 topt->flags = flags;
7407 topt->opt = opt;
2b6080f2 7408 topt->tr = tr;
577b785f 7409
5452af66 7410 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
577b785f
SR
7411 &trace_options_fops);
7412
577b785f
SR
7413}
7414
37aea98b 7415static void
2b6080f2 7416create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
577b785f
SR
7417{
7418 struct trace_option_dentry *topts;
37aea98b 7419 struct trace_options *tr_topts;
577b785f
SR
7420 struct tracer_flags *flags;
7421 struct tracer_opt *opts;
7422 int cnt;
37aea98b 7423 int i;
577b785f
SR
7424
7425 if (!tracer)
37aea98b 7426 return;
577b785f
SR
7427
7428 flags = tracer->flags;
7429
7430 if (!flags || !flags->opts)
37aea98b
SRRH
7431 return;
7432
7433 /*
7434 * If this is an instance, only create flags for tracers
7435 * the instance may have.
7436 */
7437 if (!trace_ok_for_array(tracer, tr))
7438 return;
7439
7440 for (i = 0; i < tr->nr_topts; i++) {
d39cdd20
CH
7441 /* Make sure there's no duplicate flags. */
7442 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
37aea98b
SRRH
7443 return;
7444 }
577b785f
SR
7445
7446 opts = flags->opts;
7447
7448 for (cnt = 0; opts[cnt].name; cnt++)
7449 ;
7450
0cfe8245 7451 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
577b785f 7452 if (!topts)
37aea98b
SRRH
7453 return;
7454
7455 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
7456 GFP_KERNEL);
7457 if (!tr_topts) {
7458 kfree(topts);
7459 return;
7460 }
7461
7462 tr->topts = tr_topts;
7463 tr->topts[tr->nr_topts].tracer = tracer;
7464 tr->topts[tr->nr_topts].topts = topts;
7465 tr->nr_topts++;
577b785f 7466
41d9c0be 7467 for (cnt = 0; opts[cnt].name; cnt++) {
2b6080f2 7468 create_trace_option_file(tr, &topts[cnt], flags,
577b785f 7469 &opts[cnt]);
41d9c0be
SRRH
7470 WARN_ONCE(topts[cnt].entry == NULL,
7471 "Failed to create trace option: %s",
7472 opts[cnt].name);
7473 }
577b785f
SR
7474}
7475
a8259075 7476static struct dentry *
2b6080f2
SR
7477create_trace_option_core_file(struct trace_array *tr,
7478 const char *option, long index)
a8259075
SR
7479{
7480 struct dentry *t_options;
a8259075 7481
2b6080f2 7482 t_options = trace_options_init_dentry(tr);
a8259075
SR
7483 if (!t_options)
7484 return NULL;
7485
9a38a885
SRRH
7486 return trace_create_file(option, 0644, t_options,
7487 (void *)&tr->trace_flags_index[index],
7488 &trace_options_core_fops);
a8259075
SR
7489}
7490
16270145 7491static void create_trace_options_dir(struct trace_array *tr)
a8259075
SR
7492{
7493 struct dentry *t_options;
16270145 7494 bool top_level = tr == &global_trace;
a8259075
SR
7495 int i;
7496
2b6080f2 7497 t_options = trace_options_init_dentry(tr);
a8259075
SR
7498 if (!t_options)
7499 return;
7500
16270145
SRRH
7501 for (i = 0; trace_options[i]; i++) {
7502 if (top_level ||
7503 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
7504 create_trace_option_core_file(tr, trace_options[i], i);
7505 }
a8259075
SR
7506}
7507
499e5470
SR
7508static ssize_t
7509rb_simple_read(struct file *filp, char __user *ubuf,
7510 size_t cnt, loff_t *ppos)
7511{
348f0fc2 7512 struct trace_array *tr = filp->private_data;
499e5470
SR
7513 char buf[64];
7514 int r;
7515
10246fa3 7516 r = tracer_tracing_is_on(tr);
499e5470
SR
7517 r = sprintf(buf, "%d\n", r);
7518
7519 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7520}
7521
7522static ssize_t
7523rb_simple_write(struct file *filp, const char __user *ubuf,
7524 size_t cnt, loff_t *ppos)
7525{
348f0fc2 7526 struct trace_array *tr = filp->private_data;
12883efb 7527 struct ring_buffer *buffer = tr->trace_buffer.buffer;
499e5470
SR
7528 unsigned long val;
7529 int ret;
7530
7531 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7532 if (ret)
7533 return ret;
7534
7535 if (buffer) {
2df8f8a6
SR
7536 mutex_lock(&trace_types_lock);
7537 if (val) {
10246fa3 7538 tracer_tracing_on(tr);
2b6080f2
SR
7539 if (tr->current_trace->start)
7540 tr->current_trace->start(tr);
2df8f8a6 7541 } else {
10246fa3 7542 tracer_tracing_off(tr);
2b6080f2
SR
7543 if (tr->current_trace->stop)
7544 tr->current_trace->stop(tr);
2df8f8a6
SR
7545 }
7546 mutex_unlock(&trace_types_lock);
499e5470
SR
7547 }
7548
7549 (*ppos)++;
7550
7551 return cnt;
7552}
7553
7554static const struct file_operations rb_simple_fops = {
7b85af63 7555 .open = tracing_open_generic_tr,
499e5470
SR
7556 .read = rb_simple_read,
7557 .write = rb_simple_write,
7b85af63 7558 .release = tracing_release_generic_tr,
499e5470
SR
7559 .llseek = default_llseek,
7560};
7561
277ba044
SR
7562struct dentry *trace_instance_dir;
7563
7564static void
8434dc93 7565init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
277ba044 7566
55034cd6
SRRH
7567static int
7568allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
277ba044
SR
7569{
7570 enum ring_buffer_flags rb_flags;
737223fb 7571
983f938a 7572 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
737223fb 7573
dced341b
SRRH
7574 buf->tr = tr;
7575
55034cd6
SRRH
7576 buf->buffer = ring_buffer_alloc(size, rb_flags);
7577 if (!buf->buffer)
7578 return -ENOMEM;
737223fb 7579
55034cd6
SRRH
7580 buf->data = alloc_percpu(struct trace_array_cpu);
7581 if (!buf->data) {
7582 ring_buffer_free(buf->buffer);
4397f045 7583 buf->buffer = NULL;
55034cd6
SRRH
7584 return -ENOMEM;
7585 }
737223fb 7586
737223fb
SRRH
7587 /* Allocate the first page for all buffers */
7588 set_buffer_entries(&tr->trace_buffer,
7589 ring_buffer_size(tr->trace_buffer.buffer, 0));
7590
55034cd6
SRRH
7591 return 0;
7592}
737223fb 7593
55034cd6
SRRH
7594static int allocate_trace_buffers(struct trace_array *tr, int size)
7595{
7596 int ret;
737223fb 7597
55034cd6
SRRH
7598 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7599 if (ret)
7600 return ret;
737223fb 7601
55034cd6
SRRH
7602#ifdef CONFIG_TRACER_MAX_TRACE
7603 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7604 allocate_snapshot ? size : 1);
7605 if (WARN_ON(ret)) {
737223fb 7606 ring_buffer_free(tr->trace_buffer.buffer);
24f2aaf9 7607 tr->trace_buffer.buffer = NULL;
55034cd6 7608 free_percpu(tr->trace_buffer.data);
24f2aaf9 7609 tr->trace_buffer.data = NULL;
55034cd6
SRRH
7610 return -ENOMEM;
7611 }
7612 tr->allocated_snapshot = allocate_snapshot;
737223fb 7613
55034cd6
SRRH
7614 /*
7615 * Only the top level trace array gets its snapshot allocated
7616 * from the kernel command line.
7617 */
7618 allocate_snapshot = false;
737223fb 7619#endif
55034cd6 7620 return 0;
737223fb
SRRH
7621}
7622
f0b70cc4
SRRH
7623static void free_trace_buffer(struct trace_buffer *buf)
7624{
7625 if (buf->buffer) {
7626 ring_buffer_free(buf->buffer);
7627 buf->buffer = NULL;
7628 free_percpu(buf->data);
7629 buf->data = NULL;
7630 }
7631}
7632
23aaa3c1
SRRH
7633static void free_trace_buffers(struct trace_array *tr)
7634{
7635 if (!tr)
7636 return;
7637
f0b70cc4 7638 free_trace_buffer(&tr->trace_buffer);
23aaa3c1
SRRH
7639
7640#ifdef CONFIG_TRACER_MAX_TRACE
f0b70cc4 7641 free_trace_buffer(&tr->max_buffer);
23aaa3c1
SRRH
7642#endif
7643}
7644
9a38a885
SRRH
7645static void init_trace_flags_index(struct trace_array *tr)
7646{
7647 int i;
7648
7649 /* Used by the trace options files */
7650 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7651 tr->trace_flags_index[i] = i;
7652}
7653
37aea98b
SRRH
7654static void __update_tracer_options(struct trace_array *tr)
7655{
7656 struct tracer *t;
7657
7658 for (t = trace_types; t; t = t->next)
7659 add_tracer_options(tr, t);
7660}
7661
7662static void update_tracer_options(struct trace_array *tr)
7663{
7664 mutex_lock(&trace_types_lock);
7665 __update_tracer_options(tr);
7666 mutex_unlock(&trace_types_lock);
7667}
7668
eae47358 7669static int instance_mkdir(const char *name)
737223fb 7670{
277ba044
SR
7671 struct trace_array *tr;
7672 int ret;
277ba044 7673
12ecef0c 7674 mutex_lock(&event_mutex);
277ba044
SR
7675 mutex_lock(&trace_types_lock);
7676
7677 ret = -EEXIST;
7678 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7679 if (tr->name && strcmp(tr->name, name) == 0)
7680 goto out_unlock;
7681 }
7682
7683 ret = -ENOMEM;
7684 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7685 if (!tr)
7686 goto out_unlock;
7687
7688 tr->name = kstrdup(name, GFP_KERNEL);
7689 if (!tr->name)
7690 goto out_free_tr;
7691
ccfe9e42
AL
7692 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7693 goto out_free_tr;
7694
20550622 7695 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
983f938a 7696
ccfe9e42
AL
7697 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7698
277ba044
SR
7699 raw_spin_lock_init(&tr->start_lock);
7700
0b9b12c1
SRRH
7701 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7702
277ba044
SR
7703 tr->current_trace = &nop_trace;
7704
7705 INIT_LIST_HEAD(&tr->systems);
7706 INIT_LIST_HEAD(&tr->events);
7707
737223fb 7708 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
277ba044
SR
7709 goto out_free_tr;
7710
8434dc93 7711 tr->dir = tracefs_create_dir(name, trace_instance_dir);
277ba044
SR
7712 if (!tr->dir)
7713 goto out_free_tr;
7714
7715 ret = event_trace_add_tracer(tr->dir, tr);
609e85a7 7716 if (ret) {
8434dc93 7717 tracefs_remove_recursive(tr->dir);
277ba044 7718 goto out_free_tr;
609e85a7 7719 }
277ba044 7720
04ec7bb6
SRV
7721 ftrace_init_trace_array(tr);
7722
8434dc93 7723 init_tracer_tracefs(tr, tr->dir);
9a38a885 7724 init_trace_flags_index(tr);
37aea98b 7725 __update_tracer_options(tr);
277ba044
SR
7726
7727 list_add(&tr->list, &ftrace_trace_arrays);
7728
7729 mutex_unlock(&trace_types_lock);
12ecef0c 7730 mutex_unlock(&event_mutex);
277ba044
SR
7731
7732 return 0;
7733
7734 out_free_tr:
23aaa3c1 7735 free_trace_buffers(tr);
ccfe9e42 7736 free_cpumask_var(tr->tracing_cpumask);
277ba044
SR
7737 kfree(tr->name);
7738 kfree(tr);
7739
7740 out_unlock:
7741 mutex_unlock(&trace_types_lock);
12ecef0c 7742 mutex_unlock(&event_mutex);
277ba044
SR
7743
7744 return ret;
7745
7746}
7747
eae47358 7748static int instance_rmdir(const char *name)
0c8916c3
SR
7749{
7750 struct trace_array *tr;
7751 int found = 0;
7752 int ret;
37aea98b 7753 int i;
0c8916c3 7754
12ecef0c 7755 mutex_lock(&event_mutex);
0c8916c3
SR
7756 mutex_lock(&trace_types_lock);
7757
7758 ret = -ENODEV;
7759 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7760 if (tr->name && strcmp(tr->name, name) == 0) {
7761 found = 1;
7762 break;
7763 }
7764 }
7765 if (!found)
7766 goto out_unlock;
7767
a695cb58 7768 ret = -EBUSY;
cf6ab6d9 7769 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
a695cb58
SRRH
7770 goto out_unlock;
7771
0c8916c3
SR
7772 list_del(&tr->list);
7773
20550622
SRRH
7774 /* Disable all the flags that were enabled coming in */
7775 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7776 if ((1 << i) & ZEROED_TRACE_FLAGS)
7777 set_tracer_flag(tr, 1 << i, 0);
7778 }
7779
6b450d25 7780 tracing_set_nop(tr);
a0e6369e 7781 clear_ftrace_function_probes(tr);
0c8916c3 7782 event_trace_del_tracer(tr);
d879d0b8 7783 ftrace_clear_pids(tr);
591dffda 7784 ftrace_destroy_function_files(tr);
681a4a2f 7785 tracefs_remove_recursive(tr->dir);
a9fcaaac 7786 free_trace_buffers(tr);
0c8916c3 7787
37aea98b
SRRH
7788 for (i = 0; i < tr->nr_topts; i++) {
7789 kfree(tr->topts[i].topts);
7790 }
7791 kfree(tr->topts);
7792
db9108e0 7793 free_cpumask_var(tr->tracing_cpumask);
0c8916c3
SR
7794 kfree(tr->name);
7795 kfree(tr);
7796
7797 ret = 0;
7798
7799 out_unlock:
7800 mutex_unlock(&trace_types_lock);
12ecef0c 7801 mutex_unlock(&event_mutex);
0c8916c3
SR
7802
7803 return ret;
7804}
7805
277ba044
SR
7806static __init void create_trace_instances(struct dentry *d_tracer)
7807{
eae47358
SRRH
7808 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7809 instance_mkdir,
7810 instance_rmdir);
277ba044
SR
7811 if (WARN_ON(!trace_instance_dir))
7812 return;
277ba044
SR
7813}
7814
2b6080f2 7815static void
8434dc93 7816init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
2b6080f2 7817{
121aaee7 7818 int cpu;
2b6080f2 7819
607e2ea1
SRRH
7820 trace_create_file("available_tracers", 0444, d_tracer,
7821 tr, &show_traces_fops);
7822
7823 trace_create_file("current_tracer", 0644, d_tracer,
7824 tr, &set_tracer_fops);
7825
ccfe9e42
AL
7826 trace_create_file("tracing_cpumask", 0644, d_tracer,
7827 tr, &tracing_cpumask_fops);
7828
2b6080f2
SR
7829 trace_create_file("trace_options", 0644, d_tracer,
7830 tr, &tracing_iter_fops);
7831
7832 trace_create_file("trace", 0644, d_tracer,
6484c71c 7833 tr, &tracing_fops);
2b6080f2
SR
7834
7835 trace_create_file("trace_pipe", 0444, d_tracer,
15544209 7836 tr, &tracing_pipe_fops);
2b6080f2
SR
7837
7838 trace_create_file("buffer_size_kb", 0644, d_tracer,
0bc392ee 7839 tr, &tracing_entries_fops);
2b6080f2
SR
7840
7841 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7842 tr, &tracing_total_entries_fops);
7843
238ae93d 7844 trace_create_file("free_buffer", 0200, d_tracer,
2b6080f2
SR
7845 tr, &tracing_free_buffer_fops);
7846
7847 trace_create_file("trace_marker", 0220, d_tracer,
7848 tr, &tracing_mark_fops);
7849
fa32e855
SR
7850 trace_create_file("trace_marker_raw", 0220, d_tracer,
7851 tr, &tracing_mark_raw_fops);
7852
2b6080f2
SR
7853 trace_create_file("trace_clock", 0644, d_tracer, tr,
7854 &trace_clock_fops);
7855
7856 trace_create_file("tracing_on", 0644, d_tracer,
6484c71c 7857 tr, &rb_simple_fops);
ce9bae55 7858
16270145
SRRH
7859 create_trace_options_dir(tr);
7860
f971cc9a 7861#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6d9b3fa5
SRRH
7862 trace_create_file("tracing_max_latency", 0644, d_tracer,
7863 &tr->max_latency, &tracing_max_lat_fops);
7864#endif
7865
591dffda
SRRH
7866 if (ftrace_create_function_files(tr, d_tracer))
7867 WARN(1, "Could not allocate function filter files");
7868
ce9bae55
SRRH
7869#ifdef CONFIG_TRACER_SNAPSHOT
7870 trace_create_file("snapshot", 0644, d_tracer,
6484c71c 7871 tr, &snapshot_fops);
ce9bae55 7872#endif
121aaee7
SRRH
7873
7874 for_each_tracing_cpu(cpu)
8434dc93 7875 tracing_init_tracefs_percpu(tr, cpu);
121aaee7 7876
345ddcc8 7877 ftrace_init_tracefs(tr, d_tracer);
2b6080f2
SR
7878}
7879
93faccbb 7880static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
f76180bc
SRRH
7881{
7882 struct vfsmount *mnt;
7883 struct file_system_type *type;
7884
7885 /*
7886 * To maintain backward compatibility for tools that mount
7887 * debugfs to get to the tracing facility, tracefs is automatically
7888 * mounted to the debugfs/tracing directory.
7889 */
7890 type = get_fs_type("tracefs");
7891 if (!type)
7892 return NULL;
93faccbb 7893 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
f76180bc
SRRH
7894 put_filesystem(type);
7895 if (IS_ERR(mnt))
7896 return NULL;
7897 mntget(mnt);
7898
7899 return mnt;
7900}
7901
7eeafbca
SRRH
7902/**
7903 * tracing_init_dentry - initialize top level trace array
7904 *
7905 * This is called when creating files or directories in the tracing
7906 * directory. It is called via fs_initcall() by any of the boot up code
7907 * and expects to return the dentry of the top level tracing directory.
7908 */
7909struct dentry *tracing_init_dentry(void)
7910{
7911 struct trace_array *tr = &global_trace;
7912
f76180bc 7913 /* The top level trace array uses NULL as parent */
7eeafbca 7914 if (tr->dir)
f76180bc 7915 return NULL;
7eeafbca 7916
8b129199
JW
7917 if (WARN_ON(!tracefs_initialized()) ||
7918 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7919 WARN_ON(!debugfs_initialized())))
7eeafbca
SRRH
7920 return ERR_PTR(-ENODEV);
7921
f76180bc
SRRH
7922 /*
7923 * As there may still be users that expect the tracing
7924 * files to exist in debugfs/tracing, we must automount
7925 * the tracefs file system there, so older tools still
7926 * work with the newer kerenl.
7927 */
7928 tr->dir = debugfs_create_automount("tracing", NULL,
7929 trace_automount, NULL);
7eeafbca
SRRH
7930 if (!tr->dir) {
7931 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7932 return ERR_PTR(-ENOMEM);
7933 }
7934
8434dc93 7935 return NULL;
7eeafbca
SRRH
7936}
7937
00f4b652
JL
7938extern struct trace_eval_map *__start_ftrace_eval_maps[];
7939extern struct trace_eval_map *__stop_ftrace_eval_maps[];
0c564a53 7940
5f60b351 7941static void __init trace_eval_init(void)
0c564a53 7942{
3673b8e4
SRRH
7943 int len;
7944
02fd7f68 7945 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
f57a4143 7946 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
3673b8e4
SRRH
7947}
7948
7949#ifdef CONFIG_MODULES
f57a4143 7950static void trace_module_add_evals(struct module *mod)
3673b8e4 7951{
99be647c 7952 if (!mod->num_trace_evals)
3673b8e4
SRRH
7953 return;
7954
7955 /*
7956 * Modules with bad taint do not have events created, do
7957 * not bother with enums either.
7958 */
7959 if (trace_module_has_bad_taint(mod))
7960 return;
7961
f57a4143 7962 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
3673b8e4
SRRH
7963}
7964
681bec03 7965#ifdef CONFIG_TRACE_EVAL_MAP_FILE
f57a4143 7966static void trace_module_remove_evals(struct module *mod)
9828413d 7967{
23bf8cb8
JL
7968 union trace_eval_map_item *map;
7969 union trace_eval_map_item **last = &trace_eval_maps;
9828413d 7970
99be647c 7971 if (!mod->num_trace_evals)
9828413d
SRRH
7972 return;
7973
1793ed93 7974 mutex_lock(&trace_eval_mutex);
9828413d 7975
23bf8cb8 7976 map = trace_eval_maps;
9828413d
SRRH
7977
7978 while (map) {
7979 if (map->head.mod == mod)
7980 break;
5f60b351 7981 map = trace_eval_jmp_to_tail(map);
9828413d
SRRH
7982 last = &map->tail.next;
7983 map = map->tail.next;
7984 }
7985 if (!map)
7986 goto out;
7987
5f60b351 7988 *last = trace_eval_jmp_to_tail(map)->tail.next;
9828413d
SRRH
7989 kfree(map);
7990 out:
1793ed93 7991 mutex_unlock(&trace_eval_mutex);
9828413d
SRRH
7992}
7993#else
f57a4143 7994static inline void trace_module_remove_evals(struct module *mod) { }
681bec03 7995#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9828413d 7996
3673b8e4
SRRH
7997static int trace_module_notify(struct notifier_block *self,
7998 unsigned long val, void *data)
7999{
8000 struct module *mod = data;
8001
8002 switch (val) {
8003 case MODULE_STATE_COMING:
f57a4143 8004 trace_module_add_evals(mod);
3673b8e4 8005 break;
9828413d 8006 case MODULE_STATE_GOING:
f57a4143 8007 trace_module_remove_evals(mod);
9828413d 8008 break;
3673b8e4
SRRH
8009 }
8010
8011 return 0;
0c564a53
SRRH
8012}
8013
3673b8e4
SRRH
8014static struct notifier_block trace_module_nb = {
8015 .notifier_call = trace_module_notify,
8016 .priority = 0,
8017};
9828413d 8018#endif /* CONFIG_MODULES */
3673b8e4 8019
8434dc93 8020static __init int tracer_init_tracefs(void)
bc0c38d1
SR
8021{
8022 struct dentry *d_tracer;
bc0c38d1 8023
7e53bd42
LJ
8024 trace_access_lock_init();
8025
bc0c38d1 8026 d_tracer = tracing_init_dentry();
14a5ae40 8027 if (IS_ERR(d_tracer))
ed6f1c99 8028 return 0;
bc0c38d1 8029
8434dc93 8030 init_tracer_tracefs(&global_trace, d_tracer);
501c2375 8031 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
bc0c38d1 8032
5452af66 8033 trace_create_file("tracing_thresh", 0644, d_tracer,
6508fa76 8034 &global_trace, &tracing_thresh_fops);
a8259075 8035
339ae5d3 8036 trace_create_file("README", 0444, d_tracer,
5452af66
FW
8037 NULL, &tracing_readme_fops);
8038
69abe6a5
AP
8039 trace_create_file("saved_cmdlines", 0444, d_tracer,
8040 NULL, &tracing_saved_cmdlines_fops);
5bf9a1ee 8041
939c7a4f
YY
8042 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8043 NULL, &tracing_saved_cmdlines_size_fops);
8044
99c621d7
MS
8045 trace_create_file("saved_tgids", 0444, d_tracer,
8046 NULL, &tracing_saved_tgids_fops);
8047
5f60b351 8048 trace_eval_init();
0c564a53 8049
f57a4143 8050 trace_create_eval_file(d_tracer);
9828413d 8051
3673b8e4
SRRH
8052#ifdef CONFIG_MODULES
8053 register_module_notifier(&trace_module_nb);
8054#endif
8055
bc0c38d1 8056#ifdef CONFIG_DYNAMIC_FTRACE
5452af66
FW
8057 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8058 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
bc0c38d1 8059#endif
b04cc6b1 8060
277ba044 8061 create_trace_instances(d_tracer);
5452af66 8062
37aea98b 8063 update_tracer_options(&global_trace);
09d23a1d 8064
b5ad384e 8065 return 0;
bc0c38d1
SR
8066}
8067
3f5a54e3
SR
8068static int trace_panic_handler(struct notifier_block *this,
8069 unsigned long event, void *unused)
8070{
944ac425 8071 if (ftrace_dump_on_oops)
cecbca96 8072 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
8073 return NOTIFY_OK;
8074}
8075
8076static struct notifier_block trace_panic_notifier = {
8077 .notifier_call = trace_panic_handler,
8078 .next = NULL,
8079 .priority = 150 /* priority: INT_MAX >= x >= 0 */
8080};
8081
8082static int trace_die_handler(struct notifier_block *self,
8083 unsigned long val,
8084 void *data)
8085{
8086 switch (val) {
8087 case DIE_OOPS:
944ac425 8088 if (ftrace_dump_on_oops)
cecbca96 8089 ftrace_dump(ftrace_dump_on_oops);
3f5a54e3
SR
8090 break;
8091 default:
8092 break;
8093 }
8094 return NOTIFY_OK;
8095}
8096
8097static struct notifier_block trace_die_notifier = {
8098 .notifier_call = trace_die_handler,
8099 .priority = 200
8100};
8101
8102/*
8103 * printk is set to max of 1024, we really don't need it that big.
8104 * Nothing should be printing 1000 characters anyway.
8105 */
8106#define TRACE_MAX_PRINT 1000
8107
8108/*
8109 * Define here KERN_TRACE so that we have one place to modify
8110 * it if we decide to change what log level the ftrace dump
8111 * should be at.
8112 */
428aee14 8113#define KERN_TRACE KERN_EMERG
3f5a54e3 8114
955b61e5 8115void
3f5a54e3
SR
8116trace_printk_seq(struct trace_seq *s)
8117{
8118 /* Probably should print a warning here. */
3a161d99
SRRH
8119 if (s->seq.len >= TRACE_MAX_PRINT)
8120 s->seq.len = TRACE_MAX_PRINT;
3f5a54e3 8121
820b75f6
SRRH
8122 /*
8123 * More paranoid code. Although the buffer size is set to
8124 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8125 * an extra layer of protection.
8126 */
8127 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8128 s->seq.len = s->seq.size - 1;
3f5a54e3
SR
8129
8130 /* should be zero ended, but we are paranoid. */
3a161d99 8131 s->buffer[s->seq.len] = 0;
3f5a54e3
SR
8132
8133 printk(KERN_TRACE "%s", s->buffer);
8134
f9520750 8135 trace_seq_init(s);
3f5a54e3
SR
8136}
8137
955b61e5
JW
8138void trace_init_global_iter(struct trace_iterator *iter)
8139{
8140 iter->tr = &global_trace;
2b6080f2 8141 iter->trace = iter->tr->current_trace;
ae3b5093 8142 iter->cpu_file = RING_BUFFER_ALL_CPUS;
12883efb 8143 iter->trace_buffer = &global_trace.trace_buffer;
b2f974d6
CS
8144
8145 if (iter->trace && iter->trace->open)
8146 iter->trace->open(iter);
8147
8148 /* Annotate start of buffers if we had overruns */
8149 if (ring_buffer_overruns(iter->trace_buffer->buffer))
8150 iter->iter_flags |= TRACE_FILE_ANNOTATE;
8151
8152 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8153 if (trace_clocks[iter->tr->clock_id].in_ns)
8154 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
955b61e5
JW
8155}
8156
7fe70b57 8157void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
3f5a54e3 8158{
3f5a54e3
SR
8159 /* use static because iter can be a bit big for the stack */
8160 static struct trace_iterator iter;
7fe70b57 8161 static atomic_t dump_running;
983f938a 8162 struct trace_array *tr = &global_trace;
cf586b61 8163 unsigned int old_userobj;
d769041f
SR
8164 unsigned long flags;
8165 int cnt = 0, cpu;
3f5a54e3 8166
7fe70b57
SRRH
8167 /* Only allow one dump user at a time. */
8168 if (atomic_inc_return(&dump_running) != 1) {
8169 atomic_dec(&dump_running);
8170 return;
8171 }
3f5a54e3 8172
7fe70b57
SRRH
8173 /*
8174 * Always turn off tracing when we dump.
8175 * We don't need to show trace output of what happens
8176 * between multiple crashes.
8177 *
8178 * If the user does a sysrq-z, then they can re-enable
8179 * tracing with echo 1 > tracing_on.
8180 */
0ee6b6cf 8181 tracing_off();
cf586b61 8182
7fe70b57 8183 local_irq_save(flags);
3f5a54e3 8184
38dbe0b1 8185 /* Simulate the iterator */
955b61e5
JW
8186 trace_init_global_iter(&iter);
8187
d769041f 8188 for_each_tracing_cpu(cpu) {
5e2d5ef8 8189 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
d769041f
SR
8190 }
8191
983f938a 8192 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
cf586b61 8193
b54d3de9 8194 /* don't look at user memory in panic mode */
983f938a 8195 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
b54d3de9 8196
cecbca96
FW
8197 switch (oops_dump_mode) {
8198 case DUMP_ALL:
ae3b5093 8199 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
8200 break;
8201 case DUMP_ORIG:
8202 iter.cpu_file = raw_smp_processor_id();
8203 break;
8204 case DUMP_NONE:
8205 goto out_enable;
8206 default:
8207 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
ae3b5093 8208 iter.cpu_file = RING_BUFFER_ALL_CPUS;
cecbca96
FW
8209 }
8210
8211 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3f5a54e3 8212
7fe70b57
SRRH
8213 /* Did function tracer already get disabled? */
8214 if (ftrace_is_dead()) {
8215 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8216 printk("# MAY BE MISSING FUNCTION EVENTS\n");
8217 }
8218
3f5a54e3
SR
8219 /*
8220 * We need to stop all tracing on all CPUS to read the
8221 * the next buffer. This is a bit expensive, but is
8222 * not done often. We fill all what we can read,
8223 * and then release the locks again.
8224 */
8225
3f5a54e3
SR
8226 while (!trace_empty(&iter)) {
8227
8228 if (!cnt)
8229 printk(KERN_TRACE "---------------------------------\n");
8230
8231 cnt++;
8232
8233 /* reset all but tr, trace, and overruns */
8234 memset(&iter.seq, 0,
8235 sizeof(struct trace_iterator) -
8236 offsetof(struct trace_iterator, seq));
8237 iter.iter_flags |= TRACE_FILE_LAT_FMT;
8238 iter.pos = -1;
8239
955b61e5 8240 if (trace_find_next_entry_inc(&iter) != NULL) {
74e7ff8c
LJ
8241 int ret;
8242
8243 ret = print_trace_line(&iter);
8244 if (ret != TRACE_TYPE_NO_CONSUME)
8245 trace_consume(&iter);
3f5a54e3 8246 }
b892e5c8 8247 touch_nmi_watchdog();
3f5a54e3
SR
8248
8249 trace_printk_seq(&iter.seq);
8250 }
8251
8252 if (!cnt)
8253 printk(KERN_TRACE " (ftrace buffer empty)\n");
8254 else
8255 printk(KERN_TRACE "---------------------------------\n");
8256
cecbca96 8257 out_enable:
983f938a 8258 tr->trace_flags |= old_userobj;
cf586b61 8259
7fe70b57
SRRH
8260 for_each_tracing_cpu(cpu) {
8261 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
cf586b61 8262 }
7fe70b57 8263 atomic_dec(&dump_running);
cd891ae0 8264 local_irq_restore(flags);
3f5a54e3 8265}
a8eecf22 8266EXPORT_SYMBOL_GPL(ftrace_dump);
cf586b61 8267
7e465baa
TZ
8268int trace_run_command(const char *buf, int (*createfn)(int, char **))
8269{
8270 char **argv;
8271 int argc, ret;
8272
8273 argc = 0;
8274 ret = 0;
8275 argv = argv_split(GFP_KERNEL, buf, &argc);
8276 if (!argv)
8277 return -ENOMEM;
8278
8279 if (argc)
8280 ret = createfn(argc, argv);
8281
8282 argv_free(argv);
8283
8284 return ret;
8285}
8286
8287#define WRITE_BUFSIZE 4096
8288
8289ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
8290 size_t count, loff_t *ppos,
8291 int (*createfn)(int, char **))
8292{
8293 char *kbuf, *buf, *tmp;
8294 int ret = 0;
8295 size_t done = 0;
8296 size_t size;
8297
8298 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
8299 if (!kbuf)
8300 return -ENOMEM;
8301
8302 while (done < count) {
8303 size = count - done;
8304
8305 if (size >= WRITE_BUFSIZE)
8306 size = WRITE_BUFSIZE - 1;
8307
8308 if (copy_from_user(kbuf, buffer + done, size)) {
8309 ret = -EFAULT;
8310 goto out;
8311 }
8312 kbuf[size] = '\0';
8313 buf = kbuf;
8314 do {
8315 tmp = strchr(buf, '\n');
8316 if (tmp) {
8317 *tmp = '\0';
8318 size = tmp - buf + 1;
8319 } else {
8320 size = strlen(buf);
8321 if (done + size < count) {
8322 if (buf != kbuf)
8323 break;
8324 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
8325 pr_warn("Line length is too long: Should be less than %d\n",
8326 WRITE_BUFSIZE - 2);
8327 ret = -EINVAL;
8328 goto out;
8329 }
8330 }
8331 done += size;
8332
8333 /* Remove comments */
8334 tmp = strchr(buf, '#');
8335
8336 if (tmp)
8337 *tmp = '\0';
8338
8339 ret = trace_run_command(buf, createfn);
8340 if (ret)
8341 goto out;
8342 buf += size;
8343
8344 } while (done < count);
8345 }
8346 ret = done;
8347
8348out:
8349 kfree(kbuf);
8350
8351 return ret;
8352}
8353
3928a8a2 8354__init static int tracer_alloc_buffers(void)
bc0c38d1 8355{
73c5162a 8356 int ring_buf_size;
9e01c1b7 8357 int ret = -ENOMEM;
4c11d7ae 8358
b5e87c05
SRRH
8359 /*
8360 * Make sure we don't accidently add more trace options
8361 * than we have bits for.
8362 */
9a38a885 8363 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
b5e87c05 8364
9e01c1b7
RR
8365 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
8366 goto out;
8367
ccfe9e42 8368 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
9e01c1b7 8369 goto out_free_buffer_mask;
4c11d7ae 8370
07d777fe
SR
8371 /* Only allocate trace_printk buffers if a trace_printk exists */
8372 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
81698831 8373 /* Must be called before global_trace.buffer is allocated */
07d777fe
SR
8374 trace_printk_init_buffers();
8375
73c5162a
SR
8376 /* To save memory, keep the ring buffer size to its minimum */
8377 if (ring_buffer_expanded)
8378 ring_buf_size = trace_buf_size;
8379 else
8380 ring_buf_size = 1;
8381
9e01c1b7 8382 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
ccfe9e42 8383 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
9e01c1b7 8384
2b6080f2
SR
8385 raw_spin_lock_init(&global_trace.start_lock);
8386
b32614c0
SAS
8387 /*
8388 * The prepare callbacks allocates some memory for the ring buffer. We
8389 * don't free the buffer if the if the CPU goes down. If we were to free
8390 * the buffer, then the user would lose any trace that was in the
8391 * buffer. The memory will be removed once the "instance" is removed.
8392 */
8393 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
8394 "trace/RB:preapre", trace_rb_cpu_prepare,
8395 NULL);
8396 if (ret < 0)
8397 goto out_free_cpumask;
2c4a33ab 8398 /* Used for event triggers */
147d88e0 8399 ret = -ENOMEM;
2c4a33ab
SRRH
8400 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
8401 if (!temp_buffer)
b32614c0 8402 goto out_rm_hp_state;
2c4a33ab 8403
939c7a4f
YY
8404 if (trace_create_savedcmd() < 0)
8405 goto out_free_temp_buffer;
8406
9e01c1b7 8407 /* TODO: make the number of buffers hot pluggable with CPUS */
737223fb 8408 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
3928a8a2
SR
8409 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
8410 WARN_ON(1);
939c7a4f 8411 goto out_free_savedcmd;
4c11d7ae 8412 }
a7603ff4 8413
499e5470
SR
8414 if (global_trace.buffer_disabled)
8415 tracing_off();
4c11d7ae 8416
e1e232ca
SR
8417 if (trace_boot_clock) {
8418 ret = tracing_set_clock(&global_trace, trace_boot_clock);
8419 if (ret < 0)
a395d6a7
JP
8420 pr_warn("Trace clock %s not defined, going back to default\n",
8421 trace_boot_clock);
e1e232ca
SR
8422 }
8423
ca164318
SRRH
8424 /*
8425 * register_tracer() might reference current_trace, so it
8426 * needs to be set before we register anything. This is
8427 * just a bootstrap of current_trace anyway.
8428 */
2b6080f2
SR
8429 global_trace.current_trace = &nop_trace;
8430
0b9b12c1
SRRH
8431 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8432
4104d326
SRRH
8433 ftrace_init_global_array_ops(&global_trace);
8434
9a38a885
SRRH
8435 init_trace_flags_index(&global_trace);
8436
ca164318
SRRH
8437 register_tracer(&nop_trace);
8438
dbeafd0d
SRV
8439 /* Function tracing may start here (via kernel command line) */
8440 init_function_trace();
8441
60a11774
SR
8442 /* All seems OK, enable tracing */
8443 tracing_disabled = 0;
3928a8a2 8444
3f5a54e3
SR
8445 atomic_notifier_chain_register(&panic_notifier_list,
8446 &trace_panic_notifier);
8447
8448 register_die_notifier(&trace_die_notifier);
2fc1dfbe 8449
ae63b31e
SR
8450 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
8451
8452 INIT_LIST_HEAD(&global_trace.systems);
8453 INIT_LIST_HEAD(&global_trace.events);
8454 list_add(&global_trace.list, &ftrace_trace_arrays);
8455
a4d1e688 8456 apply_trace_boot_options();
7bcfaf54 8457
77fd5c15
SRRH
8458 register_snapshot_cmd();
8459
2fc1dfbe 8460 return 0;
3f5a54e3 8461
939c7a4f
YY
8462out_free_savedcmd:
8463 free_saved_cmdlines_buffer(savedcmd);
2c4a33ab
SRRH
8464out_free_temp_buffer:
8465 ring_buffer_free(temp_buffer);
b32614c0
SAS
8466out_rm_hp_state:
8467 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
9e01c1b7 8468out_free_cpumask:
ccfe9e42 8469 free_cpumask_var(global_trace.tracing_cpumask);
9e01c1b7
RR
8470out_free_buffer_mask:
8471 free_cpumask_var(tracing_buffer_mask);
8472out:
8473 return ret;
bc0c38d1 8474}
b2821ae6 8475
e725c731 8476void __init early_trace_init(void)
5f893b26 8477{
0daa2302
SRRH
8478 if (tracepoint_printk) {
8479 tracepoint_print_iter =
8480 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8481 if (WARN_ON(!tracepoint_print_iter))
8482 tracepoint_printk = 0;
42391745
SRRH
8483 else
8484 static_key_enable(&tracepoint_printk_key.key);
0daa2302 8485 }
5f893b26 8486 tracer_alloc_buffers();
e725c731
SRV
8487}
8488
8489void __init trace_init(void)
8490{
0c564a53 8491 trace_event_init();
5f893b26
SRRH
8492}
8493
b2821ae6
SR
8494__init static int clear_boot_tracer(void)
8495{
8496 /*
8497 * The default tracer at boot buffer is an init section.
8498 * This function is called in lateinit. If we did not
8499 * find the boot tracer, then clear it out, to prevent
8500 * later registration from accessing the buffer that is
8501 * about to be freed.
8502 */
8503 if (!default_bootup_tracer)
8504 return 0;
8505
8506 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
8507 default_bootup_tracer);
8508 default_bootup_tracer = NULL;
8509
8510 return 0;
8511}
8512
8434dc93 8513fs_initcall(tracer_init_tracefs);
4bb0f0e7 8514late_initcall_sync(clear_boot_tracer);