1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer based function tracer
5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
8 * Originally taken from the RT patch by:
9 * Arnaldo Carvalho de Melo <acme@redhat.com>
11 * Based on code from the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 Nadia Yvette Chambers
15 #include <linux/ring_buffer.h>
16 #include <generated/utsrelease.h>
17 #include <linux/stacktrace.h>
18 #include <linux/writeback.h>
19 #include <linux/kallsyms.h>
20 #include <linux/security.h>
21 #include <linux/seq_file.h>
22 #include <linux/notifier.h>
23 #include <linux/irqflags.h>
24 #include <linux/debugfs.h>
25 #include <linux/tracefs.h>
26 #include <linux/pagemap.h>
27 #include <linux/hardirq.h>
28 #include <linux/linkage.h>
29 #include <linux/uaccess.h>
30 #include <linux/vmalloc.h>
31 #include <linux/ftrace.h>
32 #include <linux/module.h>
33 #include <linux/percpu.h>
34 #include <linux/splice.h>
35 #include <linux/kdebug.h>
36 #include <linux/string.h>
37 #include <linux/mount.h>
38 #include <linux/rwsem.h>
39 #include <linux/slab.h>
40 #include <linux/ctype.h>
41 #include <linux/init.h>
42 #include <linux/poll.h>
43 #include <linux/nmi.h>
45 #include <linux/trace.h>
46 #include <linux/sched/clock.h>
47 #include <linux/sched/rt.h>
50 #include "trace_output.h"
53 * On boot up, the ring buffer is set to the minimum size, so that
54 * we do not waste memory on systems that are not using tracing.
56 bool ring_buffer_expanded
;
59 * We need to change this state when a selftest is running.
60 * A selftest will lurk into the ring-buffer to count the
61 * entries inserted during the selftest although some concurrent
62 * insertions into the ring-buffer such as trace_printk could occurred
63 * at the same time, giving false positive or negative results.
65 static bool __read_mostly tracing_selftest_running
;
68 * If a tracer is running, we do not want to run SELFTEST.
70 bool __read_mostly tracing_selftest_disabled
;
72 /* Pipe tracepoints to printk */
73 struct trace_iterator
*tracepoint_print_iter
;
74 int tracepoint_printk
;
75 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key
);
77 /* For tracers that don't implement custom flags */
78 static struct tracer_opt dummy_tracer_opt
[] = {
83 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
89 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
93 static DEFINE_PER_CPU(bool, trace_taskinfo_save
);
96 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
101 static int tracing_disabled
= 1;
103 cpumask_var_t __read_mostly tracing_buffer_mask
;
106 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
108 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
109 * is set, then ftrace_dump is called. This will output the contents
110 * of the ftrace buffers to the console. This is very useful for
111 * capturing traces that lead to crashes and outputing it to a
114 * It is default off, but you can enable it with either specifying
115 * "ftrace_dump_on_oops" in the kernel command line, or setting
116 * /proc/sys/kernel/ftrace_dump_on_oops
117 * Set 1 if you want to dump buffers of all CPUs
118 * Set 2 if you want to dump the buffer of the CPU that triggered oops
121 enum ftrace_dump_mode ftrace_dump_on_oops
;
123 /* When set, tracing will stop when a WARN*() is hit */
124 int __disable_trace_on_warning
;
126 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
127 /* Map of enums to their values, for "eval_map" file */
128 struct trace_eval_map_head
{
130 unsigned long length
;
133 union trace_eval_map_item
;
135 struct trace_eval_map_tail
{
137 * "end" is first and points to NULL as it must be different
138 * than "mod" or "eval_string"
140 union trace_eval_map_item
*next
;
141 const char *end
; /* points to NULL */
144 static DEFINE_MUTEX(trace_eval_mutex
);
147 * The trace_eval_maps are saved in an array with two extra elements,
148 * one at the beginning, and one at the end. The beginning item contains
149 * the count of the saved maps (head.length), and the module they
150 * belong to if not built in (head.mod). The ending item contains a
151 * pointer to the next array of saved eval_map items.
153 union trace_eval_map_item
{
154 struct trace_eval_map map
;
155 struct trace_eval_map_head head
;
156 struct trace_eval_map_tail tail
;
159 static union trace_eval_map_item
*trace_eval_maps
;
160 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
162 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
163 static void ftrace_trace_userstack(struct ring_buffer
*buffer
,
164 unsigned long flags
, int pc
);
166 #define MAX_TRACER_SIZE 100
167 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
168 static char *default_bootup_tracer
;
170 static bool allocate_snapshot
;
172 static int __init
set_cmdline_ftrace(char *str
)
174 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
175 default_bootup_tracer
= bootup_tracer_buf
;
176 /* We are using ftrace early, expand it */
177 ring_buffer_expanded
= true;
180 __setup("ftrace=", set_cmdline_ftrace
);
182 static int __init
set_ftrace_dump_on_oops(char *str
)
184 if (*str
++ != '=' || !*str
) {
185 ftrace_dump_on_oops
= DUMP_ALL
;
189 if (!strcmp("orig_cpu", str
)) {
190 ftrace_dump_on_oops
= DUMP_ORIG
;
196 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
198 static int __init
stop_trace_on_warning(char *str
)
200 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
201 __disable_trace_on_warning
= 1;
204 __setup("traceoff_on_warning", stop_trace_on_warning
);
206 static int __init
boot_alloc_snapshot(char *str
)
208 allocate_snapshot
= true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded
= true;
213 __setup("alloc_snapshot", boot_alloc_snapshot
);
216 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
218 static int __init
set_trace_boot_options(char *str
)
220 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
223 __setup("trace_options=", set_trace_boot_options
);
225 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
226 static char *trace_boot_clock __initdata
;
228 static int __init
set_trace_boot_clock(char *str
)
230 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
231 trace_boot_clock
= trace_boot_clock_buf
;
234 __setup("trace_clock=", set_trace_boot_clock
);
236 static int __init
set_tracepoint_printk(char *str
)
238 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
239 tracepoint_printk
= 1;
242 __setup("tp_printk", set_tracepoint_printk
);
244 unsigned long long ns2usecs(u64 nsec
)
251 /* trace_flags holds trace_options default values */
252 #define TRACE_DEFAULT_FLAGS \
253 (FUNCTION_DEFAULT_FLAGS | \
254 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
255 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
256 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
257 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
259 /* trace_options that are only supported by global_trace */
260 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
261 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
263 /* trace_flags that are default zero for instances */
264 #define ZEROED_TRACE_FLAGS \
265 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
268 * The global_trace is the descriptor that holds the top-level tracing
269 * buffers for the live tracing.
271 static struct trace_array global_trace
= {
272 .trace_flags
= TRACE_DEFAULT_FLAGS
,
275 LIST_HEAD(ftrace_trace_arrays
);
277 int trace_array_get(struct trace_array
*this_tr
)
279 struct trace_array
*tr
;
282 mutex_lock(&trace_types_lock
);
283 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
290 mutex_unlock(&trace_types_lock
);
295 static void __trace_array_put(struct trace_array
*this_tr
)
297 WARN_ON(!this_tr
->ref
);
301 void trace_array_put(struct trace_array
*this_tr
)
303 mutex_lock(&trace_types_lock
);
304 __trace_array_put(this_tr
);
305 mutex_unlock(&trace_types_lock
);
308 int tracing_check_open_get_tr(struct trace_array
*tr
)
312 ret
= security_locked_down(LOCKDOWN_TRACEFS
);
316 if (tracing_disabled
)
319 if (tr
&& trace_array_get(tr
) < 0)
325 int call_filter_check_discard(struct trace_event_call
*call
, void *rec
,
326 struct ring_buffer
*buffer
,
327 struct ring_buffer_event
*event
)
329 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
330 !filter_match_preds(call
->filter
, rec
)) {
331 __trace_event_discard_commit(buffer
, event
);
338 void trace_free_pid_list(struct trace_pid_list
*pid_list
)
340 vfree(pid_list
->pids
);
345 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
346 * @filtered_pids: The list of pids to check
347 * @search_pid: The PID to find in @filtered_pids
349 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
352 trace_find_filtered_pid(struct trace_pid_list
*filtered_pids
, pid_t search_pid
)
355 * If pid_max changed after filtered_pids was created, we
356 * by default ignore all pids greater than the previous pid_max.
358 if (search_pid
>= filtered_pids
->pid_max
)
361 return test_bit(search_pid
, filtered_pids
->pids
);
365 * trace_ignore_this_task - should a task be ignored for tracing
366 * @filtered_pids: The list of pids to check
367 * @task: The task that should be ignored if not filtered
369 * Checks if @task should be traced or not from @filtered_pids.
370 * Returns true if @task should *NOT* be traced.
371 * Returns false if @task should be traced.
374 trace_ignore_this_task(struct trace_pid_list
*filtered_pids
, struct task_struct
*task
)
377 * Return false, because if filtered_pids does not exist,
378 * all pids are good to trace.
383 return !trace_find_filtered_pid(filtered_pids
, task
->pid
);
387 * trace_filter_add_remove_task - Add or remove a task from a pid_list
388 * @pid_list: The list to modify
389 * @self: The current task for fork or NULL for exit
390 * @task: The task to add or remove
392 * If adding a task, if @self is defined, the task is only added if @self
393 * is also included in @pid_list. This happens on fork and tasks should
394 * only be added when the parent is listed. If @self is NULL, then the
395 * @task pid will be removed from the list, which would happen on exit
398 void trace_filter_add_remove_task(struct trace_pid_list
*pid_list
,
399 struct task_struct
*self
,
400 struct task_struct
*task
)
405 /* For forks, we only add if the forking task is listed */
407 if (!trace_find_filtered_pid(pid_list
, self
->pid
))
411 /* Sorry, but we don't support pid_max changing after setting */
412 if (task
->pid
>= pid_list
->pid_max
)
415 /* "self" is set for forks, and NULL for exits */
417 set_bit(task
->pid
, pid_list
->pids
);
419 clear_bit(task
->pid
, pid_list
->pids
);
423 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
424 * @pid_list: The pid list to show
425 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
426 * @pos: The position of the file
428 * This is used by the seq_file "next" operation to iterate the pids
429 * listed in a trace_pid_list structure.
431 * Returns the pid+1 as we want to display pid of zero, but NULL would
432 * stop the iteration.
434 void *trace_pid_next(struct trace_pid_list
*pid_list
, void *v
, loff_t
*pos
)
436 unsigned long pid
= (unsigned long)v
;
440 /* pid already is +1 of the actual prevous bit */
441 pid
= find_next_bit(pid_list
->pids
, pid_list
->pid_max
, pid
);
443 /* Return pid + 1 to allow zero to be represented */
444 if (pid
< pid_list
->pid_max
)
445 return (void *)(pid
+ 1);
451 * trace_pid_start - Used for seq_file to start reading pid lists
452 * @pid_list: The pid list to show
453 * @pos: The position of the file
455 * This is used by seq_file "start" operation to start the iteration
458 * Returns the pid+1 as we want to display pid of zero, but NULL would
459 * stop the iteration.
461 void *trace_pid_start(struct trace_pid_list
*pid_list
, loff_t
*pos
)
466 pid
= find_first_bit(pid_list
->pids
, pid_list
->pid_max
);
467 if (pid
>= pid_list
->pid_max
)
470 /* Return pid + 1 so that zero can be the exit value */
471 for (pid
++; pid
&& l
< *pos
;
472 pid
= (unsigned long)trace_pid_next(pid_list
, (void *)pid
, &l
))
478 * trace_pid_show - show the current pid in seq_file processing
479 * @m: The seq_file structure to write into
480 * @v: A void pointer of the pid (+1) value to display
482 * Can be directly used by seq_file operations to display the current
485 int trace_pid_show(struct seq_file
*m
, void *v
)
487 unsigned long pid
= (unsigned long)v
- 1;
489 seq_printf(m
, "%lu\n", pid
);
493 /* 128 should be much more than enough */
494 #define PID_BUF_SIZE 127
496 int trace_pid_write(struct trace_pid_list
*filtered_pids
,
497 struct trace_pid_list
**new_pid_list
,
498 const char __user
*ubuf
, size_t cnt
)
500 struct trace_pid_list
*pid_list
;
501 struct trace_parser parser
;
509 if (trace_parser_get_init(&parser
, PID_BUF_SIZE
+ 1))
513 * Always recreate a new array. The write is an all or nothing
514 * operation. Always create a new array when adding new pids by
515 * the user. If the operation fails, then the current list is
518 pid_list
= kmalloc(sizeof(*pid_list
), GFP_KERNEL
);
520 trace_parser_put(&parser
);
524 pid_list
->pid_max
= READ_ONCE(pid_max
);
526 /* Only truncating will shrink pid_max */
527 if (filtered_pids
&& filtered_pids
->pid_max
> pid_list
->pid_max
)
528 pid_list
->pid_max
= filtered_pids
->pid_max
;
530 pid_list
->pids
= vzalloc((pid_list
->pid_max
+ 7) >> 3);
531 if (!pid_list
->pids
) {
532 trace_parser_put(&parser
);
538 /* copy the current bits to the new max */
539 for_each_set_bit(pid
, filtered_pids
->pids
,
540 filtered_pids
->pid_max
) {
541 set_bit(pid
, pid_list
->pids
);
550 ret
= trace_get_user(&parser
, ubuf
, cnt
, &pos
);
551 if (ret
< 0 || !trace_parser_loaded(&parser
))
559 if (kstrtoul(parser
.buffer
, 0, &val
))
561 if (val
>= pid_list
->pid_max
)
566 set_bit(pid
, pid_list
->pids
);
569 trace_parser_clear(&parser
);
572 trace_parser_put(&parser
);
575 trace_free_pid_list(pid_list
);
580 /* Cleared the list of pids */
581 trace_free_pid_list(pid_list
);
586 *new_pid_list
= pid_list
;
591 static u64
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
595 /* Early boot up does not have a buffer yet */
597 return trace_clock_local();
599 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
600 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
605 u64
ftrace_now(int cpu
)
607 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
611 * tracing_is_enabled - Show if global_trace has been disabled
613 * Shows if the global trace has been enabled or not. It uses the
614 * mirror flag "buffer_disabled" to be used in fast paths such as for
615 * the irqsoff tracer. But it may be inaccurate due to races. If you
616 * need to know the accurate state, use tracing_is_on() which is a little
617 * slower, but accurate.
619 int tracing_is_enabled(void)
622 * For quick access (irqsoff uses this in fast path), just
623 * return the mirror variable of the state of the ring buffer.
624 * It's a little racy, but we don't really care.
627 return !global_trace
.buffer_disabled
;
631 * trace_buf_size is the size in bytes that is allocated
632 * for a buffer. Note, the number of bytes is always rounded
635 * This number is purposely set to a low number of 16384.
636 * If the dump on oops happens, it will be much appreciated
637 * to not have to wait for all that output. Anyway this can be
638 * boot time and run time configurable.
640 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
642 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
644 /* trace_types holds a link list of available tracers. */
645 static struct tracer
*trace_types __read_mostly
;
648 * trace_types_lock is used to protect the trace_types list.
650 DEFINE_MUTEX(trace_types_lock
);
653 * serialize the access of the ring buffer
655 * ring buffer serializes readers, but it is low level protection.
656 * The validity of the events (which returns by ring_buffer_peek() ..etc)
657 * are not protected by ring buffer.
659 * The content of events may become garbage if we allow other process consumes
660 * these events concurrently:
661 * A) the page of the consumed events may become a normal page
662 * (not reader page) in ring buffer, and this page will be rewrited
663 * by events producer.
664 * B) The page of the consumed events may become a page for splice_read,
665 * and this page will be returned to system.
667 * These primitives allow multi process access to different cpu ring buffer
670 * These primitives don't distinguish read-only and read-consume access.
671 * Multi read-only access are also serialized.
675 static DECLARE_RWSEM(all_cpu_access_lock
);
676 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
678 static inline void trace_access_lock(int cpu
)
680 if (cpu
== RING_BUFFER_ALL_CPUS
) {
681 /* gain it for accessing the whole ring buffer. */
682 down_write(&all_cpu_access_lock
);
684 /* gain it for accessing a cpu ring buffer. */
686 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
687 down_read(&all_cpu_access_lock
);
689 /* Secondly block other access to this @cpu ring buffer. */
690 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
694 static inline void trace_access_unlock(int cpu
)
696 if (cpu
== RING_BUFFER_ALL_CPUS
) {
697 up_write(&all_cpu_access_lock
);
699 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
700 up_read(&all_cpu_access_lock
);
704 static inline void trace_access_lock_init(void)
708 for_each_possible_cpu(cpu
)
709 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
714 static DEFINE_MUTEX(access_lock
);
716 static inline void trace_access_lock(int cpu
)
719 mutex_lock(&access_lock
);
722 static inline void trace_access_unlock(int cpu
)
725 mutex_unlock(&access_lock
);
728 static inline void trace_access_lock_init(void)
734 #ifdef CONFIG_STACKTRACE
735 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
737 int skip
, int pc
, struct pt_regs
*regs
);
738 static inline void ftrace_trace_stack(struct trace_array
*tr
,
739 struct ring_buffer
*buffer
,
741 int skip
, int pc
, struct pt_regs
*regs
);
744 static inline void __ftrace_trace_stack(struct ring_buffer
*buffer
,
746 int skip
, int pc
, struct pt_regs
*regs
)
749 static inline void ftrace_trace_stack(struct trace_array
*tr
,
750 struct ring_buffer
*buffer
,
752 int skip
, int pc
, struct pt_regs
*regs
)
758 static __always_inline
void
759 trace_event_setup(struct ring_buffer_event
*event
,
760 int type
, unsigned long flags
, int pc
)
762 struct trace_entry
*ent
= ring_buffer_event_data(event
);
764 tracing_generic_entry_update(ent
, type
, flags
, pc
);
767 static __always_inline
struct ring_buffer_event
*
768 __trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
771 unsigned long flags
, int pc
)
773 struct ring_buffer_event
*event
;
775 event
= ring_buffer_lock_reserve(buffer
, len
);
777 trace_event_setup(event
, type
, flags
, pc
);
782 void tracer_tracing_on(struct trace_array
*tr
)
784 if (tr
->trace_buffer
.buffer
)
785 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
787 * This flag is looked at when buffers haven't been allocated
788 * yet, or by some tracers (like irqsoff), that just want to
789 * know if the ring buffer has been disabled, but it can handle
790 * races of where it gets disabled but we still do a record.
791 * As the check is in the fast path of the tracers, it is more
792 * important to be fast than accurate.
794 tr
->buffer_disabled
= 0;
795 /* Make the flag seen by readers */
800 * tracing_on - enable tracing buffers
802 * This function enables tracing buffers that may have been
803 * disabled with tracing_off.
805 void tracing_on(void)
807 tracer_tracing_on(&global_trace
);
809 EXPORT_SYMBOL_GPL(tracing_on
);
812 static __always_inline
void
813 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
815 __this_cpu_write(trace_taskinfo_save
, true);
817 /* If this is the temp buffer, we need to commit fully */
818 if (this_cpu_read(trace_buffered_event
) == event
) {
819 /* Length is in event->array[0] */
820 ring_buffer_write(buffer
, event
->array
[0], &event
->array
[1]);
821 /* Release the temp buffer */
822 this_cpu_dec(trace_buffered_event_cnt
);
824 ring_buffer_unlock_commit(buffer
, event
);
828 * __trace_puts - write a constant string into the trace buffer.
829 * @ip: The address of the caller
830 * @str: The constant string to write
831 * @size: The size of the string.
833 int __trace_puts(unsigned long ip
, const char *str
, int size
)
835 struct ring_buffer_event
*event
;
836 struct ring_buffer
*buffer
;
837 struct print_entry
*entry
;
838 unsigned long irq_flags
;
842 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
845 pc
= preempt_count();
847 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
850 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
852 local_save_flags(irq_flags
);
853 buffer
= global_trace
.trace_buffer
.buffer
;
854 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
859 entry
= ring_buffer_event_data(event
);
862 memcpy(&entry
->buf
, str
, size
);
864 /* Add a newline if necessary */
865 if (entry
->buf
[size
- 1] != '\n') {
866 entry
->buf
[size
] = '\n';
867 entry
->buf
[size
+ 1] = '\0';
869 entry
->buf
[size
] = '\0';
871 __buffer_unlock_commit(buffer
, event
);
872 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
876 EXPORT_SYMBOL_GPL(__trace_puts
);
879 * __trace_bputs - write the pointer to a constant string into trace buffer
880 * @ip: The address of the caller
881 * @str: The constant string to write to the buffer to
883 int __trace_bputs(unsigned long ip
, const char *str
)
885 struct ring_buffer_event
*event
;
886 struct ring_buffer
*buffer
;
887 struct bputs_entry
*entry
;
888 unsigned long irq_flags
;
889 int size
= sizeof(struct bputs_entry
);
892 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
895 pc
= preempt_count();
897 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
900 local_save_flags(irq_flags
);
901 buffer
= global_trace
.trace_buffer
.buffer
;
902 event
= __trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
907 entry
= ring_buffer_event_data(event
);
911 __buffer_unlock_commit(buffer
, event
);
912 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
916 EXPORT_SYMBOL_GPL(__trace_bputs
);
918 #ifdef CONFIG_TRACER_SNAPSHOT
919 void tracing_snapshot_instance_cond(struct trace_array
*tr
, void *cond_data
)
921 struct tracer
*tracer
= tr
->current_trace
;
925 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
926 internal_trace_puts("*** snapshot is being ignored ***\n");
930 if (!tr
->allocated_snapshot
) {
931 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
932 internal_trace_puts("*** stopping trace here! ***\n");
937 /* Note, snapshot can not be used when the tracer uses it */
938 if (tracer
->use_max_tr
) {
939 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
940 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
944 local_irq_save(flags
);
945 update_max_tr(tr
, current
, smp_processor_id(), cond_data
);
946 local_irq_restore(flags
);
949 void tracing_snapshot_instance(struct trace_array
*tr
)
951 tracing_snapshot_instance_cond(tr
, NULL
);
955 * tracing_snapshot - take a snapshot of the current buffer.
957 * This causes a swap between the snapshot buffer and the current live
958 * tracing buffer. You can use this to take snapshots of the live
959 * trace when some condition is triggered, but continue to trace.
961 * Note, make sure to allocate the snapshot with either
962 * a tracing_snapshot_alloc(), or by doing it manually
963 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
965 * If the snapshot buffer is not allocated, it will stop tracing.
966 * Basically making a permanent snapshot.
968 void tracing_snapshot(void)
970 struct trace_array
*tr
= &global_trace
;
972 tracing_snapshot_instance(tr
);
974 EXPORT_SYMBOL_GPL(tracing_snapshot
);
977 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
978 * @tr: The tracing instance to snapshot
979 * @cond_data: The data to be tested conditionally, and possibly saved
981 * This is the same as tracing_snapshot() except that the snapshot is
982 * conditional - the snapshot will only happen if the
983 * cond_snapshot.update() implementation receiving the cond_data
984 * returns true, which means that the trace array's cond_snapshot
985 * update() operation used the cond_data to determine whether the
986 * snapshot should be taken, and if it was, presumably saved it along
989 void tracing_snapshot_cond(struct trace_array
*tr
, void *cond_data
)
991 tracing_snapshot_instance_cond(tr
, cond_data
);
993 EXPORT_SYMBOL_GPL(tracing_snapshot_cond
);
996 * tracing_snapshot_cond_data - get the user data associated with a snapshot
997 * @tr: The tracing instance
999 * When the user enables a conditional snapshot using
1000 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1001 * with the snapshot. This accessor is used to retrieve it.
1003 * Should not be called from cond_snapshot.update(), since it takes
1004 * the tr->max_lock lock, which the code calling
1005 * cond_snapshot.update() has already done.
1007 * Returns the cond_data associated with the trace array's snapshot.
1009 void *tracing_cond_snapshot_data(struct trace_array
*tr
)
1011 void *cond_data
= NULL
;
1013 arch_spin_lock(&tr
->max_lock
);
1015 if (tr
->cond_snapshot
)
1016 cond_data
= tr
->cond_snapshot
->cond_data
;
1018 arch_spin_unlock(&tr
->max_lock
);
1022 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data
);
1024 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
1025 struct trace_buffer
*size_buf
, int cpu_id
);
1026 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
1028 int tracing_alloc_snapshot_instance(struct trace_array
*tr
)
1032 if (!tr
->allocated_snapshot
) {
1034 /* allocate spare buffer */
1035 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
1036 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
1040 tr
->allocated_snapshot
= true;
1046 static void free_snapshot(struct trace_array
*tr
)
1049 * We don't free the ring buffer. instead, resize it because
1050 * The max_tr ring buffer has some state (e.g. ring->clock) and
1051 * we want preserve it.
1053 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
1054 set_buffer_entries(&tr
->max_buffer
, 1);
1055 tracing_reset_online_cpus(&tr
->max_buffer
);
1056 tr
->allocated_snapshot
= false;
1060 * tracing_alloc_snapshot - allocate snapshot buffer.
1062 * This only allocates the snapshot buffer if it isn't already
1063 * allocated - it doesn't also take a snapshot.
1065 * This is meant to be used in cases where the snapshot buffer needs
1066 * to be set up for events that can't sleep but need to be able to
1067 * trigger a snapshot.
1069 int tracing_alloc_snapshot(void)
1071 struct trace_array
*tr
= &global_trace
;
1074 ret
= tracing_alloc_snapshot_instance(tr
);
1079 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
1082 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1084 * This is similar to tracing_snapshot(), but it will allocate the
1085 * snapshot buffer if it isn't already allocated. Use this only
1086 * where it is safe to sleep, as the allocation may sleep.
1088 * This causes a swap between the snapshot buffer and the current live
1089 * tracing buffer. You can use this to take snapshots of the live
1090 * trace when some condition is triggered, but continue to trace.
1092 void tracing_snapshot_alloc(void)
1096 ret
= tracing_alloc_snapshot();
1102 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
1105 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1106 * @tr: The tracing instance
1107 * @cond_data: User data to associate with the snapshot
1108 * @update: Implementation of the cond_snapshot update function
1110 * Check whether the conditional snapshot for the given instance has
1111 * already been enabled, or if the current tracer is already using a
1112 * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1113 * save the cond_data and update function inside.
1115 * Returns 0 if successful, error otherwise.
1117 int tracing_snapshot_cond_enable(struct trace_array
*tr
, void *cond_data
,
1118 cond_update_fn_t update
)
1120 struct cond_snapshot
*cond_snapshot
;
1123 cond_snapshot
= kzalloc(sizeof(*cond_snapshot
), GFP_KERNEL
);
1127 cond_snapshot
->cond_data
= cond_data
;
1128 cond_snapshot
->update
= update
;
1130 mutex_lock(&trace_types_lock
);
1132 ret
= tracing_alloc_snapshot_instance(tr
);
1136 if (tr
->current_trace
->use_max_tr
) {
1142 * The cond_snapshot can only change to NULL without the
1143 * trace_types_lock. We don't care if we race with it going
1144 * to NULL, but we want to make sure that it's not set to
1145 * something other than NULL when we get here, which we can
1146 * do safely with only holding the trace_types_lock and not
1147 * having to take the max_lock.
1149 if (tr
->cond_snapshot
) {
1154 arch_spin_lock(&tr
->max_lock
);
1155 tr
->cond_snapshot
= cond_snapshot
;
1156 arch_spin_unlock(&tr
->max_lock
);
1158 mutex_unlock(&trace_types_lock
);
1163 mutex_unlock(&trace_types_lock
);
1164 kfree(cond_snapshot
);
1167 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable
);
1170 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1171 * @tr: The tracing instance
1173 * Check whether the conditional snapshot for the given instance is
1174 * enabled; if so, free the cond_snapshot associated with it,
1175 * otherwise return -EINVAL.
1177 * Returns 0 if successful, error otherwise.
1179 int tracing_snapshot_cond_disable(struct trace_array
*tr
)
1183 arch_spin_lock(&tr
->max_lock
);
1185 if (!tr
->cond_snapshot
)
1188 kfree(tr
->cond_snapshot
);
1189 tr
->cond_snapshot
= NULL
;
1192 arch_spin_unlock(&tr
->max_lock
);
1196 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable
);
1198 void tracing_snapshot(void)
1200 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1202 EXPORT_SYMBOL_GPL(tracing_snapshot
);
1203 void tracing_snapshot_cond(struct trace_array
*tr
, void *cond_data
)
1205 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1207 EXPORT_SYMBOL_GPL(tracing_snapshot_cond
);
1208 int tracing_alloc_snapshot(void)
1210 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1213 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
1214 void tracing_snapshot_alloc(void)
1219 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
1220 void *tracing_cond_snapshot_data(struct trace_array
*tr
)
1224 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data
);
1225 int tracing_snapshot_cond_enable(struct trace_array
*tr
, void *cond_data
, cond_update_fn_t update
)
1229 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable
);
1230 int tracing_snapshot_cond_disable(struct trace_array
*tr
)
1234 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable
);
1235 #endif /* CONFIG_TRACER_SNAPSHOT */
1237 void tracer_tracing_off(struct trace_array
*tr
)
1239 if (tr
->trace_buffer
.buffer
)
1240 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
1242 * This flag is looked at when buffers haven't been allocated
1243 * yet, or by some tracers (like irqsoff), that just want to
1244 * know if the ring buffer has been disabled, but it can handle
1245 * races of where it gets disabled but we still do a record.
1246 * As the check is in the fast path of the tracers, it is more
1247 * important to be fast than accurate.
1249 tr
->buffer_disabled
= 1;
1250 /* Make the flag seen by readers */
1255 * tracing_off - turn off tracing buffers
1257 * This function stops the tracing buffers from recording data.
1258 * It does not disable any overhead the tracers themselves may
1259 * be causing. This function simply causes all recording to
1260 * the ring buffers to fail.
1262 void tracing_off(void)
1264 tracer_tracing_off(&global_trace
);
1266 EXPORT_SYMBOL_GPL(tracing_off
);
1268 void disable_trace_on_warning(void)
1270 if (__disable_trace_on_warning
)
1275 * tracer_tracing_is_on - show real state of ring buffer enabled
1276 * @tr : the trace array to know if ring buffer is enabled
1278 * Shows real state of the ring buffer if it is enabled or not.
1280 bool tracer_tracing_is_on(struct trace_array
*tr
)
1282 if (tr
->trace_buffer
.buffer
)
1283 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
1284 return !tr
->buffer_disabled
;
1288 * tracing_is_on - show state of ring buffers enabled
1290 int tracing_is_on(void)
1292 return tracer_tracing_is_on(&global_trace
);
1294 EXPORT_SYMBOL_GPL(tracing_is_on
);
1296 static int __init
set_buf_size(char *str
)
1298 unsigned long buf_size
;
1302 buf_size
= memparse(str
, &str
);
1303 /* nr_entries can not be zero */
1306 trace_buf_size
= buf_size
;
1309 __setup("trace_buf_size=", set_buf_size
);
1311 static int __init
set_tracing_thresh(char *str
)
1313 unsigned long threshold
;
1318 ret
= kstrtoul(str
, 0, &threshold
);
1321 tracing_thresh
= threshold
* 1000;
1324 __setup("tracing_thresh=", set_tracing_thresh
);
1326 unsigned long nsecs_to_usecs(unsigned long nsecs
)
1328 return nsecs
/ 1000;
1332 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1333 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1334 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1335 * of strings in the order that the evals (enum) were defined.
1340 /* These must match the bit postions in trace_iterator_flags */
1341 static const char *trace_options
[] = {
1349 int in_ns
; /* is this clock in nanoseconds? */
1350 } trace_clocks
[] = {
1351 { trace_clock_local
, "local", 1 },
1352 { trace_clock_global
, "global", 1 },
1353 { trace_clock_counter
, "counter", 0 },
1354 { trace_clock_jiffies
, "uptime", 0 },
1355 { trace_clock
, "perf", 1 },
1356 { ktime_get_mono_fast_ns
, "mono", 1 },
1357 { ktime_get_raw_fast_ns
, "mono_raw", 1 },
1358 { ktime_get_boot_fast_ns
, "boot", 1 },
1362 bool trace_clock_in_ns(struct trace_array
*tr
)
1364 if (trace_clocks
[tr
->clock_id
].in_ns
)
1371 * trace_parser_get_init - gets the buffer for trace parser
1373 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
1375 memset(parser
, 0, sizeof(*parser
));
1377 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
1378 if (!parser
->buffer
)
1381 parser
->size
= size
;
1386 * trace_parser_put - frees the buffer for trace parser
1388 void trace_parser_put(struct trace_parser
*parser
)
1390 kfree(parser
->buffer
);
1391 parser
->buffer
= NULL
;
1395 * trace_get_user - reads the user input string separated by space
1396 * (matched by isspace(ch))
1398 * For each string found the 'struct trace_parser' is updated,
1399 * and the function returns.
1401 * Returns number of bytes read.
1403 * See kernel/trace/trace.h for 'struct trace_parser' details.
1405 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
1406 size_t cnt
, loff_t
*ppos
)
1413 trace_parser_clear(parser
);
1415 ret
= get_user(ch
, ubuf
++);
1423 * The parser is not finished with the last write,
1424 * continue reading the user input without skipping spaces.
1426 if (!parser
->cont
) {
1427 /* skip white space */
1428 while (cnt
&& isspace(ch
)) {
1429 ret
= get_user(ch
, ubuf
++);
1438 /* only spaces were written */
1439 if (isspace(ch
) || !ch
) {
1446 /* read the non-space input */
1447 while (cnt
&& !isspace(ch
) && ch
) {
1448 if (parser
->idx
< parser
->size
- 1)
1449 parser
->buffer
[parser
->idx
++] = ch
;
1454 ret
= get_user(ch
, ubuf
++);
1461 /* We either got finished input or we have to wait for another call. */
1462 if (isspace(ch
) || !ch
) {
1463 parser
->buffer
[parser
->idx
] = 0;
1464 parser
->cont
= false;
1465 } else if (parser
->idx
< parser
->size
- 1) {
1466 parser
->cont
= true;
1467 parser
->buffer
[parser
->idx
++] = ch
;
1468 /* Make sure the parsed string always terminates with '\0'. */
1469 parser
->buffer
[parser
->idx
] = 0;
1482 /* TODO add a seq_buf_to_buffer() */
1483 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
1487 if (trace_seq_used(s
) <= s
->seq
.readpos
)
1490 len
= trace_seq_used(s
) - s
->seq
.readpos
;
1493 memcpy(buf
, s
->buffer
+ s
->seq
.readpos
, cnt
);
1495 s
->seq
.readpos
+= cnt
;
1499 unsigned long __read_mostly tracing_thresh
;
1501 #ifdef CONFIG_TRACER_MAX_TRACE
1503 * Copy the new maximum trace into the separate maximum-trace
1504 * structure. (this way the maximum trace is permanently saved,
1505 * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1508 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1510 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
1511 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
1512 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
1513 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
1516 max_buf
->time_start
= data
->preempt_timestamp
;
1518 max_data
->saved_latency
= tr
->max_latency
;
1519 max_data
->critical_start
= data
->critical_start
;
1520 max_data
->critical_end
= data
->critical_end
;
1522 strncpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
1523 max_data
->pid
= tsk
->pid
;
1525 * If tsk == current, then use current_uid(), as that does not use
1526 * RCU. The irq tracer can be called out of RCU scope.
1529 max_data
->uid
= current_uid();
1531 max_data
->uid
= task_uid(tsk
);
1533 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
1534 max_data
->policy
= tsk
->policy
;
1535 max_data
->rt_priority
= tsk
->rt_priority
;
1537 /* record this tasks comm */
1538 tracing_record_cmdline(tsk
);
1542 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1544 * @tsk: the task with the latency
1545 * @cpu: The cpu that initiated the trace.
1546 * @cond_data: User data associated with a conditional snapshot
1548 * Flip the buffers between the @tr and the max_tr and record information
1549 * about which task was the cause of this latency.
1552 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
,
1558 WARN_ON_ONCE(!irqs_disabled());
1560 if (!tr
->allocated_snapshot
) {
1561 /* Only the nop tracer should hit this when disabling */
1562 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1566 arch_spin_lock(&tr
->max_lock
);
1568 /* Inherit the recordable setting from trace_buffer */
1569 if (ring_buffer_record_is_set_on(tr
->trace_buffer
.buffer
))
1570 ring_buffer_record_on(tr
->max_buffer
.buffer
);
1572 ring_buffer_record_off(tr
->max_buffer
.buffer
);
1574 #ifdef CONFIG_TRACER_SNAPSHOT
1575 if (tr
->cond_snapshot
&& !tr
->cond_snapshot
->update(tr
, cond_data
))
1578 swap(tr
->trace_buffer
.buffer
, tr
->max_buffer
.buffer
);
1580 __update_max_tr(tr
, tsk
, cpu
);
1583 arch_spin_unlock(&tr
->max_lock
);
1587 * update_max_tr_single - only copy one trace over, and reset the rest
1589 * @tsk: task with the latency
1590 * @cpu: the cpu of the buffer to copy.
1592 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1595 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1602 WARN_ON_ONCE(!irqs_disabled());
1603 if (!tr
->allocated_snapshot
) {
1604 /* Only the nop tracer should hit this when disabling */
1605 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1609 arch_spin_lock(&tr
->max_lock
);
1611 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1613 if (ret
== -EBUSY
) {
1615 * We failed to swap the buffer due to a commit taking
1616 * place on this CPU. We fail to record, but we reset
1617 * the max trace buffer (no one writes directly to it)
1618 * and flag that it failed.
1620 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1621 "Failed to swap buffers due to commit in progress\n");
1624 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1626 __update_max_tr(tr
, tsk
, cpu
);
1627 arch_spin_unlock(&tr
->max_lock
);
1629 #endif /* CONFIG_TRACER_MAX_TRACE */
1631 static int wait_on_pipe(struct trace_iterator
*iter
, int full
)
1633 /* Iterators are static, they should be filled or empty */
1634 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1637 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
1641 #ifdef CONFIG_FTRACE_STARTUP_TEST
1642 static bool selftests_can_run
;
1644 struct trace_selftests
{
1645 struct list_head list
;
1646 struct tracer
*type
;
1649 static LIST_HEAD(postponed_selftests
);
1651 static int save_selftest(struct tracer
*type
)
1653 struct trace_selftests
*selftest
;
1655 selftest
= kmalloc(sizeof(*selftest
), GFP_KERNEL
);
1659 selftest
->type
= type
;
1660 list_add(&selftest
->list
, &postponed_selftests
);
1664 static int run_tracer_selftest(struct tracer
*type
)
1666 struct trace_array
*tr
= &global_trace
;
1667 struct tracer
*saved_tracer
= tr
->current_trace
;
1670 if (!type
->selftest
|| tracing_selftest_disabled
)
1674 * If a tracer registers early in boot up (before scheduling is
1675 * initialized and such), then do not run its selftests yet.
1676 * Instead, run it a little later in the boot process.
1678 if (!selftests_can_run
)
1679 return save_selftest(type
);
1682 * Run a selftest on this tracer.
1683 * Here we reset the trace buffer, and set the current
1684 * tracer to be this tracer. The tracer can then run some
1685 * internal tracing to verify that everything is in order.
1686 * If we fail, we do not register this tracer.
1688 tracing_reset_online_cpus(&tr
->trace_buffer
);
1690 tr
->current_trace
= type
;
1692 #ifdef CONFIG_TRACER_MAX_TRACE
1693 if (type
->use_max_tr
) {
1694 /* If we expanded the buffers, make sure the max is expanded too */
1695 if (ring_buffer_expanded
)
1696 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1697 RING_BUFFER_ALL_CPUS
);
1698 tr
->allocated_snapshot
= true;
1702 /* the test is responsible for initializing and enabling */
1703 pr_info("Testing tracer %s: ", type
->name
);
1704 ret
= type
->selftest(type
, tr
);
1705 /* the test is responsible for resetting too */
1706 tr
->current_trace
= saved_tracer
;
1708 printk(KERN_CONT
"FAILED!\n");
1709 /* Add the warning after printing 'FAILED' */
1713 /* Only reset on passing, to avoid touching corrupted buffers */
1714 tracing_reset_online_cpus(&tr
->trace_buffer
);
1716 #ifdef CONFIG_TRACER_MAX_TRACE
1717 if (type
->use_max_tr
) {
1718 tr
->allocated_snapshot
= false;
1720 /* Shrink the max buffer again */
1721 if (ring_buffer_expanded
)
1722 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1723 RING_BUFFER_ALL_CPUS
);
1727 printk(KERN_CONT
"PASSED\n");
1731 static __init
int init_trace_selftests(void)
1733 struct trace_selftests
*p
, *n
;
1734 struct tracer
*t
, **last
;
1737 selftests_can_run
= true;
1739 mutex_lock(&trace_types_lock
);
1741 if (list_empty(&postponed_selftests
))
1744 pr_info("Running postponed tracer tests:\n");
1746 tracing_selftest_running
= true;
1747 list_for_each_entry_safe(p
, n
, &postponed_selftests
, list
) {
1748 /* This loop can take minutes when sanitizers are enabled, so
1749 * lets make sure we allow RCU processing.
1752 ret
= run_tracer_selftest(p
->type
);
1753 /* If the test fails, then warn and remove from available_tracers */
1755 WARN(1, "tracer: %s failed selftest, disabling\n",
1757 last
= &trace_types
;
1758 for (t
= trace_types
; t
; t
= t
->next
) {
1769 tracing_selftest_running
= false;
1772 mutex_unlock(&trace_types_lock
);
1776 core_initcall(init_trace_selftests
);
1778 static inline int run_tracer_selftest(struct tracer
*type
)
1782 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1784 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
);
1786 static void __init
apply_trace_boot_options(void);
1789 * register_tracer - register a tracer with the ftrace system.
1790 * @type: the plugin for the tracer
1792 * Register a new plugin tracer.
1794 int __init
register_tracer(struct tracer
*type
)
1800 pr_info("Tracer must have a name\n");
1804 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1805 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1809 if (security_locked_down(LOCKDOWN_TRACEFS
)) {
1810 pr_warning("Can not register tracer %s due to lockdown\n",
1815 mutex_lock(&trace_types_lock
);
1817 tracing_selftest_running
= true;
1819 for (t
= trace_types
; t
; t
= t
->next
) {
1820 if (strcmp(type
->name
, t
->name
) == 0) {
1822 pr_info("Tracer %s already registered\n",
1829 if (!type
->set_flag
)
1830 type
->set_flag
= &dummy_set_flag
;
1832 /*allocate a dummy tracer_flags*/
1833 type
->flags
= kmalloc(sizeof(*type
->flags
), GFP_KERNEL
);
1838 type
->flags
->val
= 0;
1839 type
->flags
->opts
= dummy_tracer_opt
;
1841 if (!type
->flags
->opts
)
1842 type
->flags
->opts
= dummy_tracer_opt
;
1844 /* store the tracer for __set_tracer_option */
1845 type
->flags
->trace
= type
;
1847 ret
= run_tracer_selftest(type
);
1851 type
->next
= trace_types
;
1853 add_tracer_options(&global_trace
, type
);
1856 tracing_selftest_running
= false;
1857 mutex_unlock(&trace_types_lock
);
1859 if (ret
|| !default_bootup_tracer
)
1862 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1865 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1866 /* Do we want this tracer to start on bootup? */
1867 tracing_set_tracer(&global_trace
, type
->name
);
1868 default_bootup_tracer
= NULL
;
1870 apply_trace_boot_options();
1872 /* disable other selftests, since this will break it. */
1873 tracing_selftest_disabled
= true;
1874 #ifdef CONFIG_FTRACE_STARTUP_TEST
1875 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1883 static void tracing_reset_cpu(struct trace_buffer
*buf
, int cpu
)
1885 struct ring_buffer
*buffer
= buf
->buffer
;
1890 ring_buffer_record_disable(buffer
);
1892 /* Make sure all commits have finished */
1894 ring_buffer_reset_cpu(buffer
, cpu
);
1896 ring_buffer_record_enable(buffer
);
1899 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1901 struct ring_buffer
*buffer
= buf
->buffer
;
1907 ring_buffer_record_disable(buffer
);
1909 /* Make sure all commits have finished */
1912 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1914 for_each_online_cpu(cpu
)
1915 ring_buffer_reset_cpu(buffer
, cpu
);
1917 ring_buffer_record_enable(buffer
);
1920 /* Must have trace_types_lock held */
1921 void tracing_reset_all_online_cpus(void)
1923 struct trace_array
*tr
;
1925 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1926 if (!tr
->clear_trace
)
1928 tr
->clear_trace
= false;
1929 tracing_reset_online_cpus(&tr
->trace_buffer
);
1930 #ifdef CONFIG_TRACER_MAX_TRACE
1931 tracing_reset_online_cpus(&tr
->max_buffer
);
1936 static int *tgid_map
;
1938 #define SAVED_CMDLINES_DEFAULT 128
1939 #define NO_CMDLINE_MAP UINT_MAX
1940 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1941 struct saved_cmdlines_buffer
{
1942 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1943 unsigned *map_cmdline_to_pid
;
1944 unsigned cmdline_num
;
1946 char *saved_cmdlines
;
1948 static struct saved_cmdlines_buffer
*savedcmd
;
1950 /* temporary disable recording */
1951 static atomic_t trace_record_taskinfo_disabled __read_mostly
;
1953 static inline char *get_saved_cmdlines(int idx
)
1955 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1958 static inline void set_cmdline(int idx
, const char *cmdline
)
1960 strncpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1963 static int allocate_cmdlines_buffer(unsigned int val
,
1964 struct saved_cmdlines_buffer
*s
)
1966 s
->map_cmdline_to_pid
= kmalloc_array(val
,
1967 sizeof(*s
->map_cmdline_to_pid
),
1969 if (!s
->map_cmdline_to_pid
)
1972 s
->saved_cmdlines
= kmalloc_array(TASK_COMM_LEN
, val
, GFP_KERNEL
);
1973 if (!s
->saved_cmdlines
) {
1974 kfree(s
->map_cmdline_to_pid
);
1979 s
->cmdline_num
= val
;
1980 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1981 sizeof(s
->map_pid_to_cmdline
));
1982 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1983 val
* sizeof(*s
->map_cmdline_to_pid
));
1988 static int trace_create_savedcmd(void)
1992 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1996 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
2006 int is_tracing_stopped(void)
2008 return global_trace
.stop_count
;
2012 * tracing_start - quick start of the tracer
2014 * If tracing is enabled but was stopped by tracing_stop,
2015 * this will start the tracer back up.
2017 void tracing_start(void)
2019 struct ring_buffer
*buffer
;
2020 unsigned long flags
;
2022 if (tracing_disabled
)
2025 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
2026 if (--global_trace
.stop_count
) {
2027 if (global_trace
.stop_count
< 0) {
2028 /* Someone screwed up their debugging */
2030 global_trace
.stop_count
= 0;
2035 /* Prevent the buffers from switching */
2036 arch_spin_lock(&global_trace
.max_lock
);
2038 buffer
= global_trace
.trace_buffer
.buffer
;
2040 ring_buffer_record_enable(buffer
);
2042 #ifdef CONFIG_TRACER_MAX_TRACE
2043 buffer
= global_trace
.max_buffer
.buffer
;
2045 ring_buffer_record_enable(buffer
);
2048 arch_spin_unlock(&global_trace
.max_lock
);
2051 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
2054 static void tracing_start_tr(struct trace_array
*tr
)
2056 struct ring_buffer
*buffer
;
2057 unsigned long flags
;
2059 if (tracing_disabled
)
2062 /* If global, we need to also start the max tracer */
2063 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
2064 return tracing_start();
2066 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
2068 if (--tr
->stop_count
) {
2069 if (tr
->stop_count
< 0) {
2070 /* Someone screwed up their debugging */
2077 buffer
= tr
->trace_buffer
.buffer
;
2079 ring_buffer_record_enable(buffer
);
2082 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
2086 * tracing_stop - quick stop of the tracer
2088 * Light weight way to stop tracing. Use in conjunction with
2091 void tracing_stop(void)
2093 struct ring_buffer
*buffer
;
2094 unsigned long flags
;
2096 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
2097 if (global_trace
.stop_count
++)
2100 /* Prevent the buffers from switching */
2101 arch_spin_lock(&global_trace
.max_lock
);
2103 buffer
= global_trace
.trace_buffer
.buffer
;
2105 ring_buffer_record_disable(buffer
);
2107 #ifdef CONFIG_TRACER_MAX_TRACE
2108 buffer
= global_trace
.max_buffer
.buffer
;
2110 ring_buffer_record_disable(buffer
);
2113 arch_spin_unlock(&global_trace
.max_lock
);
2116 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
2119 static void tracing_stop_tr(struct trace_array
*tr
)
2121 struct ring_buffer
*buffer
;
2122 unsigned long flags
;
2124 /* If global, we need to also stop the max tracer */
2125 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
2126 return tracing_stop();
2128 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
2129 if (tr
->stop_count
++)
2132 buffer
= tr
->trace_buffer
.buffer
;
2134 ring_buffer_record_disable(buffer
);
2137 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
2140 static int trace_save_cmdline(struct task_struct
*tsk
)
2144 /* treat recording of idle task as a success */
2148 if (unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
2152 * It's not the end of the world if we don't get
2153 * the lock, but we also don't want to spin
2154 * nor do we want to disable interrupts,
2155 * so if we miss here, then better luck next time.
2157 if (!arch_spin_trylock(&trace_cmdline_lock
))
2160 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
2161 if (idx
== NO_CMDLINE_MAP
) {
2162 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
2165 * Check whether the cmdline buffer at idx has a pid
2166 * mapped. We are going to overwrite that entry so we
2167 * need to clear the map_pid_to_cmdline. Otherwise we
2168 * would read the new comm for the old pid.
2170 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
2171 if (pid
!= NO_CMDLINE_MAP
)
2172 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
2174 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
2175 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
2177 savedcmd
->cmdline_idx
= idx
;
2180 set_cmdline(idx
, tsk
->comm
);
2182 arch_spin_unlock(&trace_cmdline_lock
);
2187 static void __trace_find_cmdline(int pid
, char comm
[])
2192 strcpy(comm
, "<idle>");
2196 if (WARN_ON_ONCE(pid
< 0)) {
2197 strcpy(comm
, "<XXX>");
2201 if (pid
> PID_MAX_DEFAULT
) {
2202 strcpy(comm
, "<...>");
2206 map
= savedcmd
->map_pid_to_cmdline
[pid
];
2207 if (map
!= NO_CMDLINE_MAP
)
2208 strlcpy(comm
, get_saved_cmdlines(map
), TASK_COMM_LEN
);
2210 strcpy(comm
, "<...>");
2213 void trace_find_cmdline(int pid
, char comm
[])
2216 arch_spin_lock(&trace_cmdline_lock
);
2218 __trace_find_cmdline(pid
, comm
);
2220 arch_spin_unlock(&trace_cmdline_lock
);
2224 int trace_find_tgid(int pid
)
2226 if (unlikely(!tgid_map
|| !pid
|| pid
> PID_MAX_DEFAULT
))
2229 return tgid_map
[pid
];
2232 static int trace_save_tgid(struct task_struct
*tsk
)
2234 /* treat recording of idle task as a success */
2238 if (unlikely(!tgid_map
|| tsk
->pid
> PID_MAX_DEFAULT
))
2241 tgid_map
[tsk
->pid
] = tsk
->tgid
;
2245 static bool tracing_record_taskinfo_skip(int flags
)
2247 if (unlikely(!(flags
& (TRACE_RECORD_CMDLINE
| TRACE_RECORD_TGID
))))
2249 if (atomic_read(&trace_record_taskinfo_disabled
) || !tracing_is_on())
2251 if (!__this_cpu_read(trace_taskinfo_save
))
2257 * tracing_record_taskinfo - record the task info of a task
2259 * @task: task to record
2260 * @flags: TRACE_RECORD_CMDLINE for recording comm
2261 * TRACE_RECORD_TGID for recording tgid
2263 void tracing_record_taskinfo(struct task_struct
*task
, int flags
)
2267 if (tracing_record_taskinfo_skip(flags
))
2271 * Record as much task information as possible. If some fail, continue
2272 * to try to record the others.
2274 done
= !(flags
& TRACE_RECORD_CMDLINE
) || trace_save_cmdline(task
);
2275 done
&= !(flags
& TRACE_RECORD_TGID
) || trace_save_tgid(task
);
2277 /* If recording any information failed, retry again soon. */
2281 __this_cpu_write(trace_taskinfo_save
, false);
2285 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2287 * @prev: previous task during sched_switch
2288 * @next: next task during sched_switch
2289 * @flags: TRACE_RECORD_CMDLINE for recording comm
2290 * TRACE_RECORD_TGID for recording tgid
2292 void tracing_record_taskinfo_sched_switch(struct task_struct
*prev
,
2293 struct task_struct
*next
, int flags
)
2297 if (tracing_record_taskinfo_skip(flags
))
2301 * Record as much task information as possible. If some fail, continue
2302 * to try to record the others.
2304 done
= !(flags
& TRACE_RECORD_CMDLINE
) || trace_save_cmdline(prev
);
2305 done
&= !(flags
& TRACE_RECORD_CMDLINE
) || trace_save_cmdline(next
);
2306 done
&= !(flags
& TRACE_RECORD_TGID
) || trace_save_tgid(prev
);
2307 done
&= !(flags
& TRACE_RECORD_TGID
) || trace_save_tgid(next
);
2309 /* If recording any information failed, retry again soon. */
2313 __this_cpu_write(trace_taskinfo_save
, false);
2316 /* Helpers to record a specific task information */
2317 void tracing_record_cmdline(struct task_struct
*task
)
2319 tracing_record_taskinfo(task
, TRACE_RECORD_CMDLINE
);
2322 void tracing_record_tgid(struct task_struct
*task
)
2324 tracing_record_taskinfo(task
, TRACE_RECORD_TGID
);
2328 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2329 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2330 * simplifies those functions and keeps them in sync.
2332 enum print_line_t
trace_handle_return(struct trace_seq
*s
)
2334 return trace_seq_has_overflowed(s
) ?
2335 TRACE_TYPE_PARTIAL_LINE
: TRACE_TYPE_HANDLED
;
2337 EXPORT_SYMBOL_GPL(trace_handle_return
);
2340 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned short type
,
2341 unsigned long flags
, int pc
)
2343 struct task_struct
*tsk
= current
;
2345 entry
->preempt_count
= pc
& 0xff;
2346 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
2349 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2350 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
2352 TRACE_FLAG_IRQS_NOSUPPORT
|
2354 ((pc
& NMI_MASK
) ? TRACE_FLAG_NMI
: 0) |
2355 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
2356 ((pc
& SOFTIRQ_OFFSET
) ? TRACE_FLAG_SOFTIRQ
: 0) |
2357 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
2358 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
2360 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
2362 struct ring_buffer_event
*
2363 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
2366 unsigned long flags
, int pc
)
2368 return __trace_buffer_lock_reserve(buffer
, type
, len
, flags
, pc
);
2371 DEFINE_PER_CPU(struct ring_buffer_event
*, trace_buffered_event
);
2372 DEFINE_PER_CPU(int, trace_buffered_event_cnt
);
2373 static int trace_buffered_event_ref
;
2376 * trace_buffered_event_enable - enable buffering events
2378 * When events are being filtered, it is quicker to use a temporary
2379 * buffer to write the event data into if there's a likely chance
2380 * that it will not be committed. The discard of the ring buffer
2381 * is not as fast as committing, and is much slower than copying
2384 * When an event is to be filtered, allocate per cpu buffers to
2385 * write the event data into, and if the event is filtered and discarded
2386 * it is simply dropped, otherwise, the entire data is to be committed
2389 void trace_buffered_event_enable(void)
2391 struct ring_buffer_event
*event
;
2395 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
2397 if (trace_buffered_event_ref
++)
2400 for_each_tracing_cpu(cpu
) {
2401 page
= alloc_pages_node(cpu_to_node(cpu
),
2402 GFP_KERNEL
| __GFP_NORETRY
, 0);
2406 event
= page_address(page
);
2407 memset(event
, 0, sizeof(*event
));
2409 per_cpu(trace_buffered_event
, cpu
) = event
;
2412 if (cpu
== smp_processor_id() &&
2413 this_cpu_read(trace_buffered_event
) !=
2414 per_cpu(trace_buffered_event
, cpu
))
2421 trace_buffered_event_disable();
2424 static void enable_trace_buffered_event(void *data
)
2426 /* Probably not needed, but do it anyway */
2428 this_cpu_dec(trace_buffered_event_cnt
);
2431 static void disable_trace_buffered_event(void *data
)
2433 this_cpu_inc(trace_buffered_event_cnt
);
2437 * trace_buffered_event_disable - disable buffering events
2439 * When a filter is removed, it is faster to not use the buffered
2440 * events, and to commit directly into the ring buffer. Free up
2441 * the temp buffers when there are no more users. This requires
2442 * special synchronization with current events.
2444 void trace_buffered_event_disable(void)
2448 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
2450 if (WARN_ON_ONCE(!trace_buffered_event_ref
))
2453 if (--trace_buffered_event_ref
)
2457 /* For each CPU, set the buffer as used. */
2458 smp_call_function_many(tracing_buffer_mask
,
2459 disable_trace_buffered_event
, NULL
, 1);
2462 /* Wait for all current users to finish */
2465 for_each_tracing_cpu(cpu
) {
2466 free_page((unsigned long)per_cpu(trace_buffered_event
, cpu
));
2467 per_cpu(trace_buffered_event
, cpu
) = NULL
;
2470 * Make sure trace_buffered_event is NULL before clearing
2471 * trace_buffered_event_cnt.
2476 /* Do the work on each cpu */
2477 smp_call_function_many(tracing_buffer_mask
,
2478 enable_trace_buffered_event
, NULL
, 1);
2482 static struct ring_buffer
*temp_buffer
;
2484 struct ring_buffer_event
*
2485 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
2486 struct trace_event_file
*trace_file
,
2487 int type
, unsigned long len
,
2488 unsigned long flags
, int pc
)
2490 struct ring_buffer_event
*entry
;
2493 *current_rb
= trace_file
->tr
->trace_buffer
.buffer
;
2495 if (!ring_buffer_time_stamp_abs(*current_rb
) && (trace_file
->flags
&
2496 (EVENT_FILE_FL_SOFT_DISABLED
| EVENT_FILE_FL_FILTERED
)) &&
2497 (entry
= this_cpu_read(trace_buffered_event
))) {
2498 /* Try to use the per cpu buffer first */
2499 val
= this_cpu_inc_return(trace_buffered_event_cnt
);
2501 trace_event_setup(entry
, type
, flags
, pc
);
2502 entry
->array
[0] = len
;
2505 this_cpu_dec(trace_buffered_event_cnt
);
2508 entry
= __trace_buffer_lock_reserve(*current_rb
,
2509 type
, len
, flags
, pc
);
2511 * If tracing is off, but we have triggers enabled
2512 * we still need to look at the event data. Use the temp_buffer
2513 * to store the trace event for the tigger to use. It's recusive
2514 * safe and will not be recorded anywhere.
2516 if (!entry
&& trace_file
->flags
& EVENT_FILE_FL_TRIGGER_COND
) {
2517 *current_rb
= temp_buffer
;
2518 entry
= __trace_buffer_lock_reserve(*current_rb
,
2519 type
, len
, flags
, pc
);
2523 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
2525 static DEFINE_SPINLOCK(tracepoint_iter_lock
);
2526 static DEFINE_MUTEX(tracepoint_printk_mutex
);
2528 static void output_printk(struct trace_event_buffer
*fbuffer
)
2530 struct trace_event_call
*event_call
;
2531 struct trace_event
*event
;
2532 unsigned long flags
;
2533 struct trace_iterator
*iter
= tracepoint_print_iter
;
2535 /* We should never get here if iter is NULL */
2536 if (WARN_ON_ONCE(!iter
))
2539 event_call
= fbuffer
->trace_file
->event_call
;
2540 if (!event_call
|| !event_call
->event
.funcs
||
2541 !event_call
->event
.funcs
->trace
)
2544 event
= &fbuffer
->trace_file
->event_call
->event
;
2546 spin_lock_irqsave(&tracepoint_iter_lock
, flags
);
2547 trace_seq_init(&iter
->seq
);
2548 iter
->ent
= fbuffer
->entry
;
2549 event_call
->event
.funcs
->trace(iter
, 0, event
);
2550 trace_seq_putc(&iter
->seq
, 0);
2551 printk("%s", iter
->seq
.buffer
);
2553 spin_unlock_irqrestore(&tracepoint_iter_lock
, flags
);
2556 int tracepoint_printk_sysctl(struct ctl_table
*table
, int write
,
2557 void __user
*buffer
, size_t *lenp
,
2560 int save_tracepoint_printk
;
2563 mutex_lock(&tracepoint_printk_mutex
);
2564 save_tracepoint_printk
= tracepoint_printk
;
2566 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
2569 * This will force exiting early, as tracepoint_printk
2570 * is always zero when tracepoint_printk_iter is not allocated
2572 if (!tracepoint_print_iter
)
2573 tracepoint_printk
= 0;
2575 if (save_tracepoint_printk
== tracepoint_printk
)
2578 if (tracepoint_printk
)
2579 static_key_enable(&tracepoint_printk_key
.key
);
2581 static_key_disable(&tracepoint_printk_key
.key
);
2584 mutex_unlock(&tracepoint_printk_mutex
);
2589 void trace_event_buffer_commit(struct trace_event_buffer
*fbuffer
)
2591 if (static_key_false(&tracepoint_printk_key
.key
))
2592 output_printk(fbuffer
);
2594 event_trigger_unlock_commit(fbuffer
->trace_file
, fbuffer
->buffer
,
2595 fbuffer
->event
, fbuffer
->entry
,
2596 fbuffer
->flags
, fbuffer
->pc
);
2598 EXPORT_SYMBOL_GPL(trace_event_buffer_commit
);
2603 * trace_buffer_unlock_commit_regs()
2604 * trace_event_buffer_commit()
2605 * trace_event_raw_event_xxx()
2607 # define STACK_SKIP 3
2609 void trace_buffer_unlock_commit_regs(struct trace_array
*tr
,
2610 struct ring_buffer
*buffer
,
2611 struct ring_buffer_event
*event
,
2612 unsigned long flags
, int pc
,
2613 struct pt_regs
*regs
)
2615 __buffer_unlock_commit(buffer
, event
);
2618 * If regs is not set, then skip the necessary functions.
2619 * Note, we can still get here via blktrace, wakeup tracer
2620 * and mmiotrace, but that's ok if they lose a function or
2621 * two. They are not that meaningful.
2623 ftrace_trace_stack(tr
, buffer
, flags
, regs
? 0 : STACK_SKIP
, pc
, regs
);
2624 ftrace_trace_userstack(buffer
, flags
, pc
);
2628 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2631 trace_buffer_unlock_commit_nostack(struct ring_buffer
*buffer
,
2632 struct ring_buffer_event
*event
)
2634 __buffer_unlock_commit(buffer
, event
);
2638 trace_process_export(struct trace_export
*export
,
2639 struct ring_buffer_event
*event
)
2641 struct trace_entry
*entry
;
2642 unsigned int size
= 0;
2644 entry
= ring_buffer_event_data(event
);
2645 size
= ring_buffer_event_length(event
);
2646 export
->write(export
, entry
, size
);
2649 static DEFINE_MUTEX(ftrace_export_lock
);
2651 static struct trace_export __rcu
*ftrace_exports_list __read_mostly
;
2653 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled
);
2655 static inline void ftrace_exports_enable(void)
2657 static_branch_enable(&ftrace_exports_enabled
);
2660 static inline void ftrace_exports_disable(void)
2662 static_branch_disable(&ftrace_exports_enabled
);
2665 static void ftrace_exports(struct ring_buffer_event
*event
)
2667 struct trace_export
*export
;
2669 preempt_disable_notrace();
2671 export
= rcu_dereference_raw_check(ftrace_exports_list
);
2673 trace_process_export(export
, event
);
2674 export
= rcu_dereference_raw_check(export
->next
);
2677 preempt_enable_notrace();
2681 add_trace_export(struct trace_export
**list
, struct trace_export
*export
)
2683 rcu_assign_pointer(export
->next
, *list
);
2685 * We are entering export into the list but another
2686 * CPU might be walking that list. We need to make sure
2687 * the export->next pointer is valid before another CPU sees
2688 * the export pointer included into the list.
2690 rcu_assign_pointer(*list
, export
);
2694 rm_trace_export(struct trace_export
**list
, struct trace_export
*export
)
2696 struct trace_export
**p
;
2698 for (p
= list
; *p
!= NULL
; p
= &(*p
)->next
)
2705 rcu_assign_pointer(*p
, (*p
)->next
);
2711 add_ftrace_export(struct trace_export
**list
, struct trace_export
*export
)
2714 ftrace_exports_enable();
2716 add_trace_export(list
, export
);
2720 rm_ftrace_export(struct trace_export
**list
, struct trace_export
*export
)
2724 ret
= rm_trace_export(list
, export
);
2726 ftrace_exports_disable();
2731 int register_ftrace_export(struct trace_export
*export
)
2733 if (WARN_ON_ONCE(!export
->write
))
2736 mutex_lock(&ftrace_export_lock
);
2738 add_ftrace_export(&ftrace_exports_list
, export
);
2740 mutex_unlock(&ftrace_export_lock
);
2744 EXPORT_SYMBOL_GPL(register_ftrace_export
);
2746 int unregister_ftrace_export(struct trace_export
*export
)
2750 mutex_lock(&ftrace_export_lock
);
2752 ret
= rm_ftrace_export(&ftrace_exports_list
, export
);
2754 mutex_unlock(&ftrace_export_lock
);
2758 EXPORT_SYMBOL_GPL(unregister_ftrace_export
);
2761 trace_function(struct trace_array
*tr
,
2762 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
2765 struct trace_event_call
*call
= &event_function
;
2766 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
2767 struct ring_buffer_event
*event
;
2768 struct ftrace_entry
*entry
;
2770 event
= __trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
2774 entry
= ring_buffer_event_data(event
);
2776 entry
->parent_ip
= parent_ip
;
2778 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2779 if (static_branch_unlikely(&ftrace_exports_enabled
))
2780 ftrace_exports(event
);
2781 __buffer_unlock_commit(buffer
, event
);
2785 #ifdef CONFIG_STACKTRACE
2787 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2788 #define FTRACE_KSTACK_NESTING 4
2790 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2792 struct ftrace_stack
{
2793 unsigned long calls
[FTRACE_KSTACK_ENTRIES
];
2797 struct ftrace_stacks
{
2798 struct ftrace_stack stacks
[FTRACE_KSTACK_NESTING
];
2801 static DEFINE_PER_CPU(struct ftrace_stacks
, ftrace_stacks
);
2802 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
2804 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
2805 unsigned long flags
,
2806 int skip
, int pc
, struct pt_regs
*regs
)
2808 struct trace_event_call
*call
= &event_kernel_stack
;
2809 struct ring_buffer_event
*event
;
2810 unsigned int size
, nr_entries
;
2811 struct ftrace_stack
*fstack
;
2812 struct stack_entry
*entry
;
2816 * Add one, for this function and the call to save_stack_trace()
2817 * If regs is set, then these functions will not be in the way.
2819 #ifndef CONFIG_UNWINDER_ORC
2825 * Since events can happen in NMIs there's no safe way to
2826 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2827 * or NMI comes in, it will just have to use the default
2828 * FTRACE_STACK_SIZE.
2830 preempt_disable_notrace();
2832 stackidx
= __this_cpu_inc_return(ftrace_stack_reserve
) - 1;
2834 /* This should never happen. If it does, yell once and skip */
2835 if (WARN_ON_ONCE(stackidx
> FTRACE_KSTACK_NESTING
))
2839 * The above __this_cpu_inc_return() is 'atomic' cpu local. An
2840 * interrupt will either see the value pre increment or post
2841 * increment. If the interrupt happens pre increment it will have
2842 * restored the counter when it returns. We just need a barrier to
2843 * keep gcc from moving things around.
2847 fstack
= this_cpu_ptr(ftrace_stacks
.stacks
) + stackidx
;
2848 size
= ARRAY_SIZE(fstack
->calls
);
2851 nr_entries
= stack_trace_save_regs(regs
, fstack
->calls
,
2854 nr_entries
= stack_trace_save(fstack
->calls
, size
, skip
);
2857 size
= nr_entries
* sizeof(unsigned long);
2858 event
= __trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
2859 sizeof(*entry
) + size
, flags
, pc
);
2862 entry
= ring_buffer_event_data(event
);
2864 memcpy(&entry
->caller
, fstack
->calls
, size
);
2865 entry
->size
= nr_entries
;
2867 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2868 __buffer_unlock_commit(buffer
, event
);
2871 /* Again, don't let gcc optimize things here */
2873 __this_cpu_dec(ftrace_stack_reserve
);
2874 preempt_enable_notrace();
2878 static inline void ftrace_trace_stack(struct trace_array
*tr
,
2879 struct ring_buffer
*buffer
,
2880 unsigned long flags
,
2881 int skip
, int pc
, struct pt_regs
*regs
)
2883 if (!(tr
->trace_flags
& TRACE_ITER_STACKTRACE
))
2886 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
2889 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
2892 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
2894 if (rcu_is_watching()) {
2895 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, NULL
);
2900 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2901 * but if the above rcu_is_watching() failed, then the NMI
2902 * triggered someplace critical, and rcu_irq_enter() should
2903 * not be called from NMI.
2905 if (unlikely(in_nmi()))
2908 rcu_irq_enter_irqson();
2909 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, NULL
);
2910 rcu_irq_exit_irqson();
2914 * trace_dump_stack - record a stack back trace in the trace buffer
2915 * @skip: Number of functions to skip (helper handlers)
2917 void trace_dump_stack(int skip
)
2919 unsigned long flags
;
2921 if (tracing_disabled
|| tracing_selftest_running
)
2924 local_save_flags(flags
);
2926 #ifndef CONFIG_UNWINDER_ORC
2927 /* Skip 1 to skip this function. */
2930 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
2931 flags
, skip
, preempt_count(), NULL
);
2933 EXPORT_SYMBOL_GPL(trace_dump_stack
);
2935 #ifdef CONFIG_USER_STACKTRACE_SUPPORT
2936 static DEFINE_PER_CPU(int, user_stack_count
);
2939 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
2941 struct trace_event_call
*call
= &event_user_stack
;
2942 struct ring_buffer_event
*event
;
2943 struct userstack_entry
*entry
;
2945 if (!(global_trace
.trace_flags
& TRACE_ITER_USERSTACKTRACE
))
2949 * NMIs can not handle page faults, even with fix ups.
2950 * The save user stack can (and often does) fault.
2952 if (unlikely(in_nmi()))
2956 * prevent recursion, since the user stack tracing may
2957 * trigger other kernel events.
2960 if (__this_cpu_read(user_stack_count
))
2963 __this_cpu_inc(user_stack_count
);
2965 event
= __trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
2966 sizeof(*entry
), flags
, pc
);
2968 goto out_drop_count
;
2969 entry
= ring_buffer_event_data(event
);
2971 entry
->tgid
= current
->tgid
;
2972 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
2974 stack_trace_save_user(entry
->caller
, FTRACE_STACK_ENTRIES
);
2975 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2976 __buffer_unlock_commit(buffer
, event
);
2979 __this_cpu_dec(user_stack_count
);
2983 #else /* CONFIG_USER_STACKTRACE_SUPPORT */
2984 static void ftrace_trace_userstack(struct ring_buffer
*buffer
,
2985 unsigned long flags
, int pc
)
2988 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
2990 #endif /* CONFIG_STACKTRACE */
2992 /* created for use with alloc_percpu */
2993 struct trace_buffer_struct
{
2995 char buffer
[4][TRACE_BUF_SIZE
];
2998 static struct trace_buffer_struct
*trace_percpu_buffer
;
3001 * Thise allows for lockless recording. If we're nested too deeply, then
3002 * this returns NULL.
3004 static char *get_trace_buf(void)
3006 struct trace_buffer_struct
*buffer
= this_cpu_ptr(trace_percpu_buffer
);
3008 if (!buffer
|| buffer
->nesting
>= 4)
3013 /* Interrupts must see nesting incremented before we use the buffer */
3015 return &buffer
->buffer
[buffer
->nesting
][0];
3018 static void put_trace_buf(void)
3020 /* Don't let the decrement of nesting leak before this */
3022 this_cpu_dec(trace_percpu_buffer
->nesting
);
3025 static int alloc_percpu_trace_buffer(void)
3027 struct trace_buffer_struct
*buffers
;
3029 buffers
= alloc_percpu(struct trace_buffer_struct
);
3030 if (WARN(!buffers
, "Could not allocate percpu trace_printk buffer"))
3033 trace_percpu_buffer
= buffers
;
3037 static int buffers_allocated
;
3039 void trace_printk_init_buffers(void)
3041 if (buffers_allocated
)
3044 if (alloc_percpu_trace_buffer())
3047 /* trace_printk() is for debug use only. Don't use it in production. */
3050 pr_warn("**********************************************************\n");
3051 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3053 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3055 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3056 pr_warn("** unsafe for production use. **\n");
3058 pr_warn("** If you see this message and you are not debugging **\n");
3059 pr_warn("** the kernel, report this immediately to your vendor! **\n");
3061 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3062 pr_warn("**********************************************************\n");
3064 /* Expand the buffers to set size */
3065 tracing_update_buffers();
3067 buffers_allocated
= 1;
3070 * trace_printk_init_buffers() can be called by modules.
3071 * If that happens, then we need to start cmdline recording
3072 * directly here. If the global_trace.buffer is already
3073 * allocated here, then this was called by module code.
3075 if (global_trace
.trace_buffer
.buffer
)
3076 tracing_start_cmdline_record();
3078 EXPORT_SYMBOL_GPL(trace_printk_init_buffers
);
3080 void trace_printk_start_comm(void)
3082 /* Start tracing comms if trace printk is set */
3083 if (!buffers_allocated
)
3085 tracing_start_cmdline_record();
3088 static void trace_printk_start_stop_comm(int enabled
)
3090 if (!buffers_allocated
)
3094 tracing_start_cmdline_record();
3096 tracing_stop_cmdline_record();
3100 * trace_vbprintk - write binary msg to tracing buffer
3101 * @ip: The address of the caller
3102 * @fmt: The string format to write to the buffer
3103 * @args: Arguments for @fmt
3105 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
3107 struct trace_event_call
*call
= &event_bprint
;
3108 struct ring_buffer_event
*event
;
3109 struct ring_buffer
*buffer
;
3110 struct trace_array
*tr
= &global_trace
;
3111 struct bprint_entry
*entry
;
3112 unsigned long flags
;
3114 int len
= 0, size
, pc
;
3116 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
3119 /* Don't pollute graph traces with trace_vprintk internals */
3120 pause_graph_tracing();
3122 pc
= preempt_count();
3123 preempt_disable_notrace();
3125 tbuffer
= get_trace_buf();
3131 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
3133 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
3136 local_save_flags(flags
);
3137 size
= sizeof(*entry
) + sizeof(u32
) * len
;
3138 buffer
= tr
->trace_buffer
.buffer
;
3139 event
= __trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
3143 entry
= ring_buffer_event_data(event
);
3147 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
3148 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
3149 __buffer_unlock_commit(buffer
, event
);
3150 ftrace_trace_stack(tr
, buffer
, flags
, 6, pc
, NULL
);
3157 preempt_enable_notrace();
3158 unpause_graph_tracing();
3162 EXPORT_SYMBOL_GPL(trace_vbprintk
);
3166 __trace_array_vprintk(struct ring_buffer
*buffer
,
3167 unsigned long ip
, const char *fmt
, va_list args
)
3169 struct trace_event_call
*call
= &event_print
;
3170 struct ring_buffer_event
*event
;
3171 int len
= 0, size
, pc
;
3172 struct print_entry
*entry
;
3173 unsigned long flags
;
3176 if (tracing_disabled
|| tracing_selftest_running
)
3179 /* Don't pollute graph traces with trace_vprintk internals */
3180 pause_graph_tracing();
3182 pc
= preempt_count();
3183 preempt_disable_notrace();
3186 tbuffer
= get_trace_buf();
3192 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
3194 local_save_flags(flags
);
3195 size
= sizeof(*entry
) + len
+ 1;
3196 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
3200 entry
= ring_buffer_event_data(event
);
3203 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
3204 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
3205 __buffer_unlock_commit(buffer
, event
);
3206 ftrace_trace_stack(&global_trace
, buffer
, flags
, 6, pc
, NULL
);
3213 preempt_enable_notrace();
3214 unpause_graph_tracing();
3220 int trace_array_vprintk(struct trace_array
*tr
,
3221 unsigned long ip
, const char *fmt
, va_list args
)
3223 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
3227 int trace_array_printk(struct trace_array
*tr
,
3228 unsigned long ip
, const char *fmt
, ...)
3233 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
3237 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
3241 EXPORT_SYMBOL_GPL(trace_array_printk
);
3244 int trace_array_printk_buf(struct ring_buffer
*buffer
,
3245 unsigned long ip
, const char *fmt
, ...)
3250 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
3254 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
3260 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
3262 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
3264 EXPORT_SYMBOL_GPL(trace_vprintk
);
3266 static void trace_iterator_increment(struct trace_iterator
*iter
)
3268 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
3272 ring_buffer_read(buf_iter
, NULL
);
3275 static struct trace_entry
*
3276 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
3277 unsigned long *lost_events
)
3279 struct ring_buffer_event
*event
;
3280 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
3283 event
= ring_buffer_iter_peek(buf_iter
, ts
);
3285 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
3289 iter
->ent_size
= ring_buffer_event_length(event
);
3290 return ring_buffer_event_data(event
);
3296 static struct trace_entry
*
3297 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
3298 unsigned long *missing_events
, u64
*ent_ts
)
3300 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
3301 struct trace_entry
*ent
, *next
= NULL
;
3302 unsigned long lost_events
= 0, next_lost
= 0;
3303 int cpu_file
= iter
->cpu_file
;
3304 u64 next_ts
= 0, ts
;
3310 * If we are in a per_cpu trace file, don't bother by iterating over
3311 * all cpu and peek directly.
3313 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
3314 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
3316 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
3318 *ent_cpu
= cpu_file
;
3323 for_each_tracing_cpu(cpu
) {
3325 if (ring_buffer_empty_cpu(buffer
, cpu
))
3328 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
3331 * Pick the entry with the smallest timestamp:
3333 if (ent
&& (!next
|| ts
< next_ts
)) {
3337 next_lost
= lost_events
;
3338 next_size
= iter
->ent_size
;
3342 iter
->ent_size
= next_size
;
3345 *ent_cpu
= next_cpu
;
3351 *missing_events
= next_lost
;
3356 /* Find the next real entry, without updating the iterator itself */
3357 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
3358 int *ent_cpu
, u64
*ent_ts
)
3360 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
3363 /* Find the next real entry, and increment the iterator to the next entry */
3364 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
3366 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
3367 &iter
->lost_events
, &iter
->ts
);
3370 trace_iterator_increment(iter
);
3372 return iter
->ent
? iter
: NULL
;
3375 static void trace_consume(struct trace_iterator
*iter
)
3377 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
3378 &iter
->lost_events
);
3381 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3383 struct trace_iterator
*iter
= m
->private;
3387 WARN_ON_ONCE(iter
->leftover
);
3391 /* can't go backwards */
3396 ent
= trace_find_next_entry_inc(iter
);
3400 while (ent
&& iter
->idx
< i
)
3401 ent
= trace_find_next_entry_inc(iter
);
3408 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
3410 struct ring_buffer_event
*event
;
3411 struct ring_buffer_iter
*buf_iter
;
3412 unsigned long entries
= 0;
3415 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
3417 buf_iter
= trace_buffer_iter(iter
, cpu
);
3421 ring_buffer_iter_reset(buf_iter
);
3424 * We could have the case with the max latency tracers
3425 * that a reset never took place on a cpu. This is evident
3426 * by the timestamp being before the start of the buffer.
3428 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
3429 if (ts
>= iter
->trace_buffer
->time_start
)
3432 ring_buffer_read(buf_iter
, NULL
);
3435 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
3439 * The current tracer is copied to avoid a global locking
3442 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
3444 struct trace_iterator
*iter
= m
->private;
3445 struct trace_array
*tr
= iter
->tr
;
3446 int cpu_file
= iter
->cpu_file
;
3452 * copy the tracer to avoid using a global lock all around.
3453 * iter->trace is a copy of current_trace, the pointer to the
3454 * name may be used instead of a strcmp(), as iter->trace->name
3455 * will point to the same string as current_trace->name.
3457 mutex_lock(&trace_types_lock
);
3458 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
3459 *iter
->trace
= *tr
->current_trace
;
3460 mutex_unlock(&trace_types_lock
);
3462 #ifdef CONFIG_TRACER_MAX_TRACE
3463 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
3464 return ERR_PTR(-EBUSY
);
3467 if (!iter
->snapshot
)
3468 atomic_inc(&trace_record_taskinfo_disabled
);
3470 if (*pos
!= iter
->pos
) {
3475 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
3476 for_each_tracing_cpu(cpu
)
3477 tracing_iter_reset(iter
, cpu
);
3479 tracing_iter_reset(iter
, cpu_file
);
3482 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
3487 * If we overflowed the seq_file before, then we want
3488 * to just reuse the trace_seq buffer again.
3494 p
= s_next(m
, p
, &l
);
3498 trace_event_read_lock();
3499 trace_access_lock(cpu_file
);
3503 static void s_stop(struct seq_file
*m
, void *p
)
3505 struct trace_iterator
*iter
= m
->private;
3507 #ifdef CONFIG_TRACER_MAX_TRACE
3508 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
3512 if (!iter
->snapshot
)
3513 atomic_dec(&trace_record_taskinfo_disabled
);
3515 trace_access_unlock(iter
->cpu_file
);
3516 trace_event_read_unlock();
3520 get_total_entries_cpu(struct trace_buffer
*buf
, unsigned long *total
,
3521 unsigned long *entries
, int cpu
)
3523 unsigned long count
;
3525 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
3527 * If this buffer has skipped entries, then we hold all
3528 * entries for the trace and we need to ignore the
3529 * ones before the time stamp.
3531 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
3532 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
3533 /* total is the same as the entries */
3537 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
3542 get_total_entries(struct trace_buffer
*buf
,
3543 unsigned long *total
, unsigned long *entries
)
3551 for_each_tracing_cpu(cpu
) {
3552 get_total_entries_cpu(buf
, &t
, &e
, cpu
);
3558 unsigned long trace_total_entries_cpu(struct trace_array
*tr
, int cpu
)
3560 unsigned long total
, entries
;
3565 get_total_entries_cpu(&tr
->trace_buffer
, &total
, &entries
, cpu
);
3570 unsigned long trace_total_entries(struct trace_array
*tr
)
3572 unsigned long total
, entries
;
3577 get_total_entries(&tr
->trace_buffer
, &total
, &entries
);
3582 static void print_lat_help_header(struct seq_file
*m
)
3584 seq_puts(m
, "# _------=> CPU# \n"
3585 "# / _-----=> irqs-off \n"
3586 "# | / _----=> need-resched \n"
3587 "# || / _---=> hardirq/softirq \n"
3588 "# ||| / _--=> preempt-depth \n"
3590 "# cmd pid ||||| time | caller \n"
3591 "# \\ / ||||| \\ | / \n");
3594 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
3596 unsigned long total
;
3597 unsigned long entries
;
3599 get_total_entries(buf
, &total
, &entries
);
3600 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3601 entries
, total
, num_online_cpus());
3605 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
,
3608 bool tgid
= flags
& TRACE_ITER_RECORD_TGID
;
3610 print_event_info(buf
, m
);
3612 seq_printf(m
, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid
? "TGID " : "");
3613 seq_printf(m
, "# | | %s | | |\n", tgid
? " | " : "");
3616 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
,
3619 bool tgid
= flags
& TRACE_ITER_RECORD_TGID
;
3620 const char *space
= " ";
3621 int prec
= tgid
? 10 : 2;
3623 print_event_info(buf
, m
);
3625 seq_printf(m
, "# %.*s _-----=> irqs-off\n", prec
, space
);
3626 seq_printf(m
, "# %.*s / _----=> need-resched\n", prec
, space
);
3627 seq_printf(m
, "# %.*s| / _---=> hardirq/softirq\n", prec
, space
);
3628 seq_printf(m
, "# %.*s|| / _--=> preempt-depth\n", prec
, space
);
3629 seq_printf(m
, "# %.*s||| / delay\n", prec
, space
);
3630 seq_printf(m
, "# TASK-PID %.*sCPU# |||| TIMESTAMP FUNCTION\n", prec
, " TGID ");
3631 seq_printf(m
, "# | | %.*s | |||| | |\n", prec
, " | ");
3635 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
3637 unsigned long sym_flags
= (global_trace
.trace_flags
& TRACE_ITER_SYM_MASK
);
3638 struct trace_buffer
*buf
= iter
->trace_buffer
;
3639 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
3640 struct tracer
*type
= iter
->trace
;
3641 unsigned long entries
;
3642 unsigned long total
;
3643 const char *name
= "preemption";
3647 get_total_entries(buf
, &total
, &entries
);
3649 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
3651 seq_puts(m
, "# -----------------------------------"
3652 "---------------------------------\n");
3653 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3654 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3655 nsecs_to_usecs(data
->saved_latency
),
3659 #if defined(CONFIG_PREEMPT_NONE)
3661 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3663 #elif defined(CONFIG_PREEMPT)
3668 /* These are reserved for later use */
3671 seq_printf(m
, " #P:%d)\n", num_online_cpus());
3675 seq_puts(m
, "# -----------------\n");
3676 seq_printf(m
, "# | task: %.16s-%d "
3677 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3678 data
->comm
, data
->pid
,
3679 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
3680 data
->policy
, data
->rt_priority
);
3681 seq_puts(m
, "# -----------------\n");
3683 if (data
->critical_start
) {
3684 seq_puts(m
, "# => started at: ");
3685 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
3686 trace_print_seq(m
, &iter
->seq
);
3687 seq_puts(m
, "\n# => ended at: ");
3688 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
3689 trace_print_seq(m
, &iter
->seq
);
3690 seq_puts(m
, "\n#\n");
3696 static void test_cpu_buff_start(struct trace_iterator
*iter
)
3698 struct trace_seq
*s
= &iter
->seq
;
3699 struct trace_array
*tr
= iter
->tr
;
3701 if (!(tr
->trace_flags
& TRACE_ITER_ANNOTATE
))
3704 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
3707 if (cpumask_available(iter
->started
) &&
3708 cpumask_test_cpu(iter
->cpu
, iter
->started
))
3711 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
3714 if (cpumask_available(iter
->started
))
3715 cpumask_set_cpu(iter
->cpu
, iter
->started
);
3717 /* Don't print started cpu buffer for the first entry of the trace */
3719 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
3723 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
3725 struct trace_array
*tr
= iter
->tr
;
3726 struct trace_seq
*s
= &iter
->seq
;
3727 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
3728 struct trace_entry
*entry
;
3729 struct trace_event
*event
;
3733 test_cpu_buff_start(iter
);
3735 event
= ftrace_find_event(entry
->type
);
3737 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3738 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3739 trace_print_lat_context(iter
);
3741 trace_print_context(iter
);
3744 if (trace_seq_has_overflowed(s
))
3745 return TRACE_TYPE_PARTIAL_LINE
;
3748 return event
->funcs
->trace(iter
, sym_flags
, event
);
3750 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
3752 return trace_handle_return(s
);
3755 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
3757 struct trace_array
*tr
= iter
->tr
;
3758 struct trace_seq
*s
= &iter
->seq
;
3759 struct trace_entry
*entry
;
3760 struct trace_event
*event
;
3764 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
)
3765 trace_seq_printf(s
, "%d %d %llu ",
3766 entry
->pid
, iter
->cpu
, iter
->ts
);
3768 if (trace_seq_has_overflowed(s
))
3769 return TRACE_TYPE_PARTIAL_LINE
;
3771 event
= ftrace_find_event(entry
->type
);
3773 return event
->funcs
->raw(iter
, 0, event
);
3775 trace_seq_printf(s
, "%d ?\n", entry
->type
);
3777 return trace_handle_return(s
);
3780 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
3782 struct trace_array
*tr
= iter
->tr
;
3783 struct trace_seq
*s
= &iter
->seq
;
3784 unsigned char newline
= '\n';
3785 struct trace_entry
*entry
;
3786 struct trace_event
*event
;
3790 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3791 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
3792 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
3793 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
3794 if (trace_seq_has_overflowed(s
))
3795 return TRACE_TYPE_PARTIAL_LINE
;
3798 event
= ftrace_find_event(entry
->type
);
3800 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
3801 if (ret
!= TRACE_TYPE_HANDLED
)
3805 SEQ_PUT_FIELD(s
, newline
);
3807 return trace_handle_return(s
);
3810 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
3812 struct trace_array
*tr
= iter
->tr
;
3813 struct trace_seq
*s
= &iter
->seq
;
3814 struct trace_entry
*entry
;
3815 struct trace_event
*event
;
3819 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3820 SEQ_PUT_FIELD(s
, entry
->pid
);
3821 SEQ_PUT_FIELD(s
, iter
->cpu
);
3822 SEQ_PUT_FIELD(s
, iter
->ts
);
3823 if (trace_seq_has_overflowed(s
))
3824 return TRACE_TYPE_PARTIAL_LINE
;
3827 event
= ftrace_find_event(entry
->type
);
3828 return event
? event
->funcs
->binary(iter
, 0, event
) :
3832 int trace_empty(struct trace_iterator
*iter
)
3834 struct ring_buffer_iter
*buf_iter
;
3837 /* If we are looking at one CPU buffer, only check that one */
3838 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
3839 cpu
= iter
->cpu_file
;
3840 buf_iter
= trace_buffer_iter(iter
, cpu
);
3842 if (!ring_buffer_iter_empty(buf_iter
))
3845 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3851 for_each_tracing_cpu(cpu
) {
3852 buf_iter
= trace_buffer_iter(iter
, cpu
);
3854 if (!ring_buffer_iter_empty(buf_iter
))
3857 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3865 /* Called with trace_event_read_lock() held. */
3866 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
3868 struct trace_array
*tr
= iter
->tr
;
3869 unsigned long trace_flags
= tr
->trace_flags
;
3870 enum print_line_t ret
;
3872 if (iter
->lost_events
) {
3873 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
3874 iter
->cpu
, iter
->lost_events
);
3875 if (trace_seq_has_overflowed(&iter
->seq
))
3876 return TRACE_TYPE_PARTIAL_LINE
;
3879 if (iter
->trace
&& iter
->trace
->print_line
) {
3880 ret
= iter
->trace
->print_line(iter
);
3881 if (ret
!= TRACE_TYPE_UNHANDLED
)
3885 if (iter
->ent
->type
== TRACE_BPUTS
&&
3886 trace_flags
& TRACE_ITER_PRINTK
&&
3887 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3888 return trace_print_bputs_msg_only(iter
);
3890 if (iter
->ent
->type
== TRACE_BPRINT
&&
3891 trace_flags
& TRACE_ITER_PRINTK
&&
3892 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3893 return trace_print_bprintk_msg_only(iter
);
3895 if (iter
->ent
->type
== TRACE_PRINT
&&
3896 trace_flags
& TRACE_ITER_PRINTK
&&
3897 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3898 return trace_print_printk_msg_only(iter
);
3900 if (trace_flags
& TRACE_ITER_BIN
)
3901 return print_bin_fmt(iter
);
3903 if (trace_flags
& TRACE_ITER_HEX
)
3904 return print_hex_fmt(iter
);
3906 if (trace_flags
& TRACE_ITER_RAW
)
3907 return print_raw_fmt(iter
);
3909 return print_trace_fmt(iter
);
3912 void trace_latency_header(struct seq_file
*m
)
3914 struct trace_iterator
*iter
= m
->private;
3915 struct trace_array
*tr
= iter
->tr
;
3917 /* print nothing if the buffers are empty */
3918 if (trace_empty(iter
))
3921 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3922 print_trace_header(m
, iter
);
3924 if (!(tr
->trace_flags
& TRACE_ITER_VERBOSE
))
3925 print_lat_help_header(m
);
3928 void trace_default_header(struct seq_file
*m
)
3930 struct trace_iterator
*iter
= m
->private;
3931 struct trace_array
*tr
= iter
->tr
;
3932 unsigned long trace_flags
= tr
->trace_flags
;
3934 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
3937 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
3938 /* print nothing if the buffers are empty */
3939 if (trace_empty(iter
))
3941 print_trace_header(m
, iter
);
3942 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
3943 print_lat_help_header(m
);
3945 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
3946 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
3947 print_func_help_header_irq(iter
->trace_buffer
,
3950 print_func_help_header(iter
->trace_buffer
, m
,
3956 static void test_ftrace_alive(struct seq_file
*m
)
3958 if (!ftrace_is_dead())
3960 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3961 "# MAY BE MISSING FUNCTION EVENTS\n");
3964 #ifdef CONFIG_TRACER_MAX_TRACE
3965 static void show_snapshot_main_help(struct seq_file
*m
)
3967 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3968 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3969 "# Takes a snapshot of the main buffer.\n"
3970 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3971 "# (Doesn't have to be '2' works with any number that\n"
3972 "# is not a '0' or '1')\n");
3975 static void show_snapshot_percpu_help(struct seq_file
*m
)
3977 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3978 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3979 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3980 "# Takes a snapshot of the main buffer for this cpu.\n");
3982 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
3983 "# Must use main snapshot file to allocate.\n");
3985 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3986 "# (Doesn't have to be '2' works with any number that\n"
3987 "# is not a '0' or '1')\n");
3990 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
3992 if (iter
->tr
->allocated_snapshot
)
3993 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
3995 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
3997 seq_puts(m
, "# Snapshot commands:\n");
3998 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
3999 show_snapshot_main_help(m
);
4001 show_snapshot_percpu_help(m
);
4004 /* Should never be called */
4005 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
4008 static int s_show(struct seq_file
*m
, void *v
)
4010 struct trace_iterator
*iter
= v
;
4013 if (iter
->ent
== NULL
) {
4015 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
4017 test_ftrace_alive(m
);
4019 if (iter
->snapshot
&& trace_empty(iter
))
4020 print_snapshot_help(m
, iter
);
4021 else if (iter
->trace
&& iter
->trace
->print_header
)
4022 iter
->trace
->print_header(m
);
4024 trace_default_header(m
);
4026 } else if (iter
->leftover
) {
4028 * If we filled the seq_file buffer earlier, we
4029 * want to just show it now.
4031 ret
= trace_print_seq(m
, &iter
->seq
);
4033 /* ret should this time be zero, but you never know */
4034 iter
->leftover
= ret
;
4037 print_trace_line(iter
);
4038 ret
= trace_print_seq(m
, &iter
->seq
);
4040 * If we overflow the seq_file buffer, then it will
4041 * ask us for this data again at start up.
4043 * ret is 0 if seq_file write succeeded.
4046 iter
->leftover
= ret
;
4053 * Should be used after trace_array_get(), trace_types_lock
4054 * ensures that i_cdev was already initialized.
4056 static inline int tracing_get_cpu(struct inode
*inode
)
4058 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
4059 return (long)inode
->i_cdev
- 1;
4060 return RING_BUFFER_ALL_CPUS
;
4063 static const struct seq_operations tracer_seq_ops
= {
4070 static struct trace_iterator
*
4071 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
4073 struct trace_array
*tr
= inode
->i_private
;
4074 struct trace_iterator
*iter
;
4077 if (tracing_disabled
)
4078 return ERR_PTR(-ENODEV
);
4080 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
4082 return ERR_PTR(-ENOMEM
);
4084 iter
->buffer_iter
= kcalloc(nr_cpu_ids
, sizeof(*iter
->buffer_iter
),
4086 if (!iter
->buffer_iter
)
4090 * We make a copy of the current tracer to avoid concurrent
4091 * changes on it while we are reading.
4093 mutex_lock(&trace_types_lock
);
4094 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
4098 *iter
->trace
= *tr
->current_trace
;
4100 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
4105 #ifdef CONFIG_TRACER_MAX_TRACE
4106 /* Currently only the top directory has a snapshot */
4107 if (tr
->current_trace
->print_max
|| snapshot
)
4108 iter
->trace_buffer
= &tr
->max_buffer
;
4111 iter
->trace_buffer
= &tr
->trace_buffer
;
4112 iter
->snapshot
= snapshot
;
4114 iter
->cpu_file
= tracing_get_cpu(inode
);
4115 mutex_init(&iter
->mutex
);
4117 /* Notify the tracer early; before we stop tracing. */
4118 if (iter
->trace
&& iter
->trace
->open
)
4119 iter
->trace
->open(iter
);
4121 /* Annotate start of buffers if we had overruns */
4122 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
4123 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
4125 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4126 if (trace_clocks
[tr
->clock_id
].in_ns
)
4127 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
4129 /* stop the trace while dumping if we are not opening "snapshot" */
4130 if (!iter
->snapshot
)
4131 tracing_stop_tr(tr
);
4133 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
4134 for_each_tracing_cpu(cpu
) {
4135 iter
->buffer_iter
[cpu
] =
4136 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
,
4139 ring_buffer_read_prepare_sync();
4140 for_each_tracing_cpu(cpu
) {
4141 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
4142 tracing_iter_reset(iter
, cpu
);
4145 cpu
= iter
->cpu_file
;
4146 iter
->buffer_iter
[cpu
] =
4147 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
,
4149 ring_buffer_read_prepare_sync();
4150 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
4151 tracing_iter_reset(iter
, cpu
);
4154 mutex_unlock(&trace_types_lock
);
4159 mutex_unlock(&trace_types_lock
);
4161 kfree(iter
->buffer_iter
);
4163 seq_release_private(inode
, file
);
4164 return ERR_PTR(-ENOMEM
);
4167 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
4171 ret
= tracing_check_open_get_tr(NULL
);
4175 filp
->private_data
= inode
->i_private
;
4179 bool tracing_is_disabled(void)
4181 return (tracing_disabled
) ? true: false;
4185 * Open and update trace_array ref count.
4186 * Must have the current trace_array passed to it.
4188 int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
4190 struct trace_array
*tr
= inode
->i_private
;
4193 ret
= tracing_check_open_get_tr(tr
);
4197 filp
->private_data
= inode
->i_private
;
4202 static int tracing_release(struct inode
*inode
, struct file
*file
)
4204 struct trace_array
*tr
= inode
->i_private
;
4205 struct seq_file
*m
= file
->private_data
;
4206 struct trace_iterator
*iter
;
4209 if (!(file
->f_mode
& FMODE_READ
)) {
4210 trace_array_put(tr
);
4214 /* Writes do not use seq_file */
4216 mutex_lock(&trace_types_lock
);
4218 for_each_tracing_cpu(cpu
) {
4219 if (iter
->buffer_iter
[cpu
])
4220 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
4223 if (iter
->trace
&& iter
->trace
->close
)
4224 iter
->trace
->close(iter
);
4226 if (!iter
->snapshot
)
4227 /* reenable tracing if it was previously enabled */
4228 tracing_start_tr(tr
);
4230 __trace_array_put(tr
);
4232 mutex_unlock(&trace_types_lock
);
4234 mutex_destroy(&iter
->mutex
);
4235 free_cpumask_var(iter
->started
);
4237 kfree(iter
->buffer_iter
);
4238 seq_release_private(inode
, file
);
4243 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
4245 struct trace_array
*tr
= inode
->i_private
;
4247 trace_array_put(tr
);
4251 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
4253 struct trace_array
*tr
= inode
->i_private
;
4255 trace_array_put(tr
);
4257 return single_release(inode
, file
);
4260 static int tracing_open(struct inode
*inode
, struct file
*file
)
4262 struct trace_array
*tr
= inode
->i_private
;
4263 struct trace_iterator
*iter
;
4266 ret
= tracing_check_open_get_tr(tr
);
4270 /* If this file was open for write, then erase contents */
4271 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
4272 int cpu
= tracing_get_cpu(inode
);
4273 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
4275 #ifdef CONFIG_TRACER_MAX_TRACE
4276 if (tr
->current_trace
->print_max
)
4277 trace_buf
= &tr
->max_buffer
;
4280 if (cpu
== RING_BUFFER_ALL_CPUS
)
4281 tracing_reset_online_cpus(trace_buf
);
4283 tracing_reset_cpu(trace_buf
, cpu
);
4286 if (file
->f_mode
& FMODE_READ
) {
4287 iter
= __tracing_open(inode
, file
, false);
4289 ret
= PTR_ERR(iter
);
4290 else if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
4291 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
4295 trace_array_put(tr
);
4301 * Some tracers are not suitable for instance buffers.
4302 * A tracer is always available for the global array (toplevel)
4303 * or if it explicitly states that it is.
4306 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
4308 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
4311 /* Find the next tracer that this trace array may use */
4312 static struct tracer
*
4313 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
4315 while (t
&& !trace_ok_for_array(t
, tr
))
4322 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4324 struct trace_array
*tr
= m
->private;
4325 struct tracer
*t
= v
;
4330 t
= get_tracer_for_array(tr
, t
->next
);
4335 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
4337 struct trace_array
*tr
= m
->private;
4341 mutex_lock(&trace_types_lock
);
4343 t
= get_tracer_for_array(tr
, trace_types
);
4344 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
4350 static void t_stop(struct seq_file
*m
, void *p
)
4352 mutex_unlock(&trace_types_lock
);
4355 static int t_show(struct seq_file
*m
, void *v
)
4357 struct tracer
*t
= v
;
4362 seq_puts(m
, t
->name
);
4371 static const struct seq_operations show_traces_seq_ops
= {
4378 static int show_traces_open(struct inode
*inode
, struct file
*file
)
4380 struct trace_array
*tr
= inode
->i_private
;
4384 ret
= tracing_check_open_get_tr(tr
);
4388 ret
= seq_open(file
, &show_traces_seq_ops
);
4390 trace_array_put(tr
);
4394 m
= file
->private_data
;
4400 static int show_traces_release(struct inode
*inode
, struct file
*file
)
4402 struct trace_array
*tr
= inode
->i_private
;
4404 trace_array_put(tr
);
4405 return seq_release(inode
, file
);
4409 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
4410 size_t count
, loff_t
*ppos
)
4415 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
4419 if (file
->f_mode
& FMODE_READ
)
4420 ret
= seq_lseek(file
, offset
, whence
);
4422 file
->f_pos
= ret
= 0;
4427 static const struct file_operations tracing_fops
= {
4428 .open
= tracing_open
,
4430 .write
= tracing_write_stub
,
4431 .llseek
= tracing_lseek
,
4432 .release
= tracing_release
,
4435 static const struct file_operations show_traces_fops
= {
4436 .open
= show_traces_open
,
4438 .llseek
= seq_lseek
,
4439 .release
= show_traces_release
,
4443 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
4444 size_t count
, loff_t
*ppos
)
4446 struct trace_array
*tr
= file_inode(filp
)->i_private
;
4450 len
= snprintf(NULL
, 0, "%*pb\n",
4451 cpumask_pr_args(tr
->tracing_cpumask
)) + 1;
4452 mask_str
= kmalloc(len
, GFP_KERNEL
);
4456 len
= snprintf(mask_str
, len
, "%*pb\n",
4457 cpumask_pr_args(tr
->tracing_cpumask
));
4462 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, len
);
4471 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
4472 size_t count
, loff_t
*ppos
)
4474 struct trace_array
*tr
= file_inode(filp
)->i_private
;
4475 cpumask_var_t tracing_cpumask_new
;
4478 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
4481 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
4485 local_irq_disable();
4486 arch_spin_lock(&tr
->max_lock
);
4487 for_each_tracing_cpu(cpu
) {
4489 * Increase/decrease the disabled counter if we are
4490 * about to flip a bit in the cpumask:
4492 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
4493 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
4494 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
4495 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
4497 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
4498 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
4499 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
4500 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
4503 arch_spin_unlock(&tr
->max_lock
);
4506 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
4507 free_cpumask_var(tracing_cpumask_new
);
4512 free_cpumask_var(tracing_cpumask_new
);
4517 static const struct file_operations tracing_cpumask_fops
= {
4518 .open
= tracing_open_generic_tr
,
4519 .read
= tracing_cpumask_read
,
4520 .write
= tracing_cpumask_write
,
4521 .release
= tracing_release_generic_tr
,
4522 .llseek
= generic_file_llseek
,
4525 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
4527 struct tracer_opt
*trace_opts
;
4528 struct trace_array
*tr
= m
->private;
4532 mutex_lock(&trace_types_lock
);
4533 tracer_flags
= tr
->current_trace
->flags
->val
;
4534 trace_opts
= tr
->current_trace
->flags
->opts
;
4536 for (i
= 0; trace_options
[i
]; i
++) {
4537 if (tr
->trace_flags
& (1 << i
))
4538 seq_printf(m
, "%s\n", trace_options
[i
]);
4540 seq_printf(m
, "no%s\n", trace_options
[i
]);
4543 for (i
= 0; trace_opts
[i
].name
; i
++) {
4544 if (tracer_flags
& trace_opts
[i
].bit
)
4545 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
4547 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
4549 mutex_unlock(&trace_types_lock
);
4554 static int __set_tracer_option(struct trace_array
*tr
,
4555 struct tracer_flags
*tracer_flags
,
4556 struct tracer_opt
*opts
, int neg
)
4558 struct tracer
*trace
= tracer_flags
->trace
;
4561 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
4566 tracer_flags
->val
&= ~opts
->bit
;
4568 tracer_flags
->val
|= opts
->bit
;
4572 /* Try to assign a tracer specific option */
4573 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
4575 struct tracer
*trace
= tr
->current_trace
;
4576 struct tracer_flags
*tracer_flags
= trace
->flags
;
4577 struct tracer_opt
*opts
= NULL
;
4580 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
4581 opts
= &tracer_flags
->opts
[i
];
4583 if (strcmp(cmp
, opts
->name
) == 0)
4584 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
4590 /* Some tracers require overwrite to stay enabled */
4591 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
4593 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
4599 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
4601 if ((mask
== TRACE_ITER_RECORD_TGID
) ||
4602 (mask
== TRACE_ITER_RECORD_CMD
))
4603 lockdep_assert_held(&event_mutex
);
4605 /* do nothing if flag is already set */
4606 if (!!(tr
->trace_flags
& mask
) == !!enabled
)
4609 /* Give the tracer a chance to approve the change */
4610 if (tr
->current_trace
->flag_changed
)
4611 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
4615 tr
->trace_flags
|= mask
;
4617 tr
->trace_flags
&= ~mask
;
4619 if (mask
== TRACE_ITER_RECORD_CMD
)
4620 trace_event_enable_cmd_record(enabled
);
4622 if (mask
== TRACE_ITER_RECORD_TGID
) {
4624 tgid_map
= kvcalloc(PID_MAX_DEFAULT
+ 1,
4628 tr
->trace_flags
&= ~TRACE_ITER_RECORD_TGID
;
4632 trace_event_enable_tgid_record(enabled
);
4635 if (mask
== TRACE_ITER_EVENT_FORK
)
4636 trace_event_follow_fork(tr
, enabled
);
4638 if (mask
== TRACE_ITER_FUNC_FORK
)
4639 ftrace_pid_follow_fork(tr
, enabled
);
4641 if (mask
== TRACE_ITER_OVERWRITE
) {
4642 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
4643 #ifdef CONFIG_TRACER_MAX_TRACE
4644 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
4648 if (mask
== TRACE_ITER_PRINTK
) {
4649 trace_printk_start_stop_comm(enabled
);
4650 trace_printk_control(enabled
);
4656 static int trace_set_options(struct trace_array
*tr
, char *option
)
4661 size_t orig_len
= strlen(option
);
4664 cmp
= strstrip(option
);
4666 len
= str_has_prefix(cmp
, "no");
4672 mutex_lock(&event_mutex
);
4673 mutex_lock(&trace_types_lock
);
4675 ret
= match_string(trace_options
, -1, cmp
);
4676 /* If no option could be set, test the specific tracer options */
4678 ret
= set_tracer_option(tr
, cmp
, neg
);
4680 ret
= set_tracer_flag(tr
, 1 << ret
, !neg
);
4682 mutex_unlock(&trace_types_lock
);
4683 mutex_unlock(&event_mutex
);
4686 * If the first trailing whitespace is replaced with '\0' by strstrip,
4687 * turn it back into a space.
4689 if (orig_len
> strlen(option
))
4690 option
[strlen(option
)] = ' ';
4695 static void __init
apply_trace_boot_options(void)
4697 char *buf
= trace_boot_options_buf
;
4701 option
= strsep(&buf
, ",");
4707 trace_set_options(&global_trace
, option
);
4709 /* Put back the comma to allow this to be called again */
4716 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
4717 size_t cnt
, loff_t
*ppos
)
4719 struct seq_file
*m
= filp
->private_data
;
4720 struct trace_array
*tr
= m
->private;
4724 if (cnt
>= sizeof(buf
))
4727 if (copy_from_user(buf
, ubuf
, cnt
))
4732 ret
= trace_set_options(tr
, buf
);
4741 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
4743 struct trace_array
*tr
= inode
->i_private
;
4746 ret
= tracing_check_open_get_tr(tr
);
4750 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
4752 trace_array_put(tr
);
4757 static const struct file_operations tracing_iter_fops
= {
4758 .open
= tracing_trace_options_open
,
4760 .llseek
= seq_lseek
,
4761 .release
= tracing_single_release_tr
,
4762 .write
= tracing_trace_options_write
,
4765 static const char readme_msg
[] =
4766 "tracing mini-HOWTO:\n\n"
4767 "# echo 0 > tracing_on : quick way to disable tracing\n"
4768 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4769 " Important files:\n"
4770 " trace\t\t\t- The static contents of the buffer\n"
4771 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4772 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4773 " current_tracer\t- function and latency tracers\n"
4774 " available_tracers\t- list of configured tracers for current_tracer\n"
4775 " error_log\t- error log for failed commands (that support it)\n"
4776 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4777 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4778 " trace_clock\t\t-change the clock used to order events\n"
4779 " local: Per cpu clock but may not be synced across CPUs\n"
4780 " global: Synced across CPUs but slows tracing down.\n"
4781 " counter: Not a clock, but just an increment\n"
4782 " uptime: Jiffy counter from time of boot\n"
4783 " perf: Same clock that perf events use\n"
4784 #ifdef CONFIG_X86_64
4785 " x86-tsc: TSC cycle counter\n"
4787 "\n timestamp_mode\t-view the mode used to timestamp events\n"
4788 " delta: Delta difference against a buffer-wide timestamp\n"
4789 " absolute: Absolute (standalone) timestamp\n"
4790 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4791 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4792 " tracing_cpumask\t- Limit which CPUs to trace\n"
4793 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4794 "\t\t\t Remove sub-buffer with rmdir\n"
4795 " trace_options\t\t- Set format or modify how tracing happens\n"
4796 "\t\t\t Disable an option by prefixing 'no' to the\n"
4797 "\t\t\t option name\n"
4798 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4799 #ifdef CONFIG_DYNAMIC_FTRACE
4800 "\n available_filter_functions - list of functions that can be filtered on\n"
4801 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4802 "\t\t\t functions\n"
4803 "\t accepts: func_full_name or glob-matching-pattern\n"
4804 "\t modules: Can select a group via module\n"
4805 "\t Format: :mod:<module-name>\n"
4806 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4807 "\t triggers: a command to perform when function is hit\n"
4808 "\t Format: <function>:<trigger>[:count]\n"
4809 "\t trigger: traceon, traceoff\n"
4810 "\t\t enable_event:<system>:<event>\n"
4811 "\t\t disable_event:<system>:<event>\n"
4812 #ifdef CONFIG_STACKTRACE
4815 #ifdef CONFIG_TRACER_SNAPSHOT
4820 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4821 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4822 "\t The first one will disable tracing every time do_fault is hit\n"
4823 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4824 "\t The first time do trap is hit and it disables tracing, the\n"
4825 "\t counter will decrement to 2. If tracing is already disabled,\n"
4826 "\t the counter will not decrement. It only decrements when the\n"
4827 "\t trigger did work\n"
4828 "\t To remove trigger without count:\n"
4829 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4830 "\t To remove trigger with a count:\n"
4831 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4832 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4833 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4834 "\t modules: Can select a group via module command :mod:\n"
4835 "\t Does not accept triggers\n"
4836 #endif /* CONFIG_DYNAMIC_FTRACE */
4837 #ifdef CONFIG_FUNCTION_TRACER
4838 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4841 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4842 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4843 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4844 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4846 #ifdef CONFIG_TRACER_SNAPSHOT
4847 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4848 "\t\t\t snapshot buffer. Read the contents for more\n"
4849 "\t\t\t information\n"
4851 #ifdef CONFIG_STACK_TRACER
4852 " stack_trace\t\t- Shows the max stack trace when active\n"
4853 " stack_max_size\t- Shows current max stack size that was traced\n"
4854 "\t\t\t Write into this file to reset the max size (trigger a\n"
4855 "\t\t\t new trace)\n"
4856 #ifdef CONFIG_DYNAMIC_FTRACE
4857 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4860 #endif /* CONFIG_STACK_TRACER */
4861 #ifdef CONFIG_DYNAMIC_EVENTS
4862 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
4863 "\t\t\t Write into this file to define/undefine new trace events.\n"
4865 #ifdef CONFIG_KPROBE_EVENTS
4866 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
4867 "\t\t\t Write into this file to define/undefine new trace events.\n"
4869 #ifdef CONFIG_UPROBE_EVENTS
4870 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
4871 "\t\t\t Write into this file to define/undefine new trace events.\n"
4873 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4874 "\t accepts: event-definitions (one definition per line)\n"
4875 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4876 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4877 #ifdef CONFIG_HIST_TRIGGERS
4878 "\t s:[synthetic/]<event> <field> [<field>]\n"
4880 "\t -:[<group>/]<event>\n"
4881 #ifdef CONFIG_KPROBE_EVENTS
4882 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4883 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4885 #ifdef CONFIG_UPROBE_EVENTS
4886 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4888 "\t args: <name>=fetcharg[:type]\n"
4889 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4890 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4891 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
4893 "\t $stack<index>, $stack, $retval, $comm,\n"
4895 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
4896 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4897 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
4898 "\t <type>\\[<array-size>\\]\n"
4899 #ifdef CONFIG_HIST_TRIGGERS
4900 "\t field: <stype> <name>;\n"
4901 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4902 "\t [unsigned] char/int/long\n"
4905 " events/\t\t- Directory containing all trace event subsystems:\n"
4906 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4907 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4908 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4910 " filter\t\t- If set, only events passing filter are traced\n"
4911 " events/<system>/<event>/\t- Directory containing control files for\n"
4913 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4914 " filter\t\t- If set, only events passing filter are traced\n"
4915 " trigger\t\t- If set, a command to perform when event is hit\n"
4916 "\t Format: <trigger>[:count][if <filter>]\n"
4917 "\t trigger: traceon, traceoff\n"
4918 "\t enable_event:<system>:<event>\n"
4919 "\t disable_event:<system>:<event>\n"
4920 #ifdef CONFIG_HIST_TRIGGERS
4921 "\t enable_hist:<system>:<event>\n"
4922 "\t disable_hist:<system>:<event>\n"
4924 #ifdef CONFIG_STACKTRACE
4927 #ifdef CONFIG_TRACER_SNAPSHOT
4930 #ifdef CONFIG_HIST_TRIGGERS
4931 "\t\t hist (see below)\n"
4933 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4934 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4935 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4936 "\t events/block/block_unplug/trigger\n"
4937 "\t The first disables tracing every time block_unplug is hit.\n"
4938 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4939 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4940 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4941 "\t Like function triggers, the counter is only decremented if it\n"
4942 "\t enabled or disabled tracing.\n"
4943 "\t To remove a trigger without a count:\n"
4944 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4945 "\t To remove a trigger with a count:\n"
4946 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4947 "\t Filters can be ignored when removing a trigger.\n"
4948 #ifdef CONFIG_HIST_TRIGGERS
4949 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4950 "\t Format: hist:keys=<field1[,field2,...]>\n"
4951 "\t [:values=<field1[,field2,...]>]\n"
4952 "\t [:sort=<field1[,field2,...]>]\n"
4953 "\t [:size=#entries]\n"
4954 "\t [:pause][:continue][:clear]\n"
4955 "\t [:name=histname1]\n"
4956 "\t [:<handler>.<action>]\n"
4957 "\t [if <filter>]\n\n"
4958 "\t When a matching event is hit, an entry is added to a hash\n"
4959 "\t table using the key(s) and value(s) named, and the value of a\n"
4960 "\t sum called 'hitcount' is incremented. Keys and values\n"
4961 "\t correspond to fields in the event's format description. Keys\n"
4962 "\t can be any field, or the special string 'stacktrace'.\n"
4963 "\t Compound keys consisting of up to two fields can be specified\n"
4964 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4965 "\t fields. Sort keys consisting of up to two fields can be\n"
4966 "\t specified using the 'sort' keyword. The sort direction can\n"
4967 "\t be modified by appending '.descending' or '.ascending' to a\n"
4968 "\t sort field. The 'size' parameter can be used to specify more\n"
4969 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4970 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4971 "\t its histogram data will be shared with other triggers of the\n"
4972 "\t same name, and trigger hits will update this common data.\n\n"
4973 "\t Reading the 'hist' file for the event will dump the hash\n"
4974 "\t table in its entirety to stdout. If there are multiple hist\n"
4975 "\t triggers attached to an event, there will be a table for each\n"
4976 "\t trigger in the output. The table displayed for a named\n"
4977 "\t trigger will be the same as any other instance having the\n"
4978 "\t same name. The default format used to display a given field\n"
4979 "\t can be modified by appending any of the following modifiers\n"
4980 "\t to the field name, as applicable:\n\n"
4981 "\t .hex display a number as a hex value\n"
4982 "\t .sym display an address as a symbol\n"
4983 "\t .sym-offset display an address as a symbol and offset\n"
4984 "\t .execname display a common_pid as a program name\n"
4985 "\t .syscall display a syscall id as a syscall name\n"
4986 "\t .log2 display log2 value rather than raw number\n"
4987 "\t .usecs display a common_timestamp in microseconds\n\n"
4988 "\t The 'pause' parameter can be used to pause an existing hist\n"
4989 "\t trigger or to start a hist trigger but not log any events\n"
4990 "\t until told to do so. 'continue' can be used to start or\n"
4991 "\t restart a paused hist trigger.\n\n"
4992 "\t The 'clear' parameter will clear the contents of a running\n"
4993 "\t hist trigger and leave its current paused/active state\n"
4995 "\t The enable_hist and disable_hist triggers can be used to\n"
4996 "\t have one event conditionally start and stop another event's\n"
4997 "\t already-attached hist trigger. The syntax is analogous to\n"
4998 "\t the enable_event and disable_event triggers.\n\n"
4999 "\t Hist trigger handlers and actions are executed whenever a\n"
5000 "\t a histogram entry is added or updated. They take the form:\n\n"
5001 "\t <handler>.<action>\n\n"
5002 "\t The available handlers are:\n\n"
5003 "\t onmatch(matching.event) - invoke on addition or update\n"
5004 "\t onmax(var) - invoke if var exceeds current max\n"
5005 "\t onchange(var) - invoke action if var changes\n\n"
5006 "\t The available actions are:\n\n"
5007 "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5008 "\t save(field,...) - save current event fields\n"
5009 #ifdef CONFIG_TRACER_SNAPSHOT
5010 "\t snapshot() - snapshot the trace buffer\n"
5016 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
5017 size_t cnt
, loff_t
*ppos
)
5019 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
5020 readme_msg
, strlen(readme_msg
));
5023 static const struct file_operations tracing_readme_fops
= {
5024 .open
= tracing_open_generic
,
5025 .read
= tracing_readme_read
,
5026 .llseek
= generic_file_llseek
,
5029 static void *saved_tgids_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
5033 if (*pos
|| m
->count
)
5038 for (; ptr
<= &tgid_map
[PID_MAX_DEFAULT
]; ptr
++) {
5039 if (trace_find_tgid(*ptr
))
5046 static void *saved_tgids_start(struct seq_file
*m
, loff_t
*pos
)
5056 v
= saved_tgids_next(m
, v
, &l
);
5064 static void saved_tgids_stop(struct seq_file
*m
, void *v
)
5068 static int saved_tgids_show(struct seq_file
*m
, void *v
)
5070 int pid
= (int *)v
- tgid_map
;
5072 seq_printf(m
, "%d %d\n", pid
, trace_find_tgid(pid
));
5076 static const struct seq_operations tracing_saved_tgids_seq_ops
= {
5077 .start
= saved_tgids_start
,
5078 .stop
= saved_tgids_stop
,
5079 .next
= saved_tgids_next
,
5080 .show
= saved_tgids_show
,
5083 static int tracing_saved_tgids_open(struct inode
*inode
, struct file
*filp
)
5087 ret
= tracing_check_open_get_tr(NULL
);
5091 return seq_open(filp
, &tracing_saved_tgids_seq_ops
);
5095 static const struct file_operations tracing_saved_tgids_fops
= {
5096 .open
= tracing_saved_tgids_open
,
5098 .llseek
= seq_lseek
,
5099 .release
= seq_release
,
5102 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
5104 unsigned int *ptr
= v
;
5106 if (*pos
|| m
->count
)
5111 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
5113 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
5122 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
5128 arch_spin_lock(&trace_cmdline_lock
);
5130 v
= &savedcmd
->map_cmdline_to_pid
[0];
5132 v
= saved_cmdlines_next(m
, v
, &l
);
5140 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
5142 arch_spin_unlock(&trace_cmdline_lock
);
5146 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
5148 char buf
[TASK_COMM_LEN
];
5149 unsigned int *pid
= v
;
5151 __trace_find_cmdline(*pid
, buf
);
5152 seq_printf(m
, "%d %s\n", *pid
, buf
);
5156 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
5157 .start
= saved_cmdlines_start
,
5158 .next
= saved_cmdlines_next
,
5159 .stop
= saved_cmdlines_stop
,
5160 .show
= saved_cmdlines_show
,
5163 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
5167 ret
= tracing_check_open_get_tr(NULL
);
5171 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
5174 static const struct file_operations tracing_saved_cmdlines_fops
= {
5175 .open
= tracing_saved_cmdlines_open
,
5177 .llseek
= seq_lseek
,
5178 .release
= seq_release
,
5182 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
5183 size_t cnt
, loff_t
*ppos
)
5188 arch_spin_lock(&trace_cmdline_lock
);
5189 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
5190 arch_spin_unlock(&trace_cmdline_lock
);
5192 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5195 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
5197 kfree(s
->saved_cmdlines
);
5198 kfree(s
->map_cmdline_to_pid
);
5202 static int tracing_resize_saved_cmdlines(unsigned int val
)
5204 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
5206 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
5210 if (allocate_cmdlines_buffer(val
, s
) < 0) {
5215 arch_spin_lock(&trace_cmdline_lock
);
5216 savedcmd_temp
= savedcmd
;
5218 arch_spin_unlock(&trace_cmdline_lock
);
5219 free_saved_cmdlines_buffer(savedcmd_temp
);
5225 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
5226 size_t cnt
, loff_t
*ppos
)
5231 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5235 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
5236 if (!val
|| val
> PID_MAX_DEFAULT
)
5239 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
5248 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
5249 .open
= tracing_open_generic
,
5250 .read
= tracing_saved_cmdlines_size_read
,
5251 .write
= tracing_saved_cmdlines_size_write
,
5254 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
5255 static union trace_eval_map_item
*
5256 update_eval_map(union trace_eval_map_item
*ptr
)
5258 if (!ptr
->map
.eval_string
) {
5259 if (ptr
->tail
.next
) {
5260 ptr
= ptr
->tail
.next
;
5261 /* Set ptr to the next real item (skip head) */
5269 static void *eval_map_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
5271 union trace_eval_map_item
*ptr
= v
;
5274 * Paranoid! If ptr points to end, we don't want to increment past it.
5275 * This really should never happen.
5277 ptr
= update_eval_map(ptr
);
5278 if (WARN_ON_ONCE(!ptr
))
5285 ptr
= update_eval_map(ptr
);
5290 static void *eval_map_start(struct seq_file
*m
, loff_t
*pos
)
5292 union trace_eval_map_item
*v
;
5295 mutex_lock(&trace_eval_mutex
);
5297 v
= trace_eval_maps
;
5301 while (v
&& l
< *pos
) {
5302 v
= eval_map_next(m
, v
, &l
);
5308 static void eval_map_stop(struct seq_file
*m
, void *v
)
5310 mutex_unlock(&trace_eval_mutex
);
5313 static int eval_map_show(struct seq_file
*m
, void *v
)
5315 union trace_eval_map_item
*ptr
= v
;
5317 seq_printf(m
, "%s %ld (%s)\n",
5318 ptr
->map
.eval_string
, ptr
->map
.eval_value
,
5324 static const struct seq_operations tracing_eval_map_seq_ops
= {
5325 .start
= eval_map_start
,
5326 .next
= eval_map_next
,
5327 .stop
= eval_map_stop
,
5328 .show
= eval_map_show
,
5331 static int tracing_eval_map_open(struct inode
*inode
, struct file
*filp
)
5335 ret
= tracing_check_open_get_tr(NULL
);
5339 return seq_open(filp
, &tracing_eval_map_seq_ops
);
5342 static const struct file_operations tracing_eval_map_fops
= {
5343 .open
= tracing_eval_map_open
,
5345 .llseek
= seq_lseek
,
5346 .release
= seq_release
,
5349 static inline union trace_eval_map_item
*
5350 trace_eval_jmp_to_tail(union trace_eval_map_item
*ptr
)
5352 /* Return tail of array given the head */
5353 return ptr
+ ptr
->head
.length
+ 1;
5357 trace_insert_eval_map_file(struct module
*mod
, struct trace_eval_map
**start
,
5360 struct trace_eval_map
**stop
;
5361 struct trace_eval_map
**map
;
5362 union trace_eval_map_item
*map_array
;
5363 union trace_eval_map_item
*ptr
;
5368 * The trace_eval_maps contains the map plus a head and tail item,
5369 * where the head holds the module and length of array, and the
5370 * tail holds a pointer to the next list.
5372 map_array
= kmalloc_array(len
+ 2, sizeof(*map_array
), GFP_KERNEL
);
5374 pr_warn("Unable to allocate trace eval mapping\n");
5378 mutex_lock(&trace_eval_mutex
);
5380 if (!trace_eval_maps
)
5381 trace_eval_maps
= map_array
;
5383 ptr
= trace_eval_maps
;
5385 ptr
= trace_eval_jmp_to_tail(ptr
);
5386 if (!ptr
->tail
.next
)
5388 ptr
= ptr
->tail
.next
;
5391 ptr
->tail
.next
= map_array
;
5393 map_array
->head
.mod
= mod
;
5394 map_array
->head
.length
= len
;
5397 for (map
= start
; (unsigned long)map
< (unsigned long)stop
; map
++) {
5398 map_array
->map
= **map
;
5401 memset(map_array
, 0, sizeof(*map_array
));
5403 mutex_unlock(&trace_eval_mutex
);
5406 static void trace_create_eval_file(struct dentry
*d_tracer
)
5408 trace_create_file("eval_map", 0444, d_tracer
,
5409 NULL
, &tracing_eval_map_fops
);
5412 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5413 static inline void trace_create_eval_file(struct dentry
*d_tracer
) { }
5414 static inline void trace_insert_eval_map_file(struct module
*mod
,
5415 struct trace_eval_map
**start
, int len
) { }
5416 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5418 static void trace_insert_eval_map(struct module
*mod
,
5419 struct trace_eval_map
**start
, int len
)
5421 struct trace_eval_map
**map
;
5428 trace_event_eval_update(map
, len
);
5430 trace_insert_eval_map_file(mod
, start
, len
);
5434 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
5435 size_t cnt
, loff_t
*ppos
)
5437 struct trace_array
*tr
= filp
->private_data
;
5438 char buf
[MAX_TRACER_SIZE
+2];
5441 mutex_lock(&trace_types_lock
);
5442 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
5443 mutex_unlock(&trace_types_lock
);
5445 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5448 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
5450 tracing_reset_online_cpus(&tr
->trace_buffer
);
5454 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
5458 for_each_tracing_cpu(cpu
)
5459 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
5462 #ifdef CONFIG_TRACER_MAX_TRACE
5463 /* resize @tr's buffer to the size of @size_tr's entries */
5464 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
5465 struct trace_buffer
*size_buf
, int cpu_id
)
5469 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
5470 for_each_tracing_cpu(cpu
) {
5471 ret
= ring_buffer_resize(trace_buf
->buffer
,
5472 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
5475 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
5476 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
5479 ret
= ring_buffer_resize(trace_buf
->buffer
,
5480 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
5482 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
5483 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
5488 #endif /* CONFIG_TRACER_MAX_TRACE */
5490 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
5491 unsigned long size
, int cpu
)
5496 * If kernel or user changes the size of the ring buffer
5497 * we use the size that was given, and we can forget about
5498 * expanding it later.
5500 ring_buffer_expanded
= true;
5502 /* May be called before buffers are initialized */
5503 if (!tr
->trace_buffer
.buffer
)
5506 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
5510 #ifdef CONFIG_TRACER_MAX_TRACE
5511 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
5512 !tr
->current_trace
->use_max_tr
)
5515 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
5517 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
5518 &tr
->trace_buffer
, cpu
);
5521 * AARGH! We are left with different
5522 * size max buffer!!!!
5523 * The max buffer is our "snapshot" buffer.
5524 * When a tracer needs a snapshot (one of the
5525 * latency tracers), it swaps the max buffer
5526 * with the saved snap shot. We succeeded to
5527 * update the size of the main buffer, but failed to
5528 * update the size of the max buffer. But when we tried
5529 * to reset the main buffer to the original size, we
5530 * failed there too. This is very unlikely to
5531 * happen, but if it does, warn and kill all
5535 tracing_disabled
= 1;
5540 if (cpu
== RING_BUFFER_ALL_CPUS
)
5541 set_buffer_entries(&tr
->max_buffer
, size
);
5543 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
5546 #endif /* CONFIG_TRACER_MAX_TRACE */
5548 if (cpu
== RING_BUFFER_ALL_CPUS
)
5549 set_buffer_entries(&tr
->trace_buffer
, size
);
5551 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
5556 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
5557 unsigned long size
, int cpu_id
)
5561 mutex_lock(&trace_types_lock
);
5563 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
5564 /* make sure, this cpu is enabled in the mask */
5565 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
5571 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
5576 mutex_unlock(&trace_types_lock
);
5583 * tracing_update_buffers - used by tracing facility to expand ring buffers
5585 * To save on memory when the tracing is never used on a system with it
5586 * configured in. The ring buffers are set to a minimum size. But once
5587 * a user starts to use the tracing facility, then they need to grow
5588 * to their default size.
5590 * This function is to be called when a tracer is about to be used.
5592 int tracing_update_buffers(void)
5596 mutex_lock(&trace_types_lock
);
5597 if (!ring_buffer_expanded
)
5598 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
5599 RING_BUFFER_ALL_CPUS
);
5600 mutex_unlock(&trace_types_lock
);
5605 struct trace_option_dentry
;
5608 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
5611 * Used to clear out the tracer before deletion of an instance.
5612 * Must have trace_types_lock held.
5614 static void tracing_set_nop(struct trace_array
*tr
)
5616 if (tr
->current_trace
== &nop_trace
)
5619 tr
->current_trace
->enabled
--;
5621 if (tr
->current_trace
->reset
)
5622 tr
->current_trace
->reset(tr
);
5624 tr
->current_trace
= &nop_trace
;
5627 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
)
5629 /* Only enable if the directory has been created already. */
5633 create_trace_option_files(tr
, t
);
5636 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
5639 #ifdef CONFIG_TRACER_MAX_TRACE
5644 mutex_lock(&trace_types_lock
);
5646 if (!ring_buffer_expanded
) {
5647 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
5648 RING_BUFFER_ALL_CPUS
);
5654 for (t
= trace_types
; t
; t
= t
->next
) {
5655 if (strcmp(t
->name
, buf
) == 0)
5662 if (t
== tr
->current_trace
)
5665 #ifdef CONFIG_TRACER_SNAPSHOT
5666 if (t
->use_max_tr
) {
5667 arch_spin_lock(&tr
->max_lock
);
5668 if (tr
->cond_snapshot
)
5670 arch_spin_unlock(&tr
->max_lock
);
5675 /* Some tracers won't work on kernel command line */
5676 if (system_state
< SYSTEM_RUNNING
&& t
->noboot
) {
5677 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5682 /* Some tracers are only allowed for the top level buffer */
5683 if (!trace_ok_for_array(t
, tr
)) {
5688 /* If trace pipe files are being read, we can't change the tracer */
5689 if (tr
->trace_ref
) {
5694 trace_branch_disable();
5696 tr
->current_trace
->enabled
--;
5698 if (tr
->current_trace
->reset
)
5699 tr
->current_trace
->reset(tr
);
5701 /* Current trace needs to be nop_trace before synchronize_rcu */
5702 tr
->current_trace
= &nop_trace
;
5704 #ifdef CONFIG_TRACER_MAX_TRACE
5705 had_max_tr
= tr
->allocated_snapshot
;
5707 if (had_max_tr
&& !t
->use_max_tr
) {
5709 * We need to make sure that the update_max_tr sees that
5710 * current_trace changed to nop_trace to keep it from
5711 * swapping the buffers after we resize it.
5712 * The update_max_tr is called from interrupts disabled
5713 * so a synchronized_sched() is sufficient.
5720 #ifdef CONFIG_TRACER_MAX_TRACE
5721 if (t
->use_max_tr
&& !had_max_tr
) {
5722 ret
= tracing_alloc_snapshot_instance(tr
);
5729 ret
= tracer_init(t
, tr
);
5734 tr
->current_trace
= t
;
5735 tr
->current_trace
->enabled
++;
5736 trace_branch_enable(tr
);
5738 mutex_unlock(&trace_types_lock
);
5744 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
5745 size_t cnt
, loff_t
*ppos
)
5747 struct trace_array
*tr
= filp
->private_data
;
5748 char buf
[MAX_TRACER_SIZE
+1];
5755 if (cnt
> MAX_TRACER_SIZE
)
5756 cnt
= MAX_TRACER_SIZE
;
5758 if (copy_from_user(buf
, ubuf
, cnt
))
5763 /* strip ending whitespace. */
5764 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
5767 err
= tracing_set_tracer(tr
, buf
);
5777 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
5778 size_t cnt
, loff_t
*ppos
)
5783 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
5784 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
5785 if (r
> sizeof(buf
))
5787 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5791 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
5792 size_t cnt
, loff_t
*ppos
)
5797 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5807 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
5808 size_t cnt
, loff_t
*ppos
)
5810 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
5814 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
5815 size_t cnt
, loff_t
*ppos
)
5817 struct trace_array
*tr
= filp
->private_data
;
5820 mutex_lock(&trace_types_lock
);
5821 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
5825 if (tr
->current_trace
->update_thresh
) {
5826 ret
= tr
->current_trace
->update_thresh(tr
);
5833 mutex_unlock(&trace_types_lock
);
5838 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5841 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
5842 size_t cnt
, loff_t
*ppos
)
5844 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
5848 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
5849 size_t cnt
, loff_t
*ppos
)
5851 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
5856 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
5858 struct trace_array
*tr
= inode
->i_private
;
5859 struct trace_iterator
*iter
;
5862 ret
= tracing_check_open_get_tr(tr
);
5866 mutex_lock(&trace_types_lock
);
5868 /* create a buffer to store the information to pass to userspace */
5869 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5872 __trace_array_put(tr
);
5876 trace_seq_init(&iter
->seq
);
5877 iter
->trace
= tr
->current_trace
;
5879 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
5884 /* trace pipe does not show start of buffer */
5885 cpumask_setall(iter
->started
);
5887 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
5888 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
5890 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5891 if (trace_clocks
[tr
->clock_id
].in_ns
)
5892 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
5895 iter
->trace_buffer
= &tr
->trace_buffer
;
5896 iter
->cpu_file
= tracing_get_cpu(inode
);
5897 mutex_init(&iter
->mutex
);
5898 filp
->private_data
= iter
;
5900 if (iter
->trace
->pipe_open
)
5901 iter
->trace
->pipe_open(iter
);
5903 nonseekable_open(inode
, filp
);
5907 mutex_unlock(&trace_types_lock
);
5912 __trace_array_put(tr
);
5913 mutex_unlock(&trace_types_lock
);
5917 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
5919 struct trace_iterator
*iter
= file
->private_data
;
5920 struct trace_array
*tr
= inode
->i_private
;
5922 mutex_lock(&trace_types_lock
);
5926 if (iter
->trace
->pipe_close
)
5927 iter
->trace
->pipe_close(iter
);
5929 mutex_unlock(&trace_types_lock
);
5931 free_cpumask_var(iter
->started
);
5932 mutex_destroy(&iter
->mutex
);
5935 trace_array_put(tr
);
5941 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
5943 struct trace_array
*tr
= iter
->tr
;
5945 /* Iterators are static, they should be filled or empty */
5946 if (trace_buffer_iter(iter
, iter
->cpu_file
))
5947 return EPOLLIN
| EPOLLRDNORM
;
5949 if (tr
->trace_flags
& TRACE_ITER_BLOCK
)
5951 * Always select as readable when in blocking mode
5953 return EPOLLIN
| EPOLLRDNORM
;
5955 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
5960 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
5962 struct trace_iterator
*iter
= filp
->private_data
;
5964 return trace_poll(iter
, filp
, poll_table
);
5967 /* Must be called with iter->mutex held. */
5968 static int tracing_wait_pipe(struct file
*filp
)
5970 struct trace_iterator
*iter
= filp
->private_data
;
5973 while (trace_empty(iter
)) {
5975 if ((filp
->f_flags
& O_NONBLOCK
)) {
5980 * We block until we read something and tracing is disabled.
5981 * We still block if tracing is disabled, but we have never
5982 * read anything. This allows a user to cat this file, and
5983 * then enable tracing. But after we have read something,
5984 * we give an EOF when tracing is again disabled.
5986 * iter->pos will be 0 if we haven't read anything.
5988 if (!tracer_tracing_is_on(iter
->tr
) && iter
->pos
)
5991 mutex_unlock(&iter
->mutex
);
5993 ret
= wait_on_pipe(iter
, 0);
5995 mutex_lock(&iter
->mutex
);
6008 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
6009 size_t cnt
, loff_t
*ppos
)
6011 struct trace_iterator
*iter
= filp
->private_data
;
6015 * Avoid more than one consumer on a single file descriptor
6016 * This is just a matter of traces coherency, the ring buffer itself
6019 mutex_lock(&iter
->mutex
);
6021 /* return any leftover data */
6022 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
6026 trace_seq_init(&iter
->seq
);
6028 if (iter
->trace
->read
) {
6029 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
6035 sret
= tracing_wait_pipe(filp
);
6039 /* stop when tracing is finished */
6040 if (trace_empty(iter
)) {
6045 if (cnt
>= PAGE_SIZE
)
6046 cnt
= PAGE_SIZE
- 1;
6048 /* reset all but tr, trace, and overruns */
6049 memset(&iter
->seq
, 0,
6050 sizeof(struct trace_iterator
) -
6051 offsetof(struct trace_iterator
, seq
));
6052 cpumask_clear(iter
->started
);
6053 trace_seq_init(&iter
->seq
);
6056 trace_event_read_lock();
6057 trace_access_lock(iter
->cpu_file
);
6058 while (trace_find_next_entry_inc(iter
) != NULL
) {
6059 enum print_line_t ret
;
6060 int save_len
= iter
->seq
.seq
.len
;
6062 ret
= print_trace_line(iter
);
6063 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
6064 /* don't print partial lines */
6065 iter
->seq
.seq
.len
= save_len
;
6068 if (ret
!= TRACE_TYPE_NO_CONSUME
)
6069 trace_consume(iter
);
6071 if (trace_seq_used(&iter
->seq
) >= cnt
)
6075 * Setting the full flag means we reached the trace_seq buffer
6076 * size and we should leave by partial output condition above.
6077 * One of the trace_seq_* functions is not used properly.
6079 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
6082 trace_access_unlock(iter
->cpu_file
);
6083 trace_event_read_unlock();
6085 /* Now copy what we have to the user */
6086 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
6087 if (iter
->seq
.seq
.readpos
>= trace_seq_used(&iter
->seq
))
6088 trace_seq_init(&iter
->seq
);
6091 * If there was nothing to send to user, in spite of consuming trace
6092 * entries, go back to wait for more entries.
6098 mutex_unlock(&iter
->mutex
);
6103 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
6106 __free_page(spd
->pages
[idx
]);
6109 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
6110 .confirm
= generic_pipe_buf_confirm
,
6111 .release
= generic_pipe_buf_release
,
6112 .steal
= generic_pipe_buf_steal
,
6113 .get
= generic_pipe_buf_get
,
6117 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
6123 /* Seq buffer is page-sized, exactly what we need. */
6125 save_len
= iter
->seq
.seq
.len
;
6126 ret
= print_trace_line(iter
);
6128 if (trace_seq_has_overflowed(&iter
->seq
)) {
6129 iter
->seq
.seq
.len
= save_len
;
6134 * This should not be hit, because it should only
6135 * be set if the iter->seq overflowed. But check it
6136 * anyway to be safe.
6138 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
6139 iter
->seq
.seq
.len
= save_len
;
6143 count
= trace_seq_used(&iter
->seq
) - save_len
;
6146 iter
->seq
.seq
.len
= save_len
;
6150 if (ret
!= TRACE_TYPE_NO_CONSUME
)
6151 trace_consume(iter
);
6153 if (!trace_find_next_entry_inc(iter
)) {
6163 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
6165 struct pipe_inode_info
*pipe
,
6169 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
6170 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
6171 struct trace_iterator
*iter
= filp
->private_data
;
6172 struct splice_pipe_desc spd
= {
6174 .partial
= partial_def
,
6175 .nr_pages
= 0, /* This gets updated below. */
6176 .nr_pages_max
= PIPE_DEF_BUFFERS
,
6177 .ops
= &tracing_pipe_buf_ops
,
6178 .spd_release
= tracing_spd_release_pipe
,
6184 if (splice_grow_spd(pipe
, &spd
))
6187 mutex_lock(&iter
->mutex
);
6189 if (iter
->trace
->splice_read
) {
6190 ret
= iter
->trace
->splice_read(iter
, filp
,
6191 ppos
, pipe
, len
, flags
);
6196 ret
= tracing_wait_pipe(filp
);
6200 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
6205 trace_event_read_lock();
6206 trace_access_lock(iter
->cpu_file
);
6208 /* Fill as many pages as possible. */
6209 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
6210 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
6214 rem
= tracing_fill_pipe_page(rem
, iter
);
6216 /* Copy the data into the page, so we can start over. */
6217 ret
= trace_seq_to_buffer(&iter
->seq
,
6218 page_address(spd
.pages
[i
]),
6219 trace_seq_used(&iter
->seq
));
6221 __free_page(spd
.pages
[i
]);
6224 spd
.partial
[i
].offset
= 0;
6225 spd
.partial
[i
].len
= trace_seq_used(&iter
->seq
);
6227 trace_seq_init(&iter
->seq
);
6230 trace_access_unlock(iter
->cpu_file
);
6231 trace_event_read_unlock();
6232 mutex_unlock(&iter
->mutex
);
6237 ret
= splice_to_pipe(pipe
, &spd
);
6241 splice_shrink_spd(&spd
);
6245 mutex_unlock(&iter
->mutex
);
6250 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
6251 size_t cnt
, loff_t
*ppos
)
6253 struct inode
*inode
= file_inode(filp
);
6254 struct trace_array
*tr
= inode
->i_private
;
6255 int cpu
= tracing_get_cpu(inode
);
6260 mutex_lock(&trace_types_lock
);
6262 if (cpu
== RING_BUFFER_ALL_CPUS
) {
6263 int cpu
, buf_size_same
;
6268 /* check if all cpu sizes are same */
6269 for_each_tracing_cpu(cpu
) {
6270 /* fill in the size from first enabled cpu */
6272 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
6273 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
6279 if (buf_size_same
) {
6280 if (!ring_buffer_expanded
)
6281 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
6283 trace_buf_size
>> 10);
6285 r
= sprintf(buf
, "%lu\n", size
>> 10);
6287 r
= sprintf(buf
, "X\n");
6289 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
6291 mutex_unlock(&trace_types_lock
);
6293 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6298 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
6299 size_t cnt
, loff_t
*ppos
)
6301 struct inode
*inode
= file_inode(filp
);
6302 struct trace_array
*tr
= inode
->i_private
;
6306 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6310 /* must have at least 1 entry */
6314 /* value is in KB */
6316 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
6326 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
6327 size_t cnt
, loff_t
*ppos
)
6329 struct trace_array
*tr
= filp
->private_data
;
6332 unsigned long size
= 0, expanded_size
= 0;
6334 mutex_lock(&trace_types_lock
);
6335 for_each_tracing_cpu(cpu
) {
6336 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
6337 if (!ring_buffer_expanded
)
6338 expanded_size
+= trace_buf_size
>> 10;
6340 if (ring_buffer_expanded
)
6341 r
= sprintf(buf
, "%lu\n", size
);
6343 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
6344 mutex_unlock(&trace_types_lock
);
6346 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6350 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
6351 size_t cnt
, loff_t
*ppos
)
6354 * There is no need to read what the user has written, this function
6355 * is just to make sure that there is no error when "echo" is used
6364 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
6366 struct trace_array
*tr
= inode
->i_private
;
6368 /* disable tracing ? */
6369 if (tr
->trace_flags
& TRACE_ITER_STOP_ON_FREE
)
6370 tracer_tracing_off(tr
);
6371 /* resize the ring buffer to 0 */
6372 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
6374 trace_array_put(tr
);
6380 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
6381 size_t cnt
, loff_t
*fpos
)
6383 struct trace_array
*tr
= filp
->private_data
;
6384 struct ring_buffer_event
*event
;
6385 enum event_trigger_type tt
= ETT_NONE
;
6386 struct ring_buffer
*buffer
;
6387 struct print_entry
*entry
;
6388 unsigned long irq_flags
;
6393 /* Used in tracing_mark_raw_write() as well */
6394 #define FAULTED_STR "<faulted>"
6395 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6397 if (tracing_disabled
)
6400 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
6403 if (cnt
> TRACE_BUF_SIZE
)
6404 cnt
= TRACE_BUF_SIZE
;
6406 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
6408 local_save_flags(irq_flags
);
6409 size
= sizeof(*entry
) + cnt
+ 2; /* add '\0' and possible '\n' */
6411 /* If less than "<faulted>", then make sure we can still add that */
6412 if (cnt
< FAULTED_SIZE
)
6413 size
+= FAULTED_SIZE
- cnt
;
6415 buffer
= tr
->trace_buffer
.buffer
;
6416 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
6417 irq_flags
, preempt_count());
6418 if (unlikely(!event
))
6419 /* Ring buffer disabled, return as if not open for write */
6422 entry
= ring_buffer_event_data(event
);
6423 entry
->ip
= _THIS_IP_
;
6425 len
= __copy_from_user_inatomic(&entry
->buf
, ubuf
, cnt
);
6427 memcpy(&entry
->buf
, FAULTED_STR
, FAULTED_SIZE
);
6434 if (tr
->trace_marker_file
&& !list_empty(&tr
->trace_marker_file
->triggers
)) {
6435 /* do not add \n before testing triggers, but add \0 */
6436 entry
->buf
[cnt
] = '\0';
6437 tt
= event_triggers_call(tr
->trace_marker_file
, entry
, event
);
6440 if (entry
->buf
[cnt
- 1] != '\n') {
6441 entry
->buf
[cnt
] = '\n';
6442 entry
->buf
[cnt
+ 1] = '\0';
6444 entry
->buf
[cnt
] = '\0';
6446 __buffer_unlock_commit(buffer
, event
);
6449 event_triggers_post_call(tr
->trace_marker_file
, tt
);
6457 /* Limit it for now to 3K (including tag) */
6458 #define RAW_DATA_MAX_SIZE (1024*3)
6461 tracing_mark_raw_write(struct file
*filp
, const char __user
*ubuf
,
6462 size_t cnt
, loff_t
*fpos
)
6464 struct trace_array
*tr
= filp
->private_data
;
6465 struct ring_buffer_event
*event
;
6466 struct ring_buffer
*buffer
;
6467 struct raw_data_entry
*entry
;
6468 unsigned long irq_flags
;
6473 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6475 if (tracing_disabled
)
6478 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
6481 /* The marker must at least have a tag id */
6482 if (cnt
< sizeof(unsigned int) || cnt
> RAW_DATA_MAX_SIZE
)
6485 if (cnt
> TRACE_BUF_SIZE
)
6486 cnt
= TRACE_BUF_SIZE
;
6488 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
6490 local_save_flags(irq_flags
);
6491 size
= sizeof(*entry
) + cnt
;
6492 if (cnt
< FAULT_SIZE_ID
)
6493 size
+= FAULT_SIZE_ID
- cnt
;
6495 buffer
= tr
->trace_buffer
.buffer
;
6496 event
= __trace_buffer_lock_reserve(buffer
, TRACE_RAW_DATA
, size
,
6497 irq_flags
, preempt_count());
6499 /* Ring buffer disabled, return as if not open for write */
6502 entry
= ring_buffer_event_data(event
);
6504 len
= __copy_from_user_inatomic(&entry
->id
, ubuf
, cnt
);
6507 memcpy(&entry
->buf
, FAULTED_STR
, FAULTED_SIZE
);
6512 __buffer_unlock_commit(buffer
, event
);
6520 static int tracing_clock_show(struct seq_file
*m
, void *v
)
6522 struct trace_array
*tr
= m
->private;
6525 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
6527 "%s%s%s%s", i
? " " : "",
6528 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
6529 i
== tr
->clock_id
? "]" : "");
6535 int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
6539 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
6540 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
6543 if (i
== ARRAY_SIZE(trace_clocks
))
6546 mutex_lock(&trace_types_lock
);
6550 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
6553 * New clock may not be consistent with the previous clock.
6554 * Reset the buffer so that it doesn't have incomparable timestamps.
6556 tracing_reset_online_cpus(&tr
->trace_buffer
);
6558 #ifdef CONFIG_TRACER_MAX_TRACE
6559 if (tr
->max_buffer
.buffer
)
6560 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
6561 tracing_reset_online_cpus(&tr
->max_buffer
);
6564 mutex_unlock(&trace_types_lock
);
6569 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
6570 size_t cnt
, loff_t
*fpos
)
6572 struct seq_file
*m
= filp
->private_data
;
6573 struct trace_array
*tr
= m
->private;
6575 const char *clockstr
;
6578 if (cnt
>= sizeof(buf
))
6581 if (copy_from_user(buf
, ubuf
, cnt
))
6586 clockstr
= strstrip(buf
);
6588 ret
= tracing_set_clock(tr
, clockstr
);
6597 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
6599 struct trace_array
*tr
= inode
->i_private
;
6602 ret
= tracing_check_open_get_tr(tr
);
6606 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
6608 trace_array_put(tr
);
6613 static int tracing_time_stamp_mode_show(struct seq_file
*m
, void *v
)
6615 struct trace_array
*tr
= m
->private;
6617 mutex_lock(&trace_types_lock
);
6619 if (ring_buffer_time_stamp_abs(tr
->trace_buffer
.buffer
))
6620 seq_puts(m
, "delta [absolute]\n");
6622 seq_puts(m
, "[delta] absolute\n");
6624 mutex_unlock(&trace_types_lock
);
6629 static int tracing_time_stamp_mode_open(struct inode
*inode
, struct file
*file
)
6631 struct trace_array
*tr
= inode
->i_private
;
6634 ret
= tracing_check_open_get_tr(tr
);
6638 ret
= single_open(file
, tracing_time_stamp_mode_show
, inode
->i_private
);
6640 trace_array_put(tr
);
6645 int tracing_set_time_stamp_abs(struct trace_array
*tr
, bool abs
)
6649 mutex_lock(&trace_types_lock
);
6651 if (abs
&& tr
->time_stamp_abs_ref
++)
6655 if (WARN_ON_ONCE(!tr
->time_stamp_abs_ref
)) {
6660 if (--tr
->time_stamp_abs_ref
)
6664 ring_buffer_set_time_stamp_abs(tr
->trace_buffer
.buffer
, abs
);
6666 #ifdef CONFIG_TRACER_MAX_TRACE
6667 if (tr
->max_buffer
.buffer
)
6668 ring_buffer_set_time_stamp_abs(tr
->max_buffer
.buffer
, abs
);
6671 mutex_unlock(&trace_types_lock
);
6676 struct ftrace_buffer_info
{
6677 struct trace_iterator iter
;
6679 unsigned int spare_cpu
;
6683 #ifdef CONFIG_TRACER_SNAPSHOT
6684 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
6686 struct trace_array
*tr
= inode
->i_private
;
6687 struct trace_iterator
*iter
;
6691 ret
= tracing_check_open_get_tr(tr
);
6695 if (file
->f_mode
& FMODE_READ
) {
6696 iter
= __tracing_open(inode
, file
, true);
6698 ret
= PTR_ERR(iter
);
6700 /* Writes still need the seq_file to hold the private data */
6702 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
6705 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
6713 iter
->trace_buffer
= &tr
->max_buffer
;
6714 iter
->cpu_file
= tracing_get_cpu(inode
);
6716 file
->private_data
= m
;
6720 trace_array_put(tr
);
6726 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6729 struct seq_file
*m
= filp
->private_data
;
6730 struct trace_iterator
*iter
= m
->private;
6731 struct trace_array
*tr
= iter
->tr
;
6735 ret
= tracing_update_buffers();
6739 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6743 mutex_lock(&trace_types_lock
);
6745 if (tr
->current_trace
->use_max_tr
) {
6750 arch_spin_lock(&tr
->max_lock
);
6751 if (tr
->cond_snapshot
)
6753 arch_spin_unlock(&tr
->max_lock
);
6759 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
6763 if (tr
->allocated_snapshot
)
6767 /* Only allow per-cpu swap if the ring buffer supports it */
6768 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6769 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
6774 if (tr
->allocated_snapshot
)
6775 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
6776 &tr
->trace_buffer
, iter
->cpu_file
);
6778 ret
= tracing_alloc_snapshot_instance(tr
);
6781 local_irq_disable();
6782 /* Now, we're going to swap */
6783 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
6784 update_max_tr(tr
, current
, smp_processor_id(), NULL
);
6786 update_max_tr_single(tr
, current
, iter
->cpu_file
);
6790 if (tr
->allocated_snapshot
) {
6791 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
6792 tracing_reset_online_cpus(&tr
->max_buffer
);
6794 tracing_reset_cpu(&tr
->max_buffer
, iter
->cpu_file
);
6804 mutex_unlock(&trace_types_lock
);
6808 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
6810 struct seq_file
*m
= file
->private_data
;
6813 ret
= tracing_release(inode
, file
);
6815 if (file
->f_mode
& FMODE_READ
)
6818 /* If write only, the seq_file is just a stub */
6826 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
6827 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
6828 size_t count
, loff_t
*ppos
);
6829 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
6830 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
6831 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
6833 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
6835 struct ftrace_buffer_info
*info
;
6838 /* The following checks for tracefs lockdown */
6839 ret
= tracing_buffers_open(inode
, filp
);
6843 info
= filp
->private_data
;
6845 if (info
->iter
.trace
->use_max_tr
) {
6846 tracing_buffers_release(inode
, filp
);
6850 info
->iter
.snapshot
= true;
6851 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
6856 #endif /* CONFIG_TRACER_SNAPSHOT */
6859 static const struct file_operations tracing_thresh_fops
= {
6860 .open
= tracing_open_generic
,
6861 .read
= tracing_thresh_read
,
6862 .write
= tracing_thresh_write
,
6863 .llseek
= generic_file_llseek
,
6866 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6867 static const struct file_operations tracing_max_lat_fops
= {
6868 .open
= tracing_open_generic
,
6869 .read
= tracing_max_lat_read
,
6870 .write
= tracing_max_lat_write
,
6871 .llseek
= generic_file_llseek
,
6875 static const struct file_operations set_tracer_fops
= {
6876 .open
= tracing_open_generic
,
6877 .read
= tracing_set_trace_read
,
6878 .write
= tracing_set_trace_write
,
6879 .llseek
= generic_file_llseek
,
6882 static const struct file_operations tracing_pipe_fops
= {
6883 .open
= tracing_open_pipe
,
6884 .poll
= tracing_poll_pipe
,
6885 .read
= tracing_read_pipe
,
6886 .splice_read
= tracing_splice_read_pipe
,
6887 .release
= tracing_release_pipe
,
6888 .llseek
= no_llseek
,
6891 static const struct file_operations tracing_entries_fops
= {
6892 .open
= tracing_open_generic_tr
,
6893 .read
= tracing_entries_read
,
6894 .write
= tracing_entries_write
,
6895 .llseek
= generic_file_llseek
,
6896 .release
= tracing_release_generic_tr
,
6899 static const struct file_operations tracing_total_entries_fops
= {
6900 .open
= tracing_open_generic_tr
,
6901 .read
= tracing_total_entries_read
,
6902 .llseek
= generic_file_llseek
,
6903 .release
= tracing_release_generic_tr
,
6906 static const struct file_operations tracing_free_buffer_fops
= {
6907 .open
= tracing_open_generic_tr
,
6908 .write
= tracing_free_buffer_write
,
6909 .release
= tracing_free_buffer_release
,
6912 static const struct file_operations tracing_mark_fops
= {
6913 .open
= tracing_open_generic_tr
,
6914 .write
= tracing_mark_write
,
6915 .llseek
= generic_file_llseek
,
6916 .release
= tracing_release_generic_tr
,
6919 static const struct file_operations tracing_mark_raw_fops
= {
6920 .open
= tracing_open_generic_tr
,
6921 .write
= tracing_mark_raw_write
,
6922 .llseek
= generic_file_llseek
,
6923 .release
= tracing_release_generic_tr
,
6926 static const struct file_operations trace_clock_fops
= {
6927 .open
= tracing_clock_open
,
6929 .llseek
= seq_lseek
,
6930 .release
= tracing_single_release_tr
,
6931 .write
= tracing_clock_write
,
6934 static const struct file_operations trace_time_stamp_mode_fops
= {
6935 .open
= tracing_time_stamp_mode_open
,
6937 .llseek
= seq_lseek
,
6938 .release
= tracing_single_release_tr
,
6941 #ifdef CONFIG_TRACER_SNAPSHOT
6942 static const struct file_operations snapshot_fops
= {
6943 .open
= tracing_snapshot_open
,
6945 .write
= tracing_snapshot_write
,
6946 .llseek
= tracing_lseek
,
6947 .release
= tracing_snapshot_release
,
6950 static const struct file_operations snapshot_raw_fops
= {
6951 .open
= snapshot_raw_open
,
6952 .read
= tracing_buffers_read
,
6953 .release
= tracing_buffers_release
,
6954 .splice_read
= tracing_buffers_splice_read
,
6955 .llseek
= no_llseek
,
6958 #endif /* CONFIG_TRACER_SNAPSHOT */
6960 #define TRACING_LOG_ERRS_MAX 8
6961 #define TRACING_LOG_LOC_MAX 128
6963 #define CMD_PREFIX " Command: "
6966 const char **errs
; /* ptr to loc-specific array of err strings */
6967 u8 type
; /* index into errs -> specific err string */
6968 u8 pos
; /* MAX_FILTER_STR_VAL = 256 */
6972 struct tracing_log_err
{
6973 struct list_head list
;
6974 struct err_info info
;
6975 char loc
[TRACING_LOG_LOC_MAX
]; /* err location */
6976 char cmd
[MAX_FILTER_STR_VAL
]; /* what caused err */
6979 static DEFINE_MUTEX(tracing_err_log_lock
);
6981 static struct tracing_log_err
*get_tracing_log_err(struct trace_array
*tr
)
6983 struct tracing_log_err
*err
;
6985 if (tr
->n_err_log_entries
< TRACING_LOG_ERRS_MAX
) {
6986 err
= kzalloc(sizeof(*err
), GFP_KERNEL
);
6988 err
= ERR_PTR(-ENOMEM
);
6989 tr
->n_err_log_entries
++;
6994 err
= list_first_entry(&tr
->err_log
, struct tracing_log_err
, list
);
6995 list_del(&err
->list
);
7001 * err_pos - find the position of a string within a command for error careting
7002 * @cmd: The tracing command that caused the error
7003 * @str: The string to position the caret at within @cmd
7005 * Finds the position of the first occurence of @str within @cmd. The
7006 * return value can be passed to tracing_log_err() for caret placement
7009 * Returns the index within @cmd of the first occurence of @str or 0
7010 * if @str was not found.
7012 unsigned int err_pos(char *cmd
, const char *str
)
7016 if (WARN_ON(!strlen(cmd
)))
7019 found
= strstr(cmd
, str
);
7027 * tracing_log_err - write an error to the tracing error log
7028 * @tr: The associated trace array for the error (NULL for top level array)
7029 * @loc: A string describing where the error occurred
7030 * @cmd: The tracing command that caused the error
7031 * @errs: The array of loc-specific static error strings
7032 * @type: The index into errs[], which produces the specific static err string
7033 * @pos: The position the caret should be placed in the cmd
7035 * Writes an error into tracing/error_log of the form:
7037 * <loc>: error: <text>
7041 * tracing/error_log is a small log file containing the last
7042 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7043 * unless there has been a tracing error, and the error log can be
7044 * cleared and have its memory freed by writing the empty string in
7045 * truncation mode to it i.e. echo > tracing/error_log.
7047 * NOTE: the @errs array along with the @type param are used to
7048 * produce a static error string - this string is not copied and saved
7049 * when the error is logged - only a pointer to it is saved. See
7050 * existing callers for examples of how static strings are typically
7051 * defined for use with tracing_log_err().
7053 void tracing_log_err(struct trace_array
*tr
,
7054 const char *loc
, const char *cmd
,
7055 const char **errs
, u8 type
, u8 pos
)
7057 struct tracing_log_err
*err
;
7062 mutex_lock(&tracing_err_log_lock
);
7063 err
= get_tracing_log_err(tr
);
7064 if (PTR_ERR(err
) == -ENOMEM
) {
7065 mutex_unlock(&tracing_err_log_lock
);
7069 snprintf(err
->loc
, TRACING_LOG_LOC_MAX
, "%s: error: ", loc
);
7070 snprintf(err
->cmd
, MAX_FILTER_STR_VAL
,"\n" CMD_PREFIX
"%s\n", cmd
);
7072 err
->info
.errs
= errs
;
7073 err
->info
.type
= type
;
7074 err
->info
.pos
= pos
;
7075 err
->info
.ts
= local_clock();
7077 list_add_tail(&err
->list
, &tr
->err_log
);
7078 mutex_unlock(&tracing_err_log_lock
);
7081 static void clear_tracing_err_log(struct trace_array
*tr
)
7083 struct tracing_log_err
*err
, *next
;
7085 mutex_lock(&tracing_err_log_lock
);
7086 list_for_each_entry_safe(err
, next
, &tr
->err_log
, list
) {
7087 list_del(&err
->list
);
7091 tr
->n_err_log_entries
= 0;
7092 mutex_unlock(&tracing_err_log_lock
);
7095 static void *tracing_err_log_seq_start(struct seq_file
*m
, loff_t
*pos
)
7097 struct trace_array
*tr
= m
->private;
7099 mutex_lock(&tracing_err_log_lock
);
7101 return seq_list_start(&tr
->err_log
, *pos
);
7104 static void *tracing_err_log_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
7106 struct trace_array
*tr
= m
->private;
7108 return seq_list_next(v
, &tr
->err_log
, pos
);
7111 static void tracing_err_log_seq_stop(struct seq_file
*m
, void *v
)
7113 mutex_unlock(&tracing_err_log_lock
);
7116 static void tracing_err_log_show_pos(struct seq_file
*m
, u8 pos
)
7120 for (i
= 0; i
< sizeof(CMD_PREFIX
) - 1; i
++)
7122 for (i
= 0; i
< pos
; i
++)
7127 static int tracing_err_log_seq_show(struct seq_file
*m
, void *v
)
7129 struct tracing_log_err
*err
= v
;
7132 const char *err_text
= err
->info
.errs
[err
->info
.type
];
7133 u64 sec
= err
->info
.ts
;
7136 nsec
= do_div(sec
, NSEC_PER_SEC
);
7137 seq_printf(m
, "[%5llu.%06u] %s%s", sec
, nsec
/ 1000,
7138 err
->loc
, err_text
);
7139 seq_printf(m
, "%s", err
->cmd
);
7140 tracing_err_log_show_pos(m
, err
->info
.pos
);
7146 static const struct seq_operations tracing_err_log_seq_ops
= {
7147 .start
= tracing_err_log_seq_start
,
7148 .next
= tracing_err_log_seq_next
,
7149 .stop
= tracing_err_log_seq_stop
,
7150 .show
= tracing_err_log_seq_show
7153 static int tracing_err_log_open(struct inode
*inode
, struct file
*file
)
7155 struct trace_array
*tr
= inode
->i_private
;
7158 ret
= tracing_check_open_get_tr(tr
);
7162 /* If this file was opened for write, then erase contents */
7163 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
))
7164 clear_tracing_err_log(tr
);
7166 if (file
->f_mode
& FMODE_READ
) {
7167 ret
= seq_open(file
, &tracing_err_log_seq_ops
);
7169 struct seq_file
*m
= file
->private_data
;
7172 trace_array_put(tr
);
7178 static ssize_t
tracing_err_log_write(struct file
*file
,
7179 const char __user
*buffer
,
7180 size_t count
, loff_t
*ppos
)
7185 static int tracing_err_log_release(struct inode
*inode
, struct file
*file
)
7187 struct trace_array
*tr
= inode
->i_private
;
7189 trace_array_put(tr
);
7191 if (file
->f_mode
& FMODE_READ
)
7192 seq_release(inode
, file
);
7197 static const struct file_operations tracing_err_log_fops
= {
7198 .open
= tracing_err_log_open
,
7199 .write
= tracing_err_log_write
,
7201 .llseek
= seq_lseek
,
7202 .release
= tracing_err_log_release
,
7205 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
7207 struct trace_array
*tr
= inode
->i_private
;
7208 struct ftrace_buffer_info
*info
;
7211 ret
= tracing_check_open_get_tr(tr
);
7215 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
7217 trace_array_put(tr
);
7221 mutex_lock(&trace_types_lock
);
7224 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
7225 info
->iter
.trace
= tr
->current_trace
;
7226 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
7228 /* Force reading ring buffer for first read */
7229 info
->read
= (unsigned int)-1;
7231 filp
->private_data
= info
;
7235 mutex_unlock(&trace_types_lock
);
7237 ret
= nonseekable_open(inode
, filp
);
7239 trace_array_put(tr
);
7245 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
7247 struct ftrace_buffer_info
*info
= filp
->private_data
;
7248 struct trace_iterator
*iter
= &info
->iter
;
7250 return trace_poll(iter
, filp
, poll_table
);
7254 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
7255 size_t count
, loff_t
*ppos
)
7257 struct ftrace_buffer_info
*info
= filp
->private_data
;
7258 struct trace_iterator
*iter
= &info
->iter
;
7265 #ifdef CONFIG_TRACER_MAX_TRACE
7266 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
7271 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
7273 if (IS_ERR(info
->spare
)) {
7274 ret
= PTR_ERR(info
->spare
);
7277 info
->spare_cpu
= iter
->cpu_file
;
7283 /* Do we have previous read data to read? */
7284 if (info
->read
< PAGE_SIZE
)
7288 trace_access_lock(iter
->cpu_file
);
7289 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
7293 trace_access_unlock(iter
->cpu_file
);
7296 if (trace_empty(iter
)) {
7297 if ((filp
->f_flags
& O_NONBLOCK
))
7300 ret
= wait_on_pipe(iter
, 0);
7311 size
= PAGE_SIZE
- info
->read
;
7315 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
7327 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
7329 struct ftrace_buffer_info
*info
= file
->private_data
;
7330 struct trace_iterator
*iter
= &info
->iter
;
7332 mutex_lock(&trace_types_lock
);
7334 iter
->tr
->trace_ref
--;
7336 __trace_array_put(iter
->tr
);
7339 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
,
7340 info
->spare_cpu
, info
->spare
);
7343 mutex_unlock(&trace_types_lock
);
7349 struct ring_buffer
*buffer
;
7352 refcount_t refcount
;
7355 static void buffer_ref_release(struct buffer_ref
*ref
)
7357 if (!refcount_dec_and_test(&ref
->refcount
))
7359 ring_buffer_free_read_page(ref
->buffer
, ref
->cpu
, ref
->page
);
7363 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
7364 struct pipe_buffer
*buf
)
7366 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
7368 buffer_ref_release(ref
);
7372 static bool buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
7373 struct pipe_buffer
*buf
)
7375 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
7377 if (refcount_read(&ref
->refcount
) > INT_MAX
/2)
7380 refcount_inc(&ref
->refcount
);
7384 /* Pipe buffer operations for a buffer. */
7385 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
7386 .confirm
= generic_pipe_buf_confirm
,
7387 .release
= buffer_pipe_buf_release
,
7388 .steal
= generic_pipe_buf_nosteal
,
7389 .get
= buffer_pipe_buf_get
,
7393 * Callback from splice_to_pipe(), if we need to release some pages
7394 * at the end of the spd in case we error'ed out in filling the pipe.
7396 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
7398 struct buffer_ref
*ref
=
7399 (struct buffer_ref
*)spd
->partial
[i
].private;
7401 buffer_ref_release(ref
);
7402 spd
->partial
[i
].private = 0;
7406 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
7407 struct pipe_inode_info
*pipe
, size_t len
,
7410 struct ftrace_buffer_info
*info
= file
->private_data
;
7411 struct trace_iterator
*iter
= &info
->iter
;
7412 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
7413 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
7414 struct splice_pipe_desc spd
= {
7416 .partial
= partial_def
,
7417 .nr_pages_max
= PIPE_DEF_BUFFERS
,
7418 .ops
= &buffer_pipe_buf_ops
,
7419 .spd_release
= buffer_spd_release
,
7421 struct buffer_ref
*ref
;
7425 #ifdef CONFIG_TRACER_MAX_TRACE
7426 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
7430 if (*ppos
& (PAGE_SIZE
- 1))
7433 if (len
& (PAGE_SIZE
- 1)) {
7434 if (len
< PAGE_SIZE
)
7439 if (splice_grow_spd(pipe
, &spd
))
7443 trace_access_lock(iter
->cpu_file
);
7444 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
7446 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
7450 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
7456 refcount_set(&ref
->refcount
, 1);
7457 ref
->buffer
= iter
->trace_buffer
->buffer
;
7458 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
7459 if (IS_ERR(ref
->page
)) {
7460 ret
= PTR_ERR(ref
->page
);
7465 ref
->cpu
= iter
->cpu_file
;
7467 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
7468 len
, iter
->cpu_file
, 1);
7470 ring_buffer_free_read_page(ref
->buffer
, ref
->cpu
,
7476 page
= virt_to_page(ref
->page
);
7478 spd
.pages
[i
] = page
;
7479 spd
.partial
[i
].len
= PAGE_SIZE
;
7480 spd
.partial
[i
].offset
= 0;
7481 spd
.partial
[i
].private = (unsigned long)ref
;
7485 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
7488 trace_access_unlock(iter
->cpu_file
);
7491 /* did we read anything? */
7492 if (!spd
.nr_pages
) {
7497 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
))
7500 ret
= wait_on_pipe(iter
, iter
->tr
->buffer_percent
);
7507 ret
= splice_to_pipe(pipe
, &spd
);
7509 splice_shrink_spd(&spd
);
7514 static const struct file_operations tracing_buffers_fops
= {
7515 .open
= tracing_buffers_open
,
7516 .read
= tracing_buffers_read
,
7517 .poll
= tracing_buffers_poll
,
7518 .release
= tracing_buffers_release
,
7519 .splice_read
= tracing_buffers_splice_read
,
7520 .llseek
= no_llseek
,
7524 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
7525 size_t count
, loff_t
*ppos
)
7527 struct inode
*inode
= file_inode(filp
);
7528 struct trace_array
*tr
= inode
->i_private
;
7529 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
7530 int cpu
= tracing_get_cpu(inode
);
7531 struct trace_seq
*s
;
7533 unsigned long long t
;
7534 unsigned long usec_rem
;
7536 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
7542 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
7543 trace_seq_printf(s
, "entries: %ld\n", cnt
);
7545 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
7546 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
7548 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
7549 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
7551 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
7552 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
7554 if (trace_clocks
[tr
->clock_id
].in_ns
) {
7555 /* local or global for trace_clock */
7556 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
7557 usec_rem
= do_div(t
, USEC_PER_SEC
);
7558 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
7561 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
7562 usec_rem
= do_div(t
, USEC_PER_SEC
);
7563 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
7565 /* counter or tsc mode for trace_clock */
7566 trace_seq_printf(s
, "oldest event ts: %llu\n",
7567 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
7569 trace_seq_printf(s
, "now ts: %llu\n",
7570 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
7573 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
7574 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
7576 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
7577 trace_seq_printf(s
, "read events: %ld\n", cnt
);
7579 count
= simple_read_from_buffer(ubuf
, count
, ppos
,
7580 s
->buffer
, trace_seq_used(s
));
7587 static const struct file_operations tracing_stats_fops
= {
7588 .open
= tracing_open_generic_tr
,
7589 .read
= tracing_stats_read
,
7590 .llseek
= generic_file_llseek
,
7591 .release
= tracing_release_generic_tr
,
7594 #ifdef CONFIG_DYNAMIC_FTRACE
7597 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
7598 size_t cnt
, loff_t
*ppos
)
7600 unsigned long *p
= filp
->private_data
;
7601 char buf
[64]; /* Not too big for a shallow stack */
7604 r
= scnprintf(buf
, 63, "%ld", *p
);
7607 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
7610 static const struct file_operations tracing_dyn_info_fops
= {
7611 .open
= tracing_open_generic
,
7612 .read
= tracing_read_dyn_info
,
7613 .llseek
= generic_file_llseek
,
7615 #endif /* CONFIG_DYNAMIC_FTRACE */
7617 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7619 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
,
7620 struct trace_array
*tr
, struct ftrace_probe_ops
*ops
,
7623 tracing_snapshot_instance(tr
);
7627 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
,
7628 struct trace_array
*tr
, struct ftrace_probe_ops
*ops
,
7631 struct ftrace_func_mapper
*mapper
= data
;
7635 count
= (long *)ftrace_func_mapper_find_ip(mapper
, ip
);
7645 tracing_snapshot_instance(tr
);
7649 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
7650 struct ftrace_probe_ops
*ops
, void *data
)
7652 struct ftrace_func_mapper
*mapper
= data
;
7655 seq_printf(m
, "%ps:", (void *)ip
);
7657 seq_puts(m
, "snapshot");
7660 count
= (long *)ftrace_func_mapper_find_ip(mapper
, ip
);
7663 seq_printf(m
, ":count=%ld\n", *count
);
7665 seq_puts(m
, ":unlimited\n");
7671 ftrace_snapshot_init(struct ftrace_probe_ops
*ops
, struct trace_array
*tr
,
7672 unsigned long ip
, void *init_data
, void **data
)
7674 struct ftrace_func_mapper
*mapper
= *data
;
7677 mapper
= allocate_ftrace_func_mapper();
7683 return ftrace_func_mapper_add_ip(mapper
, ip
, init_data
);
7687 ftrace_snapshot_free(struct ftrace_probe_ops
*ops
, struct trace_array
*tr
,
7688 unsigned long ip
, void *data
)
7690 struct ftrace_func_mapper
*mapper
= data
;
7695 free_ftrace_func_mapper(mapper
, NULL
);
7699 ftrace_func_mapper_remove_ip(mapper
, ip
);
7702 static struct ftrace_probe_ops snapshot_probe_ops
= {
7703 .func
= ftrace_snapshot
,
7704 .print
= ftrace_snapshot_print
,
7707 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
7708 .func
= ftrace_count_snapshot
,
7709 .print
= ftrace_snapshot_print
,
7710 .init
= ftrace_snapshot_init
,
7711 .free
= ftrace_snapshot_free
,
7715 ftrace_trace_snapshot_callback(struct trace_array
*tr
, struct ftrace_hash
*hash
,
7716 char *glob
, char *cmd
, char *param
, int enable
)
7718 struct ftrace_probe_ops
*ops
;
7719 void *count
= (void *)-1;
7726 /* hash funcs only work with set_ftrace_filter */
7730 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
7733 return unregister_ftrace_function_probe_func(glob
+1, tr
, ops
);
7738 number
= strsep(¶m
, ":");
7740 if (!strlen(number
))
7744 * We use the callback data field (which is a pointer)
7747 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
7752 ret
= tracing_alloc_snapshot_instance(tr
);
7756 ret
= register_ftrace_function_probe(glob
, tr
, ops
, count
);
7759 return ret
< 0 ? ret
: 0;
7762 static struct ftrace_func_command ftrace_snapshot_cmd
= {
7764 .func
= ftrace_trace_snapshot_callback
,
7767 static __init
int register_snapshot_cmd(void)
7769 return register_ftrace_command(&ftrace_snapshot_cmd
);
7772 static inline __init
int register_snapshot_cmd(void) { return 0; }
7773 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7775 static struct dentry
*tracing_get_dentry(struct trace_array
*tr
)
7777 if (WARN_ON(!tr
->dir
))
7778 return ERR_PTR(-ENODEV
);
7780 /* Top directory uses NULL as the parent */
7781 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
7784 /* All sub buffers have a descriptor */
7788 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
7790 struct dentry
*d_tracer
;
7793 return tr
->percpu_dir
;
7795 d_tracer
= tracing_get_dentry(tr
);
7796 if (IS_ERR(d_tracer
))
7799 tr
->percpu_dir
= tracefs_create_dir("per_cpu", d_tracer
);
7801 WARN_ONCE(!tr
->percpu_dir
,
7802 "Could not create tracefs directory 'per_cpu/%d'\n", cpu
);
7804 return tr
->percpu_dir
;
7807 static struct dentry
*
7808 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
7809 void *data
, long cpu
, const struct file_operations
*fops
)
7811 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
7813 if (ret
) /* See tracing_get_cpu() */
7814 d_inode(ret
)->i_cdev
= (void *)(cpu
+ 1);
7819 tracing_init_tracefs_percpu(struct trace_array
*tr
, long cpu
)
7821 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
7822 struct dentry
*d_cpu
;
7823 char cpu_dir
[30]; /* 30 characters should be more than enough */
7828 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
7829 d_cpu
= tracefs_create_dir(cpu_dir
, d_percpu
);
7831 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir
);
7835 /* per cpu trace_pipe */
7836 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
7837 tr
, cpu
, &tracing_pipe_fops
);
7840 trace_create_cpu_file("trace", 0644, d_cpu
,
7841 tr
, cpu
, &tracing_fops
);
7843 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
7844 tr
, cpu
, &tracing_buffers_fops
);
7846 trace_create_cpu_file("stats", 0444, d_cpu
,
7847 tr
, cpu
, &tracing_stats_fops
);
7849 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
7850 tr
, cpu
, &tracing_entries_fops
);
7852 #ifdef CONFIG_TRACER_SNAPSHOT
7853 trace_create_cpu_file("snapshot", 0644, d_cpu
,
7854 tr
, cpu
, &snapshot_fops
);
7856 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
7857 tr
, cpu
, &snapshot_raw_fops
);
7861 #ifdef CONFIG_FTRACE_SELFTEST
7862 /* Let selftest have access to static functions in this file */
7863 #include "trace_selftest.c"
7867 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
7870 struct trace_option_dentry
*topt
= filp
->private_data
;
7873 if (topt
->flags
->val
& topt
->opt
->bit
)
7878 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
7882 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
7885 struct trace_option_dentry
*topt
= filp
->private_data
;
7889 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
7893 if (val
!= 0 && val
!= 1)
7896 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
7897 mutex_lock(&trace_types_lock
);
7898 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
7900 mutex_unlock(&trace_types_lock
);
7911 static const struct file_operations trace_options_fops
= {
7912 .open
= tracing_open_generic
,
7913 .read
= trace_options_read
,
7914 .write
= trace_options_write
,
7915 .llseek
= generic_file_llseek
,
7919 * In order to pass in both the trace_array descriptor as well as the index
7920 * to the flag that the trace option file represents, the trace_array
7921 * has a character array of trace_flags_index[], which holds the index
7922 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7923 * The address of this character array is passed to the flag option file
7924 * read/write callbacks.
7926 * In order to extract both the index and the trace_array descriptor,
7927 * get_tr_index() uses the following algorithm.
7931 * As the pointer itself contains the address of the index (remember
7934 * Then to get the trace_array descriptor, by subtracting that index
7935 * from the ptr, we get to the start of the index itself.
7937 * ptr - idx == &index[0]
7939 * Then a simple container_of() from that pointer gets us to the
7940 * trace_array descriptor.
7942 static void get_tr_index(void *data
, struct trace_array
**ptr
,
7943 unsigned int *pindex
)
7945 *pindex
= *(unsigned char *)data
;
7947 *ptr
= container_of(data
- *pindex
, struct trace_array
,
7952 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
7955 void *tr_index
= filp
->private_data
;
7956 struct trace_array
*tr
;
7960 get_tr_index(tr_index
, &tr
, &index
);
7962 if (tr
->trace_flags
& (1 << index
))
7967 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
7971 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
7974 void *tr_index
= filp
->private_data
;
7975 struct trace_array
*tr
;
7980 get_tr_index(tr_index
, &tr
, &index
);
7982 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
7986 if (val
!= 0 && val
!= 1)
7989 mutex_lock(&event_mutex
);
7990 mutex_lock(&trace_types_lock
);
7991 ret
= set_tracer_flag(tr
, 1 << index
, val
);
7992 mutex_unlock(&trace_types_lock
);
7993 mutex_unlock(&event_mutex
);
8003 static const struct file_operations trace_options_core_fops
= {
8004 .open
= tracing_open_generic
,
8005 .read
= trace_options_core_read
,
8006 .write
= trace_options_core_write
,
8007 .llseek
= generic_file_llseek
,
8010 struct dentry
*trace_create_file(const char *name
,
8012 struct dentry
*parent
,
8014 const struct file_operations
*fops
)
8018 ret
= tracefs_create_file(name
, mode
, parent
, data
, fops
);
8020 pr_warn("Could not create tracefs '%s' entry\n", name
);
8026 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
8028 struct dentry
*d_tracer
;
8033 d_tracer
= tracing_get_dentry(tr
);
8034 if (IS_ERR(d_tracer
))
8037 tr
->options
= tracefs_create_dir("options", d_tracer
);
8039 pr_warn("Could not create tracefs directory 'options'\n");
8047 create_trace_option_file(struct trace_array
*tr
,
8048 struct trace_option_dentry
*topt
,
8049 struct tracer_flags
*flags
,
8050 struct tracer_opt
*opt
)
8052 struct dentry
*t_options
;
8054 t_options
= trace_options_init_dentry(tr
);
8058 topt
->flags
= flags
;
8062 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
8063 &trace_options_fops
);
8068 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
8070 struct trace_option_dentry
*topts
;
8071 struct trace_options
*tr_topts
;
8072 struct tracer_flags
*flags
;
8073 struct tracer_opt
*opts
;
8080 flags
= tracer
->flags
;
8082 if (!flags
|| !flags
->opts
)
8086 * If this is an instance, only create flags for tracers
8087 * the instance may have.
8089 if (!trace_ok_for_array(tracer
, tr
))
8092 for (i
= 0; i
< tr
->nr_topts
; i
++) {
8093 /* Make sure there's no duplicate flags. */
8094 if (WARN_ON_ONCE(tr
->topts
[i
].tracer
->flags
== tracer
->flags
))
8100 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
8103 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
8107 tr_topts
= krealloc(tr
->topts
, sizeof(*tr
->topts
) * (tr
->nr_topts
+ 1),
8114 tr
->topts
= tr_topts
;
8115 tr
->topts
[tr
->nr_topts
].tracer
= tracer
;
8116 tr
->topts
[tr
->nr_topts
].topts
= topts
;
8119 for (cnt
= 0; opts
[cnt
].name
; cnt
++) {
8120 create_trace_option_file(tr
, &topts
[cnt
], flags
,
8122 WARN_ONCE(topts
[cnt
].entry
== NULL
,
8123 "Failed to create trace option: %s",
8128 static struct dentry
*
8129 create_trace_option_core_file(struct trace_array
*tr
,
8130 const char *option
, long index
)
8132 struct dentry
*t_options
;
8134 t_options
= trace_options_init_dentry(tr
);
8138 return trace_create_file(option
, 0644, t_options
,
8139 (void *)&tr
->trace_flags_index
[index
],
8140 &trace_options_core_fops
);
8143 static void create_trace_options_dir(struct trace_array
*tr
)
8145 struct dentry
*t_options
;
8146 bool top_level
= tr
== &global_trace
;
8149 t_options
= trace_options_init_dentry(tr
);
8153 for (i
= 0; trace_options
[i
]; i
++) {
8155 !((1 << i
) & TOP_LEVEL_TRACE_FLAGS
))
8156 create_trace_option_core_file(tr
, trace_options
[i
], i
);
8161 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
8162 size_t cnt
, loff_t
*ppos
)
8164 struct trace_array
*tr
= filp
->private_data
;
8168 r
= tracer_tracing_is_on(tr
);
8169 r
= sprintf(buf
, "%d\n", r
);
8171 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
8175 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
8176 size_t cnt
, loff_t
*ppos
)
8178 struct trace_array
*tr
= filp
->private_data
;
8179 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
8183 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
8188 mutex_lock(&trace_types_lock
);
8189 if (!!val
== tracer_tracing_is_on(tr
)) {
8190 val
= 0; /* do nothing */
8192 tracer_tracing_on(tr
);
8193 if (tr
->current_trace
->start
)
8194 tr
->current_trace
->start(tr
);
8196 tracer_tracing_off(tr
);
8197 if (tr
->current_trace
->stop
)
8198 tr
->current_trace
->stop(tr
);
8200 mutex_unlock(&trace_types_lock
);
8208 static const struct file_operations rb_simple_fops
= {
8209 .open
= tracing_open_generic_tr
,
8210 .read
= rb_simple_read
,
8211 .write
= rb_simple_write
,
8212 .release
= tracing_release_generic_tr
,
8213 .llseek
= default_llseek
,
8217 buffer_percent_read(struct file
*filp
, char __user
*ubuf
,
8218 size_t cnt
, loff_t
*ppos
)
8220 struct trace_array
*tr
= filp
->private_data
;
8224 r
= tr
->buffer_percent
;
8225 r
= sprintf(buf
, "%d\n", r
);
8227 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
8231 buffer_percent_write(struct file
*filp
, const char __user
*ubuf
,
8232 size_t cnt
, loff_t
*ppos
)
8234 struct trace_array
*tr
= filp
->private_data
;
8238 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
8248 tr
->buffer_percent
= val
;
8255 static const struct file_operations buffer_percent_fops
= {
8256 .open
= tracing_open_generic_tr
,
8257 .read
= buffer_percent_read
,
8258 .write
= buffer_percent_write
,
8259 .release
= tracing_release_generic_tr
,
8260 .llseek
= default_llseek
,
8263 static struct dentry
*trace_instance_dir
;
8266 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
);
8269 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
8271 enum ring_buffer_flags rb_flags
;
8273 rb_flags
= tr
->trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
8277 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
8281 buf
->data
= alloc_percpu(struct trace_array_cpu
);
8283 ring_buffer_free(buf
->buffer
);
8288 /* Allocate the first page for all buffers */
8289 set_buffer_entries(&tr
->trace_buffer
,
8290 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
8295 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
8299 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
8303 #ifdef CONFIG_TRACER_MAX_TRACE
8304 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
8305 allocate_snapshot
? size
: 1);
8307 ring_buffer_free(tr
->trace_buffer
.buffer
);
8308 tr
->trace_buffer
.buffer
= NULL
;
8309 free_percpu(tr
->trace_buffer
.data
);
8310 tr
->trace_buffer
.data
= NULL
;
8313 tr
->allocated_snapshot
= allocate_snapshot
;
8316 * Only the top level trace array gets its snapshot allocated
8317 * from the kernel command line.
8319 allocate_snapshot
= false;
8323 * Because of some magic with the way alloc_percpu() works on
8324 * x86_64, we need to synchronize the pgd of all the tables,
8325 * otherwise the trace events that happen in x86_64 page fault
8326 * handlers can't cope with accessing the chance that a
8327 * alloc_percpu()'d memory might be touched in the page fault trace
8328 * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
8329 * calls in tracing, because something might get triggered within a
8330 * page fault trace event!
8332 vmalloc_sync_mappings();
8337 static void free_trace_buffer(struct trace_buffer
*buf
)
8340 ring_buffer_free(buf
->buffer
);
8342 free_percpu(buf
->data
);
8347 static void free_trace_buffers(struct trace_array
*tr
)
8352 free_trace_buffer(&tr
->trace_buffer
);
8354 #ifdef CONFIG_TRACER_MAX_TRACE
8355 free_trace_buffer(&tr
->max_buffer
);
8359 static void init_trace_flags_index(struct trace_array
*tr
)
8363 /* Used by the trace options files */
8364 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++)
8365 tr
->trace_flags_index
[i
] = i
;
8368 static void __update_tracer_options(struct trace_array
*tr
)
8372 for (t
= trace_types
; t
; t
= t
->next
)
8373 add_tracer_options(tr
, t
);
8376 static void update_tracer_options(struct trace_array
*tr
)
8378 mutex_lock(&trace_types_lock
);
8379 __update_tracer_options(tr
);
8380 mutex_unlock(&trace_types_lock
);
8383 struct trace_array
*trace_array_create(const char *name
)
8385 struct trace_array
*tr
;
8388 mutex_lock(&event_mutex
);
8389 mutex_lock(&trace_types_lock
);
8392 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
8393 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
8398 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
8402 tr
->name
= kstrdup(name
, GFP_KERNEL
);
8406 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
8409 tr
->trace_flags
= global_trace
.trace_flags
& ~ZEROED_TRACE_FLAGS
;
8411 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
8413 raw_spin_lock_init(&tr
->start_lock
);
8415 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
8417 tr
->current_trace
= &nop_trace
;
8419 INIT_LIST_HEAD(&tr
->systems
);
8420 INIT_LIST_HEAD(&tr
->events
);
8421 INIT_LIST_HEAD(&tr
->hist_vars
);
8422 INIT_LIST_HEAD(&tr
->err_log
);
8424 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
8427 tr
->dir
= tracefs_create_dir(name
, trace_instance_dir
);
8431 ret
= event_trace_add_tracer(tr
->dir
, tr
);
8433 tracefs_remove_recursive(tr
->dir
);
8437 ftrace_init_trace_array(tr
);
8439 init_tracer_tracefs(tr
, tr
->dir
);
8440 init_trace_flags_index(tr
);
8441 __update_tracer_options(tr
);
8443 list_add(&tr
->list
, &ftrace_trace_arrays
);
8445 mutex_unlock(&trace_types_lock
);
8446 mutex_unlock(&event_mutex
);
8451 free_trace_buffers(tr
);
8452 free_cpumask_var(tr
->tracing_cpumask
);
8457 mutex_unlock(&trace_types_lock
);
8458 mutex_unlock(&event_mutex
);
8460 return ERR_PTR(ret
);
8462 EXPORT_SYMBOL_GPL(trace_array_create
);
8464 static int instance_mkdir(const char *name
)
8466 return PTR_ERR_OR_ZERO(trace_array_create(name
));
8469 static int __remove_instance(struct trace_array
*tr
)
8473 if (tr
->ref
|| (tr
->current_trace
&& tr
->trace_ref
))
8476 list_del(&tr
->list
);
8478 /* Disable all the flags that were enabled coming in */
8479 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++) {
8480 if ((1 << i
) & ZEROED_TRACE_FLAGS
)
8481 set_tracer_flag(tr
, 1 << i
, 0);
8484 tracing_set_nop(tr
);
8485 clear_ftrace_function_probes(tr
);
8486 event_trace_del_tracer(tr
);
8487 ftrace_clear_pids(tr
);
8488 ftrace_destroy_function_files(tr
);
8489 tracefs_remove_recursive(tr
->dir
);
8490 free_trace_buffers(tr
);
8492 for (i
= 0; i
< tr
->nr_topts
; i
++) {
8493 kfree(tr
->topts
[i
].topts
);
8497 free_cpumask_var(tr
->tracing_cpumask
);
8505 int trace_array_destroy(struct trace_array
*this_tr
)
8507 struct trace_array
*tr
;
8513 mutex_lock(&event_mutex
);
8514 mutex_lock(&trace_types_lock
);
8518 /* Making sure trace array exists before destroying it. */
8519 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
8520 if (tr
== this_tr
) {
8521 ret
= __remove_instance(tr
);
8526 mutex_unlock(&trace_types_lock
);
8527 mutex_unlock(&event_mutex
);
8531 EXPORT_SYMBOL_GPL(trace_array_destroy
);
8533 static int instance_rmdir(const char *name
)
8535 struct trace_array
*tr
;
8538 mutex_lock(&event_mutex
);
8539 mutex_lock(&trace_types_lock
);
8542 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
8543 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
8544 ret
= __remove_instance(tr
);
8549 mutex_unlock(&trace_types_lock
);
8550 mutex_unlock(&event_mutex
);
8555 static __init
void create_trace_instances(struct dentry
*d_tracer
)
8557 trace_instance_dir
= tracefs_create_instance_dir("instances", d_tracer
,
8560 if (WARN_ON(!trace_instance_dir
))
8565 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
)
8567 struct trace_event_file
*file
;
8570 trace_create_file("available_tracers", 0444, d_tracer
,
8571 tr
, &show_traces_fops
);
8573 trace_create_file("current_tracer", 0644, d_tracer
,
8574 tr
, &set_tracer_fops
);
8576 trace_create_file("tracing_cpumask", 0644, d_tracer
,
8577 tr
, &tracing_cpumask_fops
);
8579 trace_create_file("trace_options", 0644, d_tracer
,
8580 tr
, &tracing_iter_fops
);
8582 trace_create_file("trace", 0644, d_tracer
,
8585 trace_create_file("trace_pipe", 0444, d_tracer
,
8586 tr
, &tracing_pipe_fops
);
8588 trace_create_file("buffer_size_kb", 0644, d_tracer
,
8589 tr
, &tracing_entries_fops
);
8591 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
8592 tr
, &tracing_total_entries_fops
);
8594 trace_create_file("free_buffer", 0200, d_tracer
,
8595 tr
, &tracing_free_buffer_fops
);
8597 trace_create_file("trace_marker", 0220, d_tracer
,
8598 tr
, &tracing_mark_fops
);
8600 file
= __find_event_file(tr
, "ftrace", "print");
8601 if (file
&& file
->dir
)
8602 trace_create_file("trigger", 0644, file
->dir
, file
,
8603 &event_trigger_fops
);
8604 tr
->trace_marker_file
= file
;
8606 trace_create_file("trace_marker_raw", 0220, d_tracer
,
8607 tr
, &tracing_mark_raw_fops
);
8609 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
8612 trace_create_file("tracing_on", 0644, d_tracer
,
8613 tr
, &rb_simple_fops
);
8615 trace_create_file("timestamp_mode", 0444, d_tracer
, tr
,
8616 &trace_time_stamp_mode_fops
);
8618 tr
->buffer_percent
= 50;
8620 trace_create_file("buffer_percent", 0444, d_tracer
,
8621 tr
, &buffer_percent_fops
);
8623 create_trace_options_dir(tr
);
8625 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
8626 trace_create_file("tracing_max_latency", 0644, d_tracer
,
8627 &tr
->max_latency
, &tracing_max_lat_fops
);
8630 if (ftrace_create_function_files(tr
, d_tracer
))
8631 WARN(1, "Could not allocate function filter files");
8633 #ifdef CONFIG_TRACER_SNAPSHOT
8634 trace_create_file("snapshot", 0644, d_tracer
,
8635 tr
, &snapshot_fops
);
8638 trace_create_file("error_log", 0644, d_tracer
,
8639 tr
, &tracing_err_log_fops
);
8641 for_each_tracing_cpu(cpu
)
8642 tracing_init_tracefs_percpu(tr
, cpu
);
8644 ftrace_init_tracefs(tr
, d_tracer
);
8647 static struct vfsmount
*trace_automount(struct dentry
*mntpt
, void *ingore
)
8649 struct vfsmount
*mnt
;
8650 struct file_system_type
*type
;
8653 * To maintain backward compatibility for tools that mount
8654 * debugfs to get to the tracing facility, tracefs is automatically
8655 * mounted to the debugfs/tracing directory.
8657 type
= get_fs_type("tracefs");
8660 mnt
= vfs_submount(mntpt
, type
, "tracefs", NULL
);
8661 put_filesystem(type
);
8670 * tracing_init_dentry - initialize top level trace array
8672 * This is called when creating files or directories in the tracing
8673 * directory. It is called via fs_initcall() by any of the boot up code
8674 * and expects to return the dentry of the top level tracing directory.
8676 struct dentry
*tracing_init_dentry(void)
8678 struct trace_array
*tr
= &global_trace
;
8680 if (security_locked_down(LOCKDOWN_TRACEFS
)) {
8681 pr_warning("Tracing disabled due to lockdown\n");
8682 return ERR_PTR(-EPERM
);
8685 /* The top level trace array uses NULL as parent */
8689 if (WARN_ON(!tracefs_initialized()) ||
8690 (IS_ENABLED(CONFIG_DEBUG_FS
) &&
8691 WARN_ON(!debugfs_initialized())))
8692 return ERR_PTR(-ENODEV
);
8695 * As there may still be users that expect the tracing
8696 * files to exist in debugfs/tracing, we must automount
8697 * the tracefs file system there, so older tools still
8698 * work with the newer kerenl.
8700 tr
->dir
= debugfs_create_automount("tracing", NULL
,
8701 trace_automount
, NULL
);
8706 extern struct trace_eval_map
*__start_ftrace_eval_maps
[];
8707 extern struct trace_eval_map
*__stop_ftrace_eval_maps
[];
8709 static void __init
trace_eval_init(void)
8713 len
= __stop_ftrace_eval_maps
- __start_ftrace_eval_maps
;
8714 trace_insert_eval_map(NULL
, __start_ftrace_eval_maps
, len
);
8717 #ifdef CONFIG_MODULES
8718 static void trace_module_add_evals(struct module
*mod
)
8720 if (!mod
->num_trace_evals
)
8724 * Modules with bad taint do not have events created, do
8725 * not bother with enums either.
8727 if (trace_module_has_bad_taint(mod
))
8730 trace_insert_eval_map(mod
, mod
->trace_evals
, mod
->num_trace_evals
);
8733 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8734 static void trace_module_remove_evals(struct module
*mod
)
8736 union trace_eval_map_item
*map
;
8737 union trace_eval_map_item
**last
= &trace_eval_maps
;
8739 if (!mod
->num_trace_evals
)
8742 mutex_lock(&trace_eval_mutex
);
8744 map
= trace_eval_maps
;
8747 if (map
->head
.mod
== mod
)
8749 map
= trace_eval_jmp_to_tail(map
);
8750 last
= &map
->tail
.next
;
8751 map
= map
->tail
.next
;
8756 *last
= trace_eval_jmp_to_tail(map
)->tail
.next
;
8759 mutex_unlock(&trace_eval_mutex
);
8762 static inline void trace_module_remove_evals(struct module
*mod
) { }
8763 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8765 static int trace_module_notify(struct notifier_block
*self
,
8766 unsigned long val
, void *data
)
8768 struct module
*mod
= data
;
8771 case MODULE_STATE_COMING
:
8772 trace_module_add_evals(mod
);
8774 case MODULE_STATE_GOING
:
8775 trace_module_remove_evals(mod
);
8782 static struct notifier_block trace_module_nb
= {
8783 .notifier_call
= trace_module_notify
,
8786 #endif /* CONFIG_MODULES */
8788 static __init
int tracer_init_tracefs(void)
8790 struct dentry
*d_tracer
;
8792 trace_access_lock_init();
8794 d_tracer
= tracing_init_dentry();
8795 if (IS_ERR(d_tracer
))
8800 init_tracer_tracefs(&global_trace
, d_tracer
);
8801 ftrace_init_tracefs_toplevel(&global_trace
, d_tracer
);
8803 trace_create_file("tracing_thresh", 0644, d_tracer
,
8804 &global_trace
, &tracing_thresh_fops
);
8806 trace_create_file("README", 0444, d_tracer
,
8807 NULL
, &tracing_readme_fops
);
8809 trace_create_file("saved_cmdlines", 0444, d_tracer
,
8810 NULL
, &tracing_saved_cmdlines_fops
);
8812 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
8813 NULL
, &tracing_saved_cmdlines_size_fops
);
8815 trace_create_file("saved_tgids", 0444, d_tracer
,
8816 NULL
, &tracing_saved_tgids_fops
);
8820 trace_create_eval_file(d_tracer
);
8822 #ifdef CONFIG_MODULES
8823 register_module_notifier(&trace_module_nb
);
8826 #ifdef CONFIG_DYNAMIC_FTRACE
8827 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
8828 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
8831 create_trace_instances(d_tracer
);
8833 update_tracer_options(&global_trace
);
8838 static int trace_panic_handler(struct notifier_block
*this,
8839 unsigned long event
, void *unused
)
8841 if (ftrace_dump_on_oops
)
8842 ftrace_dump(ftrace_dump_on_oops
);
8846 static struct notifier_block trace_panic_notifier
= {
8847 .notifier_call
= trace_panic_handler
,
8849 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
8852 static int trace_die_handler(struct notifier_block
*self
,
8858 if (ftrace_dump_on_oops
)
8859 ftrace_dump(ftrace_dump_on_oops
);
8867 static struct notifier_block trace_die_notifier
= {
8868 .notifier_call
= trace_die_handler
,
8873 * printk is set to max of 1024, we really don't need it that big.
8874 * Nothing should be printing 1000 characters anyway.
8876 #define TRACE_MAX_PRINT 1000
8879 * Define here KERN_TRACE so that we have one place to modify
8880 * it if we decide to change what log level the ftrace dump
8883 #define KERN_TRACE KERN_EMERG
8886 trace_printk_seq(struct trace_seq
*s
)
8888 /* Probably should print a warning here. */
8889 if (s
->seq
.len
>= TRACE_MAX_PRINT
)
8890 s
->seq
.len
= TRACE_MAX_PRINT
;
8893 * More paranoid code. Although the buffer size is set to
8894 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8895 * an extra layer of protection.
8897 if (WARN_ON_ONCE(s
->seq
.len
>= s
->seq
.size
))
8898 s
->seq
.len
= s
->seq
.size
- 1;
8900 /* should be zero ended, but we are paranoid. */
8901 s
->buffer
[s
->seq
.len
] = 0;
8903 printk(KERN_TRACE
"%s", s
->buffer
);
8908 void trace_init_global_iter(struct trace_iterator
*iter
)
8910 iter
->tr
= &global_trace
;
8911 iter
->trace
= iter
->tr
->current_trace
;
8912 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
8913 iter
->trace_buffer
= &global_trace
.trace_buffer
;
8915 if (iter
->trace
&& iter
->trace
->open
)
8916 iter
->trace
->open(iter
);
8918 /* Annotate start of buffers if we had overruns */
8919 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
8920 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
8922 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8923 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
8924 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
8927 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
8929 /* use static because iter can be a bit big for the stack */
8930 static struct trace_iterator iter
;
8931 static atomic_t dump_running
;
8932 struct trace_array
*tr
= &global_trace
;
8933 unsigned int old_userobj
;
8934 unsigned long flags
;
8937 /* Only allow one dump user at a time. */
8938 if (atomic_inc_return(&dump_running
) != 1) {
8939 atomic_dec(&dump_running
);
8944 * Always turn off tracing when we dump.
8945 * We don't need to show trace output of what happens
8946 * between multiple crashes.
8948 * If the user does a sysrq-z, then they can re-enable
8949 * tracing with echo 1 > tracing_on.
8953 local_irq_save(flags
);
8954 printk_nmi_direct_enter();
8956 /* Simulate the iterator */
8957 trace_init_global_iter(&iter
);
8959 for_each_tracing_cpu(cpu
) {
8960 atomic_inc(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
8963 old_userobj
= tr
->trace_flags
& TRACE_ITER_SYM_USEROBJ
;
8965 /* don't look at user memory in panic mode */
8966 tr
->trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
8968 switch (oops_dump_mode
) {
8970 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
8973 iter
.cpu_file
= raw_smp_processor_id();
8978 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
8979 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
8982 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
8984 /* Did function tracer already get disabled? */
8985 if (ftrace_is_dead()) {
8986 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8987 printk("# MAY BE MISSING FUNCTION EVENTS\n");
8991 * We need to stop all tracing on all CPUS to read the
8992 * the next buffer. This is a bit expensive, but is
8993 * not done often. We fill all what we can read,
8994 * and then release the locks again.
8997 while (!trace_empty(&iter
)) {
9000 printk(KERN_TRACE
"---------------------------------\n");
9004 trace_iterator_reset(&iter
);
9005 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
9007 if (trace_find_next_entry_inc(&iter
) != NULL
) {
9010 ret
= print_trace_line(&iter
);
9011 if (ret
!= TRACE_TYPE_NO_CONSUME
)
9012 trace_consume(&iter
);
9014 touch_nmi_watchdog();
9016 trace_printk_seq(&iter
.seq
);
9020 printk(KERN_TRACE
" (ftrace buffer empty)\n");
9022 printk(KERN_TRACE
"---------------------------------\n");
9025 tr
->trace_flags
|= old_userobj
;
9027 for_each_tracing_cpu(cpu
) {
9028 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
9030 atomic_dec(&dump_running
);
9031 printk_nmi_direct_exit();
9032 local_irq_restore(flags
);
9034 EXPORT_SYMBOL_GPL(ftrace_dump
);
9036 int trace_run_command(const char *buf
, int (*createfn
)(int, char **))
9043 argv
= argv_split(GFP_KERNEL
, buf
, &argc
);
9048 ret
= createfn(argc
, argv
);
9055 #define WRITE_BUFSIZE 4096
9057 ssize_t
trace_parse_run_command(struct file
*file
, const char __user
*buffer
,
9058 size_t count
, loff_t
*ppos
,
9059 int (*createfn
)(int, char **))
9061 char *kbuf
, *buf
, *tmp
;
9066 kbuf
= kmalloc(WRITE_BUFSIZE
, GFP_KERNEL
);
9070 while (done
< count
) {
9071 size
= count
- done
;
9073 if (size
>= WRITE_BUFSIZE
)
9074 size
= WRITE_BUFSIZE
- 1;
9076 if (copy_from_user(kbuf
, buffer
+ done
, size
)) {
9083 tmp
= strchr(buf
, '\n');
9086 size
= tmp
- buf
+ 1;
9089 if (done
+ size
< count
) {
9092 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
9093 pr_warn("Line length is too long: Should be less than %d\n",
9101 /* Remove comments */
9102 tmp
= strchr(buf
, '#');
9107 ret
= trace_run_command(buf
, createfn
);
9112 } while (done
< count
);
9122 __init
static int tracer_alloc_buffers(void)
9128 if (security_locked_down(LOCKDOWN_TRACEFS
)) {
9129 pr_warning("Tracing disabled due to lockdown\n");
9134 * Make sure we don't accidently add more trace options
9135 * than we have bits for.
9137 BUILD_BUG_ON(TRACE_ITER_LAST_BIT
> TRACE_FLAGS_MAX_SIZE
);
9139 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
9142 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
9143 goto out_free_buffer_mask
;
9145 /* Only allocate trace_printk buffers if a trace_printk exists */
9146 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
9147 /* Must be called before global_trace.buffer is allocated */
9148 trace_printk_init_buffers();
9150 /* To save memory, keep the ring buffer size to its minimum */
9151 if (ring_buffer_expanded
)
9152 ring_buf_size
= trace_buf_size
;
9156 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
9157 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
9159 raw_spin_lock_init(&global_trace
.start_lock
);
9162 * The prepare callbacks allocates some memory for the ring buffer. We
9163 * don't free the buffer if the if the CPU goes down. If we were to free
9164 * the buffer, then the user would lose any trace that was in the
9165 * buffer. The memory will be removed once the "instance" is removed.
9167 ret
= cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE
,
9168 "trace/RB:preapre", trace_rb_cpu_prepare
,
9171 goto out_free_cpumask
;
9172 /* Used for event triggers */
9174 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
9176 goto out_rm_hp_state
;
9178 if (trace_create_savedcmd() < 0)
9179 goto out_free_temp_buffer
;
9181 /* TODO: make the number of buffers hot pluggable with CPUS */
9182 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
9183 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
9185 goto out_free_savedcmd
;
9188 if (global_trace
.buffer_disabled
)
9191 if (trace_boot_clock
) {
9192 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
9194 pr_warn("Trace clock %s not defined, going back to default\n",
9199 * register_tracer() might reference current_trace, so it
9200 * needs to be set before we register anything. This is
9201 * just a bootstrap of current_trace anyway.
9203 global_trace
.current_trace
= &nop_trace
;
9205 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
9207 ftrace_init_global_array_ops(&global_trace
);
9209 init_trace_flags_index(&global_trace
);
9211 register_tracer(&nop_trace
);
9213 /* Function tracing may start here (via kernel command line) */
9214 init_function_trace();
9216 /* All seems OK, enable tracing */
9217 tracing_disabled
= 0;
9219 atomic_notifier_chain_register(&panic_notifier_list
,
9220 &trace_panic_notifier
);
9222 register_die_notifier(&trace_die_notifier
);
9224 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
9226 INIT_LIST_HEAD(&global_trace
.systems
);
9227 INIT_LIST_HEAD(&global_trace
.events
);
9228 INIT_LIST_HEAD(&global_trace
.hist_vars
);
9229 INIT_LIST_HEAD(&global_trace
.err_log
);
9230 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
9232 apply_trace_boot_options();
9234 register_snapshot_cmd();
9239 free_saved_cmdlines_buffer(savedcmd
);
9240 out_free_temp_buffer
:
9241 ring_buffer_free(temp_buffer
);
9243 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE
);
9245 free_cpumask_var(global_trace
.tracing_cpumask
);
9246 out_free_buffer_mask
:
9247 free_cpumask_var(tracing_buffer_mask
);
9252 void __init
early_trace_init(void)
9254 if (tracepoint_printk
) {
9255 tracepoint_print_iter
=
9256 kmalloc(sizeof(*tracepoint_print_iter
), GFP_KERNEL
);
9257 if (WARN_ON(!tracepoint_print_iter
))
9258 tracepoint_printk
= 0;
9260 static_key_enable(&tracepoint_printk_key
.key
);
9262 tracer_alloc_buffers();
9265 void __init
trace_init(void)
9270 __init
static int clear_boot_tracer(void)
9273 * The default tracer at boot buffer is an init section.
9274 * This function is called in lateinit. If we did not
9275 * find the boot tracer, then clear it out, to prevent
9276 * later registration from accessing the buffer that is
9277 * about to be freed.
9279 if (!default_bootup_tracer
)
9282 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
9283 default_bootup_tracer
);
9284 default_bootup_tracer
= NULL
;
9289 fs_initcall(tracer_init_tracefs
);
9290 late_initcall_sync(clear_boot_tracer
);
9292 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
9293 __init
static int tracing_set_default_clock(void)
9295 /* sched_clock_stable() is determined in late_initcall */
9296 if (!trace_boot_clock
&& !sched_clock_stable()) {
9297 if (security_locked_down(LOCKDOWN_TRACEFS
)) {
9298 pr_warn("Can not set tracing clock due to lockdown\n");
9303 "Unstable clock detected, switching default tracing clock to \"global\"\n"
9304 "If you want to keep using the local clock, then add:\n"
9305 " \"trace_clock=local\"\n"
9306 "on the kernel command line\n");
9307 tracing_set_clock(&global_trace
, "global");
9312 late_initcall_sync(tracing_set_default_clock
);