2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/trace.h>
44 #include <linux/sched/rt.h>
47 #include "trace_output.h"
50 * On boot up, the ring buffer is set to the minimum size, so that
51 * we do not waste memory on systems that are not using tracing.
53 bool ring_buffer_expanded
;
56 * We need to change this state when a selftest is running.
57 * A selftest will lurk into the ring-buffer to count the
58 * entries inserted during the selftest although some concurrent
59 * insertions into the ring-buffer such as trace_printk could occurred
60 * at the same time, giving false positive or negative results.
62 static bool __read_mostly tracing_selftest_running
;
65 * If a tracer is running, we do not want to run SELFTEST.
67 bool __read_mostly tracing_selftest_disabled
;
69 /* Pipe tracepoints to printk */
70 struct trace_iterator
*tracepoint_print_iter
;
71 int tracepoint_printk
;
72 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key
);
74 /* For tracers that don't implement custom flags */
75 static struct tracer_opt dummy_tracer_opt
[] = {
80 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
86 * To prevent the comm cache from being overwritten when no
87 * tracing is active, only save the comm when a trace event
90 static DEFINE_PER_CPU(bool, trace_cmdline_save
);
93 * Kill all tracing for good (never come back).
94 * It is initialized to 1 but will turn to zero if the initialization
95 * of the tracer is successful. But that is the only place that sets
98 static int tracing_disabled
= 1;
100 cpumask_var_t __read_mostly tracing_buffer_mask
;
103 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
106 * is set, then ftrace_dump is called. This will output the contents
107 * of the ftrace buffers to the console. This is very useful for
108 * capturing traces that lead to crashes and outputing it to a
111 * It is default off, but you can enable it with either specifying
112 * "ftrace_dump_on_oops" in the kernel command line, or setting
113 * /proc/sys/kernel/ftrace_dump_on_oops
114 * Set 1 if you want to dump buffers of all CPUs
115 * Set 2 if you want to dump the buffer of the CPU that triggered oops
118 enum ftrace_dump_mode ftrace_dump_on_oops
;
120 /* When set, tracing will stop when a WARN*() is hit */
121 int __disable_trace_on_warning
;
123 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
124 /* Map of enums to their values, for "enum_map" file */
125 struct trace_enum_map_head
{
127 unsigned long length
;
130 union trace_enum_map_item
;
132 struct trace_enum_map_tail
{
134 * "end" is first and points to NULL as it must be different
135 * than "mod" or "enum_string"
137 union trace_enum_map_item
*next
;
138 const char *end
; /* points to NULL */
141 static DEFINE_MUTEX(trace_enum_mutex
);
144 * The trace_enum_maps are saved in an array with two extra elements,
145 * one at the beginning, and one at the end. The beginning item contains
146 * the count of the saved maps (head.length), and the module they
147 * belong to if not built in (head.mod). The ending item contains a
148 * pointer to the next array of saved enum_map items.
150 union trace_enum_map_item
{
151 struct trace_enum_map map
;
152 struct trace_enum_map_head head
;
153 struct trace_enum_map_tail tail
;
156 static union trace_enum_map_item
*trace_enum_maps
;
157 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
159 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
161 #define MAX_TRACER_SIZE 100
162 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
163 static char *default_bootup_tracer
;
165 static bool allocate_snapshot
;
167 static int __init
set_cmdline_ftrace(char *str
)
169 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
170 default_bootup_tracer
= bootup_tracer_buf
;
171 /* We are using ftrace early, expand it */
172 ring_buffer_expanded
= true;
175 __setup("ftrace=", set_cmdline_ftrace
);
177 static int __init
set_ftrace_dump_on_oops(char *str
)
179 if (*str
++ != '=' || !*str
) {
180 ftrace_dump_on_oops
= DUMP_ALL
;
184 if (!strcmp("orig_cpu", str
)) {
185 ftrace_dump_on_oops
= DUMP_ORIG
;
191 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
193 static int __init
stop_trace_on_warning(char *str
)
195 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
196 __disable_trace_on_warning
= 1;
199 __setup("traceoff_on_warning", stop_trace_on_warning
);
201 static int __init
boot_alloc_snapshot(char *str
)
203 allocate_snapshot
= true;
204 /* We also need the main ring buffer expanded */
205 ring_buffer_expanded
= true;
208 __setup("alloc_snapshot", boot_alloc_snapshot
);
211 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
213 static int __init
set_trace_boot_options(char *str
)
215 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
218 __setup("trace_options=", set_trace_boot_options
);
220 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
221 static char *trace_boot_clock __initdata
;
223 static int __init
set_trace_boot_clock(char *str
)
225 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
226 trace_boot_clock
= trace_boot_clock_buf
;
229 __setup("trace_clock=", set_trace_boot_clock
);
231 static int __init
set_tracepoint_printk(char *str
)
233 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
234 tracepoint_printk
= 1;
237 __setup("tp_printk", set_tracepoint_printk
);
239 unsigned long long ns2usecs(cycle_t nsec
)
246 /* trace_flags holds trace_options default values */
247 #define TRACE_DEFAULT_FLAGS \
248 (FUNCTION_DEFAULT_FLAGS | \
249 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
250 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
251 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
252 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
254 /* trace_options that are only supported by global_trace */
255 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
256 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
258 /* trace_flags that are default zero for instances */
259 #define ZEROED_TRACE_FLAGS \
260 TRACE_ITER_EVENT_FORK
263 * The global_trace is the descriptor that holds the tracing
264 * buffers for the live tracing. For each CPU, it contains
265 * a link list of pages that will store trace entries. The
266 * page descriptor of the pages in the memory is used to hold
267 * the link list by linking the lru item in the page descriptor
268 * to each of the pages in the buffer per CPU.
270 * For each active CPU there is a data field that holds the
271 * pages for the buffer for that CPU. Each CPU has the same number
272 * of pages allocated for its buffer.
274 static struct trace_array global_trace
= {
275 .trace_flags
= TRACE_DEFAULT_FLAGS
,
278 LIST_HEAD(ftrace_trace_arrays
);
280 int trace_array_get(struct trace_array
*this_tr
)
282 struct trace_array
*tr
;
285 mutex_lock(&trace_types_lock
);
286 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
293 mutex_unlock(&trace_types_lock
);
298 static void __trace_array_put(struct trace_array
*this_tr
)
300 WARN_ON(!this_tr
->ref
);
304 void trace_array_put(struct trace_array
*this_tr
)
306 mutex_lock(&trace_types_lock
);
307 __trace_array_put(this_tr
);
308 mutex_unlock(&trace_types_lock
);
311 int call_filter_check_discard(struct trace_event_call
*call
, void *rec
,
312 struct ring_buffer
*buffer
,
313 struct ring_buffer_event
*event
)
315 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
316 !filter_match_preds(call
->filter
, rec
)) {
317 __trace_event_discard_commit(buffer
, event
);
324 void trace_free_pid_list(struct trace_pid_list
*pid_list
)
326 vfree(pid_list
->pids
);
331 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
332 * @filtered_pids: The list of pids to check
333 * @search_pid: The PID to find in @filtered_pids
335 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
338 trace_find_filtered_pid(struct trace_pid_list
*filtered_pids
, pid_t search_pid
)
341 * If pid_max changed after filtered_pids was created, we
342 * by default ignore all pids greater than the previous pid_max.
344 if (search_pid
>= filtered_pids
->pid_max
)
347 return test_bit(search_pid
, filtered_pids
->pids
);
351 * trace_ignore_this_task - should a task be ignored for tracing
352 * @filtered_pids: The list of pids to check
353 * @task: The task that should be ignored if not filtered
355 * Checks if @task should be traced or not from @filtered_pids.
356 * Returns true if @task should *NOT* be traced.
357 * Returns false if @task should be traced.
360 trace_ignore_this_task(struct trace_pid_list
*filtered_pids
, struct task_struct
*task
)
363 * Return false, because if filtered_pids does not exist,
364 * all pids are good to trace.
369 return !trace_find_filtered_pid(filtered_pids
, task
->pid
);
373 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
374 * @pid_list: The list to modify
375 * @self: The current task for fork or NULL for exit
376 * @task: The task to add or remove
378 * If adding a task, if @self is defined, the task is only added if @self
379 * is also included in @pid_list. This happens on fork and tasks should
380 * only be added when the parent is listed. If @self is NULL, then the
381 * @task pid will be removed from the list, which would happen on exit
384 void trace_filter_add_remove_task(struct trace_pid_list
*pid_list
,
385 struct task_struct
*self
,
386 struct task_struct
*task
)
391 /* For forks, we only add if the forking task is listed */
393 if (!trace_find_filtered_pid(pid_list
, self
->pid
))
397 /* Sorry, but we don't support pid_max changing after setting */
398 if (task
->pid
>= pid_list
->pid_max
)
401 /* "self" is set for forks, and NULL for exits */
403 set_bit(task
->pid
, pid_list
->pids
);
405 clear_bit(task
->pid
, pid_list
->pids
);
409 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
410 * @pid_list: The pid list to show
411 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
412 * @pos: The position of the file
414 * This is used by the seq_file "next" operation to iterate the pids
415 * listed in a trace_pid_list structure.
417 * Returns the pid+1 as we want to display pid of zero, but NULL would
418 * stop the iteration.
420 void *trace_pid_next(struct trace_pid_list
*pid_list
, void *v
, loff_t
*pos
)
422 unsigned long pid
= (unsigned long)v
;
426 /* pid already is +1 of the actual prevous bit */
427 pid
= find_next_bit(pid_list
->pids
, pid_list
->pid_max
, pid
);
429 /* Return pid + 1 to allow zero to be represented */
430 if (pid
< pid_list
->pid_max
)
431 return (void *)(pid
+ 1);
437 * trace_pid_start - Used for seq_file to start reading pid lists
438 * @pid_list: The pid list to show
439 * @pos: The position of the file
441 * This is used by seq_file "start" operation to start the iteration
444 * Returns the pid+1 as we want to display pid of zero, but NULL would
445 * stop the iteration.
447 void *trace_pid_start(struct trace_pid_list
*pid_list
, loff_t
*pos
)
452 pid
= find_first_bit(pid_list
->pids
, pid_list
->pid_max
);
453 if (pid
>= pid_list
->pid_max
)
456 /* Return pid + 1 so that zero can be the exit value */
457 for (pid
++; pid
&& l
< *pos
;
458 pid
= (unsigned long)trace_pid_next(pid_list
, (void *)pid
, &l
))
464 * trace_pid_show - show the current pid in seq_file processing
465 * @m: The seq_file structure to write into
466 * @v: A void pointer of the pid (+1) value to display
468 * Can be directly used by seq_file operations to display the current
471 int trace_pid_show(struct seq_file
*m
, void *v
)
473 unsigned long pid
= (unsigned long)v
- 1;
475 seq_printf(m
, "%lu\n", pid
);
479 /* 128 should be much more than enough */
480 #define PID_BUF_SIZE 127
482 int trace_pid_write(struct trace_pid_list
*filtered_pids
,
483 struct trace_pid_list
**new_pid_list
,
484 const char __user
*ubuf
, size_t cnt
)
486 struct trace_pid_list
*pid_list
;
487 struct trace_parser parser
;
495 if (trace_parser_get_init(&parser
, PID_BUF_SIZE
+ 1))
499 * Always recreate a new array. The write is an all or nothing
500 * operation. Always create a new array when adding new pids by
501 * the user. If the operation fails, then the current list is
504 pid_list
= kmalloc(sizeof(*pid_list
), GFP_KERNEL
);
508 pid_list
->pid_max
= READ_ONCE(pid_max
);
510 /* Only truncating will shrink pid_max */
511 if (filtered_pids
&& filtered_pids
->pid_max
> pid_list
->pid_max
)
512 pid_list
->pid_max
= filtered_pids
->pid_max
;
514 pid_list
->pids
= vzalloc((pid_list
->pid_max
+ 7) >> 3);
515 if (!pid_list
->pids
) {
521 /* copy the current bits to the new max */
522 for_each_set_bit(pid
, filtered_pids
->pids
,
523 filtered_pids
->pid_max
) {
524 set_bit(pid
, pid_list
->pids
);
533 ret
= trace_get_user(&parser
, ubuf
, cnt
, &pos
);
534 if (ret
< 0 || !trace_parser_loaded(&parser
))
541 parser
.buffer
[parser
.idx
] = 0;
544 if (kstrtoul(parser
.buffer
, 0, &val
))
546 if (val
>= pid_list
->pid_max
)
551 set_bit(pid
, pid_list
->pids
);
554 trace_parser_clear(&parser
);
557 trace_parser_put(&parser
);
560 trace_free_pid_list(pid_list
);
565 /* Cleared the list of pids */
566 trace_free_pid_list(pid_list
);
571 *new_pid_list
= pid_list
;
576 static cycle_t
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
580 /* Early boot up does not have a buffer yet */
582 return trace_clock_local();
584 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
585 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
590 cycle_t
ftrace_now(int cpu
)
592 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
596 * tracing_is_enabled - Show if global_trace has been disabled
598 * Shows if the global trace has been enabled or not. It uses the
599 * mirror flag "buffer_disabled" to be used in fast paths such as for
600 * the irqsoff tracer. But it may be inaccurate due to races. If you
601 * need to know the accurate state, use tracing_is_on() which is a little
602 * slower, but accurate.
604 int tracing_is_enabled(void)
607 * For quick access (irqsoff uses this in fast path), just
608 * return the mirror variable of the state of the ring buffer.
609 * It's a little racy, but we don't really care.
612 return !global_trace
.buffer_disabled
;
616 * trace_buf_size is the size in bytes that is allocated
617 * for a buffer. Note, the number of bytes is always rounded
620 * This number is purposely set to a low number of 16384.
621 * If the dump on oops happens, it will be much appreciated
622 * to not have to wait for all that output. Anyway this can be
623 * boot time and run time configurable.
625 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
627 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
629 /* trace_types holds a link list of available tracers. */
630 static struct tracer
*trace_types __read_mostly
;
633 * trace_types_lock is used to protect the trace_types list.
635 DEFINE_MUTEX(trace_types_lock
);
638 * serialize the access of the ring buffer
640 * ring buffer serializes readers, but it is low level protection.
641 * The validity of the events (which returns by ring_buffer_peek() ..etc)
642 * are not protected by ring buffer.
644 * The content of events may become garbage if we allow other process consumes
645 * these events concurrently:
646 * A) the page of the consumed events may become a normal page
647 * (not reader page) in ring buffer, and this page will be rewrited
648 * by events producer.
649 * B) The page of the consumed events may become a page for splice_read,
650 * and this page will be returned to system.
652 * These primitives allow multi process access to different cpu ring buffer
655 * These primitives don't distinguish read-only and read-consume access.
656 * Multi read-only access are also serialized.
660 static DECLARE_RWSEM(all_cpu_access_lock
);
661 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
663 static inline void trace_access_lock(int cpu
)
665 if (cpu
== RING_BUFFER_ALL_CPUS
) {
666 /* gain it for accessing the whole ring buffer. */
667 down_write(&all_cpu_access_lock
);
669 /* gain it for accessing a cpu ring buffer. */
671 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
672 down_read(&all_cpu_access_lock
);
674 /* Secondly block other access to this @cpu ring buffer. */
675 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
679 static inline void trace_access_unlock(int cpu
)
681 if (cpu
== RING_BUFFER_ALL_CPUS
) {
682 up_write(&all_cpu_access_lock
);
684 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
685 up_read(&all_cpu_access_lock
);
689 static inline void trace_access_lock_init(void)
693 for_each_possible_cpu(cpu
)
694 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
699 static DEFINE_MUTEX(access_lock
);
701 static inline void trace_access_lock(int cpu
)
704 mutex_lock(&access_lock
);
707 static inline void trace_access_unlock(int cpu
)
710 mutex_unlock(&access_lock
);
713 static inline void trace_access_lock_init(void)
719 #ifdef CONFIG_STACKTRACE
720 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
722 int skip
, int pc
, struct pt_regs
*regs
);
723 static inline void ftrace_trace_stack(struct trace_array
*tr
,
724 struct ring_buffer
*buffer
,
726 int skip
, int pc
, struct pt_regs
*regs
);
729 static inline void __ftrace_trace_stack(struct ring_buffer
*buffer
,
731 int skip
, int pc
, struct pt_regs
*regs
)
734 static inline void ftrace_trace_stack(struct trace_array
*tr
,
735 struct ring_buffer
*buffer
,
737 int skip
, int pc
, struct pt_regs
*regs
)
743 static __always_inline
void
744 trace_event_setup(struct ring_buffer_event
*event
,
745 int type
, unsigned long flags
, int pc
)
747 struct trace_entry
*ent
= ring_buffer_event_data(event
);
749 tracing_generic_entry_update(ent
, flags
, pc
);
753 static __always_inline
struct ring_buffer_event
*
754 __trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
757 unsigned long flags
, int pc
)
759 struct ring_buffer_event
*event
;
761 event
= ring_buffer_lock_reserve(buffer
, len
);
763 trace_event_setup(event
, type
, flags
, pc
);
768 static void tracer_tracing_on(struct trace_array
*tr
)
770 if (tr
->trace_buffer
.buffer
)
771 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
773 * This flag is looked at when buffers haven't been allocated
774 * yet, or by some tracers (like irqsoff), that just want to
775 * know if the ring buffer has been disabled, but it can handle
776 * races of where it gets disabled but we still do a record.
777 * As the check is in the fast path of the tracers, it is more
778 * important to be fast than accurate.
780 tr
->buffer_disabled
= 0;
781 /* Make the flag seen by readers */
786 * tracing_on - enable tracing buffers
788 * This function enables tracing buffers that may have been
789 * disabled with tracing_off.
791 void tracing_on(void)
793 tracer_tracing_on(&global_trace
);
795 EXPORT_SYMBOL_GPL(tracing_on
);
798 static __always_inline
void
799 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
801 __this_cpu_write(trace_cmdline_save
, true);
803 /* If this is the temp buffer, we need to commit fully */
804 if (this_cpu_read(trace_buffered_event
) == event
) {
805 /* Length is in event->array[0] */
806 ring_buffer_write(buffer
, event
->array
[0], &event
->array
[1]);
807 /* Release the temp buffer */
808 this_cpu_dec(trace_buffered_event_cnt
);
810 ring_buffer_unlock_commit(buffer
, event
);
814 * __trace_puts - write a constant string into the trace buffer.
815 * @ip: The address of the caller
816 * @str: The constant string to write
817 * @size: The size of the string.
819 int __trace_puts(unsigned long ip
, const char *str
, int size
)
821 struct ring_buffer_event
*event
;
822 struct ring_buffer
*buffer
;
823 struct print_entry
*entry
;
824 unsigned long irq_flags
;
828 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
831 pc
= preempt_count();
833 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
836 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
838 local_save_flags(irq_flags
);
839 buffer
= global_trace
.trace_buffer
.buffer
;
840 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
845 entry
= ring_buffer_event_data(event
);
848 memcpy(&entry
->buf
, str
, size
);
850 /* Add a newline if necessary */
851 if (entry
->buf
[size
- 1] != '\n') {
852 entry
->buf
[size
] = '\n';
853 entry
->buf
[size
+ 1] = '\0';
855 entry
->buf
[size
] = '\0';
857 __buffer_unlock_commit(buffer
, event
);
858 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
862 EXPORT_SYMBOL_GPL(__trace_puts
);
865 * __trace_bputs - write the pointer to a constant string into trace buffer
866 * @ip: The address of the caller
867 * @str: The constant string to write to the buffer to
869 int __trace_bputs(unsigned long ip
, const char *str
)
871 struct ring_buffer_event
*event
;
872 struct ring_buffer
*buffer
;
873 struct bputs_entry
*entry
;
874 unsigned long irq_flags
;
875 int size
= sizeof(struct bputs_entry
);
878 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
881 pc
= preempt_count();
883 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
886 local_save_flags(irq_flags
);
887 buffer
= global_trace
.trace_buffer
.buffer
;
888 event
= __trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
893 entry
= ring_buffer_event_data(event
);
897 __buffer_unlock_commit(buffer
, event
);
898 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
902 EXPORT_SYMBOL_GPL(__trace_bputs
);
904 #ifdef CONFIG_TRACER_SNAPSHOT
906 * trace_snapshot - take a snapshot of the current buffer.
908 * This causes a swap between the snapshot buffer and the current live
909 * tracing buffer. You can use this to take snapshots of the live
910 * trace when some condition is triggered, but continue to trace.
912 * Note, make sure to allocate the snapshot with either
913 * a tracing_snapshot_alloc(), or by doing it manually
914 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
916 * If the snapshot buffer is not allocated, it will stop tracing.
917 * Basically making a permanent snapshot.
919 void tracing_snapshot(void)
921 struct trace_array
*tr
= &global_trace
;
922 struct tracer
*tracer
= tr
->current_trace
;
926 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
927 internal_trace_puts("*** snapshot is being ignored ***\n");
931 if (!tr
->allocated_snapshot
) {
932 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
933 internal_trace_puts("*** stopping trace here! ***\n");
938 /* Note, snapshot can not be used when the tracer uses it */
939 if (tracer
->use_max_tr
) {
940 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
941 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
945 local_irq_save(flags
);
946 update_max_tr(tr
, current
, smp_processor_id());
947 local_irq_restore(flags
);
949 EXPORT_SYMBOL_GPL(tracing_snapshot
);
951 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
952 struct trace_buffer
*size_buf
, int cpu_id
);
953 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
955 static int alloc_snapshot(struct trace_array
*tr
)
959 if (!tr
->allocated_snapshot
) {
961 /* allocate spare buffer */
962 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
963 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
967 tr
->allocated_snapshot
= true;
973 static void free_snapshot(struct trace_array
*tr
)
976 * We don't free the ring buffer. instead, resize it because
977 * The max_tr ring buffer has some state (e.g. ring->clock) and
978 * we want preserve it.
980 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
981 set_buffer_entries(&tr
->max_buffer
, 1);
982 tracing_reset_online_cpus(&tr
->max_buffer
);
983 tr
->allocated_snapshot
= false;
987 * tracing_alloc_snapshot - allocate snapshot buffer.
989 * This only allocates the snapshot buffer if it isn't already
990 * allocated - it doesn't also take a snapshot.
992 * This is meant to be used in cases where the snapshot buffer needs
993 * to be set up for events that can't sleep but need to be able to
994 * trigger a snapshot.
996 int tracing_alloc_snapshot(void)
998 struct trace_array
*tr
= &global_trace
;
1001 ret
= alloc_snapshot(tr
);
1006 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
1009 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
1011 * This is similar to trace_snapshot(), but it will allocate the
1012 * snapshot buffer if it isn't already allocated. Use this only
1013 * where it is safe to sleep, as the allocation may sleep.
1015 * This causes a swap between the snapshot buffer and the current live
1016 * tracing buffer. You can use this to take snapshots of the live
1017 * trace when some condition is triggered, but continue to trace.
1019 void tracing_snapshot_alloc(void)
1023 ret
= tracing_alloc_snapshot();
1029 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
1031 void tracing_snapshot(void)
1033 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1035 EXPORT_SYMBOL_GPL(tracing_snapshot
);
1036 int tracing_alloc_snapshot(void)
1038 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1041 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
1042 void tracing_snapshot_alloc(void)
1047 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
1048 #endif /* CONFIG_TRACER_SNAPSHOT */
1050 static void tracer_tracing_off(struct trace_array
*tr
)
1052 if (tr
->trace_buffer
.buffer
)
1053 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
1055 * This flag is looked at when buffers haven't been allocated
1056 * yet, or by some tracers (like irqsoff), that just want to
1057 * know if the ring buffer has been disabled, but it can handle
1058 * races of where it gets disabled but we still do a record.
1059 * As the check is in the fast path of the tracers, it is more
1060 * important to be fast than accurate.
1062 tr
->buffer_disabled
= 1;
1063 /* Make the flag seen by readers */
1068 * tracing_off - turn off tracing buffers
1070 * This function stops the tracing buffers from recording data.
1071 * It does not disable any overhead the tracers themselves may
1072 * be causing. This function simply causes all recording to
1073 * the ring buffers to fail.
1075 void tracing_off(void)
1077 tracer_tracing_off(&global_trace
);
1079 EXPORT_SYMBOL_GPL(tracing_off
);
1081 void disable_trace_on_warning(void)
1083 if (__disable_trace_on_warning
)
1088 * tracer_tracing_is_on - show real state of ring buffer enabled
1089 * @tr : the trace array to know if ring buffer is enabled
1091 * Shows real state of the ring buffer if it is enabled or not.
1093 int tracer_tracing_is_on(struct trace_array
*tr
)
1095 if (tr
->trace_buffer
.buffer
)
1096 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
1097 return !tr
->buffer_disabled
;
1101 * tracing_is_on - show state of ring buffers enabled
1103 int tracing_is_on(void)
1105 return tracer_tracing_is_on(&global_trace
);
1107 EXPORT_SYMBOL_GPL(tracing_is_on
);
1109 static int __init
set_buf_size(char *str
)
1111 unsigned long buf_size
;
1115 buf_size
= memparse(str
, &str
);
1116 /* nr_entries can not be zero */
1119 trace_buf_size
= buf_size
;
1122 __setup("trace_buf_size=", set_buf_size
);
1124 static int __init
set_tracing_thresh(char *str
)
1126 unsigned long threshold
;
1131 ret
= kstrtoul(str
, 0, &threshold
);
1134 tracing_thresh
= threshold
* 1000;
1137 __setup("tracing_thresh=", set_tracing_thresh
);
1139 unsigned long nsecs_to_usecs(unsigned long nsecs
)
1141 return nsecs
/ 1000;
1145 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1146 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1147 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1148 * of strings in the order that the enums were defined.
1153 /* These must match the bit postions in trace_iterator_flags */
1154 static const char *trace_options
[] = {
1162 int in_ns
; /* is this clock in nanoseconds? */
1163 } trace_clocks
[] = {
1164 { trace_clock_local
, "local", 1 },
1165 { trace_clock_global
, "global", 1 },
1166 { trace_clock_counter
, "counter", 0 },
1167 { trace_clock_jiffies
, "uptime", 0 },
1168 { trace_clock
, "perf", 1 },
1169 { ktime_get_mono_fast_ns
, "mono", 1 },
1170 { ktime_get_raw_fast_ns
, "mono_raw", 1 },
1175 * trace_parser_get_init - gets the buffer for trace parser
1177 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
1179 memset(parser
, 0, sizeof(*parser
));
1181 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
1182 if (!parser
->buffer
)
1185 parser
->size
= size
;
1190 * trace_parser_put - frees the buffer for trace parser
1192 void trace_parser_put(struct trace_parser
*parser
)
1194 kfree(parser
->buffer
);
1198 * trace_get_user - reads the user input string separated by space
1199 * (matched by isspace(ch))
1201 * For each string found the 'struct trace_parser' is updated,
1202 * and the function returns.
1204 * Returns number of bytes read.
1206 * See kernel/trace/trace.h for 'struct trace_parser' details.
1208 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
1209 size_t cnt
, loff_t
*ppos
)
1216 trace_parser_clear(parser
);
1218 ret
= get_user(ch
, ubuf
++);
1226 * The parser is not finished with the last write,
1227 * continue reading the user input without skipping spaces.
1229 if (!parser
->cont
) {
1230 /* skip white space */
1231 while (cnt
&& isspace(ch
)) {
1232 ret
= get_user(ch
, ubuf
++);
1239 /* only spaces were written */
1249 /* read the non-space input */
1250 while (cnt
&& !isspace(ch
)) {
1251 if (parser
->idx
< parser
->size
- 1)
1252 parser
->buffer
[parser
->idx
++] = ch
;
1257 ret
= get_user(ch
, ubuf
++);
1264 /* We either got finished input or we have to wait for another call. */
1266 parser
->buffer
[parser
->idx
] = 0;
1267 parser
->cont
= false;
1268 } else if (parser
->idx
< parser
->size
- 1) {
1269 parser
->cont
= true;
1270 parser
->buffer
[parser
->idx
++] = ch
;
1283 /* TODO add a seq_buf_to_buffer() */
1284 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
1288 if (trace_seq_used(s
) <= s
->seq
.readpos
)
1291 len
= trace_seq_used(s
) - s
->seq
.readpos
;
1294 memcpy(buf
, s
->buffer
+ s
->seq
.readpos
, cnt
);
1296 s
->seq
.readpos
+= cnt
;
1300 unsigned long __read_mostly tracing_thresh
;
1302 #ifdef CONFIG_TRACER_MAX_TRACE
1304 * Copy the new maximum trace into the separate maximum-trace
1305 * structure. (this way the maximum trace is permanently saved,
1306 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1309 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1311 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
1312 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
1313 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
1314 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
1317 max_buf
->time_start
= data
->preempt_timestamp
;
1319 max_data
->saved_latency
= tr
->max_latency
;
1320 max_data
->critical_start
= data
->critical_start
;
1321 max_data
->critical_end
= data
->critical_end
;
1323 memcpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
1324 max_data
->pid
= tsk
->pid
;
1326 * If tsk == current, then use current_uid(), as that does not use
1327 * RCU. The irq tracer can be called out of RCU scope.
1330 max_data
->uid
= current_uid();
1332 max_data
->uid
= task_uid(tsk
);
1334 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
1335 max_data
->policy
= tsk
->policy
;
1336 max_data
->rt_priority
= tsk
->rt_priority
;
1338 /* record this tasks comm */
1339 tracing_record_cmdline(tsk
);
1343 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1345 * @tsk: the task with the latency
1346 * @cpu: The cpu that initiated the trace.
1348 * Flip the buffers between the @tr and the max_tr and record information
1349 * about which task was the cause of this latency.
1352 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1354 struct ring_buffer
*buf
;
1359 WARN_ON_ONCE(!irqs_disabled());
1361 if (!tr
->allocated_snapshot
) {
1362 /* Only the nop tracer should hit this when disabling */
1363 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1367 arch_spin_lock(&tr
->max_lock
);
1369 buf
= tr
->trace_buffer
.buffer
;
1370 tr
->trace_buffer
.buffer
= tr
->max_buffer
.buffer
;
1371 tr
->max_buffer
.buffer
= buf
;
1373 __update_max_tr(tr
, tsk
, cpu
);
1374 arch_spin_unlock(&tr
->max_lock
);
1378 * update_max_tr_single - only copy one trace over, and reset the rest
1380 * @tsk - task with the latency
1381 * @cpu - the cpu of the buffer to copy.
1383 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1386 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1393 WARN_ON_ONCE(!irqs_disabled());
1394 if (!tr
->allocated_snapshot
) {
1395 /* Only the nop tracer should hit this when disabling */
1396 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1400 arch_spin_lock(&tr
->max_lock
);
1402 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1404 if (ret
== -EBUSY
) {
1406 * We failed to swap the buffer due to a commit taking
1407 * place on this CPU. We fail to record, but we reset
1408 * the max trace buffer (no one writes directly to it)
1409 * and flag that it failed.
1411 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1412 "Failed to swap buffers due to commit in progress\n");
1415 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1417 __update_max_tr(tr
, tsk
, cpu
);
1418 arch_spin_unlock(&tr
->max_lock
);
1420 #endif /* CONFIG_TRACER_MAX_TRACE */
1422 static int wait_on_pipe(struct trace_iterator
*iter
, bool full
)
1424 /* Iterators are static, they should be filled or empty */
1425 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1428 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
1432 #ifdef CONFIG_FTRACE_STARTUP_TEST
1433 static int run_tracer_selftest(struct tracer
*type
)
1435 struct trace_array
*tr
= &global_trace
;
1436 struct tracer
*saved_tracer
= tr
->current_trace
;
1439 if (!type
->selftest
|| tracing_selftest_disabled
)
1443 * Run a selftest on this tracer.
1444 * Here we reset the trace buffer, and set the current
1445 * tracer to be this tracer. The tracer can then run some
1446 * internal tracing to verify that everything is in order.
1447 * If we fail, we do not register this tracer.
1449 tracing_reset_online_cpus(&tr
->trace_buffer
);
1451 tr
->current_trace
= type
;
1453 #ifdef CONFIG_TRACER_MAX_TRACE
1454 if (type
->use_max_tr
) {
1455 /* If we expanded the buffers, make sure the max is expanded too */
1456 if (ring_buffer_expanded
)
1457 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1458 RING_BUFFER_ALL_CPUS
);
1459 tr
->allocated_snapshot
= true;
1463 /* the test is responsible for initializing and enabling */
1464 pr_info("Testing tracer %s: ", type
->name
);
1465 ret
= type
->selftest(type
, tr
);
1466 /* the test is responsible for resetting too */
1467 tr
->current_trace
= saved_tracer
;
1469 printk(KERN_CONT
"FAILED!\n");
1470 /* Add the warning after printing 'FAILED' */
1474 /* Only reset on passing, to avoid touching corrupted buffers */
1475 tracing_reset_online_cpus(&tr
->trace_buffer
);
1477 #ifdef CONFIG_TRACER_MAX_TRACE
1478 if (type
->use_max_tr
) {
1479 tr
->allocated_snapshot
= false;
1481 /* Shrink the max buffer again */
1482 if (ring_buffer_expanded
)
1483 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1484 RING_BUFFER_ALL_CPUS
);
1488 printk(KERN_CONT
"PASSED\n");
1492 static inline int run_tracer_selftest(struct tracer
*type
)
1496 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1498 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
);
1500 static void __init
apply_trace_boot_options(void);
1503 * register_tracer - register a tracer with the ftrace system.
1504 * @type - the plugin for the tracer
1506 * Register a new plugin tracer.
1508 int __init
register_tracer(struct tracer
*type
)
1514 pr_info("Tracer must have a name\n");
1518 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1519 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1523 mutex_lock(&trace_types_lock
);
1525 tracing_selftest_running
= true;
1527 for (t
= trace_types
; t
; t
= t
->next
) {
1528 if (strcmp(type
->name
, t
->name
) == 0) {
1530 pr_info("Tracer %s already registered\n",
1537 if (!type
->set_flag
)
1538 type
->set_flag
= &dummy_set_flag
;
1540 /*allocate a dummy tracer_flags*/
1541 type
->flags
= kmalloc(sizeof(*type
->flags
), GFP_KERNEL
);
1546 type
->flags
->val
= 0;
1547 type
->flags
->opts
= dummy_tracer_opt
;
1549 if (!type
->flags
->opts
)
1550 type
->flags
->opts
= dummy_tracer_opt
;
1552 /* store the tracer for __set_tracer_option */
1553 type
->flags
->trace
= type
;
1555 ret
= run_tracer_selftest(type
);
1559 type
->next
= trace_types
;
1561 add_tracer_options(&global_trace
, type
);
1564 tracing_selftest_running
= false;
1565 mutex_unlock(&trace_types_lock
);
1567 if (ret
|| !default_bootup_tracer
)
1570 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1573 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1574 /* Do we want this tracer to start on bootup? */
1575 tracing_set_tracer(&global_trace
, type
->name
);
1576 default_bootup_tracer
= NULL
;
1578 apply_trace_boot_options();
1580 /* disable other selftests, since this will break it. */
1581 tracing_selftest_disabled
= true;
1582 #ifdef CONFIG_FTRACE_STARTUP_TEST
1583 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1591 void tracing_reset(struct trace_buffer
*buf
, int cpu
)
1593 struct ring_buffer
*buffer
= buf
->buffer
;
1598 ring_buffer_record_disable(buffer
);
1600 /* Make sure all commits have finished */
1601 synchronize_sched();
1602 ring_buffer_reset_cpu(buffer
, cpu
);
1604 ring_buffer_record_enable(buffer
);
1607 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1609 struct ring_buffer
*buffer
= buf
->buffer
;
1615 ring_buffer_record_disable(buffer
);
1617 /* Make sure all commits have finished */
1618 synchronize_sched();
1620 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1622 for_each_online_cpu(cpu
)
1623 ring_buffer_reset_cpu(buffer
, cpu
);
1625 ring_buffer_record_enable(buffer
);
1628 /* Must have trace_types_lock held */
1629 void tracing_reset_all_online_cpus(void)
1631 struct trace_array
*tr
;
1633 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1634 tracing_reset_online_cpus(&tr
->trace_buffer
);
1635 #ifdef CONFIG_TRACER_MAX_TRACE
1636 tracing_reset_online_cpus(&tr
->max_buffer
);
1641 #define SAVED_CMDLINES_DEFAULT 128
1642 #define NO_CMDLINE_MAP UINT_MAX
1643 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1644 struct saved_cmdlines_buffer
{
1645 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1646 unsigned *map_cmdline_to_pid
;
1647 unsigned cmdline_num
;
1649 char *saved_cmdlines
;
1651 static struct saved_cmdlines_buffer
*savedcmd
;
1653 /* temporary disable recording */
1654 static atomic_t trace_record_cmdline_disabled __read_mostly
;
1656 static inline char *get_saved_cmdlines(int idx
)
1658 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1661 static inline void set_cmdline(int idx
, const char *cmdline
)
1663 memcpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1666 static int allocate_cmdlines_buffer(unsigned int val
,
1667 struct saved_cmdlines_buffer
*s
)
1669 s
->map_cmdline_to_pid
= kmalloc(val
* sizeof(*s
->map_cmdline_to_pid
),
1671 if (!s
->map_cmdline_to_pid
)
1674 s
->saved_cmdlines
= kmalloc(val
* TASK_COMM_LEN
, GFP_KERNEL
);
1675 if (!s
->saved_cmdlines
) {
1676 kfree(s
->map_cmdline_to_pid
);
1681 s
->cmdline_num
= val
;
1682 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1683 sizeof(s
->map_pid_to_cmdline
));
1684 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1685 val
* sizeof(*s
->map_cmdline_to_pid
));
1690 static int trace_create_savedcmd(void)
1694 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1698 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
1708 int is_tracing_stopped(void)
1710 return global_trace
.stop_count
;
1714 * tracing_start - quick start of the tracer
1716 * If tracing is enabled but was stopped by tracing_stop,
1717 * this will start the tracer back up.
1719 void tracing_start(void)
1721 struct ring_buffer
*buffer
;
1722 unsigned long flags
;
1724 if (tracing_disabled
)
1727 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1728 if (--global_trace
.stop_count
) {
1729 if (global_trace
.stop_count
< 0) {
1730 /* Someone screwed up their debugging */
1732 global_trace
.stop_count
= 0;
1737 /* Prevent the buffers from switching */
1738 arch_spin_lock(&global_trace
.max_lock
);
1740 buffer
= global_trace
.trace_buffer
.buffer
;
1742 ring_buffer_record_enable(buffer
);
1744 #ifdef CONFIG_TRACER_MAX_TRACE
1745 buffer
= global_trace
.max_buffer
.buffer
;
1747 ring_buffer_record_enable(buffer
);
1750 arch_spin_unlock(&global_trace
.max_lock
);
1753 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1756 static void tracing_start_tr(struct trace_array
*tr
)
1758 struct ring_buffer
*buffer
;
1759 unsigned long flags
;
1761 if (tracing_disabled
)
1764 /* If global, we need to also start the max tracer */
1765 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1766 return tracing_start();
1768 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1770 if (--tr
->stop_count
) {
1771 if (tr
->stop_count
< 0) {
1772 /* Someone screwed up their debugging */
1779 buffer
= tr
->trace_buffer
.buffer
;
1781 ring_buffer_record_enable(buffer
);
1784 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1788 * tracing_stop - quick stop of the tracer
1790 * Light weight way to stop tracing. Use in conjunction with
1793 void tracing_stop(void)
1795 struct ring_buffer
*buffer
;
1796 unsigned long flags
;
1798 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1799 if (global_trace
.stop_count
++)
1802 /* Prevent the buffers from switching */
1803 arch_spin_lock(&global_trace
.max_lock
);
1805 buffer
= global_trace
.trace_buffer
.buffer
;
1807 ring_buffer_record_disable(buffer
);
1809 #ifdef CONFIG_TRACER_MAX_TRACE
1810 buffer
= global_trace
.max_buffer
.buffer
;
1812 ring_buffer_record_disable(buffer
);
1815 arch_spin_unlock(&global_trace
.max_lock
);
1818 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1821 static void tracing_stop_tr(struct trace_array
*tr
)
1823 struct ring_buffer
*buffer
;
1824 unsigned long flags
;
1826 /* If global, we need to also stop the max tracer */
1827 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1828 return tracing_stop();
1830 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1831 if (tr
->stop_count
++)
1834 buffer
= tr
->trace_buffer
.buffer
;
1836 ring_buffer_record_disable(buffer
);
1839 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1842 void trace_stop_cmdline_recording(void);
1844 static int trace_save_cmdline(struct task_struct
*tsk
)
1848 if (!tsk
->pid
|| unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
1852 * It's not the end of the world if we don't get
1853 * the lock, but we also don't want to spin
1854 * nor do we want to disable interrupts,
1855 * so if we miss here, then better luck next time.
1857 if (!arch_spin_trylock(&trace_cmdline_lock
))
1860 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
1861 if (idx
== NO_CMDLINE_MAP
) {
1862 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
1865 * Check whether the cmdline buffer at idx has a pid
1866 * mapped. We are going to overwrite that entry so we
1867 * need to clear the map_pid_to_cmdline. Otherwise we
1868 * would read the new comm for the old pid.
1870 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
1871 if (pid
!= NO_CMDLINE_MAP
)
1872 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
1874 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
1875 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
1877 savedcmd
->cmdline_idx
= idx
;
1880 set_cmdline(idx
, tsk
->comm
);
1882 arch_spin_unlock(&trace_cmdline_lock
);
1887 static void __trace_find_cmdline(int pid
, char comm
[])
1892 strcpy(comm
, "<idle>");
1896 if (WARN_ON_ONCE(pid
< 0)) {
1897 strcpy(comm
, "<XXX>");
1901 if (pid
> PID_MAX_DEFAULT
) {
1902 strcpy(comm
, "<...>");
1906 map
= savedcmd
->map_pid_to_cmdline
[pid
];
1907 if (map
!= NO_CMDLINE_MAP
)
1908 strcpy(comm
, get_saved_cmdlines(map
));
1910 strcpy(comm
, "<...>");
1913 void trace_find_cmdline(int pid
, char comm
[])
1916 arch_spin_lock(&trace_cmdline_lock
);
1918 __trace_find_cmdline(pid
, comm
);
1920 arch_spin_unlock(&trace_cmdline_lock
);
1924 void tracing_record_cmdline(struct task_struct
*tsk
)
1926 if (atomic_read(&trace_record_cmdline_disabled
) || !tracing_is_on())
1929 if (!__this_cpu_read(trace_cmdline_save
))
1932 if (trace_save_cmdline(tsk
))
1933 __this_cpu_write(trace_cmdline_save
, false);
1937 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned long flags
,
1940 struct task_struct
*tsk
= current
;
1942 entry
->preempt_count
= pc
& 0xff;
1943 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
1945 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1946 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
1948 TRACE_FLAG_IRQS_NOSUPPORT
|
1950 ((pc
& NMI_MASK
) ? TRACE_FLAG_NMI
: 0) |
1951 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
1952 ((pc
& SOFTIRQ_MASK
) ? TRACE_FLAG_SOFTIRQ
: 0) |
1953 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
1954 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
1956 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
1958 struct ring_buffer_event
*
1959 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
1962 unsigned long flags
, int pc
)
1964 return __trace_buffer_lock_reserve(buffer
, type
, len
, flags
, pc
);
1967 DEFINE_PER_CPU(struct ring_buffer_event
*, trace_buffered_event
);
1968 DEFINE_PER_CPU(int, trace_buffered_event_cnt
);
1969 static int trace_buffered_event_ref
;
1972 * trace_buffered_event_enable - enable buffering events
1974 * When events are being filtered, it is quicker to use a temporary
1975 * buffer to write the event data into if there's a likely chance
1976 * that it will not be committed. The discard of the ring buffer
1977 * is not as fast as committing, and is much slower than copying
1980 * When an event is to be filtered, allocate per cpu buffers to
1981 * write the event data into, and if the event is filtered and discarded
1982 * it is simply dropped, otherwise, the entire data is to be committed
1985 void trace_buffered_event_enable(void)
1987 struct ring_buffer_event
*event
;
1991 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
1993 if (trace_buffered_event_ref
++)
1996 for_each_tracing_cpu(cpu
) {
1997 page
= alloc_pages_node(cpu_to_node(cpu
),
1998 GFP_KERNEL
| __GFP_NORETRY
, 0);
2002 event
= page_address(page
);
2003 memset(event
, 0, sizeof(*event
));
2005 per_cpu(trace_buffered_event
, cpu
) = event
;
2008 if (cpu
== smp_processor_id() &&
2009 this_cpu_read(trace_buffered_event
) !=
2010 per_cpu(trace_buffered_event
, cpu
))
2017 trace_buffered_event_disable();
2020 static void enable_trace_buffered_event(void *data
)
2022 /* Probably not needed, but do it anyway */
2024 this_cpu_dec(trace_buffered_event_cnt
);
2027 static void disable_trace_buffered_event(void *data
)
2029 this_cpu_inc(trace_buffered_event_cnt
);
2033 * trace_buffered_event_disable - disable buffering events
2035 * When a filter is removed, it is faster to not use the buffered
2036 * events, and to commit directly into the ring buffer. Free up
2037 * the temp buffers when there are no more users. This requires
2038 * special synchronization with current events.
2040 void trace_buffered_event_disable(void)
2044 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
2046 if (WARN_ON_ONCE(!trace_buffered_event_ref
))
2049 if (--trace_buffered_event_ref
)
2053 /* For each CPU, set the buffer as used. */
2054 smp_call_function_many(tracing_buffer_mask
,
2055 disable_trace_buffered_event
, NULL
, 1);
2058 /* Wait for all current users to finish */
2059 synchronize_sched();
2061 for_each_tracing_cpu(cpu
) {
2062 free_page((unsigned long)per_cpu(trace_buffered_event
, cpu
));
2063 per_cpu(trace_buffered_event
, cpu
) = NULL
;
2066 * Make sure trace_buffered_event is NULL before clearing
2067 * trace_buffered_event_cnt.
2072 /* Do the work on each cpu */
2073 smp_call_function_many(tracing_buffer_mask
,
2074 enable_trace_buffered_event
, NULL
, 1);
2078 static struct ring_buffer
*temp_buffer
;
2080 struct ring_buffer_event
*
2081 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
2082 struct trace_event_file
*trace_file
,
2083 int type
, unsigned long len
,
2084 unsigned long flags
, int pc
)
2086 struct ring_buffer_event
*entry
;
2089 *current_rb
= trace_file
->tr
->trace_buffer
.buffer
;
2091 if ((trace_file
->flags
&
2092 (EVENT_FILE_FL_SOFT_DISABLED
| EVENT_FILE_FL_FILTERED
)) &&
2093 (entry
= this_cpu_read(trace_buffered_event
))) {
2094 /* Try to use the per cpu buffer first */
2095 val
= this_cpu_inc_return(trace_buffered_event_cnt
);
2097 trace_event_setup(entry
, type
, flags
, pc
);
2098 entry
->array
[0] = len
;
2101 this_cpu_dec(trace_buffered_event_cnt
);
2104 entry
= __trace_buffer_lock_reserve(*current_rb
,
2105 type
, len
, flags
, pc
);
2107 * If tracing is off, but we have triggers enabled
2108 * we still need to look at the event data. Use the temp_buffer
2109 * to store the trace event for the tigger to use. It's recusive
2110 * safe and will not be recorded anywhere.
2112 if (!entry
&& trace_file
->flags
& EVENT_FILE_FL_TRIGGER_COND
) {
2113 *current_rb
= temp_buffer
;
2114 entry
= __trace_buffer_lock_reserve(*current_rb
,
2115 type
, len
, flags
, pc
);
2119 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
2121 static DEFINE_SPINLOCK(tracepoint_iter_lock
);
2122 static DEFINE_MUTEX(tracepoint_printk_mutex
);
2124 static void output_printk(struct trace_event_buffer
*fbuffer
)
2126 struct trace_event_call
*event_call
;
2127 struct trace_event
*event
;
2128 unsigned long flags
;
2129 struct trace_iterator
*iter
= tracepoint_print_iter
;
2131 /* We should never get here if iter is NULL */
2132 if (WARN_ON_ONCE(!iter
))
2135 event_call
= fbuffer
->trace_file
->event_call
;
2136 if (!event_call
|| !event_call
->event
.funcs
||
2137 !event_call
->event
.funcs
->trace
)
2140 event
= &fbuffer
->trace_file
->event_call
->event
;
2142 spin_lock_irqsave(&tracepoint_iter_lock
, flags
);
2143 trace_seq_init(&iter
->seq
);
2144 iter
->ent
= fbuffer
->entry
;
2145 event_call
->event
.funcs
->trace(iter
, 0, event
);
2146 trace_seq_putc(&iter
->seq
, 0);
2147 printk("%s", iter
->seq
.buffer
);
2149 spin_unlock_irqrestore(&tracepoint_iter_lock
, flags
);
2152 int tracepoint_printk_sysctl(struct ctl_table
*table
, int write
,
2153 void __user
*buffer
, size_t *lenp
,
2156 int save_tracepoint_printk
;
2159 mutex_lock(&tracepoint_printk_mutex
);
2160 save_tracepoint_printk
= tracepoint_printk
;
2162 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
2165 * This will force exiting early, as tracepoint_printk
2166 * is always zero when tracepoint_printk_iter is not allocated
2168 if (!tracepoint_print_iter
)
2169 tracepoint_printk
= 0;
2171 if (save_tracepoint_printk
== tracepoint_printk
)
2174 if (tracepoint_printk
)
2175 static_key_enable(&tracepoint_printk_key
.key
);
2177 static_key_disable(&tracepoint_printk_key
.key
);
2180 mutex_unlock(&tracepoint_printk_mutex
);
2185 void trace_event_buffer_commit(struct trace_event_buffer
*fbuffer
)
2187 if (static_key_false(&tracepoint_printk_key
.key
))
2188 output_printk(fbuffer
);
2190 event_trigger_unlock_commit(fbuffer
->trace_file
, fbuffer
->buffer
,
2191 fbuffer
->event
, fbuffer
->entry
,
2192 fbuffer
->flags
, fbuffer
->pc
);
2194 EXPORT_SYMBOL_GPL(trace_event_buffer_commit
);
2196 void trace_buffer_unlock_commit_regs(struct trace_array
*tr
,
2197 struct ring_buffer
*buffer
,
2198 struct ring_buffer_event
*event
,
2199 unsigned long flags
, int pc
,
2200 struct pt_regs
*regs
)
2202 __buffer_unlock_commit(buffer
, event
);
2205 * If regs is not set, then skip the following callers:
2206 * trace_buffer_unlock_commit_regs
2207 * event_trigger_unlock_commit
2208 * trace_event_buffer_commit
2209 * trace_event_raw_event_sched_switch
2210 * Note, we can still get here via blktrace, wakeup tracer
2211 * and mmiotrace, but that's ok if they lose a function or
2212 * two. They are that meaningful.
2214 ftrace_trace_stack(tr
, buffer
, flags
, regs
? 0 : 4, pc
, regs
);
2215 ftrace_trace_userstack(buffer
, flags
, pc
);
2219 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2222 trace_buffer_unlock_commit_nostack(struct ring_buffer
*buffer
,
2223 struct ring_buffer_event
*event
)
2225 __buffer_unlock_commit(buffer
, event
);
2229 trace_process_export(struct trace_export
*export
,
2230 struct ring_buffer_event
*event
)
2232 struct trace_entry
*entry
;
2233 unsigned int size
= 0;
2235 entry
= ring_buffer_event_data(event
);
2236 size
= ring_buffer_event_length(event
);
2237 export
->write(entry
, size
);
2240 static DEFINE_MUTEX(ftrace_export_lock
);
2242 static struct trace_export __rcu
*ftrace_exports_list __read_mostly
;
2244 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled
);
2246 static inline void ftrace_exports_enable(void)
2248 static_branch_enable(&ftrace_exports_enabled
);
2251 static inline void ftrace_exports_disable(void)
2253 static_branch_disable(&ftrace_exports_enabled
);
2256 void ftrace_exports(struct ring_buffer_event
*event
)
2258 struct trace_export
*export
;
2260 preempt_disable_notrace();
2262 export
= rcu_dereference_raw_notrace(ftrace_exports_list
);
2264 trace_process_export(export
, event
);
2265 export
= rcu_dereference_raw_notrace(export
->next
);
2268 preempt_enable_notrace();
2272 add_trace_export(struct trace_export
**list
, struct trace_export
*export
)
2274 rcu_assign_pointer(export
->next
, *list
);
2276 * We are entering export into the list but another
2277 * CPU might be walking that list. We need to make sure
2278 * the export->next pointer is valid before another CPU sees
2279 * the export pointer included into the list.
2281 rcu_assign_pointer(*list
, export
);
2285 rm_trace_export(struct trace_export
**list
, struct trace_export
*export
)
2287 struct trace_export
**p
;
2289 for (p
= list
; *p
!= NULL
; p
= &(*p
)->next
)
2296 rcu_assign_pointer(*p
, (*p
)->next
);
2302 add_ftrace_export(struct trace_export
**list
, struct trace_export
*export
)
2305 ftrace_exports_enable();
2307 add_trace_export(list
, export
);
2311 rm_ftrace_export(struct trace_export
**list
, struct trace_export
*export
)
2315 ret
= rm_trace_export(list
, export
);
2317 ftrace_exports_disable();
2322 int register_ftrace_export(struct trace_export
*export
)
2324 if (WARN_ON_ONCE(!export
->write
))
2327 mutex_lock(&ftrace_export_lock
);
2329 add_ftrace_export(&ftrace_exports_list
, export
);
2331 mutex_unlock(&ftrace_export_lock
);
2335 EXPORT_SYMBOL_GPL(register_ftrace_export
);
2337 int unregister_ftrace_export(struct trace_export
*export
)
2341 mutex_lock(&ftrace_export_lock
);
2343 ret
= rm_ftrace_export(&ftrace_exports_list
, export
);
2345 mutex_unlock(&ftrace_export_lock
);
2349 EXPORT_SYMBOL_GPL(unregister_ftrace_export
);
2352 trace_function(struct trace_array
*tr
,
2353 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
2356 struct trace_event_call
*call
= &event_function
;
2357 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
2358 struct ring_buffer_event
*event
;
2359 struct ftrace_entry
*entry
;
2361 event
= __trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
2365 entry
= ring_buffer_event_data(event
);
2367 entry
->parent_ip
= parent_ip
;
2369 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2370 if (static_branch_unlikely(&ftrace_exports_enabled
))
2371 ftrace_exports(event
);
2372 __buffer_unlock_commit(buffer
, event
);
2376 #ifdef CONFIG_STACKTRACE
2378 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2379 struct ftrace_stack
{
2380 unsigned long calls
[FTRACE_STACK_MAX_ENTRIES
];
2383 static DEFINE_PER_CPU(struct ftrace_stack
, ftrace_stack
);
2384 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
2386 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
2387 unsigned long flags
,
2388 int skip
, int pc
, struct pt_regs
*regs
)
2390 struct trace_event_call
*call
= &event_kernel_stack
;
2391 struct ring_buffer_event
*event
;
2392 struct stack_entry
*entry
;
2393 struct stack_trace trace
;
2395 int size
= FTRACE_STACK_ENTRIES
;
2397 trace
.nr_entries
= 0;
2401 * Add two, for this function and the call to save_stack_trace()
2402 * If regs is set, then these functions will not be in the way.
2408 * Since events can happen in NMIs there's no safe way to
2409 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2410 * or NMI comes in, it will just have to use the default
2411 * FTRACE_STACK_SIZE.
2413 preempt_disable_notrace();
2415 use_stack
= __this_cpu_inc_return(ftrace_stack_reserve
);
2417 * We don't need any atomic variables, just a barrier.
2418 * If an interrupt comes in, we don't care, because it would
2419 * have exited and put the counter back to what we want.
2420 * We just need a barrier to keep gcc from moving things
2424 if (use_stack
== 1) {
2425 trace
.entries
= this_cpu_ptr(ftrace_stack
.calls
);
2426 trace
.max_entries
= FTRACE_STACK_MAX_ENTRIES
;
2429 save_stack_trace_regs(regs
, &trace
);
2431 save_stack_trace(&trace
);
2433 if (trace
.nr_entries
> size
)
2434 size
= trace
.nr_entries
;
2436 /* From now on, use_stack is a boolean */
2439 size
*= sizeof(unsigned long);
2441 event
= __trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
2442 sizeof(*entry
) + size
, flags
, pc
);
2445 entry
= ring_buffer_event_data(event
);
2447 memset(&entry
->caller
, 0, size
);
2450 memcpy(&entry
->caller
, trace
.entries
,
2451 trace
.nr_entries
* sizeof(unsigned long));
2453 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
2454 trace
.entries
= entry
->caller
;
2456 save_stack_trace_regs(regs
, &trace
);
2458 save_stack_trace(&trace
);
2461 entry
->size
= trace
.nr_entries
;
2463 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2464 __buffer_unlock_commit(buffer
, event
);
2467 /* Again, don't let gcc optimize things here */
2469 __this_cpu_dec(ftrace_stack_reserve
);
2470 preempt_enable_notrace();
2474 static inline void ftrace_trace_stack(struct trace_array
*tr
,
2475 struct ring_buffer
*buffer
,
2476 unsigned long flags
,
2477 int skip
, int pc
, struct pt_regs
*regs
)
2479 if (!(tr
->trace_flags
& TRACE_ITER_STACKTRACE
))
2482 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
2485 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
2488 __ftrace_trace_stack(tr
->trace_buffer
.buffer
, flags
, skip
, pc
, NULL
);
2492 * trace_dump_stack - record a stack back trace in the trace buffer
2493 * @skip: Number of functions to skip (helper handlers)
2495 void trace_dump_stack(int skip
)
2497 unsigned long flags
;
2499 if (tracing_disabled
|| tracing_selftest_running
)
2502 local_save_flags(flags
);
2505 * Skip 3 more, seems to get us at the caller of
2509 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
2510 flags
, skip
, preempt_count(), NULL
);
2513 static DEFINE_PER_CPU(int, user_stack_count
);
2516 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
2518 struct trace_event_call
*call
= &event_user_stack
;
2519 struct ring_buffer_event
*event
;
2520 struct userstack_entry
*entry
;
2521 struct stack_trace trace
;
2523 if (!(global_trace
.trace_flags
& TRACE_ITER_USERSTACKTRACE
))
2527 * NMIs can not handle page faults, even with fix ups.
2528 * The save user stack can (and often does) fault.
2530 if (unlikely(in_nmi()))
2534 * prevent recursion, since the user stack tracing may
2535 * trigger other kernel events.
2538 if (__this_cpu_read(user_stack_count
))
2541 __this_cpu_inc(user_stack_count
);
2543 event
= __trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
2544 sizeof(*entry
), flags
, pc
);
2546 goto out_drop_count
;
2547 entry
= ring_buffer_event_data(event
);
2549 entry
->tgid
= current
->tgid
;
2550 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
2552 trace
.nr_entries
= 0;
2553 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
2555 trace
.entries
= entry
->caller
;
2557 save_stack_trace_user(&trace
);
2558 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2559 __buffer_unlock_commit(buffer
, event
);
2562 __this_cpu_dec(user_stack_count
);
2568 static void __trace_userstack(struct trace_array
*tr
, unsigned long flags
)
2570 ftrace_trace_userstack(tr
, flags
, preempt_count());
2574 #endif /* CONFIG_STACKTRACE */
2576 /* created for use with alloc_percpu */
2577 struct trace_buffer_struct
{
2579 char buffer
[4][TRACE_BUF_SIZE
];
2582 static struct trace_buffer_struct
*trace_percpu_buffer
;
2585 * Thise allows for lockless recording. If we're nested too deeply, then
2586 * this returns NULL.
2588 static char *get_trace_buf(void)
2590 struct trace_buffer_struct
*buffer
= this_cpu_ptr(trace_percpu_buffer
);
2592 if (!buffer
|| buffer
->nesting
>= 4)
2595 return &buffer
->buffer
[buffer
->nesting
++][0];
2598 static void put_trace_buf(void)
2600 this_cpu_dec(trace_percpu_buffer
->nesting
);
2603 static int alloc_percpu_trace_buffer(void)
2605 struct trace_buffer_struct
*buffers
;
2607 buffers
= alloc_percpu(struct trace_buffer_struct
);
2608 if (WARN(!buffers
, "Could not allocate percpu trace_printk buffer"))
2611 trace_percpu_buffer
= buffers
;
2615 static int buffers_allocated
;
2617 void trace_printk_init_buffers(void)
2619 if (buffers_allocated
)
2622 if (alloc_percpu_trace_buffer())
2625 /* trace_printk() is for debug use only. Don't use it in production. */
2628 pr_warn("**********************************************************\n");
2629 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2631 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2633 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2634 pr_warn("** unsafe for production use. **\n");
2636 pr_warn("** If you see this message and you are not debugging **\n");
2637 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2639 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2640 pr_warn("**********************************************************\n");
2642 /* Expand the buffers to set size */
2643 tracing_update_buffers();
2645 buffers_allocated
= 1;
2648 * trace_printk_init_buffers() can be called by modules.
2649 * If that happens, then we need to start cmdline recording
2650 * directly here. If the global_trace.buffer is already
2651 * allocated here, then this was called by module code.
2653 if (global_trace
.trace_buffer
.buffer
)
2654 tracing_start_cmdline_record();
2657 void trace_printk_start_comm(void)
2659 /* Start tracing comms if trace printk is set */
2660 if (!buffers_allocated
)
2662 tracing_start_cmdline_record();
2665 static void trace_printk_start_stop_comm(int enabled
)
2667 if (!buffers_allocated
)
2671 tracing_start_cmdline_record();
2673 tracing_stop_cmdline_record();
2677 * trace_vbprintk - write binary msg to tracing buffer
2680 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
2682 struct trace_event_call
*call
= &event_bprint
;
2683 struct ring_buffer_event
*event
;
2684 struct ring_buffer
*buffer
;
2685 struct trace_array
*tr
= &global_trace
;
2686 struct bprint_entry
*entry
;
2687 unsigned long flags
;
2689 int len
= 0, size
, pc
;
2691 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
2694 /* Don't pollute graph traces with trace_vprintk internals */
2695 pause_graph_tracing();
2697 pc
= preempt_count();
2698 preempt_disable_notrace();
2700 tbuffer
= get_trace_buf();
2706 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
2708 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
2711 local_save_flags(flags
);
2712 size
= sizeof(*entry
) + sizeof(u32
) * len
;
2713 buffer
= tr
->trace_buffer
.buffer
;
2714 event
= __trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
2718 entry
= ring_buffer_event_data(event
);
2722 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
2723 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2724 __buffer_unlock_commit(buffer
, event
);
2725 ftrace_trace_stack(tr
, buffer
, flags
, 6, pc
, NULL
);
2732 preempt_enable_notrace();
2733 unpause_graph_tracing();
2737 EXPORT_SYMBOL_GPL(trace_vbprintk
);
2740 __trace_array_vprintk(struct ring_buffer
*buffer
,
2741 unsigned long ip
, const char *fmt
, va_list args
)
2743 struct trace_event_call
*call
= &event_print
;
2744 struct ring_buffer_event
*event
;
2745 int len
= 0, size
, pc
;
2746 struct print_entry
*entry
;
2747 unsigned long flags
;
2750 if (tracing_disabled
|| tracing_selftest_running
)
2753 /* Don't pollute graph traces with trace_vprintk internals */
2754 pause_graph_tracing();
2756 pc
= preempt_count();
2757 preempt_disable_notrace();
2760 tbuffer
= get_trace_buf();
2766 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
2768 local_save_flags(flags
);
2769 size
= sizeof(*entry
) + len
+ 1;
2770 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
2774 entry
= ring_buffer_event_data(event
);
2777 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
2778 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2779 __buffer_unlock_commit(buffer
, event
);
2780 ftrace_trace_stack(&global_trace
, buffer
, flags
, 6, pc
, NULL
);
2787 preempt_enable_notrace();
2788 unpause_graph_tracing();
2793 int trace_array_vprintk(struct trace_array
*tr
,
2794 unsigned long ip
, const char *fmt
, va_list args
)
2796 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
2799 int trace_array_printk(struct trace_array
*tr
,
2800 unsigned long ip
, const char *fmt
, ...)
2805 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2809 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
2814 int trace_array_printk_buf(struct ring_buffer
*buffer
,
2815 unsigned long ip
, const char *fmt
, ...)
2820 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2824 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
2829 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
2831 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
2833 EXPORT_SYMBOL_GPL(trace_vprintk
);
2835 static void trace_iterator_increment(struct trace_iterator
*iter
)
2837 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
2841 ring_buffer_read(buf_iter
, NULL
);
2844 static struct trace_entry
*
2845 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
2846 unsigned long *lost_events
)
2848 struct ring_buffer_event
*event
;
2849 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
2852 event
= ring_buffer_iter_peek(buf_iter
, ts
);
2854 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
2858 iter
->ent_size
= ring_buffer_event_length(event
);
2859 return ring_buffer_event_data(event
);
2865 static struct trace_entry
*
2866 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
2867 unsigned long *missing_events
, u64
*ent_ts
)
2869 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
2870 struct trace_entry
*ent
, *next
= NULL
;
2871 unsigned long lost_events
= 0, next_lost
= 0;
2872 int cpu_file
= iter
->cpu_file
;
2873 u64 next_ts
= 0, ts
;
2879 * If we are in a per_cpu trace file, don't bother by iterating over
2880 * all cpu and peek directly.
2882 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
2883 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
2885 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
2887 *ent_cpu
= cpu_file
;
2892 for_each_tracing_cpu(cpu
) {
2894 if (ring_buffer_empty_cpu(buffer
, cpu
))
2897 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
2900 * Pick the entry with the smallest timestamp:
2902 if (ent
&& (!next
|| ts
< next_ts
)) {
2906 next_lost
= lost_events
;
2907 next_size
= iter
->ent_size
;
2911 iter
->ent_size
= next_size
;
2914 *ent_cpu
= next_cpu
;
2920 *missing_events
= next_lost
;
2925 /* Find the next real entry, without updating the iterator itself */
2926 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
2927 int *ent_cpu
, u64
*ent_ts
)
2929 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
2932 /* Find the next real entry, and increment the iterator to the next entry */
2933 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
2935 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
2936 &iter
->lost_events
, &iter
->ts
);
2939 trace_iterator_increment(iter
);
2941 return iter
->ent
? iter
: NULL
;
2944 static void trace_consume(struct trace_iterator
*iter
)
2946 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
2947 &iter
->lost_events
);
2950 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2952 struct trace_iterator
*iter
= m
->private;
2956 WARN_ON_ONCE(iter
->leftover
);
2960 /* can't go backwards */
2965 ent
= trace_find_next_entry_inc(iter
);
2969 while (ent
&& iter
->idx
< i
)
2970 ent
= trace_find_next_entry_inc(iter
);
2977 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
2979 struct ring_buffer_event
*event
;
2980 struct ring_buffer_iter
*buf_iter
;
2981 unsigned long entries
= 0;
2984 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
2986 buf_iter
= trace_buffer_iter(iter
, cpu
);
2990 ring_buffer_iter_reset(buf_iter
);
2993 * We could have the case with the max latency tracers
2994 * that a reset never took place on a cpu. This is evident
2995 * by the timestamp being before the start of the buffer.
2997 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
2998 if (ts
>= iter
->trace_buffer
->time_start
)
3001 ring_buffer_read(buf_iter
, NULL
);
3004 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
3008 * The current tracer is copied to avoid a global locking
3011 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
3013 struct trace_iterator
*iter
= m
->private;
3014 struct trace_array
*tr
= iter
->tr
;
3015 int cpu_file
= iter
->cpu_file
;
3021 * copy the tracer to avoid using a global lock all around.
3022 * iter->trace is a copy of current_trace, the pointer to the
3023 * name may be used instead of a strcmp(), as iter->trace->name
3024 * will point to the same string as current_trace->name.
3026 mutex_lock(&trace_types_lock
);
3027 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
3028 *iter
->trace
= *tr
->current_trace
;
3029 mutex_unlock(&trace_types_lock
);
3031 #ifdef CONFIG_TRACER_MAX_TRACE
3032 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
3033 return ERR_PTR(-EBUSY
);
3036 if (!iter
->snapshot
)
3037 atomic_inc(&trace_record_cmdline_disabled
);
3039 if (*pos
!= iter
->pos
) {
3044 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
3045 for_each_tracing_cpu(cpu
)
3046 tracing_iter_reset(iter
, cpu
);
3048 tracing_iter_reset(iter
, cpu_file
);
3051 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
3056 * If we overflowed the seq_file before, then we want
3057 * to just reuse the trace_seq buffer again.
3063 p
= s_next(m
, p
, &l
);
3067 trace_event_read_lock();
3068 trace_access_lock(cpu_file
);
3072 static void s_stop(struct seq_file
*m
, void *p
)
3074 struct trace_iterator
*iter
= m
->private;
3076 #ifdef CONFIG_TRACER_MAX_TRACE
3077 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
3081 if (!iter
->snapshot
)
3082 atomic_dec(&trace_record_cmdline_disabled
);
3084 trace_access_unlock(iter
->cpu_file
);
3085 trace_event_read_unlock();
3089 get_total_entries(struct trace_buffer
*buf
,
3090 unsigned long *total
, unsigned long *entries
)
3092 unsigned long count
;
3098 for_each_tracing_cpu(cpu
) {
3099 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
3101 * If this buffer has skipped entries, then we hold all
3102 * entries for the trace and we need to ignore the
3103 * ones before the time stamp.
3105 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
3106 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
3107 /* total is the same as the entries */
3111 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
3116 static void print_lat_help_header(struct seq_file
*m
)
3118 seq_puts(m
, "# _------=> CPU# \n"
3119 "# / _-----=> irqs-off \n"
3120 "# | / _----=> need-resched \n"
3121 "# || / _---=> hardirq/softirq \n"
3122 "# ||| / _--=> preempt-depth \n"
3124 "# cmd pid ||||| time | caller \n"
3125 "# \\ / ||||| \\ | / \n");
3128 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
3130 unsigned long total
;
3131 unsigned long entries
;
3133 get_total_entries(buf
, &total
, &entries
);
3134 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3135 entries
, total
, num_online_cpus());
3139 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
)
3141 print_event_info(buf
, m
);
3142 seq_puts(m
, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
3146 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
)
3148 print_event_info(buf
, m
);
3149 seq_puts(m
, "# _-----=> irqs-off\n"
3150 "# / _----=> need-resched\n"
3151 "# | / _---=> hardirq/softirq\n"
3152 "# || / _--=> preempt-depth\n"
3154 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
3155 "# | | | |||| | |\n");
3159 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
3161 unsigned long sym_flags
= (global_trace
.trace_flags
& TRACE_ITER_SYM_MASK
);
3162 struct trace_buffer
*buf
= iter
->trace_buffer
;
3163 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
3164 struct tracer
*type
= iter
->trace
;
3165 unsigned long entries
;
3166 unsigned long total
;
3167 const char *name
= "preemption";
3171 get_total_entries(buf
, &total
, &entries
);
3173 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
3175 seq_puts(m
, "# -----------------------------------"
3176 "---------------------------------\n");
3177 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3178 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3179 nsecs_to_usecs(data
->saved_latency
),
3183 #if defined(CONFIG_PREEMPT_NONE)
3185 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3187 #elif defined(CONFIG_PREEMPT)
3192 /* These are reserved for later use */
3195 seq_printf(m
, " #P:%d)\n", num_online_cpus());
3199 seq_puts(m
, "# -----------------\n");
3200 seq_printf(m
, "# | task: %.16s-%d "
3201 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3202 data
->comm
, data
->pid
,
3203 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
3204 data
->policy
, data
->rt_priority
);
3205 seq_puts(m
, "# -----------------\n");
3207 if (data
->critical_start
) {
3208 seq_puts(m
, "# => started at: ");
3209 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
3210 trace_print_seq(m
, &iter
->seq
);
3211 seq_puts(m
, "\n# => ended at: ");
3212 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
3213 trace_print_seq(m
, &iter
->seq
);
3214 seq_puts(m
, "\n#\n");
3220 static void test_cpu_buff_start(struct trace_iterator
*iter
)
3222 struct trace_seq
*s
= &iter
->seq
;
3223 struct trace_array
*tr
= iter
->tr
;
3225 if (!(tr
->trace_flags
& TRACE_ITER_ANNOTATE
))
3228 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
3231 if (iter
->started
&& cpumask_test_cpu(iter
->cpu
, iter
->started
))
3234 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
3238 cpumask_set_cpu(iter
->cpu
, iter
->started
);
3240 /* Don't print started cpu buffer for the first entry of the trace */
3242 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
3246 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
3248 struct trace_array
*tr
= iter
->tr
;
3249 struct trace_seq
*s
= &iter
->seq
;
3250 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
3251 struct trace_entry
*entry
;
3252 struct trace_event
*event
;
3256 test_cpu_buff_start(iter
);
3258 event
= ftrace_find_event(entry
->type
);
3260 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3261 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3262 trace_print_lat_context(iter
);
3264 trace_print_context(iter
);
3267 if (trace_seq_has_overflowed(s
))
3268 return TRACE_TYPE_PARTIAL_LINE
;
3271 return event
->funcs
->trace(iter
, sym_flags
, event
);
3273 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
3275 return trace_handle_return(s
);
3278 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
3280 struct trace_array
*tr
= iter
->tr
;
3281 struct trace_seq
*s
= &iter
->seq
;
3282 struct trace_entry
*entry
;
3283 struct trace_event
*event
;
3287 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
)
3288 trace_seq_printf(s
, "%d %d %llu ",
3289 entry
->pid
, iter
->cpu
, iter
->ts
);
3291 if (trace_seq_has_overflowed(s
))
3292 return TRACE_TYPE_PARTIAL_LINE
;
3294 event
= ftrace_find_event(entry
->type
);
3296 return event
->funcs
->raw(iter
, 0, event
);
3298 trace_seq_printf(s
, "%d ?\n", entry
->type
);
3300 return trace_handle_return(s
);
3303 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
3305 struct trace_array
*tr
= iter
->tr
;
3306 struct trace_seq
*s
= &iter
->seq
;
3307 unsigned char newline
= '\n';
3308 struct trace_entry
*entry
;
3309 struct trace_event
*event
;
3313 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3314 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
3315 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
3316 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
3317 if (trace_seq_has_overflowed(s
))
3318 return TRACE_TYPE_PARTIAL_LINE
;
3321 event
= ftrace_find_event(entry
->type
);
3323 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
3324 if (ret
!= TRACE_TYPE_HANDLED
)
3328 SEQ_PUT_FIELD(s
, newline
);
3330 return trace_handle_return(s
);
3333 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
3335 struct trace_array
*tr
= iter
->tr
;
3336 struct trace_seq
*s
= &iter
->seq
;
3337 struct trace_entry
*entry
;
3338 struct trace_event
*event
;
3342 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3343 SEQ_PUT_FIELD(s
, entry
->pid
);
3344 SEQ_PUT_FIELD(s
, iter
->cpu
);
3345 SEQ_PUT_FIELD(s
, iter
->ts
);
3346 if (trace_seq_has_overflowed(s
))
3347 return TRACE_TYPE_PARTIAL_LINE
;
3350 event
= ftrace_find_event(entry
->type
);
3351 return event
? event
->funcs
->binary(iter
, 0, event
) :
3355 int trace_empty(struct trace_iterator
*iter
)
3357 struct ring_buffer_iter
*buf_iter
;
3360 /* If we are looking at one CPU buffer, only check that one */
3361 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
3362 cpu
= iter
->cpu_file
;
3363 buf_iter
= trace_buffer_iter(iter
, cpu
);
3365 if (!ring_buffer_iter_empty(buf_iter
))
3368 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3374 for_each_tracing_cpu(cpu
) {
3375 buf_iter
= trace_buffer_iter(iter
, cpu
);
3377 if (!ring_buffer_iter_empty(buf_iter
))
3380 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3388 /* Called with trace_event_read_lock() held. */
3389 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
3391 struct trace_array
*tr
= iter
->tr
;
3392 unsigned long trace_flags
= tr
->trace_flags
;
3393 enum print_line_t ret
;
3395 if (iter
->lost_events
) {
3396 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
3397 iter
->cpu
, iter
->lost_events
);
3398 if (trace_seq_has_overflowed(&iter
->seq
))
3399 return TRACE_TYPE_PARTIAL_LINE
;
3402 if (iter
->trace
&& iter
->trace
->print_line
) {
3403 ret
= iter
->trace
->print_line(iter
);
3404 if (ret
!= TRACE_TYPE_UNHANDLED
)
3408 if (iter
->ent
->type
== TRACE_BPUTS
&&
3409 trace_flags
& TRACE_ITER_PRINTK
&&
3410 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3411 return trace_print_bputs_msg_only(iter
);
3413 if (iter
->ent
->type
== TRACE_BPRINT
&&
3414 trace_flags
& TRACE_ITER_PRINTK
&&
3415 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3416 return trace_print_bprintk_msg_only(iter
);
3418 if (iter
->ent
->type
== TRACE_PRINT
&&
3419 trace_flags
& TRACE_ITER_PRINTK
&&
3420 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3421 return trace_print_printk_msg_only(iter
);
3423 if (trace_flags
& TRACE_ITER_BIN
)
3424 return print_bin_fmt(iter
);
3426 if (trace_flags
& TRACE_ITER_HEX
)
3427 return print_hex_fmt(iter
);
3429 if (trace_flags
& TRACE_ITER_RAW
)
3430 return print_raw_fmt(iter
);
3432 return print_trace_fmt(iter
);
3435 void trace_latency_header(struct seq_file
*m
)
3437 struct trace_iterator
*iter
= m
->private;
3438 struct trace_array
*tr
= iter
->tr
;
3440 /* print nothing if the buffers are empty */
3441 if (trace_empty(iter
))
3444 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3445 print_trace_header(m
, iter
);
3447 if (!(tr
->trace_flags
& TRACE_ITER_VERBOSE
))
3448 print_lat_help_header(m
);
3451 void trace_default_header(struct seq_file
*m
)
3453 struct trace_iterator
*iter
= m
->private;
3454 struct trace_array
*tr
= iter
->tr
;
3455 unsigned long trace_flags
= tr
->trace_flags
;
3457 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
3460 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
3461 /* print nothing if the buffers are empty */
3462 if (trace_empty(iter
))
3464 print_trace_header(m
, iter
);
3465 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
3466 print_lat_help_header(m
);
3468 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
3469 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
3470 print_func_help_header_irq(iter
->trace_buffer
, m
);
3472 print_func_help_header(iter
->trace_buffer
, m
);
3477 static void test_ftrace_alive(struct seq_file
*m
)
3479 if (!ftrace_is_dead())
3481 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3482 "# MAY BE MISSING FUNCTION EVENTS\n");
3485 #ifdef CONFIG_TRACER_MAX_TRACE
3486 static void show_snapshot_main_help(struct seq_file
*m
)
3488 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3489 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3490 "# Takes a snapshot of the main buffer.\n"
3491 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3492 "# (Doesn't have to be '2' works with any number that\n"
3493 "# is not a '0' or '1')\n");
3496 static void show_snapshot_percpu_help(struct seq_file
*m
)
3498 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3499 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3500 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3501 "# Takes a snapshot of the main buffer for this cpu.\n");
3503 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
3504 "# Must use main snapshot file to allocate.\n");
3506 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3507 "# (Doesn't have to be '2' works with any number that\n"
3508 "# is not a '0' or '1')\n");
3511 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
3513 if (iter
->tr
->allocated_snapshot
)
3514 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
3516 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
3518 seq_puts(m
, "# Snapshot commands:\n");
3519 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
3520 show_snapshot_main_help(m
);
3522 show_snapshot_percpu_help(m
);
3525 /* Should never be called */
3526 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
3529 static int s_show(struct seq_file
*m
, void *v
)
3531 struct trace_iterator
*iter
= v
;
3534 if (iter
->ent
== NULL
) {
3536 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
3538 test_ftrace_alive(m
);
3540 if (iter
->snapshot
&& trace_empty(iter
))
3541 print_snapshot_help(m
, iter
);
3542 else if (iter
->trace
&& iter
->trace
->print_header
)
3543 iter
->trace
->print_header(m
);
3545 trace_default_header(m
);
3547 } else if (iter
->leftover
) {
3549 * If we filled the seq_file buffer earlier, we
3550 * want to just show it now.
3552 ret
= trace_print_seq(m
, &iter
->seq
);
3554 /* ret should this time be zero, but you never know */
3555 iter
->leftover
= ret
;
3558 print_trace_line(iter
);
3559 ret
= trace_print_seq(m
, &iter
->seq
);
3561 * If we overflow the seq_file buffer, then it will
3562 * ask us for this data again at start up.
3564 * ret is 0 if seq_file write succeeded.
3567 iter
->leftover
= ret
;
3574 * Should be used after trace_array_get(), trace_types_lock
3575 * ensures that i_cdev was already initialized.
3577 static inline int tracing_get_cpu(struct inode
*inode
)
3579 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
3580 return (long)inode
->i_cdev
- 1;
3581 return RING_BUFFER_ALL_CPUS
;
3584 static const struct seq_operations tracer_seq_ops
= {
3591 static struct trace_iterator
*
3592 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
3594 struct trace_array
*tr
= inode
->i_private
;
3595 struct trace_iterator
*iter
;
3598 if (tracing_disabled
)
3599 return ERR_PTR(-ENODEV
);
3601 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
3603 return ERR_PTR(-ENOMEM
);
3605 iter
->buffer_iter
= kcalloc(nr_cpu_ids
, sizeof(*iter
->buffer_iter
),
3607 if (!iter
->buffer_iter
)
3611 * We make a copy of the current tracer to avoid concurrent
3612 * changes on it while we are reading.
3614 mutex_lock(&trace_types_lock
);
3615 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
3619 *iter
->trace
= *tr
->current_trace
;
3621 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
3626 #ifdef CONFIG_TRACER_MAX_TRACE
3627 /* Currently only the top directory has a snapshot */
3628 if (tr
->current_trace
->print_max
|| snapshot
)
3629 iter
->trace_buffer
= &tr
->max_buffer
;
3632 iter
->trace_buffer
= &tr
->trace_buffer
;
3633 iter
->snapshot
= snapshot
;
3635 iter
->cpu_file
= tracing_get_cpu(inode
);
3636 mutex_init(&iter
->mutex
);
3638 /* Notify the tracer early; before we stop tracing. */
3639 if (iter
->trace
&& iter
->trace
->open
)
3640 iter
->trace
->open(iter
);
3642 /* Annotate start of buffers if we had overruns */
3643 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
3644 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
3646 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3647 if (trace_clocks
[tr
->clock_id
].in_ns
)
3648 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
3650 /* stop the trace while dumping if we are not opening "snapshot" */
3651 if (!iter
->snapshot
)
3652 tracing_stop_tr(tr
);
3654 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
3655 for_each_tracing_cpu(cpu
) {
3656 iter
->buffer_iter
[cpu
] =
3657 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3659 ring_buffer_read_prepare_sync();
3660 for_each_tracing_cpu(cpu
) {
3661 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3662 tracing_iter_reset(iter
, cpu
);
3665 cpu
= iter
->cpu_file
;
3666 iter
->buffer_iter
[cpu
] =
3667 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3668 ring_buffer_read_prepare_sync();
3669 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3670 tracing_iter_reset(iter
, cpu
);
3673 mutex_unlock(&trace_types_lock
);
3678 mutex_unlock(&trace_types_lock
);
3680 kfree(iter
->buffer_iter
);
3682 seq_release_private(inode
, file
);
3683 return ERR_PTR(-ENOMEM
);
3686 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
3688 if (tracing_disabled
)
3691 filp
->private_data
= inode
->i_private
;
3695 bool tracing_is_disabled(void)
3697 return (tracing_disabled
) ? true: false;
3701 * Open and update trace_array ref count.
3702 * Must have the current trace_array passed to it.
3704 static int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
3706 struct trace_array
*tr
= inode
->i_private
;
3708 if (tracing_disabled
)
3711 if (trace_array_get(tr
) < 0)
3714 filp
->private_data
= inode
->i_private
;
3719 static int tracing_release(struct inode
*inode
, struct file
*file
)
3721 struct trace_array
*tr
= inode
->i_private
;
3722 struct seq_file
*m
= file
->private_data
;
3723 struct trace_iterator
*iter
;
3726 if (!(file
->f_mode
& FMODE_READ
)) {
3727 trace_array_put(tr
);
3731 /* Writes do not use seq_file */
3733 mutex_lock(&trace_types_lock
);
3735 for_each_tracing_cpu(cpu
) {
3736 if (iter
->buffer_iter
[cpu
])
3737 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
3740 if (iter
->trace
&& iter
->trace
->close
)
3741 iter
->trace
->close(iter
);
3743 if (!iter
->snapshot
)
3744 /* reenable tracing if it was previously enabled */
3745 tracing_start_tr(tr
);
3747 __trace_array_put(tr
);
3749 mutex_unlock(&trace_types_lock
);
3751 mutex_destroy(&iter
->mutex
);
3752 free_cpumask_var(iter
->started
);
3754 kfree(iter
->buffer_iter
);
3755 seq_release_private(inode
, file
);
3760 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
3762 struct trace_array
*tr
= inode
->i_private
;
3764 trace_array_put(tr
);
3768 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
3770 struct trace_array
*tr
= inode
->i_private
;
3772 trace_array_put(tr
);
3774 return single_release(inode
, file
);
3777 static int tracing_open(struct inode
*inode
, struct file
*file
)
3779 struct trace_array
*tr
= inode
->i_private
;
3780 struct trace_iterator
*iter
;
3783 if (trace_array_get(tr
) < 0)
3786 /* If this file was open for write, then erase contents */
3787 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
3788 int cpu
= tracing_get_cpu(inode
);
3790 if (cpu
== RING_BUFFER_ALL_CPUS
)
3791 tracing_reset_online_cpus(&tr
->trace_buffer
);
3793 tracing_reset(&tr
->trace_buffer
, cpu
);
3796 if (file
->f_mode
& FMODE_READ
) {
3797 iter
= __tracing_open(inode
, file
, false);
3799 ret
= PTR_ERR(iter
);
3800 else if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
3801 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
3805 trace_array_put(tr
);
3811 * Some tracers are not suitable for instance buffers.
3812 * A tracer is always available for the global array (toplevel)
3813 * or if it explicitly states that it is.
3816 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
3818 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
3821 /* Find the next tracer that this trace array may use */
3822 static struct tracer
*
3823 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
3825 while (t
&& !trace_ok_for_array(t
, tr
))
3832 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3834 struct trace_array
*tr
= m
->private;
3835 struct tracer
*t
= v
;
3840 t
= get_tracer_for_array(tr
, t
->next
);
3845 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
3847 struct trace_array
*tr
= m
->private;
3851 mutex_lock(&trace_types_lock
);
3853 t
= get_tracer_for_array(tr
, trace_types
);
3854 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
3860 static void t_stop(struct seq_file
*m
, void *p
)
3862 mutex_unlock(&trace_types_lock
);
3865 static int t_show(struct seq_file
*m
, void *v
)
3867 struct tracer
*t
= v
;
3872 seq_puts(m
, t
->name
);
3881 static const struct seq_operations show_traces_seq_ops
= {
3888 static int show_traces_open(struct inode
*inode
, struct file
*file
)
3890 struct trace_array
*tr
= inode
->i_private
;
3894 if (tracing_disabled
)
3897 ret
= seq_open(file
, &show_traces_seq_ops
);
3901 m
= file
->private_data
;
3908 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
3909 size_t count
, loff_t
*ppos
)
3914 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
3918 if (file
->f_mode
& FMODE_READ
)
3919 ret
= seq_lseek(file
, offset
, whence
);
3921 file
->f_pos
= ret
= 0;
3926 static const struct file_operations tracing_fops
= {
3927 .open
= tracing_open
,
3929 .write
= tracing_write_stub
,
3930 .llseek
= tracing_lseek
,
3931 .release
= tracing_release
,
3934 static const struct file_operations show_traces_fops
= {
3935 .open
= show_traces_open
,
3937 .release
= seq_release
,
3938 .llseek
= seq_lseek
,
3942 * The tracer itself will not take this lock, but still we want
3943 * to provide a consistent cpumask to user-space:
3945 static DEFINE_MUTEX(tracing_cpumask_update_lock
);
3948 * Temporary storage for the character representation of the
3949 * CPU bitmask (and one more byte for the newline):
3951 static char mask_str
[NR_CPUS
+ 1];
3954 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
3955 size_t count
, loff_t
*ppos
)
3957 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3960 mutex_lock(&tracing_cpumask_update_lock
);
3962 len
= snprintf(mask_str
, count
, "%*pb\n",
3963 cpumask_pr_args(tr
->tracing_cpumask
));
3968 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, NR_CPUS
+1);
3971 mutex_unlock(&tracing_cpumask_update_lock
);
3977 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
3978 size_t count
, loff_t
*ppos
)
3980 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3981 cpumask_var_t tracing_cpumask_new
;
3984 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
3987 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
3991 mutex_lock(&tracing_cpumask_update_lock
);
3993 local_irq_disable();
3994 arch_spin_lock(&tr
->max_lock
);
3995 for_each_tracing_cpu(cpu
) {
3997 * Increase/decrease the disabled counter if we are
3998 * about to flip a bit in the cpumask:
4000 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
4001 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
4002 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
4003 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
4005 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
4006 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
4007 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
4008 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
4011 arch_spin_unlock(&tr
->max_lock
);
4014 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
4016 mutex_unlock(&tracing_cpumask_update_lock
);
4017 free_cpumask_var(tracing_cpumask_new
);
4022 free_cpumask_var(tracing_cpumask_new
);
4027 static const struct file_operations tracing_cpumask_fops
= {
4028 .open
= tracing_open_generic_tr
,
4029 .read
= tracing_cpumask_read
,
4030 .write
= tracing_cpumask_write
,
4031 .release
= tracing_release_generic_tr
,
4032 .llseek
= generic_file_llseek
,
4035 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
4037 struct tracer_opt
*trace_opts
;
4038 struct trace_array
*tr
= m
->private;
4042 mutex_lock(&trace_types_lock
);
4043 tracer_flags
= tr
->current_trace
->flags
->val
;
4044 trace_opts
= tr
->current_trace
->flags
->opts
;
4046 for (i
= 0; trace_options
[i
]; i
++) {
4047 if (tr
->trace_flags
& (1 << i
))
4048 seq_printf(m
, "%s\n", trace_options
[i
]);
4050 seq_printf(m
, "no%s\n", trace_options
[i
]);
4053 for (i
= 0; trace_opts
[i
].name
; i
++) {
4054 if (tracer_flags
& trace_opts
[i
].bit
)
4055 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
4057 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
4059 mutex_unlock(&trace_types_lock
);
4064 static int __set_tracer_option(struct trace_array
*tr
,
4065 struct tracer_flags
*tracer_flags
,
4066 struct tracer_opt
*opts
, int neg
)
4068 struct tracer
*trace
= tracer_flags
->trace
;
4071 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
4076 tracer_flags
->val
&= ~opts
->bit
;
4078 tracer_flags
->val
|= opts
->bit
;
4082 /* Try to assign a tracer specific option */
4083 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
4085 struct tracer
*trace
= tr
->current_trace
;
4086 struct tracer_flags
*tracer_flags
= trace
->flags
;
4087 struct tracer_opt
*opts
= NULL
;
4090 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
4091 opts
= &tracer_flags
->opts
[i
];
4093 if (strcmp(cmp
, opts
->name
) == 0)
4094 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
4100 /* Some tracers require overwrite to stay enabled */
4101 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
4103 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
4109 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
4111 /* do nothing if flag is already set */
4112 if (!!(tr
->trace_flags
& mask
) == !!enabled
)
4115 /* Give the tracer a chance to approve the change */
4116 if (tr
->current_trace
->flag_changed
)
4117 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
4121 tr
->trace_flags
|= mask
;
4123 tr
->trace_flags
&= ~mask
;
4125 if (mask
== TRACE_ITER_RECORD_CMD
)
4126 trace_event_enable_cmd_record(enabled
);
4128 if (mask
== TRACE_ITER_EVENT_FORK
)
4129 trace_event_follow_fork(tr
, enabled
);
4131 if (mask
== TRACE_ITER_OVERWRITE
) {
4132 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
4133 #ifdef CONFIG_TRACER_MAX_TRACE
4134 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
4138 if (mask
== TRACE_ITER_PRINTK
) {
4139 trace_printk_start_stop_comm(enabled
);
4140 trace_printk_control(enabled
);
4146 static int trace_set_options(struct trace_array
*tr
, char *option
)
4152 size_t orig_len
= strlen(option
);
4154 cmp
= strstrip(option
);
4156 if (strncmp(cmp
, "no", 2) == 0) {
4161 mutex_lock(&trace_types_lock
);
4163 for (i
= 0; trace_options
[i
]; i
++) {
4164 if (strcmp(cmp
, trace_options
[i
]) == 0) {
4165 ret
= set_tracer_flag(tr
, 1 << i
, !neg
);
4170 /* If no option could be set, test the specific tracer options */
4171 if (!trace_options
[i
])
4172 ret
= set_tracer_option(tr
, cmp
, neg
);
4174 mutex_unlock(&trace_types_lock
);
4177 * If the first trailing whitespace is replaced with '\0' by strstrip,
4178 * turn it back into a space.
4180 if (orig_len
> strlen(option
))
4181 option
[strlen(option
)] = ' ';
4186 static void __init
apply_trace_boot_options(void)
4188 char *buf
= trace_boot_options_buf
;
4192 option
= strsep(&buf
, ",");
4198 trace_set_options(&global_trace
, option
);
4200 /* Put back the comma to allow this to be called again */
4207 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
4208 size_t cnt
, loff_t
*ppos
)
4210 struct seq_file
*m
= filp
->private_data
;
4211 struct trace_array
*tr
= m
->private;
4215 if (cnt
>= sizeof(buf
))
4218 if (copy_from_user(buf
, ubuf
, cnt
))
4223 ret
= trace_set_options(tr
, buf
);
4232 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
4234 struct trace_array
*tr
= inode
->i_private
;
4237 if (tracing_disabled
)
4240 if (trace_array_get(tr
) < 0)
4243 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
4245 trace_array_put(tr
);
4250 static const struct file_operations tracing_iter_fops
= {
4251 .open
= tracing_trace_options_open
,
4253 .llseek
= seq_lseek
,
4254 .release
= tracing_single_release_tr
,
4255 .write
= tracing_trace_options_write
,
4258 static const char readme_msg
[] =
4259 "tracing mini-HOWTO:\n\n"
4260 "# echo 0 > tracing_on : quick way to disable tracing\n"
4261 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4262 " Important files:\n"
4263 " trace\t\t\t- The static contents of the buffer\n"
4264 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4265 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4266 " current_tracer\t- function and latency tracers\n"
4267 " available_tracers\t- list of configured tracers for current_tracer\n"
4268 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4269 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4270 " trace_clock\t\t-change the clock used to order events\n"
4271 " local: Per cpu clock but may not be synced across CPUs\n"
4272 " global: Synced across CPUs but slows tracing down.\n"
4273 " counter: Not a clock, but just an increment\n"
4274 " uptime: Jiffy counter from time of boot\n"
4275 " perf: Same clock that perf events use\n"
4276 #ifdef CONFIG_X86_64
4277 " x86-tsc: TSC cycle counter\n"
4279 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4280 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4281 " tracing_cpumask\t- Limit which CPUs to trace\n"
4282 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4283 "\t\t\t Remove sub-buffer with rmdir\n"
4284 " trace_options\t\t- Set format or modify how tracing happens\n"
4285 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4286 "\t\t\t option name\n"
4287 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4288 #ifdef CONFIG_DYNAMIC_FTRACE
4289 "\n available_filter_functions - list of functions that can be filtered on\n"
4290 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4291 "\t\t\t functions\n"
4292 "\t accepts: func_full_name or glob-matching-pattern\n"
4293 "\t modules: Can select a group via module\n"
4294 "\t Format: :mod:<module-name>\n"
4295 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4296 "\t triggers: a command to perform when function is hit\n"
4297 "\t Format: <function>:<trigger>[:count]\n"
4298 "\t trigger: traceon, traceoff\n"
4299 "\t\t enable_event:<system>:<event>\n"
4300 "\t\t disable_event:<system>:<event>\n"
4301 #ifdef CONFIG_STACKTRACE
4304 #ifdef CONFIG_TRACER_SNAPSHOT
4309 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4310 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4311 "\t The first one will disable tracing every time do_fault is hit\n"
4312 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4313 "\t The first time do trap is hit and it disables tracing, the\n"
4314 "\t counter will decrement to 2. If tracing is already disabled,\n"
4315 "\t the counter will not decrement. It only decrements when the\n"
4316 "\t trigger did work\n"
4317 "\t To remove trigger without count:\n"
4318 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4319 "\t To remove trigger with a count:\n"
4320 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4321 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4322 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4323 "\t modules: Can select a group via module command :mod:\n"
4324 "\t Does not accept triggers\n"
4325 #endif /* CONFIG_DYNAMIC_FTRACE */
4326 #ifdef CONFIG_FUNCTION_TRACER
4327 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4330 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4331 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4332 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4333 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4335 #ifdef CONFIG_TRACER_SNAPSHOT
4336 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4337 "\t\t\t snapshot buffer. Read the contents for more\n"
4338 "\t\t\t information\n"
4340 #ifdef CONFIG_STACK_TRACER
4341 " stack_trace\t\t- Shows the max stack trace when active\n"
4342 " stack_max_size\t- Shows current max stack size that was traced\n"
4343 "\t\t\t Write into this file to reset the max size (trigger a\n"
4344 "\t\t\t new trace)\n"
4345 #ifdef CONFIG_DYNAMIC_FTRACE
4346 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4349 #endif /* CONFIG_STACK_TRACER */
4350 #ifdef CONFIG_KPROBE_EVENT
4351 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4352 "\t\t\t Write into this file to define/undefine new trace events.\n"
4354 #ifdef CONFIG_UPROBE_EVENT
4355 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4356 "\t\t\t Write into this file to define/undefine new trace events.\n"
4358 #if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
4359 "\t accepts: event-definitions (one definition per line)\n"
4360 "\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
4361 "\t -:[<group>/]<event>\n"
4362 #ifdef CONFIG_KPROBE_EVENT
4363 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4365 #ifdef CONFIG_UPROBE_EVENT
4366 "\t place: <path>:<offset>\n"
4368 "\t args: <name>=fetcharg[:type]\n"
4369 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4370 "\t $stack<index>, $stack, $retval, $comm\n"
4371 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4372 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4374 " events/\t\t- Directory containing all trace event subsystems:\n"
4375 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4376 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4377 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4379 " filter\t\t- If set, only events passing filter are traced\n"
4380 " events/<system>/<event>/\t- Directory containing control files for\n"
4382 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4383 " filter\t\t- If set, only events passing filter are traced\n"
4384 " trigger\t\t- If set, a command to perform when event is hit\n"
4385 "\t Format: <trigger>[:count][if <filter>]\n"
4386 "\t trigger: traceon, traceoff\n"
4387 "\t enable_event:<system>:<event>\n"
4388 "\t disable_event:<system>:<event>\n"
4389 #ifdef CONFIG_HIST_TRIGGERS
4390 "\t enable_hist:<system>:<event>\n"
4391 "\t disable_hist:<system>:<event>\n"
4393 #ifdef CONFIG_STACKTRACE
4396 #ifdef CONFIG_TRACER_SNAPSHOT
4399 #ifdef CONFIG_HIST_TRIGGERS
4400 "\t\t hist (see below)\n"
4402 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4403 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4404 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4405 "\t events/block/block_unplug/trigger\n"
4406 "\t The first disables tracing every time block_unplug is hit.\n"
4407 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4408 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4409 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4410 "\t Like function triggers, the counter is only decremented if it\n"
4411 "\t enabled or disabled tracing.\n"
4412 "\t To remove a trigger without a count:\n"
4413 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4414 "\t To remove a trigger with a count:\n"
4415 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4416 "\t Filters can be ignored when removing a trigger.\n"
4417 #ifdef CONFIG_HIST_TRIGGERS
4418 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4419 "\t Format: hist:keys=<field1[,field2,...]>\n"
4420 "\t [:values=<field1[,field2,...]>]\n"
4421 "\t [:sort=<field1[,field2,...]>]\n"
4422 "\t [:size=#entries]\n"
4423 "\t [:pause][:continue][:clear]\n"
4424 "\t [:name=histname1]\n"
4425 "\t [if <filter>]\n\n"
4426 "\t When a matching event is hit, an entry is added to a hash\n"
4427 "\t table using the key(s) and value(s) named, and the value of a\n"
4428 "\t sum called 'hitcount' is incremented. Keys and values\n"
4429 "\t correspond to fields in the event's format description. Keys\n"
4430 "\t can be any field, or the special string 'stacktrace'.\n"
4431 "\t Compound keys consisting of up to two fields can be specified\n"
4432 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4433 "\t fields. Sort keys consisting of up to two fields can be\n"
4434 "\t specified using the 'sort' keyword. The sort direction can\n"
4435 "\t be modified by appending '.descending' or '.ascending' to a\n"
4436 "\t sort field. The 'size' parameter can be used to specify more\n"
4437 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4438 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4439 "\t its histogram data will be shared with other triggers of the\n"
4440 "\t same name, and trigger hits will update this common data.\n\n"
4441 "\t Reading the 'hist' file for the event will dump the hash\n"
4442 "\t table in its entirety to stdout. If there are multiple hist\n"
4443 "\t triggers attached to an event, there will be a table for each\n"
4444 "\t trigger in the output. The table displayed for a named\n"
4445 "\t trigger will be the same as any other instance having the\n"
4446 "\t same name. The default format used to display a given field\n"
4447 "\t can be modified by appending any of the following modifiers\n"
4448 "\t to the field name, as applicable:\n\n"
4449 "\t .hex display a number as a hex value\n"
4450 "\t .sym display an address as a symbol\n"
4451 "\t .sym-offset display an address as a symbol and offset\n"
4452 "\t .execname display a common_pid as a program name\n"
4453 "\t .syscall display a syscall id as a syscall name\n\n"
4454 "\t .log2 display log2 value rather than raw number\n\n"
4455 "\t The 'pause' parameter can be used to pause an existing hist\n"
4456 "\t trigger or to start a hist trigger but not log any events\n"
4457 "\t until told to do so. 'continue' can be used to start or\n"
4458 "\t restart a paused hist trigger.\n\n"
4459 "\t The 'clear' parameter will clear the contents of a running\n"
4460 "\t hist trigger and leave its current paused/active state\n"
4462 "\t The enable_hist and disable_hist triggers can be used to\n"
4463 "\t have one event conditionally start and stop another event's\n"
4464 "\t already-attached hist trigger. The syntax is analagous to\n"
4465 "\t the enable_event and disable_event triggers.\n"
4470 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
4471 size_t cnt
, loff_t
*ppos
)
4473 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
4474 readme_msg
, strlen(readme_msg
));
4477 static const struct file_operations tracing_readme_fops
= {
4478 .open
= tracing_open_generic
,
4479 .read
= tracing_readme_read
,
4480 .llseek
= generic_file_llseek
,
4483 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4485 unsigned int *ptr
= v
;
4487 if (*pos
|| m
->count
)
4492 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
4494 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
4503 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
4509 arch_spin_lock(&trace_cmdline_lock
);
4511 v
= &savedcmd
->map_cmdline_to_pid
[0];
4513 v
= saved_cmdlines_next(m
, v
, &l
);
4521 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
4523 arch_spin_unlock(&trace_cmdline_lock
);
4527 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
4529 char buf
[TASK_COMM_LEN
];
4530 unsigned int *pid
= v
;
4532 __trace_find_cmdline(*pid
, buf
);
4533 seq_printf(m
, "%d %s\n", *pid
, buf
);
4537 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
4538 .start
= saved_cmdlines_start
,
4539 .next
= saved_cmdlines_next
,
4540 .stop
= saved_cmdlines_stop
,
4541 .show
= saved_cmdlines_show
,
4544 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
4546 if (tracing_disabled
)
4549 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
4552 static const struct file_operations tracing_saved_cmdlines_fops
= {
4553 .open
= tracing_saved_cmdlines_open
,
4555 .llseek
= seq_lseek
,
4556 .release
= seq_release
,
4560 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
4561 size_t cnt
, loff_t
*ppos
)
4566 arch_spin_lock(&trace_cmdline_lock
);
4567 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
4568 arch_spin_unlock(&trace_cmdline_lock
);
4570 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4573 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
4575 kfree(s
->saved_cmdlines
);
4576 kfree(s
->map_cmdline_to_pid
);
4580 static int tracing_resize_saved_cmdlines(unsigned int val
)
4582 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
4584 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
4588 if (allocate_cmdlines_buffer(val
, s
) < 0) {
4593 arch_spin_lock(&trace_cmdline_lock
);
4594 savedcmd_temp
= savedcmd
;
4596 arch_spin_unlock(&trace_cmdline_lock
);
4597 free_saved_cmdlines_buffer(savedcmd_temp
);
4603 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
4604 size_t cnt
, loff_t
*ppos
)
4609 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4613 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4614 if (!val
|| val
> PID_MAX_DEFAULT
)
4617 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
4626 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
4627 .open
= tracing_open_generic
,
4628 .read
= tracing_saved_cmdlines_size_read
,
4629 .write
= tracing_saved_cmdlines_size_write
,
4632 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4633 static union trace_enum_map_item
*
4634 update_enum_map(union trace_enum_map_item
*ptr
)
4636 if (!ptr
->map
.enum_string
) {
4637 if (ptr
->tail
.next
) {
4638 ptr
= ptr
->tail
.next
;
4639 /* Set ptr to the next real item (skip head) */
4647 static void *enum_map_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4649 union trace_enum_map_item
*ptr
= v
;
4652 * Paranoid! If ptr points to end, we don't want to increment past it.
4653 * This really should never happen.
4655 ptr
= update_enum_map(ptr
);
4656 if (WARN_ON_ONCE(!ptr
))
4663 ptr
= update_enum_map(ptr
);
4668 static void *enum_map_start(struct seq_file
*m
, loff_t
*pos
)
4670 union trace_enum_map_item
*v
;
4673 mutex_lock(&trace_enum_mutex
);
4675 v
= trace_enum_maps
;
4679 while (v
&& l
< *pos
) {
4680 v
= enum_map_next(m
, v
, &l
);
4686 static void enum_map_stop(struct seq_file
*m
, void *v
)
4688 mutex_unlock(&trace_enum_mutex
);
4691 static int enum_map_show(struct seq_file
*m
, void *v
)
4693 union trace_enum_map_item
*ptr
= v
;
4695 seq_printf(m
, "%s %ld (%s)\n",
4696 ptr
->map
.enum_string
, ptr
->map
.enum_value
,
4702 static const struct seq_operations tracing_enum_map_seq_ops
= {
4703 .start
= enum_map_start
,
4704 .next
= enum_map_next
,
4705 .stop
= enum_map_stop
,
4706 .show
= enum_map_show
,
4709 static int tracing_enum_map_open(struct inode
*inode
, struct file
*filp
)
4711 if (tracing_disabled
)
4714 return seq_open(filp
, &tracing_enum_map_seq_ops
);
4717 static const struct file_operations tracing_enum_map_fops
= {
4718 .open
= tracing_enum_map_open
,
4720 .llseek
= seq_lseek
,
4721 .release
= seq_release
,
4724 static inline union trace_enum_map_item
*
4725 trace_enum_jmp_to_tail(union trace_enum_map_item
*ptr
)
4727 /* Return tail of array given the head */
4728 return ptr
+ ptr
->head
.length
+ 1;
4732 trace_insert_enum_map_file(struct module
*mod
, struct trace_enum_map
**start
,
4735 struct trace_enum_map
**stop
;
4736 struct trace_enum_map
**map
;
4737 union trace_enum_map_item
*map_array
;
4738 union trace_enum_map_item
*ptr
;
4743 * The trace_enum_maps contains the map plus a head and tail item,
4744 * where the head holds the module and length of array, and the
4745 * tail holds a pointer to the next list.
4747 map_array
= kmalloc(sizeof(*map_array
) * (len
+ 2), GFP_KERNEL
);
4749 pr_warn("Unable to allocate trace enum mapping\n");
4753 mutex_lock(&trace_enum_mutex
);
4755 if (!trace_enum_maps
)
4756 trace_enum_maps
= map_array
;
4758 ptr
= trace_enum_maps
;
4760 ptr
= trace_enum_jmp_to_tail(ptr
);
4761 if (!ptr
->tail
.next
)
4763 ptr
= ptr
->tail
.next
;
4766 ptr
->tail
.next
= map_array
;
4768 map_array
->head
.mod
= mod
;
4769 map_array
->head
.length
= len
;
4772 for (map
= start
; (unsigned long)map
< (unsigned long)stop
; map
++) {
4773 map_array
->map
= **map
;
4776 memset(map_array
, 0, sizeof(*map_array
));
4778 mutex_unlock(&trace_enum_mutex
);
4781 static void trace_create_enum_file(struct dentry
*d_tracer
)
4783 trace_create_file("enum_map", 0444, d_tracer
,
4784 NULL
, &tracing_enum_map_fops
);
4787 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4788 static inline void trace_create_enum_file(struct dentry
*d_tracer
) { }
4789 static inline void trace_insert_enum_map_file(struct module
*mod
,
4790 struct trace_enum_map
**start
, int len
) { }
4791 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4793 static void trace_insert_enum_map(struct module
*mod
,
4794 struct trace_enum_map
**start
, int len
)
4796 struct trace_enum_map
**map
;
4803 trace_event_enum_update(map
, len
);
4805 trace_insert_enum_map_file(mod
, start
, len
);
4809 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
4810 size_t cnt
, loff_t
*ppos
)
4812 struct trace_array
*tr
= filp
->private_data
;
4813 char buf
[MAX_TRACER_SIZE
+2];
4816 mutex_lock(&trace_types_lock
);
4817 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
4818 mutex_unlock(&trace_types_lock
);
4820 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4823 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
4825 tracing_reset_online_cpus(&tr
->trace_buffer
);
4829 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
4833 for_each_tracing_cpu(cpu
)
4834 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
4837 #ifdef CONFIG_TRACER_MAX_TRACE
4838 /* resize @tr's buffer to the size of @size_tr's entries */
4839 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
4840 struct trace_buffer
*size_buf
, int cpu_id
)
4844 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
4845 for_each_tracing_cpu(cpu
) {
4846 ret
= ring_buffer_resize(trace_buf
->buffer
,
4847 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
4850 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
4851 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
4854 ret
= ring_buffer_resize(trace_buf
->buffer
,
4855 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
4857 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
4858 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
4863 #endif /* CONFIG_TRACER_MAX_TRACE */
4865 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
4866 unsigned long size
, int cpu
)
4871 * If kernel or user changes the size of the ring buffer
4872 * we use the size that was given, and we can forget about
4873 * expanding it later.
4875 ring_buffer_expanded
= true;
4877 /* May be called before buffers are initialized */
4878 if (!tr
->trace_buffer
.buffer
)
4881 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
4885 #ifdef CONFIG_TRACER_MAX_TRACE
4886 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
4887 !tr
->current_trace
->use_max_tr
)
4890 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
4892 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
4893 &tr
->trace_buffer
, cpu
);
4896 * AARGH! We are left with different
4897 * size max buffer!!!!
4898 * The max buffer is our "snapshot" buffer.
4899 * When a tracer needs a snapshot (one of the
4900 * latency tracers), it swaps the max buffer
4901 * with the saved snap shot. We succeeded to
4902 * update the size of the main buffer, but failed to
4903 * update the size of the max buffer. But when we tried
4904 * to reset the main buffer to the original size, we
4905 * failed there too. This is very unlikely to
4906 * happen, but if it does, warn and kill all
4910 tracing_disabled
= 1;
4915 if (cpu
== RING_BUFFER_ALL_CPUS
)
4916 set_buffer_entries(&tr
->max_buffer
, size
);
4918 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
4921 #endif /* CONFIG_TRACER_MAX_TRACE */
4923 if (cpu
== RING_BUFFER_ALL_CPUS
)
4924 set_buffer_entries(&tr
->trace_buffer
, size
);
4926 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
4931 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
4932 unsigned long size
, int cpu_id
)
4936 mutex_lock(&trace_types_lock
);
4938 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
4939 /* make sure, this cpu is enabled in the mask */
4940 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
4946 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
4951 mutex_unlock(&trace_types_lock
);
4958 * tracing_update_buffers - used by tracing facility to expand ring buffers
4960 * To save on memory when the tracing is never used on a system with it
4961 * configured in. The ring buffers are set to a minimum size. But once
4962 * a user starts to use the tracing facility, then they need to grow
4963 * to their default size.
4965 * This function is to be called when a tracer is about to be used.
4967 int tracing_update_buffers(void)
4971 mutex_lock(&trace_types_lock
);
4972 if (!ring_buffer_expanded
)
4973 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
4974 RING_BUFFER_ALL_CPUS
);
4975 mutex_unlock(&trace_types_lock
);
4980 struct trace_option_dentry
;
4983 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
4986 * Used to clear out the tracer before deletion of an instance.
4987 * Must have trace_types_lock held.
4989 static void tracing_set_nop(struct trace_array
*tr
)
4991 if (tr
->current_trace
== &nop_trace
)
4994 tr
->current_trace
->enabled
--;
4996 if (tr
->current_trace
->reset
)
4997 tr
->current_trace
->reset(tr
);
4999 tr
->current_trace
= &nop_trace
;
5002 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
)
5004 /* Only enable if the directory has been created already. */
5008 create_trace_option_files(tr
, t
);
5011 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
5014 #ifdef CONFIG_TRACER_MAX_TRACE
5019 mutex_lock(&trace_types_lock
);
5021 if (!ring_buffer_expanded
) {
5022 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
5023 RING_BUFFER_ALL_CPUS
);
5029 for (t
= trace_types
; t
; t
= t
->next
) {
5030 if (strcmp(t
->name
, buf
) == 0)
5037 if (t
== tr
->current_trace
)
5040 /* Some tracers are only allowed for the top level buffer */
5041 if (!trace_ok_for_array(t
, tr
)) {
5046 /* If trace pipe files are being read, we can't change the tracer */
5047 if (tr
->current_trace
->ref
) {
5052 trace_branch_disable();
5054 tr
->current_trace
->enabled
--;
5056 if (tr
->current_trace
->reset
)
5057 tr
->current_trace
->reset(tr
);
5059 /* Current trace needs to be nop_trace before synchronize_sched */
5060 tr
->current_trace
= &nop_trace
;
5062 #ifdef CONFIG_TRACER_MAX_TRACE
5063 had_max_tr
= tr
->allocated_snapshot
;
5065 if (had_max_tr
&& !t
->use_max_tr
) {
5067 * We need to make sure that the update_max_tr sees that
5068 * current_trace changed to nop_trace to keep it from
5069 * swapping the buffers after we resize it.
5070 * The update_max_tr is called from interrupts disabled
5071 * so a synchronized_sched() is sufficient.
5073 synchronize_sched();
5078 #ifdef CONFIG_TRACER_MAX_TRACE
5079 if (t
->use_max_tr
&& !had_max_tr
) {
5080 ret
= alloc_snapshot(tr
);
5087 ret
= tracer_init(t
, tr
);
5092 tr
->current_trace
= t
;
5093 tr
->current_trace
->enabled
++;
5094 trace_branch_enable(tr
);
5096 mutex_unlock(&trace_types_lock
);
5102 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
5103 size_t cnt
, loff_t
*ppos
)
5105 struct trace_array
*tr
= filp
->private_data
;
5106 char buf
[MAX_TRACER_SIZE
+1];
5113 if (cnt
> MAX_TRACER_SIZE
)
5114 cnt
= MAX_TRACER_SIZE
;
5116 if (copy_from_user(buf
, ubuf
, cnt
))
5121 /* strip ending whitespace. */
5122 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
5125 err
= tracing_set_tracer(tr
, buf
);
5135 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
5136 size_t cnt
, loff_t
*ppos
)
5141 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
5142 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
5143 if (r
> sizeof(buf
))
5145 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5149 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
5150 size_t cnt
, loff_t
*ppos
)
5155 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5165 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
5166 size_t cnt
, loff_t
*ppos
)
5168 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
5172 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
5173 size_t cnt
, loff_t
*ppos
)
5175 struct trace_array
*tr
= filp
->private_data
;
5178 mutex_lock(&trace_types_lock
);
5179 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
5183 if (tr
->current_trace
->update_thresh
) {
5184 ret
= tr
->current_trace
->update_thresh(tr
);
5191 mutex_unlock(&trace_types_lock
);
5196 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5199 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
5200 size_t cnt
, loff_t
*ppos
)
5202 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
5206 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
5207 size_t cnt
, loff_t
*ppos
)
5209 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
5214 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
5216 struct trace_array
*tr
= inode
->i_private
;
5217 struct trace_iterator
*iter
;
5220 if (tracing_disabled
)
5223 if (trace_array_get(tr
) < 0)
5226 mutex_lock(&trace_types_lock
);
5228 /* create a buffer to store the information to pass to userspace */
5229 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5232 __trace_array_put(tr
);
5236 trace_seq_init(&iter
->seq
);
5237 iter
->trace
= tr
->current_trace
;
5239 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
5244 /* trace pipe does not show start of buffer */
5245 cpumask_setall(iter
->started
);
5247 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
5248 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
5250 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5251 if (trace_clocks
[tr
->clock_id
].in_ns
)
5252 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
5255 iter
->trace_buffer
= &tr
->trace_buffer
;
5256 iter
->cpu_file
= tracing_get_cpu(inode
);
5257 mutex_init(&iter
->mutex
);
5258 filp
->private_data
= iter
;
5260 if (iter
->trace
->pipe_open
)
5261 iter
->trace
->pipe_open(iter
);
5263 nonseekable_open(inode
, filp
);
5265 tr
->current_trace
->ref
++;
5267 mutex_unlock(&trace_types_lock
);
5273 __trace_array_put(tr
);
5274 mutex_unlock(&trace_types_lock
);
5278 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
5280 struct trace_iterator
*iter
= file
->private_data
;
5281 struct trace_array
*tr
= inode
->i_private
;
5283 mutex_lock(&trace_types_lock
);
5285 tr
->current_trace
->ref
--;
5287 if (iter
->trace
->pipe_close
)
5288 iter
->trace
->pipe_close(iter
);
5290 mutex_unlock(&trace_types_lock
);
5292 free_cpumask_var(iter
->started
);
5293 mutex_destroy(&iter
->mutex
);
5296 trace_array_put(tr
);
5302 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
5304 struct trace_array
*tr
= iter
->tr
;
5306 /* Iterators are static, they should be filled or empty */
5307 if (trace_buffer_iter(iter
, iter
->cpu_file
))
5308 return POLLIN
| POLLRDNORM
;
5310 if (tr
->trace_flags
& TRACE_ITER_BLOCK
)
5312 * Always select as readable when in blocking mode
5314 return POLLIN
| POLLRDNORM
;
5316 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
5321 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
5323 struct trace_iterator
*iter
= filp
->private_data
;
5325 return trace_poll(iter
, filp
, poll_table
);
5328 /* Must be called with iter->mutex held. */
5329 static int tracing_wait_pipe(struct file
*filp
)
5331 struct trace_iterator
*iter
= filp
->private_data
;
5334 while (trace_empty(iter
)) {
5336 if ((filp
->f_flags
& O_NONBLOCK
)) {
5341 * We block until we read something and tracing is disabled.
5342 * We still block if tracing is disabled, but we have never
5343 * read anything. This allows a user to cat this file, and
5344 * then enable tracing. But after we have read something,
5345 * we give an EOF when tracing is again disabled.
5347 * iter->pos will be 0 if we haven't read anything.
5349 if (!tracing_is_on() && iter
->pos
)
5352 mutex_unlock(&iter
->mutex
);
5354 ret
= wait_on_pipe(iter
, false);
5356 mutex_lock(&iter
->mutex
);
5369 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
5370 size_t cnt
, loff_t
*ppos
)
5372 struct trace_iterator
*iter
= filp
->private_data
;
5376 * Avoid more than one consumer on a single file descriptor
5377 * This is just a matter of traces coherency, the ring buffer itself
5380 mutex_lock(&iter
->mutex
);
5382 /* return any leftover data */
5383 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
5387 trace_seq_init(&iter
->seq
);
5389 if (iter
->trace
->read
) {
5390 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
5396 sret
= tracing_wait_pipe(filp
);
5400 /* stop when tracing is finished */
5401 if (trace_empty(iter
)) {
5406 if (cnt
>= PAGE_SIZE
)
5407 cnt
= PAGE_SIZE
- 1;
5409 /* reset all but tr, trace, and overruns */
5410 memset(&iter
->seq
, 0,
5411 sizeof(struct trace_iterator
) -
5412 offsetof(struct trace_iterator
, seq
));
5413 cpumask_clear(iter
->started
);
5416 trace_event_read_lock();
5417 trace_access_lock(iter
->cpu_file
);
5418 while (trace_find_next_entry_inc(iter
) != NULL
) {
5419 enum print_line_t ret
;
5420 int save_len
= iter
->seq
.seq
.len
;
5422 ret
= print_trace_line(iter
);
5423 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
5424 /* don't print partial lines */
5425 iter
->seq
.seq
.len
= save_len
;
5428 if (ret
!= TRACE_TYPE_NO_CONSUME
)
5429 trace_consume(iter
);
5431 if (trace_seq_used(&iter
->seq
) >= cnt
)
5435 * Setting the full flag means we reached the trace_seq buffer
5436 * size and we should leave by partial output condition above.
5437 * One of the trace_seq_* functions is not used properly.
5439 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
5442 trace_access_unlock(iter
->cpu_file
);
5443 trace_event_read_unlock();
5445 /* Now copy what we have to the user */
5446 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
5447 if (iter
->seq
.seq
.readpos
>= trace_seq_used(&iter
->seq
))
5448 trace_seq_init(&iter
->seq
);
5451 * If there was nothing to send to user, in spite of consuming trace
5452 * entries, go back to wait for more entries.
5458 mutex_unlock(&iter
->mutex
);
5463 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
5466 __free_page(spd
->pages
[idx
]);
5469 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
5471 .confirm
= generic_pipe_buf_confirm
,
5472 .release
= generic_pipe_buf_release
,
5473 .steal
= generic_pipe_buf_steal
,
5474 .get
= generic_pipe_buf_get
,
5478 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
5484 /* Seq buffer is page-sized, exactly what we need. */
5486 save_len
= iter
->seq
.seq
.len
;
5487 ret
= print_trace_line(iter
);
5489 if (trace_seq_has_overflowed(&iter
->seq
)) {
5490 iter
->seq
.seq
.len
= save_len
;
5495 * This should not be hit, because it should only
5496 * be set if the iter->seq overflowed. But check it
5497 * anyway to be safe.
5499 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
5500 iter
->seq
.seq
.len
= save_len
;
5504 count
= trace_seq_used(&iter
->seq
) - save_len
;
5507 iter
->seq
.seq
.len
= save_len
;
5511 if (ret
!= TRACE_TYPE_NO_CONSUME
)
5512 trace_consume(iter
);
5514 if (!trace_find_next_entry_inc(iter
)) {
5524 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
5526 struct pipe_inode_info
*pipe
,
5530 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
5531 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
5532 struct trace_iterator
*iter
= filp
->private_data
;
5533 struct splice_pipe_desc spd
= {
5535 .partial
= partial_def
,
5536 .nr_pages
= 0, /* This gets updated below. */
5537 .nr_pages_max
= PIPE_DEF_BUFFERS
,
5539 .ops
= &tracing_pipe_buf_ops
,
5540 .spd_release
= tracing_spd_release_pipe
,
5546 if (splice_grow_spd(pipe
, &spd
))
5549 mutex_lock(&iter
->mutex
);
5551 if (iter
->trace
->splice_read
) {
5552 ret
= iter
->trace
->splice_read(iter
, filp
,
5553 ppos
, pipe
, len
, flags
);
5558 ret
= tracing_wait_pipe(filp
);
5562 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
5567 trace_event_read_lock();
5568 trace_access_lock(iter
->cpu_file
);
5570 /* Fill as many pages as possible. */
5571 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
5572 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
5576 rem
= tracing_fill_pipe_page(rem
, iter
);
5578 /* Copy the data into the page, so we can start over. */
5579 ret
= trace_seq_to_buffer(&iter
->seq
,
5580 page_address(spd
.pages
[i
]),
5581 trace_seq_used(&iter
->seq
));
5583 __free_page(spd
.pages
[i
]);
5586 spd
.partial
[i
].offset
= 0;
5587 spd
.partial
[i
].len
= trace_seq_used(&iter
->seq
);
5589 trace_seq_init(&iter
->seq
);
5592 trace_access_unlock(iter
->cpu_file
);
5593 trace_event_read_unlock();
5594 mutex_unlock(&iter
->mutex
);
5599 ret
= splice_to_pipe(pipe
, &spd
);
5603 splice_shrink_spd(&spd
);
5607 mutex_unlock(&iter
->mutex
);
5612 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
5613 size_t cnt
, loff_t
*ppos
)
5615 struct inode
*inode
= file_inode(filp
);
5616 struct trace_array
*tr
= inode
->i_private
;
5617 int cpu
= tracing_get_cpu(inode
);
5622 mutex_lock(&trace_types_lock
);
5624 if (cpu
== RING_BUFFER_ALL_CPUS
) {
5625 int cpu
, buf_size_same
;
5630 /* check if all cpu sizes are same */
5631 for_each_tracing_cpu(cpu
) {
5632 /* fill in the size from first enabled cpu */
5634 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
5635 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
5641 if (buf_size_same
) {
5642 if (!ring_buffer_expanded
)
5643 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
5645 trace_buf_size
>> 10);
5647 r
= sprintf(buf
, "%lu\n", size
>> 10);
5649 r
= sprintf(buf
, "X\n");
5651 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
5653 mutex_unlock(&trace_types_lock
);
5655 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5660 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
5661 size_t cnt
, loff_t
*ppos
)
5663 struct inode
*inode
= file_inode(filp
);
5664 struct trace_array
*tr
= inode
->i_private
;
5668 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5672 /* must have at least 1 entry */
5676 /* value is in KB */
5678 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
5688 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
5689 size_t cnt
, loff_t
*ppos
)
5691 struct trace_array
*tr
= filp
->private_data
;
5694 unsigned long size
= 0, expanded_size
= 0;
5696 mutex_lock(&trace_types_lock
);
5697 for_each_tracing_cpu(cpu
) {
5698 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
5699 if (!ring_buffer_expanded
)
5700 expanded_size
+= trace_buf_size
>> 10;
5702 if (ring_buffer_expanded
)
5703 r
= sprintf(buf
, "%lu\n", size
);
5705 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
5706 mutex_unlock(&trace_types_lock
);
5708 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5712 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
5713 size_t cnt
, loff_t
*ppos
)
5716 * There is no need to read what the user has written, this function
5717 * is just to make sure that there is no error when "echo" is used
5726 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
5728 struct trace_array
*tr
= inode
->i_private
;
5730 /* disable tracing ? */
5731 if (tr
->trace_flags
& TRACE_ITER_STOP_ON_FREE
)
5732 tracer_tracing_off(tr
);
5733 /* resize the ring buffer to 0 */
5734 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
5736 trace_array_put(tr
);
5741 static inline int lock_user_pages(const char __user
*ubuf
, size_t cnt
,
5742 struct page
**pages
, void **map_page
,
5745 unsigned long addr
= (unsigned long)ubuf
;
5751 * Userspace is injecting traces into the kernel trace buffer.
5752 * We want to be as non intrusive as possible.
5753 * To do so, we do not want to allocate any special buffers
5754 * or take any locks, but instead write the userspace data
5755 * straight into the ring buffer.
5757 * First we need to pin the userspace buffer into memory,
5758 * which, most likely it is, because it just referenced it.
5759 * But there's no guarantee that it is. By using get_user_pages_fast()
5760 * and kmap_atomic/kunmap_atomic() we can get access to the
5761 * pages directly. We then write the data directly into the
5765 /* check if we cross pages */
5766 if ((addr
& PAGE_MASK
) != ((addr
+ cnt
) & PAGE_MASK
))
5769 *offset
= addr
& (PAGE_SIZE
- 1);
5772 ret
= get_user_pages_fast(addr
, nr_pages
, 0, pages
);
5773 if (ret
< nr_pages
) {
5775 put_page(pages
[ret
]);
5779 for (i
= 0; i
< nr_pages
; i
++)
5780 map_page
[i
] = kmap_atomic(pages
[i
]);
5785 static inline void unlock_user_pages(struct page
**pages
,
5786 void **map_page
, int nr_pages
)
5790 for (i
= nr_pages
- 1; i
>= 0; i
--) {
5791 kunmap_atomic(map_page
[i
]);
5797 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
5798 size_t cnt
, loff_t
*fpos
)
5800 struct trace_array
*tr
= filp
->private_data
;
5801 struct ring_buffer_event
*event
;
5802 struct ring_buffer
*buffer
;
5803 struct print_entry
*entry
;
5804 unsigned long irq_flags
;
5805 struct page
*pages
[2];
5813 if (tracing_disabled
)
5816 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
5819 if (cnt
> TRACE_BUF_SIZE
)
5820 cnt
= TRACE_BUF_SIZE
;
5822 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
5824 nr_pages
= lock_user_pages(ubuf
, cnt
, pages
, map_page
, &offset
);
5828 local_save_flags(irq_flags
);
5829 size
= sizeof(*entry
) + cnt
+ 2; /* possible \n added */
5830 buffer
= tr
->trace_buffer
.buffer
;
5831 event
= __trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
5832 irq_flags
, preempt_count());
5834 /* Ring buffer disabled, return as if not open for write */
5839 entry
= ring_buffer_event_data(event
);
5840 entry
->ip
= _THIS_IP_
;
5842 if (nr_pages
== 2) {
5843 len
= PAGE_SIZE
- offset
;
5844 memcpy(&entry
->buf
, map_page
[0] + offset
, len
);
5845 memcpy(&entry
->buf
[len
], map_page
[1], cnt
- len
);
5847 memcpy(&entry
->buf
, map_page
[0] + offset
, cnt
);
5849 if (entry
->buf
[cnt
- 1] != '\n') {
5850 entry
->buf
[cnt
] = '\n';
5851 entry
->buf
[cnt
+ 1] = '\0';
5853 entry
->buf
[cnt
] = '\0';
5855 __buffer_unlock_commit(buffer
, event
);
5862 unlock_user_pages(pages
, map_page
, nr_pages
);
5867 /* Limit it for now to 3K (including tag) */
5868 #define RAW_DATA_MAX_SIZE (1024*3)
5871 tracing_mark_raw_write(struct file
*filp
, const char __user
*ubuf
,
5872 size_t cnt
, loff_t
*fpos
)
5874 struct trace_array
*tr
= filp
->private_data
;
5875 struct ring_buffer_event
*event
;
5876 struct ring_buffer
*buffer
;
5877 struct raw_data_entry
*entry
;
5878 unsigned long irq_flags
;
5879 struct page
*pages
[2];
5887 if (tracing_disabled
)
5890 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
5893 /* The marker must at least have a tag id */
5894 if (cnt
< sizeof(unsigned int) || cnt
> RAW_DATA_MAX_SIZE
)
5897 if (cnt
> TRACE_BUF_SIZE
)
5898 cnt
= TRACE_BUF_SIZE
;
5900 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
5902 nr_pages
= lock_user_pages(ubuf
, cnt
, pages
, map_page
, &offset
);
5906 local_save_flags(irq_flags
);
5907 size
= sizeof(*entry
) + cnt
;
5908 buffer
= tr
->trace_buffer
.buffer
;
5909 event
= __trace_buffer_lock_reserve(buffer
, TRACE_RAW_DATA
, size
,
5910 irq_flags
, preempt_count());
5912 /* Ring buffer disabled, return as if not open for write */
5917 entry
= ring_buffer_event_data(event
);
5919 if (nr_pages
== 2) {
5920 len
= PAGE_SIZE
- offset
;
5921 memcpy(&entry
->id
, map_page
[0] + offset
, len
);
5922 memcpy(((char *)&entry
->id
) + len
, map_page
[1], cnt
- len
);
5924 memcpy(&entry
->id
, map_page
[0] + offset
, cnt
);
5926 __buffer_unlock_commit(buffer
, event
);
5933 unlock_user_pages(pages
, map_page
, nr_pages
);
5938 static int tracing_clock_show(struct seq_file
*m
, void *v
)
5940 struct trace_array
*tr
= m
->private;
5943 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
5945 "%s%s%s%s", i
? " " : "",
5946 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
5947 i
== tr
->clock_id
? "]" : "");
5953 static int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
5957 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
5958 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
5961 if (i
== ARRAY_SIZE(trace_clocks
))
5964 mutex_lock(&trace_types_lock
);
5968 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
5971 * New clock may not be consistent with the previous clock.
5972 * Reset the buffer so that it doesn't have incomparable timestamps.
5974 tracing_reset_online_cpus(&tr
->trace_buffer
);
5976 #ifdef CONFIG_TRACER_MAX_TRACE
5977 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
&& tr
->max_buffer
.buffer
)
5978 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
5979 tracing_reset_online_cpus(&tr
->max_buffer
);
5982 mutex_unlock(&trace_types_lock
);
5987 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
5988 size_t cnt
, loff_t
*fpos
)
5990 struct seq_file
*m
= filp
->private_data
;
5991 struct trace_array
*tr
= m
->private;
5993 const char *clockstr
;
5996 if (cnt
>= sizeof(buf
))
5999 if (copy_from_user(buf
, ubuf
, cnt
))
6004 clockstr
= strstrip(buf
);
6006 ret
= tracing_set_clock(tr
, clockstr
);
6015 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
6017 struct trace_array
*tr
= inode
->i_private
;
6020 if (tracing_disabled
)
6023 if (trace_array_get(tr
))
6026 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
6028 trace_array_put(tr
);
6033 struct ftrace_buffer_info
{
6034 struct trace_iterator iter
;
6039 #ifdef CONFIG_TRACER_SNAPSHOT
6040 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
6042 struct trace_array
*tr
= inode
->i_private
;
6043 struct trace_iterator
*iter
;
6047 if (trace_array_get(tr
) < 0)
6050 if (file
->f_mode
& FMODE_READ
) {
6051 iter
= __tracing_open(inode
, file
, true);
6053 ret
= PTR_ERR(iter
);
6055 /* Writes still need the seq_file to hold the private data */
6057 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
6060 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
6068 iter
->trace_buffer
= &tr
->max_buffer
;
6069 iter
->cpu_file
= tracing_get_cpu(inode
);
6071 file
->private_data
= m
;
6075 trace_array_put(tr
);
6081 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6084 struct seq_file
*m
= filp
->private_data
;
6085 struct trace_iterator
*iter
= m
->private;
6086 struct trace_array
*tr
= iter
->tr
;
6090 ret
= tracing_update_buffers();
6094 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6098 mutex_lock(&trace_types_lock
);
6100 if (tr
->current_trace
->use_max_tr
) {
6107 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
6111 if (tr
->allocated_snapshot
)
6115 /* Only allow per-cpu swap if the ring buffer supports it */
6116 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6117 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
6122 if (!tr
->allocated_snapshot
) {
6123 ret
= alloc_snapshot(tr
);
6127 local_irq_disable();
6128 /* Now, we're going to swap */
6129 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
6130 update_max_tr(tr
, current
, smp_processor_id());
6132 update_max_tr_single(tr
, current
, iter
->cpu_file
);
6136 if (tr
->allocated_snapshot
) {
6137 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
6138 tracing_reset_online_cpus(&tr
->max_buffer
);
6140 tracing_reset(&tr
->max_buffer
, iter
->cpu_file
);
6150 mutex_unlock(&trace_types_lock
);
6154 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
6156 struct seq_file
*m
= file
->private_data
;
6159 ret
= tracing_release(inode
, file
);
6161 if (file
->f_mode
& FMODE_READ
)
6164 /* If write only, the seq_file is just a stub */
6172 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
6173 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
6174 size_t count
, loff_t
*ppos
);
6175 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
6176 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
6177 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
6179 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
6181 struct ftrace_buffer_info
*info
;
6184 ret
= tracing_buffers_open(inode
, filp
);
6188 info
= filp
->private_data
;
6190 if (info
->iter
.trace
->use_max_tr
) {
6191 tracing_buffers_release(inode
, filp
);
6195 info
->iter
.snapshot
= true;
6196 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
6201 #endif /* CONFIG_TRACER_SNAPSHOT */
6204 static const struct file_operations tracing_thresh_fops
= {
6205 .open
= tracing_open_generic
,
6206 .read
= tracing_thresh_read
,
6207 .write
= tracing_thresh_write
,
6208 .llseek
= generic_file_llseek
,
6211 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6212 static const struct file_operations tracing_max_lat_fops
= {
6213 .open
= tracing_open_generic
,
6214 .read
= tracing_max_lat_read
,
6215 .write
= tracing_max_lat_write
,
6216 .llseek
= generic_file_llseek
,
6220 static const struct file_operations set_tracer_fops
= {
6221 .open
= tracing_open_generic
,
6222 .read
= tracing_set_trace_read
,
6223 .write
= tracing_set_trace_write
,
6224 .llseek
= generic_file_llseek
,
6227 static const struct file_operations tracing_pipe_fops
= {
6228 .open
= tracing_open_pipe
,
6229 .poll
= tracing_poll_pipe
,
6230 .read
= tracing_read_pipe
,
6231 .splice_read
= tracing_splice_read_pipe
,
6232 .release
= tracing_release_pipe
,
6233 .llseek
= no_llseek
,
6236 static const struct file_operations tracing_entries_fops
= {
6237 .open
= tracing_open_generic_tr
,
6238 .read
= tracing_entries_read
,
6239 .write
= tracing_entries_write
,
6240 .llseek
= generic_file_llseek
,
6241 .release
= tracing_release_generic_tr
,
6244 static const struct file_operations tracing_total_entries_fops
= {
6245 .open
= tracing_open_generic_tr
,
6246 .read
= tracing_total_entries_read
,
6247 .llseek
= generic_file_llseek
,
6248 .release
= tracing_release_generic_tr
,
6251 static const struct file_operations tracing_free_buffer_fops
= {
6252 .open
= tracing_open_generic_tr
,
6253 .write
= tracing_free_buffer_write
,
6254 .release
= tracing_free_buffer_release
,
6257 static const struct file_operations tracing_mark_fops
= {
6258 .open
= tracing_open_generic_tr
,
6259 .write
= tracing_mark_write
,
6260 .llseek
= generic_file_llseek
,
6261 .release
= tracing_release_generic_tr
,
6264 static const struct file_operations tracing_mark_raw_fops
= {
6265 .open
= tracing_open_generic_tr
,
6266 .write
= tracing_mark_raw_write
,
6267 .llseek
= generic_file_llseek
,
6268 .release
= tracing_release_generic_tr
,
6271 static const struct file_operations trace_clock_fops
= {
6272 .open
= tracing_clock_open
,
6274 .llseek
= seq_lseek
,
6275 .release
= tracing_single_release_tr
,
6276 .write
= tracing_clock_write
,
6279 #ifdef CONFIG_TRACER_SNAPSHOT
6280 static const struct file_operations snapshot_fops
= {
6281 .open
= tracing_snapshot_open
,
6283 .write
= tracing_snapshot_write
,
6284 .llseek
= tracing_lseek
,
6285 .release
= tracing_snapshot_release
,
6288 static const struct file_operations snapshot_raw_fops
= {
6289 .open
= snapshot_raw_open
,
6290 .read
= tracing_buffers_read
,
6291 .release
= tracing_buffers_release
,
6292 .splice_read
= tracing_buffers_splice_read
,
6293 .llseek
= no_llseek
,
6296 #endif /* CONFIG_TRACER_SNAPSHOT */
6298 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
6300 struct trace_array
*tr
= inode
->i_private
;
6301 struct ftrace_buffer_info
*info
;
6304 if (tracing_disabled
)
6307 if (trace_array_get(tr
) < 0)
6310 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
6312 trace_array_put(tr
);
6316 mutex_lock(&trace_types_lock
);
6319 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
6320 info
->iter
.trace
= tr
->current_trace
;
6321 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
6323 /* Force reading ring buffer for first read */
6324 info
->read
= (unsigned int)-1;
6326 filp
->private_data
= info
;
6328 tr
->current_trace
->ref
++;
6330 mutex_unlock(&trace_types_lock
);
6332 ret
= nonseekable_open(inode
, filp
);
6334 trace_array_put(tr
);
6340 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
6342 struct ftrace_buffer_info
*info
= filp
->private_data
;
6343 struct trace_iterator
*iter
= &info
->iter
;
6345 return trace_poll(iter
, filp
, poll_table
);
6349 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
6350 size_t count
, loff_t
*ppos
)
6352 struct ftrace_buffer_info
*info
= filp
->private_data
;
6353 struct trace_iterator
*iter
= &info
->iter
;
6360 #ifdef CONFIG_TRACER_MAX_TRACE
6361 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
6366 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
6371 /* Do we have previous read data to read? */
6372 if (info
->read
< PAGE_SIZE
)
6376 trace_access_lock(iter
->cpu_file
);
6377 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
6381 trace_access_unlock(iter
->cpu_file
);
6384 if (trace_empty(iter
)) {
6385 if ((filp
->f_flags
& O_NONBLOCK
))
6388 ret
= wait_on_pipe(iter
, false);
6399 size
= PAGE_SIZE
- info
->read
;
6403 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
6415 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
6417 struct ftrace_buffer_info
*info
= file
->private_data
;
6418 struct trace_iterator
*iter
= &info
->iter
;
6420 mutex_lock(&trace_types_lock
);
6422 iter
->tr
->current_trace
->ref
--;
6424 __trace_array_put(iter
->tr
);
6427 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
, info
->spare
);
6430 mutex_unlock(&trace_types_lock
);
6436 struct ring_buffer
*buffer
;
6441 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
6442 struct pipe_buffer
*buf
)
6444 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
6449 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6454 static void buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
6455 struct pipe_buffer
*buf
)
6457 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
6462 /* Pipe buffer operations for a buffer. */
6463 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
6465 .confirm
= generic_pipe_buf_confirm
,
6466 .release
= buffer_pipe_buf_release
,
6467 .steal
= generic_pipe_buf_steal
,
6468 .get
= buffer_pipe_buf_get
,
6472 * Callback from splice_to_pipe(), if we need to release some pages
6473 * at the end of the spd in case we error'ed out in filling the pipe.
6475 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
6477 struct buffer_ref
*ref
=
6478 (struct buffer_ref
*)spd
->partial
[i
].private;
6483 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6485 spd
->partial
[i
].private = 0;
6489 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
6490 struct pipe_inode_info
*pipe
, size_t len
,
6493 struct ftrace_buffer_info
*info
= file
->private_data
;
6494 struct trace_iterator
*iter
= &info
->iter
;
6495 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
6496 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
6497 struct splice_pipe_desc spd
= {
6499 .partial
= partial_def
,
6500 .nr_pages_max
= PIPE_DEF_BUFFERS
,
6502 .ops
= &buffer_pipe_buf_ops
,
6503 .spd_release
= buffer_spd_release
,
6505 struct buffer_ref
*ref
;
6506 int entries
, size
, i
;
6509 #ifdef CONFIG_TRACER_MAX_TRACE
6510 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
6514 if (*ppos
& (PAGE_SIZE
- 1))
6517 if (len
& (PAGE_SIZE
- 1)) {
6518 if (len
< PAGE_SIZE
)
6523 if (splice_grow_spd(pipe
, &spd
))
6527 trace_access_lock(iter
->cpu_file
);
6528 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
6530 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
6534 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
6541 ref
->buffer
= iter
->trace_buffer
->buffer
;
6542 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
6549 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
6550 len
, iter
->cpu_file
, 1);
6552 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6558 * zero out any left over data, this is going to
6561 size
= ring_buffer_page_len(ref
->page
);
6562 if (size
< PAGE_SIZE
)
6563 memset(ref
->page
+ size
, 0, PAGE_SIZE
- size
);
6565 page
= virt_to_page(ref
->page
);
6567 spd
.pages
[i
] = page
;
6568 spd
.partial
[i
].len
= PAGE_SIZE
;
6569 spd
.partial
[i
].offset
= 0;
6570 spd
.partial
[i
].private = (unsigned long)ref
;
6574 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
6577 trace_access_unlock(iter
->cpu_file
);
6580 /* did we read anything? */
6581 if (!spd
.nr_pages
) {
6586 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
))
6589 ret
= wait_on_pipe(iter
, true);
6596 ret
= splice_to_pipe(pipe
, &spd
);
6598 splice_shrink_spd(&spd
);
6603 static const struct file_operations tracing_buffers_fops
= {
6604 .open
= tracing_buffers_open
,
6605 .read
= tracing_buffers_read
,
6606 .poll
= tracing_buffers_poll
,
6607 .release
= tracing_buffers_release
,
6608 .splice_read
= tracing_buffers_splice_read
,
6609 .llseek
= no_llseek
,
6613 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
6614 size_t count
, loff_t
*ppos
)
6616 struct inode
*inode
= file_inode(filp
);
6617 struct trace_array
*tr
= inode
->i_private
;
6618 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
6619 int cpu
= tracing_get_cpu(inode
);
6620 struct trace_seq
*s
;
6622 unsigned long long t
;
6623 unsigned long usec_rem
;
6625 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
6631 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
6632 trace_seq_printf(s
, "entries: %ld\n", cnt
);
6634 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
6635 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
6637 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
6638 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
6640 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
6641 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
6643 if (trace_clocks
[tr
->clock_id
].in_ns
) {
6644 /* local or global for trace_clock */
6645 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
6646 usec_rem
= do_div(t
, USEC_PER_SEC
);
6647 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
6650 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
6651 usec_rem
= do_div(t
, USEC_PER_SEC
);
6652 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
6654 /* counter or tsc mode for trace_clock */
6655 trace_seq_printf(s
, "oldest event ts: %llu\n",
6656 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
6658 trace_seq_printf(s
, "now ts: %llu\n",
6659 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
6662 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
6663 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
6665 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
6666 trace_seq_printf(s
, "read events: %ld\n", cnt
);
6668 count
= simple_read_from_buffer(ubuf
, count
, ppos
,
6669 s
->buffer
, trace_seq_used(s
));
6676 static const struct file_operations tracing_stats_fops
= {
6677 .open
= tracing_open_generic_tr
,
6678 .read
= tracing_stats_read
,
6679 .llseek
= generic_file_llseek
,
6680 .release
= tracing_release_generic_tr
,
6683 #ifdef CONFIG_DYNAMIC_FTRACE
6685 int __weak
ftrace_arch_read_dyn_info(char *buf
, int size
)
6691 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
6692 size_t cnt
, loff_t
*ppos
)
6694 static char ftrace_dyn_info_buffer
[1024];
6695 static DEFINE_MUTEX(dyn_info_mutex
);
6696 unsigned long *p
= filp
->private_data
;
6697 char *buf
= ftrace_dyn_info_buffer
;
6698 int size
= ARRAY_SIZE(ftrace_dyn_info_buffer
);
6701 mutex_lock(&dyn_info_mutex
);
6702 r
= sprintf(buf
, "%ld ", *p
);
6704 r
+= ftrace_arch_read_dyn_info(buf
+r
, (size
-1)-r
);
6707 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6709 mutex_unlock(&dyn_info_mutex
);
6714 static const struct file_operations tracing_dyn_info_fops
= {
6715 .open
= tracing_open_generic
,
6716 .read
= tracing_read_dyn_info
,
6717 .llseek
= generic_file_llseek
,
6719 #endif /* CONFIG_DYNAMIC_FTRACE */
6721 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6723 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6729 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6731 unsigned long *count
= (long *)data
;
6743 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
6744 struct ftrace_probe_ops
*ops
, void *data
)
6746 long count
= (long)data
;
6748 seq_printf(m
, "%ps:", (void *)ip
);
6750 seq_puts(m
, "snapshot");
6753 seq_puts(m
, ":unlimited\n");
6755 seq_printf(m
, ":count=%ld\n", count
);
6760 static struct ftrace_probe_ops snapshot_probe_ops
= {
6761 .func
= ftrace_snapshot
,
6762 .print
= ftrace_snapshot_print
,
6765 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
6766 .func
= ftrace_count_snapshot
,
6767 .print
= ftrace_snapshot_print
,
6771 ftrace_trace_snapshot_callback(struct ftrace_hash
*hash
,
6772 char *glob
, char *cmd
, char *param
, int enable
)
6774 struct ftrace_probe_ops
*ops
;
6775 void *count
= (void *)-1;
6779 /* hash funcs only work with set_ftrace_filter */
6783 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
6785 if (glob
[0] == '!') {
6786 unregister_ftrace_function_probe_func(glob
+1, ops
);
6793 number
= strsep(¶m
, ":");
6795 if (!strlen(number
))
6799 * We use the callback data field (which is a pointer)
6802 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
6807 ret
= register_ftrace_function_probe(glob
, ops
, count
);
6810 alloc_snapshot(&global_trace
);
6812 return ret
< 0 ? ret
: 0;
6815 static struct ftrace_func_command ftrace_snapshot_cmd
= {
6817 .func
= ftrace_trace_snapshot_callback
,
6820 static __init
int register_snapshot_cmd(void)
6822 return register_ftrace_command(&ftrace_snapshot_cmd
);
6825 static inline __init
int register_snapshot_cmd(void) { return 0; }
6826 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6828 static struct dentry
*tracing_get_dentry(struct trace_array
*tr
)
6830 if (WARN_ON(!tr
->dir
))
6831 return ERR_PTR(-ENODEV
);
6833 /* Top directory uses NULL as the parent */
6834 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
6837 /* All sub buffers have a descriptor */
6841 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
6843 struct dentry
*d_tracer
;
6846 return tr
->percpu_dir
;
6848 d_tracer
= tracing_get_dentry(tr
);
6849 if (IS_ERR(d_tracer
))
6852 tr
->percpu_dir
= tracefs_create_dir("per_cpu", d_tracer
);
6854 WARN_ONCE(!tr
->percpu_dir
,
6855 "Could not create tracefs directory 'per_cpu/%d'\n", cpu
);
6857 return tr
->percpu_dir
;
6860 static struct dentry
*
6861 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
6862 void *data
, long cpu
, const struct file_operations
*fops
)
6864 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
6866 if (ret
) /* See tracing_get_cpu() */
6867 d_inode(ret
)->i_cdev
= (void *)(cpu
+ 1);
6872 tracing_init_tracefs_percpu(struct trace_array
*tr
, long cpu
)
6874 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
6875 struct dentry
*d_cpu
;
6876 char cpu_dir
[30]; /* 30 characters should be more than enough */
6881 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
6882 d_cpu
= tracefs_create_dir(cpu_dir
, d_percpu
);
6884 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir
);
6888 /* per cpu trace_pipe */
6889 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
6890 tr
, cpu
, &tracing_pipe_fops
);
6893 trace_create_cpu_file("trace", 0644, d_cpu
,
6894 tr
, cpu
, &tracing_fops
);
6896 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
6897 tr
, cpu
, &tracing_buffers_fops
);
6899 trace_create_cpu_file("stats", 0444, d_cpu
,
6900 tr
, cpu
, &tracing_stats_fops
);
6902 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
6903 tr
, cpu
, &tracing_entries_fops
);
6905 #ifdef CONFIG_TRACER_SNAPSHOT
6906 trace_create_cpu_file("snapshot", 0644, d_cpu
,
6907 tr
, cpu
, &snapshot_fops
);
6909 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
6910 tr
, cpu
, &snapshot_raw_fops
);
6914 #ifdef CONFIG_FTRACE_SELFTEST
6915 /* Let selftest have access to static functions in this file */
6916 #include "trace_selftest.c"
6920 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6923 struct trace_option_dentry
*topt
= filp
->private_data
;
6926 if (topt
->flags
->val
& topt
->opt
->bit
)
6931 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6935 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6938 struct trace_option_dentry
*topt
= filp
->private_data
;
6942 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6946 if (val
!= 0 && val
!= 1)
6949 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
6950 mutex_lock(&trace_types_lock
);
6951 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
6953 mutex_unlock(&trace_types_lock
);
6964 static const struct file_operations trace_options_fops
= {
6965 .open
= tracing_open_generic
,
6966 .read
= trace_options_read
,
6967 .write
= trace_options_write
,
6968 .llseek
= generic_file_llseek
,
6972 * In order to pass in both the trace_array descriptor as well as the index
6973 * to the flag that the trace option file represents, the trace_array
6974 * has a character array of trace_flags_index[], which holds the index
6975 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6976 * The address of this character array is passed to the flag option file
6977 * read/write callbacks.
6979 * In order to extract both the index and the trace_array descriptor,
6980 * get_tr_index() uses the following algorithm.
6984 * As the pointer itself contains the address of the index (remember
6987 * Then to get the trace_array descriptor, by subtracting that index
6988 * from the ptr, we get to the start of the index itself.
6990 * ptr - idx == &index[0]
6992 * Then a simple container_of() from that pointer gets us to the
6993 * trace_array descriptor.
6995 static void get_tr_index(void *data
, struct trace_array
**ptr
,
6996 unsigned int *pindex
)
6998 *pindex
= *(unsigned char *)data
;
7000 *ptr
= container_of(data
- *pindex
, struct trace_array
,
7005 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
7008 void *tr_index
= filp
->private_data
;
7009 struct trace_array
*tr
;
7013 get_tr_index(tr_index
, &tr
, &index
);
7015 if (tr
->trace_flags
& (1 << index
))
7020 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
7024 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
7027 void *tr_index
= filp
->private_data
;
7028 struct trace_array
*tr
;
7033 get_tr_index(tr_index
, &tr
, &index
);
7035 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
7039 if (val
!= 0 && val
!= 1)
7042 mutex_lock(&trace_types_lock
);
7043 ret
= set_tracer_flag(tr
, 1 << index
, val
);
7044 mutex_unlock(&trace_types_lock
);
7054 static const struct file_operations trace_options_core_fops
= {
7055 .open
= tracing_open_generic
,
7056 .read
= trace_options_core_read
,
7057 .write
= trace_options_core_write
,
7058 .llseek
= generic_file_llseek
,
7061 struct dentry
*trace_create_file(const char *name
,
7063 struct dentry
*parent
,
7065 const struct file_operations
*fops
)
7069 ret
= tracefs_create_file(name
, mode
, parent
, data
, fops
);
7071 pr_warn("Could not create tracefs '%s' entry\n", name
);
7077 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
7079 struct dentry
*d_tracer
;
7084 d_tracer
= tracing_get_dentry(tr
);
7085 if (IS_ERR(d_tracer
))
7088 tr
->options
= tracefs_create_dir("options", d_tracer
);
7090 pr_warn("Could not create tracefs directory 'options'\n");
7098 create_trace_option_file(struct trace_array
*tr
,
7099 struct trace_option_dentry
*topt
,
7100 struct tracer_flags
*flags
,
7101 struct tracer_opt
*opt
)
7103 struct dentry
*t_options
;
7105 t_options
= trace_options_init_dentry(tr
);
7109 topt
->flags
= flags
;
7113 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
7114 &trace_options_fops
);
7119 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
7121 struct trace_option_dentry
*topts
;
7122 struct trace_options
*tr_topts
;
7123 struct tracer_flags
*flags
;
7124 struct tracer_opt
*opts
;
7131 flags
= tracer
->flags
;
7133 if (!flags
|| !flags
->opts
)
7137 * If this is an instance, only create flags for tracers
7138 * the instance may have.
7140 if (!trace_ok_for_array(tracer
, tr
))
7143 for (i
= 0; i
< tr
->nr_topts
; i
++) {
7144 /* Make sure there's no duplicate flags. */
7145 if (WARN_ON_ONCE(tr
->topts
[i
].tracer
->flags
== tracer
->flags
))
7151 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
7154 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
7158 tr_topts
= krealloc(tr
->topts
, sizeof(*tr
->topts
) * (tr
->nr_topts
+ 1),
7165 tr
->topts
= tr_topts
;
7166 tr
->topts
[tr
->nr_topts
].tracer
= tracer
;
7167 tr
->topts
[tr
->nr_topts
].topts
= topts
;
7170 for (cnt
= 0; opts
[cnt
].name
; cnt
++) {
7171 create_trace_option_file(tr
, &topts
[cnt
], flags
,
7173 WARN_ONCE(topts
[cnt
].entry
== NULL
,
7174 "Failed to create trace option: %s",
7179 static struct dentry
*
7180 create_trace_option_core_file(struct trace_array
*tr
,
7181 const char *option
, long index
)
7183 struct dentry
*t_options
;
7185 t_options
= trace_options_init_dentry(tr
);
7189 return trace_create_file(option
, 0644, t_options
,
7190 (void *)&tr
->trace_flags_index
[index
],
7191 &trace_options_core_fops
);
7194 static void create_trace_options_dir(struct trace_array
*tr
)
7196 struct dentry
*t_options
;
7197 bool top_level
= tr
== &global_trace
;
7200 t_options
= trace_options_init_dentry(tr
);
7204 for (i
= 0; trace_options
[i
]; i
++) {
7206 !((1 << i
) & TOP_LEVEL_TRACE_FLAGS
))
7207 create_trace_option_core_file(tr
, trace_options
[i
], i
);
7212 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
7213 size_t cnt
, loff_t
*ppos
)
7215 struct trace_array
*tr
= filp
->private_data
;
7219 r
= tracer_tracing_is_on(tr
);
7220 r
= sprintf(buf
, "%d\n", r
);
7222 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
7226 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
7227 size_t cnt
, loff_t
*ppos
)
7229 struct trace_array
*tr
= filp
->private_data
;
7230 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
7234 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
7239 mutex_lock(&trace_types_lock
);
7241 tracer_tracing_on(tr
);
7242 if (tr
->current_trace
->start
)
7243 tr
->current_trace
->start(tr
);
7245 tracer_tracing_off(tr
);
7246 if (tr
->current_trace
->stop
)
7247 tr
->current_trace
->stop(tr
);
7249 mutex_unlock(&trace_types_lock
);
7257 static const struct file_operations rb_simple_fops
= {
7258 .open
= tracing_open_generic_tr
,
7259 .read
= rb_simple_read
,
7260 .write
= rb_simple_write
,
7261 .release
= tracing_release_generic_tr
,
7262 .llseek
= default_llseek
,
7265 struct dentry
*trace_instance_dir
;
7268 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
);
7271 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
7273 enum ring_buffer_flags rb_flags
;
7275 rb_flags
= tr
->trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
7279 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
7283 buf
->data
= alloc_percpu(struct trace_array_cpu
);
7285 ring_buffer_free(buf
->buffer
);
7289 /* Allocate the first page for all buffers */
7290 set_buffer_entries(&tr
->trace_buffer
,
7291 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
7296 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
7300 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
7304 #ifdef CONFIG_TRACER_MAX_TRACE
7305 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
7306 allocate_snapshot
? size
: 1);
7308 ring_buffer_free(tr
->trace_buffer
.buffer
);
7309 free_percpu(tr
->trace_buffer
.data
);
7312 tr
->allocated_snapshot
= allocate_snapshot
;
7315 * Only the top level trace array gets its snapshot allocated
7316 * from the kernel command line.
7318 allocate_snapshot
= false;
7323 static void free_trace_buffer(struct trace_buffer
*buf
)
7326 ring_buffer_free(buf
->buffer
);
7328 free_percpu(buf
->data
);
7333 static void free_trace_buffers(struct trace_array
*tr
)
7338 free_trace_buffer(&tr
->trace_buffer
);
7340 #ifdef CONFIG_TRACER_MAX_TRACE
7341 free_trace_buffer(&tr
->max_buffer
);
7345 static void init_trace_flags_index(struct trace_array
*tr
)
7349 /* Used by the trace options files */
7350 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++)
7351 tr
->trace_flags_index
[i
] = i
;
7354 static void __update_tracer_options(struct trace_array
*tr
)
7358 for (t
= trace_types
; t
; t
= t
->next
)
7359 add_tracer_options(tr
, t
);
7362 static void update_tracer_options(struct trace_array
*tr
)
7364 mutex_lock(&trace_types_lock
);
7365 __update_tracer_options(tr
);
7366 mutex_unlock(&trace_types_lock
);
7369 static int instance_mkdir(const char *name
)
7371 struct trace_array
*tr
;
7374 mutex_lock(&trace_types_lock
);
7377 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
7378 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
7383 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
7387 tr
->name
= kstrdup(name
, GFP_KERNEL
);
7391 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
7394 tr
->trace_flags
= global_trace
.trace_flags
& ~ZEROED_TRACE_FLAGS
;
7396 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
7398 raw_spin_lock_init(&tr
->start_lock
);
7400 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
7402 tr
->current_trace
= &nop_trace
;
7404 INIT_LIST_HEAD(&tr
->systems
);
7405 INIT_LIST_HEAD(&tr
->events
);
7407 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
7410 tr
->dir
= tracefs_create_dir(name
, trace_instance_dir
);
7414 ret
= event_trace_add_tracer(tr
->dir
, tr
);
7416 tracefs_remove_recursive(tr
->dir
);
7420 init_tracer_tracefs(tr
, tr
->dir
);
7421 init_trace_flags_index(tr
);
7422 __update_tracer_options(tr
);
7424 list_add(&tr
->list
, &ftrace_trace_arrays
);
7426 mutex_unlock(&trace_types_lock
);
7431 free_trace_buffers(tr
);
7432 free_cpumask_var(tr
->tracing_cpumask
);
7437 mutex_unlock(&trace_types_lock
);
7443 static int instance_rmdir(const char *name
)
7445 struct trace_array
*tr
;
7450 mutex_lock(&trace_types_lock
);
7453 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
7454 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
7463 if (tr
->ref
|| (tr
->current_trace
&& tr
->current_trace
->ref
))
7466 list_del(&tr
->list
);
7468 /* Disable all the flags that were enabled coming in */
7469 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++) {
7470 if ((1 << i
) & ZEROED_TRACE_FLAGS
)
7471 set_tracer_flag(tr
, 1 << i
, 0);
7474 tracing_set_nop(tr
);
7475 event_trace_del_tracer(tr
);
7476 ftrace_destroy_function_files(tr
);
7477 tracefs_remove_recursive(tr
->dir
);
7478 free_trace_buffers(tr
);
7480 for (i
= 0; i
< tr
->nr_topts
; i
++) {
7481 kfree(tr
->topts
[i
].topts
);
7491 mutex_unlock(&trace_types_lock
);
7496 static __init
void create_trace_instances(struct dentry
*d_tracer
)
7498 trace_instance_dir
= tracefs_create_instance_dir("instances", d_tracer
,
7501 if (WARN_ON(!trace_instance_dir
))
7506 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
)
7510 trace_create_file("available_tracers", 0444, d_tracer
,
7511 tr
, &show_traces_fops
);
7513 trace_create_file("current_tracer", 0644, d_tracer
,
7514 tr
, &set_tracer_fops
);
7516 trace_create_file("tracing_cpumask", 0644, d_tracer
,
7517 tr
, &tracing_cpumask_fops
);
7519 trace_create_file("trace_options", 0644, d_tracer
,
7520 tr
, &tracing_iter_fops
);
7522 trace_create_file("trace", 0644, d_tracer
,
7525 trace_create_file("trace_pipe", 0444, d_tracer
,
7526 tr
, &tracing_pipe_fops
);
7528 trace_create_file("buffer_size_kb", 0644, d_tracer
,
7529 tr
, &tracing_entries_fops
);
7531 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
7532 tr
, &tracing_total_entries_fops
);
7534 trace_create_file("free_buffer", 0200, d_tracer
,
7535 tr
, &tracing_free_buffer_fops
);
7537 trace_create_file("trace_marker", 0220, d_tracer
,
7538 tr
, &tracing_mark_fops
);
7540 trace_create_file("trace_marker_raw", 0220, d_tracer
,
7541 tr
, &tracing_mark_raw_fops
);
7543 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
7546 trace_create_file("tracing_on", 0644, d_tracer
,
7547 tr
, &rb_simple_fops
);
7549 create_trace_options_dir(tr
);
7551 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7552 trace_create_file("tracing_max_latency", 0644, d_tracer
,
7553 &tr
->max_latency
, &tracing_max_lat_fops
);
7556 if (ftrace_create_function_files(tr
, d_tracer
))
7557 WARN(1, "Could not allocate function filter files");
7559 #ifdef CONFIG_TRACER_SNAPSHOT
7560 trace_create_file("snapshot", 0644, d_tracer
,
7561 tr
, &snapshot_fops
);
7564 for_each_tracing_cpu(cpu
)
7565 tracing_init_tracefs_percpu(tr
, cpu
);
7567 ftrace_init_tracefs(tr
, d_tracer
);
7570 static struct vfsmount
*trace_automount(void *ingore
)
7572 struct vfsmount
*mnt
;
7573 struct file_system_type
*type
;
7576 * To maintain backward compatibility for tools that mount
7577 * debugfs to get to the tracing facility, tracefs is automatically
7578 * mounted to the debugfs/tracing directory.
7580 type
= get_fs_type("tracefs");
7583 mnt
= vfs_kern_mount(type
, 0, "tracefs", NULL
);
7584 put_filesystem(type
);
7593 * tracing_init_dentry - initialize top level trace array
7595 * This is called when creating files or directories in the tracing
7596 * directory. It is called via fs_initcall() by any of the boot up code
7597 * and expects to return the dentry of the top level tracing directory.
7599 struct dentry
*tracing_init_dentry(void)
7601 struct trace_array
*tr
= &global_trace
;
7603 /* The top level trace array uses NULL as parent */
7607 if (WARN_ON(!tracefs_initialized()) ||
7608 (IS_ENABLED(CONFIG_DEBUG_FS
) &&
7609 WARN_ON(!debugfs_initialized())))
7610 return ERR_PTR(-ENODEV
);
7613 * As there may still be users that expect the tracing
7614 * files to exist in debugfs/tracing, we must automount
7615 * the tracefs file system there, so older tools still
7616 * work with the newer kerenl.
7618 tr
->dir
= debugfs_create_automount("tracing", NULL
,
7619 trace_automount
, NULL
);
7621 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7622 return ERR_PTR(-ENOMEM
);
7628 extern struct trace_enum_map
*__start_ftrace_enum_maps
[];
7629 extern struct trace_enum_map
*__stop_ftrace_enum_maps
[];
7631 static void __init
trace_enum_init(void)
7635 len
= __stop_ftrace_enum_maps
- __start_ftrace_enum_maps
;
7636 trace_insert_enum_map(NULL
, __start_ftrace_enum_maps
, len
);
7639 #ifdef CONFIG_MODULES
7640 static void trace_module_add_enums(struct module
*mod
)
7642 if (!mod
->num_trace_enums
)
7646 * Modules with bad taint do not have events created, do
7647 * not bother with enums either.
7649 if (trace_module_has_bad_taint(mod
))
7652 trace_insert_enum_map(mod
, mod
->trace_enums
, mod
->num_trace_enums
);
7655 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7656 static void trace_module_remove_enums(struct module
*mod
)
7658 union trace_enum_map_item
*map
;
7659 union trace_enum_map_item
**last
= &trace_enum_maps
;
7661 if (!mod
->num_trace_enums
)
7664 mutex_lock(&trace_enum_mutex
);
7666 map
= trace_enum_maps
;
7669 if (map
->head
.mod
== mod
)
7671 map
= trace_enum_jmp_to_tail(map
);
7672 last
= &map
->tail
.next
;
7673 map
= map
->tail
.next
;
7678 *last
= trace_enum_jmp_to_tail(map
)->tail
.next
;
7681 mutex_unlock(&trace_enum_mutex
);
7684 static inline void trace_module_remove_enums(struct module
*mod
) { }
7685 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7687 static int trace_module_notify(struct notifier_block
*self
,
7688 unsigned long val
, void *data
)
7690 struct module
*mod
= data
;
7693 case MODULE_STATE_COMING
:
7694 trace_module_add_enums(mod
);
7696 case MODULE_STATE_GOING
:
7697 trace_module_remove_enums(mod
);
7704 static struct notifier_block trace_module_nb
= {
7705 .notifier_call
= trace_module_notify
,
7708 #endif /* CONFIG_MODULES */
7710 static __init
int tracer_init_tracefs(void)
7712 struct dentry
*d_tracer
;
7714 trace_access_lock_init();
7716 d_tracer
= tracing_init_dentry();
7717 if (IS_ERR(d_tracer
))
7720 init_tracer_tracefs(&global_trace
, d_tracer
);
7721 ftrace_init_tracefs_toplevel(&global_trace
, d_tracer
);
7723 trace_create_file("tracing_thresh", 0644, d_tracer
,
7724 &global_trace
, &tracing_thresh_fops
);
7726 trace_create_file("README", 0444, d_tracer
,
7727 NULL
, &tracing_readme_fops
);
7729 trace_create_file("saved_cmdlines", 0444, d_tracer
,
7730 NULL
, &tracing_saved_cmdlines_fops
);
7732 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
7733 NULL
, &tracing_saved_cmdlines_size_fops
);
7737 trace_create_enum_file(d_tracer
);
7739 #ifdef CONFIG_MODULES
7740 register_module_notifier(&trace_module_nb
);
7743 #ifdef CONFIG_DYNAMIC_FTRACE
7744 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
7745 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
7748 create_trace_instances(d_tracer
);
7750 update_tracer_options(&global_trace
);
7755 static int trace_panic_handler(struct notifier_block
*this,
7756 unsigned long event
, void *unused
)
7758 if (ftrace_dump_on_oops
)
7759 ftrace_dump(ftrace_dump_on_oops
);
7763 static struct notifier_block trace_panic_notifier
= {
7764 .notifier_call
= trace_panic_handler
,
7766 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
7769 static int trace_die_handler(struct notifier_block
*self
,
7775 if (ftrace_dump_on_oops
)
7776 ftrace_dump(ftrace_dump_on_oops
);
7784 static struct notifier_block trace_die_notifier
= {
7785 .notifier_call
= trace_die_handler
,
7790 * printk is set to max of 1024, we really don't need it that big.
7791 * Nothing should be printing 1000 characters anyway.
7793 #define TRACE_MAX_PRINT 1000
7796 * Define here KERN_TRACE so that we have one place to modify
7797 * it if we decide to change what log level the ftrace dump
7800 #define KERN_TRACE KERN_EMERG
7803 trace_printk_seq(struct trace_seq
*s
)
7805 /* Probably should print a warning here. */
7806 if (s
->seq
.len
>= TRACE_MAX_PRINT
)
7807 s
->seq
.len
= TRACE_MAX_PRINT
;
7810 * More paranoid code. Although the buffer size is set to
7811 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7812 * an extra layer of protection.
7814 if (WARN_ON_ONCE(s
->seq
.len
>= s
->seq
.size
))
7815 s
->seq
.len
= s
->seq
.size
- 1;
7817 /* should be zero ended, but we are paranoid. */
7818 s
->buffer
[s
->seq
.len
] = 0;
7820 printk(KERN_TRACE
"%s", s
->buffer
);
7825 void trace_init_global_iter(struct trace_iterator
*iter
)
7827 iter
->tr
= &global_trace
;
7828 iter
->trace
= iter
->tr
->current_trace
;
7829 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
7830 iter
->trace_buffer
= &global_trace
.trace_buffer
;
7832 if (iter
->trace
&& iter
->trace
->open
)
7833 iter
->trace
->open(iter
);
7835 /* Annotate start of buffers if we had overruns */
7836 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
7837 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
7839 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7840 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
7841 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
7844 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
7846 /* use static because iter can be a bit big for the stack */
7847 static struct trace_iterator iter
;
7848 static atomic_t dump_running
;
7849 struct trace_array
*tr
= &global_trace
;
7850 unsigned int old_userobj
;
7851 unsigned long flags
;
7854 /* Only allow one dump user at a time. */
7855 if (atomic_inc_return(&dump_running
) != 1) {
7856 atomic_dec(&dump_running
);
7861 * Always turn off tracing when we dump.
7862 * We don't need to show trace output of what happens
7863 * between multiple crashes.
7865 * If the user does a sysrq-z, then they can re-enable
7866 * tracing with echo 1 > tracing_on.
7870 local_irq_save(flags
);
7872 /* Simulate the iterator */
7873 trace_init_global_iter(&iter
);
7875 for_each_tracing_cpu(cpu
) {
7876 atomic_inc(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7879 old_userobj
= tr
->trace_flags
& TRACE_ITER_SYM_USEROBJ
;
7881 /* don't look at user memory in panic mode */
7882 tr
->trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
7884 switch (oops_dump_mode
) {
7886 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7889 iter
.cpu_file
= raw_smp_processor_id();
7894 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
7895 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7898 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
7900 /* Did function tracer already get disabled? */
7901 if (ftrace_is_dead()) {
7902 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7903 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7907 * We need to stop all tracing on all CPUS to read the
7908 * the next buffer. This is a bit expensive, but is
7909 * not done often. We fill all what we can read,
7910 * and then release the locks again.
7913 while (!trace_empty(&iter
)) {
7916 printk(KERN_TRACE
"---------------------------------\n");
7920 /* reset all but tr, trace, and overruns */
7921 memset(&iter
.seq
, 0,
7922 sizeof(struct trace_iterator
) -
7923 offsetof(struct trace_iterator
, seq
));
7924 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
7927 if (trace_find_next_entry_inc(&iter
) != NULL
) {
7930 ret
= print_trace_line(&iter
);
7931 if (ret
!= TRACE_TYPE_NO_CONSUME
)
7932 trace_consume(&iter
);
7934 touch_nmi_watchdog();
7936 trace_printk_seq(&iter
.seq
);
7940 printk(KERN_TRACE
" (ftrace buffer empty)\n");
7942 printk(KERN_TRACE
"---------------------------------\n");
7945 tr
->trace_flags
|= old_userobj
;
7947 for_each_tracing_cpu(cpu
) {
7948 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7950 atomic_dec(&dump_running
);
7951 local_irq_restore(flags
);
7953 EXPORT_SYMBOL_GPL(ftrace_dump
);
7955 __init
static int tracer_alloc_buffers(void)
7961 * Make sure we don't accidently add more trace options
7962 * than we have bits for.
7964 BUILD_BUG_ON(TRACE_ITER_LAST_BIT
> TRACE_FLAGS_MAX_SIZE
);
7966 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
7969 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
7970 goto out_free_buffer_mask
;
7972 /* Only allocate trace_printk buffers if a trace_printk exists */
7973 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
7974 /* Must be called before global_trace.buffer is allocated */
7975 trace_printk_init_buffers();
7977 /* To save memory, keep the ring buffer size to its minimum */
7978 if (ring_buffer_expanded
)
7979 ring_buf_size
= trace_buf_size
;
7983 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
7984 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
7986 raw_spin_lock_init(&global_trace
.start_lock
);
7988 /* Used for event triggers */
7989 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
7991 goto out_free_cpumask
;
7993 if (trace_create_savedcmd() < 0)
7994 goto out_free_temp_buffer
;
7996 /* TODO: make the number of buffers hot pluggable with CPUS */
7997 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
7998 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
8000 goto out_free_savedcmd
;
8003 if (global_trace
.buffer_disabled
)
8006 if (trace_boot_clock
) {
8007 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
8009 pr_warn("Trace clock %s not defined, going back to default\n",
8014 * register_tracer() might reference current_trace, so it
8015 * needs to be set before we register anything. This is
8016 * just a bootstrap of current_trace anyway.
8018 global_trace
.current_trace
= &nop_trace
;
8020 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
8022 ftrace_init_global_array_ops(&global_trace
);
8024 init_trace_flags_index(&global_trace
);
8026 register_tracer(&nop_trace
);
8028 /* All seems OK, enable tracing */
8029 tracing_disabled
= 0;
8031 atomic_notifier_chain_register(&panic_notifier_list
,
8032 &trace_panic_notifier
);
8034 register_die_notifier(&trace_die_notifier
);
8036 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
8038 INIT_LIST_HEAD(&global_trace
.systems
);
8039 INIT_LIST_HEAD(&global_trace
.events
);
8040 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
8042 apply_trace_boot_options();
8044 register_snapshot_cmd();
8049 free_saved_cmdlines_buffer(savedcmd
);
8050 out_free_temp_buffer
:
8051 ring_buffer_free(temp_buffer
);
8053 free_cpumask_var(global_trace
.tracing_cpumask
);
8054 out_free_buffer_mask
:
8055 free_cpumask_var(tracing_buffer_mask
);
8060 void __init
trace_init(void)
8062 if (tracepoint_printk
) {
8063 tracepoint_print_iter
=
8064 kmalloc(sizeof(*tracepoint_print_iter
), GFP_KERNEL
);
8065 if (WARN_ON(!tracepoint_print_iter
))
8066 tracepoint_printk
= 0;
8068 static_key_enable(&tracepoint_printk_key
.key
);
8070 tracer_alloc_buffers();
8074 __init
static int clear_boot_tracer(void)
8077 * The default tracer at boot buffer is an init section.
8078 * This function is called in lateinit. If we did not
8079 * find the boot tracer, then clear it out, to prevent
8080 * later registration from accessing the buffer that is
8081 * about to be freed.
8083 if (!default_bootup_tracer
)
8086 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
8087 default_bootup_tracer
);
8088 default_bootup_tracer
= NULL
;
8093 fs_initcall(tracer_init_tracefs
);
8094 late_initcall(clear_boot_tracer
);