2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded
;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running
;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled
;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator
*tracepoint_print_iter
;
70 int tracepoint_printk
;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt
[] = {
78 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save
);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled
= 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask
;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops
;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning
;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head
{
125 unsigned long length
;
128 union trace_enum_map_item
;
130 struct trace_enum_map_tail
{
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item
*next
;
136 const char *end
; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex
);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item
{
149 struct trace_enum_map map
;
150 struct trace_enum_map_head head
;
151 struct trace_enum_map_tail tail
;
154 static union trace_enum_map_item
*trace_enum_maps
;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
161 static char *default_bootup_tracer
;
163 static bool allocate_snapshot
;
165 static int __init
set_cmdline_ftrace(char *str
)
167 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
168 default_bootup_tracer
= bootup_tracer_buf
;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded
= true;
173 __setup("ftrace=", set_cmdline_ftrace
);
175 static int __init
set_ftrace_dump_on_oops(char *str
)
177 if (*str
++ != '=' || !*str
) {
178 ftrace_dump_on_oops
= DUMP_ALL
;
182 if (!strcmp("orig_cpu", str
)) {
183 ftrace_dump_on_oops
= DUMP_ORIG
;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
191 static int __init
stop_trace_on_warning(char *str
)
193 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
194 __disable_trace_on_warning
= 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning
);
199 static int __init
boot_alloc_snapshot(char *str
)
201 allocate_snapshot
= true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded
= true;
206 __setup("alloc_snapshot", boot_alloc_snapshot
);
209 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
211 static int __init
set_trace_boot_options(char *str
)
213 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
216 __setup("trace_options=", set_trace_boot_options
);
218 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
219 static char *trace_boot_clock __initdata
;
221 static int __init
set_trace_boot_clock(char *str
)
223 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
224 trace_boot_clock
= trace_boot_clock_buf
;
227 __setup("trace_clock=", set_trace_boot_clock
);
229 static int __init
set_tracepoint_printk(char *str
)
231 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
232 tracepoint_printk
= 1;
235 __setup("tp_printk", set_tracepoint_printk
);
237 unsigned long long ns2usecs(cycle_t nsec
)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace
= {
273 .trace_flags
= TRACE_DEFAULT_FLAGS
,
276 LIST_HEAD(ftrace_trace_arrays
);
278 int trace_array_get(struct trace_array
*this_tr
)
280 struct trace_array
*tr
;
283 mutex_lock(&trace_types_lock
);
284 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
291 mutex_unlock(&trace_types_lock
);
296 static void __trace_array_put(struct trace_array
*this_tr
)
298 WARN_ON(!this_tr
->ref
);
302 void trace_array_put(struct trace_array
*this_tr
)
304 mutex_lock(&trace_types_lock
);
305 __trace_array_put(this_tr
);
306 mutex_unlock(&trace_types_lock
);
309 int call_filter_check_discard(struct trace_event_call
*call
, void *rec
,
310 struct ring_buffer
*buffer
,
311 struct ring_buffer_event
*event
)
313 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
314 !filter_match_preds(call
->filter
, rec
)) {
315 __trace_event_discard_commit(buffer
, event
);
322 void trace_free_pid_list(struct trace_pid_list
*pid_list
)
324 vfree(pid_list
->pids
);
329 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
330 * @filtered_pids: The list of pids to check
331 * @search_pid: The PID to find in @filtered_pids
333 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
336 trace_find_filtered_pid(struct trace_pid_list
*filtered_pids
, pid_t search_pid
)
339 * If pid_max changed after filtered_pids was created, we
340 * by default ignore all pids greater than the previous pid_max.
342 if (search_pid
>= filtered_pids
->pid_max
)
345 return test_bit(search_pid
, filtered_pids
->pids
);
349 * trace_ignore_this_task - should a task be ignored for tracing
350 * @filtered_pids: The list of pids to check
351 * @task: The task that should be ignored if not filtered
353 * Checks if @task should be traced or not from @filtered_pids.
354 * Returns true if @task should *NOT* be traced.
355 * Returns false if @task should be traced.
358 trace_ignore_this_task(struct trace_pid_list
*filtered_pids
, struct task_struct
*task
)
361 * Return false, because if filtered_pids does not exist,
362 * all pids are good to trace.
367 return !trace_find_filtered_pid(filtered_pids
, task
->pid
);
371 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
372 * @pid_list: The list to modify
373 * @self: The current task for fork or NULL for exit
374 * @task: The task to add or remove
376 * If adding a task, if @self is defined, the task is only added if @self
377 * is also included in @pid_list. This happens on fork and tasks should
378 * only be added when the parent is listed. If @self is NULL, then the
379 * @task pid will be removed from the list, which would happen on exit
382 void trace_filter_add_remove_task(struct trace_pid_list
*pid_list
,
383 struct task_struct
*self
,
384 struct task_struct
*task
)
389 /* For forks, we only add if the forking task is listed */
391 if (!trace_find_filtered_pid(pid_list
, self
->pid
))
395 /* Sorry, but we don't support pid_max changing after setting */
396 if (task
->pid
>= pid_list
->pid_max
)
399 /* "self" is set for forks, and NULL for exits */
401 set_bit(task
->pid
, pid_list
->pids
);
403 clear_bit(task
->pid
, pid_list
->pids
);
407 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
408 * @pid_list: The pid list to show
409 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
410 * @pos: The position of the file
412 * This is used by the seq_file "next" operation to iterate the pids
413 * listed in a trace_pid_list structure.
415 * Returns the pid+1 as we want to display pid of zero, but NULL would
416 * stop the iteration.
418 void *trace_pid_next(struct trace_pid_list
*pid_list
, void *v
, loff_t
*pos
)
420 unsigned long pid
= (unsigned long)v
;
424 /* pid already is +1 of the actual prevous bit */
425 pid
= find_next_bit(pid_list
->pids
, pid_list
->pid_max
, pid
);
427 /* Return pid + 1 to allow zero to be represented */
428 if (pid
< pid_list
->pid_max
)
429 return (void *)(pid
+ 1);
435 * trace_pid_start - Used for seq_file to start reading pid lists
436 * @pid_list: The pid list to show
437 * @pos: The position of the file
439 * This is used by seq_file "start" operation to start the iteration
442 * Returns the pid+1 as we want to display pid of zero, but NULL would
443 * stop the iteration.
445 void *trace_pid_start(struct trace_pid_list
*pid_list
, loff_t
*pos
)
450 pid
= find_first_bit(pid_list
->pids
, pid_list
->pid_max
);
451 if (pid
>= pid_list
->pid_max
)
454 /* Return pid + 1 so that zero can be the exit value */
455 for (pid
++; pid
&& l
< *pos
;
456 pid
= (unsigned long)trace_pid_next(pid_list
, (void *)pid
, &l
))
462 * trace_pid_show - show the current pid in seq_file processing
463 * @m: The seq_file structure to write into
464 * @v: A void pointer of the pid (+1) value to display
466 * Can be directly used by seq_file operations to display the current
469 int trace_pid_show(struct seq_file
*m
, void *v
)
471 unsigned long pid
= (unsigned long)v
- 1;
473 seq_printf(m
, "%lu\n", pid
);
477 /* 128 should be much more than enough */
478 #define PID_BUF_SIZE 127
480 int trace_pid_write(struct trace_pid_list
*filtered_pids
,
481 struct trace_pid_list
**new_pid_list
,
482 const char __user
*ubuf
, size_t cnt
)
484 struct trace_pid_list
*pid_list
;
485 struct trace_parser parser
;
493 if (trace_parser_get_init(&parser
, PID_BUF_SIZE
+ 1))
497 * Always recreate a new array. The write is an all or nothing
498 * operation. Always create a new array when adding new pids by
499 * the user. If the operation fails, then the current list is
502 pid_list
= kmalloc(sizeof(*pid_list
), GFP_KERNEL
);
506 pid_list
->pid_max
= READ_ONCE(pid_max
);
508 /* Only truncating will shrink pid_max */
509 if (filtered_pids
&& filtered_pids
->pid_max
> pid_list
->pid_max
)
510 pid_list
->pid_max
= filtered_pids
->pid_max
;
512 pid_list
->pids
= vzalloc((pid_list
->pid_max
+ 7) >> 3);
513 if (!pid_list
->pids
) {
519 /* copy the current bits to the new max */
520 for_each_set_bit(pid
, filtered_pids
->pids
,
521 filtered_pids
->pid_max
) {
522 set_bit(pid
, pid_list
->pids
);
531 ret
= trace_get_user(&parser
, ubuf
, cnt
, &pos
);
532 if (ret
< 0 || !trace_parser_loaded(&parser
))
539 parser
.buffer
[parser
.idx
] = 0;
542 if (kstrtoul(parser
.buffer
, 0, &val
))
544 if (val
>= pid_list
->pid_max
)
549 set_bit(pid
, pid_list
->pids
);
552 trace_parser_clear(&parser
);
555 trace_parser_put(&parser
);
558 trace_free_pid_list(pid_list
);
563 /* Cleared the list of pids */
564 trace_free_pid_list(pid_list
);
569 *new_pid_list
= pid_list
;
574 static cycle_t
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
578 /* Early boot up does not have a buffer yet */
580 return trace_clock_local();
582 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
583 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
588 cycle_t
ftrace_now(int cpu
)
590 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
594 * tracing_is_enabled - Show if global_trace has been disabled
596 * Shows if the global trace has been enabled or not. It uses the
597 * mirror flag "buffer_disabled" to be used in fast paths such as for
598 * the irqsoff tracer. But it may be inaccurate due to races. If you
599 * need to know the accurate state, use tracing_is_on() which is a little
600 * slower, but accurate.
602 int tracing_is_enabled(void)
605 * For quick access (irqsoff uses this in fast path), just
606 * return the mirror variable of the state of the ring buffer.
607 * It's a little racy, but we don't really care.
610 return !global_trace
.buffer_disabled
;
614 * trace_buf_size is the size in bytes that is allocated
615 * for a buffer. Note, the number of bytes is always rounded
618 * This number is purposely set to a low number of 16384.
619 * If the dump on oops happens, it will be much appreciated
620 * to not have to wait for all that output. Anyway this can be
621 * boot time and run time configurable.
623 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
625 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
627 /* trace_types holds a link list of available tracers. */
628 static struct tracer
*trace_types __read_mostly
;
631 * trace_types_lock is used to protect the trace_types list.
633 DEFINE_MUTEX(trace_types_lock
);
636 * serialize the access of the ring buffer
638 * ring buffer serializes readers, but it is low level protection.
639 * The validity of the events (which returns by ring_buffer_peek() ..etc)
640 * are not protected by ring buffer.
642 * The content of events may become garbage if we allow other process consumes
643 * these events concurrently:
644 * A) the page of the consumed events may become a normal page
645 * (not reader page) in ring buffer, and this page will be rewrited
646 * by events producer.
647 * B) The page of the consumed events may become a page for splice_read,
648 * and this page will be returned to system.
650 * These primitives allow multi process access to different cpu ring buffer
653 * These primitives don't distinguish read-only and read-consume access.
654 * Multi read-only access are also serialized.
658 static DECLARE_RWSEM(all_cpu_access_lock
);
659 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
661 static inline void trace_access_lock(int cpu
)
663 if (cpu
== RING_BUFFER_ALL_CPUS
) {
664 /* gain it for accessing the whole ring buffer. */
665 down_write(&all_cpu_access_lock
);
667 /* gain it for accessing a cpu ring buffer. */
669 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
670 down_read(&all_cpu_access_lock
);
672 /* Secondly block other access to this @cpu ring buffer. */
673 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
677 static inline void trace_access_unlock(int cpu
)
679 if (cpu
== RING_BUFFER_ALL_CPUS
) {
680 up_write(&all_cpu_access_lock
);
682 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
683 up_read(&all_cpu_access_lock
);
687 static inline void trace_access_lock_init(void)
691 for_each_possible_cpu(cpu
)
692 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
697 static DEFINE_MUTEX(access_lock
);
699 static inline void trace_access_lock(int cpu
)
702 mutex_lock(&access_lock
);
705 static inline void trace_access_unlock(int cpu
)
708 mutex_unlock(&access_lock
);
711 static inline void trace_access_lock_init(void)
717 #ifdef CONFIG_STACKTRACE
718 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
720 int skip
, int pc
, struct pt_regs
*regs
);
721 static inline void ftrace_trace_stack(struct trace_array
*tr
,
722 struct ring_buffer
*buffer
,
724 int skip
, int pc
, struct pt_regs
*regs
);
727 static inline void __ftrace_trace_stack(struct ring_buffer
*buffer
,
729 int skip
, int pc
, struct pt_regs
*regs
)
732 static inline void ftrace_trace_stack(struct trace_array
*tr
,
733 struct ring_buffer
*buffer
,
735 int skip
, int pc
, struct pt_regs
*regs
)
741 static void tracer_tracing_on(struct trace_array
*tr
)
743 if (tr
->trace_buffer
.buffer
)
744 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
746 * This flag is looked at when buffers haven't been allocated
747 * yet, or by some tracers (like irqsoff), that just want to
748 * know if the ring buffer has been disabled, but it can handle
749 * races of where it gets disabled but we still do a record.
750 * As the check is in the fast path of the tracers, it is more
751 * important to be fast than accurate.
753 tr
->buffer_disabled
= 0;
754 /* Make the flag seen by readers */
759 * tracing_on - enable tracing buffers
761 * This function enables tracing buffers that may have been
762 * disabled with tracing_off.
764 void tracing_on(void)
766 tracer_tracing_on(&global_trace
);
768 EXPORT_SYMBOL_GPL(tracing_on
);
771 * __trace_puts - write a constant string into the trace buffer.
772 * @ip: The address of the caller
773 * @str: The constant string to write
774 * @size: The size of the string.
776 int __trace_puts(unsigned long ip
, const char *str
, int size
)
778 struct ring_buffer_event
*event
;
779 struct ring_buffer
*buffer
;
780 struct print_entry
*entry
;
781 unsigned long irq_flags
;
785 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
788 pc
= preempt_count();
790 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
793 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
795 local_save_flags(irq_flags
);
796 buffer
= global_trace
.trace_buffer
.buffer
;
797 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
802 entry
= ring_buffer_event_data(event
);
805 memcpy(&entry
->buf
, str
, size
);
807 /* Add a newline if necessary */
808 if (entry
->buf
[size
- 1] != '\n') {
809 entry
->buf
[size
] = '\n';
810 entry
->buf
[size
+ 1] = '\0';
812 entry
->buf
[size
] = '\0';
814 __buffer_unlock_commit(buffer
, event
);
815 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
819 EXPORT_SYMBOL_GPL(__trace_puts
);
822 * __trace_bputs - write the pointer to a constant string into trace buffer
823 * @ip: The address of the caller
824 * @str: The constant string to write to the buffer to
826 int __trace_bputs(unsigned long ip
, const char *str
)
828 struct ring_buffer_event
*event
;
829 struct ring_buffer
*buffer
;
830 struct bputs_entry
*entry
;
831 unsigned long irq_flags
;
832 int size
= sizeof(struct bputs_entry
);
835 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
838 pc
= preempt_count();
840 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
843 local_save_flags(irq_flags
);
844 buffer
= global_trace
.trace_buffer
.buffer
;
845 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
850 entry
= ring_buffer_event_data(event
);
854 __buffer_unlock_commit(buffer
, event
);
855 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
859 EXPORT_SYMBOL_GPL(__trace_bputs
);
861 #ifdef CONFIG_TRACER_SNAPSHOT
863 * trace_snapshot - take a snapshot of the current buffer.
865 * This causes a swap between the snapshot buffer and the current live
866 * tracing buffer. You can use this to take snapshots of the live
867 * trace when some condition is triggered, but continue to trace.
869 * Note, make sure to allocate the snapshot with either
870 * a tracing_snapshot_alloc(), or by doing it manually
871 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
873 * If the snapshot buffer is not allocated, it will stop tracing.
874 * Basically making a permanent snapshot.
876 void tracing_snapshot(void)
878 struct trace_array
*tr
= &global_trace
;
879 struct tracer
*tracer
= tr
->current_trace
;
883 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
884 internal_trace_puts("*** snapshot is being ignored ***\n");
888 if (!tr
->allocated_snapshot
) {
889 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
890 internal_trace_puts("*** stopping trace here! ***\n");
895 /* Note, snapshot can not be used when the tracer uses it */
896 if (tracer
->use_max_tr
) {
897 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
898 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
902 local_irq_save(flags
);
903 update_max_tr(tr
, current
, smp_processor_id());
904 local_irq_restore(flags
);
906 EXPORT_SYMBOL_GPL(tracing_snapshot
);
908 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
909 struct trace_buffer
*size_buf
, int cpu_id
);
910 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
912 static int alloc_snapshot(struct trace_array
*tr
)
916 if (!tr
->allocated_snapshot
) {
918 /* allocate spare buffer */
919 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
920 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
924 tr
->allocated_snapshot
= true;
930 static void free_snapshot(struct trace_array
*tr
)
933 * We don't free the ring buffer. instead, resize it because
934 * The max_tr ring buffer has some state (e.g. ring->clock) and
935 * we want preserve it.
937 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
938 set_buffer_entries(&tr
->max_buffer
, 1);
939 tracing_reset_online_cpus(&tr
->max_buffer
);
940 tr
->allocated_snapshot
= false;
944 * tracing_alloc_snapshot - allocate snapshot buffer.
946 * This only allocates the snapshot buffer if it isn't already
947 * allocated - it doesn't also take a snapshot.
949 * This is meant to be used in cases where the snapshot buffer needs
950 * to be set up for events that can't sleep but need to be able to
951 * trigger a snapshot.
953 int tracing_alloc_snapshot(void)
955 struct trace_array
*tr
= &global_trace
;
958 ret
= alloc_snapshot(tr
);
963 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
966 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
968 * This is similar to trace_snapshot(), but it will allocate the
969 * snapshot buffer if it isn't already allocated. Use this only
970 * where it is safe to sleep, as the allocation may sleep.
972 * This causes a swap between the snapshot buffer and the current live
973 * tracing buffer. You can use this to take snapshots of the live
974 * trace when some condition is triggered, but continue to trace.
976 void tracing_snapshot_alloc(void)
980 ret
= tracing_alloc_snapshot();
986 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
988 void tracing_snapshot(void)
990 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
992 EXPORT_SYMBOL_GPL(tracing_snapshot
);
993 int tracing_alloc_snapshot(void)
995 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
998 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
999 void tracing_snapshot_alloc(void)
1004 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
1005 #endif /* CONFIG_TRACER_SNAPSHOT */
1007 static void tracer_tracing_off(struct trace_array
*tr
)
1009 if (tr
->trace_buffer
.buffer
)
1010 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
1012 * This flag is looked at when buffers haven't been allocated
1013 * yet, or by some tracers (like irqsoff), that just want to
1014 * know if the ring buffer has been disabled, but it can handle
1015 * races of where it gets disabled but we still do a record.
1016 * As the check is in the fast path of the tracers, it is more
1017 * important to be fast than accurate.
1019 tr
->buffer_disabled
= 1;
1020 /* Make the flag seen by readers */
1025 * tracing_off - turn off tracing buffers
1027 * This function stops the tracing buffers from recording data.
1028 * It does not disable any overhead the tracers themselves may
1029 * be causing. This function simply causes all recording to
1030 * the ring buffers to fail.
1032 void tracing_off(void)
1034 tracer_tracing_off(&global_trace
);
1036 EXPORT_SYMBOL_GPL(tracing_off
);
1038 void disable_trace_on_warning(void)
1040 if (__disable_trace_on_warning
)
1045 * tracer_tracing_is_on - show real state of ring buffer enabled
1046 * @tr : the trace array to know if ring buffer is enabled
1048 * Shows real state of the ring buffer if it is enabled or not.
1050 int tracer_tracing_is_on(struct trace_array
*tr
)
1052 if (tr
->trace_buffer
.buffer
)
1053 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
1054 return !tr
->buffer_disabled
;
1058 * tracing_is_on - show state of ring buffers enabled
1060 int tracing_is_on(void)
1062 return tracer_tracing_is_on(&global_trace
);
1064 EXPORT_SYMBOL_GPL(tracing_is_on
);
1066 static int __init
set_buf_size(char *str
)
1068 unsigned long buf_size
;
1072 buf_size
= memparse(str
, &str
);
1073 /* nr_entries can not be zero */
1076 trace_buf_size
= buf_size
;
1079 __setup("trace_buf_size=", set_buf_size
);
1081 static int __init
set_tracing_thresh(char *str
)
1083 unsigned long threshold
;
1088 ret
= kstrtoul(str
, 0, &threshold
);
1091 tracing_thresh
= threshold
* 1000;
1094 __setup("tracing_thresh=", set_tracing_thresh
);
1096 unsigned long nsecs_to_usecs(unsigned long nsecs
)
1098 return nsecs
/ 1000;
1102 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1103 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1104 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1105 * of strings in the order that the enums were defined.
1110 /* These must match the bit postions in trace_iterator_flags */
1111 static const char *trace_options
[] = {
1119 int in_ns
; /* is this clock in nanoseconds? */
1120 } trace_clocks
[] = {
1121 { trace_clock_local
, "local", 1 },
1122 { trace_clock_global
, "global", 1 },
1123 { trace_clock_counter
, "counter", 0 },
1124 { trace_clock_jiffies
, "uptime", 0 },
1125 { trace_clock
, "perf", 1 },
1126 { ktime_get_mono_fast_ns
, "mono", 1 },
1127 { ktime_get_raw_fast_ns
, "mono_raw", 1 },
1128 { ktime_get_boot_fast_ns
, "boot", 1 },
1133 * trace_parser_get_init - gets the buffer for trace parser
1135 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
1137 memset(parser
, 0, sizeof(*parser
));
1139 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
1140 if (!parser
->buffer
)
1143 parser
->size
= size
;
1148 * trace_parser_put - frees the buffer for trace parser
1150 void trace_parser_put(struct trace_parser
*parser
)
1152 kfree(parser
->buffer
);
1156 * trace_get_user - reads the user input string separated by space
1157 * (matched by isspace(ch))
1159 * For each string found the 'struct trace_parser' is updated,
1160 * and the function returns.
1162 * Returns number of bytes read.
1164 * See kernel/trace/trace.h for 'struct trace_parser' details.
1166 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
1167 size_t cnt
, loff_t
*ppos
)
1174 trace_parser_clear(parser
);
1176 ret
= get_user(ch
, ubuf
++);
1184 * The parser is not finished with the last write,
1185 * continue reading the user input without skipping spaces.
1187 if (!parser
->cont
) {
1188 /* skip white space */
1189 while (cnt
&& isspace(ch
)) {
1190 ret
= get_user(ch
, ubuf
++);
1197 /* only spaces were written */
1207 /* read the non-space input */
1208 while (cnt
&& !isspace(ch
)) {
1209 if (parser
->idx
< parser
->size
- 1)
1210 parser
->buffer
[parser
->idx
++] = ch
;
1215 ret
= get_user(ch
, ubuf
++);
1222 /* We either got finished input or we have to wait for another call. */
1224 parser
->buffer
[parser
->idx
] = 0;
1225 parser
->cont
= false;
1226 } else if (parser
->idx
< parser
->size
- 1) {
1227 parser
->cont
= true;
1228 parser
->buffer
[parser
->idx
++] = ch
;
1241 /* TODO add a seq_buf_to_buffer() */
1242 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
1246 if (trace_seq_used(s
) <= s
->seq
.readpos
)
1249 len
= trace_seq_used(s
) - s
->seq
.readpos
;
1252 memcpy(buf
, s
->buffer
+ s
->seq
.readpos
, cnt
);
1254 s
->seq
.readpos
+= cnt
;
1258 unsigned long __read_mostly tracing_thresh
;
1260 #ifdef CONFIG_TRACER_MAX_TRACE
1262 * Copy the new maximum trace into the separate maximum-trace
1263 * structure. (this way the maximum trace is permanently saved,
1264 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1267 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1269 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
1270 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
1271 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
1272 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
1275 max_buf
->time_start
= data
->preempt_timestamp
;
1277 max_data
->saved_latency
= tr
->max_latency
;
1278 max_data
->critical_start
= data
->critical_start
;
1279 max_data
->critical_end
= data
->critical_end
;
1281 memcpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
1282 max_data
->pid
= tsk
->pid
;
1284 * If tsk == current, then use current_uid(), as that does not use
1285 * RCU. The irq tracer can be called out of RCU scope.
1288 max_data
->uid
= current_uid();
1290 max_data
->uid
= task_uid(tsk
);
1292 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
1293 max_data
->policy
= tsk
->policy
;
1294 max_data
->rt_priority
= tsk
->rt_priority
;
1296 /* record this tasks comm */
1297 tracing_record_cmdline(tsk
);
1301 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1303 * @tsk: the task with the latency
1304 * @cpu: The cpu that initiated the trace.
1306 * Flip the buffers between the @tr and the max_tr and record information
1307 * about which task was the cause of this latency.
1310 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1312 struct ring_buffer
*buf
;
1317 WARN_ON_ONCE(!irqs_disabled());
1319 if (!tr
->allocated_snapshot
) {
1320 /* Only the nop tracer should hit this when disabling */
1321 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1325 arch_spin_lock(&tr
->max_lock
);
1327 buf
= tr
->trace_buffer
.buffer
;
1328 tr
->trace_buffer
.buffer
= tr
->max_buffer
.buffer
;
1329 tr
->max_buffer
.buffer
= buf
;
1331 __update_max_tr(tr
, tsk
, cpu
);
1332 arch_spin_unlock(&tr
->max_lock
);
1336 * update_max_tr_single - only copy one trace over, and reset the rest
1338 * @tsk - task with the latency
1339 * @cpu - the cpu of the buffer to copy.
1341 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1344 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1351 WARN_ON_ONCE(!irqs_disabled());
1352 if (!tr
->allocated_snapshot
) {
1353 /* Only the nop tracer should hit this when disabling */
1354 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1358 arch_spin_lock(&tr
->max_lock
);
1360 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1362 if (ret
== -EBUSY
) {
1364 * We failed to swap the buffer due to a commit taking
1365 * place on this CPU. We fail to record, but we reset
1366 * the max trace buffer (no one writes directly to it)
1367 * and flag that it failed.
1369 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1370 "Failed to swap buffers due to commit in progress\n");
1373 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1375 __update_max_tr(tr
, tsk
, cpu
);
1376 arch_spin_unlock(&tr
->max_lock
);
1378 #endif /* CONFIG_TRACER_MAX_TRACE */
1380 static int wait_on_pipe(struct trace_iterator
*iter
, bool full
)
1382 /* Iterators are static, they should be filled or empty */
1383 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1386 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
1390 #ifdef CONFIG_FTRACE_STARTUP_TEST
1391 static int run_tracer_selftest(struct tracer
*type
)
1393 struct trace_array
*tr
= &global_trace
;
1394 struct tracer
*saved_tracer
= tr
->current_trace
;
1397 if (!type
->selftest
|| tracing_selftest_disabled
)
1401 * Run a selftest on this tracer.
1402 * Here we reset the trace buffer, and set the current
1403 * tracer to be this tracer. The tracer can then run some
1404 * internal tracing to verify that everything is in order.
1405 * If we fail, we do not register this tracer.
1407 tracing_reset_online_cpus(&tr
->trace_buffer
);
1409 tr
->current_trace
= type
;
1411 #ifdef CONFIG_TRACER_MAX_TRACE
1412 if (type
->use_max_tr
) {
1413 /* If we expanded the buffers, make sure the max is expanded too */
1414 if (ring_buffer_expanded
)
1415 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1416 RING_BUFFER_ALL_CPUS
);
1417 tr
->allocated_snapshot
= true;
1421 /* the test is responsible for initializing and enabling */
1422 pr_info("Testing tracer %s: ", type
->name
);
1423 ret
= type
->selftest(type
, tr
);
1424 /* the test is responsible for resetting too */
1425 tr
->current_trace
= saved_tracer
;
1427 printk(KERN_CONT
"FAILED!\n");
1428 /* Add the warning after printing 'FAILED' */
1432 /* Only reset on passing, to avoid touching corrupted buffers */
1433 tracing_reset_online_cpus(&tr
->trace_buffer
);
1435 #ifdef CONFIG_TRACER_MAX_TRACE
1436 if (type
->use_max_tr
) {
1437 tr
->allocated_snapshot
= false;
1439 /* Shrink the max buffer again */
1440 if (ring_buffer_expanded
)
1441 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1442 RING_BUFFER_ALL_CPUS
);
1446 printk(KERN_CONT
"PASSED\n");
1450 static inline int run_tracer_selftest(struct tracer
*type
)
1454 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1456 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
);
1458 static void __init
apply_trace_boot_options(void);
1461 * register_tracer - register a tracer with the ftrace system.
1462 * @type - the plugin for the tracer
1464 * Register a new plugin tracer.
1466 int __init
register_tracer(struct tracer
*type
)
1472 pr_info("Tracer must have a name\n");
1476 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1477 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1481 mutex_lock(&trace_types_lock
);
1483 tracing_selftest_running
= true;
1485 for (t
= trace_types
; t
; t
= t
->next
) {
1486 if (strcmp(type
->name
, t
->name
) == 0) {
1488 pr_info("Tracer %s already registered\n",
1495 if (!type
->set_flag
)
1496 type
->set_flag
= &dummy_set_flag
;
1498 /*allocate a dummy tracer_flags*/
1499 type
->flags
= kmalloc(sizeof(*type
->flags
), GFP_KERNEL
);
1504 type
->flags
->val
= 0;
1505 type
->flags
->opts
= dummy_tracer_opt
;
1507 if (!type
->flags
->opts
)
1508 type
->flags
->opts
= dummy_tracer_opt
;
1510 /* store the tracer for __set_tracer_option */
1511 type
->flags
->trace
= type
;
1513 ret
= run_tracer_selftest(type
);
1517 type
->next
= trace_types
;
1519 add_tracer_options(&global_trace
, type
);
1522 tracing_selftest_running
= false;
1523 mutex_unlock(&trace_types_lock
);
1525 if (ret
|| !default_bootup_tracer
)
1528 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1531 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1532 /* Do we want this tracer to start on bootup? */
1533 tracing_set_tracer(&global_trace
, type
->name
);
1534 default_bootup_tracer
= NULL
;
1536 apply_trace_boot_options();
1538 /* disable other selftests, since this will break it. */
1539 tracing_selftest_disabled
= true;
1540 #ifdef CONFIG_FTRACE_STARTUP_TEST
1541 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1549 void tracing_reset(struct trace_buffer
*buf
, int cpu
)
1551 struct ring_buffer
*buffer
= buf
->buffer
;
1556 ring_buffer_record_disable(buffer
);
1558 /* Make sure all commits have finished */
1559 synchronize_sched();
1560 ring_buffer_reset_cpu(buffer
, cpu
);
1562 ring_buffer_record_enable(buffer
);
1565 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1567 struct ring_buffer
*buffer
= buf
->buffer
;
1573 ring_buffer_record_disable(buffer
);
1575 /* Make sure all commits have finished */
1576 synchronize_sched();
1578 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1580 for_each_online_cpu(cpu
)
1581 ring_buffer_reset_cpu(buffer
, cpu
);
1583 ring_buffer_record_enable(buffer
);
1586 /* Must have trace_types_lock held */
1587 void tracing_reset_all_online_cpus(void)
1589 struct trace_array
*tr
;
1591 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1592 tracing_reset_online_cpus(&tr
->trace_buffer
);
1593 #ifdef CONFIG_TRACER_MAX_TRACE
1594 tracing_reset_online_cpus(&tr
->max_buffer
);
1599 #define SAVED_CMDLINES_DEFAULT 128
1600 #define NO_CMDLINE_MAP UINT_MAX
1601 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1602 struct saved_cmdlines_buffer
{
1603 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1604 unsigned *map_cmdline_to_pid
;
1605 unsigned cmdline_num
;
1607 char *saved_cmdlines
;
1609 static struct saved_cmdlines_buffer
*savedcmd
;
1611 /* temporary disable recording */
1612 static atomic_t trace_record_cmdline_disabled __read_mostly
;
1614 static inline char *get_saved_cmdlines(int idx
)
1616 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1619 static inline void set_cmdline(int idx
, const char *cmdline
)
1621 memcpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1624 static int allocate_cmdlines_buffer(unsigned int val
,
1625 struct saved_cmdlines_buffer
*s
)
1627 s
->map_cmdline_to_pid
= kmalloc(val
* sizeof(*s
->map_cmdline_to_pid
),
1629 if (!s
->map_cmdline_to_pid
)
1632 s
->saved_cmdlines
= kmalloc(val
* TASK_COMM_LEN
, GFP_KERNEL
);
1633 if (!s
->saved_cmdlines
) {
1634 kfree(s
->map_cmdline_to_pid
);
1639 s
->cmdline_num
= val
;
1640 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1641 sizeof(s
->map_pid_to_cmdline
));
1642 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1643 val
* sizeof(*s
->map_cmdline_to_pid
));
1648 static int trace_create_savedcmd(void)
1652 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1656 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
1666 int is_tracing_stopped(void)
1668 return global_trace
.stop_count
;
1672 * tracing_start - quick start of the tracer
1674 * If tracing is enabled but was stopped by tracing_stop,
1675 * this will start the tracer back up.
1677 void tracing_start(void)
1679 struct ring_buffer
*buffer
;
1680 unsigned long flags
;
1682 if (tracing_disabled
)
1685 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1686 if (--global_trace
.stop_count
) {
1687 if (global_trace
.stop_count
< 0) {
1688 /* Someone screwed up their debugging */
1690 global_trace
.stop_count
= 0;
1695 /* Prevent the buffers from switching */
1696 arch_spin_lock(&global_trace
.max_lock
);
1698 buffer
= global_trace
.trace_buffer
.buffer
;
1700 ring_buffer_record_enable(buffer
);
1702 #ifdef CONFIG_TRACER_MAX_TRACE
1703 buffer
= global_trace
.max_buffer
.buffer
;
1705 ring_buffer_record_enable(buffer
);
1708 arch_spin_unlock(&global_trace
.max_lock
);
1711 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1714 static void tracing_start_tr(struct trace_array
*tr
)
1716 struct ring_buffer
*buffer
;
1717 unsigned long flags
;
1719 if (tracing_disabled
)
1722 /* If global, we need to also start the max tracer */
1723 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1724 return tracing_start();
1726 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1728 if (--tr
->stop_count
) {
1729 if (tr
->stop_count
< 0) {
1730 /* Someone screwed up their debugging */
1737 buffer
= tr
->trace_buffer
.buffer
;
1739 ring_buffer_record_enable(buffer
);
1742 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1746 * tracing_stop - quick stop of the tracer
1748 * Light weight way to stop tracing. Use in conjunction with
1751 void tracing_stop(void)
1753 struct ring_buffer
*buffer
;
1754 unsigned long flags
;
1756 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1757 if (global_trace
.stop_count
++)
1760 /* Prevent the buffers from switching */
1761 arch_spin_lock(&global_trace
.max_lock
);
1763 buffer
= global_trace
.trace_buffer
.buffer
;
1765 ring_buffer_record_disable(buffer
);
1767 #ifdef CONFIG_TRACER_MAX_TRACE
1768 buffer
= global_trace
.max_buffer
.buffer
;
1770 ring_buffer_record_disable(buffer
);
1773 arch_spin_unlock(&global_trace
.max_lock
);
1776 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1779 static void tracing_stop_tr(struct trace_array
*tr
)
1781 struct ring_buffer
*buffer
;
1782 unsigned long flags
;
1784 /* If global, we need to also stop the max tracer */
1785 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1786 return tracing_stop();
1788 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1789 if (tr
->stop_count
++)
1792 buffer
= tr
->trace_buffer
.buffer
;
1794 ring_buffer_record_disable(buffer
);
1797 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1800 void trace_stop_cmdline_recording(void);
1802 static int trace_save_cmdline(struct task_struct
*tsk
)
1806 if (!tsk
->pid
|| unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
1810 * It's not the end of the world if we don't get
1811 * the lock, but we also don't want to spin
1812 * nor do we want to disable interrupts,
1813 * so if we miss here, then better luck next time.
1815 if (!arch_spin_trylock(&trace_cmdline_lock
))
1818 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
1819 if (idx
== NO_CMDLINE_MAP
) {
1820 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
1823 * Check whether the cmdline buffer at idx has a pid
1824 * mapped. We are going to overwrite that entry so we
1825 * need to clear the map_pid_to_cmdline. Otherwise we
1826 * would read the new comm for the old pid.
1828 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
1829 if (pid
!= NO_CMDLINE_MAP
)
1830 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
1832 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
1833 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
1835 savedcmd
->cmdline_idx
= idx
;
1838 set_cmdline(idx
, tsk
->comm
);
1840 arch_spin_unlock(&trace_cmdline_lock
);
1845 static void __trace_find_cmdline(int pid
, char comm
[])
1850 strcpy(comm
, "<idle>");
1854 if (WARN_ON_ONCE(pid
< 0)) {
1855 strcpy(comm
, "<XXX>");
1859 if (pid
> PID_MAX_DEFAULT
) {
1860 strcpy(comm
, "<...>");
1864 map
= savedcmd
->map_pid_to_cmdline
[pid
];
1865 if (map
!= NO_CMDLINE_MAP
)
1866 strcpy(comm
, get_saved_cmdlines(map
));
1868 strcpy(comm
, "<...>");
1871 void trace_find_cmdline(int pid
, char comm
[])
1874 arch_spin_lock(&trace_cmdline_lock
);
1876 __trace_find_cmdline(pid
, comm
);
1878 arch_spin_unlock(&trace_cmdline_lock
);
1882 void tracing_record_cmdline(struct task_struct
*tsk
)
1884 if (atomic_read(&trace_record_cmdline_disabled
) || !tracing_is_on())
1887 if (!__this_cpu_read(trace_cmdline_save
))
1890 if (trace_save_cmdline(tsk
))
1891 __this_cpu_write(trace_cmdline_save
, false);
1895 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned long flags
,
1898 struct task_struct
*tsk
= current
;
1900 entry
->preempt_count
= pc
& 0xff;
1901 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
1903 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1904 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
1906 TRACE_FLAG_IRQS_NOSUPPORT
|
1908 ((pc
& NMI_MASK
) ? TRACE_FLAG_NMI
: 0) |
1909 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
1910 ((pc
& SOFTIRQ_MASK
) ? TRACE_FLAG_SOFTIRQ
: 0) |
1911 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
1912 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
1914 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
1916 static __always_inline
void
1917 trace_event_setup(struct ring_buffer_event
*event
,
1918 int type
, unsigned long flags
, int pc
)
1920 struct trace_entry
*ent
= ring_buffer_event_data(event
);
1922 tracing_generic_entry_update(ent
, flags
, pc
);
1926 struct ring_buffer_event
*
1927 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
1930 unsigned long flags
, int pc
)
1932 struct ring_buffer_event
*event
;
1934 event
= ring_buffer_lock_reserve(buffer
, len
);
1936 trace_event_setup(event
, type
, flags
, pc
);
1941 DEFINE_PER_CPU(struct ring_buffer_event
*, trace_buffered_event
);
1942 DEFINE_PER_CPU(int, trace_buffered_event_cnt
);
1943 static int trace_buffered_event_ref
;
1946 * trace_buffered_event_enable - enable buffering events
1948 * When events are being filtered, it is quicker to use a temporary
1949 * buffer to write the event data into if there's a likely chance
1950 * that it will not be committed. The discard of the ring buffer
1951 * is not as fast as committing, and is much slower than copying
1954 * When an event is to be filtered, allocate per cpu buffers to
1955 * write the event data into, and if the event is filtered and discarded
1956 * it is simply dropped, otherwise, the entire data is to be committed
1959 void trace_buffered_event_enable(void)
1961 struct ring_buffer_event
*event
;
1965 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
1967 if (trace_buffered_event_ref
++)
1970 for_each_tracing_cpu(cpu
) {
1971 page
= alloc_pages_node(cpu_to_node(cpu
),
1972 GFP_KERNEL
| __GFP_NORETRY
, 0);
1976 event
= page_address(page
);
1977 memset(event
, 0, sizeof(*event
));
1979 per_cpu(trace_buffered_event
, cpu
) = event
;
1982 if (cpu
== smp_processor_id() &&
1983 this_cpu_read(trace_buffered_event
) !=
1984 per_cpu(trace_buffered_event
, cpu
))
1991 trace_buffered_event_disable();
1994 static void enable_trace_buffered_event(void *data
)
1996 /* Probably not needed, but do it anyway */
1998 this_cpu_dec(trace_buffered_event_cnt
);
2001 static void disable_trace_buffered_event(void *data
)
2003 this_cpu_inc(trace_buffered_event_cnt
);
2007 * trace_buffered_event_disable - disable buffering events
2009 * When a filter is removed, it is faster to not use the buffered
2010 * events, and to commit directly into the ring buffer. Free up
2011 * the temp buffers when there are no more users. This requires
2012 * special synchronization with current events.
2014 void trace_buffered_event_disable(void)
2018 WARN_ON_ONCE(!mutex_is_locked(&event_mutex
));
2020 if (WARN_ON_ONCE(!trace_buffered_event_ref
))
2023 if (--trace_buffered_event_ref
)
2027 /* For each CPU, set the buffer as used. */
2028 smp_call_function_many(tracing_buffer_mask
,
2029 disable_trace_buffered_event
, NULL
, 1);
2032 /* Wait for all current users to finish */
2033 synchronize_sched();
2035 for_each_tracing_cpu(cpu
) {
2036 free_page((unsigned long)per_cpu(trace_buffered_event
, cpu
));
2037 per_cpu(trace_buffered_event
, cpu
) = NULL
;
2040 * Make sure trace_buffered_event is NULL before clearing
2041 * trace_buffered_event_cnt.
2046 /* Do the work on each cpu */
2047 smp_call_function_many(tracing_buffer_mask
,
2048 enable_trace_buffered_event
, NULL
, 1);
2053 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
2055 __this_cpu_write(trace_cmdline_save
, true);
2057 /* If this is the temp buffer, we need to commit fully */
2058 if (this_cpu_read(trace_buffered_event
) == event
) {
2059 /* Length is in event->array[0] */
2060 ring_buffer_write(buffer
, event
->array
[0], &event
->array
[1]);
2061 /* Release the temp buffer */
2062 this_cpu_dec(trace_buffered_event_cnt
);
2064 ring_buffer_unlock_commit(buffer
, event
);
2067 static struct ring_buffer
*temp_buffer
;
2069 struct ring_buffer_event
*
2070 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
2071 struct trace_event_file
*trace_file
,
2072 int type
, unsigned long len
,
2073 unsigned long flags
, int pc
)
2075 struct ring_buffer_event
*entry
;
2078 *current_rb
= trace_file
->tr
->trace_buffer
.buffer
;
2080 if ((trace_file
->flags
&
2081 (EVENT_FILE_FL_SOFT_DISABLED
| EVENT_FILE_FL_FILTERED
)) &&
2082 (entry
= this_cpu_read(trace_buffered_event
))) {
2083 /* Try to use the per cpu buffer first */
2084 val
= this_cpu_inc_return(trace_buffered_event_cnt
);
2086 trace_event_setup(entry
, type
, flags
, pc
);
2087 entry
->array
[0] = len
;
2090 this_cpu_dec(trace_buffered_event_cnt
);
2093 entry
= trace_buffer_lock_reserve(*current_rb
,
2094 type
, len
, flags
, pc
);
2096 * If tracing is off, but we have triggers enabled
2097 * we still need to look at the event data. Use the temp_buffer
2098 * to store the trace event for the tigger to use. It's recusive
2099 * safe and will not be recorded anywhere.
2101 if (!entry
&& trace_file
->flags
& EVENT_FILE_FL_TRIGGER_COND
) {
2102 *current_rb
= temp_buffer
;
2103 entry
= trace_buffer_lock_reserve(*current_rb
,
2104 type
, len
, flags
, pc
);
2108 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
2110 void trace_buffer_unlock_commit_regs(struct trace_array
*tr
,
2111 struct ring_buffer
*buffer
,
2112 struct ring_buffer_event
*event
,
2113 unsigned long flags
, int pc
,
2114 struct pt_regs
*regs
)
2116 __buffer_unlock_commit(buffer
, event
);
2119 * If regs is not set, then skip the following callers:
2120 * trace_buffer_unlock_commit_regs
2121 * event_trigger_unlock_commit
2122 * trace_event_buffer_commit
2123 * trace_event_raw_event_sched_switch
2124 * Note, we can still get here via blktrace, wakeup tracer
2125 * and mmiotrace, but that's ok if they lose a function or
2126 * two. They are that meaningful.
2128 ftrace_trace_stack(tr
, buffer
, flags
, regs
? 0 : 4, pc
, regs
);
2129 ftrace_trace_userstack(buffer
, flags
, pc
);
2133 trace_function(struct trace_array
*tr
,
2134 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
2137 struct trace_event_call
*call
= &event_function
;
2138 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
2139 struct ring_buffer_event
*event
;
2140 struct ftrace_entry
*entry
;
2142 event
= trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
2146 entry
= ring_buffer_event_data(event
);
2148 entry
->parent_ip
= parent_ip
;
2150 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2151 __buffer_unlock_commit(buffer
, event
);
2154 #ifdef CONFIG_STACKTRACE
2156 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2157 struct ftrace_stack
{
2158 unsigned long calls
[FTRACE_STACK_MAX_ENTRIES
];
2161 static DEFINE_PER_CPU(struct ftrace_stack
, ftrace_stack
);
2162 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
2164 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
2165 unsigned long flags
,
2166 int skip
, int pc
, struct pt_regs
*regs
)
2168 struct trace_event_call
*call
= &event_kernel_stack
;
2169 struct ring_buffer_event
*event
;
2170 struct stack_entry
*entry
;
2171 struct stack_trace trace
;
2173 int size
= FTRACE_STACK_ENTRIES
;
2175 trace
.nr_entries
= 0;
2179 * Add two, for this function and the call to save_stack_trace()
2180 * If regs is set, then these functions will not be in the way.
2186 * Since events can happen in NMIs there's no safe way to
2187 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2188 * or NMI comes in, it will just have to use the default
2189 * FTRACE_STACK_SIZE.
2191 preempt_disable_notrace();
2193 use_stack
= __this_cpu_inc_return(ftrace_stack_reserve
);
2195 * We don't need any atomic variables, just a barrier.
2196 * If an interrupt comes in, we don't care, because it would
2197 * have exited and put the counter back to what we want.
2198 * We just need a barrier to keep gcc from moving things
2202 if (use_stack
== 1) {
2203 trace
.entries
= this_cpu_ptr(ftrace_stack
.calls
);
2204 trace
.max_entries
= FTRACE_STACK_MAX_ENTRIES
;
2207 save_stack_trace_regs(regs
, &trace
);
2209 save_stack_trace(&trace
);
2211 if (trace
.nr_entries
> size
)
2212 size
= trace
.nr_entries
;
2214 /* From now on, use_stack is a boolean */
2217 size
*= sizeof(unsigned long);
2219 event
= trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
2220 sizeof(*entry
) + size
, flags
, pc
);
2223 entry
= ring_buffer_event_data(event
);
2225 memset(&entry
->caller
, 0, size
);
2228 memcpy(&entry
->caller
, trace
.entries
,
2229 trace
.nr_entries
* sizeof(unsigned long));
2231 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
2232 trace
.entries
= entry
->caller
;
2234 save_stack_trace_regs(regs
, &trace
);
2236 save_stack_trace(&trace
);
2239 entry
->size
= trace
.nr_entries
;
2241 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2242 __buffer_unlock_commit(buffer
, event
);
2245 /* Again, don't let gcc optimize things here */
2247 __this_cpu_dec(ftrace_stack_reserve
);
2248 preempt_enable_notrace();
2252 static inline void ftrace_trace_stack(struct trace_array
*tr
,
2253 struct ring_buffer
*buffer
,
2254 unsigned long flags
,
2255 int skip
, int pc
, struct pt_regs
*regs
)
2257 if (!(tr
->trace_flags
& TRACE_ITER_STACKTRACE
))
2260 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
2263 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
2266 __ftrace_trace_stack(tr
->trace_buffer
.buffer
, flags
, skip
, pc
, NULL
);
2270 * trace_dump_stack - record a stack back trace in the trace buffer
2271 * @skip: Number of functions to skip (helper handlers)
2273 void trace_dump_stack(int skip
)
2275 unsigned long flags
;
2277 if (tracing_disabled
|| tracing_selftest_running
)
2280 local_save_flags(flags
);
2283 * Skip 3 more, seems to get us at the caller of
2287 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
2288 flags
, skip
, preempt_count(), NULL
);
2291 static DEFINE_PER_CPU(int, user_stack_count
);
2294 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
2296 struct trace_event_call
*call
= &event_user_stack
;
2297 struct ring_buffer_event
*event
;
2298 struct userstack_entry
*entry
;
2299 struct stack_trace trace
;
2301 if (!(global_trace
.trace_flags
& TRACE_ITER_USERSTACKTRACE
))
2305 * NMIs can not handle page faults, even with fix ups.
2306 * The save user stack can (and often does) fault.
2308 if (unlikely(in_nmi()))
2312 * prevent recursion, since the user stack tracing may
2313 * trigger other kernel events.
2316 if (__this_cpu_read(user_stack_count
))
2319 __this_cpu_inc(user_stack_count
);
2321 event
= trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
2322 sizeof(*entry
), flags
, pc
);
2324 goto out_drop_count
;
2325 entry
= ring_buffer_event_data(event
);
2327 entry
->tgid
= current
->tgid
;
2328 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
2330 trace
.nr_entries
= 0;
2331 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
2333 trace
.entries
= entry
->caller
;
2335 save_stack_trace_user(&trace
);
2336 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
2337 __buffer_unlock_commit(buffer
, event
);
2340 __this_cpu_dec(user_stack_count
);
2346 static void __trace_userstack(struct trace_array
*tr
, unsigned long flags
)
2348 ftrace_trace_userstack(tr
, flags
, preempt_count());
2352 #endif /* CONFIG_STACKTRACE */
2354 /* created for use with alloc_percpu */
2355 struct trace_buffer_struct
{
2357 char buffer
[4][TRACE_BUF_SIZE
];
2360 static struct trace_buffer_struct
*trace_percpu_buffer
;
2363 * Thise allows for lockless recording. If we're nested too deeply, then
2364 * this returns NULL.
2366 static char *get_trace_buf(void)
2368 struct trace_buffer_struct
*buffer
= this_cpu_ptr(trace_percpu_buffer
);
2370 if (!buffer
|| buffer
->nesting
>= 4)
2373 return &buffer
->buffer
[buffer
->nesting
++][0];
2376 static void put_trace_buf(void)
2378 this_cpu_dec(trace_percpu_buffer
->nesting
);
2381 static int alloc_percpu_trace_buffer(void)
2383 struct trace_buffer_struct
*buffers
;
2385 buffers
= alloc_percpu(struct trace_buffer_struct
);
2386 if (WARN(!buffers
, "Could not allocate percpu trace_printk buffer"))
2389 trace_percpu_buffer
= buffers
;
2393 static int buffers_allocated
;
2395 void trace_printk_init_buffers(void)
2397 if (buffers_allocated
)
2400 if (alloc_percpu_trace_buffer())
2403 /* trace_printk() is for debug use only. Don't use it in production. */
2406 pr_warn("**********************************************************\n");
2407 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2409 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2411 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2412 pr_warn("** unsafe for production use. **\n");
2414 pr_warn("** If you see this message and you are not debugging **\n");
2415 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2417 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2418 pr_warn("**********************************************************\n");
2420 /* Expand the buffers to set size */
2421 tracing_update_buffers();
2423 buffers_allocated
= 1;
2426 * trace_printk_init_buffers() can be called by modules.
2427 * If that happens, then we need to start cmdline recording
2428 * directly here. If the global_trace.buffer is already
2429 * allocated here, then this was called by module code.
2431 if (global_trace
.trace_buffer
.buffer
)
2432 tracing_start_cmdline_record();
2435 void trace_printk_start_comm(void)
2437 /* Start tracing comms if trace printk is set */
2438 if (!buffers_allocated
)
2440 tracing_start_cmdline_record();
2443 static void trace_printk_start_stop_comm(int enabled
)
2445 if (!buffers_allocated
)
2449 tracing_start_cmdline_record();
2451 tracing_stop_cmdline_record();
2455 * trace_vbprintk - write binary msg to tracing buffer
2458 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
2460 struct trace_event_call
*call
= &event_bprint
;
2461 struct ring_buffer_event
*event
;
2462 struct ring_buffer
*buffer
;
2463 struct trace_array
*tr
= &global_trace
;
2464 struct bprint_entry
*entry
;
2465 unsigned long flags
;
2467 int len
= 0, size
, pc
;
2469 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
2472 /* Don't pollute graph traces with trace_vprintk internals */
2473 pause_graph_tracing();
2475 pc
= preempt_count();
2476 preempt_disable_notrace();
2478 tbuffer
= get_trace_buf();
2484 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
2486 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
2489 local_save_flags(flags
);
2490 size
= sizeof(*entry
) + sizeof(u32
) * len
;
2491 buffer
= tr
->trace_buffer
.buffer
;
2492 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
2496 entry
= ring_buffer_event_data(event
);
2500 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
2501 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2502 __buffer_unlock_commit(buffer
, event
);
2503 ftrace_trace_stack(tr
, buffer
, flags
, 6, pc
, NULL
);
2510 preempt_enable_notrace();
2511 unpause_graph_tracing();
2515 EXPORT_SYMBOL_GPL(trace_vbprintk
);
2518 __trace_array_vprintk(struct ring_buffer
*buffer
,
2519 unsigned long ip
, const char *fmt
, va_list args
)
2521 struct trace_event_call
*call
= &event_print
;
2522 struct ring_buffer_event
*event
;
2523 int len
= 0, size
, pc
;
2524 struct print_entry
*entry
;
2525 unsigned long flags
;
2528 if (tracing_disabled
|| tracing_selftest_running
)
2531 /* Don't pollute graph traces with trace_vprintk internals */
2532 pause_graph_tracing();
2534 pc
= preempt_count();
2535 preempt_disable_notrace();
2538 tbuffer
= get_trace_buf();
2544 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
2546 local_save_flags(flags
);
2547 size
= sizeof(*entry
) + len
+ 1;
2548 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
2552 entry
= ring_buffer_event_data(event
);
2555 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
2556 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2557 __buffer_unlock_commit(buffer
, event
);
2558 ftrace_trace_stack(&global_trace
, buffer
, flags
, 6, pc
, NULL
);
2565 preempt_enable_notrace();
2566 unpause_graph_tracing();
2571 int trace_array_vprintk(struct trace_array
*tr
,
2572 unsigned long ip
, const char *fmt
, va_list args
)
2574 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
2577 int trace_array_printk(struct trace_array
*tr
,
2578 unsigned long ip
, const char *fmt
, ...)
2583 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2587 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
2592 int trace_array_printk_buf(struct ring_buffer
*buffer
,
2593 unsigned long ip
, const char *fmt
, ...)
2598 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2602 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
2607 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
2609 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
2611 EXPORT_SYMBOL_GPL(trace_vprintk
);
2613 static void trace_iterator_increment(struct trace_iterator
*iter
)
2615 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
2619 ring_buffer_read(buf_iter
, NULL
);
2622 static struct trace_entry
*
2623 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
2624 unsigned long *lost_events
)
2626 struct ring_buffer_event
*event
;
2627 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
2630 event
= ring_buffer_iter_peek(buf_iter
, ts
);
2632 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
2636 iter
->ent_size
= ring_buffer_event_length(event
);
2637 return ring_buffer_event_data(event
);
2643 static struct trace_entry
*
2644 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
2645 unsigned long *missing_events
, u64
*ent_ts
)
2647 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
2648 struct trace_entry
*ent
, *next
= NULL
;
2649 unsigned long lost_events
= 0, next_lost
= 0;
2650 int cpu_file
= iter
->cpu_file
;
2651 u64 next_ts
= 0, ts
;
2657 * If we are in a per_cpu trace file, don't bother by iterating over
2658 * all cpu and peek directly.
2660 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
2661 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
2663 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
2665 *ent_cpu
= cpu_file
;
2670 for_each_tracing_cpu(cpu
) {
2672 if (ring_buffer_empty_cpu(buffer
, cpu
))
2675 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
2678 * Pick the entry with the smallest timestamp:
2680 if (ent
&& (!next
|| ts
< next_ts
)) {
2684 next_lost
= lost_events
;
2685 next_size
= iter
->ent_size
;
2689 iter
->ent_size
= next_size
;
2692 *ent_cpu
= next_cpu
;
2698 *missing_events
= next_lost
;
2703 /* Find the next real entry, without updating the iterator itself */
2704 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
2705 int *ent_cpu
, u64
*ent_ts
)
2707 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
2710 /* Find the next real entry, and increment the iterator to the next entry */
2711 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
2713 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
2714 &iter
->lost_events
, &iter
->ts
);
2717 trace_iterator_increment(iter
);
2719 return iter
->ent
? iter
: NULL
;
2722 static void trace_consume(struct trace_iterator
*iter
)
2724 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
2725 &iter
->lost_events
);
2728 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2730 struct trace_iterator
*iter
= m
->private;
2734 WARN_ON_ONCE(iter
->leftover
);
2738 /* can't go backwards */
2743 ent
= trace_find_next_entry_inc(iter
);
2747 while (ent
&& iter
->idx
< i
)
2748 ent
= trace_find_next_entry_inc(iter
);
2755 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
2757 struct ring_buffer_event
*event
;
2758 struct ring_buffer_iter
*buf_iter
;
2759 unsigned long entries
= 0;
2762 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
2764 buf_iter
= trace_buffer_iter(iter
, cpu
);
2768 ring_buffer_iter_reset(buf_iter
);
2771 * We could have the case with the max latency tracers
2772 * that a reset never took place on a cpu. This is evident
2773 * by the timestamp being before the start of the buffer.
2775 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
2776 if (ts
>= iter
->trace_buffer
->time_start
)
2779 ring_buffer_read(buf_iter
, NULL
);
2782 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
2786 * The current tracer is copied to avoid a global locking
2789 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
2791 struct trace_iterator
*iter
= m
->private;
2792 struct trace_array
*tr
= iter
->tr
;
2793 int cpu_file
= iter
->cpu_file
;
2799 * copy the tracer to avoid using a global lock all around.
2800 * iter->trace is a copy of current_trace, the pointer to the
2801 * name may be used instead of a strcmp(), as iter->trace->name
2802 * will point to the same string as current_trace->name.
2804 mutex_lock(&trace_types_lock
);
2805 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
2806 *iter
->trace
= *tr
->current_trace
;
2807 mutex_unlock(&trace_types_lock
);
2809 #ifdef CONFIG_TRACER_MAX_TRACE
2810 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2811 return ERR_PTR(-EBUSY
);
2814 if (!iter
->snapshot
)
2815 atomic_inc(&trace_record_cmdline_disabled
);
2817 if (*pos
!= iter
->pos
) {
2822 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
2823 for_each_tracing_cpu(cpu
)
2824 tracing_iter_reset(iter
, cpu
);
2826 tracing_iter_reset(iter
, cpu_file
);
2829 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
2834 * If we overflowed the seq_file before, then we want
2835 * to just reuse the trace_seq buffer again.
2841 p
= s_next(m
, p
, &l
);
2845 trace_event_read_lock();
2846 trace_access_lock(cpu_file
);
2850 static void s_stop(struct seq_file
*m
, void *p
)
2852 struct trace_iterator
*iter
= m
->private;
2854 #ifdef CONFIG_TRACER_MAX_TRACE
2855 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2859 if (!iter
->snapshot
)
2860 atomic_dec(&trace_record_cmdline_disabled
);
2862 trace_access_unlock(iter
->cpu_file
);
2863 trace_event_read_unlock();
2867 get_total_entries(struct trace_buffer
*buf
,
2868 unsigned long *total
, unsigned long *entries
)
2870 unsigned long count
;
2876 for_each_tracing_cpu(cpu
) {
2877 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
2879 * If this buffer has skipped entries, then we hold all
2880 * entries for the trace and we need to ignore the
2881 * ones before the time stamp.
2883 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
2884 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
2885 /* total is the same as the entries */
2889 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
2894 static void print_lat_help_header(struct seq_file
*m
)
2896 seq_puts(m
, "# _------=> CPU# \n"
2897 "# / _-----=> irqs-off \n"
2898 "# | / _----=> need-resched \n"
2899 "# || / _---=> hardirq/softirq \n"
2900 "# ||| / _--=> preempt-depth \n"
2902 "# cmd pid ||||| time | caller \n"
2903 "# \\ / ||||| \\ | / \n");
2906 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
2908 unsigned long total
;
2909 unsigned long entries
;
2911 get_total_entries(buf
, &total
, &entries
);
2912 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2913 entries
, total
, num_online_cpus());
2917 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
)
2919 print_event_info(buf
, m
);
2920 seq_puts(m
, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2924 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
)
2926 print_event_info(buf
, m
);
2927 seq_puts(m
, "# _-----=> irqs-off\n"
2928 "# / _----=> need-resched\n"
2929 "# | / _---=> hardirq/softirq\n"
2930 "# || / _--=> preempt-depth\n"
2932 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2933 "# | | | |||| | |\n");
2937 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
2939 unsigned long sym_flags
= (global_trace
.trace_flags
& TRACE_ITER_SYM_MASK
);
2940 struct trace_buffer
*buf
= iter
->trace_buffer
;
2941 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
2942 struct tracer
*type
= iter
->trace
;
2943 unsigned long entries
;
2944 unsigned long total
;
2945 const char *name
= "preemption";
2949 get_total_entries(buf
, &total
, &entries
);
2951 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
2953 seq_puts(m
, "# -----------------------------------"
2954 "---------------------------------\n");
2955 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2956 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2957 nsecs_to_usecs(data
->saved_latency
),
2961 #if defined(CONFIG_PREEMPT_NONE)
2963 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2965 #elif defined(CONFIG_PREEMPT)
2970 /* These are reserved for later use */
2973 seq_printf(m
, " #P:%d)\n", num_online_cpus());
2977 seq_puts(m
, "# -----------------\n");
2978 seq_printf(m
, "# | task: %.16s-%d "
2979 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2980 data
->comm
, data
->pid
,
2981 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
2982 data
->policy
, data
->rt_priority
);
2983 seq_puts(m
, "# -----------------\n");
2985 if (data
->critical_start
) {
2986 seq_puts(m
, "# => started at: ");
2987 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
2988 trace_print_seq(m
, &iter
->seq
);
2989 seq_puts(m
, "\n# => ended at: ");
2990 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
2991 trace_print_seq(m
, &iter
->seq
);
2992 seq_puts(m
, "\n#\n");
2998 static void test_cpu_buff_start(struct trace_iterator
*iter
)
3000 struct trace_seq
*s
= &iter
->seq
;
3001 struct trace_array
*tr
= iter
->tr
;
3003 if (!(tr
->trace_flags
& TRACE_ITER_ANNOTATE
))
3006 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
3009 if (iter
->started
&& cpumask_test_cpu(iter
->cpu
, iter
->started
))
3012 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
3016 cpumask_set_cpu(iter
->cpu
, iter
->started
);
3018 /* Don't print started cpu buffer for the first entry of the trace */
3020 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
3024 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
3026 struct trace_array
*tr
= iter
->tr
;
3027 struct trace_seq
*s
= &iter
->seq
;
3028 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
3029 struct trace_entry
*entry
;
3030 struct trace_event
*event
;
3034 test_cpu_buff_start(iter
);
3036 event
= ftrace_find_event(entry
->type
);
3038 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3039 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3040 trace_print_lat_context(iter
);
3042 trace_print_context(iter
);
3045 if (trace_seq_has_overflowed(s
))
3046 return TRACE_TYPE_PARTIAL_LINE
;
3049 return event
->funcs
->trace(iter
, sym_flags
, event
);
3051 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
3053 return trace_handle_return(s
);
3056 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
3058 struct trace_array
*tr
= iter
->tr
;
3059 struct trace_seq
*s
= &iter
->seq
;
3060 struct trace_entry
*entry
;
3061 struct trace_event
*event
;
3065 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
)
3066 trace_seq_printf(s
, "%d %d %llu ",
3067 entry
->pid
, iter
->cpu
, iter
->ts
);
3069 if (trace_seq_has_overflowed(s
))
3070 return TRACE_TYPE_PARTIAL_LINE
;
3072 event
= ftrace_find_event(entry
->type
);
3074 return event
->funcs
->raw(iter
, 0, event
);
3076 trace_seq_printf(s
, "%d ?\n", entry
->type
);
3078 return trace_handle_return(s
);
3081 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
3083 struct trace_array
*tr
= iter
->tr
;
3084 struct trace_seq
*s
= &iter
->seq
;
3085 unsigned char newline
= '\n';
3086 struct trace_entry
*entry
;
3087 struct trace_event
*event
;
3091 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3092 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
3093 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
3094 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
3095 if (trace_seq_has_overflowed(s
))
3096 return TRACE_TYPE_PARTIAL_LINE
;
3099 event
= ftrace_find_event(entry
->type
);
3101 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
3102 if (ret
!= TRACE_TYPE_HANDLED
)
3106 SEQ_PUT_FIELD(s
, newline
);
3108 return trace_handle_return(s
);
3111 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
3113 struct trace_array
*tr
= iter
->tr
;
3114 struct trace_seq
*s
= &iter
->seq
;
3115 struct trace_entry
*entry
;
3116 struct trace_event
*event
;
3120 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
3121 SEQ_PUT_FIELD(s
, entry
->pid
);
3122 SEQ_PUT_FIELD(s
, iter
->cpu
);
3123 SEQ_PUT_FIELD(s
, iter
->ts
);
3124 if (trace_seq_has_overflowed(s
))
3125 return TRACE_TYPE_PARTIAL_LINE
;
3128 event
= ftrace_find_event(entry
->type
);
3129 return event
? event
->funcs
->binary(iter
, 0, event
) :
3133 int trace_empty(struct trace_iterator
*iter
)
3135 struct ring_buffer_iter
*buf_iter
;
3138 /* If we are looking at one CPU buffer, only check that one */
3139 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
3140 cpu
= iter
->cpu_file
;
3141 buf_iter
= trace_buffer_iter(iter
, cpu
);
3143 if (!ring_buffer_iter_empty(buf_iter
))
3146 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3152 for_each_tracing_cpu(cpu
) {
3153 buf_iter
= trace_buffer_iter(iter
, cpu
);
3155 if (!ring_buffer_iter_empty(buf_iter
))
3158 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
3166 /* Called with trace_event_read_lock() held. */
3167 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
3169 struct trace_array
*tr
= iter
->tr
;
3170 unsigned long trace_flags
= tr
->trace_flags
;
3171 enum print_line_t ret
;
3173 if (iter
->lost_events
) {
3174 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
3175 iter
->cpu
, iter
->lost_events
);
3176 if (trace_seq_has_overflowed(&iter
->seq
))
3177 return TRACE_TYPE_PARTIAL_LINE
;
3180 if (iter
->trace
&& iter
->trace
->print_line
) {
3181 ret
= iter
->trace
->print_line(iter
);
3182 if (ret
!= TRACE_TYPE_UNHANDLED
)
3186 if (iter
->ent
->type
== TRACE_BPUTS
&&
3187 trace_flags
& TRACE_ITER_PRINTK
&&
3188 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3189 return trace_print_bputs_msg_only(iter
);
3191 if (iter
->ent
->type
== TRACE_BPRINT
&&
3192 trace_flags
& TRACE_ITER_PRINTK
&&
3193 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3194 return trace_print_bprintk_msg_only(iter
);
3196 if (iter
->ent
->type
== TRACE_PRINT
&&
3197 trace_flags
& TRACE_ITER_PRINTK
&&
3198 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
3199 return trace_print_printk_msg_only(iter
);
3201 if (trace_flags
& TRACE_ITER_BIN
)
3202 return print_bin_fmt(iter
);
3204 if (trace_flags
& TRACE_ITER_HEX
)
3205 return print_hex_fmt(iter
);
3207 if (trace_flags
& TRACE_ITER_RAW
)
3208 return print_raw_fmt(iter
);
3210 return print_trace_fmt(iter
);
3213 void trace_latency_header(struct seq_file
*m
)
3215 struct trace_iterator
*iter
= m
->private;
3216 struct trace_array
*tr
= iter
->tr
;
3218 /* print nothing if the buffers are empty */
3219 if (trace_empty(iter
))
3222 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
3223 print_trace_header(m
, iter
);
3225 if (!(tr
->trace_flags
& TRACE_ITER_VERBOSE
))
3226 print_lat_help_header(m
);
3229 void trace_default_header(struct seq_file
*m
)
3231 struct trace_iterator
*iter
= m
->private;
3232 struct trace_array
*tr
= iter
->tr
;
3233 unsigned long trace_flags
= tr
->trace_flags
;
3235 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
3238 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
3239 /* print nothing if the buffers are empty */
3240 if (trace_empty(iter
))
3242 print_trace_header(m
, iter
);
3243 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
3244 print_lat_help_header(m
);
3246 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
3247 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
3248 print_func_help_header_irq(iter
->trace_buffer
, m
);
3250 print_func_help_header(iter
->trace_buffer
, m
);
3255 static void test_ftrace_alive(struct seq_file
*m
)
3257 if (!ftrace_is_dead())
3259 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3260 "# MAY BE MISSING FUNCTION EVENTS\n");
3263 #ifdef CONFIG_TRACER_MAX_TRACE
3264 static void show_snapshot_main_help(struct seq_file
*m
)
3266 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3267 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3268 "# Takes a snapshot of the main buffer.\n"
3269 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3270 "# (Doesn't have to be '2' works with any number that\n"
3271 "# is not a '0' or '1')\n");
3274 static void show_snapshot_percpu_help(struct seq_file
*m
)
3276 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3277 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3278 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3279 "# Takes a snapshot of the main buffer for this cpu.\n");
3281 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
3282 "# Must use main snapshot file to allocate.\n");
3284 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3285 "# (Doesn't have to be '2' works with any number that\n"
3286 "# is not a '0' or '1')\n");
3289 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
3291 if (iter
->tr
->allocated_snapshot
)
3292 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
3294 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
3296 seq_puts(m
, "# Snapshot commands:\n");
3297 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
3298 show_snapshot_main_help(m
);
3300 show_snapshot_percpu_help(m
);
3303 /* Should never be called */
3304 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
3307 static int s_show(struct seq_file
*m
, void *v
)
3309 struct trace_iterator
*iter
= v
;
3312 if (iter
->ent
== NULL
) {
3314 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
3316 test_ftrace_alive(m
);
3318 if (iter
->snapshot
&& trace_empty(iter
))
3319 print_snapshot_help(m
, iter
);
3320 else if (iter
->trace
&& iter
->trace
->print_header
)
3321 iter
->trace
->print_header(m
);
3323 trace_default_header(m
);
3325 } else if (iter
->leftover
) {
3327 * If we filled the seq_file buffer earlier, we
3328 * want to just show it now.
3330 ret
= trace_print_seq(m
, &iter
->seq
);
3332 /* ret should this time be zero, but you never know */
3333 iter
->leftover
= ret
;
3336 print_trace_line(iter
);
3337 ret
= trace_print_seq(m
, &iter
->seq
);
3339 * If we overflow the seq_file buffer, then it will
3340 * ask us for this data again at start up.
3342 * ret is 0 if seq_file write succeeded.
3345 iter
->leftover
= ret
;
3352 * Should be used after trace_array_get(), trace_types_lock
3353 * ensures that i_cdev was already initialized.
3355 static inline int tracing_get_cpu(struct inode
*inode
)
3357 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
3358 return (long)inode
->i_cdev
- 1;
3359 return RING_BUFFER_ALL_CPUS
;
3362 static const struct seq_operations tracer_seq_ops
= {
3369 static struct trace_iterator
*
3370 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
3372 struct trace_array
*tr
= inode
->i_private
;
3373 struct trace_iterator
*iter
;
3376 if (tracing_disabled
)
3377 return ERR_PTR(-ENODEV
);
3379 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
3381 return ERR_PTR(-ENOMEM
);
3383 iter
->buffer_iter
= kcalloc(nr_cpu_ids
, sizeof(*iter
->buffer_iter
),
3385 if (!iter
->buffer_iter
)
3389 * We make a copy of the current tracer to avoid concurrent
3390 * changes on it while we are reading.
3392 mutex_lock(&trace_types_lock
);
3393 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
3397 *iter
->trace
= *tr
->current_trace
;
3399 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
3404 #ifdef CONFIG_TRACER_MAX_TRACE
3405 /* Currently only the top directory has a snapshot */
3406 if (tr
->current_trace
->print_max
|| snapshot
)
3407 iter
->trace_buffer
= &tr
->max_buffer
;
3410 iter
->trace_buffer
= &tr
->trace_buffer
;
3411 iter
->snapshot
= snapshot
;
3413 iter
->cpu_file
= tracing_get_cpu(inode
);
3414 mutex_init(&iter
->mutex
);
3416 /* Notify the tracer early; before we stop tracing. */
3417 if (iter
->trace
&& iter
->trace
->open
)
3418 iter
->trace
->open(iter
);
3420 /* Annotate start of buffers if we had overruns */
3421 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
3422 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
3424 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3425 if (trace_clocks
[tr
->clock_id
].in_ns
)
3426 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
3428 /* stop the trace while dumping if we are not opening "snapshot" */
3429 if (!iter
->snapshot
)
3430 tracing_stop_tr(tr
);
3432 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
3433 for_each_tracing_cpu(cpu
) {
3434 iter
->buffer_iter
[cpu
] =
3435 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3437 ring_buffer_read_prepare_sync();
3438 for_each_tracing_cpu(cpu
) {
3439 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3440 tracing_iter_reset(iter
, cpu
);
3443 cpu
= iter
->cpu_file
;
3444 iter
->buffer_iter
[cpu
] =
3445 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3446 ring_buffer_read_prepare_sync();
3447 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3448 tracing_iter_reset(iter
, cpu
);
3451 mutex_unlock(&trace_types_lock
);
3456 mutex_unlock(&trace_types_lock
);
3458 kfree(iter
->buffer_iter
);
3460 seq_release_private(inode
, file
);
3461 return ERR_PTR(-ENOMEM
);
3464 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
3466 if (tracing_disabled
)
3469 filp
->private_data
= inode
->i_private
;
3473 bool tracing_is_disabled(void)
3475 return (tracing_disabled
) ? true: false;
3479 * Open and update trace_array ref count.
3480 * Must have the current trace_array passed to it.
3482 static int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
3484 struct trace_array
*tr
= inode
->i_private
;
3486 if (tracing_disabled
)
3489 if (trace_array_get(tr
) < 0)
3492 filp
->private_data
= inode
->i_private
;
3497 static int tracing_release(struct inode
*inode
, struct file
*file
)
3499 struct trace_array
*tr
= inode
->i_private
;
3500 struct seq_file
*m
= file
->private_data
;
3501 struct trace_iterator
*iter
;
3504 if (!(file
->f_mode
& FMODE_READ
)) {
3505 trace_array_put(tr
);
3509 /* Writes do not use seq_file */
3511 mutex_lock(&trace_types_lock
);
3513 for_each_tracing_cpu(cpu
) {
3514 if (iter
->buffer_iter
[cpu
])
3515 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
3518 if (iter
->trace
&& iter
->trace
->close
)
3519 iter
->trace
->close(iter
);
3521 if (!iter
->snapshot
)
3522 /* reenable tracing if it was previously enabled */
3523 tracing_start_tr(tr
);
3525 __trace_array_put(tr
);
3527 mutex_unlock(&trace_types_lock
);
3529 mutex_destroy(&iter
->mutex
);
3530 free_cpumask_var(iter
->started
);
3532 kfree(iter
->buffer_iter
);
3533 seq_release_private(inode
, file
);
3538 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
3540 struct trace_array
*tr
= inode
->i_private
;
3542 trace_array_put(tr
);
3546 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
3548 struct trace_array
*tr
= inode
->i_private
;
3550 trace_array_put(tr
);
3552 return single_release(inode
, file
);
3555 static int tracing_open(struct inode
*inode
, struct file
*file
)
3557 struct trace_array
*tr
= inode
->i_private
;
3558 struct trace_iterator
*iter
;
3561 if (trace_array_get(tr
) < 0)
3564 /* If this file was open for write, then erase contents */
3565 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
3566 int cpu
= tracing_get_cpu(inode
);
3568 if (cpu
== RING_BUFFER_ALL_CPUS
)
3569 tracing_reset_online_cpus(&tr
->trace_buffer
);
3571 tracing_reset(&tr
->trace_buffer
, cpu
);
3574 if (file
->f_mode
& FMODE_READ
) {
3575 iter
= __tracing_open(inode
, file
, false);
3577 ret
= PTR_ERR(iter
);
3578 else if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
3579 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
3583 trace_array_put(tr
);
3589 * Some tracers are not suitable for instance buffers.
3590 * A tracer is always available for the global array (toplevel)
3591 * or if it explicitly states that it is.
3594 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
3596 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
3599 /* Find the next tracer that this trace array may use */
3600 static struct tracer
*
3601 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
3603 while (t
&& !trace_ok_for_array(t
, tr
))
3610 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3612 struct trace_array
*tr
= m
->private;
3613 struct tracer
*t
= v
;
3618 t
= get_tracer_for_array(tr
, t
->next
);
3623 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
3625 struct trace_array
*tr
= m
->private;
3629 mutex_lock(&trace_types_lock
);
3631 t
= get_tracer_for_array(tr
, trace_types
);
3632 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
3638 static void t_stop(struct seq_file
*m
, void *p
)
3640 mutex_unlock(&trace_types_lock
);
3643 static int t_show(struct seq_file
*m
, void *v
)
3645 struct tracer
*t
= v
;
3650 seq_puts(m
, t
->name
);
3659 static const struct seq_operations show_traces_seq_ops
= {
3666 static int show_traces_open(struct inode
*inode
, struct file
*file
)
3668 struct trace_array
*tr
= inode
->i_private
;
3672 if (tracing_disabled
)
3675 ret
= seq_open(file
, &show_traces_seq_ops
);
3679 m
= file
->private_data
;
3686 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
3687 size_t count
, loff_t
*ppos
)
3692 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
3696 if (file
->f_mode
& FMODE_READ
)
3697 ret
= seq_lseek(file
, offset
, whence
);
3699 file
->f_pos
= ret
= 0;
3704 static const struct file_operations tracing_fops
= {
3705 .open
= tracing_open
,
3707 .write
= tracing_write_stub
,
3708 .llseek
= tracing_lseek
,
3709 .release
= tracing_release
,
3712 static const struct file_operations show_traces_fops
= {
3713 .open
= show_traces_open
,
3715 .release
= seq_release
,
3716 .llseek
= seq_lseek
,
3720 * The tracer itself will not take this lock, but still we want
3721 * to provide a consistent cpumask to user-space:
3723 static DEFINE_MUTEX(tracing_cpumask_update_lock
);
3726 * Temporary storage for the character representation of the
3727 * CPU bitmask (and one more byte for the newline):
3729 static char mask_str
[NR_CPUS
+ 1];
3732 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
3733 size_t count
, loff_t
*ppos
)
3735 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3738 mutex_lock(&tracing_cpumask_update_lock
);
3740 len
= snprintf(mask_str
, count
, "%*pb\n",
3741 cpumask_pr_args(tr
->tracing_cpumask
));
3746 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, NR_CPUS
+1);
3749 mutex_unlock(&tracing_cpumask_update_lock
);
3755 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
3756 size_t count
, loff_t
*ppos
)
3758 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3759 cpumask_var_t tracing_cpumask_new
;
3762 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
3765 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
3769 mutex_lock(&tracing_cpumask_update_lock
);
3771 local_irq_disable();
3772 arch_spin_lock(&tr
->max_lock
);
3773 for_each_tracing_cpu(cpu
) {
3775 * Increase/decrease the disabled counter if we are
3776 * about to flip a bit in the cpumask:
3778 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3779 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3780 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3781 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3783 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3784 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3785 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3786 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3789 arch_spin_unlock(&tr
->max_lock
);
3792 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
3794 mutex_unlock(&tracing_cpumask_update_lock
);
3795 free_cpumask_var(tracing_cpumask_new
);
3800 free_cpumask_var(tracing_cpumask_new
);
3805 static const struct file_operations tracing_cpumask_fops
= {
3806 .open
= tracing_open_generic_tr
,
3807 .read
= tracing_cpumask_read
,
3808 .write
= tracing_cpumask_write
,
3809 .release
= tracing_release_generic_tr
,
3810 .llseek
= generic_file_llseek
,
3813 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
3815 struct tracer_opt
*trace_opts
;
3816 struct trace_array
*tr
= m
->private;
3820 mutex_lock(&trace_types_lock
);
3821 tracer_flags
= tr
->current_trace
->flags
->val
;
3822 trace_opts
= tr
->current_trace
->flags
->opts
;
3824 for (i
= 0; trace_options
[i
]; i
++) {
3825 if (tr
->trace_flags
& (1 << i
))
3826 seq_printf(m
, "%s\n", trace_options
[i
]);
3828 seq_printf(m
, "no%s\n", trace_options
[i
]);
3831 for (i
= 0; trace_opts
[i
].name
; i
++) {
3832 if (tracer_flags
& trace_opts
[i
].bit
)
3833 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
3835 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
3837 mutex_unlock(&trace_types_lock
);
3842 static int __set_tracer_option(struct trace_array
*tr
,
3843 struct tracer_flags
*tracer_flags
,
3844 struct tracer_opt
*opts
, int neg
)
3846 struct tracer
*trace
= tracer_flags
->trace
;
3849 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
3854 tracer_flags
->val
&= ~opts
->bit
;
3856 tracer_flags
->val
|= opts
->bit
;
3860 /* Try to assign a tracer specific option */
3861 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
3863 struct tracer
*trace
= tr
->current_trace
;
3864 struct tracer_flags
*tracer_flags
= trace
->flags
;
3865 struct tracer_opt
*opts
= NULL
;
3868 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
3869 opts
= &tracer_flags
->opts
[i
];
3871 if (strcmp(cmp
, opts
->name
) == 0)
3872 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
3878 /* Some tracers require overwrite to stay enabled */
3879 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
3881 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
3887 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
3889 /* do nothing if flag is already set */
3890 if (!!(tr
->trace_flags
& mask
) == !!enabled
)
3893 /* Give the tracer a chance to approve the change */
3894 if (tr
->current_trace
->flag_changed
)
3895 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
3899 tr
->trace_flags
|= mask
;
3901 tr
->trace_flags
&= ~mask
;
3903 if (mask
== TRACE_ITER_RECORD_CMD
)
3904 trace_event_enable_cmd_record(enabled
);
3906 if (mask
== TRACE_ITER_EVENT_FORK
)
3907 trace_event_follow_fork(tr
, enabled
);
3909 if (mask
== TRACE_ITER_OVERWRITE
) {
3910 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
3911 #ifdef CONFIG_TRACER_MAX_TRACE
3912 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
3916 if (mask
== TRACE_ITER_PRINTK
) {
3917 trace_printk_start_stop_comm(enabled
);
3918 trace_printk_control(enabled
);
3924 static int trace_set_options(struct trace_array
*tr
, char *option
)
3930 size_t orig_len
= strlen(option
);
3932 cmp
= strstrip(option
);
3934 if (strncmp(cmp
, "no", 2) == 0) {
3939 mutex_lock(&trace_types_lock
);
3941 for (i
= 0; trace_options
[i
]; i
++) {
3942 if (strcmp(cmp
, trace_options
[i
]) == 0) {
3943 ret
= set_tracer_flag(tr
, 1 << i
, !neg
);
3948 /* If no option could be set, test the specific tracer options */
3949 if (!trace_options
[i
])
3950 ret
= set_tracer_option(tr
, cmp
, neg
);
3952 mutex_unlock(&trace_types_lock
);
3955 * If the first trailing whitespace is replaced with '\0' by strstrip,
3956 * turn it back into a space.
3958 if (orig_len
> strlen(option
))
3959 option
[strlen(option
)] = ' ';
3964 static void __init
apply_trace_boot_options(void)
3966 char *buf
= trace_boot_options_buf
;
3970 option
= strsep(&buf
, ",");
3976 trace_set_options(&global_trace
, option
);
3978 /* Put back the comma to allow this to be called again */
3985 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
3986 size_t cnt
, loff_t
*ppos
)
3988 struct seq_file
*m
= filp
->private_data
;
3989 struct trace_array
*tr
= m
->private;
3993 if (cnt
>= sizeof(buf
))
3996 if (copy_from_user(buf
, ubuf
, cnt
))
4001 ret
= trace_set_options(tr
, buf
);
4010 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
4012 struct trace_array
*tr
= inode
->i_private
;
4015 if (tracing_disabled
)
4018 if (trace_array_get(tr
) < 0)
4021 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
4023 trace_array_put(tr
);
4028 static const struct file_operations tracing_iter_fops
= {
4029 .open
= tracing_trace_options_open
,
4031 .llseek
= seq_lseek
,
4032 .release
= tracing_single_release_tr
,
4033 .write
= tracing_trace_options_write
,
4036 static const char readme_msg
[] =
4037 "tracing mini-HOWTO:\n\n"
4038 "# echo 0 > tracing_on : quick way to disable tracing\n"
4039 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4040 " Important files:\n"
4041 " trace\t\t\t- The static contents of the buffer\n"
4042 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4043 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4044 " current_tracer\t- function and latency tracers\n"
4045 " available_tracers\t- list of configured tracers for current_tracer\n"
4046 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4047 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4048 " trace_clock\t\t-change the clock used to order events\n"
4049 " local: Per cpu clock but may not be synced across CPUs\n"
4050 " global: Synced across CPUs but slows tracing down.\n"
4051 " counter: Not a clock, but just an increment\n"
4052 " uptime: Jiffy counter from time of boot\n"
4053 " perf: Same clock that perf events use\n"
4054 #ifdef CONFIG_X86_64
4055 " x86-tsc: TSC cycle counter\n"
4057 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4058 " tracing_cpumask\t- Limit which CPUs to trace\n"
4059 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4060 "\t\t\t Remove sub-buffer with rmdir\n"
4061 " trace_options\t\t- Set format or modify how tracing happens\n"
4062 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4063 "\t\t\t option name\n"
4064 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4065 #ifdef CONFIG_DYNAMIC_FTRACE
4066 "\n available_filter_functions - list of functions that can be filtered on\n"
4067 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4068 "\t\t\t functions\n"
4069 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4070 "\t modules: Can select a group via module\n"
4071 "\t Format: :mod:<module-name>\n"
4072 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4073 "\t triggers: a command to perform when function is hit\n"
4074 "\t Format: <function>:<trigger>[:count]\n"
4075 "\t trigger: traceon, traceoff\n"
4076 "\t\t enable_event:<system>:<event>\n"
4077 "\t\t disable_event:<system>:<event>\n"
4078 #ifdef CONFIG_STACKTRACE
4081 #ifdef CONFIG_TRACER_SNAPSHOT
4086 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4087 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4088 "\t The first one will disable tracing every time do_fault is hit\n"
4089 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4090 "\t The first time do trap is hit and it disables tracing, the\n"
4091 "\t counter will decrement to 2. If tracing is already disabled,\n"
4092 "\t the counter will not decrement. It only decrements when the\n"
4093 "\t trigger did work\n"
4094 "\t To remove trigger without count:\n"
4095 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4096 "\t To remove trigger with a count:\n"
4097 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4098 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4099 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4100 "\t modules: Can select a group via module command :mod:\n"
4101 "\t Does not accept triggers\n"
4102 #endif /* CONFIG_DYNAMIC_FTRACE */
4103 #ifdef CONFIG_FUNCTION_TRACER
4104 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4107 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4108 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4109 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4110 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4112 #ifdef CONFIG_TRACER_SNAPSHOT
4113 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4114 "\t\t\t snapshot buffer. Read the contents for more\n"
4115 "\t\t\t information\n"
4117 #ifdef CONFIG_STACK_TRACER
4118 " stack_trace\t\t- Shows the max stack trace when active\n"
4119 " stack_max_size\t- Shows current max stack size that was traced\n"
4120 "\t\t\t Write into this file to reset the max size (trigger a\n"
4121 "\t\t\t new trace)\n"
4122 #ifdef CONFIG_DYNAMIC_FTRACE
4123 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4126 #endif /* CONFIG_STACK_TRACER */
4127 #ifdef CONFIG_KPROBE_EVENT
4128 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4129 "\t\t\t Write into this file to define/undefine new trace events.\n"
4131 #ifdef CONFIG_UPROBE_EVENT
4132 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4133 "\t\t\t Write into this file to define/undefine new trace events.\n"
4135 #if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
4136 "\t accepts: event-definitions (one definition per line)\n"
4137 "\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
4138 "\t -:[<group>/]<event>\n"
4139 #ifdef CONFIG_KPROBE_EVENT
4140 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4142 #ifdef CONFIG_UPROBE_EVENT
4143 "\t place: <path>:<offset>\n"
4145 "\t args: <name>=fetcharg[:type]\n"
4146 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4147 "\t $stack<index>, $stack, $retval, $comm\n"
4148 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4149 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4151 " events/\t\t- Directory containing all trace event subsystems:\n"
4152 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4153 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4154 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4156 " filter\t\t- If set, only events passing filter are traced\n"
4157 " events/<system>/<event>/\t- Directory containing control files for\n"
4159 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4160 " filter\t\t- If set, only events passing filter are traced\n"
4161 " trigger\t\t- If set, a command to perform when event is hit\n"
4162 "\t Format: <trigger>[:count][if <filter>]\n"
4163 "\t trigger: traceon, traceoff\n"
4164 "\t enable_event:<system>:<event>\n"
4165 "\t disable_event:<system>:<event>\n"
4166 #ifdef CONFIG_HIST_TRIGGERS
4167 "\t enable_hist:<system>:<event>\n"
4168 "\t disable_hist:<system>:<event>\n"
4170 #ifdef CONFIG_STACKTRACE
4173 #ifdef CONFIG_TRACER_SNAPSHOT
4176 #ifdef CONFIG_HIST_TRIGGERS
4177 "\t\t hist (see below)\n"
4179 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4180 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4181 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4182 "\t events/block/block_unplug/trigger\n"
4183 "\t The first disables tracing every time block_unplug is hit.\n"
4184 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4185 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4186 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4187 "\t Like function triggers, the counter is only decremented if it\n"
4188 "\t enabled or disabled tracing.\n"
4189 "\t To remove a trigger without a count:\n"
4190 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4191 "\t To remove a trigger with a count:\n"
4192 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4193 "\t Filters can be ignored when removing a trigger.\n"
4194 #ifdef CONFIG_HIST_TRIGGERS
4195 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4196 "\t Format: hist:keys=<field1[,field2,...]>\n"
4197 "\t [:values=<field1[,field2,...]>]\n"
4198 "\t [:sort=<field1[,field2,...]>]\n"
4199 "\t [:size=#entries]\n"
4200 "\t [:pause][:continue][:clear]\n"
4201 "\t [:name=histname1]\n"
4202 "\t [if <filter>]\n\n"
4203 "\t When a matching event is hit, an entry is added to a hash\n"
4204 "\t table using the key(s) and value(s) named, and the value of a\n"
4205 "\t sum called 'hitcount' is incremented. Keys and values\n"
4206 "\t correspond to fields in the event's format description. Keys\n"
4207 "\t can be any field, or the special string 'stacktrace'.\n"
4208 "\t Compound keys consisting of up to two fields can be specified\n"
4209 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4210 "\t fields. Sort keys consisting of up to two fields can be\n"
4211 "\t specified using the 'sort' keyword. The sort direction can\n"
4212 "\t be modified by appending '.descending' or '.ascending' to a\n"
4213 "\t sort field. The 'size' parameter can be used to specify more\n"
4214 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4215 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4216 "\t its histogram data will be shared with other triggers of the\n"
4217 "\t same name, and trigger hits will update this common data.\n\n"
4218 "\t Reading the 'hist' file for the event will dump the hash\n"
4219 "\t table in its entirety to stdout. If there are multiple hist\n"
4220 "\t triggers attached to an event, there will be a table for each\n"
4221 "\t trigger in the output. The table displayed for a named\n"
4222 "\t trigger will be the same as any other instance having the\n"
4223 "\t same name. The default format used to display a given field\n"
4224 "\t can be modified by appending any of the following modifiers\n"
4225 "\t to the field name, as applicable:\n\n"
4226 "\t .hex display a number as a hex value\n"
4227 "\t .sym display an address as a symbol\n"
4228 "\t .sym-offset display an address as a symbol and offset\n"
4229 "\t .execname display a common_pid as a program name\n"
4230 "\t .syscall display a syscall id as a syscall name\n\n"
4231 "\t .log2 display log2 value rather than raw number\n\n"
4232 "\t The 'pause' parameter can be used to pause an existing hist\n"
4233 "\t trigger or to start a hist trigger but not log any events\n"
4234 "\t until told to do so. 'continue' can be used to start or\n"
4235 "\t restart a paused hist trigger.\n\n"
4236 "\t The 'clear' parameter will clear the contents of a running\n"
4237 "\t hist trigger and leave its current paused/active state\n"
4239 "\t The enable_hist and disable_hist triggers can be used to\n"
4240 "\t have one event conditionally start and stop another event's\n"
4241 "\t already-attached hist trigger. The syntax is analagous to\n"
4242 "\t the enable_event and disable_event triggers.\n"
4247 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
4248 size_t cnt
, loff_t
*ppos
)
4250 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
4251 readme_msg
, strlen(readme_msg
));
4254 static const struct file_operations tracing_readme_fops
= {
4255 .open
= tracing_open_generic
,
4256 .read
= tracing_readme_read
,
4257 .llseek
= generic_file_llseek
,
4260 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4262 unsigned int *ptr
= v
;
4264 if (*pos
|| m
->count
)
4269 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
4271 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
4280 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
4286 arch_spin_lock(&trace_cmdline_lock
);
4288 v
= &savedcmd
->map_cmdline_to_pid
[0];
4290 v
= saved_cmdlines_next(m
, v
, &l
);
4298 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
4300 arch_spin_unlock(&trace_cmdline_lock
);
4304 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
4306 char buf
[TASK_COMM_LEN
];
4307 unsigned int *pid
= v
;
4309 __trace_find_cmdline(*pid
, buf
);
4310 seq_printf(m
, "%d %s\n", *pid
, buf
);
4314 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
4315 .start
= saved_cmdlines_start
,
4316 .next
= saved_cmdlines_next
,
4317 .stop
= saved_cmdlines_stop
,
4318 .show
= saved_cmdlines_show
,
4321 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
4323 if (tracing_disabled
)
4326 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
4329 static const struct file_operations tracing_saved_cmdlines_fops
= {
4330 .open
= tracing_saved_cmdlines_open
,
4332 .llseek
= seq_lseek
,
4333 .release
= seq_release
,
4337 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
4338 size_t cnt
, loff_t
*ppos
)
4343 arch_spin_lock(&trace_cmdline_lock
);
4344 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
4345 arch_spin_unlock(&trace_cmdline_lock
);
4347 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4350 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
4352 kfree(s
->saved_cmdlines
);
4353 kfree(s
->map_cmdline_to_pid
);
4357 static int tracing_resize_saved_cmdlines(unsigned int val
)
4359 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
4361 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
4365 if (allocate_cmdlines_buffer(val
, s
) < 0) {
4370 arch_spin_lock(&trace_cmdline_lock
);
4371 savedcmd_temp
= savedcmd
;
4373 arch_spin_unlock(&trace_cmdline_lock
);
4374 free_saved_cmdlines_buffer(savedcmd_temp
);
4380 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
4381 size_t cnt
, loff_t
*ppos
)
4386 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4390 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4391 if (!val
|| val
> PID_MAX_DEFAULT
)
4394 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
4403 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
4404 .open
= tracing_open_generic
,
4405 .read
= tracing_saved_cmdlines_size_read
,
4406 .write
= tracing_saved_cmdlines_size_write
,
4409 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4410 static union trace_enum_map_item
*
4411 update_enum_map(union trace_enum_map_item
*ptr
)
4413 if (!ptr
->map
.enum_string
) {
4414 if (ptr
->tail
.next
) {
4415 ptr
= ptr
->tail
.next
;
4416 /* Set ptr to the next real item (skip head) */
4424 static void *enum_map_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4426 union trace_enum_map_item
*ptr
= v
;
4429 * Paranoid! If ptr points to end, we don't want to increment past it.
4430 * This really should never happen.
4432 ptr
= update_enum_map(ptr
);
4433 if (WARN_ON_ONCE(!ptr
))
4440 ptr
= update_enum_map(ptr
);
4445 static void *enum_map_start(struct seq_file
*m
, loff_t
*pos
)
4447 union trace_enum_map_item
*v
;
4450 mutex_lock(&trace_enum_mutex
);
4452 v
= trace_enum_maps
;
4456 while (v
&& l
< *pos
) {
4457 v
= enum_map_next(m
, v
, &l
);
4463 static void enum_map_stop(struct seq_file
*m
, void *v
)
4465 mutex_unlock(&trace_enum_mutex
);
4468 static int enum_map_show(struct seq_file
*m
, void *v
)
4470 union trace_enum_map_item
*ptr
= v
;
4472 seq_printf(m
, "%s %ld (%s)\n",
4473 ptr
->map
.enum_string
, ptr
->map
.enum_value
,
4479 static const struct seq_operations tracing_enum_map_seq_ops
= {
4480 .start
= enum_map_start
,
4481 .next
= enum_map_next
,
4482 .stop
= enum_map_stop
,
4483 .show
= enum_map_show
,
4486 static int tracing_enum_map_open(struct inode
*inode
, struct file
*filp
)
4488 if (tracing_disabled
)
4491 return seq_open(filp
, &tracing_enum_map_seq_ops
);
4494 static const struct file_operations tracing_enum_map_fops
= {
4495 .open
= tracing_enum_map_open
,
4497 .llseek
= seq_lseek
,
4498 .release
= seq_release
,
4501 static inline union trace_enum_map_item
*
4502 trace_enum_jmp_to_tail(union trace_enum_map_item
*ptr
)
4504 /* Return tail of array given the head */
4505 return ptr
+ ptr
->head
.length
+ 1;
4509 trace_insert_enum_map_file(struct module
*mod
, struct trace_enum_map
**start
,
4512 struct trace_enum_map
**stop
;
4513 struct trace_enum_map
**map
;
4514 union trace_enum_map_item
*map_array
;
4515 union trace_enum_map_item
*ptr
;
4520 * The trace_enum_maps contains the map plus a head and tail item,
4521 * where the head holds the module and length of array, and the
4522 * tail holds a pointer to the next list.
4524 map_array
= kmalloc(sizeof(*map_array
) * (len
+ 2), GFP_KERNEL
);
4526 pr_warn("Unable to allocate trace enum mapping\n");
4530 mutex_lock(&trace_enum_mutex
);
4532 if (!trace_enum_maps
)
4533 trace_enum_maps
= map_array
;
4535 ptr
= trace_enum_maps
;
4537 ptr
= trace_enum_jmp_to_tail(ptr
);
4538 if (!ptr
->tail
.next
)
4540 ptr
= ptr
->tail
.next
;
4543 ptr
->tail
.next
= map_array
;
4545 map_array
->head
.mod
= mod
;
4546 map_array
->head
.length
= len
;
4549 for (map
= start
; (unsigned long)map
< (unsigned long)stop
; map
++) {
4550 map_array
->map
= **map
;
4553 memset(map_array
, 0, sizeof(*map_array
));
4555 mutex_unlock(&trace_enum_mutex
);
4558 static void trace_create_enum_file(struct dentry
*d_tracer
)
4560 trace_create_file("enum_map", 0444, d_tracer
,
4561 NULL
, &tracing_enum_map_fops
);
4564 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4565 static inline void trace_create_enum_file(struct dentry
*d_tracer
) { }
4566 static inline void trace_insert_enum_map_file(struct module
*mod
,
4567 struct trace_enum_map
**start
, int len
) { }
4568 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4570 static void trace_insert_enum_map(struct module
*mod
,
4571 struct trace_enum_map
**start
, int len
)
4573 struct trace_enum_map
**map
;
4580 trace_event_enum_update(map
, len
);
4582 trace_insert_enum_map_file(mod
, start
, len
);
4586 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
4587 size_t cnt
, loff_t
*ppos
)
4589 struct trace_array
*tr
= filp
->private_data
;
4590 char buf
[MAX_TRACER_SIZE
+2];
4593 mutex_lock(&trace_types_lock
);
4594 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
4595 mutex_unlock(&trace_types_lock
);
4597 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4600 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
4602 tracing_reset_online_cpus(&tr
->trace_buffer
);
4606 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
4610 for_each_tracing_cpu(cpu
)
4611 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
4614 #ifdef CONFIG_TRACER_MAX_TRACE
4615 /* resize @tr's buffer to the size of @size_tr's entries */
4616 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
4617 struct trace_buffer
*size_buf
, int cpu_id
)
4621 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
4622 for_each_tracing_cpu(cpu
) {
4623 ret
= ring_buffer_resize(trace_buf
->buffer
,
4624 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
4627 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
4628 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
4631 ret
= ring_buffer_resize(trace_buf
->buffer
,
4632 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
4634 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
4635 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
4640 #endif /* CONFIG_TRACER_MAX_TRACE */
4642 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
4643 unsigned long size
, int cpu
)
4648 * If kernel or user changes the size of the ring buffer
4649 * we use the size that was given, and we can forget about
4650 * expanding it later.
4652 ring_buffer_expanded
= true;
4654 /* May be called before buffers are initialized */
4655 if (!tr
->trace_buffer
.buffer
)
4658 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
4662 #ifdef CONFIG_TRACER_MAX_TRACE
4663 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
4664 !tr
->current_trace
->use_max_tr
)
4667 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
4669 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
4670 &tr
->trace_buffer
, cpu
);
4673 * AARGH! We are left with different
4674 * size max buffer!!!!
4675 * The max buffer is our "snapshot" buffer.
4676 * When a tracer needs a snapshot (one of the
4677 * latency tracers), it swaps the max buffer
4678 * with the saved snap shot. We succeeded to
4679 * update the size of the main buffer, but failed to
4680 * update the size of the max buffer. But when we tried
4681 * to reset the main buffer to the original size, we
4682 * failed there too. This is very unlikely to
4683 * happen, but if it does, warn and kill all
4687 tracing_disabled
= 1;
4692 if (cpu
== RING_BUFFER_ALL_CPUS
)
4693 set_buffer_entries(&tr
->max_buffer
, size
);
4695 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
4698 #endif /* CONFIG_TRACER_MAX_TRACE */
4700 if (cpu
== RING_BUFFER_ALL_CPUS
)
4701 set_buffer_entries(&tr
->trace_buffer
, size
);
4703 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
4708 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
4709 unsigned long size
, int cpu_id
)
4713 mutex_lock(&trace_types_lock
);
4715 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
4716 /* make sure, this cpu is enabled in the mask */
4717 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
4723 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
4728 mutex_unlock(&trace_types_lock
);
4735 * tracing_update_buffers - used by tracing facility to expand ring buffers
4737 * To save on memory when the tracing is never used on a system with it
4738 * configured in. The ring buffers are set to a minimum size. But once
4739 * a user starts to use the tracing facility, then they need to grow
4740 * to their default size.
4742 * This function is to be called when a tracer is about to be used.
4744 int tracing_update_buffers(void)
4748 mutex_lock(&trace_types_lock
);
4749 if (!ring_buffer_expanded
)
4750 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
4751 RING_BUFFER_ALL_CPUS
);
4752 mutex_unlock(&trace_types_lock
);
4757 struct trace_option_dentry
;
4760 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
4763 * Used to clear out the tracer before deletion of an instance.
4764 * Must have trace_types_lock held.
4766 static void tracing_set_nop(struct trace_array
*tr
)
4768 if (tr
->current_trace
== &nop_trace
)
4771 tr
->current_trace
->enabled
--;
4773 if (tr
->current_trace
->reset
)
4774 tr
->current_trace
->reset(tr
);
4776 tr
->current_trace
= &nop_trace
;
4779 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
)
4781 /* Only enable if the directory has been created already. */
4785 create_trace_option_files(tr
, t
);
4788 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
4791 #ifdef CONFIG_TRACER_MAX_TRACE
4796 mutex_lock(&trace_types_lock
);
4798 if (!ring_buffer_expanded
) {
4799 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
4800 RING_BUFFER_ALL_CPUS
);
4806 for (t
= trace_types
; t
; t
= t
->next
) {
4807 if (strcmp(t
->name
, buf
) == 0)
4814 if (t
== tr
->current_trace
)
4817 /* Some tracers are only allowed for the top level buffer */
4818 if (!trace_ok_for_array(t
, tr
)) {
4823 /* If trace pipe files are being read, we can't change the tracer */
4824 if (tr
->current_trace
->ref
) {
4829 trace_branch_disable();
4831 tr
->current_trace
->enabled
--;
4833 if (tr
->current_trace
->reset
)
4834 tr
->current_trace
->reset(tr
);
4836 /* Current trace needs to be nop_trace before synchronize_sched */
4837 tr
->current_trace
= &nop_trace
;
4839 #ifdef CONFIG_TRACER_MAX_TRACE
4840 had_max_tr
= tr
->allocated_snapshot
;
4842 if (had_max_tr
&& !t
->use_max_tr
) {
4844 * We need to make sure that the update_max_tr sees that
4845 * current_trace changed to nop_trace to keep it from
4846 * swapping the buffers after we resize it.
4847 * The update_max_tr is called from interrupts disabled
4848 * so a synchronized_sched() is sufficient.
4850 synchronize_sched();
4855 #ifdef CONFIG_TRACER_MAX_TRACE
4856 if (t
->use_max_tr
&& !had_max_tr
) {
4857 ret
= alloc_snapshot(tr
);
4864 ret
= tracer_init(t
, tr
);
4869 tr
->current_trace
= t
;
4870 tr
->current_trace
->enabled
++;
4871 trace_branch_enable(tr
);
4873 mutex_unlock(&trace_types_lock
);
4879 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
4880 size_t cnt
, loff_t
*ppos
)
4882 struct trace_array
*tr
= filp
->private_data
;
4883 char buf
[MAX_TRACER_SIZE
+1];
4890 if (cnt
> MAX_TRACER_SIZE
)
4891 cnt
= MAX_TRACER_SIZE
;
4893 if (copy_from_user(buf
, ubuf
, cnt
))
4898 /* strip ending whitespace. */
4899 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
4902 err
= tracing_set_tracer(tr
, buf
);
4912 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
4913 size_t cnt
, loff_t
*ppos
)
4918 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
4919 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
4920 if (r
> sizeof(buf
))
4922 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4926 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
4927 size_t cnt
, loff_t
*ppos
)
4932 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4942 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
4943 size_t cnt
, loff_t
*ppos
)
4945 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
4949 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
4950 size_t cnt
, loff_t
*ppos
)
4952 struct trace_array
*tr
= filp
->private_data
;
4955 mutex_lock(&trace_types_lock
);
4956 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
4960 if (tr
->current_trace
->update_thresh
) {
4961 ret
= tr
->current_trace
->update_thresh(tr
);
4968 mutex_unlock(&trace_types_lock
);
4973 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
4976 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
4977 size_t cnt
, loff_t
*ppos
)
4979 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
4983 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
4984 size_t cnt
, loff_t
*ppos
)
4986 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
4991 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
4993 struct trace_array
*tr
= inode
->i_private
;
4994 struct trace_iterator
*iter
;
4997 if (tracing_disabled
)
5000 if (trace_array_get(tr
) < 0)
5003 mutex_lock(&trace_types_lock
);
5005 /* create a buffer to store the information to pass to userspace */
5006 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5009 __trace_array_put(tr
);
5013 trace_seq_init(&iter
->seq
);
5014 iter
->trace
= tr
->current_trace
;
5016 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
5021 /* trace pipe does not show start of buffer */
5022 cpumask_setall(iter
->started
);
5024 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
5025 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
5027 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5028 if (trace_clocks
[tr
->clock_id
].in_ns
)
5029 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
5032 iter
->trace_buffer
= &tr
->trace_buffer
;
5033 iter
->cpu_file
= tracing_get_cpu(inode
);
5034 mutex_init(&iter
->mutex
);
5035 filp
->private_data
= iter
;
5037 if (iter
->trace
->pipe_open
)
5038 iter
->trace
->pipe_open(iter
);
5040 nonseekable_open(inode
, filp
);
5042 tr
->current_trace
->ref
++;
5044 mutex_unlock(&trace_types_lock
);
5050 __trace_array_put(tr
);
5051 mutex_unlock(&trace_types_lock
);
5055 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
5057 struct trace_iterator
*iter
= file
->private_data
;
5058 struct trace_array
*tr
= inode
->i_private
;
5060 mutex_lock(&trace_types_lock
);
5062 tr
->current_trace
->ref
--;
5064 if (iter
->trace
->pipe_close
)
5065 iter
->trace
->pipe_close(iter
);
5067 mutex_unlock(&trace_types_lock
);
5069 free_cpumask_var(iter
->started
);
5070 mutex_destroy(&iter
->mutex
);
5073 trace_array_put(tr
);
5079 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
5081 struct trace_array
*tr
= iter
->tr
;
5083 /* Iterators are static, they should be filled or empty */
5084 if (trace_buffer_iter(iter
, iter
->cpu_file
))
5085 return POLLIN
| POLLRDNORM
;
5087 if (tr
->trace_flags
& TRACE_ITER_BLOCK
)
5089 * Always select as readable when in blocking mode
5091 return POLLIN
| POLLRDNORM
;
5093 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
5098 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
5100 struct trace_iterator
*iter
= filp
->private_data
;
5102 return trace_poll(iter
, filp
, poll_table
);
5105 /* Must be called with iter->mutex held. */
5106 static int tracing_wait_pipe(struct file
*filp
)
5108 struct trace_iterator
*iter
= filp
->private_data
;
5111 while (trace_empty(iter
)) {
5113 if ((filp
->f_flags
& O_NONBLOCK
)) {
5118 * We block until we read something and tracing is disabled.
5119 * We still block if tracing is disabled, but we have never
5120 * read anything. This allows a user to cat this file, and
5121 * then enable tracing. But after we have read something,
5122 * we give an EOF when tracing is again disabled.
5124 * iter->pos will be 0 if we haven't read anything.
5126 if (!tracing_is_on() && iter
->pos
)
5129 mutex_unlock(&iter
->mutex
);
5131 ret
= wait_on_pipe(iter
, false);
5133 mutex_lock(&iter
->mutex
);
5146 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
5147 size_t cnt
, loff_t
*ppos
)
5149 struct trace_iterator
*iter
= filp
->private_data
;
5153 * Avoid more than one consumer on a single file descriptor
5154 * This is just a matter of traces coherency, the ring buffer itself
5157 mutex_lock(&iter
->mutex
);
5159 /* return any leftover data */
5160 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
5164 trace_seq_init(&iter
->seq
);
5166 if (iter
->trace
->read
) {
5167 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
5173 sret
= tracing_wait_pipe(filp
);
5177 /* stop when tracing is finished */
5178 if (trace_empty(iter
)) {
5183 if (cnt
>= PAGE_SIZE
)
5184 cnt
= PAGE_SIZE
- 1;
5186 /* reset all but tr, trace, and overruns */
5187 memset(&iter
->seq
, 0,
5188 sizeof(struct trace_iterator
) -
5189 offsetof(struct trace_iterator
, seq
));
5190 cpumask_clear(iter
->started
);
5193 trace_event_read_lock();
5194 trace_access_lock(iter
->cpu_file
);
5195 while (trace_find_next_entry_inc(iter
) != NULL
) {
5196 enum print_line_t ret
;
5197 int save_len
= iter
->seq
.seq
.len
;
5199 ret
= print_trace_line(iter
);
5200 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
5201 /* don't print partial lines */
5202 iter
->seq
.seq
.len
= save_len
;
5205 if (ret
!= TRACE_TYPE_NO_CONSUME
)
5206 trace_consume(iter
);
5208 if (trace_seq_used(&iter
->seq
) >= cnt
)
5212 * Setting the full flag means we reached the trace_seq buffer
5213 * size and we should leave by partial output condition above.
5214 * One of the trace_seq_* functions is not used properly.
5216 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
5219 trace_access_unlock(iter
->cpu_file
);
5220 trace_event_read_unlock();
5222 /* Now copy what we have to the user */
5223 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
5224 if (iter
->seq
.seq
.readpos
>= trace_seq_used(&iter
->seq
))
5225 trace_seq_init(&iter
->seq
);
5228 * If there was nothing to send to user, in spite of consuming trace
5229 * entries, go back to wait for more entries.
5235 mutex_unlock(&iter
->mutex
);
5240 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
5243 __free_page(spd
->pages
[idx
]);
5246 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
5248 .confirm
= generic_pipe_buf_confirm
,
5249 .release
= generic_pipe_buf_release
,
5250 .steal
= generic_pipe_buf_steal
,
5251 .get
= generic_pipe_buf_get
,
5255 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
5261 /* Seq buffer is page-sized, exactly what we need. */
5263 save_len
= iter
->seq
.seq
.len
;
5264 ret
= print_trace_line(iter
);
5266 if (trace_seq_has_overflowed(&iter
->seq
)) {
5267 iter
->seq
.seq
.len
= save_len
;
5272 * This should not be hit, because it should only
5273 * be set if the iter->seq overflowed. But check it
5274 * anyway to be safe.
5276 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
5277 iter
->seq
.seq
.len
= save_len
;
5281 count
= trace_seq_used(&iter
->seq
) - save_len
;
5284 iter
->seq
.seq
.len
= save_len
;
5288 if (ret
!= TRACE_TYPE_NO_CONSUME
)
5289 trace_consume(iter
);
5291 if (!trace_find_next_entry_inc(iter
)) {
5301 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
5303 struct pipe_inode_info
*pipe
,
5307 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
5308 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
5309 struct trace_iterator
*iter
= filp
->private_data
;
5310 struct splice_pipe_desc spd
= {
5312 .partial
= partial_def
,
5313 .nr_pages
= 0, /* This gets updated below. */
5314 .nr_pages_max
= PIPE_DEF_BUFFERS
,
5316 .ops
= &tracing_pipe_buf_ops
,
5317 .spd_release
= tracing_spd_release_pipe
,
5323 if (splice_grow_spd(pipe
, &spd
))
5326 mutex_lock(&iter
->mutex
);
5328 if (iter
->trace
->splice_read
) {
5329 ret
= iter
->trace
->splice_read(iter
, filp
,
5330 ppos
, pipe
, len
, flags
);
5335 ret
= tracing_wait_pipe(filp
);
5339 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
5344 trace_event_read_lock();
5345 trace_access_lock(iter
->cpu_file
);
5347 /* Fill as many pages as possible. */
5348 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
5349 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
5353 rem
= tracing_fill_pipe_page(rem
, iter
);
5355 /* Copy the data into the page, so we can start over. */
5356 ret
= trace_seq_to_buffer(&iter
->seq
,
5357 page_address(spd
.pages
[i
]),
5358 trace_seq_used(&iter
->seq
));
5360 __free_page(spd
.pages
[i
]);
5363 spd
.partial
[i
].offset
= 0;
5364 spd
.partial
[i
].len
= trace_seq_used(&iter
->seq
);
5366 trace_seq_init(&iter
->seq
);
5369 trace_access_unlock(iter
->cpu_file
);
5370 trace_event_read_unlock();
5371 mutex_unlock(&iter
->mutex
);
5376 ret
= splice_to_pipe(pipe
, &spd
);
5380 splice_shrink_spd(&spd
);
5384 mutex_unlock(&iter
->mutex
);
5389 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
5390 size_t cnt
, loff_t
*ppos
)
5392 struct inode
*inode
= file_inode(filp
);
5393 struct trace_array
*tr
= inode
->i_private
;
5394 int cpu
= tracing_get_cpu(inode
);
5399 mutex_lock(&trace_types_lock
);
5401 if (cpu
== RING_BUFFER_ALL_CPUS
) {
5402 int cpu
, buf_size_same
;
5407 /* check if all cpu sizes are same */
5408 for_each_tracing_cpu(cpu
) {
5409 /* fill in the size from first enabled cpu */
5411 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
5412 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
5418 if (buf_size_same
) {
5419 if (!ring_buffer_expanded
)
5420 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
5422 trace_buf_size
>> 10);
5424 r
= sprintf(buf
, "%lu\n", size
>> 10);
5426 r
= sprintf(buf
, "X\n");
5428 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
5430 mutex_unlock(&trace_types_lock
);
5432 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5437 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
5438 size_t cnt
, loff_t
*ppos
)
5440 struct inode
*inode
= file_inode(filp
);
5441 struct trace_array
*tr
= inode
->i_private
;
5445 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5449 /* must have at least 1 entry */
5453 /* value is in KB */
5455 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
5465 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
5466 size_t cnt
, loff_t
*ppos
)
5468 struct trace_array
*tr
= filp
->private_data
;
5471 unsigned long size
= 0, expanded_size
= 0;
5473 mutex_lock(&trace_types_lock
);
5474 for_each_tracing_cpu(cpu
) {
5475 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
5476 if (!ring_buffer_expanded
)
5477 expanded_size
+= trace_buf_size
>> 10;
5479 if (ring_buffer_expanded
)
5480 r
= sprintf(buf
, "%lu\n", size
);
5482 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
5483 mutex_unlock(&trace_types_lock
);
5485 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5489 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
5490 size_t cnt
, loff_t
*ppos
)
5493 * There is no need to read what the user has written, this function
5494 * is just to make sure that there is no error when "echo" is used
5503 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
5505 struct trace_array
*tr
= inode
->i_private
;
5507 /* disable tracing ? */
5508 if (tr
->trace_flags
& TRACE_ITER_STOP_ON_FREE
)
5509 tracer_tracing_off(tr
);
5510 /* resize the ring buffer to 0 */
5511 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
5513 trace_array_put(tr
);
5519 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
5520 size_t cnt
, loff_t
*fpos
)
5522 unsigned long addr
= (unsigned long)ubuf
;
5523 struct trace_array
*tr
= filp
->private_data
;
5524 struct ring_buffer_event
*event
;
5525 struct ring_buffer
*buffer
;
5526 struct print_entry
*entry
;
5527 unsigned long irq_flags
;
5528 struct page
*pages
[2];
5538 if (tracing_disabled
)
5541 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
5544 if (cnt
> TRACE_BUF_SIZE
)
5545 cnt
= TRACE_BUF_SIZE
;
5548 * Userspace is injecting traces into the kernel trace buffer.
5549 * We want to be as non intrusive as possible.
5550 * To do so, we do not want to allocate any special buffers
5551 * or take any locks, but instead write the userspace data
5552 * straight into the ring buffer.
5554 * First we need to pin the userspace buffer into memory,
5555 * which, most likely it is, because it just referenced it.
5556 * But there's no guarantee that it is. By using get_user_pages_fast()
5557 * and kmap_atomic/kunmap_atomic() we can get access to the
5558 * pages directly. We then write the data directly into the
5561 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
5563 /* check if we cross pages */
5564 if ((addr
& PAGE_MASK
) != ((addr
+ cnt
) & PAGE_MASK
))
5567 offset
= addr
& (PAGE_SIZE
- 1);
5570 ret
= get_user_pages_fast(addr
, nr_pages
, 0, pages
);
5571 if (ret
< nr_pages
) {
5573 put_page(pages
[ret
]);
5578 for (i
= 0; i
< nr_pages
; i
++)
5579 map_page
[i
] = kmap_atomic(pages
[i
]);
5581 local_save_flags(irq_flags
);
5582 size
= sizeof(*entry
) + cnt
+ 2; /* possible \n added */
5583 buffer
= tr
->trace_buffer
.buffer
;
5584 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
5585 irq_flags
, preempt_count());
5587 /* Ring buffer disabled, return as if not open for write */
5592 entry
= ring_buffer_event_data(event
);
5593 entry
->ip
= _THIS_IP_
;
5595 if (nr_pages
== 2) {
5596 len
= PAGE_SIZE
- offset
;
5597 memcpy(&entry
->buf
, map_page
[0] + offset
, len
);
5598 memcpy(&entry
->buf
[len
], map_page
[1], cnt
- len
);
5600 memcpy(&entry
->buf
, map_page
[0] + offset
, cnt
);
5602 if (entry
->buf
[cnt
- 1] != '\n') {
5603 entry
->buf
[cnt
] = '\n';
5604 entry
->buf
[cnt
+ 1] = '\0';
5606 entry
->buf
[cnt
] = '\0';
5608 __buffer_unlock_commit(buffer
, event
);
5615 for (i
= nr_pages
- 1; i
>= 0; i
--) {
5616 kunmap_atomic(map_page
[i
]);
5623 static int tracing_clock_show(struct seq_file
*m
, void *v
)
5625 struct trace_array
*tr
= m
->private;
5628 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
5630 "%s%s%s%s", i
? " " : "",
5631 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
5632 i
== tr
->clock_id
? "]" : "");
5638 static int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
5642 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
5643 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
5646 if (i
== ARRAY_SIZE(trace_clocks
))
5649 mutex_lock(&trace_types_lock
);
5653 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
5656 * New clock may not be consistent with the previous clock.
5657 * Reset the buffer so that it doesn't have incomparable timestamps.
5659 tracing_reset_online_cpus(&tr
->trace_buffer
);
5661 #ifdef CONFIG_TRACER_MAX_TRACE
5662 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
&& tr
->max_buffer
.buffer
)
5663 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
5664 tracing_reset_online_cpus(&tr
->max_buffer
);
5667 mutex_unlock(&trace_types_lock
);
5672 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
5673 size_t cnt
, loff_t
*fpos
)
5675 struct seq_file
*m
= filp
->private_data
;
5676 struct trace_array
*tr
= m
->private;
5678 const char *clockstr
;
5681 if (cnt
>= sizeof(buf
))
5684 if (copy_from_user(buf
, ubuf
, cnt
))
5689 clockstr
= strstrip(buf
);
5691 ret
= tracing_set_clock(tr
, clockstr
);
5700 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
5702 struct trace_array
*tr
= inode
->i_private
;
5705 if (tracing_disabled
)
5708 if (trace_array_get(tr
))
5711 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
5713 trace_array_put(tr
);
5718 struct ftrace_buffer_info
{
5719 struct trace_iterator iter
;
5724 #ifdef CONFIG_TRACER_SNAPSHOT
5725 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
5727 struct trace_array
*tr
= inode
->i_private
;
5728 struct trace_iterator
*iter
;
5732 if (trace_array_get(tr
) < 0)
5735 if (file
->f_mode
& FMODE_READ
) {
5736 iter
= __tracing_open(inode
, file
, true);
5738 ret
= PTR_ERR(iter
);
5740 /* Writes still need the seq_file to hold the private data */
5742 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
5745 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5753 iter
->trace_buffer
= &tr
->max_buffer
;
5754 iter
->cpu_file
= tracing_get_cpu(inode
);
5756 file
->private_data
= m
;
5760 trace_array_put(tr
);
5766 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
5769 struct seq_file
*m
= filp
->private_data
;
5770 struct trace_iterator
*iter
= m
->private;
5771 struct trace_array
*tr
= iter
->tr
;
5775 ret
= tracing_update_buffers();
5779 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5783 mutex_lock(&trace_types_lock
);
5785 if (tr
->current_trace
->use_max_tr
) {
5792 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5796 if (tr
->allocated_snapshot
)
5800 /* Only allow per-cpu swap if the ring buffer supports it */
5801 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5802 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5807 if (!tr
->allocated_snapshot
) {
5808 ret
= alloc_snapshot(tr
);
5812 local_irq_disable();
5813 /* Now, we're going to swap */
5814 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5815 update_max_tr(tr
, current
, smp_processor_id());
5817 update_max_tr_single(tr
, current
, iter
->cpu_file
);
5821 if (tr
->allocated_snapshot
) {
5822 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5823 tracing_reset_online_cpus(&tr
->max_buffer
);
5825 tracing_reset(&tr
->max_buffer
, iter
->cpu_file
);
5835 mutex_unlock(&trace_types_lock
);
5839 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
5841 struct seq_file
*m
= file
->private_data
;
5844 ret
= tracing_release(inode
, file
);
5846 if (file
->f_mode
& FMODE_READ
)
5849 /* If write only, the seq_file is just a stub */
5857 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
5858 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5859 size_t count
, loff_t
*ppos
);
5860 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
5861 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5862 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
5864 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
5866 struct ftrace_buffer_info
*info
;
5869 ret
= tracing_buffers_open(inode
, filp
);
5873 info
= filp
->private_data
;
5875 if (info
->iter
.trace
->use_max_tr
) {
5876 tracing_buffers_release(inode
, filp
);
5880 info
->iter
.snapshot
= true;
5881 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
5886 #endif /* CONFIG_TRACER_SNAPSHOT */
5889 static const struct file_operations tracing_thresh_fops
= {
5890 .open
= tracing_open_generic
,
5891 .read
= tracing_thresh_read
,
5892 .write
= tracing_thresh_write
,
5893 .llseek
= generic_file_llseek
,
5896 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5897 static const struct file_operations tracing_max_lat_fops
= {
5898 .open
= tracing_open_generic
,
5899 .read
= tracing_max_lat_read
,
5900 .write
= tracing_max_lat_write
,
5901 .llseek
= generic_file_llseek
,
5905 static const struct file_operations set_tracer_fops
= {
5906 .open
= tracing_open_generic
,
5907 .read
= tracing_set_trace_read
,
5908 .write
= tracing_set_trace_write
,
5909 .llseek
= generic_file_llseek
,
5912 static const struct file_operations tracing_pipe_fops
= {
5913 .open
= tracing_open_pipe
,
5914 .poll
= tracing_poll_pipe
,
5915 .read
= tracing_read_pipe
,
5916 .splice_read
= tracing_splice_read_pipe
,
5917 .release
= tracing_release_pipe
,
5918 .llseek
= no_llseek
,
5921 static const struct file_operations tracing_entries_fops
= {
5922 .open
= tracing_open_generic_tr
,
5923 .read
= tracing_entries_read
,
5924 .write
= tracing_entries_write
,
5925 .llseek
= generic_file_llseek
,
5926 .release
= tracing_release_generic_tr
,
5929 static const struct file_operations tracing_total_entries_fops
= {
5930 .open
= tracing_open_generic_tr
,
5931 .read
= tracing_total_entries_read
,
5932 .llseek
= generic_file_llseek
,
5933 .release
= tracing_release_generic_tr
,
5936 static const struct file_operations tracing_free_buffer_fops
= {
5937 .open
= tracing_open_generic_tr
,
5938 .write
= tracing_free_buffer_write
,
5939 .release
= tracing_free_buffer_release
,
5942 static const struct file_operations tracing_mark_fops
= {
5943 .open
= tracing_open_generic_tr
,
5944 .write
= tracing_mark_write
,
5945 .llseek
= generic_file_llseek
,
5946 .release
= tracing_release_generic_tr
,
5949 static const struct file_operations trace_clock_fops
= {
5950 .open
= tracing_clock_open
,
5952 .llseek
= seq_lseek
,
5953 .release
= tracing_single_release_tr
,
5954 .write
= tracing_clock_write
,
5957 #ifdef CONFIG_TRACER_SNAPSHOT
5958 static const struct file_operations snapshot_fops
= {
5959 .open
= tracing_snapshot_open
,
5961 .write
= tracing_snapshot_write
,
5962 .llseek
= tracing_lseek
,
5963 .release
= tracing_snapshot_release
,
5966 static const struct file_operations snapshot_raw_fops
= {
5967 .open
= snapshot_raw_open
,
5968 .read
= tracing_buffers_read
,
5969 .release
= tracing_buffers_release
,
5970 .splice_read
= tracing_buffers_splice_read
,
5971 .llseek
= no_llseek
,
5974 #endif /* CONFIG_TRACER_SNAPSHOT */
5976 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
5978 struct trace_array
*tr
= inode
->i_private
;
5979 struct ftrace_buffer_info
*info
;
5982 if (tracing_disabled
)
5985 if (trace_array_get(tr
) < 0)
5988 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
5990 trace_array_put(tr
);
5994 mutex_lock(&trace_types_lock
);
5997 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
5998 info
->iter
.trace
= tr
->current_trace
;
5999 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
6001 /* Force reading ring buffer for first read */
6002 info
->read
= (unsigned int)-1;
6004 filp
->private_data
= info
;
6006 tr
->current_trace
->ref
++;
6008 mutex_unlock(&trace_types_lock
);
6010 ret
= nonseekable_open(inode
, filp
);
6012 trace_array_put(tr
);
6018 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
6020 struct ftrace_buffer_info
*info
= filp
->private_data
;
6021 struct trace_iterator
*iter
= &info
->iter
;
6023 return trace_poll(iter
, filp
, poll_table
);
6027 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
6028 size_t count
, loff_t
*ppos
)
6030 struct ftrace_buffer_info
*info
= filp
->private_data
;
6031 struct trace_iterator
*iter
= &info
->iter
;
6038 #ifdef CONFIG_TRACER_MAX_TRACE
6039 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
6044 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
6049 /* Do we have previous read data to read? */
6050 if (info
->read
< PAGE_SIZE
)
6054 trace_access_lock(iter
->cpu_file
);
6055 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
6059 trace_access_unlock(iter
->cpu_file
);
6062 if (trace_empty(iter
)) {
6063 if ((filp
->f_flags
& O_NONBLOCK
))
6066 ret
= wait_on_pipe(iter
, false);
6077 size
= PAGE_SIZE
- info
->read
;
6081 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
6093 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
6095 struct ftrace_buffer_info
*info
= file
->private_data
;
6096 struct trace_iterator
*iter
= &info
->iter
;
6098 mutex_lock(&trace_types_lock
);
6100 iter
->tr
->current_trace
->ref
--;
6102 __trace_array_put(iter
->tr
);
6105 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
, info
->spare
);
6108 mutex_unlock(&trace_types_lock
);
6114 struct ring_buffer
*buffer
;
6119 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
6120 struct pipe_buffer
*buf
)
6122 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
6127 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6132 static void buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
6133 struct pipe_buffer
*buf
)
6135 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
6140 /* Pipe buffer operations for a buffer. */
6141 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
6143 .confirm
= generic_pipe_buf_confirm
,
6144 .release
= buffer_pipe_buf_release
,
6145 .steal
= generic_pipe_buf_steal
,
6146 .get
= buffer_pipe_buf_get
,
6150 * Callback from splice_to_pipe(), if we need to release some pages
6151 * at the end of the spd in case we error'ed out in filling the pipe.
6153 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
6155 struct buffer_ref
*ref
=
6156 (struct buffer_ref
*)spd
->partial
[i
].private;
6161 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6163 spd
->partial
[i
].private = 0;
6167 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
6168 struct pipe_inode_info
*pipe
, size_t len
,
6171 struct ftrace_buffer_info
*info
= file
->private_data
;
6172 struct trace_iterator
*iter
= &info
->iter
;
6173 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
6174 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
6175 struct splice_pipe_desc spd
= {
6177 .partial
= partial_def
,
6178 .nr_pages_max
= PIPE_DEF_BUFFERS
,
6180 .ops
= &buffer_pipe_buf_ops
,
6181 .spd_release
= buffer_spd_release
,
6183 struct buffer_ref
*ref
;
6184 int entries
, size
, i
;
6187 #ifdef CONFIG_TRACER_MAX_TRACE
6188 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
6192 if (*ppos
& (PAGE_SIZE
- 1))
6195 if (len
& (PAGE_SIZE
- 1)) {
6196 if (len
< PAGE_SIZE
)
6201 if (splice_grow_spd(pipe
, &spd
))
6205 trace_access_lock(iter
->cpu_file
);
6206 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
6208 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
6212 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
6219 ref
->buffer
= iter
->trace_buffer
->buffer
;
6220 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
6227 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
6228 len
, iter
->cpu_file
, 1);
6230 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
6236 * zero out any left over data, this is going to
6239 size
= ring_buffer_page_len(ref
->page
);
6240 if (size
< PAGE_SIZE
)
6241 memset(ref
->page
+ size
, 0, PAGE_SIZE
- size
);
6243 page
= virt_to_page(ref
->page
);
6245 spd
.pages
[i
] = page
;
6246 spd
.partial
[i
].len
= PAGE_SIZE
;
6247 spd
.partial
[i
].offset
= 0;
6248 spd
.partial
[i
].private = (unsigned long)ref
;
6252 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
6255 trace_access_unlock(iter
->cpu_file
);
6258 /* did we read anything? */
6259 if (!spd
.nr_pages
) {
6264 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
))
6267 ret
= wait_on_pipe(iter
, true);
6274 ret
= splice_to_pipe(pipe
, &spd
);
6276 splice_shrink_spd(&spd
);
6281 static const struct file_operations tracing_buffers_fops
= {
6282 .open
= tracing_buffers_open
,
6283 .read
= tracing_buffers_read
,
6284 .poll
= tracing_buffers_poll
,
6285 .release
= tracing_buffers_release
,
6286 .splice_read
= tracing_buffers_splice_read
,
6287 .llseek
= no_llseek
,
6291 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
6292 size_t count
, loff_t
*ppos
)
6294 struct inode
*inode
= file_inode(filp
);
6295 struct trace_array
*tr
= inode
->i_private
;
6296 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
6297 int cpu
= tracing_get_cpu(inode
);
6298 struct trace_seq
*s
;
6300 unsigned long long t
;
6301 unsigned long usec_rem
;
6303 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
6309 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
6310 trace_seq_printf(s
, "entries: %ld\n", cnt
);
6312 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
6313 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
6315 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
6316 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
6318 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
6319 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
6321 if (trace_clocks
[tr
->clock_id
].in_ns
) {
6322 /* local or global for trace_clock */
6323 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
6324 usec_rem
= do_div(t
, USEC_PER_SEC
);
6325 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
6328 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
6329 usec_rem
= do_div(t
, USEC_PER_SEC
);
6330 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
6332 /* counter or tsc mode for trace_clock */
6333 trace_seq_printf(s
, "oldest event ts: %llu\n",
6334 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
6336 trace_seq_printf(s
, "now ts: %llu\n",
6337 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
6340 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
6341 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
6343 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
6344 trace_seq_printf(s
, "read events: %ld\n", cnt
);
6346 count
= simple_read_from_buffer(ubuf
, count
, ppos
,
6347 s
->buffer
, trace_seq_used(s
));
6354 static const struct file_operations tracing_stats_fops
= {
6355 .open
= tracing_open_generic_tr
,
6356 .read
= tracing_stats_read
,
6357 .llseek
= generic_file_llseek
,
6358 .release
= tracing_release_generic_tr
,
6361 #ifdef CONFIG_DYNAMIC_FTRACE
6363 int __weak
ftrace_arch_read_dyn_info(char *buf
, int size
)
6369 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
6370 size_t cnt
, loff_t
*ppos
)
6372 static char ftrace_dyn_info_buffer
[1024];
6373 static DEFINE_MUTEX(dyn_info_mutex
);
6374 unsigned long *p
= filp
->private_data
;
6375 char *buf
= ftrace_dyn_info_buffer
;
6376 int size
= ARRAY_SIZE(ftrace_dyn_info_buffer
);
6379 mutex_lock(&dyn_info_mutex
);
6380 r
= sprintf(buf
, "%ld ", *p
);
6382 r
+= ftrace_arch_read_dyn_info(buf
+r
, (size
-1)-r
);
6385 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6387 mutex_unlock(&dyn_info_mutex
);
6392 static const struct file_operations tracing_dyn_info_fops
= {
6393 .open
= tracing_open_generic
,
6394 .read
= tracing_read_dyn_info
,
6395 .llseek
= generic_file_llseek
,
6397 #endif /* CONFIG_DYNAMIC_FTRACE */
6399 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6401 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6407 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6409 unsigned long *count
= (long *)data
;
6421 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
6422 struct ftrace_probe_ops
*ops
, void *data
)
6424 long count
= (long)data
;
6426 seq_printf(m
, "%ps:", (void *)ip
);
6428 seq_puts(m
, "snapshot");
6431 seq_puts(m
, ":unlimited\n");
6433 seq_printf(m
, ":count=%ld\n", count
);
6438 static struct ftrace_probe_ops snapshot_probe_ops
= {
6439 .func
= ftrace_snapshot
,
6440 .print
= ftrace_snapshot_print
,
6443 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
6444 .func
= ftrace_count_snapshot
,
6445 .print
= ftrace_snapshot_print
,
6449 ftrace_trace_snapshot_callback(struct ftrace_hash
*hash
,
6450 char *glob
, char *cmd
, char *param
, int enable
)
6452 struct ftrace_probe_ops
*ops
;
6453 void *count
= (void *)-1;
6457 /* hash funcs only work with set_ftrace_filter */
6461 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
6463 if (glob
[0] == '!') {
6464 unregister_ftrace_function_probe_func(glob
+1, ops
);
6471 number
= strsep(¶m
, ":");
6473 if (!strlen(number
))
6477 * We use the callback data field (which is a pointer)
6480 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
6485 ret
= register_ftrace_function_probe(glob
, ops
, count
);
6488 alloc_snapshot(&global_trace
);
6490 return ret
< 0 ? ret
: 0;
6493 static struct ftrace_func_command ftrace_snapshot_cmd
= {
6495 .func
= ftrace_trace_snapshot_callback
,
6498 static __init
int register_snapshot_cmd(void)
6500 return register_ftrace_command(&ftrace_snapshot_cmd
);
6503 static inline __init
int register_snapshot_cmd(void) { return 0; }
6504 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6506 static struct dentry
*tracing_get_dentry(struct trace_array
*tr
)
6508 if (WARN_ON(!tr
->dir
))
6509 return ERR_PTR(-ENODEV
);
6511 /* Top directory uses NULL as the parent */
6512 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
6515 /* All sub buffers have a descriptor */
6519 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
6521 struct dentry
*d_tracer
;
6524 return tr
->percpu_dir
;
6526 d_tracer
= tracing_get_dentry(tr
);
6527 if (IS_ERR(d_tracer
))
6530 tr
->percpu_dir
= tracefs_create_dir("per_cpu", d_tracer
);
6532 WARN_ONCE(!tr
->percpu_dir
,
6533 "Could not create tracefs directory 'per_cpu/%d'\n", cpu
);
6535 return tr
->percpu_dir
;
6538 static struct dentry
*
6539 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
6540 void *data
, long cpu
, const struct file_operations
*fops
)
6542 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
6544 if (ret
) /* See tracing_get_cpu() */
6545 d_inode(ret
)->i_cdev
= (void *)(cpu
+ 1);
6550 tracing_init_tracefs_percpu(struct trace_array
*tr
, long cpu
)
6552 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
6553 struct dentry
*d_cpu
;
6554 char cpu_dir
[30]; /* 30 characters should be more than enough */
6559 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
6560 d_cpu
= tracefs_create_dir(cpu_dir
, d_percpu
);
6562 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir
);
6566 /* per cpu trace_pipe */
6567 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
6568 tr
, cpu
, &tracing_pipe_fops
);
6571 trace_create_cpu_file("trace", 0644, d_cpu
,
6572 tr
, cpu
, &tracing_fops
);
6574 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
6575 tr
, cpu
, &tracing_buffers_fops
);
6577 trace_create_cpu_file("stats", 0444, d_cpu
,
6578 tr
, cpu
, &tracing_stats_fops
);
6580 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
6581 tr
, cpu
, &tracing_entries_fops
);
6583 #ifdef CONFIG_TRACER_SNAPSHOT
6584 trace_create_cpu_file("snapshot", 0644, d_cpu
,
6585 tr
, cpu
, &snapshot_fops
);
6587 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
6588 tr
, cpu
, &snapshot_raw_fops
);
6592 #ifdef CONFIG_FTRACE_SELFTEST
6593 /* Let selftest have access to static functions in this file */
6594 #include "trace_selftest.c"
6598 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6601 struct trace_option_dentry
*topt
= filp
->private_data
;
6604 if (topt
->flags
->val
& topt
->opt
->bit
)
6609 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6613 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6616 struct trace_option_dentry
*topt
= filp
->private_data
;
6620 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6624 if (val
!= 0 && val
!= 1)
6627 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
6628 mutex_lock(&trace_types_lock
);
6629 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
6631 mutex_unlock(&trace_types_lock
);
6642 static const struct file_operations trace_options_fops
= {
6643 .open
= tracing_open_generic
,
6644 .read
= trace_options_read
,
6645 .write
= trace_options_write
,
6646 .llseek
= generic_file_llseek
,
6650 * In order to pass in both the trace_array descriptor as well as the index
6651 * to the flag that the trace option file represents, the trace_array
6652 * has a character array of trace_flags_index[], which holds the index
6653 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6654 * The address of this character array is passed to the flag option file
6655 * read/write callbacks.
6657 * In order to extract both the index and the trace_array descriptor,
6658 * get_tr_index() uses the following algorithm.
6662 * As the pointer itself contains the address of the index (remember
6665 * Then to get the trace_array descriptor, by subtracting that index
6666 * from the ptr, we get to the start of the index itself.
6668 * ptr - idx == &index[0]
6670 * Then a simple container_of() from that pointer gets us to the
6671 * trace_array descriptor.
6673 static void get_tr_index(void *data
, struct trace_array
**ptr
,
6674 unsigned int *pindex
)
6676 *pindex
= *(unsigned char *)data
;
6678 *ptr
= container_of(data
- *pindex
, struct trace_array
,
6683 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6686 void *tr_index
= filp
->private_data
;
6687 struct trace_array
*tr
;
6691 get_tr_index(tr_index
, &tr
, &index
);
6693 if (tr
->trace_flags
& (1 << index
))
6698 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6702 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6705 void *tr_index
= filp
->private_data
;
6706 struct trace_array
*tr
;
6711 get_tr_index(tr_index
, &tr
, &index
);
6713 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6717 if (val
!= 0 && val
!= 1)
6720 mutex_lock(&trace_types_lock
);
6721 ret
= set_tracer_flag(tr
, 1 << index
, val
);
6722 mutex_unlock(&trace_types_lock
);
6732 static const struct file_operations trace_options_core_fops
= {
6733 .open
= tracing_open_generic
,
6734 .read
= trace_options_core_read
,
6735 .write
= trace_options_core_write
,
6736 .llseek
= generic_file_llseek
,
6739 struct dentry
*trace_create_file(const char *name
,
6741 struct dentry
*parent
,
6743 const struct file_operations
*fops
)
6747 ret
= tracefs_create_file(name
, mode
, parent
, data
, fops
);
6749 pr_warn("Could not create tracefs '%s' entry\n", name
);
6755 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
6757 struct dentry
*d_tracer
;
6762 d_tracer
= tracing_get_dentry(tr
);
6763 if (IS_ERR(d_tracer
))
6766 tr
->options
= tracefs_create_dir("options", d_tracer
);
6768 pr_warn("Could not create tracefs directory 'options'\n");
6776 create_trace_option_file(struct trace_array
*tr
,
6777 struct trace_option_dentry
*topt
,
6778 struct tracer_flags
*flags
,
6779 struct tracer_opt
*opt
)
6781 struct dentry
*t_options
;
6783 t_options
= trace_options_init_dentry(tr
);
6787 topt
->flags
= flags
;
6791 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
6792 &trace_options_fops
);
6797 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
6799 struct trace_option_dentry
*topts
;
6800 struct trace_options
*tr_topts
;
6801 struct tracer_flags
*flags
;
6802 struct tracer_opt
*opts
;
6809 flags
= tracer
->flags
;
6811 if (!flags
|| !flags
->opts
)
6815 * If this is an instance, only create flags for tracers
6816 * the instance may have.
6818 if (!trace_ok_for_array(tracer
, tr
))
6821 for (i
= 0; i
< tr
->nr_topts
; i
++) {
6822 /* Make sure there's no duplicate flags. */
6823 if (WARN_ON_ONCE(tr
->topts
[i
].tracer
->flags
== tracer
->flags
))
6829 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
6832 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
6836 tr_topts
= krealloc(tr
->topts
, sizeof(*tr
->topts
) * (tr
->nr_topts
+ 1),
6843 tr
->topts
= tr_topts
;
6844 tr
->topts
[tr
->nr_topts
].tracer
= tracer
;
6845 tr
->topts
[tr
->nr_topts
].topts
= topts
;
6848 for (cnt
= 0; opts
[cnt
].name
; cnt
++) {
6849 create_trace_option_file(tr
, &topts
[cnt
], flags
,
6851 WARN_ONCE(topts
[cnt
].entry
== NULL
,
6852 "Failed to create trace option: %s",
6857 static struct dentry
*
6858 create_trace_option_core_file(struct trace_array
*tr
,
6859 const char *option
, long index
)
6861 struct dentry
*t_options
;
6863 t_options
= trace_options_init_dentry(tr
);
6867 return trace_create_file(option
, 0644, t_options
,
6868 (void *)&tr
->trace_flags_index
[index
],
6869 &trace_options_core_fops
);
6872 static void create_trace_options_dir(struct trace_array
*tr
)
6874 struct dentry
*t_options
;
6875 bool top_level
= tr
== &global_trace
;
6878 t_options
= trace_options_init_dentry(tr
);
6882 for (i
= 0; trace_options
[i
]; i
++) {
6884 !((1 << i
) & TOP_LEVEL_TRACE_FLAGS
))
6885 create_trace_option_core_file(tr
, trace_options
[i
], i
);
6890 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
6891 size_t cnt
, loff_t
*ppos
)
6893 struct trace_array
*tr
= filp
->private_data
;
6897 r
= tracer_tracing_is_on(tr
);
6898 r
= sprintf(buf
, "%d\n", r
);
6900 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6904 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
6905 size_t cnt
, loff_t
*ppos
)
6907 struct trace_array
*tr
= filp
->private_data
;
6908 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
6912 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6917 mutex_lock(&trace_types_lock
);
6919 tracer_tracing_on(tr
);
6920 if (tr
->current_trace
->start
)
6921 tr
->current_trace
->start(tr
);
6923 tracer_tracing_off(tr
);
6924 if (tr
->current_trace
->stop
)
6925 tr
->current_trace
->stop(tr
);
6927 mutex_unlock(&trace_types_lock
);
6935 static const struct file_operations rb_simple_fops
= {
6936 .open
= tracing_open_generic_tr
,
6937 .read
= rb_simple_read
,
6938 .write
= rb_simple_write
,
6939 .release
= tracing_release_generic_tr
,
6940 .llseek
= default_llseek
,
6943 struct dentry
*trace_instance_dir
;
6946 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
);
6949 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
6951 enum ring_buffer_flags rb_flags
;
6953 rb_flags
= tr
->trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
6957 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
6961 buf
->data
= alloc_percpu(struct trace_array_cpu
);
6963 ring_buffer_free(buf
->buffer
);
6967 /* Allocate the first page for all buffers */
6968 set_buffer_entries(&tr
->trace_buffer
,
6969 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
6974 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
6978 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
6982 #ifdef CONFIG_TRACER_MAX_TRACE
6983 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
6984 allocate_snapshot
? size
: 1);
6986 ring_buffer_free(tr
->trace_buffer
.buffer
);
6987 free_percpu(tr
->trace_buffer
.data
);
6990 tr
->allocated_snapshot
= allocate_snapshot
;
6993 * Only the top level trace array gets its snapshot allocated
6994 * from the kernel command line.
6996 allocate_snapshot
= false;
7001 static void free_trace_buffer(struct trace_buffer
*buf
)
7004 ring_buffer_free(buf
->buffer
);
7006 free_percpu(buf
->data
);
7011 static void free_trace_buffers(struct trace_array
*tr
)
7016 free_trace_buffer(&tr
->trace_buffer
);
7018 #ifdef CONFIG_TRACER_MAX_TRACE
7019 free_trace_buffer(&tr
->max_buffer
);
7023 static void init_trace_flags_index(struct trace_array
*tr
)
7027 /* Used by the trace options files */
7028 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++)
7029 tr
->trace_flags_index
[i
] = i
;
7032 static void __update_tracer_options(struct trace_array
*tr
)
7036 for (t
= trace_types
; t
; t
= t
->next
)
7037 add_tracer_options(tr
, t
);
7040 static void update_tracer_options(struct trace_array
*tr
)
7042 mutex_lock(&trace_types_lock
);
7043 __update_tracer_options(tr
);
7044 mutex_unlock(&trace_types_lock
);
7047 static int instance_mkdir(const char *name
)
7049 struct trace_array
*tr
;
7052 mutex_lock(&trace_types_lock
);
7055 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
7056 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
7061 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
7065 tr
->name
= kstrdup(name
, GFP_KERNEL
);
7069 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
7072 tr
->trace_flags
= global_trace
.trace_flags
& ~ZEROED_TRACE_FLAGS
;
7074 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
7076 raw_spin_lock_init(&tr
->start_lock
);
7078 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
7080 tr
->current_trace
= &nop_trace
;
7082 INIT_LIST_HEAD(&tr
->systems
);
7083 INIT_LIST_HEAD(&tr
->events
);
7085 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
7088 tr
->dir
= tracefs_create_dir(name
, trace_instance_dir
);
7092 ret
= event_trace_add_tracer(tr
->dir
, tr
);
7094 tracefs_remove_recursive(tr
->dir
);
7098 init_tracer_tracefs(tr
, tr
->dir
);
7099 init_trace_flags_index(tr
);
7100 __update_tracer_options(tr
);
7102 list_add(&tr
->list
, &ftrace_trace_arrays
);
7104 mutex_unlock(&trace_types_lock
);
7109 free_trace_buffers(tr
);
7110 free_cpumask_var(tr
->tracing_cpumask
);
7115 mutex_unlock(&trace_types_lock
);
7121 static int instance_rmdir(const char *name
)
7123 struct trace_array
*tr
;
7128 mutex_lock(&trace_types_lock
);
7131 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
7132 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
7141 if (tr
->ref
|| (tr
->current_trace
&& tr
->current_trace
->ref
))
7144 list_del(&tr
->list
);
7146 /* Disable all the flags that were enabled coming in */
7147 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++) {
7148 if ((1 << i
) & ZEROED_TRACE_FLAGS
)
7149 set_tracer_flag(tr
, 1 << i
, 0);
7152 tracing_set_nop(tr
);
7153 event_trace_del_tracer(tr
);
7154 ftrace_destroy_function_files(tr
);
7155 tracefs_remove_recursive(tr
->dir
);
7156 free_trace_buffers(tr
);
7158 for (i
= 0; i
< tr
->nr_topts
; i
++) {
7159 kfree(tr
->topts
[i
].topts
);
7169 mutex_unlock(&trace_types_lock
);
7174 static __init
void create_trace_instances(struct dentry
*d_tracer
)
7176 trace_instance_dir
= tracefs_create_instance_dir("instances", d_tracer
,
7179 if (WARN_ON(!trace_instance_dir
))
7184 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
)
7188 trace_create_file("available_tracers", 0444, d_tracer
,
7189 tr
, &show_traces_fops
);
7191 trace_create_file("current_tracer", 0644, d_tracer
,
7192 tr
, &set_tracer_fops
);
7194 trace_create_file("tracing_cpumask", 0644, d_tracer
,
7195 tr
, &tracing_cpumask_fops
);
7197 trace_create_file("trace_options", 0644, d_tracer
,
7198 tr
, &tracing_iter_fops
);
7200 trace_create_file("trace", 0644, d_tracer
,
7203 trace_create_file("trace_pipe", 0444, d_tracer
,
7204 tr
, &tracing_pipe_fops
);
7206 trace_create_file("buffer_size_kb", 0644, d_tracer
,
7207 tr
, &tracing_entries_fops
);
7209 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
7210 tr
, &tracing_total_entries_fops
);
7212 trace_create_file("free_buffer", 0200, d_tracer
,
7213 tr
, &tracing_free_buffer_fops
);
7215 trace_create_file("trace_marker", 0220, d_tracer
,
7216 tr
, &tracing_mark_fops
);
7218 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
7221 trace_create_file("tracing_on", 0644, d_tracer
,
7222 tr
, &rb_simple_fops
);
7224 create_trace_options_dir(tr
);
7226 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7227 trace_create_file("tracing_max_latency", 0644, d_tracer
,
7228 &tr
->max_latency
, &tracing_max_lat_fops
);
7231 if (ftrace_create_function_files(tr
, d_tracer
))
7232 WARN(1, "Could not allocate function filter files");
7234 #ifdef CONFIG_TRACER_SNAPSHOT
7235 trace_create_file("snapshot", 0644, d_tracer
,
7236 tr
, &snapshot_fops
);
7239 for_each_tracing_cpu(cpu
)
7240 tracing_init_tracefs_percpu(tr
, cpu
);
7242 ftrace_init_tracefs(tr
, d_tracer
);
7245 static struct vfsmount
*trace_automount(void *ingore
)
7247 struct vfsmount
*mnt
;
7248 struct file_system_type
*type
;
7251 * To maintain backward compatibility for tools that mount
7252 * debugfs to get to the tracing facility, tracefs is automatically
7253 * mounted to the debugfs/tracing directory.
7255 type
= get_fs_type("tracefs");
7258 mnt
= vfs_kern_mount(type
, 0, "tracefs", NULL
);
7259 put_filesystem(type
);
7268 * tracing_init_dentry - initialize top level trace array
7270 * This is called when creating files or directories in the tracing
7271 * directory. It is called via fs_initcall() by any of the boot up code
7272 * and expects to return the dentry of the top level tracing directory.
7274 struct dentry
*tracing_init_dentry(void)
7276 struct trace_array
*tr
= &global_trace
;
7278 /* The top level trace array uses NULL as parent */
7282 if (WARN_ON(!tracefs_initialized()) ||
7283 (IS_ENABLED(CONFIG_DEBUG_FS
) &&
7284 WARN_ON(!debugfs_initialized())))
7285 return ERR_PTR(-ENODEV
);
7288 * As there may still be users that expect the tracing
7289 * files to exist in debugfs/tracing, we must automount
7290 * the tracefs file system there, so older tools still
7291 * work with the newer kerenl.
7293 tr
->dir
= debugfs_create_automount("tracing", NULL
,
7294 trace_automount
, NULL
);
7296 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7297 return ERR_PTR(-ENOMEM
);
7303 extern struct trace_enum_map
*__start_ftrace_enum_maps
[];
7304 extern struct trace_enum_map
*__stop_ftrace_enum_maps
[];
7306 static void __init
trace_enum_init(void)
7310 len
= __stop_ftrace_enum_maps
- __start_ftrace_enum_maps
;
7311 trace_insert_enum_map(NULL
, __start_ftrace_enum_maps
, len
);
7314 #ifdef CONFIG_MODULES
7315 static void trace_module_add_enums(struct module
*mod
)
7317 if (!mod
->num_trace_enums
)
7321 * Modules with bad taint do not have events created, do
7322 * not bother with enums either.
7324 if (trace_module_has_bad_taint(mod
))
7327 trace_insert_enum_map(mod
, mod
->trace_enums
, mod
->num_trace_enums
);
7330 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7331 static void trace_module_remove_enums(struct module
*mod
)
7333 union trace_enum_map_item
*map
;
7334 union trace_enum_map_item
**last
= &trace_enum_maps
;
7336 if (!mod
->num_trace_enums
)
7339 mutex_lock(&trace_enum_mutex
);
7341 map
= trace_enum_maps
;
7344 if (map
->head
.mod
== mod
)
7346 map
= trace_enum_jmp_to_tail(map
);
7347 last
= &map
->tail
.next
;
7348 map
= map
->tail
.next
;
7353 *last
= trace_enum_jmp_to_tail(map
)->tail
.next
;
7356 mutex_unlock(&trace_enum_mutex
);
7359 static inline void trace_module_remove_enums(struct module
*mod
) { }
7360 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7362 static int trace_module_notify(struct notifier_block
*self
,
7363 unsigned long val
, void *data
)
7365 struct module
*mod
= data
;
7368 case MODULE_STATE_COMING
:
7369 trace_module_add_enums(mod
);
7371 case MODULE_STATE_GOING
:
7372 trace_module_remove_enums(mod
);
7379 static struct notifier_block trace_module_nb
= {
7380 .notifier_call
= trace_module_notify
,
7383 #endif /* CONFIG_MODULES */
7385 static __init
int tracer_init_tracefs(void)
7387 struct dentry
*d_tracer
;
7389 trace_access_lock_init();
7391 d_tracer
= tracing_init_dentry();
7392 if (IS_ERR(d_tracer
))
7395 init_tracer_tracefs(&global_trace
, d_tracer
);
7396 ftrace_init_tracefs_toplevel(&global_trace
, d_tracer
);
7398 trace_create_file("tracing_thresh", 0644, d_tracer
,
7399 &global_trace
, &tracing_thresh_fops
);
7401 trace_create_file("README", 0444, d_tracer
,
7402 NULL
, &tracing_readme_fops
);
7404 trace_create_file("saved_cmdlines", 0444, d_tracer
,
7405 NULL
, &tracing_saved_cmdlines_fops
);
7407 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
7408 NULL
, &tracing_saved_cmdlines_size_fops
);
7412 trace_create_enum_file(d_tracer
);
7414 #ifdef CONFIG_MODULES
7415 register_module_notifier(&trace_module_nb
);
7418 #ifdef CONFIG_DYNAMIC_FTRACE
7419 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
7420 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
7423 create_trace_instances(d_tracer
);
7425 update_tracer_options(&global_trace
);
7430 static int trace_panic_handler(struct notifier_block
*this,
7431 unsigned long event
, void *unused
)
7433 if (ftrace_dump_on_oops
)
7434 ftrace_dump(ftrace_dump_on_oops
);
7438 static struct notifier_block trace_panic_notifier
= {
7439 .notifier_call
= trace_panic_handler
,
7441 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
7444 static int trace_die_handler(struct notifier_block
*self
,
7450 if (ftrace_dump_on_oops
)
7451 ftrace_dump(ftrace_dump_on_oops
);
7459 static struct notifier_block trace_die_notifier
= {
7460 .notifier_call
= trace_die_handler
,
7465 * printk is set to max of 1024, we really don't need it that big.
7466 * Nothing should be printing 1000 characters anyway.
7468 #define TRACE_MAX_PRINT 1000
7471 * Define here KERN_TRACE so that we have one place to modify
7472 * it if we decide to change what log level the ftrace dump
7475 #define KERN_TRACE KERN_EMERG
7478 trace_printk_seq(struct trace_seq
*s
)
7480 /* Probably should print a warning here. */
7481 if (s
->seq
.len
>= TRACE_MAX_PRINT
)
7482 s
->seq
.len
= TRACE_MAX_PRINT
;
7485 * More paranoid code. Although the buffer size is set to
7486 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7487 * an extra layer of protection.
7489 if (WARN_ON_ONCE(s
->seq
.len
>= s
->seq
.size
))
7490 s
->seq
.len
= s
->seq
.size
- 1;
7492 /* should be zero ended, but we are paranoid. */
7493 s
->buffer
[s
->seq
.len
] = 0;
7495 printk(KERN_TRACE
"%s", s
->buffer
);
7500 void trace_init_global_iter(struct trace_iterator
*iter
)
7502 iter
->tr
= &global_trace
;
7503 iter
->trace
= iter
->tr
->current_trace
;
7504 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
7505 iter
->trace_buffer
= &global_trace
.trace_buffer
;
7507 if (iter
->trace
&& iter
->trace
->open
)
7508 iter
->trace
->open(iter
);
7510 /* Annotate start of buffers if we had overruns */
7511 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
7512 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
7514 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7515 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
7516 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
7519 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
7521 /* use static because iter can be a bit big for the stack */
7522 static struct trace_iterator iter
;
7523 static atomic_t dump_running
;
7524 struct trace_array
*tr
= &global_trace
;
7525 unsigned int old_userobj
;
7526 unsigned long flags
;
7529 /* Only allow one dump user at a time. */
7530 if (atomic_inc_return(&dump_running
) != 1) {
7531 atomic_dec(&dump_running
);
7536 * Always turn off tracing when we dump.
7537 * We don't need to show trace output of what happens
7538 * between multiple crashes.
7540 * If the user does a sysrq-z, then they can re-enable
7541 * tracing with echo 1 > tracing_on.
7545 local_irq_save(flags
);
7547 /* Simulate the iterator */
7548 trace_init_global_iter(&iter
);
7550 for_each_tracing_cpu(cpu
) {
7551 atomic_inc(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7554 old_userobj
= tr
->trace_flags
& TRACE_ITER_SYM_USEROBJ
;
7556 /* don't look at user memory in panic mode */
7557 tr
->trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
7559 switch (oops_dump_mode
) {
7561 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7564 iter
.cpu_file
= raw_smp_processor_id();
7569 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
7570 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7573 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
7575 /* Did function tracer already get disabled? */
7576 if (ftrace_is_dead()) {
7577 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7578 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7582 * We need to stop all tracing on all CPUS to read the
7583 * the next buffer. This is a bit expensive, but is
7584 * not done often. We fill all what we can read,
7585 * and then release the locks again.
7588 while (!trace_empty(&iter
)) {
7591 printk(KERN_TRACE
"---------------------------------\n");
7595 /* reset all but tr, trace, and overruns */
7596 memset(&iter
.seq
, 0,
7597 sizeof(struct trace_iterator
) -
7598 offsetof(struct trace_iterator
, seq
));
7599 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
7602 if (trace_find_next_entry_inc(&iter
) != NULL
) {
7605 ret
= print_trace_line(&iter
);
7606 if (ret
!= TRACE_TYPE_NO_CONSUME
)
7607 trace_consume(&iter
);
7609 touch_nmi_watchdog();
7611 trace_printk_seq(&iter
.seq
);
7615 printk(KERN_TRACE
" (ftrace buffer empty)\n");
7617 printk(KERN_TRACE
"---------------------------------\n");
7620 tr
->trace_flags
|= old_userobj
;
7622 for_each_tracing_cpu(cpu
) {
7623 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7625 atomic_dec(&dump_running
);
7626 local_irq_restore(flags
);
7628 EXPORT_SYMBOL_GPL(ftrace_dump
);
7630 __init
static int tracer_alloc_buffers(void)
7636 * Make sure we don't accidently add more trace options
7637 * than we have bits for.
7639 BUILD_BUG_ON(TRACE_ITER_LAST_BIT
> TRACE_FLAGS_MAX_SIZE
);
7641 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
7644 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
7645 goto out_free_buffer_mask
;
7647 /* Only allocate trace_printk buffers if a trace_printk exists */
7648 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
7649 /* Must be called before global_trace.buffer is allocated */
7650 trace_printk_init_buffers();
7652 /* To save memory, keep the ring buffer size to its minimum */
7653 if (ring_buffer_expanded
)
7654 ring_buf_size
= trace_buf_size
;
7658 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
7659 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
7661 raw_spin_lock_init(&global_trace
.start_lock
);
7664 * The prepare callbacks allocates some memory for the ring buffer. We
7665 * don't free the buffer if the if the CPU goes down. If we were to free
7666 * the buffer, then the user would lose any trace that was in the
7667 * buffer. The memory will be removed once the "instance" is removed.
7669 ret
= cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE
,
7670 "trace/RB:preapre", trace_rb_cpu_prepare
,
7673 goto out_free_cpumask
;
7674 /* Used for event triggers */
7675 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
7677 goto out_rm_hp_state
;
7679 if (trace_create_savedcmd() < 0)
7680 goto out_free_temp_buffer
;
7682 /* TODO: make the number of buffers hot pluggable with CPUS */
7683 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
7684 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
7686 goto out_free_savedcmd
;
7689 if (global_trace
.buffer_disabled
)
7692 if (trace_boot_clock
) {
7693 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
7695 pr_warn("Trace clock %s not defined, going back to default\n",
7700 * register_tracer() might reference current_trace, so it
7701 * needs to be set before we register anything. This is
7702 * just a bootstrap of current_trace anyway.
7704 global_trace
.current_trace
= &nop_trace
;
7706 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
7708 ftrace_init_global_array_ops(&global_trace
);
7710 init_trace_flags_index(&global_trace
);
7712 register_tracer(&nop_trace
);
7714 /* All seems OK, enable tracing */
7715 tracing_disabled
= 0;
7717 atomic_notifier_chain_register(&panic_notifier_list
,
7718 &trace_panic_notifier
);
7720 register_die_notifier(&trace_die_notifier
);
7722 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
7724 INIT_LIST_HEAD(&global_trace
.systems
);
7725 INIT_LIST_HEAD(&global_trace
.events
);
7726 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
7728 apply_trace_boot_options();
7730 register_snapshot_cmd();
7735 free_saved_cmdlines_buffer(savedcmd
);
7736 out_free_temp_buffer
:
7737 ring_buffer_free(temp_buffer
);
7739 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE
);
7741 free_cpumask_var(global_trace
.tracing_cpumask
);
7742 out_free_buffer_mask
:
7743 free_cpumask_var(tracing_buffer_mask
);
7748 void __init
trace_init(void)
7750 if (tracepoint_printk
) {
7751 tracepoint_print_iter
=
7752 kmalloc(sizeof(*tracepoint_print_iter
), GFP_KERNEL
);
7753 if (WARN_ON(!tracepoint_print_iter
))
7754 tracepoint_printk
= 0;
7756 tracer_alloc_buffers();
7760 __init
static int clear_boot_tracer(void)
7763 * The default tracer at boot buffer is an init section.
7764 * This function is called in lateinit. If we did not
7765 * find the boot tracer, then clear it out, to prevent
7766 * later registration from accessing the buffer that is
7767 * about to be freed.
7769 if (!default_bootup_tracer
)
7772 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
7773 default_bootup_tracer
);
7774 default_bootup_tracer
= NULL
;
7779 fs_initcall(tracer_init_tracefs
);
7780 late_initcall(clear_boot_tracer
);