2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
23 #include "trace_probe.h"
25 #define KPROBE_EVENT_SYSTEM "kprobes"
28 * Kprobe event core functions
31 struct list_head list
;
32 struct kretprobe rp
; /* Use rp.kp for kprobe use */
34 const char *symbol
; /* symbol name */
35 struct trace_probe tp
;
38 #define SIZEOF_TRACE_KPROBE(n) \
39 (offsetof(struct trace_kprobe, tp.args) + \
40 (sizeof(struct probe_arg) * (n)))
43 static nokprobe_inline
bool trace_kprobe_is_return(struct trace_kprobe
*tk
)
45 return tk
->rp
.handler
!= NULL
;
48 static nokprobe_inline
const char *trace_kprobe_symbol(struct trace_kprobe
*tk
)
50 return tk
->symbol
? tk
->symbol
: "unknown";
53 static nokprobe_inline
unsigned long trace_kprobe_offset(struct trace_kprobe
*tk
)
55 return tk
->rp
.kp
.offset
;
58 static nokprobe_inline
bool trace_kprobe_has_gone(struct trace_kprobe
*tk
)
60 return !!(kprobe_gone(&tk
->rp
.kp
));
63 static nokprobe_inline
bool trace_kprobe_within_module(struct trace_kprobe
*tk
,
66 int len
= strlen(mod
->name
);
67 const char *name
= trace_kprobe_symbol(tk
);
68 return strncmp(mod
->name
, name
, len
) == 0 && name
[len
] == ':';
71 static nokprobe_inline
bool trace_kprobe_is_on_module(struct trace_kprobe
*tk
)
73 return !!strchr(trace_kprobe_symbol(tk
), ':');
76 static int register_kprobe_event(struct trace_kprobe
*tk
);
77 static int unregister_kprobe_event(struct trace_kprobe
*tk
);
79 static DEFINE_MUTEX(probe_lock
);
80 static LIST_HEAD(probe_list
);
82 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
83 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
84 struct pt_regs
*regs
);
86 /* Memory fetching by symbol */
93 unsigned long update_symbol_cache(struct symbol_cache
*sc
)
95 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
98 sc
->addr
+= sc
->offset
;
103 void free_symbol_cache(struct symbol_cache
*sc
)
109 struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
111 struct symbol_cache
*sc
;
113 if (!sym
|| strlen(sym
) == 0)
116 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
120 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
126 update_symbol_cache(sc
);
132 * Kprobes-specific fetch functions
134 #define DEFINE_FETCH_stack(type) \
135 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
136 void *offset, void *dest) \
138 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
139 (unsigned int)((unsigned long)offset)); \
141 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
143 DEFINE_BASIC_FETCH_FUNCS(stack
)
144 /* No string on the stack entry */
145 #define fetch_stack_string NULL
146 #define fetch_stack_string_size NULL
148 #define DEFINE_FETCH_memory(type) \
149 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
150 void *addr, void *dest) \
153 if (probe_kernel_address(addr, retval)) \
156 *(type *)dest = retval; \
158 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
160 DEFINE_BASIC_FETCH_FUNCS(memory
)
162 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
163 * length and relative data location.
165 static void FETCH_FUNC_NAME(memory
, string
)(struct pt_regs
*regs
,
166 void *addr
, void *dest
)
169 int maxlen
= get_rloc_len(*(u32
*)dest
);
170 u8
*dst
= get_rloc_data(dest
);
172 mm_segment_t old_fs
= get_fs();
178 * Try to get string again, since the string can be changed while
185 ret
= __copy_from_user_inatomic(dst
++, src
++, 1);
186 while (dst
[-1] && ret
== 0 && src
- (u8
*)addr
< maxlen
);
192 if (ret
< 0) { /* Failed to fetch string */
193 ((u8
*)get_rloc_data(dest
))[0] = '\0';
194 *(u32
*)dest
= make_data_rloc(0, get_rloc_offs(*(u32
*)dest
));
196 *(u32
*)dest
= make_data_rloc(src
- (u8
*)addr
,
197 get_rloc_offs(*(u32
*)dest
));
200 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory
, string
));
202 /* Return the length of string -- including null terminal byte */
203 static void FETCH_FUNC_NAME(memory
, string_size
)(struct pt_regs
*regs
,
204 void *addr
, void *dest
)
215 ret
= __copy_from_user_inatomic(&c
, (u8
*)addr
+ len
, 1);
217 } while (c
&& ret
== 0 && len
< MAX_STRING_SIZE
);
222 if (ret
< 0) /* Failed to check the length */
227 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory
, string_size
));
229 #define DEFINE_FETCH_symbol(type) \
230 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
232 struct symbol_cache *sc = data; \
234 fetch_memory_##type(regs, (void *)sc->addr, dest); \
238 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
240 DEFINE_BASIC_FETCH_FUNCS(symbol
)
241 DEFINE_FETCH_symbol(string
)
242 DEFINE_FETCH_symbol(string_size
)
244 /* kprobes don't support file_offset fetch methods */
245 #define fetch_file_offset_u8 NULL
246 #define fetch_file_offset_u16 NULL
247 #define fetch_file_offset_u32 NULL
248 #define fetch_file_offset_u64 NULL
249 #define fetch_file_offset_string NULL
250 #define fetch_file_offset_string_size NULL
252 /* Fetch type information table */
253 const struct fetch_type kprobes_fetch_type_table
[] = {
255 [FETCH_TYPE_STRING
] = __ASSIGN_FETCH_TYPE("string", string
, string
,
256 sizeof(u32
), 1, "__data_loc char[]"),
257 [FETCH_TYPE_STRSIZE
] = __ASSIGN_FETCH_TYPE("string_size", u32
,
258 string_size
, sizeof(u32
), 0, "u32"),
260 ASSIGN_FETCH_TYPE(u8
, u8
, 0),
261 ASSIGN_FETCH_TYPE(u16
, u16
, 0),
262 ASSIGN_FETCH_TYPE(u32
, u32
, 0),
263 ASSIGN_FETCH_TYPE(u64
, u64
, 0),
264 ASSIGN_FETCH_TYPE(s8
, u8
, 1),
265 ASSIGN_FETCH_TYPE(s16
, u16
, 1),
266 ASSIGN_FETCH_TYPE(s32
, u32
, 1),
267 ASSIGN_FETCH_TYPE(s64
, u64
, 1),
269 ASSIGN_FETCH_TYPE_END
273 * Allocate new trace_probe and initialize it (including kprobes).
275 static struct trace_kprobe
*alloc_trace_kprobe(const char *group
,
280 int nargs
, bool is_return
)
282 struct trace_kprobe
*tk
;
285 tk
= kzalloc(SIZEOF_TRACE_KPROBE(nargs
), GFP_KERNEL
);
290 tk
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
293 tk
->rp
.kp
.symbol_name
= tk
->symbol
;
294 tk
->rp
.kp
.offset
= offs
;
296 tk
->rp
.kp
.addr
= addr
;
299 tk
->rp
.handler
= kretprobe_dispatcher
;
301 tk
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
303 if (!event
|| !is_good_name(event
)) {
308 tk
->tp
.call
.class = &tk
->tp
.class;
309 tk
->tp
.call
.name
= kstrdup(event
, GFP_KERNEL
);
310 if (!tk
->tp
.call
.name
)
313 if (!group
|| !is_good_name(group
)) {
318 tk
->tp
.class.system
= kstrdup(group
, GFP_KERNEL
);
319 if (!tk
->tp
.class.system
)
322 INIT_LIST_HEAD(&tk
->list
);
323 INIT_LIST_HEAD(&tk
->tp
.files
);
326 kfree(tk
->tp
.call
.name
);
332 static void free_trace_kprobe(struct trace_kprobe
*tk
)
336 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
337 traceprobe_free_probe_arg(&tk
->tp
.args
[i
]);
339 kfree(tk
->tp
.call
.class->system
);
340 kfree(tk
->tp
.call
.name
);
345 static struct trace_kprobe
*find_trace_kprobe(const char *event
,
348 struct trace_kprobe
*tk
;
350 list_for_each_entry(tk
, &probe_list
, list
)
351 if (strcmp(ftrace_event_name(&tk
->tp
.call
), event
) == 0 &&
352 strcmp(tk
->tp
.call
.class->system
, group
) == 0)
359 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
362 enable_trace_kprobe(struct trace_kprobe
*tk
, struct ftrace_event_file
*file
)
367 struct event_file_link
*link
;
369 link
= kmalloc(sizeof(*link
), GFP_KERNEL
);
376 list_add_tail_rcu(&link
->list
, &tk
->tp
.files
);
378 tk
->tp
.flags
|= TP_FLAG_TRACE
;
380 tk
->tp
.flags
|= TP_FLAG_PROFILE
;
382 if (trace_probe_is_registered(&tk
->tp
) && !trace_kprobe_has_gone(tk
)) {
383 if (trace_kprobe_is_return(tk
))
384 ret
= enable_kretprobe(&tk
->rp
);
386 ret
= enable_kprobe(&tk
->rp
.kp
);
393 * Disable trace_probe
394 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
397 disable_trace_kprobe(struct trace_kprobe
*tk
, struct ftrace_event_file
*file
)
399 struct event_file_link
*link
= NULL
;
404 link
= find_event_file_link(&tk
->tp
, file
);
410 list_del_rcu(&link
->list
);
412 if (!list_empty(&tk
->tp
.files
))
415 tk
->tp
.flags
&= ~TP_FLAG_TRACE
;
417 tk
->tp
.flags
&= ~TP_FLAG_PROFILE
;
419 if (!trace_probe_is_enabled(&tk
->tp
) && trace_probe_is_registered(&tk
->tp
)) {
420 if (trace_kprobe_is_return(tk
))
421 disable_kretprobe(&tk
->rp
);
423 disable_kprobe(&tk
->rp
.kp
);
429 * Synchronize with kprobe_trace_func/kretprobe_trace_func
430 * to ensure disabled (all running handlers are finished).
431 * This is not only for kfree(), but also the caller,
432 * trace_remove_event_call() supposes it for releasing
433 * event_call related objects, which will be accessed in
434 * the kprobe_trace_func/kretprobe_trace_func.
437 kfree(link
); /* Ignored if link == NULL */
443 /* Internal register function - just handle k*probes and flags */
444 static int __register_trace_kprobe(struct trace_kprobe
*tk
)
448 if (trace_probe_is_registered(&tk
->tp
))
451 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
452 traceprobe_update_arg(&tk
->tp
.args
[i
]);
454 /* Set/clear disabled flag according to tp->flag */
455 if (trace_probe_is_enabled(&tk
->tp
))
456 tk
->rp
.kp
.flags
&= ~KPROBE_FLAG_DISABLED
;
458 tk
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
460 if (trace_kprobe_is_return(tk
))
461 ret
= register_kretprobe(&tk
->rp
);
463 ret
= register_kprobe(&tk
->rp
.kp
);
466 tk
->tp
.flags
|= TP_FLAG_REGISTERED
;
468 pr_warning("Could not insert probe at %s+%lu: %d\n",
469 trace_kprobe_symbol(tk
), trace_kprobe_offset(tk
), ret
);
470 if (ret
== -ENOENT
&& trace_kprobe_is_on_module(tk
)) {
471 pr_warning("This probe might be able to register after"
472 "target module is loaded. Continue.\n");
474 } else if (ret
== -EILSEQ
) {
475 pr_warning("Probing address(0x%p) is not an "
476 "instruction boundary.\n",
485 /* Internal unregister function - just handle k*probes and flags */
486 static void __unregister_trace_kprobe(struct trace_kprobe
*tk
)
488 if (trace_probe_is_registered(&tk
->tp
)) {
489 if (trace_kprobe_is_return(tk
))
490 unregister_kretprobe(&tk
->rp
);
492 unregister_kprobe(&tk
->rp
.kp
);
493 tk
->tp
.flags
&= ~TP_FLAG_REGISTERED
;
494 /* Cleanup kprobe for reuse */
495 if (tk
->rp
.kp
.symbol_name
)
496 tk
->rp
.kp
.addr
= NULL
;
500 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
501 static int unregister_trace_kprobe(struct trace_kprobe
*tk
)
503 /* Enabled event can not be unregistered */
504 if (trace_probe_is_enabled(&tk
->tp
))
507 /* Will fail if probe is being used by ftrace or perf */
508 if (unregister_kprobe_event(tk
))
511 __unregister_trace_kprobe(tk
);
517 /* Register a trace_probe and probe_event */
518 static int register_trace_kprobe(struct trace_kprobe
*tk
)
520 struct trace_kprobe
*old_tk
;
523 mutex_lock(&probe_lock
);
525 /* Delete old (same name) event if exist */
526 old_tk
= find_trace_kprobe(ftrace_event_name(&tk
->tp
.call
),
527 tk
->tp
.call
.class->system
);
529 ret
= unregister_trace_kprobe(old_tk
);
532 free_trace_kprobe(old_tk
);
535 /* Register new event */
536 ret
= register_kprobe_event(tk
);
538 pr_warning("Failed to register probe event(%d)\n", ret
);
542 /* Register k*probe */
543 ret
= __register_trace_kprobe(tk
);
545 unregister_kprobe_event(tk
);
547 list_add_tail(&tk
->list
, &probe_list
);
550 mutex_unlock(&probe_lock
);
554 /* Module notifier call back, checking event on the module */
555 static int trace_kprobe_module_callback(struct notifier_block
*nb
,
556 unsigned long val
, void *data
)
558 struct module
*mod
= data
;
559 struct trace_kprobe
*tk
;
562 if (val
!= MODULE_STATE_COMING
)
565 /* Update probes on coming module */
566 mutex_lock(&probe_lock
);
567 list_for_each_entry(tk
, &probe_list
, list
) {
568 if (trace_kprobe_within_module(tk
, mod
)) {
569 /* Don't need to check busy - this should have gone. */
570 __unregister_trace_kprobe(tk
);
571 ret
= __register_trace_kprobe(tk
);
573 pr_warning("Failed to re-register probe %s on"
575 ftrace_event_name(&tk
->tp
.call
),
579 mutex_unlock(&probe_lock
);
584 static struct notifier_block trace_kprobe_module_nb
= {
585 .notifier_call
= trace_kprobe_module_callback
,
586 .priority
= 1 /* Invoked after kprobe module callback */
589 static int create_trace_kprobe(int argc
, char **argv
)
593 * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
594 * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
596 * $retval : fetch return value
597 * $stack : fetch stack address
598 * $stackN : fetch Nth of stack (N:0-)
599 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
600 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
601 * %REG : fetch register REG
602 * Dereferencing memory fetch:
603 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
604 * Alias name of args:
605 * NAME=FETCHARG : set NAME as alias of FETCHARG.
607 * FETCHARG:TYPE : use TYPE instead of unsigned long.
609 struct trace_kprobe
*tk
;
611 bool is_return
= false, is_delete
= false;
612 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
614 unsigned long offset
= 0;
616 char buf
[MAX_EVENT_NAME_LEN
];
618 /* argc must be >= 1 */
619 if (argv
[0][0] == 'p')
621 else if (argv
[0][0] == 'r')
623 else if (argv
[0][0] == '-')
626 pr_info("Probe definition must be started with 'p', 'r' or"
631 if (argv
[0][1] == ':') {
633 if (strchr(event
, '/')) {
635 event
= strchr(group
, '/') + 1;
637 if (strlen(group
) == 0) {
638 pr_info("Group name is not specified\n");
642 if (strlen(event
) == 0) {
643 pr_info("Event name is not specified\n");
648 group
= KPROBE_EVENT_SYSTEM
;
652 pr_info("Delete command needs an event name.\n");
655 mutex_lock(&probe_lock
);
656 tk
= find_trace_kprobe(event
, group
);
658 mutex_unlock(&probe_lock
);
659 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
662 /* delete an event */
663 ret
= unregister_trace_kprobe(tk
);
665 free_trace_kprobe(tk
);
666 mutex_unlock(&probe_lock
);
671 pr_info("Probe point is not specified.\n");
674 if (isdigit(argv
[1][0])) {
676 pr_info("Return probe point must be a symbol.\n");
679 /* an address specified */
680 ret
= kstrtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
682 pr_info("Failed to parse address.\n");
686 /* a symbol specified */
688 /* TODO: support .init module functions */
689 ret
= traceprobe_split_symbol_offset(symbol
, &offset
);
691 pr_info("Failed to parse symbol.\n");
694 if (offset
&& is_return
) {
695 pr_info("Return probe must be used without offset.\n");
699 argc
-= 2; argv
+= 2;
703 /* Make a new event name */
705 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
706 is_return
? 'r' : 'p', symbol
, offset
);
708 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
709 is_return
? 'r' : 'p', addr
);
712 tk
= alloc_trace_kprobe(group
, event
, addr
, symbol
, offset
, argc
,
715 pr_info("Failed to allocate trace_probe.(%d)\n",
720 /* parse arguments */
722 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
723 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
725 /* Increment count for freeing args in error case */
728 /* Parse argument name */
729 arg
= strchr(argv
[i
], '=');
732 parg
->name
= kstrdup(argv
[i
], GFP_KERNEL
);
735 /* If argument name is omitted, set "argN" */
736 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
737 parg
->name
= kstrdup(buf
, GFP_KERNEL
);
741 pr_info("Failed to allocate argument[%d] name.\n", i
);
746 if (!is_good_name(parg
->name
)) {
747 pr_info("Invalid argument[%d] name: %s\n",
753 if (traceprobe_conflict_field_name(parg
->name
,
755 pr_info("Argument[%d] name '%s' conflicts with "
756 "another field.\n", i
, argv
[i
]);
761 /* Parse fetch argument */
762 ret
= traceprobe_parse_probe_arg(arg
, &tk
->tp
.size
, parg
,
765 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
770 ret
= register_trace_kprobe(tk
);
776 free_trace_kprobe(tk
);
780 static int release_all_trace_kprobes(void)
782 struct trace_kprobe
*tk
;
785 mutex_lock(&probe_lock
);
786 /* Ensure no probe is in use. */
787 list_for_each_entry(tk
, &probe_list
, list
)
788 if (trace_probe_is_enabled(&tk
->tp
)) {
792 /* TODO: Use batch unregistration */
793 while (!list_empty(&probe_list
)) {
794 tk
= list_entry(probe_list
.next
, struct trace_kprobe
, list
);
795 ret
= unregister_trace_kprobe(tk
);
798 free_trace_kprobe(tk
);
802 mutex_unlock(&probe_lock
);
807 /* Probes listing interfaces */
808 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
810 mutex_lock(&probe_lock
);
811 return seq_list_start(&probe_list
, *pos
);
814 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
816 return seq_list_next(v
, &probe_list
, pos
);
819 static void probes_seq_stop(struct seq_file
*m
, void *v
)
821 mutex_unlock(&probe_lock
);
824 static int probes_seq_show(struct seq_file
*m
, void *v
)
826 struct trace_kprobe
*tk
= v
;
829 seq_printf(m
, "%c", trace_kprobe_is_return(tk
) ? 'r' : 'p');
830 seq_printf(m
, ":%s/%s", tk
->tp
.call
.class->system
,
831 ftrace_event_name(&tk
->tp
.call
));
834 seq_printf(m
, " 0x%p", tk
->rp
.kp
.addr
);
835 else if (tk
->rp
.kp
.offset
)
836 seq_printf(m
, " %s+%u", trace_kprobe_symbol(tk
),
839 seq_printf(m
, " %s", trace_kprobe_symbol(tk
));
841 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
842 seq_printf(m
, " %s=%s", tk
->tp
.args
[i
].name
, tk
->tp
.args
[i
].comm
);
848 static const struct seq_operations probes_seq_op
= {
849 .start
= probes_seq_start
,
850 .next
= probes_seq_next
,
851 .stop
= probes_seq_stop
,
852 .show
= probes_seq_show
855 static int probes_open(struct inode
*inode
, struct file
*file
)
859 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
860 ret
= release_all_trace_kprobes();
865 return seq_open(file
, &probes_seq_op
);
868 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
869 size_t count
, loff_t
*ppos
)
871 return traceprobe_probes_write(file
, buffer
, count
, ppos
,
872 create_trace_kprobe
);
875 static const struct file_operations kprobe_events_ops
= {
876 .owner
= THIS_MODULE
,
880 .release
= seq_release
,
881 .write
= probes_write
,
884 /* Probes profiling interfaces */
885 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
887 struct trace_kprobe
*tk
= v
;
889 seq_printf(m
, " %-44s %15lu %15lu\n",
890 ftrace_event_name(&tk
->tp
.call
), tk
->nhit
,
896 static const struct seq_operations profile_seq_op
= {
897 .start
= probes_seq_start
,
898 .next
= probes_seq_next
,
899 .stop
= probes_seq_stop
,
900 .show
= probes_profile_seq_show
903 static int profile_open(struct inode
*inode
, struct file
*file
)
905 return seq_open(file
, &profile_seq_op
);
908 static const struct file_operations kprobe_profile_ops
= {
909 .owner
= THIS_MODULE
,
910 .open
= profile_open
,
913 .release
= seq_release
,
917 static nokprobe_inline
void
918 __kprobe_trace_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
,
919 struct ftrace_event_file
*ftrace_file
)
921 struct kprobe_trace_entry_head
*entry
;
922 struct ring_buffer_event
*event
;
923 struct ring_buffer
*buffer
;
925 unsigned long irq_flags
;
926 struct ftrace_event_call
*call
= &tk
->tp
.call
;
928 WARN_ON(call
!= ftrace_file
->event_call
);
930 if (ftrace_trigger_soft_disabled(ftrace_file
))
933 local_save_flags(irq_flags
);
934 pc
= preempt_count();
936 dsize
= __get_data_size(&tk
->tp
, regs
);
937 size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
939 event
= trace_event_buffer_lock_reserve(&buffer
, ftrace_file
,
941 size
, irq_flags
, pc
);
945 entry
= ring_buffer_event_data(event
);
946 entry
->ip
= (unsigned long)tk
->rp
.kp
.addr
;
947 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
949 event_trigger_unlock_commit_regs(ftrace_file
, buffer
, event
,
950 entry
, irq_flags
, pc
, regs
);
954 kprobe_trace_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
)
956 struct event_file_link
*link
;
958 list_for_each_entry_rcu(link
, &tk
->tp
.files
, list
)
959 __kprobe_trace_func(tk
, regs
, link
->file
);
961 NOKPROBE_SYMBOL(kprobe_trace_func
);
963 /* Kretprobe handler */
964 static nokprobe_inline
void
965 __kretprobe_trace_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
966 struct pt_regs
*regs
,
967 struct ftrace_event_file
*ftrace_file
)
969 struct kretprobe_trace_entry_head
*entry
;
970 struct ring_buffer_event
*event
;
971 struct ring_buffer
*buffer
;
973 unsigned long irq_flags
;
974 struct ftrace_event_call
*call
= &tk
->tp
.call
;
976 WARN_ON(call
!= ftrace_file
->event_call
);
978 if (ftrace_trigger_soft_disabled(ftrace_file
))
981 local_save_flags(irq_flags
);
982 pc
= preempt_count();
984 dsize
= __get_data_size(&tk
->tp
, regs
);
985 size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
987 event
= trace_event_buffer_lock_reserve(&buffer
, ftrace_file
,
989 size
, irq_flags
, pc
);
993 entry
= ring_buffer_event_data(event
);
994 entry
->func
= (unsigned long)tk
->rp
.kp
.addr
;
995 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
996 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
998 event_trigger_unlock_commit_regs(ftrace_file
, buffer
, event
,
999 entry
, irq_flags
, pc
, regs
);
1003 kretprobe_trace_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
1004 struct pt_regs
*regs
)
1006 struct event_file_link
*link
;
1008 list_for_each_entry_rcu(link
, &tk
->tp
.files
, list
)
1009 __kretprobe_trace_func(tk
, ri
, regs
, link
->file
);
1011 NOKPROBE_SYMBOL(kretprobe_trace_func
);
1013 /* Event entry printers */
1014 static enum print_line_t
1015 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
1016 struct trace_event
*event
)
1018 struct kprobe_trace_entry_head
*field
;
1019 struct trace_seq
*s
= &iter
->seq
;
1020 struct trace_probe
*tp
;
1024 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
1025 tp
= container_of(event
, struct trace_probe
, call
.event
);
1027 if (!trace_seq_printf(s
, "%s: (", ftrace_event_name(&tp
->call
)))
1030 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1033 if (!trace_seq_puts(s
, ")"))
1036 data
= (u8
*)&field
[1];
1037 for (i
= 0; i
< tp
->nr_args
; i
++)
1038 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1039 data
+ tp
->args
[i
].offset
, field
))
1042 if (!trace_seq_puts(s
, "\n"))
1045 return TRACE_TYPE_HANDLED
;
1047 return TRACE_TYPE_PARTIAL_LINE
;
1050 static enum print_line_t
1051 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
1052 struct trace_event
*event
)
1054 struct kretprobe_trace_entry_head
*field
;
1055 struct trace_seq
*s
= &iter
->seq
;
1056 struct trace_probe
*tp
;
1060 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
1061 tp
= container_of(event
, struct trace_probe
, call
.event
);
1063 if (!trace_seq_printf(s
, "%s: (", ftrace_event_name(&tp
->call
)))
1066 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1069 if (!trace_seq_puts(s
, " <- "))
1072 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1075 if (!trace_seq_puts(s
, ")"))
1078 data
= (u8
*)&field
[1];
1079 for (i
= 0; i
< tp
->nr_args
; i
++)
1080 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1081 data
+ tp
->args
[i
].offset
, field
))
1084 if (!trace_seq_puts(s
, "\n"))
1087 return TRACE_TYPE_HANDLED
;
1089 return TRACE_TYPE_PARTIAL_LINE
;
1093 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1096 struct kprobe_trace_entry_head field
;
1097 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event_call
->data
;
1099 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1100 /* Set argument names as fields */
1101 for (i
= 0; i
< tk
->tp
.nr_args
; i
++) {
1102 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
1104 ret
= trace_define_field(event_call
, parg
->type
->fmttype
,
1106 sizeof(field
) + parg
->offset
,
1108 parg
->type
->is_signed
,
1116 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1119 struct kretprobe_trace_entry_head field
;
1120 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event_call
->data
;
1122 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1123 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1124 /* Set argument names as fields */
1125 for (i
= 0; i
< tk
->tp
.nr_args
; i
++) {
1126 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
1128 ret
= trace_define_field(event_call
, parg
->type
->fmttype
,
1130 sizeof(field
) + parg
->offset
,
1132 parg
->type
->is_signed
,
1140 #ifdef CONFIG_PERF_EVENTS
1142 /* Kprobe profile handler */
1144 kprobe_perf_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
)
1146 struct ftrace_event_call
*call
= &tk
->tp
.call
;
1147 struct kprobe_trace_entry_head
*entry
;
1148 struct hlist_head
*head
;
1149 int size
, __size
, dsize
;
1152 head
= this_cpu_ptr(call
->perf_events
);
1153 if (hlist_empty(head
))
1156 dsize
= __get_data_size(&tk
->tp
, regs
);
1157 __size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
1158 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1159 size
-= sizeof(u32
);
1161 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1165 entry
->ip
= (unsigned long)tk
->rp
.kp
.addr
;
1166 memset(&entry
[1], 0, dsize
);
1167 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
1168 perf_trace_buf_submit(entry
, size
, rctx
, 0, 1, regs
, head
, NULL
);
1170 NOKPROBE_SYMBOL(kprobe_perf_func
);
1172 /* Kretprobe profile handler */
1174 kretprobe_perf_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
1175 struct pt_regs
*regs
)
1177 struct ftrace_event_call
*call
= &tk
->tp
.call
;
1178 struct kretprobe_trace_entry_head
*entry
;
1179 struct hlist_head
*head
;
1180 int size
, __size
, dsize
;
1183 head
= this_cpu_ptr(call
->perf_events
);
1184 if (hlist_empty(head
))
1187 dsize
= __get_data_size(&tk
->tp
, regs
);
1188 __size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
1189 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1190 size
-= sizeof(u32
);
1192 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1196 entry
->func
= (unsigned long)tk
->rp
.kp
.addr
;
1197 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1198 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
1199 perf_trace_buf_submit(entry
, size
, rctx
, 0, 1, regs
, head
, NULL
);
1201 NOKPROBE_SYMBOL(kretprobe_perf_func
);
1202 #endif /* CONFIG_PERF_EVENTS */
1205 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1207 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1208 * lockless, but we can't race with this __init function.
1210 static int kprobe_register(struct ftrace_event_call
*event
,
1211 enum trace_reg type
, void *data
)
1213 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event
->data
;
1214 struct ftrace_event_file
*file
= data
;
1217 case TRACE_REG_REGISTER
:
1218 return enable_trace_kprobe(tk
, file
);
1219 case TRACE_REG_UNREGISTER
:
1220 return disable_trace_kprobe(tk
, file
);
1222 #ifdef CONFIG_PERF_EVENTS
1223 case TRACE_REG_PERF_REGISTER
:
1224 return enable_trace_kprobe(tk
, NULL
);
1225 case TRACE_REG_PERF_UNREGISTER
:
1226 return disable_trace_kprobe(tk
, NULL
);
1227 case TRACE_REG_PERF_OPEN
:
1228 case TRACE_REG_PERF_CLOSE
:
1229 case TRACE_REG_PERF_ADD
:
1230 case TRACE_REG_PERF_DEL
:
1237 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1239 struct trace_kprobe
*tk
= container_of(kp
, struct trace_kprobe
, rp
.kp
);
1243 if (tk
->tp
.flags
& TP_FLAG_TRACE
)
1244 kprobe_trace_func(tk
, regs
);
1245 #ifdef CONFIG_PERF_EVENTS
1246 if (tk
->tp
.flags
& TP_FLAG_PROFILE
)
1247 kprobe_perf_func(tk
, regs
);
1249 return 0; /* We don't tweek kernel, so just return 0 */
1251 NOKPROBE_SYMBOL(kprobe_dispatcher
);
1254 kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1256 struct trace_kprobe
*tk
= container_of(ri
->rp
, struct trace_kprobe
, rp
);
1260 if (tk
->tp
.flags
& TP_FLAG_TRACE
)
1261 kretprobe_trace_func(tk
, ri
, regs
);
1262 #ifdef CONFIG_PERF_EVENTS
1263 if (tk
->tp
.flags
& TP_FLAG_PROFILE
)
1264 kretprobe_perf_func(tk
, ri
, regs
);
1266 return 0; /* We don't tweek kernel, so just return 0 */
1268 NOKPROBE_SYMBOL(kretprobe_dispatcher
);
1270 static struct trace_event_functions kretprobe_funcs
= {
1271 .trace
= print_kretprobe_event
1274 static struct trace_event_functions kprobe_funcs
= {
1275 .trace
= print_kprobe_event
1278 static int register_kprobe_event(struct trace_kprobe
*tk
)
1280 struct ftrace_event_call
*call
= &tk
->tp
.call
;
1283 /* Initialize ftrace_event_call */
1284 INIT_LIST_HEAD(&call
->class->fields
);
1285 if (trace_kprobe_is_return(tk
)) {
1286 call
->event
.funcs
= &kretprobe_funcs
;
1287 call
->class->define_fields
= kretprobe_event_define_fields
;
1289 call
->event
.funcs
= &kprobe_funcs
;
1290 call
->class->define_fields
= kprobe_event_define_fields
;
1292 if (set_print_fmt(&tk
->tp
, trace_kprobe_is_return(tk
)) < 0)
1294 ret
= register_ftrace_event(&call
->event
);
1296 kfree(call
->print_fmt
);
1300 call
->class->reg
= kprobe_register
;
1302 ret
= trace_add_event_call(call
);
1304 pr_info("Failed to register kprobe event: %s\n",
1305 ftrace_event_name(call
));
1306 kfree(call
->print_fmt
);
1307 unregister_ftrace_event(&call
->event
);
1312 static int unregister_kprobe_event(struct trace_kprobe
*tk
)
1316 /* tp->event is unregistered in trace_remove_event_call() */
1317 ret
= trace_remove_event_call(&tk
->tp
.call
);
1319 kfree(tk
->tp
.call
.print_fmt
);
1323 /* Make a debugfs interface for controlling probe points */
1324 static __init
int init_kprobe_trace(void)
1326 struct dentry
*d_tracer
;
1327 struct dentry
*entry
;
1329 if (register_module_notifier(&trace_kprobe_module_nb
))
1332 d_tracer
= tracing_init_dentry();
1336 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1337 NULL
, &kprobe_events_ops
);
1339 /* Event list interface */
1341 pr_warning("Could not create debugfs "
1342 "'kprobe_events' entry\n");
1344 /* Profile interface */
1345 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1346 NULL
, &kprobe_profile_ops
);
1349 pr_warning("Could not create debugfs "
1350 "'kprobe_profile' entry\n");
1353 fs_initcall(init_kprobe_trace
);
1356 #ifdef CONFIG_FTRACE_STARTUP_TEST
1359 * The "__used" keeps gcc from removing the function symbol
1360 * from the kallsyms table.
1362 static __used
int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1363 int a4
, int a5
, int a6
)
1365 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1368 static struct ftrace_event_file
*
1369 find_trace_probe_file(struct trace_kprobe
*tk
, struct trace_array
*tr
)
1371 struct ftrace_event_file
*file
;
1373 list_for_each_entry(file
, &tr
->events
, list
)
1374 if (file
->event_call
== &tk
->tp
.call
)
1381 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1382 * stage, we can do this lockless.
1384 static __init
int kprobe_trace_self_tests_init(void)
1387 int (*target
)(int, int, int, int, int, int);
1388 struct trace_kprobe
*tk
;
1389 struct ftrace_event_file
*file
;
1391 target
= kprobe_trace_selftest_target
;
1393 pr_info("Testing kprobe tracing: ");
1395 ret
= traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1396 "$stack $stack0 +0($stack)",
1397 create_trace_kprobe
);
1398 if (WARN_ON_ONCE(ret
)) {
1399 pr_warn("error on probing function entry.\n");
1402 /* Enable trace point */
1403 tk
= find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM
);
1404 if (WARN_ON_ONCE(tk
== NULL
)) {
1405 pr_warn("error on getting new probe.\n");
1408 file
= find_trace_probe_file(tk
, top_trace_array());
1409 if (WARN_ON_ONCE(file
== NULL
)) {
1410 pr_warn("error on getting probe file.\n");
1413 enable_trace_kprobe(tk
, file
);
1417 ret
= traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1418 "$retval", create_trace_kprobe
);
1419 if (WARN_ON_ONCE(ret
)) {
1420 pr_warn("error on probing function return.\n");
1423 /* Enable trace point */
1424 tk
= find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM
);
1425 if (WARN_ON_ONCE(tk
== NULL
)) {
1426 pr_warn("error on getting 2nd new probe.\n");
1429 file
= find_trace_probe_file(tk
, top_trace_array());
1430 if (WARN_ON_ONCE(file
== NULL
)) {
1431 pr_warn("error on getting probe file.\n");
1434 enable_trace_kprobe(tk
, file
);
1441 ret
= target(1, 2, 3, 4, 5, 6);
1443 /* Disable trace points before removing it */
1444 tk
= find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM
);
1445 if (WARN_ON_ONCE(tk
== NULL
)) {
1446 pr_warn("error on getting test probe.\n");
1449 file
= find_trace_probe_file(tk
, top_trace_array());
1450 if (WARN_ON_ONCE(file
== NULL
)) {
1451 pr_warn("error on getting probe file.\n");
1454 disable_trace_kprobe(tk
, file
);
1457 tk
= find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM
);
1458 if (WARN_ON_ONCE(tk
== NULL
)) {
1459 pr_warn("error on getting 2nd test probe.\n");
1462 file
= find_trace_probe_file(tk
, top_trace_array());
1463 if (WARN_ON_ONCE(file
== NULL
)) {
1464 pr_warn("error on getting probe file.\n");
1467 disable_trace_kprobe(tk
, file
);
1470 ret
= traceprobe_command("-:testprobe", create_trace_kprobe
);
1471 if (WARN_ON_ONCE(ret
)) {
1472 pr_warn("error on deleting a probe.\n");
1476 ret
= traceprobe_command("-:testprobe2", create_trace_kprobe
);
1477 if (WARN_ON_ONCE(ret
)) {
1478 pr_warn("error on deleting a probe.\n");
1483 release_all_trace_kprobes();
1485 pr_cont("NG: Some tests are failed. Please check them.\n");
1491 late_initcall(kprobe_trace_self_tests_init
);