2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 #define pr_fmt(fmt) "trace_kprobe: " fmt
21 #include <linux/module.h>
22 #include <linux/uaccess.h>
23 #include <linux/rculist.h>
25 #include "trace_probe.h"
27 #define KPROBE_EVENT_SYSTEM "kprobes"
28 #define KRETPROBE_MAXACTIVE_MAX 4096
31 * Kprobe event core functions
34 struct list_head list
;
35 struct kretprobe rp
; /* Use rp.kp for kprobe use */
36 unsigned long __percpu
*nhit
;
37 const char *symbol
; /* symbol name */
38 struct trace_probe tp
;
41 #define SIZEOF_TRACE_KPROBE(n) \
42 (offsetof(struct trace_kprobe, tp.args) + \
43 (sizeof(struct probe_arg) * (n)))
46 static nokprobe_inline
bool trace_kprobe_is_return(struct trace_kprobe
*tk
)
48 return tk
->rp
.handler
!= NULL
;
51 static nokprobe_inline
const char *trace_kprobe_symbol(struct trace_kprobe
*tk
)
53 return tk
->symbol
? tk
->symbol
: "unknown";
56 static nokprobe_inline
unsigned long trace_kprobe_offset(struct trace_kprobe
*tk
)
58 return tk
->rp
.kp
.offset
;
61 static nokprobe_inline
bool trace_kprobe_has_gone(struct trace_kprobe
*tk
)
63 return !!(kprobe_gone(&tk
->rp
.kp
));
66 static nokprobe_inline
bool trace_kprobe_within_module(struct trace_kprobe
*tk
,
69 int len
= strlen(mod
->name
);
70 const char *name
= trace_kprobe_symbol(tk
);
71 return strncmp(mod
->name
, name
, len
) == 0 && name
[len
] == ':';
74 static nokprobe_inline
bool trace_kprobe_is_on_module(struct trace_kprobe
*tk
)
76 return !!strchr(trace_kprobe_symbol(tk
), ':');
79 static nokprobe_inline
unsigned long trace_kprobe_nhit(struct trace_kprobe
*tk
)
81 unsigned long nhit
= 0;
84 for_each_possible_cpu(cpu
)
85 nhit
+= *per_cpu_ptr(tk
->nhit
, cpu
);
90 static int register_kprobe_event(struct trace_kprobe
*tk
);
91 static int unregister_kprobe_event(struct trace_kprobe
*tk
);
93 static DEFINE_MUTEX(probe_lock
);
94 static LIST_HEAD(probe_list
);
96 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
97 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
98 struct pt_regs
*regs
);
100 /* Memory fetching by symbol */
101 struct symbol_cache
{
107 unsigned long update_symbol_cache(struct symbol_cache
*sc
)
109 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
112 sc
->addr
+= sc
->offset
;
117 void free_symbol_cache(struct symbol_cache
*sc
)
123 struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
125 struct symbol_cache
*sc
;
127 if (!sym
|| strlen(sym
) == 0)
130 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
134 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
140 update_symbol_cache(sc
);
146 * Kprobes-specific fetch functions
148 #define DEFINE_FETCH_stack(type) \
149 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
150 void *offset, void *dest) \
152 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
153 (unsigned int)((unsigned long)offset)); \
155 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
157 DEFINE_BASIC_FETCH_FUNCS(stack
)
158 /* No string on the stack entry */
159 #define fetch_stack_string NULL
160 #define fetch_stack_string_size NULL
162 #define DEFINE_FETCH_memory(type) \
163 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
164 void *addr, void *dest) \
167 if (probe_kernel_address(addr, retval)) \
170 *(type *)dest = retval; \
172 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
174 DEFINE_BASIC_FETCH_FUNCS(memory
)
176 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
177 * length and relative data location.
179 static void FETCH_FUNC_NAME(memory
, string
)(struct pt_regs
*regs
,
180 void *addr
, void *dest
)
182 int maxlen
= get_rloc_len(*(u32
*)dest
);
183 u8
*dst
= get_rloc_data(dest
);
190 * Try to get string again, since the string can be changed while
193 ret
= strncpy_from_unsafe(dst
, addr
, maxlen
);
195 if (ret
< 0) { /* Failed to fetch string */
197 *(u32
*)dest
= make_data_rloc(0, get_rloc_offs(*(u32
*)dest
));
199 *(u32
*)dest
= make_data_rloc(ret
, get_rloc_offs(*(u32
*)dest
));
202 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory
, string
));
204 /* Return the length of string -- including null terminal byte */
205 static void FETCH_FUNC_NAME(memory
, string_size
)(struct pt_regs
*regs
,
206 void *addr
, void *dest
)
217 ret
= __copy_from_user_inatomic(&c
, (u8
*)addr
+ len
, 1);
219 } while (c
&& ret
== 0 && len
< MAX_STRING_SIZE
);
224 if (ret
< 0) /* Failed to check the length */
229 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory
, string_size
));
231 #define DEFINE_FETCH_symbol(type) \
232 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
234 struct symbol_cache *sc = data; \
236 fetch_memory_##type(regs, (void *)sc->addr, dest); \
240 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
242 DEFINE_BASIC_FETCH_FUNCS(symbol
)
243 DEFINE_FETCH_symbol(string
)
244 DEFINE_FETCH_symbol(string_size
)
246 /* kprobes don't support file_offset fetch methods */
247 #define fetch_file_offset_u8 NULL
248 #define fetch_file_offset_u16 NULL
249 #define fetch_file_offset_u32 NULL
250 #define fetch_file_offset_u64 NULL
251 #define fetch_file_offset_string NULL
252 #define fetch_file_offset_string_size NULL
254 /* Fetch type information table */
255 static const struct fetch_type kprobes_fetch_type_table
[] = {
257 [FETCH_TYPE_STRING
] = __ASSIGN_FETCH_TYPE("string", string
, string
,
258 sizeof(u32
), 1, "__data_loc char[]"),
259 [FETCH_TYPE_STRSIZE
] = __ASSIGN_FETCH_TYPE("string_size", u32
,
260 string_size
, sizeof(u32
), 0, "u32"),
262 ASSIGN_FETCH_TYPE(u8
, u8
, 0),
263 ASSIGN_FETCH_TYPE(u16
, u16
, 0),
264 ASSIGN_FETCH_TYPE(u32
, u32
, 0),
265 ASSIGN_FETCH_TYPE(u64
, u64
, 0),
266 ASSIGN_FETCH_TYPE(s8
, u8
, 1),
267 ASSIGN_FETCH_TYPE(s16
, u16
, 1),
268 ASSIGN_FETCH_TYPE(s32
, u32
, 1),
269 ASSIGN_FETCH_TYPE(s64
, u64
, 1),
270 ASSIGN_FETCH_TYPE_ALIAS(x8
, u8
, u8
, 0),
271 ASSIGN_FETCH_TYPE_ALIAS(x16
, u16
, u16
, 0),
272 ASSIGN_FETCH_TYPE_ALIAS(x32
, u32
, u32
, 0),
273 ASSIGN_FETCH_TYPE_ALIAS(x64
, u64
, u64
, 0),
275 ASSIGN_FETCH_TYPE_END
279 * Allocate new trace_probe and initialize it (including kprobes).
281 static struct trace_kprobe
*alloc_trace_kprobe(const char *group
,
287 int nargs
, bool is_return
)
289 struct trace_kprobe
*tk
;
292 tk
= kzalloc(SIZEOF_TRACE_KPROBE(nargs
), GFP_KERNEL
);
296 tk
->nhit
= alloc_percpu(unsigned long);
301 tk
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
304 tk
->rp
.kp
.symbol_name
= tk
->symbol
;
305 tk
->rp
.kp
.offset
= offs
;
307 tk
->rp
.kp
.addr
= addr
;
310 tk
->rp
.handler
= kretprobe_dispatcher
;
312 tk
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
314 tk
->rp
.maxactive
= maxactive
;
316 if (!event
|| !is_good_name(event
)) {
321 tk
->tp
.call
.class = &tk
->tp
.class;
322 tk
->tp
.call
.name
= kstrdup(event
, GFP_KERNEL
);
323 if (!tk
->tp
.call
.name
)
326 if (!group
|| !is_good_name(group
)) {
331 tk
->tp
.class.system
= kstrdup(group
, GFP_KERNEL
);
332 if (!tk
->tp
.class.system
)
335 INIT_LIST_HEAD(&tk
->list
);
336 INIT_LIST_HEAD(&tk
->tp
.files
);
339 kfree(tk
->tp
.call
.name
);
341 free_percpu(tk
->nhit
);
346 static void free_trace_kprobe(struct trace_kprobe
*tk
)
350 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
351 traceprobe_free_probe_arg(&tk
->tp
.args
[i
]);
353 kfree(tk
->tp
.call
.class->system
);
354 kfree(tk
->tp
.call
.name
);
356 free_percpu(tk
->nhit
);
360 static struct trace_kprobe
*find_trace_kprobe(const char *event
,
363 struct trace_kprobe
*tk
;
365 list_for_each_entry(tk
, &probe_list
, list
)
366 if (strcmp(trace_event_name(&tk
->tp
.call
), event
) == 0 &&
367 strcmp(tk
->tp
.call
.class->system
, group
) == 0)
374 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
377 enable_trace_kprobe(struct trace_kprobe
*tk
, struct trace_event_file
*file
)
382 struct event_file_link
*link
;
384 link
= kmalloc(sizeof(*link
), GFP_KERNEL
);
391 list_add_tail_rcu(&link
->list
, &tk
->tp
.files
);
393 tk
->tp
.flags
|= TP_FLAG_TRACE
;
395 tk
->tp
.flags
|= TP_FLAG_PROFILE
;
397 if (trace_probe_is_registered(&tk
->tp
) && !trace_kprobe_has_gone(tk
)) {
398 if (trace_kprobe_is_return(tk
))
399 ret
= enable_kretprobe(&tk
->rp
);
401 ret
= enable_kprobe(&tk
->rp
.kp
);
408 * Disable trace_probe
409 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
412 disable_trace_kprobe(struct trace_kprobe
*tk
, struct trace_event_file
*file
)
414 struct event_file_link
*link
= NULL
;
419 link
= find_event_file_link(&tk
->tp
, file
);
425 list_del_rcu(&link
->list
);
427 if (!list_empty(&tk
->tp
.files
))
430 tk
->tp
.flags
&= ~TP_FLAG_TRACE
;
432 tk
->tp
.flags
&= ~TP_FLAG_PROFILE
;
434 if (!trace_probe_is_enabled(&tk
->tp
) && trace_probe_is_registered(&tk
->tp
)) {
435 if (trace_kprobe_is_return(tk
))
436 disable_kretprobe(&tk
->rp
);
438 disable_kprobe(&tk
->rp
.kp
);
444 * Synchronize with kprobe_trace_func/kretprobe_trace_func
445 * to ensure disabled (all running handlers are finished).
446 * This is not only for kfree(), but also the caller,
447 * trace_remove_event_call() supposes it for releasing
448 * event_call related objects, which will be accessed in
449 * the kprobe_trace_func/kretprobe_trace_func.
452 kfree(link
); /* Ignored if link == NULL */
458 /* Internal register function - just handle k*probes and flags */
459 static int __register_trace_kprobe(struct trace_kprobe
*tk
)
463 if (trace_probe_is_registered(&tk
->tp
))
466 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
467 traceprobe_update_arg(&tk
->tp
.args
[i
]);
469 /* Set/clear disabled flag according to tp->flag */
470 if (trace_probe_is_enabled(&tk
->tp
))
471 tk
->rp
.kp
.flags
&= ~KPROBE_FLAG_DISABLED
;
473 tk
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
475 if (trace_kprobe_is_return(tk
))
476 ret
= register_kretprobe(&tk
->rp
);
478 ret
= register_kprobe(&tk
->rp
.kp
);
481 tk
->tp
.flags
|= TP_FLAG_REGISTERED
;
483 pr_warn("Could not insert probe at %s+%lu: %d\n",
484 trace_kprobe_symbol(tk
), trace_kprobe_offset(tk
), ret
);
485 if (ret
== -ENOENT
&& trace_kprobe_is_on_module(tk
)) {
486 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
488 } else if (ret
== -EILSEQ
) {
489 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
498 /* Internal unregister function - just handle k*probes and flags */
499 static void __unregister_trace_kprobe(struct trace_kprobe
*tk
)
501 if (trace_probe_is_registered(&tk
->tp
)) {
502 if (trace_kprobe_is_return(tk
))
503 unregister_kretprobe(&tk
->rp
);
505 unregister_kprobe(&tk
->rp
.kp
);
506 tk
->tp
.flags
&= ~TP_FLAG_REGISTERED
;
507 /* Cleanup kprobe for reuse */
508 if (tk
->rp
.kp
.symbol_name
)
509 tk
->rp
.kp
.addr
= NULL
;
513 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
514 static int unregister_trace_kprobe(struct trace_kprobe
*tk
)
516 /* Enabled event can not be unregistered */
517 if (trace_probe_is_enabled(&tk
->tp
))
520 /* Will fail if probe is being used by ftrace or perf */
521 if (unregister_kprobe_event(tk
))
524 __unregister_trace_kprobe(tk
);
530 /* Register a trace_probe and probe_event */
531 static int register_trace_kprobe(struct trace_kprobe
*tk
)
533 struct trace_kprobe
*old_tk
;
536 mutex_lock(&probe_lock
);
538 /* Delete old (same name) event if exist */
539 old_tk
= find_trace_kprobe(trace_event_name(&tk
->tp
.call
),
540 tk
->tp
.call
.class->system
);
542 ret
= unregister_trace_kprobe(old_tk
);
545 free_trace_kprobe(old_tk
);
548 /* Register new event */
549 ret
= register_kprobe_event(tk
);
551 pr_warn("Failed to register probe event(%d)\n", ret
);
555 /* Register k*probe */
556 ret
= __register_trace_kprobe(tk
);
558 unregister_kprobe_event(tk
);
560 list_add_tail(&tk
->list
, &probe_list
);
563 mutex_unlock(&probe_lock
);
567 /* Module notifier call back, checking event on the module */
568 static int trace_kprobe_module_callback(struct notifier_block
*nb
,
569 unsigned long val
, void *data
)
571 struct module
*mod
= data
;
572 struct trace_kprobe
*tk
;
575 if (val
!= MODULE_STATE_COMING
)
578 /* Update probes on coming module */
579 mutex_lock(&probe_lock
);
580 list_for_each_entry(tk
, &probe_list
, list
) {
581 if (trace_kprobe_within_module(tk
, mod
)) {
582 /* Don't need to check busy - this should have gone. */
583 __unregister_trace_kprobe(tk
);
584 ret
= __register_trace_kprobe(tk
);
586 pr_warn("Failed to re-register probe %s on %s: %d\n",
587 trace_event_name(&tk
->tp
.call
),
591 mutex_unlock(&probe_lock
);
596 static struct notifier_block trace_kprobe_module_nb
= {
597 .notifier_call
= trace_kprobe_module_callback
,
598 .priority
= 1 /* Invoked after kprobe module callback */
601 static int create_trace_kprobe(int argc
, char **argv
)
606 * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
608 * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
610 * $retval : fetch return value
611 * $stack : fetch stack address
612 * $stackN : fetch Nth of stack (N:0-)
613 * $comm : fetch current task comm
614 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
615 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
616 * %REG : fetch register REG
617 * Dereferencing memory fetch:
618 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
619 * Alias name of args:
620 * NAME=FETCHARG : set NAME as alias of FETCHARG.
622 * FETCHARG:TYPE : use TYPE instead of unsigned long.
624 struct trace_kprobe
*tk
;
626 bool is_return
= false, is_delete
= false;
627 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
630 unsigned long offset
= 0;
632 char buf
[MAX_EVENT_NAME_LEN
];
634 /* argc must be >= 1 */
635 if (argv
[0][0] == 'p')
637 else if (argv
[0][0] == 'r')
639 else if (argv
[0][0] == '-')
642 pr_info("Probe definition must be started with 'p', 'r' or"
647 event
= strchr(&argv
[0][1], ':');
652 if (is_return
&& isdigit(argv
[0][1])) {
653 ret
= kstrtouint(&argv
[0][1], 0, &maxactive
);
655 pr_info("Failed to parse maxactive.\n");
658 /* kretprobes instances are iterated over via a list. The
659 * maximum should stay reasonable.
661 if (maxactive
> KRETPROBE_MAXACTIVE_MAX
) {
662 pr_info("Maxactive is too big (%d > %d).\n",
663 maxactive
, KRETPROBE_MAXACTIVE_MAX
);
669 if (strchr(event
, '/')) {
671 event
= strchr(group
, '/') + 1;
673 if (strlen(group
) == 0) {
674 pr_info("Group name is not specified\n");
678 if (strlen(event
) == 0) {
679 pr_info("Event name is not specified\n");
684 group
= KPROBE_EVENT_SYSTEM
;
688 pr_info("Delete command needs an event name.\n");
691 mutex_lock(&probe_lock
);
692 tk
= find_trace_kprobe(event
, group
);
694 mutex_unlock(&probe_lock
);
695 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
698 /* delete an event */
699 ret
= unregister_trace_kprobe(tk
);
701 free_trace_kprobe(tk
);
702 mutex_unlock(&probe_lock
);
707 pr_info("Probe point is not specified.\n");
710 if (isdigit(argv
[1][0])) {
711 /* an address specified */
712 ret
= kstrtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
714 pr_info("Failed to parse address.\n");
718 /* a symbol specified */
720 /* TODO: support .init module functions */
721 ret
= traceprobe_split_symbol_offset(symbol
, &offset
);
723 pr_info("Failed to parse symbol.\n");
726 if (offset
&& is_return
&&
727 !function_offset_within_entry(NULL
, symbol
, offset
)) {
728 pr_info("Given offset is not valid for return probe.\n");
732 argc
-= 2; argv
+= 2;
736 /* Make a new event name */
738 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
739 is_return
? 'r' : 'p', symbol
, offset
);
741 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
742 is_return
? 'r' : 'p', addr
);
745 tk
= alloc_trace_kprobe(group
, event
, addr
, symbol
, offset
, maxactive
,
748 pr_info("Failed to allocate trace_probe.(%d)\n",
753 /* parse arguments */
755 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
756 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
758 /* Increment count for freeing args in error case */
761 /* Parse argument name */
762 arg
= strchr(argv
[i
], '=');
765 parg
->name
= kstrdup(argv
[i
], GFP_KERNEL
);
768 /* If argument name is omitted, set "argN" */
769 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
770 parg
->name
= kstrdup(buf
, GFP_KERNEL
);
774 pr_info("Failed to allocate argument[%d] name.\n", i
);
779 if (!is_good_name(parg
->name
)) {
780 pr_info("Invalid argument[%d] name: %s\n",
786 if (traceprobe_conflict_field_name(parg
->name
,
788 pr_info("Argument[%d] name '%s' conflicts with "
789 "another field.\n", i
, argv
[i
]);
794 /* Parse fetch argument */
795 ret
= traceprobe_parse_probe_arg(arg
, &tk
->tp
.size
, parg
,
797 kprobes_fetch_type_table
);
799 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
804 ret
= register_trace_kprobe(tk
);
810 free_trace_kprobe(tk
);
814 static int release_all_trace_kprobes(void)
816 struct trace_kprobe
*tk
;
819 mutex_lock(&probe_lock
);
820 /* Ensure no probe is in use. */
821 list_for_each_entry(tk
, &probe_list
, list
)
822 if (trace_probe_is_enabled(&tk
->tp
)) {
826 /* TODO: Use batch unregistration */
827 while (!list_empty(&probe_list
)) {
828 tk
= list_entry(probe_list
.next
, struct trace_kprobe
, list
);
829 ret
= unregister_trace_kprobe(tk
);
832 free_trace_kprobe(tk
);
836 mutex_unlock(&probe_lock
);
841 /* Probes listing interfaces */
842 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
844 mutex_lock(&probe_lock
);
845 return seq_list_start(&probe_list
, *pos
);
848 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
850 return seq_list_next(v
, &probe_list
, pos
);
853 static void probes_seq_stop(struct seq_file
*m
, void *v
)
855 mutex_unlock(&probe_lock
);
858 static int probes_seq_show(struct seq_file
*m
, void *v
)
860 struct trace_kprobe
*tk
= v
;
863 seq_putc(m
, trace_kprobe_is_return(tk
) ? 'r' : 'p');
864 seq_printf(m
, ":%s/%s", tk
->tp
.call
.class->system
,
865 trace_event_name(&tk
->tp
.call
));
868 seq_printf(m
, " 0x%p", tk
->rp
.kp
.addr
);
869 else if (tk
->rp
.kp
.offset
)
870 seq_printf(m
, " %s+%u", trace_kprobe_symbol(tk
),
873 seq_printf(m
, " %s", trace_kprobe_symbol(tk
));
875 for (i
= 0; i
< tk
->tp
.nr_args
; i
++)
876 seq_printf(m
, " %s=%s", tk
->tp
.args
[i
].name
, tk
->tp
.args
[i
].comm
);
882 static const struct seq_operations probes_seq_op
= {
883 .start
= probes_seq_start
,
884 .next
= probes_seq_next
,
885 .stop
= probes_seq_stop
,
886 .show
= probes_seq_show
889 static int probes_open(struct inode
*inode
, struct file
*file
)
893 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
894 ret
= release_all_trace_kprobes();
899 return seq_open(file
, &probes_seq_op
);
902 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
903 size_t count
, loff_t
*ppos
)
905 return traceprobe_probes_write(file
, buffer
, count
, ppos
,
906 create_trace_kprobe
);
909 static const struct file_operations kprobe_events_ops
= {
910 .owner
= THIS_MODULE
,
914 .release
= seq_release
,
915 .write
= probes_write
,
918 /* Probes profiling interfaces */
919 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
921 struct trace_kprobe
*tk
= v
;
923 seq_printf(m
, " %-44s %15lu %15lu\n",
924 trace_event_name(&tk
->tp
.call
),
925 trace_kprobe_nhit(tk
),
931 static const struct seq_operations profile_seq_op
= {
932 .start
= probes_seq_start
,
933 .next
= probes_seq_next
,
934 .stop
= probes_seq_stop
,
935 .show
= probes_profile_seq_show
938 static int profile_open(struct inode
*inode
, struct file
*file
)
940 return seq_open(file
, &profile_seq_op
);
943 static const struct file_operations kprobe_profile_ops
= {
944 .owner
= THIS_MODULE
,
945 .open
= profile_open
,
948 .release
= seq_release
,
952 static nokprobe_inline
void
953 __kprobe_trace_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
,
954 struct trace_event_file
*trace_file
)
956 struct kprobe_trace_entry_head
*entry
;
957 struct ring_buffer_event
*event
;
958 struct ring_buffer
*buffer
;
960 unsigned long irq_flags
;
961 struct trace_event_call
*call
= &tk
->tp
.call
;
963 WARN_ON(call
!= trace_file
->event_call
);
965 if (trace_trigger_soft_disabled(trace_file
))
968 local_save_flags(irq_flags
);
969 pc
= preempt_count();
971 dsize
= __get_data_size(&tk
->tp
, regs
);
972 size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
974 event
= trace_event_buffer_lock_reserve(&buffer
, trace_file
,
976 size
, irq_flags
, pc
);
980 entry
= ring_buffer_event_data(event
);
981 entry
->ip
= (unsigned long)tk
->rp
.kp
.addr
;
982 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
984 event_trigger_unlock_commit_regs(trace_file
, buffer
, event
,
985 entry
, irq_flags
, pc
, regs
);
989 kprobe_trace_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
)
991 struct event_file_link
*link
;
993 list_for_each_entry_rcu(link
, &tk
->tp
.files
, list
)
994 __kprobe_trace_func(tk
, regs
, link
->file
);
996 NOKPROBE_SYMBOL(kprobe_trace_func
);
998 /* Kretprobe handler */
999 static nokprobe_inline
void
1000 __kretprobe_trace_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
1001 struct pt_regs
*regs
,
1002 struct trace_event_file
*trace_file
)
1004 struct kretprobe_trace_entry_head
*entry
;
1005 struct ring_buffer_event
*event
;
1006 struct ring_buffer
*buffer
;
1007 int size
, pc
, dsize
;
1008 unsigned long irq_flags
;
1009 struct trace_event_call
*call
= &tk
->tp
.call
;
1011 WARN_ON(call
!= trace_file
->event_call
);
1013 if (trace_trigger_soft_disabled(trace_file
))
1016 local_save_flags(irq_flags
);
1017 pc
= preempt_count();
1019 dsize
= __get_data_size(&tk
->tp
, regs
);
1020 size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
1022 event
= trace_event_buffer_lock_reserve(&buffer
, trace_file
,
1024 size
, irq_flags
, pc
);
1028 entry
= ring_buffer_event_data(event
);
1029 entry
->func
= (unsigned long)tk
->rp
.kp
.addr
;
1030 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1031 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
1033 event_trigger_unlock_commit_regs(trace_file
, buffer
, event
,
1034 entry
, irq_flags
, pc
, regs
);
1038 kretprobe_trace_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
1039 struct pt_regs
*regs
)
1041 struct event_file_link
*link
;
1043 list_for_each_entry_rcu(link
, &tk
->tp
.files
, list
)
1044 __kretprobe_trace_func(tk
, ri
, regs
, link
->file
);
1046 NOKPROBE_SYMBOL(kretprobe_trace_func
);
1048 /* Event entry printers */
1049 static enum print_line_t
1050 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
1051 struct trace_event
*event
)
1053 struct kprobe_trace_entry_head
*field
;
1054 struct trace_seq
*s
= &iter
->seq
;
1055 struct trace_probe
*tp
;
1059 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
1060 tp
= container_of(event
, struct trace_probe
, call
.event
);
1062 trace_seq_printf(s
, "%s: (", trace_event_name(&tp
->call
));
1064 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1067 trace_seq_putc(s
, ')');
1069 data
= (u8
*)&field
[1];
1070 for (i
= 0; i
< tp
->nr_args
; i
++)
1071 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1072 data
+ tp
->args
[i
].offset
, field
))
1075 trace_seq_putc(s
, '\n');
1077 return trace_handle_return(s
);
1080 static enum print_line_t
1081 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
1082 struct trace_event
*event
)
1084 struct kretprobe_trace_entry_head
*field
;
1085 struct trace_seq
*s
= &iter
->seq
;
1086 struct trace_probe
*tp
;
1090 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
1091 tp
= container_of(event
, struct trace_probe
, call
.event
);
1093 trace_seq_printf(s
, "%s: (", trace_event_name(&tp
->call
));
1095 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1098 trace_seq_puts(s
, " <- ");
1100 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1103 trace_seq_putc(s
, ')');
1105 data
= (u8
*)&field
[1];
1106 for (i
= 0; i
< tp
->nr_args
; i
++)
1107 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1108 data
+ tp
->args
[i
].offset
, field
))
1111 trace_seq_putc(s
, '\n');
1114 return trace_handle_return(s
);
1118 static int kprobe_event_define_fields(struct trace_event_call
*event_call
)
1121 struct kprobe_trace_entry_head field
;
1122 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event_call
->data
;
1124 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1125 /* Set argument names as fields */
1126 for (i
= 0; i
< tk
->tp
.nr_args
; i
++) {
1127 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
1129 ret
= trace_define_field(event_call
, parg
->type
->fmttype
,
1131 sizeof(field
) + parg
->offset
,
1133 parg
->type
->is_signed
,
1141 static int kretprobe_event_define_fields(struct trace_event_call
*event_call
)
1144 struct kretprobe_trace_entry_head field
;
1145 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event_call
->data
;
1147 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1148 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1149 /* Set argument names as fields */
1150 for (i
= 0; i
< tk
->tp
.nr_args
; i
++) {
1151 struct probe_arg
*parg
= &tk
->tp
.args
[i
];
1153 ret
= trace_define_field(event_call
, parg
->type
->fmttype
,
1155 sizeof(field
) + parg
->offset
,
1157 parg
->type
->is_signed
,
1165 #ifdef CONFIG_PERF_EVENTS
1167 /* Kprobe profile handler */
1169 kprobe_perf_func(struct trace_kprobe
*tk
, struct pt_regs
*regs
)
1171 struct trace_event_call
*call
= &tk
->tp
.call
;
1172 struct bpf_prog
*prog
= call
->prog
;
1173 struct kprobe_trace_entry_head
*entry
;
1174 struct hlist_head
*head
;
1175 int size
, __size
, dsize
;
1178 if (prog
&& !trace_call_bpf(prog
, regs
))
1181 head
= this_cpu_ptr(call
->perf_events
);
1182 if (hlist_empty(head
))
1185 dsize
= __get_data_size(&tk
->tp
, regs
);
1186 __size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
1187 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1188 size
-= sizeof(u32
);
1190 entry
= perf_trace_buf_alloc(size
, NULL
, &rctx
);
1194 entry
->ip
= (unsigned long)tk
->rp
.kp
.addr
;
1195 memset(&entry
[1], 0, dsize
);
1196 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
1197 perf_trace_buf_submit(entry
, size
, rctx
, call
->event
.type
, 1, regs
,
1200 NOKPROBE_SYMBOL(kprobe_perf_func
);
1202 /* Kretprobe profile handler */
1204 kretprobe_perf_func(struct trace_kprobe
*tk
, struct kretprobe_instance
*ri
,
1205 struct pt_regs
*regs
)
1207 struct trace_event_call
*call
= &tk
->tp
.call
;
1208 struct bpf_prog
*prog
= call
->prog
;
1209 struct kretprobe_trace_entry_head
*entry
;
1210 struct hlist_head
*head
;
1211 int size
, __size
, dsize
;
1214 if (prog
&& !trace_call_bpf(prog
, regs
))
1217 head
= this_cpu_ptr(call
->perf_events
);
1218 if (hlist_empty(head
))
1221 dsize
= __get_data_size(&tk
->tp
, regs
);
1222 __size
= sizeof(*entry
) + tk
->tp
.size
+ dsize
;
1223 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1224 size
-= sizeof(u32
);
1226 entry
= perf_trace_buf_alloc(size
, NULL
, &rctx
);
1230 entry
->func
= (unsigned long)tk
->rp
.kp
.addr
;
1231 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1232 store_trace_args(sizeof(*entry
), &tk
->tp
, regs
, (u8
*)&entry
[1], dsize
);
1233 perf_trace_buf_submit(entry
, size
, rctx
, call
->event
.type
, 1, regs
,
1236 NOKPROBE_SYMBOL(kretprobe_perf_func
);
1237 #endif /* CONFIG_PERF_EVENTS */
1240 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1242 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1243 * lockless, but we can't race with this __init function.
1245 static int kprobe_register(struct trace_event_call
*event
,
1246 enum trace_reg type
, void *data
)
1248 struct trace_kprobe
*tk
= (struct trace_kprobe
*)event
->data
;
1249 struct trace_event_file
*file
= data
;
1252 case TRACE_REG_REGISTER
:
1253 return enable_trace_kprobe(tk
, file
);
1254 case TRACE_REG_UNREGISTER
:
1255 return disable_trace_kprobe(tk
, file
);
1257 #ifdef CONFIG_PERF_EVENTS
1258 case TRACE_REG_PERF_REGISTER
:
1259 return enable_trace_kprobe(tk
, NULL
);
1260 case TRACE_REG_PERF_UNREGISTER
:
1261 return disable_trace_kprobe(tk
, NULL
);
1262 case TRACE_REG_PERF_OPEN
:
1263 case TRACE_REG_PERF_CLOSE
:
1264 case TRACE_REG_PERF_ADD
:
1265 case TRACE_REG_PERF_DEL
:
1272 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1274 struct trace_kprobe
*tk
= container_of(kp
, struct trace_kprobe
, rp
.kp
);
1276 raw_cpu_inc(*tk
->nhit
);
1278 if (tk
->tp
.flags
& TP_FLAG_TRACE
)
1279 kprobe_trace_func(tk
, regs
);
1280 #ifdef CONFIG_PERF_EVENTS
1281 if (tk
->tp
.flags
& TP_FLAG_PROFILE
)
1282 kprobe_perf_func(tk
, regs
);
1284 return 0; /* We don't tweek kernel, so just return 0 */
1286 NOKPROBE_SYMBOL(kprobe_dispatcher
);
1289 kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1291 struct trace_kprobe
*tk
= container_of(ri
->rp
, struct trace_kprobe
, rp
);
1293 raw_cpu_inc(*tk
->nhit
);
1295 if (tk
->tp
.flags
& TP_FLAG_TRACE
)
1296 kretprobe_trace_func(tk
, ri
, regs
);
1297 #ifdef CONFIG_PERF_EVENTS
1298 if (tk
->tp
.flags
& TP_FLAG_PROFILE
)
1299 kretprobe_perf_func(tk
, ri
, regs
);
1301 return 0; /* We don't tweek kernel, so just return 0 */
1303 NOKPROBE_SYMBOL(kretprobe_dispatcher
);
1305 static struct trace_event_functions kretprobe_funcs
= {
1306 .trace
= print_kretprobe_event
1309 static struct trace_event_functions kprobe_funcs
= {
1310 .trace
= print_kprobe_event
1313 static int register_kprobe_event(struct trace_kprobe
*tk
)
1315 struct trace_event_call
*call
= &tk
->tp
.call
;
1318 /* Initialize trace_event_call */
1319 INIT_LIST_HEAD(&call
->class->fields
);
1320 if (trace_kprobe_is_return(tk
)) {
1321 call
->event
.funcs
= &kretprobe_funcs
;
1322 call
->class->define_fields
= kretprobe_event_define_fields
;
1324 call
->event
.funcs
= &kprobe_funcs
;
1325 call
->class->define_fields
= kprobe_event_define_fields
;
1327 if (set_print_fmt(&tk
->tp
, trace_kprobe_is_return(tk
)) < 0)
1329 ret
= register_trace_event(&call
->event
);
1331 kfree(call
->print_fmt
);
1334 call
->flags
= TRACE_EVENT_FL_KPROBE
;
1335 call
->class->reg
= kprobe_register
;
1337 ret
= trace_add_event_call(call
);
1339 pr_info("Failed to register kprobe event: %s\n",
1340 trace_event_name(call
));
1341 kfree(call
->print_fmt
);
1342 unregister_trace_event(&call
->event
);
1347 static int unregister_kprobe_event(struct trace_kprobe
*tk
)
1351 /* tp->event is unregistered in trace_remove_event_call() */
1352 ret
= trace_remove_event_call(&tk
->tp
.call
);
1354 kfree(tk
->tp
.call
.print_fmt
);
1358 /* Make a tracefs interface for controlling probe points */
1359 static __init
int init_kprobe_trace(void)
1361 struct dentry
*d_tracer
;
1362 struct dentry
*entry
;
1364 if (register_module_notifier(&trace_kprobe_module_nb
))
1367 d_tracer
= tracing_init_dentry();
1368 if (IS_ERR(d_tracer
))
1371 entry
= tracefs_create_file("kprobe_events", 0644, d_tracer
,
1372 NULL
, &kprobe_events_ops
);
1374 /* Event list interface */
1376 pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1378 /* Profile interface */
1379 entry
= tracefs_create_file("kprobe_profile", 0444, d_tracer
,
1380 NULL
, &kprobe_profile_ops
);
1383 pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
1386 fs_initcall(init_kprobe_trace
);
1389 #ifdef CONFIG_FTRACE_STARTUP_TEST
1391 * The "__used" keeps gcc from removing the function symbol
1392 * from the kallsyms table. 'noinline' makes sure that there
1393 * isn't an inlined version used by the test method below
1395 static __used __init noinline
int
1396 kprobe_trace_selftest_target(int a1
, int a2
, int a3
, int a4
, int a5
, int a6
)
1398 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1401 static __init
struct trace_event_file
*
1402 find_trace_probe_file(struct trace_kprobe
*tk
, struct trace_array
*tr
)
1404 struct trace_event_file
*file
;
1406 list_for_each_entry(file
, &tr
->events
, list
)
1407 if (file
->event_call
== &tk
->tp
.call
)
1414 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1415 * stage, we can do this lockless.
1417 static __init
int kprobe_trace_self_tests_init(void)
1420 int (*target
)(int, int, int, int, int, int);
1421 struct trace_kprobe
*tk
;
1422 struct trace_event_file
*file
;
1424 if (tracing_is_disabled())
1427 target
= kprobe_trace_selftest_target
;
1429 pr_info("Testing kprobe tracing: ");
1431 ret
= traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1432 "$stack $stack0 +0($stack)",
1433 create_trace_kprobe
);
1434 if (WARN_ON_ONCE(ret
)) {
1435 pr_warn("error on probing function entry.\n");
1438 /* Enable trace point */
1439 tk
= find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM
);
1440 if (WARN_ON_ONCE(tk
== NULL
)) {
1441 pr_warn("error on getting new probe.\n");
1444 file
= find_trace_probe_file(tk
, top_trace_array());
1445 if (WARN_ON_ONCE(file
== NULL
)) {
1446 pr_warn("error on getting probe file.\n");
1449 enable_trace_kprobe(tk
, file
);
1453 ret
= traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1454 "$retval", create_trace_kprobe
);
1455 if (WARN_ON_ONCE(ret
)) {
1456 pr_warn("error on probing function return.\n");
1459 /* Enable trace point */
1460 tk
= find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM
);
1461 if (WARN_ON_ONCE(tk
== NULL
)) {
1462 pr_warn("error on getting 2nd new probe.\n");
1465 file
= find_trace_probe_file(tk
, top_trace_array());
1466 if (WARN_ON_ONCE(file
== NULL
)) {
1467 pr_warn("error on getting probe file.\n");
1470 enable_trace_kprobe(tk
, file
);
1477 ret
= target(1, 2, 3, 4, 5, 6);
1480 * Not expecting an error here, the check is only to prevent the
1481 * optimizer from removing the call to target() as otherwise there
1482 * are no side-effects and the call is never performed.
1487 /* Disable trace points before removing it */
1488 tk
= find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM
);
1489 if (WARN_ON_ONCE(tk
== NULL
)) {
1490 pr_warn("error on getting test probe.\n");
1493 if (trace_kprobe_nhit(tk
) != 1) {
1494 pr_warn("incorrect number of testprobe hits\n");
1498 file
= find_trace_probe_file(tk
, top_trace_array());
1499 if (WARN_ON_ONCE(file
== NULL
)) {
1500 pr_warn("error on getting probe file.\n");
1503 disable_trace_kprobe(tk
, file
);
1506 tk
= find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM
);
1507 if (WARN_ON_ONCE(tk
== NULL
)) {
1508 pr_warn("error on getting 2nd test probe.\n");
1511 if (trace_kprobe_nhit(tk
) != 1) {
1512 pr_warn("incorrect number of testprobe2 hits\n");
1516 file
= find_trace_probe_file(tk
, top_trace_array());
1517 if (WARN_ON_ONCE(file
== NULL
)) {
1518 pr_warn("error on getting probe file.\n");
1521 disable_trace_kprobe(tk
, file
);
1524 ret
= traceprobe_command("-:testprobe", create_trace_kprobe
);
1525 if (WARN_ON_ONCE(ret
)) {
1526 pr_warn("error on deleting a probe.\n");
1530 ret
= traceprobe_command("-:testprobe2", create_trace_kprobe
);
1531 if (WARN_ON_ONCE(ret
)) {
1532 pr_warn("error on deleting a probe.\n");
1537 release_all_trace_kprobes();
1539 * Wait for the optimizer work to finish. Otherwise it might fiddle
1540 * with probes in already freed __init text.
1542 wait_for_kprobe_optimizer();
1544 pr_cont("NG: Some tests are failed. Please check them.\n");
1550 late_initcall(kprobe_trace_self_tests_init
);