2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
23 #include "trace_probe.h"
25 #define KPROBE_EVENT_SYSTEM "kprobes"
28 * Kprobe event core functions
32 struct list_head list
;
33 struct kretprobe rp
; /* Use rp.kp for kprobe use */
35 unsigned int flags
; /* For TP_FLAG_* */
36 const char *symbol
; /* symbol name */
37 struct ftrace_event_class
class;
38 struct ftrace_event_call call
;
39 ssize_t size
; /* trace entry size */
41 struct probe_arg args
[];
44 #define SIZEOF_TRACE_PROBE(n) \
45 (offsetof(struct trace_probe, args) + \
46 (sizeof(struct probe_arg) * (n)))
49 static __kprobes
bool trace_probe_is_return(struct trace_probe
*tp
)
51 return tp
->rp
.handler
!= NULL
;
54 static __kprobes
const char *trace_probe_symbol(struct trace_probe
*tp
)
56 return tp
->symbol
? tp
->symbol
: "unknown";
59 static __kprobes
unsigned long trace_probe_offset(struct trace_probe
*tp
)
61 return tp
->rp
.kp
.offset
;
64 static __kprobes
bool trace_probe_is_enabled(struct trace_probe
*tp
)
66 return !!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
));
69 static __kprobes
bool trace_probe_is_registered(struct trace_probe
*tp
)
71 return !!(tp
->flags
& TP_FLAG_REGISTERED
);
74 static __kprobes
bool trace_probe_has_gone(struct trace_probe
*tp
)
76 return !!(kprobe_gone(&tp
->rp
.kp
));
79 static __kprobes
bool trace_probe_within_module(struct trace_probe
*tp
,
82 int len
= strlen(mod
->name
);
83 const char *name
= trace_probe_symbol(tp
);
84 return strncmp(mod
->name
, name
, len
) == 0 && name
[len
] == ':';
87 static __kprobes
bool trace_probe_is_on_module(struct trace_probe
*tp
)
89 return !!strchr(trace_probe_symbol(tp
), ':');
92 static int register_probe_event(struct trace_probe
*tp
);
93 static void unregister_probe_event(struct trace_probe
*tp
);
95 static DEFINE_MUTEX(probe_lock
);
96 static LIST_HEAD(probe_list
);
98 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
99 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
100 struct pt_regs
*regs
);
103 * Allocate new trace_probe and initialize it (including kprobes).
105 static struct trace_probe
*alloc_trace_probe(const char *group
,
110 int nargs
, bool is_return
)
112 struct trace_probe
*tp
;
115 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
120 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
123 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
124 tp
->rp
.kp
.offset
= offs
;
126 tp
->rp
.kp
.addr
= addr
;
129 tp
->rp
.handler
= kretprobe_dispatcher
;
131 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
133 if (!event
|| !is_good_name(event
)) {
138 tp
->call
.class = &tp
->class;
139 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
143 if (!group
|| !is_good_name(group
)) {
148 tp
->class.system
= kstrdup(group
, GFP_KERNEL
);
149 if (!tp
->class.system
)
152 INIT_LIST_HEAD(&tp
->list
);
155 kfree(tp
->call
.name
);
161 static void free_trace_probe(struct trace_probe
*tp
)
165 for (i
= 0; i
< tp
->nr_args
; i
++)
166 traceprobe_free_probe_arg(&tp
->args
[i
]);
168 kfree(tp
->call
.class->system
);
169 kfree(tp
->call
.name
);
174 static struct trace_probe
*find_trace_probe(const char *event
,
177 struct trace_probe
*tp
;
179 list_for_each_entry(tp
, &probe_list
, list
)
180 if (strcmp(tp
->call
.name
, event
) == 0 &&
181 strcmp(tp
->call
.class->system
, group
) == 0)
186 /* Enable trace_probe - @flag must be TP_FLAG_TRACE or TP_FLAG_PROFILE */
187 static int enable_trace_probe(struct trace_probe
*tp
, int flag
)
192 if (trace_probe_is_enabled(tp
) && trace_probe_is_registered(tp
) &&
193 !trace_probe_has_gone(tp
)) {
194 if (trace_probe_is_return(tp
))
195 ret
= enable_kretprobe(&tp
->rp
);
197 ret
= enable_kprobe(&tp
->rp
.kp
);
203 /* Disable trace_probe - @flag must be TP_FLAG_TRACE or TP_FLAG_PROFILE */
204 static void disable_trace_probe(struct trace_probe
*tp
, int flag
)
207 if (!trace_probe_is_enabled(tp
) && trace_probe_is_registered(tp
)) {
208 if (trace_probe_is_return(tp
))
209 disable_kretprobe(&tp
->rp
);
211 disable_kprobe(&tp
->rp
.kp
);
215 /* Internal register function - just handle k*probes and flags */
216 static int __register_trace_probe(struct trace_probe
*tp
)
220 if (trace_probe_is_registered(tp
))
223 for (i
= 0; i
< tp
->nr_args
; i
++)
224 traceprobe_update_arg(&tp
->args
[i
]);
226 /* Set/clear disabled flag according to tp->flag */
227 if (trace_probe_is_enabled(tp
))
228 tp
->rp
.kp
.flags
&= ~KPROBE_FLAG_DISABLED
;
230 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
232 if (trace_probe_is_return(tp
))
233 ret
= register_kretprobe(&tp
->rp
);
235 ret
= register_kprobe(&tp
->rp
.kp
);
238 tp
->flags
|= TP_FLAG_REGISTERED
;
240 pr_warning("Could not insert probe at %s+%lu: %d\n",
241 trace_probe_symbol(tp
), trace_probe_offset(tp
), ret
);
242 if (ret
== -ENOENT
&& trace_probe_is_on_module(tp
)) {
243 pr_warning("This probe might be able to register after"
244 "target module is loaded. Continue.\n");
246 } else if (ret
== -EILSEQ
) {
247 pr_warning("Probing address(0x%p) is not an "
248 "instruction boundary.\n",
257 /* Internal unregister function - just handle k*probes and flags */
258 static void __unregister_trace_probe(struct trace_probe
*tp
)
260 if (trace_probe_is_registered(tp
)) {
261 if (trace_probe_is_return(tp
))
262 unregister_kretprobe(&tp
->rp
);
264 unregister_kprobe(&tp
->rp
.kp
);
265 tp
->flags
&= ~TP_FLAG_REGISTERED
;
266 /* Cleanup kprobe for reuse */
267 if (tp
->rp
.kp
.symbol_name
)
268 tp
->rp
.kp
.addr
= NULL
;
272 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
273 static int unregister_trace_probe(struct trace_probe
*tp
)
275 /* Enabled event can not be unregistered */
276 if (trace_probe_is_enabled(tp
))
279 __unregister_trace_probe(tp
);
281 unregister_probe_event(tp
);
286 /* Register a trace_probe and probe_event */
287 static int register_trace_probe(struct trace_probe
*tp
)
289 struct trace_probe
*old_tp
;
292 mutex_lock(&probe_lock
);
294 /* Delete old (same name) event if exist */
295 old_tp
= find_trace_probe(tp
->call
.name
, tp
->call
.class->system
);
297 ret
= unregister_trace_probe(old_tp
);
300 free_trace_probe(old_tp
);
303 /* Register new event */
304 ret
= register_probe_event(tp
);
306 pr_warning("Failed to register probe event(%d)\n", ret
);
310 /* Register k*probe */
311 ret
= __register_trace_probe(tp
);
313 unregister_probe_event(tp
);
315 list_add_tail(&tp
->list
, &probe_list
);
318 mutex_unlock(&probe_lock
);
322 /* Module notifier call back, checking event on the module */
323 static int trace_probe_module_callback(struct notifier_block
*nb
,
324 unsigned long val
, void *data
)
326 struct module
*mod
= data
;
327 struct trace_probe
*tp
;
330 if (val
!= MODULE_STATE_COMING
)
333 /* Update probes on coming module */
334 mutex_lock(&probe_lock
);
335 list_for_each_entry(tp
, &probe_list
, list
) {
336 if (trace_probe_within_module(tp
, mod
)) {
337 /* Don't need to check busy - this should have gone. */
338 __unregister_trace_probe(tp
);
339 ret
= __register_trace_probe(tp
);
341 pr_warning("Failed to re-register probe %s on"
343 tp
->call
.name
, mod
->name
, ret
);
346 mutex_unlock(&probe_lock
);
351 static struct notifier_block trace_probe_module_nb
= {
352 .notifier_call
= trace_probe_module_callback
,
353 .priority
= 1 /* Invoked after kprobe module callback */
356 static int create_trace_probe(int argc
, char **argv
)
360 * - Add kprobe: p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
361 * - Add kretprobe: r[:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
363 * $retval : fetch return value
364 * $stack : fetch stack address
365 * $stackN : fetch Nth of stack (N:0-)
366 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
367 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
368 * %REG : fetch register REG
369 * Dereferencing memory fetch:
370 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
371 * Alias name of args:
372 * NAME=FETCHARG : set NAME as alias of FETCHARG.
374 * FETCHARG:TYPE : use TYPE instead of unsigned long.
376 struct trace_probe
*tp
;
378 bool is_return
= false, is_delete
= false;
379 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
381 unsigned long offset
= 0;
383 char buf
[MAX_EVENT_NAME_LEN
];
385 /* argc must be >= 1 */
386 if (argv
[0][0] == 'p')
388 else if (argv
[0][0] == 'r')
390 else if (argv
[0][0] == '-')
393 pr_info("Probe definition must be started with 'p', 'r' or"
398 if (argv
[0][1] == ':') {
400 if (strchr(event
, '/')) {
402 event
= strchr(group
, '/') + 1;
404 if (strlen(group
) == 0) {
405 pr_info("Group name is not specified\n");
409 if (strlen(event
) == 0) {
410 pr_info("Event name is not specified\n");
415 group
= KPROBE_EVENT_SYSTEM
;
419 pr_info("Delete command needs an event name.\n");
422 mutex_lock(&probe_lock
);
423 tp
= find_trace_probe(event
, group
);
425 mutex_unlock(&probe_lock
);
426 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
429 /* delete an event */
430 ret
= unregister_trace_probe(tp
);
432 free_trace_probe(tp
);
433 mutex_unlock(&probe_lock
);
438 pr_info("Probe point is not specified.\n");
441 if (isdigit(argv
[1][0])) {
443 pr_info("Return probe point must be a symbol.\n");
446 /* an address specified */
447 ret
= kstrtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
449 pr_info("Failed to parse address.\n");
453 /* a symbol specified */
455 /* TODO: support .init module functions */
456 ret
= traceprobe_split_symbol_offset(symbol
, &offset
);
458 pr_info("Failed to parse symbol.\n");
461 if (offset
&& is_return
) {
462 pr_info("Return probe must be used without offset.\n");
466 argc
-= 2; argv
+= 2;
470 /* Make a new event name */
472 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
473 is_return
? 'r' : 'p', symbol
, offset
);
475 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
476 is_return
? 'r' : 'p', addr
);
479 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
482 pr_info("Failed to allocate trace_probe.(%d)\n",
487 /* parse arguments */
489 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
490 /* Increment count for freeing args in error case */
493 /* Parse argument name */
494 arg
= strchr(argv
[i
], '=');
497 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
500 /* If argument name is omitted, set "argN" */
501 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
502 tp
->args
[i
].name
= kstrdup(buf
, GFP_KERNEL
);
505 if (!tp
->args
[i
].name
) {
506 pr_info("Failed to allocate argument[%d] name.\n", i
);
511 if (!is_good_name(tp
->args
[i
].name
)) {
512 pr_info("Invalid argument[%d] name: %s\n",
513 i
, tp
->args
[i
].name
);
518 if (traceprobe_conflict_field_name(tp
->args
[i
].name
,
520 pr_info("Argument[%d] name '%s' conflicts with "
521 "another field.\n", i
, argv
[i
]);
526 /* Parse fetch argument */
527 ret
= traceprobe_parse_probe_arg(arg
, &tp
->size
, &tp
->args
[i
],
530 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
535 ret
= register_trace_probe(tp
);
541 free_trace_probe(tp
);
545 static int release_all_trace_probes(void)
547 struct trace_probe
*tp
;
550 mutex_lock(&probe_lock
);
551 /* Ensure no probe is in use. */
552 list_for_each_entry(tp
, &probe_list
, list
)
553 if (trace_probe_is_enabled(tp
)) {
557 /* TODO: Use batch unregistration */
558 while (!list_empty(&probe_list
)) {
559 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
560 unregister_trace_probe(tp
);
561 free_trace_probe(tp
);
565 mutex_unlock(&probe_lock
);
570 /* Probes listing interfaces */
571 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
573 mutex_lock(&probe_lock
);
574 return seq_list_start(&probe_list
, *pos
);
577 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
579 return seq_list_next(v
, &probe_list
, pos
);
582 static void probes_seq_stop(struct seq_file
*m
, void *v
)
584 mutex_unlock(&probe_lock
);
587 static int probes_seq_show(struct seq_file
*m
, void *v
)
589 struct trace_probe
*tp
= v
;
592 seq_printf(m
, "%c", trace_probe_is_return(tp
) ? 'r' : 'p');
593 seq_printf(m
, ":%s/%s", tp
->call
.class->system
, tp
->call
.name
);
596 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
597 else if (tp
->rp
.kp
.offset
)
598 seq_printf(m
, " %s+%u", trace_probe_symbol(tp
),
601 seq_printf(m
, " %s", trace_probe_symbol(tp
));
603 for (i
= 0; i
< tp
->nr_args
; i
++)
604 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, tp
->args
[i
].comm
);
610 static const struct seq_operations probes_seq_op
= {
611 .start
= probes_seq_start
,
612 .next
= probes_seq_next
,
613 .stop
= probes_seq_stop
,
614 .show
= probes_seq_show
617 static int probes_open(struct inode
*inode
, struct file
*file
)
621 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
622 ret
= release_all_trace_probes();
627 return seq_open(file
, &probes_seq_op
);
630 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
631 size_t count
, loff_t
*ppos
)
633 return traceprobe_probes_write(file
, buffer
, count
, ppos
,
637 static const struct file_operations kprobe_events_ops
= {
638 .owner
= THIS_MODULE
,
642 .release
= seq_release
,
643 .write
= probes_write
,
646 /* Probes profiling interfaces */
647 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
649 struct trace_probe
*tp
= v
;
651 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
657 static const struct seq_operations profile_seq_op
= {
658 .start
= probes_seq_start
,
659 .next
= probes_seq_next
,
660 .stop
= probes_seq_stop
,
661 .show
= probes_profile_seq_show
664 static int profile_open(struct inode
*inode
, struct file
*file
)
666 return seq_open(file
, &profile_seq_op
);
669 static const struct file_operations kprobe_profile_ops
= {
670 .owner
= THIS_MODULE
,
671 .open
= profile_open
,
674 .release
= seq_release
,
677 /* Sum up total data length for dynamic arraies (strings) */
678 static __kprobes
int __get_data_size(struct trace_probe
*tp
,
679 struct pt_regs
*regs
)
684 for (i
= 0; i
< tp
->nr_args
; i
++)
685 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
686 call_fetch(&tp
->args
[i
].fetch_size
, regs
, &len
);
693 /* Store the value of each argument */
694 static __kprobes
void store_trace_args(int ent_size
, struct trace_probe
*tp
,
695 struct pt_regs
*regs
,
696 u8
*data
, int maxlen
)
700 u32
*dl
; /* Data (relative) location */
702 for (i
= 0; i
< tp
->nr_args
; i
++) {
703 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
705 * First, we set the relative location and
706 * maximum data length to *dl
708 dl
= (u32
*)(data
+ tp
->args
[i
].offset
);
709 *dl
= make_data_rloc(maxlen
, end
- tp
->args
[i
].offset
);
710 /* Then try to fetch string or dynamic array data */
711 call_fetch(&tp
->args
[i
].fetch
, regs
, dl
);
712 /* Reduce maximum length */
713 end
+= get_rloc_len(*dl
);
714 maxlen
-= get_rloc_len(*dl
);
715 /* Trick here, convert data_rloc to data_loc */
716 *dl
= convert_rloc_to_loc(*dl
,
717 ent_size
+ tp
->args
[i
].offset
);
719 /* Just fetching data normally */
720 call_fetch(&tp
->args
[i
].fetch
, regs
,
721 data
+ tp
->args
[i
].offset
);
726 static __kprobes
void
727 kprobe_trace_func(struct trace_probe
*tp
, struct pt_regs
*regs
)
729 struct kprobe_trace_entry_head
*entry
;
730 struct ring_buffer_event
*event
;
731 struct ring_buffer
*buffer
;
733 unsigned long irq_flags
;
734 struct ftrace_event_call
*call
= &tp
->call
;
736 local_save_flags(irq_flags
);
737 pc
= preempt_count();
739 dsize
= __get_data_size(tp
, regs
);
740 size
= sizeof(*entry
) + tp
->size
+ dsize
;
742 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
743 size
, irq_flags
, pc
);
747 entry
= ring_buffer_event_data(event
);
748 entry
->ip
= (unsigned long)tp
->rp
.kp
.addr
;
749 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
751 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
752 trace_buffer_unlock_commit_regs(buffer
, event
,
753 irq_flags
, pc
, regs
);
756 /* Kretprobe handler */
757 static __kprobes
void
758 kretprobe_trace_func(struct trace_probe
*tp
, struct kretprobe_instance
*ri
,
759 struct pt_regs
*regs
)
761 struct kretprobe_trace_entry_head
*entry
;
762 struct ring_buffer_event
*event
;
763 struct ring_buffer
*buffer
;
765 unsigned long irq_flags
;
766 struct ftrace_event_call
*call
= &tp
->call
;
768 local_save_flags(irq_flags
);
769 pc
= preempt_count();
771 dsize
= __get_data_size(tp
, regs
);
772 size
= sizeof(*entry
) + tp
->size
+ dsize
;
774 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
775 size
, irq_flags
, pc
);
779 entry
= ring_buffer_event_data(event
);
780 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
781 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
782 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
784 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
785 trace_buffer_unlock_commit_regs(buffer
, event
,
786 irq_flags
, pc
, regs
);
789 /* Event entry printers */
791 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
792 struct trace_event
*event
)
794 struct kprobe_trace_entry_head
*field
;
795 struct trace_seq
*s
= &iter
->seq
;
796 struct trace_probe
*tp
;
800 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
801 tp
= container_of(event
, struct trace_probe
, call
.event
);
803 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
806 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
809 if (!trace_seq_puts(s
, ")"))
812 data
= (u8
*)&field
[1];
813 for (i
= 0; i
< tp
->nr_args
; i
++)
814 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
815 data
+ tp
->args
[i
].offset
, field
))
818 if (!trace_seq_puts(s
, "\n"))
821 return TRACE_TYPE_HANDLED
;
823 return TRACE_TYPE_PARTIAL_LINE
;
827 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
828 struct trace_event
*event
)
830 struct kretprobe_trace_entry_head
*field
;
831 struct trace_seq
*s
= &iter
->seq
;
832 struct trace_probe
*tp
;
836 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
837 tp
= container_of(event
, struct trace_probe
, call
.event
);
839 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
842 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
845 if (!trace_seq_puts(s
, " <- "))
848 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
851 if (!trace_seq_puts(s
, ")"))
854 data
= (u8
*)&field
[1];
855 for (i
= 0; i
< tp
->nr_args
; i
++)
856 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
857 data
+ tp
->args
[i
].offset
, field
))
860 if (!trace_seq_puts(s
, "\n"))
863 return TRACE_TYPE_HANDLED
;
865 return TRACE_TYPE_PARTIAL_LINE
;
869 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
872 struct kprobe_trace_entry_head field
;
873 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
875 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
876 /* Set argument names as fields */
877 for (i
= 0; i
< tp
->nr_args
; i
++) {
878 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
880 sizeof(field
) + tp
->args
[i
].offset
,
881 tp
->args
[i
].type
->size
,
882 tp
->args
[i
].type
->is_signed
,
890 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
893 struct kretprobe_trace_entry_head field
;
894 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
896 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
897 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
898 /* Set argument names as fields */
899 for (i
= 0; i
< tp
->nr_args
; i
++) {
900 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
902 sizeof(field
) + tp
->args
[i
].offset
,
903 tp
->args
[i
].type
->size
,
904 tp
->args
[i
].type
->is_signed
,
912 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
917 const char *fmt
, *arg
;
919 if (!trace_probe_is_return(tp
)) {
921 arg
= "REC->" FIELD_STRING_IP
;
923 fmt
= "(%lx <- %lx)";
924 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
927 /* When len=0, we just calculate the needed length */
928 #define LEN_OR_ZERO (len ? len - pos : 0)
930 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
932 for (i
= 0; i
< tp
->nr_args
; i
++) {
933 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%s",
934 tp
->args
[i
].name
, tp
->args
[i
].type
->fmt
);
937 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
939 for (i
= 0; i
< tp
->nr_args
; i
++) {
940 if (strcmp(tp
->args
[i
].type
->name
, "string") == 0)
941 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
,
945 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
951 /* return the length of print_fmt */
955 static int set_print_fmt(struct trace_probe
*tp
)
960 /* First: called with 0 length to calculate the needed length */
961 len
= __set_print_fmt(tp
, NULL
, 0);
962 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
966 /* Second: actually write the @print_fmt */
967 __set_print_fmt(tp
, print_fmt
, len
+ 1);
968 tp
->call
.print_fmt
= print_fmt
;
973 #ifdef CONFIG_PERF_EVENTS
975 /* Kprobe profile handler */
976 static __kprobes
void
977 kprobe_perf_func(struct trace_probe
*tp
, struct pt_regs
*regs
)
979 struct ftrace_event_call
*call
= &tp
->call
;
980 struct kprobe_trace_entry_head
*entry
;
981 struct hlist_head
*head
;
982 int size
, __size
, dsize
;
985 dsize
= __get_data_size(tp
, regs
);
986 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
987 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
989 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
990 "profile buffer not large enough"))
993 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
997 entry
->ip
= (unsigned long)tp
->rp
.kp
.addr
;
998 memset(&entry
[1], 0, dsize
);
999 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1001 head
= this_cpu_ptr(call
->perf_events
);
1002 perf_trace_buf_submit(entry
, size
, rctx
,
1003 entry
->ip
, 1, regs
, head
, NULL
);
1006 /* Kretprobe profile handler */
1007 static __kprobes
void
1008 kretprobe_perf_func(struct trace_probe
*tp
, struct kretprobe_instance
*ri
,
1009 struct pt_regs
*regs
)
1011 struct ftrace_event_call
*call
= &tp
->call
;
1012 struct kretprobe_trace_entry_head
*entry
;
1013 struct hlist_head
*head
;
1014 int size
, __size
, dsize
;
1017 dsize
= __get_data_size(tp
, regs
);
1018 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1019 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1020 size
-= sizeof(u32
);
1021 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1022 "profile buffer not large enough"))
1025 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1029 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1030 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1031 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1033 head
= this_cpu_ptr(call
->perf_events
);
1034 perf_trace_buf_submit(entry
, size
, rctx
,
1035 entry
->ret_ip
, 1, regs
, head
, NULL
);
1037 #endif /* CONFIG_PERF_EVENTS */
1040 int kprobe_register(struct ftrace_event_call
*event
,
1041 enum trace_reg type
, void *data
)
1043 struct trace_probe
*tp
= (struct trace_probe
*)event
->data
;
1046 case TRACE_REG_REGISTER
:
1047 return enable_trace_probe(tp
, TP_FLAG_TRACE
);
1048 case TRACE_REG_UNREGISTER
:
1049 disable_trace_probe(tp
, TP_FLAG_TRACE
);
1052 #ifdef CONFIG_PERF_EVENTS
1053 case TRACE_REG_PERF_REGISTER
:
1054 return enable_trace_probe(tp
, TP_FLAG_PROFILE
);
1055 case TRACE_REG_PERF_UNREGISTER
:
1056 disable_trace_probe(tp
, TP_FLAG_PROFILE
);
1058 case TRACE_REG_PERF_OPEN
:
1059 case TRACE_REG_PERF_CLOSE
:
1060 case TRACE_REG_PERF_ADD
:
1061 case TRACE_REG_PERF_DEL
:
1069 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1071 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1075 if (tp
->flags
& TP_FLAG_TRACE
)
1076 kprobe_trace_func(tp
, regs
);
1077 #ifdef CONFIG_PERF_EVENTS
1078 if (tp
->flags
& TP_FLAG_PROFILE
)
1079 kprobe_perf_func(tp
, regs
);
1081 return 0; /* We don't tweek kernel, so just return 0 */
1085 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1087 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1091 if (tp
->flags
& TP_FLAG_TRACE
)
1092 kretprobe_trace_func(tp
, ri
, regs
);
1093 #ifdef CONFIG_PERF_EVENTS
1094 if (tp
->flags
& TP_FLAG_PROFILE
)
1095 kretprobe_perf_func(tp
, ri
, regs
);
1097 return 0; /* We don't tweek kernel, so just return 0 */
1100 static struct trace_event_functions kretprobe_funcs
= {
1101 .trace
= print_kretprobe_event
1104 static struct trace_event_functions kprobe_funcs
= {
1105 .trace
= print_kprobe_event
1108 static int register_probe_event(struct trace_probe
*tp
)
1110 struct ftrace_event_call
*call
= &tp
->call
;
1113 /* Initialize ftrace_event_call */
1114 INIT_LIST_HEAD(&call
->class->fields
);
1115 if (trace_probe_is_return(tp
)) {
1116 call
->event
.funcs
= &kretprobe_funcs
;
1117 call
->class->define_fields
= kretprobe_event_define_fields
;
1119 call
->event
.funcs
= &kprobe_funcs
;
1120 call
->class->define_fields
= kprobe_event_define_fields
;
1122 if (set_print_fmt(tp
) < 0)
1124 ret
= register_ftrace_event(&call
->event
);
1126 kfree(call
->print_fmt
);
1130 call
->class->reg
= kprobe_register
;
1132 ret
= trace_add_event_call(call
);
1134 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1135 kfree(call
->print_fmt
);
1136 unregister_ftrace_event(&call
->event
);
1141 static void unregister_probe_event(struct trace_probe
*tp
)
1143 /* tp->event is unregistered in trace_remove_event_call() */
1144 trace_remove_event_call(&tp
->call
);
1145 kfree(tp
->call
.print_fmt
);
1148 /* Make a debugfs interface for controlling probe points */
1149 static __init
int init_kprobe_trace(void)
1151 struct dentry
*d_tracer
;
1152 struct dentry
*entry
;
1154 if (register_module_notifier(&trace_probe_module_nb
))
1157 d_tracer
= tracing_init_dentry();
1161 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1162 NULL
, &kprobe_events_ops
);
1164 /* Event list interface */
1166 pr_warning("Could not create debugfs "
1167 "'kprobe_events' entry\n");
1169 /* Profile interface */
1170 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1171 NULL
, &kprobe_profile_ops
);
1174 pr_warning("Could not create debugfs "
1175 "'kprobe_profile' entry\n");
1178 fs_initcall(init_kprobe_trace
);
1181 #ifdef CONFIG_FTRACE_STARTUP_TEST
1184 * The "__used" keeps gcc from removing the function symbol
1185 * from the kallsyms table.
1187 static __used
int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1188 int a4
, int a5
, int a6
)
1190 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1193 static __init
int kprobe_trace_self_tests_init(void)
1196 int (*target
)(int, int, int, int, int, int);
1197 struct trace_probe
*tp
;
1199 target
= kprobe_trace_selftest_target
;
1201 pr_info("Testing kprobe tracing: ");
1203 ret
= traceprobe_command("p:testprobe kprobe_trace_selftest_target "
1204 "$stack $stack0 +0($stack)",
1205 create_trace_probe
);
1206 if (WARN_ON_ONCE(ret
)) {
1207 pr_warning("error on probing function entry.\n");
1210 /* Enable trace point */
1211 tp
= find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM
);
1212 if (WARN_ON_ONCE(tp
== NULL
)) {
1213 pr_warning("error on getting new probe.\n");
1216 enable_trace_probe(tp
, TP_FLAG_TRACE
);
1219 ret
= traceprobe_command("r:testprobe2 kprobe_trace_selftest_target "
1220 "$retval", create_trace_probe
);
1221 if (WARN_ON_ONCE(ret
)) {
1222 pr_warning("error on probing function return.\n");
1225 /* Enable trace point */
1226 tp
= find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM
);
1227 if (WARN_ON_ONCE(tp
== NULL
)) {
1228 pr_warning("error on getting new probe.\n");
1231 enable_trace_probe(tp
, TP_FLAG_TRACE
);
1237 ret
= target(1, 2, 3, 4, 5, 6);
1239 /* Disable trace points before removing it */
1240 tp
= find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM
);
1241 if (WARN_ON_ONCE(tp
== NULL
)) {
1242 pr_warning("error on getting test probe.\n");
1245 disable_trace_probe(tp
, TP_FLAG_TRACE
);
1247 tp
= find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM
);
1248 if (WARN_ON_ONCE(tp
== NULL
)) {
1249 pr_warning("error on getting 2nd test probe.\n");
1252 disable_trace_probe(tp
, TP_FLAG_TRACE
);
1254 ret
= traceprobe_command("-:testprobe", create_trace_probe
);
1255 if (WARN_ON_ONCE(ret
)) {
1256 pr_warning("error on deleting a probe.\n");
1260 ret
= traceprobe_command("-:testprobe2", create_trace_probe
);
1261 if (WARN_ON_ONCE(ret
)) {
1262 pr_warning("error on deleting a probe.\n");
1267 release_all_trace_probes();
1269 pr_cont("NG: Some tests are failed. Please check them.\n");
1275 late_initcall(kprobe_trace_self_tests_init
);