1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) 2008-2014 Mathieu Desnoyers
5 #include <linux/module.h>
6 #include <linux/mutex.h>
7 #include <linux/types.h>
8 #include <linux/jhash.h>
9 #include <linux/list.h>
10 #include <linux/rcupdate.h>
11 #include <linux/tracepoint.h>
12 #include <linux/err.h>
13 #include <linux/slab.h>
14 #include <linux/sched/signal.h>
15 #include <linux/sched/task.h>
16 #include <linux/static_key.h>
18 extern tracepoint_ptr_t __start___tracepoints_ptrs
[];
19 extern tracepoint_ptr_t __stop___tracepoints_ptrs
[];
21 DEFINE_SRCU(tracepoint_srcu
);
22 EXPORT_SYMBOL_GPL(tracepoint_srcu
);
24 /* Set to 1 to enable tracepoint debug output */
25 static const int tracepoint_debug
;
29 * Tracepoint module list mutex protects the local module list.
31 static DEFINE_MUTEX(tracepoint_module_list_mutex
);
33 /* Local list of struct tp_module */
34 static LIST_HEAD(tracepoint_module_list
);
35 #endif /* CONFIG_MODULES */
38 * tracepoints_mutex protects the builtin and module tracepoints.
39 * tracepoints_mutex nests inside tracepoint_module_list_mutex.
41 static DEFINE_MUTEX(tracepoints_mutex
);
43 static struct rcu_head
*early_probes
;
44 static bool ok_to_free_tracepoints
;
48 * It is used to delay the free of multiple probes array until a quiescent
53 struct tracepoint_func probes
[];
56 /* Called in removal of a func but failed to allocate a new tp_funcs */
57 static void tp_stub_func(void)
62 static inline void *allocate_probes(int count
)
64 struct tp_probes
*p
= kmalloc(struct_size(p
, probes
, count
),
66 return p
== NULL
? NULL
: p
->probes
;
69 static void srcu_free_old_probes(struct rcu_head
*head
)
71 kfree(container_of(head
, struct tp_probes
, rcu
));
74 static void rcu_free_old_probes(struct rcu_head
*head
)
76 call_srcu(&tracepoint_srcu
, head
, srcu_free_old_probes
);
79 static __init
int release_early_probes(void)
83 ok_to_free_tracepoints
= true;
85 while (early_probes
) {
87 early_probes
= tmp
->next
;
88 call_rcu(tmp
, rcu_free_old_probes
);
94 /* SRCU is initialized at core_initcall */
95 postcore_initcall(release_early_probes
);
97 static inline void release_probes(struct tracepoint_func
*old
)
100 struct tp_probes
*tp_probes
= container_of(old
,
101 struct tp_probes
, probes
[0]);
104 * We can't free probes if SRCU is not initialized yet.
105 * Postpone the freeing till after SRCU is initialized.
107 if (unlikely(!ok_to_free_tracepoints
)) {
108 tp_probes
->rcu
.next
= early_probes
;
109 early_probes
= &tp_probes
->rcu
;
114 * Tracepoint probes are protected by both sched RCU and SRCU,
115 * by calling the SRCU callback in the sched RCU callback we
116 * cover both cases. So let us chain the SRCU and sched RCU
117 * callbacks to wait for both grace periods.
119 call_rcu(&tp_probes
->rcu
, rcu_free_old_probes
);
123 static void debug_print_probes(struct tracepoint_func
*funcs
)
127 if (!tracepoint_debug
|| !funcs
)
130 for (i
= 0; funcs
[i
].func
; i
++)
131 printk(KERN_DEBUG
"Probe %d : %p\n", i
, funcs
[i
].func
);
134 static struct tracepoint_func
*
135 func_add(struct tracepoint_func
**funcs
, struct tracepoint_func
*tp_func
,
138 struct tracepoint_func
*old
, *new;
139 int iter_probes
; /* Iterate over old probe array. */
140 int nr_probes
= 0; /* Counter for probes */
141 int pos
= -1; /* Insertion position into new array */
143 if (WARN_ON(!tp_func
->func
))
144 return ERR_PTR(-EINVAL
);
146 debug_print_probes(*funcs
);
149 /* (N -> N+1), (N != 0, 1) probes */
150 for (iter_probes
= 0; old
[iter_probes
].func
; iter_probes
++) {
151 if (old
[iter_probes
].func
== tp_stub_func
)
152 continue; /* Skip stub functions. */
153 if (old
[iter_probes
].func
== tp_func
->func
&&
154 old
[iter_probes
].data
== tp_func
->data
)
155 return ERR_PTR(-EEXIST
);
159 /* + 2 : one for new probe, one for NULL func */
160 new = allocate_probes(nr_probes
+ 2);
162 return ERR_PTR(-ENOMEM
);
165 for (iter_probes
= 0; old
[iter_probes
].func
; iter_probes
++) {
166 if (old
[iter_probes
].func
== tp_stub_func
)
168 /* Insert before probes of lower priority */
169 if (pos
< 0 && old
[iter_probes
].prio
< prio
)
171 new[nr_probes
++] = old
[iter_probes
];
175 /* nr_probes now points to the end of the new array */
178 nr_probes
= 1; /* must point at end of array */
181 new[nr_probes
].func
= NULL
;
183 debug_print_probes(*funcs
);
187 static void *func_remove(struct tracepoint_func
**funcs
,
188 struct tracepoint_func
*tp_func
)
190 int nr_probes
= 0, nr_del
= 0, i
;
191 struct tracepoint_func
*old
, *new;
196 return ERR_PTR(-ENOENT
);
198 debug_print_probes(*funcs
);
199 /* (N -> M), (N > 1, M >= 0) probes */
201 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
202 if ((old
[nr_probes
].func
== tp_func
->func
&&
203 old
[nr_probes
].data
== tp_func
->data
) ||
204 old
[nr_probes
].func
== tp_stub_func
)
210 * If probe is NULL, then nr_probes = nr_del = 0, and then the
211 * entire entry will be removed.
213 if (nr_probes
- nr_del
== 0) {
214 /* N -> 0, (N > 1) */
216 debug_print_probes(*funcs
);
220 /* N -> M, (N > 1, M > 0) */
222 new = allocate_probes(nr_probes
- nr_del
+ 1);
224 for (i
= 0; old
[i
].func
; i
++) {
225 if ((old
[i
].func
!= tp_func
->func
||
226 old
[i
].data
!= tp_func
->data
) &&
227 old
[i
].func
!= tp_stub_func
)
230 new[nr_probes
- nr_del
].func
= NULL
;
234 * Failed to allocate, replace the old function
235 * with calls to tp_stub_func.
237 for (i
= 0; old
[i
].func
; i
++) {
238 if (old
[i
].func
== tp_func
->func
&&
239 old
[i
].data
== tp_func
->data
)
240 WRITE_ONCE(old
[i
].func
, tp_stub_func
);
245 debug_print_probes(*funcs
);
249 static void tracepoint_update_call(struct tracepoint
*tp
, struct tracepoint_func
*tp_funcs
, bool sync
)
251 void *func
= tp
->iterator
;
253 /* Synthetic events do not have static call sites */
254 if (!tp
->static_call_key
)
257 if (!tp_funcs
[1].func
) {
258 func
= tp_funcs
[0].func
;
260 * If going from the iterator back to a single caller,
261 * we need to synchronize with __DO_TRACE to make sure
262 * that the data passed to the callback is the one that
263 * belongs to that callback.
266 tracepoint_synchronize_unregister();
269 __static_call_update(tp
->static_call_key
, tp
->static_call_tramp
, func
);
273 * Add the probe function to a tracepoint.
275 static int tracepoint_add_func(struct tracepoint
*tp
,
276 struct tracepoint_func
*func
, int prio
)
278 struct tracepoint_func
*old
, *tp_funcs
;
281 if (tp
->regfunc
&& !static_key_enabled(&tp
->key
)) {
287 tp_funcs
= rcu_dereference_protected(tp
->funcs
,
288 lockdep_is_held(&tracepoints_mutex
));
289 old
= func_add(&tp_funcs
, func
, prio
);
291 WARN_ON_ONCE(PTR_ERR(old
) != -ENOMEM
);
296 * rcu_assign_pointer has as smp_store_release() which makes sure
297 * that the new probe callbacks array is consistent before setting
298 * a pointer to it. This array is referenced by __DO_TRACE from
299 * include/linux/tracepoint.h using rcu_dereference_sched().
301 rcu_assign_pointer(tp
->funcs
, tp_funcs
);
302 tracepoint_update_call(tp
, tp_funcs
, false);
303 static_key_enable(&tp
->key
);
310 * Remove a probe function from a tracepoint.
311 * Note: only waiting an RCU period after setting elem->call to the empty
312 * function insures that the original callback is not used anymore. This insured
313 * by preempt_disable around the call site.
315 static int tracepoint_remove_func(struct tracepoint
*tp
,
316 struct tracepoint_func
*func
)
318 struct tracepoint_func
*old
, *tp_funcs
;
320 tp_funcs
= rcu_dereference_protected(tp
->funcs
,
321 lockdep_is_held(&tracepoints_mutex
));
322 old
= func_remove(&tp_funcs
, func
);
323 if (WARN_ON_ONCE(IS_ERR(old
)))
327 /* Failed allocating new tp_funcs, replaced func with stub */
331 /* Removed last function */
332 if (tp
->unregfunc
&& static_key_enabled(&tp
->key
))
335 static_key_disable(&tp
->key
);
336 rcu_assign_pointer(tp
->funcs
, tp_funcs
);
338 rcu_assign_pointer(tp
->funcs
, tp_funcs
);
339 tracepoint_update_call(tp
, tp_funcs
,
340 tp_funcs
[0].func
!= old
[0].func
);
347 * tracepoint_probe_register_prio - Connect a probe to a tracepoint with priority
349 * @probe: probe handler
350 * @data: tracepoint data
351 * @prio: priority of this function over other registered functions
353 * Returns 0 if ok, error value on error.
354 * Note: if @tp is within a module, the caller is responsible for
355 * unregistering the probe before the module is gone. This can be
356 * performed either with a tracepoint module going notifier, or from
357 * within module exit functions.
359 int tracepoint_probe_register_prio(struct tracepoint
*tp
, void *probe
,
360 void *data
, int prio
)
362 struct tracepoint_func tp_func
;
365 mutex_lock(&tracepoints_mutex
);
366 tp_func
.func
= probe
;
369 ret
= tracepoint_add_func(tp
, &tp_func
, prio
);
370 mutex_unlock(&tracepoints_mutex
);
373 EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio
);
376 * tracepoint_probe_register - Connect a probe to a tracepoint
378 * @probe: probe handler
379 * @data: tracepoint data
381 * Returns 0 if ok, error value on error.
382 * Note: if @tp is within a module, the caller is responsible for
383 * unregistering the probe before the module is gone. This can be
384 * performed either with a tracepoint module going notifier, or from
385 * within module exit functions.
387 int tracepoint_probe_register(struct tracepoint
*tp
, void *probe
, void *data
)
389 return tracepoint_probe_register_prio(tp
, probe
, data
, TRACEPOINT_DEFAULT_PRIO
);
391 EXPORT_SYMBOL_GPL(tracepoint_probe_register
);
394 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint
396 * @probe: probe function pointer
397 * @data: tracepoint data
399 * Returns 0 if ok, error value on error.
401 int tracepoint_probe_unregister(struct tracepoint
*tp
, void *probe
, void *data
)
403 struct tracepoint_func tp_func
;
406 mutex_lock(&tracepoints_mutex
);
407 tp_func
.func
= probe
;
409 ret
= tracepoint_remove_func(tp
, &tp_func
);
410 mutex_unlock(&tracepoints_mutex
);
413 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister
);
415 static void for_each_tracepoint_range(
416 tracepoint_ptr_t
*begin
, tracepoint_ptr_t
*end
,
417 void (*fct
)(struct tracepoint
*tp
, void *priv
),
420 tracepoint_ptr_t
*iter
;
424 for (iter
= begin
; iter
< end
; iter
++)
425 fct(tracepoint_ptr_deref(iter
), priv
);
428 #ifdef CONFIG_MODULES
429 bool trace_module_has_bad_taint(struct module
*mod
)
431 return mod
->taints
& ~((1 << TAINT_OOT_MODULE
) | (1 << TAINT_CRAP
) |
432 (1 << TAINT_UNSIGNED_MODULE
));
435 static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list
);
438 * register_tracepoint_notifier - register tracepoint coming/going notifier
439 * @nb: notifier block
441 * Notifiers registered with this function are called on module
442 * coming/going with the tracepoint_module_list_mutex held.
443 * The notifier block callback should expect a "struct tp_module" data
446 int register_tracepoint_module_notifier(struct notifier_block
*nb
)
448 struct tp_module
*tp_mod
;
451 mutex_lock(&tracepoint_module_list_mutex
);
452 ret
= blocking_notifier_chain_register(&tracepoint_notify_list
, nb
);
455 list_for_each_entry(tp_mod
, &tracepoint_module_list
, list
)
456 (void) nb
->notifier_call(nb
, MODULE_STATE_COMING
, tp_mod
);
458 mutex_unlock(&tracepoint_module_list_mutex
);
461 EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier
);
464 * unregister_tracepoint_notifier - unregister tracepoint coming/going notifier
465 * @nb: notifier block
467 * The notifier block callback should expect a "struct tp_module" data
470 int unregister_tracepoint_module_notifier(struct notifier_block
*nb
)
472 struct tp_module
*tp_mod
;
475 mutex_lock(&tracepoint_module_list_mutex
);
476 ret
= blocking_notifier_chain_unregister(&tracepoint_notify_list
, nb
);
479 list_for_each_entry(tp_mod
, &tracepoint_module_list
, list
)
480 (void) nb
->notifier_call(nb
, MODULE_STATE_GOING
, tp_mod
);
482 mutex_unlock(&tracepoint_module_list_mutex
);
486 EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier
);
489 * Ensure the tracer unregistered the module's probes before the module
490 * teardown is performed. Prevents leaks of probe and data pointers.
492 static void tp_module_going_check_quiescent(struct tracepoint
*tp
, void *priv
)
494 WARN_ON_ONCE(tp
->funcs
);
497 static int tracepoint_module_coming(struct module
*mod
)
499 struct tp_module
*tp_mod
;
502 if (!mod
->num_tracepoints
)
506 * We skip modules that taint the kernel, especially those with different
507 * module headers (for forced load), to make sure we don't cause a crash.
508 * Staging, out-of-tree, and unsigned GPL modules are fine.
510 if (trace_module_has_bad_taint(mod
))
512 mutex_lock(&tracepoint_module_list_mutex
);
513 tp_mod
= kmalloc(sizeof(struct tp_module
), GFP_KERNEL
);
519 list_add_tail(&tp_mod
->list
, &tracepoint_module_list
);
520 blocking_notifier_call_chain(&tracepoint_notify_list
,
521 MODULE_STATE_COMING
, tp_mod
);
523 mutex_unlock(&tracepoint_module_list_mutex
);
527 static void tracepoint_module_going(struct module
*mod
)
529 struct tp_module
*tp_mod
;
531 if (!mod
->num_tracepoints
)
534 mutex_lock(&tracepoint_module_list_mutex
);
535 list_for_each_entry(tp_mod
, &tracepoint_module_list
, list
) {
536 if (tp_mod
->mod
== mod
) {
537 blocking_notifier_call_chain(&tracepoint_notify_list
,
538 MODULE_STATE_GOING
, tp_mod
);
539 list_del(&tp_mod
->list
);
542 * Called the going notifier before checking for
545 for_each_tracepoint_range(mod
->tracepoints_ptrs
,
546 mod
->tracepoints_ptrs
+ mod
->num_tracepoints
,
547 tp_module_going_check_quiescent
, NULL
);
552 * In the case of modules that were tainted at "coming", we'll simply
553 * walk through the list without finding it. We cannot use the "tainted"
554 * flag on "going", in case a module taints the kernel only after being
557 mutex_unlock(&tracepoint_module_list_mutex
);
560 static int tracepoint_module_notify(struct notifier_block
*self
,
561 unsigned long val
, void *data
)
563 struct module
*mod
= data
;
567 case MODULE_STATE_COMING
:
568 ret
= tracepoint_module_coming(mod
);
570 case MODULE_STATE_LIVE
:
572 case MODULE_STATE_GOING
:
573 tracepoint_module_going(mod
);
575 case MODULE_STATE_UNFORMED
:
578 return notifier_from_errno(ret
);
581 static struct notifier_block tracepoint_module_nb
= {
582 .notifier_call
= tracepoint_module_notify
,
586 static __init
int init_tracepoints(void)
590 ret
= register_module_notifier(&tracepoint_module_nb
);
592 pr_warn("Failed to register tracepoint module enter notifier\n");
596 __initcall(init_tracepoints
);
597 #endif /* CONFIG_MODULES */
600 * for_each_kernel_tracepoint - iteration on all kernel tracepoints
602 * @priv: private data
604 void for_each_kernel_tracepoint(void (*fct
)(struct tracepoint
*tp
, void *priv
),
607 for_each_tracepoint_range(__start___tracepoints_ptrs
,
608 __stop___tracepoints_ptrs
, fct
, priv
);
610 EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint
);
612 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
614 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
615 static int sys_tracepoint_refcount
;
617 int syscall_regfunc(void)
619 struct task_struct
*p
, *t
;
621 if (!sys_tracepoint_refcount
) {
622 read_lock(&tasklist_lock
);
623 for_each_process_thread(p
, t
) {
624 set_task_syscall_work(t
, SYSCALL_TRACEPOINT
);
626 read_unlock(&tasklist_lock
);
628 sys_tracepoint_refcount
++;
633 void syscall_unregfunc(void)
635 struct task_struct
*p
, *t
;
637 sys_tracepoint_refcount
--;
638 if (!sys_tracepoint_refcount
) {
639 read_lock(&tasklist_lock
);
640 for_each_process_thread(p
, t
) {
641 clear_task_syscall_work(t
, SYSCALL_TRACEPOINT
);
643 read_unlock(&tasklist_lock
);