]>
Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
97e1c18e | 2 | /* |
de7b2973 | 3 | * Copyright (C) 2008-2014 Mathieu Desnoyers |
97e1c18e MD |
4 | */ |
5 | #include <linux/module.h> | |
6 | #include <linux/mutex.h> | |
7 | #include <linux/types.h> | |
8 | #include <linux/jhash.h> | |
9 | #include <linux/list.h> | |
10 | #include <linux/rcupdate.h> | |
11 | #include <linux/tracepoint.h> | |
12 | #include <linux/err.h> | |
13 | #include <linux/slab.h> | |
3f07c014 | 14 | #include <linux/sched/signal.h> |
29930025 | 15 | #include <linux/sched/task.h> |
c5905afb | 16 | #include <linux/static_key.h> |
97e1c18e | 17 | |
9c0be3f6 MD |
18 | extern tracepoint_ptr_t __start___tracepoints_ptrs[]; |
19 | extern tracepoint_ptr_t __stop___tracepoints_ptrs[]; | |
97e1c18e | 20 | |
e6753f23 JFG |
21 | DEFINE_SRCU(tracepoint_srcu); |
22 | EXPORT_SYMBOL_GPL(tracepoint_srcu); | |
23 | ||
97e1c18e MD |
24 | /* Set to 1 to enable tracepoint debug output */ |
25 | static const int tracepoint_debug; | |
26 | ||
de7b2973 | 27 | #ifdef CONFIG_MODULES |
97e1c18e | 28 | /* |
de7b2973 | 29 | * Tracepoint module list mutex protects the local module list. |
97e1c18e | 30 | */ |
de7b2973 | 31 | static DEFINE_MUTEX(tracepoint_module_list_mutex); |
97e1c18e | 32 | |
de7b2973 | 33 | /* Local list of struct tp_module */ |
b75ef8b4 MD |
34 | static LIST_HEAD(tracepoint_module_list); |
35 | #endif /* CONFIG_MODULES */ | |
36 | ||
97e1c18e | 37 | /* |
de7b2973 MD |
38 | * tracepoints_mutex protects the builtin and module tracepoints. |
39 | * tracepoints_mutex nests inside tracepoint_module_list_mutex. | |
97e1c18e | 40 | */ |
de7b2973 | 41 | static DEFINE_MUTEX(tracepoints_mutex); |
97e1c18e | 42 | |
f8a79d5c SRV |
43 | static struct rcu_head *early_probes; |
44 | static bool ok_to_free_tracepoints; | |
45 | ||
97e1c18e MD |
46 | /* |
47 | * Note about RCU : | |
fd589a8f | 48 | * It is used to delay the free of multiple probes array until a quiescent |
97e1c18e | 49 | * state is reached. |
97e1c18e | 50 | */ |
19dba33c | 51 | struct tp_probes { |
0dea6d52 | 52 | struct rcu_head rcu; |
9d0a49c7 | 53 | struct tracepoint_func probes[]; |
19dba33c | 54 | }; |
97e1c18e | 55 | |
befe6d94 SRV |
56 | /* Called in removal of a func but failed to allocate a new tp_funcs */ |
57 | static void tp_stub_func(void) | |
58 | { | |
59 | return; | |
60 | } | |
61 | ||
19dba33c | 62 | static inline void *allocate_probes(int count) |
97e1c18e | 63 | { |
f0553dcb GS |
64 | struct tp_probes *p = kmalloc(struct_size(p, probes, count), |
65 | GFP_KERNEL); | |
19dba33c | 66 | return p == NULL ? NULL : p->probes; |
97e1c18e MD |
67 | } |
68 | ||
e6753f23 | 69 | static void srcu_free_old_probes(struct rcu_head *head) |
97e1c18e | 70 | { |
0dea6d52 | 71 | kfree(container_of(head, struct tp_probes, rcu)); |
19dba33c LJ |
72 | } |
73 | ||
e6753f23 JFG |
74 | static void rcu_free_old_probes(struct rcu_head *head) |
75 | { | |
76 | call_srcu(&tracepoint_srcu, head, srcu_free_old_probes); | |
77 | } | |
78 | ||
f8a79d5c SRV |
79 | static __init int release_early_probes(void) |
80 | { | |
81 | struct rcu_head *tmp; | |
82 | ||
83 | ok_to_free_tracepoints = true; | |
84 | ||
85 | while (early_probes) { | |
86 | tmp = early_probes; | |
87 | early_probes = tmp->next; | |
74401729 | 88 | call_rcu(tmp, rcu_free_old_probes); |
f8a79d5c SRV |
89 | } |
90 | ||
91 | return 0; | |
92 | } | |
93 | ||
94 | /* SRCU is initialized at core_initcall */ | |
95 | postcore_initcall(release_early_probes); | |
96 | ||
38516ab5 | 97 | static inline void release_probes(struct tracepoint_func *old) |
19dba33c LJ |
98 | { |
99 | if (old) { | |
100 | struct tp_probes *tp_probes = container_of(old, | |
101 | struct tp_probes, probes[0]); | |
f8a79d5c SRV |
102 | |
103 | /* | |
104 | * We can't free probes if SRCU is not initialized yet. | |
105 | * Postpone the freeing till after SRCU is initialized. | |
106 | */ | |
107 | if (unlikely(!ok_to_free_tracepoints)) { | |
108 | tp_probes->rcu.next = early_probes; | |
109 | early_probes = &tp_probes->rcu; | |
110 | return; | |
111 | } | |
112 | ||
e6753f23 JFG |
113 | /* |
114 | * Tracepoint probes are protected by both sched RCU and SRCU, | |
115 | * by calling the SRCU callback in the sched RCU callback we | |
116 | * cover both cases. So let us chain the SRCU and sched RCU | |
117 | * callbacks to wait for both grace periods. | |
118 | */ | |
74401729 | 119 | call_rcu(&tp_probes->rcu, rcu_free_old_probes); |
19dba33c | 120 | } |
97e1c18e MD |
121 | } |
122 | ||
de7b2973 | 123 | static void debug_print_probes(struct tracepoint_func *funcs) |
97e1c18e MD |
124 | { |
125 | int i; | |
126 | ||
de7b2973 | 127 | if (!tracepoint_debug || !funcs) |
97e1c18e MD |
128 | return; |
129 | ||
de7b2973 MD |
130 | for (i = 0; funcs[i].func; i++) |
131 | printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func); | |
97e1c18e MD |
132 | } |
133 | ||
7904b5c4 SRRH |
134 | static struct tracepoint_func * |
135 | func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func, | |
136 | int prio) | |
97e1c18e | 137 | { |
38516ab5 | 138 | struct tracepoint_func *old, *new; |
7211f0a2 SRV |
139 | int iter_probes; /* Iterate over old probe array. */ |
140 | int nr_probes = 0; /* Counter for probes */ | |
141 | int pos = -1; /* Insertion position into new array */ | |
97e1c18e | 142 | |
de7b2973 | 143 | if (WARN_ON(!tp_func->func)) |
4c69e6ea | 144 | return ERR_PTR(-EINVAL); |
97e1c18e | 145 | |
de7b2973 MD |
146 | debug_print_probes(*funcs); |
147 | old = *funcs; | |
97e1c18e MD |
148 | if (old) { |
149 | /* (N -> N+1), (N != 0, 1) probes */ | |
7211f0a2 SRV |
150 | for (iter_probes = 0; old[iter_probes].func; iter_probes++) { |
151 | if (old[iter_probes].func == tp_stub_func) | |
152 | continue; /* Skip stub functions. */ | |
153 | if (old[iter_probes].func == tp_func->func && | |
154 | old[iter_probes].data == tp_func->data) | |
97e1c18e | 155 | return ERR_PTR(-EEXIST); |
7211f0a2 | 156 | nr_probes++; |
7904b5c4 | 157 | } |
97e1c18e | 158 | } |
7211f0a2 SRV |
159 | /* + 2 : one for new probe, one for NULL func */ |
160 | new = allocate_probes(nr_probes + 2); | |
97e1c18e MD |
161 | if (new == NULL) |
162 | return ERR_PTR(-ENOMEM); | |
7904b5c4 | 163 | if (old) { |
7211f0a2 SRV |
164 | nr_probes = 0; |
165 | for (iter_probes = 0; old[iter_probes].func; iter_probes++) { | |
166 | if (old[iter_probes].func == tp_stub_func) | |
167 | continue; | |
168 | /* Insert before probes of lower priority */ | |
169 | if (pos < 0 && old[iter_probes].prio < prio) | |
170 | pos = nr_probes++; | |
171 | new[nr_probes++] = old[iter_probes]; | |
7904b5c4 | 172 | } |
7211f0a2 SRV |
173 | if (pos < 0) |
174 | pos = nr_probes++; | |
175 | /* nr_probes now points to the end of the new array */ | |
176 | } else { | |
7904b5c4 | 177 | pos = 0; |
7211f0a2 SRV |
178 | nr_probes = 1; /* must point at end of array */ |
179 | } | |
7904b5c4 | 180 | new[pos] = *tp_func; |
7211f0a2 | 181 | new[nr_probes].func = NULL; |
de7b2973 MD |
182 | *funcs = new; |
183 | debug_print_probes(*funcs); | |
97e1c18e MD |
184 | return old; |
185 | } | |
186 | ||
de7b2973 MD |
187 | static void *func_remove(struct tracepoint_func **funcs, |
188 | struct tracepoint_func *tp_func) | |
97e1c18e MD |
189 | { |
190 | int nr_probes = 0, nr_del = 0, i; | |
38516ab5 | 191 | struct tracepoint_func *old, *new; |
97e1c18e | 192 | |
de7b2973 | 193 | old = *funcs; |
97e1c18e | 194 | |
f66af459 | 195 | if (!old) |
19dba33c | 196 | return ERR_PTR(-ENOENT); |
f66af459 | 197 | |
de7b2973 | 198 | debug_print_probes(*funcs); |
97e1c18e | 199 | /* (N -> M), (N > 1, M >= 0) probes */ |
de7b2973 | 200 | if (tp_func->func) { |
4c69e6ea | 201 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { |
befe6d94 SRV |
202 | if ((old[nr_probes].func == tp_func->func && |
203 | old[nr_probes].data == tp_func->data) || | |
204 | old[nr_probes].func == tp_stub_func) | |
4c69e6ea S |
205 | nr_del++; |
206 | } | |
97e1c18e MD |
207 | } |
208 | ||
4c69e6ea S |
209 | /* |
210 | * If probe is NULL, then nr_probes = nr_del = 0, and then the | |
211 | * entire entry will be removed. | |
212 | */ | |
97e1c18e MD |
213 | if (nr_probes - nr_del == 0) { |
214 | /* N -> 0, (N > 1) */ | |
de7b2973 MD |
215 | *funcs = NULL; |
216 | debug_print_probes(*funcs); | |
97e1c18e MD |
217 | return old; |
218 | } else { | |
219 | int j = 0; | |
220 | /* N -> M, (N > 1, M > 0) */ | |
221 | /* + 1 for NULL */ | |
19dba33c | 222 | new = allocate_probes(nr_probes - nr_del + 1); |
befe6d94 | 223 | if (new) { |
7211f0a2 SRV |
224 | for (i = 0; old[i].func; i++) { |
225 | if ((old[i].func != tp_func->func || | |
226 | old[i].data != tp_func->data) && | |
227 | old[i].func != tp_stub_func) | |
befe6d94 | 228 | new[j++] = old[i]; |
7211f0a2 | 229 | } |
befe6d94 SRV |
230 | new[nr_probes - nr_del].func = NULL; |
231 | *funcs = new; | |
232 | } else { | |
233 | /* | |
234 | * Failed to allocate, replace the old function | |
235 | * with calls to tp_stub_func. | |
236 | */ | |
7211f0a2 | 237 | for (i = 0; old[i].func; i++) { |
befe6d94 | 238 | if (old[i].func == tp_func->func && |
7211f0a2 SRV |
239 | old[i].data == tp_func->data) |
240 | WRITE_ONCE(old[i].func, tp_stub_func); | |
241 | } | |
befe6d94 SRV |
242 | *funcs = old; |
243 | } | |
97e1c18e | 244 | } |
de7b2973 | 245 | debug_print_probes(*funcs); |
97e1c18e MD |
246 | return old; |
247 | } | |
248 | ||
547305a6 | 249 | static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs, bool sync) |
d25e37d8 SRV |
250 | { |
251 | void *func = tp->iterator; | |
252 | ||
253 | /* Synthetic events do not have static call sites */ | |
254 | if (!tp->static_call_key) | |
255 | return; | |
256 | ||
547305a6 | 257 | if (!tp_funcs[1].func) { |
d25e37d8 | 258 | func = tp_funcs[0].func; |
547305a6 SRV |
259 | /* |
260 | * If going from the iterator back to a single caller, | |
261 | * we need to synchronize with __DO_TRACE to make sure | |
262 | * that the data passed to the callback is the one that | |
263 | * belongs to that callback. | |
264 | */ | |
265 | if (sync) | |
266 | tracepoint_synchronize_unregister(); | |
267 | } | |
d25e37d8 SRV |
268 | |
269 | __static_call_update(tp->static_call_key, tp->static_call_tramp, func); | |
270 | } | |
271 | ||
97e1c18e | 272 | /* |
de7b2973 | 273 | * Add the probe function to a tracepoint. |
97e1c18e | 274 | */ |
de7b2973 | 275 | static int tracepoint_add_func(struct tracepoint *tp, |
7904b5c4 | 276 | struct tracepoint_func *func, int prio) |
97e1c18e | 277 | { |
de7b2973 | 278 | struct tracepoint_func *old, *tp_funcs; |
8cf868af | 279 | int ret; |
97e1c18e | 280 | |
8cf868af SRRH |
281 | if (tp->regfunc && !static_key_enabled(&tp->key)) { |
282 | ret = tp->regfunc(); | |
283 | if (ret < 0) | |
284 | return ret; | |
285 | } | |
97e1c18e | 286 | |
b725dfea MD |
287 | tp_funcs = rcu_dereference_protected(tp->funcs, |
288 | lockdep_is_held(&tracepoints_mutex)); | |
7904b5c4 | 289 | old = func_add(&tp_funcs, func, prio); |
de7b2973 | 290 | if (IS_ERR(old)) { |
d66a270b | 291 | WARN_ON_ONCE(PTR_ERR(old) != -ENOMEM); |
de7b2973 MD |
292 | return PTR_ERR(old); |
293 | } | |
97419875 | 294 | |
97e1c18e | 295 | /* |
243d1a79 PM |
296 | * rcu_assign_pointer has as smp_store_release() which makes sure |
297 | * that the new probe callbacks array is consistent before setting | |
298 | * a pointer to it. This array is referenced by __DO_TRACE from | |
299 | * include/linux/tracepoint.h using rcu_dereference_sched(). | |
97e1c18e | 300 | */ |
de7b2973 | 301 | rcu_assign_pointer(tp->funcs, tp_funcs); |
547305a6 | 302 | tracepoint_update_call(tp, tp_funcs, false); |
d25e37d8 SRV |
303 | static_key_enable(&tp->key); |
304 | ||
8058bd0f | 305 | release_probes(old); |
de7b2973 | 306 | return 0; |
97e1c18e MD |
307 | } |
308 | ||
309 | /* | |
de7b2973 | 310 | * Remove a probe function from a tracepoint. |
97e1c18e MD |
311 | * Note: only waiting an RCU period after setting elem->call to the empty |
312 | * function insures that the original callback is not used anymore. This insured | |
313 | * by preempt_disable around the call site. | |
314 | */ | |
de7b2973 MD |
315 | static int tracepoint_remove_func(struct tracepoint *tp, |
316 | struct tracepoint_func *func) | |
97e1c18e | 317 | { |
de7b2973 | 318 | struct tracepoint_func *old, *tp_funcs; |
97e1c18e | 319 | |
b725dfea MD |
320 | tp_funcs = rcu_dereference_protected(tp->funcs, |
321 | lockdep_is_held(&tracepoints_mutex)); | |
de7b2973 | 322 | old = func_remove(&tp_funcs, func); |
befe6d94 | 323 | if (WARN_ON_ONCE(IS_ERR(old))) |
de7b2973 | 324 | return PTR_ERR(old); |
befe6d94 SRV |
325 | |
326 | if (tp_funcs == old) | |
327 | /* Failed allocating new tp_funcs, replaced func with stub */ | |
328 | return 0; | |
b75ef8b4 | 329 | |
de7b2973 MD |
330 | if (!tp_funcs) { |
331 | /* Removed last function */ | |
332 | if (tp->unregfunc && static_key_enabled(&tp->key)) | |
333 | tp->unregfunc(); | |
b75ef8b4 | 334 | |
d25e37d8 | 335 | static_key_disable(&tp->key); |
547305a6 | 336 | rcu_assign_pointer(tp->funcs, tp_funcs); |
d25e37d8 | 337 | } else { |
547305a6 SRV |
338 | rcu_assign_pointer(tp->funcs, tp_funcs); |
339 | tracepoint_update_call(tp, tp_funcs, | |
340 | tp_funcs[0].func != old[0].func); | |
127cafbb | 341 | } |
8058bd0f | 342 | release_probes(old); |
de7b2973 | 343 | return 0; |
127cafbb LJ |
344 | } |
345 | ||
97e1c18e | 346 | /** |
f39e2391 | 347 | * tracepoint_probe_register_prio - Connect a probe to a tracepoint with priority |
de7b2973 | 348 | * @tp: tracepoint |
97e1c18e | 349 | * @probe: probe handler |
cac92ba7 | 350 | * @data: tracepoint data |
7904b5c4 | 351 | * @prio: priority of this function over other registered functions |
97e1c18e | 352 | * |
de7b2973 MD |
353 | * Returns 0 if ok, error value on error. |
354 | * Note: if @tp is within a module, the caller is responsible for | |
355 | * unregistering the probe before the module is gone. This can be | |
356 | * performed either with a tracepoint module going notifier, or from | |
357 | * within module exit functions. | |
97e1c18e | 358 | */ |
7904b5c4 SRRH |
359 | int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe, |
360 | void *data, int prio) | |
97e1c18e | 361 | { |
de7b2973 MD |
362 | struct tracepoint_func tp_func; |
363 | int ret; | |
97e1c18e MD |
364 | |
365 | mutex_lock(&tracepoints_mutex); | |
de7b2973 MD |
366 | tp_func.func = probe; |
367 | tp_func.data = data; | |
7904b5c4 SRRH |
368 | tp_func.prio = prio; |
369 | ret = tracepoint_add_func(tp, &tp_func, prio); | |
b75ef8b4 | 370 | mutex_unlock(&tracepoints_mutex); |
b196e2b9 | 371 | return ret; |
97e1c18e | 372 | } |
7904b5c4 SRRH |
373 | EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio); |
374 | ||
375 | /** | |
376 | * tracepoint_probe_register - Connect a probe to a tracepoint | |
377 | * @tp: tracepoint | |
378 | * @probe: probe handler | |
379 | * @data: tracepoint data | |
7904b5c4 SRRH |
380 | * |
381 | * Returns 0 if ok, error value on error. | |
382 | * Note: if @tp is within a module, the caller is responsible for | |
383 | * unregistering the probe before the module is gone. This can be | |
384 | * performed either with a tracepoint module going notifier, or from | |
385 | * within module exit functions. | |
386 | */ | |
387 | int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data) | |
388 | { | |
389 | return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO); | |
390 | } | |
97e1c18e MD |
391 | EXPORT_SYMBOL_GPL(tracepoint_probe_register); |
392 | ||
393 | /** | |
394 | * tracepoint_probe_unregister - Disconnect a probe from a tracepoint | |
de7b2973 | 395 | * @tp: tracepoint |
97e1c18e | 396 | * @probe: probe function pointer |
cac92ba7 | 397 | * @data: tracepoint data |
97e1c18e | 398 | * |
de7b2973 | 399 | * Returns 0 if ok, error value on error. |
97e1c18e | 400 | */ |
de7b2973 | 401 | int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data) |
97e1c18e | 402 | { |
de7b2973 MD |
403 | struct tracepoint_func tp_func; |
404 | int ret; | |
97e1c18e MD |
405 | |
406 | mutex_lock(&tracepoints_mutex); | |
de7b2973 MD |
407 | tp_func.func = probe; |
408 | tp_func.data = data; | |
409 | ret = tracepoint_remove_func(tp, &tp_func); | |
b75ef8b4 | 410 | mutex_unlock(&tracepoints_mutex); |
de7b2973 | 411 | return ret; |
97e1c18e MD |
412 | } |
413 | EXPORT_SYMBOL_GPL(tracepoint_probe_unregister); | |
414 | ||
9c0be3f6 MD |
415 | static void for_each_tracepoint_range( |
416 | tracepoint_ptr_t *begin, tracepoint_ptr_t *end, | |
46e0c9be AB |
417 | void (*fct)(struct tracepoint *tp, void *priv), |
418 | void *priv) | |
419 | { | |
9c0be3f6 MD |
420 | tracepoint_ptr_t *iter; |
421 | ||
46e0c9be AB |
422 | if (!begin) |
423 | return; | |
9c0be3f6 MD |
424 | for (iter = begin; iter < end; iter++) |
425 | fct(tracepoint_ptr_deref(iter), priv); | |
46e0c9be AB |
426 | } |
427 | ||
227a8375 | 428 | #ifdef CONFIG_MODULES |
45ab2813 SRRH |
429 | bool trace_module_has_bad_taint(struct module *mod) |
430 | { | |
66cc69e3 MD |
431 | return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) | |
432 | (1 << TAINT_UNSIGNED_MODULE)); | |
45ab2813 SRRH |
433 | } |
434 | ||
de7b2973 MD |
435 | static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list); |
436 | ||
437 | /** | |
438 | * register_tracepoint_notifier - register tracepoint coming/going notifier | |
439 | * @nb: notifier block | |
440 | * | |
441 | * Notifiers registered with this function are called on module | |
442 | * coming/going with the tracepoint_module_list_mutex held. | |
443 | * The notifier block callback should expect a "struct tp_module" data | |
444 | * pointer. | |
445 | */ | |
446 | int register_tracepoint_module_notifier(struct notifier_block *nb) | |
447 | { | |
448 | struct tp_module *tp_mod; | |
449 | int ret; | |
450 | ||
451 | mutex_lock(&tracepoint_module_list_mutex); | |
452 | ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb); | |
453 | if (ret) | |
454 | goto end; | |
455 | list_for_each_entry(tp_mod, &tracepoint_module_list, list) | |
456 | (void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod); | |
457 | end: | |
458 | mutex_unlock(&tracepoint_module_list_mutex); | |
459 | return ret; | |
460 | } | |
461 | EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier); | |
462 | ||
463 | /** | |
464 | * unregister_tracepoint_notifier - unregister tracepoint coming/going notifier | |
465 | * @nb: notifier block | |
466 | * | |
467 | * The notifier block callback should expect a "struct tp_module" data | |
468 | * pointer. | |
469 | */ | |
470 | int unregister_tracepoint_module_notifier(struct notifier_block *nb) | |
471 | { | |
472 | struct tp_module *tp_mod; | |
473 | int ret; | |
474 | ||
475 | mutex_lock(&tracepoint_module_list_mutex); | |
476 | ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb); | |
477 | if (ret) | |
478 | goto end; | |
479 | list_for_each_entry(tp_mod, &tracepoint_module_list, list) | |
480 | (void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod); | |
481 | end: | |
482 | mutex_unlock(&tracepoint_module_list_mutex); | |
483 | return ret; | |
484 | ||
485 | } | |
486 | EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier); | |
487 | ||
488 | /* | |
489 | * Ensure the tracer unregistered the module's probes before the module | |
490 | * teardown is performed. Prevents leaks of probe and data pointers. | |
491 | */ | |
46e0c9be | 492 | static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv) |
de7b2973 | 493 | { |
46e0c9be | 494 | WARN_ON_ONCE(tp->funcs); |
de7b2973 MD |
495 | } |
496 | ||
b75ef8b4 MD |
497 | static int tracepoint_module_coming(struct module *mod) |
498 | { | |
0dea6d52 | 499 | struct tp_module *tp_mod; |
b75ef8b4 MD |
500 | int ret = 0; |
501 | ||
7dec935a SRRH |
502 | if (!mod->num_tracepoints) |
503 | return 0; | |
504 | ||
b75ef8b4 | 505 | /* |
c10076c4 SR |
506 | * We skip modules that taint the kernel, especially those with different |
507 | * module headers (for forced load), to make sure we don't cause a crash. | |
66cc69e3 | 508 | * Staging, out-of-tree, and unsigned GPL modules are fine. |
b75ef8b4 | 509 | */ |
45ab2813 | 510 | if (trace_module_has_bad_taint(mod)) |
b75ef8b4 | 511 | return 0; |
de7b2973 | 512 | mutex_lock(&tracepoint_module_list_mutex); |
b75ef8b4 MD |
513 | tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL); |
514 | if (!tp_mod) { | |
515 | ret = -ENOMEM; | |
516 | goto end; | |
517 | } | |
eb7d035c | 518 | tp_mod->mod = mod; |
0dea6d52 | 519 | list_add_tail(&tp_mod->list, &tracepoint_module_list); |
de7b2973 MD |
520 | blocking_notifier_call_chain(&tracepoint_notify_list, |
521 | MODULE_STATE_COMING, tp_mod); | |
b75ef8b4 | 522 | end: |
de7b2973 | 523 | mutex_unlock(&tracepoint_module_list_mutex); |
b75ef8b4 MD |
524 | return ret; |
525 | } | |
526 | ||
de7b2973 | 527 | static void tracepoint_module_going(struct module *mod) |
b75ef8b4 | 528 | { |
de7b2973 | 529 | struct tp_module *tp_mod; |
b75ef8b4 | 530 | |
7dec935a | 531 | if (!mod->num_tracepoints) |
de7b2973 | 532 | return; |
7dec935a | 533 | |
de7b2973 MD |
534 | mutex_lock(&tracepoint_module_list_mutex); |
535 | list_for_each_entry(tp_mod, &tracepoint_module_list, list) { | |
eb7d035c | 536 | if (tp_mod->mod == mod) { |
de7b2973 MD |
537 | blocking_notifier_call_chain(&tracepoint_notify_list, |
538 | MODULE_STATE_GOING, tp_mod); | |
539 | list_del(&tp_mod->list); | |
540 | kfree(tp_mod); | |
541 | /* | |
542 | * Called the going notifier before checking for | |
543 | * quiescence. | |
544 | */ | |
46e0c9be AB |
545 | for_each_tracepoint_range(mod->tracepoints_ptrs, |
546 | mod->tracepoints_ptrs + mod->num_tracepoints, | |
547 | tp_module_going_check_quiescent, NULL); | |
b75ef8b4 MD |
548 | break; |
549 | } | |
550 | } | |
551 | /* | |
552 | * In the case of modules that were tainted at "coming", we'll simply | |
553 | * walk through the list without finding it. We cannot use the "tainted" | |
554 | * flag on "going", in case a module taints the kernel only after being | |
555 | * loaded. | |
556 | */ | |
de7b2973 | 557 | mutex_unlock(&tracepoint_module_list_mutex); |
b75ef8b4 | 558 | } |
227a8375 | 559 | |
de7b2973 MD |
560 | static int tracepoint_module_notify(struct notifier_block *self, |
561 | unsigned long val, void *data) | |
32f85742 MD |
562 | { |
563 | struct module *mod = data; | |
b75ef8b4 | 564 | int ret = 0; |
32f85742 MD |
565 | |
566 | switch (val) { | |
567 | case MODULE_STATE_COMING: | |
b75ef8b4 MD |
568 | ret = tracepoint_module_coming(mod); |
569 | break; | |
570 | case MODULE_STATE_LIVE: | |
571 | break; | |
32f85742 | 572 | case MODULE_STATE_GOING: |
de7b2973 MD |
573 | tracepoint_module_going(mod); |
574 | break; | |
575 | case MODULE_STATE_UNFORMED: | |
32f85742 MD |
576 | break; |
577 | } | |
0340a6b7 | 578 | return notifier_from_errno(ret); |
32f85742 MD |
579 | } |
580 | ||
de7b2973 | 581 | static struct notifier_block tracepoint_module_nb = { |
32f85742 MD |
582 | .notifier_call = tracepoint_module_notify, |
583 | .priority = 0, | |
584 | }; | |
585 | ||
de7b2973 | 586 | static __init int init_tracepoints(void) |
32f85742 | 587 | { |
de7b2973 MD |
588 | int ret; |
589 | ||
590 | ret = register_module_notifier(&tracepoint_module_nb); | |
eb7d035c | 591 | if (ret) |
a395d6a7 | 592 | pr_warn("Failed to register tracepoint module enter notifier\n"); |
eb7d035c | 593 | |
de7b2973 | 594 | return ret; |
32f85742 MD |
595 | } |
596 | __initcall(init_tracepoints); | |
227a8375 | 597 | #endif /* CONFIG_MODULES */ |
a871bd33 | 598 | |
de7b2973 MD |
599 | /** |
600 | * for_each_kernel_tracepoint - iteration on all kernel tracepoints | |
601 | * @fct: callback | |
602 | * @priv: private data | |
603 | */ | |
604 | void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv), | |
605 | void *priv) | |
606 | { | |
607 | for_each_tracepoint_range(__start___tracepoints_ptrs, | |
608 | __stop___tracepoints_ptrs, fct, priv); | |
609 | } | |
610 | EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint); | |
611 | ||
3d27d8cb | 612 | #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS |
60d970c2 | 613 | |
97419875 | 614 | /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ |
a871bd33 JB |
615 | static int sys_tracepoint_refcount; |
616 | ||
8cf868af | 617 | int syscall_regfunc(void) |
a871bd33 | 618 | { |
8063e41d | 619 | struct task_struct *p, *t; |
a871bd33 | 620 | |
a871bd33 | 621 | if (!sys_tracepoint_refcount) { |
8063e41d ON |
622 | read_lock(&tasklist_lock); |
623 | for_each_process_thread(p, t) { | |
524666cb | 624 | set_task_syscall_work(t, SYSCALL_TRACEPOINT); |
8063e41d ON |
625 | } |
626 | read_unlock(&tasklist_lock); | |
a871bd33 JB |
627 | } |
628 | sys_tracepoint_refcount++; | |
8cf868af SRRH |
629 | |
630 | return 0; | |
a871bd33 JB |
631 | } |
632 | ||
633 | void syscall_unregfunc(void) | |
634 | { | |
8063e41d | 635 | struct task_struct *p, *t; |
a871bd33 | 636 | |
a871bd33 JB |
637 | sys_tracepoint_refcount--; |
638 | if (!sys_tracepoint_refcount) { | |
8063e41d ON |
639 | read_lock(&tasklist_lock); |
640 | for_each_process_thread(p, t) { | |
524666cb | 641 | clear_task_syscall_work(t, SYSCALL_TRACEPOINT); |
8063e41d ON |
642 | } |
643 | read_unlock(&tasklist_lock); | |
a871bd33 | 644 | } |
a871bd33 | 645 | } |
60d970c2 | 646 | #endif |