]>
Commit | Line | Data |
---|---|---|
8256e47c MD |
1 | /* |
2 | * Copyright (C) 2007 Mathieu Desnoyers | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | */ | |
18 | #include <linux/module.h> | |
19 | #include <linux/mutex.h> | |
20 | #include <linux/types.h> | |
21 | #include <linux/jhash.h> | |
22 | #include <linux/list.h> | |
23 | #include <linux/rcupdate.h> | |
24 | #include <linux/marker.h> | |
25 | #include <linux/err.h> | |
1aeb272c | 26 | #include <linux/slab.h> |
8256e47c MD |
27 | |
28 | extern struct marker __start___markers[]; | |
29 | extern struct marker __stop___markers[]; | |
30 | ||
fb40bd78 | 31 | /* Set to 1 to enable marker debug output */ |
ab883af5 | 32 | static const int marker_debug; |
fb40bd78 | 33 | |
8256e47c | 34 | /* |
314de8a9 | 35 | * markers_mutex nests inside module_mutex. Markers mutex protects the builtin |
fb40bd78 | 36 | * and module markers and the hash table. |
8256e47c MD |
37 | */ |
38 | static DEFINE_MUTEX(markers_mutex); | |
39 | ||
8256e47c MD |
40 | /* |
41 | * Marker hash table, containing the active markers. | |
42 | * Protected by module_mutex. | |
43 | */ | |
44 | #define MARKER_HASH_BITS 6 | |
45 | #define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) | |
4de62748 | 46 | static struct hlist_head marker_table[MARKER_TABLE_SIZE]; |
8256e47c | 47 | |
fb40bd78 MD |
48 | /* |
49 | * Note about RCU : | |
50 | * It is used to make sure every handler has finished using its private data | |
51 | * between two consecutive operation (add or remove) on a given marker. It is | |
52 | * also used to delay the free of multiple probes array until a quiescent state | |
53 | * is reached. | |
54 | * marker entries modifications are protected by the markers_mutex. | |
55 | */ | |
8256e47c MD |
56 | struct marker_entry { |
57 | struct hlist_node hlist; | |
58 | char *format; | |
dc102a8f MD |
59 | /* Probe wrapper */ |
60 | void (*call)(const struct marker *mdata, void *call_private, ...); | |
fb40bd78 MD |
61 | struct marker_probe_closure single; |
62 | struct marker_probe_closure *multi; | |
8256e47c | 63 | int refcount; /* Number of times armed. 0 if disarmed. */ |
ed86a590 MD |
64 | struct rcu_head rcu; |
65 | void *oldptr; | |
1b7ae37c | 66 | int rcu_pending; |
de4fc64f | 67 | unsigned char ptype:1; |
0eec481e | 68 | unsigned char format_allocated:1; |
8256e47c MD |
69 | char name[0]; /* Contains name'\0'format'\0' */ |
70 | }; | |
71 | ||
8256e47c MD |
72 | /** |
73 | * __mark_empty_function - Empty probe callback | |
fb40bd78 MD |
74 | * @probe_private: probe private data |
75 | * @call_private: call site private data | |
8256e47c MD |
76 | * @fmt: format string |
77 | * @...: variable argument list | |
78 | * | |
79 | * Empty callback provided as a probe to the markers. By providing this to a | |
80 | * disabled marker, we make sure the execution flow is always valid even | |
81 | * though the function pointer change and the marker enabling are two distinct | |
82 | * operations that modifies the execution flow of preemptible code. | |
83 | */ | |
fb40bd78 MD |
84 | void __mark_empty_function(void *probe_private, void *call_private, |
85 | const char *fmt, va_list *args) | |
8256e47c MD |
86 | { |
87 | } | |
88 | EXPORT_SYMBOL_GPL(__mark_empty_function); | |
89 | ||
fb40bd78 MD |
90 | /* |
91 | * marker_probe_cb Callback that prepares the variable argument list for probes. | |
92 | * @mdata: pointer of type struct marker | |
93 | * @call_private: caller site private data | |
fb40bd78 MD |
94 | * @...: Variable argument list. |
95 | * | |
96 | * Since we do not use "typical" pointer based RCU in the 1 argument case, we | |
97 | * need to put a full smp_rmb() in this branch. This is why we do not use | |
98 | * rcu_dereference() for the pointer read. | |
99 | */ | |
dc102a8f | 100 | void marker_probe_cb(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
101 | { |
102 | va_list args; | |
103 | char ptype; | |
104 | ||
105 | /* | |
e2d3b75d MD |
106 | * rcu_read_lock_sched does two things : disabling preemption to make |
107 | * sure the teardown of the callbacks can be done correctly when they | |
108 | * are in modules and they insure RCU read coherency. | |
fb40bd78 | 109 | */ |
e2d3b75d | 110 | rcu_read_lock_sched(); |
58336114 | 111 | ptype = mdata->ptype; |
fb40bd78 MD |
112 | if (likely(!ptype)) { |
113 | marker_probe_func *func; | |
114 | /* Must read the ptype before ptr. They are not data dependant, | |
115 | * so we put an explicit smp_rmb() here. */ | |
116 | smp_rmb(); | |
58336114 | 117 | func = mdata->single.func; |
fb40bd78 MD |
118 | /* Must read the ptr before private data. They are not data |
119 | * dependant, so we put an explicit smp_rmb() here. */ | |
120 | smp_rmb(); | |
dc102a8f MD |
121 | va_start(args, call_private); |
122 | func(mdata->single.probe_private, call_private, mdata->format, | |
123 | &args); | |
fb40bd78 MD |
124 | va_end(args); |
125 | } else { | |
126 | struct marker_probe_closure *multi; | |
127 | int i; | |
5def9a3a MD |
128 | /* |
129 | * Read mdata->ptype before mdata->multi. | |
130 | */ | |
131 | smp_rmb(); | |
132 | multi = mdata->multi; | |
fb40bd78 MD |
133 | /* |
134 | * multi points to an array, therefore accessing the array | |
135 | * depends on reading multi. However, even in this case, | |
136 | * we must insure that the pointer is read _before_ the array | |
137 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
138 | * in the fast path, so put the explicit barrier here. | |
139 | */ | |
140 | smp_read_barrier_depends(); | |
fb40bd78 | 141 | for (i = 0; multi[i].func; i++) { |
dc102a8f MD |
142 | va_start(args, call_private); |
143 | multi[i].func(multi[i].probe_private, call_private, | |
144 | mdata->format, &args); | |
fb40bd78 MD |
145 | va_end(args); |
146 | } | |
147 | } | |
e2d3b75d | 148 | rcu_read_unlock_sched(); |
fb40bd78 MD |
149 | } |
150 | EXPORT_SYMBOL_GPL(marker_probe_cb); | |
151 | ||
152 | /* | |
153 | * marker_probe_cb Callback that does not prepare the variable argument list. | |
154 | * @mdata: pointer of type struct marker | |
155 | * @call_private: caller site private data | |
fb40bd78 MD |
156 | * @...: Variable argument list. |
157 | * | |
158 | * Should be connected to markers "MARK_NOARGS". | |
159 | */ | |
505e371d | 160 | static void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) |
fb40bd78 MD |
161 | { |
162 | va_list args; /* not initialized */ | |
163 | char ptype; | |
164 | ||
e2d3b75d | 165 | rcu_read_lock_sched(); |
58336114 | 166 | ptype = mdata->ptype; |
fb40bd78 MD |
167 | if (likely(!ptype)) { |
168 | marker_probe_func *func; | |
169 | /* Must read the ptype before ptr. They are not data dependant, | |
170 | * so we put an explicit smp_rmb() here. */ | |
171 | smp_rmb(); | |
58336114 | 172 | func = mdata->single.func; |
fb40bd78 MD |
173 | /* Must read the ptr before private data. They are not data |
174 | * dependant, so we put an explicit smp_rmb() here. */ | |
175 | smp_rmb(); | |
dc102a8f MD |
176 | func(mdata->single.probe_private, call_private, mdata->format, |
177 | &args); | |
fb40bd78 MD |
178 | } else { |
179 | struct marker_probe_closure *multi; | |
180 | int i; | |
5def9a3a MD |
181 | /* |
182 | * Read mdata->ptype before mdata->multi. | |
183 | */ | |
184 | smp_rmb(); | |
185 | multi = mdata->multi; | |
fb40bd78 MD |
186 | /* |
187 | * multi points to an array, therefore accessing the array | |
188 | * depends on reading multi. However, even in this case, | |
189 | * we must insure that the pointer is read _before_ the array | |
190 | * data. Same as rcu_dereference, but we need a full smp_rmb() | |
191 | * in the fast path, so put the explicit barrier here. | |
192 | */ | |
193 | smp_read_barrier_depends(); | |
fb40bd78 | 194 | for (i = 0; multi[i].func; i++) |
dc102a8f MD |
195 | multi[i].func(multi[i].probe_private, call_private, |
196 | mdata->format, &args); | |
fb40bd78 | 197 | } |
e2d3b75d | 198 | rcu_read_unlock_sched(); |
fb40bd78 | 199 | } |
fb40bd78 | 200 | |
ed86a590 MD |
201 | static void free_old_closure(struct rcu_head *head) |
202 | { | |
203 | struct marker_entry *entry = container_of(head, | |
204 | struct marker_entry, rcu); | |
205 | kfree(entry->oldptr); | |
206 | /* Make sure we free the data before setting the pending flag to 0 */ | |
207 | smp_wmb(); | |
208 | entry->rcu_pending = 0; | |
209 | } | |
210 | ||
fb40bd78 MD |
211 | static void debug_print_probes(struct marker_entry *entry) |
212 | { | |
213 | int i; | |
214 | ||
215 | if (!marker_debug) | |
216 | return; | |
217 | ||
218 | if (!entry->ptype) { | |
219 | printk(KERN_DEBUG "Single probe : %p %p\n", | |
220 | entry->single.func, | |
221 | entry->single.probe_private); | |
222 | } else { | |
223 | for (i = 0; entry->multi[i].func; i++) | |
224 | printk(KERN_DEBUG "Multi probe %d : %p %p\n", i, | |
225 | entry->multi[i].func, | |
226 | entry->multi[i].probe_private); | |
227 | } | |
228 | } | |
229 | ||
230 | static struct marker_probe_closure * | |
231 | marker_entry_add_probe(struct marker_entry *entry, | |
232 | marker_probe_func *probe, void *probe_private) | |
233 | { | |
234 | int nr_probes = 0; | |
235 | struct marker_probe_closure *old, *new; | |
236 | ||
237 | WARN_ON(!probe); | |
238 | ||
239 | debug_print_probes(entry); | |
240 | old = entry->multi; | |
241 | if (!entry->ptype) { | |
242 | if (entry->single.func == probe && | |
243 | entry->single.probe_private == probe_private) | |
244 | return ERR_PTR(-EBUSY); | |
245 | if (entry->single.func == __mark_empty_function) { | |
246 | /* 0 -> 1 probes */ | |
247 | entry->single.func = probe; | |
248 | entry->single.probe_private = probe_private; | |
249 | entry->refcount = 1; | |
250 | entry->ptype = 0; | |
251 | debug_print_probes(entry); | |
252 | return NULL; | |
253 | } else { | |
254 | /* 1 -> 2 probes */ | |
255 | nr_probes = 1; | |
256 | old = NULL; | |
257 | } | |
258 | } else { | |
259 | /* (N -> N+1), (N != 0, 1) probes */ | |
260 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) | |
261 | if (old[nr_probes].func == probe | |
262 | && old[nr_probes].probe_private | |
263 | == probe_private) | |
264 | return ERR_PTR(-EBUSY); | |
265 | } | |
266 | /* + 2 : one for new probe, one for NULL func */ | |
267 | new = kzalloc((nr_probes + 2) * sizeof(struct marker_probe_closure), | |
268 | GFP_KERNEL); | |
269 | if (new == NULL) | |
270 | return ERR_PTR(-ENOMEM); | |
271 | if (!old) | |
272 | new[0] = entry->single; | |
273 | else | |
274 | memcpy(new, old, | |
275 | nr_probes * sizeof(struct marker_probe_closure)); | |
276 | new[nr_probes].func = probe; | |
277 | new[nr_probes].probe_private = probe_private; | |
278 | entry->refcount = nr_probes + 1; | |
279 | entry->multi = new; | |
280 | entry->ptype = 1; | |
281 | debug_print_probes(entry); | |
282 | return old; | |
283 | } | |
284 | ||
285 | static struct marker_probe_closure * | |
286 | marker_entry_remove_probe(struct marker_entry *entry, | |
287 | marker_probe_func *probe, void *probe_private) | |
288 | { | |
289 | int nr_probes = 0, nr_del = 0, i; | |
290 | struct marker_probe_closure *old, *new; | |
291 | ||
292 | old = entry->multi; | |
293 | ||
294 | debug_print_probes(entry); | |
295 | if (!entry->ptype) { | |
296 | /* 0 -> N is an error */ | |
297 | WARN_ON(entry->single.func == __mark_empty_function); | |
298 | /* 1 -> 0 probes */ | |
299 | WARN_ON(probe && entry->single.func != probe); | |
300 | WARN_ON(entry->single.probe_private != probe_private); | |
301 | entry->single.func = __mark_empty_function; | |
302 | entry->refcount = 0; | |
303 | entry->ptype = 0; | |
304 | debug_print_probes(entry); | |
305 | return NULL; | |
306 | } else { | |
307 | /* (N -> M), (N > 1, M >= 0) probes */ | |
308 | for (nr_probes = 0; old[nr_probes].func; nr_probes++) { | |
309 | if ((!probe || old[nr_probes].func == probe) | |
310 | && old[nr_probes].probe_private | |
311 | == probe_private) | |
312 | nr_del++; | |
313 | } | |
314 | } | |
315 | ||
316 | if (nr_probes - nr_del == 0) { | |
317 | /* N -> 0, (N > 1) */ | |
318 | entry->single.func = __mark_empty_function; | |
319 | entry->refcount = 0; | |
320 | entry->ptype = 0; | |
321 | } else if (nr_probes - nr_del == 1) { | |
322 | /* N -> 1, (N > 1) */ | |
323 | for (i = 0; old[i].func; i++) | |
324 | if ((probe && old[i].func != probe) || | |
325 | old[i].probe_private != probe_private) | |
326 | entry->single = old[i]; | |
327 | entry->refcount = 1; | |
328 | entry->ptype = 0; | |
329 | } else { | |
330 | int j = 0; | |
331 | /* N -> M, (N > 1, M > 1) */ | |
332 | /* + 1 for NULL */ | |
333 | new = kzalloc((nr_probes - nr_del + 1) | |
334 | * sizeof(struct marker_probe_closure), GFP_KERNEL); | |
335 | if (new == NULL) | |
336 | return ERR_PTR(-ENOMEM); | |
337 | for (i = 0; old[i].func; i++) | |
338 | if ((probe && old[i].func != probe) || | |
339 | old[i].probe_private != probe_private) | |
340 | new[j++] = old[i]; | |
341 | entry->refcount = nr_probes - nr_del; | |
342 | entry->ptype = 1; | |
343 | entry->multi = new; | |
344 | } | |
345 | debug_print_probes(entry); | |
346 | return old; | |
347 | } | |
348 | ||
8256e47c MD |
349 | /* |
350 | * Get marker if the marker is present in the marker hash table. | |
351 | * Must be called with markers_mutex held. | |
352 | * Returns NULL if not present. | |
353 | */ | |
354 | static struct marker_entry *get_marker(const char *name) | |
355 | { | |
356 | struct hlist_head *head; | |
357 | struct hlist_node *node; | |
358 | struct marker_entry *e; | |
359 | u32 hash = jhash(name, strlen(name), 0); | |
360 | ||
361 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
362 | hlist_for_each_entry(e, node, head, hlist) { | |
363 | if (!strcmp(name, e->name)) | |
364 | return e; | |
365 | } | |
366 | return NULL; | |
367 | } | |
368 | ||
369 | /* | |
370 | * Add the marker to the marker hash table. Must be called with markers_mutex | |
371 | * held. | |
372 | */ | |
fb40bd78 | 373 | static struct marker_entry *add_marker(const char *name, const char *format) |
8256e47c MD |
374 | { |
375 | struct hlist_head *head; | |
376 | struct hlist_node *node; | |
377 | struct marker_entry *e; | |
378 | size_t name_len = strlen(name) + 1; | |
379 | size_t format_len = 0; | |
380 | u32 hash = jhash(name, name_len-1, 0); | |
381 | ||
382 | if (format) | |
383 | format_len = strlen(format) + 1; | |
384 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
385 | hlist_for_each_entry(e, node, head, hlist) { | |
386 | if (!strcmp(name, e->name)) { | |
387 | printk(KERN_NOTICE | |
fb40bd78 MD |
388 | "Marker %s busy\n", name); |
389 | return ERR_PTR(-EBUSY); /* Already there */ | |
8256e47c MD |
390 | } |
391 | } | |
392 | /* | |
393 | * Using kmalloc here to allocate a variable length element. Could | |
394 | * cause some memory fragmentation if overused. | |
395 | */ | |
396 | e = kmalloc(sizeof(struct marker_entry) + name_len + format_len, | |
397 | GFP_KERNEL); | |
398 | if (!e) | |
fb40bd78 | 399 | return ERR_PTR(-ENOMEM); |
8256e47c MD |
400 | memcpy(&e->name[0], name, name_len); |
401 | if (format) { | |
402 | e->format = &e->name[name_len]; | |
403 | memcpy(e->format, format, format_len); | |
fb40bd78 MD |
404 | if (strcmp(e->format, MARK_NOARGS) == 0) |
405 | e->call = marker_probe_cb_noarg; | |
406 | else | |
407 | e->call = marker_probe_cb; | |
8256e47c MD |
408 | trace_mark(core_marker_format, "name %s format %s", |
409 | e->name, e->format); | |
fb40bd78 | 410 | } else { |
8256e47c | 411 | e->format = NULL; |
fb40bd78 MD |
412 | e->call = marker_probe_cb; |
413 | } | |
414 | e->single.func = __mark_empty_function; | |
415 | e->single.probe_private = NULL; | |
416 | e->multi = NULL; | |
417 | e->ptype = 0; | |
0eec481e | 418 | e->format_allocated = 0; |
8256e47c | 419 | e->refcount = 0; |
ed86a590 | 420 | e->rcu_pending = 0; |
8256e47c | 421 | hlist_add_head(&e->hlist, head); |
fb40bd78 | 422 | return e; |
8256e47c MD |
423 | } |
424 | ||
425 | /* | |
426 | * Remove the marker from the marker hash table. Must be called with mutex_lock | |
427 | * held. | |
428 | */ | |
fb40bd78 | 429 | static int remove_marker(const char *name) |
8256e47c MD |
430 | { |
431 | struct hlist_head *head; | |
432 | struct hlist_node *node; | |
433 | struct marker_entry *e; | |
434 | int found = 0; | |
435 | size_t len = strlen(name) + 1; | |
8256e47c MD |
436 | u32 hash = jhash(name, len-1, 0); |
437 | ||
438 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
439 | hlist_for_each_entry(e, node, head, hlist) { | |
440 | if (!strcmp(name, e->name)) { | |
441 | found = 1; | |
442 | break; | |
443 | } | |
444 | } | |
fb40bd78 MD |
445 | if (!found) |
446 | return -ENOENT; | |
447 | if (e->single.func != __mark_empty_function) | |
448 | return -EBUSY; | |
449 | hlist_del(&e->hlist); | |
0eec481e LJ |
450 | if (e->format_allocated) |
451 | kfree(e->format); | |
ed86a590 MD |
452 | /* Make sure the call_rcu has been executed */ |
453 | if (e->rcu_pending) | |
454 | rcu_barrier_sched(); | |
fb40bd78 MD |
455 | kfree(e); |
456 | return 0; | |
8256e47c MD |
457 | } |
458 | ||
459 | /* | |
460 | * Set the mark_entry format to the format found in the element. | |
461 | */ | |
0eec481e | 462 | static int marker_set_format(struct marker_entry *entry, const char *format) |
8256e47c | 463 | { |
0eec481e LJ |
464 | entry->format = kstrdup(format, GFP_KERNEL); |
465 | if (!entry->format) | |
8256e47c | 466 | return -ENOMEM; |
0eec481e LJ |
467 | entry->format_allocated = 1; |
468 | ||
8256e47c | 469 | trace_mark(core_marker_format, "name %s format %s", |
0eec481e | 470 | entry->name, entry->format); |
8256e47c MD |
471 | return 0; |
472 | } | |
473 | ||
474 | /* | |
475 | * Sets the probe callback corresponding to one marker. | |
476 | */ | |
0eec481e | 477 | static int set_marker(struct marker_entry *entry, struct marker *elem, |
fb40bd78 | 478 | int active) |
8256e47c MD |
479 | { |
480 | int ret; | |
0eec481e | 481 | WARN_ON(strcmp(entry->name, elem->name) != 0); |
8256e47c | 482 | |
0eec481e LJ |
483 | if (entry->format) { |
484 | if (strcmp(entry->format, elem->format) != 0) { | |
8256e47c MD |
485 | printk(KERN_NOTICE |
486 | "Format mismatch for probe %s " | |
487 | "(%s), marker (%s)\n", | |
0eec481e LJ |
488 | entry->name, |
489 | entry->format, | |
8256e47c MD |
490 | elem->format); |
491 | return -EPERM; | |
492 | } | |
493 | } else { | |
494 | ret = marker_set_format(entry, elem->format); | |
495 | if (ret) | |
496 | return ret; | |
497 | } | |
fb40bd78 MD |
498 | |
499 | /* | |
500 | * probe_cb setup (statically known) is done here. It is | |
501 | * asynchronous with the rest of execution, therefore we only | |
502 | * pass from a "safe" callback (with argument) to an "unsafe" | |
503 | * callback (does not set arguments). | |
504 | */ | |
0eec481e | 505 | elem->call = entry->call; |
fb40bd78 MD |
506 | /* |
507 | * Sanity check : | |
508 | * We only update the single probe private data when the ptr is | |
509 | * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) | |
510 | */ | |
511 | WARN_ON(elem->single.func != __mark_empty_function | |
0eec481e LJ |
512 | && elem->single.probe_private != entry->single.probe_private |
513 | && !elem->ptype); | |
514 | elem->single.probe_private = entry->single.probe_private; | |
fb40bd78 MD |
515 | /* |
516 | * Make sure the private data is valid when we update the | |
517 | * single probe ptr. | |
518 | */ | |
519 | smp_wmb(); | |
0eec481e | 520 | elem->single.func = entry->single.func; |
fb40bd78 MD |
521 | /* |
522 | * We also make sure that the new probe callbacks array is consistent | |
523 | * before setting a pointer to it. | |
524 | */ | |
0eec481e | 525 | rcu_assign_pointer(elem->multi, entry->multi); |
fb40bd78 MD |
526 | /* |
527 | * Update the function or multi probe array pointer before setting the | |
528 | * ptype. | |
529 | */ | |
530 | smp_wmb(); | |
0eec481e | 531 | elem->ptype = entry->ptype; |
fb40bd78 MD |
532 | elem->state = active; |
533 | ||
8256e47c MD |
534 | return 0; |
535 | } | |
536 | ||
537 | /* | |
538 | * Disable a marker and its probe callback. | |
fd3c36f8 MD |
539 | * Note: only waiting an RCU period after setting elem->call to the empty |
540 | * function insures that the original callback is not used anymore. This insured | |
e2d3b75d | 541 | * by rcu_read_lock_sched around the call site. |
8256e47c MD |
542 | */ |
543 | static void disable_marker(struct marker *elem) | |
544 | { | |
fb40bd78 | 545 | /* leave "call" as is. It is known statically. */ |
8256e47c | 546 | elem->state = 0; |
fb40bd78 MD |
547 | elem->single.func = __mark_empty_function; |
548 | /* Update the function before setting the ptype */ | |
549 | smp_wmb(); | |
550 | elem->ptype = 0; /* single probe */ | |
8256e47c MD |
551 | /* |
552 | * Leave the private data and id there, because removal is racy and | |
fd3c36f8 MD |
553 | * should be done only after an RCU period. These are never used until |
554 | * the next initialization anyway. | |
8256e47c MD |
555 | */ |
556 | } | |
557 | ||
558 | /** | |
559 | * marker_update_probe_range - Update a probe range | |
560 | * @begin: beginning of the range | |
561 | * @end: end of the range | |
8256e47c MD |
562 | * |
563 | * Updates the probe callback corresponding to a range of markers. | |
8256e47c MD |
564 | */ |
565 | void marker_update_probe_range(struct marker *begin, | |
fb40bd78 | 566 | struct marker *end) |
8256e47c MD |
567 | { |
568 | struct marker *iter; | |
569 | struct marker_entry *mark_entry; | |
570 | ||
314de8a9 | 571 | mutex_lock(&markers_mutex); |
8256e47c MD |
572 | for (iter = begin; iter < end; iter++) { |
573 | mark_entry = get_marker(iter->name); | |
fb40bd78 | 574 | if (mark_entry) { |
0eec481e | 575 | set_marker(mark_entry, iter, !!mark_entry->refcount); |
8256e47c MD |
576 | /* |
577 | * ignore error, continue | |
578 | */ | |
8256e47c MD |
579 | } else { |
580 | disable_marker(iter); | |
581 | } | |
582 | } | |
314de8a9 | 583 | mutex_unlock(&markers_mutex); |
8256e47c MD |
584 | } |
585 | ||
586 | /* | |
587 | * Update probes, removing the faulty probes. | |
fb40bd78 MD |
588 | * |
589 | * Internal callback only changed before the first probe is connected to it. | |
590 | * Single probe private data can only be changed on 0 -> 1 and 2 -> 1 | |
591 | * transitions. All other transitions will leave the old private data valid. | |
592 | * This makes the non-atomicity of the callback/private data updates valid. | |
593 | * | |
594 | * "special case" updates : | |
595 | * 0 -> 1 callback | |
596 | * 1 -> 0 callback | |
597 | * 1 -> 2 callbacks | |
598 | * 2 -> 1 callbacks | |
599 | * Other updates all behave the same, just like the 2 -> 3 or 3 -> 2 updates. | |
600 | * Site effect : marker_set_format may delete the marker entry (creating a | |
601 | * replacement). | |
8256e47c | 602 | */ |
fb40bd78 | 603 | static void marker_update_probes(void) |
8256e47c | 604 | { |
8256e47c | 605 | /* Core kernel markers */ |
fb40bd78 | 606 | marker_update_probe_range(__start___markers, __stop___markers); |
8256e47c | 607 | /* Markers in modules. */ |
fb40bd78 | 608 | module_update_markers(); |
8256e47c MD |
609 | } |
610 | ||
611 | /** | |
612 | * marker_probe_register - Connect a probe to a marker | |
613 | * @name: marker name | |
614 | * @format: format string | |
615 | * @probe: probe handler | |
fb40bd78 | 616 | * @probe_private: probe private data |
8256e47c MD |
617 | * |
618 | * private data must be a valid allocated memory address, or NULL. | |
619 | * Returns 0 if ok, error value on error. | |
fb40bd78 | 620 | * The probe address must at least be aligned on the architecture pointer size. |
8256e47c MD |
621 | */ |
622 | int marker_probe_register(const char *name, const char *format, | |
fb40bd78 | 623 | marker_probe_func *probe, void *probe_private) |
8256e47c MD |
624 | { |
625 | struct marker_entry *entry; | |
314de8a9 | 626 | int ret = 0; |
fb40bd78 | 627 | struct marker_probe_closure *old; |
8256e47c MD |
628 | |
629 | mutex_lock(&markers_mutex); | |
630 | entry = get_marker(name); | |
fb40bd78 MD |
631 | if (!entry) { |
632 | entry = add_marker(name, format); | |
48043bcd | 633 | if (IS_ERR(entry)) |
fb40bd78 | 634 | ret = PTR_ERR(entry); |
48043bcd LJ |
635 | } else if (format) { |
636 | if (!entry->format) | |
0eec481e | 637 | ret = marker_set_format(entry, format); |
48043bcd LJ |
638 | else if (strcmp(entry->format, format)) |
639 | ret = -EPERM; | |
8256e47c | 640 | } |
48043bcd LJ |
641 | if (ret) |
642 | goto end; | |
643 | ||
ed86a590 MD |
644 | /* |
645 | * If we detect that a call_rcu is pending for this marker, | |
646 | * make sure it's executed now. | |
647 | */ | |
648 | if (entry->rcu_pending) | |
649 | rcu_barrier_sched(); | |
fb40bd78 MD |
650 | old = marker_entry_add_probe(entry, probe, probe_private); |
651 | if (IS_ERR(old)) { | |
652 | ret = PTR_ERR(old); | |
8256e47c | 653 | goto end; |
fb40bd78 | 654 | } |
314de8a9 | 655 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
656 | marker_update_probes(); /* may update entry */ |
657 | mutex_lock(&markers_mutex); | |
658 | entry = get_marker(name); | |
659 | WARN_ON(!entry); | |
ed86a590 MD |
660 | if (entry->rcu_pending) |
661 | rcu_barrier_sched(); | |
662 | entry->oldptr = old; | |
663 | entry->rcu_pending = 1; | |
664 | /* write rcu_pending before calling the RCU callback */ | |
665 | smp_wmb(); | |
666 | call_rcu_sched(&entry->rcu, free_old_closure); | |
8256e47c MD |
667 | end: |
668 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
669 | return ret; |
670 | } | |
671 | EXPORT_SYMBOL_GPL(marker_probe_register); | |
672 | ||
673 | /** | |
674 | * marker_probe_unregister - Disconnect a probe from a marker | |
675 | * @name: marker name | |
fb40bd78 MD |
676 | * @probe: probe function pointer |
677 | * @probe_private: probe private data | |
8256e47c MD |
678 | * |
679 | * Returns the private data given to marker_probe_register, or an ERR_PTR(). | |
fb40bd78 MD |
680 | * We do not need to call a synchronize_sched to make sure the probes have |
681 | * finished running before doing a module unload, because the module unload | |
682 | * itself uses stop_machine(), which insures that every preempt disabled section | |
683 | * have finished. | |
8256e47c | 684 | */ |
fb40bd78 MD |
685 | int marker_probe_unregister(const char *name, |
686 | marker_probe_func *probe, void *probe_private) | |
8256e47c | 687 | { |
8256e47c | 688 | struct marker_entry *entry; |
fb40bd78 | 689 | struct marker_probe_closure *old; |
544adb41 | 690 | int ret = -ENOENT; |
8256e47c MD |
691 | |
692 | mutex_lock(&markers_mutex); | |
693 | entry = get_marker(name); | |
544adb41 | 694 | if (!entry) |
8256e47c | 695 | goto end; |
ed86a590 MD |
696 | if (entry->rcu_pending) |
697 | rcu_barrier_sched(); | |
fb40bd78 | 698 | old = marker_entry_remove_probe(entry, probe, probe_private); |
314de8a9 | 699 | mutex_unlock(&markers_mutex); |
fb40bd78 MD |
700 | marker_update_probes(); /* may update entry */ |
701 | mutex_lock(&markers_mutex); | |
702 | entry = get_marker(name); | |
544adb41 JJ |
703 | if (!entry) |
704 | goto end; | |
ed86a590 MD |
705 | if (entry->rcu_pending) |
706 | rcu_barrier_sched(); | |
707 | entry->oldptr = old; | |
708 | entry->rcu_pending = 1; | |
709 | /* write rcu_pending before calling the RCU callback */ | |
710 | smp_wmb(); | |
711 | call_rcu_sched(&entry->rcu, free_old_closure); | |
fb40bd78 | 712 | remove_marker(name); /* Ignore busy error message */ |
544adb41 | 713 | ret = 0; |
8256e47c MD |
714 | end: |
715 | mutex_unlock(&markers_mutex); | |
fb40bd78 | 716 | return ret; |
8256e47c MD |
717 | } |
718 | EXPORT_SYMBOL_GPL(marker_probe_unregister); | |
719 | ||
fb40bd78 MD |
720 | static struct marker_entry * |
721 | get_marker_from_private_data(marker_probe_func *probe, void *probe_private) | |
8256e47c | 722 | { |
8256e47c | 723 | struct marker_entry *entry; |
8256e47c | 724 | unsigned int i; |
fb40bd78 MD |
725 | struct hlist_head *head; |
726 | struct hlist_node *node; | |
8256e47c | 727 | |
8256e47c MD |
728 | for (i = 0; i < MARKER_TABLE_SIZE; i++) { |
729 | head = &marker_table[i]; | |
730 | hlist_for_each_entry(entry, node, head, hlist) { | |
fb40bd78 MD |
731 | if (!entry->ptype) { |
732 | if (entry->single.func == probe | |
733 | && entry->single.probe_private | |
734 | == probe_private) | |
735 | return entry; | |
736 | } else { | |
737 | struct marker_probe_closure *closure; | |
738 | closure = entry->multi; | |
739 | for (i = 0; closure[i].func; i++) { | |
740 | if (closure[i].func == probe && | |
741 | closure[i].probe_private | |
742 | == probe_private) | |
743 | return entry; | |
744 | } | |
8256e47c MD |
745 | } |
746 | } | |
747 | } | |
fb40bd78 | 748 | return NULL; |
8256e47c | 749 | } |
8256e47c MD |
750 | |
751 | /** | |
fb40bd78 MD |
752 | * marker_probe_unregister_private_data - Disconnect a probe from a marker |
753 | * @probe: probe function | |
754 | * @probe_private: probe private data | |
8256e47c | 755 | * |
fb40bd78 MD |
756 | * Unregister a probe by providing the registered private data. |
757 | * Only removes the first marker found in hash table. | |
758 | * Return 0 on success or error value. | |
759 | * We do not need to call a synchronize_sched to make sure the probes have | |
760 | * finished running before doing a module unload, because the module unload | |
761 | * itself uses stop_machine(), which insures that every preempt disabled section | |
762 | * have finished. | |
8256e47c | 763 | */ |
fb40bd78 MD |
764 | int marker_probe_unregister_private_data(marker_probe_func *probe, |
765 | void *probe_private) | |
8256e47c MD |
766 | { |
767 | struct marker_entry *entry; | |
314de8a9 | 768 | int ret = 0; |
fb40bd78 | 769 | struct marker_probe_closure *old; |
8256e47c MD |
770 | |
771 | mutex_lock(&markers_mutex); | |
fb40bd78 | 772 | entry = get_marker_from_private_data(probe, probe_private); |
8256e47c MD |
773 | if (!entry) { |
774 | ret = -ENOENT; | |
775 | goto end; | |
776 | } | |
ed86a590 MD |
777 | if (entry->rcu_pending) |
778 | rcu_barrier_sched(); | |
fb40bd78 | 779 | old = marker_entry_remove_probe(entry, NULL, probe_private); |
8256e47c | 780 | mutex_unlock(&markers_mutex); |
fb40bd78 | 781 | marker_update_probes(); /* may update entry */ |
8256e47c | 782 | mutex_lock(&markers_mutex); |
fb40bd78 MD |
783 | entry = get_marker_from_private_data(probe, probe_private); |
784 | WARN_ON(!entry); | |
ed86a590 MD |
785 | if (entry->rcu_pending) |
786 | rcu_barrier_sched(); | |
787 | entry->oldptr = old; | |
788 | entry->rcu_pending = 1; | |
789 | /* write rcu_pending before calling the RCU callback */ | |
790 | smp_wmb(); | |
791 | call_rcu_sched(&entry->rcu, free_old_closure); | |
fb40bd78 | 792 | remove_marker(entry->name); /* Ignore busy error message */ |
8256e47c MD |
793 | end: |
794 | mutex_unlock(&markers_mutex); | |
8256e47c MD |
795 | return ret; |
796 | } | |
fb40bd78 | 797 | EXPORT_SYMBOL_GPL(marker_probe_unregister_private_data); |
8256e47c MD |
798 | |
799 | /** | |
800 | * marker_get_private_data - Get a marker's probe private data | |
801 | * @name: marker name | |
fb40bd78 MD |
802 | * @probe: probe to match |
803 | * @num: get the nth matching probe's private data | |
8256e47c | 804 | * |
fb40bd78 MD |
805 | * Returns the nth private data pointer (starting from 0) matching, or an |
806 | * ERR_PTR. | |
8256e47c MD |
807 | * Returns the private data pointer, or an ERR_PTR. |
808 | * The private data pointer should _only_ be dereferenced if the caller is the | |
809 | * owner of the data, or its content could vanish. This is mostly used to | |
810 | * confirm that a caller is the owner of a registered probe. | |
811 | */ | |
fb40bd78 MD |
812 | void *marker_get_private_data(const char *name, marker_probe_func *probe, |
813 | int num) | |
8256e47c MD |
814 | { |
815 | struct hlist_head *head; | |
816 | struct hlist_node *node; | |
817 | struct marker_entry *e; | |
818 | size_t name_len = strlen(name) + 1; | |
819 | u32 hash = jhash(name, name_len-1, 0); | |
fb40bd78 | 820 | int i; |
8256e47c MD |
821 | |
822 | head = &marker_table[hash & ((1 << MARKER_HASH_BITS)-1)]; | |
823 | hlist_for_each_entry(e, node, head, hlist) { | |
824 | if (!strcmp(name, e->name)) { | |
fb40bd78 MD |
825 | if (!e->ptype) { |
826 | if (num == 0 && e->single.func == probe) | |
827 | return e->single.probe_private; | |
fb40bd78 MD |
828 | } else { |
829 | struct marker_probe_closure *closure; | |
830 | int match = 0; | |
831 | closure = e->multi; | |
832 | for (i = 0; closure[i].func; i++) { | |
833 | if (closure[i].func != probe) | |
834 | continue; | |
835 | if (match++ == num) | |
836 | return closure[i].probe_private; | |
837 | } | |
838 | } | |
5d9881ea | 839 | break; |
8256e47c MD |
840 | } |
841 | } | |
842 | return ERR_PTR(-ENOENT); | |
843 | } | |
844 | EXPORT_SYMBOL_GPL(marker_get_private_data); |