1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kdebug.h>
3 #include <linux/kprobes.h>
4 #include <linux/export.h>
5 #include <linux/notifier.h>
6 #include <linux/rcupdate.h>
7 #include <linux/vmalloc.h>
8 #include <linux/reboot.h>
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list
);
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
22 static int notifier_chain_register(struct notifier_block
**nl
,
23 struct notifier_block
*n
)
25 while ((*nl
) != NULL
) {
26 if (unlikely((*nl
) == n
)) {
27 WARN(1, "double register detected");
30 if (n
->priority
> (*nl
)->priority
)
35 rcu_assign_pointer(*nl
, n
);
39 static int notifier_chain_unregister(struct notifier_block
**nl
,
40 struct notifier_block
*n
)
42 while ((*nl
) != NULL
) {
44 rcu_assign_pointer(*nl
, n
->next
);
53 * notifier_call_chain - Informs the registered notifiers about an event.
54 * @nl: Pointer to head of the blocking notifier chain
55 * @val: Value passed unmodified to notifier function
56 * @v: Pointer passed unmodified to notifier function
57 * @nr_to_call: Number of notifier functions to be called. Don't care
58 * value of this parameter is -1.
59 * @nr_calls: Records the number of notifications sent. Don't care
60 * value of this field is NULL.
61 * @returns: notifier_call_chain returns the value returned by the
62 * last notifier function called.
64 static int notifier_call_chain(struct notifier_block
**nl
,
65 unsigned long val
, void *v
,
66 int nr_to_call
, int *nr_calls
)
68 int ret
= NOTIFY_DONE
;
69 struct notifier_block
*nb
, *next_nb
;
71 nb
= rcu_dereference_raw(*nl
);
73 while (nb
&& nr_to_call
) {
74 next_nb
= rcu_dereference_raw(nb
->next
);
76 #ifdef CONFIG_DEBUG_NOTIFIERS
77 if (unlikely(!func_ptr_is_kernel_text(nb
->notifier_call
))) {
78 WARN(1, "Invalid notifier called!");
83 ret
= nb
->notifier_call(nb
, val
, v
);
88 if (ret
& NOTIFY_STOP_MASK
)
95 NOKPROBE_SYMBOL(notifier_call_chain
);
98 * notifier_call_chain_robust - Inform the registered notifiers about an event
99 * and rollback on error.
100 * @nl: Pointer to head of the blocking notifier chain
101 * @val_up: Value passed unmodified to the notifier function
102 * @val_down: Value passed unmodified to the notifier function when recovering
103 * from an error on @val_up
104 * @v Pointer passed unmodified to the notifier function
106 * NOTE: It is important the @nl chain doesn't change between the two
107 * invocations of notifier_call_chain() such that we visit the
108 * exact same notifier callbacks; this rules out any RCU usage.
110 * Returns: the return value of the @val_up call.
112 static int notifier_call_chain_robust(struct notifier_block
**nl
,
113 unsigned long val_up
, unsigned long val_down
,
118 ret
= notifier_call_chain(nl
, val_up
, v
, -1, &nr
);
119 if (ret
& NOTIFY_STOP_MASK
)
120 notifier_call_chain(nl
, val_down
, v
, nr
-1, NULL
);
126 * Atomic notifier chain routines. Registration and unregistration
127 * use a spinlock, and call_chain is synchronized by RCU (no locks).
131 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
132 * @nh: Pointer to head of the atomic notifier chain
133 * @n: New entry in notifier chain
135 * Adds a notifier to an atomic notifier chain.
137 * Currently always returns zero.
139 int atomic_notifier_chain_register(struct atomic_notifier_head
*nh
,
140 struct notifier_block
*n
)
145 spin_lock_irqsave(&nh
->lock
, flags
);
146 ret
= notifier_chain_register(&nh
->head
, n
);
147 spin_unlock_irqrestore(&nh
->lock
, flags
);
150 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register
);
153 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
154 * @nh: Pointer to head of the atomic notifier chain
155 * @n: Entry to remove from notifier chain
157 * Removes a notifier from an atomic notifier chain.
159 * Returns zero on success or %-ENOENT on failure.
161 int atomic_notifier_chain_unregister(struct atomic_notifier_head
*nh
,
162 struct notifier_block
*n
)
167 spin_lock_irqsave(&nh
->lock
, flags
);
168 ret
= notifier_chain_unregister(&nh
->head
, n
);
169 spin_unlock_irqrestore(&nh
->lock
, flags
);
173 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister
);
176 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
177 * @nh: Pointer to head of the atomic notifier chain
178 * @val: Value passed unmodified to notifier function
179 * @v: Pointer passed unmodified to notifier function
181 * Calls each function in a notifier chain in turn. The functions
182 * run in an atomic context, so they must not block.
183 * This routine uses RCU to synchronize with changes to the chain.
185 * If the return value of the notifier can be and'ed
186 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
187 * will return immediately, with the return value of
188 * the notifier function which halted execution.
189 * Otherwise the return value is the return value
190 * of the last notifier function called.
192 int atomic_notifier_call_chain(struct atomic_notifier_head
*nh
,
193 unsigned long val
, void *v
)
198 ret
= notifier_call_chain(&nh
->head
, val
, v
, -1, NULL
);
203 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain
);
204 NOKPROBE_SYMBOL(atomic_notifier_call_chain
);
207 * Blocking notifier chain routines. All access to the chain is
208 * synchronized by an rwsem.
212 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
213 * @nh: Pointer to head of the blocking notifier chain
214 * @n: New entry in notifier chain
216 * Adds a notifier to a blocking notifier chain.
217 * Must be called in process context.
219 * Currently always returns zero.
221 int blocking_notifier_chain_register(struct blocking_notifier_head
*nh
,
222 struct notifier_block
*n
)
227 * This code gets used during boot-up, when task switching is
228 * not yet working and interrupts must remain disabled. At
229 * such times we must not call down_write().
231 if (unlikely(system_state
== SYSTEM_BOOTING
))
232 return notifier_chain_register(&nh
->head
, n
);
234 down_write(&nh
->rwsem
);
235 ret
= notifier_chain_register(&nh
->head
, n
);
236 up_write(&nh
->rwsem
);
239 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register
);
242 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
243 * @nh: Pointer to head of the blocking notifier chain
244 * @n: Entry to remove from notifier chain
246 * Removes a notifier from a blocking notifier chain.
247 * Must be called from process context.
249 * Returns zero on success or %-ENOENT on failure.
251 int blocking_notifier_chain_unregister(struct blocking_notifier_head
*nh
,
252 struct notifier_block
*n
)
257 * This code gets used during boot-up, when task switching is
258 * not yet working and interrupts must remain disabled. At
259 * such times we must not call down_write().
261 if (unlikely(system_state
== SYSTEM_BOOTING
))
262 return notifier_chain_unregister(&nh
->head
, n
);
264 down_write(&nh
->rwsem
);
265 ret
= notifier_chain_unregister(&nh
->head
, n
);
266 up_write(&nh
->rwsem
);
269 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister
);
271 int blocking_notifier_call_chain_robust(struct blocking_notifier_head
*nh
,
272 unsigned long val_up
, unsigned long val_down
, void *v
)
274 int ret
= NOTIFY_DONE
;
277 * We check the head outside the lock, but if this access is
278 * racy then it does not matter what the result of the test
279 * is, we re-check the list after having taken the lock anyway:
281 if (rcu_access_pointer(nh
->head
)) {
282 down_read(&nh
->rwsem
);
283 ret
= notifier_call_chain_robust(&nh
->head
, val_up
, val_down
, v
);
288 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust
);
291 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
292 * @nh: Pointer to head of the blocking notifier chain
293 * @val: Value passed unmodified to notifier function
294 * @v: Pointer passed unmodified to notifier function
296 * Calls each function in a notifier chain in turn. The functions
297 * run in a process context, so they are allowed to block.
299 * If the return value of the notifier can be and'ed
300 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
301 * will return immediately, with the return value of
302 * the notifier function which halted execution.
303 * Otherwise the return value is the return value
304 * of the last notifier function called.
306 int blocking_notifier_call_chain(struct blocking_notifier_head
*nh
,
307 unsigned long val
, void *v
)
309 int ret
= NOTIFY_DONE
;
312 * We check the head outside the lock, but if this access is
313 * racy then it does not matter what the result of the test
314 * is, we re-check the list after having taken the lock anyway:
316 if (rcu_access_pointer(nh
->head
)) {
317 down_read(&nh
->rwsem
);
318 ret
= notifier_call_chain(&nh
->head
, val
, v
, -1, NULL
);
323 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain
);
326 * Raw notifier chain routines. There is no protection;
327 * the caller must provide it. Use at your own risk!
331 * raw_notifier_chain_register - Add notifier to a raw notifier chain
332 * @nh: Pointer to head of the raw notifier chain
333 * @n: New entry in notifier chain
335 * Adds a notifier to a raw notifier chain.
336 * All locking must be provided by the caller.
338 * Currently always returns zero.
340 int raw_notifier_chain_register(struct raw_notifier_head
*nh
,
341 struct notifier_block
*n
)
343 return notifier_chain_register(&nh
->head
, n
);
345 EXPORT_SYMBOL_GPL(raw_notifier_chain_register
);
348 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
349 * @nh: Pointer to head of the raw notifier chain
350 * @n: Entry to remove from notifier chain
352 * Removes a notifier from a raw notifier chain.
353 * All locking must be provided by the caller.
355 * Returns zero on success or %-ENOENT on failure.
357 int raw_notifier_chain_unregister(struct raw_notifier_head
*nh
,
358 struct notifier_block
*n
)
360 return notifier_chain_unregister(&nh
->head
, n
);
362 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister
);
364 int raw_notifier_call_chain_robust(struct raw_notifier_head
*nh
,
365 unsigned long val_up
, unsigned long val_down
, void *v
)
367 return notifier_call_chain_robust(&nh
->head
, val_up
, val_down
, v
);
369 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust
);
372 * raw_notifier_call_chain - Call functions in a raw notifier chain
373 * @nh: Pointer to head of the raw notifier chain
374 * @val: Value passed unmodified to notifier function
375 * @v: Pointer passed unmodified to notifier function
377 * Calls each function in a notifier chain in turn. The functions
378 * run in an undefined context.
379 * All locking must be provided by the caller.
381 * If the return value of the notifier can be and'ed
382 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
383 * will return immediately, with the return value of
384 * the notifier function which halted execution.
385 * Otherwise the return value is the return value
386 * of the last notifier function called.
388 int raw_notifier_call_chain(struct raw_notifier_head
*nh
,
389 unsigned long val
, void *v
)
391 return notifier_call_chain(&nh
->head
, val
, v
, -1, NULL
);
393 EXPORT_SYMBOL_GPL(raw_notifier_call_chain
);
397 * SRCU notifier chain routines. Registration and unregistration
398 * use a mutex, and call_chain is synchronized by SRCU (no locks).
402 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
403 * @nh: Pointer to head of the SRCU notifier chain
404 * @n: New entry in notifier chain
406 * Adds a notifier to an SRCU notifier chain.
407 * Must be called in process context.
409 * Currently always returns zero.
411 int srcu_notifier_chain_register(struct srcu_notifier_head
*nh
,
412 struct notifier_block
*n
)
417 * This code gets used during boot-up, when task switching is
418 * not yet working and interrupts must remain disabled. At
419 * such times we must not call mutex_lock().
421 if (unlikely(system_state
== SYSTEM_BOOTING
))
422 return notifier_chain_register(&nh
->head
, n
);
424 mutex_lock(&nh
->mutex
);
425 ret
= notifier_chain_register(&nh
->head
, n
);
426 mutex_unlock(&nh
->mutex
);
429 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register
);
432 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
433 * @nh: Pointer to head of the SRCU notifier chain
434 * @n: Entry to remove from notifier chain
436 * Removes a notifier from an SRCU notifier chain.
437 * Must be called from process context.
439 * Returns zero on success or %-ENOENT on failure.
441 int srcu_notifier_chain_unregister(struct srcu_notifier_head
*nh
,
442 struct notifier_block
*n
)
447 * This code gets used during boot-up, when task switching is
448 * not yet working and interrupts must remain disabled. At
449 * such times we must not call mutex_lock().
451 if (unlikely(system_state
== SYSTEM_BOOTING
))
452 return notifier_chain_unregister(&nh
->head
, n
);
454 mutex_lock(&nh
->mutex
);
455 ret
= notifier_chain_unregister(&nh
->head
, n
);
456 mutex_unlock(&nh
->mutex
);
457 synchronize_srcu(&nh
->srcu
);
460 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister
);
463 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
464 * @nh: Pointer to head of the SRCU notifier chain
465 * @val: Value passed unmodified to notifier function
466 * @v: Pointer passed unmodified to notifier function
468 * Calls each function in a notifier chain in turn. The functions
469 * run in a process context, so they are allowed to block.
471 * If the return value of the notifier can be and'ed
472 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
473 * will return immediately, with the return value of
474 * the notifier function which halted execution.
475 * Otherwise the return value is the return value
476 * of the last notifier function called.
478 int srcu_notifier_call_chain(struct srcu_notifier_head
*nh
,
479 unsigned long val
, void *v
)
484 idx
= srcu_read_lock(&nh
->srcu
);
485 ret
= notifier_call_chain(&nh
->head
, val
, v
, -1, NULL
);
486 srcu_read_unlock(&nh
->srcu
, idx
);
489 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain
);
492 * srcu_init_notifier_head - Initialize an SRCU notifier head
493 * @nh: Pointer to head of the srcu notifier chain
495 * Unlike other sorts of notifier heads, SRCU notifier heads require
496 * dynamic initialization. Be sure to call this routine before
497 * calling any of the other SRCU notifier routines for this head.
499 * If an SRCU notifier head is deallocated, it must first be cleaned
500 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
501 * per-cpu data (used by the SRCU mechanism) will leak.
503 void srcu_init_notifier_head(struct srcu_notifier_head
*nh
)
505 mutex_init(&nh
->mutex
);
506 if (init_srcu_struct(&nh
->srcu
) < 0)
510 EXPORT_SYMBOL_GPL(srcu_init_notifier_head
);
512 #endif /* CONFIG_SRCU */
514 static ATOMIC_NOTIFIER_HEAD(die_chain
);
516 int notrace
notify_die(enum die_val val
, const char *str
,
517 struct pt_regs
*regs
, long err
, int trap
, int sig
)
519 struct die_args args
= {
527 RCU_LOCKDEP_WARN(!rcu_is_watching(),
528 "notify_die called but RCU thinks we're quiescent");
529 return atomic_notifier_call_chain(&die_chain
, val
, &args
);
531 NOKPROBE_SYMBOL(notify_die
);
533 int register_die_notifier(struct notifier_block
*nb
)
535 return atomic_notifier_chain_register(&die_chain
, nb
);
537 EXPORT_SYMBOL_GPL(register_die_notifier
);
539 int unregister_die_notifier(struct notifier_block
*nb
)
541 return atomic_notifier_chain_unregister(&die_chain
, nb
);
543 EXPORT_SYMBOL_GPL(unregister_die_notifier
);