]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - include/linux/rcupdate.h
kernel: Remove undead ifdef CONFIG_DEBUG_LOCK_ALLOC
[mirror_ubuntu-bionic-kernel.git] / include / linux / rcupdate.h
CommitLineData
1da177e4 1/*
a71fca58 2 * Read-Copy Update mechanism for mutual exclusion
1da177e4
LT
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
01c1c660 18 * Copyright IBM Corporation, 2001
1da177e4
LT
19 *
20 * Author: Dipankar Sarma <dipankar@in.ibm.com>
a71fca58 21 *
595182bc 22 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
1da177e4
LT
23 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
24 * Papers:
25 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
26 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
27 *
28 * For detailed explanation of Read-Copy Update mechanism see -
a71fca58 29 * http://lse.sourceforge.net/locking/rcupdate.html
1da177e4
LT
30 *
31 */
32
33#ifndef __LINUX_RCUPDATE_H
34#define __LINUX_RCUPDATE_H
35
1da177e4
LT
36#include <linux/cache.h>
37#include <linux/spinlock.h>
38#include <linux/threads.h>
1da177e4
LT
39#include <linux/cpumask.h>
40#include <linux/seqlock.h>
851a67b8 41#include <linux/lockdep.h>
4446a36f 42#include <linux/completion.h>
551d55a9 43#include <linux/debugobjects.h>
ca5ecddf 44#include <linux/compiler.h>
1da177e4 45
e5ab6772
DY
46#ifdef CONFIG_RCU_TORTURE_TEST
47extern int rcutorture_runnable; /* for sysctl */
48#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
49
a3dc3fb1
PM
50#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
51#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
52
1da177e4
LT
53/**
54 * struct rcu_head - callback structure for use with RCU
55 * @next: next update requests in a list
56 * @func: actual update function to call after the grace period.
57 */
58struct rcu_head {
59 struct rcu_head *next;
60 void (*func)(struct rcu_head *head);
61};
62
03b042bf 63/* Exported common interfaces */
7b0b759b
PM
64extern void call_rcu_sched(struct rcu_head *head,
65 void (*func)(struct rcu_head *rcu));
66extern void synchronize_sched(void);
03b042bf
PM
67extern void rcu_barrier_bh(void);
68extern void rcu_barrier_sched(void);
69extern void synchronize_sched_expedited(void);
70extern int sched_expedited_torture_stats(char *page);
71
7b0b759b
PM
72static inline void __rcu_read_lock_bh(void)
73{
74 local_bh_disable();
75}
76
77static inline void __rcu_read_unlock_bh(void)
78{
79 local_bh_enable();
80}
a6826048 81
a3dc3fb1
PM
82#ifdef CONFIG_PREEMPT_RCU
83
7b0b759b
PM
84extern void __rcu_read_lock(void);
85extern void __rcu_read_unlock(void);
86void synchronize_rcu(void);
87
a3dc3fb1
PM
88/*
89 * Defined as a macro as it is a very low level header included from
90 * areas that don't even know about current. This gives the rcu_read_lock()
91 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
92 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
93 */
94#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
95
7b0b759b
PM
96#else /* #ifdef CONFIG_PREEMPT_RCU */
97
98static inline void __rcu_read_lock(void)
99{
100 preempt_disable();
101}
102
103static inline void __rcu_read_unlock(void)
104{
105 preempt_enable();
106}
107
108static inline void synchronize_rcu(void)
109{
110 synchronize_sched();
111}
112
113static inline int rcu_preempt_depth(void)
114{
115 return 0;
116}
117
118#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
119
120/* Internal to kernel */
121extern void rcu_init(void);
122extern void rcu_sched_qs(int cpu);
123extern void rcu_bh_qs(int cpu);
124extern void rcu_check_callbacks(int cpu, int user);
125struct notifier_block;
126
127#ifdef CONFIG_NO_HZ
128
129extern void rcu_enter_nohz(void);
130extern void rcu_exit_nohz(void);
131
132#else /* #ifdef CONFIG_NO_HZ */
133
134static inline void rcu_enter_nohz(void)
135{
136}
137
138static inline void rcu_exit_nohz(void)
139{
140}
141
142#endif /* #else #ifdef CONFIG_NO_HZ */
a3dc3fb1 143
f41d911f 144#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
64db4cff 145#include <linux/rcutree.h>
a57eb940 146#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
9b1d82fa 147#include <linux/rcutiny.h>
64db4cff
PM
148#else
149#error "Unknown RCU implementation specified to kernel configuration"
6b3ef48a 150#endif
01c1c660 151
551d55a9
MD
152/*
153 * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
154 * initialization and destruction of rcu_head on the stack. rcu_head structures
155 * allocated dynamically in the heap or defined statically don't need any
156 * initialization.
157 */
158#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
159extern void init_rcu_head_on_stack(struct rcu_head *head);
160extern void destroy_rcu_head_on_stack(struct rcu_head *head);
161#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
4376030a
MD
162static inline void init_rcu_head_on_stack(struct rcu_head *head)
163{
164}
165
166static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
167{
168}
551d55a9 169#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
4376030a 170
bc33f24b 171#ifdef CONFIG_DEBUG_LOCK_ALLOC
632ee200 172
bc33f24b 173extern struct lockdep_map rcu_lock_map;
632ee200
PM
174# define rcu_read_acquire() \
175 lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
bc33f24b 176# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
632ee200
PM
177
178extern struct lockdep_map rcu_bh_lock_map;
179# define rcu_read_acquire_bh() \
180 lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
181# define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_)
182
183extern struct lockdep_map rcu_sched_lock_map;
184# define rcu_read_acquire_sched() \
185 lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
186# define rcu_read_release_sched() \
187 lock_release(&rcu_sched_lock_map, 1, _THIS_IP_)
188
bc293d62 189extern int debug_lockdep_rcu_enabled(void);
54dbf96c 190
632ee200 191/**
ca5ecddf 192 * rcu_read_lock_held() - might we be in RCU read-side critical section?
632ee200 193 *
d20200b5
PM
194 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
195 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
632ee200 196 * this assumes we are in an RCU read-side critical section unless it can
ca5ecddf
PM
197 * prove otherwise. This is useful for debug checks in functions that
198 * require that they be called within an RCU read-side critical section.
54dbf96c 199 *
ca5ecddf 200 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
32c141a0 201 * and while lockdep is disabled.
632ee200
PM
202 */
203static inline int rcu_read_lock_held(void)
204{
54dbf96c
PM
205 if (!debug_lockdep_rcu_enabled())
206 return 1;
207 return lock_is_held(&rcu_lock_map);
632ee200
PM
208}
209
e3818b8d
PM
210/*
211 * rcu_read_lock_bh_held() is defined out of line to avoid #include-file
212 * hell.
632ee200 213 */
e3818b8d 214extern int rcu_read_lock_bh_held(void);
632ee200
PM
215
216/**
ca5ecddf 217 * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
632ee200 218 *
d20200b5
PM
219 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
220 * RCU-sched read-side critical section. In absence of
221 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
222 * critical section unless it can prove otherwise. Note that disabling
223 * of preemption (including disabling irqs) counts as an RCU-sched
ca5ecddf
PM
224 * read-side critical section. This is useful for debug checks in functions
225 * that required that they be called within an RCU-sched read-side
226 * critical section.
54dbf96c 227 *
32c141a0
PM
228 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
229 * and while lockdep is disabled.
632ee200 230 */
e6033e3b 231#ifdef CONFIG_PREEMPT
632ee200
PM
232static inline int rcu_read_lock_sched_held(void)
233{
234 int lockdep_opinion = 0;
235
54dbf96c
PM
236 if (!debug_lockdep_rcu_enabled())
237 return 1;
632ee200
PM
238 if (debug_locks)
239 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
0cff810f 240 return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
632ee200 241}
e6033e3b
PM
242#else /* #ifdef CONFIG_PREEMPT */
243static inline int rcu_read_lock_sched_held(void)
244{
245 return 1;
632ee200 246}
e6033e3b 247#endif /* #else #ifdef CONFIG_PREEMPT */
632ee200
PM
248
249#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
250
251# define rcu_read_acquire() do { } while (0)
252# define rcu_read_release() do { } while (0)
253# define rcu_read_acquire_bh() do { } while (0)
254# define rcu_read_release_bh() do { } while (0)
255# define rcu_read_acquire_sched() do { } while (0)
256# define rcu_read_release_sched() do { } while (0)
257
258static inline int rcu_read_lock_held(void)
259{
260 return 1;
261}
262
263static inline int rcu_read_lock_bh_held(void)
264{
265 return 1;
266}
267
e6033e3b 268#ifdef CONFIG_PREEMPT
632ee200
PM
269static inline int rcu_read_lock_sched_held(void)
270{
bbad9379 271 return preempt_count() != 0 || irqs_disabled();
632ee200 272}
e6033e3b
PM
273#else /* #ifdef CONFIG_PREEMPT */
274static inline int rcu_read_lock_sched_held(void)
275{
276 return 1;
632ee200 277}
e6033e3b 278#endif /* #else #ifdef CONFIG_PREEMPT */
632ee200
PM
279
280#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
281
282#ifdef CONFIG_PROVE_RCU
283
ee84b824
PM
284extern int rcu_my_thread_group_empty(void);
285
4221a991
TH
286/**
287 * rcu_lockdep_assert - emit lockdep splat if specified condition not met
288 * @c: condition to check
289 */
290#define rcu_lockdep_assert(c) \
2b3fc35f
LJ
291 do { \
292 static bool __warned; \
293 if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
294 __warned = true; \
295 lockdep_rcu_dereference(__FILE__, __LINE__); \
296 } \
297 } while (0)
298
ca5ecddf
PM
299#else /* #ifdef CONFIG_PROVE_RCU */
300
4221a991 301#define rcu_lockdep_assert(c) do { } while (0)
ca5ecddf
PM
302
303#endif /* #else #ifdef CONFIG_PROVE_RCU */
304
305/*
306 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
307 * and rcu_assign_pointer(). Some of these could be folded into their
308 * callers, but they are left separate in order to ease introduction of
309 * multiple flavors of pointers to match the multiple flavors of RCU
310 * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
311 * the future.
312 */
313#define __rcu_access_pointer(p, space) \
314 ({ \
315 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
316 (void) (((typeof (*p) space *)p) == p); \
317 ((typeof(*p) __force __kernel *)(_________p1)); \
318 })
319#define __rcu_dereference_check(p, c, space) \
320 ({ \
321 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
4221a991 322 rcu_lockdep_assert(c); \
ca5ecddf
PM
323 (void) (((typeof (*p) space *)p) == p); \
324 smp_read_barrier_depends(); \
325 ((typeof(*p) __force __kernel *)(_________p1)); \
326 })
327#define __rcu_dereference_protected(p, c, space) \
328 ({ \
4221a991 329 rcu_lockdep_assert(c); \
ca5ecddf
PM
330 (void) (((typeof (*p) space *)p) == p); \
331 ((typeof(*p) __force __kernel *)(p)); \
332 })
333
334#define __rcu_dereference_index_check(p, c) \
335 ({ \
336 typeof(p) _________p1 = ACCESS_ONCE(p); \
4221a991 337 rcu_lockdep_assert(c); \
ca5ecddf
PM
338 smp_read_barrier_depends(); \
339 (_________p1); \
340 })
341#define __rcu_assign_pointer(p, v, space) \
342 ({ \
343 if (!__builtin_constant_p(v) || \
344 ((v) != NULL)) \
345 smp_wmb(); \
346 (p) = (typeof(*v) __force space *)(v); \
347 })
348
349
350/**
351 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
352 * @p: The pointer to read
353 *
354 * Return the value of the specified RCU-protected pointer, but omit the
355 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
356 * when the value of this pointer is accessed, but the pointer is not
357 * dereferenced, for example, when testing an RCU-protected pointer against
358 * NULL. Although rcu_access_pointer() may also be used in cases where
359 * update-side locks prevent the value of the pointer from changing, you
360 * should instead use rcu_dereference_protected() for this use case.
361 */
362#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
363
632ee200 364/**
ca5ecddf 365 * rcu_dereference_check() - rcu_dereference with debug checking
c08c68dd
DH
366 * @p: The pointer to read, prior to dereferencing
367 * @c: The conditions under which the dereference will take place
632ee200 368 *
c08c68dd 369 * Do an rcu_dereference(), but check that the conditions under which the
ca5ecddf
PM
370 * dereference will take place are correct. Typically the conditions
371 * indicate the various locking conditions that should be held at that
372 * point. The check should return true if the conditions are satisfied.
373 * An implicit check for being in an RCU read-side critical section
374 * (rcu_read_lock()) is included.
c08c68dd
DH
375 *
376 * For example:
377 *
ca5ecddf 378 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
c08c68dd
DH
379 *
380 * could be used to indicate to lockdep that foo->bar may only be dereferenced
ca5ecddf 381 * if either rcu_read_lock() is held, or that the lock required to replace
c08c68dd
DH
382 * the bar struct at foo->bar is held.
383 *
384 * Note that the list of conditions may also include indications of when a lock
385 * need not be held, for example during initialisation or destruction of the
386 * target struct:
387 *
ca5ecddf 388 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
c08c68dd 389 * atomic_read(&foo->usage) == 0);
ca5ecddf
PM
390 *
391 * Inserts memory barriers on architectures that require them
392 * (currently only the Alpha), prevents the compiler from refetching
393 * (and from merging fetches), and, more importantly, documents exactly
394 * which pointers are protected by RCU and checks that the pointer is
395 * annotated as __rcu.
632ee200
PM
396 */
397#define rcu_dereference_check(p, c) \
ca5ecddf
PM
398 __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
399
400/**
401 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
402 * @p: The pointer to read, prior to dereferencing
403 * @c: The conditions under which the dereference will take place
404 *
405 * This is the RCU-bh counterpart to rcu_dereference_check().
406 */
407#define rcu_dereference_bh_check(p, c) \
408 __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
632ee200 409
b62730ba 410/**
ca5ecddf
PM
411 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
412 * @p: The pointer to read, prior to dereferencing
413 * @c: The conditions under which the dereference will take place
414 *
415 * This is the RCU-sched counterpart to rcu_dereference_check().
416 */
417#define rcu_dereference_sched_check(p, c) \
418 __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
419 __rcu)
420
421#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
422
423/**
424 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
425 * @p: The pointer to read, prior to dereferencing
426 * @c: The conditions under which the dereference will take place
427 *
428 * Similar to rcu_dereference_check(), but omits the sparse checking.
429 * This allows rcu_dereference_index_check() to be used on integers,
430 * which can then be used as array indices. Attempting to use
431 * rcu_dereference_check() on an integer will give compiler warnings
432 * because the sparse address-space mechanism relies on dereferencing
433 * the RCU-protected pointer. Dereferencing integers is not something
434 * that even gcc will put up with.
435 *
436 * Note that this function does not implicitly check for RCU read-side
437 * critical sections. If this function gains lots of uses, it might
438 * make sense to provide versions for each flavor of RCU, but it does
439 * not make sense as of early 2010.
440 */
441#define rcu_dereference_index_check(p, c) \
442 __rcu_dereference_index_check((p), (c))
443
444/**
445 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
446 * @p: The pointer to read, prior to dereferencing
447 * @c: The conditions under which the dereference will take place
b62730ba
PM
448 *
449 * Return the value of the specified RCU-protected pointer, but omit
450 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
451 * is useful in cases where update-side locks prevent the value of the
452 * pointer from changing. Please note that this primitive does -not-
453 * prevent the compiler from repeating this reference or combining it
454 * with other references, so it should not be used without protection
455 * of appropriate locks.
ca5ecddf
PM
456 *
457 * This function is only for update-side use. Using this function
458 * when protected only by rcu_read_lock() will result in infrequent
459 * but very ugly failures.
b62730ba
PM
460 */
461#define rcu_dereference_protected(p, c) \
ca5ecddf 462 __rcu_dereference_protected((p), (c), __rcu)
b62730ba 463
ca5ecddf
PM
464/**
465 * rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented
466 * @p: The pointer to read, prior to dereferencing
467 * @c: The conditions under which the dereference will take place
468 *
469 * This is the RCU-bh counterpart to rcu_dereference_protected().
470 */
471#define rcu_dereference_bh_protected(p, c) \
472 __rcu_dereference_protected((p), (c), __rcu)
632ee200 473
ca5ecddf
PM
474/**
475 * rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented
476 * @p: The pointer to read, prior to dereferencing
477 * @c: The conditions under which the dereference will take place
478 *
479 * This is the RCU-sched counterpart to rcu_dereference_protected().
480 */
481#define rcu_dereference_sched_protected(p, c) \
482 __rcu_dereference_protected((p), (c), __rcu)
632ee200 483
bc33f24b 484
b62730ba 485/**
ca5ecddf
PM
486 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
487 * @p: The pointer to read, prior to dereferencing
b62730ba 488 *
ca5ecddf 489 * This is a simple wrapper around rcu_dereference_check().
b62730ba 490 */
ca5ecddf 491#define rcu_dereference(p) rcu_dereference_check(p, 0)
b62730ba 492
1da177e4 493/**
ca5ecddf
PM
494 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
495 * @p: The pointer to read, prior to dereferencing
496 *
497 * Makes rcu_dereference_check() do the dirty work.
498 */
499#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
500
501/**
502 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
503 * @p: The pointer to read, prior to dereferencing
504 *
505 * Makes rcu_dereference_check() do the dirty work.
506 */
507#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
508
509/**
510 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
1da177e4 511 *
9b06e818 512 * When synchronize_rcu() is invoked on one CPU while other CPUs
1da177e4 513 * are within RCU read-side critical sections, then the
9b06e818 514 * synchronize_rcu() is guaranteed to block until after all the other
1da177e4
LT
515 * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
516 * on one CPU while other CPUs are within RCU read-side critical
517 * sections, invocation of the corresponding RCU callback is deferred
518 * until after the all the other CPUs exit their critical sections.
519 *
520 * Note, however, that RCU callbacks are permitted to run concurrently
77d8485a 521 * with new RCU read-side critical sections. One way that this can happen
1da177e4
LT
522 * is via the following sequence of events: (1) CPU 0 enters an RCU
523 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
524 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
525 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
526 * callback is invoked. This is legal, because the RCU read-side critical
527 * section that was running concurrently with the call_rcu() (and which
528 * therefore might be referencing something that the corresponding RCU
529 * callback would free up) has completed before the corresponding
530 * RCU callback is invoked.
531 *
532 * RCU read-side critical sections may be nested. Any deferred actions
533 * will be deferred until the outermost RCU read-side critical section
534 * completes.
535 *
9079fd7c
PM
536 * You can avoid reading and understanding the next paragraph by
537 * following this rule: don't put anything in an rcu_read_lock() RCU
538 * read-side critical section that would block in a !PREEMPT kernel.
539 * But if you want the full story, read on!
540 *
541 * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
542 * is illegal to block while in an RCU read-side critical section. In
543 * preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
544 * in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
545 * be preempted, but explicit blocking is illegal. Finally, in preemptible
546 * RCU implementations in real-time (CONFIG_PREEMPT_RT) kernel builds,
547 * RCU read-side critical sections may be preempted and they may also
548 * block, but only when acquiring spinlocks that are subject to priority
549 * inheritance.
1da177e4 550 */
bc33f24b
PM
551static inline void rcu_read_lock(void)
552{
553 __rcu_read_lock();
554 __acquire(RCU);
555 rcu_read_acquire();
556}
1da177e4 557
1da177e4
LT
558/*
559 * So where is rcu_write_lock()? It does not exist, as there is no
560 * way for writers to lock out RCU readers. This is a feature, not
561 * a bug -- this property is what provides RCU's performance benefits.
562 * Of course, writers must coordinate with each other. The normal
563 * spinlock primitives work well for this, but any other technique may be
564 * used as well. RCU does not care how the writers keep out of each
565 * others' way, as long as they do so.
566 */
3d76c082
PM
567
568/**
ca5ecddf 569 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
3d76c082
PM
570 *
571 * See rcu_read_lock() for more information.
572 */
bc33f24b
PM
573static inline void rcu_read_unlock(void)
574{
575 rcu_read_release();
576 __release(RCU);
577 __rcu_read_unlock();
578}
1da177e4
LT
579
580/**
ca5ecddf 581 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
1da177e4
LT
582 *
583 * This is equivalent of rcu_read_lock(), but to be used when updates
ca5ecddf
PM
584 * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
585 * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
586 * softirq handler to be a quiescent state, a process in RCU read-side
587 * critical section must be protected by disabling softirqs. Read-side
588 * critical sections in interrupt context can use just rcu_read_lock(),
589 * though this should at least be commented to avoid confusing people
590 * reading the code.
1da177e4 591 */
bc33f24b
PM
592static inline void rcu_read_lock_bh(void)
593{
594 __rcu_read_lock_bh();
595 __acquire(RCU_BH);
632ee200 596 rcu_read_acquire_bh();
bc33f24b 597}
1da177e4
LT
598
599/*
600 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
601 *
602 * See rcu_read_lock_bh() for more information.
603 */
bc33f24b
PM
604static inline void rcu_read_unlock_bh(void)
605{
632ee200 606 rcu_read_release_bh();
bc33f24b
PM
607 __release(RCU_BH);
608 __rcu_read_unlock_bh();
609}
1da177e4 610
1c50b728 611/**
ca5ecddf 612 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
1c50b728 613 *
ca5ecddf
PM
614 * This is equivalent of rcu_read_lock(), but to be used when updates
615 * are being done using call_rcu_sched() or synchronize_rcu_sched().
616 * Read-side critical sections can also be introduced by anything that
617 * disables preemption, including local_irq_disable() and friends.
1c50b728 618 */
d6714c22
PM
619static inline void rcu_read_lock_sched(void)
620{
621 preempt_disable();
bc33f24b 622 __acquire(RCU_SCHED);
632ee200 623 rcu_read_acquire_sched();
d6714c22 624}
1eba8f84
PM
625
626/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
7c614d64 627static inline notrace void rcu_read_lock_sched_notrace(void)
d6714c22
PM
628{
629 preempt_disable_notrace();
bc33f24b 630 __acquire(RCU_SCHED);
d6714c22 631}
1c50b728
MD
632
633/*
634 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
635 *
636 * See rcu_read_lock_sched for more information.
637 */
d6714c22
PM
638static inline void rcu_read_unlock_sched(void)
639{
632ee200 640 rcu_read_release_sched();
bc33f24b 641 __release(RCU_SCHED);
d6714c22
PM
642 preempt_enable();
643}
1eba8f84
PM
644
645/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
7c614d64 646static inline notrace void rcu_read_unlock_sched_notrace(void)
d6714c22 647{
bc33f24b 648 __release(RCU_SCHED);
d6714c22
PM
649 preempt_enable_notrace();
650}
1c50b728 651
1da177e4 652/**
ca5ecddf
PM
653 * rcu_assign_pointer() - assign to RCU-protected pointer
654 * @p: pointer to assign to
655 * @v: value to assign (publish)
c26d34a5 656 *
ca5ecddf
PM
657 * Assigns the specified value to the specified RCU-protected
658 * pointer, ensuring that any concurrent RCU readers will see
659 * any prior initialization. Returns the value assigned.
1da177e4
LT
660 *
661 * Inserts memory barriers on architectures that require them
662 * (pretty much all of them other than x86), and also prevents
663 * the compiler from reordering the code that initializes the
664 * structure after the pointer assignment. More importantly, this
665 * call documents which pointers will be dereferenced by RCU read-side
666 * code.
667 */
d99c4f6b 668#define rcu_assign_pointer(p, v) \
ca5ecddf
PM
669 __rcu_assign_pointer((p), (v), __rcu)
670
671/**
672 * RCU_INIT_POINTER() - initialize an RCU protected pointer
673 *
674 * Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep
675 * splats.
676 */
677#define RCU_INIT_POINTER(p, v) \
678 p = (typeof(*v) __force __rcu *)(v)
1da177e4 679
4446a36f
PM
680/* Infrastructure to implement the synchronize_() primitives. */
681
682struct rcu_synchronize {
683 struct rcu_head head;
684 struct completion completion;
685};
686
687extern void wakeme_after_rcu(struct rcu_head *head);
688
7b0b759b
PM
689#ifdef CONFIG_PREEMPT_RCU
690
01c1c660 691/**
ca5ecddf 692 * call_rcu() - Queue an RCU callback for invocation after a grace period.
01c1c660 693 * @head: structure to be used for queueing the RCU updates.
77d8485a 694 * @func: actual callback function to be invoked after the grace period
01c1c660 695 *
77d8485a
PM
696 * The callback function will be invoked some time after a full grace
697 * period elapses, in other words after all pre-existing RCU read-side
698 * critical sections have completed. However, the callback function
699 * might well execute concurrently with RCU read-side critical sections
700 * that started after call_rcu() was invoked. RCU read-side critical
01c1c660
PM
701 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
702 * and may be nested.
703 */
704extern void call_rcu(struct rcu_head *head,
705 void (*func)(struct rcu_head *head));
706
7b0b759b
PM
707#else /* #ifdef CONFIG_PREEMPT_RCU */
708
709/* In classic RCU, call_rcu() is just call_rcu_sched(). */
710#define call_rcu call_rcu_sched
711
712#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
713
01c1c660 714/**
ca5ecddf 715 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
01c1c660 716 * @head: structure to be used for queueing the RCU updates.
77d8485a 717 * @func: actual callback function to be invoked after the grace period
01c1c660 718 *
77d8485a 719 * The callback function will be invoked some time after a full grace
01c1c660
PM
720 * period elapses, in other words after all currently executing RCU
721 * read-side critical sections have completed. call_rcu_bh() assumes
722 * that the read-side critical sections end on completion of a softirq
723 * handler. This means that read-side critical sections in process
724 * context must not be interrupted by softirqs. This interface is to be
725 * used when most of the read-side critical sections are in softirq context.
726 * RCU read-side critical sections are delimited by :
727 * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
728 * OR
729 * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
730 * These may be nested.
731 */
732extern void call_rcu_bh(struct rcu_head *head,
733 void (*func)(struct rcu_head *head));
734
551d55a9
MD
735/*
736 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
737 * by call_rcu() and rcu callback execution, and are therefore not part of the
738 * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
739 */
740
741#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
742# define STATE_RCU_HEAD_READY 0
743# define STATE_RCU_HEAD_QUEUED 1
744
745extern struct debug_obj_descr rcuhead_debug_descr;
746
747static inline void debug_rcu_head_queue(struct rcu_head *head)
748{
749 debug_object_activate(head, &rcuhead_debug_descr);
750 debug_object_active_state(head, &rcuhead_debug_descr,
751 STATE_RCU_HEAD_READY,
752 STATE_RCU_HEAD_QUEUED);
753}
754
755static inline void debug_rcu_head_unqueue(struct rcu_head *head)
756{
757 debug_object_active_state(head, &rcuhead_debug_descr,
758 STATE_RCU_HEAD_QUEUED,
759 STATE_RCU_HEAD_READY);
760 debug_object_deactivate(head, &rcuhead_debug_descr);
761}
762#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
763static inline void debug_rcu_head_queue(struct rcu_head *head)
764{
765}
766
767static inline void debug_rcu_head_unqueue(struct rcu_head *head)
768{
769}
770#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
771
1da177e4 772#endif /* __LINUX_RCUPDATE_H */