2 * Read-Copy Update mechanism for mutual exclusion
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright IBM Corporation, 2008
20 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
21 * Manfred Spraul <manfred@colorfullife.com>
22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
24 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
25 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
27 * For detailed explanation of Read-Copy Update mechanism see -
30 #include <linux/types.h>
31 #include <linux/kernel.h>
32 #include <linux/init.h>
33 #include <linux/spinlock.h>
34 #include <linux/smp.h>
35 #include <linux/rcupdate_wait.h>
36 #include <linux/interrupt.h>
37 #include <linux/sched.h>
38 #include <linux/sched/debug.h>
39 #include <linux/nmi.h>
40 #include <linux/atomic.h>
41 #include <linux/bitops.h>
42 #include <linux/export.h>
43 #include <linux/completion.h>
44 #include <linux/moduleparam.h>
45 #include <linux/percpu.h>
46 #include <linux/notifier.h>
47 #include <linux/cpu.h>
48 #include <linux/mutex.h>
49 #include <linux/time.h>
50 #include <linux/kernel_stat.h>
51 #include <linux/wait.h>
52 #include <linux/kthread.h>
53 #include <uapi/linux/sched/types.h>
54 #include <linux/prefetch.h>
55 #include <linux/delay.h>
56 #include <linux/stop_machine.h>
57 #include <linux/random.h>
58 #include <linux/trace_events.h>
59 #include <linux/suspend.h>
60 #include <linux/ftrace.h>
65 #ifdef MODULE_PARAM_PREFIX
66 #undef MODULE_PARAM_PREFIX
68 #define MODULE_PARAM_PREFIX "rcutree."
70 /* Data structures. */
73 * In order to export the rcu_state name to the tracing tools, it
74 * needs to be added in the __tracepoint_string section.
75 * This requires defining a separate variable tp_<sname>_varname
76 * that points to the string being used, and this will allow
77 * the tracing userspace tools to be able to decipher the string
78 * address to the matching string.
81 # define DEFINE_RCU_TPS(sname) \
82 static char sname##_varname[] = #sname; \
83 static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;
84 # define RCU_STATE_NAME(sname) sname##_varname
86 # define DEFINE_RCU_TPS(sname)
87 # define RCU_STATE_NAME(sname) __stringify(sname)
90 #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
91 DEFINE_RCU_TPS(sname) \
92 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
93 struct rcu_state sname##_state = { \
94 .level = { &sname##_state.node[0] }, \
95 .rda = &sname##_data, \
97 .gp_state = RCU_GP_IDLE, \
98 .gpnum = 0UL - 300UL, \
99 .completed = 0UL - 300UL, \
100 .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \
101 .name = RCU_STATE_NAME(sname), \
103 .exp_mutex = __MUTEX_INITIALIZER(sname##_state.exp_mutex), \
104 .exp_wake_mutex = __MUTEX_INITIALIZER(sname##_state.exp_wake_mutex), \
107 RCU_STATE_INITIALIZER(rcu_sched
, 's', call_rcu_sched
);
108 RCU_STATE_INITIALIZER(rcu_bh
, 'b', call_rcu_bh
);
110 static struct rcu_state
*const rcu_state_p
;
111 LIST_HEAD(rcu_struct_flavors
);
113 /* Dump rcu_node combining tree at boot to verify correct setup. */
114 static bool dump_tree
;
115 module_param(dump_tree
, bool, 0444);
116 /* Control rcu_node-tree auto-balancing at boot time. */
117 static bool rcu_fanout_exact
;
118 module_param(rcu_fanout_exact
, bool, 0444);
119 /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
120 static int rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
121 module_param(rcu_fanout_leaf
, int, 0444);
122 int rcu_num_lvls __read_mostly
= RCU_NUM_LVLS
;
123 /* Number of rcu_nodes at specified level. */
124 int num_rcu_lvl
[] = NUM_RCU_LVL_INIT
;
125 int rcu_num_nodes __read_mostly
= NUM_RCU_NODES
; /* Total # rcu_nodes in use. */
126 /* panic() on RCU Stall sysctl. */
127 int sysctl_panic_on_rcu_stall __read_mostly
;
130 * The rcu_scheduler_active variable is initialized to the value
131 * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the
132 * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE,
133 * RCU can assume that there is but one task, allowing RCU to (for example)
134 * optimize synchronize_rcu() to a simple barrier(). When this variable
135 * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required
136 * to detect real grace periods. This variable is also used to suppress
137 * boot-time false positives from lockdep-RCU error checking. Finally, it
138 * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU
139 * is fully initialized, including all of its kthreads having been spawned.
141 int rcu_scheduler_active __read_mostly
;
142 EXPORT_SYMBOL_GPL(rcu_scheduler_active
);
145 * The rcu_scheduler_fully_active variable transitions from zero to one
146 * during the early_initcall() processing, which is after the scheduler
147 * is capable of creating new tasks. So RCU processing (for example,
148 * creating tasks for RCU priority boosting) must be delayed until after
149 * rcu_scheduler_fully_active transitions from zero to one. We also
150 * currently delay invocation of any RCU callbacks until after this point.
152 * It might later prove better for people registering RCU callbacks during
153 * early boot to take responsibility for these callbacks, but one step at
156 static int rcu_scheduler_fully_active __read_mostly
;
158 static void rcu_init_new_rnp(struct rcu_node
*rnp_leaf
);
159 static void rcu_cleanup_dead_rnp(struct rcu_node
*rnp_leaf
);
160 static void rcu_boost_kthread_setaffinity(struct rcu_node
*rnp
, int outgoingcpu
);
161 static void invoke_rcu_core(void);
162 static void invoke_rcu_callbacks(struct rcu_state
*rsp
, struct rcu_data
*rdp
);
163 static void rcu_report_exp_rdp(struct rcu_state
*rsp
,
164 struct rcu_data
*rdp
, bool wake
);
165 static void sync_sched_exp_online_cleanup(int cpu
);
167 /* rcuc/rcub kthread realtime priority */
168 static int kthread_prio
= IS_ENABLED(CONFIG_RCU_BOOST
) ? 1 : 0;
169 module_param(kthread_prio
, int, 0644);
171 /* Delay in jiffies for grace-period initialization delays, debug only. */
173 static int gp_preinit_delay
;
174 module_param(gp_preinit_delay
, int, 0444);
175 static int gp_init_delay
;
176 module_param(gp_init_delay
, int, 0444);
177 static int gp_cleanup_delay
;
178 module_param(gp_cleanup_delay
, int, 0444);
181 * Number of grace periods between delays, normalized by the duration of
182 * the delay. The longer the delay, the more the grace periods between
183 * each delay. The reason for this normalization is that it means that,
184 * for non-zero delays, the overall slowdown of grace periods is constant
185 * regardless of the duration of the delay. This arrangement balances
186 * the need for long delays to increase some race probabilities with the
187 * need for fast grace periods to increase other race probabilities.
189 #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */
192 * Track the rcutorture test sequence number and the update version
193 * number within a given test. The rcutorture_testseq is incremented
194 * on every rcutorture module load and unload, so has an odd value
195 * when a test is running. The rcutorture_vernum is set to zero
196 * when rcutorture starts and is incremented on each rcutorture update.
197 * These variables enable correlating rcutorture output with the
198 * RCU tracing information.
200 unsigned long rcutorture_testseq
;
201 unsigned long rcutorture_vernum
;
204 * Compute the mask of online CPUs for the specified rcu_node structure.
205 * This will not be stable unless the rcu_node structure's ->lock is
206 * held, but the bit corresponding to the current CPU will be stable
209 unsigned long rcu_rnp_online_cpus(struct rcu_node
*rnp
)
211 return READ_ONCE(rnp
->qsmaskinitnext
);
215 * Return true if an RCU grace period is in progress. The READ_ONCE()s
216 * permit this function to be invoked without holding the root rcu_node
217 * structure's ->lock, but of course results can be subject to change.
219 static int rcu_gp_in_progress(struct rcu_state
*rsp
)
221 return READ_ONCE(rsp
->completed
) != READ_ONCE(rsp
->gpnum
);
225 * Note a quiescent state. Because we do not need to know
226 * how many quiescent states passed, just if there was at least
227 * one since the start of the grace period, this just sets a flag.
228 * The caller must have disabled preemption.
230 void rcu_sched_qs(void)
232 RCU_LOCKDEP_WARN(preemptible(), "rcu_sched_qs() invoked with preemption enabled!!!");
233 if (!__this_cpu_read(rcu_sched_data
.cpu_no_qs
.s
))
235 trace_rcu_grace_period(TPS("rcu_sched"),
236 __this_cpu_read(rcu_sched_data
.gpnum
),
238 __this_cpu_write(rcu_sched_data
.cpu_no_qs
.b
.norm
, false);
239 if (!__this_cpu_read(rcu_sched_data
.cpu_no_qs
.b
.exp
))
241 __this_cpu_write(rcu_sched_data
.cpu_no_qs
.b
.exp
, false);
242 rcu_report_exp_rdp(&rcu_sched_state
,
243 this_cpu_ptr(&rcu_sched_data
), true);
248 RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
249 if (__this_cpu_read(rcu_bh_data
.cpu_no_qs
.s
)) {
250 trace_rcu_grace_period(TPS("rcu_bh"),
251 __this_cpu_read(rcu_bh_data
.gpnum
),
253 __this_cpu_write(rcu_bh_data
.cpu_no_qs
.b
.norm
, false);
258 * Steal a bit from the bottom of ->dynticks for idle entry/exit
259 * control. Initially this is for TLB flushing.
261 #define RCU_DYNTICK_CTRL_MASK 0x1
262 #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1)
263 #ifndef rcu_eqs_special_exit
264 #define rcu_eqs_special_exit() do { } while (0)
267 static DEFINE_PER_CPU(struct rcu_dynticks
, rcu_dynticks
) = {
268 .dynticks_nesting
= DYNTICK_TASK_EXIT_IDLE
,
269 .dynticks
= ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR
),
273 * There's a few places, currently just in the tracing infrastructure,
274 * that uses rcu_irq_enter() to make sure RCU is watching. But there's
275 * a small location where that will not even work. In those cases
276 * rcu_irq_enter_disabled() needs to be checked to make sure rcu_irq_enter()
279 static DEFINE_PER_CPU(bool, disable_rcu_irq_enter
);
281 bool rcu_irq_enter_disabled(void)
283 return this_cpu_read(disable_rcu_irq_enter
);
287 * Record entry into an extended quiescent state. This is only to be
288 * called when not already in an extended quiescent state.
290 static void rcu_dynticks_eqs_enter(void)
292 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
296 * CPUs seeing atomic_add_return() must see prior RCU read-side
297 * critical sections, and we also must force ordering with the
300 seq
= atomic_add_return(RCU_DYNTICK_CTRL_CTR
, &rdtp
->dynticks
);
301 /* Better be in an extended quiescent state! */
302 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
303 (seq
& RCU_DYNTICK_CTRL_CTR
));
304 /* Better not have special action (TLB flush) pending! */
305 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
306 (seq
& RCU_DYNTICK_CTRL_MASK
));
310 * Record exit from an extended quiescent state. This is only to be
311 * called from an extended quiescent state.
313 static void rcu_dynticks_eqs_exit(void)
315 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
319 * CPUs seeing atomic_add_return() must see prior idle sojourns,
320 * and we also must force ordering with the next RCU read-side
323 seq
= atomic_add_return(RCU_DYNTICK_CTRL_CTR
, &rdtp
->dynticks
);
324 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
325 !(seq
& RCU_DYNTICK_CTRL_CTR
));
326 if (seq
& RCU_DYNTICK_CTRL_MASK
) {
327 atomic_andnot(RCU_DYNTICK_CTRL_MASK
, &rdtp
->dynticks
);
328 smp_mb__after_atomic(); /* _exit after clearing mask. */
329 /* Prefer duplicate flushes to losing a flush. */
330 rcu_eqs_special_exit();
335 * Reset the current CPU's ->dynticks counter to indicate that the
336 * newly onlined CPU is no longer in an extended quiescent state.
337 * This will either leave the counter unchanged, or increment it
338 * to the next non-quiescent value.
340 * The non-atomic test/increment sequence works because the upper bits
341 * of the ->dynticks counter are manipulated only by the corresponding CPU,
342 * or when the corresponding CPU is offline.
344 static void rcu_dynticks_eqs_online(void)
346 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
348 if (atomic_read(&rdtp
->dynticks
) & RCU_DYNTICK_CTRL_CTR
)
350 atomic_add(RCU_DYNTICK_CTRL_CTR
, &rdtp
->dynticks
);
354 * Is the current CPU in an extended quiescent state?
356 * No ordering, as we are sampling CPU-local information.
358 bool rcu_dynticks_curr_cpu_in_eqs(void)
360 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
362 return !(atomic_read(&rdtp
->dynticks
) & RCU_DYNTICK_CTRL_CTR
);
366 * Snapshot the ->dynticks counter with full ordering so as to allow
367 * stable comparison of this counter with past and future snapshots.
369 int rcu_dynticks_snap(struct rcu_dynticks
*rdtp
)
371 int snap
= atomic_add_return(0, &rdtp
->dynticks
);
373 return snap
& ~RCU_DYNTICK_CTRL_MASK
;
377 * Return true if the snapshot returned from rcu_dynticks_snap()
378 * indicates that RCU is in an extended quiescent state.
380 static bool rcu_dynticks_in_eqs(int snap
)
382 return !(snap
& RCU_DYNTICK_CTRL_CTR
);
386 * Return true if the CPU corresponding to the specified rcu_dynticks
387 * structure has spent some time in an extended quiescent state since
388 * rcu_dynticks_snap() returned the specified snapshot.
390 static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks
*rdtp
, int snap
)
392 return snap
!= rcu_dynticks_snap(rdtp
);
396 * Do a double-increment of the ->dynticks counter to emulate a
397 * momentary idle-CPU quiescent state.
399 static void rcu_dynticks_momentary_idle(void)
401 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
402 int special
= atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR
,
405 /* It is illegal to call this from idle state. */
406 WARN_ON_ONCE(!(special
& RCU_DYNTICK_CTRL_CTR
));
410 * Set the special (bottom) bit of the specified CPU so that it
411 * will take special action (such as flushing its TLB) on the
412 * next exit from an extended quiescent state. Returns true if
413 * the bit was successfully set, or false if the CPU was not in
414 * an extended quiescent state.
416 bool rcu_eqs_special_set(int cpu
)
420 struct rcu_dynticks
*rdtp
= &per_cpu(rcu_dynticks
, cpu
);
423 old
= atomic_read(&rdtp
->dynticks
);
424 if (old
& RCU_DYNTICK_CTRL_CTR
)
426 new = old
| RCU_DYNTICK_CTRL_MASK
;
427 } while (atomic_cmpxchg(&rdtp
->dynticks
, old
, new) != old
);
432 * Let the RCU core know that this CPU has gone through the scheduler,
433 * which is a quiescent state. This is called when the need for a
434 * quiescent state is urgent, so we burn an atomic operation and full
435 * memory barriers to let the RCU core know about it, regardless of what
436 * this CPU might (or might not) do in the near future.
438 * We inform the RCU core by emulating a zero-duration dyntick-idle period.
440 * The caller must have disabled interrupts.
442 static void rcu_momentary_dyntick_idle(void)
444 raw_cpu_write(rcu_dynticks
.rcu_need_heavy_qs
, false);
445 rcu_dynticks_momentary_idle();
449 * Note a context switch. This is a quiescent state for RCU-sched,
450 * and requires special handling for preemptible RCU.
451 * The caller must have disabled interrupts.
453 void rcu_note_context_switch(bool preempt
)
455 barrier(); /* Avoid RCU read-side critical sections leaking down. */
456 trace_rcu_utilization(TPS("Start context switch"));
458 rcu_preempt_note_context_switch(preempt
);
459 /* Load rcu_urgent_qs before other flags. */
460 if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks
.rcu_urgent_qs
)))
462 this_cpu_write(rcu_dynticks
.rcu_urgent_qs
, false);
463 if (unlikely(raw_cpu_read(rcu_dynticks
.rcu_need_heavy_qs
)))
464 rcu_momentary_dyntick_idle();
465 this_cpu_inc(rcu_dynticks
.rcu_qs_ctr
);
467 rcu_note_voluntary_context_switch_lite(current
);
469 trace_rcu_utilization(TPS("End context switch"));
470 barrier(); /* Avoid RCU read-side critical sections leaking up. */
472 EXPORT_SYMBOL_GPL(rcu_note_context_switch
);
475 * Register a quiescent state for all RCU flavors. If there is an
476 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
477 * dyntick-idle quiescent state visible to other CPUs (but only for those
478 * RCU flavors in desperate need of a quiescent state, which will normally
479 * be none of them). Either way, do a lightweight quiescent state for
482 * The barrier() calls are redundant in the common case when this is
483 * called externally, but just in case this is called from within this
487 void rcu_all_qs(void)
491 if (!raw_cpu_read(rcu_dynticks
.rcu_urgent_qs
))
494 /* Load rcu_urgent_qs before other flags. */
495 if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks
.rcu_urgent_qs
))) {
499 this_cpu_write(rcu_dynticks
.rcu_urgent_qs
, false);
500 barrier(); /* Avoid RCU read-side critical sections leaking down. */
501 if (unlikely(raw_cpu_read(rcu_dynticks
.rcu_need_heavy_qs
))) {
502 local_irq_save(flags
);
503 rcu_momentary_dyntick_idle();
504 local_irq_restore(flags
);
506 if (unlikely(raw_cpu_read(rcu_sched_data
.cpu_no_qs
.b
.exp
)))
508 this_cpu_inc(rcu_dynticks
.rcu_qs_ctr
);
509 barrier(); /* Avoid RCU read-side critical sections leaking up. */
512 EXPORT_SYMBOL_GPL(rcu_all_qs
);
514 #define DEFAULT_RCU_BLIMIT 10 /* Maximum callbacks per rcu_do_batch. */
515 static long blimit
= DEFAULT_RCU_BLIMIT
;
516 #define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
517 static long qhimark
= DEFAULT_RCU_QHIMARK
;
518 #define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
519 static long qlowmark
= DEFAULT_RCU_QLOMARK
;
521 module_param(blimit
, long, 0444);
522 module_param(qhimark
, long, 0444);
523 module_param(qlowmark
, long, 0444);
525 static ulong jiffies_till_first_fqs
= ULONG_MAX
;
526 static ulong jiffies_till_next_fqs
= ULONG_MAX
;
527 static bool rcu_kick_kthreads
;
529 module_param(jiffies_till_first_fqs
, ulong
, 0644);
530 module_param(jiffies_till_next_fqs
, ulong
, 0644);
531 module_param(rcu_kick_kthreads
, bool, 0644);
534 * How long the grace period must be before we start recruiting
535 * quiescent-state help from rcu_note_context_switch().
537 static ulong jiffies_till_sched_qs
= HZ
/ 20;
538 module_param(jiffies_till_sched_qs
, ulong
, 0644);
540 static bool rcu_start_gp_advanced(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
541 struct rcu_data
*rdp
);
542 static void force_qs_rnp(struct rcu_state
*rsp
, int (*f
)(struct rcu_data
*rsp
));
543 static void force_quiescent_state(struct rcu_state
*rsp
);
544 static int rcu_pending(void);
547 * Return the number of RCU batches started thus far for debug & stats.
549 unsigned long rcu_batches_started(void)
551 return rcu_state_p
->gpnum
;
553 EXPORT_SYMBOL_GPL(rcu_batches_started
);
556 * Return the number of RCU-sched batches started thus far for debug & stats.
558 unsigned long rcu_batches_started_sched(void)
560 return rcu_sched_state
.gpnum
;
562 EXPORT_SYMBOL_GPL(rcu_batches_started_sched
);
565 * Return the number of RCU BH batches started thus far for debug & stats.
567 unsigned long rcu_batches_started_bh(void)
569 return rcu_bh_state
.gpnum
;
571 EXPORT_SYMBOL_GPL(rcu_batches_started_bh
);
574 * Return the number of RCU batches completed thus far for debug & stats.
576 unsigned long rcu_batches_completed(void)
578 return rcu_state_p
->completed
;
580 EXPORT_SYMBOL_GPL(rcu_batches_completed
);
583 * Return the number of RCU-sched batches completed thus far for debug & stats.
585 unsigned long rcu_batches_completed_sched(void)
587 return rcu_sched_state
.completed
;
589 EXPORT_SYMBOL_GPL(rcu_batches_completed_sched
);
592 * Return the number of RCU BH batches completed thus far for debug & stats.
594 unsigned long rcu_batches_completed_bh(void)
596 return rcu_bh_state
.completed
;
598 EXPORT_SYMBOL_GPL(rcu_batches_completed_bh
);
601 * Return the number of RCU expedited batches completed thus far for
602 * debug & stats. Odd numbers mean that a batch is in progress, even
603 * numbers mean idle. The value returned will thus be roughly double
604 * the cumulative batches since boot.
606 unsigned long rcu_exp_batches_completed(void)
608 return rcu_state_p
->expedited_sequence
;
610 EXPORT_SYMBOL_GPL(rcu_exp_batches_completed
);
613 * Return the number of RCU-sched expedited batches completed thus far
614 * for debug & stats. Similar to rcu_exp_batches_completed().
616 unsigned long rcu_exp_batches_completed_sched(void)
618 return rcu_sched_state
.expedited_sequence
;
620 EXPORT_SYMBOL_GPL(rcu_exp_batches_completed_sched
);
623 * Force a quiescent state.
625 void rcu_force_quiescent_state(void)
627 force_quiescent_state(rcu_state_p
);
629 EXPORT_SYMBOL_GPL(rcu_force_quiescent_state
);
632 * Force a quiescent state for RCU BH.
634 void rcu_bh_force_quiescent_state(void)
636 force_quiescent_state(&rcu_bh_state
);
638 EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state
);
641 * Force a quiescent state for RCU-sched.
643 void rcu_sched_force_quiescent_state(void)
645 force_quiescent_state(&rcu_sched_state
);
647 EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state
);
650 * Show the state of the grace-period kthreads.
652 void show_rcu_gp_kthreads(void)
654 struct rcu_state
*rsp
;
656 for_each_rcu_flavor(rsp
) {
657 pr_info("%s: wait state: %d ->state: %#lx\n",
658 rsp
->name
, rsp
->gp_state
, rsp
->gp_kthread
->state
);
659 /* sched_show_task(rsp->gp_kthread); */
662 EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads
);
665 * Record the number of times rcutorture tests have been initiated and
666 * terminated. This information allows the debugfs tracing stats to be
667 * correlated to the rcutorture messages, even when the rcutorture module
668 * is being repeatedly loaded and unloaded. In other words, we cannot
669 * store this state in rcutorture itself.
671 void rcutorture_record_test_transition(void)
673 rcutorture_testseq
++;
674 rcutorture_vernum
= 0;
676 EXPORT_SYMBOL_GPL(rcutorture_record_test_transition
);
679 * Send along grace-period-related data for rcutorture diagnostics.
681 void rcutorture_get_gp_data(enum rcutorture_type test_type
, int *flags
,
682 unsigned long *gpnum
, unsigned long *completed
)
684 struct rcu_state
*rsp
= NULL
;
693 case RCU_SCHED_FLAVOR
:
694 rsp
= &rcu_sched_state
;
701 *flags
= READ_ONCE(rsp
->gp_flags
);
702 *gpnum
= READ_ONCE(rsp
->gpnum
);
703 *completed
= READ_ONCE(rsp
->completed
);
705 EXPORT_SYMBOL_GPL(rcutorture_get_gp_data
);
708 * Record the number of writer passes through the current rcutorture test.
709 * This is also used to correlate debugfs tracing stats with the rcutorture
712 void rcutorture_record_progress(unsigned long vernum
)
716 EXPORT_SYMBOL_GPL(rcutorture_record_progress
);
719 * Return the root node of the specified rcu_state structure.
721 static struct rcu_node
*rcu_get_root(struct rcu_state
*rsp
)
723 return &rsp
->node
[0];
727 * Is there any need for future grace periods?
728 * Interrupts must be disabled. If the caller does not hold the root
729 * rnp_node structure's ->lock, the results are advisory only.
731 static int rcu_future_needs_gp(struct rcu_state
*rsp
)
733 struct rcu_node
*rnp
= rcu_get_root(rsp
);
734 int idx
= (READ_ONCE(rnp
->completed
) + 1) & 0x1;
735 int *fp
= &rnp
->need_future_gp
[idx
];
737 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_future_needs_gp() invoked with irqs enabled!!!");
738 return READ_ONCE(*fp
);
742 * Does the current CPU require a not-yet-started grace period?
743 * The caller must have disabled interrupts to prevent races with
744 * normal callback registry.
747 cpu_needs_another_gp(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
749 RCU_LOCKDEP_WARN(!irqs_disabled(), "cpu_needs_another_gp() invoked with irqs enabled!!!");
750 if (rcu_gp_in_progress(rsp
))
751 return false; /* No, a grace period is already in progress. */
752 if (rcu_future_needs_gp(rsp
))
753 return true; /* Yes, a no-CBs CPU needs one. */
754 if (!rcu_segcblist_is_enabled(&rdp
->cblist
))
755 return false; /* No, this is a no-CBs (or offline) CPU. */
756 if (!rcu_segcblist_restempty(&rdp
->cblist
, RCU_NEXT_READY_TAIL
))
757 return true; /* Yes, CPU has newly registered callbacks. */
758 if (rcu_segcblist_future_gp_needed(&rdp
->cblist
,
759 READ_ONCE(rsp
->completed
)))
760 return true; /* Yes, CBs for future grace period. */
761 return false; /* No grace period needed. */
765 * rcu_eqs_enter_common - current CPU is entering an extended quiescent state
767 * Enter idle, doing appropriate accounting. The caller must have
768 * disabled interrupts.
770 static void rcu_eqs_enter_common(bool user
)
772 struct rcu_state
*rsp
;
773 struct rcu_data
*rdp
;
774 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
776 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_eqs_enter_common() invoked with irqs enabled!!!");
777 trace_rcu_dyntick(TPS("Start"), rdtp
->dynticks_nesting
, 0);
778 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
779 !user
&& !is_idle_task(current
)) {
780 struct task_struct
*idle __maybe_unused
=
781 idle_task(smp_processor_id());
783 trace_rcu_dyntick(TPS("Error on entry: not idle task"), rdtp
->dynticks_nesting
, 0);
784 rcu_ftrace_dump(DUMP_ORIG
);
785 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
786 current
->pid
, current
->comm
,
787 idle
->pid
, idle
->comm
); /* must be idle task! */
789 for_each_rcu_flavor(rsp
) {
790 rdp
= this_cpu_ptr(rsp
->rda
);
791 do_nocb_deferred_wakeup(rdp
);
793 rcu_prepare_for_idle();
794 __this_cpu_inc(disable_rcu_irq_enter
);
795 rdtp
->dynticks_nesting
= 0; /* Breaks tracing momentarily. */
796 rcu_dynticks_eqs_enter(); /* After this, tracing works again. */
797 __this_cpu_dec(disable_rcu_irq_enter
);
798 rcu_dynticks_task_enter();
801 * It is illegal to enter an extended quiescent state while
802 * in an RCU read-side critical section.
804 RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map
),
805 "Illegal idle entry in RCU read-side critical section.");
806 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map
),
807 "Illegal idle entry in RCU-bh read-side critical section.");
808 RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map
),
809 "Illegal idle entry in RCU-sched read-side critical section.");
813 * Enter an RCU extended quiescent state, which can be either the
814 * idle loop or adaptive-tickless usermode execution.
816 static void rcu_eqs_enter(bool user
)
818 struct rcu_dynticks
*rdtp
;
820 rdtp
= this_cpu_ptr(&rcu_dynticks
);
821 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
822 (rdtp
->dynticks_nesting
& DYNTICK_TASK_NEST_MASK
) == 0);
823 if ((rdtp
->dynticks_nesting
& DYNTICK_TASK_NEST_MASK
) == DYNTICK_TASK_NEST_VALUE
)
824 rcu_eqs_enter_common(user
);
826 rdtp
->dynticks_nesting
-= DYNTICK_TASK_NEST_VALUE
;
830 * rcu_idle_enter - inform RCU that current CPU is entering idle
832 * Enter idle mode, in other words, -leave- the mode in which RCU
833 * read-side critical sections can occur. (Though RCU read-side
834 * critical sections can occur in irq handlers in idle, a possibility
835 * handled by irq_enter() and irq_exit().)
837 * We crowbar the ->dynticks_nesting field to zero to allow for
838 * the possibility of usermode upcalls having messed up our count
839 * of interrupt nesting level during the prior busy period.
841 void rcu_idle_enter(void)
843 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_idle_enter() invoked with irqs enabled!!!");
844 rcu_eqs_enter(false);
847 #ifdef CONFIG_NO_HZ_FULL
849 * rcu_user_enter - inform RCU that we are resuming userspace.
851 * Enter RCU idle mode right before resuming userspace. No use of RCU
852 * is permitted between this call and rcu_user_exit(). This way the
853 * CPU doesn't need to maintain the tick for RCU maintenance purposes
854 * when the CPU runs in userspace.
856 void rcu_user_enter(void)
858 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_user_enter() invoked with irqs enabled!!!");
861 #endif /* CONFIG_NO_HZ_FULL */
864 * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
866 * Exit from an interrupt handler, which might possibly result in entering
867 * idle mode, in other words, leaving the mode in which read-side critical
868 * sections can occur. The caller must have disabled interrupts.
870 * This code assumes that the idle loop never does anything that might
871 * result in unbalanced calls to irq_enter() and irq_exit(). If your
872 * architecture violates this assumption, RCU will give you what you
873 * deserve, good and hard. But very infrequently and irreproducibly.
875 * Use things like work queues to work around this limitation.
877 * You have been warned.
879 void rcu_irq_exit(void)
881 struct rcu_dynticks
*rdtp
;
883 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_exit() invoked with irqs enabled!!!");
884 rdtp
= this_cpu_ptr(&rcu_dynticks
);
886 /* Page faults can happen in NMI handlers, so check... */
887 if (rdtp
->dynticks_nmi_nesting
)
890 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
891 rdtp
->dynticks_nesting
< 1);
892 if (rdtp
->dynticks_nesting
<= 1) {
893 rcu_eqs_enter_common(true);
895 trace_rcu_dyntick(TPS("--="), rdtp
->dynticks_nesting
, rdtp
->dynticks_nesting
- 1);
896 rdtp
->dynticks_nesting
--;
901 * Wrapper for rcu_irq_exit() where interrupts are enabled.
903 void rcu_irq_exit_irqson(void)
907 local_irq_save(flags
);
909 local_irq_restore(flags
);
913 * rcu_eqs_exit_common - current CPU moving away from extended quiescent state
915 * If the new value of the ->dynticks_nesting counter was previously zero,
916 * we really have exited idle, and must do the appropriate accounting.
917 * The caller must have disabled interrupts.
919 static void rcu_eqs_exit_common(long long oldval
, int user
)
921 RCU_TRACE(struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);)
923 rcu_dynticks_task_exit();
924 rcu_dynticks_eqs_exit();
925 rcu_cleanup_after_idle();
926 trace_rcu_dyntick(TPS("End"), oldval
, rdtp
->dynticks_nesting
);
927 if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
928 !user
&& !is_idle_task(current
)) {
929 struct task_struct
*idle __maybe_unused
=
930 idle_task(smp_processor_id());
932 trace_rcu_dyntick(TPS("Error on exit: not idle task"),
933 oldval
, rdtp
->dynticks_nesting
);
934 rcu_ftrace_dump(DUMP_ORIG
);
935 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
936 current
->pid
, current
->comm
,
937 idle
->pid
, idle
->comm
); /* must be idle task! */
942 * Exit an RCU extended quiescent state, which can be either the
943 * idle loop or adaptive-tickless usermode execution.
945 static void rcu_eqs_exit(bool user
)
947 struct rcu_dynticks
*rdtp
;
950 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_eqs_exit() invoked with irqs enabled!!!");
951 rdtp
= this_cpu_ptr(&rcu_dynticks
);
952 oldval
= rdtp
->dynticks_nesting
;
953 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) && oldval
< 0);
954 if (oldval
& DYNTICK_TASK_NEST_MASK
) {
955 rdtp
->dynticks_nesting
+= DYNTICK_TASK_NEST_VALUE
;
957 __this_cpu_inc(disable_rcu_irq_enter
);
958 rdtp
->dynticks_nesting
= DYNTICK_TASK_EXIT_IDLE
;
959 rcu_eqs_exit_common(oldval
, user
);
960 __this_cpu_dec(disable_rcu_irq_enter
);
965 * rcu_idle_exit - inform RCU that current CPU is leaving idle
967 * Exit idle mode, in other words, -enter- the mode in which RCU
968 * read-side critical sections can occur.
970 * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to
971 * allow for the possibility of usermode upcalls messing up our count
972 * of interrupt nesting level during the busy period that is just
975 void rcu_idle_exit(void)
979 local_irq_save(flags
);
981 local_irq_restore(flags
);
984 #ifdef CONFIG_NO_HZ_FULL
986 * rcu_user_exit - inform RCU that we are exiting userspace.
988 * Exit RCU idle mode while entering the kernel because it can
989 * run a RCU read side critical section anytime.
991 void rcu_user_exit(void)
995 #endif /* CONFIG_NO_HZ_FULL */
998 * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
1000 * Enter an interrupt handler, which might possibly result in exiting
1001 * idle mode, in other words, entering the mode in which read-side critical
1002 * sections can occur. The caller must have disabled interrupts.
1004 * Note that the Linux kernel is fully capable of entering an interrupt
1005 * handler that it never exits, for example when doing upcalls to
1006 * user mode! This code assumes that the idle loop never does upcalls to
1007 * user mode. If your architecture does do upcalls from the idle loop (or
1008 * does anything else that results in unbalanced calls to the irq_enter()
1009 * and irq_exit() functions), RCU will give you what you deserve, good
1010 * and hard. But very infrequently and irreproducibly.
1012 * Use things like work queues to work around this limitation.
1014 * You have been warned.
1016 void rcu_irq_enter(void)
1018 struct rcu_dynticks
*rdtp
;
1021 RCU_LOCKDEP_WARN(!irqs_disabled(), "rcu_irq_enter() invoked with irqs enabled!!!");
1022 rdtp
= this_cpu_ptr(&rcu_dynticks
);
1024 /* Page faults can happen in NMI handlers, so check... */
1025 if (rdtp
->dynticks_nmi_nesting
)
1028 oldval
= rdtp
->dynticks_nesting
;
1029 rdtp
->dynticks_nesting
++;
1030 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG
) &&
1031 rdtp
->dynticks_nesting
== 0);
1033 trace_rcu_dyntick(TPS("++="), oldval
, rdtp
->dynticks_nesting
);
1035 rcu_eqs_exit_common(oldval
, true);
1039 * Wrapper for rcu_irq_enter() where interrupts are enabled.
1041 void rcu_irq_enter_irqson(void)
1043 unsigned long flags
;
1045 local_irq_save(flags
);
1047 local_irq_restore(flags
);
1051 * rcu_nmi_enter - inform RCU of entry to NMI context
1053 * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
1054 * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
1055 * that the CPU is active. This implementation permits nested NMIs, as
1056 * long as the nesting level does not overflow an int. (You will probably
1057 * run out of stack space first.)
1059 void rcu_nmi_enter(void)
1061 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
1064 /* Complain about underflow. */
1065 WARN_ON_ONCE(rdtp
->dynticks_nmi_nesting
< 0);
1068 * If idle from RCU viewpoint, atomically increment ->dynticks
1069 * to mark non-idle and increment ->dynticks_nmi_nesting by one.
1070 * Otherwise, increment ->dynticks_nmi_nesting by two. This means
1071 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
1072 * to be in the outermost NMI handler that interrupted an RCU-idle
1073 * period (observation due to Andy Lutomirski).
1075 if (rcu_dynticks_curr_cpu_in_eqs()) {
1076 rcu_dynticks_eqs_exit();
1079 rdtp
->dynticks_nmi_nesting
+= incby
;
1084 * rcu_nmi_exit - inform RCU of exit from NMI context
1086 * If we are returning from the outermost NMI handler that interrupted an
1087 * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
1088 * to let the RCU grace-period handling know that the CPU is back to
1091 void rcu_nmi_exit(void)
1093 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
1096 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
1097 * (We are exiting an NMI handler, so RCU better be paying attention
1100 WARN_ON_ONCE(rdtp
->dynticks_nmi_nesting
<= 0);
1101 WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
1104 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
1105 * leave it in non-RCU-idle state.
1107 if (rdtp
->dynticks_nmi_nesting
!= 1) {
1108 rdtp
->dynticks_nmi_nesting
-= 2;
1112 /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
1113 rdtp
->dynticks_nmi_nesting
= 0;
1114 rcu_dynticks_eqs_enter();
1118 * rcu_is_watching - see if RCU thinks that the current CPU is idle
1120 * Return true if RCU is watching the running CPU, which means that this
1121 * CPU can safely enter RCU read-side critical sections. In other words,
1122 * if the current CPU is in its idle loop and is neither in an interrupt
1123 * or NMI handler, return true.
1125 bool notrace
rcu_is_watching(void)
1129 preempt_disable_notrace();
1130 ret
= !rcu_dynticks_curr_cpu_in_eqs();
1131 preempt_enable_notrace();
1134 EXPORT_SYMBOL_GPL(rcu_is_watching
);
1137 * If a holdout task is actually running, request an urgent quiescent
1138 * state from its CPU. This is unsynchronized, so migrations can cause
1139 * the request to go to the wrong CPU. Which is OK, all that will happen
1140 * is that the CPU's next context switch will be a bit slower and next
1141 * time around this task will generate another request.
1143 void rcu_request_urgent_qs_task(struct task_struct
*t
)
1150 return; /* This task is not running on that CPU. */
1151 smp_store_release(per_cpu_ptr(&rcu_dynticks
.rcu_urgent_qs
, cpu
), true);
1154 #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
1157 * Is the current CPU online? Disable preemption to avoid false positives
1158 * that could otherwise happen due to the current CPU number being sampled,
1159 * this task being preempted, its old CPU being taken offline, resuming
1160 * on some other CPU, then determining that its old CPU is now offline.
1161 * It is OK to use RCU on an offline processor during initial boot, hence
1162 * the check for rcu_scheduler_fully_active. Note also that it is OK
1163 * for a CPU coming online to use RCU for one jiffy prior to marking itself
1164 * online in the cpu_online_mask. Similarly, it is OK for a CPU going
1165 * offline to continue to use RCU for one jiffy after marking itself
1166 * offline in the cpu_online_mask. This leniency is necessary given the
1167 * non-atomic nature of the online and offline processing, for example,
1168 * the fact that a CPU enters the scheduler after completing the teardown
1171 * This is also why RCU internally marks CPUs online during in the
1172 * preparation phase and offline after the CPU has been taken down.
1174 * Disable checking if in an NMI handler because we cannot safely report
1175 * errors from NMI handlers anyway.
1177 bool rcu_lockdep_current_cpu_online(void)
1179 struct rcu_data
*rdp
;
1180 struct rcu_node
*rnp
;
1186 rdp
= this_cpu_ptr(&rcu_sched_data
);
1188 ret
= (rdp
->grpmask
& rcu_rnp_online_cpus(rnp
)) ||
1189 !rcu_scheduler_fully_active
;
1193 EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online
);
1195 #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
1198 * rcu_is_cpu_rrupt_from_idle - see if idle or immediately interrupted from idle
1200 * If the current CPU is idle or running at a first-level (not nested)
1201 * interrupt from idle, return true. The caller must have at least
1202 * disabled preemption.
1204 static int rcu_is_cpu_rrupt_from_idle(void)
1206 return __this_cpu_read(rcu_dynticks
.dynticks_nesting
) <= 1;
1210 * Snapshot the specified CPU's dynticks counter so that we can later
1211 * credit them with an implicit quiescent state. Return 1 if this CPU
1212 * is in dynticks idle mode, which is an extended quiescent state.
1214 static int dyntick_save_progress_counter(struct rcu_data
*rdp
)
1216 rdp
->dynticks_snap
= rcu_dynticks_snap(rdp
->dynticks
);
1217 if (rcu_dynticks_in_eqs(rdp
->dynticks_snap
)) {
1218 trace_rcu_fqs(rdp
->rsp
->name
, rdp
->gpnum
, rdp
->cpu
, TPS("dti"));
1219 if (ULONG_CMP_LT(READ_ONCE(rdp
->gpnum
) + ULONG_MAX
/ 4,
1220 rdp
->mynode
->gpnum
))
1221 WRITE_ONCE(rdp
->gpwrap
, true);
1228 * Return true if the specified CPU has passed through a quiescent
1229 * state by virtue of being in or having passed through an dynticks
1230 * idle state since the last call to dyntick_save_progress_counter()
1231 * for this same CPU, or by virtue of having been offline.
1233 static int rcu_implicit_dynticks_qs(struct rcu_data
*rdp
)
1238 unsigned long rjtsc
;
1239 struct rcu_node
*rnp
;
1242 * If the CPU passed through or entered a dynticks idle phase with
1243 * no active irq/NMI handlers, then we can safely pretend that the CPU
1244 * already acknowledged the request to pass through a quiescent
1245 * state. Either way, that CPU cannot possibly be in an RCU
1246 * read-side critical section that started before the beginning
1247 * of the current RCU grace period.
1249 if (rcu_dynticks_in_eqs_since(rdp
->dynticks
, rdp
->dynticks_snap
)) {
1250 trace_rcu_fqs(rdp
->rsp
->name
, rdp
->gpnum
, rdp
->cpu
, TPS("dti"));
1251 rdp
->dynticks_fqs
++;
1255 /* Compute and saturate jiffies_till_sched_qs. */
1256 jtsq
= jiffies_till_sched_qs
;
1257 rjtsc
= rcu_jiffies_till_stall_check();
1258 if (jtsq
> rjtsc
/ 2) {
1259 WRITE_ONCE(jiffies_till_sched_qs
, rjtsc
);
1261 } else if (jtsq
< 1) {
1262 WRITE_ONCE(jiffies_till_sched_qs
, 1);
1267 * Has this CPU encountered a cond_resched_rcu_qs() since the
1268 * beginning of the grace period? For this to be the case,
1269 * the CPU has to have noticed the current grace period. This
1270 * might not be the case for nohz_full CPUs looping in the kernel.
1273 ruqp
= per_cpu_ptr(&rcu_dynticks
.rcu_urgent_qs
, rdp
->cpu
);
1274 if (time_after(jiffies
, rdp
->rsp
->gp_start
+ jtsq
) &&
1275 READ_ONCE(rdp
->rcu_qs_ctr_snap
) != per_cpu(rcu_dynticks
.rcu_qs_ctr
, rdp
->cpu
) &&
1276 READ_ONCE(rdp
->gpnum
) == rnp
->gpnum
&& !rdp
->gpwrap
) {
1277 trace_rcu_fqs(rdp
->rsp
->name
, rdp
->gpnum
, rdp
->cpu
, TPS("rqc"));
1280 /* Load rcu_qs_ctr before store to rcu_urgent_qs. */
1281 smp_store_release(ruqp
, true);
1284 /* Check for the CPU being offline. */
1285 if (!(rdp
->grpmask
& rcu_rnp_online_cpus(rnp
))) {
1286 trace_rcu_fqs(rdp
->rsp
->name
, rdp
->gpnum
, rdp
->cpu
, TPS("ofl"));
1292 * A CPU running for an extended time within the kernel can
1293 * delay RCU grace periods. When the CPU is in NO_HZ_FULL mode,
1294 * even context-switching back and forth between a pair of
1295 * in-kernel CPU-bound tasks cannot advance grace periods.
1296 * So if the grace period is old enough, make the CPU pay attention.
1297 * Note that the unsynchronized assignments to the per-CPU
1298 * rcu_need_heavy_qs variable are safe. Yes, setting of
1299 * bits can be lost, but they will be set again on the next
1300 * force-quiescent-state pass. So lost bit sets do not result
1301 * in incorrect behavior, merely in a grace period lasting
1302 * a few jiffies longer than it might otherwise. Because
1303 * there are at most four threads involved, and because the
1304 * updates are only once every few jiffies, the probability of
1305 * lossage (and thus of slight grace-period extension) is
1308 * Note that if the jiffies_till_sched_qs boot/sysfs parameter
1309 * is set too high, we override with half of the RCU CPU stall
1312 rnhqp
= &per_cpu(rcu_dynticks
.rcu_need_heavy_qs
, rdp
->cpu
);
1313 if (!READ_ONCE(*rnhqp
) &&
1314 (time_after(jiffies
, rdp
->rsp
->gp_start
+ jtsq
) ||
1315 time_after(jiffies
, rdp
->rsp
->jiffies_resched
))) {
1316 WRITE_ONCE(*rnhqp
, true);
1317 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1318 smp_store_release(ruqp
, true);
1319 rdp
->rsp
->jiffies_resched
+= 5; /* Re-enable beating. */
1323 * If more than halfway to RCU CPU stall-warning time, do
1324 * a resched_cpu() to try to loosen things up a bit.
1326 if (jiffies
- rdp
->rsp
->gp_start
> rcu_jiffies_till_stall_check() / 2)
1327 resched_cpu(rdp
->cpu
);
1332 static void record_gp_stall_check_time(struct rcu_state
*rsp
)
1334 unsigned long j
= jiffies
;
1338 smp_wmb(); /* Record start time before stall time. */
1339 j1
= rcu_jiffies_till_stall_check();
1340 WRITE_ONCE(rsp
->jiffies_stall
, j
+ j1
);
1341 rsp
->jiffies_resched
= j
+ j1
/ 2;
1342 rsp
->n_force_qs_gpstart
= READ_ONCE(rsp
->n_force_qs
);
1346 * Convert a ->gp_state value to a character string.
1348 static const char *gp_state_getname(short gs
)
1350 if (gs
< 0 || gs
>= ARRAY_SIZE(gp_state_names
))
1352 return gp_state_names
[gs
];
1356 * Complain about starvation of grace-period kthread.
1358 static void rcu_check_gp_kthread_starvation(struct rcu_state
*rsp
)
1364 gpa
= READ_ONCE(rsp
->gp_activity
);
1365 if (j
- gpa
> 2 * HZ
) {
1366 pr_err("%s kthread starved for %ld jiffies! g%lu c%lu f%#x %s(%d) ->state=%#lx ->cpu=%d\n",
1368 rsp
->gpnum
, rsp
->completed
,
1370 gp_state_getname(rsp
->gp_state
), rsp
->gp_state
,
1371 rsp
->gp_kthread
? rsp
->gp_kthread
->state
: ~0,
1372 rsp
->gp_kthread
? task_cpu(rsp
->gp_kthread
) : -1);
1373 if (rsp
->gp_kthread
) {
1374 sched_show_task(rsp
->gp_kthread
);
1375 wake_up_process(rsp
->gp_kthread
);
1381 * Dump stacks of all tasks running on stalled CPUs. First try using
1382 * NMIs, but fall back to manual remote stack tracing on architectures
1383 * that don't support NMI-based stack dumps. The NMI-triggered stack
1384 * traces are more accurate because they are printed by the target CPU.
1386 static void rcu_dump_cpu_stacks(struct rcu_state
*rsp
)
1389 unsigned long flags
;
1390 struct rcu_node
*rnp
;
1392 rcu_for_each_leaf_node(rsp
, rnp
) {
1393 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1394 for_each_leaf_node_possible_cpu(rnp
, cpu
)
1395 if (rnp
->qsmask
& leaf_node_cpu_bit(rnp
, cpu
))
1396 if (!trigger_single_cpu_backtrace(cpu
))
1398 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1403 * If too much time has passed in the current grace period, and if
1404 * so configured, go kick the relevant kthreads.
1406 static void rcu_stall_kick_kthreads(struct rcu_state
*rsp
)
1410 if (!rcu_kick_kthreads
)
1412 j
= READ_ONCE(rsp
->jiffies_kick_kthreads
);
1413 if (time_after(jiffies
, j
) && rsp
->gp_kthread
&&
1414 (rcu_gp_in_progress(rsp
) || READ_ONCE(rsp
->gp_flags
))) {
1415 WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp
->name
);
1416 rcu_ftrace_dump(DUMP_ALL
);
1417 wake_up_process(rsp
->gp_kthread
);
1418 WRITE_ONCE(rsp
->jiffies_kick_kthreads
, j
+ HZ
);
1422 static inline void panic_on_rcu_stall(void)
1424 if (sysctl_panic_on_rcu_stall
)
1425 panic("RCU Stall\n");
1428 static void print_other_cpu_stall(struct rcu_state
*rsp
, unsigned long gpnum
)
1432 unsigned long flags
;
1436 struct rcu_node
*rnp
= rcu_get_root(rsp
);
1439 /* Kick and suppress, if so configured. */
1440 rcu_stall_kick_kthreads(rsp
);
1441 if (rcu_cpu_stall_suppress
)
1444 /* Only let one CPU complain about others per time interval. */
1446 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1447 delta
= jiffies
- READ_ONCE(rsp
->jiffies_stall
);
1448 if (delta
< RCU_STALL_RAT_DELAY
|| !rcu_gp_in_progress(rsp
)) {
1449 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1452 WRITE_ONCE(rsp
->jiffies_stall
,
1453 jiffies
+ 3 * rcu_jiffies_till_stall_check() + 3);
1454 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1457 * OK, time to rat on our buddy...
1458 * See Documentation/RCU/stallwarn.txt for info on how to debug
1459 * RCU CPU stall warnings.
1461 pr_err("INFO: %s detected stalls on CPUs/tasks:",
1463 print_cpu_stall_info_begin();
1464 rcu_for_each_leaf_node(rsp
, rnp
) {
1465 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1466 ndetected
+= rcu_print_task_stall(rnp
);
1467 if (rnp
->qsmask
!= 0) {
1468 for_each_leaf_node_possible_cpu(rnp
, cpu
)
1469 if (rnp
->qsmask
& leaf_node_cpu_bit(rnp
, cpu
)) {
1470 print_cpu_stall_info(rsp
, cpu
);
1474 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1477 print_cpu_stall_info_end();
1478 for_each_possible_cpu(cpu
)
1479 totqlen
+= rcu_segcblist_n_cbs(&per_cpu_ptr(rsp
->rda
,
1481 pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n",
1482 smp_processor_id(), (long)(jiffies
- rsp
->gp_start
),
1483 (long)rsp
->gpnum
, (long)rsp
->completed
, totqlen
);
1485 rcu_dump_cpu_stacks(rsp
);
1487 /* Complain about tasks blocking the grace period. */
1488 rcu_print_detail_task_stall(rsp
);
1490 if (READ_ONCE(rsp
->gpnum
) != gpnum
||
1491 READ_ONCE(rsp
->completed
) == gpnum
) {
1492 pr_err("INFO: Stall ended before state dump start\n");
1495 gpa
= READ_ONCE(rsp
->gp_activity
);
1496 pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld, root ->qsmask %#lx\n",
1497 rsp
->name
, j
- gpa
, j
, gpa
,
1498 jiffies_till_next_fqs
,
1499 rcu_get_root(rsp
)->qsmask
);
1500 /* In this case, the current CPU might be at fault. */
1501 sched_show_task(current
);
1505 rcu_check_gp_kthread_starvation(rsp
);
1507 panic_on_rcu_stall();
1509 force_quiescent_state(rsp
); /* Kick them all. */
1512 static void print_cpu_stall(struct rcu_state
*rsp
)
1515 unsigned long flags
;
1516 struct rcu_node
*rnp
= rcu_get_root(rsp
);
1519 /* Kick and suppress, if so configured. */
1520 rcu_stall_kick_kthreads(rsp
);
1521 if (rcu_cpu_stall_suppress
)
1525 * OK, time to rat on ourselves...
1526 * See Documentation/RCU/stallwarn.txt for info on how to debug
1527 * RCU CPU stall warnings.
1529 pr_err("INFO: %s self-detected stall on CPU", rsp
->name
);
1530 print_cpu_stall_info_begin();
1531 print_cpu_stall_info(rsp
, smp_processor_id());
1532 print_cpu_stall_info_end();
1533 for_each_possible_cpu(cpu
)
1534 totqlen
+= rcu_segcblist_n_cbs(&per_cpu_ptr(rsp
->rda
,
1536 pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n",
1537 jiffies
- rsp
->gp_start
,
1538 (long)rsp
->gpnum
, (long)rsp
->completed
, totqlen
);
1540 rcu_check_gp_kthread_starvation(rsp
);
1542 rcu_dump_cpu_stacks(rsp
);
1544 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
1545 if (ULONG_CMP_GE(jiffies
, READ_ONCE(rsp
->jiffies_stall
)))
1546 WRITE_ONCE(rsp
->jiffies_stall
,
1547 jiffies
+ 3 * rcu_jiffies_till_stall_check() + 3);
1548 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1550 panic_on_rcu_stall();
1553 * Attempt to revive the RCU machinery by forcing a context switch.
1555 * A context switch would normally allow the RCU state machine to make
1556 * progress and it could be we're stuck in kernel space without context
1557 * switches for an entirely unreasonable amount of time.
1559 resched_cpu(smp_processor_id());
1562 static void check_cpu_stall(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
1564 unsigned long completed
;
1565 unsigned long gpnum
;
1569 struct rcu_node
*rnp
;
1571 if ((rcu_cpu_stall_suppress
&& !rcu_kick_kthreads
) ||
1572 !rcu_gp_in_progress(rsp
))
1574 rcu_stall_kick_kthreads(rsp
);
1578 * Lots of memory barriers to reject false positives.
1580 * The idea is to pick up rsp->gpnum, then rsp->jiffies_stall,
1581 * then rsp->gp_start, and finally rsp->completed. These values
1582 * are updated in the opposite order with memory barriers (or
1583 * equivalent) during grace-period initialization and cleanup.
1584 * Now, a false positive can occur if we get an new value of
1585 * rsp->gp_start and a old value of rsp->jiffies_stall. But given
1586 * the memory barriers, the only way that this can happen is if one
1587 * grace period ends and another starts between these two fetches.
1588 * Detect this by comparing rsp->completed with the previous fetch
1591 * Given this check, comparisons of jiffies, rsp->jiffies_stall,
1592 * and rsp->gp_start suffice to forestall false positives.
1594 gpnum
= READ_ONCE(rsp
->gpnum
);
1595 smp_rmb(); /* Pick up ->gpnum first... */
1596 js
= READ_ONCE(rsp
->jiffies_stall
);
1597 smp_rmb(); /* ...then ->jiffies_stall before the rest... */
1598 gps
= READ_ONCE(rsp
->gp_start
);
1599 smp_rmb(); /* ...and finally ->gp_start before ->completed. */
1600 completed
= READ_ONCE(rsp
->completed
);
1601 if (ULONG_CMP_GE(completed
, gpnum
) ||
1602 ULONG_CMP_LT(j
, js
) ||
1603 ULONG_CMP_GE(gps
, js
))
1604 return; /* No stall or GP completed since entering function. */
1606 if (rcu_gp_in_progress(rsp
) &&
1607 (READ_ONCE(rnp
->qsmask
) & rdp
->grpmask
)) {
1609 /* We haven't checked in, so go dump stack. */
1610 print_cpu_stall(rsp
);
1612 } else if (rcu_gp_in_progress(rsp
) &&
1613 ULONG_CMP_GE(j
, js
+ RCU_STALL_RAT_DELAY
)) {
1615 /* They had a few time units to dump stack, so complain. */
1616 print_other_cpu_stall(rsp
, gpnum
);
1621 * rcu_cpu_stall_reset - prevent further stall warnings in current grace period
1623 * Set the stall-warning timeout way off into the future, thus preventing
1624 * any RCU CPU stall-warning messages from appearing in the current set of
1625 * RCU grace periods.
1627 * The caller must disable hard irqs.
1629 void rcu_cpu_stall_reset(void)
1631 struct rcu_state
*rsp
;
1633 for_each_rcu_flavor(rsp
)
1634 WRITE_ONCE(rsp
->jiffies_stall
, jiffies
+ ULONG_MAX
/ 2);
1638 * Determine the value that ->completed will have at the end of the
1639 * next subsequent grace period. This is used to tag callbacks so that
1640 * a CPU can invoke callbacks in a timely fashion even if that CPU has
1641 * been dyntick-idle for an extended period with callbacks under the
1642 * influence of RCU_FAST_NO_HZ.
1644 * The caller must hold rnp->lock with interrupts disabled.
1646 static unsigned long rcu_cbs_completed(struct rcu_state
*rsp
,
1647 struct rcu_node
*rnp
)
1649 lockdep_assert_held(&rnp
->lock
);
1652 * If RCU is idle, we just wait for the next grace period.
1653 * But we can only be sure that RCU is idle if we are looking
1654 * at the root rcu_node structure -- otherwise, a new grace
1655 * period might have started, but just not yet gotten around
1656 * to initializing the current non-root rcu_node structure.
1658 if (rcu_get_root(rsp
) == rnp
&& rnp
->gpnum
== rnp
->completed
)
1659 return rnp
->completed
+ 1;
1662 * Otherwise, wait for a possible partial grace period and
1663 * then the subsequent full grace period.
1665 return rnp
->completed
+ 2;
1669 * Trace-event helper function for rcu_start_future_gp() and
1670 * rcu_nocb_wait_gp().
1672 static void trace_rcu_future_gp(struct rcu_node
*rnp
, struct rcu_data
*rdp
,
1673 unsigned long c
, const char *s
)
1675 trace_rcu_future_grace_period(rdp
->rsp
->name
, rnp
->gpnum
,
1676 rnp
->completed
, c
, rnp
->level
,
1677 rnp
->grplo
, rnp
->grphi
, s
);
1681 * Start some future grace period, as needed to handle newly arrived
1682 * callbacks. The required future grace periods are recorded in each
1683 * rcu_node structure's ->need_future_gp field. Returns true if there
1684 * is reason to awaken the grace-period kthread.
1686 * The caller must hold the specified rcu_node structure's ->lock.
1688 static bool __maybe_unused
1689 rcu_start_future_gp(struct rcu_node
*rnp
, struct rcu_data
*rdp
,
1690 unsigned long *c_out
)
1694 struct rcu_node
*rnp_root
= rcu_get_root(rdp
->rsp
);
1696 lockdep_assert_held(&rnp
->lock
);
1699 * Pick up grace-period number for new callbacks. If this
1700 * grace period is already marked as needed, return to the caller.
1702 c
= rcu_cbs_completed(rdp
->rsp
, rnp
);
1703 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Startleaf"));
1704 if (rnp
->need_future_gp
[c
& 0x1]) {
1705 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Prestartleaf"));
1710 * If either this rcu_node structure or the root rcu_node structure
1711 * believe that a grace period is in progress, then we must wait
1712 * for the one following, which is in "c". Because our request
1713 * will be noticed at the end of the current grace period, we don't
1714 * need to explicitly start one. We only do the lockless check
1715 * of rnp_root's fields if the current rcu_node structure thinks
1716 * there is no grace period in flight, and because we hold rnp->lock,
1717 * the only possible change is when rnp_root's two fields are
1718 * equal, in which case rnp_root->gpnum might be concurrently
1719 * incremented. But that is OK, as it will just result in our
1720 * doing some extra useless work.
1722 if (rnp
->gpnum
!= rnp
->completed
||
1723 READ_ONCE(rnp_root
->gpnum
) != READ_ONCE(rnp_root
->completed
)) {
1724 rnp
->need_future_gp
[c
& 0x1]++;
1725 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Startedleaf"));
1730 * There might be no grace period in progress. If we don't already
1731 * hold it, acquire the root rcu_node structure's lock in order to
1732 * start one (if needed).
1734 if (rnp
!= rnp_root
)
1735 raw_spin_lock_rcu_node(rnp_root
);
1738 * Get a new grace-period number. If there really is no grace
1739 * period in progress, it will be smaller than the one we obtained
1740 * earlier. Adjust callbacks as needed.
1742 c
= rcu_cbs_completed(rdp
->rsp
, rnp_root
);
1743 if (!rcu_is_nocb_cpu(rdp
->cpu
))
1744 (void)rcu_segcblist_accelerate(&rdp
->cblist
, c
);
1747 * If the needed for the required grace period is already
1748 * recorded, trace and leave.
1750 if (rnp_root
->need_future_gp
[c
& 0x1]) {
1751 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Prestartedroot"));
1755 /* Record the need for the future grace period. */
1756 rnp_root
->need_future_gp
[c
& 0x1]++;
1758 /* If a grace period is not already in progress, start one. */
1759 if (rnp_root
->gpnum
!= rnp_root
->completed
) {
1760 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Startedleafroot"));
1762 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("Startedroot"));
1763 ret
= rcu_start_gp_advanced(rdp
->rsp
, rnp_root
, rdp
);
1766 if (rnp
!= rnp_root
)
1767 raw_spin_unlock_rcu_node(rnp_root
);
1775 * Clean up any old requests for the just-ended grace period. Also return
1776 * whether any additional grace periods have been requested.
1778 static int rcu_future_gp_cleanup(struct rcu_state
*rsp
, struct rcu_node
*rnp
)
1780 int c
= rnp
->completed
;
1782 struct rcu_data
*rdp
= this_cpu_ptr(rsp
->rda
);
1784 rnp
->need_future_gp
[c
& 0x1] = 0;
1785 needmore
= rnp
->need_future_gp
[(c
+ 1) & 0x1];
1786 trace_rcu_future_gp(rnp
, rdp
, c
,
1787 needmore
? TPS("CleanupMore") : TPS("Cleanup"));
1792 * Awaken the grace-period kthread for the specified flavor of RCU.
1793 * Don't do a self-awaken, and don't bother awakening when there is
1794 * nothing for the grace-period kthread to do (as in several CPUs
1795 * raced to awaken, and we lost), and finally don't try to awaken
1796 * a kthread that has not yet been created.
1798 static void rcu_gp_kthread_wake(struct rcu_state
*rsp
)
1800 if (current
== rsp
->gp_kthread
||
1801 !READ_ONCE(rsp
->gp_flags
) ||
1804 swake_up(&rsp
->gp_wq
);
1808 * If there is room, assign a ->completed number to any callbacks on
1809 * this CPU that have not already been assigned. Also accelerate any
1810 * callbacks that were previously assigned a ->completed number that has
1811 * since proven to be too conservative, which can happen if callbacks get
1812 * assigned a ->completed number while RCU is idle, but with reference to
1813 * a non-root rcu_node structure. This function is idempotent, so it does
1814 * not hurt to call it repeatedly. Returns an flag saying that we should
1815 * awaken the RCU grace-period kthread.
1817 * The caller must hold rnp->lock with interrupts disabled.
1819 static bool rcu_accelerate_cbs(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
1820 struct rcu_data
*rdp
)
1824 lockdep_assert_held(&rnp
->lock
);
1826 /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
1827 if (!rcu_segcblist_pend_cbs(&rdp
->cblist
))
1831 * Callbacks are often registered with incomplete grace-period
1832 * information. Something about the fact that getting exact
1833 * information requires acquiring a global lock... RCU therefore
1834 * makes a conservative estimate of the grace period number at which
1835 * a given callback will become ready to invoke. The following
1836 * code checks this estimate and improves it when possible, thus
1837 * accelerating callback invocation to an earlier grace-period
1840 if (rcu_segcblist_accelerate(&rdp
->cblist
, rcu_cbs_completed(rsp
, rnp
)))
1841 ret
= rcu_start_future_gp(rnp
, rdp
, NULL
);
1843 /* Trace depending on how much we were able to accelerate. */
1844 if (rcu_segcblist_restempty(&rdp
->cblist
, RCU_WAIT_TAIL
))
1845 trace_rcu_grace_period(rsp
->name
, rdp
->gpnum
, TPS("AccWaitCB"));
1847 trace_rcu_grace_period(rsp
->name
, rdp
->gpnum
, TPS("AccReadyCB"));
1852 * Move any callbacks whose grace period has completed to the
1853 * RCU_DONE_TAIL sublist, then compact the remaining sublists and
1854 * assign ->completed numbers to any callbacks in the RCU_NEXT_TAIL
1855 * sublist. This function is idempotent, so it does not hurt to
1856 * invoke it repeatedly. As long as it is not invoked -too- often...
1857 * Returns true if the RCU grace-period kthread needs to be awakened.
1859 * The caller must hold rnp->lock with interrupts disabled.
1861 static bool rcu_advance_cbs(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
1862 struct rcu_data
*rdp
)
1864 lockdep_assert_held(&rnp
->lock
);
1866 /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
1867 if (!rcu_segcblist_pend_cbs(&rdp
->cblist
))
1871 * Find all callbacks whose ->completed numbers indicate that they
1872 * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
1874 rcu_segcblist_advance(&rdp
->cblist
, rnp
->completed
);
1876 /* Classify any remaining callbacks. */
1877 return rcu_accelerate_cbs(rsp
, rnp
, rdp
);
1881 * Update CPU-local rcu_data state to record the beginnings and ends of
1882 * grace periods. The caller must hold the ->lock of the leaf rcu_node
1883 * structure corresponding to the current CPU, and must have irqs disabled.
1884 * Returns true if the grace-period kthread needs to be awakened.
1886 static bool __note_gp_changes(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
1887 struct rcu_data
*rdp
)
1892 lockdep_assert_held(&rnp
->lock
);
1894 /* Handle the ends of any preceding grace periods first. */
1895 if (rdp
->completed
== rnp
->completed
&&
1896 !unlikely(READ_ONCE(rdp
->gpwrap
))) {
1898 /* No grace period end, so just accelerate recent callbacks. */
1899 ret
= rcu_accelerate_cbs(rsp
, rnp
, rdp
);
1903 /* Advance callbacks. */
1904 ret
= rcu_advance_cbs(rsp
, rnp
, rdp
);
1906 /* Remember that we saw this grace-period completion. */
1907 rdp
->completed
= rnp
->completed
;
1908 trace_rcu_grace_period(rsp
->name
, rdp
->gpnum
, TPS("cpuend"));
1911 if (rdp
->gpnum
!= rnp
->gpnum
|| unlikely(READ_ONCE(rdp
->gpwrap
))) {
1913 * If the current grace period is waiting for this CPU,
1914 * set up to detect a quiescent state, otherwise don't
1915 * go looking for one.
1917 rdp
->gpnum
= rnp
->gpnum
;
1918 trace_rcu_grace_period(rsp
->name
, rdp
->gpnum
, TPS("cpustart"));
1919 need_gp
= !!(rnp
->qsmask
& rdp
->grpmask
);
1920 rdp
->cpu_no_qs
.b
.norm
= need_gp
;
1921 rdp
->rcu_qs_ctr_snap
= __this_cpu_read(rcu_dynticks
.rcu_qs_ctr
);
1922 rdp
->core_needs_qs
= need_gp
;
1923 zero_cpu_stall_ticks(rdp
);
1924 WRITE_ONCE(rdp
->gpwrap
, false);
1929 static void note_gp_changes(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
1931 unsigned long flags
;
1933 struct rcu_node
*rnp
;
1935 local_irq_save(flags
);
1937 if ((rdp
->gpnum
== READ_ONCE(rnp
->gpnum
) &&
1938 rdp
->completed
== READ_ONCE(rnp
->completed
) &&
1939 !unlikely(READ_ONCE(rdp
->gpwrap
))) || /* w/out lock. */
1940 !raw_spin_trylock_rcu_node(rnp
)) { /* irqs already off, so later. */
1941 local_irq_restore(flags
);
1944 needwake
= __note_gp_changes(rsp
, rnp
, rdp
);
1945 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
1947 rcu_gp_kthread_wake(rsp
);
1950 static void rcu_gp_slow(struct rcu_state
*rsp
, int delay
)
1953 !(rsp
->gpnum
% (rcu_num_nodes
* PER_RCU_NODE_PERIOD
* delay
)))
1954 schedule_timeout_uninterruptible(delay
);
1958 * Initialize a new grace period. Return false if no grace period required.
1960 static bool rcu_gp_init(struct rcu_state
*rsp
)
1962 unsigned long oldmask
;
1963 struct rcu_data
*rdp
;
1964 struct rcu_node
*rnp
= rcu_get_root(rsp
);
1966 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
1967 raw_spin_lock_irq_rcu_node(rnp
);
1968 if (!READ_ONCE(rsp
->gp_flags
)) {
1969 /* Spurious wakeup, tell caller to go back to sleep. */
1970 raw_spin_unlock_irq_rcu_node(rnp
);
1973 WRITE_ONCE(rsp
->gp_flags
, 0); /* Clear all flags: New grace period. */
1975 if (WARN_ON_ONCE(rcu_gp_in_progress(rsp
))) {
1977 * Grace period already in progress, don't start another.
1978 * Not supposed to be able to happen.
1980 raw_spin_unlock_irq_rcu_node(rnp
);
1984 /* Advance to a new grace period and initialize state. */
1985 record_gp_stall_check_time(rsp
);
1986 /* Record GP times before starting GP, hence smp_store_release(). */
1987 smp_store_release(&rsp
->gpnum
, rsp
->gpnum
+ 1);
1988 trace_rcu_grace_period(rsp
->name
, rsp
->gpnum
, TPS("start"));
1989 raw_spin_unlock_irq_rcu_node(rnp
);
1992 * Apply per-leaf buffered online and offline operations to the
1993 * rcu_node tree. Note that this new grace period need not wait
1994 * for subsequent online CPUs, and that quiescent-state forcing
1995 * will handle subsequent offline CPUs.
1997 rcu_for_each_leaf_node(rsp
, rnp
) {
1998 rcu_gp_slow(rsp
, gp_preinit_delay
);
1999 raw_spin_lock_irq_rcu_node(rnp
);
2000 if (rnp
->qsmaskinit
== rnp
->qsmaskinitnext
&&
2001 !rnp
->wait_blkd_tasks
) {
2002 /* Nothing to do on this leaf rcu_node structure. */
2003 raw_spin_unlock_irq_rcu_node(rnp
);
2007 /* Record old state, apply changes to ->qsmaskinit field. */
2008 oldmask
= rnp
->qsmaskinit
;
2009 rnp
->qsmaskinit
= rnp
->qsmaskinitnext
;
2011 /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
2012 if (!oldmask
!= !rnp
->qsmaskinit
) {
2013 if (!oldmask
) /* First online CPU for this rcu_node. */
2014 rcu_init_new_rnp(rnp
);
2015 else if (rcu_preempt_has_tasks(rnp
)) /* blocked tasks */
2016 rnp
->wait_blkd_tasks
= true;
2017 else /* Last offline CPU and can propagate. */
2018 rcu_cleanup_dead_rnp(rnp
);
2022 * If all waited-on tasks from prior grace period are
2023 * done, and if all this rcu_node structure's CPUs are
2024 * still offline, propagate up the rcu_node tree and
2025 * clear ->wait_blkd_tasks. Otherwise, if one of this
2026 * rcu_node structure's CPUs has since come back online,
2027 * simply clear ->wait_blkd_tasks (but rcu_cleanup_dead_rnp()
2028 * checks for this, so just call it unconditionally).
2030 if (rnp
->wait_blkd_tasks
&&
2031 (!rcu_preempt_has_tasks(rnp
) ||
2033 rnp
->wait_blkd_tasks
= false;
2034 rcu_cleanup_dead_rnp(rnp
);
2037 raw_spin_unlock_irq_rcu_node(rnp
);
2041 * Set the quiescent-state-needed bits in all the rcu_node
2042 * structures for all currently online CPUs in breadth-first order,
2043 * starting from the root rcu_node structure, relying on the layout
2044 * of the tree within the rsp->node[] array. Note that other CPUs
2045 * will access only the leaves of the hierarchy, thus seeing that no
2046 * grace period is in progress, at least until the corresponding
2047 * leaf node has been initialized.
2049 * The grace period cannot complete until the initialization
2050 * process finishes, because this kthread handles both.
2052 rcu_for_each_node_breadth_first(rsp
, rnp
) {
2053 rcu_gp_slow(rsp
, gp_init_delay
);
2054 raw_spin_lock_irq_rcu_node(rnp
);
2055 rdp
= this_cpu_ptr(rsp
->rda
);
2056 rcu_preempt_check_blocked_tasks(rnp
);
2057 rnp
->qsmask
= rnp
->qsmaskinit
;
2058 WRITE_ONCE(rnp
->gpnum
, rsp
->gpnum
);
2059 if (WARN_ON_ONCE(rnp
->completed
!= rsp
->completed
))
2060 WRITE_ONCE(rnp
->completed
, rsp
->completed
);
2061 if (rnp
== rdp
->mynode
)
2062 (void)__note_gp_changes(rsp
, rnp
, rdp
);
2063 rcu_preempt_boost_start_gp(rnp
);
2064 trace_rcu_grace_period_init(rsp
->name
, rnp
->gpnum
,
2065 rnp
->level
, rnp
->grplo
,
2066 rnp
->grphi
, rnp
->qsmask
);
2067 raw_spin_unlock_irq_rcu_node(rnp
);
2068 cond_resched_rcu_qs();
2069 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2076 * Helper function for swait_event_idle() wakeup at force-quiescent-state
2079 static bool rcu_gp_fqs_check_wake(struct rcu_state
*rsp
, int *gfp
)
2081 struct rcu_node
*rnp
= rcu_get_root(rsp
);
2083 /* Someone like call_rcu() requested a force-quiescent-state scan. */
2084 *gfp
= READ_ONCE(rsp
->gp_flags
);
2085 if (*gfp
& RCU_GP_FLAG_FQS
)
2088 /* The current grace period has completed. */
2089 if (!READ_ONCE(rnp
->qsmask
) && !rcu_preempt_blocked_readers_cgp(rnp
))
2096 * Do one round of quiescent-state forcing.
2098 static void rcu_gp_fqs(struct rcu_state
*rsp
, bool first_time
)
2100 struct rcu_node
*rnp
= rcu_get_root(rsp
);
2102 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2105 /* Collect dyntick-idle snapshots. */
2106 force_qs_rnp(rsp
, dyntick_save_progress_counter
);
2108 /* Handle dyntick-idle and offline CPUs. */
2109 force_qs_rnp(rsp
, rcu_implicit_dynticks_qs
);
2111 /* Clear flag to prevent immediate re-entry. */
2112 if (READ_ONCE(rsp
->gp_flags
) & RCU_GP_FLAG_FQS
) {
2113 raw_spin_lock_irq_rcu_node(rnp
);
2114 WRITE_ONCE(rsp
->gp_flags
,
2115 READ_ONCE(rsp
->gp_flags
) & ~RCU_GP_FLAG_FQS
);
2116 raw_spin_unlock_irq_rcu_node(rnp
);
2121 * Clean up after the old grace period.
2123 static void rcu_gp_cleanup(struct rcu_state
*rsp
)
2125 unsigned long gp_duration
;
2126 bool needgp
= false;
2128 struct rcu_data
*rdp
;
2129 struct rcu_node
*rnp
= rcu_get_root(rsp
);
2130 struct swait_queue_head
*sq
;
2132 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2133 raw_spin_lock_irq_rcu_node(rnp
);
2134 gp_duration
= jiffies
- rsp
->gp_start
;
2135 if (gp_duration
> rsp
->gp_max
)
2136 rsp
->gp_max
= gp_duration
;
2139 * We know the grace period is complete, but to everyone else
2140 * it appears to still be ongoing. But it is also the case
2141 * that to everyone else it looks like there is nothing that
2142 * they can do to advance the grace period. It is therefore
2143 * safe for us to drop the lock in order to mark the grace
2144 * period as completed in all of the rcu_node structures.
2146 raw_spin_unlock_irq_rcu_node(rnp
);
2149 * Propagate new ->completed value to rcu_node structures so
2150 * that other CPUs don't have to wait until the start of the next
2151 * grace period to process their callbacks. This also avoids
2152 * some nasty RCU grace-period initialization races by forcing
2153 * the end of the current grace period to be completely recorded in
2154 * all of the rcu_node structures before the beginning of the next
2155 * grace period is recorded in any of the rcu_node structures.
2157 rcu_for_each_node_breadth_first(rsp
, rnp
) {
2158 raw_spin_lock_irq_rcu_node(rnp
);
2159 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp
));
2160 WARN_ON_ONCE(rnp
->qsmask
);
2161 WRITE_ONCE(rnp
->completed
, rsp
->gpnum
);
2162 rdp
= this_cpu_ptr(rsp
->rda
);
2163 if (rnp
== rdp
->mynode
)
2164 needgp
= __note_gp_changes(rsp
, rnp
, rdp
) || needgp
;
2165 /* smp_mb() provided by prior unlock-lock pair. */
2166 nocb
+= rcu_future_gp_cleanup(rsp
, rnp
);
2167 sq
= rcu_nocb_gp_get(rnp
);
2168 raw_spin_unlock_irq_rcu_node(rnp
);
2169 rcu_nocb_gp_cleanup(sq
);
2170 cond_resched_rcu_qs();
2171 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2172 rcu_gp_slow(rsp
, gp_cleanup_delay
);
2174 rnp
= rcu_get_root(rsp
);
2175 raw_spin_lock_irq_rcu_node(rnp
); /* Order GP before ->completed update. */
2176 rcu_nocb_gp_set(rnp
, nocb
);
2178 /* Declare grace period done. */
2179 WRITE_ONCE(rsp
->completed
, rsp
->gpnum
);
2180 trace_rcu_grace_period(rsp
->name
, rsp
->completed
, TPS("end"));
2181 rsp
->gp_state
= RCU_GP_IDLE
;
2182 rdp
= this_cpu_ptr(rsp
->rda
);
2183 /* Advance CBs to reduce false positives below. */
2184 needgp
= rcu_advance_cbs(rsp
, rnp
, rdp
) || needgp
;
2185 if (needgp
|| cpu_needs_another_gp(rsp
, rdp
)) {
2186 WRITE_ONCE(rsp
->gp_flags
, RCU_GP_FLAG_INIT
);
2187 trace_rcu_grace_period(rsp
->name
,
2188 READ_ONCE(rsp
->gpnum
),
2191 raw_spin_unlock_irq_rcu_node(rnp
);
2195 * Body of kthread that handles grace periods.
2197 static int __noreturn
rcu_gp_kthread(void *arg
)
2203 struct rcu_state
*rsp
= arg
;
2204 struct rcu_node
*rnp
= rcu_get_root(rsp
);
2206 rcu_bind_gp_kthread();
2209 /* Handle grace-period start. */
2211 trace_rcu_grace_period(rsp
->name
,
2212 READ_ONCE(rsp
->gpnum
),
2214 rsp
->gp_state
= RCU_GP_WAIT_GPS
;
2215 swait_event_idle(rsp
->gp_wq
, READ_ONCE(rsp
->gp_flags
) &
2217 rsp
->gp_state
= RCU_GP_DONE_GPS
;
2218 /* Locking provides needed memory barrier. */
2219 if (rcu_gp_init(rsp
))
2221 cond_resched_rcu_qs();
2222 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2223 WARN_ON(signal_pending(current
));
2224 trace_rcu_grace_period(rsp
->name
,
2225 READ_ONCE(rsp
->gpnum
),
2229 /* Handle quiescent-state forcing. */
2230 first_gp_fqs
= true;
2231 j
= jiffies_till_first_fqs
;
2234 jiffies_till_first_fqs
= HZ
;
2239 rsp
->jiffies_force_qs
= jiffies
+ j
;
2240 WRITE_ONCE(rsp
->jiffies_kick_kthreads
,
2243 trace_rcu_grace_period(rsp
->name
,
2244 READ_ONCE(rsp
->gpnum
),
2246 rsp
->gp_state
= RCU_GP_WAIT_FQS
;
2247 ret
= swait_event_idle_timeout(rsp
->gp_wq
,
2248 rcu_gp_fqs_check_wake(rsp
, &gf
), j
);
2249 rsp
->gp_state
= RCU_GP_DOING_FQS
;
2250 /* Locking provides needed memory barriers. */
2251 /* If grace period done, leave loop. */
2252 if (!READ_ONCE(rnp
->qsmask
) &&
2253 !rcu_preempt_blocked_readers_cgp(rnp
))
2255 /* If time for quiescent-state forcing, do it. */
2256 if (ULONG_CMP_GE(jiffies
, rsp
->jiffies_force_qs
) ||
2257 (gf
& RCU_GP_FLAG_FQS
)) {
2258 trace_rcu_grace_period(rsp
->name
,
2259 READ_ONCE(rsp
->gpnum
),
2261 rcu_gp_fqs(rsp
, first_gp_fqs
);
2262 first_gp_fqs
= false;
2263 trace_rcu_grace_period(rsp
->name
,
2264 READ_ONCE(rsp
->gpnum
),
2266 cond_resched_rcu_qs();
2267 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2268 ret
= 0; /* Force full wait till next FQS. */
2269 j
= jiffies_till_next_fqs
;
2272 jiffies_till_next_fqs
= HZ
;
2275 jiffies_till_next_fqs
= 1;
2278 /* Deal with stray signal. */
2279 cond_resched_rcu_qs();
2280 WRITE_ONCE(rsp
->gp_activity
, jiffies
);
2281 WARN_ON(signal_pending(current
));
2282 trace_rcu_grace_period(rsp
->name
,
2283 READ_ONCE(rsp
->gpnum
),
2285 ret
= 1; /* Keep old FQS timing. */
2287 if (time_after(jiffies
, rsp
->jiffies_force_qs
))
2290 j
= rsp
->jiffies_force_qs
- j
;
2294 /* Handle grace-period end. */
2295 rsp
->gp_state
= RCU_GP_CLEANUP
;
2296 rcu_gp_cleanup(rsp
);
2297 rsp
->gp_state
= RCU_GP_CLEANED
;
2302 * Start a new RCU grace period if warranted, re-initializing the hierarchy
2303 * in preparation for detecting the next grace period. The caller must hold
2304 * the root node's ->lock and hard irqs must be disabled.
2306 * Note that it is legal for a dying CPU (which is marked as offline) to
2307 * invoke this function. This can happen when the dying CPU reports its
2310 * Returns true if the grace-period kthread must be awakened.
2313 rcu_start_gp_advanced(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
2314 struct rcu_data
*rdp
)
2316 lockdep_assert_held(&rnp
->lock
);
2317 if (!rsp
->gp_kthread
|| !cpu_needs_another_gp(rsp
, rdp
)) {
2319 * Either we have not yet spawned the grace-period
2320 * task, this CPU does not need another grace period,
2321 * or a grace period is already in progress.
2322 * Either way, don't start a new grace period.
2326 WRITE_ONCE(rsp
->gp_flags
, RCU_GP_FLAG_INIT
);
2327 trace_rcu_grace_period(rsp
->name
, READ_ONCE(rsp
->gpnum
),
2331 * We can't do wakeups while holding the rnp->lock, as that
2332 * could cause possible deadlocks with the rq->lock. Defer
2333 * the wakeup to our caller.
2339 * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's
2340 * callbacks. Note that rcu_start_gp_advanced() cannot do this because it
2341 * is invoked indirectly from rcu_advance_cbs(), which would result in
2342 * endless recursion -- or would do so if it wasn't for the self-deadlock
2343 * that is encountered beforehand.
2345 * Returns true if the grace-period kthread needs to be awakened.
2347 static bool rcu_start_gp(struct rcu_state
*rsp
)
2349 struct rcu_data
*rdp
= this_cpu_ptr(rsp
->rda
);
2350 struct rcu_node
*rnp
= rcu_get_root(rsp
);
2354 * If there is no grace period in progress right now, any
2355 * callbacks we have up to this point will be satisfied by the
2356 * next grace period. Also, advancing the callbacks reduces the
2357 * probability of false positives from cpu_needs_another_gp()
2358 * resulting in pointless grace periods. So, advance callbacks
2359 * then start the grace period!
2361 ret
= rcu_advance_cbs(rsp
, rnp
, rdp
) || ret
;
2362 ret
= rcu_start_gp_advanced(rsp
, rnp
, rdp
) || ret
;
2367 * Report a full set of quiescent states to the specified rcu_state data
2368 * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period
2369 * kthread if another grace period is required. Whether we wake
2370 * the grace-period kthread or it awakens itself for the next round
2371 * of quiescent-state forcing, that kthread will clean up after the
2372 * just-completed grace period. Note that the caller must hold rnp->lock,
2373 * which is released before return.
2375 static void rcu_report_qs_rsp(struct rcu_state
*rsp
, unsigned long flags
)
2376 __releases(rcu_get_root(rsp
)->lock
)
2378 lockdep_assert_held(&rcu_get_root(rsp
)->lock
);
2379 WARN_ON_ONCE(!rcu_gp_in_progress(rsp
));
2380 WRITE_ONCE(rsp
->gp_flags
, READ_ONCE(rsp
->gp_flags
) | RCU_GP_FLAG_FQS
);
2381 raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp
), flags
);
2382 rcu_gp_kthread_wake(rsp
);
2386 * Similar to rcu_report_qs_rdp(), for which it is a helper function.
2387 * Allows quiescent states for a group of CPUs to be reported at one go
2388 * to the specified rcu_node structure, though all the CPUs in the group
2389 * must be represented by the same rcu_node structure (which need not be a
2390 * leaf rcu_node structure, though it often will be). The gps parameter
2391 * is the grace-period snapshot, which means that the quiescent states
2392 * are valid only if rnp->gpnum is equal to gps. That structure's lock
2393 * must be held upon entry, and it is released before return.
2396 rcu_report_qs_rnp(unsigned long mask
, struct rcu_state
*rsp
,
2397 struct rcu_node
*rnp
, unsigned long gps
, unsigned long flags
)
2398 __releases(rnp
->lock
)
2400 unsigned long oldmask
= 0;
2401 struct rcu_node
*rnp_c
;
2403 lockdep_assert_held(&rnp
->lock
);
2405 /* Walk up the rcu_node hierarchy. */
2407 if (!(rnp
->qsmask
& mask
) || rnp
->gpnum
!= gps
) {
2410 * Our bit has already been cleared, or the
2411 * relevant grace period is already over, so done.
2413 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2416 WARN_ON_ONCE(oldmask
); /* Any child must be all zeroed! */
2417 WARN_ON_ONCE(rnp
->level
!= rcu_num_lvls
- 1 &&
2418 rcu_preempt_blocked_readers_cgp(rnp
));
2419 rnp
->qsmask
&= ~mask
;
2420 trace_rcu_quiescent_state_report(rsp
->name
, rnp
->gpnum
,
2421 mask
, rnp
->qsmask
, rnp
->level
,
2422 rnp
->grplo
, rnp
->grphi
,
2424 if (rnp
->qsmask
!= 0 || rcu_preempt_blocked_readers_cgp(rnp
)) {
2426 /* Other bits still set at this level, so done. */
2427 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2430 mask
= rnp
->grpmask
;
2431 if (rnp
->parent
== NULL
) {
2433 /* No more levels. Exit loop holding root lock. */
2437 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2440 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
2441 oldmask
= rnp_c
->qsmask
;
2445 * Get here if we are the last CPU to pass through a quiescent
2446 * state for this grace period. Invoke rcu_report_qs_rsp()
2447 * to clean up and start the next grace period if one is needed.
2449 rcu_report_qs_rsp(rsp
, flags
); /* releases rnp->lock. */
2453 * Record a quiescent state for all tasks that were previously queued
2454 * on the specified rcu_node structure and that were blocking the current
2455 * RCU grace period. The caller must hold the specified rnp->lock with
2456 * irqs disabled, and this lock is released upon return, but irqs remain
2459 static void rcu_report_unblock_qs_rnp(struct rcu_state
*rsp
,
2460 struct rcu_node
*rnp
, unsigned long flags
)
2461 __releases(rnp
->lock
)
2465 struct rcu_node
*rnp_p
;
2467 lockdep_assert_held(&rnp
->lock
);
2468 if (rcu_state_p
== &rcu_sched_state
|| rsp
!= rcu_state_p
||
2469 rnp
->qsmask
!= 0 || rcu_preempt_blocked_readers_cgp(rnp
)) {
2470 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2471 return; /* Still need more quiescent states! */
2474 rnp_p
= rnp
->parent
;
2475 if (rnp_p
== NULL
) {
2477 * Only one rcu_node structure in the tree, so don't
2478 * try to report up to its nonexistent parent!
2480 rcu_report_qs_rsp(rsp
, flags
);
2484 /* Report up the rest of the hierarchy, tracking current ->gpnum. */
2486 mask
= rnp
->grpmask
;
2487 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
2488 raw_spin_lock_rcu_node(rnp_p
); /* irqs already disabled. */
2489 rcu_report_qs_rnp(mask
, rsp
, rnp_p
, gps
, flags
);
2493 * Record a quiescent state for the specified CPU to that CPU's rcu_data
2494 * structure. This must be called from the specified CPU.
2497 rcu_report_qs_rdp(int cpu
, struct rcu_state
*rsp
, struct rcu_data
*rdp
)
2499 unsigned long flags
;
2502 struct rcu_node
*rnp
;
2505 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
2506 if (rdp
->cpu_no_qs
.b
.norm
|| rdp
->gpnum
!= rnp
->gpnum
||
2507 rnp
->completed
== rnp
->gpnum
|| rdp
->gpwrap
) {
2510 * The grace period in which this quiescent state was
2511 * recorded has ended, so don't report it upwards.
2512 * We will instead need a new quiescent state that lies
2513 * within the current grace period.
2515 rdp
->cpu_no_qs
.b
.norm
= true; /* need qs for new gp. */
2516 rdp
->rcu_qs_ctr_snap
= __this_cpu_read(rcu_dynticks
.rcu_qs_ctr
);
2517 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2520 mask
= rdp
->grpmask
;
2521 if ((rnp
->qsmask
& mask
) == 0) {
2522 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2524 rdp
->core_needs_qs
= false;
2527 * This GP can't end until cpu checks in, so all of our
2528 * callbacks can be processed during the next GP.
2530 needwake
= rcu_accelerate_cbs(rsp
, rnp
, rdp
);
2532 rcu_report_qs_rnp(mask
, rsp
, rnp
, rnp
->gpnum
, flags
);
2533 /* ^^^ Released rnp->lock */
2535 rcu_gp_kthread_wake(rsp
);
2540 * Check to see if there is a new grace period of which this CPU
2541 * is not yet aware, and if so, set up local rcu_data state for it.
2542 * Otherwise, see if this CPU has just passed through its first
2543 * quiescent state for this grace period, and record that fact if so.
2546 rcu_check_quiescent_state(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
2548 /* Check for grace-period ends and beginnings. */
2549 note_gp_changes(rsp
, rdp
);
2552 * Does this CPU still need to do its part for current grace period?
2553 * If no, return and let the other CPUs do their part as well.
2555 if (!rdp
->core_needs_qs
)
2559 * Was there a quiescent state since the beginning of the grace
2560 * period? If no, then exit and wait for the next call.
2562 if (rdp
->cpu_no_qs
.b
.norm
)
2566 * Tell RCU we are done (but rcu_report_qs_rdp() will be the
2569 rcu_report_qs_rdp(rdp
->cpu
, rsp
, rdp
);
2573 * Trace the fact that this CPU is going offline.
2575 static void rcu_cleanup_dying_cpu(struct rcu_state
*rsp
)
2577 RCU_TRACE(unsigned long mask
;)
2578 RCU_TRACE(struct rcu_data
*rdp
= this_cpu_ptr(rsp
->rda
);)
2579 RCU_TRACE(struct rcu_node
*rnp
= rdp
->mynode
;)
2581 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
))
2584 RCU_TRACE(mask
= rdp
->grpmask
;)
2585 trace_rcu_grace_period(rsp
->name
,
2586 rnp
->gpnum
+ 1 - !!(rnp
->qsmask
& mask
),
2591 * All CPUs for the specified rcu_node structure have gone offline,
2592 * and all tasks that were preempted within an RCU read-side critical
2593 * section while running on one of those CPUs have since exited their RCU
2594 * read-side critical section. Some other CPU is reporting this fact with
2595 * the specified rcu_node structure's ->lock held and interrupts disabled.
2596 * This function therefore goes up the tree of rcu_node structures,
2597 * clearing the corresponding bits in the ->qsmaskinit fields. Note that
2598 * the leaf rcu_node structure's ->qsmaskinit field has already been
2601 * This function does check that the specified rcu_node structure has
2602 * all CPUs offline and no blocked tasks, so it is OK to invoke it
2603 * prematurely. That said, invoking it after the fact will cost you
2604 * a needless lock acquisition. So once it has done its work, don't
2607 static void rcu_cleanup_dead_rnp(struct rcu_node
*rnp_leaf
)
2610 struct rcu_node
*rnp
= rnp_leaf
;
2612 lockdep_assert_held(&rnp
->lock
);
2613 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
) ||
2614 rnp
->qsmaskinit
|| rcu_preempt_has_tasks(rnp
))
2617 mask
= rnp
->grpmask
;
2621 raw_spin_lock_rcu_node(rnp
); /* irqs already disabled. */
2622 rnp
->qsmaskinit
&= ~mask
;
2623 rnp
->qsmask
&= ~mask
;
2624 if (rnp
->qsmaskinit
) {
2625 raw_spin_unlock_rcu_node(rnp
);
2626 /* irqs remain disabled. */
2629 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
2634 * The CPU has been completely removed, and some other CPU is reporting
2635 * this fact from process context. Do the remainder of the cleanup.
2636 * There can only be one CPU hotplug operation at a time, so no need for
2639 static void rcu_cleanup_dead_cpu(int cpu
, struct rcu_state
*rsp
)
2641 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
2642 struct rcu_node
*rnp
= rdp
->mynode
; /* Outgoing CPU's rdp & rnp. */
2644 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU
))
2647 /* Adjust any no-longer-needed kthreads. */
2648 rcu_boost_kthread_setaffinity(rnp
, -1);
2652 * Invoke any RCU callbacks that have made it to the end of their grace
2653 * period. Thottle as specified by rdp->blimit.
2655 static void rcu_do_batch(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
2657 unsigned long flags
;
2658 struct rcu_head
*rhp
;
2659 struct rcu_cblist rcl
= RCU_CBLIST_INITIALIZER(rcl
);
2662 /* If no callbacks are ready, just return. */
2663 if (!rcu_segcblist_ready_cbs(&rdp
->cblist
)) {
2664 trace_rcu_batch_start(rsp
->name
,
2665 rcu_segcblist_n_lazy_cbs(&rdp
->cblist
),
2666 rcu_segcblist_n_cbs(&rdp
->cblist
), 0);
2667 trace_rcu_batch_end(rsp
->name
, 0,
2668 !rcu_segcblist_empty(&rdp
->cblist
),
2669 need_resched(), is_idle_task(current
),
2670 rcu_is_callbacks_kthread());
2675 * Extract the list of ready callbacks, disabling to prevent
2676 * races with call_rcu() from interrupt handlers. Leave the
2677 * callback counts, as rcu_barrier() needs to be conservative.
2679 local_irq_save(flags
);
2680 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2682 trace_rcu_batch_start(rsp
->name
, rcu_segcblist_n_lazy_cbs(&rdp
->cblist
),
2683 rcu_segcblist_n_cbs(&rdp
->cblist
), bl
);
2684 rcu_segcblist_extract_done_cbs(&rdp
->cblist
, &rcl
);
2685 local_irq_restore(flags
);
2687 /* Invoke callbacks. */
2688 rhp
= rcu_cblist_dequeue(&rcl
);
2689 for (; rhp
; rhp
= rcu_cblist_dequeue(&rcl
)) {
2690 debug_rcu_head_unqueue(rhp
);
2691 if (__rcu_reclaim(rsp
->name
, rhp
))
2692 rcu_cblist_dequeued_lazy(&rcl
);
2694 * Stop only if limit reached and CPU has something to do.
2695 * Note: The rcl structure counts down from zero.
2697 if (-rcl
.len
>= bl
&&
2699 (!is_idle_task(current
) && !rcu_is_callbacks_kthread())))
2703 local_irq_save(flags
);
2705 trace_rcu_batch_end(rsp
->name
, count
, !!rcl
.head
, need_resched(),
2706 is_idle_task(current
), rcu_is_callbacks_kthread());
2708 /* Update counts and requeue any remaining callbacks. */
2709 rcu_segcblist_insert_done_cbs(&rdp
->cblist
, &rcl
);
2710 smp_mb(); /* List handling before counting for rcu_barrier(). */
2711 rdp
->n_cbs_invoked
+= count
;
2712 rcu_segcblist_insert_count(&rdp
->cblist
, &rcl
);
2714 /* Reinstate batch limit if we have worked down the excess. */
2715 count
= rcu_segcblist_n_cbs(&rdp
->cblist
);
2716 if (rdp
->blimit
== LONG_MAX
&& count
<= qlowmark
)
2717 rdp
->blimit
= blimit
;
2719 /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
2720 if (count
== 0 && rdp
->qlen_last_fqs_check
!= 0) {
2721 rdp
->qlen_last_fqs_check
= 0;
2722 rdp
->n_force_qs_snap
= rsp
->n_force_qs
;
2723 } else if (count
< rdp
->qlen_last_fqs_check
- qhimark
)
2724 rdp
->qlen_last_fqs_check
= count
;
2725 WARN_ON_ONCE(rcu_segcblist_empty(&rdp
->cblist
) != (count
== 0));
2727 local_irq_restore(flags
);
2729 /* Re-invoke RCU core processing if there are callbacks remaining. */
2730 if (rcu_segcblist_ready_cbs(&rdp
->cblist
))
2735 * Check to see if this CPU is in a non-context-switch quiescent state
2736 * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
2737 * Also schedule RCU core processing.
2739 * This function must be called from hardirq context. It is normally
2740 * invoked from the scheduling-clock interrupt.
2742 void rcu_check_callbacks(int user
)
2744 trace_rcu_utilization(TPS("Start scheduler-tick"));
2745 increment_cpu_stall_ticks();
2746 if (user
|| rcu_is_cpu_rrupt_from_idle()) {
2749 * Get here if this CPU took its interrupt from user
2750 * mode or from the idle loop, and if this is not a
2751 * nested interrupt. In this case, the CPU is in
2752 * a quiescent state, so note it.
2754 * No memory barrier is required here because both
2755 * rcu_sched_qs() and rcu_bh_qs() reference only CPU-local
2756 * variables that other CPUs neither access nor modify,
2757 * at least not while the corresponding CPU is online.
2763 } else if (!in_softirq()) {
2766 * Get here if this CPU did not take its interrupt from
2767 * softirq, in other words, if it is not interrupting
2768 * a rcu_bh read-side critical section. This is an _bh
2769 * critical section, so note it.
2774 rcu_preempt_check_callbacks();
2778 rcu_note_voluntary_context_switch(current
);
2779 trace_rcu_utilization(TPS("End scheduler-tick"));
2783 * Scan the leaf rcu_node structures, processing dyntick state for any that
2784 * have not yet encountered a quiescent state, using the function specified.
2785 * Also initiate boosting for any threads blocked on the root rcu_node.
2787 * The caller must have suppressed start of new grace periods.
2789 static void force_qs_rnp(struct rcu_state
*rsp
, int (*f
)(struct rcu_data
*rsp
))
2792 unsigned long flags
;
2794 struct rcu_node
*rnp
;
2796 rcu_for_each_leaf_node(rsp
, rnp
) {
2797 cond_resched_rcu_qs();
2799 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
2800 if (rnp
->qsmask
== 0) {
2801 if (rcu_state_p
== &rcu_sched_state
||
2802 rsp
!= rcu_state_p
||
2803 rcu_preempt_blocked_readers_cgp(rnp
)) {
2805 * No point in scanning bits because they
2806 * are all zero. But we might need to
2807 * priority-boost blocked readers.
2809 rcu_initiate_boost(rnp
, flags
);
2810 /* rcu_initiate_boost() releases rnp->lock */
2814 (rnp
->parent
->qsmask
& rnp
->grpmask
)) {
2816 * Race between grace-period
2817 * initialization and task exiting RCU
2818 * read-side critical section: Report.
2820 rcu_report_unblock_qs_rnp(rsp
, rnp
, flags
);
2821 /* rcu_report_unblock_qs_rnp() rlses ->lock */
2825 for_each_leaf_node_possible_cpu(rnp
, cpu
) {
2826 unsigned long bit
= leaf_node_cpu_bit(rnp
, cpu
);
2827 if ((rnp
->qsmask
& bit
) != 0) {
2828 if (f(per_cpu_ptr(rsp
->rda
, cpu
)))
2833 /* Idle/offline CPUs, report (releases rnp->lock. */
2834 rcu_report_qs_rnp(mask
, rsp
, rnp
, rnp
->gpnum
, flags
);
2836 /* Nothing to do here, so just drop the lock. */
2837 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
2843 * Force quiescent states on reluctant CPUs, and also detect which
2844 * CPUs are in dyntick-idle mode.
2846 static void force_quiescent_state(struct rcu_state
*rsp
)
2848 unsigned long flags
;
2850 struct rcu_node
*rnp
;
2851 struct rcu_node
*rnp_old
= NULL
;
2853 /* Funnel through hierarchy to reduce memory contention. */
2854 rnp
= __this_cpu_read(rsp
->rda
->mynode
);
2855 for (; rnp
!= NULL
; rnp
= rnp
->parent
) {
2856 ret
= (READ_ONCE(rsp
->gp_flags
) & RCU_GP_FLAG_FQS
) ||
2857 !raw_spin_trylock(&rnp
->fqslock
);
2858 if (rnp_old
!= NULL
)
2859 raw_spin_unlock(&rnp_old
->fqslock
);
2861 rsp
->n_force_qs_lh
++;
2866 /* rnp_old == rcu_get_root(rsp), rnp == NULL. */
2868 /* Reached the root of the rcu_node tree, acquire lock. */
2869 raw_spin_lock_irqsave_rcu_node(rnp_old
, flags
);
2870 raw_spin_unlock(&rnp_old
->fqslock
);
2871 if (READ_ONCE(rsp
->gp_flags
) & RCU_GP_FLAG_FQS
) {
2872 rsp
->n_force_qs_lh
++;
2873 raw_spin_unlock_irqrestore_rcu_node(rnp_old
, flags
);
2874 return; /* Someone beat us to it. */
2876 WRITE_ONCE(rsp
->gp_flags
, READ_ONCE(rsp
->gp_flags
) | RCU_GP_FLAG_FQS
);
2877 raw_spin_unlock_irqrestore_rcu_node(rnp_old
, flags
);
2878 rcu_gp_kthread_wake(rsp
);
2882 * This does the RCU core processing work for the specified rcu_state
2883 * and rcu_data structures. This may be called only from the CPU to
2884 * whom the rdp belongs.
2887 __rcu_process_callbacks(struct rcu_state
*rsp
)
2889 unsigned long flags
;
2891 struct rcu_data
*rdp
= raw_cpu_ptr(rsp
->rda
);
2893 WARN_ON_ONCE(!rdp
->beenonline
);
2895 /* Update RCU state based on any recent quiescent states. */
2896 rcu_check_quiescent_state(rsp
, rdp
);
2898 /* Does this CPU require a not-yet-started grace period? */
2899 local_irq_save(flags
);
2900 if (cpu_needs_another_gp(rsp
, rdp
)) {
2901 raw_spin_lock_rcu_node(rcu_get_root(rsp
)); /* irqs disabled. */
2902 needwake
= rcu_start_gp(rsp
);
2903 raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp
), flags
);
2905 rcu_gp_kthread_wake(rsp
);
2907 local_irq_restore(flags
);
2910 /* If there are callbacks ready, invoke them. */
2911 if (rcu_segcblist_ready_cbs(&rdp
->cblist
))
2912 invoke_rcu_callbacks(rsp
, rdp
);
2914 /* Do any needed deferred wakeups of rcuo kthreads. */
2915 do_nocb_deferred_wakeup(rdp
);
2919 * Do RCU core processing for the current CPU.
2921 static __latent_entropy
void rcu_process_callbacks(struct softirq_action
*unused
)
2923 struct rcu_state
*rsp
;
2925 if (cpu_is_offline(smp_processor_id()))
2927 trace_rcu_utilization(TPS("Start RCU core"));
2928 for_each_rcu_flavor(rsp
)
2929 __rcu_process_callbacks(rsp
);
2930 trace_rcu_utilization(TPS("End RCU core"));
2934 * Schedule RCU callback invocation. If the specified type of RCU
2935 * does not support RCU priority boosting, just do a direct call,
2936 * otherwise wake up the per-CPU kernel kthread. Note that because we
2937 * are running on the current CPU with softirqs disabled, the
2938 * rcu_cpu_kthread_task cannot disappear out from under us.
2940 static void invoke_rcu_callbacks(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
2942 if (unlikely(!READ_ONCE(rcu_scheduler_fully_active
)))
2944 if (likely(!rsp
->boost
)) {
2945 rcu_do_batch(rsp
, rdp
);
2948 invoke_rcu_callbacks_kthread();
2951 static void invoke_rcu_core(void)
2953 if (cpu_online(smp_processor_id()))
2954 raise_softirq(RCU_SOFTIRQ
);
2958 * Handle any core-RCU processing required by a call_rcu() invocation.
2960 static void __call_rcu_core(struct rcu_state
*rsp
, struct rcu_data
*rdp
,
2961 struct rcu_head
*head
, unsigned long flags
)
2966 * If called from an extended quiescent state, invoke the RCU
2967 * core in order to force a re-evaluation of RCU's idleness.
2969 if (!rcu_is_watching())
2972 /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
2973 if (irqs_disabled_flags(flags
) || cpu_is_offline(smp_processor_id()))
2977 * Force the grace period if too many callbacks or too long waiting.
2978 * Enforce hysteresis, and don't invoke force_quiescent_state()
2979 * if some other CPU has recently done so. Also, don't bother
2980 * invoking force_quiescent_state() if the newly enqueued callback
2981 * is the only one waiting for a grace period to complete.
2983 if (unlikely(rcu_segcblist_n_cbs(&rdp
->cblist
) >
2984 rdp
->qlen_last_fqs_check
+ qhimark
)) {
2986 /* Are we ignoring a completed grace period? */
2987 note_gp_changes(rsp
, rdp
);
2989 /* Start a new grace period if one not already started. */
2990 if (!rcu_gp_in_progress(rsp
)) {
2991 struct rcu_node
*rnp_root
= rcu_get_root(rsp
);
2993 raw_spin_lock_rcu_node(rnp_root
);
2994 needwake
= rcu_start_gp(rsp
);
2995 raw_spin_unlock_rcu_node(rnp_root
);
2997 rcu_gp_kthread_wake(rsp
);
2999 /* Give the grace period a kick. */
3000 rdp
->blimit
= LONG_MAX
;
3001 if (rsp
->n_force_qs
== rdp
->n_force_qs_snap
&&
3002 rcu_segcblist_first_pend_cb(&rdp
->cblist
) != head
)
3003 force_quiescent_state(rsp
);
3004 rdp
->n_force_qs_snap
= rsp
->n_force_qs
;
3005 rdp
->qlen_last_fqs_check
= rcu_segcblist_n_cbs(&rdp
->cblist
);
3011 * RCU callback function to leak a callback.
3013 static void rcu_leak_callback(struct rcu_head
*rhp
)
3018 * Helper function for call_rcu() and friends. The cpu argument will
3019 * normally be -1, indicating "currently running CPU". It may specify
3020 * a CPU only if that CPU is a no-CBs CPU. Currently, only _rcu_barrier()
3021 * is expected to specify a CPU.
3024 __call_rcu(struct rcu_head
*head
, rcu_callback_t func
,
3025 struct rcu_state
*rsp
, int cpu
, bool lazy
)
3027 unsigned long flags
;
3028 struct rcu_data
*rdp
;
3030 /* Misaligned rcu_head! */
3031 WARN_ON_ONCE((unsigned long)head
& (sizeof(void *) - 1));
3033 if (debug_rcu_head_queue(head
)) {
3035 * Probable double call_rcu(), so leak the callback.
3036 * Use rcu:rcu_callback trace event to find the previous
3037 * time callback was passed to __call_rcu().
3039 WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pF()!!!\n",
3041 WRITE_ONCE(head
->func
, rcu_leak_callback
);
3046 local_irq_save(flags
);
3047 rdp
= this_cpu_ptr(rsp
->rda
);
3049 /* Add the callback to our list. */
3050 if (unlikely(!rcu_segcblist_is_enabled(&rdp
->cblist
)) || cpu
!= -1) {
3054 rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3055 if (likely(rdp
->mynode
)) {
3056 /* Post-boot, so this should be for a no-CBs CPU. */
3057 offline
= !__call_rcu_nocb(rdp
, head
, lazy
, flags
);
3058 WARN_ON_ONCE(offline
);
3059 /* Offline CPU, _call_rcu() illegal, leak callback. */
3060 local_irq_restore(flags
);
3064 * Very early boot, before rcu_init(). Initialize if needed
3065 * and then drop through to queue the callback.
3068 WARN_ON_ONCE(!rcu_is_watching());
3069 if (rcu_segcblist_empty(&rdp
->cblist
))
3070 rcu_segcblist_init(&rdp
->cblist
);
3072 rcu_segcblist_enqueue(&rdp
->cblist
, head
, lazy
);
3074 rcu_idle_count_callbacks_posted();
3076 if (__is_kfree_rcu_offset((unsigned long)func
))
3077 trace_rcu_kfree_callback(rsp
->name
, head
, (unsigned long)func
,
3078 rcu_segcblist_n_lazy_cbs(&rdp
->cblist
),
3079 rcu_segcblist_n_cbs(&rdp
->cblist
));
3081 trace_rcu_callback(rsp
->name
, head
,
3082 rcu_segcblist_n_lazy_cbs(&rdp
->cblist
),
3083 rcu_segcblist_n_cbs(&rdp
->cblist
));
3085 /* Go handle any RCU core processing required. */
3086 __call_rcu_core(rsp
, rdp
, head
, flags
);
3087 local_irq_restore(flags
);
3091 * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
3092 * @head: structure to be used for queueing the RCU updates.
3093 * @func: actual callback function to be invoked after the grace period
3095 * The callback function will be invoked some time after a full grace
3096 * period elapses, in other words after all currently executing RCU
3097 * read-side critical sections have completed. call_rcu_sched() assumes
3098 * that the read-side critical sections end on enabling of preemption
3099 * or on voluntary preemption.
3100 * RCU read-side critical sections are delimited by:
3102 * - rcu_read_lock_sched() and rcu_read_unlock_sched(), OR
3103 * - anything that disables preemption.
3105 * These may be nested.
3107 * See the description of call_rcu() for more detailed information on
3108 * memory ordering guarantees.
3110 void call_rcu_sched(struct rcu_head
*head
, rcu_callback_t func
)
3112 __call_rcu(head
, func
, &rcu_sched_state
, -1, 0);
3114 EXPORT_SYMBOL_GPL(call_rcu_sched
);
3117 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
3118 * @head: structure to be used for queueing the RCU updates.
3119 * @func: actual callback function to be invoked after the grace period
3121 * The callback function will be invoked some time after a full grace
3122 * period elapses, in other words after all currently executing RCU
3123 * read-side critical sections have completed. call_rcu_bh() assumes
3124 * that the read-side critical sections end on completion of a softirq
3125 * handler. This means that read-side critical sections in process
3126 * context must not be interrupted by softirqs. This interface is to be
3127 * used when most of the read-side critical sections are in softirq context.
3128 * RCU read-side critical sections are delimited by:
3130 * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context, OR
3131 * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
3133 * These may be nested.
3135 * See the description of call_rcu() for more detailed information on
3136 * memory ordering guarantees.
3138 void call_rcu_bh(struct rcu_head
*head
, rcu_callback_t func
)
3140 __call_rcu(head
, func
, &rcu_bh_state
, -1, 0);
3142 EXPORT_SYMBOL_GPL(call_rcu_bh
);
3145 * Queue an RCU callback for lazy invocation after a grace period.
3146 * This will likely be later named something like "call_rcu_lazy()",
3147 * but this change will require some way of tagging the lazy RCU
3148 * callbacks in the list of pending callbacks. Until then, this
3149 * function may only be called from __kfree_rcu().
3151 void kfree_call_rcu(struct rcu_head
*head
,
3152 rcu_callback_t func
)
3154 __call_rcu(head
, func
, rcu_state_p
, -1, 1);
3156 EXPORT_SYMBOL_GPL(kfree_call_rcu
);
3159 * Because a context switch is a grace period for RCU-sched and RCU-bh,
3160 * any blocking grace-period wait automatically implies a grace period
3161 * if there is only one CPU online at any point time during execution
3162 * of either synchronize_sched() or synchronize_rcu_bh(). It is OK to
3163 * occasionally incorrectly indicate that there are multiple CPUs online
3164 * when there was in fact only one the whole time, as this just adds
3165 * some overhead: RCU still operates correctly.
3167 static inline int rcu_blocking_is_gp(void)
3171 might_sleep(); /* Check for RCU read-side critical section. */
3173 ret
= num_online_cpus() <= 1;
3179 * synchronize_sched - wait until an rcu-sched grace period has elapsed.
3181 * Control will return to the caller some time after a full rcu-sched
3182 * grace period has elapsed, in other words after all currently executing
3183 * rcu-sched read-side critical sections have completed. These read-side
3184 * critical sections are delimited by rcu_read_lock_sched() and
3185 * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(),
3186 * local_irq_disable(), and so on may be used in place of
3187 * rcu_read_lock_sched().
3189 * This means that all preempt_disable code sequences, including NMI and
3190 * non-threaded hardware-interrupt handlers, in progress on entry will
3191 * have completed before this primitive returns. However, this does not
3192 * guarantee that softirq handlers will have completed, since in some
3193 * kernels, these handlers can run in process context, and can block.
3195 * Note that this guarantee implies further memory-ordering guarantees.
3196 * On systems with more than one CPU, when synchronize_sched() returns,
3197 * each CPU is guaranteed to have executed a full memory barrier since the
3198 * end of its last RCU-sched read-side critical section whose beginning
3199 * preceded the call to synchronize_sched(). In addition, each CPU having
3200 * an RCU read-side critical section that extends beyond the return from
3201 * synchronize_sched() is guaranteed to have executed a full memory barrier
3202 * after the beginning of synchronize_sched() and before the beginning of
3203 * that RCU read-side critical section. Note that these guarantees include
3204 * CPUs that are offline, idle, or executing in user mode, as well as CPUs
3205 * that are executing in the kernel.
3207 * Furthermore, if CPU A invoked synchronize_sched(), which returned
3208 * to its caller on CPU B, then both CPU A and CPU B are guaranteed
3209 * to have executed a full memory barrier during the execution of
3210 * synchronize_sched() -- even if CPU A and CPU B are the same CPU (but
3211 * again only if the system has more than one CPU).
3213 void synchronize_sched(void)
3215 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map
) ||
3216 lock_is_held(&rcu_lock_map
) ||
3217 lock_is_held(&rcu_sched_lock_map
),
3218 "Illegal synchronize_sched() in RCU-sched read-side critical section");
3219 if (rcu_blocking_is_gp())
3221 if (rcu_gp_is_expedited())
3222 synchronize_sched_expedited();
3224 wait_rcu_gp(call_rcu_sched
);
3226 EXPORT_SYMBOL_GPL(synchronize_sched
);
3229 * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
3231 * Control will return to the caller some time after a full rcu_bh grace
3232 * period has elapsed, in other words after all currently executing rcu_bh
3233 * read-side critical sections have completed. RCU read-side critical
3234 * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(),
3235 * and may be nested.
3237 * See the description of synchronize_sched() for more detailed information
3238 * on memory ordering guarantees.
3240 void synchronize_rcu_bh(void)
3242 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map
) ||
3243 lock_is_held(&rcu_lock_map
) ||
3244 lock_is_held(&rcu_sched_lock_map
),
3245 "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
3246 if (rcu_blocking_is_gp())
3248 if (rcu_gp_is_expedited())
3249 synchronize_rcu_bh_expedited();
3251 wait_rcu_gp(call_rcu_bh
);
3253 EXPORT_SYMBOL_GPL(synchronize_rcu_bh
);
3256 * get_state_synchronize_rcu - Snapshot current RCU state
3258 * Returns a cookie that is used by a later call to cond_synchronize_rcu()
3259 * to determine whether or not a full grace period has elapsed in the
3262 unsigned long get_state_synchronize_rcu(void)
3265 * Any prior manipulation of RCU-protected data must happen
3266 * before the load from ->gpnum.
3271 * Make sure this load happens before the purportedly
3272 * time-consuming work between get_state_synchronize_rcu()
3273 * and cond_synchronize_rcu().
3275 return smp_load_acquire(&rcu_state_p
->gpnum
);
3277 EXPORT_SYMBOL_GPL(get_state_synchronize_rcu
);
3280 * cond_synchronize_rcu - Conditionally wait for an RCU grace period
3282 * @oldstate: return value from earlier call to get_state_synchronize_rcu()
3284 * If a full RCU grace period has elapsed since the earlier call to
3285 * get_state_synchronize_rcu(), just return. Otherwise, invoke
3286 * synchronize_rcu() to wait for a full grace period.
3288 * Yes, this function does not take counter wrap into account. But
3289 * counter wrap is harmless. If the counter wraps, we have waited for
3290 * more than 2 billion grace periods (and way more on a 64-bit system!),
3291 * so waiting for one additional grace period should be just fine.
3293 void cond_synchronize_rcu(unsigned long oldstate
)
3295 unsigned long newstate
;
3298 * Ensure that this load happens before any RCU-destructive
3299 * actions the caller might carry out after we return.
3301 newstate
= smp_load_acquire(&rcu_state_p
->completed
);
3302 if (ULONG_CMP_GE(oldstate
, newstate
))
3305 EXPORT_SYMBOL_GPL(cond_synchronize_rcu
);
3308 * get_state_synchronize_sched - Snapshot current RCU-sched state
3310 * Returns a cookie that is used by a later call to cond_synchronize_sched()
3311 * to determine whether or not a full grace period has elapsed in the
3314 unsigned long get_state_synchronize_sched(void)
3317 * Any prior manipulation of RCU-protected data must happen
3318 * before the load from ->gpnum.
3323 * Make sure this load happens before the purportedly
3324 * time-consuming work between get_state_synchronize_sched()
3325 * and cond_synchronize_sched().
3327 return smp_load_acquire(&rcu_sched_state
.gpnum
);
3329 EXPORT_SYMBOL_GPL(get_state_synchronize_sched
);
3332 * cond_synchronize_sched - Conditionally wait for an RCU-sched grace period
3334 * @oldstate: return value from earlier call to get_state_synchronize_sched()
3336 * If a full RCU-sched grace period has elapsed since the earlier call to
3337 * get_state_synchronize_sched(), just return. Otherwise, invoke
3338 * synchronize_sched() to wait for a full grace period.
3340 * Yes, this function does not take counter wrap into account. But
3341 * counter wrap is harmless. If the counter wraps, we have waited for
3342 * more than 2 billion grace periods (and way more on a 64-bit system!),
3343 * so waiting for one additional grace period should be just fine.
3345 void cond_synchronize_sched(unsigned long oldstate
)
3347 unsigned long newstate
;
3350 * Ensure that this load happens before any RCU-destructive
3351 * actions the caller might carry out after we return.
3353 newstate
= smp_load_acquire(&rcu_sched_state
.completed
);
3354 if (ULONG_CMP_GE(oldstate
, newstate
))
3355 synchronize_sched();
3357 EXPORT_SYMBOL_GPL(cond_synchronize_sched
);
3360 * Check to see if there is any immediate RCU-related work to be done
3361 * by the current CPU, for the specified type of RCU, returning 1 if so.
3362 * The checks are in order of increasing expense: checks that can be
3363 * carried out against CPU-local state are performed first. However,
3364 * we must check for CPU stalls first, else we might not get a chance.
3366 static int __rcu_pending(struct rcu_state
*rsp
, struct rcu_data
*rdp
)
3368 struct rcu_node
*rnp
= rdp
->mynode
;
3370 rdp
->n_rcu_pending
++;
3372 /* Check for CPU stalls, if enabled. */
3373 check_cpu_stall(rsp
, rdp
);
3375 /* Is this CPU a NO_HZ_FULL CPU that should ignore RCU? */
3376 if (rcu_nohz_full_cpu(rsp
))
3379 /* Is the RCU core waiting for a quiescent state from this CPU? */
3380 if (rcu_scheduler_fully_active
&&
3381 rdp
->core_needs_qs
&& rdp
->cpu_no_qs
.b
.norm
&&
3382 rdp
->rcu_qs_ctr_snap
== __this_cpu_read(rcu_dynticks
.rcu_qs_ctr
)) {
3383 rdp
->n_rp_core_needs_qs
++;
3384 } else if (rdp
->core_needs_qs
&& !rdp
->cpu_no_qs
.b
.norm
) {
3385 rdp
->n_rp_report_qs
++;
3389 /* Does this CPU have callbacks ready to invoke? */
3390 if (rcu_segcblist_ready_cbs(&rdp
->cblist
)) {
3391 rdp
->n_rp_cb_ready
++;
3395 /* Has RCU gone idle with this CPU needing another grace period? */
3396 if (cpu_needs_another_gp(rsp
, rdp
)) {
3397 rdp
->n_rp_cpu_needs_gp
++;
3401 /* Has another RCU grace period completed? */
3402 if (READ_ONCE(rnp
->completed
) != rdp
->completed
) { /* outside lock */
3403 rdp
->n_rp_gp_completed
++;
3407 /* Has a new RCU grace period started? */
3408 if (READ_ONCE(rnp
->gpnum
) != rdp
->gpnum
||
3409 unlikely(READ_ONCE(rdp
->gpwrap
))) { /* outside lock */
3410 rdp
->n_rp_gp_started
++;
3414 /* Does this CPU need a deferred NOCB wakeup? */
3415 if (rcu_nocb_need_deferred_wakeup(rdp
)) {
3416 rdp
->n_rp_nocb_defer_wakeup
++;
3421 rdp
->n_rp_need_nothing
++;
3426 * Check to see if there is any immediate RCU-related work to be done
3427 * by the current CPU, returning 1 if so. This function is part of the
3428 * RCU implementation; it is -not- an exported member of the RCU API.
3430 static int rcu_pending(void)
3432 struct rcu_state
*rsp
;
3434 for_each_rcu_flavor(rsp
)
3435 if (__rcu_pending(rsp
, this_cpu_ptr(rsp
->rda
)))
3441 * Return true if the specified CPU has any callback. If all_lazy is
3442 * non-NULL, store an indication of whether all callbacks are lazy.
3443 * (If there are no callbacks, all of them are deemed to be lazy.)
3445 static bool __maybe_unused
rcu_cpu_has_callbacks(bool *all_lazy
)
3449 struct rcu_data
*rdp
;
3450 struct rcu_state
*rsp
;
3452 for_each_rcu_flavor(rsp
) {
3453 rdp
= this_cpu_ptr(rsp
->rda
);
3454 if (rcu_segcblist_empty(&rdp
->cblist
))
3457 if (rcu_segcblist_n_nonlazy_cbs(&rdp
->cblist
) || !all_lazy
) {
3468 * Helper function for _rcu_barrier() tracing. If tracing is disabled,
3469 * the compiler is expected to optimize this away.
3471 static void _rcu_barrier_trace(struct rcu_state
*rsp
, const char *s
,
3472 int cpu
, unsigned long done
)
3474 trace_rcu_barrier(rsp
->name
, s
, cpu
,
3475 atomic_read(&rsp
->barrier_cpu_count
), done
);
3479 * RCU callback function for _rcu_barrier(). If we are last, wake
3480 * up the task executing _rcu_barrier().
3482 static void rcu_barrier_callback(struct rcu_head
*rhp
)
3484 struct rcu_data
*rdp
= container_of(rhp
, struct rcu_data
, barrier_head
);
3485 struct rcu_state
*rsp
= rdp
->rsp
;
3487 if (atomic_dec_and_test(&rsp
->barrier_cpu_count
)) {
3488 _rcu_barrier_trace(rsp
, TPS("LastCB"), -1,
3489 rsp
->barrier_sequence
);
3490 complete(&rsp
->barrier_completion
);
3492 _rcu_barrier_trace(rsp
, TPS("CB"), -1, rsp
->barrier_sequence
);
3497 * Called with preemption disabled, and from cross-cpu IRQ context.
3499 static void rcu_barrier_func(void *type
)
3501 struct rcu_state
*rsp
= type
;
3502 struct rcu_data
*rdp
= raw_cpu_ptr(rsp
->rda
);
3504 _rcu_barrier_trace(rsp
, TPS("IRQ"), -1, rsp
->barrier_sequence
);
3505 rdp
->barrier_head
.func
= rcu_barrier_callback
;
3506 debug_rcu_head_queue(&rdp
->barrier_head
);
3507 if (rcu_segcblist_entrain(&rdp
->cblist
, &rdp
->barrier_head
, 0)) {
3508 atomic_inc(&rsp
->barrier_cpu_count
);
3510 debug_rcu_head_unqueue(&rdp
->barrier_head
);
3511 _rcu_barrier_trace(rsp
, TPS("IRQNQ"), -1,
3512 rsp
->barrier_sequence
);
3517 * Orchestrate the specified type of RCU barrier, waiting for all
3518 * RCU callbacks of the specified type to complete.
3520 static void _rcu_barrier(struct rcu_state
*rsp
)
3523 struct rcu_data
*rdp
;
3524 unsigned long s
= rcu_seq_snap(&rsp
->barrier_sequence
);
3526 _rcu_barrier_trace(rsp
, TPS("Begin"), -1, s
);
3528 /* Take mutex to serialize concurrent rcu_barrier() requests. */
3529 mutex_lock(&rsp
->barrier_mutex
);
3531 /* Did someone else do our work for us? */
3532 if (rcu_seq_done(&rsp
->barrier_sequence
, s
)) {
3533 _rcu_barrier_trace(rsp
, TPS("EarlyExit"), -1,
3534 rsp
->barrier_sequence
);
3535 smp_mb(); /* caller's subsequent code after above check. */
3536 mutex_unlock(&rsp
->barrier_mutex
);
3540 /* Mark the start of the barrier operation. */
3541 rcu_seq_start(&rsp
->barrier_sequence
);
3542 _rcu_barrier_trace(rsp
, TPS("Inc1"), -1, rsp
->barrier_sequence
);
3545 * Initialize the count to one rather than to zero in order to
3546 * avoid a too-soon return to zero in case of a short grace period
3547 * (or preemption of this task). Exclude CPU-hotplug operations
3548 * to ensure that no offline CPU has callbacks queued.
3550 init_completion(&rsp
->barrier_completion
);
3551 atomic_set(&rsp
->barrier_cpu_count
, 1);
3555 * Force each CPU with callbacks to register a new callback.
3556 * When that callback is invoked, we will know that all of the
3557 * corresponding CPU's preceding callbacks have been invoked.
3559 for_each_possible_cpu(cpu
) {
3560 if (!cpu_online(cpu
) && !rcu_is_nocb_cpu(cpu
))
3562 rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3563 if (rcu_is_nocb_cpu(cpu
)) {
3564 if (!rcu_nocb_cpu_needs_barrier(rsp
, cpu
)) {
3565 _rcu_barrier_trace(rsp
, TPS("OfflineNoCB"), cpu
,
3566 rsp
->barrier_sequence
);
3568 _rcu_barrier_trace(rsp
, TPS("OnlineNoCB"), cpu
,
3569 rsp
->barrier_sequence
);
3570 smp_mb__before_atomic();
3571 atomic_inc(&rsp
->barrier_cpu_count
);
3572 __call_rcu(&rdp
->barrier_head
,
3573 rcu_barrier_callback
, rsp
, cpu
, 0);
3575 } else if (rcu_segcblist_n_cbs(&rdp
->cblist
)) {
3576 _rcu_barrier_trace(rsp
, TPS("OnlineQ"), cpu
,
3577 rsp
->barrier_sequence
);
3578 smp_call_function_single(cpu
, rcu_barrier_func
, rsp
, 1);
3580 _rcu_barrier_trace(rsp
, TPS("OnlineNQ"), cpu
,
3581 rsp
->barrier_sequence
);
3587 * Now that we have an rcu_barrier_callback() callback on each
3588 * CPU, and thus each counted, remove the initial count.
3590 if (atomic_dec_and_test(&rsp
->barrier_cpu_count
))
3591 complete(&rsp
->barrier_completion
);
3593 /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
3594 wait_for_completion(&rsp
->barrier_completion
);
3596 /* Mark the end of the barrier operation. */
3597 _rcu_barrier_trace(rsp
, TPS("Inc2"), -1, rsp
->barrier_sequence
);
3598 rcu_seq_end(&rsp
->barrier_sequence
);
3600 /* Other rcu_barrier() invocations can now safely proceed. */
3601 mutex_unlock(&rsp
->barrier_mutex
);
3605 * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
3607 void rcu_barrier_bh(void)
3609 _rcu_barrier(&rcu_bh_state
);
3611 EXPORT_SYMBOL_GPL(rcu_barrier_bh
);
3614 * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
3616 void rcu_barrier_sched(void)
3618 _rcu_barrier(&rcu_sched_state
);
3620 EXPORT_SYMBOL_GPL(rcu_barrier_sched
);
3623 * Propagate ->qsinitmask bits up the rcu_node tree to account for the
3624 * first CPU in a given leaf rcu_node structure coming online. The caller
3625 * must hold the corresponding leaf rcu_node ->lock with interrrupts
3628 static void rcu_init_new_rnp(struct rcu_node
*rnp_leaf
)
3631 struct rcu_node
*rnp
= rnp_leaf
;
3633 lockdep_assert_held(&rnp
->lock
);
3635 mask
= rnp
->grpmask
;
3639 raw_spin_lock_rcu_node(rnp
); /* Interrupts already disabled. */
3640 rnp
->qsmaskinit
|= mask
;
3641 raw_spin_unlock_rcu_node(rnp
); /* Interrupts remain disabled. */
3646 * Do boot-time initialization of a CPU's per-CPU RCU data.
3649 rcu_boot_init_percpu_data(int cpu
, struct rcu_state
*rsp
)
3651 unsigned long flags
;
3652 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3653 struct rcu_node
*rnp
= rcu_get_root(rsp
);
3655 /* Set up local state, ensuring consistent view of global state. */
3656 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3657 rdp
->grpmask
= leaf_node_cpu_bit(rdp
->mynode
, cpu
);
3658 rdp
->dynticks
= &per_cpu(rcu_dynticks
, cpu
);
3659 WARN_ON_ONCE(rdp
->dynticks
->dynticks_nesting
!= DYNTICK_TASK_EXIT_IDLE
);
3660 WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp
->dynticks
)));
3663 rcu_boot_init_nocb_percpu_data(rdp
);
3664 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3668 * Initialize a CPU's per-CPU RCU data. Note that only one online or
3669 * offline event can be happening at a given time. Note also that we
3670 * can accept some slop in the rsp->completed access due to the fact
3671 * that this CPU cannot possibly have any RCU callbacks in flight yet.
3674 rcu_init_percpu_data(int cpu
, struct rcu_state
*rsp
)
3676 unsigned long flags
;
3677 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3678 struct rcu_node
*rnp
= rcu_get_root(rsp
);
3680 /* Set up local state, ensuring consistent view of global state. */
3681 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3682 rdp
->qlen_last_fqs_check
= 0;
3683 rdp
->n_force_qs_snap
= rsp
->n_force_qs
;
3684 rdp
->blimit
= blimit
;
3685 if (rcu_segcblist_empty(&rdp
->cblist
) && /* No early-boot CBs? */
3686 !init_nocb_callback_list(rdp
))
3687 rcu_segcblist_init(&rdp
->cblist
); /* Re-enable callbacks. */
3688 rdp
->dynticks
->dynticks_nesting
= DYNTICK_TASK_EXIT_IDLE
;
3689 rcu_dynticks_eqs_online();
3690 raw_spin_unlock_rcu_node(rnp
); /* irqs remain disabled. */
3693 * Add CPU to leaf rcu_node pending-online bitmask. Any needed
3694 * propagation up the rcu_node tree will happen at the beginning
3695 * of the next grace period.
3698 raw_spin_lock_rcu_node(rnp
); /* irqs already disabled. */
3699 rdp
->beenonline
= true; /* We have now been online. */
3700 rdp
->gpnum
= rnp
->completed
; /* Make CPU later note any new GP. */
3701 rdp
->completed
= rnp
->completed
;
3702 rdp
->cpu_no_qs
.b
.norm
= true;
3703 rdp
->rcu_qs_ctr_snap
= per_cpu(rcu_dynticks
.rcu_qs_ctr
, cpu
);
3704 rdp
->core_needs_qs
= false;
3705 trace_rcu_grace_period(rsp
->name
, rdp
->gpnum
, TPS("cpuonl"));
3706 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3710 * Invoked early in the CPU-online process, when pretty much all
3711 * services are available. The incoming CPU is not present.
3713 int rcutree_prepare_cpu(unsigned int cpu
)
3715 struct rcu_state
*rsp
;
3717 for_each_rcu_flavor(rsp
)
3718 rcu_init_percpu_data(cpu
, rsp
);
3720 rcu_prepare_kthreads(cpu
);
3721 rcu_spawn_all_nocb_kthreads(cpu
);
3727 * Update RCU priority boot kthread affinity for CPU-hotplug changes.
3729 static void rcutree_affinity_setting(unsigned int cpu
, int outgoing
)
3731 struct rcu_data
*rdp
= per_cpu_ptr(rcu_state_p
->rda
, cpu
);
3733 rcu_boost_kthread_setaffinity(rdp
->mynode
, outgoing
);
3737 * Near the end of the CPU-online process. Pretty much all services
3738 * enabled, and the CPU is now very much alive.
3740 int rcutree_online_cpu(unsigned int cpu
)
3742 sync_sched_exp_online_cleanup(cpu
);
3743 rcutree_affinity_setting(cpu
, -1);
3744 if (IS_ENABLED(CONFIG_TREE_SRCU
))
3745 srcu_online_cpu(cpu
);
3750 * Near the beginning of the process. The CPU is still very much alive
3751 * with pretty much all services enabled.
3753 int rcutree_offline_cpu(unsigned int cpu
)
3755 rcutree_affinity_setting(cpu
, cpu
);
3756 if (IS_ENABLED(CONFIG_TREE_SRCU
))
3757 srcu_offline_cpu(cpu
);
3762 * Near the end of the offline process. We do only tracing here.
3764 int rcutree_dying_cpu(unsigned int cpu
)
3766 struct rcu_state
*rsp
;
3768 for_each_rcu_flavor(rsp
)
3769 rcu_cleanup_dying_cpu(rsp
);
3774 * The outgoing CPU is gone and we are running elsewhere.
3776 int rcutree_dead_cpu(unsigned int cpu
)
3778 struct rcu_state
*rsp
;
3780 for_each_rcu_flavor(rsp
) {
3781 rcu_cleanup_dead_cpu(cpu
, rsp
);
3782 do_nocb_deferred_wakeup(per_cpu_ptr(rsp
->rda
, cpu
));
3788 * Mark the specified CPU as being online so that subsequent grace periods
3789 * (both expedited and normal) will wait on it. Note that this means that
3790 * incoming CPUs are not allowed to use RCU read-side critical sections
3791 * until this function is called. Failing to observe this restriction
3792 * will result in lockdep splats.
3794 * Note that this function is special in that it is invoked directly
3795 * from the incoming CPU rather than from the cpuhp_step mechanism.
3796 * This is because this function must be invoked at a precise location.
3798 void rcu_cpu_starting(unsigned int cpu
)
3800 unsigned long flags
;
3803 unsigned long oldmask
;
3804 struct rcu_data
*rdp
;
3805 struct rcu_node
*rnp
;
3806 struct rcu_state
*rsp
;
3808 for_each_rcu_flavor(rsp
) {
3809 rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3811 mask
= rdp
->grpmask
;
3812 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3813 rnp
->qsmaskinitnext
|= mask
;
3814 oldmask
= rnp
->expmaskinitnext
;
3815 rnp
->expmaskinitnext
|= mask
;
3816 oldmask
^= rnp
->expmaskinitnext
;
3817 nbits
= bitmap_weight(&oldmask
, BITS_PER_LONG
);
3818 /* Allow lockless access for expedited grace periods. */
3819 smp_store_release(&rsp
->ncpus
, rsp
->ncpus
+ nbits
); /* ^^^ */
3820 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3822 smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
3825 #ifdef CONFIG_HOTPLUG_CPU
3827 * The CPU is exiting the idle loop into the arch_cpu_idle_dead()
3828 * function. We now remove it from the rcu_node tree's ->qsmaskinit
3831 static void rcu_cleanup_dying_idle_cpu(int cpu
, struct rcu_state
*rsp
)
3833 unsigned long flags
;
3835 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3836 struct rcu_node
*rnp
= rdp
->mynode
; /* Outgoing CPU's rdp & rnp. */
3838 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
3839 mask
= rdp
->grpmask
;
3840 raw_spin_lock_irqsave_rcu_node(rnp
, flags
); /* Enforce GP memory-order guarantee. */
3841 rnp
->qsmaskinitnext
&= ~mask
;
3842 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3846 * The outgoing function has no further need of RCU, so remove it from
3847 * the list of CPUs that RCU must track.
3849 * Note that this function is special in that it is invoked directly
3850 * from the outgoing CPU rather than from the cpuhp_step mechanism.
3851 * This is because this function must be invoked at a precise location.
3853 void rcu_report_dead(unsigned int cpu
)
3855 struct rcu_state
*rsp
;
3857 /* QS for any half-done expedited RCU-sched GP. */
3859 rcu_report_exp_rdp(&rcu_sched_state
,
3860 this_cpu_ptr(rcu_sched_state
.rda
), true);
3862 for_each_rcu_flavor(rsp
)
3863 rcu_cleanup_dying_idle_cpu(cpu
, rsp
);
3866 /* Migrate the dead CPU's callbacks to the current CPU. */
3867 static void rcu_migrate_callbacks(int cpu
, struct rcu_state
*rsp
)
3869 unsigned long flags
;
3870 struct rcu_data
*my_rdp
;
3871 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
3872 struct rcu_node
*rnp_root
= rcu_get_root(rdp
->rsp
);
3874 if (rcu_is_nocb_cpu(cpu
) || rcu_segcblist_empty(&rdp
->cblist
))
3875 return; /* No callbacks to migrate. */
3877 local_irq_save(flags
);
3878 my_rdp
= this_cpu_ptr(rsp
->rda
);
3879 if (rcu_nocb_adopt_orphan_cbs(my_rdp
, rdp
, flags
)) {
3880 local_irq_restore(flags
);
3883 raw_spin_lock_rcu_node(rnp_root
); /* irqs already disabled. */
3884 rcu_advance_cbs(rsp
, rnp_root
, rdp
); /* Leverage recent GPs. */
3885 rcu_advance_cbs(rsp
, rnp_root
, my_rdp
); /* Assign GP to pending CBs. */
3886 rcu_segcblist_merge(&my_rdp
->cblist
, &rdp
->cblist
);
3887 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp
->cblist
) !=
3888 !rcu_segcblist_n_cbs(&my_rdp
->cblist
));
3889 raw_spin_unlock_irqrestore_rcu_node(rnp_root
, flags
);
3890 WARN_ONCE(rcu_segcblist_n_cbs(&rdp
->cblist
) != 0 ||
3891 !rcu_segcblist_empty(&rdp
->cblist
),
3892 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
3893 cpu
, rcu_segcblist_n_cbs(&rdp
->cblist
),
3894 rcu_segcblist_first_cb(&rdp
->cblist
));
3898 * The outgoing CPU has just passed through the dying-idle state,
3899 * and we are being invoked from the CPU that was IPIed to continue the
3900 * offline operation. We need to migrate the outgoing CPU's callbacks.
3902 void rcutree_migrate_callbacks(int cpu
)
3904 struct rcu_state
*rsp
;
3906 for_each_rcu_flavor(rsp
)
3907 rcu_migrate_callbacks(cpu
, rsp
);
3912 * On non-huge systems, use expedited RCU grace periods to make suspend
3913 * and hibernation run faster.
3915 static int rcu_pm_notify(struct notifier_block
*self
,
3916 unsigned long action
, void *hcpu
)
3919 case PM_HIBERNATION_PREPARE
:
3920 case PM_SUSPEND_PREPARE
:
3921 if (nr_cpu_ids
<= 256) /* Expediting bad for large systems. */
3924 case PM_POST_HIBERNATION
:
3925 case PM_POST_SUSPEND
:
3926 if (nr_cpu_ids
<= 256) /* Expediting bad for large systems. */
3927 rcu_unexpedite_gp();
3936 * Spawn the kthreads that handle each RCU flavor's grace periods.
3938 static int __init
rcu_spawn_gp_kthread(void)
3940 unsigned long flags
;
3941 int kthread_prio_in
= kthread_prio
;
3942 struct rcu_node
*rnp
;
3943 struct rcu_state
*rsp
;
3944 struct sched_param sp
;
3945 struct task_struct
*t
;
3947 /* Force priority into range. */
3948 if (IS_ENABLED(CONFIG_RCU_BOOST
) && kthread_prio
< 1)
3950 else if (kthread_prio
< 0)
3952 else if (kthread_prio
> 99)
3954 if (kthread_prio
!= kthread_prio_in
)
3955 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
3956 kthread_prio
, kthread_prio_in
);
3958 rcu_scheduler_fully_active
= 1;
3959 for_each_rcu_flavor(rsp
) {
3960 t
= kthread_create(rcu_gp_kthread
, rsp
, "%s", rsp
->name
);
3962 rnp
= rcu_get_root(rsp
);
3963 raw_spin_lock_irqsave_rcu_node(rnp
, flags
);
3964 rsp
->gp_kthread
= t
;
3966 sp
.sched_priority
= kthread_prio
;
3967 sched_setscheduler_nocheck(t
, SCHED_FIFO
, &sp
);
3969 raw_spin_unlock_irqrestore_rcu_node(rnp
, flags
);
3972 rcu_spawn_nocb_kthreads();
3973 rcu_spawn_boost_kthreads();
3976 early_initcall(rcu_spawn_gp_kthread
);
3979 * This function is invoked towards the end of the scheduler's
3980 * initialization process. Before this is called, the idle task might
3981 * contain synchronous grace-period primitives (during which time, this idle
3982 * task is booting the system, and such primitives are no-ops). After this
3983 * function is called, any synchronous grace-period primitives are run as
3984 * expedited, with the requesting task driving the grace period forward.
3985 * A later core_initcall() rcu_set_runtime_mode() will switch to full
3986 * runtime RCU functionality.
3988 void rcu_scheduler_starting(void)
3990 WARN_ON(num_online_cpus() != 1);
3991 WARN_ON(nr_context_switches() > 0);
3992 rcu_test_sync_prims();
3993 rcu_scheduler_active
= RCU_SCHEDULER_INIT
;
3994 rcu_test_sync_prims();
3998 * Helper function for rcu_init() that initializes one rcu_state structure.
4000 static void __init
rcu_init_one(struct rcu_state
*rsp
)
4002 static const char * const buf
[] = RCU_NODE_NAME_INIT
;
4003 static const char * const fqs
[] = RCU_FQS_NAME_INIT
;
4004 static struct lock_class_key rcu_node_class
[RCU_NUM_LVLS
];
4005 static struct lock_class_key rcu_fqs_class
[RCU_NUM_LVLS
];
4007 int levelspread
[RCU_NUM_LVLS
]; /* kids/node in each level. */
4011 struct rcu_node
*rnp
;
4013 BUILD_BUG_ON(RCU_NUM_LVLS
> ARRAY_SIZE(buf
)); /* Fix buf[] init! */
4015 /* Silence gcc 4.8 false positive about array index out of range. */
4016 if (rcu_num_lvls
<= 0 || rcu_num_lvls
> RCU_NUM_LVLS
)
4017 panic("rcu_init_one: rcu_num_lvls out of range");
4019 /* Initialize the level-tracking arrays. */
4021 for (i
= 1; i
< rcu_num_lvls
; i
++)
4022 rsp
->level
[i
] = rsp
->level
[i
- 1] + num_rcu_lvl
[i
- 1];
4023 rcu_init_levelspread(levelspread
, num_rcu_lvl
);
4025 /* Initialize the elements themselves, starting from the leaves. */
4027 for (i
= rcu_num_lvls
- 1; i
>= 0; i
--) {
4028 cpustride
*= levelspread
[i
];
4029 rnp
= rsp
->level
[i
];
4030 for (j
= 0; j
< num_rcu_lvl
[i
]; j
++, rnp
++) {
4031 raw_spin_lock_init(&ACCESS_PRIVATE(rnp
, lock
));
4032 lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp
, lock
),
4033 &rcu_node_class
[i
], buf
[i
]);
4034 raw_spin_lock_init(&rnp
->fqslock
);
4035 lockdep_set_class_and_name(&rnp
->fqslock
,
4036 &rcu_fqs_class
[i
], fqs
[i
]);
4037 rnp
->gpnum
= rsp
->gpnum
;
4038 rnp
->completed
= rsp
->completed
;
4040 rnp
->qsmaskinit
= 0;
4041 rnp
->grplo
= j
* cpustride
;
4042 rnp
->grphi
= (j
+ 1) * cpustride
- 1;
4043 if (rnp
->grphi
>= nr_cpu_ids
)
4044 rnp
->grphi
= nr_cpu_ids
- 1;
4050 rnp
->grpnum
= j
% levelspread
[i
- 1];
4051 rnp
->grpmask
= 1UL << rnp
->grpnum
;
4052 rnp
->parent
= rsp
->level
[i
- 1] +
4053 j
/ levelspread
[i
- 1];
4056 INIT_LIST_HEAD(&rnp
->blkd_tasks
);
4057 rcu_init_one_nocb(rnp
);
4058 init_waitqueue_head(&rnp
->exp_wq
[0]);
4059 init_waitqueue_head(&rnp
->exp_wq
[1]);
4060 init_waitqueue_head(&rnp
->exp_wq
[2]);
4061 init_waitqueue_head(&rnp
->exp_wq
[3]);
4062 spin_lock_init(&rnp
->exp_lock
);
4066 init_swait_queue_head(&rsp
->gp_wq
);
4067 init_swait_queue_head(&rsp
->expedited_wq
);
4068 rnp
= rsp
->level
[rcu_num_lvls
- 1];
4069 for_each_possible_cpu(i
) {
4070 while (i
> rnp
->grphi
)
4072 per_cpu_ptr(rsp
->rda
, i
)->mynode
= rnp
;
4073 rcu_boot_init_percpu_data(i
, rsp
);
4075 list_add(&rsp
->flavors
, &rcu_struct_flavors
);
4079 * Compute the rcu_node tree geometry from kernel parameters. This cannot
4080 * replace the definitions in tree.h because those are needed to size
4081 * the ->node array in the rcu_state structure.
4083 static void __init
rcu_init_geometry(void)
4087 int rcu_capacity
[RCU_NUM_LVLS
];
4090 * Initialize any unspecified boot parameters.
4091 * The default values of jiffies_till_first_fqs and
4092 * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
4093 * value, which is a function of HZ, then adding one for each
4094 * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
4096 d
= RCU_JIFFIES_TILL_FORCE_QS
+ nr_cpu_ids
/ RCU_JIFFIES_FQS_DIV
;
4097 if (jiffies_till_first_fqs
== ULONG_MAX
)
4098 jiffies_till_first_fqs
= d
;
4099 if (jiffies_till_next_fqs
== ULONG_MAX
)
4100 jiffies_till_next_fqs
= d
;
4102 /* If the compile-time values are accurate, just leave. */
4103 if (rcu_fanout_leaf
== RCU_FANOUT_LEAF
&&
4104 nr_cpu_ids
== NR_CPUS
)
4106 pr_info("RCU: Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
4107 rcu_fanout_leaf
, nr_cpu_ids
);
4110 * The boot-time rcu_fanout_leaf parameter must be at least two
4111 * and cannot exceed the number of bits in the rcu_node masks.
4112 * Complain and fall back to the compile-time values if this
4113 * limit is exceeded.
4115 if (rcu_fanout_leaf
< 2 ||
4116 rcu_fanout_leaf
> sizeof(unsigned long) * 8) {
4117 rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
4123 * Compute number of nodes that can be handled an rcu_node tree
4124 * with the given number of levels.
4126 rcu_capacity
[0] = rcu_fanout_leaf
;
4127 for (i
= 1; i
< RCU_NUM_LVLS
; i
++)
4128 rcu_capacity
[i
] = rcu_capacity
[i
- 1] * RCU_FANOUT
;
4131 * The tree must be able to accommodate the configured number of CPUs.
4132 * If this limit is exceeded, fall back to the compile-time values.
4134 if (nr_cpu_ids
> rcu_capacity
[RCU_NUM_LVLS
- 1]) {
4135 rcu_fanout_leaf
= RCU_FANOUT_LEAF
;
4140 /* Calculate the number of levels in the tree. */
4141 for (i
= 0; nr_cpu_ids
> rcu_capacity
[i
]; i
++) {
4143 rcu_num_lvls
= i
+ 1;
4145 /* Calculate the number of rcu_nodes at each level of the tree. */
4146 for (i
= 0; i
< rcu_num_lvls
; i
++) {
4147 int cap
= rcu_capacity
[(rcu_num_lvls
- 1) - i
];
4148 num_rcu_lvl
[i
] = DIV_ROUND_UP(nr_cpu_ids
, cap
);
4151 /* Calculate the total number of rcu_node structures. */
4153 for (i
= 0; i
< rcu_num_lvls
; i
++)
4154 rcu_num_nodes
+= num_rcu_lvl
[i
];
4158 * Dump out the structure of the rcu_node combining tree associated
4159 * with the rcu_state structure referenced by rsp.
4161 static void __init
rcu_dump_rcu_node_tree(struct rcu_state
*rsp
)
4164 struct rcu_node
*rnp
;
4166 pr_info("rcu_node tree layout dump\n");
4168 rcu_for_each_node_breadth_first(rsp
, rnp
) {
4169 if (rnp
->level
!= level
) {
4174 pr_cont("%d:%d ^%d ", rnp
->grplo
, rnp
->grphi
, rnp
->grpnum
);
4179 void __init
rcu_init(void)
4183 rcu_early_boot_tests();
4185 rcu_bootup_announce();
4186 rcu_init_geometry();
4187 rcu_init_one(&rcu_bh_state
);
4188 rcu_init_one(&rcu_sched_state
);
4190 rcu_dump_rcu_node_tree(&rcu_sched_state
);
4191 __rcu_init_preempt();
4192 open_softirq(RCU_SOFTIRQ
, rcu_process_callbacks
);
4195 * We don't need protection against CPU-hotplug here because
4196 * this is called early in boot, before either interrupts
4197 * or the scheduler are operational.
4199 pm_notifier(rcu_pm_notify
, 0);
4200 for_each_online_cpu(cpu
) {
4201 rcutree_prepare_cpu(cpu
);
4202 rcu_cpu_starting(cpu
);
4203 if (IS_ENABLED(CONFIG_TREE_SRCU
))
4204 srcu_online_cpu(cpu
);
4208 #include "tree_exp.h"
4209 #include "tree_plugin.h"