2 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
3 * Internal non-public definitions that provide either classic
4 * or preemptible semantics.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you can access it online at
18 * http://www.gnu.org/licenses/gpl-2.0.html.
20 * Copyright Red Hat, 2009
21 * Copyright IBM Corporation, 2009
23 * Author: Ingo Molnar <mingo@elte.hu>
24 * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
27 #include <linux/delay.h>
28 #include <linux/gfp.h>
29 #include <linux/oom.h>
30 #include <linux/smpboot.h>
31 #include "../time/tick-internal.h"
33 #ifdef CONFIG_RCU_BOOST
35 #include "../locking/rtmutex_common.h"
38 * Control variables for per-CPU and per-rcu_node kthreads. These
39 * handle all flavors of RCU.
41 static DEFINE_PER_CPU(struct task_struct
*, rcu_cpu_kthread_task
);
42 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status
);
43 DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops
);
44 DEFINE_PER_CPU(char, rcu_cpu_has_work
);
46 #else /* #ifdef CONFIG_RCU_BOOST */
49 * Some architectures do not define rt_mutexes, but if !CONFIG_RCU_BOOST,
50 * all uses are in dead code. Provide a definition to keep the compiler
51 * happy, but add WARN_ON_ONCE() to complain if used in the wrong place.
52 * This probably needs to be excluded from -rt builds.
54 #define rt_mutex_owner(a) ({ WARN_ON_ONCE(1); NULL; })
56 #endif /* #else #ifdef CONFIG_RCU_BOOST */
58 #ifdef CONFIG_RCU_NOCB_CPU
59 static cpumask_var_t rcu_nocb_mask
; /* CPUs to have callbacks offloaded. */
60 static bool have_rcu_nocb_mask
; /* Was rcu_nocb_mask allocated? */
61 static bool __read_mostly rcu_nocb_poll
; /* Offload kthread are to poll. */
62 #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
65 * Check the RCU kernel configuration parameters and print informative
66 * messages about anything out of the ordinary. If you like #ifdef, you
67 * will love this function.
69 static void __init
rcu_bootup_announce_oddness(void)
71 if (IS_ENABLED(CONFIG_RCU_TRACE
))
72 pr_info("\tRCU debugfs-based tracing is enabled.\n");
73 if ((IS_ENABLED(CONFIG_64BIT
) && RCU_FANOUT
!= 64) ||
74 (!IS_ENABLED(CONFIG_64BIT
) && RCU_FANOUT
!= 32))
75 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
78 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
79 if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ
))
80 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
81 if (IS_ENABLED(CONFIG_PROVE_RCU
))
82 pr_info("\tRCU lockdep checking is enabled.\n");
83 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_RUNNABLE
))
84 pr_info("\tRCU torture testing starts during boot.\n");
85 if (RCU_NUM_LVLS
>= 4)
86 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
87 if (RCU_FANOUT_LEAF
!= 16)
88 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
90 if (rcu_fanout_leaf
!= RCU_FANOUT_LEAF
)
91 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf
);
92 if (nr_cpu_ids
!= NR_CPUS
)
93 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS
, nr_cpu_ids
);
94 if (IS_ENABLED(CONFIG_RCU_BOOST
))
95 pr_info("\tRCU kthread priority: %d.\n", kthread_prio
);
98 #ifdef CONFIG_PREEMPT_RCU
100 RCU_STATE_INITIALIZER(rcu_preempt
, 'p', call_rcu
);
101 static struct rcu_state
*const rcu_state_p
= &rcu_preempt_state
;
102 static struct rcu_data __percpu
*const rcu_data_p
= &rcu_preempt_data
;
104 static void rcu_report_exp_rnp(struct rcu_state
*rsp
, struct rcu_node
*rnp
,
108 * Tell them what RCU they are running.
110 static void __init
rcu_bootup_announce(void)
112 pr_info("Preemptible hierarchical RCU implementation.\n");
113 rcu_bootup_announce_oddness();
116 /* Flags for rcu_preempt_ctxt_queue() decision table. */
117 #define RCU_GP_TASKS 0x8
118 #define RCU_EXP_TASKS 0x4
119 #define RCU_GP_BLKD 0x2
120 #define RCU_EXP_BLKD 0x1
123 * Queues a task preempted within an RCU-preempt read-side critical
124 * section into the appropriate location within the ->blkd_tasks list,
125 * depending on the states of any ongoing normal and expedited grace
126 * periods. The ->gp_tasks pointer indicates which element the normal
127 * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
128 * indicates which element the expedited grace period is waiting on (again,
129 * NULL if none). If a grace period is waiting on a given element in the
130 * ->blkd_tasks list, it also waits on all subsequent elements. Thus,
131 * adding a task to the tail of the list blocks any grace period that is
132 * already waiting on one of the elements. In contrast, adding a task
133 * to the head of the list won't block any grace period that is already
134 * waiting on one of the elements.
136 * This queuing is imprecise, and can sometimes make an ongoing grace
137 * period wait for a task that is not strictly speaking blocking it.
138 * Given the choice, we needlessly block a normal grace period rather than
139 * blocking an expedited grace period.
141 * Note that an endless sequence of expedited grace periods still cannot
142 * indefinitely postpone a normal grace period. Eventually, all of the
143 * fixed number of preempted tasks blocking the normal grace period that are
144 * not also blocking the expedited grace period will resume and complete
145 * their RCU read-side critical sections. At that point, the ->gp_tasks
146 * pointer will equal the ->exp_tasks pointer, at which point the end of
147 * the corresponding expedited grace period will also be the end of the
148 * normal grace period.
150 static void rcu_preempt_ctxt_queue(struct rcu_node
*rnp
, struct rcu_data
*rdp
,
151 unsigned long flags
) __releases(rnp
->lock
)
153 int blkd_state
= (rnp
->gp_tasks
? RCU_GP_TASKS
: 0) +
154 (rnp
->exp_tasks
? RCU_EXP_TASKS
: 0) +
155 (rnp
->qsmask
& rdp
->grpmask
? RCU_GP_BLKD
: 0) +
156 (rnp
->expmask
& rdp
->grpmask
? RCU_EXP_BLKD
: 0);
157 struct task_struct
*t
= current
;
160 * Decide where to queue the newly blocked task. In theory,
161 * this could be an if-statement. In practice, when I tried
162 * that, it was quite messy.
164 switch (blkd_state
) {
167 case RCU_EXP_TASKS
+ RCU_GP_BLKD
:
169 case RCU_GP_TASKS
+ RCU_EXP_TASKS
:
172 * Blocking neither GP, or first task blocking the normal
173 * GP but not blocking the already-waiting expedited GP.
174 * Queue at the head of the list to avoid unnecessarily
175 * blocking the already-waiting GPs.
177 list_add(&t
->rcu_node_entry
, &rnp
->blkd_tasks
);
182 case RCU_GP_BLKD
+ RCU_EXP_BLKD
:
183 case RCU_GP_TASKS
+ RCU_EXP_BLKD
:
184 case RCU_GP_TASKS
+ RCU_GP_BLKD
+ RCU_EXP_BLKD
:
185 case RCU_GP_TASKS
+ RCU_EXP_TASKS
+ RCU_GP_BLKD
+ RCU_EXP_BLKD
:
188 * First task arriving that blocks either GP, or first task
189 * arriving that blocks the expedited GP (with the normal
190 * GP already waiting), or a task arriving that blocks
191 * both GPs with both GPs already waiting. Queue at the
192 * tail of the list to avoid any GP waiting on any of the
193 * already queued tasks that are not blocking it.
195 list_add_tail(&t
->rcu_node_entry
, &rnp
->blkd_tasks
);
198 case RCU_EXP_TASKS
+ RCU_EXP_BLKD
:
199 case RCU_EXP_TASKS
+ RCU_GP_BLKD
+ RCU_EXP_BLKD
:
200 case RCU_GP_TASKS
+ RCU_EXP_TASKS
+ RCU_EXP_BLKD
:
203 * Second or subsequent task blocking the expedited GP.
204 * The task either does not block the normal GP, or is the
205 * first task blocking the normal GP. Queue just after
206 * the first task blocking the expedited GP.
208 list_add(&t
->rcu_node_entry
, rnp
->exp_tasks
);
211 case RCU_GP_TASKS
+ RCU_GP_BLKD
:
212 case RCU_GP_TASKS
+ RCU_EXP_TASKS
+ RCU_GP_BLKD
:
215 * Second or subsequent task blocking the normal GP.
216 * The task does not block the expedited GP. Queue just
217 * after the first task blocking the normal GP.
219 list_add(&t
->rcu_node_entry
, rnp
->gp_tasks
);
224 /* Yet another exercise in excessive paranoia. */
230 * We have now queued the task. If it was the first one to
231 * block either grace period, update the ->gp_tasks and/or
232 * ->exp_tasks pointers, respectively, to reference the newly
235 if (!rnp
->gp_tasks
&& (blkd_state
& RCU_GP_BLKD
))
236 rnp
->gp_tasks
= &t
->rcu_node_entry
;
237 if (!rnp
->exp_tasks
&& (blkd_state
& RCU_EXP_BLKD
))
238 rnp
->exp_tasks
= &t
->rcu_node_entry
;
239 raw_spin_unlock(&rnp
->lock
);
242 * Report the quiescent state for the expedited GP. This expedited
243 * GP should not be able to end until we report, so there should be
244 * no need to check for a subsequent expedited GP. (Though we are
245 * still in a quiescent state in any case.)
247 if (blkd_state
& RCU_EXP_BLKD
&&
248 t
->rcu_read_unlock_special
.b
.exp_need_qs
) {
249 t
->rcu_read_unlock_special
.b
.exp_need_qs
= false;
250 rcu_report_exp_rdp(rdp
->rsp
, rdp
, true);
252 WARN_ON_ONCE(t
->rcu_read_unlock_special
.b
.exp_need_qs
);
254 local_irq_restore(flags
);
258 * Record a preemptible-RCU quiescent state for the specified CPU. Note
259 * that this just means that the task currently running on the CPU is
260 * not in a quiescent state. There might be any number of tasks blocked
261 * while in an RCU read-side critical section.
263 * As with the other rcu_*_qs() functions, callers to this function
264 * must disable preemption.
266 static void rcu_preempt_qs(void)
268 if (!__this_cpu_read(rcu_data_p
->passed_quiesce
)) {
269 trace_rcu_grace_period(TPS("rcu_preempt"),
270 __this_cpu_read(rcu_data_p
->gpnum
),
272 __this_cpu_write(rcu_data_p
->passed_quiesce
, 1);
273 barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
274 current
->rcu_read_unlock_special
.b
.need_qs
= false;
279 * We have entered the scheduler, and the current task might soon be
280 * context-switched away from. If this task is in an RCU read-side
281 * critical section, we will no longer be able to rely on the CPU to
282 * record that fact, so we enqueue the task on the blkd_tasks list.
283 * The task will dequeue itself when it exits the outermost enclosing
284 * RCU read-side critical section. Therefore, the current grace period
285 * cannot be permitted to complete until the blkd_tasks list entries
286 * predating the current grace period drain, in other words, until
287 * rnp->gp_tasks becomes NULL.
289 * Caller must disable preemption.
291 static void rcu_preempt_note_context_switch(void)
293 struct task_struct
*t
= current
;
295 struct rcu_data
*rdp
;
296 struct rcu_node
*rnp
;
298 if (t
->rcu_read_lock_nesting
> 0 &&
299 !t
->rcu_read_unlock_special
.b
.blocked
) {
301 /* Possibly blocking in an RCU read-side critical section. */
302 rdp
= this_cpu_ptr(rcu_state_p
->rda
);
304 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
305 smp_mb__after_unlock_lock();
306 t
->rcu_read_unlock_special
.b
.blocked
= true;
307 t
->rcu_blocked_node
= rnp
;
310 * Verify the CPU's sanity, trace the preemption, and
311 * then queue the task as required based on the states
312 * of any ongoing and expedited grace periods.
314 WARN_ON_ONCE((rdp
->grpmask
& rcu_rnp_online_cpus(rnp
)) == 0);
315 WARN_ON_ONCE(!list_empty(&t
->rcu_node_entry
));
316 trace_rcu_preempt_task(rdp
->rsp
->name
,
318 (rnp
->qsmask
& rdp
->grpmask
)
321 rcu_preempt_ctxt_queue(rnp
, rdp
, flags
);
322 } else if (t
->rcu_read_lock_nesting
< 0 &&
323 t
->rcu_read_unlock_special
.s
) {
326 * Complete exit from RCU read-side critical section on
327 * behalf of preempted instance of __rcu_read_unlock().
329 rcu_read_unlock_special(t
);
333 * Either we were not in an RCU read-side critical section to
334 * begin with, or we have now recorded that critical section
335 * globally. Either way, we can now note a quiescent state
336 * for this CPU. Again, if we were in an RCU read-side critical
337 * section, and if that critical section was blocking the current
338 * grace period, then the fact that the task has been enqueued
339 * means that we continue to block the current grace period.
345 * Check for preempted RCU readers blocking the current grace period
346 * for the specified rcu_node structure. If the caller needs a reliable
347 * answer, it must hold the rcu_node's ->lock.
349 static int rcu_preempt_blocked_readers_cgp(struct rcu_node
*rnp
)
351 return rnp
->gp_tasks
!= NULL
;
355 * Advance a ->blkd_tasks-list pointer to the next entry, instead
356 * returning NULL if at the end of the list.
358 static struct list_head
*rcu_next_node_entry(struct task_struct
*t
,
359 struct rcu_node
*rnp
)
361 struct list_head
*np
;
363 np
= t
->rcu_node_entry
.next
;
364 if (np
== &rnp
->blkd_tasks
)
370 * Return true if the specified rcu_node structure has tasks that were
371 * preempted within an RCU read-side critical section.
373 static bool rcu_preempt_has_tasks(struct rcu_node
*rnp
)
375 return !list_empty(&rnp
->blkd_tasks
);
379 * Handle special cases during rcu_read_unlock(), such as needing to
380 * notify RCU core processing or task having blocked during the RCU
381 * read-side critical section.
383 void rcu_read_unlock_special(struct task_struct
*t
)
389 struct list_head
*np
;
390 bool drop_boost_mutex
= false;
391 struct rcu_data
*rdp
;
392 struct rcu_node
*rnp
;
393 union rcu_special special
;
395 /* NMI handlers cannot block and cannot safely manipulate state. */
399 local_irq_save(flags
);
402 * If RCU core is waiting for this CPU to exit its critical section,
403 * report the fact that it has exited. Because irqs are disabled,
404 * t->rcu_read_unlock_special cannot change.
406 special
= t
->rcu_read_unlock_special
;
407 if (special
.b
.need_qs
) {
409 t
->rcu_read_unlock_special
.b
.need_qs
= false;
410 if (!t
->rcu_read_unlock_special
.s
) {
411 local_irq_restore(flags
);
417 * Respond to a request for an expedited grace period, but only if
418 * we were not preempted, meaning that we were running on the same
419 * CPU throughout. If we were preempted, the exp_need_qs flag
420 * would have been cleared at the time of the first preemption,
421 * and the quiescent state would be reported when we were dequeued.
423 if (special
.b
.exp_need_qs
) {
424 WARN_ON_ONCE(special
.b
.blocked
);
425 t
->rcu_read_unlock_special
.b
.exp_need_qs
= false;
426 rdp
= this_cpu_ptr(rcu_state_p
->rda
);
427 rcu_report_exp_rdp(rcu_state_p
, rdp
, true);
428 if (!t
->rcu_read_unlock_special
.s
) {
429 local_irq_restore(flags
);
434 /* Hardware IRQ handlers cannot block, complain if they get here. */
435 if (in_irq() || in_serving_softirq()) {
436 lockdep_rcu_suspicious(__FILE__
, __LINE__
,
437 "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n");
438 pr_alert("->rcu_read_unlock_special: %#x (b: %d, enq: %d nq: %d)\n",
439 t
->rcu_read_unlock_special
.s
,
440 t
->rcu_read_unlock_special
.b
.blocked
,
441 t
->rcu_read_unlock_special
.b
.exp_need_qs
,
442 t
->rcu_read_unlock_special
.b
.need_qs
);
443 local_irq_restore(flags
);
447 /* Clean up if blocked during RCU read-side critical section. */
448 if (special
.b
.blocked
) {
449 t
->rcu_read_unlock_special
.b
.blocked
= false;
452 * Remove this task from the list it blocked on. The task
453 * now remains queued on the rcu_node corresponding to
454 * the CPU it first blocked on, so the first attempt to
455 * acquire the task's rcu_node's ->lock will succeed.
456 * Keep the loop and add a WARN_ON() out of sheer paranoia.
459 rnp
= t
->rcu_blocked_node
;
460 raw_spin_lock(&rnp
->lock
); /* irqs already disabled. */
461 smp_mb__after_unlock_lock();
462 if (rnp
== t
->rcu_blocked_node
)
465 raw_spin_unlock(&rnp
->lock
); /* irqs remain disabled. */
467 empty_norm
= !rcu_preempt_blocked_readers_cgp(rnp
);
468 empty_exp
= sync_rcu_preempt_exp_done(rnp
);
469 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
470 np
= rcu_next_node_entry(t
, rnp
);
471 list_del_init(&t
->rcu_node_entry
);
472 t
->rcu_blocked_node
= NULL
;
473 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
475 if (&t
->rcu_node_entry
== rnp
->gp_tasks
)
477 if (&t
->rcu_node_entry
== rnp
->exp_tasks
)
479 if (IS_ENABLED(CONFIG_RCU_BOOST
)) {
480 if (&t
->rcu_node_entry
== rnp
->boost_tasks
)
481 rnp
->boost_tasks
= np
;
482 /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
483 drop_boost_mutex
= rt_mutex_owner(&rnp
->boost_mtx
) == t
;
487 * If this was the last task on the current list, and if
488 * we aren't waiting on any CPUs, report the quiescent state.
489 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
490 * so we must take a snapshot of the expedited state.
492 empty_exp_now
= sync_rcu_preempt_exp_done(rnp
);
493 if (!empty_norm
&& !rcu_preempt_blocked_readers_cgp(rnp
)) {
494 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
501 rcu_report_unblock_qs_rnp(rcu_state_p
, rnp
, flags
);
503 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
506 /* Unboost if we were boosted. */
507 if (IS_ENABLED(CONFIG_RCU_BOOST
) && drop_boost_mutex
)
508 rt_mutex_unlock(&rnp
->boost_mtx
);
511 * If this was the last task on the expedited lists,
512 * then we need to report up the rcu_node hierarchy.
514 if (!empty_exp
&& empty_exp_now
)
515 rcu_report_exp_rnp(rcu_state_p
, rnp
, true);
517 local_irq_restore(flags
);
522 * Dump detailed information for all tasks blocking the current RCU
523 * grace period on the specified rcu_node structure.
525 static void rcu_print_detail_task_stall_rnp(struct rcu_node
*rnp
)
528 struct task_struct
*t
;
530 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
531 if (!rcu_preempt_blocked_readers_cgp(rnp
)) {
532 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
535 t
= list_entry(rnp
->gp_tasks
->prev
,
536 struct task_struct
, rcu_node_entry
);
537 list_for_each_entry_continue(t
, &rnp
->blkd_tasks
, rcu_node_entry
)
539 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
543 * Dump detailed information for all tasks blocking the current RCU
546 static void rcu_print_detail_task_stall(struct rcu_state
*rsp
)
548 struct rcu_node
*rnp
= rcu_get_root(rsp
);
550 rcu_print_detail_task_stall_rnp(rnp
);
551 rcu_for_each_leaf_node(rsp
, rnp
)
552 rcu_print_detail_task_stall_rnp(rnp
);
555 static void rcu_print_task_stall_begin(struct rcu_node
*rnp
)
557 pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
558 rnp
->level
, rnp
->grplo
, rnp
->grphi
);
561 static void rcu_print_task_stall_end(void)
567 * Scan the current list of tasks blocked within RCU read-side critical
568 * sections, printing out the tid of each.
570 static int rcu_print_task_stall(struct rcu_node
*rnp
)
572 struct task_struct
*t
;
575 if (!rcu_preempt_blocked_readers_cgp(rnp
))
577 rcu_print_task_stall_begin(rnp
);
578 t
= list_entry(rnp
->gp_tasks
->prev
,
579 struct task_struct
, rcu_node_entry
);
580 list_for_each_entry_continue(t
, &rnp
->blkd_tasks
, rcu_node_entry
) {
581 pr_cont(" P%d", t
->pid
);
584 rcu_print_task_stall_end();
589 * Check that the list of blocked tasks for the newly completed grace
590 * period is in fact empty. It is a serious bug to complete a grace
591 * period that still has RCU readers blocked! This function must be
592 * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
593 * must be held by the caller.
595 * Also, if there are blocked tasks on the list, they automatically
596 * block the newly created grace period, so set up ->gp_tasks accordingly.
598 static void rcu_preempt_check_blocked_tasks(struct rcu_node
*rnp
)
600 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp
));
601 if (rcu_preempt_has_tasks(rnp
))
602 rnp
->gp_tasks
= rnp
->blkd_tasks
.next
;
603 WARN_ON_ONCE(rnp
->qsmask
);
607 * Check for a quiescent state from the current CPU. When a task blocks,
608 * the task is recorded in the corresponding CPU's rcu_node structure,
609 * which is checked elsewhere.
611 * Caller must disable hard irqs.
613 static void rcu_preempt_check_callbacks(void)
615 struct task_struct
*t
= current
;
617 if (t
->rcu_read_lock_nesting
== 0) {
621 if (t
->rcu_read_lock_nesting
> 0 &&
622 __this_cpu_read(rcu_data_p
->qs_pending
) &&
623 !__this_cpu_read(rcu_data_p
->passed_quiesce
))
624 t
->rcu_read_unlock_special
.b
.need_qs
= true;
627 #ifdef CONFIG_RCU_BOOST
629 static void rcu_preempt_do_callbacks(void)
631 rcu_do_batch(rcu_state_p
, this_cpu_ptr(rcu_data_p
));
634 #endif /* #ifdef CONFIG_RCU_BOOST */
637 * Queue a preemptible-RCU callback for invocation after a grace period.
639 void call_rcu(struct rcu_head
*head
, void (*func
)(struct rcu_head
*rcu
))
641 __call_rcu(head
, func
, rcu_state_p
, -1, 0);
643 EXPORT_SYMBOL_GPL(call_rcu
);
646 * synchronize_rcu - wait until a grace period has elapsed.
648 * Control will return to the caller some time after a full grace
649 * period has elapsed, in other words after all currently executing RCU
650 * read-side critical sections have completed. Note, however, that
651 * upon return from synchronize_rcu(), the caller might well be executing
652 * concurrently with new RCU read-side critical sections that began while
653 * synchronize_rcu() was waiting. RCU read-side critical sections are
654 * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
656 * See the description of synchronize_sched() for more detailed information
657 * on memory ordering guarantees.
659 void synchronize_rcu(void)
661 RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map
) ||
662 lock_is_held(&rcu_lock_map
) ||
663 lock_is_held(&rcu_sched_lock_map
),
664 "Illegal synchronize_rcu() in RCU read-side critical section");
665 if (!rcu_scheduler_active
)
667 if (rcu_gp_is_expedited())
668 synchronize_rcu_expedited();
670 wait_rcu_gp(call_rcu
);
672 EXPORT_SYMBOL_GPL(synchronize_rcu
);
675 * Remote handler for smp_call_function_single(). If there is an
676 * RCU read-side critical section in effect, request that the
677 * next rcu_read_unlock() record the quiescent state up the
678 * ->expmask fields in the rcu_node tree. Otherwise, immediately
679 * report the quiescent state.
681 static void sync_rcu_exp_handler(void *info
)
683 struct rcu_data
*rdp
;
684 struct rcu_state
*rsp
= info
;
685 struct task_struct
*t
= current
;
688 * Within an RCU read-side critical section, request that the next
689 * rcu_read_unlock() report. Unless this RCU read-side critical
690 * section has already blocked, in which case it is already set
691 * up for the expedited grace period to wait on it.
693 if (t
->rcu_read_lock_nesting
> 0 &&
694 !t
->rcu_read_unlock_special
.b
.blocked
) {
695 t
->rcu_read_unlock_special
.b
.exp_need_qs
= true;
700 * We are either exiting an RCU read-side critical section (negative
701 * values of t->rcu_read_lock_nesting) or are not in one at all
702 * (zero value of t->rcu_read_lock_nesting). Or we are in an RCU
703 * read-side critical section that blocked before this expedited
704 * grace period started. Either way, we can immediately report
705 * the quiescent state.
707 rdp
= this_cpu_ptr(rsp
->rda
);
708 rcu_report_exp_rdp(rsp
, rdp
, true);
712 * Select the nodes that the upcoming expedited grace period needs
715 static void sync_rcu_exp_select_cpus(struct rcu_state
*rsp
)
720 unsigned long mask_ofl_test
;
721 unsigned long mask_ofl_ipi
;
723 struct rcu_node
*rnp
;
725 sync_exp_reset_tree(rsp
);
726 rcu_for_each_leaf_node(rsp
, rnp
) {
727 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
728 smp_mb__after_unlock_lock();
730 /* Each pass checks a CPU for identity, offline, and idle. */
732 for (cpu
= rnp
->grplo
; cpu
<= rnp
->grphi
; cpu
++) {
733 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
734 struct rcu_dynticks
*rdtp
= &per_cpu(rcu_dynticks
, cpu
);
736 if (raw_smp_processor_id() == cpu
||
737 cpu_is_offline(cpu
) ||
738 !(atomic_add_return(0, &rdtp
->dynticks
) & 0x1))
739 mask_ofl_test
|= rdp
->grpmask
;
741 mask_ofl_ipi
= rnp
->expmask
& ~mask_ofl_test
;
744 * Need to wait for any blocked tasks as well. Note that
745 * additional blocking tasks will also block the expedited
746 * GP until such time as the ->expmask bits are cleared.
748 if (rcu_preempt_has_tasks(rnp
))
749 rnp
->exp_tasks
= rnp
->blkd_tasks
.next
;
750 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
752 /* IPI the remaining CPUs for expedited quiescent state. */
754 for (cpu
= rnp
->grplo
; cpu
<= rnp
->grphi
; cpu
++, mask
<<= 1) {
755 if (!(mask_ofl_ipi
& mask
))
757 ret
= smp_call_function_single(cpu
,
758 sync_rcu_exp_handler
,
761 mask_ofl_ipi
&= ~mask
;
763 /* Report quiescent states for those that went offline. */
764 mask_ofl_test
|= mask_ofl_ipi
;
766 rcu_report_exp_cpu_mult(rsp
, rnp
, mask_ofl_test
, false);
771 * synchronize_rcu_expedited - Brute-force RCU grace period
773 * Wait for an RCU-preempt grace period, but expedite it. The basic
774 * idea is to invoke synchronize_sched_expedited() to push all the tasks to
775 * the ->blkd_tasks lists and wait for this list to drain. This consumes
776 * significant time on all CPUs and is unfriendly to real-time workloads,
777 * so is thus not recommended for any sort of common-case code.
778 * In fact, if you are using synchronize_rcu_expedited() in a loop,
779 * please restructure your code to batch your updates, and then Use a
780 * single synchronize_rcu() instead.
782 void synchronize_rcu_expedited(void)
784 struct rcu_node
*rnp
;
785 struct rcu_node
*rnp_unlock
;
786 struct rcu_state
*rsp
= rcu_state_p
;
789 s
= rcu_exp_gp_seq_snap(rsp
);
791 rnp_unlock
= exp_funnel_lock(rsp
, s
);
792 if (rnp_unlock
== NULL
)
793 return; /* Someone else did our work for us. */
795 rcu_exp_gp_seq_start(rsp
);
797 /* Initialize the rcu_node tree in preparation for the wait. */
798 sync_rcu_exp_select_cpus(rsp
);
800 /* Wait for snapshotted ->blkd_tasks lists to drain. */
801 rnp
= rcu_get_root(rsp
);
802 wait_event(rsp
->expedited_wq
,
803 sync_rcu_preempt_exp_done(rnp
));
805 /* Clean up and exit. */
806 rcu_exp_gp_seq_end(rsp
);
807 mutex_unlock(&rnp_unlock
->exp_funnel_mutex
);
809 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited
);
812 * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
814 * Note that this primitive does not necessarily wait for an RCU grace period
815 * to complete. For example, if there are no RCU callbacks queued anywhere
816 * in the system, then rcu_barrier() is within its rights to return
817 * immediately, without waiting for anything, much less an RCU grace period.
819 void rcu_barrier(void)
821 _rcu_barrier(rcu_state_p
);
823 EXPORT_SYMBOL_GPL(rcu_barrier
);
826 * Initialize preemptible RCU's state structures.
828 static void __init
__rcu_init_preempt(void)
830 rcu_init_one(rcu_state_p
, rcu_data_p
);
834 * Check for a task exiting while in a preemptible-RCU read-side
835 * critical section, clean up if so. No need to issue warnings,
836 * as debug_check_no_locks_held() already does this if lockdep
841 struct task_struct
*t
= current
;
843 if (likely(list_empty(¤t
->rcu_node_entry
)))
845 t
->rcu_read_lock_nesting
= 1;
847 t
->rcu_read_unlock_special
.b
.blocked
= true;
851 #else /* #ifdef CONFIG_PREEMPT_RCU */
853 static struct rcu_state
*const rcu_state_p
= &rcu_sched_state
;
854 static struct rcu_data __percpu
*const rcu_data_p
= &rcu_sched_data
;
857 * Tell them what RCU they are running.
859 static void __init
rcu_bootup_announce(void)
861 pr_info("Hierarchical RCU implementation.\n");
862 rcu_bootup_announce_oddness();
866 * Because preemptible RCU does not exist, we never have to check for
867 * CPUs being in quiescent states.
869 static void rcu_preempt_note_context_switch(void)
874 * Because preemptible RCU does not exist, there are never any preempted
877 static int rcu_preempt_blocked_readers_cgp(struct rcu_node
*rnp
)
883 * Because there is no preemptible RCU, there can be no readers blocked.
885 static bool rcu_preempt_has_tasks(struct rcu_node
*rnp
)
891 * Because preemptible RCU does not exist, we never have to check for
892 * tasks blocked within RCU read-side critical sections.
894 static void rcu_print_detail_task_stall(struct rcu_state
*rsp
)
899 * Because preemptible RCU does not exist, we never have to check for
900 * tasks blocked within RCU read-side critical sections.
902 static int rcu_print_task_stall(struct rcu_node
*rnp
)
908 * Because there is no preemptible RCU, there can be no readers blocked,
909 * so there is no need to check for blocked tasks. So check only for
910 * bogus qsmask values.
912 static void rcu_preempt_check_blocked_tasks(struct rcu_node
*rnp
)
914 WARN_ON_ONCE(rnp
->qsmask
);
918 * Because preemptible RCU does not exist, it never has any callbacks
921 static void rcu_preempt_check_callbacks(void)
926 * Wait for an rcu-preempt grace period, but make it happen quickly.
927 * But because preemptible RCU does not exist, map to rcu-sched.
929 void synchronize_rcu_expedited(void)
931 synchronize_sched_expedited();
933 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited
);
936 * Because preemptible RCU does not exist, rcu_barrier() is just
937 * another name for rcu_barrier_sched().
939 void rcu_barrier(void)
943 EXPORT_SYMBOL_GPL(rcu_barrier
);
946 * Because preemptible RCU does not exist, it need not be initialized.
948 static void __init
__rcu_init_preempt(void)
953 * Because preemptible RCU does not exist, tasks cannot possibly exit
954 * while in preemptible RCU read-side critical sections.
960 #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
962 #ifdef CONFIG_RCU_BOOST
964 #include "../locking/rtmutex_common.h"
966 #ifdef CONFIG_RCU_TRACE
968 static void rcu_initiate_boost_trace(struct rcu_node
*rnp
)
970 if (!rcu_preempt_has_tasks(rnp
))
971 rnp
->n_balk_blkd_tasks
++;
972 else if (rnp
->exp_tasks
== NULL
&& rnp
->gp_tasks
== NULL
)
973 rnp
->n_balk_exp_gp_tasks
++;
974 else if (rnp
->gp_tasks
!= NULL
&& rnp
->boost_tasks
!= NULL
)
975 rnp
->n_balk_boost_tasks
++;
976 else if (rnp
->gp_tasks
!= NULL
&& rnp
->qsmask
!= 0)
977 rnp
->n_balk_notblocked
++;
978 else if (rnp
->gp_tasks
!= NULL
&&
979 ULONG_CMP_LT(jiffies
, rnp
->boost_time
))
980 rnp
->n_balk_notyet
++;
985 #else /* #ifdef CONFIG_RCU_TRACE */
987 static void rcu_initiate_boost_trace(struct rcu_node
*rnp
)
991 #endif /* #else #ifdef CONFIG_RCU_TRACE */
993 static void rcu_wake_cond(struct task_struct
*t
, int status
)
996 * If the thread is yielding, only wake it when this
997 * is invoked from idle
999 if (status
!= RCU_KTHREAD_YIELDING
|| is_idle_task(current
))
1004 * Carry out RCU priority boosting on the task indicated by ->exp_tasks
1005 * or ->boost_tasks, advancing the pointer to the next task in the
1006 * ->blkd_tasks list.
1008 * Note that irqs must be enabled: boosting the task can block.
1009 * Returns 1 if there are more tasks needing to be boosted.
1011 static int rcu_boost(struct rcu_node
*rnp
)
1013 unsigned long flags
;
1014 struct task_struct
*t
;
1015 struct list_head
*tb
;
1017 if (READ_ONCE(rnp
->exp_tasks
) == NULL
&&
1018 READ_ONCE(rnp
->boost_tasks
) == NULL
)
1019 return 0; /* Nothing left to boost. */
1021 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
1022 smp_mb__after_unlock_lock();
1025 * Recheck under the lock: all tasks in need of boosting
1026 * might exit their RCU read-side critical sections on their own.
1028 if (rnp
->exp_tasks
== NULL
&& rnp
->boost_tasks
== NULL
) {
1029 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1034 * Preferentially boost tasks blocking expedited grace periods.
1035 * This cannot starve the normal grace periods because a second
1036 * expedited grace period must boost all blocked tasks, including
1037 * those blocking the pre-existing normal grace period.
1039 if (rnp
->exp_tasks
!= NULL
) {
1040 tb
= rnp
->exp_tasks
;
1041 rnp
->n_exp_boosts
++;
1043 tb
= rnp
->boost_tasks
;
1044 rnp
->n_normal_boosts
++;
1046 rnp
->n_tasks_boosted
++;
1049 * We boost task t by manufacturing an rt_mutex that appears to
1050 * be held by task t. We leave a pointer to that rt_mutex where
1051 * task t can find it, and task t will release the mutex when it
1052 * exits its outermost RCU read-side critical section. Then
1053 * simply acquiring this artificial rt_mutex will boost task
1054 * t's priority. (Thanks to tglx for suggesting this approach!)
1056 * Note that task t must acquire rnp->lock to remove itself from
1057 * the ->blkd_tasks list, which it will do from exit() if from
1058 * nowhere else. We therefore are guaranteed that task t will
1059 * stay around at least until we drop rnp->lock. Note that
1060 * rnp->lock also resolves races between our priority boosting
1061 * and task t's exiting its outermost RCU read-side critical
1064 t
= container_of(tb
, struct task_struct
, rcu_node_entry
);
1065 rt_mutex_init_proxy_locked(&rnp
->boost_mtx
, t
);
1066 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1067 /* Lock only for side effect: boosts task t's priority. */
1068 rt_mutex_lock(&rnp
->boost_mtx
);
1069 rt_mutex_unlock(&rnp
->boost_mtx
); /* Then keep lockdep happy. */
1071 return READ_ONCE(rnp
->exp_tasks
) != NULL
||
1072 READ_ONCE(rnp
->boost_tasks
) != NULL
;
1076 * Priority-boosting kthread, one per leaf rcu_node.
1078 static int rcu_boost_kthread(void *arg
)
1080 struct rcu_node
*rnp
= (struct rcu_node
*)arg
;
1084 trace_rcu_utilization(TPS("Start boost kthread@init"));
1086 rnp
->boost_kthread_status
= RCU_KTHREAD_WAITING
;
1087 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1088 rcu_wait(rnp
->boost_tasks
|| rnp
->exp_tasks
);
1089 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1090 rnp
->boost_kthread_status
= RCU_KTHREAD_RUNNING
;
1091 more2boost
= rcu_boost(rnp
);
1097 rnp
->boost_kthread_status
= RCU_KTHREAD_YIELDING
;
1098 trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1099 schedule_timeout_interruptible(2);
1100 trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
1105 trace_rcu_utilization(TPS("End boost kthread@notreached"));
1110 * Check to see if it is time to start boosting RCU readers that are
1111 * blocking the current grace period, and, if so, tell the per-rcu_node
1112 * kthread to start boosting them. If there is an expedited grace
1113 * period in progress, it is always time to boost.
1115 * The caller must hold rnp->lock, which this function releases.
1116 * The ->boost_kthread_task is immortal, so we don't need to worry
1117 * about it going away.
1119 static void rcu_initiate_boost(struct rcu_node
*rnp
, unsigned long flags
)
1120 __releases(rnp
->lock
)
1122 struct task_struct
*t
;
1124 if (!rcu_preempt_blocked_readers_cgp(rnp
) && rnp
->exp_tasks
== NULL
) {
1125 rnp
->n_balk_exp_gp_tasks
++;
1126 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1129 if (rnp
->exp_tasks
!= NULL
||
1130 (rnp
->gp_tasks
!= NULL
&&
1131 rnp
->boost_tasks
== NULL
&&
1133 ULONG_CMP_GE(jiffies
, rnp
->boost_time
))) {
1134 if (rnp
->exp_tasks
== NULL
)
1135 rnp
->boost_tasks
= rnp
->gp_tasks
;
1136 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1137 t
= rnp
->boost_kthread_task
;
1139 rcu_wake_cond(t
, rnp
->boost_kthread_status
);
1141 rcu_initiate_boost_trace(rnp
);
1142 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1147 * Wake up the per-CPU kthread to invoke RCU callbacks.
1149 static void invoke_rcu_callbacks_kthread(void)
1151 unsigned long flags
;
1153 local_irq_save(flags
);
1154 __this_cpu_write(rcu_cpu_has_work
, 1);
1155 if (__this_cpu_read(rcu_cpu_kthread_task
) != NULL
&&
1156 current
!= __this_cpu_read(rcu_cpu_kthread_task
)) {
1157 rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task
),
1158 __this_cpu_read(rcu_cpu_kthread_status
));
1160 local_irq_restore(flags
);
1164 * Is the current CPU running the RCU-callbacks kthread?
1165 * Caller must have preemption disabled.
1167 static bool rcu_is_callbacks_kthread(void)
1169 return __this_cpu_read(rcu_cpu_kthread_task
) == current
;
1172 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1175 * Do priority-boost accounting for the start of a new grace period.
1177 static void rcu_preempt_boost_start_gp(struct rcu_node
*rnp
)
1179 rnp
->boost_time
= jiffies
+ RCU_BOOST_DELAY_JIFFIES
;
1183 * Create an RCU-boost kthread for the specified node if one does not
1184 * already exist. We only create this kthread for preemptible RCU.
1185 * Returns zero if all is well, a negated errno otherwise.
1187 static int rcu_spawn_one_boost_kthread(struct rcu_state
*rsp
,
1188 struct rcu_node
*rnp
)
1190 int rnp_index
= rnp
- &rsp
->node
[0];
1191 unsigned long flags
;
1192 struct sched_param sp
;
1193 struct task_struct
*t
;
1195 if (rcu_state_p
!= rsp
)
1198 if (!rcu_scheduler_fully_active
|| rcu_rnp_online_cpus(rnp
) == 0)
1202 if (rnp
->boost_kthread_task
!= NULL
)
1204 t
= kthread_create(rcu_boost_kthread
, (void *)rnp
,
1205 "rcub/%d", rnp_index
);
1208 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
1209 smp_mb__after_unlock_lock();
1210 rnp
->boost_kthread_task
= t
;
1211 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1212 sp
.sched_priority
= kthread_prio
;
1213 sched_setscheduler_nocheck(t
, SCHED_FIFO
, &sp
);
1214 wake_up_process(t
); /* get to TASK_INTERRUPTIBLE quickly. */
1218 static void rcu_kthread_do_work(void)
1220 rcu_do_batch(&rcu_sched_state
, this_cpu_ptr(&rcu_sched_data
));
1221 rcu_do_batch(&rcu_bh_state
, this_cpu_ptr(&rcu_bh_data
));
1222 rcu_preempt_do_callbacks();
1225 static void rcu_cpu_kthread_setup(unsigned int cpu
)
1227 struct sched_param sp
;
1229 sp
.sched_priority
= kthread_prio
;
1230 sched_setscheduler_nocheck(current
, SCHED_FIFO
, &sp
);
1233 static void rcu_cpu_kthread_park(unsigned int cpu
)
1235 per_cpu(rcu_cpu_kthread_status
, cpu
) = RCU_KTHREAD_OFFCPU
;
1238 static int rcu_cpu_kthread_should_run(unsigned int cpu
)
1240 return __this_cpu_read(rcu_cpu_has_work
);
1244 * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
1245 * RCU softirq used in flavors and configurations of RCU that do not
1246 * support RCU priority boosting.
1248 static void rcu_cpu_kthread(unsigned int cpu
)
1250 unsigned int *statusp
= this_cpu_ptr(&rcu_cpu_kthread_status
);
1251 char work
, *workp
= this_cpu_ptr(&rcu_cpu_has_work
);
1254 for (spincnt
= 0; spincnt
< 10; spincnt
++) {
1255 trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait"));
1257 *statusp
= RCU_KTHREAD_RUNNING
;
1258 this_cpu_inc(rcu_cpu_kthread_loops
);
1259 local_irq_disable();
1264 rcu_kthread_do_work();
1267 trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
1268 *statusp
= RCU_KTHREAD_WAITING
;
1272 *statusp
= RCU_KTHREAD_YIELDING
;
1273 trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
1274 schedule_timeout_interruptible(2);
1275 trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
1276 *statusp
= RCU_KTHREAD_WAITING
;
1280 * Set the per-rcu_node kthread's affinity to cover all CPUs that are
1281 * served by the rcu_node in question. The CPU hotplug lock is still
1282 * held, so the value of rnp->qsmaskinit will be stable.
1284 * We don't include outgoingcpu in the affinity set, use -1 if there is
1285 * no outgoing CPU. If there are no CPUs left in the affinity set,
1286 * this function allows the kthread to execute on any CPU.
1288 static void rcu_boost_kthread_setaffinity(struct rcu_node
*rnp
, int outgoingcpu
)
1290 struct task_struct
*t
= rnp
->boost_kthread_task
;
1291 unsigned long mask
= rcu_rnp_online_cpus(rnp
);
1297 if (!zalloc_cpumask_var(&cm
, GFP_KERNEL
))
1299 for (cpu
= rnp
->grplo
; cpu
<= rnp
->grphi
; cpu
++, mask
>>= 1)
1300 if ((mask
& 0x1) && cpu
!= outgoingcpu
)
1301 cpumask_set_cpu(cpu
, cm
);
1302 if (cpumask_weight(cm
) == 0)
1304 set_cpus_allowed_ptr(t
, cm
);
1305 free_cpumask_var(cm
);
1308 static struct smp_hotplug_thread rcu_cpu_thread_spec
= {
1309 .store
= &rcu_cpu_kthread_task
,
1310 .thread_should_run
= rcu_cpu_kthread_should_run
,
1311 .thread_fn
= rcu_cpu_kthread
,
1312 .thread_comm
= "rcuc/%u",
1313 .setup
= rcu_cpu_kthread_setup
,
1314 .park
= rcu_cpu_kthread_park
,
1318 * Spawn boost kthreads -- called as soon as the scheduler is running.
1320 static void __init
rcu_spawn_boost_kthreads(void)
1322 struct rcu_node
*rnp
;
1325 for_each_possible_cpu(cpu
)
1326 per_cpu(rcu_cpu_has_work
, cpu
) = 0;
1327 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec
));
1328 rcu_for_each_leaf_node(rcu_state_p
, rnp
)
1329 (void)rcu_spawn_one_boost_kthread(rcu_state_p
, rnp
);
1332 static void rcu_prepare_kthreads(int cpu
)
1334 struct rcu_data
*rdp
= per_cpu_ptr(rcu_state_p
->rda
, cpu
);
1335 struct rcu_node
*rnp
= rdp
->mynode
;
1337 /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
1338 if (rcu_scheduler_fully_active
)
1339 (void)rcu_spawn_one_boost_kthread(rcu_state_p
, rnp
);
1342 #else /* #ifdef CONFIG_RCU_BOOST */
1344 static void rcu_initiate_boost(struct rcu_node
*rnp
, unsigned long flags
)
1345 __releases(rnp
->lock
)
1347 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
1350 static void invoke_rcu_callbacks_kthread(void)
1355 static bool rcu_is_callbacks_kthread(void)
1360 static void rcu_preempt_boost_start_gp(struct rcu_node
*rnp
)
1364 static void rcu_boost_kthread_setaffinity(struct rcu_node
*rnp
, int outgoingcpu
)
1368 static void __init
rcu_spawn_boost_kthreads(void)
1372 static void rcu_prepare_kthreads(int cpu
)
1376 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1378 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1381 * Check to see if any future RCU-related work will need to be done
1382 * by the current CPU, even if none need be done immediately, returning
1383 * 1 if so. This function is part of the RCU implementation; it is -not-
1384 * an exported member of the RCU API.
1386 * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
1387 * any flavor of RCU.
1389 int rcu_needs_cpu(u64 basemono
, u64
*nextevt
)
1391 *nextevt
= KTIME_MAX
;
1392 return IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL
)
1393 ? 0 : rcu_cpu_has_callbacks(NULL
);
1397 * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
1400 static void rcu_cleanup_after_idle(void)
1405 * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
1408 static void rcu_prepare_for_idle(void)
1413 * Don't bother keeping a running count of the number of RCU callbacks
1414 * posted because CONFIG_RCU_FAST_NO_HZ=n.
1416 static void rcu_idle_count_callbacks_posted(void)
1420 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1423 * This code is invoked when a CPU goes idle, at which point we want
1424 * to have the CPU do everything required for RCU so that it can enter
1425 * the energy-efficient dyntick-idle mode. This is handled by a
1426 * state machine implemented by rcu_prepare_for_idle() below.
1428 * The following three proprocessor symbols control this state machine:
1430 * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
1431 * to sleep in dyntick-idle mode with RCU callbacks pending. This
1432 * is sized to be roughly one RCU grace period. Those energy-efficiency
1433 * benchmarkers who might otherwise be tempted to set this to a large
1434 * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
1435 * system. And if you are -that- concerned about energy efficiency,
1436 * just power the system down and be done with it!
1437 * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1438 * permitted to sleep in dyntick-idle mode with only lazy RCU
1439 * callbacks pending. Setting this too high can OOM your system.
1441 * The values below work well in practice. If future workloads require
1442 * adjustment, they can be converted into kernel config parameters, though
1443 * making the state machine smarter might be a better option.
1445 #define RCU_IDLE_GP_DELAY 4 /* Roughly one grace period. */
1446 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
1448 static int rcu_idle_gp_delay
= RCU_IDLE_GP_DELAY
;
1449 module_param(rcu_idle_gp_delay
, int, 0644);
1450 static int rcu_idle_lazy_gp_delay
= RCU_IDLE_LAZY_GP_DELAY
;
1451 module_param(rcu_idle_lazy_gp_delay
, int, 0644);
1454 * Try to advance callbacks for all flavors of RCU on the current CPU, but
1455 * only if it has been awhile since the last time we did so. Afterwards,
1456 * if there are any callbacks ready for immediate invocation, return true.
1458 static bool __maybe_unused
rcu_try_advance_all_cbs(void)
1460 bool cbs_ready
= false;
1461 struct rcu_data
*rdp
;
1462 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
1463 struct rcu_node
*rnp
;
1464 struct rcu_state
*rsp
;
1466 /* Exit early if we advanced recently. */
1467 if (jiffies
== rdtp
->last_advance_all
)
1469 rdtp
->last_advance_all
= jiffies
;
1471 for_each_rcu_flavor(rsp
) {
1472 rdp
= this_cpu_ptr(rsp
->rda
);
1476 * Don't bother checking unless a grace period has
1477 * completed since we last checked and there are
1478 * callbacks not yet ready to invoke.
1480 if ((rdp
->completed
!= rnp
->completed
||
1481 unlikely(READ_ONCE(rdp
->gpwrap
))) &&
1482 rdp
->nxttail
[RCU_DONE_TAIL
] != rdp
->nxttail
[RCU_NEXT_TAIL
])
1483 note_gp_changes(rsp
, rdp
);
1485 if (cpu_has_callbacks_ready_to_invoke(rdp
))
1492 * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
1493 * to invoke. If the CPU has callbacks, try to advance them. Tell the
1494 * caller to set the timeout based on whether or not there are non-lazy
1497 * The caller must have disabled interrupts.
1499 int rcu_needs_cpu(u64 basemono
, u64
*nextevt
)
1501 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
1504 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL
)) {
1505 *nextevt
= KTIME_MAX
;
1509 /* Snapshot to detect later posting of non-lazy callback. */
1510 rdtp
->nonlazy_posted_snap
= rdtp
->nonlazy_posted
;
1512 /* If no callbacks, RCU doesn't need the CPU. */
1513 if (!rcu_cpu_has_callbacks(&rdtp
->all_lazy
)) {
1514 *nextevt
= KTIME_MAX
;
1518 /* Attempt to advance callbacks. */
1519 if (rcu_try_advance_all_cbs()) {
1520 /* Some ready to invoke, so initiate later invocation. */
1524 rdtp
->last_accelerate
= jiffies
;
1526 /* Request timer delay depending on laziness, and round. */
1527 if (!rdtp
->all_lazy
) {
1528 dj
= round_up(rcu_idle_gp_delay
+ jiffies
,
1529 rcu_idle_gp_delay
) - jiffies
;
1531 dj
= round_jiffies(rcu_idle_lazy_gp_delay
+ jiffies
) - jiffies
;
1533 *nextevt
= basemono
+ dj
* TICK_NSEC
;
1538 * Prepare a CPU for idle from an RCU perspective. The first major task
1539 * is to sense whether nohz mode has been enabled or disabled via sysfs.
1540 * The second major task is to check to see if a non-lazy callback has
1541 * arrived at a CPU that previously had only lazy callbacks. The third
1542 * major task is to accelerate (that is, assign grace-period numbers to)
1543 * any recently arrived callbacks.
1545 * The caller must have disabled interrupts.
1547 static void rcu_prepare_for_idle(void)
1550 struct rcu_data
*rdp
;
1551 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
1552 struct rcu_node
*rnp
;
1553 struct rcu_state
*rsp
;
1556 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL
))
1559 /* Handle nohz enablement switches conservatively. */
1560 tne
= READ_ONCE(tick_nohz_active
);
1561 if (tne
!= rdtp
->tick_nohz_enabled_snap
) {
1562 if (rcu_cpu_has_callbacks(NULL
))
1563 invoke_rcu_core(); /* force nohz to see update. */
1564 rdtp
->tick_nohz_enabled_snap
= tne
;
1570 /* If this is a no-CBs CPU, no callbacks, just return. */
1571 if (rcu_is_nocb_cpu(smp_processor_id()))
1575 * If a non-lazy callback arrived at a CPU having only lazy
1576 * callbacks, invoke RCU core for the side-effect of recalculating
1577 * idle duration on re-entry to idle.
1579 if (rdtp
->all_lazy
&&
1580 rdtp
->nonlazy_posted
!= rdtp
->nonlazy_posted_snap
) {
1581 rdtp
->all_lazy
= false;
1582 rdtp
->nonlazy_posted_snap
= rdtp
->nonlazy_posted
;
1588 * If we have not yet accelerated this jiffy, accelerate all
1589 * callbacks on this CPU.
1591 if (rdtp
->last_accelerate
== jiffies
)
1593 rdtp
->last_accelerate
= jiffies
;
1594 for_each_rcu_flavor(rsp
) {
1595 rdp
= this_cpu_ptr(rsp
->rda
);
1596 if (!*rdp
->nxttail
[RCU_DONE_TAIL
])
1599 raw_spin_lock(&rnp
->lock
); /* irqs already disabled. */
1600 smp_mb__after_unlock_lock();
1601 needwake
= rcu_accelerate_cbs(rsp
, rnp
, rdp
);
1602 raw_spin_unlock(&rnp
->lock
); /* irqs remain disabled. */
1604 rcu_gp_kthread_wake(rsp
);
1609 * Clean up for exit from idle. Attempt to advance callbacks based on
1610 * any grace periods that elapsed while the CPU was idle, and if any
1611 * callbacks are now ready to invoke, initiate invocation.
1613 static void rcu_cleanup_after_idle(void)
1615 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL
) ||
1616 rcu_is_nocb_cpu(smp_processor_id()))
1618 if (rcu_try_advance_all_cbs())
1623 * Keep a running count of the number of non-lazy callbacks posted
1624 * on this CPU. This running counter (which is never decremented) allows
1625 * rcu_prepare_for_idle() to detect when something out of the idle loop
1626 * posts a callback, even if an equal number of callbacks are invoked.
1627 * Of course, callbacks should only be posted from within a trace event
1628 * designed to be called from idle or from within RCU_NONIDLE().
1630 static void rcu_idle_count_callbacks_posted(void)
1632 __this_cpu_add(rcu_dynticks
.nonlazy_posted
, 1);
1636 * Data for flushing lazy RCU callbacks at OOM time.
1638 static atomic_t oom_callback_count
;
1639 static DECLARE_WAIT_QUEUE_HEAD(oom_callback_wq
);
1642 * RCU OOM callback -- decrement the outstanding count and deliver the
1643 * wake-up if we are the last one.
1645 static void rcu_oom_callback(struct rcu_head
*rhp
)
1647 if (atomic_dec_and_test(&oom_callback_count
))
1648 wake_up(&oom_callback_wq
);
1652 * Post an rcu_oom_notify callback on the current CPU if it has at
1653 * least one lazy callback. This will unnecessarily post callbacks
1654 * to CPUs that already have a non-lazy callback at the end of their
1655 * callback list, but this is an infrequent operation, so accept some
1656 * extra overhead to keep things simple.
1658 static void rcu_oom_notify_cpu(void *unused
)
1660 struct rcu_state
*rsp
;
1661 struct rcu_data
*rdp
;
1663 for_each_rcu_flavor(rsp
) {
1664 rdp
= raw_cpu_ptr(rsp
->rda
);
1665 if (rdp
->qlen_lazy
!= 0) {
1666 atomic_inc(&oom_callback_count
);
1667 rsp
->call(&rdp
->oom_head
, rcu_oom_callback
);
1673 * If low on memory, ensure that each CPU has a non-lazy callback.
1674 * This will wake up CPUs that have only lazy callbacks, in turn
1675 * ensuring that they free up the corresponding memory in a timely manner.
1676 * Because an uncertain amount of memory will be freed in some uncertain
1677 * timeframe, we do not claim to have freed anything.
1679 static int rcu_oom_notify(struct notifier_block
*self
,
1680 unsigned long notused
, void *nfreed
)
1684 /* Wait for callbacks from earlier instance to complete. */
1685 wait_event(oom_callback_wq
, atomic_read(&oom_callback_count
) == 0);
1686 smp_mb(); /* Ensure callback reuse happens after callback invocation. */
1689 * Prevent premature wakeup: ensure that all increments happen
1690 * before there is a chance of the counter reaching zero.
1692 atomic_set(&oom_callback_count
, 1);
1694 for_each_online_cpu(cpu
) {
1695 smp_call_function_single(cpu
, rcu_oom_notify_cpu
, NULL
, 1);
1696 cond_resched_rcu_qs();
1699 /* Unconditionally decrement: no need to wake ourselves up. */
1700 atomic_dec(&oom_callback_count
);
1705 static struct notifier_block rcu_oom_nb
= {
1706 .notifier_call
= rcu_oom_notify
1709 static int __init
rcu_register_oom_notifier(void)
1711 register_oom_notifier(&rcu_oom_nb
);
1714 early_initcall(rcu_register_oom_notifier
);
1716 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1718 #ifdef CONFIG_RCU_FAST_NO_HZ
1720 static void print_cpu_stall_fast_no_hz(char *cp
, int cpu
)
1722 struct rcu_dynticks
*rdtp
= &per_cpu(rcu_dynticks
, cpu
);
1723 unsigned long nlpd
= rdtp
->nonlazy_posted
- rdtp
->nonlazy_posted_snap
;
1725 sprintf(cp
, "last_accelerate: %04lx/%04lx, nonlazy_posted: %ld, %c%c",
1726 rdtp
->last_accelerate
& 0xffff, jiffies
& 0xffff,
1728 rdtp
->all_lazy
? 'L' : '.',
1729 rdtp
->tick_nohz_enabled_snap
? '.' : 'D');
1732 #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
1734 static void print_cpu_stall_fast_no_hz(char *cp
, int cpu
)
1739 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
1741 /* Initiate the stall-info list. */
1742 static void print_cpu_stall_info_begin(void)
1748 * Print out diagnostic information for the specified stalled CPU.
1750 * If the specified CPU is aware of the current RCU grace period
1751 * (flavor specified by rsp), then print the number of scheduling
1752 * clock interrupts the CPU has taken during the time that it has
1753 * been aware. Otherwise, print the number of RCU grace periods
1754 * that this CPU is ignorant of, for example, "1" if the CPU was
1755 * aware of the previous grace period.
1757 * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
1759 static void print_cpu_stall_info(struct rcu_state
*rsp
, int cpu
)
1761 char fast_no_hz
[72];
1762 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
1763 struct rcu_dynticks
*rdtp
= rdp
->dynticks
;
1765 unsigned long ticks_value
;
1767 if (rsp
->gpnum
== rdp
->gpnum
) {
1768 ticks_title
= "ticks this GP";
1769 ticks_value
= rdp
->ticks_this_gp
;
1771 ticks_title
= "GPs behind";
1772 ticks_value
= rsp
->gpnum
- rdp
->gpnum
;
1774 print_cpu_stall_fast_no_hz(fast_no_hz
, cpu
);
1775 pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1776 cpu
, ticks_value
, ticks_title
,
1777 atomic_read(&rdtp
->dynticks
) & 0xfff,
1778 rdtp
->dynticks_nesting
, rdtp
->dynticks_nmi_nesting
,
1779 rdp
->softirq_snap
, kstat_softirqs_cpu(RCU_SOFTIRQ
, cpu
),
1780 READ_ONCE(rsp
->n_force_qs
) - rsp
->n_force_qs_gpstart
,
1784 /* Terminate the stall-info list. */
1785 static void print_cpu_stall_info_end(void)
1790 /* Zero ->ticks_this_gp for all flavors of RCU. */
1791 static void zero_cpu_stall_ticks(struct rcu_data
*rdp
)
1793 rdp
->ticks_this_gp
= 0;
1794 rdp
->softirq_snap
= kstat_softirqs_cpu(RCU_SOFTIRQ
, smp_processor_id());
1797 /* Increment ->ticks_this_gp for all flavors of RCU. */
1798 static void increment_cpu_stall_ticks(void)
1800 struct rcu_state
*rsp
;
1802 for_each_rcu_flavor(rsp
)
1803 raw_cpu_inc(rsp
->rda
->ticks_this_gp
);
1806 #ifdef CONFIG_RCU_NOCB_CPU
1809 * Offload callback processing from the boot-time-specified set of CPUs
1810 * specified by rcu_nocb_mask. For each CPU in the set, there is a
1811 * kthread created that pulls the callbacks from the corresponding CPU,
1812 * waits for a grace period to elapse, and invokes the callbacks.
1813 * The no-CBs CPUs do a wake_up() on their kthread when they insert
1814 * a callback into any empty list, unless the rcu_nocb_poll boot parameter
1815 * has been specified, in which case each kthread actively polls its
1816 * CPU. (Which isn't so great for energy efficiency, but which does
1817 * reduce RCU's overhead on that CPU.)
1819 * This is intended to be used in conjunction with Frederic Weisbecker's
1820 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
1821 * running CPU-bound user-mode computations.
1823 * Offloading of callback processing could also in theory be used as
1824 * an energy-efficiency measure because CPUs with no RCU callbacks
1825 * queued are more aggressive about entering dyntick-idle mode.
1829 /* Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters. */
1830 static int __init
rcu_nocb_setup(char *str
)
1832 alloc_bootmem_cpumask_var(&rcu_nocb_mask
);
1833 have_rcu_nocb_mask
= true;
1834 cpulist_parse(str
, rcu_nocb_mask
);
1837 __setup("rcu_nocbs=", rcu_nocb_setup
);
1839 static int __init
parse_rcu_nocb_poll(char *arg
)
1844 early_param("rcu_nocb_poll", parse_rcu_nocb_poll
);
1847 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
1850 static void rcu_nocb_gp_cleanup(struct rcu_state
*rsp
, struct rcu_node
*rnp
)
1852 wake_up_all(&rnp
->nocb_gp_wq
[rnp
->completed
& 0x1]);
1856 * Set the root rcu_node structure's ->need_future_gp field
1857 * based on the sum of those of all rcu_node structures. This does
1858 * double-count the root rcu_node structure's requests, but this
1859 * is necessary to handle the possibility of a rcu_nocb_kthread()
1860 * having awakened during the time that the rcu_node structures
1861 * were being updated for the end of the previous grace period.
1863 static void rcu_nocb_gp_set(struct rcu_node
*rnp
, int nrq
)
1865 rnp
->need_future_gp
[(rnp
->completed
+ 1) & 0x1] += nrq
;
1868 static void rcu_init_one_nocb(struct rcu_node
*rnp
)
1870 init_waitqueue_head(&rnp
->nocb_gp_wq
[0]);
1871 init_waitqueue_head(&rnp
->nocb_gp_wq
[1]);
1874 #ifndef CONFIG_RCU_NOCB_CPU_ALL
1875 /* Is the specified CPU a no-CBs CPU? */
1876 bool rcu_is_nocb_cpu(int cpu
)
1878 if (have_rcu_nocb_mask
)
1879 return cpumask_test_cpu(cpu
, rcu_nocb_mask
);
1882 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
1885 * Kick the leader kthread for this NOCB group.
1887 static void wake_nocb_leader(struct rcu_data
*rdp
, bool force
)
1889 struct rcu_data
*rdp_leader
= rdp
->nocb_leader
;
1891 if (!READ_ONCE(rdp_leader
->nocb_kthread
))
1893 if (READ_ONCE(rdp_leader
->nocb_leader_sleep
) || force
) {
1894 /* Prior smp_mb__after_atomic() orders against prior enqueue. */
1895 WRITE_ONCE(rdp_leader
->nocb_leader_sleep
, false);
1896 wake_up(&rdp_leader
->nocb_wq
);
1901 * Does the specified CPU need an RCU callback for the specified flavor
1904 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state
*rsp
, int cpu
)
1906 struct rcu_data
*rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
1908 #ifdef CONFIG_PROVE_RCU
1909 struct rcu_head
*rhp
;
1910 #endif /* #ifdef CONFIG_PROVE_RCU */
1913 * Check count of all no-CBs callbacks awaiting invocation.
1914 * There needs to be a barrier before this function is called,
1915 * but associated with a prior determination that no more
1916 * callbacks would be posted. In the worst case, the first
1917 * barrier in _rcu_barrier() suffices (but the caller cannot
1918 * necessarily rely on this, not a substitute for the caller
1919 * getting the concurrency design right!). There must also be
1920 * a barrier between the following load an posting of a callback
1921 * (if a callback is in fact needed). This is associated with an
1922 * atomic_inc() in the caller.
1924 ret
= atomic_long_read(&rdp
->nocb_q_count
);
1926 #ifdef CONFIG_PROVE_RCU
1927 rhp
= READ_ONCE(rdp
->nocb_head
);
1929 rhp
= READ_ONCE(rdp
->nocb_gp_head
);
1931 rhp
= READ_ONCE(rdp
->nocb_follower_head
);
1933 /* Having no rcuo kthread but CBs after scheduler starts is bad! */
1934 if (!READ_ONCE(rdp
->nocb_kthread
) && rhp
&&
1935 rcu_scheduler_fully_active
) {
1936 /* RCU callback enqueued before CPU first came online??? */
1937 pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
1941 #endif /* #ifdef CONFIG_PROVE_RCU */
1947 * Enqueue the specified string of rcu_head structures onto the specified
1948 * CPU's no-CBs lists. The CPU is specified by rdp, the head of the
1949 * string by rhp, and the tail of the string by rhtp. The non-lazy/lazy
1950 * counts are supplied by rhcount and rhcount_lazy.
1952 * If warranted, also wake up the kthread servicing this CPUs queues.
1954 static void __call_rcu_nocb_enqueue(struct rcu_data
*rdp
,
1955 struct rcu_head
*rhp
,
1956 struct rcu_head
**rhtp
,
1957 int rhcount
, int rhcount_lazy
,
1958 unsigned long flags
)
1961 struct rcu_head
**old_rhpp
;
1962 struct task_struct
*t
;
1964 /* Enqueue the callback on the nocb list and update counts. */
1965 atomic_long_add(rhcount
, &rdp
->nocb_q_count
);
1966 /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
1967 old_rhpp
= xchg(&rdp
->nocb_tail
, rhtp
);
1968 WRITE_ONCE(*old_rhpp
, rhp
);
1969 atomic_long_add(rhcount_lazy
, &rdp
->nocb_q_count_lazy
);
1970 smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
1972 /* If we are not being polled and there is a kthread, awaken it ... */
1973 t
= READ_ONCE(rdp
->nocb_kthread
);
1974 if (rcu_nocb_poll
|| !t
) {
1975 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
1976 TPS("WakeNotPoll"));
1979 len
= atomic_long_read(&rdp
->nocb_q_count
);
1980 if (old_rhpp
== &rdp
->nocb_head
) {
1981 if (!irqs_disabled_flags(flags
)) {
1982 /* ... if queue was empty ... */
1983 wake_nocb_leader(rdp
, false);
1984 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
1987 rdp
->nocb_defer_wakeup
= RCU_NOGP_WAKE
;
1988 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
1989 TPS("WakeEmptyIsDeferred"));
1991 rdp
->qlen_last_fqs_check
= 0;
1992 } else if (len
> rdp
->qlen_last_fqs_check
+ qhimark
) {
1993 /* ... or if many callbacks queued. */
1994 if (!irqs_disabled_flags(flags
)) {
1995 wake_nocb_leader(rdp
, true);
1996 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
1999 rdp
->nocb_defer_wakeup
= RCU_NOGP_WAKE_FORCE
;
2000 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
2001 TPS("WakeOvfIsDeferred"));
2003 rdp
->qlen_last_fqs_check
= LONG_MAX
/ 2;
2005 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
, TPS("WakeNot"));
2011 * This is a helper for __call_rcu(), which invokes this when the normal
2012 * callback queue is inoperable. If this is not a no-CBs CPU, this
2013 * function returns failure back to __call_rcu(), which can complain
2016 * Otherwise, this function queues the callback where the corresponding
2017 * "rcuo" kthread can find it.
2019 static bool __call_rcu_nocb(struct rcu_data
*rdp
, struct rcu_head
*rhp
,
2020 bool lazy
, unsigned long flags
)
2023 if (!rcu_is_nocb_cpu(rdp
->cpu
))
2025 __call_rcu_nocb_enqueue(rdp
, rhp
, &rhp
->next
, 1, lazy
, flags
);
2026 if (__is_kfree_rcu_offset((unsigned long)rhp
->func
))
2027 trace_rcu_kfree_callback(rdp
->rsp
->name
, rhp
,
2028 (unsigned long)rhp
->func
,
2029 -atomic_long_read(&rdp
->nocb_q_count_lazy
),
2030 -atomic_long_read(&rdp
->nocb_q_count
));
2032 trace_rcu_callback(rdp
->rsp
->name
, rhp
,
2033 -atomic_long_read(&rdp
->nocb_q_count_lazy
),
2034 -atomic_long_read(&rdp
->nocb_q_count
));
2037 * If called from an extended quiescent state with interrupts
2038 * disabled, invoke the RCU core in order to allow the idle-entry
2039 * deferred-wakeup check to function.
2041 if (irqs_disabled_flags(flags
) &&
2042 !rcu_is_watching() &&
2043 cpu_online(smp_processor_id()))
2050 * Adopt orphaned callbacks on a no-CBs CPU, or return 0 if this is
2053 static bool __maybe_unused
rcu_nocb_adopt_orphan_cbs(struct rcu_state
*rsp
,
2054 struct rcu_data
*rdp
,
2055 unsigned long flags
)
2057 long ql
= rsp
->qlen
;
2058 long qll
= rsp
->qlen_lazy
;
2060 /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
2061 if (!rcu_is_nocb_cpu(smp_processor_id()))
2066 /* First, enqueue the donelist, if any. This preserves CB ordering. */
2067 if (rsp
->orphan_donelist
!= NULL
) {
2068 __call_rcu_nocb_enqueue(rdp
, rsp
->orphan_donelist
,
2069 rsp
->orphan_donetail
, ql
, qll
, flags
);
2071 rsp
->orphan_donelist
= NULL
;
2072 rsp
->orphan_donetail
= &rsp
->orphan_donelist
;
2074 if (rsp
->orphan_nxtlist
!= NULL
) {
2075 __call_rcu_nocb_enqueue(rdp
, rsp
->orphan_nxtlist
,
2076 rsp
->orphan_nxttail
, ql
, qll
, flags
);
2078 rsp
->orphan_nxtlist
= NULL
;
2079 rsp
->orphan_nxttail
= &rsp
->orphan_nxtlist
;
2085 * If necessary, kick off a new grace period, and either way wait
2086 * for a subsequent grace period to complete.
2088 static void rcu_nocb_wait_gp(struct rcu_data
*rdp
)
2092 unsigned long flags
;
2094 struct rcu_node
*rnp
= rdp
->mynode
;
2096 raw_spin_lock_irqsave(&rnp
->lock
, flags
);
2097 smp_mb__after_unlock_lock();
2098 needwake
= rcu_start_future_gp(rnp
, rdp
, &c
);
2099 raw_spin_unlock_irqrestore(&rnp
->lock
, flags
);
2101 rcu_gp_kthread_wake(rdp
->rsp
);
2104 * Wait for the grace period. Do so interruptibly to avoid messing
2105 * up the load average.
2107 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("StartWait"));
2109 wait_event_interruptible(
2110 rnp
->nocb_gp_wq
[c
& 0x1],
2111 (d
= ULONG_CMP_GE(READ_ONCE(rnp
->completed
), c
)));
2114 WARN_ON(signal_pending(current
));
2115 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("ResumeWait"));
2117 trace_rcu_future_gp(rnp
, rdp
, c
, TPS("EndWait"));
2118 smp_mb(); /* Ensure that CB invocation happens after GP end. */
2122 * Leaders come here to wait for additional callbacks to show up.
2123 * This function does not return until callbacks appear.
2125 static void nocb_leader_wait(struct rcu_data
*my_rdp
)
2127 bool firsttime
= true;
2129 struct rcu_data
*rdp
;
2130 struct rcu_head
**tail
;
2134 /* Wait for callbacks to appear. */
2135 if (!rcu_nocb_poll
) {
2136 trace_rcu_nocb_wake(my_rdp
->rsp
->name
, my_rdp
->cpu
, "Sleep");
2137 wait_event_interruptible(my_rdp
->nocb_wq
,
2138 !READ_ONCE(my_rdp
->nocb_leader_sleep
));
2139 /* Memory barrier handled by smp_mb() calls below and repoll. */
2140 } else if (firsttime
) {
2141 firsttime
= false; /* Don't drown trace log with "Poll"! */
2142 trace_rcu_nocb_wake(my_rdp
->rsp
->name
, my_rdp
->cpu
, "Poll");
2146 * Each pass through the following loop checks a follower for CBs.
2147 * We are our own first follower. Any CBs found are moved to
2148 * nocb_gp_head, where they await a grace period.
2151 for (rdp
= my_rdp
; rdp
; rdp
= rdp
->nocb_next_follower
) {
2152 rdp
->nocb_gp_head
= READ_ONCE(rdp
->nocb_head
);
2153 if (!rdp
->nocb_gp_head
)
2154 continue; /* No CBs here, try next follower. */
2156 /* Move callbacks to wait-for-GP list, which is empty. */
2157 WRITE_ONCE(rdp
->nocb_head
, NULL
);
2158 rdp
->nocb_gp_tail
= xchg(&rdp
->nocb_tail
, &rdp
->nocb_head
);
2163 * If there were no callbacks, sleep a bit, rescan after a
2164 * memory barrier, and go retry.
2166 if (unlikely(!gotcbs
)) {
2168 trace_rcu_nocb_wake(my_rdp
->rsp
->name
, my_rdp
->cpu
,
2170 WARN_ON(signal_pending(current
));
2171 schedule_timeout_interruptible(1);
2173 /* Rescan in case we were a victim of memory ordering. */
2174 my_rdp
->nocb_leader_sleep
= true;
2175 smp_mb(); /* Ensure _sleep true before scan. */
2176 for (rdp
= my_rdp
; rdp
; rdp
= rdp
->nocb_next_follower
)
2177 if (READ_ONCE(rdp
->nocb_head
)) {
2178 /* Found CB, so short-circuit next wait. */
2179 my_rdp
->nocb_leader_sleep
= false;
2185 /* Wait for one grace period. */
2186 rcu_nocb_wait_gp(my_rdp
);
2189 * We left ->nocb_leader_sleep unset to reduce cache thrashing.
2190 * We set it now, but recheck for new callbacks while
2191 * traversing our follower list.
2193 my_rdp
->nocb_leader_sleep
= true;
2194 smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
2196 /* Each pass through the following loop wakes a follower, if needed. */
2197 for (rdp
= my_rdp
; rdp
; rdp
= rdp
->nocb_next_follower
) {
2198 if (READ_ONCE(rdp
->nocb_head
))
2199 my_rdp
->nocb_leader_sleep
= false;/* No need to sleep.*/
2200 if (!rdp
->nocb_gp_head
)
2201 continue; /* No CBs, so no need to wake follower. */
2203 /* Append callbacks to follower's "done" list. */
2204 tail
= xchg(&rdp
->nocb_follower_tail
, rdp
->nocb_gp_tail
);
2205 *tail
= rdp
->nocb_gp_head
;
2206 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2207 if (rdp
!= my_rdp
&& tail
== &rdp
->nocb_follower_head
) {
2209 * List was empty, wake up the follower.
2210 * Memory barriers supplied by atomic_long_add().
2212 wake_up(&rdp
->nocb_wq
);
2216 /* If we (the leader) don't have CBs, go wait some more. */
2217 if (!my_rdp
->nocb_follower_head
)
2222 * Followers come here to wait for additional callbacks to show up.
2223 * This function does not return until callbacks appear.
2225 static void nocb_follower_wait(struct rcu_data
*rdp
)
2227 bool firsttime
= true;
2230 if (!rcu_nocb_poll
) {
2231 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
2233 wait_event_interruptible(rdp
->nocb_wq
,
2234 READ_ONCE(rdp
->nocb_follower_head
));
2235 } else if (firsttime
) {
2236 /* Don't drown trace log with "Poll"! */
2238 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
, "Poll");
2240 if (smp_load_acquire(&rdp
->nocb_follower_head
)) {
2241 /* ^^^ Ensure CB invocation follows _head test. */
2245 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
2247 WARN_ON(signal_pending(current
));
2248 schedule_timeout_interruptible(1);
2253 * Per-rcu_data kthread, but only for no-CBs CPUs. Each kthread invokes
2254 * callbacks queued by the corresponding no-CBs CPU, however, there is
2255 * an optional leader-follower relationship so that the grace-period
2256 * kthreads don't have to do quite so many wakeups.
2258 static int rcu_nocb_kthread(void *arg
)
2261 struct rcu_head
*list
;
2262 struct rcu_head
*next
;
2263 struct rcu_head
**tail
;
2264 struct rcu_data
*rdp
= arg
;
2266 /* Each pass through this loop invokes one batch of callbacks */
2268 /* Wait for callbacks. */
2269 if (rdp
->nocb_leader
== rdp
)
2270 nocb_leader_wait(rdp
);
2272 nocb_follower_wait(rdp
);
2274 /* Pull the ready-to-invoke callbacks onto local list. */
2275 list
= READ_ONCE(rdp
->nocb_follower_head
);
2277 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
, "WokeNonEmpty");
2278 WRITE_ONCE(rdp
->nocb_follower_head
, NULL
);
2279 tail
= xchg(&rdp
->nocb_follower_tail
, &rdp
->nocb_follower_head
);
2281 /* Each pass through the following loop invokes a callback. */
2282 trace_rcu_batch_start(rdp
->rsp
->name
,
2283 atomic_long_read(&rdp
->nocb_q_count_lazy
),
2284 atomic_long_read(&rdp
->nocb_q_count
), -1);
2288 /* Wait for enqueuing to complete, if needed. */
2289 while (next
== NULL
&& &list
->next
!= tail
) {
2290 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
2292 schedule_timeout_interruptible(1);
2293 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
,
2297 debug_rcu_head_unqueue(list
);
2299 if (__rcu_reclaim(rdp
->rsp
->name
, list
))
2305 trace_rcu_batch_end(rdp
->rsp
->name
, c
, !!list
, 0, 0, 1);
2306 smp_mb__before_atomic(); /* _add after CB invocation. */
2307 atomic_long_add(-c
, &rdp
->nocb_q_count
);
2308 atomic_long_add(-cl
, &rdp
->nocb_q_count_lazy
);
2309 rdp
->n_nocbs_invoked
+= c
;
2314 /* Is a deferred wakeup of rcu_nocb_kthread() required? */
2315 static int rcu_nocb_need_deferred_wakeup(struct rcu_data
*rdp
)
2317 return READ_ONCE(rdp
->nocb_defer_wakeup
);
2320 /* Do a deferred wakeup of rcu_nocb_kthread(). */
2321 static void do_nocb_deferred_wakeup(struct rcu_data
*rdp
)
2325 if (!rcu_nocb_need_deferred_wakeup(rdp
))
2327 ndw
= READ_ONCE(rdp
->nocb_defer_wakeup
);
2328 WRITE_ONCE(rdp
->nocb_defer_wakeup
, RCU_NOGP_WAKE_NOT
);
2329 wake_nocb_leader(rdp
, ndw
== RCU_NOGP_WAKE_FORCE
);
2330 trace_rcu_nocb_wake(rdp
->rsp
->name
, rdp
->cpu
, TPS("DeferredWake"));
2333 void __init
rcu_init_nohz(void)
2336 bool need_rcu_nocb_mask
= true;
2337 struct rcu_state
*rsp
;
2339 #ifdef CONFIG_RCU_NOCB_CPU_NONE
2340 need_rcu_nocb_mask
= false;
2341 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_NONE */
2343 #if defined(CONFIG_NO_HZ_FULL)
2344 if (tick_nohz_full_running
&& cpumask_weight(tick_nohz_full_mask
))
2345 need_rcu_nocb_mask
= true;
2346 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2348 if (!have_rcu_nocb_mask
&& need_rcu_nocb_mask
) {
2349 if (!zalloc_cpumask_var(&rcu_nocb_mask
, GFP_KERNEL
)) {
2350 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
2353 have_rcu_nocb_mask
= true;
2355 if (!have_rcu_nocb_mask
)
2358 #ifdef CONFIG_RCU_NOCB_CPU_ZERO
2359 pr_info("\tOffload RCU callbacks from CPU 0\n");
2360 cpumask_set_cpu(0, rcu_nocb_mask
);
2361 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ZERO */
2362 #ifdef CONFIG_RCU_NOCB_CPU_ALL
2363 pr_info("\tOffload RCU callbacks from all CPUs\n");
2364 cpumask_copy(rcu_nocb_mask
, cpu_possible_mask
);
2365 #endif /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
2366 #if defined(CONFIG_NO_HZ_FULL)
2367 if (tick_nohz_full_running
)
2368 cpumask_or(rcu_nocb_mask
, rcu_nocb_mask
, tick_nohz_full_mask
);
2369 #endif /* #if defined(CONFIG_NO_HZ_FULL) */
2371 if (!cpumask_subset(rcu_nocb_mask
, cpu_possible_mask
)) {
2372 pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n");
2373 cpumask_and(rcu_nocb_mask
, cpu_possible_mask
,
2376 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
2377 cpumask_pr_args(rcu_nocb_mask
));
2379 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
2381 for_each_rcu_flavor(rsp
) {
2382 for_each_cpu(cpu
, rcu_nocb_mask
)
2383 init_nocb_callback_list(per_cpu_ptr(rsp
->rda
, cpu
));
2384 rcu_organize_nocb_kthreads(rsp
);
2388 /* Initialize per-rcu_data variables for no-CBs CPUs. */
2389 static void __init
rcu_boot_init_nocb_percpu_data(struct rcu_data
*rdp
)
2391 rdp
->nocb_tail
= &rdp
->nocb_head
;
2392 init_waitqueue_head(&rdp
->nocb_wq
);
2393 rdp
->nocb_follower_tail
= &rdp
->nocb_follower_head
;
2397 * If the specified CPU is a no-CBs CPU that does not already have its
2398 * rcuo kthread for the specified RCU flavor, spawn it. If the CPUs are
2399 * brought online out of order, this can require re-organizing the
2400 * leader-follower relationships.
2402 static void rcu_spawn_one_nocb_kthread(struct rcu_state
*rsp
, int cpu
)
2404 struct rcu_data
*rdp
;
2405 struct rcu_data
*rdp_last
;
2406 struct rcu_data
*rdp_old_leader
;
2407 struct rcu_data
*rdp_spawn
= per_cpu_ptr(rsp
->rda
, cpu
);
2408 struct task_struct
*t
;
2411 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
2412 * then nothing to do.
2414 if (!rcu_is_nocb_cpu(cpu
) || rdp_spawn
->nocb_kthread
)
2417 /* If we didn't spawn the leader first, reorganize! */
2418 rdp_old_leader
= rdp_spawn
->nocb_leader
;
2419 if (rdp_old_leader
!= rdp_spawn
&& !rdp_old_leader
->nocb_kthread
) {
2421 rdp
= rdp_old_leader
;
2423 rdp
->nocb_leader
= rdp_spawn
;
2424 if (rdp_last
&& rdp
!= rdp_spawn
)
2425 rdp_last
->nocb_next_follower
= rdp
;
2426 if (rdp
== rdp_spawn
) {
2427 rdp
= rdp
->nocb_next_follower
;
2430 rdp
= rdp
->nocb_next_follower
;
2431 rdp_last
->nocb_next_follower
= NULL
;
2434 rdp_spawn
->nocb_next_follower
= rdp_old_leader
;
2437 /* Spawn the kthread for this CPU and RCU flavor. */
2438 t
= kthread_run(rcu_nocb_kthread
, rdp_spawn
,
2439 "rcuo%c/%d", rsp
->abbr
, cpu
);
2441 WRITE_ONCE(rdp_spawn
->nocb_kthread
, t
);
2445 * If the specified CPU is a no-CBs CPU that does not already have its
2446 * rcuo kthreads, spawn them.
2448 static void rcu_spawn_all_nocb_kthreads(int cpu
)
2450 struct rcu_state
*rsp
;
2452 if (rcu_scheduler_fully_active
)
2453 for_each_rcu_flavor(rsp
)
2454 rcu_spawn_one_nocb_kthread(rsp
, cpu
);
2458 * Once the scheduler is running, spawn rcuo kthreads for all online
2459 * no-CBs CPUs. This assumes that the early_initcall()s happen before
2460 * non-boot CPUs come online -- if this changes, we will need to add
2461 * some mutual exclusion.
2463 static void __init
rcu_spawn_nocb_kthreads(void)
2467 for_each_online_cpu(cpu
)
2468 rcu_spawn_all_nocb_kthreads(cpu
);
2471 /* How many follower CPU IDs per leader? Default of -1 for sqrt(nr_cpu_ids). */
2472 static int rcu_nocb_leader_stride
= -1;
2473 module_param(rcu_nocb_leader_stride
, int, 0444);
2476 * Initialize leader-follower relationships for all no-CBs CPU.
2478 static void __init
rcu_organize_nocb_kthreads(struct rcu_state
*rsp
)
2481 int ls
= rcu_nocb_leader_stride
;
2482 int nl
= 0; /* Next leader. */
2483 struct rcu_data
*rdp
;
2484 struct rcu_data
*rdp_leader
= NULL
; /* Suppress misguided gcc warn. */
2485 struct rcu_data
*rdp_prev
= NULL
;
2487 if (!have_rcu_nocb_mask
)
2490 ls
= int_sqrt(nr_cpu_ids
);
2491 rcu_nocb_leader_stride
= ls
;
2495 * Each pass through this loop sets up one rcu_data structure and
2496 * spawns one rcu_nocb_kthread().
2498 for_each_cpu(cpu
, rcu_nocb_mask
) {
2499 rdp
= per_cpu_ptr(rsp
->rda
, cpu
);
2500 if (rdp
->cpu
>= nl
) {
2501 /* New leader, set up for followers & next leader. */
2502 nl
= DIV_ROUND_UP(rdp
->cpu
+ 1, ls
) * ls
;
2503 rdp
->nocb_leader
= rdp
;
2506 /* Another follower, link to previous leader. */
2507 rdp
->nocb_leader
= rdp_leader
;
2508 rdp_prev
->nocb_next_follower
= rdp
;
2514 /* Prevent __call_rcu() from enqueuing callbacks on no-CBs CPUs */
2515 static bool init_nocb_callback_list(struct rcu_data
*rdp
)
2517 if (!rcu_is_nocb_cpu(rdp
->cpu
))
2520 /* If there are early-boot callbacks, move them to nocb lists. */
2522 rdp
->nocb_head
= rdp
->nxtlist
;
2523 rdp
->nocb_tail
= rdp
->nxttail
[RCU_NEXT_TAIL
];
2524 atomic_long_set(&rdp
->nocb_q_count
, rdp
->qlen
);
2525 atomic_long_set(&rdp
->nocb_q_count_lazy
, rdp
->qlen_lazy
);
2526 rdp
->nxtlist
= NULL
;
2530 rdp
->nxttail
[RCU_NEXT_TAIL
] = NULL
;
2534 #else /* #ifdef CONFIG_RCU_NOCB_CPU */
2536 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state
*rsp
, int cpu
)
2538 WARN_ON_ONCE(1); /* Should be dead code. */
2542 static void rcu_nocb_gp_cleanup(struct rcu_state
*rsp
, struct rcu_node
*rnp
)
2546 static void rcu_nocb_gp_set(struct rcu_node
*rnp
, int nrq
)
2550 static void rcu_init_one_nocb(struct rcu_node
*rnp
)
2554 static bool __call_rcu_nocb(struct rcu_data
*rdp
, struct rcu_head
*rhp
,
2555 bool lazy
, unsigned long flags
)
2560 static bool __maybe_unused
rcu_nocb_adopt_orphan_cbs(struct rcu_state
*rsp
,
2561 struct rcu_data
*rdp
,
2562 unsigned long flags
)
2567 static void __init
rcu_boot_init_nocb_percpu_data(struct rcu_data
*rdp
)
2571 static int rcu_nocb_need_deferred_wakeup(struct rcu_data
*rdp
)
2576 static void do_nocb_deferred_wakeup(struct rcu_data
*rdp
)
2580 static void rcu_spawn_all_nocb_kthreads(int cpu
)
2584 static void __init
rcu_spawn_nocb_kthreads(void)
2588 static bool init_nocb_callback_list(struct rcu_data
*rdp
)
2593 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
2596 * An adaptive-ticks CPU can potentially execute in kernel mode for an
2597 * arbitrarily long period of time with the scheduling-clock tick turned
2598 * off. RCU will be paying attention to this CPU because it is in the
2599 * kernel, but the CPU cannot be guaranteed to be executing the RCU state
2600 * machine because the scheduling-clock tick has been disabled. Therefore,
2601 * if an adaptive-ticks CPU is failing to respond to the current grace
2602 * period and has not be idle from an RCU perspective, kick it.
2604 static void __maybe_unused
rcu_kick_nohz_cpu(int cpu
)
2606 #ifdef CONFIG_NO_HZ_FULL
2607 if (tick_nohz_full_cpu(cpu
))
2608 smp_send_reschedule(cpu
);
2609 #endif /* #ifdef CONFIG_NO_HZ_FULL */
2613 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
2615 static int full_sysidle_state
; /* Current system-idle state. */
2616 #define RCU_SYSIDLE_NOT 0 /* Some CPU is not idle. */
2617 #define RCU_SYSIDLE_SHORT 1 /* All CPUs idle for brief period. */
2618 #define RCU_SYSIDLE_LONG 2 /* All CPUs idle for long enough. */
2619 #define RCU_SYSIDLE_FULL 3 /* All CPUs idle, ready for sysidle. */
2620 #define RCU_SYSIDLE_FULL_NOTED 4 /* Actually entered sysidle state. */
2623 * Invoked to note exit from irq or task transition to idle. Note that
2624 * usermode execution does -not- count as idle here! After all, we want
2625 * to detect full-system idle states, not RCU quiescent states and grace
2626 * periods. The caller must have disabled interrupts.
2628 static void rcu_sysidle_enter(int irq
)
2631 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
2633 /* If there are no nohz_full= CPUs, no need to track this. */
2634 if (!tick_nohz_full_enabled())
2637 /* Adjust nesting, check for fully idle. */
2639 rdtp
->dynticks_idle_nesting
--;
2640 WARN_ON_ONCE(rdtp
->dynticks_idle_nesting
< 0);
2641 if (rdtp
->dynticks_idle_nesting
!= 0)
2642 return; /* Still not fully idle. */
2644 if ((rdtp
->dynticks_idle_nesting
& DYNTICK_TASK_NEST_MASK
) ==
2645 DYNTICK_TASK_NEST_VALUE
) {
2646 rdtp
->dynticks_idle_nesting
= 0;
2648 rdtp
->dynticks_idle_nesting
-= DYNTICK_TASK_NEST_VALUE
;
2649 WARN_ON_ONCE(rdtp
->dynticks_idle_nesting
< 0);
2650 return; /* Still not fully idle. */
2654 /* Record start of fully idle period. */
2656 WRITE_ONCE(rdtp
->dynticks_idle_jiffies
, j
);
2657 smp_mb__before_atomic();
2658 atomic_inc(&rdtp
->dynticks_idle
);
2659 smp_mb__after_atomic();
2660 WARN_ON_ONCE(atomic_read(&rdtp
->dynticks_idle
) & 0x1);
2664 * Unconditionally force exit from full system-idle state. This is
2665 * invoked when a normal CPU exits idle, but must be called separately
2666 * for the timekeeping CPU (tick_do_timer_cpu). The reason for this
2667 * is that the timekeeping CPU is permitted to take scheduling-clock
2668 * interrupts while the system is in system-idle state, and of course
2669 * rcu_sysidle_exit() has no way of distinguishing a scheduling-clock
2670 * interrupt from any other type of interrupt.
2672 void rcu_sysidle_force_exit(void)
2674 int oldstate
= READ_ONCE(full_sysidle_state
);
2678 * Each pass through the following loop attempts to exit full
2679 * system-idle state. If contention proves to be a problem,
2680 * a trylock-based contention tree could be used here.
2682 while (oldstate
> RCU_SYSIDLE_SHORT
) {
2683 newoldstate
= cmpxchg(&full_sysidle_state
,
2684 oldstate
, RCU_SYSIDLE_NOT
);
2685 if (oldstate
== newoldstate
&&
2686 oldstate
== RCU_SYSIDLE_FULL_NOTED
) {
2687 rcu_kick_nohz_cpu(tick_do_timer_cpu
);
2688 return; /* We cleared it, done! */
2690 oldstate
= newoldstate
;
2692 smp_mb(); /* Order initial oldstate fetch vs. later non-idle work. */
2696 * Invoked to note entry to irq or task transition from idle. Note that
2697 * usermode execution does -not- count as idle here! The caller must
2698 * have disabled interrupts.
2700 static void rcu_sysidle_exit(int irq
)
2702 struct rcu_dynticks
*rdtp
= this_cpu_ptr(&rcu_dynticks
);
2704 /* If there are no nohz_full= CPUs, no need to track this. */
2705 if (!tick_nohz_full_enabled())
2708 /* Adjust nesting, check for already non-idle. */
2710 rdtp
->dynticks_idle_nesting
++;
2711 WARN_ON_ONCE(rdtp
->dynticks_idle_nesting
<= 0);
2712 if (rdtp
->dynticks_idle_nesting
!= 1)
2713 return; /* Already non-idle. */
2716 * Allow for irq misnesting. Yes, it really is possible
2717 * to enter an irq handler then never leave it, and maybe
2718 * also vice versa. Handle both possibilities.
2720 if (rdtp
->dynticks_idle_nesting
& DYNTICK_TASK_NEST_MASK
) {
2721 rdtp
->dynticks_idle_nesting
+= DYNTICK_TASK_NEST_VALUE
;
2722 WARN_ON_ONCE(rdtp
->dynticks_idle_nesting
<= 0);
2723 return; /* Already non-idle. */
2725 rdtp
->dynticks_idle_nesting
= DYNTICK_TASK_EXIT_IDLE
;
2729 /* Record end of idle period. */
2730 smp_mb__before_atomic();
2731 atomic_inc(&rdtp
->dynticks_idle
);
2732 smp_mb__after_atomic();
2733 WARN_ON_ONCE(!(atomic_read(&rdtp
->dynticks_idle
) & 0x1));
2736 * If we are the timekeeping CPU, we are permitted to be non-idle
2737 * during a system-idle state. This must be the case, because
2738 * the timekeeping CPU has to take scheduling-clock interrupts
2739 * during the time that the system is transitioning to full
2740 * system-idle state. This means that the timekeeping CPU must
2741 * invoke rcu_sysidle_force_exit() directly if it does anything
2742 * more than take a scheduling-clock interrupt.
2744 if (smp_processor_id() == tick_do_timer_cpu
)
2747 /* Update system-idle state: We are clearly no longer fully idle! */
2748 rcu_sysidle_force_exit();
2752 * Check to see if the current CPU is idle. Note that usermode execution
2753 * does not count as idle. The caller must have disabled interrupts,
2754 * and must be running on tick_do_timer_cpu.
2756 static void rcu_sysidle_check_cpu(struct rcu_data
*rdp
, bool *isidle
,
2757 unsigned long *maxj
)
2761 struct rcu_dynticks
*rdtp
= rdp
->dynticks
;
2763 /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
2764 if (!tick_nohz_full_enabled())
2768 * If some other CPU has already reported non-idle, if this is
2769 * not the flavor of RCU that tracks sysidle state, or if this
2770 * is an offline or the timekeeping CPU, nothing to do.
2772 if (!*isidle
|| rdp
->rsp
!= rcu_state_p
||
2773 cpu_is_offline(rdp
->cpu
) || rdp
->cpu
== tick_do_timer_cpu
)
2775 /* Verify affinity of current kthread. */
2776 WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu
);
2778 /* Pick up current idle and NMI-nesting counter and check. */
2779 cur
= atomic_read(&rdtp
->dynticks_idle
);
2781 *isidle
= false; /* We are not idle! */
2784 smp_mb(); /* Read counters before timestamps. */
2786 /* Pick up timestamps. */
2787 j
= READ_ONCE(rdtp
->dynticks_idle_jiffies
);
2788 /* If this CPU entered idle more recently, update maxj timestamp. */
2789 if (ULONG_CMP_LT(*maxj
, j
))
2794 * Is this the flavor of RCU that is handling full-system idle?
2796 static bool is_sysidle_rcu_state(struct rcu_state
*rsp
)
2798 return rsp
== rcu_state_p
;
2802 * Return a delay in jiffies based on the number of CPUs, rcu_node
2803 * leaf fanout, and jiffies tick rate. The idea is to allow larger
2804 * systems more time to transition to full-idle state in order to
2805 * avoid the cache thrashing that otherwise occur on the state variable.
2806 * Really small systems (less than a couple of tens of CPUs) should
2807 * instead use a single global atomically incremented counter, and later
2808 * versions of this will automatically reconfigure themselves accordingly.
2810 static unsigned long rcu_sysidle_delay(void)
2812 if (nr_cpu_ids
<= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL
)
2814 return DIV_ROUND_UP(nr_cpu_ids
* HZ
, rcu_fanout_leaf
* 1000);
2818 * Advance the full-system-idle state. This is invoked when all of
2819 * the non-timekeeping CPUs are idle.
2821 static void rcu_sysidle(unsigned long j
)
2823 /* Check the current state. */
2824 switch (READ_ONCE(full_sysidle_state
)) {
2825 case RCU_SYSIDLE_NOT
:
2827 /* First time all are idle, so note a short idle period. */
2828 WRITE_ONCE(full_sysidle_state
, RCU_SYSIDLE_SHORT
);
2831 case RCU_SYSIDLE_SHORT
:
2834 * Idle for a bit, time to advance to next state?
2835 * cmpxchg failure means race with non-idle, let them win.
2837 if (ULONG_CMP_GE(jiffies
, j
+ rcu_sysidle_delay()))
2838 (void)cmpxchg(&full_sysidle_state
,
2839 RCU_SYSIDLE_SHORT
, RCU_SYSIDLE_LONG
);
2842 case RCU_SYSIDLE_LONG
:
2845 * Do an additional check pass before advancing to full.
2846 * cmpxchg failure means race with non-idle, let them win.
2848 if (ULONG_CMP_GE(jiffies
, j
+ rcu_sysidle_delay()))
2849 (void)cmpxchg(&full_sysidle_state
,
2850 RCU_SYSIDLE_LONG
, RCU_SYSIDLE_FULL
);
2859 * Found a non-idle non-timekeeping CPU, so kick the system-idle state
2860 * back to the beginning.
2862 static void rcu_sysidle_cancel(void)
2865 if (full_sysidle_state
> RCU_SYSIDLE_SHORT
)
2866 WRITE_ONCE(full_sysidle_state
, RCU_SYSIDLE_NOT
);
2870 * Update the sysidle state based on the results of a force-quiescent-state
2871 * scan of the CPUs' dyntick-idle state.
2873 static void rcu_sysidle_report(struct rcu_state
*rsp
, int isidle
,
2874 unsigned long maxj
, bool gpkt
)
2876 if (rsp
!= rcu_state_p
)
2877 return; /* Wrong flavor, ignore. */
2878 if (gpkt
&& nr_cpu_ids
<= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL
)
2879 return; /* Running state machine from timekeeping CPU. */
2881 rcu_sysidle(maxj
); /* More idle! */
2883 rcu_sysidle_cancel(); /* Idle is over. */
2887 * Wrapper for rcu_sysidle_report() when called from the grace-period
2888 * kthread's context.
2890 static void rcu_sysidle_report_gp(struct rcu_state
*rsp
, int isidle
,
2893 /* If there are no nohz_full= CPUs, no need to track this. */
2894 if (!tick_nohz_full_enabled())
2897 rcu_sysidle_report(rsp
, isidle
, maxj
, true);
2900 /* Callback and function for forcing an RCU grace period. */
2901 struct rcu_sysidle_head
{
2906 static void rcu_sysidle_cb(struct rcu_head
*rhp
)
2908 struct rcu_sysidle_head
*rshp
;
2911 * The following memory barrier is needed to replace the
2912 * memory barriers that would normally be in the memory
2915 smp_mb(); /* grace period precedes setting inuse. */
2917 rshp
= container_of(rhp
, struct rcu_sysidle_head
, rh
);
2918 WRITE_ONCE(rshp
->inuse
, 0);
2922 * Check to see if the system is fully idle, other than the timekeeping CPU.
2923 * The caller must have disabled interrupts. This is not intended to be
2924 * called unless tick_nohz_full_enabled().
2926 bool rcu_sys_is_idle(void)
2928 static struct rcu_sysidle_head rsh
;
2929 int rss
= READ_ONCE(full_sysidle_state
);
2931 if (WARN_ON_ONCE(smp_processor_id() != tick_do_timer_cpu
))
2934 /* Handle small-system case by doing a full scan of CPUs. */
2935 if (nr_cpu_ids
<= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL
) {
2936 int oldrss
= rss
- 1;
2939 * One pass to advance to each state up to _FULL.
2940 * Give up if any pass fails to advance the state.
2942 while (rss
< RCU_SYSIDLE_FULL
&& oldrss
< rss
) {
2945 unsigned long maxj
= jiffies
- ULONG_MAX
/ 4;
2946 struct rcu_data
*rdp
;
2948 /* Scan all the CPUs looking for nonidle CPUs. */
2949 for_each_possible_cpu(cpu
) {
2950 rdp
= per_cpu_ptr(rcu_state_p
->rda
, cpu
);
2951 rcu_sysidle_check_cpu(rdp
, &isidle
, &maxj
);
2955 rcu_sysidle_report(rcu_state_p
, isidle
, maxj
, false);
2957 rss
= READ_ONCE(full_sysidle_state
);
2961 /* If this is the first observation of an idle period, record it. */
2962 if (rss
== RCU_SYSIDLE_FULL
) {
2963 rss
= cmpxchg(&full_sysidle_state
,
2964 RCU_SYSIDLE_FULL
, RCU_SYSIDLE_FULL_NOTED
);
2965 return rss
== RCU_SYSIDLE_FULL
;
2968 smp_mb(); /* ensure rss load happens before later caller actions. */
2970 /* If already fully idle, tell the caller (in case of races). */
2971 if (rss
== RCU_SYSIDLE_FULL_NOTED
)
2975 * If we aren't there yet, and a grace period is not in flight,
2976 * initiate a grace period. Either way, tell the caller that
2977 * we are not there yet. We use an xchg() rather than an assignment
2978 * to make up for the memory barriers that would otherwise be
2979 * provided by the memory allocator.
2981 if (nr_cpu_ids
> CONFIG_NO_HZ_FULL_SYSIDLE_SMALL
&&
2982 !rcu_gp_in_progress(rcu_state_p
) &&
2983 !rsh
.inuse
&& xchg(&rsh
.inuse
, 1) == 0)
2984 call_rcu(&rsh
.rh
, rcu_sysidle_cb
);
2989 * Initialize dynticks sysidle state for CPUs coming online.
2991 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks
*rdtp
)
2993 rdtp
->dynticks_idle_nesting
= DYNTICK_TASK_NEST_VALUE
;
2996 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2998 static void rcu_sysidle_enter(int irq
)
3002 static void rcu_sysidle_exit(int irq
)
3006 static void rcu_sysidle_check_cpu(struct rcu_data
*rdp
, bool *isidle
,
3007 unsigned long *maxj
)
3011 static bool is_sysidle_rcu_state(struct rcu_state
*rsp
)
3016 static void rcu_sysidle_report_gp(struct rcu_state
*rsp
, int isidle
,
3021 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks
*rdtp
)
3025 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3028 * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
3029 * grace-period kthread will do force_quiescent_state() processing?
3030 * The idea is to avoid waking up RCU core processing on such a
3031 * CPU unless the grace period has extended for too long.
3033 * This code relies on the fact that all NO_HZ_FULL CPUs are also
3034 * CONFIG_RCU_NOCB_CPU CPUs.
3036 static bool rcu_nohz_full_cpu(struct rcu_state
*rsp
)
3038 #ifdef CONFIG_NO_HZ_FULL
3039 if (tick_nohz_full_cpu(smp_processor_id()) &&
3040 (!rcu_gp_in_progress(rsp
) ||
3041 ULONG_CMP_LT(jiffies
, READ_ONCE(rsp
->gp_start
) + HZ
)))
3043 #endif /* #ifdef CONFIG_NO_HZ_FULL */
3048 * Bind the grace-period kthread for the sysidle flavor of RCU to the
3051 static void rcu_bind_gp_kthread(void)
3053 int __maybe_unused cpu
;
3055 if (!tick_nohz_full_enabled())
3057 #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
3058 cpu
= tick_do_timer_cpu
;
3059 if (cpu
>= 0 && cpu
< nr_cpu_ids
)
3060 set_cpus_allowed_ptr(current
, cpumask_of(cpu
));
3061 #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3062 housekeeping_affine(current
);
3063 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
3066 /* Record the current task on dyntick-idle entry. */
3067 static void rcu_dynticks_task_enter(void)
3069 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3070 WRITE_ONCE(current
->rcu_tasks_idle_cpu
, smp_processor_id());
3071 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
3074 /* Record no current task on dyntick-idle exit. */
3075 static void rcu_dynticks_task_exit(void)
3077 #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
3078 WRITE_ONCE(current
->rcu_tasks_idle_cpu
, -1);
3079 #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */