kernel/rcutiny.c

   1 /*
   2  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright IBM Corporation, 2008
  19  *
  20  * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  21  *
  22  * For detailed explanation of Read-Copy Update mechanism see -
  23  *              Documentation/RCU
  24  */
  25 #include <linux/completion.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/notifier.h>
  28 #include <linux/rcupdate.h>
  29 #include <linux/kernel.h>
  30 #include <linux/export.h>
  31 #include <linux/mutex.h>
  32 #include <linux/sched.h>
  33 #include <linux/types.h>
  34 #include <linux/init.h>
  35 #include <linux/time.h>
  36 #include <linux/cpu.h>
  37 #include <linux/prefetch.h>
  38
  39 #ifdef CONFIG_RCU_TRACE
  40 #include <trace/events/rcu.h>
  41 #endif /* #else #ifdef CONFIG_RCU_TRACE */
  42
  43 #include "rcu.h"
  44
  45 /* Forward declarations for rcutiny_plugin.h. */
  46 struct rcu_ctrlblk;
  47 static void invoke_rcu_callbacks(void);
  48 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp);
  49 static void rcu_process_callbacks(struct softirq_action *unused);
  50 static void __call_rcu(struct rcu_head *head,
  51                        void (*func)(struct rcu_head *rcu),
  52                        struct rcu_ctrlblk *rcp);
  53
  54 #include "rcutiny_plugin.h"
  55
  56 static long long rcu_dynticks_nesting = LLONG_MAX / 2;
  57
  58 /* Common code for rcu_idle_enter() and rcu_irq_exit(), see kernel/rcutree.c. */
  59 static void rcu_idle_enter_common(void)
  60 {
  61         if (rcu_dynticks_nesting) {
  62                 RCU_TRACE(trace_rcu_dyntick("--=", rcu_dynticks_nesting));
  63                 return;
  64         }
  65         RCU_TRACE(trace_rcu_dyntick("Start", rcu_dynticks_nesting));
  66         if (!idle_cpu(smp_processor_id())) {
  67                 WARN_ON_ONCE(1);        /* must be idle task! */
  68                 RCU_TRACE(trace_rcu_dyntick("Error on entry: not idle task",
  69                                             rcu_dynticks_nesting));
  70                 ftrace_dump(DUMP_ALL);
  71         }
  72         rcu_sched_qs(0); /* implies rcu_bh_qsctr_inc(0) */
  73 }
  74
  75 /*
  76  * Enter idle, which is an extended quiescent state if we have fully
  77  * entered that mode (i.e., if the new value of dynticks_nesting is zero).
  78  */
  79 void rcu_idle_enter(void)
  80 {
  81         unsigned long flags;
  82
  83         local_irq_save(flags);
  84         rcu_dynticks_nesting = 0;
  85         rcu_idle_enter_common();
  86         local_irq_restore(flags);
  87 }
  88
  89 /*
  90  * Exit an interrupt handler towards idle.
  91  */
  92 void rcu_irq_exit(void)
  93 {
  94         unsigned long flags;
  95
  96         local_irq_save(flags);
  97         rcu_dynticks_nesting--;
  98         WARN_ON_ONCE(rcu_dynticks_nesting < 0);
  99         rcu_idle_enter_common();
 100         local_irq_restore(flags);
 101 }
 102
 103 /* Common code for rcu_idle_exit() and rcu_irq_enter(), see kernel/rcutree.c. */
 104 static void rcu_idle_exit_common(long long oldval)
 105 {
 106         if (oldval) {
 107                 RCU_TRACE(trace_rcu_dyntick("++=", rcu_dynticks_nesting));
 108                 return;
 109         }
 110         RCU_TRACE(trace_rcu_dyntick("End", oldval));
 111         if (!idle_cpu(smp_processor_id())) {
 112                 WARN_ON_ONCE(1);        /* must be idle task! */
 113                 RCU_TRACE(trace_rcu_dyntick("Error on exit: not idle task",
 114                           oldval));
 115                 ftrace_dump(DUMP_ALL);
 116         }
 117 }
 118
 119 /*
 120  * Exit idle, so that we are no longer in an extended quiescent state.
 121  */
 122 void rcu_idle_exit(void)
 123 {
 124         unsigned long flags;
 125         long long oldval;
 126
 127         local_irq_save(flags);
 128         oldval = rcu_dynticks_nesting;
 129         WARN_ON_ONCE(oldval != 0);
 130         rcu_dynticks_nesting = LLONG_MAX / 2;
 131         rcu_idle_exit_common(oldval);
 132         local_irq_restore(flags);
 133 }
 134
 135 /*
 136  * Enter an interrupt handler, moving away from idle.
 137  */
 138 void rcu_irq_enter(void)
 139 {
 140         unsigned long flags;
 141         long long oldval;
 142
 143         local_irq_save(flags);
 144         oldval = rcu_dynticks_nesting;
 145         rcu_dynticks_nesting++;
 146         WARN_ON_ONCE(rcu_dynticks_nesting == 0);
 147         rcu_idle_exit_common(oldval);
 148         local_irq_restore(flags);
 149 }
 150
 151 #ifdef CONFIG_PROVE_RCU
 152
 153 /*
 154  * Test whether RCU thinks that the current CPU is idle.
 155  */
 156 int rcu_is_cpu_idle(void)
 157 {
 158         return !rcu_dynticks_nesting;
 159 }
 160 EXPORT_SYMBOL(rcu_is_cpu_idle);
 161
 162 #endif /* #ifdef CONFIG_PROVE_RCU */
 163
 164 /*
 165  * Test whether the current CPU was interrupted from idle.  Nested
 166  * interrupts don't count, we must be running at the first interrupt
 167  * level.
 168  */
 169 int rcu_is_cpu_rrupt_from_idle(void)
 170 {
 171         return rcu_dynticks_nesting <= 0;
 172 }
 173
 174 /*
 175  * Helper function for rcu_sched_qs() and rcu_bh_qs().
 176  * Also irqs are disabled to avoid confusion due to interrupt handlers
 177  * invoking call_rcu().
 178  */
 179 static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
 180 {
 181         if (rcp->rcucblist != NULL &&
 182             rcp->donetail != rcp->curtail) {
 183                 rcp->donetail = rcp->curtail;
 184                 return 1;
 185         }
 186
 187         return 0;
 188 }
 189
 190 /*
 191  * Record an rcu quiescent state.  And an rcu_bh quiescent state while we
 192  * are at it, given that any rcu quiescent state is also an rcu_bh
 193  * quiescent state.  Use "+" instead of "||" to defeat short circuiting.
 194  */
 195 void rcu_sched_qs(int cpu)
 196 {
 197         unsigned long flags;
 198
 199         local_irq_save(flags);
 200         if (rcu_qsctr_help(&rcu_sched_ctrlblk) +
 201             rcu_qsctr_help(&rcu_bh_ctrlblk))
 202                 invoke_rcu_callbacks();
 203         local_irq_restore(flags);
 204 }
 205
 206 /*
 207  * Record an rcu_bh quiescent state.
 208  */
 209 void rcu_bh_qs(int cpu)
 210 {
 211         unsigned long flags;
 212
 213         local_irq_save(flags);
 214         if (rcu_qsctr_help(&rcu_bh_ctrlblk))
 215                 invoke_rcu_callbacks();
 216         local_irq_restore(flags);
 217 }
 218
 219 /*
 220  * Check to see if the scheduling-clock interrupt came from an extended
 221  * quiescent state, and, if so, tell RCU about it.  This function must
 222  * be called from hardirq context.  It is normally called from the
 223  * scheduling-clock interrupt.
 224  */
 225 void rcu_check_callbacks(int cpu, int user)
 226 {
 227         if (user || rcu_is_cpu_rrupt_from_idle())
 228                 rcu_sched_qs(cpu);
 229         else if (!in_softirq())
 230                 rcu_bh_qs(cpu);
 231         rcu_preempt_check_callbacks();
 232 }
 233
 234 /*
 235  * Invoke the RCU callbacks on the specified rcu_ctrlkblk structure
 236  * whose grace period has elapsed.
 237  */
 238 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
 239 {
 240         char *rn = NULL;
 241         struct rcu_head *next, *list;
 242         unsigned long flags;
 243         RCU_TRACE(int cb_count = 0);
 244
 245         /* If no RCU callbacks ready to invoke, just return. */
 246         if (&rcp->rcucblist == rcp->donetail) {
 247                 RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, -1));
 248                 RCU_TRACE(trace_rcu_batch_end(rcp->name, 0));
 249                 return;
 250         }
 251
 252         /* Move the ready-to-invoke callbacks to a local list. */
 253         local_irq_save(flags);
 254         RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, -1));
 255         list = rcp->rcucblist;
 256         rcp->rcucblist = *rcp->donetail;
 257         *rcp->donetail = NULL;
 258         if (rcp->curtail == rcp->donetail)
 259                 rcp->curtail = &rcp->rcucblist;
 260         rcu_preempt_remove_callbacks(rcp);
 261         rcp->donetail = &rcp->rcucblist;
 262         local_irq_restore(flags);
 263
 264         /* Invoke the callbacks on the local list. */
 265         RCU_TRACE(rn = rcp->name);
 266         while (list) {
 267                 next = list->next;
 268                 prefetch(next);
 269                 debug_rcu_head_unqueue(list);
 270                 local_bh_disable();
 271                 __rcu_reclaim(rn, list);
 272                 local_bh_enable();
 273                 list = next;
 274                 RCU_TRACE(cb_count++);
 275         }
 276         RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count));
 277         RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count));
 278 }
 279
 280 static void rcu_process_callbacks(struct softirq_action *unused)
 281 {
 282         __rcu_process_callbacks(&rcu_sched_ctrlblk);
 283         __rcu_process_callbacks(&rcu_bh_ctrlblk);
 284         rcu_preempt_process_callbacks();
 285 }
 286
 287 /*
 288  * Wait for a grace period to elapse.  But it is illegal to invoke
 289  * synchronize_sched() from within an RCU read-side critical section.
 290  * Therefore, any legal call to synchronize_sched() is a quiescent
 291  * state, and so on a UP system, synchronize_sched() need do nothing.
 292  * Ditto for synchronize_rcu_bh().  (But Lai Jiangshan points out the
 293  * benefits of doing might_sleep() to reduce latency.)
 294  *
 295  * Cool, huh?  (Due to Josh Triplett.)
 296  *
 297  * But we want to make this a static inline later.  The cond_resched()
 298  * currently makes this problematic.
 299  */
 300 void synchronize_sched(void)
 301 {
 302         cond_resched();
 303 }
 304 EXPORT_SYMBOL_GPL(synchronize_sched);
 305
 306 /*
 307  * Helper function for call_rcu() and call_rcu_bh().
 308  */
 309 static void __call_rcu(struct rcu_head *head,
 310                        void (*func)(struct rcu_head *rcu),
 311                        struct rcu_ctrlblk *rcp)
 312 {
 313         unsigned long flags;
 314
 315         debug_rcu_head_queue(head);
 316         head->func = func;
 317         head->next = NULL;
 318
 319         local_irq_save(flags);
 320         *rcp->curtail = head;
 321         rcp->curtail = &head->next;
 322         RCU_TRACE(rcp->qlen++);
 323         local_irq_restore(flags);
 324 }
 325
 326 /*
 327  * Post an RCU callback to be invoked after the end of an RCU-sched grace
 328  * period.  But since we have but one CPU, that would be after any
 329  * quiescent state.
 330  */
 331 void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 332 {
 333         __call_rcu(head, func, &rcu_sched_ctrlblk);
 334 }
 335 EXPORT_SYMBOL_GPL(call_rcu_sched);
 336
 337 /*
 338  * Post an RCU bottom-half callback to be invoked after any subsequent
 339  * quiescent state.
 340  */
 341 void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
 342 {
 343         __call_rcu(head, func, &rcu_bh_ctrlblk);
 344 }
 345 EXPORT_SYMBOL_GPL(call_rcu_bh);