kernel/rcu/tiny.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   4  *
   5  * Copyright IBM Corporation, 2008
   6  *
   7  * Author: Paul E. McKenney <paulmck@linux.ibm.com>
   8  *
   9  * For detailed explanation of Read-Copy Update mechanism see -
  10  *              Documentation/RCU
  11  */
  12 #include <linux/completion.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/notifier.h>
  15 #include <linux/rcupdate_wait.h>
  16 #include <linux/kernel.h>
  17 #include <linux/export.h>
  18 #include <linux/mutex.h>
  19 #include <linux/sched.h>
  20 #include <linux/types.h>
  21 #include <linux/init.h>
  22 #include <linux/time.h>
  23 #include <linux/cpu.h>
  24 #include <linux/prefetch.h>
  25 #include <linux/slab.h>
  26
  27 #include "rcu.h"
  28
  29 /* Global control variables for rcupdate callback mechanism. */
  30 struct rcu_ctrlblk {
  31         struct rcu_head *rcucblist;     /* List of pending callbacks (CBs). */
  32         struct rcu_head **donetail;     /* ->next pointer of last "done" CB. */
  33         struct rcu_head **curtail;      /* ->next pointer of last CB. */
  34 };
  35
  36 /* Definition for rcupdate control block. */
  37 static struct rcu_ctrlblk rcu_ctrlblk = {
  38         .donetail       = &rcu_ctrlblk.rcucblist,
  39         .curtail        = &rcu_ctrlblk.rcucblist,
  40 };
  41
  42 void rcu_barrier(void)
  43 {
  44         wait_rcu_gp(call_rcu);
  45 }
  46 EXPORT_SYMBOL(rcu_barrier);
  47
  48 /* Record an rcu quiescent state.  */
  49 void rcu_qs(void)
  50 {
  51         unsigned long flags;
  52
  53         local_irq_save(flags);
  54         if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  55                 rcu_ctrlblk.donetail = rcu_ctrlblk.curtail;
  56                 raise_softirq_irqoff(RCU_SOFTIRQ);
  57         }
  58         local_irq_restore(flags);
  59 }
  60
  61 /*
  62  * Check to see if the scheduling-clock interrupt came from an extended
  63  * quiescent state, and, if so, tell RCU about it.  This function must
  64  * be called from hardirq context.  It is normally called from the
  65  * scheduling-clock interrupt.
  66  */
  67 void rcu_sched_clock_irq(int user)
  68 {
  69         if (user) {
  70                 rcu_qs();
  71         } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) {
  72                 set_tsk_need_resched(current);
  73                 set_preempt_need_resched();
  74         }
  75 }
  76
  77 /*
  78  * Reclaim the specified callback, either by invoking it for non-kfree cases or
  79  * freeing it directly (for kfree). Return true if kfreeing, false otherwise.
  80  */
  81 static inline bool rcu_reclaim_tiny(struct rcu_head *head)
  82 {
  83         rcu_callback_t f;
  84         unsigned long offset = (unsigned long)head->func;
  85
  86         rcu_lock_acquire(&rcu_callback_map);
  87         if (__is_kfree_rcu_offset(offset)) {
  88                 trace_rcu_invoke_kfree_callback("", head, offset);
  89                 kfree((void *)head - offset);
  90                 rcu_lock_release(&rcu_callback_map);
  91                 return true;
  92         }
  93
  94         trace_rcu_invoke_callback("", head);
  95         f = head->func;
  96         WRITE_ONCE(head->func, (rcu_callback_t)0L);
  97         f(head);
  98         rcu_lock_release(&rcu_callback_map);
  99         return false;
 100 }
 101
 102 /* Invoke the RCU callbacks whose grace period has elapsed.  */
 103 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
 104 {
 105         struct rcu_head *next, *list;
 106         unsigned long flags;
 107
 108         /* Move the ready-to-invoke callbacks to a local list. */
 109         local_irq_save(flags);
 110         if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) {
 111                 /* No callbacks ready, so just leave. */
 112                 local_irq_restore(flags);
 113                 return;
 114         }
 115         list = rcu_ctrlblk.rcucblist;
 116         rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail;
 117         *rcu_ctrlblk.donetail = NULL;
 118         if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail)
 119                 rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist;
 120         rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist;
 121         local_irq_restore(flags);
 122
 123         /* Invoke the callbacks on the local list. */
 124         while (list) {
 125                 next = list->next;
 126                 prefetch(next);
 127                 debug_rcu_head_unqueue(list);
 128                 local_bh_disable();
 129                 rcu_reclaim_tiny(list);
 130                 local_bh_enable();
 131                 list = next;
 132         }
 133 }
 134
 135 /*
 136  * Wait for a grace period to elapse.  But it is illegal to invoke
 137  * synchronize_rcu() from within an RCU read-side critical section.
 138  * Therefore, any legal call to synchronize_rcu() is a quiescent
 139  * state, and so on a UP system, synchronize_rcu() need do nothing.
 140  * (But Lai Jiangshan points out the benefits of doing might_sleep()
 141  * to reduce latency.)
 142  *
 143  * Cool, huh?  (Due to Josh Triplett.)
 144  */
 145 void synchronize_rcu(void)
 146 {
 147         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 148                          lock_is_held(&rcu_lock_map) ||
 149                          lock_is_held(&rcu_sched_lock_map),
 150                          "Illegal synchronize_rcu() in RCU read-side critical section");
 151 }
 152 EXPORT_SYMBOL_GPL(synchronize_rcu);
 153
 154 /*
 155  * Post an RCU callback to be invoked after the end of an RCU grace
 156  * period.  But since we have but one CPU, that would be after any
 157  * quiescent state.
 158  */
 159 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 160 {
 161         unsigned long flags;
 162
 163         debug_rcu_head_queue(head);
 164         head->func = func;
 165         head->next = NULL;
 166
 167         local_irq_save(flags);
 168         *rcu_ctrlblk.curtail = head;
 169         rcu_ctrlblk.curtail = &head->next;
 170         local_irq_restore(flags);
 171
 172         if (unlikely(is_idle_task(current))) {
 173                 /* force scheduling for rcu_qs() */
 174                 resched_cpu(0);
 175         }
 176 }
 177 EXPORT_SYMBOL_GPL(call_rcu);
 178
 179 void __init rcu_init(void)
 180 {
 181         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 182         rcu_early_boot_tests();
 183         srcu_init();
 184 }