rcu: Per-CPU operation cleanups to rcu_*_qs() functions

[mirror_ubuntu-bionic-kernel.git] / kernel / rcu / tree.c

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 1b70cb6fbe3ccda0466f3f0004865d82cdd9399d..4c340625ffd4bfa449b6285ac2e44f26f77efef2 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -188,22 +188,24 @@ static int rcu_gp_in_progress(struct rcu_state *rsp)
  * one since the start of the grace period, this just sets a flag.
  * The caller must have disabled preemption.
  */
-void rcu_sched_qs(int cpu)
+void rcu_sched_qs(void)
 {
-       struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
-
-       if (rdp->passed_quiesce == 0)
-               trace_rcu_grace_period(TPS("rcu_sched"), rdp->gpnum, TPS("cpuqs"));
-       rdp->passed_quiesce = 1;
+       if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) {
+               trace_rcu_grace_period(TPS("rcu_sched"),
+                                      __this_cpu_read(rcu_sched_data.gpnum),
+                                      TPS("cpuqs"));
+               __this_cpu_write(rcu_sched_data.passed_quiesce, 1);
+       }
 }
 
-void rcu_bh_qs(int cpu)
+void rcu_bh_qs(void)
 {
-       struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
-
-       if (rdp->passed_quiesce == 0)
-               trace_rcu_grace_period(TPS("rcu_bh"), rdp->gpnum, TPS("cpuqs"));
-       rdp->passed_quiesce = 1;
+       if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) {
+               trace_rcu_grace_period(TPS("rcu_bh"),
+                                      __this_cpu_read(rcu_bh_data.gpnum),
+                                      TPS("cpuqs"));
+               __this_cpu_write(rcu_bh_data.passed_quiesce, 1);
+       }
 }
 
 static DEFINE_PER_CPU(int, rcu_sched_qs_mask);
@@ -278,7 +280,7 @@ static void rcu_momentary_dyntick_idle(void)
 void rcu_note_context_switch(int cpu)
 {
        trace_rcu_utilization(TPS("Start context switch"));
-       rcu_sched_qs(cpu);
+       rcu_sched_qs();
        rcu_preempt_note_context_switch(cpu);
        if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
                rcu_momentary_dyntick_idle();
@@ -526,6 +528,7 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
        atomic_inc(&rdtp->dynticks);
        smp_mb__after_atomic();  /* Force ordering with next sojourn. */
        WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+       rcu_dynticks_task_enter();
 
        /*
         * It is illegal to enter an extended quiescent state while
@@ -642,6 +645,7 @@ void rcu_irq_exit(void)
 static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
                               int user)
 {
+       rcu_dynticks_task_exit();
        smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
        atomic_inc(&rdtp->dynticks);
        /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
@@ -1647,7 +1651,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
                                            rnp->level, rnp->grplo,
                                            rnp->grphi, rnp->qsmask);
                raw_spin_unlock_irq(&rnp->lock);
-               cond_resched();
+               cond_resched_rcu_qs();
        }
 
        mutex_unlock(&rsp->onoff_mutex);
@@ -1736,7 +1740,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
                /* smp_mb() provided by prior unlock-lock pair. */
                nocb += rcu_future_gp_cleanup(rsp, rnp);
                raw_spin_unlock_irq(&rnp->lock);
-               cond_resched();
+               cond_resched_rcu_qs();
        }
        rnp = rcu_get_root(rsp);
        raw_spin_lock_irq(&rnp->lock);
@@ -1785,7 +1789,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
                        /* Locking provides needed memory barrier. */
                        if (rcu_gp_init(rsp))
                                break;
-                       cond_resched();
+                       cond_resched_rcu_qs();
                        flush_signals(current);
                        trace_rcu_grace_period(rsp->name,
                                               ACCESS_ONCE(rsp->gpnum),
@@ -1828,10 +1832,10 @@ static int __noreturn rcu_gp_kthread(void *arg)
                                trace_rcu_grace_period(rsp->name,
                                                       ACCESS_ONCE(rsp->gpnum),
                                                       TPS("fqsend"));
-                               cond_resched();
+                               cond_resched_rcu_qs();
                        } else {
                                /* Deal with stray signal. */
-                               cond_resched();
+                               cond_resched_rcu_qs();
                                flush_signals(current);
                                trace_rcu_grace_period(rsp->name,
                                                       ACCESS_ONCE(rsp->gpnum),
@@ -2393,8 +2397,8 @@ void rcu_check_callbacks(int cpu, int user)
                 * at least not while the corresponding CPU is online.
                 */
 
-               rcu_sched_qs(cpu);
-               rcu_bh_qs(cpu);
+               rcu_sched_qs();
+               rcu_bh_qs();
 
        } else if (!in_softirq()) {
 
@@ -2405,11 +2409,13 @@ void rcu_check_callbacks(int cpu, int user)
                 * critical section, so note it.
                 */
 
-               rcu_bh_qs(cpu);
+               rcu_bh_qs();
        }
        rcu_preempt_check_callbacks(cpu);
        if (rcu_pending(cpu))
                invoke_rcu_core();
+       if (user)
+               rcu_note_voluntary_context_switch(current);
        trace_rcu_utilization(TPS("End scheduler-tick"));
 }
 
@@ -2432,7 +2438,7 @@ static void force_qs_rnp(struct rcu_state *rsp,
        struct rcu_node *rnp;
 
        rcu_for_each_leaf_node(rsp, rnp) {
-               cond_resched();
+               cond_resched_rcu_qs();
                mask = 0;
                raw_spin_lock_irqsave(&rnp->lock, flags);
                smp_mb__after_unlock_lock();