Merge git://git.kernel.org/pub/scm/linux/kernel/git/nab/target-pending

[mirror_ubuntu-artful-kernel.git] / kernel / rcu / tree_plugin.h

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 0a62a8f1caacfab8a2a39c729f70e445f8048cbf..c9a48657512ae35a833d6cc1c56ff9687a899b17 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1350,10 +1350,10 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
                 */
                if ((rdp->completed != rnp->completed ||
                     unlikely(READ_ONCE(rdp->gpwrap))) &&
-                   rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
+                   rcu_segcblist_pend_cbs(&rdp->cblist))
                        note_gp_changes(rsp, rdp);
 
-               if (cpu_has_callbacks_ready_to_invoke(rdp))
+               if (rcu_segcblist_ready_cbs(&rdp->cblist))
                        cbs_ready = true;
        }
        return cbs_ready;
@@ -1461,7 +1461,7 @@ static void rcu_prepare_for_idle(void)
        rdtp->last_accelerate = jiffies;
        for_each_rcu_flavor(rsp) {
                rdp = this_cpu_ptr(rsp->rda);
-               if (!*rdp->nxttail[RCU_DONE_TAIL])
+               if (rcu_segcblist_pend_cbs(&rdp->cblist))
                        continue;
                rnp = rdp->mynode;
                raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
@@ -1529,7 +1529,7 @@ static void rcu_oom_notify_cpu(void *unused)
 
        for_each_rcu_flavor(rsp) {
                rdp = raw_cpu_ptr(rsp->rda);
-               if (rdp->qlen_lazy != 0) {
+               if (rcu_segcblist_n_lazy_cbs(&rdp->cblist)) {
                        atomic_inc(&oom_callback_count);
                        rsp->call(&rdp->oom_head, rcu_oom_callback);
                }
@@ -1709,7 +1709,7 @@ __setup("rcu_nocbs=", rcu_nocb_setup);
 
 static int __init parse_rcu_nocb_poll(char *arg)
 {
-       rcu_nocb_poll = 1;
+       rcu_nocb_poll = true;
        return 0;
 }
 early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
@@ -1860,7 +1860,9 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeEmpty"));
                } else {
-                       rdp->nocb_defer_wakeup = RCU_NOGP_WAKE;
+                       WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE);
+                       /* Store ->nocb_defer_wakeup before ->rcu_urgent_qs. */
+                       smp_store_release(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs), true);
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeEmptyIsDeferred"));
                }
@@ -1872,7 +1874,9 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeOvf"));
                } else {
-                       rdp->nocb_defer_wakeup = RCU_NOGP_WAKE_FORCE;
+                       WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOGP_WAKE_FORCE);
+                       /* Store ->nocb_defer_wakeup before ->rcu_urgent_qs. */
+                       smp_store_release(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs), true);
                        trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,
                                            TPS("WakeOvfIsDeferred"));
                }
@@ -1930,30 +1934,26 @@ static bool __maybe_unused rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
                                                     struct rcu_data *rdp,
                                                     unsigned long flags)
 {
-       long ql = rsp->qlen;
-       long qll = rsp->qlen_lazy;
+       long ql = rsp->orphan_done.len;
+       long qll = rsp->orphan_done.len_lazy;
 
        /* If this is not a no-CBs CPU, tell the caller to do it the old way. */
        if (!rcu_is_nocb_cpu(smp_processor_id()))
                return false;
-       rsp->qlen = 0;
-       rsp->qlen_lazy = 0;
 
        /* First, enqueue the donelist, if any.  This preserves CB ordering. */
-       if (rsp->orphan_donelist != NULL) {
-               __call_rcu_nocb_enqueue(rdp, rsp->orphan_donelist,
-                                       rsp->orphan_donetail, ql, qll, flags);
-               ql = qll = 0;
-               rsp->orphan_donelist = NULL;
-               rsp->orphan_donetail = &rsp->orphan_donelist;
+       if (rsp->orphan_done.head) {
+               __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_done),
+                                       rcu_cblist_tail(&rsp->orphan_done),
+                                       ql, qll, flags);
        }
-       if (rsp->orphan_nxtlist != NULL) {
-               __call_rcu_nocb_enqueue(rdp, rsp->orphan_nxtlist,
-                                       rsp->orphan_nxttail, ql, qll, flags);
-               ql = qll = 0;
-               rsp->orphan_nxtlist = NULL;
-               rsp->orphan_nxttail = &rsp->orphan_nxtlist;
+       if (rsp->orphan_pend.head) {
+               __call_rcu_nocb_enqueue(rdp, rcu_cblist_head(&rsp->orphan_pend),
+                                       rcu_cblist_tail(&rsp->orphan_pend),
+                                       ql, qll, flags);
        }
+       rcu_cblist_init(&rsp->orphan_done);
+       rcu_cblist_init(&rsp->orphan_pend);
        return true;
 }
 
@@ -2395,16 +2395,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
                return false;
 
        /* If there are early-boot callbacks, move them to nocb lists. */
-       if (rdp->nxtlist) {
-               rdp->nocb_head = rdp->nxtlist;
-               rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
-               atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
-               atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
-               rdp->nxtlist = NULL;
-               rdp->qlen = 0;
-               rdp->qlen_lazy = 0;
+       if (!rcu_segcblist_empty(&rdp->cblist)) {
+               rdp->nocb_head = rcu_segcblist_head(&rdp->cblist);
+               rdp->nocb_tail = rcu_segcblist_tail(&rdp->cblist);
+               atomic_long_set(&rdp->nocb_q_count,
+                               rcu_segcblist_n_cbs(&rdp->cblist));
+               atomic_long_set(&rdp->nocb_q_count_lazy,
+                               rcu_segcblist_n_lazy_cbs(&rdp->cblist));
+               rcu_segcblist_init(&rdp->cblist);
        }
-       rdp->nxttail[RCU_NEXT_TAIL] = NULL;
+       rcu_segcblist_disable(&rdp->cblist);
        return true;
 }