entry: Explicitly flush pending rcuog wakeup before last rescheduling point

Following the idle loop model, cleanly check for pending rcuog wakeup
before the last rescheduling point on resuming to user mode. This
way we can avoid to do it from rcu_user_enter() with the last resort
self-IPI hack that enforces rescheduling.

Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: stable@vger.kernel.org
Link: https://lkml.kernel.org/r/20210131230548.32970-5-frederic@kernel.org

diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index f09cae37ddd5ef617a3a1e27e37b2c9983982799..8442e5c9cfa269a6b0fab833a26c09d3970ca889 100644 (file)
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -184,6 +184,10 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
                 * enabled above.
                 */
                local_irq_disable_exit_to_user();
+
+               /* Check if any of the above work has queued a deferred wakeup */
+               rcu_nocb_flush_deferred_wakeup();
+
                ti_work = READ_ONCE(current_thread_info()->flags);
        }
 
@@ -197,6 +201,9 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs)
 
        lockdep_assert_irqs_disabled();
 
+       /* Flush pending rcuog wakeup before the last need_resched() check */
+       rcu_nocb_flush_deferred_wakeup();
+
        if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
                ti_work = exit_to_user_mode_loop(regs, ti_work);
 

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 4b1e5bd1649286bb893a4df9c63e6279c08bdae8..2ebc211fffcb9e5aa83a0e4b31259548e263852f 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -707,13 +707,15 @@ noinstr void rcu_user_enter(void)
        lockdep_assert_irqs_disabled();
 
        /*
-        * We may be past the last rescheduling opportunity in the entry code.
-        * Trigger a self IPI that will fire and reschedule once we resume to
-        * user/guest mode.
+        * Other than generic entry implementation, we may be past the last
+        * rescheduling opportunity in the entry code. Trigger a self IPI
+        * that will fire and reschedule once we resume in user/guest mode.
         */
        instrumentation_begin();
-       if (do_nocb_deferred_wakeup(rdp) && need_resched())
-               irq_work_queue(this_cpu_ptr(&late_wakeup_work));
+       if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) {
+               if (do_nocb_deferred_wakeup(rdp) && need_resched())
+                       irq_work_queue(this_cpu_ptr(&late_wakeup_work));
+       }
        instrumentation_end();
 
        rcu_eqs_enter(true);