timekeeping: Split jiffies seqlock

[mirror_ubuntu-jammy-kernel.git] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 8b192e67aabc9d16d8a7b08c0356641b5462c783..4be756b88a48e894e0cbf4dbe5480efa314a3e86 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -58,20 +58,23 @@ static void tick_do_update_jiffies64(ktime_t now)
 
        /*
         * Do a quick check without holding jiffies_lock:
+        * The READ_ONCE() pairs with two updates done later in this function.
         */
-       delta = ktime_sub(now, last_jiffies_update);
+       delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
        if (delta < tick_period)
                return;
 
        /* Reevaluate with jiffies_lock held */
-       write_seqlock(&jiffies_lock);
+       raw_spin_lock(&jiffies_lock);
+       write_seqcount_begin(&jiffies_seq);
 
        delta = ktime_sub(now, last_jiffies_update);
        if (delta >= tick_period) {
 
                delta = ktime_sub(delta, tick_period);
-               last_jiffies_update = ktime_add(last_jiffies_update,
-                                               tick_period);
+               /* Pairs with the lockless read in this function. */
+               WRITE_ONCE(last_jiffies_update,
+                          ktime_add(last_jiffies_update, tick_period));
 
                /* Slow path for long timeouts */
                if (unlikely(delta >= tick_period)) {
@@ -79,18 +82,22 @@ static void tick_do_update_jiffies64(ktime_t now)
 
                        ticks = ktime_divns(delta, incr);
 
-                       last_jiffies_update = ktime_add_ns(last_jiffies_update,
-                                                          incr * ticks);
+                       /* Pairs with the lockless read in this function. */
+                       WRITE_ONCE(last_jiffies_update,
+                                  ktime_add_ns(last_jiffies_update,
+                                               incr * ticks));
                }
                do_timer(++ticks);
 
                /* Keep the tick_next_period variable up to date */
                tick_next_period = ktime_add(last_jiffies_update, tick_period);
        } else {
-               write_sequnlock(&jiffies_lock);
+               write_seqcount_end(&jiffies_seq);
+               raw_spin_unlock(&jiffies_lock);
                return;
        }
-       write_sequnlock(&jiffies_lock);
+       write_seqcount_end(&jiffies_seq);
+       raw_spin_unlock(&jiffies_lock);
        update_wall_time();
 }
 
@@ -101,12 +108,14 @@ static ktime_t tick_init_jiffy_update(void)
 {
        ktime_t period;
 
-       write_seqlock(&jiffies_lock);
+       raw_spin_lock(&jiffies_lock);
+       write_seqcount_begin(&jiffies_seq);
        /* Did we start the jiffies update yet ? */
        if (last_jiffies_update == 0)
                last_jiffies_update = tick_next_period;
        period = last_jiffies_update;
-       write_sequnlock(&jiffies_lock);
+       write_seqcount_end(&jiffies_seq);
+       raw_spin_unlock(&jiffies_lock);
        return period;
 }
 
@@ -672,10 +681,10 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 
        /* Read jiffies and the time when jiffies were updated last */
        do {
-               seq = read_seqbegin(&jiffies_lock);
+               seq = read_seqcount_begin(&jiffies_seq);
                basemono = last_jiffies_update;
                basejiff = jiffies;
-       } while (read_seqretry(&jiffies_lock, seq));
+       } while (read_seqcount_retry(&jiffies_seq, seq));
        ts->last_jiffies = basejiff;
        ts->timer_expires_base = basemono;