hrtimer: Optimize the hrtimer code by using static keys for migration_enable/nohz_active

The hrtimer_cpu_base::migration_enable and ::nohz_active fields
were originally introduced to avoid accessing global variables
for these decisions.

Still that results in a (cache hot) load and conditional branch,
which can be avoided by using static keys.

Implement it with static keys and optimize for the most critical
case of high performance networking which tends to disable the
timer migration functionality.

No change in functionality.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Anna-Maria Gleixner <anna-maria@linutronix.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: keescook@chromium.org
Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1801142327490.2371@nanos
Link: https://lkml.kernel.org/r/20171221104205.7269-2-anna-maria@linutronix.de
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 012c37fdb688cb2c169530eb821c02abed2bcdd5..79b2a8d29d8cd4ca0fdd2524596fbac3b4400c30 100644 (file)
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -153,8 +153,6 @@ enum  hrtimer_base_type {
  * @cpu:               cpu number
  * @active_bases:      Bitfield to mark bases with active timers
  * @clock_was_set_seq: Sequence counter of clock was set events
- * @migration_enabled: The migration of hrtimers to other cpus is enabled
- * @nohz_active:       The nohz functionality is enabled
  * @expires_next:      absolute time of the next event which was scheduled
  *                     via clock_set_next_event()
  * @next_timer:                Pointer to the first expiring timer
@@ -178,8 +176,6 @@ struct hrtimer_cpu_base {
        unsigned int                    cpu;
        unsigned int                    active_bases;
        unsigned int                    clock_was_set_seq;
-       bool                            migration_enabled;
-       bool                            nohz_active;
 #ifdef CONFIG_HIGH_RES_TIMERS
        unsigned int                    in_hrtirq       : 1,
                                        hres_active     : 1,

diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index d32520840fde9b1d0e63a4be7d96e092ddd06acf..1d06d2bde73370cddb91ea5d8b3137145ec545ce 100644 (file)
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -178,23 +178,16 @@ hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
 #endif
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
-static inline
-struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
-                                        int pinned)
-{
-       if (pinned || !base->migration_enabled)
-               return base;
-       return &per_cpu(hrtimer_bases, get_nohz_timer_target());
-}
-#else
 static inline
 struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
                                         int pinned)
 {
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+       if (static_branch_likely(&timers_migration_enabled) && !pinned)
+               return &per_cpu(hrtimer_bases, get_nohz_timer_target());
+#endif
        return base;
 }
-#endif
 
 /*
  * We switch the timer base to a power-optimized selected CPU target,
@@ -969,7 +962,7 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
                 * Kick to reschedule the next tick to handle the new timer
                 * on dynticks target.
                 */
-               if (new_base->cpu_base->nohz_active)
+               if (is_timers_nohz_active())
                        wake_up_nohz_cpu(new_base->cpu_base->cpu);
        } else {
                hrtimer_reprogram(timer, new_base);

diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index f8e1845aa464b2c0b11dead0ad9baa37cae62822..f690628e068c56406dea1bbd6a8deb86bac87195 100644 (file)
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -150,14 +150,19 @@ static inline void tick_nohz_init(void) { }
 
 #ifdef CONFIG_NO_HZ_COMMON
 extern unsigned long tick_nohz_active;
-#else
+extern void timers_update_nohz(void);
+extern struct static_key_false timers_nohz_active;
+static inline bool is_timers_nohz_active(void)
+{
+       return static_branch_likely(&timers_nohz_active);
+}
+# ifdef CONFIG_SMP
+extern struct static_key_false timers_migration_enabled;
+# endif
+#else /* CONFIG_NO_HZ_COMMON */
+static inline void timers_update_nohz(void) { }
 #define tick_nohz_active (0)
-#endif
-
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-extern void timers_update_migration(bool update_nohz);
-#else
-static inline void timers_update_migration(bool update_nohz) { }
+static inline bool is_timers_nohz_active(void) { return false; }
 #endif
 
 DECLARE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases);

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f7cc7abfcf252f0cb6fadb3b9437b9fd096b8baf..29a5733eff83ec9e40432a2e9353bf19253d10b7 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1107,7 +1107,7 @@ static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
        ts->nohz_mode = mode;
        /* One update is enough */
        if (!test_and_set_bit(0, &tick_nohz_active))
-               timers_update_migration(true);
+               timers_update_nohz();
 }
 
 /**

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 0bcf00e3ce482b17392e30904decf86d459c4928..d530f72b32f9a37b0c005dccfe1b8e5740f37934 100644 (file)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -200,8 +200,6 @@ struct timer_base {
        unsigned long           clk;
        unsigned long           next_expiry;
        unsigned int            cpu;
-       bool                    migration_enabled;
-       bool                    nohz_active;
        bool                    is_idle;
        bool                    must_forward_clk;
        DECLARE_BITMAP(pending_map, WHEEL_SIZE);
@@ -210,45 +208,57 @@ struct timer_base {
 
 static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
 
-#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+#ifdef CONFIG_NO_HZ_COMMON
+
+DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
+static DEFINE_MUTEX(timer_keys_mutex);
+
+static void timer_update_keys(struct work_struct *work);
+static DECLARE_WORK(timer_update_work, timer_update_keys);
+
+#ifdef CONFIG_SMP
 unsigned int sysctl_timer_migration = 1;
 
-void timers_update_migration(bool update_nohz)
+DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
+
+static void timers_update_migration(void)
 {
-       bool on = sysctl_timer_migration && tick_nohz_active;
-       unsigned int cpu;
+       if (sysctl_timer_migration && tick_nohz_active)
+               static_branch_enable(&timers_migration_enabled);
+       else
+               static_branch_disable(&timers_migration_enabled);
+}
+#else
+static inline void timers_update_migration(void) { }
+#endif /* !CONFIG_SMP */
 
-       /* Avoid the loop, if nothing to update */
-       if (this_cpu_read(timer_bases[BASE_STD].migration_enabled) == on)
-               return;
+static void timer_update_keys(struct work_struct *work)
+{
+       mutex_lock(&timer_keys_mutex);
+       timers_update_migration();
+       static_branch_enable(&timers_nohz_active);
+       mutex_unlock(&timer_keys_mutex);
+}
 
-       for_each_possible_cpu(cpu) {
-               per_cpu(timer_bases[BASE_STD].migration_enabled, cpu) = on;
-               per_cpu(timer_bases[BASE_DEF].migration_enabled, cpu) = on;
-               per_cpu(hrtimer_bases.migration_enabled, cpu) = on;
-               if (!update_nohz)
-                       continue;
-               per_cpu(timer_bases[BASE_STD].nohz_active, cpu) = true;
-               per_cpu(timer_bases[BASE_DEF].nohz_active, cpu) = true;
-               per_cpu(hrtimer_bases.nohz_active, cpu) = true;
-       }
+void timers_update_nohz(void)
+{
+       schedule_work(&timer_update_work);
 }
 
 int timer_migration_handler(struct ctl_table *table, int write,
                            void __user *buffer, size_t *lenp,
                            loff_t *ppos)
 {
-       static DEFINE_MUTEX(mutex);
        int ret;
 
-       mutex_lock(&mutex);
+       mutex_lock(&timer_keys_mutex);
        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        if (!ret && write)
-               timers_update_migration(false);
-       mutex_unlock(&mutex);
+               timers_update_migration();
+       mutex_unlock(&timer_keys_mutex);
        return ret;
 }
-#endif
+#endif /* NO_HZ_COMMON */
 
 static unsigned long round_jiffies_common(unsigned long j, int cpu,
                bool force_up)
@@ -534,7 +544,7 @@ __internal_add_timer(struct timer_base *base, struct timer_list *timer)
 static void
 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
 {
-       if (!IS_ENABLED(CONFIG_NO_HZ_COMMON) || !base->nohz_active)
+       if (!is_timers_nohz_active())
                return;
 
        /*
@@ -849,21 +859,20 @@ static inline struct timer_base *get_timer_base(u32 tflags)
        return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
 static inline struct timer_base *
 get_target_base(struct timer_base *base, unsigned tflags)
 {
-#ifdef CONFIG_SMP
-       if ((tflags & TIMER_PINNED) || !base->migration_enabled)
-               return get_timer_this_cpu_base(tflags);
-       return get_timer_cpu_base(tflags, get_nohz_timer_target());
-#else
-       return get_timer_this_cpu_base(tflags);
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
+       if (static_branch_likely(&timers_migration_enabled) &&
+           !(tflags & TIMER_PINNED))
+               return get_timer_cpu_base(tflags, get_nohz_timer_target());
 #endif
+       return get_timer_this_cpu_base(tflags);
 }
 
 static inline void forward_timer_base(struct timer_base *base)
 {
+#ifdef CONFIG_NO_HZ_COMMON
        unsigned long jnow;
 
        /*
@@ -887,16 +896,8 @@ static inline void forward_timer_base(struct timer_base *base)
                base->clk = jnow;
        else
                base->clk = base->next_expiry;
-}
-#else
-static inline struct timer_base *
-get_target_base(struct timer_base *base, unsigned tflags)
-{
-       return get_timer_this_cpu_base(tflags);
-}
-
-static inline void forward_timer_base(struct timer_base *base) { }
 #endif
+}
 
 
 /*