x86: Make cpu_tss available to external modules

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / process.c

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 046e2d620bbe7be507808e0f7188c45249f2d69c..bfc99b3b6522078ba2aea707b24d6aa513e1bf7d 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -9,7 +9,7 @@
 #include <linux/sched.h>
 #include <linux/module.h>
 #include <linux/pm.h>
-#include <linux/clockchips.h>
+#include <linux/tick.h>
 #include <linux/random.h>
 #include <linux/user-return-notifier.h>
 #include <linux/dmi.h>
@@ -24,6 +24,7 @@
 #include <asm/syscalls.h>
 #include <asm/idle.h>
 #include <asm/uaccess.h>
+#include <asm/mwait.h>
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/debugreg.h>
@@ -37,7 +38,26 @@
  * section. Since TSS's are completely CPU-local, we want them
  * on exact cacheline boundaries, to eliminate cacheline ping-pong.
  */
-__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss) = INIT_TSS;
+__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
+       .x86_tss = {
+               .sp0 = TOP_OF_INIT_STACK,
+#ifdef CONFIG_X86_32
+               .ss0 = __KERNEL_DS,
+               .ss1 = __KERNEL_CS,
+               .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
+#endif
+        },
+#ifdef CONFIG_X86_32
+        /*
+         * Note that the .io_bitmap member must be extra-big. This is because
+         * the CPU will access an additional byte beyond the end of the IO
+         * permission bitmap. The extra byte must be all 1 bits, and must
+         * be within the limit.
+         */
+       .io_bitmap              = { [0 ... IO_BITMAP_LONGS] = ~0 },
+#endif
+};
+EXPORT_PER_CPU_SYMBOL(cpu_tss);
 
 #ifdef CONFIG_X86_64
 static DEFINE_PER_CPU(unsigned char, is_idle);
@@ -69,8 +89,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 
        dst->thread.fpu_counter = 0;
        dst->thread.fpu.has_fpu = 0;
-       dst->thread.fpu.last_cpu = ~0;
        dst->thread.fpu.state = NULL;
+       task_disable_lazy_fpu_restore(dst);
        if (tsk_used_math(src)) {
                int err = fpu_alloc(&dst->thread.fpu);
                if (err)
@@ -109,7 +129,7 @@ void exit_thread(void)
        unsigned long *bp = t->io_bitmap_ptr;
 
        if (bp) {
-               struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
+               struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
 
                t->io_bitmap_ptr = NULL;
                clear_thread_flag(TIF_IO_BITMAP);
@@ -131,13 +151,18 @@ void flush_thread(void)
 
        flush_ptrace_hw_breakpoint(tsk);
        memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
-       drop_init_fpu(tsk);
-       /*
-        * Free the FPU state for non xsave platforms. They get reallocated
-        * lazily at the first use.
-        */
-       if (!use_eager_fpu())
+
+       if (!use_eager_fpu()) {
+               /* FPU state will be reallocated lazily at the first use. */
+               drop_fpu(tsk);
                free_thread_xstate(tsk);
+       } else if (!used_math()) {
+               /* kthread execs. TODO: cleanup this horror. */
+               if (WARN_ON(init_fpu(tsk)))
+                       force_sig(SIGKILL, tsk);
+               user_fpu_begin();
+               restore_init_xstate();
+       }
 }
 
 static void hard_disable_TSC(void)
@@ -377,14 +402,11 @@ static void amd_e400_idle(void)
 
                if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
                        cpumask_set_cpu(cpu, amd_e400_c1e_mask);
-                       /*
-                        * Force broadcast so ACPI can not interfere.
-                        */
-                       clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE,
-                                          &cpu);
+                       /* Force broadcast so ACPI can not interfere. */
+                       tick_broadcast_force();
                        pr_info("Switch to broadcast mode on CPU%d\n", cpu);
                }
-               clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
+               tick_broadcast_enter();
 
                default_idle();
 
@@ -393,12 +415,59 @@ static void amd_e400_idle(void)
                 * called with interrupts disabled.
                 */
                local_irq_disable();
-               clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
+               tick_broadcast_exit();
                local_irq_enable();
        } else
                default_idle();
 }
 
+/*
+ * Intel Core2 and older machines prefer MWAIT over HALT for C1.
+ * We can't rely on cpuidle installing MWAIT, because it will not load
+ * on systems that support only C1 -- so the boot default must be MWAIT.
+ *
+ * Some AMD machines are the opposite, they depend on using HALT.
+ *
+ * So for default C1, which is used during boot until cpuidle loads,
+ * use MWAIT-C1 on Intel HW that has it, else use HALT.
+ */
+static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
+{
+       if (c->x86_vendor != X86_VENDOR_INTEL)
+               return 0;
+
+       if (!cpu_has(c, X86_FEATURE_MWAIT))
+               return 0;
+
+       return 1;
+}
+
+/*
+ * MONITOR/MWAIT with no hints, used for default default C1 state.
+ * This invokes MWAIT with interrutps enabled and no flags,
+ * which is backwards compatible with the original MWAIT implementation.
+ */
+
+static void mwait_idle(void)
+{
+       if (!current_set_polling_and_test()) {
+               if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
+                       smp_mb(); /* quirk */
+                       clflush((void *)&current_thread_info()->flags);
+                       smp_mb(); /* quirk */
+               }
+
+               __monitor((void *)&current_thread_info()->flags, 0, 0);
+               if (!need_resched())
+                       __sti_mwait(0, 0);
+               else
+                       local_irq_enable();
+       } else {
+               local_irq_enable();
+       }
+       __current_clr_polling();
+}
+
 void select_idle_routine(const struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
@@ -412,6 +481,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
                /* E400: APIC timer interrupt does not wake up CPU from C1e */
                pr_info("using AMD E400 aware idle routine\n");
                x86_idle = amd_e400_idle;
+       } else if (prefer_mwait_c1_over_halt(c)) {
+               pr_info("using mwait in idle threads\n");
+               x86_idle = mwait_idle;
        } else
                x86_idle = default_idle;
 }