x86/fpu: Rename fpu-internal.h to fpu/internal.h

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / process_64.c

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 4baaa972f52aaed15b3dddd94b4df4f653126d81..c50e013b57d20fe34b58932ce57dc671241e1435 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -38,8 +38,7 @@
 
 #include <asm/pgtable.h>
 #include <asm/processor.h>
-#include <asm/i387.h>
-#include <asm/fpu-internal.h>
+#include <asm/fpu/internal.h>
 #include <asm/mmu_context.h>
 #include <asm/prctl.h>
 #include <asm/desc.h>
@@ -274,12 +273,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
        struct thread_struct *prev = &prev_p->thread;
        struct thread_struct *next = &next_p->thread;
+       struct fpu *prev_fpu = &prev->fpu;
+       struct fpu *next_fpu = &next->fpu;
        int cpu = smp_processor_id();
        struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
        unsigned fsindex, gsindex;
-       fpu_switch_t fpu;
+       fpu_switch_t fpu_switch;
 
-       fpu = switch_fpu_prepare(prev_p, next_p, cpu);
+       fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
 
        /* We must save %fs and %gs before load_TLS() because
         * %fs and %gs may be cleared by load_TLS().
@@ -299,7 +300,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         * Leave lazy mode, flushing any hypercalls made here.  This
         * must be done after loading TLS entries in the GDT but before
         * loading segments that might reference them, and and it must
-        * be done before math_state_restore, so the TS bit is up to
+        * be done before fpu__restore(), so the TS bit is up to
         * date.
         */
        arch_end_context_switch(next_p);
@@ -391,7 +392,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
        prev->gsindex = gsindex;
 
-       switch_fpu_finish(next_p, fpu);
+       switch_fpu_finish(next_fpu, fpu_switch);
 
        /*
         * Switch the PDA and FPU contexts.
@@ -419,6 +420,34 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                     task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
                __switch_to_xtra(prev_p, next_p, tss);
 
+       if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
+               /*
+                * AMD CPUs have a misfeature: SYSRET sets the SS selector but
+                * does not update the cached descriptor.  As a result, if we
+                * do SYSRET while SS is NULL, we'll end up in user mode with
+                * SS apparently equal to __USER_DS but actually unusable.
+                *
+                * The straightforward workaround would be to fix it up just
+                * before SYSRET, but that would slow down the system call
+                * fast paths.  Instead, we ensure that SS is never NULL in
+                * system call context.  We do this by replacing NULL SS
+                * selectors at every context switch.  SYSCALL sets up a valid
+                * SS, so the only way to get NULL is to re-enter the kernel
+                * from CPL 3 through an interrupt.  Since that can't happen
+                * in the same task as a running syscall, we are guaranteed to
+                * context switch between every interrupt vector entry and a
+                * subsequent SYSRET.
+                *
+                * We read SS first because SS reads are much faster than
+                * writes.  Out of caution, we force SS to __KERNEL_DS even if
+                * it previously had a different non-NULL value.
+                */
+               unsigned short ss_sel;
+               savesegment(ss, ss_sel);
+               if (ss_sel != __KERNEL_DS)
+                       loadsegment(ss, __KERNEL_DS);
+       }
+
        return prev_p;
 }