x86/enter: Use IBRS on syscall and interrupts

CVE-2017-5715 (Spectre v2 Intel)

Set IBRS upon kernel entrance via syscall and interrupts. Clear it upon exit.

Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: Andy Whitcroft <apw@canonical.com>
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Signed-off-by: Andy Whitcroft <apw@canonical.com>
Acked-by: Colin Ian King <colin.king@canonical.com>
Acked-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 82b1bf92a65065552e2e4241bac52106d2e5f885..f89e813a4de3625c3e270b5eb8856441fb3d01ac 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -37,6 +37,7 @@
 #include <asm/export.h>
 #include <asm/frame.h>
 #include <asm/nospec-branch.h>
+#include <asm/spec_ctrl.h>
 #include <linux/err.h>
 
 #include "calling.h"
@@ -236,6 +237,8 @@ GLOBAL(entry_SYSCALL_64_after_hwframe)
        sub     $(6*8), %rsp                    /* pt_regs->bp, bx, r12-15 not saved */
        UNWIND_HINT_REGS extra=0
 
+       ENABLE_IBRS
+
        /*
         * If we need to do entry work or if we guess we'll need to do
         * exit work, go straight to the slow path.
@@ -292,6 +295,7 @@ entry_SYSCALL_64_fastpath:
        TRACE_IRQS_ON           /* user mode is traced as IRQs on */
        movq    RIP(%rsp), %rcx
        movq    EFLAGS(%rsp), %r11
+       DISABLE_IBRS
        addq    $6*8, %rsp      /* skip extra regs -- they were preserved */
        UNWIND_HINT_EMPTY
        jmp     .Lpop_c_regs_except_rcx_r11_and_sysret
@@ -385,6 +389,8 @@ return_from_SYSCALL_64:
         * perf profiles. Nothing jumps here.
         */
 syscall_return_via_sysret:
+       DISABLE_IBRS
+
        /* rcx and r11 are already restored (see code above) */
        UNWIND_HINT_EMPTY
        POP_EXTRA_REGS
@@ -678,6 +684,10 @@ END(irq_entries_start)
        /*
         * IRQ from user mode.
         *
+        */
+       ENABLE_IBRS
+
+       /*
         * We need to tell lockdep that IRQs are off.  We can't do this until
         * we fix gsbase, and we should do it before enter_from_user_mode
         * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
@@ -761,7 +771,7 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode)
         * We are on the trampoline stack.  All regs except RDI are live.
         * We can do future final exit work right here.
         */
-
+       DISABLE_IBRS
        SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
 
        /* Restore RDI. */
@@ -1295,6 +1305,7 @@ ENTRY(paranoid_entry)
 
 1:
        SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
+       ENABLE_IBRS_CLOBBER
 
        ret
 END(paranoid_entry)
@@ -1349,6 +1360,8 @@ ENTRY(error_entry)
        /* We have user CR3.  Change to kernel CR3. */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 
+       ENABLE_IBRS
+
 .Lerror_entry_from_usermode_after_swapgs:
        /* Put us onto the real thread stack. */
        popq    %r12                            /* save return addr in %12 */
@@ -1395,6 +1408,7 @@ ENTRY(error_entry)
         */
        SWAPGS
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
+       ENABLE_IBRS_CLOBBER
        jmp .Lerror_entry_done
 
 .Lbstep_iret:
@@ -1409,6 +1423,7 @@ ENTRY(error_entry)
         */
        SWAPGS
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
+       ENABLE_IBRS_CLOBBER
 
        /*
         * Pretend that the exception came from user mode: set up pt_regs
@@ -1536,6 +1551,7 @@ ENTRY(nmi)
        UNWIND_HINT_REGS
        ENCODE_FRAME_POINTER
 
+       ENABLE_IBRS
        /*
         * At this point we no longer need to worry about stack damage
         * due to nesting -- we're on the normal thread stack and we're

diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 2b5e7685823c59421f27dc0f9ab601d3efc0609f..ee4f3edb3c50255d23f986503d3023e56684c8d9 100644 (file)
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -13,6 +13,7 @@
 #include <asm/irqflags.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
+#include <asm/spec_ctrl.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -95,6 +96,8 @@ ENTRY(entry_SYSENTER_compat)
        pushq   $0                      /* pt_regs->r15 = 0 */
        cld
 
+       ENABLE_IBRS
+
        /*
         * SYSENTER doesn't filter flags, so we need to clear NT and AC
         * ourselves.  To save a few cycles, we can check whether
@@ -194,6 +197,7 @@ ENTRY(entry_SYSCALL_compat)
 
        /* Use %rsp as scratch reg. User ESP is stashed in r8 */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
+       ENABLE_IBRS
 
        /* Switch to the kernel stack */
        movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp
@@ -249,6 +253,7 @@ sysret32_from_system_call:
        popq    %rsi                    /* pt_regs->si */
        popq    %rdi                    /* pt_regs->di */
 
+       DISABLE_IBRS
         /*
          * USERGS_SYSRET32 does:
          *  GSBASE = user's GS base
@@ -348,6 +353,8 @@ ENTRY(entry_INT80_compat)
        pushq   %r15                    /* pt_regs->r15 */
        cld
 
+       ENABLE_IBRS
+
        /*
         * User mode is traced as though IRQs are on, and the interrupt
         * gate turned them off.