arm64: entry: Hook up entry trampoline to exception vectors

Commit 4bf3286d29f3 upstream.

Hook up the entry trampoline to our exception vectors so that all
exceptions from and returns to EL0 go via the trampoline, which swizzles
the vector base register accordingly. Transitioning to and from the
kernel clobbers x30, so we use tpidrro_el0 and far_el1 as scratch
registers for native tasks.

Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Tested-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
(cherry picked from commit 6472f1a3a54f061864a5091788778793661aa7c0)

CVE-2017-5753
CVE-2017-5715
CVE-2017-5754

Signed-off-by: Paolo Pisati <paolo.pisati@canonical.com>
Acked-by: Brad Figg <brad.figg@canonical.com>
Acked-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 9b53f5a5c4106574e3f2b55eb82edeff3d2318ec..0210893435dd81c652848948c37f40bb560041c8 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -73,6 +73,17 @@
 
        .macro kernel_ventry, el, label, regsize = 64
        .align 7
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+       .if     \el == 0
+       .if     \regsize == 64
+       mrs     x30, tpidrro_el0
+       msr     tpidrro_el0, xzr
+       .else
+       mov     x30, xzr
+       .endif
+       .endif
+#endif
+
        sub     sp, sp, #S_FRAME_SIZE
 #ifdef CONFIG_VMAP_STACK
        /*
@@ -119,6 +130,11 @@
        b       el\()\el\()_\label
        .endm
 
+       .macro tramp_alias, dst, sym
+       mov_q   \dst, TRAMP_VALIAS
+       add     \dst, \dst, #(\sym - .entry.tramp.text)
+       .endm
+
        .macro  kernel_entry, el, regsize = 64
        .if     \regsize == 32
        mov     w0, w0                          // zero upper 32 bits of x0
@@ -271,18 +287,20 @@ alternative_else_nop_endif
        .if     \el == 0
        ldr     x23, [sp, #S_SP]                // load return stack pointer
        msr     sp_el0, x23
+       tst     x22, #PSR_MODE32_BIT            // native task?
+       b.eq    3f
+
 #ifdef CONFIG_ARM64_ERRATUM_845719
 alternative_if ARM64_WORKAROUND_845719
-       tbz     x22, #4, 1f
 #ifdef CONFIG_PID_IN_CONTEXTIDR
        mrs     x29, contextidr_el1
        msr     contextidr_el1, x29
 #else
        msr contextidr_el1, xzr
 #endif
-1:
 alternative_else_nop_endif
 #endif
+3:
        .endif
 
        msr     elr_el1, x21                    // set up the return data
@@ -304,7 +322,22 @@ alternative_else_nop_endif
        ldp     x28, x29, [sp, #16 * 14]
        ldr     lr, [sp, #S_LR]
        add     sp, sp, #S_FRAME_SIZE           // restore sp
-       eret                                    // return to kernel
+
+#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
+       eret
+#else
+       .if     \el == 0
+       bne     4f
+       msr     far_el1, x30
+       tramp_alias     x30, tramp_exit_native
+       br      x30
+4:
+       tramp_alias     x30, tramp_exit_compat
+       br      x30
+       .else
+       eret
+       .endif
+#endif
        .endm
 
        .macro  irq_stack_entry