arm64: entry: Make the trampoline cleanup optional

commit d739da1694a0eaef0358a42b76904b611539b77b upstream.

Subsequent patches will add additional sets of vectors that use
the same tricks as the kpti vectors to reach the full-fat vectors.
The full-fat vectors contain some cleanup for kpti that is patched
in by alternatives when kpti is in use. Once there are additional
vectors, the cleanup will be needed in more cases.

But on big/little systems, the cleanup would be harmful if no
trampoline vector were in use. Instead of forcing CPUs that don't
need a trampoline vector to use one, make the trampoline cleanup
optional.

Entry at the top of the vectors will skip the cleanup. The trampoline
vectors can then skip the first instruction, triggering the cleanup
to run.

Reviewed-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
CVE-2022-23960
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@canonical.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 0a0347859fd59e5374d31c4c71f766f9d2adbae4..180bfd2f38285bd3f3a82861811d92f0927b4f05 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -40,14 +40,18 @@
 .Lventry_start\@:
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
        .if     \el == 0
-alternative_if ARM64_UNMAP_KERNEL_AT_EL0
+       /*
+        * This must be the first instruction of the EL0 vector entries. It is
+        * skipped by the trampoline vectors, to trigger the cleanup.
+        */
+       b       .Lskip_tramp_vectors_cleanup\@
        .if     \regsize == 64
        mrs     x30, tpidrro_el0
        msr     tpidrro_el0, xzr
        .else
        mov     x30, xzr
        .endif
-alternative_else_nop_endif
+.Lskip_tramp_vectors_cleanup\@:
        .endif
 #endif
 
@@ -661,7 +665,7 @@ alternative_if_not ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM
        prfm    plil1strm, [x30, #(1b - tramp_vectors)]
 alternative_else_nop_endif
        msr     vbar_el1, x30
-       add     x30, x30, #(1b - tramp_vectors)
+       add     x30, x30, #(1b - tramp_vectors + 4)
        isb
        ret
 .org 1b + 128  // Did we overflow the ventry slot?