arm64: Implement branch predictor hardening for affected Cortex-A CPUs

Commit aa6acde65e03 upstream.

Cortex-A57, A72, A73 and A75 are susceptible to branch predictor aliasing
and can theoretically be attacked by malicious code.

This patch implements a PSCI-based mitigation for these CPUs when available.
The call into firmware will invalidate the branch predictor state, preventing
any malicious entries from affecting other victim contexts.

Co-developed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
(cherry picked from commit 48993dfa1af8c719576a18c0e2ca1d611297e34e)

CVE-2017-5753
CVE-2017-5715
CVE-2017-5754

Signed-off-by: Paolo Pisati <paolo.pisati@canonical.com>
Acked-by: Brad Figg <brad.figg@canonical.com>
Acked-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/arch/arm64/kernel/bpi.S b/arch/arm64/kernel/bpi.S
index 06a931eb26737a7e6190b7f8b4843c657ba6fe3e..dec95bd82e319e9fe3e701d602d532f1b2508227 100644 (file)
--- a/arch/arm64/kernel/bpi.S
+++ b/arch/arm64/kernel/bpi.S
@@ -53,3 +53,27 @@ ENTRY(__bp_harden_hyp_vecs_start)
        vectors __kvm_hyp_vector
        .endr
 ENTRY(__bp_harden_hyp_vecs_end)
+ENTRY(__psci_hyp_bp_inval_start)
+       sub     sp, sp, #(8 * 18)
+       stp     x16, x17, [sp, #(16 * 0)]
+       stp     x14, x15, [sp, #(16 * 1)]
+       stp     x12, x13, [sp, #(16 * 2)]
+       stp     x10, x11, [sp, #(16 * 3)]
+       stp     x8, x9, [sp, #(16 * 4)]
+       stp     x6, x7, [sp, #(16 * 5)]
+       stp     x4, x5, [sp, #(16 * 6)]
+       stp     x2, x3, [sp, #(16 * 7)]
+       stp     x0, x1, [sp, #(16 * 8)]
+       mov     x0, #0x84000000
+       smc     #0
+       ldp     x16, x17, [sp, #(16 * 0)]
+       ldp     x14, x15, [sp, #(16 * 1)]
+       ldp     x12, x13, [sp, #(16 * 2)]
+       ldp     x10, x11, [sp, #(16 * 3)]
+       ldp     x8, x9, [sp, #(16 * 4)]
+       ldp     x6, x7, [sp, #(16 * 5)]
+       ldp     x4, x5, [sp, #(16 * 6)]
+       ldp     x2, x3, [sp, #(16 * 7)]
+       ldp     x0, x1, [sp, #(16 * 8)]
+       add     sp, sp, #(8 * 18)
+ENTRY(__psci_hyp_bp_inval_end)

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 3f2fee9d4590ef04ecb7214382d13addd79e6c59..e09027094f318bfbb7262f6e2d63e504127658dc 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -67,6 +67,8 @@ static int cpu_enable_trap_ctr_access(void *__unused)
 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
 
 #ifdef CONFIG_KVM
+extern char __psci_hyp_bp_inval_start[], __psci_hyp_bp_inval_end[];
+
 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
                                const char *hyp_vecs_end)
 {
@@ -108,6 +110,9 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
        spin_unlock(&bp_lock);
 }
 #else
+#define __psci_hyp_bp_inval_start      NULL
+#define __psci_hyp_bp_inval_end                NULL
+
 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
                                      const char *hyp_vecs_start,
                                      const char *hyp_vecs_end)
@@ -132,6 +137,21 @@ static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
 
        __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
 }
+
+#include <linux/psci.h>
+
+static int enable_psci_bp_hardening(void *data)
+{
+       const struct arm64_cpu_capabilities *entry = data;
+
+       if (psci_ops.get_version)
+               install_bp_hardening_cb(entry,
+                                      (bp_hardening_cb_t)psci_ops.get_version,
+                                      __psci_hyp_bp_inval_start,
+                                      __psci_hyp_bp_inval_end);
+
+       return 0;
+}
 #endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */
 
 #define MIDR_RANGE(model, min, max) \
@@ -281,6 +301,28 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                .capability = ARM64_WORKAROUND_858921,
                MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
        },
+#endif
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+       {
+               .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+               MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
+               .enable = enable_psci_bp_hardening,
+       },
+       {
+               .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+               MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
+               .enable = enable_psci_bp_hardening,
+       },
+       {
+               .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+               MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
+               .enable = enable_psci_bp_hardening,
+       },
+       {
+               .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+               MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
+               .enable = enable_psci_bp_hardening,
+       },
 #endif
        {
        }