arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3

BugLink: https://bugs.launchpad.net/bugs/1851876
[ Upstream commit 8f04e8e6e29c93421a95b61cad62e3918425eac7 ]

On CPUs with support for PSTATE.SSBS, the kernel can toggle the SSBD
state without needing to call into firmware.

This patch hooks into the existing SSBD infrastructure so that SSBS is
used on CPUs that support it, but it's all made horribly complicated by
the very real possibility of big/little systems that don't uniformly
provide the new capability.

Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
[ardb: add #include of asm/compat.h]
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 4f6ea704a8a0824a5f38ba136e4b3f20c9914e99..4a05e0a36d0be40725f1d52e9dfd122a8e39942a 100644 (file)
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -156,6 +156,10 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc,
 {
        start_thread_common(regs, pc);
        regs->pstate = PSR_MODE_EL0t;
+
+       if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
+               regs->pstate |= PSR_SSBS_BIT;
+
        regs->sp = sp;
 }
 
@@ -172,6 +176,9 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
        regs->pstate |= COMPAT_PSR_E_BIT;
 #endif
 
+       if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
+               regs->pstate |= PSR_AA32_SSBS_BIT;
+
        regs->compat_sp = sp;
 }
 #endif

diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 1b2a253de6a1556d4682dc19f26cec1b15604f75..b466d763a90d9dbe16ded05a1772b2de46b5ac4e 100644 (file)
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -50,6 +50,7 @@
 #define PSR_AA32_I_BIT         0x00000080
 #define PSR_AA32_A_BIT         0x00000100
 #define PSR_AA32_E_BIT         0x00000200
+#define PSR_AA32_SSBS_BIT      0x00800000
 #define PSR_AA32_DIT_BIT       0x01000000
 #define PSR_AA32_Q_BIT         0x08000000
 #define PSR_AA32_V_BIT         0x10000000

diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 1b9f773899f725260dad5f146c4f3d2de49840a9..96b443703b8997b6aae5a1b597c05963370a1d48 100644 (file)
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -86,11 +86,14 @@
 
 #define REG_PSTATE_PAN_IMM             sys_reg(0, 0, 4, 0, 4)
 #define REG_PSTATE_UAO_IMM             sys_reg(0, 0, 4, 0, 3)
+#define REG_PSTATE_SSBS_IMM            sys_reg(0, 3, 4, 0, 1)
 
 #define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM |        \
                                      (!!x)<<8 | 0x1f)
 #define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM |        \
                                      (!!x)<<8 | 0x1f)
+#define SET_PSTATE_SSBS(x) __emit_inst(0xd5000000 | REG_PSTATE_SSBS_IMM | \
+                                      (!!x)<<8 | 0x1f)
 
 #define SYS_DC_ISW                     sys_insn(1, 0, 7, 6, 2)
 #define SYS_DC_CSW                     sys_insn(1, 0, 7, 10, 2)

diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 5dff8eccd17d4c4ff2e7514519eed1bf959b0fa6..b0fd1d3001543f9462d33a4ca1f0b342a7b7a68f 100644 (file)
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -46,6 +46,7 @@
 #define PSR_I_BIT      0x00000080
 #define PSR_A_BIT      0x00000100
 #define PSR_D_BIT      0x00000200
+#define PSR_SSBS_BIT   0x00001000
 #define PSR_PAN_BIT    0x00400000
 #define PSR_UAO_BIT    0x00800000
 #define PSR_V_BIT      0x10000000

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 6e565d8d4f71910841faf9a37e1b63d3addd0cfd..105741487a8679e6817a22acf7c311c715010d59 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -304,6 +304,14 @@ void __init arm64_enable_wa2_handling(struct alt_instr *alt,
 
 void arm64_set_ssbd_mitigation(bool state)
 {
+       if (this_cpu_has_cap(ARM64_SSBS)) {
+               if (state)
+                       asm volatile(SET_PSTATE_SSBS(0));
+               else
+                       asm volatile(SET_PSTATE_SSBS(1));
+               return;
+       }
+
        switch (psci_ops.conduit) {
        case PSCI_CONDUIT_HVC:
                arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
@@ -328,6 +336,11 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
 
        WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 
+       if (this_cpu_has_cap(ARM64_SSBS)) {
+               required = false;
+               goto out_printmsg;
+       }
+
        if (psci_ops.smccc_version == SMCCC_VERSION_1_0) {
                ssbd_state = ARM64_SSBD_UNKNOWN;
                return false;
@@ -376,7 +389,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
 
        switch (ssbd_state) {
        case ARM64_SSBD_FORCE_DISABLE:
-               pr_info_once("%s disabled from command-line\n", entry->desc);
                arm64_set_ssbd_mitigation(false);
                required = false;
                break;
@@ -389,7 +401,6 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
                break;
 
        case ARM64_SSBD_FORCE_ENABLE:
-               pr_info_once("%s forced from command-line\n", entry->desc);
                arm64_set_ssbd_mitigation(true);
                required = true;
                break;
@@ -399,6 +410,17 @@ static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
                break;
        }
 
+out_printmsg:
+       switch (ssbd_state) {
+       case ARM64_SSBD_FORCE_DISABLE:
+               pr_info_once("%s disabled from command-line\n", entry->desc);
+               break;
+
+       case ARM64_SSBD_FORCE_ENABLE:
+               pr_info_once("%s forced from command-line\n", entry->desc);
+               break;
+       }
+
        return required;
 }
 #endif /* CONFIG_ARM64_SSBD */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index e0b4122ded52e6dbd012c10c006becb04ad0c6e6..3a42007553412fb206a0e58556953d24ac1f99d3 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -980,6 +980,48 @@ static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
                write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
 }
 
+#ifdef CONFIG_ARM64_SSBD
+static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
+{
+       if (user_mode(regs))
+               return 1;
+
+       if (instr & BIT(CRm_shift))
+               regs->pstate |= PSR_SSBS_BIT;
+       else
+               regs->pstate &= ~PSR_SSBS_BIT;
+
+       arm64_skip_faulting_instruction(regs, 4);
+       return 0;
+}
+
+static struct undef_hook ssbs_emulation_hook = {
+       .instr_mask     = ~(1U << CRm_shift),
+       .instr_val      = 0xd500001f | REG_PSTATE_SSBS_IMM,
+       .fn             = ssbs_emulation_handler,
+};
+
+static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)
+{
+       static bool undef_hook_registered = false;
+       static DEFINE_SPINLOCK(hook_lock);
+
+       spin_lock(&hook_lock);
+       if (!undef_hook_registered) {
+               register_undef_hook(&ssbs_emulation_hook);
+               undef_hook_registered = true;
+       }
+       spin_unlock(&hook_lock);
+
+       if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
+               sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
+               arm64_set_ssbd_mitigation(false);
+       } else {
+               arm64_set_ssbd_mitigation(true);
+       }
+}
+#endif /* CONFIG_ARM64_SSBD */
+
 static const struct arm64_cpu_capabilities arm64_features[] = {
        {
                .desc = "GIC system register CPU interface",
@@ -1176,6 +1218,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 #ifdef CONFIG_ARM64_SVE
        HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
 #endif
+#ifdef CONFIG_ARM64_SSBD
        {
                .desc = "Speculative Store Bypassing Safe (SSBS)",
                .capability = ARM64_SSBS,
@@ -1185,7 +1228,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
                .field_pos = ID_AA64PFR1_SSBS_SHIFT,
                .sign = FTR_UNSIGNED,
                .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
+               .cpu_enable = cpu_enable_ssbs,
        },
+#endif
        {},
 };
 

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 80b3286c75a460334697b31ec5f2f31f74e6727c..f8f1f8ed8e45db78e391e747f8b0aa6241aca0ac 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -357,6 +357,10 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
                if (IS_ENABLED(CONFIG_ARM64_UAO) &&
                    cpus_have_const_cap(ARM64_HAS_UAO))
                        childregs->pstate |= PSR_UAO_BIT;
+
+               if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
+                       childregs->pstate |= PSR_SSBS_BIT;
+
                p->thread.cpu_context.x19 = stack_start;
                p->thread.cpu_context.x20 = stk_sz;
        }

diff --git a/arch/arm64/kernel/ssbd.c b/arch/arm64/kernel/ssbd.c
index 0560738c1d5ccad415d1f50d2ffec56ba7e9ff08..58de005cd756fc873af0eb15d3e3dddf6db3d910 100644 (file)
--- a/arch/arm64/kernel/ssbd.c
+++ b/arch/arm64/kernel/ssbd.c
@@ -3,13 +3,32 @@
  * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
  */
 
+#include <linux/compat.h>
 #include <linux/errno.h>
 #include <linux/prctl.h>
 #include <linux/sched.h>
+#include <linux/sched/task_stack.h>
 #include <linux/thread_info.h>
 
+#include <asm/compat.h>
 #include <asm/cpufeature.h>
 
+static void ssbd_ssbs_enable(struct task_struct *task)
+{
+       u64 val = is_compat_thread(task_thread_info(task)) ?
+                 PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
+
+       task_pt_regs(task)->pstate |= val;
+}
+
+static void ssbd_ssbs_disable(struct task_struct *task)
+{
+       u64 val = is_compat_thread(task_thread_info(task)) ?
+                 PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
+
+       task_pt_regs(task)->pstate &= ~val;
+}
+
 /*
  * prctl interface for SSBD
  */
@@ -45,12 +64,14 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
                        return -EPERM;
                task_clear_spec_ssb_disable(task);
                clear_tsk_thread_flag(task, TIF_SSBD);
+               ssbd_ssbs_enable(task);
                break;
        case PR_SPEC_DISABLE:
                if (state == ARM64_SSBD_FORCE_DISABLE)
                        return -EPERM;
                task_set_spec_ssb_disable(task);
                set_tsk_thread_flag(task, TIF_SSBD);
+               ssbd_ssbs_disable(task);
                break;
        case PR_SPEC_FORCE_DISABLE:
                if (state == ARM64_SSBD_FORCE_DISABLE)
@@ -58,6 +79,7 @@ static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
                task_set_spec_ssb_disable(task);
                task_set_spec_ssb_force_disable(task);
                set_tsk_thread_flag(task, TIF_SSBD);
+               ssbd_ssbs_disable(task);
                break;
        default:
                return -ERANGE;