x86/speculation: Move arch_smt_update() call to after mitigation decisions

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / cpu / bugs.c

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 35743822102fc8d78b5d5b071e3c6263b433ea80..086bb04cf15fef7939e6563d64754c1c6075042e 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -30,10 +30,12 @@
 #include <asm/set_memory.h>
 #include <asm/intel-family.h>
 #include <asm/e820/api.h>
+#include <asm/hypervisor.h>
 
 static void __init spectre_v2_select_mitigation(void);
 static void __init ssb_select_mitigation(void);
 static void __init l1tf_select_mitigation(void);
+static void __init mds_select_mitigation(void);
 
 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
 u64 x86_spec_ctrl_base;
@@ -60,6 +62,13 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 /* Control unconditional IBPB in switch_mm() */
 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 
+/* Control MDS CPU buffer clear before returning to user space */
+DEFINE_STATIC_KEY_FALSE(mds_user_clear);
+EXPORT_SYMBOL_GPL(mds_user_clear);
+/* Control MDS CPU buffer clear before idling (halt, mwait) */
+DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
+EXPORT_SYMBOL_GPL(mds_idle_clear);
+
 void __init check_bugs(void)
 {
        identify_boot_cpu();
@@ -98,6 +107,10 @@ void __init check_bugs(void)
 
        l1tf_select_mitigation();
 
+       mds_select_mitigation();
+
+       arch_smt_update();
+
 #ifdef CONFIG_X86_32
        /*
         * Check whether we are able to run this kernel safely on SMP.
@@ -203,6 +216,60 @@ static void x86_amd_ssb_disable(void)
                wrmsrl(MSR_AMD64_LS_CFG, msrval);
 }
 
+#undef pr_fmt
+#define pr_fmt(fmt)    "MDS: " fmt
+
+/* Default mitigation for L1TF-affected CPUs */
+static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
+static bool mds_nosmt __ro_after_init = false;
+
+static const char * const mds_strings[] = {
+       [MDS_MITIGATION_OFF]    = "Vulnerable",
+       [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
+       [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
+};
+
+static void mds_select_mitigation(void)
+{
+       if (!boot_cpu_has_bug(X86_BUG_MDS)) {
+               mds_mitigation = MDS_MITIGATION_OFF;
+               return;
+       }
+
+       if (mds_mitigation == MDS_MITIGATION_FULL) {
+               if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
+                       mds_mitigation = MDS_MITIGATION_VMWERV;
+
+               static_branch_enable(&mds_user_clear);
+
+               if (mds_nosmt && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
+                       cpu_smt_disable(false);
+       }
+
+       pr_info("%s\n", mds_strings[mds_mitigation]);
+}
+
+static int __init mds_cmdline(char *str)
+{
+       if (!boot_cpu_has_bug(X86_BUG_MDS))
+               return 0;
+
+       if (!str)
+               return -EINVAL;
+
+       if (!strcmp(str, "off"))
+               mds_mitigation = MDS_MITIGATION_OFF;
+       else if (!strcmp(str, "full"))
+               mds_mitigation = MDS_MITIGATION_FULL;
+       else if (!strcmp(str, "full,nosmt")) {
+               mds_mitigation = MDS_MITIGATION_FULL;
+               mds_nosmt = true;
+       }
+
+       return 0;
+}
+early_param("mds", mds_cmdline);
+
 #undef pr_fmt
 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 
@@ -212,7 +279,7 @@ static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
        SPECTRE_V2_USER_NONE;
 
-#ifdef RETPOLINE
+#ifdef CONFIG_RETPOLINE
 static bool spectre_v2_bad_module;
 
 bool retpoline_module_ok(bool has_retpoline)
@@ -569,9 +636,6 @@ specv2_set_mode:
 
        /* Set up IBPB and STIBP depending on the general spectre V2 command */
        spectre_v2_user_select_mitigation(cmd);
-
-       /* Enable STIBP if appropriate */
-       arch_smt_update();
 }
 
 static void update_stibp_msr(void * __unused)
@@ -605,6 +669,24 @@ static void update_indir_branch_cond(void)
                static_branch_disable(&switch_to_cond_stibp);
 }
 
+/* Update the static key controlling the MDS CPU buffer clear in idle */
+static void update_mds_branch_idle(void)
+{
+       /*
+        * Enable the idle clearing on CPUs which are affected only by
+        * MDBDS and not any other MDS variant. The other variants cannot
+        * be mitigated when SMT is enabled, so clearing the buffers on
+        * idle would be a window dressing exercise.
+        */
+       if (!boot_cpu_has(X86_BUG_MSBDS_ONLY))
+               return;
+
+       if (sched_smt_active())
+               static_branch_enable(&mds_idle_clear);
+       else
+               static_branch_disable(&mds_idle_clear);
+}
+
 void arch_smt_update(void)
 {
        /* Enhanced IBRS implies STIBP. No update required. */
@@ -626,6 +708,15 @@ void arch_smt_update(void)
                break;
        }
 
+       switch(mds_mitigation) {
+       case MDS_MITIGATION_FULL:
+       case MDS_MITIGATION_VMWERV:
+               update_mds_branch_idle();
+               break;
+       case MDS_MITIGATION_OFF:
+               break;
+       }
+
        mutex_unlock(&spec_ctrl_mutex);
 }
 
@@ -1016,6 +1107,10 @@ static void __init l1tf_select_mitigation(void)
        half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
        if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
                pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
+               pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
+                               half_pa);
+               pr_info("However, doing so will make a part of your RAM unusable.\n");
+               pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
                return;
        }
 
@@ -1086,6 +1181,22 @@ static ssize_t l1tf_show_state(char *buf)
 }
 #endif
 
+static ssize_t mds_show_state(char *buf)
+{
+       if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
+               return sprintf(buf, "%s; SMT Host state unknown\n",
+                              mds_strings[mds_mitigation]);
+       }
+
+       if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
+               return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
+                              sched_smt_active() ? "mitigated" : "disabled");
+       }
+
+       return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
+                      sched_smt_active() ? "vulnerable" : "disabled");
+}
+
 static char *stibp_state(void)
 {
        if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
@@ -1149,6 +1260,10 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
                if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
                        return l1tf_show_state(buf);
                break;
+
+       case X86_BUG_MDS:
+               return mds_show_state(buf);
+
        default:
                break;
        }
@@ -1180,4 +1295,9 @@ ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *b
 {
        return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
 }
+
+ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
+{
+       return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
+}
 #endif