arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15
  16 #include <asm/nospec-branch.h>
  17 #include <asm/cmdline.h>
  18 #include <asm/bugs.h>
  19 #include <asm/processor.h>
  20 #include <asm/processor-flags.h>
  21 #include <asm/fpu/internal.h>
  22 #include <asm/msr.h>
  23 #include <asm/paravirt.h>
  24 #include <asm/alternative.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/set_memory.h>
  27 #include <asm/intel-family.h>
  28
  29 static void __init spectre_v2_select_mitigation(void);
  30
  31 void __init check_bugs(void)
  32 {
  33         identify_boot_cpu();
  34
  35         if (!IS_ENABLED(CONFIG_SMP)) {
  36                 pr_info("CPU: ");
  37                 print_cpu_info(&boot_cpu_data);
  38         }
  39
  40         /* Select the proper spectre mitigation before patching alternatives */
  41         spectre_v2_select_mitigation();
  42
  43 #ifdef CONFIG_X86_32
  44         /*
  45          * Check whether we are able to run this kernel safely on SMP.
  46          *
  47          * - i386 is no longer supported.
  48          * - In order to run on anything without a TSC, we need to be
  49          *   compiled for a i486.
  50          */
  51         if (boot_cpu_data.x86 < 4)
  52                 panic("Kernel requires i486+ for 'invlpg' and other features");
  53
  54         init_utsname()->machine[1] =
  55                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
  56         alternative_instructions();
  57
  58         fpu__init_check_bugs();
  59 #else /* CONFIG_X86_64 */
  60         alternative_instructions();
  61
  62         /*
  63          * Make sure the first 2MB area is not mapped by huge pages
  64          * There are typically fixed size MTRRs in there and overlapping
  65          * MTRRs into large pages causes slow downs.
  66          *
  67          * Right now we don't do that with gbpages because there seems
  68          * very little benefit for that case.
  69          */
  70         if (!direct_gbpages)
  71                 set_memory_4k((unsigned long)__va(0), 1);
  72 #endif
  73 }
  74
  75 /* The kernel command line selection */
  76 enum spectre_v2_mitigation_cmd {
  77         SPECTRE_V2_CMD_NONE,
  78         SPECTRE_V2_CMD_AUTO,
  79         SPECTRE_V2_CMD_FORCE,
  80         SPECTRE_V2_CMD_RETPOLINE,
  81         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
  82         SPECTRE_V2_CMD_RETPOLINE_AMD,
  83 };
  84
  85 static const char *spectre_v2_strings[] = {
  86         [SPECTRE_V2_NONE]                       = "Vulnerable",
  87         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
  88         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
  89         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
  90         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
  91 };
  92
  93 #undef pr_fmt
  94 #define pr_fmt(fmt)     "Spectre V2 : " fmt
  95
  96 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
  97
  98 #ifdef RETPOLINE
  99 static bool spectre_v2_bad_module;
 100
 101 bool retpoline_module_ok(bool has_retpoline)
 102 {
 103         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 104                 return true;
 105
 106         pr_err("System may be vulnerable to spectre v2\n");
 107         spectre_v2_bad_module = true;
 108         return false;
 109 }
 110
 111 static inline const char *spectre_v2_module_string(void)
 112 {
 113         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 114 }
 115 #else
 116 static inline const char *spectre_v2_module_string(void) { return ""; }
 117 #endif
 118
 119 static void __init spec2_print_if_insecure(const char *reason)
 120 {
 121         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 122                 pr_info("%s selected on command line.\n", reason);
 123 }
 124
 125 static void __init spec2_print_if_secure(const char *reason)
 126 {
 127         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 128                 pr_info("%s selected on command line.\n", reason);
 129 }
 130
 131 static inline bool retp_compiler(void)
 132 {
 133         return __is_defined(RETPOLINE);
 134 }
 135
 136 static inline bool match_option(const char *arg, int arglen, const char *opt)
 137 {
 138         int len = strlen(opt);
 139
 140         return len == arglen && !strncmp(arg, opt, len);
 141 }
 142
 143 static const struct {
 144         const char *option;
 145         enum spectre_v2_mitigation_cmd cmd;
 146         bool secure;
 147 } mitigation_options[] = {
 148         { "off",               SPECTRE_V2_CMD_NONE,              false },
 149         { "on",                SPECTRE_V2_CMD_FORCE,             true },
 150         { "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
 151         { "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 152         { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 153         { "auto",              SPECTRE_V2_CMD_AUTO,              false },
 154 };
 155
 156 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 157 {
 158         char arg[20];
 159         int ret, i;
 160         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 161
 162         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 163                 return SPECTRE_V2_CMD_NONE;
 164         else {
 165                 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 166                 if (ret < 0)
 167                         return SPECTRE_V2_CMD_AUTO;
 168
 169                 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 170                         if (!match_option(arg, ret, mitigation_options[i].option))
 171                                 continue;
 172                         cmd = mitigation_options[i].cmd;
 173                         break;
 174                 }
 175
 176                 if (i >= ARRAY_SIZE(mitigation_options)) {
 177                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 178                         return SPECTRE_V2_CMD_AUTO;
 179                 }
 180         }
 181
 182         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 183              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 184              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 185             !IS_ENABLED(CONFIG_RETPOLINE)) {
 186                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 187                 return SPECTRE_V2_CMD_AUTO;
 188         }
 189
 190         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 191             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 192                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 193                 return SPECTRE_V2_CMD_AUTO;
 194         }
 195
 196         if (mitigation_options[i].secure)
 197                 spec2_print_if_secure(mitigation_options[i].option);
 198         else
 199                 spec2_print_if_insecure(mitigation_options[i].option);
 200
 201         return cmd;
 202 }
 203
 204 /* Check for Skylake-like CPUs (for RSB handling) */
 205 static bool __init is_skylake_era(void)
 206 {
 207         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 208             boot_cpu_data.x86 == 6) {
 209                 switch (boot_cpu_data.x86_model) {
 210                 case INTEL_FAM6_SKYLAKE_MOBILE:
 211                 case INTEL_FAM6_SKYLAKE_DESKTOP:
 212                 case INTEL_FAM6_SKYLAKE_X:
 213                 case INTEL_FAM6_KABYLAKE_MOBILE:
 214                 case INTEL_FAM6_KABYLAKE_DESKTOP:
 215                         return true;
 216                 }
 217         }
 218         return false;
 219 }
 220
 221 static void __init spectre_v2_select_mitigation(void)
 222 {
 223         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 224         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 225
 226         /*
 227          * If the CPU is not affected and the command line mode is NONE or AUTO
 228          * then nothing to do.
 229          */
 230         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 231             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 232                 return;
 233
 234         switch (cmd) {
 235         case SPECTRE_V2_CMD_NONE:
 236                 return;
 237
 238         case SPECTRE_V2_CMD_FORCE:
 239         case SPECTRE_V2_CMD_AUTO:
 240                 if (IS_ENABLED(CONFIG_RETPOLINE))
 241                         goto retpoline_auto;
 242                 break;
 243         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 244                 if (IS_ENABLED(CONFIG_RETPOLINE))
 245                         goto retpoline_amd;
 246                 break;
 247         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 248                 if (IS_ENABLED(CONFIG_RETPOLINE))
 249                         goto retpoline_generic;
 250                 break;
 251         case SPECTRE_V2_CMD_RETPOLINE:
 252                 if (IS_ENABLED(CONFIG_RETPOLINE))
 253                         goto retpoline_auto;
 254                 break;
 255         }
 256         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 257         return;
 258
 259 retpoline_auto:
 260         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 261         retpoline_amd:
 262                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 263                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 264                         goto retpoline_generic;
 265                 }
 266                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 267                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 268                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 269                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 270         } else {
 271         retpoline_generic:
 272                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 273                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 274                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 275         }
 276
 277         spectre_v2_enabled = mode;
 278         pr_info("%s\n", spectre_v2_strings[mode]);
 279
 280         /*
 281          * If neither SMEP nor PTI are available, there is a risk of
 282          * hitting userspace addresses in the RSB after a context switch
 283          * from a shallow call stack to a deeper one. To prevent this fill
 284          * the entire RSB, even when using IBRS.
 285          *
 286          * Skylake era CPUs have a separate issue with *underflow* of the
 287          * RSB, when they will predict 'ret' targets from the generic BTB.
 288          * The proper mitigation for this is IBRS. If IBRS is not supported
 289          * or deactivated in favour of retpolines the RSB fill on context
 290          * switch is required.
 291          */
 292         if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 293              !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 294                 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 295                 pr_info("Spectre v2 mitigation: Filling RSB on context switch\n");
 296         }
 297
 298         /* Initialize Indirect Branch Prediction Barrier if supported */
 299         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 300                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 301                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
 302         }
 303 }
 304
 305 #undef pr_fmt
 306
 307 #ifdef CONFIG_SYSFS
 308 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 309 {
 310         if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
 311                 return sprintf(buf, "Not affected\n");
 312         if (boot_cpu_has(X86_FEATURE_PTI))
 313                 return sprintf(buf, "Mitigation: PTI\n");
 314         return sprintf(buf, "Vulnerable\n");
 315 }
 316
 317 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 318 {
 319         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
 320                 return sprintf(buf, "Not affected\n");
 321         return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 322 }
 323
 324 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 325 {
 326         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 327                 return sprintf(buf, "Not affected\n");
 328
 329         return sprintf(buf, "%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 330                        boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
 331                        spectre_v2_module_string());
 332 }
 333 #endif