arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/smp.h>
  14
  15 #include <asm/nospec-branch.h>
  16 #include <asm/cmdline.h>
  17 #include <asm/bugs.h>
  18 #include <asm/processor.h>
  19 #include <asm/processor-flags.h>
  20 #include <asm/fpu/internal.h>
  21 #include <asm/msr.h>
  22 #include <asm/paravirt.h>
  23 #include <asm/alternative.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/set_memory.h>
  26 #include <asm/intel-family.h>
  27
  28 static void __init spectre_v2_select_mitigation(void);
  29 static void __init ssb_select_mitigation(void);
  30
  31 /*
  32  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
  33  * writes to SPEC_CTRL contain whatever reserved bits have been set.
  34  */
  35 static u64 __ro_after_init x86_spec_ctrl_base;
  36
  37 void __init check_bugs(void)
  38 {
  39         identify_boot_cpu();
  40
  41         if (!IS_ENABLED(CONFIG_SMP)) {
  42                 pr_info("CPU: ");
  43                 print_cpu_info(&boot_cpu_data);
  44         }
  45
  46         /*
  47          * Read the SPEC_CTRL MSR to account for reserved bits which may
  48          * have unknown values.
  49          */
  50         if (ibrs_inuse)
  51                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  52
  53         /* Select the proper spectre mitigation before patching alternatives */
  54         spectre_v2_select_mitigation();
  55
  56         /*
  57          * Select proper mitigation for any exposure to the Speculative Store
  58          * Bypass vulnerability.
  59          */
  60         ssb_select_mitigation();
  61
  62 #ifdef CONFIG_X86_32
  63         /*
  64          * Check whether we are able to run this kernel safely on SMP.
  65          *
  66          * - i386 is no longer supported.
  67          * - In order to run on anything without a TSC, we need to be
  68          *   compiled for a i486.
  69          */
  70         if (boot_cpu_data.x86 < 4)
  71                 panic("Kernel requires i486+ for 'invlpg' and other features");
  72
  73         init_utsname()->machine[1] =
  74                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
  75         alternative_instructions();
  76
  77         fpu__init_check_bugs();
  78 #else /* CONFIG_X86_64 */
  79         alternative_instructions();
  80
  81         /*
  82          * Make sure the first 2MB area is not mapped by huge pages
  83          * There are typically fixed size MTRRs in there and overlapping
  84          * MTRRs into large pages causes slow downs.
  85          *
  86          * Right now we don't do that with gbpages because there seems
  87          * very little benefit for that case.
  88          */
  89         if (!direct_gbpages)
  90                 set_memory_4k((unsigned long)__va(0), 1);
  91 #endif
  92 }
  93
  94 /* The kernel command line selection */
  95 enum spectre_v2_mitigation_cmd {
  96         SPECTRE_V2_CMD_NONE,
  97         SPECTRE_V2_CMD_AUTO,
  98         SPECTRE_V2_CMD_FORCE,
  99         SPECTRE_V2_CMD_RETPOLINE,
 100         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 101         SPECTRE_V2_CMD_RETPOLINE_AMD,
 102 };
 103
 104 static const char *spectre_v2_strings[] = {
 105         [SPECTRE_V2_NONE]                       = "Vulnerable",
 106         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 107         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 108         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 109         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 110 };
 111
 112 #undef pr_fmt
 113 #define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
 114
 115 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
 116
 117 void x86_spec_ctrl_set(u64 val)
 118 {
 119         if (val & ~SPEC_CTRL_IBRS)
 120                 WARN_ONCE(1, "SPEC_CTRL MSR value 0x%16llx is unknown.\n", val);
 121         else
 122                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | val);
 123 }
 124 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set);
 125
 126 u64 x86_spec_ctrl_get_default(void)
 127 {
 128         return x86_spec_ctrl_base;
 129 }
 130 EXPORT_SYMBOL_GPL(x86_spec_ctrl_get_default);
 131
 132 void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl)
 133 {
 134         if (!ibrs_inuse)
 135                 return;
 136         if (x86_spec_ctrl_base != guest_spec_ctrl)
 137                 wrmsrl(MSR_IA32_SPEC_CTRL, guest_spec_ctrl);
 138 }
 139 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set_guest);
 140
 141 void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl)
 142 {
 143         if (!ibrs_inuse)
 144                 return;
 145         if (x86_spec_ctrl_base != guest_spec_ctrl)
 146                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 147 }
 148 EXPORT_SYMBOL_GPL(x86_spec_ctrl_restore_host);
 149
 150 static void __init spec2_print_if_insecure(const char *reason)
 151 {
 152         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 153                 pr_info("%s\n", reason);
 154 }
 155
 156 static void __init spec2_print_if_secure(const char *reason)
 157 {
 158         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 159                 pr_info("%s\n", reason);
 160 }
 161
 162 static inline bool retp_compiler(void)
 163 {
 164         return __is_defined(RETPOLINE);
 165 }
 166
 167 static inline bool match_option(const char *arg, int arglen, const char *opt)
 168 {
 169         int len = strlen(opt);
 170
 171         return len == arglen && !strncmp(arg, opt, len);
 172 }
 173
 174 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 175 {
 176         char arg[20];
 177         int ret;
 178
 179         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
 180                                   sizeof(arg));
 181         if (ret > 0)  {
 182                 if (match_option(arg, ret, "off")) {
 183                         goto disable;
 184                 } else if (match_option(arg, ret, "on")) {
 185                         spec2_print_if_secure("force enabled on command line.");
 186                         return SPECTRE_V2_CMD_FORCE;
 187                 } else if (match_option(arg, ret, "retpoline")) {
 188                         spec2_print_if_insecure("retpoline selected on command line.");
 189                         return SPECTRE_V2_CMD_RETPOLINE;
 190                 } else if (match_option(arg, ret, "retpoline,amd")) {
 191                         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 192                                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 193                                 return SPECTRE_V2_CMD_AUTO;
 194                         }
 195                         spec2_print_if_insecure("AMD retpoline selected on command line.");
 196                         return SPECTRE_V2_CMD_RETPOLINE_AMD;
 197                 } else if (match_option(arg, ret, "retpoline,generic")) {
 198                         spec2_print_if_insecure("generic retpoline selected on command line.");
 199                         return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
 200                 } else if (match_option(arg, ret, "auto")) {
 201                         return SPECTRE_V2_CMD_AUTO;
 202                 }
 203         }
 204
 205         if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 206                 return SPECTRE_V2_CMD_AUTO;
 207 disable:
 208         spec2_print_if_insecure("disabled on command line.");
 209         return SPECTRE_V2_CMD_NONE;
 210 }
 211
 212 /* Check for Skylake-like CPUs (for RSB handling) */
 213 static bool __init is_skylake_era(void)
 214 {
 215         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 216             boot_cpu_data.x86 == 6) {
 217                 switch (boot_cpu_data.x86_model) {
 218                 case INTEL_FAM6_SKYLAKE_MOBILE:
 219                 case INTEL_FAM6_SKYLAKE_DESKTOP:
 220                 case INTEL_FAM6_SKYLAKE_X:
 221                 case INTEL_FAM6_KABYLAKE_MOBILE:
 222                 case INTEL_FAM6_KABYLAKE_DESKTOP:
 223                         return true;
 224                 }
 225         }
 226         return false;
 227 }
 228
 229 static void __init spectre_v2_select_mitigation(void)
 230 {
 231         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 232         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 233
 234         /*
 235          * If the CPU is not affected and the command line mode is NONE or AUTO
 236          * then nothing to do.
 237          */
 238         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 239             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 240                 return;
 241
 242         switch (cmd) {
 243         case SPECTRE_V2_CMD_NONE:
 244                 return;
 245
 246         case SPECTRE_V2_CMD_FORCE:
 247                 /* FALLTRHU */
 248         case SPECTRE_V2_CMD_AUTO:
 249                 goto retpoline_auto;
 250
 251         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 252                 if (IS_ENABLED(CONFIG_RETPOLINE))
 253                         goto retpoline_amd;
 254                 break;
 255         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 256                 if (IS_ENABLED(CONFIG_RETPOLINE))
 257                         goto retpoline_generic;
 258                 break;
 259         case SPECTRE_V2_CMD_RETPOLINE:
 260                 if (IS_ENABLED(CONFIG_RETPOLINE))
 261                         goto retpoline_auto;
 262                 break;
 263         }
 264         pr_err("kernel not compiled with retpoline; no mitigation available!");
 265         return;
 266
 267 retpoline_auto:
 268         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 269         retpoline_amd:
 270                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 271                         pr_err("LFENCE not serializing. Switching to generic retpoline\n");
 272                         goto retpoline_generic;
 273                 }
 274                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 275                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 276                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 277                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 278         } else {
 279         retpoline_generic:
 280                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 281                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 282                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 283         }
 284
 285         spectre_v2_enabled = mode;
 286         pr_info("%s\n", spectre_v2_strings[mode]);
 287
 288         pr_info("Speculation control IBPB %s IBRS %s",
 289                 ibpb_supported ? "supported" : "not-supported",
 290                 ibrs_supported ? "supported" : "not-supported");
 291
 292         /*
 293          * If we have a full retpoline mode and then disable IBPB in kernel mode
 294          * we do not require both.
 295          */
 296         if (mode == SPECTRE_V2_RETPOLINE_AMD ||
 297             mode == SPECTRE_V2_RETPOLINE_GENERIC)
 298         {
 299                 if (ibrs_supported) {
 300                         pr_info("Retpoline compiled kernel.  Defaulting IBRS to disabled");
 301                         set_ibrs_disabled();
 302                         if (!ibrs_inuse)
 303                                 sysctl_ibrs_enabled = 0;
 304                 }
 305         }
 306
 307         /*
 308          * If neither SMEP or KPTI are available, there is a risk of
 309          * hitting userspace addresses in the RSB after a context switch
 310          * from a shallow call stack to a deeper one. To prevent this fill
 311          * the entire RSB, even when using IBRS.
 312          *
 313          * Skylake era CPUs have a separate issue with *underflow* of the
 314          * RSB, when they will predict 'ret' targets from the generic BTB.
 315          * The proper mitigation for this is IBRS. If IBRS is not supported
 316          * or deactivated in favour of retpolines the RSB fill on context
 317          * switch is required.
 318          */
 319         if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 320              !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 321                 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 322                 pr_info("Filling RSB on context switch\n");
 323         }
 324 }
 325
 326 #undef pr_fmt
 327 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 328
 329 static enum ssb_mitigation ssb_mode = SPEC_STORE_BYPASS_NONE;
 330
 331 /* The kernel command line selection */
 332 enum ssb_mitigation_cmd {
 333         SPEC_STORE_BYPASS_CMD_NONE,
 334         SPEC_STORE_BYPASS_CMD_AUTO,
 335         SPEC_STORE_BYPASS_CMD_ON,
 336 };
 337
 338 static const char *ssb_strings[] = {
 339         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 340         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled"
 341 };
 342
 343 static const struct {
 344         const char *option;
 345         enum ssb_mitigation_cmd cmd;
 346 } ssb_mitigation_options[] = {
 347         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
 348         { "on",         SPEC_STORE_BYPASS_CMD_ON },   /* Disable Speculative Store Bypass */
 349         { "off",        SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
 350 };
 351
 352 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 353 {
 354         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 355         char arg[20];
 356         int ret, i;
 357
 358         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 359                 return SPEC_STORE_BYPASS_CMD_NONE;
 360         } else {
 361                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 362                                           arg, sizeof(arg));
 363                 if (ret < 0)
 364                         return SPEC_STORE_BYPASS_CMD_AUTO;
 365
 366                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 367                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 368                                 continue;
 369
 370                         cmd = ssb_mitigation_options[i].cmd;
 371                         break;
 372                 }
 373
 374                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 375                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 376                         return SPEC_STORE_BYPASS_CMD_AUTO;
 377                 }
 378         }
 379
 380         return cmd;
 381 }
 382
 383 static enum ssb_mitigation_cmd __init __ssb_select_mitigation(void)
 384 {
 385         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 386         enum ssb_mitigation_cmd cmd;
 387
 388         if (!boot_cpu_has(X86_FEATURE_RDS))
 389                 return mode;
 390
 391         cmd = ssb_parse_cmdline();
 392         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 393             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 394              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 395                 return mode;
 396
 397         switch (cmd) {
 398         case SPEC_STORE_BYPASS_CMD_AUTO:
 399         case SPEC_STORE_BYPASS_CMD_ON:
 400                 mode = SPEC_STORE_BYPASS_DISABLE;
 401                 break;
 402         case SPEC_STORE_BYPASS_CMD_NONE:
 403                 break;
 404         }
 405
 406         if (mode != SPEC_STORE_BYPASS_NONE)
 407                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 408         return mode;
 409 }
 410
 411 static void ssb_select_mitigation()
 412 {
 413         ssb_mode = __ssb_select_mitigation();
 414
 415         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 416                 pr_info("%s\n", ssb_strings[ssb_mode]);
 417 }
 418
 419 #undef pr_fmt
 420
 421 #ifdef CONFIG_SYSFS
 422 ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 423                         char *buf, unsigned int bug)
 424 {
 425         if (!boot_cpu_has_bug(bug))
 426                 return sprintf(buf, "Not affected\n");
 427
 428         switch (bug) {
 429         case X86_BUG_CPU_MELTDOWN:
 430                 if (boot_cpu_has(X86_FEATURE_PTI))
 431                         return sprintf(buf, "Mitigation: PTI\n");
 432                 break;
 433
 434         case X86_BUG_SPECTRE_V1:
 435                 if (osb_is_enabled)
 436                         return sprintf(buf, "Mitigation: OSB (observable speculation barrier, Intel v6)\n");
 437
 438         case X86_BUG_SPECTRE_V2:
 439                 return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled], ibpb_inuse ? ", IBPB (Intel v4)" : "");
 440
 441         case X86_BUG_SPEC_STORE_BYPASS:
 442                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 443
 444         default:
 445                 break;
 446         }
 447
 448         return sprintf(buf, "Vulnerable\n");
 449 }
 450
 451 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
 452                           char *buf)
 453 {
 454         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 455 }
 456
 457 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 458                             char *buf)
 459 {
 460         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 461 }
 462
 463 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 464                             char *buf)
 465 {
 466         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 467 }
 468
 469 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 470 {
 471         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 472 }
 473 #endif