arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/smp.h>
  14
  15 #include <asm/nospec-branch.h>
  16 #include <asm/cmdline.h>
  17 #include <asm/bugs.h>
  18 #include <asm/processor.h>
  19 #include <asm/processor-flags.h>
  20 #include <asm/fpu/internal.h>
  21 #include <asm/msr.h>
  22 #include <asm/paravirt.h>
  23 #include <asm/alternative.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/set_memory.h>
  26 #include <asm/intel-family.h>
  27
  28 static void __init spectre_v2_select_mitigation(void);
  29 static void __init ssb_select_mitigation(void);
  30
  31 /*
  32  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
  33  * writes to SPEC_CTRL contain whatever reserved bits have been set.
  34  */
  35 static u64 __ro_after_init x86_spec_ctrl_base;
  36
  37 /*
  38  * The vendor and possibly platform specific bits which can be modified in
  39  * x86_spec_ctrl_base.
  40  */
  41 static u64 __ro_after_init x86_spec_ctrl_mask = ~SPEC_CTRL_IBRS;
  42
  43 void __init check_bugs(void)
  44 {
  45         identify_boot_cpu();
  46
  47         if (!IS_ENABLED(CONFIG_SMP)) {
  48                 pr_info("CPU: ");
  49                 print_cpu_info(&boot_cpu_data);
  50         }
  51
  52         /*
  53          * Read the SPEC_CTRL MSR to account for reserved bits which may
  54          * have unknown values.
  55          */
  56         if (ibrs_inuse)
  57                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  58
  59         /* Select the proper spectre mitigation before patching alternatives */
  60         spectre_v2_select_mitigation();
  61
  62         /*
  63          * Select proper mitigation for any exposure to the Speculative Store
  64          * Bypass vulnerability.
  65          */
  66         ssb_select_mitigation();
  67
  68 #ifdef CONFIG_X86_32
  69         /*
  70          * Check whether we are able to run this kernel safely on SMP.
  71          *
  72          * - i386 is no longer supported.
  73          * - In order to run on anything without a TSC, we need to be
  74          *   compiled for a i486.
  75          */
  76         if (boot_cpu_data.x86 < 4)
  77                 panic("Kernel requires i486+ for 'invlpg' and other features");
  78
  79         init_utsname()->machine[1] =
  80                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
  81         alternative_instructions();
  82
  83         fpu__init_check_bugs();
  84 #else /* CONFIG_X86_64 */
  85         alternative_instructions();
  86
  87         /*
  88          * Make sure the first 2MB area is not mapped by huge pages
  89          * There are typically fixed size MTRRs in there and overlapping
  90          * MTRRs into large pages causes slow downs.
  91          *
  92          * Right now we don't do that with gbpages because there seems
  93          * very little benefit for that case.
  94          */
  95         if (!direct_gbpages)
  96                 set_memory_4k((unsigned long)__va(0), 1);
  97 #endif
  98 }
  99
 100 /* The kernel command line selection */
 101 enum spectre_v2_mitigation_cmd {
 102         SPECTRE_V2_CMD_NONE,
 103         SPECTRE_V2_CMD_AUTO,
 104         SPECTRE_V2_CMD_FORCE,
 105         SPECTRE_V2_CMD_RETPOLINE,
 106         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 107         SPECTRE_V2_CMD_RETPOLINE_AMD,
 108 };
 109
 110 static const char *spectre_v2_strings[] = {
 111         [SPECTRE_V2_NONE]                       = "Vulnerable",
 112         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 113         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 114         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 115         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 116 };
 117
 118 #undef pr_fmt
 119 #define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
 120
 121 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
 122
 123 void x86_spec_ctrl_set(u64 val)
 124 {
 125         if (val & x86_spec_ctrl_mask)
 126                 WARN_ONCE(1, "SPEC_CTRL MSR value 0x%16llx is unknown.\n", val);
 127         else
 128                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | val);
 129 }
 130 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set);
 131
 132 u64 x86_spec_ctrl_get_default(void)
 133 {
 134         return x86_spec_ctrl_base;
 135 }
 136 EXPORT_SYMBOL_GPL(x86_spec_ctrl_get_default);
 137
 138 void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl)
 139 {
 140         if (!ibrs_inuse)
 141                 return;
 142         if (x86_spec_ctrl_base != guest_spec_ctrl)
 143                 wrmsrl(MSR_IA32_SPEC_CTRL, guest_spec_ctrl);
 144 }
 145 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set_guest);
 146
 147 void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl)
 148 {
 149         if (!ibrs_inuse)
 150                 return;
 151         if (x86_spec_ctrl_base != guest_spec_ctrl)
 152                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 153 }
 154 EXPORT_SYMBOL_GPL(x86_spec_ctrl_restore_host);
 155
 156 static void __init spec2_print_if_insecure(const char *reason)
 157 {
 158         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 159                 pr_info("%s\n", reason);
 160 }
 161
 162 static void __init spec2_print_if_secure(const char *reason)
 163 {
 164         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 165                 pr_info("%s\n", reason);
 166 }
 167
 168 static inline bool retp_compiler(void)
 169 {
 170         return __is_defined(RETPOLINE);
 171 }
 172
 173 static inline bool match_option(const char *arg, int arglen, const char *opt)
 174 {
 175         int len = strlen(opt);
 176
 177         return len == arglen && !strncmp(arg, opt, len);
 178 }
 179
 180 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 181 {
 182         char arg[20];
 183         int ret;
 184
 185         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
 186                                   sizeof(arg));
 187         if (ret > 0)  {
 188                 if (match_option(arg, ret, "off")) {
 189                         goto disable;
 190                 } else if (match_option(arg, ret, "on")) {
 191                         spec2_print_if_secure("force enabled on command line.");
 192                         return SPECTRE_V2_CMD_FORCE;
 193                 } else if (match_option(arg, ret, "retpoline")) {
 194                         spec2_print_if_insecure("retpoline selected on command line.");
 195                         return SPECTRE_V2_CMD_RETPOLINE;
 196                 } else if (match_option(arg, ret, "retpoline,amd")) {
 197                         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 198                                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 199                                 return SPECTRE_V2_CMD_AUTO;
 200                         }
 201                         spec2_print_if_insecure("AMD retpoline selected on command line.");
 202                         return SPECTRE_V2_CMD_RETPOLINE_AMD;
 203                 } else if (match_option(arg, ret, "retpoline,generic")) {
 204                         spec2_print_if_insecure("generic retpoline selected on command line.");
 205                         return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
 206                 } else if (match_option(arg, ret, "auto")) {
 207                         return SPECTRE_V2_CMD_AUTO;
 208                 }
 209         }
 210
 211         if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 212                 return SPECTRE_V2_CMD_AUTO;
 213 disable:
 214         spec2_print_if_insecure("disabled on command line.");
 215         return SPECTRE_V2_CMD_NONE;
 216 }
 217
 218 /* Check for Skylake-like CPUs (for RSB handling) */
 219 static bool __init is_skylake_era(void)
 220 {
 221         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 222             boot_cpu_data.x86 == 6) {
 223                 switch (boot_cpu_data.x86_model) {
 224                 case INTEL_FAM6_SKYLAKE_MOBILE:
 225                 case INTEL_FAM6_SKYLAKE_DESKTOP:
 226                 case INTEL_FAM6_SKYLAKE_X:
 227                 case INTEL_FAM6_KABYLAKE_MOBILE:
 228                 case INTEL_FAM6_KABYLAKE_DESKTOP:
 229                         return true;
 230                 }
 231         }
 232         return false;
 233 }
 234
 235 static void __init spectre_v2_select_mitigation(void)
 236 {
 237         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 238         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 239
 240         /*
 241          * If the CPU is not affected and the command line mode is NONE or AUTO
 242          * then nothing to do.
 243          */
 244         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 245             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 246                 return;
 247
 248         switch (cmd) {
 249         case SPECTRE_V2_CMD_NONE:
 250                 return;
 251
 252         case SPECTRE_V2_CMD_FORCE:
 253                 /* FALLTRHU */
 254         case SPECTRE_V2_CMD_AUTO:
 255                 goto retpoline_auto;
 256
 257         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 258                 if (IS_ENABLED(CONFIG_RETPOLINE))
 259                         goto retpoline_amd;
 260                 break;
 261         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 262                 if (IS_ENABLED(CONFIG_RETPOLINE))
 263                         goto retpoline_generic;
 264                 break;
 265         case SPECTRE_V2_CMD_RETPOLINE:
 266                 if (IS_ENABLED(CONFIG_RETPOLINE))
 267                         goto retpoline_auto;
 268                 break;
 269         }
 270         pr_err("kernel not compiled with retpoline; no mitigation available!");
 271         return;
 272
 273 retpoline_auto:
 274         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 275         retpoline_amd:
 276                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 277                         pr_err("LFENCE not serializing. Switching to generic retpoline\n");
 278                         goto retpoline_generic;
 279                 }
 280                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 281                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 282                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 283                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 284         } else {
 285         retpoline_generic:
 286                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 287                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 288                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 289         }
 290
 291         spectre_v2_enabled = mode;
 292         pr_info("%s\n", spectre_v2_strings[mode]);
 293
 294         pr_info("Speculation control IBPB %s IBRS %s",
 295                 ibpb_supported ? "supported" : "not-supported",
 296                 ibrs_supported ? "supported" : "not-supported");
 297
 298         /*
 299          * If we have a full retpoline mode and then disable IBPB in kernel mode
 300          * we do not require both.
 301          */
 302         if (mode == SPECTRE_V2_RETPOLINE_AMD ||
 303             mode == SPECTRE_V2_RETPOLINE_GENERIC)
 304         {
 305                 if (ibrs_supported) {
 306                         pr_info("Retpoline compiled kernel.  Defaulting IBRS to disabled");
 307                         set_ibrs_disabled();
 308                         if (!ibrs_inuse)
 309                                 sysctl_ibrs_enabled = 0;
 310                 }
 311         }
 312
 313         /*
 314          * If neither SMEP or KPTI are available, there is a risk of
 315          * hitting userspace addresses in the RSB after a context switch
 316          * from a shallow call stack to a deeper one. To prevent this fill
 317          * the entire RSB, even when using IBRS.
 318          *
 319          * Skylake era CPUs have a separate issue with *underflow* of the
 320          * RSB, when they will predict 'ret' targets from the generic BTB.
 321          * The proper mitigation for this is IBRS. If IBRS is not supported
 322          * or deactivated in favour of retpolines the RSB fill on context
 323          * switch is required.
 324          */
 325         if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 326              !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 327                 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 328                 pr_info("Filling RSB on context switch\n");
 329         }
 330 }
 331
 332 #undef pr_fmt
 333 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 334
 335 static enum ssb_mitigation ssb_mode = SPEC_STORE_BYPASS_NONE;
 336
 337 /* The kernel command line selection */
 338 enum ssb_mitigation_cmd {
 339         SPEC_STORE_BYPASS_CMD_NONE,
 340         SPEC_STORE_BYPASS_CMD_AUTO,
 341         SPEC_STORE_BYPASS_CMD_ON,
 342 };
 343
 344 static const char *ssb_strings[] = {
 345         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 346         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled"
 347 };
 348
 349 static const struct {
 350         const char *option;
 351         enum ssb_mitigation_cmd cmd;
 352 } ssb_mitigation_options[] = {
 353         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
 354         { "on",         SPEC_STORE_BYPASS_CMD_ON },   /* Disable Speculative Store Bypass */
 355         { "off",        SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
 356 };
 357
 358 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 359 {
 360         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 361         char arg[20];
 362         int ret, i;
 363
 364         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 365                 return SPEC_STORE_BYPASS_CMD_NONE;
 366         } else {
 367                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 368                                           arg, sizeof(arg));
 369                 if (ret < 0)
 370                         return SPEC_STORE_BYPASS_CMD_AUTO;
 371
 372                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 373                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 374                                 continue;
 375
 376                         cmd = ssb_mitigation_options[i].cmd;
 377                         break;
 378                 }
 379
 380                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 381                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 382                         return SPEC_STORE_BYPASS_CMD_AUTO;
 383                 }
 384         }
 385
 386         return cmd;
 387 }
 388
 389 static enum ssb_mitigation_cmd __init __ssb_select_mitigation(void)
 390 {
 391         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 392         enum ssb_mitigation_cmd cmd;
 393
 394         if (!boot_cpu_has(X86_FEATURE_RDS))
 395                 return mode;
 396
 397         cmd = ssb_parse_cmdline();
 398         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 399             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 400              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 401                 return mode;
 402
 403         switch (cmd) {
 404         case SPEC_STORE_BYPASS_CMD_AUTO:
 405         case SPEC_STORE_BYPASS_CMD_ON:
 406                 mode = SPEC_STORE_BYPASS_DISABLE;
 407                 break;
 408         case SPEC_STORE_BYPASS_CMD_NONE:
 409                 break;
 410         }
 411
 412         /*
 413          * We have three CPU feature flags that are in play here:
 414          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 415          *  - X86_FEATURE_RDS - CPU is able to turn off speculative store bypass
 416          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 417          */
 418         if (mode != SPEC_STORE_BYPASS_NONE) {
 419                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 420                 /*
 421                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD uses
 422                  * a completely different MSR and bit dependent on family.
 423                  */
 424                 switch (boot_cpu_data.x86_vendor) {
 425                 case X86_VENDOR_INTEL:
 426                         x86_spec_ctrl_base |= SPEC_CTRL_RDS;
 427                         x86_spec_ctrl_mask &= ~SPEC_CTRL_RDS;
 428                         x86_spec_ctrl_set(SPEC_CTRL_RDS);
 429                         break;
 430                 case X86_VENDOR_AMD:
 431                         break;
 432                 }
 433         }
 434
 435         return mode;
 436 }
 437
 438 static void ssb_select_mitigation()
 439 {
 440         ssb_mode = __ssb_select_mitigation();
 441
 442         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 443                 pr_info("%s\n", ssb_strings[ssb_mode]);
 444 }
 445
 446 #undef pr_fmt
 447
 448 void x86_spec_ctrl_setup_ap(void)
 449 {
 450         if (ibrs_inuse)
 451                 x86_spec_ctrl_set(x86_spec_ctrl_base & ~x86_spec_ctrl_mask);
 452 }
 453
 454 #ifdef CONFIG_SYSFS
 455 ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 456                         char *buf, unsigned int bug)
 457 {
 458         if (!boot_cpu_has_bug(bug))
 459                 return sprintf(buf, "Not affected\n");
 460
 461         switch (bug) {
 462         case X86_BUG_CPU_MELTDOWN:
 463                 if (boot_cpu_has(X86_FEATURE_PTI))
 464                         return sprintf(buf, "Mitigation: PTI\n");
 465                 break;
 466
 467         case X86_BUG_SPECTRE_V1:
 468                 if (osb_is_enabled)
 469                         return sprintf(buf, "Mitigation: OSB (observable speculation barrier, Intel v6)\n");
 470
 471         case X86_BUG_SPECTRE_V2:
 472                 return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled], ibpb_inuse ? ", IBPB (Intel v4)" : "");
 473
 474         case X86_BUG_SPEC_STORE_BYPASS:
 475                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 476
 477         default:
 478                 break;
 479         }
 480
 481         return sprintf(buf, "Vulnerable\n");
 482 }
 483
 484 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
 485                           char *buf)
 486 {
 487         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 488 }
 489
 490 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 491                             char *buf)
 492 {
 493         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 494 }
 495
 496 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 497                             char *buf)
 498 {
 499         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 500 }
 501
 502 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 503 {
 504         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 505 }
 506 #endif