arch/x86/kernel/cpu/bugs.c

   1 /*
   2  *  Copyright (C) 1994  Linus Torvalds
   3  *
   4  *  Cyrix stuff, June 1998 by:
   5  *      - Rafael R. Reilova (moved everything from head.S),
   6  *        <rreilova@ececs.uc.edu>
   7  *      - Channing Corn (tests & fixes),
   8  *      - Andrew D. Balsa (code cleanup).
   9  */
  10 #include <linux/init.h>
  11 #include <linux/utsname.h>
  12 #include <linux/cpu.h>
  13 #include <linux/smp.h>
  14
  15 #include <asm/nospec-branch.h>
  16 #include <asm/cmdline.h>
  17 #include <asm/bugs.h>
  18 #include <asm/processor.h>
  19 #include <asm/processor-flags.h>
  20 #include <asm/fpu/internal.h>
  21 #include <asm/msr.h>
  22 #include <asm/paravirt.h>
  23 #include <asm/alternative.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/set_memory.h>
  26 #include <asm/intel-family.h>
  27
  28 static void __init spectre_v2_select_mitigation(void);
  29
  30 /*
  31  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
  32  * writes to SPEC_CTRL contain whatever reserved bits have been set.
  33  */
  34 static u64 __ro_after_init x86_spec_ctrl_base;
  35
  36 void __init check_bugs(void)
  37 {
  38         identify_boot_cpu();
  39
  40         if (!IS_ENABLED(CONFIG_SMP)) {
  41                 pr_info("CPU: ");
  42                 print_cpu_info(&boot_cpu_data);
  43         }
  44
  45         /*
  46          * Read the SPEC_CTRL MSR to account for reserved bits which may
  47          * have unknown values.
  48          */
  49         if (ibrs_inuse)
  50                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  51
  52         /* Select the proper spectre mitigation before patching alternatives */
  53         spectre_v2_select_mitigation();
  54
  55 #ifdef CONFIG_X86_32
  56         /*
  57          * Check whether we are able to run this kernel safely on SMP.
  58          *
  59          * - i386 is no longer supported.
  60          * - In order to run on anything without a TSC, we need to be
  61          *   compiled for a i486.
  62          */
  63         if (boot_cpu_data.x86 < 4)
  64                 panic("Kernel requires i486+ for 'invlpg' and other features");
  65
  66         init_utsname()->machine[1] =
  67                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
  68         alternative_instructions();
  69
  70         fpu__init_check_bugs();
  71 #else /* CONFIG_X86_64 */
  72         alternative_instructions();
  73
  74         /*
  75          * Make sure the first 2MB area is not mapped by huge pages
  76          * There are typically fixed size MTRRs in there and overlapping
  77          * MTRRs into large pages causes slow downs.
  78          *
  79          * Right now we don't do that with gbpages because there seems
  80          * very little benefit for that case.
  81          */
  82         if (!direct_gbpages)
  83                 set_memory_4k((unsigned long)__va(0), 1);
  84 #endif
  85 }
  86
  87 /* The kernel command line selection */
  88 enum spectre_v2_mitigation_cmd {
  89         SPECTRE_V2_CMD_NONE,
  90         SPECTRE_V2_CMD_AUTO,
  91         SPECTRE_V2_CMD_FORCE,
  92         SPECTRE_V2_CMD_RETPOLINE,
  93         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
  94         SPECTRE_V2_CMD_RETPOLINE_AMD,
  95 };
  96
  97 static const char *spectre_v2_strings[] = {
  98         [SPECTRE_V2_NONE]                       = "Vulnerable",
  99         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 100         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 101         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 102         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 103 };
 104
 105 #undef pr_fmt
 106 #define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
 107
 108 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
 109
 110 void x86_spec_ctrl_set(u64 val)
 111 {
 112         if (val & ~SPEC_CTRL_IBRS)
 113                 WARN_ONCE(1, "SPEC_CTRL MSR value 0x%16llx is unknown.\n", val);
 114         else
 115                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | val);
 116 }
 117 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set);
 118
 119 u64 x86_spec_ctrl_get_default(void)
 120 {
 121         return x86_spec_ctrl_base;
 122 }
 123 EXPORT_SYMBOL_GPL(x86_spec_ctrl_get_default);
 124
 125 void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl)
 126 {
 127         if (!ibrs_inuse)
 128                 return;
 129         if (x86_spec_ctrl_base != guest_spec_ctrl)
 130                 wrmsrl(MSR_IA32_SPEC_CTRL, guest_spec_ctrl);
 131 }
 132 EXPORT_SYMBOL_GPL(x86_spec_ctrl_set_guest);
 133
 134 void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl)
 135 {
 136         if (!ibrs_inuse)
 137                 return;
 138         if (x86_spec_ctrl_base != guest_spec_ctrl)
 139                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 140 }
 141 EXPORT_SYMBOL_GPL(x86_spec_ctrl_restore_host);
 142
 143 static void __init spec2_print_if_insecure(const char *reason)
 144 {
 145         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 146                 pr_info("%s\n", reason);
 147 }
 148
 149 static void __init spec2_print_if_secure(const char *reason)
 150 {
 151         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 152                 pr_info("%s\n", reason);
 153 }
 154
 155 static inline bool retp_compiler(void)
 156 {
 157         return __is_defined(RETPOLINE);
 158 }
 159
 160 static inline bool match_option(const char *arg, int arglen, const char *opt)
 161 {
 162         int len = strlen(opt);
 163
 164         return len == arglen && !strncmp(arg, opt, len);
 165 }
 166
 167 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 168 {
 169         char arg[20];
 170         int ret;
 171
 172         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
 173                                   sizeof(arg));
 174         if (ret > 0)  {
 175                 if (match_option(arg, ret, "off")) {
 176                         goto disable;
 177                 } else if (match_option(arg, ret, "on")) {
 178                         spec2_print_if_secure("force enabled on command line.");
 179                         return SPECTRE_V2_CMD_FORCE;
 180                 } else if (match_option(arg, ret, "retpoline")) {
 181                         spec2_print_if_insecure("retpoline selected on command line.");
 182                         return SPECTRE_V2_CMD_RETPOLINE;
 183                 } else if (match_option(arg, ret, "retpoline,amd")) {
 184                         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 185                                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 186                                 return SPECTRE_V2_CMD_AUTO;
 187                         }
 188                         spec2_print_if_insecure("AMD retpoline selected on command line.");
 189                         return SPECTRE_V2_CMD_RETPOLINE_AMD;
 190                 } else if (match_option(arg, ret, "retpoline,generic")) {
 191                         spec2_print_if_insecure("generic retpoline selected on command line.");
 192                         return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
 193                 } else if (match_option(arg, ret, "auto")) {
 194                         return SPECTRE_V2_CMD_AUTO;
 195                 }
 196         }
 197
 198         if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 199                 return SPECTRE_V2_CMD_AUTO;
 200 disable:
 201         spec2_print_if_insecure("disabled on command line.");
 202         return SPECTRE_V2_CMD_NONE;
 203 }
 204
 205 /* Check for Skylake-like CPUs (for RSB handling) */
 206 static bool __init is_skylake_era(void)
 207 {
 208         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 209             boot_cpu_data.x86 == 6) {
 210                 switch (boot_cpu_data.x86_model) {
 211                 case INTEL_FAM6_SKYLAKE_MOBILE:
 212                 case INTEL_FAM6_SKYLAKE_DESKTOP:
 213                 case INTEL_FAM6_SKYLAKE_X:
 214                 case INTEL_FAM6_KABYLAKE_MOBILE:
 215                 case INTEL_FAM6_KABYLAKE_DESKTOP:
 216                         return true;
 217                 }
 218         }
 219         return false;
 220 }
 221
 222 static void __init spectre_v2_select_mitigation(void)
 223 {
 224         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 225         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 226
 227         /*
 228          * If the CPU is not affected and the command line mode is NONE or AUTO
 229          * then nothing to do.
 230          */
 231         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 232             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 233                 return;
 234
 235         switch (cmd) {
 236         case SPECTRE_V2_CMD_NONE:
 237                 return;
 238
 239         case SPECTRE_V2_CMD_FORCE:
 240                 /* FALLTRHU */
 241         case SPECTRE_V2_CMD_AUTO:
 242                 goto retpoline_auto;
 243
 244         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 245                 if (IS_ENABLED(CONFIG_RETPOLINE))
 246                         goto retpoline_amd;
 247                 break;
 248         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 249                 if (IS_ENABLED(CONFIG_RETPOLINE))
 250                         goto retpoline_generic;
 251                 break;
 252         case SPECTRE_V2_CMD_RETPOLINE:
 253                 if (IS_ENABLED(CONFIG_RETPOLINE))
 254                         goto retpoline_auto;
 255                 break;
 256         }
 257         pr_err("kernel not compiled with retpoline; no mitigation available!");
 258         return;
 259
 260 retpoline_auto:
 261         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 262         retpoline_amd:
 263                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 264                         pr_err("LFENCE not serializing. Switching to generic retpoline\n");
 265                         goto retpoline_generic;
 266                 }
 267                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 268                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 269                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 270                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 271         } else {
 272         retpoline_generic:
 273                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 274                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 275                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 276         }
 277
 278         spectre_v2_enabled = mode;
 279         pr_info("%s\n", spectre_v2_strings[mode]);
 280
 281         pr_info("Speculation control IBPB %s IBRS %s",
 282                 ibpb_supported ? "supported" : "not-supported",
 283                 ibrs_supported ? "supported" : "not-supported");
 284
 285         /*
 286          * If we have a full retpoline mode and then disable IBPB in kernel mode
 287          * we do not require both.
 288          */
 289         if (mode == SPECTRE_V2_RETPOLINE_AMD ||
 290             mode == SPECTRE_V2_RETPOLINE_GENERIC)
 291         {
 292                 if (ibrs_supported) {
 293                         pr_info("Retpoline compiled kernel.  Defaulting IBRS to disabled");
 294                         set_ibrs_disabled();
 295                         if (!ibrs_inuse)
 296                                 sysctl_ibrs_enabled = 0;
 297                 }
 298         }
 299
 300         /*
 301          * If neither SMEP or KPTI are available, there is a risk of
 302          * hitting userspace addresses in the RSB after a context switch
 303          * from a shallow call stack to a deeper one. To prevent this fill
 304          * the entire RSB, even when using IBRS.
 305          *
 306          * Skylake era CPUs have a separate issue with *underflow* of the
 307          * RSB, when they will predict 'ret' targets from the generic BTB.
 308          * The proper mitigation for this is IBRS. If IBRS is not supported
 309          * or deactivated in favour of retpolines the RSB fill on context
 310          * switch is required.
 311          */
 312         if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 313              !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 314                 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 315                 pr_info("Filling RSB on context switch\n");
 316         }
 317 }
 318
 319 #undef pr_fmt
 320
 321 #ifdef CONFIG_SYSFS
 322 ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 323                         char *buf, unsigned int bug)
 324 {
 325         if (!boot_cpu_has_bug(bug))
 326                 return sprintf(buf, "Not affected\n");
 327
 328         switch (bug) {
 329         case X86_BUG_CPU_MELTDOWN:
 330                 if (boot_cpu_has(X86_FEATURE_PTI))
 331                         return sprintf(buf, "Mitigation: PTI\n");
 332                 break;
 333
 334         case X86_BUG_SPECTRE_V1:
 335                 if (osb_is_enabled)
 336                         return sprintf(buf, "Mitigation: OSB (observable speculation barrier, Intel v6)\n");
 337
 338         case X86_BUG_SPECTRE_V2:
 339                 return sprintf(buf, "%s%s\n", spectre_v2_strings[spectre_v2_enabled], ibpb_inuse ? ", IBPB (Intel v4)" : "");
 340
 341         default:
 342                 break;
 343         }
 344
 345         return sprintf(buf, "Vulnerable\n");
 346 }
 347
 348 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
 349                           char *buf)
 350 {
 351         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 352 }
 353
 354 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 355                             char *buf)
 356 {
 357         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 358 }
 359
 360 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 361                             char *buf)
 362 {
 363         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 364 }
 365 #endif