]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/x86/kernel/cpu/bugs.c
x86/speculation/mds: Add sysfs reporting for MDS
[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / cpu / bugs.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
1353ebb4 2/*
1353ebb4
JF
3 * Copyright (C) 1994 Linus Torvalds
4 *
5 * Cyrix stuff, June 1998 by:
6 * - Rafael R. Reilova (moved everything from head.S),
7 * <rreilova@ececs.uc.edu>
8 * - Channing Corn (tests & fixes),
9 * - Andrew D. Balsa (code cleanup).
10 */
11#include <linux/init.h>
12#include <linux/utsname.h>
61dc0f55 13#include <linux/cpu.h>
d280282b 14#include <linux/module.h>
574dcf89
TG
15#include <linux/nospec.h>
16#include <linux/prctl.h>
fc8944e3 17#include <linux/sched/smt.h>
da285121 18
d7a6a163 19#include <asm/spec-ctrl.h>
da285121 20#include <asm/cmdline.h>
91eb1b79 21#include <asm/bugs.h>
1353ebb4 22#include <asm/processor.h>
7ebad705 23#include <asm/processor-flags.h>
952f07ec 24#include <asm/fpu/internal.h>
1353ebb4 25#include <asm/msr.h>
b9cfedcd 26#include <asm/vmx.h>
1353ebb4
JF
27#include <asm/paravirt.h>
28#include <asm/alternative.h>
62a67e12 29#include <asm/pgtable.h>
d1163651 30#include <asm/set_memory.h>
c995efd5 31#include <asm/intel-family.h>
05516ad8 32#include <asm/e820/api.h>
1122cc78 33#include <asm/hypervisor.h>
1353ebb4 34
da285121 35static void __init spectre_v2_select_mitigation(void);
e63490c8 36static void __init ssb_select_mitigation(void);
05516ad8 37static void __init l1tf_select_mitigation(void);
fc5ee888 38static void __init mds_select_mitigation(void);
da285121 39
7011f443
JK
40/* The base value of the SPEC_CTRL MSR that always has to be preserved. */
41u64 x86_spec_ctrl_base;
4ac9b1f9 42EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
7011f443 43static DEFINE_MUTEX(spec_ctrl_mutex);
296b454a 44
d0c3bedd
KRW
45/*
46 * The vendor and possibly platform specific bits which can be modified in
47 * x86_spec_ctrl_base.
48 */
e5f984ed 49static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
d0c3bedd 50
c37b94dd
KRW
51/*
52 * AMD specific MSR info for Speculative Store Bypass control.
8fe36c9d 53 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
c37b94dd
KRW
54 */
55u64 __ro_after_init x86_amd_ls_cfg_base;
8fe36c9d 56u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
c37b94dd 57
8edb388f 58/* Control conditional STIBP in switch_to() */
6cd930f0 59DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
58b2d4c8
TG
60/* Control conditional IBPB in switch_mm() */
61DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
62/* Control unconditional IBPB in switch_mm() */
63DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
6cd930f0 64
5ab15133
TG
65/* Control MDS CPU buffer clear before returning to user space */
66DEFINE_STATIC_KEY_FALSE(mds_user_clear);
abf9852b 67EXPORT_SYMBOL_GPL(mds_user_clear);
f3eb8f09
TG
68/* Control MDS CPU buffer clear before idling (halt, mwait) */
69DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
70EXPORT_SYMBOL_GPL(mds_idle_clear);
5ab15133 71
1353ebb4
JF
72void __init check_bugs(void)
73{
74 identify_boot_cpu();
55a36b65 75
483ec3c6
TG
76 /*
77 * identify_boot_cpu() initialized SMT support information, let the
78 * core code know.
79 */
ade31b9e 80 cpu_smt_check_topology_early();
483ec3c6 81
62a67e12
BP
82 if (!IS_ENABLED(CONFIG_SMP)) {
83 pr_info("CPU: ");
84 print_cpu_info(&boot_cpu_data);
85 }
86
296b454a
KRW
87 /*
88 * Read the SPEC_CTRL MSR to account for reserved bits which may
c37b94dd
KRW
89 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
90 * init code as it is not enumerated and depends on the family.
296b454a 91 */
50f9b919 92 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
296b454a
KRW
93 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
94
e5f984ed
TG
95 /* Allow STIBP in MSR_SPEC_CTRL if supported */
96 if (boot_cpu_has(X86_FEATURE_STIBP))
97 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
98
da285121
DW
99 /* Select the proper spectre mitigation before patching alternatives */
100 spectre_v2_select_mitigation();
101
e63490c8
KRW
102 /*
103 * Select proper mitigation for any exposure to the Speculative Store
104 * Bypass vulnerability.
105 */
106 ssb_select_mitigation();
107
05516ad8
AK
108 l1tf_select_mitigation();
109
fc5ee888
TG
110 mds_select_mitigation();
111
62a67e12 112#ifdef CONFIG_X86_32
55a36b65
BP
113 /*
114 * Check whether we are able to run this kernel safely on SMP.
115 *
116 * - i386 is no longer supported.
117 * - In order to run on anything without a TSC, we need to be
118 * compiled for a i486.
119 */
120 if (boot_cpu_data.x86 < 4)
121 panic("Kernel requires i486+ for 'invlpg' and other features");
122
bfe4bb15
MV
123 init_utsname()->machine[1] =
124 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
1353ebb4 125 alternative_instructions();
304bceda 126
4d164092 127 fpu__init_check_bugs();
62a67e12
BP
128#else /* CONFIG_X86_64 */
129 alternative_instructions();
130
131 /*
132 * Make sure the first 2MB area is not mapped by huge pages
133 * There are typically fixed size MTRRs in there and overlapping
134 * MTRRs into large pages causes slow downs.
135 *
136 * Right now we don't do that with gbpages because there seems
137 * very little benefit for that case.
138 */
139 if (!direct_gbpages)
140 set_memory_4k((unsigned long)__va(0), 1);
141#endif
1353ebb4 142}
61dc0f55 143
12376b62
BP
144void
145x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
19fff03f 146{
e5f984ed 147 u64 msrval, guestval, hostval = x86_spec_ctrl_base;
12376b62 148 struct thread_info *ti = current_thread_info();
5407b7f8 149
50f9b919 150 /* Is MSR_SPEC_CTRL implemented ? */
12376b62 151 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
e5f984ed
TG
152 /*
153 * Restrict guest_spec_ctrl to supported values. Clear the
154 * modifiable bits in the host base value and or the
155 * modifiable bits from the guest value.
156 */
157 guestval = hostval & ~x86_spec_ctrl_mask;
158 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
159
12376b62 160 /* SSBD controlled in MSR_SPEC_CTRL */
5005c716
TL
161 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
162 static_cpu_has(X86_FEATURE_AMD_SSBD))
e5f984ed 163 hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
12376b62 164
e0b04783
TC
165 /* Conditional STIBP enabled? */
166 if (static_branch_unlikely(&switch_to_cond_stibp))
167 hostval |= stibp_tif_to_spec_ctrl(ti->flags);
168
e5f984ed
TG
169 if (hostval != guestval) {
170 msrval = setguest ? guestval : hostval;
171 wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
12376b62
BP
172 }
173 }
1238ed31
TG
174
175 /*
176 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
177 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
178 */
179 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
180 !static_cpu_has(X86_FEATURE_VIRT_SSBD))
181 return;
182
183 /*
184 * If the host has SSBD mitigation enabled, force it in the host's
185 * virtual MSR value. If its not permanently enabled, evaluate
186 * current's TIF_SSBD thread flag.
187 */
188 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
189 hostval = SPEC_CTRL_SSBD;
190 else
191 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
192
193 /* Sanitize the guest value */
194 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
195
196 if (hostval != guestval) {
197 unsigned long tif;
198
199 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
200 ssbd_spec_ctrl_to_tif(hostval);
201
8fce7184 202 speculation_ctrl_update(tif);
1238ed31 203 }
19fff03f 204}
12376b62 205EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
19fff03f 206
8fe36c9d 207static void x86_amd_ssb_disable(void)
c37b94dd 208{
8fe36c9d 209 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
c37b94dd 210
65e02bbd
TL
211 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
212 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
213 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
c37b94dd
KRW
214 wrmsrl(MSR_AMD64_LS_CFG, msrval);
215}
216
fc5ee888
TG
217#undef pr_fmt
218#define pr_fmt(fmt) "MDS: " fmt
219
220/* Default mitigation for L1TF-affected CPUs */
221static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
222
223static const char * const mds_strings[] = {
224 [MDS_MITIGATION_OFF] = "Vulnerable",
225 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers"
226};
227
228static void mds_select_mitigation(void)
229{
230 if (!boot_cpu_has_bug(X86_BUG_MDS)) {
231 mds_mitigation = MDS_MITIGATION_OFF;
232 return;
233 }
234
235 if (mds_mitigation == MDS_MITIGATION_FULL) {
236 if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
237 static_branch_enable(&mds_user_clear);
238 else
239 mds_mitigation = MDS_MITIGATION_OFF;
240 }
241 pr_info("%s\n", mds_strings[mds_mitigation]);
242}
243
244static int __init mds_cmdline(char *str)
245{
246 if (!boot_cpu_has_bug(X86_BUG_MDS))
247 return 0;
248
249 if (!str)
250 return -EINVAL;
251
252 if (!strcmp(str, "off"))
253 mds_mitigation = MDS_MITIGATION_OFF;
254 else if (!strcmp(str, "full"))
255 mds_mitigation = MDS_MITIGATION_FULL;
256
257 return 0;
258}
259early_param("mds", mds_cmdline);
260
aef0bebc
TG
261#undef pr_fmt
262#define pr_fmt(fmt) "Spectre V2 : " fmt
263
264static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
265 SPECTRE_V2_NONE;
266
6cd930f0
TG
267static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
268 SPECTRE_V2_USER_NONE;
269
e9a556ad 270#ifdef CONFIG_RETPOLINE
bb3c2578
TG
271static bool spectre_v2_bad_module;
272
d280282b
AK
273bool retpoline_module_ok(bool has_retpoline)
274{
275 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
276 return true;
277
c8b8e109 278 pr_err("System may be vulnerable to spectre v2\n");
d280282b
AK
279 spectre_v2_bad_module = true;
280 return false;
281}
bb3c2578
TG
282
283static inline const char *spectre_v2_module_string(void)
284{
285 return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
286}
287#else
288static inline const char *spectre_v2_module_string(void) { return ""; }
d280282b 289#endif
da285121 290
da285121
DW
291static inline bool match_option(const char *arg, int arglen, const char *opt)
292{
293 int len = strlen(opt);
294
295 return len == arglen && !strncmp(arg, opt, len);
296}
297
aef0bebc
TG
298/* The kernel command line selection for spectre v2 */
299enum spectre_v2_mitigation_cmd {
300 SPECTRE_V2_CMD_NONE,
301 SPECTRE_V2_CMD_AUTO,
302 SPECTRE_V2_CMD_FORCE,
303 SPECTRE_V2_CMD_RETPOLINE,
304 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
305 SPECTRE_V2_CMD_RETPOLINE_AMD,
306};
307
6cd930f0
TG
308enum spectre_v2_user_cmd {
309 SPECTRE_V2_USER_CMD_NONE,
310 SPECTRE_V2_USER_CMD_AUTO,
311 SPECTRE_V2_USER_CMD_FORCE,
2cdf6a58 312 SPECTRE_V2_USER_CMD_PRCTL,
fd1b7023 313 SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1c3cf627 314 SPECTRE_V2_USER_CMD_SECCOMP,
fd1b7023 315 SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
6cd930f0
TG
316};
317
318static const char * const spectre_v2_user_strings[] = {
fbed8eca
TL
319 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable",
320 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection",
321 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection",
322 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl",
323 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl",
6cd930f0
TG
324};
325
326static const struct {
327 const char *option;
328 enum spectre_v2_user_cmd cmd;
329 bool secure;
330} v2_user_options[] __initdata = {
fd1b7023
TG
331 { "auto", SPECTRE_V2_USER_CMD_AUTO, false },
332 { "off", SPECTRE_V2_USER_CMD_NONE, false },
333 { "on", SPECTRE_V2_USER_CMD_FORCE, true },
334 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false },
335 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false },
336 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false },
337 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false },
6cd930f0
TG
338};
339
340static void __init spec_v2_user_print_cond(const char *reason, bool secure)
341{
342 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
343 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
344}
345
346static enum spectre_v2_user_cmd __init
347spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
348{
349 char arg[20];
350 int ret, i;
351
352 switch (v2_cmd) {
353 case SPECTRE_V2_CMD_NONE:
354 return SPECTRE_V2_USER_CMD_NONE;
355 case SPECTRE_V2_CMD_FORCE:
356 return SPECTRE_V2_USER_CMD_FORCE;
357 default:
358 break;
359 }
360
361 ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
362 arg, sizeof(arg));
363 if (ret < 0)
364 return SPECTRE_V2_USER_CMD_AUTO;
365
366 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
367 if (match_option(arg, ret, v2_user_options[i].option)) {
368 spec_v2_user_print_cond(v2_user_options[i].option,
369 v2_user_options[i].secure);
370 return v2_user_options[i].cmd;
371 }
372 }
373
374 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
375 return SPECTRE_V2_USER_CMD_AUTO;
376}
377
378static void __init
379spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
380{
381 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
382 bool smt_possible = IS_ENABLED(CONFIG_SMP);
fd1b7023 383 enum spectre_v2_user_cmd cmd;
6cd930f0
TG
384
385 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
386 return;
387
388 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
389 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
390 smt_possible = false;
391
fd1b7023
TG
392 cmd = spectre_v2_parse_user_cmdline(v2_cmd);
393 switch (cmd) {
6cd930f0
TG
394 case SPECTRE_V2_USER_CMD_NONE:
395 goto set_mode;
396 case SPECTRE_V2_USER_CMD_FORCE:
397 mode = SPECTRE_V2_USER_STRICT;
398 break;
2cdf6a58 399 case SPECTRE_V2_USER_CMD_PRCTL:
fd1b7023 400 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
2cdf6a58
TG
401 mode = SPECTRE_V2_USER_PRCTL;
402 break;
1c3cf627
TG
403 case SPECTRE_V2_USER_CMD_AUTO:
404 case SPECTRE_V2_USER_CMD_SECCOMP:
fd1b7023 405 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1c3cf627
TG
406 if (IS_ENABLED(CONFIG_SECCOMP))
407 mode = SPECTRE_V2_USER_SECCOMP;
408 else
409 mode = SPECTRE_V2_USER_PRCTL;
410 break;
6cd930f0
TG
411 }
412
fbed8eca
TL
413 /*
414 * At this point, an STIBP mode other than "off" has been set.
415 * If STIBP support is not being forced, check if STIBP always-on
416 * is preferred.
417 */
418 if (mode != SPECTRE_V2_USER_STRICT &&
419 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
420 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
421
6cd930f0
TG
422 /* Initialize Indirect Branch Prediction Barrier */
423 if (boot_cpu_has(X86_FEATURE_IBPB)) {
424 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
58b2d4c8 425
fd1b7023
TG
426 switch (cmd) {
427 case SPECTRE_V2_USER_CMD_FORCE:
428 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
429 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
58b2d4c8
TG
430 static_branch_enable(&switch_mm_always_ibpb);
431 break;
fd1b7023
TG
432 case SPECTRE_V2_USER_CMD_PRCTL:
433 case SPECTRE_V2_USER_CMD_AUTO:
434 case SPECTRE_V2_USER_CMD_SECCOMP:
2cdf6a58
TG
435 static_branch_enable(&switch_mm_cond_ibpb);
436 break;
58b2d4c8
TG
437 default:
438 break;
439 }
440
441 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
fd1b7023
TG
442 static_key_enabled(&switch_mm_always_ibpb) ?
443 "always-on" : "conditional");
6cd930f0
TG
444 }
445
8edb388f 446 /* If enhanced IBRS is enabled no STIBP required */
6cd930f0
TG
447 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
448 return;
449
2cdf6a58 450 /*
8edb388f 451 * If SMT is not possible or STIBP is not available clear the STIBP
2cdf6a58
TG
452 * mode.
453 */
454 if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
455 mode = SPECTRE_V2_USER_NONE;
6cd930f0
TG
456set_mode:
457 spectre_v2_user = mode;
458 /* Only print the STIBP mode when SMT possible */
459 if (smt_possible)
460 pr_info("%s\n", spectre_v2_user_strings[mode]);
461}
462
0eb5928b 463static const char * const spectre_v2_strings[] = {
aef0bebc
TG
464 [SPECTRE_V2_NONE] = "Vulnerable",
465 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
466 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
467 [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
468};
469
17c33e7c
KA
470static const struct {
471 const char *option;
472 enum spectre_v2_mitigation_cmd cmd;
473 bool secure;
5a76cb46 474} mitigation_options[] __initdata = {
aef0bebc
TG
475 { "off", SPECTRE_V2_CMD_NONE, false },
476 { "on", SPECTRE_V2_CMD_FORCE, true },
477 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
478 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false },
479 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
480 { "auto", SPECTRE_V2_CMD_AUTO, false },
17c33e7c
KA
481};
482
10640103 483static void __init spec_v2_print_cond(const char *reason, bool secure)
aef0bebc 484{
10640103 485 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
aef0bebc
TG
486 pr_info("%s selected on command line.\n", reason);
487}
488
da285121
DW
489static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
490{
aef0bebc 491 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
da285121 492 char arg[20];
17c33e7c 493 int ret, i;
17c33e7c
KA
494
495 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
496 return SPECTRE_V2_CMD_NONE;
17c33e7c 497
4e8e955a
TC
498 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
499 if (ret < 0)
500 return SPECTRE_V2_CMD_AUTO;
501
502 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
503 if (!match_option(arg, ret, mitigation_options[i].option))
504 continue;
505 cmd = mitigation_options[i].cmd;
506 break;
507 }
508
509 if (i >= ARRAY_SIZE(mitigation_options)) {
510 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
511 return SPECTRE_V2_CMD_AUTO;
da285121
DW
512 }
513
17c33e7c
KA
514 if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
515 cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
516 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
517 !IS_ENABLED(CONFIG_RETPOLINE)) {
713f1b95 518 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
da285121 519 return SPECTRE_V2_CMD_AUTO;
17c33e7c
KA
520 }
521
522 if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
523 boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
524 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
525 return SPECTRE_V2_CMD_AUTO;
526 }
527
10640103
TG
528 spec_v2_print_cond(mitigation_options[i].option,
529 mitigation_options[i].secure);
17c33e7c 530 return cmd;
da285121
DW
531}
532
533static void __init spectre_v2_select_mitigation(void)
534{
535 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
536 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
537
538 /*
539 * If the CPU is not affected and the command line mode is NONE or AUTO
540 * then nothing to do.
541 */
542 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
543 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
544 return;
545
546 switch (cmd) {
547 case SPECTRE_V2_CMD_NONE:
548 return;
549
550 case SPECTRE_V2_CMD_FORCE:
da285121 551 case SPECTRE_V2_CMD_AUTO:
1044fde9
SP
552 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
553 mode = SPECTRE_V2_IBRS_ENHANCED;
554 /* Force it so VMEXIT will restore correctly */
555 x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
556 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
557 goto specv2_set_mode;
558 }
d0f293e0
DL
559 if (IS_ENABLED(CONFIG_RETPOLINE))
560 goto retpoline_auto;
561 break;
da285121
DW
562 case SPECTRE_V2_CMD_RETPOLINE_AMD:
563 if (IS_ENABLED(CONFIG_RETPOLINE))
564 goto retpoline_amd;
565 break;
566 case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
567 if (IS_ENABLED(CONFIG_RETPOLINE))
568 goto retpoline_generic;
569 break;
570 case SPECTRE_V2_CMD_RETPOLINE:
571 if (IS_ENABLED(CONFIG_RETPOLINE))
572 goto retpoline_auto;
573 break;
574 }
713f1b95 575 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
da285121
DW
576 return;
577
578retpoline_auto:
579 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
580 retpoline_amd:
581 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
713f1b95 582 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
da285121
DW
583 goto retpoline_generic;
584 }
51dd01b5 585 mode = SPECTRE_V2_RETPOLINE_AMD;
da285121
DW
586 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
587 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
588 } else {
589 retpoline_generic:
51dd01b5 590 mode = SPECTRE_V2_RETPOLINE_GENERIC;
da285121
DW
591 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
592 }
593
1044fde9 594specv2_set_mode:
da285121
DW
595 spectre_v2_enabled = mode;
596 pr_info("%s\n", spectre_v2_strings[mode]);
c995efd5
DW
597
598 /*
14623a43
JK
599 * If spectre v2 protection has been enabled, unconditionally fill
600 * RSB during a context switch; this protects against two independent
601 * issues:
c995efd5 602 *
14623a43
JK
603 * - RSB underflow (and switch to BTB) on Skylake+
604 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
c995efd5 605 */
14623a43
JK
606 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
607 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
bd12e896 608
390b99c3
DW
609 /*
610 * Retpoline means the kernel is safe because it has no indirect
1044fde9
SP
611 * branches. Enhanced IBRS protects firmware too, so, enable restricted
612 * speculation around firmware calls only when Enhanced IBRS isn't
613 * supported.
614 *
615 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
616 * the user might select retpoline on the kernel command line and if
617 * the CPU supports Enhanced IBRS, kernel might un-intentionally not
618 * enable IBRS around firmware calls.
390b99c3 619 */
1044fde9 620 if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
390b99c3
DW
621 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
622 pr_info("Enabling Restricted Speculation for firmware calls\n");
623 }
7011f443 624
6cd930f0
TG
625 /* Set up IBPB and STIBP depending on the general spectre V2 command */
626 spectre_v2_user_select_mitigation(cmd);
627
7011f443
JK
628 /* Enable STIBP if appropriate */
629 arch_smt_update();
da285121
DW
630}
631
984d266f 632static void update_stibp_msr(void * __unused)
aef0bebc 633{
984d266f 634 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
aef0bebc
TG
635}
636
984d266f
TG
637/* Update x86_spec_ctrl_base in case SMT state changed. */
638static void update_stibp_strict(void)
aef0bebc 639{
984d266f
TG
640 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
641
642 if (sched_smt_active())
643 mask |= SPEC_CTRL_STIBP;
644
645 if (mask == x86_spec_ctrl_base)
646 return;
647
648 pr_info("Update user space SMT mitigation: STIBP %s\n",
649 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
650 x86_spec_ctrl_base = mask;
651 on_each_cpu(update_stibp_msr, NULL, 1);
aef0bebc
TG
652}
653
2cdf6a58
TG
654/* Update the static key controlling the evaluation of TIF_SPEC_IB */
655static void update_indir_branch_cond(void)
656{
657 if (sched_smt_active())
658 static_branch_enable(&switch_to_cond_stibp);
659 else
660 static_branch_disable(&switch_to_cond_stibp);
661}
662
fc5ee888
TG
663/* Update the static key controlling the MDS CPU buffer clear in idle */
664static void update_mds_branch_idle(void)
665{
666 /*
667 * Enable the idle clearing on CPUs which are affected only by
668 * MDBDS and not any other MDS variant. The other variants cannot
669 * be mitigated when SMT is enabled, so clearing the buffers on
670 * idle would be a window dressing exercise.
671 */
672 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY))
673 return;
674
675 if (sched_smt_active())
676 static_branch_enable(&mds_idle_clear);
677 else
678 static_branch_disable(&mds_idle_clear);
679}
680
aef0bebc
TG
681void arch_smt_update(void)
682{
984d266f
TG
683 /* Enhanced IBRS implies STIBP. No update required. */
684 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
aef0bebc
TG
685 return;
686
687 mutex_lock(&spec_ctrl_mutex);
688
984d266f
TG
689 switch (spectre_v2_user) {
690 case SPECTRE_V2_USER_NONE:
691 break;
692 case SPECTRE_V2_USER_STRICT:
fbed8eca 693 case SPECTRE_V2_USER_STRICT_PREFERRED:
984d266f
TG
694 update_stibp_strict();
695 break;
ac40ad3b 696 case SPECTRE_V2_USER_PRCTL:
1c3cf627 697 case SPECTRE_V2_USER_SECCOMP:
2cdf6a58 698 update_indir_branch_cond();
ac40ad3b 699 break;
aef0bebc 700 }
984d266f 701
fc5ee888
TG
702 if (mds_mitigation == MDS_MITIGATION_FULL)
703 update_mds_branch_idle();
704
aef0bebc
TG
705 mutex_unlock(&spec_ctrl_mutex);
706}
707
e63490c8
KRW
708#undef pr_fmt
709#define pr_fmt(fmt) "Speculative Store Bypass: " fmt
710
b5e6d77d 711static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
e63490c8
KRW
712
713/* The kernel command line selection */
714enum ssb_mitigation_cmd {
715 SPEC_STORE_BYPASS_CMD_NONE,
716 SPEC_STORE_BYPASS_CMD_AUTO,
717 SPEC_STORE_BYPASS_CMD_ON,
574dcf89 718 SPEC_STORE_BYPASS_CMD_PRCTL,
c7416003 719 SPEC_STORE_BYPASS_CMD_SECCOMP,
e63490c8
KRW
720};
721
0eb5928b 722static const char * const ssb_strings[] = {
e63490c8 723 [SPEC_STORE_BYPASS_NONE] = "Vulnerable",
574dcf89 724 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled",
c7416003
KC
725 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl",
726 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
e63490c8
KRW
727};
728
729static const struct {
730 const char *option;
731 enum ssb_mitigation_cmd cmd;
5a76cb46 732} ssb_mitigation_options[] __initdata = {
c7416003
KC
733 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
734 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */
735 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
736 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */
737 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
e63490c8
KRW
738};
739
740static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
741{
742 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
743 char arg[20];
744 int ret, i;
745
746 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
747 return SPEC_STORE_BYPASS_CMD_NONE;
748 } else {
749 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
750 arg, sizeof(arg));
751 if (ret < 0)
752 return SPEC_STORE_BYPASS_CMD_AUTO;
753
754 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
755 if (!match_option(arg, ret, ssb_mitigation_options[i].option))
756 continue;
757
758 cmd = ssb_mitigation_options[i].cmd;
759 break;
760 }
761
762 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
763 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
764 return SPEC_STORE_BYPASS_CMD_AUTO;
765 }
766 }
767
768 return cmd;
769}
770
dbe3009d 771static enum ssb_mitigation __init __ssb_select_mitigation(void)
e63490c8
KRW
772{
773 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
774 enum ssb_mitigation_cmd cmd;
775
8fe36c9d 776 if (!boot_cpu_has(X86_FEATURE_SSBD))
e63490c8
KRW
777 return mode;
778
779 cmd = ssb_parse_cmdline();
780 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
781 (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
782 cmd == SPEC_STORE_BYPASS_CMD_AUTO))
783 return mode;
784
785 switch (cmd) {
786 case SPEC_STORE_BYPASS_CMD_AUTO:
c7416003
KC
787 case SPEC_STORE_BYPASS_CMD_SECCOMP:
788 /*
789 * Choose prctl+seccomp as the default mode if seccomp is
790 * enabled.
791 */
792 if (IS_ENABLED(CONFIG_SECCOMP))
793 mode = SPEC_STORE_BYPASS_SECCOMP;
794 else
795 mode = SPEC_STORE_BYPASS_PRCTL;
574dcf89 796 break;
e63490c8
KRW
797 case SPEC_STORE_BYPASS_CMD_ON:
798 mode = SPEC_STORE_BYPASS_DISABLE;
799 break;
574dcf89
TG
800 case SPEC_STORE_BYPASS_CMD_PRCTL:
801 mode = SPEC_STORE_BYPASS_PRCTL;
802 break;
e63490c8
KRW
803 case SPEC_STORE_BYPASS_CMD_NONE:
804 break;
805 }
806
23b9eab9
KRW
807 /*
808 * We have three CPU feature flags that are in play here:
809 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
8fe36c9d 810 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
23b9eab9
KRW
811 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
812 */
574dcf89 813 if (mode == SPEC_STORE_BYPASS_DISABLE) {
e63490c8 814 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
23b9eab9 815 /*
3b881627
KRW
816 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
817 * use a completely different MSR and bit dependent on family.
23b9eab9 818 */
5005c716
TL
819 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
820 !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
733c54c9 821 x86_amd_ssb_disable();
5005c716 822 } else {
8fe36c9d 823 x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
e5f984ed 824 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
208efa83 825 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
23b9eab9
KRW
826 }
827 }
828
e63490c8
KRW
829 return mode;
830}
831
043d480e 832static void ssb_select_mitigation(void)
e63490c8
KRW
833{
834 ssb_mode = __ssb_select_mitigation();
835
836 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
837 pr_info("%s\n", ssb_strings[ssb_mode]);
838}
839
da285121 840#undef pr_fmt
c7416003 841#define pr_fmt(fmt) "Speculation prctl: " fmt
da285121 842
801d6893 843static void task_update_spec_tif(struct task_struct *tsk)
574dcf89 844{
801d6893
TG
845 /* Force the update of the real TIF bits */
846 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
d4595948
TG
847
848 /*
849 * Immediately update the speculation control MSRs for the current
850 * task, but for a non-current task delay setting the CPU
851 * mitigation until it is scheduled next.
852 *
853 * This can only happen for SECCOMP mitigation. For PRCTL it's
854 * always the current task.
855 */
801d6893 856 if (tsk == current)
d4595948
TG
857 speculation_ctrl_update_current();
858}
859
860static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
861{
c7416003
KC
862 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
863 ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
574dcf89
TG
864 return -ENXIO;
865
733f4234
TG
866 switch (ctrl) {
867 case PR_SPEC_ENABLE:
868 /* If speculation is force disabled, enable is not allowed */
869 if (task_spec_ssb_force_disable(task))
870 return -EPERM;
871 task_clear_spec_ssb_disable(task);
801d6893 872 task_update_spec_tif(task);
733f4234
TG
873 break;
874 case PR_SPEC_DISABLE:
875 task_set_spec_ssb_disable(task);
801d6893 876 task_update_spec_tif(task);
733f4234
TG
877 break;
878 case PR_SPEC_FORCE_DISABLE:
879 task_set_spec_ssb_disable(task);
880 task_set_spec_ssb_force_disable(task);
801d6893 881 task_update_spec_tif(task);
733f4234
TG
882 break;
883 default:
884 return -ERANGE;
885 }
574dcf89
TG
886 return 0;
887}
888
ac40ad3b
TG
889static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
890{
891 switch (ctrl) {
892 case PR_SPEC_ENABLE:
893 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
894 return 0;
895 /*
896 * Indirect branch speculation is always disabled in strict
897 * mode.
898 */
fbed8eca
TL
899 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
900 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
ac40ad3b
TG
901 return -EPERM;
902 task_clear_spec_ib_disable(task);
903 task_update_spec_tif(task);
904 break;
905 case PR_SPEC_DISABLE:
906 case PR_SPEC_FORCE_DISABLE:
907 /*
908 * Indirect branch speculation is always allowed when
909 * mitigation is force disabled.
910 */
911 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
912 return -EPERM;
fbed8eca
TL
913 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
914 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
ac40ad3b
TG
915 return 0;
916 task_set_spec_ib_disable(task);
917 if (ctrl == PR_SPEC_FORCE_DISABLE)
918 task_set_spec_ib_force_disable(task);
919 task_update_spec_tif(task);
920 break;
921 default:
922 return -ERANGE;
923 }
924 return 0;
925}
926
5b38e244
TG
927int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
928 unsigned long ctrl)
929{
930 switch (which) {
931 case PR_SPEC_STORE_BYPASS:
932 return ssb_prctl_set(task, ctrl);
ac40ad3b
TG
933 case PR_SPEC_INDIRECT_BRANCH:
934 return ib_prctl_set(task, ctrl);
5b38e244
TG
935 default:
936 return -ENODEV;
937 }
938}
939
940#ifdef CONFIG_SECCOMP
941void arch_seccomp_spec_mitigate(struct task_struct *task)
942{
c7416003
KC
943 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
944 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1c3cf627
TG
945 if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
946 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
5b38e244
TG
947}
948#endif
949
199bfed2 950static int ssb_prctl_get(struct task_struct *task)
574dcf89
TG
951{
952 switch (ssb_mode) {
953 case SPEC_STORE_BYPASS_DISABLE:
954 return PR_SPEC_DISABLE;
c7416003 955 case SPEC_STORE_BYPASS_SECCOMP:
574dcf89 956 case SPEC_STORE_BYPASS_PRCTL:
733f4234
TG
957 if (task_spec_ssb_force_disable(task))
958 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
959 if (task_spec_ssb_disable(task))
574dcf89
TG
960 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
961 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
962 default:
963 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
964 return PR_SPEC_ENABLE;
965 return PR_SPEC_NOT_AFFECTED;
966 }
967}
968
ac40ad3b
TG
969static int ib_prctl_get(struct task_struct *task)
970{
971 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
972 return PR_SPEC_NOT_AFFECTED;
973
974 switch (spectre_v2_user) {
975 case SPECTRE_V2_USER_NONE:
976 return PR_SPEC_ENABLE;
977 case SPECTRE_V2_USER_PRCTL:
1c3cf627 978 case SPECTRE_V2_USER_SECCOMP:
ac40ad3b
TG
979 if (task_spec_ib_force_disable(task))
980 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
981 if (task_spec_ib_disable(task))
982 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
983 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
984 case SPECTRE_V2_USER_STRICT:
fbed8eca 985 case SPECTRE_V2_USER_STRICT_PREFERRED:
ac40ad3b
TG
986 return PR_SPEC_DISABLE;
987 default:
988 return PR_SPEC_NOT_AFFECTED;
989 }
990}
991
199bfed2 992int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
574dcf89
TG
993{
994 switch (which) {
995 case PR_SPEC_STORE_BYPASS:
199bfed2 996 return ssb_prctl_get(task);
ac40ad3b
TG
997 case PR_SPEC_INDIRECT_BRANCH:
998 return ib_prctl_get(task);
574dcf89
TG
999 default:
1000 return -ENODEV;
1001 }
1002}
1003
23b9eab9
KRW
1004void x86_spec_ctrl_setup_ap(void)
1005{
50f9b919 1006 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
208efa83 1007 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
c37b94dd
KRW
1008
1009 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
8fe36c9d 1010 x86_amd_ssb_disable();
23b9eab9
KRW
1011}
1012
73ca9f47
KRW
1013#undef pr_fmt
1014#define pr_fmt(fmt) "L1TF: " fmt
b9cfedcd 1015
24fcb53c
JK
1016/* Default mitigation for L1TF-affected CPUs */
1017enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
b9cfedcd 1018#if IS_ENABLED(CONFIG_KVM_INTEL)
24fcb53c
JK
1019EXPORT_SYMBOL_GPL(l1tf_mitigation);
1020
522c7bed 1021enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
b9cfedcd
TG
1022EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1023#endif
1024
f1f016ed
AK
1025/*
1026 * These CPUs all support 44bits physical address space internally in the
1027 * cache but CPUID can report a smaller number of physical address bits.
1028 *
1029 * The L1TF mitigation uses the top most address bit for the inversion of
1030 * non present PTEs. When the installed memory reaches into the top most
1031 * address bit due to memory holes, which has been observed on machines
1032 * which report 36bits physical address bits and have 32G RAM installed,
1033 * then the mitigation range check in l1tf_select_mitigation() triggers.
1034 * This is a false positive because the mitigation is still possible due to
1035 * the fact that the cache uses 44bit internally. Use the cache bits
1036 * instead of the reported physical bits and adjust them on the affected
1037 * machines to 44bit if the reported bits are less than 44.
1038 */
1039static void override_cache_bits(struct cpuinfo_x86 *c)
1040{
1041 if (c->x86 != 6)
1042 return;
1043
1044 switch (c->x86_model) {
1045 case INTEL_FAM6_NEHALEM:
1046 case INTEL_FAM6_WESTMERE:
1047 case INTEL_FAM6_SANDYBRIDGE:
1048 case INTEL_FAM6_IVYBRIDGE:
1049 case INTEL_FAM6_HASWELL_CORE:
1050 case INTEL_FAM6_HASWELL_ULT:
1051 case INTEL_FAM6_HASWELL_GT3E:
1052 case INTEL_FAM6_BROADWELL_CORE:
1053 case INTEL_FAM6_BROADWELL_GT3E:
1054 case INTEL_FAM6_SKYLAKE_MOBILE:
1055 case INTEL_FAM6_SKYLAKE_DESKTOP:
1056 case INTEL_FAM6_KABYLAKE_MOBILE:
1057 case INTEL_FAM6_KABYLAKE_DESKTOP:
1058 if (c->x86_cache_bits < 44)
1059 c->x86_cache_bits = 44;
1060 break;
1061 }
1062}
1063
73ca9f47
KRW
1064static void __init l1tf_select_mitigation(void)
1065{
1066 u64 half_pa;
1067
1068 if (!boot_cpu_has_bug(X86_BUG_L1TF))
1069 return;
1070
f1f016ed
AK
1071 override_cache_bits(&boot_cpu_data);
1072
24fcb53c
JK
1073 switch (l1tf_mitigation) {
1074 case L1TF_MITIGATION_OFF:
1075 case L1TF_MITIGATION_FLUSH_NOWARN:
1076 case L1TF_MITIGATION_FLUSH:
1077 break;
1078 case L1TF_MITIGATION_FLUSH_NOSMT:
1079 case L1TF_MITIGATION_FULL:
1080 cpu_smt_disable(false);
1081 break;
1082 case L1TF_MITIGATION_FULL_FORCE:
1083 cpu_smt_disable(true);
1084 break;
1085 }
1086
73ca9f47
KRW
1087#if CONFIG_PGTABLE_LEVELS == 2
1088 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1089 return;
1090#endif
1091
73ca9f47
KRW
1092 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1093 if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
1094 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
f80e341a
VB
1095 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1096 half_pa);
1097 pr_info("However, doing so will make a part of your RAM unusable.\n");
1098 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
73ca9f47
KRW
1099 return;
1100 }
1101
1102 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1103}
24fcb53c
JK
1104
1105static int __init l1tf_cmdline(char *str)
1106{
1107 if (!boot_cpu_has_bug(X86_BUG_L1TF))
1108 return 0;
1109
1110 if (!str)
1111 return -EINVAL;
1112
1113 if (!strcmp(str, "off"))
1114 l1tf_mitigation = L1TF_MITIGATION_OFF;
1115 else if (!strcmp(str, "flush,nowarn"))
1116 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1117 else if (!strcmp(str, "flush"))
1118 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1119 else if (!strcmp(str, "flush,nosmt"))
1120 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1121 else if (!strcmp(str, "full"))
1122 l1tf_mitigation = L1TF_MITIGATION_FULL;
1123 else if (!strcmp(str, "full,force"))
1124 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1125
1126 return 0;
1127}
1128early_param("l1tf", l1tf_cmdline);
1129
73ca9f47
KRW
1130#undef pr_fmt
1131
61dc0f55 1132#ifdef CONFIG_SYSFS
d2b8fc2d 1133
b9cfedcd
TG
1134#define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1135
1136#if IS_ENABLED(CONFIG_KVM_INTEL)
0eb5928b 1137static const char * const l1tf_vmx_states[] = {
1ead4979
TG
1138 [VMENTER_L1D_FLUSH_AUTO] = "auto",
1139 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable",
1140 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes",
1141 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes",
1142 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled",
364a4311 1143 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary"
b9cfedcd
TG
1144};
1145
1146static ssize_t l1tf_show_state(char *buf)
1147{
1148 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1149 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1150
3899f7b1
PB
1151 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1152 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
8bc59ba0 1153 sched_smt_active())) {
3899f7b1
PB
1154 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1155 l1tf_vmx_states[l1tf_vmx_mitigation]);
8bc59ba0 1156 }
3899f7b1
PB
1157
1158 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1159 l1tf_vmx_states[l1tf_vmx_mitigation],
8bc59ba0 1160 sched_smt_active() ? "vulnerable" : "disabled");
b9cfedcd
TG
1161}
1162#else
1163static ssize_t l1tf_show_state(char *buf)
1164{
1165 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1166}
1167#endif
1168
1122cc78
TG
1169static ssize_t mds_show_state(char *buf)
1170{
1171 if (!hypervisor_is_type(X86_HYPER_NATIVE)) {
1172 return sprintf(buf, "%s; SMT Host state unknown\n",
1173 mds_strings[mds_mitigation]);
1174 }
1175
1176 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1177 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1178 sched_smt_active() ? "mitigated" : "disabled");
1179 }
1180
1181 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1182 sched_smt_active() ? "vulnerable" : "disabled");
1183}
1184
2daa53e8
TC
1185static char *stibp_state(void)
1186{
4a8d91c8
TC
1187 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1188 return "";
1189
6cd930f0
TG
1190 switch (spectre_v2_user) {
1191 case SPECTRE_V2_USER_NONE:
1192 return ", STIBP: disabled";
1193 case SPECTRE_V2_USER_STRICT:
1194 return ", STIBP: forced";
fbed8eca
TL
1195 case SPECTRE_V2_USER_STRICT_PREFERRED:
1196 return ", STIBP: always-on";
ac40ad3b 1197 case SPECTRE_V2_USER_PRCTL:
1c3cf627 1198 case SPECTRE_V2_USER_SECCOMP:
2cdf6a58
TG
1199 if (static_key_enabled(&switch_to_cond_stibp))
1200 return ", STIBP: conditional";
6cd930f0
TG
1201 }
1202 return "";
2daa53e8
TC
1203}
1204
1205static char *ibpb_state(void)
1206{
58b2d4c8 1207 if (boot_cpu_has(X86_FEATURE_IBPB)) {
2cdf6a58 1208 if (static_key_enabled(&switch_mm_always_ibpb))
58b2d4c8 1209 return ", IBPB: always-on";
2cdf6a58
TG
1210 if (static_key_enabled(&switch_mm_cond_ibpb))
1211 return ", IBPB: conditional";
1212 return ", IBPB: disabled";
58b2d4c8
TG
1213 }
1214 return "";
2daa53e8
TC
1215}
1216
ace051d5 1217static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
043d480e 1218 char *buf, unsigned int bug)
61dc0f55 1219{
d2b8fc2d 1220 if (!boot_cpu_has_bug(bug))
61dc0f55 1221 return sprintf(buf, "Not affected\n");
d2b8fc2d
KRW
1222
1223 switch (bug) {
1224 case X86_BUG_CPU_MELTDOWN:
1225 if (boot_cpu_has(X86_FEATURE_PTI))
1226 return sprintf(buf, "Mitigation: PTI\n");
1227
1228 break;
1229
1230 case X86_BUG_SPECTRE_V1:
1231 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
1232
1233 case X86_BUG_SPECTRE_V2:
0b96b80d 1234 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
2daa53e8 1235 ibpb_state(),
d2b8fc2d 1236 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
2daa53e8 1237 stibp_state(),
5e2fafd5 1238 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
d2b8fc2d
KRW
1239 spectre_v2_module_string());
1240
e63490c8
KRW
1241 case X86_BUG_SPEC_STORE_BYPASS:
1242 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1243
05516ad8
AK
1244 case X86_BUG_L1TF:
1245 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
b9cfedcd 1246 return l1tf_show_state(buf);
05516ad8 1247 break;
1122cc78
TG
1248
1249 case X86_BUG_MDS:
1250 return mds_show_state(buf);
1251
d2b8fc2d
KRW
1252 default:
1253 break;
1254 }
1255
61dc0f55
TG
1256 return sprintf(buf, "Vulnerable\n");
1257}
1258
d2b8fc2d
KRW
1259ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1260{
1261 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1262}
1263
713f1b95 1264ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
61dc0f55 1265{
d2b8fc2d 1266 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
61dc0f55
TG
1267}
1268
713f1b95 1269ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
61dc0f55 1270{
d2b8fc2d 1271 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
61dc0f55 1272}
d7de9182
KRW
1273
1274ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1275{
1276 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1277}
05516ad8
AK
1278
1279ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1280{
1281 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1282}
1122cc78
TG
1283
1284ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1285{
1286 return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1287}
61dc0f55 1288#endif