1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1994 Linus Torvalds
5 * Cyrix stuff, June 1998 by:
6 * - Rafael R. Reilova (moved everything from head.S),
7 * <rreilova@ececs.uc.edu>
8 * - Channing Corn (tests & fixes),
9 * - Andrew D. Balsa (code cleanup).
11 #include <linux/init.h>
12 #include <linux/utsname.h>
13 #include <linux/cpu.h>
14 #include <linux/module.h>
15 #include <linux/nospec.h>
16 #include <linux/prctl.h>
17 #include <linux/sched/smt.h>
18 #include <linux/pgtable.h>
19 #include <linux/bpf.h>
21 #include <asm/spec-ctrl.h>
22 #include <asm/cmdline.h>
24 #include <asm/processor.h>
25 #include <asm/processor-flags.h>
26 #include <asm/fpu/api.h>
29 #include <asm/paravirt.h>
30 #include <asm/alternative.h>
31 #include <asm/set_memory.h>
32 #include <asm/intel-family.h>
33 #include <asm/e820/api.h>
34 #include <asm/hypervisor.h>
35 #include <asm/tlbflush.h>
39 static void __init
spectre_v1_select_mitigation(void);
40 static void __init
spectre_v2_select_mitigation(void);
41 static void __init
retbleed_select_mitigation(void);
42 static void __init
spectre_v2_user_select_mitigation(void);
43 static void __init
ssb_select_mitigation(void);
44 static void __init
l1tf_select_mitigation(void);
45 static void __init
mds_select_mitigation(void);
46 static void __init
md_clear_update_mitigation(void);
47 static void __init
md_clear_select_mitigation(void);
48 static void __init
taa_select_mitigation(void);
49 static void __init
mmio_select_mitigation(void);
50 static void __init
srbds_select_mitigation(void);
51 static void __init
l1d_flush_select_mitigation(void);
53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
54 u64 x86_spec_ctrl_base
;
55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base
);
57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
58 DEFINE_PER_CPU(u64
, x86_spec_ctrl_current
);
59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current
);
61 static DEFINE_MUTEX(spec_ctrl_mutex
);
64 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
65 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
67 void write_spec_ctrl_current(u64 val
, bool force
)
69 if (this_cpu_read(x86_spec_ctrl_current
) == val
)
72 this_cpu_write(x86_spec_ctrl_current
, val
);
75 * When KERNEL_IBRS this MSR is written on return-to-user, unless
76 * forced the update can be delayed until that time.
78 if (force
|| !cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS
))
79 wrmsrl(MSR_IA32_SPEC_CTRL
, val
);
82 u64
spec_ctrl_current(void)
84 return this_cpu_read(x86_spec_ctrl_current
);
86 EXPORT_SYMBOL_GPL(spec_ctrl_current
);
89 * AMD specific MSR info for Speculative Store Bypass control.
90 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
92 u64 __ro_after_init x86_amd_ls_cfg_base
;
93 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask
;
95 /* Control conditional STIBP in switch_to() */
96 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp
);
97 /* Control conditional IBPB in switch_mm() */
98 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb
);
99 /* Control unconditional IBPB in switch_mm() */
100 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb
);
102 /* Control MDS CPU buffer clear before returning to user space */
103 DEFINE_STATIC_KEY_FALSE(mds_user_clear
);
104 EXPORT_SYMBOL_GPL(mds_user_clear
);
105 /* Control MDS CPU buffer clear before idling (halt, mwait) */
106 DEFINE_STATIC_KEY_FALSE(mds_idle_clear
);
107 EXPORT_SYMBOL_GPL(mds_idle_clear
);
110 * Controls whether l1d flush based mitigations are enabled,
111 * based on hw features and admin setting via boot parameter
114 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush
);
116 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
117 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear
);
118 EXPORT_SYMBOL_GPL(mmio_stale_data_clear
);
120 void __init
check_bugs(void)
125 * identify_boot_cpu() initialized SMT support information, let the
128 cpu_smt_check_topology();
130 if (!IS_ENABLED(CONFIG_SMP
)) {
132 print_cpu_info(&boot_cpu_data
);
136 * Read the SPEC_CTRL MSR to account for reserved bits which may
137 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
138 * init code as it is not enumerated and depends on the family.
140 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
141 rdmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
143 /* Select the proper CPU mitigations before patching alternatives: */
144 spectre_v1_select_mitigation();
145 spectre_v2_select_mitigation();
147 * retbleed_select_mitigation() relies on the state set by
148 * spectre_v2_select_mitigation(); specifically it wants to know about
151 retbleed_select_mitigation();
153 * spectre_v2_user_select_mitigation() relies on the state set by
154 * retbleed_select_mitigation(); specifically the STIBP selection is
157 spectre_v2_user_select_mitigation();
158 ssb_select_mitigation();
159 l1tf_select_mitigation();
160 md_clear_select_mitigation();
161 srbds_select_mitigation();
162 l1d_flush_select_mitigation();
168 * Check whether we are able to run this kernel safely on SMP.
170 * - i386 is no longer supported.
171 * - In order to run on anything without a TSC, we need to be
172 * compiled for a i486.
174 if (boot_cpu_data
.x86
< 4)
175 panic("Kernel requires i486+ for 'invlpg' and other features");
177 init_utsname()->machine
[1] =
178 '0' + (boot_cpu_data
.x86
> 6 ? 6 : boot_cpu_data
.x86
);
179 alternative_instructions();
181 fpu__init_check_bugs();
182 #else /* CONFIG_X86_64 */
183 alternative_instructions();
186 * Make sure the first 2MB area is not mapped by huge pages
187 * There are typically fixed size MTRRs in there and overlapping
188 * MTRRs into large pages causes slow downs.
190 * Right now we don't do that with gbpages because there seems
191 * very little benefit for that case.
194 set_memory_4k((unsigned long)__va(0), 1);
199 * NOTE: This function is *only* called for SVM. VMX spec_ctrl handling is
203 x86_virt_spec_ctrl(u64 guest_spec_ctrl
, u64 guest_virt_spec_ctrl
, bool setguest
)
205 u64 msrval
, guestval
= guest_spec_ctrl
, hostval
= spec_ctrl_current();
206 struct thread_info
*ti
= current_thread_info();
208 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
)) {
209 if (hostval
!= guestval
) {
210 msrval
= setguest
? guestval
: hostval
;
211 wrmsrl(MSR_IA32_SPEC_CTRL
, msrval
);
216 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
217 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
219 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD
) &&
220 !static_cpu_has(X86_FEATURE_VIRT_SSBD
))
224 * If the host has SSBD mitigation enabled, force it in the host's
225 * virtual MSR value. If its not permanently enabled, evaluate
226 * current's TIF_SSBD thread flag.
228 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
))
229 hostval
= SPEC_CTRL_SSBD
;
231 hostval
= ssbd_tif_to_spec_ctrl(ti
->flags
);
233 /* Sanitize the guest value */
234 guestval
= guest_virt_spec_ctrl
& SPEC_CTRL_SSBD
;
236 if (hostval
!= guestval
) {
239 tif
= setguest
? ssbd_spec_ctrl_to_tif(guestval
) :
240 ssbd_spec_ctrl_to_tif(hostval
);
242 speculation_ctrl_update(tif
);
245 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl
);
247 static void x86_amd_ssb_disable(void)
249 u64 msrval
= x86_amd_ls_cfg_base
| x86_amd_ls_cfg_ssbd_mask
;
251 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD
))
252 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL
, SPEC_CTRL_SSBD
);
253 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD
))
254 wrmsrl(MSR_AMD64_LS_CFG
, msrval
);
258 #define pr_fmt(fmt) "MDS: " fmt
260 /* Default mitigation for MDS-affected CPUs */
261 static enum mds_mitigations mds_mitigation __ro_after_init
= MDS_MITIGATION_FULL
;
262 static bool mds_nosmt __ro_after_init
= false;
264 static const char * const mds_strings
[] = {
265 [MDS_MITIGATION_OFF
] = "Vulnerable",
266 [MDS_MITIGATION_FULL
] = "Mitigation: Clear CPU buffers",
267 [MDS_MITIGATION_VMWERV
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
270 static void __init
mds_select_mitigation(void)
272 if (!boot_cpu_has_bug(X86_BUG_MDS
) || cpu_mitigations_off()) {
273 mds_mitigation
= MDS_MITIGATION_OFF
;
277 if (mds_mitigation
== MDS_MITIGATION_FULL
) {
278 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR
))
279 mds_mitigation
= MDS_MITIGATION_VMWERV
;
281 static_branch_enable(&mds_user_clear
);
283 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY
) &&
284 (mds_nosmt
|| cpu_mitigations_auto_nosmt()))
285 cpu_smt_disable(false);
289 static int __init
mds_cmdline(char *str
)
291 if (!boot_cpu_has_bug(X86_BUG_MDS
))
297 if (!strcmp(str
, "off"))
298 mds_mitigation
= MDS_MITIGATION_OFF
;
299 else if (!strcmp(str
, "full"))
300 mds_mitigation
= MDS_MITIGATION_FULL
;
301 else if (!strcmp(str
, "full,nosmt")) {
302 mds_mitigation
= MDS_MITIGATION_FULL
;
308 early_param("mds", mds_cmdline
);
311 #define pr_fmt(fmt) "TAA: " fmt
313 enum taa_mitigations
{
315 TAA_MITIGATION_UCODE_NEEDED
,
317 TAA_MITIGATION_TSX_DISABLED
,
320 /* Default mitigation for TAA-affected CPUs */
321 static enum taa_mitigations taa_mitigation __ro_after_init
= TAA_MITIGATION_VERW
;
322 static bool taa_nosmt __ro_after_init
;
324 static const char * const taa_strings
[] = {
325 [TAA_MITIGATION_OFF
] = "Vulnerable",
326 [TAA_MITIGATION_UCODE_NEEDED
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
327 [TAA_MITIGATION_VERW
] = "Mitigation: Clear CPU buffers",
328 [TAA_MITIGATION_TSX_DISABLED
] = "Mitigation: TSX disabled",
331 static void __init
taa_select_mitigation(void)
335 if (!boot_cpu_has_bug(X86_BUG_TAA
)) {
336 taa_mitigation
= TAA_MITIGATION_OFF
;
340 /* TSX previously disabled by tsx=off */
341 if (!boot_cpu_has(X86_FEATURE_RTM
)) {
342 taa_mitigation
= TAA_MITIGATION_TSX_DISABLED
;
346 if (cpu_mitigations_off()) {
347 taa_mitigation
= TAA_MITIGATION_OFF
;
352 * TAA mitigation via VERW is turned off if both
353 * tsx_async_abort=off and mds=off are specified.
355 if (taa_mitigation
== TAA_MITIGATION_OFF
&&
356 mds_mitigation
== MDS_MITIGATION_OFF
)
359 if (boot_cpu_has(X86_FEATURE_MD_CLEAR
))
360 taa_mitigation
= TAA_MITIGATION_VERW
;
362 taa_mitigation
= TAA_MITIGATION_UCODE_NEEDED
;
365 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
366 * A microcode update fixes this behavior to clear CPU buffers. It also
367 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
368 * ARCH_CAP_TSX_CTRL_MSR bit.
370 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
371 * update is required.
373 ia32_cap
= x86_read_arch_cap_msr();
374 if ( (ia32_cap
& ARCH_CAP_MDS_NO
) &&
375 !(ia32_cap
& ARCH_CAP_TSX_CTRL_MSR
))
376 taa_mitigation
= TAA_MITIGATION_UCODE_NEEDED
;
379 * TSX is enabled, select alternate mitigation for TAA which is
380 * the same as MDS. Enable MDS static branch to clear CPU buffers.
382 * For guests that can't determine whether the correct microcode is
383 * present on host, enable the mitigation for UCODE_NEEDED as well.
385 static_branch_enable(&mds_user_clear
);
387 if (taa_nosmt
|| cpu_mitigations_auto_nosmt())
388 cpu_smt_disable(false);
391 static int __init
tsx_async_abort_parse_cmdline(char *str
)
393 if (!boot_cpu_has_bug(X86_BUG_TAA
))
399 if (!strcmp(str
, "off")) {
400 taa_mitigation
= TAA_MITIGATION_OFF
;
401 } else if (!strcmp(str
, "full")) {
402 taa_mitigation
= TAA_MITIGATION_VERW
;
403 } else if (!strcmp(str
, "full,nosmt")) {
404 taa_mitigation
= TAA_MITIGATION_VERW
;
410 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline
);
413 #define pr_fmt(fmt) "MMIO Stale Data: " fmt
415 enum mmio_mitigations
{
417 MMIO_MITIGATION_UCODE_NEEDED
,
418 MMIO_MITIGATION_VERW
,
421 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
422 static enum mmio_mitigations mmio_mitigation __ro_after_init
= MMIO_MITIGATION_VERW
;
423 static bool mmio_nosmt __ro_after_init
= false;
425 static const char * const mmio_strings
[] = {
426 [MMIO_MITIGATION_OFF
] = "Vulnerable",
427 [MMIO_MITIGATION_UCODE_NEEDED
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
428 [MMIO_MITIGATION_VERW
] = "Mitigation: Clear CPU buffers",
431 static void __init
mmio_select_mitigation(void)
435 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
) ||
436 cpu_mitigations_off()) {
437 mmio_mitigation
= MMIO_MITIGATION_OFF
;
441 if (mmio_mitigation
== MMIO_MITIGATION_OFF
)
444 ia32_cap
= x86_read_arch_cap_msr();
447 * Enable CPU buffer clear mitigation for host and VMM, if also affected
448 * by MDS or TAA. Otherwise, enable mitigation for VMM only.
450 if (boot_cpu_has_bug(X86_BUG_MDS
) || (boot_cpu_has_bug(X86_BUG_TAA
) &&
451 boot_cpu_has(X86_FEATURE_RTM
)))
452 static_branch_enable(&mds_user_clear
);
454 static_branch_enable(&mmio_stale_data_clear
);
457 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
458 * be propagated to uncore buffers, clearing the Fill buffers on idle
459 * is required irrespective of SMT state.
461 if (!(ia32_cap
& ARCH_CAP_FBSDP_NO
))
462 static_branch_enable(&mds_idle_clear
);
465 * Check if the system has the right microcode.
467 * CPU Fill buffer clear mitigation is enumerated by either an explicit
468 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
471 if ((ia32_cap
& ARCH_CAP_FB_CLEAR
) ||
472 (boot_cpu_has(X86_FEATURE_MD_CLEAR
) &&
473 boot_cpu_has(X86_FEATURE_FLUSH_L1D
) &&
474 !(ia32_cap
& ARCH_CAP_MDS_NO
)))
475 mmio_mitigation
= MMIO_MITIGATION_VERW
;
477 mmio_mitigation
= MMIO_MITIGATION_UCODE_NEEDED
;
479 if (mmio_nosmt
|| cpu_mitigations_auto_nosmt())
480 cpu_smt_disable(false);
483 static int __init
mmio_stale_data_parse_cmdline(char *str
)
485 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
491 if (!strcmp(str
, "off")) {
492 mmio_mitigation
= MMIO_MITIGATION_OFF
;
493 } else if (!strcmp(str
, "full")) {
494 mmio_mitigation
= MMIO_MITIGATION_VERW
;
495 } else if (!strcmp(str
, "full,nosmt")) {
496 mmio_mitigation
= MMIO_MITIGATION_VERW
;
502 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline
);
505 #define pr_fmt(fmt) "" fmt
507 static void __init
md_clear_update_mitigation(void)
509 if (cpu_mitigations_off())
512 if (!static_key_enabled(&mds_user_clear
))
516 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
517 * mitigation, if necessary.
519 if (mds_mitigation
== MDS_MITIGATION_OFF
&&
520 boot_cpu_has_bug(X86_BUG_MDS
)) {
521 mds_mitigation
= MDS_MITIGATION_FULL
;
522 mds_select_mitigation();
524 if (taa_mitigation
== TAA_MITIGATION_OFF
&&
525 boot_cpu_has_bug(X86_BUG_TAA
)) {
526 taa_mitigation
= TAA_MITIGATION_VERW
;
527 taa_select_mitigation();
529 if (mmio_mitigation
== MMIO_MITIGATION_OFF
&&
530 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
)) {
531 mmio_mitigation
= MMIO_MITIGATION_VERW
;
532 mmio_select_mitigation();
535 if (boot_cpu_has_bug(X86_BUG_MDS
))
536 pr_info("MDS: %s\n", mds_strings
[mds_mitigation
]);
537 if (boot_cpu_has_bug(X86_BUG_TAA
))
538 pr_info("TAA: %s\n", taa_strings
[taa_mitigation
]);
539 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
540 pr_info("MMIO Stale Data: %s\n", mmio_strings
[mmio_mitigation
]);
543 static void __init
md_clear_select_mitigation(void)
545 mds_select_mitigation();
546 taa_select_mitigation();
547 mmio_select_mitigation();
550 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
551 * and print their mitigation after MDS, TAA and MMIO Stale Data
552 * mitigation selection is done.
554 md_clear_update_mitigation();
558 #define pr_fmt(fmt) "SRBDS: " fmt
560 enum srbds_mitigations
{
561 SRBDS_MITIGATION_OFF
,
562 SRBDS_MITIGATION_UCODE_NEEDED
,
563 SRBDS_MITIGATION_FULL
,
564 SRBDS_MITIGATION_TSX_OFF
,
565 SRBDS_MITIGATION_HYPERVISOR
,
568 static enum srbds_mitigations srbds_mitigation __ro_after_init
= SRBDS_MITIGATION_FULL
;
570 static const char * const srbds_strings
[] = {
571 [SRBDS_MITIGATION_OFF
] = "Vulnerable",
572 [SRBDS_MITIGATION_UCODE_NEEDED
] = "Vulnerable: No microcode",
573 [SRBDS_MITIGATION_FULL
] = "Mitigation: Microcode",
574 [SRBDS_MITIGATION_TSX_OFF
] = "Mitigation: TSX disabled",
575 [SRBDS_MITIGATION_HYPERVISOR
] = "Unknown: Dependent on hypervisor status",
578 static bool srbds_off
;
580 void update_srbds_msr(void)
584 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
587 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
))
590 if (srbds_mitigation
== SRBDS_MITIGATION_UCODE_NEEDED
)
593 rdmsrl(MSR_IA32_MCU_OPT_CTRL
, mcu_ctrl
);
595 switch (srbds_mitigation
) {
596 case SRBDS_MITIGATION_OFF
:
597 case SRBDS_MITIGATION_TSX_OFF
:
598 mcu_ctrl
|= RNGDS_MITG_DIS
;
600 case SRBDS_MITIGATION_FULL
:
601 mcu_ctrl
&= ~RNGDS_MITG_DIS
;
607 wrmsrl(MSR_IA32_MCU_OPT_CTRL
, mcu_ctrl
);
610 static void __init
srbds_select_mitigation(void)
614 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
618 * Check to see if this is one of the MDS_NO systems supporting TSX that
619 * are only exposed to SRBDS when TSX is enabled or when CPU is affected
620 * by Processor MMIO Stale Data vulnerability.
622 ia32_cap
= x86_read_arch_cap_msr();
623 if ((ia32_cap
& ARCH_CAP_MDS_NO
) && !boot_cpu_has(X86_FEATURE_RTM
) &&
624 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
625 srbds_mitigation
= SRBDS_MITIGATION_TSX_OFF
;
626 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR
))
627 srbds_mitigation
= SRBDS_MITIGATION_HYPERVISOR
;
628 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL
))
629 srbds_mitigation
= SRBDS_MITIGATION_UCODE_NEEDED
;
630 else if (cpu_mitigations_off() || srbds_off
)
631 srbds_mitigation
= SRBDS_MITIGATION_OFF
;
634 pr_info("%s\n", srbds_strings
[srbds_mitigation
]);
637 static int __init
srbds_parse_cmdline(char *str
)
642 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
645 srbds_off
= !strcmp(str
, "off");
648 early_param("srbds", srbds_parse_cmdline
);
651 #define pr_fmt(fmt) "L1D Flush : " fmt
653 enum l1d_flush_mitigations
{
658 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata
= L1D_FLUSH_OFF
;
660 static void __init
l1d_flush_select_mitigation(void)
662 if (!l1d_flush_mitigation
|| !boot_cpu_has(X86_FEATURE_FLUSH_L1D
))
665 static_branch_enable(&switch_mm_cond_l1d_flush
);
666 pr_info("Conditional flush on switch_mm() enabled\n");
669 static int __init
l1d_flush_parse_cmdline(char *str
)
671 if (!strcmp(str
, "on"))
672 l1d_flush_mitigation
= L1D_FLUSH_ON
;
676 early_param("l1d_flush", l1d_flush_parse_cmdline
);
679 #define pr_fmt(fmt) "Spectre V1 : " fmt
681 enum spectre_v1_mitigation
{
682 SPECTRE_V1_MITIGATION_NONE
,
683 SPECTRE_V1_MITIGATION_AUTO
,
686 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init
=
687 SPECTRE_V1_MITIGATION_AUTO
;
689 static const char * const spectre_v1_strings
[] = {
690 [SPECTRE_V1_MITIGATION_NONE
] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
691 [SPECTRE_V1_MITIGATION_AUTO
] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
695 * Does SMAP provide full mitigation against speculative kernel access to
698 static bool smap_works_speculatively(void)
700 if (!boot_cpu_has(X86_FEATURE_SMAP
))
704 * On CPUs which are vulnerable to Meltdown, SMAP does not
705 * prevent speculative access to user data in the L1 cache.
706 * Consider SMAP to be non-functional as a mitigation on these
709 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN
))
715 static void __init
spectre_v1_select_mitigation(void)
717 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1
) || cpu_mitigations_off()) {
718 spectre_v1_mitigation
= SPECTRE_V1_MITIGATION_NONE
;
722 if (spectre_v1_mitigation
== SPECTRE_V1_MITIGATION_AUTO
) {
724 * With Spectre v1, a user can speculatively control either
725 * path of a conditional swapgs with a user-controlled GS
726 * value. The mitigation is to add lfences to both code paths.
728 * If FSGSBASE is enabled, the user can put a kernel address in
729 * GS, in which case SMAP provides no protection.
731 * If FSGSBASE is disabled, the user can only put a user space
732 * address in GS. That makes an attack harder, but still
733 * possible if there's no SMAP protection.
735 if (boot_cpu_has(X86_FEATURE_FSGSBASE
) ||
736 !smap_works_speculatively()) {
738 * Mitigation can be provided from SWAPGS itself or
739 * PTI as the CR3 write in the Meltdown mitigation
742 * If neither is there, mitigate with an LFENCE to
743 * stop speculation through swapgs.
745 if (boot_cpu_has_bug(X86_BUG_SWAPGS
) &&
746 !boot_cpu_has(X86_FEATURE_PTI
))
747 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER
);
750 * Enable lfences in the kernel entry (non-swapgs)
751 * paths, to prevent user entry from speculatively
754 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL
);
758 pr_info("%s\n", spectre_v1_strings
[spectre_v1_mitigation
]);
761 static int __init
nospectre_v1_cmdline(char *str
)
763 spectre_v1_mitigation
= SPECTRE_V1_MITIGATION_NONE
;
766 early_param("nospectre_v1", nospectre_v1_cmdline
);
768 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init
=
772 #define pr_fmt(fmt) "RETBleed: " fmt
774 enum retbleed_mitigation
{
775 RETBLEED_MITIGATION_NONE
,
776 RETBLEED_MITIGATION_UNRET
,
777 RETBLEED_MITIGATION_IBPB
,
778 RETBLEED_MITIGATION_IBRS
,
779 RETBLEED_MITIGATION_EIBRS
,
782 enum retbleed_mitigation_cmd
{
789 const char * const retbleed_strings
[] = {
790 [RETBLEED_MITIGATION_NONE
] = "Vulnerable",
791 [RETBLEED_MITIGATION_UNRET
] = "Mitigation: untrained return thunk",
792 [RETBLEED_MITIGATION_IBPB
] = "Mitigation: IBPB",
793 [RETBLEED_MITIGATION_IBRS
] = "Mitigation: IBRS",
794 [RETBLEED_MITIGATION_EIBRS
] = "Mitigation: Enhanced IBRS",
797 static enum retbleed_mitigation retbleed_mitigation __ro_after_init
=
798 RETBLEED_MITIGATION_NONE
;
799 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init
=
802 static int __ro_after_init retbleed_nosmt
= false;
804 static int __init
retbleed_parse_cmdline(char *str
)
810 char *next
= strchr(str
, ',');
816 if (!strcmp(str
, "off")) {
817 retbleed_cmd
= RETBLEED_CMD_OFF
;
818 } else if (!strcmp(str
, "auto")) {
819 retbleed_cmd
= RETBLEED_CMD_AUTO
;
820 } else if (!strcmp(str
, "unret")) {
821 retbleed_cmd
= RETBLEED_CMD_UNRET
;
822 } else if (!strcmp(str
, "ibpb")) {
823 retbleed_cmd
= RETBLEED_CMD_IBPB
;
824 } else if (!strcmp(str
, "nosmt")) {
825 retbleed_nosmt
= true;
827 pr_err("Ignoring unknown retbleed option (%s).", str
);
835 early_param("retbleed", retbleed_parse_cmdline
);
837 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
838 #define RETBLEED_COMPILER_MSG "WARNING: kernel not compiled with RETPOLINE or -mfunction-return capable compiler; falling back to IBPB!\n"
839 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
841 static void __init
retbleed_select_mitigation(void)
843 bool mitigate_smt
= false;
845 if (!boot_cpu_has_bug(X86_BUG_RETBLEED
) || cpu_mitigations_off())
848 switch (retbleed_cmd
) {
849 case RETBLEED_CMD_OFF
:
852 case RETBLEED_CMD_UNRET
:
853 retbleed_mitigation
= RETBLEED_MITIGATION_UNRET
;
856 case RETBLEED_CMD_IBPB
:
857 retbleed_mitigation
= RETBLEED_MITIGATION_IBPB
;
860 case RETBLEED_CMD_AUTO
:
862 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
||
863 boot_cpu_data
.x86_vendor
== X86_VENDOR_HYGON
)
864 retbleed_mitigation
= RETBLEED_MITIGATION_UNRET
;
867 * The Intel mitigation (IBRS or eIBRS) was already selected in
868 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will
869 * be set accordingly below.
875 switch (retbleed_mitigation
) {
876 case RETBLEED_MITIGATION_UNRET
:
878 if (!IS_ENABLED(CONFIG_RETPOLINE
) ||
879 !IS_ENABLED(CONFIG_CC_HAS_RETURN_THUNK
)) {
880 pr_err(RETBLEED_COMPILER_MSG
);
881 retbleed_mitigation
= RETBLEED_MITIGATION_IBPB
;
882 goto retbleed_force_ibpb
;
885 setup_force_cpu_cap(X86_FEATURE_RETHUNK
);
886 setup_force_cpu_cap(X86_FEATURE_UNRET
);
888 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
&&
889 boot_cpu_data
.x86_vendor
!= X86_VENDOR_HYGON
)
890 pr_err(RETBLEED_UNTRAIN_MSG
);
895 case RETBLEED_MITIGATION_IBPB
:
897 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB
);
905 if (mitigate_smt
&& !boot_cpu_has(X86_FEATURE_STIBP
) &&
906 (retbleed_nosmt
|| cpu_mitigations_auto_nosmt()))
907 cpu_smt_disable(false);
910 * Let IBRS trump all on Intel without affecting the effects of the
911 * retbleed= cmdline option.
913 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
) {
914 switch (spectre_v2_enabled
) {
915 case SPECTRE_V2_IBRS
:
916 retbleed_mitigation
= RETBLEED_MITIGATION_IBRS
;
918 case SPECTRE_V2_EIBRS
:
919 case SPECTRE_V2_EIBRS_RETPOLINE
:
920 case SPECTRE_V2_EIBRS_LFENCE
:
921 retbleed_mitigation
= RETBLEED_MITIGATION_EIBRS
;
924 pr_err(RETBLEED_INTEL_MSG
);
928 pr_info("%s\n", retbleed_strings
[retbleed_mitigation
]);
932 #define pr_fmt(fmt) "Spectre V2 : " fmt
934 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init
=
935 SPECTRE_V2_USER_NONE
;
936 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init
=
937 SPECTRE_V2_USER_NONE
;
939 #ifdef CONFIG_RETPOLINE
940 static bool spectre_v2_bad_module
;
942 bool retpoline_module_ok(bool has_retpoline
)
944 if (spectre_v2_enabled
== SPECTRE_V2_NONE
|| has_retpoline
)
947 pr_err("System may be vulnerable to spectre v2\n");
948 spectre_v2_bad_module
= true;
952 static inline const char *spectre_v2_module_string(void)
954 return spectre_v2_bad_module
? " - vulnerable module loaded" : "";
957 static inline const char *spectre_v2_module_string(void) { return ""; }
960 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
961 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
962 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
964 #ifdef CONFIG_BPF_SYSCALL
965 void unpriv_ebpf_notify(int new_state
)
970 /* Unprivileged eBPF is enabled */
972 switch (spectre_v2_enabled
) {
973 case SPECTRE_V2_EIBRS
:
974 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG
);
976 case SPECTRE_V2_EIBRS_LFENCE
:
977 if (sched_smt_active())
978 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG
);
986 static inline bool match_option(const char *arg
, int arglen
, const char *opt
)
988 int len
= strlen(opt
);
990 return len
== arglen
&& !strncmp(arg
, opt
, len
);
993 /* The kernel command line selection for spectre v2 */
994 enum spectre_v2_mitigation_cmd
{
997 SPECTRE_V2_CMD_FORCE
,
998 SPECTRE_V2_CMD_RETPOLINE
,
999 SPECTRE_V2_CMD_RETPOLINE_GENERIC
,
1000 SPECTRE_V2_CMD_RETPOLINE_LFENCE
,
1001 SPECTRE_V2_CMD_EIBRS
,
1002 SPECTRE_V2_CMD_EIBRS_RETPOLINE
,
1003 SPECTRE_V2_CMD_EIBRS_LFENCE
,
1004 SPECTRE_V2_CMD_IBRS
,
1007 enum spectre_v2_user_cmd
{
1008 SPECTRE_V2_USER_CMD_NONE
,
1009 SPECTRE_V2_USER_CMD_AUTO
,
1010 SPECTRE_V2_USER_CMD_FORCE
,
1011 SPECTRE_V2_USER_CMD_PRCTL
,
1012 SPECTRE_V2_USER_CMD_PRCTL_IBPB
,
1013 SPECTRE_V2_USER_CMD_SECCOMP
,
1014 SPECTRE_V2_USER_CMD_SECCOMP_IBPB
,
1017 static const char * const spectre_v2_user_strings
[] = {
1018 [SPECTRE_V2_USER_NONE
] = "User space: Vulnerable",
1019 [SPECTRE_V2_USER_STRICT
] = "User space: Mitigation: STIBP protection",
1020 [SPECTRE_V2_USER_STRICT_PREFERRED
] = "User space: Mitigation: STIBP always-on protection",
1021 [SPECTRE_V2_USER_PRCTL
] = "User space: Mitigation: STIBP via prctl",
1022 [SPECTRE_V2_USER_SECCOMP
] = "User space: Mitigation: STIBP via seccomp and prctl",
1025 static const struct {
1027 enum spectre_v2_user_cmd cmd
;
1029 } v2_user_options
[] __initconst
= {
1030 { "auto", SPECTRE_V2_USER_CMD_AUTO
, false },
1031 { "off", SPECTRE_V2_USER_CMD_NONE
, false },
1032 { "on", SPECTRE_V2_USER_CMD_FORCE
, true },
1033 { "prctl", SPECTRE_V2_USER_CMD_PRCTL
, false },
1034 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB
, false },
1035 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP
, false },
1036 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB
, false },
1039 static void __init
spec_v2_user_print_cond(const char *reason
, bool secure
)
1041 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
1042 pr_info("spectre_v2_user=%s forced on command line.\n", reason
);
1045 static __ro_after_init
enum spectre_v2_mitigation_cmd spectre_v2_cmd
;
1047 static enum spectre_v2_user_cmd __init
1048 spectre_v2_parse_user_cmdline(void)
1053 switch (spectre_v2_cmd
) {
1054 case SPECTRE_V2_CMD_NONE
:
1055 return SPECTRE_V2_USER_CMD_NONE
;
1056 case SPECTRE_V2_CMD_FORCE
:
1057 return SPECTRE_V2_USER_CMD_FORCE
;
1062 ret
= cmdline_find_option(boot_command_line
, "spectre_v2_user",
1065 return SPECTRE_V2_USER_CMD_AUTO
;
1067 for (i
= 0; i
< ARRAY_SIZE(v2_user_options
); i
++) {
1068 if (match_option(arg
, ret
, v2_user_options
[i
].option
)) {
1069 spec_v2_user_print_cond(v2_user_options
[i
].option
,
1070 v2_user_options
[i
].secure
);
1071 return v2_user_options
[i
].cmd
;
1075 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg
);
1076 return SPECTRE_V2_USER_CMD_AUTO
;
1079 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode
)
1081 return mode
== SPECTRE_V2_IBRS
||
1082 mode
== SPECTRE_V2_EIBRS
||
1083 mode
== SPECTRE_V2_EIBRS_RETPOLINE
||
1084 mode
== SPECTRE_V2_EIBRS_LFENCE
;
1088 spectre_v2_user_select_mitigation(void)
1090 enum spectre_v2_user_mitigation mode
= SPECTRE_V2_USER_NONE
;
1091 bool smt_possible
= IS_ENABLED(CONFIG_SMP
);
1092 enum spectre_v2_user_cmd cmd
;
1094 if (!boot_cpu_has(X86_FEATURE_IBPB
) && !boot_cpu_has(X86_FEATURE_STIBP
))
1097 if (cpu_smt_control
== CPU_SMT_FORCE_DISABLED
||
1098 cpu_smt_control
== CPU_SMT_NOT_SUPPORTED
)
1099 smt_possible
= false;
1101 cmd
= spectre_v2_parse_user_cmdline();
1103 case SPECTRE_V2_USER_CMD_NONE
:
1105 case SPECTRE_V2_USER_CMD_FORCE
:
1106 mode
= SPECTRE_V2_USER_STRICT
;
1108 case SPECTRE_V2_USER_CMD_PRCTL
:
1109 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
1110 mode
= SPECTRE_V2_USER_PRCTL
;
1112 case SPECTRE_V2_USER_CMD_AUTO
:
1113 case SPECTRE_V2_USER_CMD_SECCOMP
:
1114 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
1115 if (IS_ENABLED(CONFIG_SECCOMP
))
1116 mode
= SPECTRE_V2_USER_SECCOMP
;
1118 mode
= SPECTRE_V2_USER_PRCTL
;
1122 /* Initialize Indirect Branch Prediction Barrier */
1123 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
1124 setup_force_cpu_cap(X86_FEATURE_USE_IBPB
);
1126 spectre_v2_user_ibpb
= mode
;
1128 case SPECTRE_V2_USER_CMD_FORCE
:
1129 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
1130 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
1131 static_branch_enable(&switch_mm_always_ibpb
);
1132 spectre_v2_user_ibpb
= SPECTRE_V2_USER_STRICT
;
1134 case SPECTRE_V2_USER_CMD_PRCTL
:
1135 case SPECTRE_V2_USER_CMD_AUTO
:
1136 case SPECTRE_V2_USER_CMD_SECCOMP
:
1137 static_branch_enable(&switch_mm_cond_ibpb
);
1143 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1144 static_key_enabled(&switch_mm_always_ibpb
) ?
1145 "always-on" : "conditional");
1149 * If no STIBP, IBRS or enhanced IBRS is enabled, or SMT impossible,
1150 * STIBP is not required.
1152 if (!boot_cpu_has(X86_FEATURE_STIBP
) ||
1154 spectre_v2_in_ibrs_mode(spectre_v2_enabled
))
1158 * At this point, an STIBP mode other than "off" has been set.
1159 * If STIBP support is not being forced, check if STIBP always-on
1162 if (mode
!= SPECTRE_V2_USER_STRICT
&&
1163 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON
))
1164 mode
= SPECTRE_V2_USER_STRICT_PREFERRED
;
1166 if (retbleed_mitigation
== RETBLEED_MITIGATION_UNRET
) {
1167 if (mode
!= SPECTRE_V2_USER_STRICT
&&
1168 mode
!= SPECTRE_V2_USER_STRICT_PREFERRED
)
1169 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation'\n");
1170 mode
= SPECTRE_V2_USER_STRICT_PREFERRED
;
1173 spectre_v2_user_stibp
= mode
;
1176 pr_info("%s\n", spectre_v2_user_strings
[mode
]);
1179 static const char * const spectre_v2_strings
[] = {
1180 [SPECTRE_V2_NONE
] = "Vulnerable",
1181 [SPECTRE_V2_RETPOLINE
] = "Mitigation: Retpolines",
1182 [SPECTRE_V2_LFENCE
] = "Mitigation: LFENCE",
1183 [SPECTRE_V2_EIBRS
] = "Mitigation: Enhanced IBRS",
1184 [SPECTRE_V2_EIBRS_LFENCE
] = "Mitigation: Enhanced IBRS + LFENCE",
1185 [SPECTRE_V2_EIBRS_RETPOLINE
] = "Mitigation: Enhanced IBRS + Retpolines",
1186 [SPECTRE_V2_IBRS
] = "Mitigation: IBRS",
1189 static const struct {
1191 enum spectre_v2_mitigation_cmd cmd
;
1193 } mitigation_options
[] __initconst
= {
1194 { "off", SPECTRE_V2_CMD_NONE
, false },
1195 { "on", SPECTRE_V2_CMD_FORCE
, true },
1196 { "retpoline", SPECTRE_V2_CMD_RETPOLINE
, false },
1197 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE
, false },
1198 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE
, false },
1199 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC
, false },
1200 { "eibrs", SPECTRE_V2_CMD_EIBRS
, false },
1201 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE
, false },
1202 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE
, false },
1203 { "auto", SPECTRE_V2_CMD_AUTO
, false },
1204 { "ibrs", SPECTRE_V2_CMD_IBRS
, false },
1207 static void __init
spec_v2_print_cond(const char *reason
, bool secure
)
1209 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
1210 pr_info("%s selected on command line.\n", reason
);
1213 static enum spectre_v2_mitigation_cmd __init
spectre_v2_parse_cmdline(void)
1215 enum spectre_v2_mitigation_cmd cmd
= SPECTRE_V2_CMD_AUTO
;
1219 if (cmdline_find_option_bool(boot_command_line
, "nospectre_v2") ||
1220 cpu_mitigations_off())
1221 return SPECTRE_V2_CMD_NONE
;
1223 ret
= cmdline_find_option(boot_command_line
, "spectre_v2", arg
, sizeof(arg
));
1225 return SPECTRE_V2_CMD_AUTO
;
1227 for (i
= 0; i
< ARRAY_SIZE(mitigation_options
); i
++) {
1228 if (!match_option(arg
, ret
, mitigation_options
[i
].option
))
1230 cmd
= mitigation_options
[i
].cmd
;
1234 if (i
>= ARRAY_SIZE(mitigation_options
)) {
1235 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
1236 return SPECTRE_V2_CMD_AUTO
;
1239 if ((cmd
== SPECTRE_V2_CMD_RETPOLINE
||
1240 cmd
== SPECTRE_V2_CMD_RETPOLINE_LFENCE
||
1241 cmd
== SPECTRE_V2_CMD_RETPOLINE_GENERIC
||
1242 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
||
1243 cmd
== SPECTRE_V2_CMD_EIBRS_RETPOLINE
) &&
1244 !IS_ENABLED(CONFIG_RETPOLINE
)) {
1245 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1246 mitigation_options
[i
].option
);
1247 return SPECTRE_V2_CMD_AUTO
;
1250 if ((cmd
== SPECTRE_V2_CMD_EIBRS
||
1251 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
||
1252 cmd
== SPECTRE_V2_CMD_EIBRS_RETPOLINE
) &&
1253 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED
)) {
1254 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
1255 mitigation_options
[i
].option
);
1256 return SPECTRE_V2_CMD_AUTO
;
1259 if ((cmd
== SPECTRE_V2_CMD_RETPOLINE_LFENCE
||
1260 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
) &&
1261 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC
)) {
1262 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1263 mitigation_options
[i
].option
);
1264 return SPECTRE_V2_CMD_AUTO
;
1267 if (cmd
== SPECTRE_V2_CMD_IBRS
&& boot_cpu_data
.x86_vendor
!= X86_VENDOR_INTEL
) {
1268 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1269 mitigation_options
[i
].option
);
1270 return SPECTRE_V2_CMD_AUTO
;
1273 if (cmd
== SPECTRE_V2_CMD_IBRS
&& !boot_cpu_has(X86_FEATURE_IBRS
)) {
1274 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1275 mitigation_options
[i
].option
);
1276 return SPECTRE_V2_CMD_AUTO
;
1279 if (cmd
== SPECTRE_V2_CMD_IBRS
&& boot_cpu_has(X86_FEATURE_XENPV
)) {
1280 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1281 mitigation_options
[i
].option
);
1282 return SPECTRE_V2_CMD_AUTO
;
1285 spec_v2_print_cond(mitigation_options
[i
].option
,
1286 mitigation_options
[i
].secure
);
1290 static enum spectre_v2_mitigation __init
spectre_v2_select_retpoline(void)
1292 if (!IS_ENABLED(CONFIG_RETPOLINE
)) {
1293 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1294 return SPECTRE_V2_NONE
;
1297 return SPECTRE_V2_RETPOLINE
;
1300 static void __init
spectre_v2_select_mitigation(void)
1302 enum spectre_v2_mitigation_cmd cmd
= spectre_v2_parse_cmdline();
1303 enum spectre_v2_mitigation mode
= SPECTRE_V2_NONE
;
1306 * If the CPU is not affected and the command line mode is NONE or AUTO
1307 * then nothing to do.
1309 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) &&
1310 (cmd
== SPECTRE_V2_CMD_NONE
|| cmd
== SPECTRE_V2_CMD_AUTO
))
1314 case SPECTRE_V2_CMD_NONE
:
1317 case SPECTRE_V2_CMD_FORCE
:
1318 case SPECTRE_V2_CMD_AUTO
:
1319 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED
)) {
1320 mode
= SPECTRE_V2_EIBRS
;
1324 if (boot_cpu_has_bug(X86_BUG_RETBLEED
) &&
1325 retbleed_cmd
!= RETBLEED_CMD_OFF
&&
1326 boot_cpu_has(X86_FEATURE_IBRS
) &&
1327 boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
) {
1328 mode
= SPECTRE_V2_IBRS
;
1332 mode
= spectre_v2_select_retpoline();
1335 case SPECTRE_V2_CMD_RETPOLINE_LFENCE
:
1336 pr_err(SPECTRE_V2_LFENCE_MSG
);
1337 mode
= SPECTRE_V2_LFENCE
;
1340 case SPECTRE_V2_CMD_RETPOLINE_GENERIC
:
1341 mode
= SPECTRE_V2_RETPOLINE
;
1344 case SPECTRE_V2_CMD_RETPOLINE
:
1345 mode
= spectre_v2_select_retpoline();
1348 case SPECTRE_V2_CMD_IBRS
:
1349 mode
= SPECTRE_V2_IBRS
;
1352 case SPECTRE_V2_CMD_EIBRS
:
1353 mode
= SPECTRE_V2_EIBRS
;
1356 case SPECTRE_V2_CMD_EIBRS_LFENCE
:
1357 mode
= SPECTRE_V2_EIBRS_LFENCE
;
1360 case SPECTRE_V2_CMD_EIBRS_RETPOLINE
:
1361 mode
= SPECTRE_V2_EIBRS_RETPOLINE
;
1365 if (mode
== SPECTRE_V2_EIBRS
&& unprivileged_ebpf_enabled())
1366 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG
);
1368 if (spectre_v2_in_ibrs_mode(mode
)) {
1369 x86_spec_ctrl_base
|= SPEC_CTRL_IBRS
;
1370 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1374 case SPECTRE_V2_NONE
:
1375 case SPECTRE_V2_EIBRS
:
1378 case SPECTRE_V2_IBRS
:
1379 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS
);
1382 case SPECTRE_V2_LFENCE
:
1383 case SPECTRE_V2_EIBRS_LFENCE
:
1384 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE
);
1387 case SPECTRE_V2_RETPOLINE
:
1388 case SPECTRE_V2_EIBRS_RETPOLINE
:
1389 setup_force_cpu_cap(X86_FEATURE_RETPOLINE
);
1393 spectre_v2_enabled
= mode
;
1394 pr_info("%s\n", spectre_v2_strings
[mode
]);
1397 * If Spectre v2 protection has been enabled, fill the RSB during a
1398 * context switch. In general there are two types of RSB attacks
1399 * across context switches, for which the CALLs/RETs may be unbalanced.
1403 * Some Intel parts have "bottomless RSB". When the RSB is empty,
1404 * speculated return targets may come from the branch predictor,
1405 * which could have a user-poisoned BTB or BHB entry.
1407 * AMD has it even worse: *all* returns are speculated from the BTB,
1408 * regardless of the state of the RSB.
1410 * When IBRS or eIBRS is enabled, the "user -> kernel" attack
1411 * scenario is mitigated by the IBRS branch prediction isolation
1412 * properties, so the RSB buffer filling wouldn't be necessary to
1413 * protect against this type of attack.
1415 * The "user -> user" attack scenario is mitigated by RSB filling.
1417 * 2) Poisoned RSB entry
1419 * If the 'next' in-kernel return stack is shorter than 'prev',
1420 * 'next' could be tricked into speculating with a user-poisoned RSB
1423 * The "user -> kernel" attack scenario is mitigated by SMEP and
1426 * The "user -> user" scenario, also known as SpectreBHB, requires
1429 * So to mitigate all cases, unconditionally fill RSB on context
1432 * FIXME: Is this pointless for retbleed-affected AMD?
1434 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW
);
1435 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1438 * Similar to context switches, there are two types of RSB attacks
1443 * 2) Poisoned RSB entry
1445 * When retpoline is enabled, both are mitigated by filling/clearing
1448 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1449 * prediction isolation protections, RSB still needs to be cleared
1450 * because of #2. Note that SMEP provides no protection here, unlike
1451 * user-space-poisoned RSB entries.
1453 * eIBRS, on the other hand, has RSB-poisoning protections, so it
1454 * doesn't need RSB clearing after vmexit.
1456 if (boot_cpu_has(X86_FEATURE_RETPOLINE
) ||
1457 boot_cpu_has(X86_FEATURE_KERNEL_IBRS
))
1458 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT
);
1461 * Retpoline protects the kernel, but doesn't protect firmware. IBRS
1462 * and Enhanced IBRS protect firmware too, so enable IBRS around
1463 * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
1466 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1467 * the user might select retpoline on the kernel command line and if
1468 * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1469 * enable IBRS around firmware calls.
1471 if (boot_cpu_has(X86_FEATURE_IBRS
) && !spectre_v2_in_ibrs_mode(mode
)) {
1472 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW
);
1473 pr_info("Enabling Restricted Speculation for firmware calls\n");
1476 /* Set up IBPB and STIBP depending on the general spectre V2 command */
1477 spectre_v2_cmd
= cmd
;
1480 static void update_stibp_msr(void * __unused
)
1482 u64 val
= spec_ctrl_current() | (x86_spec_ctrl_base
& SPEC_CTRL_STIBP
);
1483 write_spec_ctrl_current(val
, true);
1486 /* Update x86_spec_ctrl_base in case SMT state changed. */
1487 static void update_stibp_strict(void)
1489 u64 mask
= x86_spec_ctrl_base
& ~SPEC_CTRL_STIBP
;
1491 if (sched_smt_active())
1492 mask
|= SPEC_CTRL_STIBP
;
1494 if (mask
== x86_spec_ctrl_base
)
1497 pr_info("Update user space SMT mitigation: STIBP %s\n",
1498 mask
& SPEC_CTRL_STIBP
? "always-on" : "off");
1499 x86_spec_ctrl_base
= mask
;
1500 on_each_cpu(update_stibp_msr
, NULL
, 1);
1503 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1504 static void update_indir_branch_cond(void)
1506 if (sched_smt_active())
1507 static_branch_enable(&switch_to_cond_stibp
);
1509 static_branch_disable(&switch_to_cond_stibp
);
1513 #define pr_fmt(fmt) fmt
1515 /* Update the static key controlling the MDS CPU buffer clear in idle */
1516 static void update_mds_branch_idle(void)
1518 u64 ia32_cap
= x86_read_arch_cap_msr();
1521 * Enable the idle clearing if SMT is active on CPUs which are
1522 * affected only by MSBDS and not any other MDS variant.
1524 * The other variants cannot be mitigated when SMT is enabled, so
1525 * clearing the buffers on idle just to prevent the Store Buffer
1526 * repartitioning leak would be a window dressing exercise.
1528 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY
))
1531 if (sched_smt_active()) {
1532 static_branch_enable(&mds_idle_clear
);
1533 } else if (mmio_mitigation
== MMIO_MITIGATION_OFF
||
1534 (ia32_cap
& ARCH_CAP_FBSDP_NO
)) {
1535 static_branch_disable(&mds_idle_clear
);
1539 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1540 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1541 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1543 void cpu_bugs_smt_update(void)
1545 mutex_lock(&spec_ctrl_mutex
);
1547 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1548 spectre_v2_enabled
== SPECTRE_V2_EIBRS_LFENCE
)
1549 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG
);
1551 switch (spectre_v2_user_stibp
) {
1552 case SPECTRE_V2_USER_NONE
:
1554 case SPECTRE_V2_USER_STRICT
:
1555 case SPECTRE_V2_USER_STRICT_PREFERRED
:
1556 update_stibp_strict();
1558 case SPECTRE_V2_USER_PRCTL
:
1559 case SPECTRE_V2_USER_SECCOMP
:
1560 update_indir_branch_cond();
1564 switch (mds_mitigation
) {
1565 case MDS_MITIGATION_FULL
:
1566 case MDS_MITIGATION_VMWERV
:
1567 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY
))
1568 pr_warn_once(MDS_MSG_SMT
);
1569 update_mds_branch_idle();
1571 case MDS_MITIGATION_OFF
:
1575 switch (taa_mitigation
) {
1576 case TAA_MITIGATION_VERW
:
1577 case TAA_MITIGATION_UCODE_NEEDED
:
1578 if (sched_smt_active())
1579 pr_warn_once(TAA_MSG_SMT
);
1581 case TAA_MITIGATION_TSX_DISABLED
:
1582 case TAA_MITIGATION_OFF
:
1586 switch (mmio_mitigation
) {
1587 case MMIO_MITIGATION_VERW
:
1588 case MMIO_MITIGATION_UCODE_NEEDED
:
1589 if (sched_smt_active())
1590 pr_warn_once(MMIO_MSG_SMT
);
1592 case MMIO_MITIGATION_OFF
:
1596 mutex_unlock(&spec_ctrl_mutex
);
1600 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt
1602 static enum ssb_mitigation ssb_mode __ro_after_init
= SPEC_STORE_BYPASS_NONE
;
1604 /* The kernel command line selection */
1605 enum ssb_mitigation_cmd
{
1606 SPEC_STORE_BYPASS_CMD_NONE
,
1607 SPEC_STORE_BYPASS_CMD_AUTO
,
1608 SPEC_STORE_BYPASS_CMD_ON
,
1609 SPEC_STORE_BYPASS_CMD_PRCTL
,
1610 SPEC_STORE_BYPASS_CMD_SECCOMP
,
1613 static const char * const ssb_strings
[] = {
1614 [SPEC_STORE_BYPASS_NONE
] = "Vulnerable",
1615 [SPEC_STORE_BYPASS_DISABLE
] = "Mitigation: Speculative Store Bypass disabled",
1616 [SPEC_STORE_BYPASS_PRCTL
] = "Mitigation: Speculative Store Bypass disabled via prctl",
1617 [SPEC_STORE_BYPASS_SECCOMP
] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1620 static const struct {
1622 enum ssb_mitigation_cmd cmd
;
1623 } ssb_mitigation_options
[] __initconst
= {
1624 { "auto", SPEC_STORE_BYPASS_CMD_AUTO
}, /* Platform decides */
1625 { "on", SPEC_STORE_BYPASS_CMD_ON
}, /* Disable Speculative Store Bypass */
1626 { "off", SPEC_STORE_BYPASS_CMD_NONE
}, /* Don't touch Speculative Store Bypass */
1627 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL
}, /* Disable Speculative Store Bypass via prctl */
1628 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP
}, /* Disable Speculative Store Bypass via prctl and seccomp */
1631 static enum ssb_mitigation_cmd __init
ssb_parse_cmdline(void)
1633 enum ssb_mitigation_cmd cmd
= SPEC_STORE_BYPASS_CMD_AUTO
;
1637 if (cmdline_find_option_bool(boot_command_line
, "nospec_store_bypass_disable") ||
1638 cpu_mitigations_off()) {
1639 return SPEC_STORE_BYPASS_CMD_NONE
;
1641 ret
= cmdline_find_option(boot_command_line
, "spec_store_bypass_disable",
1644 return SPEC_STORE_BYPASS_CMD_AUTO
;
1646 for (i
= 0; i
< ARRAY_SIZE(ssb_mitigation_options
); i
++) {
1647 if (!match_option(arg
, ret
, ssb_mitigation_options
[i
].option
))
1650 cmd
= ssb_mitigation_options
[i
].cmd
;
1654 if (i
>= ARRAY_SIZE(ssb_mitigation_options
)) {
1655 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
1656 return SPEC_STORE_BYPASS_CMD_AUTO
;
1663 static enum ssb_mitigation __init
__ssb_select_mitigation(void)
1665 enum ssb_mitigation mode
= SPEC_STORE_BYPASS_NONE
;
1666 enum ssb_mitigation_cmd cmd
;
1668 if (!boot_cpu_has(X86_FEATURE_SSBD
))
1671 cmd
= ssb_parse_cmdline();
1672 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
) &&
1673 (cmd
== SPEC_STORE_BYPASS_CMD_NONE
||
1674 cmd
== SPEC_STORE_BYPASS_CMD_AUTO
))
1678 case SPEC_STORE_BYPASS_CMD_AUTO
:
1679 case SPEC_STORE_BYPASS_CMD_SECCOMP
:
1681 * Choose prctl+seccomp as the default mode if seccomp is
1684 if (IS_ENABLED(CONFIG_SECCOMP
))
1685 mode
= SPEC_STORE_BYPASS_SECCOMP
;
1687 mode
= SPEC_STORE_BYPASS_PRCTL
;
1689 case SPEC_STORE_BYPASS_CMD_ON
:
1690 mode
= SPEC_STORE_BYPASS_DISABLE
;
1692 case SPEC_STORE_BYPASS_CMD_PRCTL
:
1693 mode
= SPEC_STORE_BYPASS_PRCTL
;
1695 case SPEC_STORE_BYPASS_CMD_NONE
:
1700 * We have three CPU feature flags that are in play here:
1701 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1702 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1703 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1705 if (mode
== SPEC_STORE_BYPASS_DISABLE
) {
1706 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
);
1708 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1709 * use a completely different MSR and bit dependent on family.
1711 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD
) &&
1712 !static_cpu_has(X86_FEATURE_AMD_SSBD
)) {
1713 x86_amd_ssb_disable();
1715 x86_spec_ctrl_base
|= SPEC_CTRL_SSBD
;
1716 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1723 static void ssb_select_mitigation(void)
1725 ssb_mode
= __ssb_select_mitigation();
1727 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
1728 pr_info("%s\n", ssb_strings
[ssb_mode
]);
1732 #define pr_fmt(fmt) "Speculation prctl: " fmt
1734 static void task_update_spec_tif(struct task_struct
*tsk
)
1736 /* Force the update of the real TIF bits */
1737 set_tsk_thread_flag(tsk
, TIF_SPEC_FORCE_UPDATE
);
1740 * Immediately update the speculation control MSRs for the current
1741 * task, but for a non-current task delay setting the CPU
1742 * mitigation until it is scheduled next.
1744 * This can only happen for SECCOMP mitigation. For PRCTL it's
1745 * always the current task.
1748 speculation_ctrl_update_current();
1751 static int l1d_flush_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1754 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush
))
1758 case PR_SPEC_ENABLE
:
1759 set_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
);
1761 case PR_SPEC_DISABLE
:
1762 clear_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
);
1769 static int ssb_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1771 if (ssb_mode
!= SPEC_STORE_BYPASS_PRCTL
&&
1772 ssb_mode
!= SPEC_STORE_BYPASS_SECCOMP
)
1776 case PR_SPEC_ENABLE
:
1777 /* If speculation is force disabled, enable is not allowed */
1778 if (task_spec_ssb_force_disable(task
))
1780 task_clear_spec_ssb_disable(task
);
1781 task_clear_spec_ssb_noexec(task
);
1782 task_update_spec_tif(task
);
1784 case PR_SPEC_DISABLE
:
1785 task_set_spec_ssb_disable(task
);
1786 task_clear_spec_ssb_noexec(task
);
1787 task_update_spec_tif(task
);
1789 case PR_SPEC_FORCE_DISABLE
:
1790 task_set_spec_ssb_disable(task
);
1791 task_set_spec_ssb_force_disable(task
);
1792 task_clear_spec_ssb_noexec(task
);
1793 task_update_spec_tif(task
);
1795 case PR_SPEC_DISABLE_NOEXEC
:
1796 if (task_spec_ssb_force_disable(task
))
1798 task_set_spec_ssb_disable(task
);
1799 task_set_spec_ssb_noexec(task
);
1800 task_update_spec_tif(task
);
1808 static bool is_spec_ib_user_controlled(void)
1810 return spectre_v2_user_ibpb
== SPECTRE_V2_USER_PRCTL
||
1811 spectre_v2_user_ibpb
== SPECTRE_V2_USER_SECCOMP
||
1812 spectre_v2_user_stibp
== SPECTRE_V2_USER_PRCTL
||
1813 spectre_v2_user_stibp
== SPECTRE_V2_USER_SECCOMP
;
1816 static int ib_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1819 case PR_SPEC_ENABLE
:
1820 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1821 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1825 * With strict mode for both IBPB and STIBP, the instruction
1826 * code paths avoid checking this task flag and instead,
1827 * unconditionally run the instruction. However, STIBP and IBPB
1828 * are independent and either can be set to conditionally
1829 * enabled regardless of the mode of the other.
1831 * If either is set to conditional, allow the task flag to be
1832 * updated, unless it was force-disabled by a previous prctl
1833 * call. Currently, this is possible on an AMD CPU which has the
1834 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
1835 * kernel is booted with 'spectre_v2_user=seccomp', then
1836 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
1837 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
1839 if (!is_spec_ib_user_controlled() ||
1840 task_spec_ib_force_disable(task
))
1843 task_clear_spec_ib_disable(task
);
1844 task_update_spec_tif(task
);
1846 case PR_SPEC_DISABLE
:
1847 case PR_SPEC_FORCE_DISABLE
:
1849 * Indirect branch speculation is always allowed when
1850 * mitigation is force disabled.
1852 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1853 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1856 if (!is_spec_ib_user_controlled())
1859 task_set_spec_ib_disable(task
);
1860 if (ctrl
== PR_SPEC_FORCE_DISABLE
)
1861 task_set_spec_ib_force_disable(task
);
1862 task_update_spec_tif(task
);
1870 int arch_prctl_spec_ctrl_set(struct task_struct
*task
, unsigned long which
,
1874 case PR_SPEC_STORE_BYPASS
:
1875 return ssb_prctl_set(task
, ctrl
);
1876 case PR_SPEC_INDIRECT_BRANCH
:
1877 return ib_prctl_set(task
, ctrl
);
1878 case PR_SPEC_L1D_FLUSH
:
1879 return l1d_flush_prctl_set(task
, ctrl
);
1885 #ifdef CONFIG_SECCOMP
1886 void arch_seccomp_spec_mitigate(struct task_struct
*task
)
1888 if (ssb_mode
== SPEC_STORE_BYPASS_SECCOMP
)
1889 ssb_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1890 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_SECCOMP
||
1891 spectre_v2_user_stibp
== SPECTRE_V2_USER_SECCOMP
)
1892 ib_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1896 static int l1d_flush_prctl_get(struct task_struct
*task
)
1898 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush
))
1899 return PR_SPEC_FORCE_DISABLE
;
1901 if (test_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
))
1902 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1904 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1907 static int ssb_prctl_get(struct task_struct
*task
)
1910 case SPEC_STORE_BYPASS_DISABLE
:
1911 return PR_SPEC_DISABLE
;
1912 case SPEC_STORE_BYPASS_SECCOMP
:
1913 case SPEC_STORE_BYPASS_PRCTL
:
1914 if (task_spec_ssb_force_disable(task
))
1915 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1916 if (task_spec_ssb_noexec(task
))
1917 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE_NOEXEC
;
1918 if (task_spec_ssb_disable(task
))
1919 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1920 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1922 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
1923 return PR_SPEC_ENABLE
;
1924 return PR_SPEC_NOT_AFFECTED
;
1928 static int ib_prctl_get(struct task_struct
*task
)
1930 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
))
1931 return PR_SPEC_NOT_AFFECTED
;
1933 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1934 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1935 return PR_SPEC_ENABLE
;
1936 else if (is_spec_ib_user_controlled()) {
1937 if (task_spec_ib_force_disable(task
))
1938 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1939 if (task_spec_ib_disable(task
))
1940 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1941 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1942 } else if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_STRICT
||
1943 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT
||
1944 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT_PREFERRED
)
1945 return PR_SPEC_DISABLE
;
1947 return PR_SPEC_NOT_AFFECTED
;
1950 int arch_prctl_spec_ctrl_get(struct task_struct
*task
, unsigned long which
)
1953 case PR_SPEC_STORE_BYPASS
:
1954 return ssb_prctl_get(task
);
1955 case PR_SPEC_INDIRECT_BRANCH
:
1956 return ib_prctl_get(task
);
1957 case PR_SPEC_L1D_FLUSH
:
1958 return l1d_flush_prctl_get(task
);
1964 void x86_spec_ctrl_setup_ap(void)
1966 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
1967 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1969 if (ssb_mode
== SPEC_STORE_BYPASS_DISABLE
)
1970 x86_amd_ssb_disable();
1973 bool itlb_multihit_kvm_mitigation
;
1974 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation
);
1977 #define pr_fmt(fmt) "L1TF: " fmt
1979 /* Default mitigation for L1TF-affected CPUs */
1980 enum l1tf_mitigations l1tf_mitigation __ro_after_init
= L1TF_MITIGATION_FLUSH
;
1981 #if IS_ENABLED(CONFIG_KVM_INTEL)
1982 EXPORT_SYMBOL_GPL(l1tf_mitigation
);
1984 enum vmx_l1d_flush_state l1tf_vmx_mitigation
= VMENTER_L1D_FLUSH_AUTO
;
1985 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation
);
1988 * These CPUs all support 44bits physical address space internally in the
1989 * cache but CPUID can report a smaller number of physical address bits.
1991 * The L1TF mitigation uses the top most address bit for the inversion of
1992 * non present PTEs. When the installed memory reaches into the top most
1993 * address bit due to memory holes, which has been observed on machines
1994 * which report 36bits physical address bits and have 32G RAM installed,
1995 * then the mitigation range check in l1tf_select_mitigation() triggers.
1996 * This is a false positive because the mitigation is still possible due to
1997 * the fact that the cache uses 44bit internally. Use the cache bits
1998 * instead of the reported physical bits and adjust them on the affected
1999 * machines to 44bit if the reported bits are less than 44.
2001 static void override_cache_bits(struct cpuinfo_x86
*c
)
2006 switch (c
->x86_model
) {
2007 case INTEL_FAM6_NEHALEM
:
2008 case INTEL_FAM6_WESTMERE
:
2009 case INTEL_FAM6_SANDYBRIDGE
:
2010 case INTEL_FAM6_IVYBRIDGE
:
2011 case INTEL_FAM6_HASWELL
:
2012 case INTEL_FAM6_HASWELL_L
:
2013 case INTEL_FAM6_HASWELL_G
:
2014 case INTEL_FAM6_BROADWELL
:
2015 case INTEL_FAM6_BROADWELL_G
:
2016 case INTEL_FAM6_SKYLAKE_L
:
2017 case INTEL_FAM6_SKYLAKE
:
2018 case INTEL_FAM6_KABYLAKE_L
:
2019 case INTEL_FAM6_KABYLAKE
:
2020 if (c
->x86_cache_bits
< 44)
2021 c
->x86_cache_bits
= 44;
2026 static void __init
l1tf_select_mitigation(void)
2030 if (!boot_cpu_has_bug(X86_BUG_L1TF
))
2033 if (cpu_mitigations_off())
2034 l1tf_mitigation
= L1TF_MITIGATION_OFF
;
2035 else if (cpu_mitigations_auto_nosmt())
2036 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOSMT
;
2038 override_cache_bits(&boot_cpu_data
);
2040 switch (l1tf_mitigation
) {
2041 case L1TF_MITIGATION_OFF
:
2042 case L1TF_MITIGATION_FLUSH_NOWARN
:
2043 case L1TF_MITIGATION_FLUSH
:
2045 case L1TF_MITIGATION_FLUSH_NOSMT
:
2046 case L1TF_MITIGATION_FULL
:
2047 cpu_smt_disable(false);
2049 case L1TF_MITIGATION_FULL_FORCE
:
2050 cpu_smt_disable(true);
2054 #if CONFIG_PGTABLE_LEVELS == 2
2055 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2059 half_pa
= (u64
)l1tf_pfn_limit() << PAGE_SHIFT
;
2060 if (l1tf_mitigation
!= L1TF_MITIGATION_OFF
&&
2061 e820__mapped_any(half_pa
, ULLONG_MAX
- half_pa
, E820_TYPE_RAM
)) {
2062 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2063 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2065 pr_info("However, doing so will make a part of your RAM unusable.\n");
2066 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2070 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV
);
2073 static int __init
l1tf_cmdline(char *str
)
2075 if (!boot_cpu_has_bug(X86_BUG_L1TF
))
2081 if (!strcmp(str
, "off"))
2082 l1tf_mitigation
= L1TF_MITIGATION_OFF
;
2083 else if (!strcmp(str
, "flush,nowarn"))
2084 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOWARN
;
2085 else if (!strcmp(str
, "flush"))
2086 l1tf_mitigation
= L1TF_MITIGATION_FLUSH
;
2087 else if (!strcmp(str
, "flush,nosmt"))
2088 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOSMT
;
2089 else if (!strcmp(str
, "full"))
2090 l1tf_mitigation
= L1TF_MITIGATION_FULL
;
2091 else if (!strcmp(str
, "full,force"))
2092 l1tf_mitigation
= L1TF_MITIGATION_FULL_FORCE
;
2096 early_param("l1tf", l1tf_cmdline
);
2099 #define pr_fmt(fmt) fmt
2103 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2105 #if IS_ENABLED(CONFIG_KVM_INTEL)
2106 static const char * const l1tf_vmx_states
[] = {
2107 [VMENTER_L1D_FLUSH_AUTO
] = "auto",
2108 [VMENTER_L1D_FLUSH_NEVER
] = "vulnerable",
2109 [VMENTER_L1D_FLUSH_COND
] = "conditional cache flushes",
2110 [VMENTER_L1D_FLUSH_ALWAYS
] = "cache flushes",
2111 [VMENTER_L1D_FLUSH_EPT_DISABLED
] = "EPT disabled",
2112 [VMENTER_L1D_FLUSH_NOT_REQUIRED
] = "flush not necessary"
2115 static ssize_t
l1tf_show_state(char *buf
)
2117 if (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_AUTO
)
2118 return sprintf(buf
, "%s\n", L1TF_DEFAULT_MSG
);
2120 if (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_EPT_DISABLED
||
2121 (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_NEVER
&&
2122 sched_smt_active())) {
2123 return sprintf(buf
, "%s; VMX: %s\n", L1TF_DEFAULT_MSG
,
2124 l1tf_vmx_states
[l1tf_vmx_mitigation
]);
2127 return sprintf(buf
, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG
,
2128 l1tf_vmx_states
[l1tf_vmx_mitigation
],
2129 sched_smt_active() ? "vulnerable" : "disabled");
2132 static ssize_t
itlb_multihit_show_state(char *buf
)
2134 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL
) ||
2135 !boot_cpu_has(X86_FEATURE_VMX
))
2136 return sprintf(buf
, "KVM: Mitigation: VMX unsupported\n");
2137 else if (!(cr4_read_shadow() & X86_CR4_VMXE
))
2138 return sprintf(buf
, "KVM: Mitigation: VMX disabled\n");
2139 else if (itlb_multihit_kvm_mitigation
)
2140 return sprintf(buf
, "KVM: Mitigation: Split huge pages\n");
2142 return sprintf(buf
, "KVM: Vulnerable\n");
2145 static ssize_t
l1tf_show_state(char *buf
)
2147 return sprintf(buf
, "%s\n", L1TF_DEFAULT_MSG
);
2150 static ssize_t
itlb_multihit_show_state(char *buf
)
2152 return sprintf(buf
, "Processor vulnerable\n");
2156 static ssize_t
mds_show_state(char *buf
)
2158 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2159 return sprintf(buf
, "%s; SMT Host state unknown\n",
2160 mds_strings
[mds_mitigation
]);
2163 if (boot_cpu_has(X86_BUG_MSBDS_ONLY
)) {
2164 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
2165 (mds_mitigation
== MDS_MITIGATION_OFF
? "vulnerable" :
2166 sched_smt_active() ? "mitigated" : "disabled"));
2169 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
2170 sched_smt_active() ? "vulnerable" : "disabled");
2173 static ssize_t
tsx_async_abort_show_state(char *buf
)
2175 if ((taa_mitigation
== TAA_MITIGATION_TSX_DISABLED
) ||
2176 (taa_mitigation
== TAA_MITIGATION_OFF
))
2177 return sprintf(buf
, "%s\n", taa_strings
[taa_mitigation
]);
2179 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2180 return sprintf(buf
, "%s; SMT Host state unknown\n",
2181 taa_strings
[taa_mitigation
]);
2184 return sprintf(buf
, "%s; SMT %s\n", taa_strings
[taa_mitigation
],
2185 sched_smt_active() ? "vulnerable" : "disabled");
2188 static ssize_t
mmio_stale_data_show_state(char *buf
)
2190 if (mmio_mitigation
== MMIO_MITIGATION_OFF
)
2191 return sysfs_emit(buf
, "%s\n", mmio_strings
[mmio_mitigation
]);
2193 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2194 return sysfs_emit(buf
, "%s; SMT Host state unknown\n",
2195 mmio_strings
[mmio_mitigation
]);
2198 return sysfs_emit(buf
, "%s; SMT %s\n", mmio_strings
[mmio_mitigation
],
2199 sched_smt_active() ? "vulnerable" : "disabled");
2202 static char *stibp_state(void)
2204 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled
))
2207 switch (spectre_v2_user_stibp
) {
2208 case SPECTRE_V2_USER_NONE
:
2209 return ", STIBP: disabled";
2210 case SPECTRE_V2_USER_STRICT
:
2211 return ", STIBP: forced";
2212 case SPECTRE_V2_USER_STRICT_PREFERRED
:
2213 return ", STIBP: always-on";
2214 case SPECTRE_V2_USER_PRCTL
:
2215 case SPECTRE_V2_USER_SECCOMP
:
2216 if (static_key_enabled(&switch_to_cond_stibp
))
2217 return ", STIBP: conditional";
2222 static char *ibpb_state(void)
2224 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
2225 if (static_key_enabled(&switch_mm_always_ibpb
))
2226 return ", IBPB: always-on";
2227 if (static_key_enabled(&switch_mm_cond_ibpb
))
2228 return ", IBPB: conditional";
2229 return ", IBPB: disabled";
2234 static ssize_t
spectre_v2_show_state(char *buf
)
2236 if (spectre_v2_enabled
== SPECTRE_V2_LFENCE
)
2237 return sprintf(buf
, "Vulnerable: LFENCE\n");
2239 if (spectre_v2_enabled
== SPECTRE_V2_EIBRS
&& unprivileged_ebpf_enabled())
2240 return sprintf(buf
, "Vulnerable: eIBRS with unprivileged eBPF\n");
2242 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2243 spectre_v2_enabled
== SPECTRE_V2_EIBRS_LFENCE
)
2244 return sprintf(buf
, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2246 return sprintf(buf
, "%s%s%s%s%s%s\n",
2247 spectre_v2_strings
[spectre_v2_enabled
],
2249 boot_cpu_has(X86_FEATURE_USE_IBRS_FW
) ? ", IBRS_FW" : "",
2251 boot_cpu_has(X86_FEATURE_RSB_CTXSW
) ? ", RSB filling" : "",
2252 spectre_v2_module_string());
2255 static ssize_t
srbds_show_state(char *buf
)
2257 return sprintf(buf
, "%s\n", srbds_strings
[srbds_mitigation
]);
2260 static ssize_t
retbleed_show_state(char *buf
)
2262 if (retbleed_mitigation
== RETBLEED_MITIGATION_UNRET
) {
2263 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
&&
2264 boot_cpu_data
.x86_vendor
!= X86_VENDOR_HYGON
)
2265 return sprintf(buf
, "Vulnerable: untrained return thunk on non-Zen uarch\n");
2267 return sprintf(buf
, "%s; SMT %s\n",
2268 retbleed_strings
[retbleed_mitigation
],
2269 !sched_smt_active() ? "disabled" :
2270 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT
||
2271 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT_PREFERRED
?
2272 "enabled with STIBP protection" : "vulnerable");
2275 return sprintf(buf
, "%s\n", retbleed_strings
[retbleed_mitigation
]);
2278 static ssize_t
cpu_show_common(struct device
*dev
, struct device_attribute
*attr
,
2279 char *buf
, unsigned int bug
)
2281 if (!boot_cpu_has_bug(bug
))
2282 return sprintf(buf
, "Not affected\n");
2285 case X86_BUG_CPU_MELTDOWN
:
2286 if (boot_cpu_has(X86_FEATURE_PTI
))
2287 return sprintf(buf
, "Mitigation: PTI\n");
2289 if (hypervisor_is_type(X86_HYPER_XEN_PV
))
2290 return sprintf(buf
, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2294 case X86_BUG_SPECTRE_V1
:
2295 return sprintf(buf
, "%s\n", spectre_v1_strings
[spectre_v1_mitigation
]);
2297 case X86_BUG_SPECTRE_V2
:
2298 return spectre_v2_show_state(buf
);
2300 case X86_BUG_SPEC_STORE_BYPASS
:
2301 return sprintf(buf
, "%s\n", ssb_strings
[ssb_mode
]);
2304 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV
))
2305 return l1tf_show_state(buf
);
2309 return mds_show_state(buf
);
2312 return tsx_async_abort_show_state(buf
);
2314 case X86_BUG_ITLB_MULTIHIT
:
2315 return itlb_multihit_show_state(buf
);
2318 return srbds_show_state(buf
);
2320 case X86_BUG_MMIO_STALE_DATA
:
2321 return mmio_stale_data_show_state(buf
);
2323 case X86_BUG_RETBLEED
:
2324 return retbleed_show_state(buf
);
2330 return sprintf(buf
, "Vulnerable\n");
2333 ssize_t
cpu_show_meltdown(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2335 return cpu_show_common(dev
, attr
, buf
, X86_BUG_CPU_MELTDOWN
);
2338 ssize_t
cpu_show_spectre_v1(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2340 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V1
);
2343 ssize_t
cpu_show_spectre_v2(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2345 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V2
);
2348 ssize_t
cpu_show_spec_store_bypass(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2350 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPEC_STORE_BYPASS
);
2353 ssize_t
cpu_show_l1tf(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2355 return cpu_show_common(dev
, attr
, buf
, X86_BUG_L1TF
);
2358 ssize_t
cpu_show_mds(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2360 return cpu_show_common(dev
, attr
, buf
, X86_BUG_MDS
);
2363 ssize_t
cpu_show_tsx_async_abort(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2365 return cpu_show_common(dev
, attr
, buf
, X86_BUG_TAA
);
2368 ssize_t
cpu_show_itlb_multihit(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2370 return cpu_show_common(dev
, attr
, buf
, X86_BUG_ITLB_MULTIHIT
);
2373 ssize_t
cpu_show_srbds(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2375 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SRBDS
);
2378 ssize_t
cpu_show_mmio_stale_data(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2380 return cpu_show_common(dev
, attr
, buf
, X86_BUG_MMIO_STALE_DATA
);
2383 ssize_t
cpu_show_retbleed(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2385 return cpu_show_common(dev
, attr
, buf
, X86_BUG_RETBLEED
);