1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1994 Linus Torvalds
5 * Cyrix stuff, June 1998 by:
6 * - Rafael R. Reilova (moved everything from head.S),
7 * <rreilova@ececs.uc.edu>
8 * - Channing Corn (tests & fixes),
9 * - Andrew D. Balsa (code cleanup).
11 #include <linux/init.h>
12 #include <linux/utsname.h>
13 #include <linux/cpu.h>
14 #include <linux/module.h>
15 #include <linux/nospec.h>
16 #include <linux/prctl.h>
17 #include <linux/sched/smt.h>
18 #include <linux/pgtable.h>
19 #include <linux/bpf.h>
21 #include <asm/spec-ctrl.h>
22 #include <asm/cmdline.h>
24 #include <asm/processor.h>
25 #include <asm/processor-flags.h>
26 #include <asm/fpu/api.h>
29 #include <asm/paravirt.h>
30 #include <asm/alternative.h>
31 #include <asm/set_memory.h>
32 #include <asm/intel-family.h>
33 #include <asm/e820/api.h>
34 #include <asm/hypervisor.h>
35 #include <asm/tlbflush.h>
39 static void __init
spectre_v1_select_mitigation(void);
40 static void __init
spectre_v2_select_mitigation(void);
41 static void __init
retbleed_select_mitigation(void);
42 static void __init
spectre_v2_user_select_mitigation(void);
43 static void __init
ssb_select_mitigation(void);
44 static void __init
l1tf_select_mitigation(void);
45 static void __init
mds_select_mitigation(void);
46 static void __init
md_clear_update_mitigation(void);
47 static void __init
md_clear_select_mitigation(void);
48 static void __init
taa_select_mitigation(void);
49 static void __init
mmio_select_mitigation(void);
50 static void __init
srbds_select_mitigation(void);
51 static void __init
l1d_flush_select_mitigation(void);
53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
54 u64 x86_spec_ctrl_base
;
55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base
);
57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
58 DEFINE_PER_CPU(u64
, x86_spec_ctrl_current
);
59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current
);
61 static DEFINE_MUTEX(spec_ctrl_mutex
);
64 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
65 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
67 void write_spec_ctrl_current(u64 val
, bool force
)
69 if (this_cpu_read(x86_spec_ctrl_current
) == val
)
72 this_cpu_write(x86_spec_ctrl_current
, val
);
75 * When KERNEL_IBRS this MSR is written on return-to-user, unless
76 * forced the update can be delayed until that time.
78 if (force
|| !cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS
))
79 wrmsrl(MSR_IA32_SPEC_CTRL
, val
);
82 u64
spec_ctrl_current(void)
84 return this_cpu_read(x86_spec_ctrl_current
);
86 EXPORT_SYMBOL_GPL(spec_ctrl_current
);
89 * AMD specific MSR info for Speculative Store Bypass control.
90 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
92 u64 __ro_after_init x86_amd_ls_cfg_base
;
93 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask
;
95 /* Control conditional STIBP in switch_to() */
96 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp
);
97 /* Control conditional IBPB in switch_mm() */
98 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb
);
99 /* Control unconditional IBPB in switch_mm() */
100 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb
);
102 /* Control MDS CPU buffer clear before returning to user space */
103 DEFINE_STATIC_KEY_FALSE(mds_user_clear
);
104 EXPORT_SYMBOL_GPL(mds_user_clear
);
105 /* Control MDS CPU buffer clear before idling (halt, mwait) */
106 DEFINE_STATIC_KEY_FALSE(mds_idle_clear
);
107 EXPORT_SYMBOL_GPL(mds_idle_clear
);
110 * Controls whether l1d flush based mitigations are enabled,
111 * based on hw features and admin setting via boot parameter
114 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush
);
116 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
117 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear
);
118 EXPORT_SYMBOL_GPL(mmio_stale_data_clear
);
120 void __init
check_bugs(void)
125 * identify_boot_cpu() initialized SMT support information, let the
128 cpu_smt_check_topology();
130 if (!IS_ENABLED(CONFIG_SMP
)) {
132 print_cpu_info(&boot_cpu_data
);
136 * Read the SPEC_CTRL MSR to account for reserved bits which may
137 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
138 * init code as it is not enumerated and depends on the family.
140 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
141 rdmsrl(MSR_IA32_SPEC_CTRL
, x86_spec_ctrl_base
);
143 /* Select the proper CPU mitigations before patching alternatives: */
144 spectre_v1_select_mitigation();
145 spectre_v2_select_mitigation();
147 * retbleed_select_mitigation() relies on the state set by
148 * spectre_v2_select_mitigation(); specifically it wants to know about
151 retbleed_select_mitigation();
153 * spectre_v2_user_select_mitigation() relies on the state set by
154 * retbleed_select_mitigation(); specifically the STIBP selection is
157 spectre_v2_user_select_mitigation();
158 ssb_select_mitigation();
159 l1tf_select_mitigation();
160 md_clear_select_mitigation();
161 srbds_select_mitigation();
162 l1d_flush_select_mitigation();
168 * Check whether we are able to run this kernel safely on SMP.
170 * - i386 is no longer supported.
171 * - In order to run on anything without a TSC, we need to be
172 * compiled for a i486.
174 if (boot_cpu_data
.x86
< 4)
175 panic("Kernel requires i486+ for 'invlpg' and other features");
177 init_utsname()->machine
[1] =
178 '0' + (boot_cpu_data
.x86
> 6 ? 6 : boot_cpu_data
.x86
);
179 alternative_instructions();
181 fpu__init_check_bugs();
182 #else /* CONFIG_X86_64 */
183 alternative_instructions();
186 * Make sure the first 2MB area is not mapped by huge pages
187 * There are typically fixed size MTRRs in there and overlapping
188 * MTRRs into large pages causes slow downs.
190 * Right now we don't do that with gbpages because there seems
191 * very little benefit for that case.
194 set_memory_4k((unsigned long)__va(0), 1);
199 * NOTE: This function is *only* called for SVM. VMX spec_ctrl handling is
203 x86_virt_spec_ctrl(u64 guest_spec_ctrl
, u64 guest_virt_spec_ctrl
, bool setguest
)
205 u64 msrval
, guestval
= guest_spec_ctrl
, hostval
= spec_ctrl_current();
206 struct thread_info
*ti
= current_thread_info();
208 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
)) {
209 if (hostval
!= guestval
) {
210 msrval
= setguest
? guestval
: hostval
;
211 wrmsrl(MSR_IA32_SPEC_CTRL
, msrval
);
216 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
217 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
219 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD
) &&
220 !static_cpu_has(X86_FEATURE_VIRT_SSBD
))
224 * If the host has SSBD mitigation enabled, force it in the host's
225 * virtual MSR value. If its not permanently enabled, evaluate
226 * current's TIF_SSBD thread flag.
228 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
))
229 hostval
= SPEC_CTRL_SSBD
;
231 hostval
= ssbd_tif_to_spec_ctrl(ti
->flags
);
233 /* Sanitize the guest value */
234 guestval
= guest_virt_spec_ctrl
& SPEC_CTRL_SSBD
;
236 if (hostval
!= guestval
) {
239 tif
= setguest
? ssbd_spec_ctrl_to_tif(guestval
) :
240 ssbd_spec_ctrl_to_tif(hostval
);
242 speculation_ctrl_update(tif
);
245 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl
);
247 static void x86_amd_ssb_disable(void)
249 u64 msrval
= x86_amd_ls_cfg_base
| x86_amd_ls_cfg_ssbd_mask
;
251 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD
))
252 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL
, SPEC_CTRL_SSBD
);
253 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD
))
254 wrmsrl(MSR_AMD64_LS_CFG
, msrval
);
258 #define pr_fmt(fmt) "MDS: " fmt
260 /* Default mitigation for MDS-affected CPUs */
261 static enum mds_mitigations mds_mitigation __ro_after_init
= MDS_MITIGATION_FULL
;
262 static bool mds_nosmt __ro_after_init
= false;
264 static const char * const mds_strings
[] = {
265 [MDS_MITIGATION_OFF
] = "Vulnerable",
266 [MDS_MITIGATION_FULL
] = "Mitigation: Clear CPU buffers",
267 [MDS_MITIGATION_VMWERV
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
270 static void __init
mds_select_mitigation(void)
272 if (!boot_cpu_has_bug(X86_BUG_MDS
) || cpu_mitigations_off()) {
273 mds_mitigation
= MDS_MITIGATION_OFF
;
277 if (mds_mitigation
== MDS_MITIGATION_FULL
) {
278 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR
))
279 mds_mitigation
= MDS_MITIGATION_VMWERV
;
281 static_branch_enable(&mds_user_clear
);
283 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY
) &&
284 (mds_nosmt
|| cpu_mitigations_auto_nosmt()))
285 cpu_smt_disable(false);
289 static int __init
mds_cmdline(char *str
)
291 if (!boot_cpu_has_bug(X86_BUG_MDS
))
297 if (!strcmp(str
, "off"))
298 mds_mitigation
= MDS_MITIGATION_OFF
;
299 else if (!strcmp(str
, "full"))
300 mds_mitigation
= MDS_MITIGATION_FULL
;
301 else if (!strcmp(str
, "full,nosmt")) {
302 mds_mitigation
= MDS_MITIGATION_FULL
;
308 early_param("mds", mds_cmdline
);
311 #define pr_fmt(fmt) "TAA: " fmt
313 enum taa_mitigations
{
315 TAA_MITIGATION_UCODE_NEEDED
,
317 TAA_MITIGATION_TSX_DISABLED
,
320 /* Default mitigation for TAA-affected CPUs */
321 static enum taa_mitigations taa_mitigation __ro_after_init
= TAA_MITIGATION_VERW
;
322 static bool taa_nosmt __ro_after_init
;
324 static const char * const taa_strings
[] = {
325 [TAA_MITIGATION_OFF
] = "Vulnerable",
326 [TAA_MITIGATION_UCODE_NEEDED
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
327 [TAA_MITIGATION_VERW
] = "Mitigation: Clear CPU buffers",
328 [TAA_MITIGATION_TSX_DISABLED
] = "Mitigation: TSX disabled",
331 static void __init
taa_select_mitigation(void)
335 if (!boot_cpu_has_bug(X86_BUG_TAA
)) {
336 taa_mitigation
= TAA_MITIGATION_OFF
;
340 /* TSX previously disabled by tsx=off */
341 if (!boot_cpu_has(X86_FEATURE_RTM
)) {
342 taa_mitigation
= TAA_MITIGATION_TSX_DISABLED
;
346 if (cpu_mitigations_off()) {
347 taa_mitigation
= TAA_MITIGATION_OFF
;
352 * TAA mitigation via VERW is turned off if both
353 * tsx_async_abort=off and mds=off are specified.
355 if (taa_mitigation
== TAA_MITIGATION_OFF
&&
356 mds_mitigation
== MDS_MITIGATION_OFF
)
359 if (boot_cpu_has(X86_FEATURE_MD_CLEAR
))
360 taa_mitigation
= TAA_MITIGATION_VERW
;
362 taa_mitigation
= TAA_MITIGATION_UCODE_NEEDED
;
365 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
366 * A microcode update fixes this behavior to clear CPU buffers. It also
367 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
368 * ARCH_CAP_TSX_CTRL_MSR bit.
370 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
371 * update is required.
373 ia32_cap
= x86_read_arch_cap_msr();
374 if ( (ia32_cap
& ARCH_CAP_MDS_NO
) &&
375 !(ia32_cap
& ARCH_CAP_TSX_CTRL_MSR
))
376 taa_mitigation
= TAA_MITIGATION_UCODE_NEEDED
;
379 * TSX is enabled, select alternate mitigation for TAA which is
380 * the same as MDS. Enable MDS static branch to clear CPU buffers.
382 * For guests that can't determine whether the correct microcode is
383 * present on host, enable the mitigation for UCODE_NEEDED as well.
385 static_branch_enable(&mds_user_clear
);
387 if (taa_nosmt
|| cpu_mitigations_auto_nosmt())
388 cpu_smt_disable(false);
391 static int __init
tsx_async_abort_parse_cmdline(char *str
)
393 if (!boot_cpu_has_bug(X86_BUG_TAA
))
399 if (!strcmp(str
, "off")) {
400 taa_mitigation
= TAA_MITIGATION_OFF
;
401 } else if (!strcmp(str
, "full")) {
402 taa_mitigation
= TAA_MITIGATION_VERW
;
403 } else if (!strcmp(str
, "full,nosmt")) {
404 taa_mitigation
= TAA_MITIGATION_VERW
;
410 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline
);
413 #define pr_fmt(fmt) "MMIO Stale Data: " fmt
415 enum mmio_mitigations
{
417 MMIO_MITIGATION_UCODE_NEEDED
,
418 MMIO_MITIGATION_VERW
,
421 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
422 static enum mmio_mitigations mmio_mitigation __ro_after_init
= MMIO_MITIGATION_VERW
;
423 static bool mmio_nosmt __ro_after_init
= false;
425 static const char * const mmio_strings
[] = {
426 [MMIO_MITIGATION_OFF
] = "Vulnerable",
427 [MMIO_MITIGATION_UCODE_NEEDED
] = "Vulnerable: Clear CPU buffers attempted, no microcode",
428 [MMIO_MITIGATION_VERW
] = "Mitigation: Clear CPU buffers",
431 static void __init
mmio_select_mitigation(void)
435 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
) ||
436 cpu_mitigations_off()) {
437 mmio_mitigation
= MMIO_MITIGATION_OFF
;
441 if (mmio_mitigation
== MMIO_MITIGATION_OFF
)
444 ia32_cap
= x86_read_arch_cap_msr();
447 * Enable CPU buffer clear mitigation for host and VMM, if also affected
448 * by MDS or TAA. Otherwise, enable mitigation for VMM only.
450 if (boot_cpu_has_bug(X86_BUG_MDS
) || (boot_cpu_has_bug(X86_BUG_TAA
) &&
451 boot_cpu_has(X86_FEATURE_RTM
)))
452 static_branch_enable(&mds_user_clear
);
454 static_branch_enable(&mmio_stale_data_clear
);
457 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
458 * be propagated to uncore buffers, clearing the Fill buffers on idle
459 * is required irrespective of SMT state.
461 if (!(ia32_cap
& ARCH_CAP_FBSDP_NO
))
462 static_branch_enable(&mds_idle_clear
);
465 * Check if the system has the right microcode.
467 * CPU Fill buffer clear mitigation is enumerated by either an explicit
468 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
471 if ((ia32_cap
& ARCH_CAP_FB_CLEAR
) ||
472 (boot_cpu_has(X86_FEATURE_MD_CLEAR
) &&
473 boot_cpu_has(X86_FEATURE_FLUSH_L1D
) &&
474 !(ia32_cap
& ARCH_CAP_MDS_NO
)))
475 mmio_mitigation
= MMIO_MITIGATION_VERW
;
477 mmio_mitigation
= MMIO_MITIGATION_UCODE_NEEDED
;
479 if (mmio_nosmt
|| cpu_mitigations_auto_nosmt())
480 cpu_smt_disable(false);
483 static int __init
mmio_stale_data_parse_cmdline(char *str
)
485 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
491 if (!strcmp(str
, "off")) {
492 mmio_mitigation
= MMIO_MITIGATION_OFF
;
493 } else if (!strcmp(str
, "full")) {
494 mmio_mitigation
= MMIO_MITIGATION_VERW
;
495 } else if (!strcmp(str
, "full,nosmt")) {
496 mmio_mitigation
= MMIO_MITIGATION_VERW
;
502 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline
);
505 #define pr_fmt(fmt) "" fmt
507 static void __init
md_clear_update_mitigation(void)
509 if (cpu_mitigations_off())
512 if (!static_key_enabled(&mds_user_clear
))
516 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data
517 * mitigation, if necessary.
519 if (mds_mitigation
== MDS_MITIGATION_OFF
&&
520 boot_cpu_has_bug(X86_BUG_MDS
)) {
521 mds_mitigation
= MDS_MITIGATION_FULL
;
522 mds_select_mitigation();
524 if (taa_mitigation
== TAA_MITIGATION_OFF
&&
525 boot_cpu_has_bug(X86_BUG_TAA
)) {
526 taa_mitigation
= TAA_MITIGATION_VERW
;
527 taa_select_mitigation();
529 if (mmio_mitigation
== MMIO_MITIGATION_OFF
&&
530 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
)) {
531 mmio_mitigation
= MMIO_MITIGATION_VERW
;
532 mmio_select_mitigation();
535 if (boot_cpu_has_bug(X86_BUG_MDS
))
536 pr_info("MDS: %s\n", mds_strings
[mds_mitigation
]);
537 if (boot_cpu_has_bug(X86_BUG_TAA
))
538 pr_info("TAA: %s\n", taa_strings
[taa_mitigation
]);
539 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
540 pr_info("MMIO Stale Data: %s\n", mmio_strings
[mmio_mitigation
]);
543 static void __init
md_clear_select_mitigation(void)
545 mds_select_mitigation();
546 taa_select_mitigation();
547 mmio_select_mitigation();
550 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update
551 * and print their mitigation after MDS, TAA and MMIO Stale Data
552 * mitigation selection is done.
554 md_clear_update_mitigation();
558 #define pr_fmt(fmt) "SRBDS: " fmt
560 enum srbds_mitigations
{
561 SRBDS_MITIGATION_OFF
,
562 SRBDS_MITIGATION_UCODE_NEEDED
,
563 SRBDS_MITIGATION_FULL
,
564 SRBDS_MITIGATION_TSX_OFF
,
565 SRBDS_MITIGATION_HYPERVISOR
,
568 static enum srbds_mitigations srbds_mitigation __ro_after_init
= SRBDS_MITIGATION_FULL
;
570 static const char * const srbds_strings
[] = {
571 [SRBDS_MITIGATION_OFF
] = "Vulnerable",
572 [SRBDS_MITIGATION_UCODE_NEEDED
] = "Vulnerable: No microcode",
573 [SRBDS_MITIGATION_FULL
] = "Mitigation: Microcode",
574 [SRBDS_MITIGATION_TSX_OFF
] = "Mitigation: TSX disabled",
575 [SRBDS_MITIGATION_HYPERVISOR
] = "Unknown: Dependent on hypervisor status",
578 static bool srbds_off
;
580 void update_srbds_msr(void)
584 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
587 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
))
590 if (srbds_mitigation
== SRBDS_MITIGATION_UCODE_NEEDED
)
593 rdmsrl(MSR_IA32_MCU_OPT_CTRL
, mcu_ctrl
);
595 switch (srbds_mitigation
) {
596 case SRBDS_MITIGATION_OFF
:
597 case SRBDS_MITIGATION_TSX_OFF
:
598 mcu_ctrl
|= RNGDS_MITG_DIS
;
600 case SRBDS_MITIGATION_FULL
:
601 mcu_ctrl
&= ~RNGDS_MITG_DIS
;
607 wrmsrl(MSR_IA32_MCU_OPT_CTRL
, mcu_ctrl
);
610 static void __init
srbds_select_mitigation(void)
614 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
618 * Check to see if this is one of the MDS_NO systems supporting TSX that
619 * are only exposed to SRBDS when TSX is enabled or when CPU is affected
620 * by Processor MMIO Stale Data vulnerability.
622 ia32_cap
= x86_read_arch_cap_msr();
623 if ((ia32_cap
& ARCH_CAP_MDS_NO
) && !boot_cpu_has(X86_FEATURE_RTM
) &&
624 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA
))
625 srbds_mitigation
= SRBDS_MITIGATION_TSX_OFF
;
626 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR
))
627 srbds_mitigation
= SRBDS_MITIGATION_HYPERVISOR
;
628 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL
))
629 srbds_mitigation
= SRBDS_MITIGATION_UCODE_NEEDED
;
630 else if (cpu_mitigations_off() || srbds_off
)
631 srbds_mitigation
= SRBDS_MITIGATION_OFF
;
634 pr_info("%s\n", srbds_strings
[srbds_mitigation
]);
637 static int __init
srbds_parse_cmdline(char *str
)
642 if (!boot_cpu_has_bug(X86_BUG_SRBDS
))
645 srbds_off
= !strcmp(str
, "off");
648 early_param("srbds", srbds_parse_cmdline
);
651 #define pr_fmt(fmt) "L1D Flush : " fmt
653 enum l1d_flush_mitigations
{
658 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata
= L1D_FLUSH_OFF
;
660 static void __init
l1d_flush_select_mitigation(void)
662 if (!l1d_flush_mitigation
|| !boot_cpu_has(X86_FEATURE_FLUSH_L1D
))
665 static_branch_enable(&switch_mm_cond_l1d_flush
);
666 pr_info("Conditional flush on switch_mm() enabled\n");
669 static int __init
l1d_flush_parse_cmdline(char *str
)
671 if (!strcmp(str
, "on"))
672 l1d_flush_mitigation
= L1D_FLUSH_ON
;
676 early_param("l1d_flush", l1d_flush_parse_cmdline
);
679 #define pr_fmt(fmt) "Spectre V1 : " fmt
681 enum spectre_v1_mitigation
{
682 SPECTRE_V1_MITIGATION_NONE
,
683 SPECTRE_V1_MITIGATION_AUTO
,
686 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init
=
687 SPECTRE_V1_MITIGATION_AUTO
;
689 static const char * const spectre_v1_strings
[] = {
690 [SPECTRE_V1_MITIGATION_NONE
] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
691 [SPECTRE_V1_MITIGATION_AUTO
] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
695 * Does SMAP provide full mitigation against speculative kernel access to
698 static bool smap_works_speculatively(void)
700 if (!boot_cpu_has(X86_FEATURE_SMAP
))
704 * On CPUs which are vulnerable to Meltdown, SMAP does not
705 * prevent speculative access to user data in the L1 cache.
706 * Consider SMAP to be non-functional as a mitigation on these
709 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN
))
715 static void __init
spectre_v1_select_mitigation(void)
717 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1
) || cpu_mitigations_off()) {
718 spectre_v1_mitigation
= SPECTRE_V1_MITIGATION_NONE
;
722 if (spectre_v1_mitigation
== SPECTRE_V1_MITIGATION_AUTO
) {
724 * With Spectre v1, a user can speculatively control either
725 * path of a conditional swapgs with a user-controlled GS
726 * value. The mitigation is to add lfences to both code paths.
728 * If FSGSBASE is enabled, the user can put a kernel address in
729 * GS, in which case SMAP provides no protection.
731 * If FSGSBASE is disabled, the user can only put a user space
732 * address in GS. That makes an attack harder, but still
733 * possible if there's no SMAP protection.
735 if (boot_cpu_has(X86_FEATURE_FSGSBASE
) ||
736 !smap_works_speculatively()) {
738 * Mitigation can be provided from SWAPGS itself or
739 * PTI as the CR3 write in the Meltdown mitigation
742 * If neither is there, mitigate with an LFENCE to
743 * stop speculation through swapgs.
745 if (boot_cpu_has_bug(X86_BUG_SWAPGS
) &&
746 !boot_cpu_has(X86_FEATURE_PTI
))
747 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER
);
750 * Enable lfences in the kernel entry (non-swapgs)
751 * paths, to prevent user entry from speculatively
754 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL
);
758 pr_info("%s\n", spectre_v1_strings
[spectre_v1_mitigation
]);
761 static int __init
nospectre_v1_cmdline(char *str
)
763 spectre_v1_mitigation
= SPECTRE_V1_MITIGATION_NONE
;
766 early_param("nospectre_v1", nospectre_v1_cmdline
);
768 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init
=
772 #define pr_fmt(fmt) "RETBleed: " fmt
774 enum retbleed_mitigation
{
775 RETBLEED_MITIGATION_NONE
,
776 RETBLEED_MITIGATION_UNRET
,
777 RETBLEED_MITIGATION_IBPB
,
778 RETBLEED_MITIGATION_IBRS
,
779 RETBLEED_MITIGATION_EIBRS
,
782 enum retbleed_mitigation_cmd
{
789 const char * const retbleed_strings
[] = {
790 [RETBLEED_MITIGATION_NONE
] = "Vulnerable",
791 [RETBLEED_MITIGATION_UNRET
] = "Mitigation: untrained return thunk",
792 [RETBLEED_MITIGATION_IBPB
] = "Mitigation: IBPB",
793 [RETBLEED_MITIGATION_IBRS
] = "Mitigation: IBRS",
794 [RETBLEED_MITIGATION_EIBRS
] = "Mitigation: Enhanced IBRS",
797 static enum retbleed_mitigation retbleed_mitigation __ro_after_init
=
798 RETBLEED_MITIGATION_NONE
;
799 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init
=
802 static int __ro_after_init retbleed_nosmt
= false;
804 static int __init
retbleed_parse_cmdline(char *str
)
810 char *next
= strchr(str
, ',');
816 if (!strcmp(str
, "off")) {
817 retbleed_cmd
= RETBLEED_CMD_OFF
;
818 } else if (!strcmp(str
, "auto")) {
819 retbleed_cmd
= RETBLEED_CMD_AUTO
;
820 } else if (!strcmp(str
, "unret")) {
821 retbleed_cmd
= RETBLEED_CMD_UNRET
;
822 } else if (!strcmp(str
, "ibpb")) {
823 retbleed_cmd
= RETBLEED_CMD_IBPB
;
824 } else if (!strcmp(str
, "nosmt")) {
825 retbleed_nosmt
= true;
827 pr_err("Ignoring unknown retbleed option (%s).", str
);
835 early_param("retbleed", retbleed_parse_cmdline
);
837 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
838 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
840 static void __init
retbleed_select_mitigation(void)
842 bool mitigate_smt
= false;
844 if (!boot_cpu_has_bug(X86_BUG_RETBLEED
) || cpu_mitigations_off())
847 switch (retbleed_cmd
) {
848 case RETBLEED_CMD_OFF
:
851 case RETBLEED_CMD_UNRET
:
852 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY
)) {
853 retbleed_mitigation
= RETBLEED_MITIGATION_UNRET
;
855 pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n");
860 case RETBLEED_CMD_IBPB
:
861 if (!boot_cpu_has(X86_FEATURE_IBPB
)) {
862 pr_err("WARNING: CPU does not support IBPB.\n");
864 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY
)) {
865 retbleed_mitigation
= RETBLEED_MITIGATION_IBPB
;
867 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");
873 case RETBLEED_CMD_AUTO
:
875 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_AMD
||
876 boot_cpu_data
.x86_vendor
== X86_VENDOR_HYGON
) {
877 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY
))
878 retbleed_mitigation
= RETBLEED_MITIGATION_UNRET
;
879 else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY
) && boot_cpu_has(X86_FEATURE_IBPB
))
880 retbleed_mitigation
= RETBLEED_MITIGATION_IBPB
;
884 * The Intel mitigation (IBRS or eIBRS) was already selected in
885 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will
886 * be set accordingly below.
892 switch (retbleed_mitigation
) {
893 case RETBLEED_MITIGATION_UNRET
:
894 setup_force_cpu_cap(X86_FEATURE_RETHUNK
);
895 setup_force_cpu_cap(X86_FEATURE_UNRET
);
897 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
&&
898 boot_cpu_data
.x86_vendor
!= X86_VENDOR_HYGON
)
899 pr_err(RETBLEED_UNTRAIN_MSG
);
904 case RETBLEED_MITIGATION_IBPB
:
905 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB
);
913 if (mitigate_smt
&& !boot_cpu_has(X86_FEATURE_STIBP
) &&
914 (retbleed_nosmt
|| cpu_mitigations_auto_nosmt()))
915 cpu_smt_disable(false);
918 * Let IBRS trump all on Intel without affecting the effects of the
919 * retbleed= cmdline option.
921 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
) {
922 switch (spectre_v2_enabled
) {
923 case SPECTRE_V2_IBRS
:
924 retbleed_mitigation
= RETBLEED_MITIGATION_IBRS
;
926 case SPECTRE_V2_EIBRS
:
927 case SPECTRE_V2_EIBRS_RETPOLINE
:
928 case SPECTRE_V2_EIBRS_LFENCE
:
929 retbleed_mitigation
= RETBLEED_MITIGATION_EIBRS
;
932 pr_err(RETBLEED_INTEL_MSG
);
936 pr_info("%s\n", retbleed_strings
[retbleed_mitigation
]);
940 #define pr_fmt(fmt) "Spectre V2 : " fmt
942 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init
=
943 SPECTRE_V2_USER_NONE
;
944 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init
=
945 SPECTRE_V2_USER_NONE
;
947 #ifdef CONFIG_RETPOLINE
948 static bool spectre_v2_bad_module
;
950 bool retpoline_module_ok(bool has_retpoline
)
952 if (spectre_v2_enabled
== SPECTRE_V2_NONE
|| has_retpoline
)
955 pr_err("System may be vulnerable to spectre v2\n");
956 spectre_v2_bad_module
= true;
960 static inline const char *spectre_v2_module_string(void)
962 return spectre_v2_bad_module
? " - vulnerable module loaded" : "";
965 static inline const char *spectre_v2_module_string(void) { return ""; }
968 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
969 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
970 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
972 #ifdef CONFIG_BPF_SYSCALL
973 void unpriv_ebpf_notify(int new_state
)
978 /* Unprivileged eBPF is enabled */
980 switch (spectre_v2_enabled
) {
981 case SPECTRE_V2_EIBRS
:
982 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG
);
984 case SPECTRE_V2_EIBRS_LFENCE
:
985 if (sched_smt_active())
986 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG
);
994 static inline bool match_option(const char *arg
, int arglen
, const char *opt
)
996 int len
= strlen(opt
);
998 return len
== arglen
&& !strncmp(arg
, opt
, len
);
1001 /* The kernel command line selection for spectre v2 */
1002 enum spectre_v2_mitigation_cmd
{
1003 SPECTRE_V2_CMD_NONE
,
1004 SPECTRE_V2_CMD_AUTO
,
1005 SPECTRE_V2_CMD_FORCE
,
1006 SPECTRE_V2_CMD_RETPOLINE
,
1007 SPECTRE_V2_CMD_RETPOLINE_GENERIC
,
1008 SPECTRE_V2_CMD_RETPOLINE_LFENCE
,
1009 SPECTRE_V2_CMD_EIBRS
,
1010 SPECTRE_V2_CMD_EIBRS_RETPOLINE
,
1011 SPECTRE_V2_CMD_EIBRS_LFENCE
,
1012 SPECTRE_V2_CMD_IBRS
,
1015 enum spectre_v2_user_cmd
{
1016 SPECTRE_V2_USER_CMD_NONE
,
1017 SPECTRE_V2_USER_CMD_AUTO
,
1018 SPECTRE_V2_USER_CMD_FORCE
,
1019 SPECTRE_V2_USER_CMD_PRCTL
,
1020 SPECTRE_V2_USER_CMD_PRCTL_IBPB
,
1021 SPECTRE_V2_USER_CMD_SECCOMP
,
1022 SPECTRE_V2_USER_CMD_SECCOMP_IBPB
,
1025 static const char * const spectre_v2_user_strings
[] = {
1026 [SPECTRE_V2_USER_NONE
] = "User space: Vulnerable",
1027 [SPECTRE_V2_USER_STRICT
] = "User space: Mitigation: STIBP protection",
1028 [SPECTRE_V2_USER_STRICT_PREFERRED
] = "User space: Mitigation: STIBP always-on protection",
1029 [SPECTRE_V2_USER_PRCTL
] = "User space: Mitigation: STIBP via prctl",
1030 [SPECTRE_V2_USER_SECCOMP
] = "User space: Mitigation: STIBP via seccomp and prctl",
1033 static const struct {
1035 enum spectre_v2_user_cmd cmd
;
1037 } v2_user_options
[] __initconst
= {
1038 { "auto", SPECTRE_V2_USER_CMD_AUTO
, false },
1039 { "off", SPECTRE_V2_USER_CMD_NONE
, false },
1040 { "on", SPECTRE_V2_USER_CMD_FORCE
, true },
1041 { "prctl", SPECTRE_V2_USER_CMD_PRCTL
, false },
1042 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB
, false },
1043 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP
, false },
1044 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB
, false },
1047 static void __init
spec_v2_user_print_cond(const char *reason
, bool secure
)
1049 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
1050 pr_info("spectre_v2_user=%s forced on command line.\n", reason
);
1053 static __ro_after_init
enum spectre_v2_mitigation_cmd spectre_v2_cmd
;
1055 static enum spectre_v2_user_cmd __init
1056 spectre_v2_parse_user_cmdline(void)
1061 switch (spectre_v2_cmd
) {
1062 case SPECTRE_V2_CMD_NONE
:
1063 return SPECTRE_V2_USER_CMD_NONE
;
1064 case SPECTRE_V2_CMD_FORCE
:
1065 return SPECTRE_V2_USER_CMD_FORCE
;
1070 ret
= cmdline_find_option(boot_command_line
, "spectre_v2_user",
1073 return SPECTRE_V2_USER_CMD_AUTO
;
1075 for (i
= 0; i
< ARRAY_SIZE(v2_user_options
); i
++) {
1076 if (match_option(arg
, ret
, v2_user_options
[i
].option
)) {
1077 spec_v2_user_print_cond(v2_user_options
[i
].option
,
1078 v2_user_options
[i
].secure
);
1079 return v2_user_options
[i
].cmd
;
1083 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg
);
1084 return SPECTRE_V2_USER_CMD_AUTO
;
1087 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode
)
1089 return mode
== SPECTRE_V2_IBRS
||
1090 mode
== SPECTRE_V2_EIBRS
||
1091 mode
== SPECTRE_V2_EIBRS_RETPOLINE
||
1092 mode
== SPECTRE_V2_EIBRS_LFENCE
;
1096 spectre_v2_user_select_mitigation(void)
1098 enum spectre_v2_user_mitigation mode
= SPECTRE_V2_USER_NONE
;
1099 bool smt_possible
= IS_ENABLED(CONFIG_SMP
);
1100 enum spectre_v2_user_cmd cmd
;
1102 if (!boot_cpu_has(X86_FEATURE_IBPB
) && !boot_cpu_has(X86_FEATURE_STIBP
))
1105 if (cpu_smt_control
== CPU_SMT_FORCE_DISABLED
||
1106 cpu_smt_control
== CPU_SMT_NOT_SUPPORTED
)
1107 smt_possible
= false;
1109 cmd
= spectre_v2_parse_user_cmdline();
1111 case SPECTRE_V2_USER_CMD_NONE
:
1113 case SPECTRE_V2_USER_CMD_FORCE
:
1114 mode
= SPECTRE_V2_USER_STRICT
;
1116 case SPECTRE_V2_USER_CMD_PRCTL
:
1117 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
1118 mode
= SPECTRE_V2_USER_PRCTL
;
1120 case SPECTRE_V2_USER_CMD_AUTO
:
1121 case SPECTRE_V2_USER_CMD_SECCOMP
:
1122 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
1123 if (IS_ENABLED(CONFIG_SECCOMP
))
1124 mode
= SPECTRE_V2_USER_SECCOMP
;
1126 mode
= SPECTRE_V2_USER_PRCTL
;
1130 /* Initialize Indirect Branch Prediction Barrier */
1131 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
1132 setup_force_cpu_cap(X86_FEATURE_USE_IBPB
);
1134 spectre_v2_user_ibpb
= mode
;
1136 case SPECTRE_V2_USER_CMD_FORCE
:
1137 case SPECTRE_V2_USER_CMD_PRCTL_IBPB
:
1138 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
:
1139 static_branch_enable(&switch_mm_always_ibpb
);
1140 spectre_v2_user_ibpb
= SPECTRE_V2_USER_STRICT
;
1142 case SPECTRE_V2_USER_CMD_PRCTL
:
1143 case SPECTRE_V2_USER_CMD_AUTO
:
1144 case SPECTRE_V2_USER_CMD_SECCOMP
:
1145 static_branch_enable(&switch_mm_cond_ibpb
);
1151 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1152 static_key_enabled(&switch_mm_always_ibpb
) ?
1153 "always-on" : "conditional");
1157 * If no STIBP, IBRS or enhanced IBRS is enabled, or SMT impossible,
1158 * STIBP is not required.
1160 if (!boot_cpu_has(X86_FEATURE_STIBP
) ||
1162 spectre_v2_in_ibrs_mode(spectre_v2_enabled
))
1166 * At this point, an STIBP mode other than "off" has been set.
1167 * If STIBP support is not being forced, check if STIBP always-on
1170 if (mode
!= SPECTRE_V2_USER_STRICT
&&
1171 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON
))
1172 mode
= SPECTRE_V2_USER_STRICT_PREFERRED
;
1174 if (retbleed_mitigation
== RETBLEED_MITIGATION_UNRET
) {
1175 if (mode
!= SPECTRE_V2_USER_STRICT
&&
1176 mode
!= SPECTRE_V2_USER_STRICT_PREFERRED
)
1177 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation'\n");
1178 mode
= SPECTRE_V2_USER_STRICT_PREFERRED
;
1181 spectre_v2_user_stibp
= mode
;
1184 pr_info("%s\n", spectre_v2_user_strings
[mode
]);
1187 static const char * const spectre_v2_strings
[] = {
1188 [SPECTRE_V2_NONE
] = "Vulnerable",
1189 [SPECTRE_V2_RETPOLINE
] = "Mitigation: Retpolines",
1190 [SPECTRE_V2_LFENCE
] = "Mitigation: LFENCE",
1191 [SPECTRE_V2_EIBRS
] = "Mitigation: Enhanced IBRS",
1192 [SPECTRE_V2_EIBRS_LFENCE
] = "Mitigation: Enhanced IBRS + LFENCE",
1193 [SPECTRE_V2_EIBRS_RETPOLINE
] = "Mitigation: Enhanced IBRS + Retpolines",
1194 [SPECTRE_V2_IBRS
] = "Mitigation: IBRS",
1197 static const struct {
1199 enum spectre_v2_mitigation_cmd cmd
;
1201 } mitigation_options
[] __initconst
= {
1202 { "off", SPECTRE_V2_CMD_NONE
, false },
1203 { "on", SPECTRE_V2_CMD_FORCE
, true },
1204 { "retpoline", SPECTRE_V2_CMD_RETPOLINE
, false },
1205 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE
, false },
1206 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE
, false },
1207 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC
, false },
1208 { "eibrs", SPECTRE_V2_CMD_EIBRS
, false },
1209 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE
, false },
1210 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE
, false },
1211 { "auto", SPECTRE_V2_CMD_AUTO
, false },
1212 { "ibrs", SPECTRE_V2_CMD_IBRS
, false },
1215 static void __init
spec_v2_print_cond(const char *reason
, bool secure
)
1217 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) != secure
)
1218 pr_info("%s selected on command line.\n", reason
);
1221 static enum spectre_v2_mitigation_cmd __init
spectre_v2_parse_cmdline(void)
1223 enum spectre_v2_mitigation_cmd cmd
= SPECTRE_V2_CMD_AUTO
;
1227 if (cmdline_find_option_bool(boot_command_line
, "nospectre_v2") ||
1228 cpu_mitigations_off())
1229 return SPECTRE_V2_CMD_NONE
;
1231 ret
= cmdline_find_option(boot_command_line
, "spectre_v2", arg
, sizeof(arg
));
1233 return SPECTRE_V2_CMD_AUTO
;
1235 for (i
= 0; i
< ARRAY_SIZE(mitigation_options
); i
++) {
1236 if (!match_option(arg
, ret
, mitigation_options
[i
].option
))
1238 cmd
= mitigation_options
[i
].cmd
;
1242 if (i
>= ARRAY_SIZE(mitigation_options
)) {
1243 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
1244 return SPECTRE_V2_CMD_AUTO
;
1247 if ((cmd
== SPECTRE_V2_CMD_RETPOLINE
||
1248 cmd
== SPECTRE_V2_CMD_RETPOLINE_LFENCE
||
1249 cmd
== SPECTRE_V2_CMD_RETPOLINE_GENERIC
||
1250 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
||
1251 cmd
== SPECTRE_V2_CMD_EIBRS_RETPOLINE
) &&
1252 !IS_ENABLED(CONFIG_RETPOLINE
)) {
1253 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1254 mitigation_options
[i
].option
);
1255 return SPECTRE_V2_CMD_AUTO
;
1258 if ((cmd
== SPECTRE_V2_CMD_EIBRS
||
1259 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
||
1260 cmd
== SPECTRE_V2_CMD_EIBRS_RETPOLINE
) &&
1261 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED
)) {
1262 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
1263 mitigation_options
[i
].option
);
1264 return SPECTRE_V2_CMD_AUTO
;
1267 if ((cmd
== SPECTRE_V2_CMD_RETPOLINE_LFENCE
||
1268 cmd
== SPECTRE_V2_CMD_EIBRS_LFENCE
) &&
1269 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC
)) {
1270 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1271 mitigation_options
[i
].option
);
1272 return SPECTRE_V2_CMD_AUTO
;
1275 if (cmd
== SPECTRE_V2_CMD_IBRS
&& !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY
)) {
1276 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1277 mitigation_options
[i
].option
);
1278 return SPECTRE_V2_CMD_AUTO
;
1281 if (cmd
== SPECTRE_V2_CMD_IBRS
&& boot_cpu_data
.x86_vendor
!= X86_VENDOR_INTEL
) {
1282 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1283 mitigation_options
[i
].option
);
1284 return SPECTRE_V2_CMD_AUTO
;
1287 if (cmd
== SPECTRE_V2_CMD_IBRS
&& !boot_cpu_has(X86_FEATURE_IBRS
)) {
1288 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1289 mitigation_options
[i
].option
);
1290 return SPECTRE_V2_CMD_AUTO
;
1293 if (cmd
== SPECTRE_V2_CMD_IBRS
&& boot_cpu_has(X86_FEATURE_XENPV
)) {
1294 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1295 mitigation_options
[i
].option
);
1296 return SPECTRE_V2_CMD_AUTO
;
1299 spec_v2_print_cond(mitigation_options
[i
].option
,
1300 mitigation_options
[i
].secure
);
1304 static enum spectre_v2_mitigation __init
spectre_v2_select_retpoline(void)
1306 if (!IS_ENABLED(CONFIG_RETPOLINE
)) {
1307 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1308 return SPECTRE_V2_NONE
;
1311 return SPECTRE_V2_RETPOLINE
;
1314 /* Disable in-kernel use of non-RSB RET predictors */
1315 static void __init
spec_ctrl_disable_kernel_rrsba(void)
1319 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL
))
1322 ia32_cap
= x86_read_arch_cap_msr();
1324 if (ia32_cap
& ARCH_CAP_RRSBA
) {
1325 x86_spec_ctrl_base
|= SPEC_CTRL_RRSBA_DIS_S
;
1326 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1330 static void __init
spectre_v2_select_mitigation(void)
1332 enum spectre_v2_mitigation_cmd cmd
= spectre_v2_parse_cmdline();
1333 enum spectre_v2_mitigation mode
= SPECTRE_V2_NONE
;
1336 * If the CPU is not affected and the command line mode is NONE or AUTO
1337 * then nothing to do.
1339 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
) &&
1340 (cmd
== SPECTRE_V2_CMD_NONE
|| cmd
== SPECTRE_V2_CMD_AUTO
))
1344 case SPECTRE_V2_CMD_NONE
:
1347 case SPECTRE_V2_CMD_FORCE
:
1348 case SPECTRE_V2_CMD_AUTO
:
1349 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED
)) {
1350 mode
= SPECTRE_V2_EIBRS
;
1354 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY
) &&
1355 boot_cpu_has_bug(X86_BUG_RETBLEED
) &&
1356 retbleed_cmd
!= RETBLEED_CMD_OFF
&&
1357 boot_cpu_has(X86_FEATURE_IBRS
) &&
1358 boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
) {
1359 mode
= SPECTRE_V2_IBRS
;
1363 mode
= spectre_v2_select_retpoline();
1366 case SPECTRE_V2_CMD_RETPOLINE_LFENCE
:
1367 pr_err(SPECTRE_V2_LFENCE_MSG
);
1368 mode
= SPECTRE_V2_LFENCE
;
1371 case SPECTRE_V2_CMD_RETPOLINE_GENERIC
:
1372 mode
= SPECTRE_V2_RETPOLINE
;
1375 case SPECTRE_V2_CMD_RETPOLINE
:
1376 mode
= spectre_v2_select_retpoline();
1379 case SPECTRE_V2_CMD_IBRS
:
1380 mode
= SPECTRE_V2_IBRS
;
1383 case SPECTRE_V2_CMD_EIBRS
:
1384 mode
= SPECTRE_V2_EIBRS
;
1387 case SPECTRE_V2_CMD_EIBRS_LFENCE
:
1388 mode
= SPECTRE_V2_EIBRS_LFENCE
;
1391 case SPECTRE_V2_CMD_EIBRS_RETPOLINE
:
1392 mode
= SPECTRE_V2_EIBRS_RETPOLINE
;
1396 if (mode
== SPECTRE_V2_EIBRS
&& unprivileged_ebpf_enabled())
1397 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG
);
1399 if (spectre_v2_in_ibrs_mode(mode
)) {
1400 x86_spec_ctrl_base
|= SPEC_CTRL_IBRS
;
1401 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1405 case SPECTRE_V2_NONE
:
1406 case SPECTRE_V2_EIBRS
:
1409 case SPECTRE_V2_IBRS
:
1410 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS
);
1413 case SPECTRE_V2_LFENCE
:
1414 case SPECTRE_V2_EIBRS_LFENCE
:
1415 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE
);
1418 case SPECTRE_V2_RETPOLINE
:
1419 case SPECTRE_V2_EIBRS_RETPOLINE
:
1420 setup_force_cpu_cap(X86_FEATURE_RETPOLINE
);
1425 * Disable alternate RSB predictions in kernel when indirect CALLs and
1426 * JMPs gets protection against BHI and Intramode-BTI, but RET
1427 * prediction from a non-RSB predictor is still a risk.
1429 if (mode
== SPECTRE_V2_EIBRS_LFENCE
||
1430 mode
== SPECTRE_V2_EIBRS_RETPOLINE
||
1431 mode
== SPECTRE_V2_RETPOLINE
)
1432 spec_ctrl_disable_kernel_rrsba();
1434 spectre_v2_enabled
= mode
;
1435 pr_info("%s\n", spectre_v2_strings
[mode
]);
1438 * If Spectre v2 protection has been enabled, fill the RSB during a
1439 * context switch. In general there are two types of RSB attacks
1440 * across context switches, for which the CALLs/RETs may be unbalanced.
1444 * Some Intel parts have "bottomless RSB". When the RSB is empty,
1445 * speculated return targets may come from the branch predictor,
1446 * which could have a user-poisoned BTB or BHB entry.
1448 * AMD has it even worse: *all* returns are speculated from the BTB,
1449 * regardless of the state of the RSB.
1451 * When IBRS or eIBRS is enabled, the "user -> kernel" attack
1452 * scenario is mitigated by the IBRS branch prediction isolation
1453 * properties, so the RSB buffer filling wouldn't be necessary to
1454 * protect against this type of attack.
1456 * The "user -> user" attack scenario is mitigated by RSB filling.
1458 * 2) Poisoned RSB entry
1460 * If the 'next' in-kernel return stack is shorter than 'prev',
1461 * 'next' could be tricked into speculating with a user-poisoned RSB
1464 * The "user -> kernel" attack scenario is mitigated by SMEP and
1467 * The "user -> user" scenario, also known as SpectreBHB, requires
1470 * So to mitigate all cases, unconditionally fill RSB on context
1473 * FIXME: Is this pointless for retbleed-affected AMD?
1475 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW
);
1476 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1479 * Similar to context switches, there are two types of RSB attacks
1484 * 2) Poisoned RSB entry
1486 * When retpoline is enabled, both are mitigated by filling/clearing
1489 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1490 * prediction isolation protections, RSB still needs to be cleared
1491 * because of #2. Note that SMEP provides no protection here, unlike
1492 * user-space-poisoned RSB entries.
1494 * eIBRS, on the other hand, has RSB-poisoning protections, so it
1495 * doesn't need RSB clearing after vmexit.
1497 if (boot_cpu_has(X86_FEATURE_RETPOLINE
) ||
1498 boot_cpu_has(X86_FEATURE_KERNEL_IBRS
))
1499 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT
);
1502 * Retpoline protects the kernel, but doesn't protect firmware. IBRS
1503 * and Enhanced IBRS protect firmware too, so enable IBRS around
1504 * firmware calls only when IBRS / Enhanced IBRS aren't otherwise
1507 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1508 * the user might select retpoline on the kernel command line and if
1509 * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1510 * enable IBRS around firmware calls.
1512 if (boot_cpu_has(X86_FEATURE_IBRS
) && !spectre_v2_in_ibrs_mode(mode
)) {
1513 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW
);
1514 pr_info("Enabling Restricted Speculation for firmware calls\n");
1517 /* Set up IBPB and STIBP depending on the general spectre V2 command */
1518 spectre_v2_cmd
= cmd
;
1521 static void update_stibp_msr(void * __unused
)
1523 u64 val
= spec_ctrl_current() | (x86_spec_ctrl_base
& SPEC_CTRL_STIBP
);
1524 write_spec_ctrl_current(val
, true);
1527 /* Update x86_spec_ctrl_base in case SMT state changed. */
1528 static void update_stibp_strict(void)
1530 u64 mask
= x86_spec_ctrl_base
& ~SPEC_CTRL_STIBP
;
1532 if (sched_smt_active())
1533 mask
|= SPEC_CTRL_STIBP
;
1535 if (mask
== x86_spec_ctrl_base
)
1538 pr_info("Update user space SMT mitigation: STIBP %s\n",
1539 mask
& SPEC_CTRL_STIBP
? "always-on" : "off");
1540 x86_spec_ctrl_base
= mask
;
1541 on_each_cpu(update_stibp_msr
, NULL
, 1);
1544 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1545 static void update_indir_branch_cond(void)
1547 if (sched_smt_active())
1548 static_branch_enable(&switch_to_cond_stibp
);
1550 static_branch_disable(&switch_to_cond_stibp
);
1554 #define pr_fmt(fmt) fmt
1556 /* Update the static key controlling the MDS CPU buffer clear in idle */
1557 static void update_mds_branch_idle(void)
1559 u64 ia32_cap
= x86_read_arch_cap_msr();
1562 * Enable the idle clearing if SMT is active on CPUs which are
1563 * affected only by MSBDS and not any other MDS variant.
1565 * The other variants cannot be mitigated when SMT is enabled, so
1566 * clearing the buffers on idle just to prevent the Store Buffer
1567 * repartitioning leak would be a window dressing exercise.
1569 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY
))
1572 if (sched_smt_active()) {
1573 static_branch_enable(&mds_idle_clear
);
1574 } else if (mmio_mitigation
== MMIO_MITIGATION_OFF
||
1575 (ia32_cap
& ARCH_CAP_FBSDP_NO
)) {
1576 static_branch_disable(&mds_idle_clear
);
1580 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1581 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1582 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1584 void cpu_bugs_smt_update(void)
1586 mutex_lock(&spec_ctrl_mutex
);
1588 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1589 spectre_v2_enabled
== SPECTRE_V2_EIBRS_LFENCE
)
1590 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG
);
1592 switch (spectre_v2_user_stibp
) {
1593 case SPECTRE_V2_USER_NONE
:
1595 case SPECTRE_V2_USER_STRICT
:
1596 case SPECTRE_V2_USER_STRICT_PREFERRED
:
1597 update_stibp_strict();
1599 case SPECTRE_V2_USER_PRCTL
:
1600 case SPECTRE_V2_USER_SECCOMP
:
1601 update_indir_branch_cond();
1605 switch (mds_mitigation
) {
1606 case MDS_MITIGATION_FULL
:
1607 case MDS_MITIGATION_VMWERV
:
1608 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY
))
1609 pr_warn_once(MDS_MSG_SMT
);
1610 update_mds_branch_idle();
1612 case MDS_MITIGATION_OFF
:
1616 switch (taa_mitigation
) {
1617 case TAA_MITIGATION_VERW
:
1618 case TAA_MITIGATION_UCODE_NEEDED
:
1619 if (sched_smt_active())
1620 pr_warn_once(TAA_MSG_SMT
);
1622 case TAA_MITIGATION_TSX_DISABLED
:
1623 case TAA_MITIGATION_OFF
:
1627 switch (mmio_mitigation
) {
1628 case MMIO_MITIGATION_VERW
:
1629 case MMIO_MITIGATION_UCODE_NEEDED
:
1630 if (sched_smt_active())
1631 pr_warn_once(MMIO_MSG_SMT
);
1633 case MMIO_MITIGATION_OFF
:
1637 mutex_unlock(&spec_ctrl_mutex
);
1641 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt
1643 static enum ssb_mitigation ssb_mode __ro_after_init
= SPEC_STORE_BYPASS_NONE
;
1645 /* The kernel command line selection */
1646 enum ssb_mitigation_cmd
{
1647 SPEC_STORE_BYPASS_CMD_NONE
,
1648 SPEC_STORE_BYPASS_CMD_AUTO
,
1649 SPEC_STORE_BYPASS_CMD_ON
,
1650 SPEC_STORE_BYPASS_CMD_PRCTL
,
1651 SPEC_STORE_BYPASS_CMD_SECCOMP
,
1654 static const char * const ssb_strings
[] = {
1655 [SPEC_STORE_BYPASS_NONE
] = "Vulnerable",
1656 [SPEC_STORE_BYPASS_DISABLE
] = "Mitigation: Speculative Store Bypass disabled",
1657 [SPEC_STORE_BYPASS_PRCTL
] = "Mitigation: Speculative Store Bypass disabled via prctl",
1658 [SPEC_STORE_BYPASS_SECCOMP
] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
1661 static const struct {
1663 enum ssb_mitigation_cmd cmd
;
1664 } ssb_mitigation_options
[] __initconst
= {
1665 { "auto", SPEC_STORE_BYPASS_CMD_AUTO
}, /* Platform decides */
1666 { "on", SPEC_STORE_BYPASS_CMD_ON
}, /* Disable Speculative Store Bypass */
1667 { "off", SPEC_STORE_BYPASS_CMD_NONE
}, /* Don't touch Speculative Store Bypass */
1668 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL
}, /* Disable Speculative Store Bypass via prctl */
1669 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP
}, /* Disable Speculative Store Bypass via prctl and seccomp */
1672 static enum ssb_mitigation_cmd __init
ssb_parse_cmdline(void)
1674 enum ssb_mitigation_cmd cmd
= SPEC_STORE_BYPASS_CMD_AUTO
;
1678 if (cmdline_find_option_bool(boot_command_line
, "nospec_store_bypass_disable") ||
1679 cpu_mitigations_off()) {
1680 return SPEC_STORE_BYPASS_CMD_NONE
;
1682 ret
= cmdline_find_option(boot_command_line
, "spec_store_bypass_disable",
1685 return SPEC_STORE_BYPASS_CMD_AUTO
;
1687 for (i
= 0; i
< ARRAY_SIZE(ssb_mitigation_options
); i
++) {
1688 if (!match_option(arg
, ret
, ssb_mitigation_options
[i
].option
))
1691 cmd
= ssb_mitigation_options
[i
].cmd
;
1695 if (i
>= ARRAY_SIZE(ssb_mitigation_options
)) {
1696 pr_err("unknown option (%s). Switching to AUTO select\n", arg
);
1697 return SPEC_STORE_BYPASS_CMD_AUTO
;
1704 static enum ssb_mitigation __init
__ssb_select_mitigation(void)
1706 enum ssb_mitigation mode
= SPEC_STORE_BYPASS_NONE
;
1707 enum ssb_mitigation_cmd cmd
;
1709 if (!boot_cpu_has(X86_FEATURE_SSBD
))
1712 cmd
= ssb_parse_cmdline();
1713 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
) &&
1714 (cmd
== SPEC_STORE_BYPASS_CMD_NONE
||
1715 cmd
== SPEC_STORE_BYPASS_CMD_AUTO
))
1719 case SPEC_STORE_BYPASS_CMD_AUTO
:
1720 case SPEC_STORE_BYPASS_CMD_SECCOMP
:
1722 * Choose prctl+seccomp as the default mode if seccomp is
1725 if (IS_ENABLED(CONFIG_SECCOMP
))
1726 mode
= SPEC_STORE_BYPASS_SECCOMP
;
1728 mode
= SPEC_STORE_BYPASS_PRCTL
;
1730 case SPEC_STORE_BYPASS_CMD_ON
:
1731 mode
= SPEC_STORE_BYPASS_DISABLE
;
1733 case SPEC_STORE_BYPASS_CMD_PRCTL
:
1734 mode
= SPEC_STORE_BYPASS_PRCTL
;
1736 case SPEC_STORE_BYPASS_CMD_NONE
:
1741 * We have three CPU feature flags that are in play here:
1742 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1743 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1744 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1746 if (mode
== SPEC_STORE_BYPASS_DISABLE
) {
1747 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE
);
1749 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1750 * use a completely different MSR and bit dependent on family.
1752 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD
) &&
1753 !static_cpu_has(X86_FEATURE_AMD_SSBD
)) {
1754 x86_amd_ssb_disable();
1756 x86_spec_ctrl_base
|= SPEC_CTRL_SSBD
;
1757 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
1764 static void ssb_select_mitigation(void)
1766 ssb_mode
= __ssb_select_mitigation();
1768 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
1769 pr_info("%s\n", ssb_strings
[ssb_mode
]);
1773 #define pr_fmt(fmt) "Speculation prctl: " fmt
1775 static void task_update_spec_tif(struct task_struct
*tsk
)
1777 /* Force the update of the real TIF bits */
1778 set_tsk_thread_flag(tsk
, TIF_SPEC_FORCE_UPDATE
);
1781 * Immediately update the speculation control MSRs for the current
1782 * task, but for a non-current task delay setting the CPU
1783 * mitigation until it is scheduled next.
1785 * This can only happen for SECCOMP mitigation. For PRCTL it's
1786 * always the current task.
1789 speculation_ctrl_update_current();
1792 static int l1d_flush_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1795 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush
))
1799 case PR_SPEC_ENABLE
:
1800 set_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
);
1802 case PR_SPEC_DISABLE
:
1803 clear_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
);
1810 static int ssb_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1812 if (ssb_mode
!= SPEC_STORE_BYPASS_PRCTL
&&
1813 ssb_mode
!= SPEC_STORE_BYPASS_SECCOMP
)
1817 case PR_SPEC_ENABLE
:
1818 /* If speculation is force disabled, enable is not allowed */
1819 if (task_spec_ssb_force_disable(task
))
1821 task_clear_spec_ssb_disable(task
);
1822 task_clear_spec_ssb_noexec(task
);
1823 task_update_spec_tif(task
);
1825 case PR_SPEC_DISABLE
:
1826 task_set_spec_ssb_disable(task
);
1827 task_clear_spec_ssb_noexec(task
);
1828 task_update_spec_tif(task
);
1830 case PR_SPEC_FORCE_DISABLE
:
1831 task_set_spec_ssb_disable(task
);
1832 task_set_spec_ssb_force_disable(task
);
1833 task_clear_spec_ssb_noexec(task
);
1834 task_update_spec_tif(task
);
1836 case PR_SPEC_DISABLE_NOEXEC
:
1837 if (task_spec_ssb_force_disable(task
))
1839 task_set_spec_ssb_disable(task
);
1840 task_set_spec_ssb_noexec(task
);
1841 task_update_spec_tif(task
);
1849 static bool is_spec_ib_user_controlled(void)
1851 return spectre_v2_user_ibpb
== SPECTRE_V2_USER_PRCTL
||
1852 spectre_v2_user_ibpb
== SPECTRE_V2_USER_SECCOMP
||
1853 spectre_v2_user_stibp
== SPECTRE_V2_USER_PRCTL
||
1854 spectre_v2_user_stibp
== SPECTRE_V2_USER_SECCOMP
;
1857 static int ib_prctl_set(struct task_struct
*task
, unsigned long ctrl
)
1860 case PR_SPEC_ENABLE
:
1861 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1862 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1866 * With strict mode for both IBPB and STIBP, the instruction
1867 * code paths avoid checking this task flag and instead,
1868 * unconditionally run the instruction. However, STIBP and IBPB
1869 * are independent and either can be set to conditionally
1870 * enabled regardless of the mode of the other.
1872 * If either is set to conditional, allow the task flag to be
1873 * updated, unless it was force-disabled by a previous prctl
1874 * call. Currently, this is possible on an AMD CPU which has the
1875 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
1876 * kernel is booted with 'spectre_v2_user=seccomp', then
1877 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
1878 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
1880 if (!is_spec_ib_user_controlled() ||
1881 task_spec_ib_force_disable(task
))
1884 task_clear_spec_ib_disable(task
);
1885 task_update_spec_tif(task
);
1887 case PR_SPEC_DISABLE
:
1888 case PR_SPEC_FORCE_DISABLE
:
1890 * Indirect branch speculation is always allowed when
1891 * mitigation is force disabled.
1893 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1894 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1897 if (!is_spec_ib_user_controlled())
1900 task_set_spec_ib_disable(task
);
1901 if (ctrl
== PR_SPEC_FORCE_DISABLE
)
1902 task_set_spec_ib_force_disable(task
);
1903 task_update_spec_tif(task
);
1911 int arch_prctl_spec_ctrl_set(struct task_struct
*task
, unsigned long which
,
1915 case PR_SPEC_STORE_BYPASS
:
1916 return ssb_prctl_set(task
, ctrl
);
1917 case PR_SPEC_INDIRECT_BRANCH
:
1918 return ib_prctl_set(task
, ctrl
);
1919 case PR_SPEC_L1D_FLUSH
:
1920 return l1d_flush_prctl_set(task
, ctrl
);
1926 #ifdef CONFIG_SECCOMP
1927 void arch_seccomp_spec_mitigate(struct task_struct
*task
)
1929 if (ssb_mode
== SPEC_STORE_BYPASS_SECCOMP
)
1930 ssb_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1931 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_SECCOMP
||
1932 spectre_v2_user_stibp
== SPECTRE_V2_USER_SECCOMP
)
1933 ib_prctl_set(task
, PR_SPEC_FORCE_DISABLE
);
1937 static int l1d_flush_prctl_get(struct task_struct
*task
)
1939 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush
))
1940 return PR_SPEC_FORCE_DISABLE
;
1942 if (test_ti_thread_flag(&task
->thread_info
, TIF_SPEC_L1D_FLUSH
))
1943 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1945 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1948 static int ssb_prctl_get(struct task_struct
*task
)
1951 case SPEC_STORE_BYPASS_DISABLE
:
1952 return PR_SPEC_DISABLE
;
1953 case SPEC_STORE_BYPASS_SECCOMP
:
1954 case SPEC_STORE_BYPASS_PRCTL
:
1955 if (task_spec_ssb_force_disable(task
))
1956 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1957 if (task_spec_ssb_noexec(task
))
1958 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE_NOEXEC
;
1959 if (task_spec_ssb_disable(task
))
1960 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1961 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1963 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS
))
1964 return PR_SPEC_ENABLE
;
1965 return PR_SPEC_NOT_AFFECTED
;
1969 static int ib_prctl_get(struct task_struct
*task
)
1971 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2
))
1972 return PR_SPEC_NOT_AFFECTED
;
1974 if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_NONE
&&
1975 spectre_v2_user_stibp
== SPECTRE_V2_USER_NONE
)
1976 return PR_SPEC_ENABLE
;
1977 else if (is_spec_ib_user_controlled()) {
1978 if (task_spec_ib_force_disable(task
))
1979 return PR_SPEC_PRCTL
| PR_SPEC_FORCE_DISABLE
;
1980 if (task_spec_ib_disable(task
))
1981 return PR_SPEC_PRCTL
| PR_SPEC_DISABLE
;
1982 return PR_SPEC_PRCTL
| PR_SPEC_ENABLE
;
1983 } else if (spectre_v2_user_ibpb
== SPECTRE_V2_USER_STRICT
||
1984 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT
||
1985 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT_PREFERRED
)
1986 return PR_SPEC_DISABLE
;
1988 return PR_SPEC_NOT_AFFECTED
;
1991 int arch_prctl_spec_ctrl_get(struct task_struct
*task
, unsigned long which
)
1994 case PR_SPEC_STORE_BYPASS
:
1995 return ssb_prctl_get(task
);
1996 case PR_SPEC_INDIRECT_BRANCH
:
1997 return ib_prctl_get(task
);
1998 case PR_SPEC_L1D_FLUSH
:
1999 return l1d_flush_prctl_get(task
);
2005 void x86_spec_ctrl_setup_ap(void)
2007 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL
))
2008 write_spec_ctrl_current(x86_spec_ctrl_base
, true);
2010 if (ssb_mode
== SPEC_STORE_BYPASS_DISABLE
)
2011 x86_amd_ssb_disable();
2014 bool itlb_multihit_kvm_mitigation
;
2015 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation
);
2018 #define pr_fmt(fmt) "L1TF: " fmt
2020 /* Default mitigation for L1TF-affected CPUs */
2021 enum l1tf_mitigations l1tf_mitigation __ro_after_init
= L1TF_MITIGATION_FLUSH
;
2022 #if IS_ENABLED(CONFIG_KVM_INTEL)
2023 EXPORT_SYMBOL_GPL(l1tf_mitigation
);
2025 enum vmx_l1d_flush_state l1tf_vmx_mitigation
= VMENTER_L1D_FLUSH_AUTO
;
2026 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation
);
2029 * These CPUs all support 44bits physical address space internally in the
2030 * cache but CPUID can report a smaller number of physical address bits.
2032 * The L1TF mitigation uses the top most address bit for the inversion of
2033 * non present PTEs. When the installed memory reaches into the top most
2034 * address bit due to memory holes, which has been observed on machines
2035 * which report 36bits physical address bits and have 32G RAM installed,
2036 * then the mitigation range check in l1tf_select_mitigation() triggers.
2037 * This is a false positive because the mitigation is still possible due to
2038 * the fact that the cache uses 44bit internally. Use the cache bits
2039 * instead of the reported physical bits and adjust them on the affected
2040 * machines to 44bit if the reported bits are less than 44.
2042 static void override_cache_bits(struct cpuinfo_x86
*c
)
2047 switch (c
->x86_model
) {
2048 case INTEL_FAM6_NEHALEM
:
2049 case INTEL_FAM6_WESTMERE
:
2050 case INTEL_FAM6_SANDYBRIDGE
:
2051 case INTEL_FAM6_IVYBRIDGE
:
2052 case INTEL_FAM6_HASWELL
:
2053 case INTEL_FAM6_HASWELL_L
:
2054 case INTEL_FAM6_HASWELL_G
:
2055 case INTEL_FAM6_BROADWELL
:
2056 case INTEL_FAM6_BROADWELL_G
:
2057 case INTEL_FAM6_SKYLAKE_L
:
2058 case INTEL_FAM6_SKYLAKE
:
2059 case INTEL_FAM6_KABYLAKE_L
:
2060 case INTEL_FAM6_KABYLAKE
:
2061 if (c
->x86_cache_bits
< 44)
2062 c
->x86_cache_bits
= 44;
2067 static void __init
l1tf_select_mitigation(void)
2071 if (!boot_cpu_has_bug(X86_BUG_L1TF
))
2074 if (cpu_mitigations_off())
2075 l1tf_mitigation
= L1TF_MITIGATION_OFF
;
2076 else if (cpu_mitigations_auto_nosmt())
2077 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOSMT
;
2079 override_cache_bits(&boot_cpu_data
);
2081 switch (l1tf_mitigation
) {
2082 case L1TF_MITIGATION_OFF
:
2083 case L1TF_MITIGATION_FLUSH_NOWARN
:
2084 case L1TF_MITIGATION_FLUSH
:
2086 case L1TF_MITIGATION_FLUSH_NOSMT
:
2087 case L1TF_MITIGATION_FULL
:
2088 cpu_smt_disable(false);
2090 case L1TF_MITIGATION_FULL_FORCE
:
2091 cpu_smt_disable(true);
2095 #if CONFIG_PGTABLE_LEVELS == 2
2096 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2100 half_pa
= (u64
)l1tf_pfn_limit() << PAGE_SHIFT
;
2101 if (l1tf_mitigation
!= L1TF_MITIGATION_OFF
&&
2102 e820__mapped_any(half_pa
, ULLONG_MAX
- half_pa
, E820_TYPE_RAM
)) {
2103 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2104 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2106 pr_info("However, doing so will make a part of your RAM unusable.\n");
2107 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2111 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV
);
2114 static int __init
l1tf_cmdline(char *str
)
2116 if (!boot_cpu_has_bug(X86_BUG_L1TF
))
2122 if (!strcmp(str
, "off"))
2123 l1tf_mitigation
= L1TF_MITIGATION_OFF
;
2124 else if (!strcmp(str
, "flush,nowarn"))
2125 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOWARN
;
2126 else if (!strcmp(str
, "flush"))
2127 l1tf_mitigation
= L1TF_MITIGATION_FLUSH
;
2128 else if (!strcmp(str
, "flush,nosmt"))
2129 l1tf_mitigation
= L1TF_MITIGATION_FLUSH_NOSMT
;
2130 else if (!strcmp(str
, "full"))
2131 l1tf_mitigation
= L1TF_MITIGATION_FULL
;
2132 else if (!strcmp(str
, "full,force"))
2133 l1tf_mitigation
= L1TF_MITIGATION_FULL_FORCE
;
2137 early_param("l1tf", l1tf_cmdline
);
2140 #define pr_fmt(fmt) fmt
2144 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2146 #if IS_ENABLED(CONFIG_KVM_INTEL)
2147 static const char * const l1tf_vmx_states
[] = {
2148 [VMENTER_L1D_FLUSH_AUTO
] = "auto",
2149 [VMENTER_L1D_FLUSH_NEVER
] = "vulnerable",
2150 [VMENTER_L1D_FLUSH_COND
] = "conditional cache flushes",
2151 [VMENTER_L1D_FLUSH_ALWAYS
] = "cache flushes",
2152 [VMENTER_L1D_FLUSH_EPT_DISABLED
] = "EPT disabled",
2153 [VMENTER_L1D_FLUSH_NOT_REQUIRED
] = "flush not necessary"
2156 static ssize_t
l1tf_show_state(char *buf
)
2158 if (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_AUTO
)
2159 return sprintf(buf
, "%s\n", L1TF_DEFAULT_MSG
);
2161 if (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_EPT_DISABLED
||
2162 (l1tf_vmx_mitigation
== VMENTER_L1D_FLUSH_NEVER
&&
2163 sched_smt_active())) {
2164 return sprintf(buf
, "%s; VMX: %s\n", L1TF_DEFAULT_MSG
,
2165 l1tf_vmx_states
[l1tf_vmx_mitigation
]);
2168 return sprintf(buf
, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG
,
2169 l1tf_vmx_states
[l1tf_vmx_mitigation
],
2170 sched_smt_active() ? "vulnerable" : "disabled");
2173 static ssize_t
itlb_multihit_show_state(char *buf
)
2175 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL
) ||
2176 !boot_cpu_has(X86_FEATURE_VMX
))
2177 return sprintf(buf
, "KVM: Mitigation: VMX unsupported\n");
2178 else if (!(cr4_read_shadow() & X86_CR4_VMXE
))
2179 return sprintf(buf
, "KVM: Mitigation: VMX disabled\n");
2180 else if (itlb_multihit_kvm_mitigation
)
2181 return sprintf(buf
, "KVM: Mitigation: Split huge pages\n");
2183 return sprintf(buf
, "KVM: Vulnerable\n");
2186 static ssize_t
l1tf_show_state(char *buf
)
2188 return sprintf(buf
, "%s\n", L1TF_DEFAULT_MSG
);
2191 static ssize_t
itlb_multihit_show_state(char *buf
)
2193 return sprintf(buf
, "Processor vulnerable\n");
2197 static ssize_t
mds_show_state(char *buf
)
2199 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2200 return sprintf(buf
, "%s; SMT Host state unknown\n",
2201 mds_strings
[mds_mitigation
]);
2204 if (boot_cpu_has(X86_BUG_MSBDS_ONLY
)) {
2205 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
2206 (mds_mitigation
== MDS_MITIGATION_OFF
? "vulnerable" :
2207 sched_smt_active() ? "mitigated" : "disabled"));
2210 return sprintf(buf
, "%s; SMT %s\n", mds_strings
[mds_mitigation
],
2211 sched_smt_active() ? "vulnerable" : "disabled");
2214 static ssize_t
tsx_async_abort_show_state(char *buf
)
2216 if ((taa_mitigation
== TAA_MITIGATION_TSX_DISABLED
) ||
2217 (taa_mitigation
== TAA_MITIGATION_OFF
))
2218 return sprintf(buf
, "%s\n", taa_strings
[taa_mitigation
]);
2220 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2221 return sprintf(buf
, "%s; SMT Host state unknown\n",
2222 taa_strings
[taa_mitigation
]);
2225 return sprintf(buf
, "%s; SMT %s\n", taa_strings
[taa_mitigation
],
2226 sched_smt_active() ? "vulnerable" : "disabled");
2229 static ssize_t
mmio_stale_data_show_state(char *buf
)
2231 if (mmio_mitigation
== MMIO_MITIGATION_OFF
)
2232 return sysfs_emit(buf
, "%s\n", mmio_strings
[mmio_mitigation
]);
2234 if (boot_cpu_has(X86_FEATURE_HYPERVISOR
)) {
2235 return sysfs_emit(buf
, "%s; SMT Host state unknown\n",
2236 mmio_strings
[mmio_mitigation
]);
2239 return sysfs_emit(buf
, "%s; SMT %s\n", mmio_strings
[mmio_mitigation
],
2240 sched_smt_active() ? "vulnerable" : "disabled");
2243 static char *stibp_state(void)
2245 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled
))
2248 switch (spectre_v2_user_stibp
) {
2249 case SPECTRE_V2_USER_NONE
:
2250 return ", STIBP: disabled";
2251 case SPECTRE_V2_USER_STRICT
:
2252 return ", STIBP: forced";
2253 case SPECTRE_V2_USER_STRICT_PREFERRED
:
2254 return ", STIBP: always-on";
2255 case SPECTRE_V2_USER_PRCTL
:
2256 case SPECTRE_V2_USER_SECCOMP
:
2257 if (static_key_enabled(&switch_to_cond_stibp
))
2258 return ", STIBP: conditional";
2263 static char *ibpb_state(void)
2265 if (boot_cpu_has(X86_FEATURE_IBPB
)) {
2266 if (static_key_enabled(&switch_mm_always_ibpb
))
2267 return ", IBPB: always-on";
2268 if (static_key_enabled(&switch_mm_cond_ibpb
))
2269 return ", IBPB: conditional";
2270 return ", IBPB: disabled";
2275 static ssize_t
spectre_v2_show_state(char *buf
)
2277 if (spectre_v2_enabled
== SPECTRE_V2_LFENCE
)
2278 return sprintf(buf
, "Vulnerable: LFENCE\n");
2280 if (spectre_v2_enabled
== SPECTRE_V2_EIBRS
&& unprivileged_ebpf_enabled())
2281 return sprintf(buf
, "Vulnerable: eIBRS with unprivileged eBPF\n");
2283 if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2284 spectre_v2_enabled
== SPECTRE_V2_EIBRS_LFENCE
)
2285 return sprintf(buf
, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2287 return sprintf(buf
, "%s%s%s%s%s%s\n",
2288 spectre_v2_strings
[spectre_v2_enabled
],
2290 boot_cpu_has(X86_FEATURE_USE_IBRS_FW
) ? ", IBRS_FW" : "",
2292 boot_cpu_has(X86_FEATURE_RSB_CTXSW
) ? ", RSB filling" : "",
2293 spectre_v2_module_string());
2296 static ssize_t
srbds_show_state(char *buf
)
2298 return sprintf(buf
, "%s\n", srbds_strings
[srbds_mitigation
]);
2301 static ssize_t
retbleed_show_state(char *buf
)
2303 if (retbleed_mitigation
== RETBLEED_MITIGATION_UNRET
) {
2304 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
&&
2305 boot_cpu_data
.x86_vendor
!= X86_VENDOR_HYGON
)
2306 return sprintf(buf
, "Vulnerable: untrained return thunk on non-Zen uarch\n");
2308 return sprintf(buf
, "%s; SMT %s\n",
2309 retbleed_strings
[retbleed_mitigation
],
2310 !sched_smt_active() ? "disabled" :
2311 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT
||
2312 spectre_v2_user_stibp
== SPECTRE_V2_USER_STRICT_PREFERRED
?
2313 "enabled with STIBP protection" : "vulnerable");
2316 return sprintf(buf
, "%s\n", retbleed_strings
[retbleed_mitigation
]);
2319 static ssize_t
cpu_show_common(struct device
*dev
, struct device_attribute
*attr
,
2320 char *buf
, unsigned int bug
)
2322 if (!boot_cpu_has_bug(bug
))
2323 return sprintf(buf
, "Not affected\n");
2326 case X86_BUG_CPU_MELTDOWN
:
2327 if (boot_cpu_has(X86_FEATURE_PTI
))
2328 return sprintf(buf
, "Mitigation: PTI\n");
2330 if (hypervisor_is_type(X86_HYPER_XEN_PV
))
2331 return sprintf(buf
, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2335 case X86_BUG_SPECTRE_V1
:
2336 return sprintf(buf
, "%s\n", spectre_v1_strings
[spectre_v1_mitigation
]);
2338 case X86_BUG_SPECTRE_V2
:
2339 return spectre_v2_show_state(buf
);
2341 case X86_BUG_SPEC_STORE_BYPASS
:
2342 return sprintf(buf
, "%s\n", ssb_strings
[ssb_mode
]);
2345 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV
))
2346 return l1tf_show_state(buf
);
2350 return mds_show_state(buf
);
2353 return tsx_async_abort_show_state(buf
);
2355 case X86_BUG_ITLB_MULTIHIT
:
2356 return itlb_multihit_show_state(buf
);
2359 return srbds_show_state(buf
);
2361 case X86_BUG_MMIO_STALE_DATA
:
2362 return mmio_stale_data_show_state(buf
);
2364 case X86_BUG_RETBLEED
:
2365 return retbleed_show_state(buf
);
2371 return sprintf(buf
, "Vulnerable\n");
2374 ssize_t
cpu_show_meltdown(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2376 return cpu_show_common(dev
, attr
, buf
, X86_BUG_CPU_MELTDOWN
);
2379 ssize_t
cpu_show_spectre_v1(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2381 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V1
);
2384 ssize_t
cpu_show_spectre_v2(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2386 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPECTRE_V2
);
2389 ssize_t
cpu_show_spec_store_bypass(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2391 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SPEC_STORE_BYPASS
);
2394 ssize_t
cpu_show_l1tf(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2396 return cpu_show_common(dev
, attr
, buf
, X86_BUG_L1TF
);
2399 ssize_t
cpu_show_mds(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2401 return cpu_show_common(dev
, attr
, buf
, X86_BUG_MDS
);
2404 ssize_t
cpu_show_tsx_async_abort(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2406 return cpu_show_common(dev
, attr
, buf
, X86_BUG_TAA
);
2409 ssize_t
cpu_show_itlb_multihit(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2411 return cpu_show_common(dev
, attr
, buf
, X86_BUG_ITLB_MULTIHIT
);
2414 ssize_t
cpu_show_srbds(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2416 return cpu_show_common(dev
, attr
, buf
, X86_BUG_SRBDS
);
2419 ssize_t
cpu_show_mmio_stale_data(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2421 return cpu_show_common(dev
, attr
, buf
, X86_BUG_MMIO_STALE_DATA
);
2424 ssize_t
cpu_show_retbleed(struct device
*dev
, struct device_attribute
*attr
, char *buf
)
2426 return cpu_show_common(dev
, attr
, buf
, X86_BUG_RETBLEED
);