1 /* SPDX-License-Identifier: GPL-2.0 */
3 #ifndef __NOSPEC_BRANCH_H__
4 #define __NOSPEC_BRANCH_H__
6 #include <asm/alternative.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/cpufeatures.h>
11 * Fill the CPU return stack buffer.
13 * Each entry in the RSB, if used for a speculative 'ret', contains an
14 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
16 * This is required in various cases for retpoline and IBRS-based
17 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
18 * eliminate potentially bogus entries from the RSB, and sometimes
19 * purely to ensure that it doesn't get empty, which on some CPUs would
20 * allow predictions from other (unwanted!) sources to be used.
22 * We define a CPP macro such that it can be used from both .S files and
23 * inline assembly. It's possible to do a .macro and then include that
24 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
27 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
28 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
31 * Google experimented with loop-unrolling and this turned out to be
32 * the optimal version — two calls, each with their own speculation
33 * trap should their return address end up getting used, in a loop.
35 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
39 773: /* speculation trap */ \
45 775: /* speculation trap */ \
52 add $(BITS_PER_LONG/8) * nr, sp;
57 * This should be used immediately before a retpoline alternative. It tells
58 * objtool where the retpolines are so that it can make sense of the control
59 * flow by just reading the original instruction(s) and ignoring the
62 .macro ANNOTATE_NOSPEC_ALTERNATIVE
64 .pushsection
.discard
.nospec
65 .long .Lannotate_\@
- .
70 * These are the bare retpoline primitives for indirect jmp and call.
71 * Do not use these directly; they only exist to make the ALTERNATIVE
72 * invocation below less ugly.
74 .macro RETPOLINE_JMP reg
:req
86 * This is a wrapper around RETPOLINE_JMP so the called function in reg
87 * returns to the instruction after the macro.
89 .macro RETPOLINE_CALL reg
:req
91 .Ldo_retpoline_jmp_\@
:
94 call
.Ldo_retpoline_jmp_\@
98 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
99 * indirect jmp/call which may be susceptible to the Spectre variant 2
102 .macro JMP_NOSPEC reg
:req
103 #ifdef CONFIG_RETPOLINE
104 ANNOTATE_NOSPEC_ALTERNATIVE
105 ALTERNATIVE_2
__stringify(jmp
*\reg
), \
106 __stringify(RETPOLINE_JMP
\reg
), X86_FEATURE_RETPOLINE
, \
107 __stringify(lfence
; jmp
*\reg
), X86_FEATURE_RETPOLINE_AMD
113 .macro CALL_NOSPEC reg
:req
114 #ifdef CONFIG_RETPOLINE
115 ANNOTATE_NOSPEC_ALTERNATIVE
116 ALTERNATIVE_2
__stringify(call
*\reg
), \
117 __stringify(RETPOLINE_CALL
\reg
), X86_FEATURE_RETPOLINE
,\
118 __stringify(lfence
; call
*\reg
), X86_FEATURE_RETPOLINE_AMD
125 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
126 * monstrosity above, manually.
128 .macro FILL_RETURN_BUFFER reg
:req nr
:req ftr
:req
129 #ifdef CONFIG_RETPOLINE
130 ANNOTATE_NOSPEC_ALTERNATIVE
131 ALTERNATIVE
"jmp .Lskip_rsb_\@", \
132 __stringify(__FILL_RETURN_BUFFER(\reg
,\nr
,%_ASM_SP
)) \
138 #else /* __ASSEMBLY__ */
140 #define ANNOTATE_NOSPEC_ALTERNATIVE \
142 ".pushsection .discard.nospec\n\t" \
143 ".long 999b - .\n\t" \
146 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
149 * Since the inline asm uses the %V modifier which is only in newer GCC,
150 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
152 # define CALL_NOSPEC \
153 ANNOTATE_NOSPEC_ALTERNATIVE \
155 "call *%[thunk_target]\n", \
156 "call __x86_indirect_thunk_%V[thunk_target]\n", \
157 X86_FEATURE_RETPOLINE)
158 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
160 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
162 * For i386 we use the original ret-equivalent retpoline, because
163 * otherwise we'll run out of registers. We don't care about CET
166 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
169 "901: call 903f;\n" \
174 "903: addl $4, %%esp;\n" \
175 " pushl %[thunk_target];\n" \
178 "904: call 901b;\n", \
179 X86_FEATURE_RETPOLINE)
181 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
182 #else /* No retpoline for C / inline asm */
183 # define CALL_NOSPEC "call *%[thunk_target]\n"
184 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
187 /* The Spectre V2 mitigation variants */
188 enum spectre_v2_mitigation
{
190 SPECTRE_V2_RETPOLINE_MINIMAL
,
191 SPECTRE_V2_RETPOLINE_MINIMAL_AMD
,
192 SPECTRE_V2_RETPOLINE_GENERIC
,
193 SPECTRE_V2_RETPOLINE_AMD
,
198 * On VMEXIT we must ensure that no RSB predictions learned in the guest
199 * can be followed in the host, by overwriting the RSB completely. Both
200 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
201 * CPUs with IBRS_ATT *might* it be avoided.
203 static inline void vmexit_fill_RSB(void)
205 #ifdef CONFIG_RETPOLINE
206 unsigned long loops
= RSB_CLEAR_LOOPS
/ 2;
208 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
209 ALTERNATIVE("jmp 910f",
210 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS
, %1)),
211 X86_FEATURE_RETPOLINE
)
213 : "=&r" (loops
), ASM_CALL_CONSTRAINT
214 : "r" (loops
) : "memory" );
217 #endif /* __ASSEMBLY__ */
218 #endif /* __NOSPEC_BRANCH_H__ */