]>
Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
7a32fc51 BP |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ | |
76b04384 DW |
5 | |
6 | #include <asm/alternative.h> | |
7 | #include <asm/alternative-asm.h> | |
8 | #include <asm/cpufeatures.h> | |
ea00f301 | 9 | #include <asm/msr-index.h> |
76b04384 | 10 | |
d1c99108 DW |
11 | /* |
12 | * Fill the CPU return stack buffer. | |
13 | * | |
14 | * Each entry in the RSB, if used for a speculative 'ret', contains an | |
15 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. | |
16 | * | |
17 | * This is required in various cases for retpoline and IBRS-based | |
18 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to | |
19 | * eliminate potentially bogus entries from the RSB, and sometimes | |
20 | * purely to ensure that it doesn't get empty, which on some CPUs would | |
21 | * allow predictions from other (unwanted!) sources to be used. | |
22 | * | |
23 | * We define a CPP macro such that it can be used from both .S files and | |
24 | * inline assembly. It's possible to do a .macro and then include that | |
25 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. | |
26 | */ | |
27 | ||
28 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ | |
29 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ | |
30 | ||
31 | /* | |
32 | * Google experimented with loop-unrolling and this turned out to be | |
33 | * the optimal version — two calls, each with their own speculation | |
34 | * trap should their return address end up getting used, in a loop. | |
35 | */ | |
36 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ | |
37 | mov $(nr/2), reg; \ | |
38 | 771: \ | |
39 | call 772f; \ | |
40 | 773: /* speculation trap */ \ | |
41 | pause; \ | |
42 | lfence; \ | |
43 | jmp 773b; \ | |
44 | 772: \ | |
45 | call 774f; \ | |
46 | 775: /* speculation trap */ \ | |
47 | pause; \ | |
48 | lfence; \ | |
49 | jmp 775b; \ | |
50 | 774: \ | |
51 | dec reg; \ | |
52 | jnz 771b; \ | |
53 | add $(BITS_PER_LONG/8) * nr, sp; | |
54 | ||
76b04384 DW |
55 | #ifdef __ASSEMBLY__ |
56 | ||
57 | /* | |
58 | * This should be used immediately before a retpoline alternative. It tells | |
59 | * objtool where the retpolines are so that it can make sense of the control | |
60 | * flow by just reading the original instruction(s) and ignoring the | |
61 | * alternatives. | |
62 | */ | |
63 | .macro ANNOTATE_NOSPEC_ALTERNATIVE | |
64 | .Lannotate_\@: | |
65 | .pushsection .discard.nospec | |
66 | .long .Lannotate_\@ - . | |
67 | .popsection | |
68 | .endm | |
69 | ||
9e0e3c51 PZ |
70 | /* |
71 | * This should be used immediately before an indirect jump/call. It tells | |
72 | * objtool the subsequent indirect jump/call is vouched safe for retpoline | |
73 | * builds. | |
74 | */ | |
75 | .macro ANNOTATE_RETPOLINE_SAFE | |
76 | .Lannotate_\@: | |
77 | .pushsection .discard.retpoline_safe | |
78 | _ASM_PTR .Lannotate_\@ | |
79 | .popsection | |
80 | .endm | |
81 | ||
76b04384 DW |
82 | /* |
83 | * These are the bare retpoline primitives for indirect jmp and call. | |
84 | * Do not use these directly; they only exist to make the ALTERNATIVE | |
85 | * invocation below less ugly. | |
86 | */ | |
87 | .macro RETPOLINE_JMP reg:req | |
88 | call .Ldo_rop_\@ | |
89 | .Lspec_trap_\@: | |
90 | pause | |
28d437d5 | 91 | lfence |
76b04384 DW |
92 | jmp .Lspec_trap_\@ |
93 | .Ldo_rop_\@: | |
94 | mov \reg, (%_ASM_SP) | |
95 | ret | |
96 | .endm | |
97 | ||
98 | /* | |
99 | * This is a wrapper around RETPOLINE_JMP so the called function in reg | |
100 | * returns to the instruction after the macro. | |
101 | */ | |
102 | .macro RETPOLINE_CALL reg:req | |
103 | jmp .Ldo_call_\@ | |
104 | .Ldo_retpoline_jmp_\@: | |
105 | RETPOLINE_JMP \reg | |
106 | .Ldo_call_\@: | |
107 | call .Ldo_retpoline_jmp_\@ | |
108 | .endm | |
109 | ||
110 | /* | |
111 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | |
112 | * indirect jmp/call which may be susceptible to the Spectre variant 2 | |
113 | * attack. | |
114 | */ | |
115 | .macro JMP_NOSPEC reg:req | |
116 | #ifdef CONFIG_RETPOLINE | |
117 | ANNOTATE_NOSPEC_ALTERNATIVE | |
9e0e3c51 | 118 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
76b04384 | 119 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
9e0e3c51 | 120 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
121 | #else |
122 | jmp *\reg | |
123 | #endif | |
124 | .endm | |
125 | ||
126 | .macro CALL_NOSPEC reg:req | |
127 | #ifdef CONFIG_RETPOLINE | |
128 | ANNOTATE_NOSPEC_ALTERNATIVE | |
9e0e3c51 | 129 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
76b04384 | 130 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
9e0e3c51 | 131 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
132 | #else |
133 | call *\reg | |
134 | #endif | |
117cc7a9 DW |
135 | .endm |
136 | ||
d1c99108 DW |
137 | /* |
138 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP | |
139 | * monstrosity above, manually. | |
140 | */ | |
141 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req | |
117cc7a9 | 142 | #ifdef CONFIG_RETPOLINE |
d1c99108 DW |
143 | ANNOTATE_NOSPEC_ALTERNATIVE |
144 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ | |
145 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ | |
146 | \ftr | |
147 | .Lskip_rsb_\@: | |
117cc7a9 | 148 | #endif |
76b04384 DW |
149 | .endm |
150 | ||
151 | #else /* __ASSEMBLY__ */ | |
152 | ||
153 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ | |
154 | "999:\n\t" \ | |
155 | ".pushsection .discard.nospec\n\t" \ | |
156 | ".long 999b - .\n\t" \ | |
157 | ".popsection\n\t" | |
158 | ||
9e0e3c51 PZ |
159 | #define ANNOTATE_RETPOLINE_SAFE \ |
160 | "999:\n\t" \ | |
161 | ".pushsection .discard.retpoline_safe\n\t" \ | |
162 | _ASM_PTR " 999b\n\t" \ | |
163 | ".popsection\n\t" | |
164 | ||
76b04384 DW |
165 | #if defined(CONFIG_X86_64) && defined(RETPOLINE) |
166 | ||
167 | /* | |
168 | * Since the inline asm uses the %V modifier which is only in newer GCC, | |
169 | * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. | |
170 | */ | |
171 | # define CALL_NOSPEC \ | |
172 | ANNOTATE_NOSPEC_ALTERNATIVE \ | |
0cbb76d6 | 173 | ALTERNATIVE_2( \ |
9e0e3c51 | 174 | ANNOTATE_RETPOLINE_SAFE \ |
76b04384 DW |
175 | "call *%[thunk_target]\n", \ |
176 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ | |
0cbb76d6 ZD |
177 | X86_FEATURE_RETPOLINE, \ |
178 | "lfence;\n" \ | |
179 | ANNOTATE_RETPOLINE_SAFE \ | |
180 | "call *%[thunk_target]\n", \ | |
181 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
182 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
183 | ||
184 | #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) | |
185 | /* | |
186 | * For i386 we use the original ret-equivalent retpoline, because | |
187 | * otherwise we'll run out of registers. We don't care about CET | |
188 | * here, anyway. | |
189 | */ | |
a14bff13 | 190 | # define CALL_NOSPEC \ |
0cbb76d6 ZD |
191 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
192 | ALTERNATIVE_2( \ | |
a14bff13 AW |
193 | ANNOTATE_RETPOLINE_SAFE \ |
194 | "call *%[thunk_target]\n", \ | |
76b04384 DW |
195 | " jmp 904f;\n" \ |
196 | " .align 16\n" \ | |
197 | "901: call 903f;\n" \ | |
198 | "902: pause;\n" \ | |
28d437d5 | 199 | " lfence;\n" \ |
76b04384 DW |
200 | " jmp 902b;\n" \ |
201 | " .align 16\n" \ | |
202 | "903: addl $4, %%esp;\n" \ | |
203 | " pushl %[thunk_target];\n" \ | |
204 | " ret;\n" \ | |
205 | " .align 16\n" \ | |
206 | "904: call 901b;\n", \ | |
0cbb76d6 ZD |
207 | X86_FEATURE_RETPOLINE, \ |
208 | "lfence;\n" \ | |
209 | ANNOTATE_RETPOLINE_SAFE \ | |
210 | "call *%[thunk_target]\n", \ | |
211 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
212 | |
213 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
117cc7a9 | 214 | #else /* No retpoline for C / inline asm */ |
76b04384 DW |
215 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
216 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
217 | #endif | |
218 | ||
da285121 DW |
219 | /* The Spectre V2 mitigation variants */ |
220 | enum spectre_v2_mitigation { | |
221 | SPECTRE_V2_NONE, | |
222 | SPECTRE_V2_RETPOLINE_MINIMAL, | |
223 | SPECTRE_V2_RETPOLINE_MINIMAL_AMD, | |
224 | SPECTRE_V2_RETPOLINE_GENERIC, | |
225 | SPECTRE_V2_RETPOLINE_AMD, | |
706d5168 | 226 | SPECTRE_V2_IBRS_ENHANCED, |
da285121 DW |
227 | }; |
228 | ||
24f7fc83 KRW |
229 | /* The Speculative Store Bypass disable variants */ |
230 | enum ssb_mitigation { | |
231 | SPEC_STORE_BYPASS_NONE, | |
232 | SPEC_STORE_BYPASS_DISABLE, | |
a73ec77e | 233 | SPEC_STORE_BYPASS_PRCTL, |
f21b53b2 | 234 | SPEC_STORE_BYPASS_SECCOMP, |
24f7fc83 KRW |
235 | }; |
236 | ||
736e80a4 MH |
237 | extern char __indirect_thunk_start[]; |
238 | extern char __indirect_thunk_end[]; | |
239 | ||
117cc7a9 DW |
240 | /* |
241 | * On VMEXIT we must ensure that no RSB predictions learned in the guest | |
242 | * can be followed in the host, by overwriting the RSB completely. Both | |
243 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future | |
af189c95 | 244 | * CPUs with IBRS_ALL *might* it be avoided. |
117cc7a9 DW |
245 | */ |
246 | static inline void vmexit_fill_RSB(void) | |
247 | { | |
248 | #ifdef CONFIG_RETPOLINE | |
d1c99108 DW |
249 | unsigned long loops; |
250 | ||
251 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE | |
252 | ALTERNATIVE("jmp 910f", | |
253 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), | |
254 | X86_FEATURE_RETPOLINE) | |
255 | "910:" | |
256 | : "=r" (loops), ASM_CALL_CONSTRAINT | |
257 | : : "memory" ); | |
117cc7a9 DW |
258 | #endif |
259 | } | |
3f7d8755 | 260 | |
1aa7a573 LT |
261 | static __always_inline |
262 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) | |
263 | { | |
264 | asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) | |
265 | : : "c" (msr), | |
5f2b745f JM |
266 | "a" ((u32)val), |
267 | "d" ((u32)(val >> 32)), | |
1aa7a573 LT |
268 | [feature] "i" (feature) |
269 | : "memory"); | |
270 | } | |
dd84441a | 271 | |
20ffa1ca DW |
272 | static inline void indirect_branch_prediction_barrier(void) |
273 | { | |
1b86883c KRW |
274 | u64 val = PRED_CMD_IBPB; |
275 | ||
276 | alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); | |
20ffa1ca DW |
277 | } |
278 | ||
fa8ac498 TG |
279 | /* The Intel SPEC CTRL MSR base value cache */ |
280 | extern u64 x86_spec_ctrl_base; | |
281 | ||
dd84441a DW |
282 | /* |
283 | * With retpoline, we must use IBRS to restrict branch prediction | |
284 | * before calling into firmware. | |
d72f4e29 IM |
285 | * |
286 | * (Implemented as CPP macros due to header hell.) | |
dd84441a | 287 | */ |
d72f4e29 IM |
288 | #define firmware_restrict_branch_speculation_start() \ |
289 | do { \ | |
fa8ac498 | 290 | u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ |
1b86883c | 291 | \ |
d72f4e29 | 292 | preempt_disable(); \ |
1b86883c | 293 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
d72f4e29 IM |
294 | X86_FEATURE_USE_IBRS_FW); \ |
295 | } while (0) | |
dd84441a | 296 | |
d72f4e29 IM |
297 | #define firmware_restrict_branch_speculation_end() \ |
298 | do { \ | |
fa8ac498 | 299 | u64 val = x86_spec_ctrl_base; \ |
1b86883c KRW |
300 | \ |
301 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ | |
d72f4e29 IM |
302 | X86_FEATURE_USE_IBRS_FW); \ |
303 | preempt_enable(); \ | |
304 | } while (0) | |
20ffa1ca | 305 | |
76b04384 | 306 | #endif /* __ASSEMBLY__ */ |
a493a87f DB |
307 | |
308 | /* | |
309 | * Below is used in the eBPF JIT compiler and emits the byte sequence | |
310 | * for the following assembly: | |
311 | * | |
312 | * With retpolines configured: | |
313 | * | |
314 | * callq do_rop | |
315 | * spec_trap: | |
316 | * pause | |
317 | * lfence | |
318 | * jmp spec_trap | |
319 | * do_rop: | |
03f5781b WY |
320 | * mov %rax,(%rsp) for x86_64 |
321 | * mov %edx,(%esp) for x86_32 | |
a493a87f DB |
322 | * retq |
323 | * | |
324 | * Without retpolines configured: | |
325 | * | |
03f5781b WY |
326 | * jmp *%rax for x86_64 |
327 | * jmp *%edx for x86_32 | |
a493a87f DB |
328 | */ |
329 | #ifdef CONFIG_RETPOLINE | |
36256009 DB |
330 | # ifdef CONFIG_X86_64 |
331 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 | |
332 | # define RETPOLINE_RAX_BPF_JIT() \ | |
03f5781b | 333 | do { \ |
a493a87f DB |
334 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ |
335 | /* spec_trap: */ \ | |
336 | EMIT2(0xF3, 0x90); /* pause */ \ | |
337 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
338 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
339 | /* do_rop: */ \ | |
340 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ | |
03f5781b WY |
341 | EMIT1(0xC3); /* retq */ \ |
342 | } while (0) | |
36256009 DB |
343 | # else /* !CONFIG_X86_64 */ |
344 | # define RETPOLINE_EDX_BPF_JIT() \ | |
03f5781b WY |
345 | do { \ |
346 | EMIT1_off32(0xE8, 7); /* call do_rop */ \ | |
347 | /* spec_trap: */ \ | |
348 | EMIT2(0xF3, 0x90); /* pause */ \ | |
349 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
350 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
351 | /* do_rop: */ \ | |
352 | EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ | |
353 | EMIT1(0xC3); /* ret */ \ | |
354 | } while (0) | |
36256009 | 355 | # endif |
03f5781b | 356 | #else /* !CONFIG_RETPOLINE */ |
36256009 DB |
357 | # ifdef CONFIG_X86_64 |
358 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 | |
359 | # define RETPOLINE_RAX_BPF_JIT() \ | |
360 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ | |
361 | # else /* !CONFIG_X86_64 */ | |
362 | # define RETPOLINE_EDX_BPF_JIT() \ | |
363 | EMIT2(0xFF, 0xE2) /* jmp *%edx */ | |
364 | # endif | |
a493a87f DB |
365 | #endif |
366 | ||
7a32fc51 | 367 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |