]>
Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
7a32fc51 BP |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ | |
76b04384 | 5 | |
fa1202ef TG |
6 | #include <linux/static_key.h> |
7 | ||
76b04384 DW |
8 | #include <asm/alternative.h> |
9 | #include <asm/alternative-asm.h> | |
10 | #include <asm/cpufeatures.h> | |
ea00f301 | 11 | #include <asm/msr-index.h> |
76b04384 | 12 | |
ff05ab23 PZ |
13 | /* |
14 | * This should be used immediately before a retpoline alternative. It tells | |
15 | * objtool where the retpolines are so that it can make sense of the control | |
16 | * flow by just reading the original instruction(s) and ignoring the | |
17 | * alternatives. | |
18 | */ | |
19 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ | |
20 | ANNOTATE_IGNORE_ALTERNATIVE | |
21 | ||
d1c99108 DW |
22 | /* |
23 | * Fill the CPU return stack buffer. | |
24 | * | |
25 | * Each entry in the RSB, if used for a speculative 'ret', contains an | |
26 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. | |
27 | * | |
28 | * This is required in various cases for retpoline and IBRS-based | |
29 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to | |
30 | * eliminate potentially bogus entries from the RSB, and sometimes | |
31 | * purely to ensure that it doesn't get empty, which on some CPUs would | |
32 | * allow predictions from other (unwanted!) sources to be used. | |
33 | * | |
34 | * We define a CPP macro such that it can be used from both .S files and | |
35 | * inline assembly. It's possible to do a .macro and then include that | |
36 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. | |
37 | */ | |
38 | ||
39 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ | |
40 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ | |
41 | ||
42 | /* | |
43 | * Google experimented with loop-unrolling and this turned out to be | |
44 | * the optimal version — two calls, each with their own speculation | |
45 | * trap should their return address end up getting used, in a loop. | |
46 | */ | |
47 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ | |
48 | mov $(nr/2), reg; \ | |
49 | 771: \ | |
50 | call 772f; \ | |
51 | 773: /* speculation trap */ \ | |
52 | pause; \ | |
53 | lfence; \ | |
54 | jmp 773b; \ | |
55 | 772: \ | |
56 | call 774f; \ | |
57 | 775: /* speculation trap */ \ | |
58 | pause; \ | |
59 | lfence; \ | |
60 | jmp 775b; \ | |
61 | 774: \ | |
62 | dec reg; \ | |
63 | jnz 771b; \ | |
64 | add $(BITS_PER_LONG/8) * nr, sp; | |
65 | ||
76b04384 DW |
66 | #ifdef __ASSEMBLY__ |
67 | ||
9e0e3c51 PZ |
68 | /* |
69 | * This should be used immediately before an indirect jump/call. It tells | |
70 | * objtool the subsequent indirect jump/call is vouched safe for retpoline | |
71 | * builds. | |
72 | */ | |
73 | .macro ANNOTATE_RETPOLINE_SAFE | |
74 | .Lannotate_\@: | |
75 | .pushsection .discard.retpoline_safe | |
76 | _ASM_PTR .Lannotate_\@ | |
77 | .popsection | |
78 | .endm | |
79 | ||
76b04384 DW |
80 | /* |
81 | * These are the bare retpoline primitives for indirect jmp and call. | |
82 | * Do not use these directly; they only exist to make the ALTERNATIVE | |
83 | * invocation below less ugly. | |
84 | */ | |
85 | .macro RETPOLINE_JMP reg:req | |
86 | call .Ldo_rop_\@ | |
87 | .Lspec_trap_\@: | |
88 | pause | |
28d437d5 | 89 | lfence |
76b04384 DW |
90 | jmp .Lspec_trap_\@ |
91 | .Ldo_rop_\@: | |
92 | mov \reg, (%_ASM_SP) | |
93 | ret | |
94 | .endm | |
95 | ||
96 | /* | |
97 | * This is a wrapper around RETPOLINE_JMP so the called function in reg | |
98 | * returns to the instruction after the macro. | |
99 | */ | |
100 | .macro RETPOLINE_CALL reg:req | |
101 | jmp .Ldo_call_\@ | |
102 | .Ldo_retpoline_jmp_\@: | |
103 | RETPOLINE_JMP \reg | |
104 | .Ldo_call_\@: | |
105 | call .Ldo_retpoline_jmp_\@ | |
106 | .endm | |
107 | ||
108 | /* | |
109 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | |
110 | * indirect jmp/call which may be susceptible to the Spectre variant 2 | |
111 | * attack. | |
112 | */ | |
113 | .macro JMP_NOSPEC reg:req | |
114 | #ifdef CONFIG_RETPOLINE | |
115 | ANNOTATE_NOSPEC_ALTERNATIVE | |
9e0e3c51 | 116 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
76b04384 | 117 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
9e0e3c51 | 118 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
119 | #else |
120 | jmp *\reg | |
121 | #endif | |
122 | .endm | |
123 | ||
124 | .macro CALL_NOSPEC reg:req | |
125 | #ifdef CONFIG_RETPOLINE | |
126 | ANNOTATE_NOSPEC_ALTERNATIVE | |
9e0e3c51 | 127 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
76b04384 | 128 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
9e0e3c51 | 129 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
130 | #else |
131 | call *\reg | |
132 | #endif | |
117cc7a9 DW |
133 | .endm |
134 | ||
d1c99108 DW |
135 | /* |
136 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP | |
137 | * monstrosity above, manually. | |
138 | */ | |
139 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req | |
117cc7a9 | 140 | #ifdef CONFIG_RETPOLINE |
d1c99108 DW |
141 | ANNOTATE_NOSPEC_ALTERNATIVE |
142 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ | |
143 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ | |
144 | \ftr | |
145 | .Lskip_rsb_\@: | |
117cc7a9 | 146 | #endif |
76b04384 DW |
147 | .endm |
148 | ||
149 | #else /* __ASSEMBLY__ */ | |
150 | ||
9e0e3c51 PZ |
151 | #define ANNOTATE_RETPOLINE_SAFE \ |
152 | "999:\n\t" \ | |
153 | ".pushsection .discard.retpoline_safe\n\t" \ | |
154 | _ASM_PTR " 999b\n\t" \ | |
155 | ".popsection\n\t" | |
156 | ||
4cd24de3 ZD |
157 | #ifdef CONFIG_RETPOLINE |
158 | #ifdef CONFIG_X86_64 | |
76b04384 DW |
159 | |
160 | /* | |
4cd24de3 ZD |
161 | * Inline asm uses the %V modifier which is only in newer GCC |
162 | * which is ensured when CONFIG_RETPOLINE is defined. | |
76b04384 DW |
163 | */ |
164 | # define CALL_NOSPEC \ | |
165 | ANNOTATE_NOSPEC_ALTERNATIVE \ | |
0cbb76d6 | 166 | ALTERNATIVE_2( \ |
9e0e3c51 | 167 | ANNOTATE_RETPOLINE_SAFE \ |
76b04384 DW |
168 | "call *%[thunk_target]\n", \ |
169 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ | |
0cbb76d6 ZD |
170 | X86_FEATURE_RETPOLINE, \ |
171 | "lfence;\n" \ | |
172 | ANNOTATE_RETPOLINE_SAFE \ | |
173 | "call *%[thunk_target]\n", \ | |
174 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
175 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
176 | ||
4cd24de3 | 177 | #else /* CONFIG_X86_32 */ |
76b04384 DW |
178 | /* |
179 | * For i386 we use the original ret-equivalent retpoline, because | |
180 | * otherwise we'll run out of registers. We don't care about CET | |
181 | * here, anyway. | |
182 | */ | |
a14bff13 | 183 | # define CALL_NOSPEC \ |
0cbb76d6 ZD |
184 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
185 | ALTERNATIVE_2( \ | |
a14bff13 AW |
186 | ANNOTATE_RETPOLINE_SAFE \ |
187 | "call *%[thunk_target]\n", \ | |
76b04384 DW |
188 | " jmp 904f;\n" \ |
189 | " .align 16\n" \ | |
190 | "901: call 903f;\n" \ | |
191 | "902: pause;\n" \ | |
28d437d5 | 192 | " lfence;\n" \ |
76b04384 DW |
193 | " jmp 902b;\n" \ |
194 | " .align 16\n" \ | |
b63f20a7 | 195 | "903: lea 4(%%esp), %%esp;\n" \ |
76b04384 DW |
196 | " pushl %[thunk_target];\n" \ |
197 | " ret;\n" \ | |
198 | " .align 16\n" \ | |
199 | "904: call 901b;\n", \ | |
0cbb76d6 ZD |
200 | X86_FEATURE_RETPOLINE, \ |
201 | "lfence;\n" \ | |
202 | ANNOTATE_RETPOLINE_SAFE \ | |
203 | "call *%[thunk_target]\n", \ | |
204 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
205 | |
206 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
4cd24de3 | 207 | #endif |
117cc7a9 | 208 | #else /* No retpoline for C / inline asm */ |
76b04384 DW |
209 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
210 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
211 | #endif | |
212 | ||
da285121 DW |
213 | /* The Spectre V2 mitigation variants */ |
214 | enum spectre_v2_mitigation { | |
215 | SPECTRE_V2_NONE, | |
da285121 DW |
216 | SPECTRE_V2_RETPOLINE_GENERIC, |
217 | SPECTRE_V2_RETPOLINE_AMD, | |
706d5168 | 218 | SPECTRE_V2_IBRS_ENHANCED, |
da285121 DW |
219 | }; |
220 | ||
fa1202ef TG |
221 | /* The indirect branch speculation control variants */ |
222 | enum spectre_v2_user_mitigation { | |
223 | SPECTRE_V2_USER_NONE, | |
224 | SPECTRE_V2_USER_STRICT, | |
20c3a2c3 | 225 | SPECTRE_V2_USER_STRICT_PREFERRED, |
9137bb27 | 226 | SPECTRE_V2_USER_PRCTL, |
6b3e64c2 | 227 | SPECTRE_V2_USER_SECCOMP, |
fa1202ef TG |
228 | }; |
229 | ||
24f7fc83 KRW |
230 | /* The Speculative Store Bypass disable variants */ |
231 | enum ssb_mitigation { | |
232 | SPEC_STORE_BYPASS_NONE, | |
233 | SPEC_STORE_BYPASS_DISABLE, | |
a73ec77e | 234 | SPEC_STORE_BYPASS_PRCTL, |
f21b53b2 | 235 | SPEC_STORE_BYPASS_SECCOMP, |
24f7fc83 KRW |
236 | }; |
237 | ||
736e80a4 MH |
238 | extern char __indirect_thunk_start[]; |
239 | extern char __indirect_thunk_end[]; | |
240 | ||
117cc7a9 DW |
241 | /* |
242 | * On VMEXIT we must ensure that no RSB predictions learned in the guest | |
243 | * can be followed in the host, by overwriting the RSB completely. Both | |
244 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future | |
af189c95 | 245 | * CPUs with IBRS_ALL *might* it be avoided. |
117cc7a9 DW |
246 | */ |
247 | static inline void vmexit_fill_RSB(void) | |
248 | { | |
249 | #ifdef CONFIG_RETPOLINE | |
d1c99108 DW |
250 | unsigned long loops; |
251 | ||
252 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE | |
253 | ALTERNATIVE("jmp 910f", | |
254 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), | |
255 | X86_FEATURE_RETPOLINE) | |
256 | "910:" | |
257 | : "=r" (loops), ASM_CALL_CONSTRAINT | |
258 | : : "memory" ); | |
117cc7a9 DW |
259 | #endif |
260 | } | |
3f7d8755 | 261 | |
1aa7a573 LT |
262 | static __always_inline |
263 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) | |
264 | { | |
265 | asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) | |
266 | : : "c" (msr), | |
5f2b745f JM |
267 | "a" ((u32)val), |
268 | "d" ((u32)(val >> 32)), | |
1aa7a573 LT |
269 | [feature] "i" (feature) |
270 | : "memory"); | |
271 | } | |
dd84441a | 272 | |
20ffa1ca DW |
273 | static inline void indirect_branch_prediction_barrier(void) |
274 | { | |
1b86883c KRW |
275 | u64 val = PRED_CMD_IBPB; |
276 | ||
277 | alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); | |
20ffa1ca DW |
278 | } |
279 | ||
fa8ac498 TG |
280 | /* The Intel SPEC CTRL MSR base value cache */ |
281 | extern u64 x86_spec_ctrl_base; | |
282 | ||
dd84441a DW |
283 | /* |
284 | * With retpoline, we must use IBRS to restrict branch prediction | |
285 | * before calling into firmware. | |
d72f4e29 IM |
286 | * |
287 | * (Implemented as CPP macros due to header hell.) | |
dd84441a | 288 | */ |
d72f4e29 IM |
289 | #define firmware_restrict_branch_speculation_start() \ |
290 | do { \ | |
fa8ac498 | 291 | u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ |
1b86883c | 292 | \ |
d72f4e29 | 293 | preempt_disable(); \ |
1b86883c | 294 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
d72f4e29 IM |
295 | X86_FEATURE_USE_IBRS_FW); \ |
296 | } while (0) | |
dd84441a | 297 | |
d72f4e29 IM |
298 | #define firmware_restrict_branch_speculation_end() \ |
299 | do { \ | |
fa8ac498 | 300 | u64 val = x86_spec_ctrl_base; \ |
1b86883c KRW |
301 | \ |
302 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ | |
d72f4e29 IM |
303 | X86_FEATURE_USE_IBRS_FW); \ |
304 | preempt_enable(); \ | |
305 | } while (0) | |
20ffa1ca | 306 | |
fa1202ef | 307 | DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); |
4c71a2b6 TG |
308 | DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); |
309 | DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); | |
fa1202ef | 310 | |
04dcbdb8 | 311 | DECLARE_STATIC_KEY_FALSE(mds_user_clear); |
07f07f55 | 312 | DECLARE_STATIC_KEY_FALSE(mds_idle_clear); |
04dcbdb8 | 313 | |
6a9e5292 TG |
314 | #include <asm/segment.h> |
315 | ||
316 | /** | |
317 | * mds_clear_cpu_buffers - Mitigation for MDS vulnerability | |
318 | * | |
319 | * This uses the otherwise unused and obsolete VERW instruction in | |
320 | * combination with microcode which triggers a CPU buffer flush when the | |
321 | * instruction is executed. | |
322 | */ | |
323 | static inline void mds_clear_cpu_buffers(void) | |
324 | { | |
325 | static const u16 ds = __KERNEL_DS; | |
326 | ||
327 | /* | |
328 | * Has to be the memory-operand variant because only that | |
329 | * guarantees the CPU buffer flush functionality according to | |
330 | * documentation. The register-operand variant does not. | |
331 | * Works with any segment selector, but a valid writable | |
332 | * data segment is the fastest variant. | |
333 | * | |
334 | * "cc" clobber is required because VERW modifies ZF. | |
335 | */ | |
336 | asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); | |
337 | } | |
338 | ||
04dcbdb8 TG |
339 | /** |
340 | * mds_user_clear_cpu_buffers - Mitigation for MDS vulnerability | |
341 | * | |
342 | * Clear CPU buffers if the corresponding static key is enabled | |
343 | */ | |
344 | static inline void mds_user_clear_cpu_buffers(void) | |
345 | { | |
346 | if (static_branch_likely(&mds_user_clear)) | |
347 | mds_clear_cpu_buffers(); | |
348 | } | |
349 | ||
07f07f55 TG |
350 | /** |
351 | * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability | |
352 | * | |
353 | * Clear CPU buffers if the corresponding static key is enabled | |
354 | */ | |
355 | static inline void mds_idle_clear_cpu_buffers(void) | |
356 | { | |
357 | if (static_branch_likely(&mds_idle_clear)) | |
358 | mds_clear_cpu_buffers(); | |
359 | } | |
360 | ||
76b04384 | 361 | #endif /* __ASSEMBLY__ */ |
a493a87f DB |
362 | |
363 | /* | |
364 | * Below is used in the eBPF JIT compiler and emits the byte sequence | |
365 | * for the following assembly: | |
366 | * | |
367 | * With retpolines configured: | |
368 | * | |
369 | * callq do_rop | |
370 | * spec_trap: | |
371 | * pause | |
372 | * lfence | |
373 | * jmp spec_trap | |
374 | * do_rop: | |
03f5781b WY |
375 | * mov %rax,(%rsp) for x86_64 |
376 | * mov %edx,(%esp) for x86_32 | |
a493a87f DB |
377 | * retq |
378 | * | |
379 | * Without retpolines configured: | |
380 | * | |
03f5781b WY |
381 | * jmp *%rax for x86_64 |
382 | * jmp *%edx for x86_32 | |
a493a87f DB |
383 | */ |
384 | #ifdef CONFIG_RETPOLINE | |
36256009 DB |
385 | # ifdef CONFIG_X86_64 |
386 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 | |
387 | # define RETPOLINE_RAX_BPF_JIT() \ | |
03f5781b | 388 | do { \ |
a493a87f DB |
389 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ |
390 | /* spec_trap: */ \ | |
391 | EMIT2(0xF3, 0x90); /* pause */ \ | |
392 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
393 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
394 | /* do_rop: */ \ | |
395 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ | |
03f5781b WY |
396 | EMIT1(0xC3); /* retq */ \ |
397 | } while (0) | |
36256009 DB |
398 | # else /* !CONFIG_X86_64 */ |
399 | # define RETPOLINE_EDX_BPF_JIT() \ | |
03f5781b WY |
400 | do { \ |
401 | EMIT1_off32(0xE8, 7); /* call do_rop */ \ | |
402 | /* spec_trap: */ \ | |
403 | EMIT2(0xF3, 0x90); /* pause */ \ | |
404 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
405 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
406 | /* do_rop: */ \ | |
407 | EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \ | |
408 | EMIT1(0xC3); /* ret */ \ | |
409 | } while (0) | |
36256009 | 410 | # endif |
03f5781b | 411 | #else /* !CONFIG_RETPOLINE */ |
36256009 DB |
412 | # ifdef CONFIG_X86_64 |
413 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 | |
414 | # define RETPOLINE_RAX_BPF_JIT() \ | |
415 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ | |
416 | # else /* !CONFIG_X86_64 */ | |
417 | # define RETPOLINE_EDX_BPF_JIT() \ | |
418 | EMIT2(0xFF, 0xE2) /* jmp *%edx */ | |
419 | # endif | |
a493a87f DB |
420 | #endif |
421 | ||
7a32fc51 | 422 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |