]>
Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
c55d51ae BP |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ | |
76b04384 | 5 | |
6cd930f0 TG |
6 | #include <linux/static_key.h> |
7 | ||
76b04384 DW |
8 | #include <asm/alternative.h> |
9 | #include <asm/alternative-asm.h> | |
10 | #include <asm/cpufeatures.h> | |
8effdc23 | 11 | #include <asm/msr-index.h> |
76b04384 | 12 | |
10c16289 DW |
13 | /* |
14 | * Fill the CPU return stack buffer. | |
15 | * | |
16 | * Each entry in the RSB, if used for a speculative 'ret', contains an | |
17 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. | |
18 | * | |
19 | * This is required in various cases for retpoline and IBRS-based | |
20 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to | |
21 | * eliminate potentially bogus entries from the RSB, and sometimes | |
22 | * purely to ensure that it doesn't get empty, which on some CPUs would | |
23 | * allow predictions from other (unwanted!) sources to be used. | |
24 | * | |
25 | * We define a CPP macro such that it can be used from both .S files and | |
26 | * inline assembly. It's possible to do a .macro and then include that | |
27 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. | |
28 | */ | |
29 | ||
30 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ | |
31 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ | |
32 | ||
33 | /* | |
34 | * Google experimented with loop-unrolling and this turned out to be | |
35 | * the optimal version — two calls, each with their own speculation | |
36 | * trap should their return address end up getting used, in a loop. | |
37 | */ | |
38 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ | |
39 | mov $(nr/2), reg; \ | |
40 | 771: \ | |
41 | call 772f; \ | |
42 | 773: /* speculation trap */ \ | |
43 | pause; \ | |
44 | lfence; \ | |
45 | jmp 773b; \ | |
46 | 772: \ | |
47 | call 774f; \ | |
48 | 775: /* speculation trap */ \ | |
49 | pause; \ | |
50 | lfence; \ | |
51 | jmp 775b; \ | |
52 | 774: \ | |
53 | dec reg; \ | |
54 | jnz 771b; \ | |
55 | add $(BITS_PER_LONG/8) * nr, sp; | |
56 | ||
76b04384 DW |
57 | #ifdef __ASSEMBLY__ |
58 | ||
59 | /* | |
60 | * This should be used immediately before a retpoline alternative. It tells | |
61 | * objtool where the retpolines are so that it can make sense of the control | |
62 | * flow by just reading the original instruction(s) and ignoring the | |
63 | * alternatives. | |
64 | */ | |
65 | .macro ANNOTATE_NOSPEC_ALTERNATIVE | |
66 | .Lannotate_\@: | |
67 | .pushsection .discard.nospec | |
68 | .long .Lannotate_\@ - . | |
69 | .popsection | |
70 | .endm | |
71 | ||
b207cde6 PZ |
72 | /* |
73 | * This should be used immediately before an indirect jump/call. It tells | |
74 | * objtool the subsequent indirect jump/call is vouched safe for retpoline | |
75 | * builds. | |
76 | */ | |
77 | .macro ANNOTATE_RETPOLINE_SAFE | |
78 | .Lannotate_\@: | |
79 | .pushsection .discard.retpoline_safe | |
80 | _ASM_PTR .Lannotate_\@ | |
81 | .popsection | |
82 | .endm | |
83 | ||
76b04384 DW |
84 | /* |
85 | * These are the bare retpoline primitives for indirect jmp and call. | |
86 | * Do not use these directly; they only exist to make the ALTERNATIVE | |
87 | * invocation below less ugly. | |
88 | */ | |
89 | .macro RETPOLINE_JMP reg:req | |
90 | call .Ldo_rop_\@ | |
91 | .Lspec_trap_\@: | |
92 | pause | |
28d437d5 | 93 | lfence |
76b04384 DW |
94 | jmp .Lspec_trap_\@ |
95 | .Ldo_rop_\@: | |
96 | mov \reg, (%_ASM_SP) | |
97 | ret | |
98 | .endm | |
99 | ||
100 | /* | |
101 | * This is a wrapper around RETPOLINE_JMP so the called function in reg | |
102 | * returns to the instruction after the macro. | |
103 | */ | |
104 | .macro RETPOLINE_CALL reg:req | |
105 | jmp .Ldo_call_\@ | |
106 | .Ldo_retpoline_jmp_\@: | |
107 | RETPOLINE_JMP \reg | |
108 | .Ldo_call_\@: | |
109 | call .Ldo_retpoline_jmp_\@ | |
110 | .endm | |
111 | ||
112 | /* | |
113 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | |
114 | * indirect jmp/call which may be susceptible to the Spectre variant 2 | |
115 | * attack. | |
116 | */ | |
117 | .macro JMP_NOSPEC reg:req | |
118 | #ifdef CONFIG_RETPOLINE | |
119 | ANNOTATE_NOSPEC_ALTERNATIVE | |
b207cde6 | 120 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \ |
76b04384 | 121 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ |
b207cde6 | 122 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
123 | #else |
124 | jmp *\reg | |
125 | #endif | |
126 | .endm | |
127 | ||
128 | .macro CALL_NOSPEC reg:req | |
129 | #ifdef CONFIG_RETPOLINE | |
130 | ANNOTATE_NOSPEC_ALTERNATIVE | |
b207cde6 | 131 | ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \ |
76b04384 | 132 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ |
b207cde6 | 133 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD |
76b04384 DW |
134 | #else |
135 | call *\reg | |
136 | #endif | |
117cc7a9 DW |
137 | .endm |
138 | ||
10c16289 DW |
139 | /* |
140 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP | |
141 | * monstrosity above, manually. | |
142 | */ | |
143 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req | |
117cc7a9 | 144 | #ifdef CONFIG_RETPOLINE |
10c16289 DW |
145 | ANNOTATE_NOSPEC_ALTERNATIVE |
146 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ | |
147 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ | |
148 | \ftr | |
149 | .Lskip_rsb_\@: | |
117cc7a9 | 150 | #endif |
76b04384 DW |
151 | .endm |
152 | ||
153 | #else /* __ASSEMBLY__ */ | |
154 | ||
155 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ | |
156 | "999:\n\t" \ | |
157 | ".pushsection .discard.nospec\n\t" \ | |
158 | ".long 999b - .\n\t" \ | |
159 | ".popsection\n\t" | |
160 | ||
b207cde6 PZ |
161 | #define ANNOTATE_RETPOLINE_SAFE \ |
162 | "999:\n\t" \ | |
163 | ".pushsection .discard.retpoline_safe\n\t" \ | |
164 | _ASM_PTR " 999b\n\t" \ | |
165 | ".popsection\n\t" | |
166 | ||
db698bd0 ZD |
167 | #ifdef CONFIG_RETPOLINE |
168 | #ifdef CONFIG_X86_64 | |
76b04384 DW |
169 | |
170 | /* | |
db698bd0 ZD |
171 | * Inline asm uses the %V modifier which is only in newer GCC |
172 | * which is ensured when CONFIG_RETPOLINE is defined. | |
76b04384 DW |
173 | */ |
174 | # define CALL_NOSPEC \ | |
175 | ANNOTATE_NOSPEC_ALTERNATIVE \ | |
e1474558 | 176 | ALTERNATIVE_2( \ |
b207cde6 | 177 | ANNOTATE_RETPOLINE_SAFE \ |
76b04384 DW |
178 | "call *%[thunk_target]\n", \ |
179 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ | |
e1474558 ZD |
180 | X86_FEATURE_RETPOLINE, \ |
181 | "lfence;\n" \ | |
182 | ANNOTATE_RETPOLINE_SAFE \ | |
183 | "call *%[thunk_target]\n", \ | |
184 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
185 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) |
186 | ||
db698bd0 | 187 | #else /* CONFIG_X86_32 */ |
76b04384 DW |
188 | /* |
189 | * For i386 we use the original ret-equivalent retpoline, because | |
190 | * otherwise we'll run out of registers. We don't care about CET | |
191 | * here, anyway. | |
192 | */ | |
2748b99a | 193 | # define CALL_NOSPEC \ |
e1474558 ZD |
194 | ANNOTATE_NOSPEC_ALTERNATIVE \ |
195 | ALTERNATIVE_2( \ | |
2748b99a AW |
196 | ANNOTATE_RETPOLINE_SAFE \ |
197 | "call *%[thunk_target]\n", \ | |
76b04384 DW |
198 | " jmp 904f;\n" \ |
199 | " .align 16\n" \ | |
200 | "901: call 903f;\n" \ | |
201 | "902: pause;\n" \ | |
28d437d5 | 202 | " lfence;\n" \ |
76b04384 DW |
203 | " jmp 902b;\n" \ |
204 | " .align 16\n" \ | |
205 | "903: addl $4, %%esp;\n" \ | |
206 | " pushl %[thunk_target];\n" \ | |
207 | " ret;\n" \ | |
208 | " .align 16\n" \ | |
209 | "904: call 901b;\n", \ | |
e1474558 ZD |
210 | X86_FEATURE_RETPOLINE, \ |
211 | "lfence;\n" \ | |
212 | ANNOTATE_RETPOLINE_SAFE \ | |
213 | "call *%[thunk_target]\n", \ | |
214 | X86_FEATURE_RETPOLINE_AMD) | |
76b04384 DW |
215 | |
216 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
db698bd0 | 217 | #endif |
117cc7a9 | 218 | #else /* No retpoline for C / inline asm */ |
76b04384 DW |
219 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
220 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
221 | #endif | |
222 | ||
da285121 DW |
223 | /* The Spectre V2 mitigation variants */ |
224 | enum spectre_v2_mitigation { | |
225 | SPECTRE_V2_NONE, | |
da285121 DW |
226 | SPECTRE_V2_RETPOLINE_GENERIC, |
227 | SPECTRE_V2_RETPOLINE_AMD, | |
1044fde9 | 228 | SPECTRE_V2_IBRS_ENHANCED, |
da285121 DW |
229 | }; |
230 | ||
6cd930f0 TG |
231 | /* The indirect branch speculation control variants */ |
232 | enum spectre_v2_user_mitigation { | |
233 | SPECTRE_V2_USER_NONE, | |
234 | SPECTRE_V2_USER_STRICT, | |
fbed8eca | 235 | SPECTRE_V2_USER_STRICT_PREFERRED, |
ac40ad3b | 236 | SPECTRE_V2_USER_PRCTL, |
1c3cf627 | 237 | SPECTRE_V2_USER_SECCOMP, |
6cd930f0 TG |
238 | }; |
239 | ||
e63490c8 KRW |
240 | /* The Speculative Store Bypass disable variants */ |
241 | enum ssb_mitigation { | |
242 | SPEC_STORE_BYPASS_NONE, | |
243 | SPEC_STORE_BYPASS_DISABLE, | |
574dcf89 | 244 | SPEC_STORE_BYPASS_PRCTL, |
c7416003 | 245 | SPEC_STORE_BYPASS_SECCOMP, |
e63490c8 KRW |
246 | }; |
247 | ||
736e80a4 MH |
248 | extern char __indirect_thunk_start[]; |
249 | extern char __indirect_thunk_end[]; | |
250 | ||
117cc7a9 DW |
251 | /* |
252 | * On VMEXIT we must ensure that no RSB predictions learned in the guest | |
253 | * can be followed in the host, by overwriting the RSB completely. Both | |
254 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future | |
894655b1 | 255 | * CPUs with IBRS_ALL *might* it be avoided. |
117cc7a9 DW |
256 | */ |
257 | static inline void vmexit_fill_RSB(void) | |
258 | { | |
259 | #ifdef CONFIG_RETPOLINE | |
10c16289 DW |
260 | unsigned long loops; |
261 | ||
262 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE | |
263 | ALTERNATIVE("jmp 910f", | |
264 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), | |
265 | X86_FEATURE_RETPOLINE) | |
266 | "910:" | |
267 | : "=r" (loops), ASM_CALL_CONSTRAINT | |
268 | : : "memory" ); | |
117cc7a9 DW |
269 | #endif |
270 | } | |
3f7d8755 | 271 | |
73e523ad LT |
272 | static __always_inline |
273 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) | |
274 | { | |
275 | asm volatile(ALTERNATIVE("", "wrmsr", %c[feature]) | |
276 | : : "c" (msr), | |
277 | "a" ((u32)val), | |
278 | "d" ((u32)(val >> 32)), | |
279 | [feature] "i" (feature) | |
280 | : "memory"); | |
281 | } | |
390b99c3 | 282 | |
bd12e896 DW |
283 | static inline void indirect_branch_prediction_barrier(void) |
284 | { | |
296b454a KRW |
285 | u64 val = PRED_CMD_IBPB; |
286 | ||
287 | alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB); | |
390b99c3 DW |
288 | } |
289 | ||
4ac9b1f9 TG |
290 | /* The Intel SPEC CTRL MSR base value cache */ |
291 | extern u64 x86_spec_ctrl_base; | |
292 | ||
390b99c3 DW |
293 | /* |
294 | * With retpoline, we must use IBRS to restrict branch prediction | |
295 | * before calling into firmware. | |
02702fc9 IM |
296 | * |
297 | * (Implemented as CPP macros due to header hell.) | |
390b99c3 | 298 | */ |
02702fc9 IM |
299 | #define firmware_restrict_branch_speculation_start() \ |
300 | do { \ | |
4ac9b1f9 | 301 | u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \ |
296b454a | 302 | \ |
02702fc9 | 303 | preempt_disable(); \ |
296b454a | 304 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ |
02702fc9 IM |
305 | X86_FEATURE_USE_IBRS_FW); \ |
306 | } while (0) | |
390b99c3 | 307 | |
02702fc9 IM |
308 | #define firmware_restrict_branch_speculation_end() \ |
309 | do { \ | |
4ac9b1f9 | 310 | u64 val = x86_spec_ctrl_base; \ |
296b454a KRW |
311 | \ |
312 | alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \ | |
02702fc9 IM |
313 | X86_FEATURE_USE_IBRS_FW); \ |
314 | preempt_enable(); \ | |
315 | } while (0) | |
bd12e896 | 316 | |
6cd930f0 | 317 | DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); |
58b2d4c8 TG |
318 | DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); |
319 | DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); | |
6cd930f0 | 320 | |
4446d382 TG |
321 | #include <asm/segment.h> |
322 | ||
323 | /** | |
324 | * mds_clear_cpu_buffers - Mitigation for MDS vulnerability | |
325 | * | |
326 | * This uses the otherwise unused and obsolete VERW instruction in | |
327 | * combination with microcode which triggers a CPU buffer flush when the | |
328 | * instruction is executed. | |
329 | */ | |
330 | static inline void mds_clear_cpu_buffers(void) | |
331 | { | |
332 | static const u16 ds = __KERNEL_DS; | |
333 | ||
334 | /* | |
335 | * Has to be the memory-operand variant because only that | |
336 | * guarantees the CPU buffer flush functionality according to | |
337 | * documentation. The register-operand variant does not. | |
338 | * Works with any segment selector, but a valid writable | |
339 | * data segment is the fastest variant. | |
340 | * | |
341 | * "cc" clobber is required because VERW modifies ZF. | |
342 | */ | |
343 | asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); | |
344 | } | |
345 | ||
76b04384 | 346 | #endif /* __ASSEMBLY__ */ |
29fed5fa DB |
347 | |
348 | /* | |
349 | * Below is used in the eBPF JIT compiler and emits the byte sequence | |
350 | * for the following assembly: | |
351 | * | |
352 | * With retpolines configured: | |
353 | * | |
354 | * callq do_rop | |
355 | * spec_trap: | |
356 | * pause | |
357 | * lfence | |
358 | * jmp spec_trap | |
359 | * do_rop: | |
360 | * mov %rax,(%rsp) | |
361 | * retq | |
362 | * | |
363 | * Without retpolines configured: | |
364 | * | |
365 | * jmp *%rax | |
366 | */ | |
367 | #ifdef CONFIG_RETPOLINE | |
368 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 | |
369 | # define RETPOLINE_RAX_BPF_JIT() \ | |
370 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ | |
371 | /* spec_trap: */ \ | |
372 | EMIT2(0xF3, 0x90); /* pause */ \ | |
373 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
374 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
375 | /* do_rop: */ \ | |
376 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ | |
377 | EMIT1(0xC3); /* retq */ | |
378 | #else | |
379 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 | |
380 | # define RETPOLINE_RAX_BPF_JIT() \ | |
381 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ | |
382 | #endif | |
383 | ||
c55d51ae | 384 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |