]>
Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
c55d51ae BP |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ | |
76b04384 DW |
5 | |
6 | #include <asm/alternative.h> | |
7 | #include <asm/alternative-asm.h> | |
8 | #include <asm/cpufeatures.h> | |
8effdc23 | 9 | #include <asm/msr-index.h> |
76b04384 | 10 | |
10c16289 DW |
11 | /* |
12 | * Fill the CPU return stack buffer. | |
13 | * | |
14 | * Each entry in the RSB, if used for a speculative 'ret', contains an | |
15 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. | |
16 | * | |
17 | * This is required in various cases for retpoline and IBRS-based | |
18 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to | |
19 | * eliminate potentially bogus entries from the RSB, and sometimes | |
20 | * purely to ensure that it doesn't get empty, which on some CPUs would | |
21 | * allow predictions from other (unwanted!) sources to be used. | |
22 | * | |
23 | * We define a CPP macro such that it can be used from both .S files and | |
24 | * inline assembly. It's possible to do a .macro and then include that | |
25 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. | |
26 | */ | |
27 | ||
28 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ | |
29 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ | |
30 | ||
31 | /* | |
32 | * Google experimented with loop-unrolling and this turned out to be | |
33 | * the optimal version — two calls, each with their own speculation | |
34 | * trap should their return address end up getting used, in a loop. | |
35 | */ | |
36 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ | |
37 | mov $(nr/2), reg; \ | |
38 | 771: \ | |
39 | call 772f; \ | |
40 | 773: /* speculation trap */ \ | |
41 | pause; \ | |
42 | lfence; \ | |
43 | jmp 773b; \ | |
44 | 772: \ | |
45 | call 774f; \ | |
46 | 775: /* speculation trap */ \ | |
47 | pause; \ | |
48 | lfence; \ | |
49 | jmp 775b; \ | |
50 | 774: \ | |
51 | dec reg; \ | |
52 | jnz 771b; \ | |
53 | add $(BITS_PER_LONG/8) * nr, sp; | |
54 | ||
76b04384 DW |
55 | #ifdef __ASSEMBLY__ |
56 | ||
57 | /* | |
58 | * This should be used immediately before a retpoline alternative. It tells | |
59 | * objtool where the retpolines are so that it can make sense of the control | |
60 | * flow by just reading the original instruction(s) and ignoring the | |
61 | * alternatives. | |
62 | */ | |
63 | .macro ANNOTATE_NOSPEC_ALTERNATIVE | |
64 | .Lannotate_\@: | |
65 | .pushsection .discard.nospec | |
66 | .long .Lannotate_\@ - . | |
67 | .popsection | |
68 | .endm | |
69 | ||
70 | /* | |
71 | * These are the bare retpoline primitives for indirect jmp and call. | |
72 | * Do not use these directly; they only exist to make the ALTERNATIVE | |
73 | * invocation below less ugly. | |
74 | */ | |
75 | .macro RETPOLINE_JMP reg:req | |
76 | call .Ldo_rop_\@ | |
77 | .Lspec_trap_\@: | |
78 | pause | |
28d437d5 | 79 | lfence |
76b04384 DW |
80 | jmp .Lspec_trap_\@ |
81 | .Ldo_rop_\@: | |
82 | mov \reg, (%_ASM_SP) | |
83 | ret | |
84 | .endm | |
85 | ||
86 | /* | |
87 | * This is a wrapper around RETPOLINE_JMP so the called function in reg | |
88 | * returns to the instruction after the macro. | |
89 | */ | |
90 | .macro RETPOLINE_CALL reg:req | |
91 | jmp .Ldo_call_\@ | |
92 | .Ldo_retpoline_jmp_\@: | |
93 | RETPOLINE_JMP \reg | |
94 | .Ldo_call_\@: | |
95 | call .Ldo_retpoline_jmp_\@ | |
96 | .endm | |
97 | ||
98 | /* | |
99 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | |
100 | * indirect jmp/call which may be susceptible to the Spectre variant 2 | |
101 | * attack. | |
102 | */ | |
103 | .macro JMP_NOSPEC reg:req | |
104 | #ifdef CONFIG_RETPOLINE | |
105 | ANNOTATE_NOSPEC_ALTERNATIVE | |
106 | ALTERNATIVE_2 __stringify(jmp *\reg), \ | |
107 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ | |
108 | __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD | |
109 | #else | |
110 | jmp *\reg | |
111 | #endif | |
112 | .endm | |
113 | ||
114 | .macro CALL_NOSPEC reg:req | |
115 | #ifdef CONFIG_RETPOLINE | |
116 | ANNOTATE_NOSPEC_ALTERNATIVE | |
117 | ALTERNATIVE_2 __stringify(call *\reg), \ | |
118 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ | |
119 | __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD | |
120 | #else | |
121 | call *\reg | |
122 | #endif | |
117cc7a9 DW |
123 | .endm |
124 | ||
10c16289 DW |
125 | /* |
126 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP | |
127 | * monstrosity above, manually. | |
128 | */ | |
129 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req | |
117cc7a9 | 130 | #ifdef CONFIG_RETPOLINE |
10c16289 DW |
131 | ANNOTATE_NOSPEC_ALTERNATIVE |
132 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ | |
133 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ | |
134 | \ftr | |
135 | .Lskip_rsb_\@: | |
117cc7a9 | 136 | #endif |
76b04384 DW |
137 | .endm |
138 | ||
139 | #else /* __ASSEMBLY__ */ | |
140 | ||
141 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ | |
142 | "999:\n\t" \ | |
143 | ".pushsection .discard.nospec\n\t" \ | |
144 | ".long 999b - .\n\t" \ | |
145 | ".popsection\n\t" | |
146 | ||
147 | #if defined(CONFIG_X86_64) && defined(RETPOLINE) | |
148 | ||
149 | /* | |
150 | * Since the inline asm uses the %V modifier which is only in newer GCC, | |
151 | * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. | |
152 | */ | |
153 | # define CALL_NOSPEC \ | |
154 | ANNOTATE_NOSPEC_ALTERNATIVE \ | |
155 | ALTERNATIVE( \ | |
156 | "call *%[thunk_target]\n", \ | |
157 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ | |
158 | X86_FEATURE_RETPOLINE) | |
159 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) | |
160 | ||
161 | #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) | |
162 | /* | |
163 | * For i386 we use the original ret-equivalent retpoline, because | |
164 | * otherwise we'll run out of registers. We don't care about CET | |
165 | * here, anyway. | |
166 | */ | |
167 | # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ | |
168 | " jmp 904f;\n" \ | |
169 | " .align 16\n" \ | |
170 | "901: call 903f;\n" \ | |
171 | "902: pause;\n" \ | |
28d437d5 | 172 | " lfence;\n" \ |
76b04384 DW |
173 | " jmp 902b;\n" \ |
174 | " .align 16\n" \ | |
175 | "903: addl $4, %%esp;\n" \ | |
176 | " pushl %[thunk_target];\n" \ | |
177 | " ret;\n" \ | |
178 | " .align 16\n" \ | |
179 | "904: call 901b;\n", \ | |
180 | X86_FEATURE_RETPOLINE) | |
181 | ||
182 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
117cc7a9 | 183 | #else /* No retpoline for C / inline asm */ |
76b04384 DW |
184 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
185 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
186 | #endif | |
187 | ||
da285121 DW |
188 | /* The Spectre V2 mitigation variants */ |
189 | enum spectre_v2_mitigation { | |
190 | SPECTRE_V2_NONE, | |
191 | SPECTRE_V2_RETPOLINE_MINIMAL, | |
192 | SPECTRE_V2_RETPOLINE_MINIMAL_AMD, | |
193 | SPECTRE_V2_RETPOLINE_GENERIC, | |
194 | SPECTRE_V2_RETPOLINE_AMD, | |
195 | SPECTRE_V2_IBRS, | |
196 | }; | |
197 | ||
736e80a4 MH |
198 | extern char __indirect_thunk_start[]; |
199 | extern char __indirect_thunk_end[]; | |
200 | ||
117cc7a9 DW |
201 | /* |
202 | * On VMEXIT we must ensure that no RSB predictions learned in the guest | |
203 | * can be followed in the host, by overwriting the RSB completely. Both | |
204 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future | |
894655b1 | 205 | * CPUs with IBRS_ALL *might* it be avoided. |
117cc7a9 DW |
206 | */ |
207 | static inline void vmexit_fill_RSB(void) | |
208 | { | |
209 | #ifdef CONFIG_RETPOLINE | |
10c16289 DW |
210 | unsigned long loops; |
211 | ||
212 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE | |
213 | ALTERNATIVE("jmp 910f", | |
214 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), | |
215 | X86_FEATURE_RETPOLINE) | |
216 | "910:" | |
217 | : "=r" (loops), ASM_CALL_CONSTRAINT | |
218 | : : "memory" ); | |
117cc7a9 DW |
219 | #endif |
220 | } | |
3f7d8755 | 221 | |
390b99c3 DW |
222 | #define alternative_msr_write(_msr, _val, _feature) \ |
223 | asm volatile(ALTERNATIVE("", \ | |
224 | "movl %[msr], %%ecx\n\t" \ | |
225 | "movl %[val], %%eax\n\t" \ | |
226 | "movl $0, %%edx\n\t" \ | |
227 | "wrmsr", \ | |
228 | _feature) \ | |
229 | : : [msr] "i" (_msr), [val] "i" (_val) \ | |
230 | : "eax", "ecx", "edx", "memory") | |
231 | ||
bd12e896 DW |
232 | static inline void indirect_branch_prediction_barrier(void) |
233 | { | |
390b99c3 DW |
234 | alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB, |
235 | X86_FEATURE_USE_IBPB); | |
236 | } | |
237 | ||
238 | /* | |
239 | * With retpoline, we must use IBRS to restrict branch prediction | |
240 | * before calling into firmware. | |
241 | */ | |
242 | static inline void firmware_restrict_branch_speculation_start(void) | |
243 | { | |
244 | preempt_disable(); | |
245 | alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS, | |
246 | X86_FEATURE_USE_IBRS_FW); | |
247 | } | |
248 | ||
249 | static inline void firmware_restrict_branch_speculation_end(void) | |
250 | { | |
251 | alternative_msr_write(MSR_IA32_SPEC_CTRL, 0, | |
252 | X86_FEATURE_USE_IBRS_FW); | |
253 | preempt_enable(); | |
bd12e896 DW |
254 | } |
255 | ||
76b04384 | 256 | #endif /* __ASSEMBLY__ */ |
29fed5fa DB |
257 | |
258 | /* | |
259 | * Below is used in the eBPF JIT compiler and emits the byte sequence | |
260 | * for the following assembly: | |
261 | * | |
262 | * With retpolines configured: | |
263 | * | |
264 | * callq do_rop | |
265 | * spec_trap: | |
266 | * pause | |
267 | * lfence | |
268 | * jmp spec_trap | |
269 | * do_rop: | |
270 | * mov %rax,(%rsp) | |
271 | * retq | |
272 | * | |
273 | * Without retpolines configured: | |
274 | * | |
275 | * jmp *%rax | |
276 | */ | |
277 | #ifdef CONFIG_RETPOLINE | |
278 | # define RETPOLINE_RAX_BPF_JIT_SIZE 17 | |
279 | # define RETPOLINE_RAX_BPF_JIT() \ | |
280 | EMIT1_off32(0xE8, 7); /* callq do_rop */ \ | |
281 | /* spec_trap: */ \ | |
282 | EMIT2(0xF3, 0x90); /* pause */ \ | |
283 | EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \ | |
284 | EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \ | |
285 | /* do_rop: */ \ | |
286 | EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \ | |
287 | EMIT1(0xC3); /* retq */ | |
288 | #else | |
289 | # define RETPOLINE_RAX_BPF_JIT_SIZE 2 | |
290 | # define RETPOLINE_RAX_BPF_JIT() \ | |
291 | EMIT2(0xFF, 0xE0); /* jmp *%rax */ | |
292 | #endif | |
293 | ||
c55d51ae | 294 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |