]>
Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
c55d51ae BP |
3 | #ifndef _ASM_X86_NOSPEC_BRANCH_H_ |
4 | #define _ASM_X86_NOSPEC_BRANCH_H_ | |
76b04384 DW |
5 | |
6 | #include <asm/alternative.h> | |
7 | #include <asm/alternative-asm.h> | |
8 | #include <asm/cpufeatures.h> | |
9 | ||
117cc7a9 DW |
10 | /* |
11 | * Fill the CPU return stack buffer. | |
12 | * | |
13 | * Each entry in the RSB, if used for a speculative 'ret', contains an | |
28d437d5 | 14 | * infinite 'pause; lfence; jmp' loop to capture speculative execution. |
117cc7a9 DW |
15 | * |
16 | * This is required in various cases for retpoline and IBRS-based | |
17 | * mitigations for the Spectre variant 2 vulnerability. Sometimes to | |
18 | * eliminate potentially bogus entries from the RSB, and sometimes | |
19 | * purely to ensure that it doesn't get empty, which on some CPUs would | |
20 | * allow predictions from other (unwanted!) sources to be used. | |
21 | * | |
22 | * We define a CPP macro such that it can be used from both .S files and | |
23 | * inline assembly. It's possible to do a .macro and then include that | |
24 | * from C via asm(".include <asm/nospec-branch.h>") but let's not go there. | |
25 | */ | |
26 | ||
27 | #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ | |
28 | #define RSB_FILL_LOOPS 16 /* To avoid underflow */ | |
29 | ||
30 | /* | |
31 | * Google experimented with loop-unrolling and this turned out to be | |
32 | * the optimal version — two calls, each with their own speculation | |
33 | * trap should their return address end up getting used, in a loop. | |
34 | */ | |
35 | #define __FILL_RETURN_BUFFER(reg, nr, sp) \ | |
36 | mov $(nr/2), reg; \ | |
37 | 771: \ | |
38 | call 772f; \ | |
39 | 773: /* speculation trap */ \ | |
40 | pause; \ | |
28d437d5 | 41 | lfence; \ |
117cc7a9 DW |
42 | jmp 773b; \ |
43 | 772: \ | |
44 | call 774f; \ | |
45 | 775: /* speculation trap */ \ | |
46 | pause; \ | |
28d437d5 | 47 | lfence; \ |
117cc7a9 DW |
48 | jmp 775b; \ |
49 | 774: \ | |
50 | dec reg; \ | |
51 | jnz 771b; \ | |
52 | add $(BITS_PER_LONG/8) * nr, sp; | |
53 | ||
76b04384 DW |
54 | #ifdef __ASSEMBLY__ |
55 | ||
56 | /* | |
57 | * This should be used immediately before a retpoline alternative. It tells | |
58 | * objtool where the retpolines are so that it can make sense of the control | |
59 | * flow by just reading the original instruction(s) and ignoring the | |
60 | * alternatives. | |
61 | */ | |
62 | .macro ANNOTATE_NOSPEC_ALTERNATIVE | |
63 | .Lannotate_\@: | |
64 | .pushsection .discard.nospec | |
65 | .long .Lannotate_\@ - . | |
66 | .popsection | |
67 | .endm | |
68 | ||
69 | /* | |
70 | * These are the bare retpoline primitives for indirect jmp and call. | |
71 | * Do not use these directly; they only exist to make the ALTERNATIVE | |
72 | * invocation below less ugly. | |
73 | */ | |
74 | .macro RETPOLINE_JMP reg:req | |
75 | call .Ldo_rop_\@ | |
76 | .Lspec_trap_\@: | |
77 | pause | |
28d437d5 | 78 | lfence |
76b04384 DW |
79 | jmp .Lspec_trap_\@ |
80 | .Ldo_rop_\@: | |
81 | mov \reg, (%_ASM_SP) | |
82 | ret | |
83 | .endm | |
84 | ||
85 | /* | |
86 | * This is a wrapper around RETPOLINE_JMP so the called function in reg | |
87 | * returns to the instruction after the macro. | |
88 | */ | |
89 | .macro RETPOLINE_CALL reg:req | |
90 | jmp .Ldo_call_\@ | |
91 | .Ldo_retpoline_jmp_\@: | |
92 | RETPOLINE_JMP \reg | |
93 | .Ldo_call_\@: | |
94 | call .Ldo_retpoline_jmp_\@ | |
95 | .endm | |
96 | ||
97 | /* | |
98 | * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple | |
99 | * indirect jmp/call which may be susceptible to the Spectre variant 2 | |
100 | * attack. | |
101 | */ | |
102 | .macro JMP_NOSPEC reg:req | |
103 | #ifdef CONFIG_RETPOLINE | |
104 | ANNOTATE_NOSPEC_ALTERNATIVE | |
105 | ALTERNATIVE_2 __stringify(jmp *\reg), \ | |
106 | __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \ | |
107 | __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD | |
108 | #else | |
109 | jmp *\reg | |
110 | #endif | |
111 | .endm | |
112 | ||
113 | .macro CALL_NOSPEC reg:req | |
114 | #ifdef CONFIG_RETPOLINE | |
115 | ANNOTATE_NOSPEC_ALTERNATIVE | |
116 | ALTERNATIVE_2 __stringify(call *\reg), \ | |
117 | __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\ | |
118 | __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD | |
119 | #else | |
120 | call *\reg | |
121 | #endif | |
117cc7a9 DW |
122 | .endm |
123 | ||
124 | /* | |
125 | * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP | |
126 | * monstrosity above, manually. | |
127 | */ | |
128 | .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req | |
129 | #ifdef CONFIG_RETPOLINE | |
130 | ANNOTATE_NOSPEC_ALTERNATIVE | |
131 | ALTERNATIVE "jmp .Lskip_rsb_\@", \ | |
132 | __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \ | |
133 | \ftr | |
134 | .Lskip_rsb_\@: | |
135 | #endif | |
76b04384 DW |
136 | .endm |
137 | ||
138 | #else /* __ASSEMBLY__ */ | |
139 | ||
140 | #define ANNOTATE_NOSPEC_ALTERNATIVE \ | |
141 | "999:\n\t" \ | |
142 | ".pushsection .discard.nospec\n\t" \ | |
143 | ".long 999b - .\n\t" \ | |
144 | ".popsection\n\t" | |
145 | ||
146 | #if defined(CONFIG_X86_64) && defined(RETPOLINE) | |
147 | ||
148 | /* | |
149 | * Since the inline asm uses the %V modifier which is only in newer GCC, | |
150 | * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE. | |
151 | */ | |
152 | # define CALL_NOSPEC \ | |
153 | ANNOTATE_NOSPEC_ALTERNATIVE \ | |
154 | ALTERNATIVE( \ | |
155 | "call *%[thunk_target]\n", \ | |
156 | "call __x86_indirect_thunk_%V[thunk_target]\n", \ | |
157 | X86_FEATURE_RETPOLINE) | |
158 | # define THUNK_TARGET(addr) [thunk_target] "r" (addr) | |
159 | ||
160 | #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE) | |
161 | /* | |
162 | * For i386 we use the original ret-equivalent retpoline, because | |
163 | * otherwise we'll run out of registers. We don't care about CET | |
164 | * here, anyway. | |
165 | */ | |
166 | # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \ | |
167 | " jmp 904f;\n" \ | |
168 | " .align 16\n" \ | |
169 | "901: call 903f;\n" \ | |
170 | "902: pause;\n" \ | |
28d437d5 | 171 | " lfence;\n" \ |
76b04384 DW |
172 | " jmp 902b;\n" \ |
173 | " .align 16\n" \ | |
174 | "903: addl $4, %%esp;\n" \ | |
175 | " pushl %[thunk_target];\n" \ | |
176 | " ret;\n" \ | |
177 | " .align 16\n" \ | |
178 | "904: call 901b;\n", \ | |
179 | X86_FEATURE_RETPOLINE) | |
180 | ||
181 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
117cc7a9 | 182 | #else /* No retpoline for C / inline asm */ |
76b04384 DW |
183 | # define CALL_NOSPEC "call *%[thunk_target]\n" |
184 | # define THUNK_TARGET(addr) [thunk_target] "rm" (addr) | |
185 | #endif | |
186 | ||
da285121 DW |
187 | /* The Spectre V2 mitigation variants */ |
188 | enum spectre_v2_mitigation { | |
189 | SPECTRE_V2_NONE, | |
190 | SPECTRE_V2_RETPOLINE_MINIMAL, | |
191 | SPECTRE_V2_RETPOLINE_MINIMAL_AMD, | |
192 | SPECTRE_V2_RETPOLINE_GENERIC, | |
193 | SPECTRE_V2_RETPOLINE_AMD, | |
194 | SPECTRE_V2_IBRS, | |
195 | }; | |
196 | ||
736e80a4 MH |
197 | extern char __indirect_thunk_start[]; |
198 | extern char __indirect_thunk_end[]; | |
199 | ||
117cc7a9 DW |
200 | /* |
201 | * On VMEXIT we must ensure that no RSB predictions learned in the guest | |
202 | * can be followed in the host, by overwriting the RSB completely. Both | |
203 | * retpoline and IBRS mitigations for Spectre v2 need this; only on future | |
204 | * CPUs with IBRS_ATT *might* it be avoided. | |
205 | */ | |
206 | static inline void vmexit_fill_RSB(void) | |
207 | { | |
208 | #ifdef CONFIG_RETPOLINE | |
3f7d8755 | 209 | unsigned long loops; |
117cc7a9 DW |
210 | |
211 | asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE | |
212 | ALTERNATIVE("jmp 910f", | |
213 | __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)), | |
214 | X86_FEATURE_RETPOLINE) | |
215 | "910:" | |
3f7d8755 AK |
216 | : "=r" (loops), ASM_CALL_CONSTRAINT |
217 | : : "memory" ); | |
117cc7a9 DW |
218 | #endif |
219 | } | |
3f7d8755 | 220 | |
bd12e896 DW |
221 | static inline void indirect_branch_prediction_barrier(void) |
222 | { | |
223 | asm volatile(ALTERNATIVE("", | |
224 | "movl %[msr], %%ecx\n\t" | |
225 | "movl %[val], %%eax\n\t" | |
226 | "movl $0, %%edx\n\t" | |
227 | "wrmsr", | |
228 | X86_FEATURE_IBPB) | |
229 | : : [msr] "i" (MSR_IA32_PRED_CMD), | |
230 | [val] "i" (PRED_CMD_IBPB) | |
231 | : "eax", "ecx", "edx", "memory"); | |
232 | } | |
233 | ||
76b04384 | 234 | #endif /* __ASSEMBLY__ */ |
c55d51ae | 235 | #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ |