]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
478dc89c | 2 | #include <linux/jump_label.h> |
8c1f7558 | 3 | #include <asm/unwind_hints.h> |
8a09317b DH |
4 | #include <asm/cpufeatures.h> |
5 | #include <asm/page_types.h> | |
6fd166aa PZ |
6 | #include <asm/percpu.h> |
7 | #include <asm/asm-offsets.h> | |
8 | #include <asm/processor-flags.h> | |
6627eb25 | 9 | #include <asm/ptrace-abi.h> |
478dc89c | 10 | |
0c2bd5a5 | 11 | /* |
063f8913 IM |
12 | |
13 | x86 function call convention, 64-bit: | |
14 | ------------------------------------- | |
15 | arguments | callee-saved | extra caller-saved | return | |
16 | [callee-clobbered] | | [callee-clobbered] | | |
17 | --------------------------------------------------------------------------- | |
18 | rdi rsi rdx rcx r8-9 | rbx rbp [*] r12-15 | r10-11 | rax, rdx [**] | |
19 | ||
20 | ( rsp is obviously invariant across normal function calls. (gcc can 'merge' | |
21 | functions when it sees tail-call optimization possibilities) rflags is | |
22 | clobbered. Leftover arguments are passed over the stack frame.) | |
23 | ||
24 | [*] In the frame-pointers case rbp is fixed to the stack frame. | |
25 | ||
26 | [**] for struct return values wider than 64 bits the return convention is a | |
27 | bit more complex: up to 128 bits width we return small structures | |
28 | straight in rax, rdx. For structures larger than that (3 words or | |
29 | larger) the caller puts a pointer to an on-stack return struct | |
30 | [allocated in the caller's stack frame] into the first argument - i.e. | |
31 | into rdi. All other arguments shift up by one in this case. | |
32 | Fortunately this case is rare in the kernel. | |
33 | ||
34 | For 32-bit we have the following conventions - kernel is built with | |
35 | -mregparm=3 and -freg-struct-return: | |
36 | ||
37 | x86 function calling convention, 32-bit: | |
38 | ---------------------------------------- | |
39 | arguments | callee-saved | extra caller-saved | return | |
40 | [callee-clobbered] | | [callee-clobbered] | | |
41 | ------------------------------------------------------------------------- | |
42 | eax edx ecx | ebx edi esi ebp [*] | <none> | eax, edx [**] | |
43 | ||
44 | ( here too esp is obviously invariant across normal function calls. eflags | |
45 | is clobbered. Leftover arguments are passed over the stack frame. ) | |
46 | ||
47 | [*] In the frame-pointers case ebp is fixed to the stack frame. | |
48 | ||
49 | [**] We build with -freg-struct-return, which on 32-bit means similar | |
50 | semantics as on 64-bit: edx can be used for a second return value | |
51 | (i.e. covering integer and structure sizes up to 64 bits) - after that | |
52 | it gets more complex and more expensive: 3-word or larger struct returns | |
53 | get done in the caller's frame and the pointer to the return struct goes | |
54 | into regparm0, i.e. eax - the other arguments shift up and the | |
55 | function's register parameters degenerate to regparm=2 in essence. | |
56 | ||
57 | */ | |
58 | ||
1a338ac3 PZ |
59 | #ifdef CONFIG_X86_64 |
60 | ||
063f8913 | 61 | /* |
1b2b23d8 TG |
62 | * 64-bit system call stack frame layout defines and helpers, |
63 | * for assembly code: | |
0c2bd5a5 | 64 | */ |
1da177e4 | 65 | |
29e97589 | 66 | .macro PUSH_REGS rdx=%rdx rax=%rax save_ret=0 |
9e809d15 DB |
67 | .if \save_ret |
68 | pushq %rsi /* pt_regs->si */ | |
69 | movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */ | |
70 | movq %rdi, 8(%rsp) /* pt_regs->di (overwriting original return address) */ | |
71 | .else | |
3f01daec DB |
72 | pushq %rdi /* pt_regs->di */ |
73 | pushq %rsi /* pt_regs->si */ | |
9e809d15 | 74 | .endif |
30907fd1 | 75 | pushq \rdx /* pt_regs->dx */ |
3f01daec | 76 | pushq %rcx /* pt_regs->cx */ |
30907fd1 | 77 | pushq \rax /* pt_regs->ax */ |
3f01daec | 78 | pushq %r8 /* pt_regs->r8 */ |
3f01daec | 79 | pushq %r9 /* pt_regs->r9 */ |
3f01daec | 80 | pushq %r10 /* pt_regs->r10 */ |
3f01daec | 81 | pushq %r11 /* pt_regs->r11 */ |
3f01daec | 82 | pushq %rbx /* pt_regs->rbx */ |
3f01daec | 83 | pushq %rbp /* pt_regs->rbp */ |
3f01daec | 84 | pushq %r12 /* pt_regs->r12 */ |
3f01daec | 85 | pushq %r13 /* pt_regs->r13 */ |
3f01daec | 86 | pushq %r14 /* pt_regs->r14 */ |
3f01daec | 87 | pushq %r15 /* pt_regs->r15 */ |
3f01daec | 88 | UNWIND_HINT_REGS |
06a9750e | 89 | |
9e809d15 DB |
90 | .if \save_ret |
91 | pushq %rsi /* return address on top of stack */ | |
92 | .endif | |
29e97589 | 93 | .endm |
06a9750e | 94 | |
29e97589 | 95 | .macro CLEAR_REGS |
06a9750e JP |
96 | /* |
97 | * Sanitize registers of values that a speculation attack might | |
98 | * otherwise want to exploit. The lower registers are likely clobbered | |
99 | * well before they could be put to use in a speculative execution | |
100 | * gadget. | |
101 | */ | |
102 | xorl %edx, %edx /* nospec dx */ | |
103 | xorl %ecx, %ecx /* nospec cx */ | |
104 | xorl %r8d, %r8d /* nospec r8 */ | |
105 | xorl %r9d, %r9d /* nospec r9 */ | |
106 | xorl %r10d, %r10d /* nospec r10 */ | |
107 | xorl %r11d, %r11d /* nospec r11 */ | |
108 | xorl %ebx, %ebx /* nospec rbx */ | |
109 | xorl %ebp, %ebp /* nospec rbp */ | |
110 | xorl %r12d, %r12d /* nospec r12 */ | |
111 | xorl %r13d, %r13d /* nospec r13 */ | |
112 | xorl %r14d, %r14d /* nospec r14 */ | |
113 | xorl %r15d, %r15d /* nospec r15 */ | |
114 | ||
92816f57 | 115 | .endm |
3f01daec | 116 | |
29e97589 PAI |
117 | .macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 |
118 | PUSH_REGS rdx=\rdx, rax=\rax, save_ret=\save_ret | |
119 | CLEAR_REGS | |
120 | .endm | |
121 | ||
92816f57 | 122 | .macro POP_REGS pop_rdi=1 skip_r11rcx=0 |
e872045b AL |
123 | popq %r15 |
124 | popq %r14 | |
125 | popq %r13 | |
126 | popq %r12 | |
127 | popq %rbp | |
128 | popq %rbx | |
502af0d7 DB |
129 | .if \skip_r11rcx |
130 | popq %rsi | |
131 | .else | |
e872045b | 132 | popq %r11 |
502af0d7 | 133 | .endif |
e872045b AL |
134 | popq %r10 |
135 | popq %r9 | |
136 | popq %r8 | |
137 | popq %rax | |
502af0d7 DB |
138 | .if \skip_r11rcx |
139 | popq %rsi | |
140 | .else | |
e872045b | 141 | popq %rcx |
502af0d7 | 142 | .endif |
e872045b AL |
143 | popq %rdx |
144 | popq %rsi | |
502af0d7 | 145 | .if \pop_rdi |
e872045b | 146 | popq %rdi |
502af0d7 | 147 | .endif |
92816f57 | 148 | .endm |
1a338ac3 | 149 | |
8a09317b DH |
150 | #ifdef CONFIG_PAGE_TABLE_ISOLATION |
151 | ||
6fd166aa PZ |
152 | /* |
153 | * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two | |
154 | * halves: | |
155 | */ | |
f10ee3dc TG |
156 | #define PTI_USER_PGTABLE_BIT PAGE_SHIFT |
157 | #define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) | |
158 | #define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT | |
159 | #define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) | |
160 | #define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) | |
8a09317b | 161 | |
6fd166aa PZ |
162 | .macro SET_NOFLUSH_BIT reg:req |
163 | bts $X86_CR3_PCID_NOFLUSH_BIT, \reg | |
8a09317b DH |
164 | .endm |
165 | ||
6fd166aa PZ |
166 | .macro ADJUST_KERNEL_CR3 reg:req |
167 | ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID | |
168 | /* Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ | |
f10ee3dc | 169 | andq $(~PTI_USER_PGTABLE_AND_PCID_MASK), \reg |
8a09317b DH |
170 | .endm |
171 | ||
172 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req | |
aa8c6248 | 173 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
8a09317b DH |
174 | mov %cr3, \scratch_reg |
175 | ADJUST_KERNEL_CR3 \scratch_reg | |
176 | mov \scratch_reg, %cr3 | |
aa8c6248 | 177 | .Lend_\@: |
8a09317b DH |
178 | .endm |
179 | ||
6fd166aa PZ |
180 | #define THIS_CPU_user_pcid_flush_mask \ |
181 | PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask | |
182 | ||
183 | .macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req | |
aa8c6248 | 184 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
8a09317b | 185 | mov %cr3, \scratch_reg |
6fd166aa PZ |
186 | |
187 | ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID | |
188 | ||
189 | /* | |
190 | * Test if the ASID needs a flush. | |
191 | */ | |
192 | movq \scratch_reg, \scratch_reg2 | |
193 | andq $(0x7FF), \scratch_reg /* mask ASID */ | |
194 | bt \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
195 | jnc .Lnoflush_\@ | |
196 | ||
197 | /* Flush needed, clear the bit */ | |
198 | btr \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
199 | movq \scratch_reg2, \scratch_reg | |
f10ee3dc | 200 | jmp .Lwrcr3_pcid_\@ |
6fd166aa PZ |
201 | |
202 | .Lnoflush_\@: | |
203 | movq \scratch_reg2, \scratch_reg | |
204 | SET_NOFLUSH_BIT \scratch_reg | |
205 | ||
f10ee3dc TG |
206 | .Lwrcr3_pcid_\@: |
207 | /* Flip the ASID to the user version */ | |
208 | orq $(PTI_USER_PCID_MASK), \scratch_reg | |
209 | ||
6fd166aa | 210 | .Lwrcr3_\@: |
f10ee3dc TG |
211 | /* Flip the PGD to the user version */ |
212 | orq $(PTI_USER_PGTABLE_MASK), \scratch_reg | |
8a09317b | 213 | mov \scratch_reg, %cr3 |
aa8c6248 | 214 | .Lend_\@: |
8a09317b DH |
215 | .endm |
216 | ||
6fd166aa PZ |
217 | .macro SWITCH_TO_USER_CR3_STACK scratch_reg:req |
218 | pushq %rax | |
219 | SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax | |
220 | popq %rax | |
221 | .endm | |
222 | ||
8a09317b | 223 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req |
aa8c6248 | 224 | ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI |
8a09317b DH |
225 | movq %cr3, \scratch_reg |
226 | movq \scratch_reg, \save_reg | |
227 | /* | |
f10ee3dc TG |
228 | * Test the user pagetable bit. If set, then the user page tables |
229 | * are active. If clear CR3 already has the kernel page table | |
230 | * active. | |
8a09317b | 231 | */ |
f10ee3dc TG |
232 | bt $PTI_USER_PGTABLE_BIT, \scratch_reg |
233 | jnc .Ldone_\@ | |
8a09317b DH |
234 | |
235 | ADJUST_KERNEL_CR3 \scratch_reg | |
236 | movq \scratch_reg, %cr3 | |
237 | ||
238 | .Ldone_\@: | |
239 | .endm | |
240 | ||
21e94459 | 241 | .macro RESTORE_CR3 scratch_reg:req save_reg:req |
aa8c6248 | 242 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI |
21e94459 PZ |
243 | |
244 | ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID | |
245 | ||
246 | /* | |
247 | * KERNEL pages can always resume with NOFLUSH as we do | |
248 | * explicit flushes. | |
249 | */ | |
f10ee3dc | 250 | bt $PTI_USER_PGTABLE_BIT, \save_reg |
21e94459 PZ |
251 | jnc .Lnoflush_\@ |
252 | ||
253 | /* | |
254 | * Check if there's a pending flush for the user ASID we're | |
255 | * about to set. | |
256 | */ | |
257 | movq \save_reg, \scratch_reg | |
258 | andq $(0x7FF), \scratch_reg | |
259 | bt \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
260 | jnc .Lnoflush_\@ | |
261 | ||
262 | btr \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
263 | jmp .Lwrcr3_\@ | |
264 | ||
265 | .Lnoflush_\@: | |
266 | SET_NOFLUSH_BIT \save_reg | |
267 | ||
268 | .Lwrcr3_\@: | |
8a09317b DH |
269 | /* |
270 | * The CR3 write could be avoided when not changing its value, | |
271 | * but would require a CR3 read *and* a scratch register. | |
272 | */ | |
273 | movq \save_reg, %cr3 | |
aa8c6248 | 274 | .Lend_\@: |
8a09317b DH |
275 | .endm |
276 | ||
277 | #else /* CONFIG_PAGE_TABLE_ISOLATION=n: */ | |
278 | ||
279 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req | |
280 | .endm | |
6fd166aa PZ |
281 | .macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req |
282 | .endm | |
283 | .macro SWITCH_TO_USER_CR3_STACK scratch_reg:req | |
8a09317b DH |
284 | .endm |
285 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req | |
286 | .endm | |
21e94459 | 287 | .macro RESTORE_CR3 scratch_reg:req save_reg:req |
8a09317b DH |
288 | .endm |
289 | ||
290 | #endif | |
291 | ||
18ec54fd JP |
292 | /* |
293 | * Mitigate Spectre v1 for conditional swapgs code paths. | |
294 | * | |
295 | * FENCE_SWAPGS_USER_ENTRY is used in the user entry swapgs code path, to | |
296 | * prevent a speculative swapgs when coming from kernel space. | |
297 | * | |
298 | * FENCE_SWAPGS_KERNEL_ENTRY is used in the kernel entry non-swapgs code path, | |
299 | * to prevent the swapgs from getting speculatively skipped when coming from | |
300 | * user space. | |
301 | */ | |
302 | .macro FENCE_SWAPGS_USER_ENTRY | |
303 | ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_USER | |
304 | .endm | |
305 | .macro FENCE_SWAPGS_KERNEL_ENTRY | |
306 | ALTERNATIVE "", "lfence", X86_FEATURE_FENCE_SWAPGS_KERNEL | |
307 | .endm | |
308 | ||
afaef01c AP |
309 | .macro STACKLEAK_ERASE_NOCLOBBER |
310 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK | |
311 | PUSH_AND_CLEAR_REGS | |
312 | call stackleak_erase | |
313 | POP_REGS | |
314 | #endif | |
315 | .endm | |
316 | ||
c82965f9 CB |
317 | .macro SAVE_AND_SET_GSBASE scratch_reg:req save_reg:req |
318 | rdgsbase \save_reg | |
319 | GET_PERCPU_BASE \scratch_reg | |
320 | wrgsbase \scratch_reg | |
321 | .endm | |
322 | ||
633260fa TG |
323 | #else /* CONFIG_X86_64 */ |
324 | # undef UNWIND_HINT_IRET_REGS | |
325 | # define UNWIND_HINT_IRET_REGS | |
326 | #endif /* !CONFIG_X86_64 */ | |
1a338ac3 | 327 | |
afaef01c AP |
328 | .macro STACKLEAK_ERASE |
329 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK | |
330 | call stackleak_erase | |
331 | #endif | |
332 | .endm | |
eaad9812 CB |
333 | |
334 | #ifdef CONFIG_SMP | |
335 | ||
336 | /* | |
337 | * CPU/node NR is loaded from the limit (size) field of a special segment | |
338 | * descriptor entry in GDT. | |
339 | */ | |
340 | .macro LOAD_CPU_AND_NODE_SEG_LIMIT reg:req | |
341 | movq $__CPUNODE_SEG, \reg | |
342 | lsl \reg, \reg | |
343 | .endm | |
344 | ||
345 | /* | |
346 | * Fetch the per-CPU GSBASE value for this processor and put it in @reg. | |
347 | * We normally use %gs for accessing per-CPU data, but we are setting up | |
348 | * %gs here and obviously can not use %gs itself to access per-CPU data. | |
6a3ea3e6 SC |
349 | * |
350 | * Do not use RDPID, because KVM loads guest's TSC_AUX on vm-entry and | |
351 | * may not restore the host's value until the CPU returns to userspace. | |
352 | * Thus the kernel would consume a guest's TSC_AUX if an NMI arrives | |
353 | * while running KVM's run loop. | |
eaad9812 CB |
354 | */ |
355 | .macro GET_PERCPU_BASE reg:req | |
6a3ea3e6 | 356 | LOAD_CPU_AND_NODE_SEG_LIMIT \reg |
eaad9812 CB |
357 | andq $VDSO_CPUNODE_MASK, \reg |
358 | movq __per_cpu_offset(, \reg, 8), \reg | |
359 | .endm | |
360 | ||
361 | #else | |
362 | ||
363 | .macro GET_PERCPU_BASE reg:req | |
364 | movq pcpu_unit_offsets(%rip), \reg | |
365 | .endm | |
366 | ||
367 | #endif /* CONFIG_SMP */ |