]>
Commit | Line | Data |
---|---|---|
321d628a FG |
1 | From 98a0f7841c6cdd3f882548ed3975df7b22cfa482 Mon Sep 17 00:00:00 2001 |
2 | From: Andy Lutomirski <luto@kernel.org> | |
3 | Date: Mon, 4 Dec 2017 15:07:23 +0100 | |
e4cdf2a5 | 4 | Subject: [PATCH 153/241] x86/entry/64: Use a per-CPU trampoline stack for IDT |
321d628a FG |
5 | entries |
6 | MIME-Version: 1.0 | |
7 | Content-Type: text/plain; charset=UTF-8 | |
8 | Content-Transfer-Encoding: 8bit | |
9 | ||
10 | CVE-2017-5754 | |
11 | ||
12 | Historically, IDT entries from usermode have always gone directly | |
13 | to the running task's kernel stack. Rearrange it so that we enter on | |
14 | a per-CPU trampoline stack and then manually switch to the task's stack. | |
15 | This touches a couple of extra cachelines, but it gives us a chance | |
16 | to run some code before we touch the kernel stack. | |
17 | ||
18 | The asm isn't exactly beautiful, but I think that fully refactoring | |
19 | it can wait. | |
20 | ||
21 | Signed-off-by: Andy Lutomirski <luto@kernel.org> | |
22 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
23 | Reviewed-by: Borislav Petkov <bp@suse.de> | |
24 | Reviewed-by: Thomas Gleixner <tglx@linutronix.de> | |
25 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
26 | Cc: Borislav Petkov <bp@alien8.de> | |
27 | Cc: Borislav Petkov <bpetkov@suse.de> | |
28 | Cc: Brian Gerst <brgerst@gmail.com> | |
29 | Cc: Dave Hansen <dave.hansen@intel.com> | |
30 | Cc: Dave Hansen <dave.hansen@linux.intel.com> | |
31 | Cc: David Laight <David.Laight@aculab.com> | |
32 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
33 | Cc: Eduardo Valentin <eduval@amazon.com> | |
34 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
35 | Cc: H. Peter Anvin <hpa@zytor.com> | |
36 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
37 | Cc: Juergen Gross <jgross@suse.com> | |
38 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
39 | Cc: Peter Zijlstra <peterz@infradead.org> | |
40 | Cc: Rik van Riel <riel@redhat.com> | |
41 | Cc: Will Deacon <will.deacon@arm.com> | |
42 | Cc: aliguori@amazon.com | |
43 | Cc: daniel.gruss@iaik.tugraz.at | |
44 | Cc: hughd@google.com | |
45 | Cc: keescook@google.com | |
46 | Link: https://lkml.kernel.org/r/20171204150606.225330557@linutronix.de | |
47 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
48 | (cherry picked from commit 7f2590a110b837af5679d08fc25c6227c5a8c497) | |
49 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
50 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
51 | (cherry picked from commit bfb2d0ede023853fb8c24d3dae8974cb2f7117c3) | |
52 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
53 | --- | |
54 | arch/x86/include/asm/switch_to.h | 4 ++- | |
55 | arch/x86/include/asm/traps.h | 1 - | |
56 | arch/x86/kernel/cpu/common.c | 6 ++-- | |
57 | arch/x86/kernel/traps.c | 21 +++++++------ | |
58 | arch/x86/entry/entry_64.S | 67 ++++++++++++++++++++++++++++++---------- | |
59 | arch/x86/entry/entry_64_compat.S | 5 ++- | |
60 | 6 files changed, 72 insertions(+), 32 deletions(-) | |
61 | ||
62 | diff --git a/arch/x86/include/asm/switch_to.h b/arch/x86/include/asm/switch_to.h | |
63 | index 010cd6e4eafc..ca2fc84ad278 100644 | |
64 | --- a/arch/x86/include/asm/switch_to.h | |
65 | +++ b/arch/x86/include/asm/switch_to.h | |
66 | @@ -89,10 +89,12 @@ static inline void refresh_sysenter_cs(struct thread_struct *thread) | |
67 | /* This is used when switching tasks or entering/exiting vm86 mode. */ | |
68 | static inline void update_sp0(struct task_struct *task) | |
69 | { | |
70 | + /* On x86_64, sp0 always points to the entry trampoline stack, which is constant: */ | |
71 | #ifdef CONFIG_X86_32 | |
72 | load_sp0(task->thread.sp0); | |
73 | #else | |
74 | - load_sp0(task_top_of_stack(task)); | |
75 | + if (static_cpu_has(X86_FEATURE_XENPV)) | |
76 | + load_sp0(task_top_of_stack(task)); | |
77 | #endif | |
78 | } | |
79 | ||
80 | diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h | |
81 | index b052a7621ca1..c3b652672d6f 100644 | |
82 | --- a/arch/x86/include/asm/traps.h | |
83 | +++ b/arch/x86/include/asm/traps.h | |
84 | @@ -92,7 +92,6 @@ dotraplinkage void do_segment_not_present(struct pt_regs *, long); | |
85 | dotraplinkage void do_stack_segment(struct pt_regs *, long); | |
86 | #ifdef CONFIG_X86_64 | |
87 | dotraplinkage void do_double_fault(struct pt_regs *, long); | |
88 | -asmlinkage struct pt_regs *sync_regs(struct pt_regs *); | |
89 | #endif | |
90 | dotraplinkage void do_general_protection(struct pt_regs *, long); | |
91 | dotraplinkage void do_page_fault(struct pt_regs *, unsigned long); | |
92 | diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c | |
93 | index 4a38de4c6ede..404e4b75db6e 100644 | |
94 | --- a/arch/x86/kernel/cpu/common.c | |
95 | +++ b/arch/x86/kernel/cpu/common.c | |
96 | @@ -1623,11 +1623,13 @@ void cpu_init(void) | |
97 | setup_cpu_entry_area(cpu); | |
98 | ||
99 | /* | |
100 | - * Initialize the TSS. Don't bother initializing sp0, as the initial | |
101 | - * task never enters user mode. | |
102 | + * Initialize the TSS. sp0 points to the entry trampoline stack | |
103 | + * regardless of what task is running. | |
104 | */ | |
105 | set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss); | |
106 | load_TR_desc(); | |
107 | + load_sp0((unsigned long)&get_cpu_entry_area(cpu)->tss + | |
108 | + offsetofend(struct tss_struct, SYSENTER_stack)); | |
109 | ||
110 | load_mm_ldt(&init_mm); | |
111 | ||
112 | diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c | |
113 | index b69db1ee8733..d9debdafe7a6 100644 | |
114 | --- a/arch/x86/kernel/traps.c | |
115 | +++ b/arch/x86/kernel/traps.c | |
116 | @@ -631,14 +631,15 @@ NOKPROBE_SYMBOL(do_int3); | |
117 | ||
118 | #ifdef CONFIG_X86_64 | |
119 | /* | |
120 | - * Help handler running on IST stack to switch off the IST stack if the | |
121 | - * interrupted code was in user mode. The actual stack switch is done in | |
122 | - * entry_64.S | |
123 | + * Help handler running on a per-cpu (IST or entry trampoline) stack | |
124 | + * to switch to the normal thread stack if the interrupted code was in | |
125 | + * user mode. The actual stack switch is done in entry_64.S | |
126 | */ | |
127 | asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs) | |
128 | { | |
129 | - struct pt_regs *regs = task_pt_regs(current); | |
130 | - *regs = *eregs; | |
131 | + struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1; | |
132 | + if (regs != eregs) | |
133 | + *regs = *eregs; | |
134 | return regs; | |
135 | } | |
136 | NOKPROBE_SYMBOL(sync_regs); | |
137 | @@ -654,13 +655,13 @@ struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s) | |
138 | /* | |
139 | * This is called from entry_64.S early in handling a fault | |
140 | * caused by a bad iret to user mode. To handle the fault | |
141 | - * correctly, we want move our stack frame to task_pt_regs | |
142 | - * and we want to pretend that the exception came from the | |
143 | - * iret target. | |
144 | + * correctly, we want to move our stack frame to where it would | |
145 | + * be had we entered directly on the entry stack (rather than | |
146 | + * just below the IRET frame) and we want to pretend that the | |
147 | + * exception came from the IRET target. | |
148 | */ | |
149 | struct bad_iret_stack *new_stack = | |
150 | - container_of(task_pt_regs(current), | |
151 | - struct bad_iret_stack, regs); | |
152 | + (struct bad_iret_stack *)this_cpu_read(cpu_tss.x86_tss.sp0) - 1; | |
153 | ||
154 | /* Copy the IRET target to the new stack. */ | |
155 | memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8); | |
156 | diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S | |
157 | index 6c73e96daf78..f70fedc58bac 100644 | |
158 | --- a/arch/x86/entry/entry_64.S | |
159 | +++ b/arch/x86/entry/entry_64.S | |
160 | @@ -559,6 +559,13 @@ END(irq_entries_start) | |
161 | /* 0(%rsp): ~(interrupt number) */ | |
162 | .macro interrupt func | |
163 | cld | |
164 | + | |
165 | + testb $3, CS-ORIG_RAX(%rsp) | |
166 | + jz 1f | |
167 | + SWAPGS | |
168 | + call switch_to_thread_stack | |
169 | +1: | |
170 | + | |
171 | ALLOC_PT_GPREGS_ON_STACK | |
172 | SAVE_C_REGS | |
173 | SAVE_EXTRA_REGS | |
174 | @@ -568,12 +575,8 @@ END(irq_entries_start) | |
175 | jz 1f | |
176 | ||
177 | /* | |
178 | - * IRQ from user mode. Switch to kernel gsbase and inform context | |
179 | - * tracking that we're in kernel mode. | |
180 | - */ | |
181 | - SWAPGS | |
182 | - | |
183 | - /* | |
184 | + * IRQ from user mode. | |
185 | + * | |
186 | * We need to tell lockdep that IRQs are off. We can't do this until | |
187 | * we fix gsbase, and we should do it before enter_from_user_mode | |
188 | * (which can take locks). Since TRACE_IRQS_OFF idempotent, | |
189 | @@ -840,6 +843,32 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt | |
190 | */ | |
191 | #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8) | |
192 | ||
193 | +/* | |
194 | + * Switch to the thread stack. This is called with the IRET frame and | |
195 | + * orig_ax on the stack. (That is, RDI..R12 are not on the stack and | |
196 | + * space has not been allocated for them.) | |
197 | + */ | |
198 | +ENTRY(switch_to_thread_stack) | |
199 | + UNWIND_HINT_FUNC | |
200 | + | |
201 | + pushq %rdi | |
202 | + movq %rsp, %rdi | |
203 | + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp | |
204 | + UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI | |
205 | + | |
206 | + pushq 7*8(%rdi) /* regs->ss */ | |
207 | + pushq 6*8(%rdi) /* regs->rsp */ | |
208 | + pushq 5*8(%rdi) /* regs->eflags */ | |
209 | + pushq 4*8(%rdi) /* regs->cs */ | |
210 | + pushq 3*8(%rdi) /* regs->ip */ | |
211 | + pushq 2*8(%rdi) /* regs->orig_ax */ | |
212 | + pushq 8(%rdi) /* return address */ | |
213 | + UNWIND_HINT_FUNC | |
214 | + | |
215 | + movq (%rdi), %rdi | |
216 | + ret | |
217 | +END(switch_to_thread_stack) | |
218 | + | |
219 | .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 | |
220 | ENTRY(\sym) | |
221 | UNWIND_HINT_IRET_REGS offset=\has_error_code*8 | |
222 | @@ -857,11 +886,12 @@ ENTRY(\sym) | |
223 | ||
224 | ALLOC_PT_GPREGS_ON_STACK | |
225 | ||
226 | - .if \paranoid | |
227 | - .if \paranoid == 1 | |
228 | + .if \paranoid < 2 | |
229 | testb $3, CS(%rsp) /* If coming from userspace, switch stacks */ | |
230 | - jnz 1f | |
231 | + jnz .Lfrom_usermode_switch_stack_\@ | |
232 | .endif | |
233 | + | |
234 | + .if \paranoid | |
235 | call paranoid_entry | |
236 | .else | |
237 | call error_entry | |
238 | @@ -903,20 +933,15 @@ ENTRY(\sym) | |
239 | jmp error_exit | |
240 | .endif | |
241 | ||
242 | - .if \paranoid == 1 | |
243 | + .if \paranoid < 2 | |
244 | /* | |
245 | - * Paranoid entry from userspace. Switch stacks and treat it | |
246 | + * Entry from userspace. Switch stacks and treat it | |
247 | * as a normal entry. This means that paranoid handlers | |
248 | * run in real process context if user_mode(regs). | |
249 | */ | |
250 | -1: | |
251 | +.Lfrom_usermode_switch_stack_\@: | |
252 | call error_entry | |
253 | ||
254 | - | |
255 | - movq %rsp, %rdi /* pt_regs pointer */ | |
256 | - call sync_regs | |
257 | - movq %rax, %rsp /* switch stack */ | |
258 | - | |
259 | movq %rsp, %rdi /* pt_regs pointer */ | |
260 | ||
261 | .if \has_error_code | |
262 | @@ -1177,6 +1202,14 @@ ENTRY(error_entry) | |
263 | SWAPGS | |
264 | ||
265 | .Lerror_entry_from_usermode_after_swapgs: | |
266 | + /* Put us onto the real thread stack. */ | |
267 | + popq %r12 /* save return addr in %12 */ | |
268 | + movq %rsp, %rdi /* arg0 = pt_regs pointer */ | |
269 | + call sync_regs | |
270 | + movq %rax, %rsp /* switch stack */ | |
271 | + ENCODE_FRAME_POINTER | |
272 | + pushq %r12 | |
273 | + | |
274 | /* | |
275 | * We need to tell lockdep that IRQs are off. We can't do this until | |
276 | * we fix gsbase, and we should do it before enter_from_user_mode | |
277 | diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S | |
278 | index 1f76b66518ee..2270601b6218 100644 | |
279 | --- a/arch/x86/entry/entry_64_compat.S | |
280 | +++ b/arch/x86/entry/entry_64_compat.S | |
281 | @@ -305,8 +305,11 @@ ENTRY(entry_INT80_compat) | |
282 | */ | |
283 | movl %eax, %eax | |
284 | ||
285 | - /* Construct struct pt_regs on stack (iret frame is already on stack) */ | |
286 | pushq %rax /* pt_regs->orig_ax */ | |
287 | + | |
288 | + /* switch to thread stack expects orig_ax to be pushed */ | |
289 | + call switch_to_thread_stack | |
290 | + | |
291 | pushq %rdi /* pt_regs->di */ | |
292 | pushq %rsi /* pt_regs->si */ | |
293 | pushq %rdx /* pt_regs->dx */ | |
294 | -- | |
295 | 2.14.2 | |
296 |