1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Josh Poimboeuf <jpoimboe@redhat.com>
3 Date: Tue, 11 Jul 2017 10:33:44 -0500
4 Subject: [PATCH] x86/entry/64: Add unwind hint annotations
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
11 Add unwind hint annotations to entry_64.S. This will enable the ORC
12 unwinder to unwind through any location in the entry code including
13 syscalls, interrupts, and exceptions.
15 Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
16 Cc: Andy Lutomirski <luto@kernel.org>
17 Cc: Borislav Petkov <bp@alien8.de>
18 Cc: Brian Gerst <brgerst@gmail.com>
19 Cc: Denys Vlasenko <dvlasenk@redhat.com>
20 Cc: H. Peter Anvin <hpa@zytor.com>
21 Cc: Jiri Slaby <jslaby@suse.cz>
22 Cc: Linus Torvalds <torvalds@linux-foundation.org>
23 Cc: Mike Galbraith <efault@gmx.de>
24 Cc: Peter Zijlstra <peterz@infradead.org>
25 Cc: Thomas Gleixner <tglx@linutronix.de>
26 Cc: live-patching@vger.kernel.org
27 Link: http://lkml.kernel.org/r/b9f6d478aadf68ba57c739dcfac34ec0dc021c4c.1499786555.git.jpoimboe@redhat.com
28 Signed-off-by: Ingo Molnar <mingo@kernel.org>
29 (cherry picked from commit 8c1f75587a18ca032da8f6376d1ed882d7095289)
30 Signed-off-by: Andy Whitcroft <apw@canonical.com>
31 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
32 (cherry picked from commit a8448e6971c1e71b22c651131d14f8be76e6d399)
33 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
35 arch/x86/entry/Makefile | 1 -
36 arch/x86/entry/calling.h | 5 ++++
37 arch/x86/entry/entry_64.S | 71 ++++++++++++++++++++++++++++++++++++++++-------
38 3 files changed, 66 insertions(+), 11 deletions(-)
40 diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
41 index 9976fcecd17e..af28a8a24366 100644
42 --- a/arch/x86/entry/Makefile
43 +++ b/arch/x86/entry/Makefile
45 # Makefile for the x86 low level entry code
48 -OBJECT_FILES_NON_STANDARD_entry_$(BITS).o := y
49 OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y
51 CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,)
52 diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
53 index 05ed3d393da7..640aafebdc00 100644
54 --- a/arch/x86/entry/calling.h
55 +++ b/arch/x86/entry/calling.h
57 #include <linux/jump_label.h>
58 +#include <asm/unwind_hints.h>
62 @@ -112,6 +113,7 @@ For 32-bit we have the following conventions - kernel is built with
63 movq %rdx, 12*8+\offset(%rsp)
64 movq %rsi, 13*8+\offset(%rsp)
65 movq %rdi, 14*8+\offset(%rsp)
66 + UNWIND_HINT_REGS offset=\offset extra=0
68 .macro SAVE_C_REGS offset=0
69 SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
70 @@ -136,6 +138,7 @@ For 32-bit we have the following conventions - kernel is built with
71 movq %r12, 3*8+\offset(%rsp)
72 movq %rbp, 4*8+\offset(%rsp)
73 movq %rbx, 5*8+\offset(%rsp)
74 + UNWIND_HINT_REGS offset=\offset
77 .macro RESTORE_EXTRA_REGS offset=0
78 @@ -145,6 +148,7 @@ For 32-bit we have the following conventions - kernel is built with
79 movq 3*8+\offset(%rsp), %r12
80 movq 4*8+\offset(%rsp), %rbp
81 movq 5*8+\offset(%rsp), %rbx
82 + UNWIND_HINT_REGS offset=\offset extra=0
85 .macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1
86 @@ -167,6 +171,7 @@ For 32-bit we have the following conventions - kernel is built with
90 + UNWIND_HINT_IRET_REGS offset=16*8
93 RESTORE_C_REGS_HELPER 1,1,1,1,1
94 diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
95 index 184b70712545..64b233ab7cad 100644
96 --- a/arch/x86/entry/entry_64.S
97 +++ b/arch/x86/entry/entry_64.S
100 #include <asm/pgtable_types.h>
101 #include <asm/export.h>
102 +#include <asm/frame.h>
103 #include <linux/err.h>
108 #ifdef CONFIG_PARAVIRT
109 ENTRY(native_usergs_sysret64)
113 -ENDPROC(native_usergs_sysret64)
114 +END(native_usergs_sysret64)
115 #endif /* CONFIG_PARAVIRT */
117 .macro TRACE_IRQS_IRETQ
118 @@ -134,6 +136,7 @@ ENDPROC(native_usergs_sysret64)
121 ENTRY(entry_SYSCALL_64)
124 * Interrupts are off on entry.
125 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
126 @@ -169,6 +172,7 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
127 pushq %r10 /* pt_regs->r10 */
128 pushq %r11 /* pt_regs->r11 */
129 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */
130 + UNWIND_HINT_REGS extra=0
133 * If we need to do entry work or if we guess we'll need to do
134 @@ -223,6 +227,7 @@ entry_SYSCALL_64_fastpath:
135 movq EFLAGS(%rsp), %r11
136 RESTORE_C_REGS_EXCEPT_RCX_R11
142 @@ -316,6 +321,7 @@ syscall_return_via_sysret:
143 /* rcx and r11 are already restored (see code above) */
144 RESTORE_C_REGS_EXCEPT_RCX_R11
149 opportunistic_sysret_failed:
150 @@ -343,6 +349,7 @@ ENTRY(stub_ptregs_64)
151 DISABLE_INTERRUPTS(CLBR_ANY)
154 + UNWIND_HINT_REGS extra=0
155 jmp entry_SYSCALL64_slow_path
158 @@ -351,6 +358,7 @@ END(stub_ptregs_64)
160 .macro ptregs_stub func
163 leaq \func(%rip), %rax
166 @@ -367,6 +375,7 @@ END(ptregs_\func)
169 ENTRY(__switch_to_asm)
172 * Save callee-saved registers
173 * This must match the order in inactive_task_frame
174 @@ -406,6 +415,7 @@ END(__switch_to_asm)
175 * r12: kernel thread arg
180 call schedule_tail /* rdi: 'prev' task parameter */
182 @@ -413,6 +423,7 @@ ENTRY(ret_from_fork)
183 jnz 1f /* kernel threads are uncommon */
188 call syscall_return_slowpath /* returns with IRQs disabled */
189 TRACE_IRQS_ON /* user mode is traced as IRQS on */
190 @@ -440,10 +451,11 @@ END(ret_from_fork)
191 ENTRY(irq_entries_start)
192 vector=FIRST_EXTERNAL_VECTOR
193 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
194 + UNWIND_HINT_IRET_REGS
195 pushq $(~vector+0x80) /* Note: always in signed byte range */
201 END(irq_entries_start)
203 @@ -465,9 +477,14 @@ END(irq_entries_start)
205 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
207 -.macro ENTER_IRQ_STACK old_rsp
208 +.macro ENTER_IRQ_STACK regs=1 old_rsp
209 DEBUG_ENTRY_ASSERT_IRQS_OFF
213 + UNWIND_HINT_REGS base=\old_rsp
216 incl PER_CPU_VAR(irq_count)
217 jnz .Lirq_stack_push_old_rsp_\@
219 @@ -504,16 +521,24 @@ END(irq_entries_start)
221 .Lirq_stack_push_old_rsp_\@:
225 + UNWIND_HINT_REGS indirect=1
230 * Undoes ENTER_IRQ_STACK.
232 -.macro LEAVE_IRQ_STACK
233 +.macro LEAVE_IRQ_STACK regs=1
234 DEBUG_ENTRY_ASSERT_IRQS_OFF
235 /* We need to be off the IRQ stack before decrementing irq_count. */
243 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
244 * the irq stack but we're not on it.
245 @@ -624,6 +649,7 @@ restore_c_regs_and_iret:
249 + UNWIND_HINT_IRET_REGS
251 * Are we returning to a stack segment from the LDT? Note: in
252 * 64-bit mode SS:RSP on the exception stack is always valid.
253 @@ -696,6 +722,7 @@ native_irq_return_ldt:
254 orq PER_CPU_VAR(espfix_stack), %rax
257 + UNWIND_HINT_IRET_REGS offset=8
260 * At this point, we cannot write to the stack any more, but we can
261 @@ -717,6 +744,7 @@ END(common_interrupt)
263 .macro apicinterrupt3 num sym do_sym
265 + UNWIND_HINT_IRET_REGS
269 @@ -803,6 +831,8 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt
271 .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
273 + UNWIND_HINT_IRET_REGS offset=8
276 .if \shift_ist != -1 && \paranoid == 0
277 .error "using shift_ist requires paranoid=1"
278 @@ -826,6 +856,7 @@ ENTRY(\sym)
283 /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
286 @@ -923,6 +954,7 @@ idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
289 ENTRY(native_load_gs_index)
292 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
294 @@ -931,8 +963,9 @@ ENTRY(native_load_gs_index)
295 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
300 -END(native_load_gs_index)
301 +ENDPROC(native_load_gs_index)
302 EXPORT_SYMBOL(native_load_gs_index)
304 _ASM_EXTABLE(.Lgs_change, bad_gs)
305 @@ -955,12 +988,12 @@ bad_gs:
306 ENTRY(do_softirq_own_stack)
309 - ENTER_IRQ_STACK old_rsp=%r11
310 + ENTER_IRQ_STACK regs=0 old_rsp=%r11
313 + LEAVE_IRQ_STACK regs=0
316 -END(do_softirq_own_stack)
317 +ENDPROC(do_softirq_own_stack)
320 idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
321 @@ -984,7 +1017,9 @@ ENTRY(xen_do_hypervisor_callback) /* do_hypervisor_callback(struct *pt_regs) */
322 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
323 * see the correct pointer to the pt_regs
326 movq %rdi, %rsp /* we don't return, adjust the stack frame */
329 ENTER_IRQ_STACK old_rsp=%r10
330 call xen_evtchn_do_upcall
331 @@ -1010,6 +1045,7 @@ END(xen_do_hypervisor_callback)
332 * with its current contents: any discrepancy means we in category 1.
334 ENTRY(xen_failsafe_callback)
339 @@ -1029,11 +1065,13 @@ ENTRY(xen_failsafe_callback)
343 + UNWIND_HINT_IRET_REGS offset=8
344 jmp general_protection
345 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
349 + UNWIND_HINT_IRET_REGS
350 pushq $-1 /* orig_ax = -1 => not a system call */
351 ALLOC_PT_GPREGS_ON_STACK
353 @@ -1079,6 +1117,7 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vec
354 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
356 ENTRY(paranoid_entry)
361 @@ -1106,6 +1145,7 @@ END(paranoid_entry)
362 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
366 DISABLE_INTERRUPTS(CLBR_ANY)
368 testl %ebx, %ebx /* swapgs needed? */
369 @@ -1127,6 +1167,7 @@ END(paranoid_exit)
370 * Return: EBX=0: came from user mode; EBX=1: otherwise
377 @@ -1211,6 +1252,7 @@ END(error_entry)
378 * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
382 DISABLE_INTERRUPTS(CLBR_ANY)
385 @@ -1220,6 +1262,7 @@ END(error_exit)
387 /* Runs on exception stack */
389 + UNWIND_HINT_IRET_REGS
391 * Fix up the exception frame if we're on Xen.
392 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
393 @@ -1293,11 +1336,13 @@ ENTRY(nmi)
396 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
397 + UNWIND_HINT_IRET_REGS base=%rdx offset=8
398 pushq 5*8(%rdx) /* pt_regs->ss */
399 pushq 4*8(%rdx) /* pt_regs->rsp */
400 pushq 3*8(%rdx) /* pt_regs->flags */
401 pushq 2*8(%rdx) /* pt_regs->cs */
402 pushq 1*8(%rdx) /* pt_regs->rip */
403 + UNWIND_HINT_IRET_REGS
404 pushq $-1 /* pt_regs->orig_ax */
405 pushq %rdi /* pt_regs->di */
406 pushq %rsi /* pt_regs->si */
407 @@ -1314,6 +1359,7 @@ ENTRY(nmi)
408 pushq %r13 /* pt_regs->r13 */
409 pushq %r14 /* pt_regs->r14 */
410 pushq %r15 /* pt_regs->r15 */
415 @@ -1468,6 +1514,7 @@ first_nmi:
419 + UNWIND_HINT_IRET_REGS
421 /* Everything up to here is safe from nested NMIs */
423 @@ -1483,6 +1530,7 @@ first_nmi:
424 pushq $__KERNEL_CS /* CS */
426 INTERRUPT_RETURN /* continues at repeat_nmi below */
427 + UNWIND_HINT_IRET_REGS
431 @@ -1532,6 +1580,7 @@ end_repeat_nmi:
432 * exceptions might do.
437 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
439 @@ -1569,17 +1618,19 @@ nmi_restore:
448 ENTRY(rewind_stack_do_exit)
450 /* Prevent any naive code from trying to unwind to our caller. */
453 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
454 - leaq -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%rax), %rsp
455 + leaq -PTREGS_SIZE(%rax), %rsp
456 + UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
460 END(rewind_stack_do_exit)