1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Andy Lutomirski <luto@kernel.org>
3 Date: Mon, 4 Dec 2017 15:07:12 +0100
4 Subject: [PATCH] x86/entry/64: Allocate and enable the SYSENTER stack
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
11 This will simplify future changes that want scratch variables early in
12 the SYSENTER handler -- they'll be able to spill registers to the
13 stack. It also lets us get rid of a SWAPGS_UNSAFE_STACK user.
15 This does not depend on CONFIG_IA32_EMULATION=y because we'll want the
16 stack space even without IA32 emulation.
18 As far as I can tell, the reason that this wasn't done from day 1 is
19 that we use IST for #DB and #BP, which is IMO rather nasty and causes
20 a lot more problems than it solves. But, since #DB uses IST, we don't
21 actually need a real stack for SYSENTER (because SYSENTER with TF set
22 will invoke #DB on the IST stack rather than the SYSENTER stack).
24 I want to remove IST usage from these vectors some day, and this patch
25 is a prerequisite for that as well.
27 Signed-off-by: Andy Lutomirski <luto@kernel.org>
28 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
29 Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
30 Reviewed-by: Borislav Petkov <bp@suse.de>
31 Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
32 Cc: Borislav Petkov <bp@alien8.de>
33 Cc: Borislav Petkov <bpetkov@suse.de>
34 Cc: Brian Gerst <brgerst@gmail.com>
35 Cc: Dave Hansen <dave.hansen@intel.com>
36 Cc: Dave Hansen <dave.hansen@linux.intel.com>
37 Cc: David Laight <David.Laight@aculab.com>
38 Cc: Denys Vlasenko <dvlasenk@redhat.com>
39 Cc: Eduardo Valentin <eduval@amazon.com>
40 Cc: Greg KH <gregkh@linuxfoundation.org>
41 Cc: H. Peter Anvin <hpa@zytor.com>
42 Cc: Josh Poimboeuf <jpoimboe@redhat.com>
43 Cc: Juergen Gross <jgross@suse.com>
44 Cc: Linus Torvalds <torvalds@linux-foundation.org>
45 Cc: Peter Zijlstra <peterz@infradead.org>
46 Cc: Rik van Riel <riel@redhat.com>
47 Cc: Will Deacon <will.deacon@arm.com>
48 Cc: aliguori@amazon.com
49 Cc: daniel.gruss@iaik.tugraz.at
51 Cc: keescook@google.com
52 Link: https://lkml.kernel.org/r/20171204150605.312726423@linutronix.de
53 Signed-off-by: Ingo Molnar <mingo@kernel.org>
54 (cherry picked from commit 1a79797b58cddfa948420a7553241c79c013e3ca)
55 Signed-off-by: Andy Whitcroft <apw@canonical.com>
56 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
57 (cherry picked from commit 8e621515fa8d1649b031f34b9d498dcd865db1c3)
58 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
60 arch/x86/include/asm/processor.h | 3 ---
61 arch/x86/kernel/asm-offsets.c | 5 +++++
62 arch/x86/kernel/asm-offsets_32.c | 5 -----
63 arch/x86/kernel/cpu/common.c | 4 +++-
64 arch/x86/kernel/process.c | 2 --
65 arch/x86/kernel/traps.c | 3 +--
66 arch/x86/entry/entry_64_compat.S | 2 +-
67 7 files changed, 10 insertions(+), 14 deletions(-)
69 diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
70 index 79739e5f939a..5225917f9760 100644
71 --- a/arch/x86/include/asm/processor.h
72 +++ b/arch/x86/include/asm/processor.h
73 @@ -333,14 +333,11 @@ struct tss_struct {
75 unsigned long io_bitmap[IO_BITMAP_LONGS + 1];
79 * Space for the temporary SYSENTER stack.
81 unsigned long SYSENTER_stack_canary;
82 unsigned long SYSENTER_stack[64];
85 } ____cacheline_aligned;
87 DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
88 diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
89 index de827d6ac8c2..031bd35bd911 100644
90 --- a/arch/x86/kernel/asm-offsets.c
91 +++ b/arch/x86/kernel/asm-offsets.c
92 @@ -92,4 +92,9 @@ void common(void) {
95 DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
97 + /* Offset from cpu_tss to SYSENTER_stack */
98 + OFFSET(CPU_TSS_SYSENTER_stack, tss_struct, SYSENTER_stack);
99 + /* Size of SYSENTER_stack */
100 + DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
102 diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
103 index 880aa093268d..d09b161a3bd0 100644
104 --- a/arch/x86/kernel/asm-offsets_32.c
105 +++ b/arch/x86/kernel/asm-offsets_32.c
106 @@ -52,11 +52,6 @@ void foo(void)
107 DEFINE(TSS_sysenter_sp0, offsetof(struct tss_struct, x86_tss.sp0) -
108 offsetofend(struct tss_struct, SYSENTER_stack));
110 - /* Offset from cpu_tss to SYSENTER_stack */
111 - OFFSET(CPU_TSS_SYSENTER_stack, tss_struct, SYSENTER_stack);
112 - /* Size of SYSENTER_stack */
113 - DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
115 #ifdef CONFIG_CC_STACKPROTECTOR
117 OFFSET(stack_canary_offset, stack_canary, canary);
118 diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
119 index 121fe3570d6f..aa97e4cd3a33 100644
120 --- a/arch/x86/kernel/cpu/common.c
121 +++ b/arch/x86/kernel/cpu/common.c
122 @@ -1362,7 +1362,9 @@ void syscall_init(void)
123 * AMD doesn't allow SYSENTER in long mode (either 32- or 64-bit).
125 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
126 - wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
127 + wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
128 + (unsigned long)this_cpu_ptr(&cpu_tss) +
129 + offsetofend(struct tss_struct, SYSENTER_stack));
130 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
132 wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
133 diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
134 index ccf3a4f4ef68..aa86e810fb54 100644
135 --- a/arch/x86/kernel/process.c
136 +++ b/arch/x86/kernel/process.c
137 @@ -70,9 +70,7 @@ __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
139 .io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 },
141 -#ifdef CONFIG_X86_32
142 .SYSENTER_stack_canary = STACK_END_MAGIC,
145 EXPORT_PER_CPU_SYMBOL(cpu_tss);
147 diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
148 index 3a46cab2696e..7b1d0df624cf 100644
149 --- a/arch/x86/kernel/traps.c
150 +++ b/arch/x86/kernel/traps.c
151 @@ -806,14 +806,13 @@ dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
152 debug_stack_usage_dec();
155 -#if defined(CONFIG_X86_32)
157 * This is the most likely code path that involves non-trivial use
158 * of the SYSENTER stack. Check that we haven't overrun it.
160 WARN(this_cpu_read(cpu_tss.SYSENTER_stack_canary) != STACK_END_MAGIC,
161 "Overran or corrupted SYSENTER stack\n");
166 NOKPROBE_SYMBOL(do_debug);
167 diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
168 index be745b7a3e3e..1f76b66518ee 100644
169 --- a/arch/x86/entry/entry_64_compat.S
170 +++ b/arch/x86/entry/entry_64_compat.S
173 ENTRY(entry_SYSENTER_compat)
174 /* Interrupts are off on entry. */
175 - SWAPGS_UNSAFE_STACK
177 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp