]>
Commit | Line | Data |
---|---|---|
321d628a FG |
1 | From 548bfd4d539c4e13eb86236f8f09596e3663c38b Mon Sep 17 00:00:00 2001 |
2 | From: Andy Lutomirski <luto@kernel.org> | |
3 | Date: Mon, 4 Dec 2017 15:07:26 +0100 | |
e4cdf2a5 | 4 | Subject: [PATCH 156/241] x86/entry/64: Move the IST stacks into struct |
321d628a FG |
5 | cpu_entry_area |
6 | MIME-Version: 1.0 | |
7 | Content-Type: text/plain; charset=UTF-8 | |
8 | Content-Transfer-Encoding: 8bit | |
9 | ||
10 | CVE-2017-5754 | |
11 | ||
12 | The IST stacks are needed when an IST exception occurs and are accessed | |
13 | before any kernel code at all runs. Move them into struct cpu_entry_area. | |
14 | ||
15 | The IST stacks are unlike the rest of cpu_entry_area: they're used even for | |
16 | entries from kernel mode. This means that they should be set up before we | |
17 | load the final IDT. Move cpu_entry_area setup to trap_init() for the boot | |
18 | CPU and set it up for all possible CPUs at once in native_smp_prepare_cpus(). | |
19 | ||
20 | Signed-off-by: Andy Lutomirski <luto@kernel.org> | |
21 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
22 | Reviewed-by: Thomas Gleixner <tglx@linutronix.de> | |
23 | Reviewed-by: Borislav Petkov <bp@suse.de> | |
24 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
25 | Cc: Borislav Petkov <bp@alien8.de> | |
26 | Cc: Borislav Petkov <bpetkov@suse.de> | |
27 | Cc: Brian Gerst <brgerst@gmail.com> | |
28 | Cc: Dave Hansen <dave.hansen@intel.com> | |
29 | Cc: Dave Hansen <dave.hansen@linux.intel.com> | |
30 | Cc: David Laight <David.Laight@aculab.com> | |
31 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
32 | Cc: Eduardo Valentin <eduval@amazon.com> | |
33 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
34 | Cc: H. Peter Anvin <hpa@zytor.com> | |
35 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
36 | Cc: Juergen Gross <jgross@suse.com> | |
37 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
38 | Cc: Peter Zijlstra <peterz@infradead.org> | |
39 | Cc: Rik van Riel <riel@redhat.com> | |
40 | Cc: Will Deacon <will.deacon@arm.com> | |
41 | Cc: aliguori@amazon.com | |
42 | Cc: daniel.gruss@iaik.tugraz.at | |
43 | Cc: hughd@google.com | |
44 | Cc: keescook@google.com | |
45 | Link: https://lkml.kernel.org/r/20171204150606.480598743@linutronix.de | |
46 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
47 | (backported from commit 40e7f949e0d9a33968ebde5d67f7e3a47c97742a) | |
48 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
49 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
50 | (cherry picked from commit 88e7277709f2e7c023e66ff9ae158aeff4cf7c8f) | |
51 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
52 | --- | |
53 | arch/x86/include/asm/fixmap.h | 12 +++++++ | |
54 | arch/x86/kernel/cpu/common.c | 74 ++++++++++++++++++++++++------------------- | |
55 | arch/x86/kernel/traps.c | 3 ++ | |
56 | 3 files changed, 57 insertions(+), 32 deletions(-) | |
57 | ||
58 | diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h | |
59 | index 189d12d8afe0..953aed54cb5e 100644 | |
60 | --- a/arch/x86/include/asm/fixmap.h | |
61 | +++ b/arch/x86/include/asm/fixmap.h | |
62 | @@ -63,10 +63,22 @@ struct cpu_entry_area { | |
63 | struct tss_struct tss; | |
64 | ||
65 | char entry_trampoline[PAGE_SIZE]; | |
66 | + | |
67 | +#ifdef CONFIG_X86_64 | |
68 | + /* | |
69 | + * Exception stacks used for IST entries. | |
70 | + * | |
71 | + * In the future, this should have a separate slot for each stack | |
72 | + * with guard pages between them. | |
73 | + */ | |
74 | + char exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]; | |
75 | +#endif | |
76 | }; | |
77 | ||
78 | #define CPU_ENTRY_AREA_PAGES (sizeof(struct cpu_entry_area) / PAGE_SIZE) | |
79 | ||
80 | +extern void setup_cpu_entry_areas(void); | |
81 | + | |
82 | /* | |
83 | * Here we define all the compile-time 'special' virtual | |
84 | * addresses. The point is to have a constant address at | |
85 | diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c | |
86 | index c2b2ee73b8a1..f487766855d3 100644 | |
87 | --- a/arch/x86/kernel/cpu/common.c | |
88 | +++ b/arch/x86/kernel/cpu/common.c | |
89 | @@ -466,24 +466,36 @@ void load_percpu_segment(int cpu) | |
90 | load_stack_canary_segment(); | |
91 | } | |
92 | ||
93 | -static void set_percpu_fixmap_pages(int fixmap_index, void *ptr, | |
94 | - int pages, pgprot_t prot) | |
95 | -{ | |
96 | - int i; | |
97 | - | |
98 | - for (i = 0; i < pages; i++) { | |
99 | - __set_fixmap(fixmap_index - i, | |
100 | - per_cpu_ptr_to_phys(ptr + i * PAGE_SIZE), prot); | |
101 | - } | |
102 | -} | |
103 | - | |
104 | #ifdef CONFIG_X86_32 | |
105 | /* The 32-bit entry code needs to find cpu_entry_area. */ | |
106 | DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area); | |
107 | #endif | |
108 | ||
109 | +#ifdef CONFIG_X86_64 | |
110 | +/* | |
111 | + * Special IST stacks which the CPU switches to when it calls | |
112 | + * an IST-marked descriptor entry. Up to 7 stacks (hardware | |
113 | + * limit), all of them are 4K, except the debug stack which | |
114 | + * is 8K. | |
115 | + */ | |
116 | +static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = { | |
117 | + [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ, | |
118 | + [DEBUG_STACK - 1] = DEBUG_STKSZ | |
119 | +}; | |
120 | + | |
121 | +static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks | |
122 | + [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); | |
123 | +#endif | |
124 | + | |
125 | +static void __init | |
126 | +set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot) | |
127 | +{ | |
128 | + for ( ; pages; pages--, idx--, ptr += PAGE_SIZE) | |
129 | + __set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot); | |
130 | +} | |
131 | + | |
132 | /* Setup the fixmap mappings only once per-processor */ | |
133 | -static inline void setup_cpu_entry_area(int cpu) | |
134 | +static void __init setup_cpu_entry_area(int cpu) | |
135 | { | |
136 | #ifdef CONFIG_X86_64 | |
137 | extern char _entry_trampoline[]; | |
138 | @@ -532,15 +544,31 @@ static inline void setup_cpu_entry_area(int cpu) | |
139 | PAGE_KERNEL); | |
140 | ||
141 | #ifdef CONFIG_X86_32 | |
142 | - this_cpu_write(cpu_entry_area, get_cpu_entry_area(cpu)); | |
143 | + per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); | |
144 | #endif | |
145 | ||
146 | #ifdef CONFIG_X86_64 | |
147 | + BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); | |
148 | + BUILD_BUG_ON(sizeof(exception_stacks) != | |
149 | + sizeof(((struct cpu_entry_area *)0)->exception_stacks)); | |
150 | + set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks), | |
151 | + &per_cpu(exception_stacks, cpu), | |
152 | + sizeof(exception_stacks) / PAGE_SIZE, | |
153 | + PAGE_KERNEL); | |
154 | + | |
155 | __set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline), | |
156 | __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); | |
157 | #endif | |
158 | } | |
159 | ||
160 | +void __init setup_cpu_entry_areas(void) | |
161 | +{ | |
162 | + unsigned int cpu; | |
163 | + | |
164 | + for_each_possible_cpu(cpu) | |
165 | + setup_cpu_entry_area(cpu); | |
166 | +} | |
167 | + | |
168 | /* Load the original GDT from the per-cpu structure */ | |
169 | void load_direct_gdt(int cpu) | |
170 | { | |
171 | @@ -1386,20 +1414,6 @@ DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; | |
172 | DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; | |
173 | EXPORT_PER_CPU_SYMBOL(__preempt_count); | |
174 | ||
175 | -/* | |
176 | - * Special IST stacks which the CPU switches to when it calls | |
177 | - * an IST-marked descriptor entry. Up to 7 stacks (hardware | |
178 | - * limit), all of them are 4K, except the debug stack which | |
179 | - * is 8K. | |
180 | - */ | |
181 | -static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = { | |
182 | - [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ, | |
183 | - [DEBUG_STACK - 1] = DEBUG_STKSZ | |
184 | -}; | |
185 | - | |
186 | -static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks | |
187 | - [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); | |
188 | - | |
189 | /* May not be marked __init: used by software suspend */ | |
190 | void syscall_init(void) | |
191 | { | |
192 | @@ -1608,7 +1622,7 @@ void cpu_init(void) | |
193 | * set up and load the per-CPU TSS | |
194 | */ | |
195 | if (!oist->ist[0]) { | |
196 | - char *estacks = per_cpu(exception_stacks, cpu); | |
197 | + char *estacks = get_cpu_entry_area(cpu)->exception_stacks; | |
198 | ||
199 | for (v = 0; v < N_EXCEPTION_STACKS; v++) { | |
200 | estacks += exception_stack_sizes[v]; | |
201 | @@ -1633,8 +1647,6 @@ void cpu_init(void) | |
202 | BUG_ON(me->mm); | |
203 | enter_lazy_tlb(&init_mm, me); | |
204 | ||
205 | - setup_cpu_entry_area(cpu); | |
206 | - | |
207 | /* | |
208 | * Initialize the TSS. sp0 points to the entry trampoline stack | |
209 | * regardless of what task is running. | |
210 | @@ -1693,8 +1705,6 @@ void cpu_init(void) | |
211 | BUG_ON(curr->mm); | |
212 | enter_lazy_tlb(&init_mm, curr); | |
213 | ||
214 | - setup_cpu_entry_area(cpu); | |
215 | - | |
216 | /* | |
217 | * Initialize the TSS. Don't bother initializing sp0, as the initial | |
218 | * task never enters user mode. | |
219 | diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c | |
220 | index d9debdafe7a6..fd4d47e8672e 100644 | |
221 | --- a/arch/x86/kernel/traps.c | |
222 | +++ b/arch/x86/kernel/traps.c | |
223 | @@ -992,6 +992,9 @@ void __init trap_init(void) | |
224 | { | |
225 | int i; | |
226 | ||
227 | + /* Init cpu_entry_area before IST entries are set up */ | |
228 | + setup_cpu_entry_areas(); | |
229 | + | |
230 | #ifdef CONFIG_EISA | |
231 | void __iomem *p = early_ioremap(0x0FFFD9, 4); | |
232 | ||
233 | -- | |
234 | 2.14.2 | |
235 |