]>
Commit | Line | Data |
---|---|---|
59d5af67 | 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
321d628a FG |
2 | From: Peter Zijlstra <peterz@infradead.org> |
3 | Date: Mon, 4 Dec 2017 15:07:59 +0100 | |
59d5af67 | 4 | Subject: [PATCH] x86/mm: Use/Fix PCID to optimize user/kernel switches |
321d628a FG |
5 | MIME-Version: 1.0 |
6 | Content-Type: text/plain; charset=UTF-8 | |
7 | Content-Transfer-Encoding: 8bit | |
8 | ||
9 | CVE-2017-5754 | |
10 | ||
11 | We can use PCID to retain the TLBs across CR3 switches; including those now | |
12 | part of the user/kernel switch. This increases performance of kernel | |
13 | entry/exit at the cost of more expensive/complicated TLB flushing. | |
14 | ||
15 | Now that we have two address spaces, one for kernel and one for user space, | |
16 | we need two PCIDs per mm. We use the top PCID bit to indicate a user PCID | |
17 | (just like we use the PFN LSB for the PGD). Since we do TLB invalidation | |
18 | from kernel space, the existing code will only invalidate the kernel PCID, | |
19 | we augment that by marking the corresponding user PCID invalid, and upon | |
20 | switching back to userspace, use a flushing CR3 write for the switch. | |
21 | ||
22 | In order to access the user_pcid_flush_mask we use PER_CPU storage, which | |
23 | means the previously established SWAPGS vs CR3 ordering is now mandatory | |
24 | and required. | |
25 | ||
26 | Having to do this memory access does require additional registers, most | |
27 | sites have a functioning stack and we can spill one (RAX), sites without | |
28 | functional stack need to otherwise provide the second scratch register. | |
29 | ||
30 | Note: PCID is generally available on Intel Sandybridge and later CPUs. | |
31 | Note: Up until this point TLB flushing was broken in this series. | |
32 | ||
33 | Based-on-code-from: Dave Hansen <dave.hansen@linux.intel.com> | |
34 | Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> | |
35 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
36 | Cc: Andy Lutomirski <luto@kernel.org> | |
37 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
38 | Cc: Borislav Petkov <bp@alien8.de> | |
39 | Cc: Brian Gerst <brgerst@gmail.com> | |
40 | Cc: Dave Hansen <dave.hansen@linux.intel.com> | |
41 | Cc: David Laight <David.Laight@aculab.com> | |
42 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
43 | Cc: Eduardo Valentin <eduval@amazon.com> | |
44 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
45 | Cc: H. Peter Anvin <hpa@zytor.com> | |
46 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
47 | Cc: Juergen Gross <jgross@suse.com> | |
48 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
49 | Cc: Peter Zijlstra <peterz@infradead.org> | |
50 | Cc: Will Deacon <will.deacon@arm.com> | |
51 | Cc: aliguori@amazon.com | |
52 | Cc: daniel.gruss@iaik.tugraz.at | |
53 | Cc: hughd@google.com | |
54 | Cc: keescook@google.com | |
55 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
56 | (backported from commit 6fd166aae78c0ab738d49bda653cbd9e3b1491cf) | |
57 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
58 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
59 | (cherry picked from commit ac7471365d49c0a91d4b63453eb848cc19f17589) | |
60 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
61 | --- | |
62 | arch/x86/entry/calling.h | 72 ++++++++++++++++++----- | |
63 | arch/x86/include/asm/processor-flags.h | 5 ++ | |
64 | arch/x86/include/asm/tlbflush.h | 91 +++++++++++++++++++++++++---- | |
65 | arch/x86/include/uapi/asm/processor-flags.h | 7 ++- | |
66 | arch/x86/kernel/asm-offsets.c | 4 ++ | |
67 | arch/x86/mm/init.c | 2 +- | |
68 | arch/x86/mm/tlb.c | 1 + | |
69 | arch/x86/entry/entry_64.S | 9 +-- | |
70 | arch/x86/entry/entry_64_compat.S | 4 +- | |
71 | 9 files changed, 162 insertions(+), 33 deletions(-) | |
72 | ||
73 | diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h | |
74 | index bb56f5346ae8..ce5fb309926d 100644 | |
75 | --- a/arch/x86/entry/calling.h | |
76 | +++ b/arch/x86/entry/calling.h | |
77 | @@ -2,6 +2,9 @@ | |
78 | #include <asm/unwind_hints.h> | |
79 | #include <asm/cpufeatures.h> | |
80 | #include <asm/page_types.h> | |
81 | +#include <asm/percpu.h> | |
82 | +#include <asm/asm-offsets.h> | |
83 | +#include <asm/processor-flags.h> | |
84 | ||
85 | /* | |
86 | ||
87 | @@ -190,17 +193,21 @@ For 32-bit we have the following conventions - kernel is built with | |
88 | ||
89 | #ifdef CONFIG_PAGE_TABLE_ISOLATION | |
90 | ||
91 | -/* PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two halves: */ | |
92 | -#define PTI_SWITCH_MASK (1<<PAGE_SHIFT) | |
93 | +/* | |
94 | + * PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two | |
95 | + * halves: | |
96 | + */ | |
97 | +#define PTI_SWITCH_PGTABLES_MASK (1<<PAGE_SHIFT) | |
98 | +#define PTI_SWITCH_MASK (PTI_SWITCH_PGTABLES_MASK|(1<<X86_CR3_PTI_SWITCH_BIT)) | |
99 | ||
100 | -.macro ADJUST_KERNEL_CR3 reg:req | |
101 | - /* Clear "PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ | |
102 | - andq $(~PTI_SWITCH_MASK), \reg | |
103 | +.macro SET_NOFLUSH_BIT reg:req | |
104 | + bts $X86_CR3_PCID_NOFLUSH_BIT, \reg | |
105 | .endm | |
106 | ||
107 | -.macro ADJUST_USER_CR3 reg:req | |
108 | - /* Move CR3 up a page to the user page tables: */ | |
109 | - orq $(PTI_SWITCH_MASK), \reg | |
110 | +.macro ADJUST_KERNEL_CR3 reg:req | |
111 | + ALTERNATIVE "", "SET_NOFLUSH_BIT \reg", X86_FEATURE_PCID | |
112 | + /* Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 at kernel pagetables: */ | |
113 | + andq $(~PTI_SWITCH_MASK), \reg | |
114 | .endm | |
115 | ||
116 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req | |
117 | @@ -211,21 +218,58 @@ For 32-bit we have the following conventions - kernel is built with | |
118 | .Lend_\@: | |
119 | .endm | |
120 | ||
121 | -.macro SWITCH_TO_USER_CR3 scratch_reg:req | |
122 | +#define THIS_CPU_user_pcid_flush_mask \ | |
123 | + PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask | |
124 | + | |
125 | +.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req | |
126 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI | |
127 | mov %cr3, \scratch_reg | |
128 | - ADJUST_USER_CR3 \scratch_reg | |
129 | + | |
130 | + ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID | |
131 | + | |
132 | + /* | |
133 | + * Test if the ASID needs a flush. | |
134 | + */ | |
135 | + movq \scratch_reg, \scratch_reg2 | |
136 | + andq $(0x7FF), \scratch_reg /* mask ASID */ | |
137 | + bt \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
138 | + jnc .Lnoflush_\@ | |
139 | + | |
140 | + /* Flush needed, clear the bit */ | |
141 | + btr \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
142 | + movq \scratch_reg2, \scratch_reg | |
143 | + jmp .Lwrcr3_\@ | |
144 | + | |
145 | +.Lnoflush_\@: | |
146 | + movq \scratch_reg2, \scratch_reg | |
147 | + SET_NOFLUSH_BIT \scratch_reg | |
148 | + | |
149 | +.Lwrcr3_\@: | |
150 | + /* Flip the PGD and ASID to the user version */ | |
151 | + orq $(PTI_SWITCH_MASK), \scratch_reg | |
152 | mov \scratch_reg, %cr3 | |
153 | .Lend_\@: | |
154 | .endm | |
155 | ||
156 | +.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req | |
157 | + pushq %rax | |
158 | + SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax | |
159 | + popq %rax | |
160 | +.endm | |
161 | + | |
162 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req | |
163 | ALTERNATIVE "jmp .Ldone_\@", "", X86_FEATURE_PTI | |
164 | movq %cr3, \scratch_reg | |
165 | movq \scratch_reg, \save_reg | |
166 | /* | |
167 | - * Is the switch bit zero? This means the address is | |
168 | - * up in real PAGE_TABLE_ISOLATION patches in a moment. | |
169 | + * Is the "switch mask" all zero? That means that both of | |
170 | + * these are zero: | |
171 | + * | |
172 | + * 1. The user/kernel PCID bit, and | |
173 | + * 2. The user/kernel "bit" that points CR3 to the | |
174 | + * bottom half of the 8k PGD | |
175 | + * | |
176 | + * That indicates a kernel CR3 value, not a user CR3. | |
177 | */ | |
178 | testq $(PTI_SWITCH_MASK), \scratch_reg | |
179 | jz .Ldone_\@ | |
180 | @@ -250,7 +294,9 @@ For 32-bit we have the following conventions - kernel is built with | |
181 | ||
182 | .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req | |
183 | .endm | |
184 | -.macro SWITCH_TO_USER_CR3 scratch_reg:req | |
185 | +.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req | |
186 | +.endm | |
187 | +.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req | |
188 | .endm | |
189 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req | |
190 | .endm | |
191 | diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h | |
192 | index 791b60199aa4..fb9708d13761 100644 | |
193 | --- a/arch/x86/include/asm/processor-flags.h | |
194 | +++ b/arch/x86/include/asm/processor-flags.h | |
195 | @@ -36,6 +36,11 @@ | |
196 | #define CR3_ADDR_MASK 0x7FFFFFFFFFFFF000ull | |
197 | #define CR3_PCID_MASK 0xFFFull | |
198 | #define CR3_NOFLUSH (1UL << 63) | |
199 | + | |
200 | +#ifdef CONFIG_PAGE_TABLE_ISOLATION | |
201 | +# define X86_CR3_PTI_SWITCH_BIT 11 | |
202 | +#endif | |
203 | + | |
204 | #else | |
205 | /* | |
206 | * CR3_ADDR_MASK needs at least bits 31:5 set on PAE systems, and we save | |
207 | diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h | |
208 | index 3769ce182eac..2b7b32c243f1 100644 | |
209 | --- a/arch/x86/include/asm/tlbflush.h | |
210 | +++ b/arch/x86/include/asm/tlbflush.h | |
211 | @@ -9,6 +9,8 @@ | |
212 | #include <asm/special_insns.h> | |
213 | #include <asm/smp.h> | |
214 | #include <asm/invpcid.h> | |
215 | +#include <asm/pti.h> | |
216 | +#include <asm/processor-flags.h> | |
217 | ||
218 | static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
219 | { | |
220 | @@ -23,24 +25,54 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) | |
221 | ||
222 | /* There are 12 bits of space for ASIDS in CR3 */ | |
223 | #define CR3_HW_ASID_BITS 12 | |
224 | + | |
225 | /* | |
226 | * When enabled, PAGE_TABLE_ISOLATION consumes a single bit for | |
227 | * user/kernel switches | |
228 | */ | |
229 | -#define PTI_CONSUMED_ASID_BITS 0 | |
230 | +#ifdef CONFIG_PAGE_TABLE_ISOLATION | |
231 | +# define PTI_CONSUMED_PCID_BITS 1 | |
232 | +#else | |
233 | +# define PTI_CONSUMED_PCID_BITS 0 | |
234 | +#endif | |
235 | + | |
236 | +#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS) | |
237 | ||
238 | -#define CR3_AVAIL_ASID_BITS (CR3_HW_ASID_BITS - PTI_CONSUMED_ASID_BITS) | |
239 | /* | |
240 | * ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account | |
241 | * for them being zero-based. Another -1 is because ASID 0 is reserved for | |
242 | * use by non-PCID-aware users. | |
243 | */ | |
244 | -#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_ASID_BITS) - 2) | |
245 | +#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2) | |
246 | + | |
247 | +/* | |
248 | + * 6 because 6 should be plenty and struct tlb_state will fit in two cache | |
249 | + * lines. | |
250 | + */ | |
251 | +#define TLB_NR_DYN_ASIDS 6 | |
252 | ||
253 | static inline u16 kern_pcid(u16 asid) | |
254 | { | |
255 | VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); | |
256 | + | |
257 | +#ifdef CONFIG_PAGE_TABLE_ISOLATION | |
258 | + /* | |
259 | + * Make sure that the dynamic ASID space does not confict with the | |
260 | + * bit we are using to switch between user and kernel ASIDs. | |
261 | + */ | |
262 | + BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_SWITCH_BIT)); | |
263 | + | |
264 | /* | |
265 | + * The ASID being passed in here should have respected the | |
266 | + * MAX_ASID_AVAILABLE and thus never have the switch bit set. | |
267 | + */ | |
268 | + VM_WARN_ON_ONCE(asid & (1 << X86_CR3_PTI_SWITCH_BIT)); | |
269 | +#endif | |
270 | + /* | |
271 | + * The dynamically-assigned ASIDs that get passed in are small | |
272 | + * (<TLB_NR_DYN_ASIDS). They never have the high switch bit set, | |
273 | + * so do not bother to clear it. | |
274 | + * | |
275 | * If PCID is on, ASID-aware code paths put the ASID+1 into the | |
276 | * PCID bits. This serves two purposes. It prevents a nasty | |
277 | * situation in which PCID-unaware code saves CR3, loads some other | |
278 | @@ -85,12 +117,6 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) | |
279 | */ | |
280 | DECLARE_STATIC_KEY_TRUE(tlb_use_lazy_mode); | |
281 | ||
282 | -/* | |
283 | - * 6 because 6 should be plenty and struct tlb_state will fit in | |
284 | - * two cache lines. | |
285 | - */ | |
286 | -#define TLB_NR_DYN_ASIDS 6 | |
287 | - | |
288 | struct tlb_context { | |
289 | u64 ctx_id; | |
290 | u64 tlb_gen; | |
291 | @@ -135,6 +161,13 @@ struct tlb_state { | |
292 | */ | |
293 | bool invalidate_other; | |
294 | ||
295 | + /* | |
296 | + * Mask that contains TLB_NR_DYN_ASIDS+1 bits to indicate | |
297 | + * the corresponding user PCID needs a flush next time we | |
298 | + * switch to it; see SWITCH_TO_USER_CR3. | |
299 | + */ | |
300 | + unsigned short user_pcid_flush_mask; | |
301 | + | |
302 | /* | |
303 | * Access to this CR4 shadow and to H/W CR4 is protected by | |
304 | * disabling interrupts when modifying either one. | |
305 | @@ -238,15 +271,42 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask) | |
306 | } | |
307 | ||
308 | ||
309 | +/* | |
310 | + * Given an ASID, flush the corresponding user ASID. We can delay this | |
311 | + * until the next time we switch to it. | |
312 | + * | |
313 | + * See SWITCH_TO_USER_CR3. | |
314 | + */ | |
315 | +static inline void invalidate_user_asid(u16 asid) | |
316 | +{ | |
317 | + /* There is no user ASID if address space separation is off */ | |
318 | + if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) | |
319 | + return; | |
320 | + | |
321 | + /* | |
322 | + * We only have a single ASID if PCID is off and the CR3 | |
323 | + * write will have flushed it. | |
324 | + */ | |
325 | + if (!cpu_feature_enabled(X86_FEATURE_PCID)) | |
326 | + return; | |
327 | + | |
328 | + if (!static_cpu_has(X86_FEATURE_PTI)) | |
329 | + return; | |
330 | + | |
331 | + __set_bit(kern_pcid(asid), | |
332 | + (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask)); | |
333 | +} | |
334 | + | |
335 | /* | |
336 | * flush the entire current user mapping | |
337 | */ | |
338 | static inline void __native_flush_tlb(void) | |
339 | { | |
340 | + invalidate_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid)); | |
341 | /* | |
342 | - * If current->mm == NULL then we borrow a mm which may change during a | |
343 | - * task switch and therefore we must not be preempted while we write CR3 | |
344 | - * back: | |
345 | + * If current->mm == NULL then we borrow a mm which may change | |
346 | + * during a task switch and therefore we must not be preempted | |
347 | + * while we write CR3 back: | |
348 | */ | |
349 | preempt_disable(); | |
350 | native_write_cr3(__native_read_cr3()); | |
351 | @@ -290,7 +350,14 @@ static inline void __native_flush_tlb_global(void) | |
352 | */ | |
353 | static inline void __native_flush_tlb_single(unsigned long addr) | |
354 | { | |
355 | + u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); | |
356 | + | |
357 | asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); | |
358 | + | |
359 | + if (!static_cpu_has(X86_FEATURE_PTI)) | |
360 | + return; | |
361 | + | |
362 | + invalidate_user_asid(loaded_mm_asid); | |
363 | } | |
364 | ||
365 | /* | |
366 | diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h | |
367 | index 39946d0a1d41..69077da3dbf1 100644 | |
368 | --- a/arch/x86/include/uapi/asm/processor-flags.h | |
369 | +++ b/arch/x86/include/uapi/asm/processor-flags.h | |
370 | @@ -77,7 +77,12 @@ | |
371 | #define X86_CR3_PWT _BITUL(X86_CR3_PWT_BIT) | |
372 | #define X86_CR3_PCD_BIT 4 /* Page Cache Disable */ | |
373 | #define X86_CR3_PCD _BITUL(X86_CR3_PCD_BIT) | |
374 | -#define X86_CR3_PCID_MASK _AC(0x00000fff,UL) /* PCID Mask */ | |
375 | + | |
376 | +#define X86_CR3_PCID_BITS 12 | |
377 | +#define X86_CR3_PCID_MASK (_AC((1UL << X86_CR3_PCID_BITS) - 1, UL)) | |
378 | + | |
379 | +#define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */ | |
380 | +#define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT) | |
381 | ||
382 | /* | |
383 | * Intel CPU features in CR4 | |
384 | diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c | |
385 | index 25b4832e9c28..87c3bafcef2c 100644 | |
386 | --- a/arch/x86/kernel/asm-offsets.c | |
387 | +++ b/arch/x86/kernel/asm-offsets.c | |
388 | @@ -16,6 +16,7 @@ | |
389 | #include <asm/sigframe.h> | |
390 | #include <asm/bootparam.h> | |
391 | #include <asm/suspend.h> | |
392 | +#include <asm/tlbflush.h> | |
393 | ||
394 | #ifdef CONFIG_XEN | |
395 | #include <xen/interface/xen.h> | |
396 | @@ -93,6 +94,9 @@ void common(void) { | |
397 | BLANK(); | |
398 | DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); | |
399 | ||
400 | + /* TLB state for the entry code */ | |
401 | + OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask); | |
402 | + | |
403 | /* Layout info for cpu_entry_area */ | |
404 | OFFSET(CPU_ENTRY_AREA_tss, cpu_entry_area, tss); | |
405 | OFFSET(CPU_ENTRY_AREA_entry_trampoline, cpu_entry_area, entry_trampoline); | |
406 | diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c | |
407 | index af75069fb116..caeb8a7bf0a4 100644 | |
408 | --- a/arch/x86/mm/init.c | |
409 | +++ b/arch/x86/mm/init.c | |
410 | @@ -855,7 +855,7 @@ void __init zone_sizes_init(void) | |
411 | free_area_init_nodes(max_zone_pfns); | |
412 | } | |
413 | ||
414 | -DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { | |
415 | +__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { | |
416 | .loaded_mm = &init_mm, | |
417 | .next_asid = 1, | |
418 | .cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */ | |
419 | diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c | |
420 | index 353f2f4e1d96..06f3854d0a4f 100644 | |
421 | --- a/arch/x86/mm/tlb.c | |
422 | +++ b/arch/x86/mm/tlb.c | |
423 | @@ -106,6 +106,7 @@ static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) | |
424 | unsigned long new_mm_cr3; | |
425 | ||
426 | if (need_flush) { | |
427 | + invalidate_user_asid(new_asid); | |
428 | new_mm_cr3 = build_cr3(pgdir, new_asid); | |
429 | } else { | |
430 | new_mm_cr3 = build_cr3_noflush(pgdir, new_asid); | |
431 | diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S | |
432 | index 292ccc6ec48d..fb43f14ed299 100644 | |
433 | --- a/arch/x86/entry/entry_64.S | |
434 | +++ b/arch/x86/entry/entry_64.S | |
435 | @@ -22,7 +22,6 @@ | |
436 | #include <asm/segment.h> | |
437 | #include <asm/cache.h> | |
438 | #include <asm/errno.h> | |
439 | -#include "calling.h" | |
440 | #include <asm/asm-offsets.h> | |
441 | #include <asm/msr.h> | |
442 | #include <asm/unistd.h> | |
443 | @@ -39,6 +38,8 @@ | |
444 | #include <asm/frame.h> | |
445 | #include <linux/err.h> | |
446 | ||
447 | +#include "calling.h" | |
448 | + | |
449 | .code64 | |
450 | .section .entry.text, "ax" | |
451 | ||
452 | @@ -405,7 +406,7 @@ syscall_return_via_sysret: | |
453 | * We are on the trampoline stack. All regs except RDI are live. | |
454 | * We can do future final exit work right here. | |
455 | */ | |
456 | - SWITCH_TO_USER_CR3 scratch_reg=%rdi | |
457 | + SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi | |
458 | ||
459 | popq %rdi | |
460 | popq %rsp | |
461 | @@ -743,7 +744,7 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode) | |
462 | * We can do future final exit work right here. | |
463 | */ | |
464 | ||
465 | - SWITCH_TO_USER_CR3 scratch_reg=%rdi | |
466 | + SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi | |
467 | ||
468 | /* Restore RDI. */ | |
469 | popq %rdi | |
470 | @@ -856,7 +857,7 @@ native_irq_return_ldt: | |
471 | */ | |
472 | orq PER_CPU_VAR(espfix_stack), %rax | |
473 | ||
474 | - SWITCH_TO_USER_CR3 scratch_reg=%rdi /* to user CR3 */ | |
475 | + SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi | |
476 | SWAPGS /* to user GS */ | |
477 | popq %rdi /* Restore user RDI */ | |
478 | ||
479 | diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S | |
480 | index 43f856aeee67..973527e34887 100644 | |
481 | --- a/arch/x86/entry/entry_64_compat.S | |
482 | +++ b/arch/x86/entry/entry_64_compat.S | |
483 | @@ -274,9 +274,9 @@ sysret32_from_system_call: | |
484 | * switch until after after the last reference to the process | |
485 | * stack. | |
486 | * | |
487 | - * %r8 is zeroed before the sysret, thus safe to clobber. | |
488 | + * %r8/%r9 are zeroed before the sysret, thus safe to clobber. | |
489 | */ | |
490 | - SWITCH_TO_USER_CR3 scratch_reg=%r8 | |
491 | + SWITCH_TO_USER_CR3_NOSTACK scratch_reg=%r8 scratch_reg2=%r9 | |
492 | ||
493 | xorq %r8, %r8 | |
494 | xorq %r9, %r9 | |
495 | -- | |
496 | 2.14.2 | |
497 |