]>
Commit | Line | Data |
---|---|---|
321d628a FG |
1 | From eb4a670cb54266bfab7bb4d9fd9e5da7b296ecdf Mon Sep 17 00:00:00 2001 |
2 | From: Peter Zijlstra <peterz@infradead.org> | |
3 | Date: Mon, 4 Dec 2017 15:08:00 +0100 | |
4 | Subject: [PATCH 210/231] x86/mm: Optimize RESTORE_CR3 | |
5 | MIME-Version: 1.0 | |
6 | Content-Type: text/plain; charset=UTF-8 | |
7 | Content-Transfer-Encoding: 8bit | |
8 | ||
9 | CVE-2017-5754 | |
10 | ||
11 | Most NMI/paranoid exceptions will not in fact change pagetables and would | |
12 | thus not require TLB flushing, however RESTORE_CR3 uses flushing CR3 | |
13 | writes. | |
14 | ||
15 | Restores to kernel PCIDs can be NOFLUSH, because we explicitly flush the | |
16 | kernel mappings and now that we track which user PCIDs need flushing we can | |
17 | avoid those too when possible. | |
18 | ||
19 | This does mean RESTORE_CR3 needs an additional scratch_reg, luckily both | |
20 | sites have plenty available. | |
21 | ||
22 | Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> | |
23 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
24 | Cc: Andy Lutomirski <luto@kernel.org> | |
25 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
26 | Cc: Borislav Petkov <bp@alien8.de> | |
27 | Cc: Brian Gerst <brgerst@gmail.com> | |
28 | Cc: Dave Hansen <dave.hansen@linux.intel.com> | |
29 | Cc: David Laight <David.Laight@aculab.com> | |
30 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
31 | Cc: Eduardo Valentin <eduval@amazon.com> | |
32 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
33 | Cc: H. Peter Anvin <hpa@zytor.com> | |
34 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
35 | Cc: Juergen Gross <jgross@suse.com> | |
36 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
37 | Cc: Peter Zijlstra <peterz@infradead.org> | |
38 | Cc: Will Deacon <will.deacon@arm.com> | |
39 | Cc: aliguori@amazon.com | |
40 | Cc: daniel.gruss@iaik.tugraz.at | |
41 | Cc: hughd@google.com | |
42 | Cc: keescook@google.com | |
43 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
44 | (cherry picked from commit 21e94459110252d41b45c0c8ba50fd72a664d50c) | |
45 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
46 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
47 | (cherry picked from commit 6ebe6e2896841282357d43c09394b0ca47c41e4a) | |
48 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
49 | --- | |
50 | arch/x86/entry/calling.h | 30 ++++++++++++++++++++++++++++-- | |
51 | arch/x86/entry/entry_64.S | 4 ++-- | |
52 | 2 files changed, 30 insertions(+), 4 deletions(-) | |
53 | ||
54 | diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h | |
55 | index ce5fb309926d..015e0a84bb99 100644 | |
56 | --- a/arch/x86/entry/calling.h | |
57 | +++ b/arch/x86/entry/calling.h | |
58 | @@ -280,8 +280,34 @@ For 32-bit we have the following conventions - kernel is built with | |
59 | .Ldone_\@: | |
60 | .endm | |
61 | ||
62 | -.macro RESTORE_CR3 save_reg:req | |
63 | +.macro RESTORE_CR3 scratch_reg:req save_reg:req | |
64 | ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI | |
65 | + | |
66 | + ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID | |
67 | + | |
68 | + /* | |
69 | + * KERNEL pages can always resume with NOFLUSH as we do | |
70 | + * explicit flushes. | |
71 | + */ | |
72 | + bt $X86_CR3_PTI_SWITCH_BIT, \save_reg | |
73 | + jnc .Lnoflush_\@ | |
74 | + | |
75 | + /* | |
76 | + * Check if there's a pending flush for the user ASID we're | |
77 | + * about to set. | |
78 | + */ | |
79 | + movq \save_reg, \scratch_reg | |
80 | + andq $(0x7FF), \scratch_reg | |
81 | + bt \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
82 | + jnc .Lnoflush_\@ | |
83 | + | |
84 | + btr \scratch_reg, THIS_CPU_user_pcid_flush_mask | |
85 | + jmp .Lwrcr3_\@ | |
86 | + | |
87 | +.Lnoflush_\@: | |
88 | + SET_NOFLUSH_BIT \save_reg | |
89 | + | |
90 | +.Lwrcr3_\@: | |
91 | /* | |
92 | * The CR3 write could be avoided when not changing its value, | |
93 | * but would require a CR3 read *and* a scratch register. | |
94 | @@ -300,7 +326,7 @@ For 32-bit we have the following conventions - kernel is built with | |
95 | .endm | |
96 | .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req | |
97 | .endm | |
98 | -.macro RESTORE_CR3 save_reg:req | |
99 | +.macro RESTORE_CR3 scratch_reg:req save_reg:req | |
100 | .endm | |
101 | ||
102 | #endif | |
103 | diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S | |
104 | index fb43f14ed299..b48f2c78a9bf 100644 | |
105 | --- a/arch/x86/entry/entry_64.S | |
106 | +++ b/arch/x86/entry/entry_64.S | |
107 | @@ -1300,7 +1300,7 @@ ENTRY(paranoid_exit) | |
108 | testl %ebx, %ebx /* swapgs needed? */ | |
109 | jnz .Lparanoid_exit_no_swapgs | |
110 | TRACE_IRQS_IRETQ | |
111 | - RESTORE_CR3 save_reg=%r14 | |
112 | + RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 | |
113 | SWAPGS_UNSAFE_STACK | |
114 | jmp .Lparanoid_exit_restore | |
115 | .Lparanoid_exit_no_swapgs: | |
116 | @@ -1742,7 +1742,7 @@ end_repeat_nmi: | |
117 | movq $-1, %rsi | |
118 | call do_nmi | |
119 | ||
120 | - RESTORE_CR3 save_reg=%r14 | |
121 | + RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 | |
122 | ||
123 | testl %ebx, %ebx /* swapgs needed? */ | |
124 | jnz nmi_restore | |
125 | -- | |
126 | 2.14.2 | |
127 |