]>
Commit | Line | Data |
---|---|---|
59d5af67 | 1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
321d628a FG |
2 | From: Dave Hansen <dave.hansen@linux.intel.com> |
3 | Date: Mon, 4 Dec 2017 15:07:57 +0100 | |
59d5af67 | 4 | Subject: [PATCH] x86/mm: Allow flushing for future ASID switches |
321d628a FG |
5 | MIME-Version: 1.0 |
6 | Content-Type: text/plain; charset=UTF-8 | |
7 | Content-Transfer-Encoding: 8bit | |
8 | ||
9 | CVE-2017-5754 | |
10 | ||
11 | If changing the page tables in such a way that an invalidation of all | |
12 | contexts (aka. PCIDs / ASIDs) is required, they can be actively invalidated | |
13 | by: | |
14 | ||
15 | 1. INVPCID for each PCID (works for single pages too). | |
16 | ||
17 | 2. Load CR3 with each PCID without the NOFLUSH bit set | |
18 | ||
19 | 3. Load CR3 with the NOFLUSH bit set for each and do INVLPG for each address. | |
20 | ||
21 | But, none of these are really feasible since there are ~6 ASIDs (12 with | |
22 | PAGE_TABLE_ISOLATION) at the time that invalidation is required. | |
23 | Instead of actively invalidating them, invalidate the *current* context and | |
24 | also mark the cpu_tlbstate _quickly_ to indicate future invalidation to be | |
25 | required. | |
26 | ||
27 | At the next context-switch, look for this indicator | |
28 | ('invalidate_other' being set) invalidate all of the | |
29 | cpu_tlbstate.ctxs[] entries. | |
30 | ||
31 | This ensures that any future context switches will do a full flush | |
32 | of the TLB, picking up the previous changes. | |
33 | ||
34 | [ tglx: Folded more fixups from Peter ] | |
35 | ||
36 | Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com> | |
37 | Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> | |
38 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
39 | Cc: Andy Lutomirski <luto@kernel.org> | |
40 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
41 | Cc: Borislav Petkov <bp@alien8.de> | |
42 | Cc: Brian Gerst <brgerst@gmail.com> | |
43 | Cc: David Laight <David.Laight@aculab.com> | |
44 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
45 | Cc: Eduardo Valentin <eduval@amazon.com> | |
46 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
47 | Cc: H. Peter Anvin <hpa@zytor.com> | |
48 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
49 | Cc: Juergen Gross <jgross@suse.com> | |
50 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
51 | Cc: Peter Zijlstra <peterz@infradead.org> | |
52 | Cc: Will Deacon <will.deacon@arm.com> | |
53 | Cc: aliguori@amazon.com | |
54 | Cc: daniel.gruss@iaik.tugraz.at | |
55 | Cc: hughd@google.com | |
56 | Cc: keescook@google.com | |
57 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
58 | (cherry picked from commit 2ea907c4fe7b78e5840c1dc07800eae93248cad1) | |
59 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
60 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
61 | (cherry picked from commit fbb7e6e9e7e7cedecc164d660d08563f88103b56) | |
62 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
63 | --- | |
64 | arch/x86/include/asm/tlbflush.h | 37 +++++++++++++++++++++++++++++-------- | |
65 | arch/x86/mm/tlb.c | 35 +++++++++++++++++++++++++++++++++++ | |
66 | 2 files changed, 64 insertions(+), 8 deletions(-) | |
67 | ||
68 | diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h | |
69 | index 503f87c30c15..3769ce182eac 100644 | |
70 | --- a/arch/x86/include/asm/tlbflush.h | |
71 | +++ b/arch/x86/include/asm/tlbflush.h | |
72 | @@ -124,6 +124,17 @@ struct tlb_state { | |
73 | */ | |
74 | bool is_lazy; | |
75 | ||
76 | + /* | |
77 | + * If set we changed the page tables in such a way that we | |
78 | + * needed an invalidation of all contexts (aka. PCIDs / ASIDs). | |
79 | + * This tells us to go invalidate all the non-loaded ctxs[] | |
80 | + * on the next context switch. | |
81 | + * | |
82 | + * The current ctx was kept up-to-date as it ran and does not | |
83 | + * need to be invalidated. | |
84 | + */ | |
85 | + bool invalidate_other; | |
86 | + | |
87 | /* | |
88 | * Access to this CR4 shadow and to H/W CR4 is protected by | |
89 | * disabling interrupts when modifying either one. | |
90 | @@ -201,6 +212,14 @@ static inline unsigned long cr4_read_shadow(void) | |
91 | return this_cpu_read(cpu_tlbstate.cr4); | |
92 | } | |
93 | ||
94 | +/* | |
95 | + * Mark all other ASIDs as invalid, preserves the current. | |
96 | + */ | |
97 | +static inline void invalidate_other_asid(void) | |
98 | +{ | |
99 | + this_cpu_write(cpu_tlbstate.invalidate_other, true); | |
100 | +} | |
101 | + | |
102 | /* | |
103 | * Save some of cr4 feature set we're using (e.g. Pentium 4MB | |
104 | * enable and PPro Global page enable), so that any CPU's that boot | |
105 | @@ -287,14 +306,6 @@ static inline void __flush_tlb_all(void) | |
106 | */ | |
107 | __flush_tlb(); | |
108 | } | |
109 | - | |
110 | - /* | |
111 | - * Note: if we somehow had PCID but not PGE, then this wouldn't work -- | |
112 | - * we'd end up flushing kernel translations for the current ASID but | |
113 | - * we might fail to flush kernel translations for other cached ASIDs. | |
114 | - * | |
115 | - * To avoid this issue, we force PCID off if PGE is off. | |
116 | - */ | |
117 | } | |
118 | ||
119 | /* | |
120 | @@ -304,6 +315,16 @@ static inline void __flush_tlb_one(unsigned long addr) | |
121 | { | |
122 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); | |
123 | __flush_tlb_single(addr); | |
124 | + | |
125 | + if (!static_cpu_has(X86_FEATURE_PTI)) | |
126 | + return; | |
127 | + | |
128 | + /* | |
129 | + * __flush_tlb_single() will have cleared the TLB entry for this ASID, | |
130 | + * but since kernel space is replicated across all, we must also | |
131 | + * invalidate all others. | |
132 | + */ | |
133 | + invalidate_other_asid(); | |
134 | } | |
135 | ||
136 | #define TLB_FLUSH_ALL -1UL | |
137 | diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c | |
138 | index 87d4f961bcb4..ce87b69fb4e0 100644 | |
139 | --- a/arch/x86/mm/tlb.c | |
140 | +++ b/arch/x86/mm/tlb.c | |
141 | @@ -28,6 +28,38 @@ | |
142 | * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi | |
143 | */ | |
144 | ||
145 | +/* | |
146 | + * We get here when we do something requiring a TLB invalidation | |
147 | + * but could not go invalidate all of the contexts. We do the | |
148 | + * necessary invalidation by clearing out the 'ctx_id' which | |
149 | + * forces a TLB flush when the context is loaded. | |
150 | + */ | |
151 | +void clear_asid_other(void) | |
152 | +{ | |
153 | + u16 asid; | |
154 | + | |
155 | + /* | |
156 | + * This is only expected to be set if we have disabled | |
157 | + * kernel _PAGE_GLOBAL pages. | |
158 | + */ | |
159 | + if (!static_cpu_has(X86_FEATURE_PTI)) { | |
160 | + WARN_ON_ONCE(1); | |
161 | + return; | |
162 | + } | |
163 | + | |
164 | + for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { | |
165 | + /* Do not need to flush the current asid */ | |
166 | + if (asid == this_cpu_read(cpu_tlbstate.loaded_mm_asid)) | |
167 | + continue; | |
168 | + /* | |
169 | + * Make sure the next time we go to switch to | |
170 | + * this asid, we do a flush: | |
171 | + */ | |
172 | + this_cpu_write(cpu_tlbstate.ctxs[asid].ctx_id, 0); | |
173 | + } | |
174 | + this_cpu_write(cpu_tlbstate.invalidate_other, false); | |
175 | +} | |
176 | + | |
177 | atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1); | |
178 | ||
179 | DEFINE_STATIC_KEY_TRUE(tlb_use_lazy_mode); | |
180 | @@ -43,6 +75,9 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, | |
181 | return; | |
182 | } | |
183 | ||
184 | + if (this_cpu_read(cpu_tlbstate.invalidate_other)) | |
185 | + clear_asid_other(); | |
186 | + | |
187 | for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { | |
188 | if (this_cpu_read(cpu_tlbstate.ctxs[asid].ctx_id) != | |
189 | next->context.ctx_id) | |
190 | -- | |
191 | 2.14.2 | |
192 |