]>
Commit | Line | Data |
---|---|---|
321d628a FG |
1 | From 68338a3b7267b4fc346630b2d82a3599b5fbf54e Mon Sep 17 00:00:00 2001 |
2 | From: Hugh Dickins <hughd@google.com> | |
3 | Date: Mon, 4 Dec 2017 15:07:50 +0100 | |
b378f209 | 4 | Subject: [PATCH 203/233] x86/events/intel/ds: Map debug buffers in |
321d628a FG |
5 | cpu_entry_area |
6 | MIME-Version: 1.0 | |
7 | Content-Type: text/plain; charset=UTF-8 | |
8 | Content-Transfer-Encoding: 8bit | |
9 | ||
10 | CVE-2017-5754 | |
11 | ||
12 | The BTS and PEBS buffers both have their virtual addresses programmed into | |
13 | the hardware. This means that any access to them is performed via the page | |
14 | tables. The times that the hardware accesses these are entirely dependent | |
15 | on how the performance monitoring hardware events are set up. In other | |
16 | words, there is no way for the kernel to tell when the hardware might | |
17 | access these buffers. | |
18 | ||
19 | To avoid perf crashes, place 'debug_store' allocate pages and map them into | |
20 | the cpu_entry_area. | |
21 | ||
22 | The PEBS fixup buffer does not need this treatment. | |
23 | ||
24 | [ tglx: Got rid of the kaiser_add_mapping() complication ] | |
25 | ||
26 | Signed-off-by: Hugh Dickins <hughd@google.com> | |
27 | Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com> | |
28 | Signed-off-by: Thomas Gleixner <tglx@linutronix.de> | |
29 | Cc: Andy Lutomirski <luto@kernel.org> | |
30 | Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com> | |
31 | Cc: Borislav Petkov <bp@alien8.de> | |
32 | Cc: Brian Gerst <brgerst@gmail.com> | |
33 | Cc: David Laight <David.Laight@aculab.com> | |
34 | Cc: Denys Vlasenko <dvlasenk@redhat.com> | |
35 | Cc: Eduardo Valentin <eduval@amazon.com> | |
36 | Cc: Greg KH <gregkh@linuxfoundation.org> | |
37 | Cc: H. Peter Anvin <hpa@zytor.com> | |
38 | Cc: Josh Poimboeuf <jpoimboe@redhat.com> | |
39 | Cc: Juergen Gross <jgross@suse.com> | |
40 | Cc: Linus Torvalds <torvalds@linux-foundation.org> | |
41 | Cc: Peter Zijlstra <peterz@infradead.org> | |
42 | Cc: Will Deacon <will.deacon@arm.com> | |
43 | Cc: aliguori@amazon.com | |
44 | Cc: daniel.gruss@iaik.tugraz.at | |
45 | Cc: keescook@google.com | |
46 | Signed-off-by: Ingo Molnar <mingo@kernel.org> | |
47 | (cherry picked from commit c1961a4631daef4aeabee8e368b1b13e8f173c91) | |
48 | Signed-off-by: Andy Whitcroft <apw@canonical.com> | |
49 | Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com> | |
50 | (cherry picked from commit 569dedbb62e16e3268f006dcf745b8d27690ef91) | |
51 | Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com> | |
52 | --- | |
53 | arch/x86/events/perf_event.h | 2 + | |
54 | arch/x86/events/intel/ds.c | 125 +++++++++++++++++++++++++++---------------- | |
55 | 2 files changed, 82 insertions(+), 45 deletions(-) | |
56 | ||
57 | diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h | |
58 | index 308bc14f58af..eb0876475f18 100644 | |
59 | --- a/arch/x86/events/perf_event.h | |
60 | +++ b/arch/x86/events/perf_event.h | |
61 | @@ -199,6 +199,8 @@ struct cpu_hw_events { | |
62 | * Intel DebugStore bits | |
63 | */ | |
64 | struct debug_store *ds; | |
65 | + void *ds_pebs_vaddr; | |
66 | + void *ds_bts_vaddr; | |
67 | u64 pebs_enabled; | |
68 | int n_pebs; | |
69 | int n_large_pebs; | |
70 | diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c | |
71 | index 21a4ed789ec0..85df1f12c49e 100644 | |
72 | --- a/arch/x86/events/intel/ds.c | |
73 | +++ b/arch/x86/events/intel/ds.c | |
74 | @@ -2,6 +2,7 @@ | |
75 | #include <linux/types.h> | |
76 | #include <linux/slab.h> | |
77 | ||
78 | +#include <asm/cpu_entry_area.h> | |
79 | #include <asm/perf_event.h> | |
80 | #include <asm/insn.h> | |
81 | ||
82 | @@ -279,17 +280,52 @@ void fini_debug_store_on_cpu(int cpu) | |
83 | ||
84 | static DEFINE_PER_CPU(void *, insn_buffer); | |
85 | ||
86 | -static int alloc_pebs_buffer(int cpu) | |
87 | +static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot) | |
88 | { | |
89 | - struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; | |
90 | + phys_addr_t pa; | |
91 | + size_t msz = 0; | |
92 | + | |
93 | + pa = virt_to_phys(addr); | |
94 | + for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE) | |
95 | + cea_set_pte(cea, pa, prot); | |
96 | +} | |
97 | + | |
98 | +static void ds_clear_cea(void *cea, size_t size) | |
99 | +{ | |
100 | + size_t msz = 0; | |
101 | + | |
102 | + for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE) | |
103 | + cea_set_pte(cea, 0, PAGE_NONE); | |
104 | +} | |
105 | + | |
106 | +static void *dsalloc_pages(size_t size, gfp_t flags, int cpu) | |
107 | +{ | |
108 | + unsigned int order = get_order(size); | |
109 | int node = cpu_to_node(cpu); | |
110 | - int max; | |
111 | - void *buffer, *ibuffer; | |
112 | + struct page *page; | |
113 | + | |
114 | + page = __alloc_pages_node(node, flags | __GFP_ZERO, order); | |
115 | + return page ? page_address(page) : NULL; | |
116 | +} | |
117 | + | |
118 | +static void dsfree_pages(const void *buffer, size_t size) | |
119 | +{ | |
120 | + if (buffer) | |
121 | + free_pages((unsigned long)buffer, get_order(size)); | |
122 | +} | |
123 | + | |
124 | +static int alloc_pebs_buffer(int cpu) | |
125 | +{ | |
126 | + struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu); | |
127 | + struct debug_store *ds = hwev->ds; | |
128 | + size_t bsiz = x86_pmu.pebs_buffer_size; | |
129 | + int max, node = cpu_to_node(cpu); | |
130 | + void *buffer, *ibuffer, *cea; | |
131 | ||
132 | if (!x86_pmu.pebs) | |
133 | return 0; | |
134 | ||
135 | - buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node); | |
136 | + buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu); | |
137 | if (unlikely(!buffer)) | |
138 | return -ENOMEM; | |
139 | ||
140 | @@ -300,25 +336,27 @@ static int alloc_pebs_buffer(int cpu) | |
141 | if (x86_pmu.intel_cap.pebs_format < 2) { | |
142 | ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node); | |
143 | if (!ibuffer) { | |
144 | - kfree(buffer); | |
145 | + dsfree_pages(buffer, bsiz); | |
146 | return -ENOMEM; | |
147 | } | |
148 | per_cpu(insn_buffer, cpu) = ibuffer; | |
149 | } | |
150 | - | |
151 | - max = x86_pmu.pebs_buffer_size / x86_pmu.pebs_record_size; | |
152 | - | |
153 | - ds->pebs_buffer_base = (u64)(unsigned long)buffer; | |
154 | + hwev->ds_pebs_vaddr = buffer; | |
155 | + /* Update the cpu entry area mapping */ | |
156 | + cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer; | |
157 | + ds->pebs_buffer_base = (unsigned long) cea; | |
158 | + ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL); | |
159 | ds->pebs_index = ds->pebs_buffer_base; | |
160 | - ds->pebs_absolute_maximum = ds->pebs_buffer_base + | |
161 | - max * x86_pmu.pebs_record_size; | |
162 | - | |
163 | + max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size); | |
164 | + ds->pebs_absolute_maximum = ds->pebs_buffer_base + max; | |
165 | return 0; | |
166 | } | |
167 | ||
168 | static void release_pebs_buffer(int cpu) | |
169 | { | |
170 | - struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; | |
171 | + struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu); | |
172 | + struct debug_store *ds = hwev->ds; | |
173 | + void *cea; | |
174 | ||
175 | if (!ds || !x86_pmu.pebs) | |
176 | return; | |
177 | @@ -326,73 +364,70 @@ static void release_pebs_buffer(int cpu) | |
178 | kfree(per_cpu(insn_buffer, cpu)); | |
179 | per_cpu(insn_buffer, cpu) = NULL; | |
180 | ||
181 | - kfree((void *)(unsigned long)ds->pebs_buffer_base); | |
182 | + /* Clear the fixmap */ | |
183 | + cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer; | |
184 | + ds_clear_cea(cea, x86_pmu.pebs_buffer_size); | |
185 | ds->pebs_buffer_base = 0; | |
186 | + dsfree_pages(hwev->ds_pebs_vaddr, x86_pmu.pebs_buffer_size); | |
187 | + hwev->ds_pebs_vaddr = NULL; | |
188 | } | |
189 | ||
190 | static int alloc_bts_buffer(int cpu) | |
191 | { | |
192 | - struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; | |
193 | - int node = cpu_to_node(cpu); | |
194 | - int max, thresh; | |
195 | - void *buffer; | |
196 | + struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu); | |
197 | + struct debug_store *ds = hwev->ds; | |
198 | + void *buffer, *cea; | |
199 | + int max; | |
200 | ||
201 | if (!x86_pmu.bts) | |
202 | return 0; | |
203 | ||
204 | - buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node); | |
205 | + buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu); | |
206 | if (unlikely(!buffer)) { | |
207 | WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__); | |
208 | return -ENOMEM; | |
209 | } | |
210 | - | |
211 | - max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE; | |
212 | - thresh = max / 16; | |
213 | - | |
214 | - ds->bts_buffer_base = (u64)(unsigned long)buffer; | |
215 | + hwev->ds_bts_vaddr = buffer; | |
216 | + /* Update the fixmap */ | |
217 | + cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer; | |
218 | + ds->bts_buffer_base = (unsigned long) cea; | |
219 | + ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL); | |
220 | ds->bts_index = ds->bts_buffer_base; | |
221 | - ds->bts_absolute_maximum = ds->bts_buffer_base + | |
222 | - max * BTS_RECORD_SIZE; | |
223 | - ds->bts_interrupt_threshold = ds->bts_absolute_maximum - | |
224 | - thresh * BTS_RECORD_SIZE; | |
225 | - | |
226 | + max = BTS_RECORD_SIZE * (BTS_BUFFER_SIZE / BTS_RECORD_SIZE); | |
227 | + ds->bts_absolute_maximum = ds->bts_buffer_base + max; | |
228 | + ds->bts_interrupt_threshold = ds->bts_absolute_maximum - (max / 16); | |
229 | return 0; | |
230 | } | |
231 | ||
232 | static void release_bts_buffer(int cpu) | |
233 | { | |
234 | - struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; | |
235 | + struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu); | |
236 | + struct debug_store *ds = hwev->ds; | |
237 | + void *cea; | |
238 | ||
239 | if (!ds || !x86_pmu.bts) | |
240 | return; | |
241 | ||
242 | - kfree((void *)(unsigned long)ds->bts_buffer_base); | |
243 | + /* Clear the fixmap */ | |
244 | + cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer; | |
245 | + ds_clear_cea(cea, BTS_BUFFER_SIZE); | |
246 | ds->bts_buffer_base = 0; | |
247 | + dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE); | |
248 | + hwev->ds_bts_vaddr = NULL; | |
249 | } | |
250 | ||
251 | static int alloc_ds_buffer(int cpu) | |
252 | { | |
253 | - int node = cpu_to_node(cpu); | |
254 | - struct debug_store *ds; | |
255 | - | |
256 | - ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node); | |
257 | - if (unlikely(!ds)) | |
258 | - return -ENOMEM; | |
259 | + struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store; | |
260 | ||
261 | + memset(ds, 0, sizeof(*ds)); | |
262 | per_cpu(cpu_hw_events, cpu).ds = ds; | |
263 | - | |
264 | return 0; | |
265 | } | |
266 | ||
267 | static void release_ds_buffer(int cpu) | |
268 | { | |
269 | - struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; | |
270 | - | |
271 | - if (!ds) | |
272 | - return; | |
273 | - | |
274 | per_cpu(cpu_hw_events, cpu).ds = NULL; | |
275 | - kfree(ds); | |
276 | } | |
277 | ||
278 | void release_ds_buffers(void) | |
279 | -- | |
280 | 2.14.2 | |
281 |