2 * BTS PMU driver for perf
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/bitops.h>
20 #include <linux/types.h>
21 #include <linux/slab.h>
22 #include <linux/debugfs.h>
23 #include <linux/device.h>
24 #include <linux/coredump.h>
26 #include <asm-generic/sizes.h>
27 #include <asm/perf_event.h>
29 #include "perf_event.h"
32 struct perf_output_handle handle
;
33 struct debug_store ds_back
;
37 static DEFINE_PER_CPU(struct bts_ctx
, bts_ctx
);
39 #define BTS_RECORD_SIZE 24
40 #define BTS_SAFETY_MARGIN 4080
46 unsigned long displacement
;
50 size_t real_size
; /* multiple of BTS_RECORD_SIZE */
51 unsigned int nr_pages
;
60 struct bts_phys buf
[0];
65 void intel_pmu_enable_bts(u64 config
);
66 void intel_pmu_disable_bts(void);
68 static size_t buf_size(struct page
*page
)
70 return 1 << (PAGE_SHIFT
+ page_private(page
));
74 bts_buffer_setup_aux(int cpu
, void **pages
, int nr_pages
, bool overwrite
)
76 struct bts_buffer
*buf
;
78 int node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
80 size_t size
= nr_pages
<< PAGE_SHIFT
;
83 /* count all the high order buffers */
84 for (pg
= 0, nbuf
= 0; pg
< nr_pages
;) {
85 page
= virt_to_page(pages
[pg
]);
86 if (WARN_ON_ONCE(!PagePrivate(page
) && nr_pages
> 1))
88 pg
+= 1 << page_private(page
);
93 * to avoid interrupts in overwrite mode, only allow one physical
95 if (overwrite
&& nbuf
> 1)
98 buf
= kzalloc_node(offsetof(struct bts_buffer
, buf
[nbuf
]), GFP_KERNEL
, node
);
102 buf
->nr_pages
= nr_pages
;
104 buf
->snapshot
= overwrite
;
105 buf
->data_pages
= pages
;
106 buf
->real_size
= size
- size
% BTS_RECORD_SIZE
;
108 for (pg
= 0, nbuf
= 0, offset
= 0, pad
= 0; nbuf
< buf
->nr_bufs
; nbuf
++) {
109 unsigned int __nr_pages
;
111 page
= virt_to_page(pages
[pg
]);
112 __nr_pages
= PagePrivate(page
) ? 1 << page_private(page
) : 1;
113 buf
->buf
[nbuf
].page
= page
;
114 buf
->buf
[nbuf
].offset
= offset
;
115 buf
->buf
[nbuf
].displacement
= (pad
? BTS_RECORD_SIZE
- pad
: 0);
116 buf
->buf
[nbuf
].size
= buf_size(page
) - buf
->buf
[nbuf
].displacement
;
117 pad
= buf
->buf
[nbuf
].size
% BTS_RECORD_SIZE
;
118 buf
->buf
[nbuf
].size
-= pad
;
121 offset
+= __nr_pages
<< PAGE_SHIFT
;
127 static void bts_buffer_free_aux(void *data
)
132 static unsigned long bts_buffer_offset(struct bts_buffer
*buf
, unsigned int idx
)
134 return buf
->buf
[idx
].offset
+ buf
->buf
[idx
].displacement
;
138 bts_config_buffer(struct bts_buffer
*buf
)
140 int cpu
= raw_smp_processor_id();
141 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
142 struct bts_phys
*phys
= &buf
->buf
[buf
->cur_buf
];
143 unsigned long index
, thresh
= 0, end
= phys
->size
;
144 struct page
*page
= phys
->page
;
146 index
= local_read(&buf
->head
);
148 if (!buf
->snapshot
) {
149 if (buf
->end
< phys
->offset
+ buf_size(page
))
150 end
= buf
->end
- phys
->offset
- phys
->displacement
;
152 index
-= phys
->offset
+ phys
->displacement
;
154 if (end
- index
> BTS_SAFETY_MARGIN
)
155 thresh
= end
- BTS_SAFETY_MARGIN
;
156 else if (end
- index
> BTS_RECORD_SIZE
)
157 thresh
= end
- BTS_RECORD_SIZE
;
162 ds
->bts_buffer_base
= (u64
)page_address(page
) + phys
->displacement
;
163 ds
->bts_index
= ds
->bts_buffer_base
+ index
;
164 ds
->bts_absolute_maximum
= ds
->bts_buffer_base
+ end
;
165 ds
->bts_interrupt_threshold
= !buf
->snapshot
166 ? ds
->bts_buffer_base
+ thresh
167 : ds
->bts_absolute_maximum
+ BTS_RECORD_SIZE
;
170 static void bts_buffer_pad_out(struct bts_phys
*phys
, unsigned long head
)
172 unsigned long index
= head
- phys
->offset
;
174 memset(page_address(phys
->page
) + index
, 0, phys
->size
- index
);
177 static bool bts_buffer_is_full(struct bts_buffer
*buf
, struct bts_ctx
*bts
)
182 if (local_read(&buf
->data_size
) >= bts
->handle
.size
||
183 bts
->handle
.size
- local_read(&buf
->data_size
) < BTS_RECORD_SIZE
)
189 static void bts_update(struct bts_ctx
*bts
)
191 int cpu
= raw_smp_processor_id();
192 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
193 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
194 unsigned long index
= ds
->bts_index
- ds
->bts_buffer_base
, old
, head
;
199 head
= index
+ bts_buffer_offset(buf
, buf
->cur_buf
);
200 old
= local_xchg(&buf
->head
, head
);
202 if (!buf
->snapshot
) {
206 if (ds
->bts_index
>= ds
->bts_absolute_maximum
)
207 local_inc(&buf
->lost
);
210 * old and head are always in the same physical buffer, so we
211 * can subtract them to get the data size.
213 local_add(head
- old
, &buf
->data_size
);
215 local_set(&buf
->data_size
, head
);
219 static void __bts_event_start(struct perf_event
*event
)
221 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
222 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
225 if (!buf
|| bts_buffer_is_full(buf
, bts
))
231 config
|= ARCH_PERFMON_EVENTSEL_INT
;
232 if (!event
->attr
.exclude_kernel
)
233 config
|= ARCH_PERFMON_EVENTSEL_OS
;
234 if (!event
->attr
.exclude_user
)
235 config
|= ARCH_PERFMON_EVENTSEL_USR
;
237 bts_config_buffer(buf
);
240 * local barrier to make sure that ds configuration made it
241 * before we enable BTS
245 intel_pmu_enable_bts(config
);
248 static void bts_event_start(struct perf_event
*event
, int flags
)
250 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
252 __bts_event_start(event
);
254 /* PMI handler: this counter is running and likely generating PMIs */
255 ACCESS_ONCE(bts
->started
) = 1;
258 static void __bts_event_stop(struct perf_event
*event
)
261 * No extra synchronization is mandated by the documentation to have
262 * BTS data stores globally visible.
264 intel_pmu_disable_bts();
266 if (event
->hw
.state
& PERF_HES_STOPPED
)
269 ACCESS_ONCE(event
->hw
.state
) |= PERF_HES_STOPPED
;
272 static void bts_event_stop(struct perf_event
*event
, int flags
)
274 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
276 /* PMI handler: don't restart this counter */
277 ACCESS_ONCE(bts
->started
) = 0;
279 __bts_event_stop(event
);
281 if (flags
& PERF_EF_UPDATE
)
285 void intel_bts_enable_local(void)
287 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
289 if (bts
->handle
.event
&& bts
->started
)
290 __bts_event_start(bts
->handle
.event
);
293 void intel_bts_disable_local(void)
295 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
297 if (bts
->handle
.event
)
298 __bts_event_stop(bts
->handle
.event
);
302 bts_buffer_reset(struct bts_buffer
*buf
, struct perf_output_handle
*handle
)
304 unsigned long head
, space
, next_space
, pad
, gap
, skip
, wakeup
;
305 unsigned int next_buf
;
306 struct bts_phys
*phys
, *next_phys
;
312 head
= handle
->head
& ((buf
->nr_pages
<< PAGE_SHIFT
) - 1);
313 if (WARN_ON_ONCE(head
!= local_read(&buf
->head
)))
316 phys
= &buf
->buf
[buf
->cur_buf
];
317 space
= phys
->offset
+ phys
->displacement
+ phys
->size
- head
;
319 if (space
> handle
->size
) {
320 space
= handle
->size
;
321 space
-= space
% BTS_RECORD_SIZE
;
323 if (space
<= BTS_SAFETY_MARGIN
) {
324 /* See if next phys buffer has more space */
325 next_buf
= buf
->cur_buf
+ 1;
326 if (next_buf
>= buf
->nr_bufs
)
328 next_phys
= &buf
->buf
[next_buf
];
329 gap
= buf_size(phys
->page
) - phys
->displacement
- phys
->size
+
330 next_phys
->displacement
;
332 if (handle
->size
>= skip
) {
333 next_space
= next_phys
->size
;
334 if (next_space
+ skip
> handle
->size
) {
335 next_space
= handle
->size
- skip
;
336 next_space
-= next_space
% BTS_RECORD_SIZE
;
338 if (next_space
> space
|| !space
) {
340 bts_buffer_pad_out(phys
, head
);
341 ret
= perf_aux_output_skip(handle
, skip
);
344 /* Advance to next phys buffer */
347 head
= phys
->offset
+ phys
->displacement
;
349 * After this, cur_buf and head won't match ds
350 * anymore, so we must not be racing with
353 buf
->cur_buf
= next_buf
;
354 local_set(&buf
->head
, head
);
359 /* Don't go far beyond wakeup watermark */
360 wakeup
= BTS_SAFETY_MARGIN
+ BTS_RECORD_SIZE
+ handle
->wakeup
-
362 if (space
> wakeup
) {
364 space
-= space
% BTS_RECORD_SIZE
;
367 buf
->end
= head
+ space
;
370 * If we have no space, the lost notification would have been sent when
371 * we hit absolute_maximum - see bts_update()
379 int intel_bts_interrupt(void)
381 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
382 struct perf_event
*event
= bts
->handle
.event
;
383 struct bts_buffer
*buf
;
387 if (!event
|| !bts
->started
)
390 buf
= perf_get_aux(&bts
->handle
);
392 * Skip snapshot counters: they don't use the interrupt, but
393 * there's no other way of telling, because the pointer will
396 if (!buf
|| buf
->snapshot
)
399 old_head
= local_read(&buf
->head
);
403 if (old_head
== local_read(&buf
->head
))
406 perf_aux_output_end(&bts
->handle
, local_xchg(&buf
->data_size
, 0),
407 !!local_xchg(&buf
->lost
, 0));
409 buf
= perf_aux_output_begin(&bts
->handle
, event
);
413 err
= bts_buffer_reset(buf
, &bts
->handle
);
415 perf_aux_output_end(&bts
->handle
, 0, false);
420 static void bts_event_del(struct perf_event
*event
, int mode
)
422 struct cpu_hw_events
*cpuc
= this_cpu_ptr(&cpu_hw_events
);
423 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
424 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
426 bts_event_stop(event
, PERF_EF_UPDATE
);
431 local_xchg(&buf
->data_size
,
432 buf
->nr_pages
<< PAGE_SHIFT
);
433 perf_aux_output_end(&bts
->handle
, local_xchg(&buf
->data_size
, 0),
434 !!local_xchg(&buf
->lost
, 0));
437 cpuc
->ds
->bts_index
= bts
->ds_back
.bts_buffer_base
;
438 cpuc
->ds
->bts_buffer_base
= bts
->ds_back
.bts_buffer_base
;
439 cpuc
->ds
->bts_absolute_maximum
= bts
->ds_back
.bts_absolute_maximum
;
440 cpuc
->ds
->bts_interrupt_threshold
= bts
->ds_back
.bts_interrupt_threshold
;
443 static int bts_event_add(struct perf_event
*event
, int mode
)
445 struct bts_buffer
*buf
;
446 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
447 struct cpu_hw_events
*cpuc
= this_cpu_ptr(&cpu_hw_events
);
448 struct hw_perf_event
*hwc
= &event
->hw
;
451 event
->hw
.state
= PERF_HES_STOPPED
;
453 if (test_bit(INTEL_PMC_IDX_FIXED_BTS
, cpuc
->active_mask
))
456 if (bts
->handle
.event
)
459 buf
= perf_aux_output_begin(&bts
->handle
, event
);
463 ret
= bts_buffer_reset(buf
, &bts
->handle
);
465 perf_aux_output_end(&bts
->handle
, 0, false);
469 bts
->ds_back
.bts_buffer_base
= cpuc
->ds
->bts_buffer_base
;
470 bts
->ds_back
.bts_absolute_maximum
= cpuc
->ds
->bts_absolute_maximum
;
471 bts
->ds_back
.bts_interrupt_threshold
= cpuc
->ds
->bts_interrupt_threshold
;
473 if (mode
& PERF_EF_START
) {
474 bts_event_start(event
, 0);
475 if (hwc
->state
& PERF_HES_STOPPED
) {
476 bts_event_del(event
, 0);
484 static void bts_event_destroy(struct perf_event
*event
)
486 x86_del_exclusive(x86_lbr_exclusive_bts
);
489 static int bts_event_init(struct perf_event
*event
)
491 if (event
->attr
.type
!= bts_pmu
.type
)
494 if (x86_add_exclusive(x86_lbr_exclusive_bts
))
497 event
->destroy
= bts_event_destroy
;
502 static void bts_event_read(struct perf_event
*event
)
506 static __init
int bts_init(void)
508 if (!boot_cpu_has(X86_FEATURE_DTES64
) || !x86_pmu
.bts
)
511 bts_pmu
.capabilities
= PERF_PMU_CAP_AUX_NO_SG
| PERF_PMU_CAP_ITRACE
;
512 bts_pmu
.task_ctx_nr
= perf_sw_context
;
513 bts_pmu
.event_init
= bts_event_init
;
514 bts_pmu
.add
= bts_event_add
;
515 bts_pmu
.del
= bts_event_del
;
516 bts_pmu
.start
= bts_event_start
;
517 bts_pmu
.stop
= bts_event_stop
;
518 bts_pmu
.read
= bts_event_read
;
519 bts_pmu
.setup_aux
= bts_buffer_setup_aux
;
520 bts_pmu
.free_aux
= bts_buffer_free_aux
;
522 return perf_pmu_register(&bts_pmu
, "intel_bts", -1);
525 module_init(bts_init
);