2 * intel-bts.c: Intel Processor Trace support
3 * Copyright (c) 2013-2015, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 #include <linux/kernel.h>
17 #include <linux/types.h>
18 #include <linux/bitops.h>
19 #include <linux/log2.h>
21 #include "../../util/cpumap.h"
22 #include "../../util/evsel.h"
23 #include "../../util/evlist.h"
24 #include "../../util/session.h"
25 #include "../../util/util.h"
26 #include "../../util/pmu.h"
27 #include "../../util/debug.h"
28 #include "../../util/tsc.h"
29 #include "../../util/auxtrace.h"
30 #include "../../util/intel-bts.h"
32 #define KiB(x) ((x) * 1024)
33 #define MiB(x) ((x) * 1024 * 1024)
34 #define KiB_MASK(x) (KiB(x) - 1)
35 #define MiB_MASK(x) (MiB(x) - 1)
37 #define INTEL_BTS_DFLT_SAMPLE_SIZE KiB(4)
39 #define INTEL_BTS_MAX_SAMPLE_SIZE KiB(60)
41 struct intel_bts_snapshot_ref
{
47 struct intel_bts_recording
{
48 struct auxtrace_record itr
;
49 struct perf_pmu
*intel_bts_pmu
;
50 struct perf_evlist
*evlist
;
54 struct intel_bts_snapshot_ref
*snapshot_refs
;
63 static size_t intel_bts_info_priv_size(struct auxtrace_record
*itr __maybe_unused
)
65 return INTEL_BTS_AUXTRACE_PRIV_SIZE
;
68 static int intel_bts_info_fill(struct auxtrace_record
*itr
,
69 struct perf_session
*session
,
70 struct auxtrace_info_event
*auxtrace_info
,
73 struct intel_bts_recording
*btsr
=
74 container_of(itr
, struct intel_bts_recording
, itr
);
75 struct perf_pmu
*intel_bts_pmu
= btsr
->intel_bts_pmu
;
76 struct perf_event_mmap_page
*pc
;
77 struct perf_tsc_conversion tc
= { .time_mult
= 0, };
78 bool cap_user_time_zero
= false;
81 if (priv_size
!= INTEL_BTS_AUXTRACE_PRIV_SIZE
)
84 if (!session
->evlist
->nr_mmaps
)
87 pc
= session
->evlist
->mmap
[0].base
;
89 err
= perf_read_tsc_conversion(pc
, &tc
);
91 if (err
!= -EOPNOTSUPP
)
94 cap_user_time_zero
= tc
.time_mult
!= 0;
96 if (!cap_user_time_zero
)
97 ui__warning("Intel BTS: TSC not available\n");
100 auxtrace_info
->type
= PERF_AUXTRACE_INTEL_BTS
;
101 auxtrace_info
->priv
[INTEL_BTS_PMU_TYPE
] = intel_bts_pmu
->type
;
102 auxtrace_info
->priv
[INTEL_BTS_TIME_SHIFT
] = tc
.time_shift
;
103 auxtrace_info
->priv
[INTEL_BTS_TIME_MULT
] = tc
.time_mult
;
104 auxtrace_info
->priv
[INTEL_BTS_TIME_ZERO
] = tc
.time_zero
;
105 auxtrace_info
->priv
[INTEL_BTS_CAP_USER_TIME_ZERO
] = cap_user_time_zero
;
106 auxtrace_info
->priv
[INTEL_BTS_SNAPSHOT_MODE
] = btsr
->snapshot_mode
;
111 static int intel_bts_recording_options(struct auxtrace_record
*itr
,
112 struct perf_evlist
*evlist
,
113 struct record_opts
*opts
)
115 struct intel_bts_recording
*btsr
=
116 container_of(itr
, struct intel_bts_recording
, itr
);
117 struct perf_pmu
*intel_bts_pmu
= btsr
->intel_bts_pmu
;
118 struct perf_evsel
*evsel
, *intel_bts_evsel
= NULL
;
119 const struct cpu_map
*cpus
= evlist
->cpus
;
120 bool privileged
= geteuid() == 0 || perf_event_paranoid() < 0;
122 btsr
->evlist
= evlist
;
123 btsr
->snapshot_mode
= opts
->auxtrace_snapshot_mode
;
125 evlist__for_each(evlist
, evsel
) {
126 if (evsel
->attr
.type
== intel_bts_pmu
->type
) {
127 if (intel_bts_evsel
) {
128 pr_err("There may be only one " INTEL_BTS_PMU_NAME
" event\n");
131 evsel
->attr
.freq
= 0;
132 evsel
->attr
.sample_period
= 1;
133 intel_bts_evsel
= evsel
;
134 opts
->full_auxtrace
= true;
138 if (opts
->auxtrace_snapshot_mode
&& !opts
->full_auxtrace
) {
139 pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME
" PMU event (-e " INTEL_BTS_PMU_NAME
")\n");
143 if (!opts
->full_auxtrace
)
146 if (opts
->full_auxtrace
&& !cpu_map__empty(cpus
)) {
147 pr_err(INTEL_BTS_PMU_NAME
" does not support per-cpu recording\n");
151 /* Set default sizes for snapshot mode */
152 if (opts
->auxtrace_snapshot_mode
) {
153 if (!opts
->auxtrace_snapshot_size
&& !opts
->auxtrace_mmap_pages
) {
155 opts
->auxtrace_mmap_pages
= MiB(4) / page_size
;
157 opts
->auxtrace_mmap_pages
= KiB(128) / page_size
;
158 if (opts
->mmap_pages
== UINT_MAX
)
159 opts
->mmap_pages
= KiB(256) / page_size
;
161 } else if (!opts
->auxtrace_mmap_pages
&& !privileged
&&
162 opts
->mmap_pages
== UINT_MAX
) {
163 opts
->mmap_pages
= KiB(256) / page_size
;
165 if (!opts
->auxtrace_snapshot_size
)
166 opts
->auxtrace_snapshot_size
=
167 opts
->auxtrace_mmap_pages
* (size_t)page_size
;
168 if (!opts
->auxtrace_mmap_pages
) {
169 size_t sz
= opts
->auxtrace_snapshot_size
;
171 sz
= round_up(sz
, page_size
) / page_size
;
172 opts
->auxtrace_mmap_pages
= roundup_pow_of_two(sz
);
174 if (opts
->auxtrace_snapshot_size
>
175 opts
->auxtrace_mmap_pages
* (size_t)page_size
) {
176 pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
177 opts
->auxtrace_snapshot_size
,
178 opts
->auxtrace_mmap_pages
* (size_t)page_size
);
181 if (!opts
->auxtrace_snapshot_size
|| !opts
->auxtrace_mmap_pages
) {
182 pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
185 pr_debug2("Intel BTS snapshot size: %zu\n",
186 opts
->auxtrace_snapshot_size
);
189 /* Set default sizes for full trace mode */
190 if (opts
->full_auxtrace
&& !opts
->auxtrace_mmap_pages
) {
192 opts
->auxtrace_mmap_pages
= MiB(4) / page_size
;
194 opts
->auxtrace_mmap_pages
= KiB(128) / page_size
;
195 if (opts
->mmap_pages
== UINT_MAX
)
196 opts
->mmap_pages
= KiB(256) / page_size
;
200 /* Validate auxtrace_mmap_pages */
201 if (opts
->auxtrace_mmap_pages
) {
202 size_t sz
= opts
->auxtrace_mmap_pages
* (size_t)page_size
;
205 if (opts
->auxtrace_snapshot_mode
)
210 if (sz
< min_sz
|| !is_power_of_2(sz
)) {
211 pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
217 if (intel_bts_evsel
) {
219 * To obtain the auxtrace buffer file descriptor, the auxtrace event
222 perf_evlist__to_front(evlist
, intel_bts_evsel
);
224 * In the case of per-cpu mmaps, we need the CPU on the
227 if (!cpu_map__empty(cpus
))
228 perf_evsel__set_sample_bit(intel_bts_evsel
, CPU
);
231 /* Add dummy event to keep tracking */
232 if (opts
->full_auxtrace
) {
233 struct perf_evsel
*tracking_evsel
;
236 err
= parse_events(evlist
, "dummy:u", NULL
);
240 tracking_evsel
= perf_evlist__last(evlist
);
242 perf_evlist__set_tracking_event(evlist
, tracking_evsel
);
244 tracking_evsel
->attr
.freq
= 0;
245 tracking_evsel
->attr
.sample_period
= 1;
251 static int intel_bts_parse_snapshot_options(struct auxtrace_record
*itr
,
252 struct record_opts
*opts
,
255 struct intel_bts_recording
*btsr
=
256 container_of(itr
, struct intel_bts_recording
, itr
);
257 unsigned long long snapshot_size
= 0;
261 snapshot_size
= strtoull(str
, &endptr
, 0);
262 if (*endptr
|| snapshot_size
> SIZE_MAX
)
266 opts
->auxtrace_snapshot_mode
= true;
267 opts
->auxtrace_snapshot_size
= snapshot_size
;
269 btsr
->snapshot_size
= snapshot_size
;
274 static u64
intel_bts_reference(struct auxtrace_record
*itr __maybe_unused
)
279 static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording
*btsr
,
282 const size_t sz
= sizeof(struct intel_bts_snapshot_ref
);
283 int cnt
= btsr
->snapshot_ref_cnt
, new_cnt
= cnt
* 2;
284 struct intel_bts_snapshot_ref
*refs
;
289 while (new_cnt
<= idx
)
292 refs
= calloc(new_cnt
, sz
);
296 memcpy(refs
, btsr
->snapshot_refs
, cnt
* sz
);
298 btsr
->snapshot_refs
= refs
;
299 btsr
->snapshot_ref_cnt
= new_cnt
;
304 static void intel_bts_free_snapshot_refs(struct intel_bts_recording
*btsr
)
308 for (i
= 0; i
< btsr
->snapshot_ref_cnt
; i
++)
309 zfree(&btsr
->snapshot_refs
[i
].ref_buf
);
310 zfree(&btsr
->snapshot_refs
);
313 static void intel_bts_recording_free(struct auxtrace_record
*itr
)
315 struct intel_bts_recording
*btsr
=
316 container_of(itr
, struct intel_bts_recording
, itr
);
318 intel_bts_free_snapshot_refs(btsr
);
322 static int intel_bts_snapshot_start(struct auxtrace_record
*itr
)
324 struct intel_bts_recording
*btsr
=
325 container_of(itr
, struct intel_bts_recording
, itr
);
326 struct perf_evsel
*evsel
;
328 evlist__for_each(btsr
->evlist
, evsel
) {
329 if (evsel
->attr
.type
== btsr
->intel_bts_pmu
->type
)
330 return perf_evlist__disable_event(btsr
->evlist
, evsel
);
335 static int intel_bts_snapshot_finish(struct auxtrace_record
*itr
)
337 struct intel_bts_recording
*btsr
=
338 container_of(itr
, struct intel_bts_recording
, itr
);
339 struct perf_evsel
*evsel
;
341 evlist__for_each(btsr
->evlist
, evsel
) {
342 if (evsel
->attr
.type
== btsr
->intel_bts_pmu
->type
)
343 return perf_evlist__enable_event(btsr
->evlist
, evsel
);
348 static bool intel_bts_first_wrap(u64
*data
, size_t buf_size
)
357 for (i
= a
; i
< b
; i
++) {
365 static int intel_bts_find_snapshot(struct auxtrace_record
*itr
, int idx
,
366 struct auxtrace_mmap
*mm
, unsigned char *data
,
369 struct intel_bts_recording
*btsr
=
370 container_of(itr
, struct intel_bts_recording
, itr
);
374 pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
375 __func__
, idx
, (size_t)*old
, (size_t)*head
);
377 if (idx
>= btsr
->snapshot_ref_cnt
) {
378 err
= intel_bts_alloc_snapshot_refs(btsr
, idx
);
383 wrapped
= btsr
->snapshot_refs
[idx
].wrapped
;
384 if (!wrapped
&& intel_bts_first_wrap((u64
*)data
, mm
->len
)) {
385 btsr
->snapshot_refs
[idx
].wrapped
= true;
390 * In full trace mode 'head' continually increases. However in snapshot
391 * mode 'head' is an offset within the buffer. Here 'old' and 'head'
392 * are adjusted to match the full trace case which expects that 'old' is
393 * always less than 'head'.
407 pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
408 __func__
, wrapped
? "" : "not ", (size_t)*old
, (size_t)*head
);
413 pr_err("%s: failed, error %d\n", __func__
, err
);
417 static int intel_bts_read_finish(struct auxtrace_record
*itr
, int idx
)
419 struct intel_bts_recording
*btsr
=
420 container_of(itr
, struct intel_bts_recording
, itr
);
421 struct perf_evsel
*evsel
;
423 evlist__for_each(btsr
->evlist
, evsel
) {
424 if (evsel
->attr
.type
== btsr
->intel_bts_pmu
->type
)
425 return perf_evlist__enable_event_idx(btsr
->evlist
,
431 struct auxtrace_record
*intel_bts_recording_init(int *err
)
433 struct perf_pmu
*intel_bts_pmu
= perf_pmu__find(INTEL_BTS_PMU_NAME
);
434 struct intel_bts_recording
*btsr
;
439 btsr
= zalloc(sizeof(struct intel_bts_recording
));
445 btsr
->intel_bts_pmu
= intel_bts_pmu
;
446 btsr
->itr
.recording_options
= intel_bts_recording_options
;
447 btsr
->itr
.info_priv_size
= intel_bts_info_priv_size
;
448 btsr
->itr
.info_fill
= intel_bts_info_fill
;
449 btsr
->itr
.free
= intel_bts_recording_free
;
450 btsr
->itr
.snapshot_start
= intel_bts_snapshot_start
;
451 btsr
->itr
.snapshot_finish
= intel_bts_snapshot_finish
;
452 btsr
->itr
.find_snapshot
= intel_bts_find_snapshot
;
453 btsr
->itr
.parse_snapshot_options
= intel_bts_parse_snapshot_options
;
454 btsr
->itr
.reference
= intel_bts_reference
;
455 btsr
->itr
.read_finish
= intel_bts_read_finish
;
456 btsr
->itr
.alignment
= sizeof(struct branch
);