]>
Commit | Line | Data |
---|---|---|
90e457f7 AH |
1 | /* |
2 | * intel_pt.c: Intel Processor Trace support | |
3 | * Copyright (c) 2013-2015, Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | */ | |
15 | ||
16 | #include <stdbool.h> | |
17 | #include <linux/kernel.h> | |
18 | #include <linux/types.h> | |
19 | #include <linux/bitops.h> | |
20 | #include <linux/log2.h> | |
11fa7cb8 | 21 | #include <cpuid.h> |
90e457f7 AH |
22 | |
23 | #include "../../perf.h" | |
24 | #include "../../util/session.h" | |
25 | #include "../../util/event.h" | |
26 | #include "../../util/evlist.h" | |
27 | #include "../../util/evsel.h" | |
28 | #include "../../util/cpumap.h" | |
4b6ab94e | 29 | #include <subcmd/parse-options.h> |
90e457f7 AH |
30 | #include "../../util/parse-events.h" |
31 | #include "../../util/pmu.h" | |
32 | #include "../../util/debug.h" | |
33 | #include "../../util/auxtrace.h" | |
34 | #include "../../util/tsc.h" | |
35 | #include "../../util/intel-pt.h" | |
36 | ||
37 | #define KiB(x) ((x) * 1024) | |
38 | #define MiB(x) ((x) * 1024 * 1024) | |
39 | #define KiB_MASK(x) (KiB(x) - 1) | |
40 | #define MiB_MASK(x) (MiB(x) - 1) | |
41 | ||
42 | #define INTEL_PT_DEFAULT_SAMPLE_SIZE KiB(4) | |
43 | ||
44 | #define INTEL_PT_MAX_SAMPLE_SIZE KiB(60) | |
45 | ||
46 | #define INTEL_PT_PSB_PERIOD_NEAR 256 | |
47 | ||
48 | struct intel_pt_snapshot_ref { | |
49 | void *ref_buf; | |
50 | size_t ref_offset; | |
51 | bool wrapped; | |
52 | }; | |
53 | ||
54 | struct intel_pt_recording { | |
55 | struct auxtrace_record itr; | |
56 | struct perf_pmu *intel_pt_pmu; | |
57 | int have_sched_switch; | |
58 | struct perf_evlist *evlist; | |
59 | bool snapshot_mode; | |
60 | bool snapshot_init_done; | |
61 | size_t snapshot_size; | |
62 | size_t snapshot_ref_buf_size; | |
63 | int snapshot_ref_cnt; | |
64 | struct intel_pt_snapshot_ref *snapshot_refs; | |
65 | }; | |
66 | ||
67 | static int intel_pt_parse_terms_with_default(struct list_head *formats, | |
68 | const char *str, | |
69 | u64 *config) | |
70 | { | |
71 | struct list_head *terms; | |
72 | struct perf_event_attr attr = { .size = 0, }; | |
73 | int err; | |
74 | ||
75 | terms = malloc(sizeof(struct list_head)); | |
76 | if (!terms) | |
77 | return -ENOMEM; | |
78 | ||
79 | INIT_LIST_HEAD(terms); | |
80 | ||
81 | err = parse_events_terms(terms, str); | |
82 | if (err) | |
83 | goto out_free; | |
84 | ||
85 | attr.config = *config; | |
86 | err = perf_pmu__config_terms(formats, &attr, terms, true, NULL); | |
87 | if (err) | |
88 | goto out_free; | |
89 | ||
90 | *config = attr.config; | |
91 | out_free: | |
92 | parse_events__free_terms(terms); | |
93 | return err; | |
94 | } | |
95 | ||
96 | static int intel_pt_parse_terms(struct list_head *formats, const char *str, | |
97 | u64 *config) | |
98 | { | |
99 | *config = 0; | |
100 | return intel_pt_parse_terms_with_default(formats, str, config); | |
101 | } | |
102 | ||
bc9b6bf0 | 103 | static u64 intel_pt_masked_bits(u64 mask, u64 bits) |
90e457f7 | 104 | { |
bc9b6bf0 AH |
105 | const u64 top_bit = 1ULL << 63; |
106 | u64 res = 0; | |
107 | int i; | |
108 | ||
109 | for (i = 0; i < 64; i++) { | |
110 | if (mask & top_bit) { | |
111 | res <<= 1; | |
112 | if (bits & top_bit) | |
113 | res |= 1; | |
114 | } | |
115 | mask <<= 1; | |
116 | bits <<= 1; | |
117 | } | |
118 | ||
119 | return res; | |
120 | } | |
121 | ||
122 | static int intel_pt_read_config(struct perf_pmu *intel_pt_pmu, const char *str, | |
123 | struct perf_evlist *evlist, u64 *res) | |
124 | { | |
125 | struct perf_evsel *evsel; | |
126 | u64 mask; | |
127 | ||
128 | *res = 0; | |
129 | ||
130 | mask = perf_pmu__format_bits(&intel_pt_pmu->format, str); | |
131 | if (!mask) | |
132 | return -EINVAL; | |
133 | ||
134 | evlist__for_each(evlist, evsel) { | |
135 | if (evsel->attr.type == intel_pt_pmu->type) { | |
136 | *res = intel_pt_masked_bits(mask, evsel->attr.config); | |
137 | return 0; | |
138 | } | |
139 | } | |
140 | ||
141 | return -EINVAL; | |
142 | } | |
143 | ||
144 | static size_t intel_pt_psb_period(struct perf_pmu *intel_pt_pmu, | |
145 | struct perf_evlist *evlist) | |
146 | { | |
147 | u64 val; | |
148 | int err, topa_multiple_entries; | |
149 | size_t psb_period; | |
150 | ||
151 | if (perf_pmu__scan_file(intel_pt_pmu, "caps/topa_multiple_entries", | |
152 | "%d", &topa_multiple_entries) != 1) | |
153 | topa_multiple_entries = 0; | |
154 | ||
155 | /* | |
156 | * Use caps/topa_multiple_entries to indicate early hardware that had | |
157 | * extra frequent PSBs. | |
158 | */ | |
159 | if (!topa_multiple_entries) { | |
160 | psb_period = 256; | |
161 | goto out; | |
162 | } | |
163 | ||
164 | err = intel_pt_read_config(intel_pt_pmu, "psb_period", evlist, &val); | |
165 | if (err) | |
166 | val = 0; | |
167 | ||
168 | psb_period = 1 << (val + 11); | |
169 | out: | |
170 | pr_debug2("%s psb_period %zu\n", intel_pt_pmu->name, psb_period); | |
171 | return psb_period; | |
172 | } | |
173 | ||
174 | static int intel_pt_pick_bit(int bits, int target) | |
175 | { | |
176 | int pos, pick = -1; | |
177 | ||
178 | for (pos = 0; bits; bits >>= 1, pos++) { | |
179 | if (bits & 1) { | |
180 | if (pos <= target || pick < 0) | |
181 | pick = pos; | |
182 | if (pos >= target) | |
183 | break; | |
184 | } | |
185 | } | |
186 | ||
187 | return pick; | |
90e457f7 AH |
188 | } |
189 | ||
190 | static u64 intel_pt_default_config(struct perf_pmu *intel_pt_pmu) | |
191 | { | |
bc9b6bf0 | 192 | char buf[256]; |
b45fc0bf | 193 | int mtc, mtc_periods = 0, mtc_period; |
bc9b6bf0 AH |
194 | int psb_cyc, psb_periods, psb_period; |
195 | int pos = 0; | |
90e457f7 AH |
196 | u64 config; |
197 | ||
bc9b6bf0 AH |
198 | pos += scnprintf(buf + pos, sizeof(buf) - pos, "tsc"); |
199 | ||
b45fc0bf AH |
200 | if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc", "%d", |
201 | &mtc) != 1) | |
202 | mtc = 1; | |
203 | ||
204 | if (mtc) { | |
205 | if (perf_pmu__scan_file(intel_pt_pmu, "caps/mtc_periods", "%x", | |
206 | &mtc_periods) != 1) | |
207 | mtc_periods = 0; | |
208 | if (mtc_periods) { | |
209 | mtc_period = intel_pt_pick_bit(mtc_periods, 3); | |
210 | pos += scnprintf(buf + pos, sizeof(buf) - pos, | |
211 | ",mtc,mtc_period=%d", mtc_period); | |
212 | } | |
213 | } | |
214 | ||
bc9b6bf0 AH |
215 | if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_cyc", "%d", |
216 | &psb_cyc) != 1) | |
217 | psb_cyc = 1; | |
218 | ||
b45fc0bf | 219 | if (psb_cyc && mtc_periods) { |
bc9b6bf0 AH |
220 | if (perf_pmu__scan_file(intel_pt_pmu, "caps/psb_periods", "%x", |
221 | &psb_periods) != 1) | |
222 | psb_periods = 0; | |
223 | if (psb_periods) { | |
224 | psb_period = intel_pt_pick_bit(psb_periods, 3); | |
225 | pos += scnprintf(buf + pos, sizeof(buf) - pos, | |
226 | ",psb_period=%d", psb_period); | |
227 | } | |
228 | } | |
229 | ||
230 | pr_debug2("%s default config: %s\n", intel_pt_pmu->name, buf); | |
231 | ||
232 | intel_pt_parse_terms(&intel_pt_pmu->format, buf, &config); | |
233 | ||
90e457f7 AH |
234 | return config; |
235 | } | |
236 | ||
237 | static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr, | |
238 | struct record_opts *opts, | |
239 | const char *str) | |
240 | { | |
241 | struct intel_pt_recording *ptr = | |
242 | container_of(itr, struct intel_pt_recording, itr); | |
243 | unsigned long long snapshot_size = 0; | |
244 | char *endptr; | |
245 | ||
246 | if (str) { | |
247 | snapshot_size = strtoull(str, &endptr, 0); | |
248 | if (*endptr || snapshot_size > SIZE_MAX) | |
249 | return -1; | |
250 | } | |
251 | ||
252 | opts->auxtrace_snapshot_mode = true; | |
253 | opts->auxtrace_snapshot_size = snapshot_size; | |
254 | ||
255 | ptr->snapshot_size = snapshot_size; | |
256 | ||
257 | return 0; | |
258 | } | |
259 | ||
260 | struct perf_event_attr * | |
261 | intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu) | |
262 | { | |
263 | struct perf_event_attr *attr; | |
264 | ||
265 | attr = zalloc(sizeof(struct perf_event_attr)); | |
266 | if (!attr) | |
267 | return NULL; | |
268 | ||
269 | attr->config = intel_pt_default_config(intel_pt_pmu); | |
270 | ||
271 | intel_pt_pmu->selectable = true; | |
272 | ||
273 | return attr; | |
274 | } | |
275 | ||
276 | static size_t intel_pt_info_priv_size(struct auxtrace_record *itr __maybe_unused) | |
277 | { | |
278 | return INTEL_PT_AUXTRACE_PRIV_SIZE; | |
279 | } | |
280 | ||
11fa7cb8 AH |
281 | static void intel_pt_tsc_ctc_ratio(u32 *n, u32 *d) |
282 | { | |
283 | unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; | |
284 | ||
285 | __get_cpuid(0x15, &eax, &ebx, &ecx, &edx); | |
286 | *n = ebx; | |
287 | *d = eax; | |
288 | } | |
289 | ||
90e457f7 AH |
290 | static int intel_pt_info_fill(struct auxtrace_record *itr, |
291 | struct perf_session *session, | |
292 | struct auxtrace_info_event *auxtrace_info, | |
293 | size_t priv_size) | |
294 | { | |
295 | struct intel_pt_recording *ptr = | |
296 | container_of(itr, struct intel_pt_recording, itr); | |
297 | struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; | |
298 | struct perf_event_mmap_page *pc; | |
299 | struct perf_tsc_conversion tc = { .time_mult = 0, }; | |
300 | bool cap_user_time_zero = false, per_cpu_mmaps; | |
11fa7cb8 AH |
301 | u64 tsc_bit, mtc_bit, mtc_freq_bits, cyc_bit, noretcomp_bit; |
302 | u32 tsc_ctc_ratio_n, tsc_ctc_ratio_d; | |
90e457f7 AH |
303 | int err; |
304 | ||
305 | if (priv_size != INTEL_PT_AUXTRACE_PRIV_SIZE) | |
306 | return -EINVAL; | |
307 | ||
308 | intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); | |
309 | intel_pt_parse_terms(&intel_pt_pmu->format, "noretcomp", | |
310 | &noretcomp_bit); | |
11fa7cb8 AH |
311 | intel_pt_parse_terms(&intel_pt_pmu->format, "mtc", &mtc_bit); |
312 | mtc_freq_bits = perf_pmu__format_bits(&intel_pt_pmu->format, | |
313 | "mtc_period"); | |
314 | intel_pt_parse_terms(&intel_pt_pmu->format, "cyc", &cyc_bit); | |
315 | ||
316 | intel_pt_tsc_ctc_ratio(&tsc_ctc_ratio_n, &tsc_ctc_ratio_d); | |
90e457f7 AH |
317 | |
318 | if (!session->evlist->nr_mmaps) | |
319 | return -EINVAL; | |
320 | ||
321 | pc = session->evlist->mmap[0].base; | |
322 | if (pc) { | |
323 | err = perf_read_tsc_conversion(pc, &tc); | |
324 | if (err) { | |
325 | if (err != -EOPNOTSUPP) | |
326 | return err; | |
327 | } else { | |
328 | cap_user_time_zero = tc.time_mult != 0; | |
329 | } | |
330 | if (!cap_user_time_zero) | |
331 | ui__warning("Intel Processor Trace: TSC not available\n"); | |
332 | } | |
333 | ||
334 | per_cpu_mmaps = !cpu_map__empty(session->evlist->cpus); | |
335 | ||
336 | auxtrace_info->type = PERF_AUXTRACE_INTEL_PT; | |
337 | auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type; | |
338 | auxtrace_info->priv[INTEL_PT_TIME_SHIFT] = tc.time_shift; | |
339 | auxtrace_info->priv[INTEL_PT_TIME_MULT] = tc.time_mult; | |
340 | auxtrace_info->priv[INTEL_PT_TIME_ZERO] = tc.time_zero; | |
341 | auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO] = cap_user_time_zero; | |
342 | auxtrace_info->priv[INTEL_PT_TSC_BIT] = tsc_bit; | |
343 | auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT] = noretcomp_bit; | |
344 | auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH] = ptr->have_sched_switch; | |
345 | auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE] = ptr->snapshot_mode; | |
346 | auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS] = per_cpu_mmaps; | |
11fa7cb8 AH |
347 | auxtrace_info->priv[INTEL_PT_MTC_BIT] = mtc_bit; |
348 | auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS] = mtc_freq_bits; | |
349 | auxtrace_info->priv[INTEL_PT_TSC_CTC_N] = tsc_ctc_ratio_n; | |
350 | auxtrace_info->priv[INTEL_PT_TSC_CTC_D] = tsc_ctc_ratio_d; | |
351 | auxtrace_info->priv[INTEL_PT_CYC_BIT] = cyc_bit; | |
90e457f7 AH |
352 | |
353 | return 0; | |
354 | } | |
355 | ||
356 | static int intel_pt_track_switches(struct perf_evlist *evlist) | |
357 | { | |
358 | const char *sched_switch = "sched:sched_switch"; | |
359 | struct perf_evsel *evsel; | |
360 | int err; | |
361 | ||
362 | if (!perf_evlist__can_select_event(evlist, sched_switch)) | |
363 | return -EPERM; | |
364 | ||
365 | err = parse_events(evlist, sched_switch, NULL); | |
366 | if (err) { | |
367 | pr_debug2("%s: failed to parse %s, error %d\n", | |
368 | __func__, sched_switch, err); | |
369 | return err; | |
370 | } | |
371 | ||
372 | evsel = perf_evlist__last(evlist); | |
373 | ||
374 | perf_evsel__set_sample_bit(evsel, CPU); | |
375 | perf_evsel__set_sample_bit(evsel, TIME); | |
376 | ||
377 | evsel->system_wide = true; | |
378 | evsel->no_aux_samples = true; | |
379 | evsel->immediate = true; | |
380 | ||
381 | return 0; | |
382 | } | |
383 | ||
bc9b6bf0 AH |
384 | static void intel_pt_valid_str(char *str, size_t len, u64 valid) |
385 | { | |
386 | unsigned int val, last = 0, state = 1; | |
387 | int p = 0; | |
388 | ||
389 | str[0] = '\0'; | |
390 | ||
391 | for (val = 0; val <= 64; val++, valid >>= 1) { | |
392 | if (valid & 1) { | |
393 | last = val; | |
394 | switch (state) { | |
395 | case 0: | |
396 | p += scnprintf(str + p, len - p, ","); | |
397 | /* Fall through */ | |
398 | case 1: | |
399 | p += scnprintf(str + p, len - p, "%u", val); | |
400 | state = 2; | |
401 | break; | |
402 | case 2: | |
403 | state = 3; | |
404 | break; | |
405 | case 3: | |
406 | state = 4; | |
407 | break; | |
408 | default: | |
409 | break; | |
410 | } | |
411 | } else { | |
412 | switch (state) { | |
413 | case 3: | |
414 | p += scnprintf(str + p, len - p, ",%u", last); | |
415 | state = 0; | |
416 | break; | |
417 | case 4: | |
418 | p += scnprintf(str + p, len - p, "-%u", last); | |
419 | state = 0; | |
420 | break; | |
421 | default: | |
422 | break; | |
423 | } | |
424 | if (state != 1) | |
425 | state = 0; | |
426 | } | |
427 | } | |
428 | } | |
429 | ||
430 | static int intel_pt_val_config_term(struct perf_pmu *intel_pt_pmu, | |
431 | const char *caps, const char *name, | |
432 | const char *supported, u64 config) | |
433 | { | |
434 | char valid_str[256]; | |
435 | unsigned int shift; | |
436 | unsigned long long valid; | |
437 | u64 bits; | |
438 | int ok; | |
439 | ||
440 | if (perf_pmu__scan_file(intel_pt_pmu, caps, "%llx", &valid) != 1) | |
441 | valid = 0; | |
442 | ||
443 | if (supported && | |
444 | perf_pmu__scan_file(intel_pt_pmu, supported, "%d", &ok) == 1 && !ok) | |
445 | valid = 0; | |
446 | ||
447 | valid |= 1; | |
448 | ||
449 | bits = perf_pmu__format_bits(&intel_pt_pmu->format, name); | |
450 | ||
451 | config &= bits; | |
452 | ||
453 | for (shift = 0; bits && !(bits & 1); shift++) | |
454 | bits >>= 1; | |
455 | ||
456 | config >>= shift; | |
457 | ||
458 | if (config > 63) | |
459 | goto out_err; | |
460 | ||
461 | if (valid & (1 << config)) | |
462 | return 0; | |
463 | out_err: | |
464 | intel_pt_valid_str(valid_str, sizeof(valid_str), valid); | |
465 | pr_err("Invalid %s for %s. Valid values are: %s\n", | |
466 | name, INTEL_PT_PMU_NAME, valid_str); | |
467 | return -EINVAL; | |
468 | } | |
469 | ||
470 | static int intel_pt_validate_config(struct perf_pmu *intel_pt_pmu, | |
471 | struct perf_evsel *evsel) | |
472 | { | |
b45fc0bf AH |
473 | int err; |
474 | ||
bc9b6bf0 AH |
475 | if (!evsel) |
476 | return 0; | |
477 | ||
0de802ab AH |
478 | err = intel_pt_val_config_term(intel_pt_pmu, "caps/cycle_thresholds", |
479 | "cyc_thresh", "caps/psb_cyc", | |
480 | evsel->attr.config); | |
481 | if (err) | |
482 | return err; | |
483 | ||
b45fc0bf AH |
484 | err = intel_pt_val_config_term(intel_pt_pmu, "caps/mtc_periods", |
485 | "mtc_period", "caps/mtc", | |
486 | evsel->attr.config); | |
487 | if (err) | |
488 | return err; | |
489 | ||
bc9b6bf0 AH |
490 | return intel_pt_val_config_term(intel_pt_pmu, "caps/psb_periods", |
491 | "psb_period", "caps/psb_cyc", | |
492 | evsel->attr.config); | |
493 | } | |
494 | ||
90e457f7 AH |
495 | static int intel_pt_recording_options(struct auxtrace_record *itr, |
496 | struct perf_evlist *evlist, | |
497 | struct record_opts *opts) | |
498 | { | |
499 | struct intel_pt_recording *ptr = | |
500 | container_of(itr, struct intel_pt_recording, itr); | |
501 | struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu; | |
502 | bool have_timing_info; | |
503 | struct perf_evsel *evsel, *intel_pt_evsel = NULL; | |
504 | const struct cpu_map *cpus = evlist->cpus; | |
505 | bool privileged = geteuid() == 0 || perf_event_paranoid() < 0; | |
506 | u64 tsc_bit; | |
bc9b6bf0 | 507 | int err; |
90e457f7 AH |
508 | |
509 | ptr->evlist = evlist; | |
510 | ptr->snapshot_mode = opts->auxtrace_snapshot_mode; | |
511 | ||
512 | evlist__for_each(evlist, evsel) { | |
513 | if (evsel->attr.type == intel_pt_pmu->type) { | |
514 | if (intel_pt_evsel) { | |
515 | pr_err("There may be only one " INTEL_PT_PMU_NAME " event\n"); | |
516 | return -EINVAL; | |
517 | } | |
518 | evsel->attr.freq = 0; | |
519 | evsel->attr.sample_period = 1; | |
520 | intel_pt_evsel = evsel; | |
521 | opts->full_auxtrace = true; | |
522 | } | |
523 | } | |
524 | ||
525 | if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) { | |
526 | pr_err("Snapshot mode (-S option) requires " INTEL_PT_PMU_NAME " PMU event (-e " INTEL_PT_PMU_NAME ")\n"); | |
527 | return -EINVAL; | |
528 | } | |
529 | ||
530 | if (opts->use_clockid) { | |
531 | pr_err("Cannot use clockid (-k option) with " INTEL_PT_PMU_NAME "\n"); | |
532 | return -EINVAL; | |
533 | } | |
534 | ||
535 | if (!opts->full_auxtrace) | |
536 | return 0; | |
537 | ||
bc9b6bf0 AH |
538 | err = intel_pt_validate_config(intel_pt_pmu, intel_pt_evsel); |
539 | if (err) | |
540 | return err; | |
541 | ||
90e457f7 AH |
542 | /* Set default sizes for snapshot mode */ |
543 | if (opts->auxtrace_snapshot_mode) { | |
544 | size_t psb_period = intel_pt_psb_period(intel_pt_pmu, evlist); | |
545 | ||
546 | if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) { | |
547 | if (privileged) { | |
548 | opts->auxtrace_mmap_pages = MiB(4) / page_size; | |
549 | } else { | |
550 | opts->auxtrace_mmap_pages = KiB(128) / page_size; | |
551 | if (opts->mmap_pages == UINT_MAX) | |
552 | opts->mmap_pages = KiB(256) / page_size; | |
553 | } | |
554 | } else if (!opts->auxtrace_mmap_pages && !privileged && | |
555 | opts->mmap_pages == UINT_MAX) { | |
556 | opts->mmap_pages = KiB(256) / page_size; | |
557 | } | |
558 | if (!opts->auxtrace_snapshot_size) | |
559 | opts->auxtrace_snapshot_size = | |
560 | opts->auxtrace_mmap_pages * (size_t)page_size; | |
561 | if (!opts->auxtrace_mmap_pages) { | |
562 | size_t sz = opts->auxtrace_snapshot_size; | |
563 | ||
564 | sz = round_up(sz, page_size) / page_size; | |
565 | opts->auxtrace_mmap_pages = roundup_pow_of_two(sz); | |
566 | } | |
567 | if (opts->auxtrace_snapshot_size > | |
568 | opts->auxtrace_mmap_pages * (size_t)page_size) { | |
569 | pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n", | |
570 | opts->auxtrace_snapshot_size, | |
571 | opts->auxtrace_mmap_pages * (size_t)page_size); | |
572 | return -EINVAL; | |
573 | } | |
574 | if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) { | |
575 | pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n"); | |
576 | return -EINVAL; | |
577 | } | |
578 | pr_debug2("Intel PT snapshot size: %zu\n", | |
579 | opts->auxtrace_snapshot_size); | |
580 | if (psb_period && | |
581 | opts->auxtrace_snapshot_size <= psb_period + | |
582 | INTEL_PT_PSB_PERIOD_NEAR) | |
583 | ui__warning("Intel PT snapshot size (%zu) may be too small for PSB period (%zu)\n", | |
584 | opts->auxtrace_snapshot_size, psb_period); | |
585 | } | |
586 | ||
587 | /* Set default sizes for full trace mode */ | |
588 | if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { | |
589 | if (privileged) { | |
590 | opts->auxtrace_mmap_pages = MiB(4) / page_size; | |
591 | } else { | |
592 | opts->auxtrace_mmap_pages = KiB(128) / page_size; | |
593 | if (opts->mmap_pages == UINT_MAX) | |
594 | opts->mmap_pages = KiB(256) / page_size; | |
595 | } | |
596 | } | |
597 | ||
598 | /* Validate auxtrace_mmap_pages */ | |
599 | if (opts->auxtrace_mmap_pages) { | |
600 | size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; | |
601 | size_t min_sz; | |
602 | ||
603 | if (opts->auxtrace_snapshot_mode) | |
604 | min_sz = KiB(4); | |
605 | else | |
606 | min_sz = KiB(8); | |
607 | ||
608 | if (sz < min_sz || !is_power_of_2(sz)) { | |
609 | pr_err("Invalid mmap size for Intel Processor Trace: must be at least %zuKiB and a power of 2\n", | |
610 | min_sz / 1024); | |
611 | return -EINVAL; | |
612 | } | |
613 | } | |
614 | ||
615 | intel_pt_parse_terms(&intel_pt_pmu->format, "tsc", &tsc_bit); | |
616 | ||
617 | if (opts->full_auxtrace && (intel_pt_evsel->attr.config & tsc_bit)) | |
618 | have_timing_info = true; | |
619 | else | |
620 | have_timing_info = false; | |
621 | ||
622 | /* | |
623 | * Per-cpu recording needs sched_switch events to distinguish different | |
624 | * threads. | |
625 | */ | |
626 | if (have_timing_info && !cpu_map__empty(cpus)) { | |
86c27869 AH |
627 | if (perf_can_record_switch_events()) { |
628 | bool cpu_wide = !target__none(&opts->target) && | |
629 | !target__has_task(&opts->target); | |
630 | ||
631 | if (!cpu_wide && perf_can_record_cpu_wide()) { | |
632 | struct perf_evsel *switch_evsel; | |
633 | ||
634 | err = parse_events(evlist, "dummy:u", NULL); | |
635 | if (err) | |
636 | return err; | |
637 | ||
638 | switch_evsel = perf_evlist__last(evlist); | |
639 | ||
640 | switch_evsel->attr.freq = 0; | |
641 | switch_evsel->attr.sample_period = 1; | |
642 | switch_evsel->attr.context_switch = 1; | |
643 | ||
644 | switch_evsel->system_wide = true; | |
645 | switch_evsel->no_aux_samples = true; | |
646 | switch_evsel->immediate = true; | |
647 | ||
648 | perf_evsel__set_sample_bit(switch_evsel, TID); | |
649 | perf_evsel__set_sample_bit(switch_evsel, TIME); | |
650 | perf_evsel__set_sample_bit(switch_evsel, CPU); | |
651 | ||
652 | opts->record_switch_events = false; | |
653 | ptr->have_sched_switch = 3; | |
654 | } else { | |
655 | opts->record_switch_events = true; | |
656 | if (cpu_wide) | |
657 | ptr->have_sched_switch = 3; | |
658 | else | |
659 | ptr->have_sched_switch = 2; | |
660 | } | |
661 | } else { | |
662 | err = intel_pt_track_switches(evlist); | |
663 | if (err == -EPERM) | |
664 | pr_debug2("Unable to select sched:sched_switch\n"); | |
665 | else if (err) | |
666 | return err; | |
667 | else | |
668 | ptr->have_sched_switch = 1; | |
669 | } | |
90e457f7 AH |
670 | } |
671 | ||
672 | if (intel_pt_evsel) { | |
673 | /* | |
674 | * To obtain the auxtrace buffer file descriptor, the auxtrace | |
675 | * event must come first. | |
676 | */ | |
677 | perf_evlist__to_front(evlist, intel_pt_evsel); | |
678 | /* | |
679 | * In the case of per-cpu mmaps, we need the CPU on the | |
680 | * AUX event. | |
681 | */ | |
682 | if (!cpu_map__empty(cpus)) | |
683 | perf_evsel__set_sample_bit(intel_pt_evsel, CPU); | |
684 | } | |
685 | ||
686 | /* Add dummy event to keep tracking */ | |
687 | if (opts->full_auxtrace) { | |
688 | struct perf_evsel *tracking_evsel; | |
90e457f7 AH |
689 | |
690 | err = parse_events(evlist, "dummy:u", NULL); | |
691 | if (err) | |
692 | return err; | |
693 | ||
694 | tracking_evsel = perf_evlist__last(evlist); | |
695 | ||
696 | perf_evlist__set_tracking_event(evlist, tracking_evsel); | |
697 | ||
698 | tracking_evsel->attr.freq = 0; | |
699 | tracking_evsel->attr.sample_period = 1; | |
700 | ||
701 | /* In per-cpu case, always need the time of mmap events etc */ | |
86c27869 | 702 | if (!cpu_map__empty(cpus)) { |
90e457f7 | 703 | perf_evsel__set_sample_bit(tracking_evsel, TIME); |
86c27869 AH |
704 | /* And the CPU for switch events */ |
705 | perf_evsel__set_sample_bit(tracking_evsel, CPU); | |
706 | } | |
90e457f7 AH |
707 | } |
708 | ||
709 | /* | |
710 | * Warn the user when we do not have enough information to decode i.e. | |
711 | * per-cpu with no sched_switch (except workload-only). | |
712 | */ | |
713 | if (!ptr->have_sched_switch && !cpu_map__empty(cpus) && | |
714 | !target__none(&opts->target)) | |
715 | ui__warning("Intel Processor Trace decoding will not be possible except for kernel tracing!\n"); | |
716 | ||
717 | return 0; | |
718 | } | |
719 | ||
720 | static int intel_pt_snapshot_start(struct auxtrace_record *itr) | |
721 | { | |
722 | struct intel_pt_recording *ptr = | |
723 | container_of(itr, struct intel_pt_recording, itr); | |
724 | struct perf_evsel *evsel; | |
725 | ||
726 | evlist__for_each(ptr->evlist, evsel) { | |
727 | if (evsel->attr.type == ptr->intel_pt_pmu->type) | |
d2190a80 | 728 | return perf_evsel__disable(evsel); |
90e457f7 AH |
729 | } |
730 | return -EINVAL; | |
731 | } | |
732 | ||
733 | static int intel_pt_snapshot_finish(struct auxtrace_record *itr) | |
734 | { | |
735 | struct intel_pt_recording *ptr = | |
736 | container_of(itr, struct intel_pt_recording, itr); | |
737 | struct perf_evsel *evsel; | |
738 | ||
739 | evlist__for_each(ptr->evlist, evsel) { | |
740 | if (evsel->attr.type == ptr->intel_pt_pmu->type) | |
d2190a80 | 741 | return perf_evsel__enable(evsel); |
90e457f7 AH |
742 | } |
743 | return -EINVAL; | |
744 | } | |
745 | ||
746 | static int intel_pt_alloc_snapshot_refs(struct intel_pt_recording *ptr, int idx) | |
747 | { | |
748 | const size_t sz = sizeof(struct intel_pt_snapshot_ref); | |
749 | int cnt = ptr->snapshot_ref_cnt, new_cnt = cnt * 2; | |
750 | struct intel_pt_snapshot_ref *refs; | |
751 | ||
752 | if (!new_cnt) | |
753 | new_cnt = 16; | |
754 | ||
755 | while (new_cnt <= idx) | |
756 | new_cnt *= 2; | |
757 | ||
758 | refs = calloc(new_cnt, sz); | |
759 | if (!refs) | |
760 | return -ENOMEM; | |
761 | ||
762 | memcpy(refs, ptr->snapshot_refs, cnt * sz); | |
763 | ||
764 | ptr->snapshot_refs = refs; | |
765 | ptr->snapshot_ref_cnt = new_cnt; | |
766 | ||
767 | return 0; | |
768 | } | |
769 | ||
770 | static void intel_pt_free_snapshot_refs(struct intel_pt_recording *ptr) | |
771 | { | |
772 | int i; | |
773 | ||
774 | for (i = 0; i < ptr->snapshot_ref_cnt; i++) | |
775 | zfree(&ptr->snapshot_refs[i].ref_buf); | |
776 | zfree(&ptr->snapshot_refs); | |
777 | } | |
778 | ||
779 | static void intel_pt_recording_free(struct auxtrace_record *itr) | |
780 | { | |
781 | struct intel_pt_recording *ptr = | |
782 | container_of(itr, struct intel_pt_recording, itr); | |
783 | ||
784 | intel_pt_free_snapshot_refs(ptr); | |
785 | free(ptr); | |
786 | } | |
787 | ||
788 | static int intel_pt_alloc_snapshot_ref(struct intel_pt_recording *ptr, int idx, | |
789 | size_t snapshot_buf_size) | |
790 | { | |
791 | size_t ref_buf_size = ptr->snapshot_ref_buf_size; | |
792 | void *ref_buf; | |
793 | ||
794 | ref_buf = zalloc(ref_buf_size); | |
795 | if (!ref_buf) | |
796 | return -ENOMEM; | |
797 | ||
798 | ptr->snapshot_refs[idx].ref_buf = ref_buf; | |
799 | ptr->snapshot_refs[idx].ref_offset = snapshot_buf_size - ref_buf_size; | |
800 | ||
801 | return 0; | |
802 | } | |
803 | ||
804 | static size_t intel_pt_snapshot_ref_buf_size(struct intel_pt_recording *ptr, | |
805 | size_t snapshot_buf_size) | |
806 | { | |
807 | const size_t max_size = 256 * 1024; | |
808 | size_t buf_size = 0, psb_period; | |
809 | ||
810 | if (ptr->snapshot_size <= 64 * 1024) | |
811 | return 0; | |
812 | ||
813 | psb_period = intel_pt_psb_period(ptr->intel_pt_pmu, ptr->evlist); | |
814 | if (psb_period) | |
815 | buf_size = psb_period * 2; | |
816 | ||
817 | if (!buf_size || buf_size > max_size) | |
818 | buf_size = max_size; | |
819 | ||
820 | if (buf_size >= snapshot_buf_size) | |
821 | return 0; | |
822 | ||
823 | if (buf_size >= ptr->snapshot_size / 2) | |
824 | return 0; | |
825 | ||
826 | return buf_size; | |
827 | } | |
828 | ||
829 | static int intel_pt_snapshot_init(struct intel_pt_recording *ptr, | |
830 | size_t snapshot_buf_size) | |
831 | { | |
832 | if (ptr->snapshot_init_done) | |
833 | return 0; | |
834 | ||
835 | ptr->snapshot_init_done = true; | |
836 | ||
837 | ptr->snapshot_ref_buf_size = intel_pt_snapshot_ref_buf_size(ptr, | |
838 | snapshot_buf_size); | |
839 | ||
840 | return 0; | |
841 | } | |
842 | ||
843 | /** | |
844 | * intel_pt_compare_buffers - compare bytes in a buffer to a circular buffer. | |
845 | * @buf1: first buffer | |
846 | * @compare_size: number of bytes to compare | |
847 | * @buf2: second buffer (a circular buffer) | |
848 | * @offs2: offset in second buffer | |
849 | * @buf2_size: size of second buffer | |
850 | * | |
851 | * The comparison allows for the possibility that the bytes to compare in the | |
852 | * circular buffer are not contiguous. It is assumed that @compare_size <= | |
853 | * @buf2_size. This function returns %false if the bytes are identical, %true | |
854 | * otherwise. | |
855 | */ | |
856 | static bool intel_pt_compare_buffers(void *buf1, size_t compare_size, | |
857 | void *buf2, size_t offs2, size_t buf2_size) | |
858 | { | |
859 | size_t end2 = offs2 + compare_size, part_size; | |
860 | ||
861 | if (end2 <= buf2_size) | |
862 | return memcmp(buf1, buf2 + offs2, compare_size); | |
863 | ||
864 | part_size = end2 - buf2_size; | |
865 | if (memcmp(buf1, buf2 + offs2, part_size)) | |
866 | return true; | |
867 | ||
868 | compare_size -= part_size; | |
869 | ||
870 | return memcmp(buf1 + part_size, buf2, compare_size); | |
871 | } | |
872 | ||
873 | static bool intel_pt_compare_ref(void *ref_buf, size_t ref_offset, | |
874 | size_t ref_size, size_t buf_size, | |
875 | void *data, size_t head) | |
876 | { | |
877 | size_t ref_end = ref_offset + ref_size; | |
878 | ||
879 | if (ref_end > buf_size) { | |
880 | if (head > ref_offset || head < ref_end - buf_size) | |
881 | return true; | |
882 | } else if (head > ref_offset && head < ref_end) { | |
883 | return true; | |
884 | } | |
885 | ||
886 | return intel_pt_compare_buffers(ref_buf, ref_size, data, ref_offset, | |
887 | buf_size); | |
888 | } | |
889 | ||
890 | static void intel_pt_copy_ref(void *ref_buf, size_t ref_size, size_t buf_size, | |
891 | void *data, size_t head) | |
892 | { | |
893 | if (head >= ref_size) { | |
894 | memcpy(ref_buf, data + head - ref_size, ref_size); | |
895 | } else { | |
896 | memcpy(ref_buf, data, head); | |
897 | ref_size -= head; | |
898 | memcpy(ref_buf + head, data + buf_size - ref_size, ref_size); | |
899 | } | |
900 | } | |
901 | ||
902 | static bool intel_pt_wrapped(struct intel_pt_recording *ptr, int idx, | |
903 | struct auxtrace_mmap *mm, unsigned char *data, | |
904 | u64 head) | |
905 | { | |
906 | struct intel_pt_snapshot_ref *ref = &ptr->snapshot_refs[idx]; | |
907 | bool wrapped; | |
908 | ||
909 | wrapped = intel_pt_compare_ref(ref->ref_buf, ref->ref_offset, | |
910 | ptr->snapshot_ref_buf_size, mm->len, | |
911 | data, head); | |
912 | ||
913 | intel_pt_copy_ref(ref->ref_buf, ptr->snapshot_ref_buf_size, mm->len, | |
914 | data, head); | |
915 | ||
916 | return wrapped; | |
917 | } | |
918 | ||
919 | static bool intel_pt_first_wrap(u64 *data, size_t buf_size) | |
920 | { | |
921 | int i, a, b; | |
922 | ||
923 | b = buf_size >> 3; | |
924 | a = b - 512; | |
925 | if (a < 0) | |
926 | a = 0; | |
927 | ||
928 | for (i = a; i < b; i++) { | |
929 | if (data[i]) | |
930 | return true; | |
931 | } | |
932 | ||
933 | return false; | |
934 | } | |
935 | ||
936 | static int intel_pt_find_snapshot(struct auxtrace_record *itr, int idx, | |
937 | struct auxtrace_mmap *mm, unsigned char *data, | |
938 | u64 *head, u64 *old) | |
939 | { | |
940 | struct intel_pt_recording *ptr = | |
941 | container_of(itr, struct intel_pt_recording, itr); | |
942 | bool wrapped; | |
943 | int err; | |
944 | ||
945 | pr_debug3("%s: mmap index %d old head %zu new head %zu\n", | |
946 | __func__, idx, (size_t)*old, (size_t)*head); | |
947 | ||
948 | err = intel_pt_snapshot_init(ptr, mm->len); | |
949 | if (err) | |
950 | goto out_err; | |
951 | ||
952 | if (idx >= ptr->snapshot_ref_cnt) { | |
953 | err = intel_pt_alloc_snapshot_refs(ptr, idx); | |
954 | if (err) | |
955 | goto out_err; | |
956 | } | |
957 | ||
958 | if (ptr->snapshot_ref_buf_size) { | |
959 | if (!ptr->snapshot_refs[idx].ref_buf) { | |
960 | err = intel_pt_alloc_snapshot_ref(ptr, idx, mm->len); | |
961 | if (err) | |
962 | goto out_err; | |
963 | } | |
964 | wrapped = intel_pt_wrapped(ptr, idx, mm, data, *head); | |
965 | } else { | |
966 | wrapped = ptr->snapshot_refs[idx].wrapped; | |
967 | if (!wrapped && intel_pt_first_wrap((u64 *)data, mm->len)) { | |
968 | ptr->snapshot_refs[idx].wrapped = true; | |
969 | wrapped = true; | |
970 | } | |
971 | } | |
972 | ||
973 | /* | |
974 | * In full trace mode 'head' continually increases. However in snapshot | |
975 | * mode 'head' is an offset within the buffer. Here 'old' and 'head' | |
976 | * are adjusted to match the full trace case which expects that 'old' is | |
977 | * always less than 'head'. | |
978 | */ | |
979 | if (wrapped) { | |
980 | *old = *head; | |
981 | *head += mm->len; | |
982 | } else { | |
983 | if (mm->mask) | |
984 | *old &= mm->mask; | |
985 | else | |
986 | *old %= mm->len; | |
987 | if (*old > *head) | |
988 | *head += mm->len; | |
989 | } | |
990 | ||
991 | pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n", | |
992 | __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head); | |
993 | ||
994 | return 0; | |
995 | ||
996 | out_err: | |
997 | pr_err("%s: failed, error %d\n", __func__, err); | |
998 | return err; | |
999 | } | |
1000 | ||
1001 | static u64 intel_pt_reference(struct auxtrace_record *itr __maybe_unused) | |
1002 | { | |
1003 | return rdtsc(); | |
1004 | } | |
1005 | ||
1006 | static int intel_pt_read_finish(struct auxtrace_record *itr, int idx) | |
1007 | { | |
1008 | struct intel_pt_recording *ptr = | |
1009 | container_of(itr, struct intel_pt_recording, itr); | |
1010 | struct perf_evsel *evsel; | |
1011 | ||
1012 | evlist__for_each(ptr->evlist, evsel) { | |
1013 | if (evsel->attr.type == ptr->intel_pt_pmu->type) | |
1014 | return perf_evlist__enable_event_idx(ptr->evlist, evsel, | |
1015 | idx); | |
1016 | } | |
1017 | return -EINVAL; | |
1018 | } | |
1019 | ||
1020 | struct auxtrace_record *intel_pt_recording_init(int *err) | |
1021 | { | |
1022 | struct perf_pmu *intel_pt_pmu = perf_pmu__find(INTEL_PT_PMU_NAME); | |
1023 | struct intel_pt_recording *ptr; | |
1024 | ||
1025 | if (!intel_pt_pmu) | |
1026 | return NULL; | |
1027 | ||
1028 | ptr = zalloc(sizeof(struct intel_pt_recording)); | |
1029 | if (!ptr) { | |
1030 | *err = -ENOMEM; | |
1031 | return NULL; | |
1032 | } | |
1033 | ||
1034 | ptr->intel_pt_pmu = intel_pt_pmu; | |
1035 | ptr->itr.recording_options = intel_pt_recording_options; | |
1036 | ptr->itr.info_priv_size = intel_pt_info_priv_size; | |
1037 | ptr->itr.info_fill = intel_pt_info_fill; | |
1038 | ptr->itr.free = intel_pt_recording_free; | |
1039 | ptr->itr.snapshot_start = intel_pt_snapshot_start; | |
1040 | ptr->itr.snapshot_finish = intel_pt_snapshot_finish; | |
1041 | ptr->itr.find_snapshot = intel_pt_find_snapshot; | |
1042 | ptr->itr.parse_snapshot_options = intel_pt_parse_snapshot_options; | |
1043 | ptr->itr.reference = intel_pt_reference; | |
1044 | ptr->itr.read_finish = intel_pt_read_finish; | |
1045 | return &ptr->itr; | |
1046 | } |