]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - tools/perf/util/auxtrace.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf: Cleanup for HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT
[mirror_ubuntu-jammy-kernel.git] / tools / perf / util / auxtrace.c
CommitLineData
2025cf9e 1// SPDX-License-Identifier: GPL-2.0-only
718c602d
AH		 2/*
3 * auxtrace.c: AUX area trace support
4 * Copyright (c) 2013-2015, Intel Corporation.
718c602d
AH		 5 */
6
fd20e811 7#include <inttypes.h>
718c602d
AH		 8#include <sys/types.h>
9#include <sys/mman.h>
10#include <stdbool.h>
1b36c03e
AH		 11#include <string.h>
12#include <limits.h>
13#include <errno.h>
718c602d
AH		 14
15#include <linux/kernel.h>
16#include <linux/perf_event.h>
17#include <linux/types.h>
18#include <linux/bitops.h>
19#include <linux/log2.h>
e5027893 20#include <linux/string.h>
16bd4321 21#include <linux/time64.h>
718c602d 22
e5027893 23#include <sys/param.h>
9e0cc4fe 24#include <stdlib.h>
85ed4729 25#include <stdio.h>
e5027893 26#include <linux/list.h>
7f7c536f 27#include <linux/zalloc.h>
9e0cc4fe 28
718c602d 29#include "evlist.h"
1b36c03e
AH		 30#include "dso.h"
31#include "map.h"
32#include "pmu.h"
33#include "evsel.h"
eb7a52d4 34#include "evsel_config.h"
7cadca8e 35#include "symbol.h"
40c7d246 36#include "util/perf_api_probe.h"
ea49e01c 37#include "util/synthetic-events.h"
718c602d
AH		 38#include "thread_map.h"
39#include "asm/bug.h"
40#include "auxtrace.h"
41
c3278f02
AH		 42#include <linux/hash.h>
43
9e0cc4fe 44#include "event.h"
aeb00b1a 45#include "record.h"
85ed4729 46#include "session.h"
9e0cc4fe 47#include "debug.h"
4b6ab94e 48#include <subcmd/parse-options.h>
9e0cc4fe 49
440a23b3 50#include "cs-etm.h"
5efb1d54 51#include "intel-pt.h"
d0170af7 52#include "intel-bts.h"
ffd3d18c 53#include "arm-spe.h"
b96e6615 54#include "s390-cpumsf.h"
e0fcfb08 55#include "util/mmap.h"
5efb1d54 56
3052ba56 57#include <linux/ctype.h>
3d689ed6 58#include "symbol/kallsyms.h"
fb71c86c 59#include <internal/lib.h>
3d689ed6 60
eb7a52d4
AH		 61/*
62 * Make a group from 'leader' to 'last', requiring that the events were not
63 * already grouped to a different leader.
64 */
64b4778b 65static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct evsel *last)
eb7a52d4
AH		 66{
67 struct evsel *evsel;
68 bool grp;
69
c754c382 70 if (!evsel__is_group_leader(leader))
eb7a52d4
AH		 71 return -EINVAL;
72
73 grp = false;
74 evlist__for_each_entry(evlist, evsel) {
75 if (grp) {
fba7c866
JO		 76 if (!(evsel__leader(evsel) == leader ||
77 (evsel__leader(evsel) == evsel &&
eb7a52d4
AH		 78 evsel->core.nr_members <= 1)))
79 return -EINVAL;
80 } else if (evsel == leader) {
81 grp = true;
82 }
83 if (evsel == last)
84 break;
85 }
86
87 grp = false;
88 evlist__for_each_entry(evlist, evsel) {
89 if (grp) {
fba7c866
JO		 90 if (!evsel__has_leader(evsel, leader)) {
91 evsel__set_leader(evsel, leader);
eb7a52d4
AH		 92 if (leader->core.nr_members < 1)
93 leader->core.nr_members = 1;
94 leader->core.nr_members += 1;
95 }
96 } else if (evsel == leader) {
97 grp = true;
98 }
99 if (evsel == last)
100 break;
101 }
102
103 return 0;
104}
105
2e2967f4
AH		 106static bool auxtrace__dont_decode(struct perf_session *session)
107{
108 return !session->itrace_synth_opts ||
109 session->itrace_synth_opts->dont_decode;
110}
111
718c602d
AH		 112int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
113 struct auxtrace_mmap_params *mp,
114 void *userpg, int fd)
115{
116 struct perf_event_mmap_page *pc = userpg;
117
718c602d
AH		 118 WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");
119
120 mm->userpg = userpg;
121 mm->mask = mp->mask;
122 mm->len = mp->len;
123 mm->prev = 0;
124 mm->idx = mp->idx;
125 mm->tid = mp->tid;
126 mm->cpu = mp->cpu;
127
128 if (!mp->len) {
129 mm->base = NULL;
130 return 0;
131 }
132
133 pc->aux_offset = mp->offset;
134 pc->aux_size = mp->len;
135
136 mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);
137 if (mm->base == MAP_FAILED) {
138 pr_debug2("failed to mmap AUX area\n");
139 mm->base = NULL;
140 return -1;
141 }
142
143 return 0;
144}
145
146void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)
147{
148 if (mm->base) {
149 munmap(mm->base, mm->len);
150 mm->base = NULL;
151 }
152}
153
154void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
155 off_t auxtrace_offset,
156 unsigned int auxtrace_pages,
157 bool auxtrace_overwrite)
158{
159 if (auxtrace_pages) {
160 mp->offset = auxtrace_offset;
161 mp->len = auxtrace_pages * (size_t)page_size;
162 mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;
163 mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);
164 pr_debug2("AUX area mmap length %zu\n", mp->len);
165 } else {
166 mp->len = 0;
167 }
168}
169
170void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
63503dba 171 struct evlist *evlist, int idx,
718c602d
AH		 172 bool per_cpu)
173{
174 mp->idx = idx;
175
176 if (per_cpu) {
f72f901d 177 mp->cpu = evlist->core.cpus->map[idx];
03617c22 178 if (evlist->core.threads)
a2f354e3 179 mp->tid = perf_thread_map__pid(evlist->core.threads, 0);
718c602d
AH		 180 else
181 mp->tid = -1;
182 } else {
183 mp->cpu = -1;
a2f354e3 184 mp->tid = perf_thread_map__pid(evlist->core.threads, idx);
718c602d
AH		 185 }
186}
9e0cc4fe 187
e5027893
AH		 188#define AUXTRACE_INIT_NR_QUEUES 32
189
190static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)
191{
192 struct auxtrace_queue *queue_array;
193 unsigned int max_nr_queues, i;
194
195 max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);
196 if (nr_queues > max_nr_queues)
197 return NULL;
198
199 queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));
200 if (!queue_array)
201 return NULL;
202
203 for (i = 0; i < nr_queues; i++) {
204 INIT_LIST_HEAD(&queue_array[i].head);
205 queue_array[i].priv = NULL;
206 }
207
208 return queue_array;
209}
210
211int auxtrace_queues__init(struct auxtrace_queues *queues)
212{
213 queues->nr_queues = AUXTRACE_INIT_NR_QUEUES;
214 queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);
215 if (!queues->queue_array)
216 return -ENOMEM;
217 return 0;
218}
219
220static int auxtrace_queues__grow(struct auxtrace_queues *queues,
221 unsigned int new_nr_queues)
222{
223 unsigned int nr_queues = queues->nr_queues;
224 struct auxtrace_queue *queue_array;
225 unsigned int i;
226
227 if (!nr_queues)
228 nr_queues = AUXTRACE_INIT_NR_QUEUES;
229
230 while (nr_queues && nr_queues < new_nr_queues)
231 nr_queues <<= 1;
232
233 if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)
234 return -EINVAL;
235
236 queue_array = auxtrace_alloc_queue_array(nr_queues);
237 if (!queue_array)
238 return -ENOMEM;
239
240 for (i = 0; i < queues->nr_queues; i++) {
241 list_splice_tail(&queues->queue_array[i].head,
242 &queue_array[i].head);
99cbbe56
AH		 243 queue_array[i].tid = queues->queue_array[i].tid;
244 queue_array[i].cpu = queues->queue_array[i].cpu;
245 queue_array[i].set = queues->queue_array[i].set;
e5027893
AH		 246 queue_array[i].priv = queues->queue_array[i].priv;
247 }
248
249 queues->nr_queues = nr_queues;
250 queues->queue_array = queue_array;
251
252 return 0;
253}
254
255static void *auxtrace_copy_data(u64 size, struct perf_session *session)
256{
8ceb41d7 257 int fd = perf_data__fd(session->data);
e5027893
AH		 258 void *p;
259 ssize_t ret;
260
261 if (size > SSIZE_MAX)
262 return NULL;
263
264 p = malloc(size);
265 if (!p)
266 return NULL;
267
268 ret = readn(fd, p, size);
269 if (ret != (ssize_t)size) {
270 free(p);
271 return NULL;
272 }
273
274 return p;
275}
276
a356a597
AH		 277static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues,
278 unsigned int idx,
279 struct auxtrace_buffer *buffer)
e5027893
AH		 280{
281 struct auxtrace_queue *queue;
282 int err;
283
284 if (idx >= queues->nr_queues) {
285 err = auxtrace_queues__grow(queues, idx + 1);
286 if (err)
287 return err;
288 }
289
290 queue = &queues->queue_array[idx];
291
292 if (!queue->set) {
293 queue->set = true;
294 queue->tid = buffer->tid;
295 queue->cpu = buffer->cpu;
e5027893
AH		 296 }
297
298 buffer->buffer_nr = queues->next_buffer_nr++;
299
300 list_add_tail(&buffer->list, &queue->head);
301
302 queues->new_data = true;
303 queues->populated = true;
304
305 return 0;
306}
307
308/* Limit buffers to 32MiB on 32-bit */
309#define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)
310
311static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,
312 unsigned int idx,
313 struct auxtrace_buffer *buffer)
314{
315 u64 sz = buffer->size;
316 bool consecutive = false;
317 struct auxtrace_buffer *b;
318 int err;
319
320 while (sz > BUFFER_LIMIT_FOR_32_BIT) {
321 b = memdup(buffer, sizeof(struct auxtrace_buffer));
322 if (!b)
323 return -ENOMEM;
324 b->size = BUFFER_LIMIT_FOR_32_BIT;
325 b->consecutive = consecutive;
a356a597 326 err = auxtrace_queues__queue_buffer(queues, idx, b);
e5027893
AH		 327 if (err) {
328 auxtrace_buffer__free(b);
329 return err;
330 }
331 buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;
332 sz -= BUFFER_LIMIT_FOR_32_BIT;
333 consecutive = true;
334 }
335
336 buffer->size = sz;
337 buffer->consecutive = consecutive;
338
339 return 0;
340}
341
b238db65
AH		 342static bool filter_cpu(struct perf_session *session, int cpu)
343{
344 unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap;
345
346 return cpu_bitmap && cpu != -1 && !test_bit(cpu, cpu_bitmap);
347}
348
a356a597
AH		 349static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,
350 struct perf_session *session,
351 unsigned int idx,
4c454843
AH		 352 struct auxtrace_buffer *buffer,
353 struct auxtrace_buffer **buffer_ptr)
e5027893 354{
0d75f123
AH		 355 int err = -ENOMEM;
356
b238db65
AH		 357 if (filter_cpu(session, buffer->cpu))
358 return 0;
359
0d75f123
AH		 360 buffer = memdup(buffer, sizeof(*buffer));
361 if (!buffer)
362 return -ENOMEM;
4c454843 363
e5027893
AH		 364 if (session->one_mmap) {
365 buffer->data = buffer->data_offset - session->one_mmap_offset +
366 session->one_mmap_addr;
8ceb41d7 367 } else if (perf_data__is_pipe(session->data)) {
e5027893
AH		 368 buffer->data = auxtrace_copy_data(buffer->size, session);
369 if (!buffer->data)
0d75f123 370 goto out_free;
e5027893
AH		 371 buffer->data_needs_freeing = true;
372 } else if (BITS_PER_LONG == 32 &&
373 buffer->size > BUFFER_LIMIT_FOR_32_BIT) {
e5027893
AH		 374 err = auxtrace_queues__split_buffer(queues, idx, buffer);
375 if (err)
0d75f123 376 goto out_free;
e5027893
AH		 377 }
378
4c454843
AH		 379 err = auxtrace_queues__queue_buffer(queues, idx, buffer);
380 if (err)
0d75f123 381 goto out_free;
4c454843
AH		 382
383 /* FIXME: Doesn't work for split buffer */
384 if (buffer_ptr)
385 *buffer_ptr = buffer;
386
387 return 0;
0d75f123
AH		 388
389out_free:
390 auxtrace_buffer__free(buffer);
391 return err;
e5027893
AH		 392}
393
394int auxtrace_queues__add_event(struct auxtrace_queues *queues,
395 struct perf_session *session,
396 union perf_event *event, off_t data_offset,
397 struct auxtrace_buffer **buffer_ptr)
398{
0d75f123
AH		 399 struct auxtrace_buffer buffer = {
400 .pid = -1,
401 .tid = event->auxtrace.tid,
402 .cpu = event->auxtrace.cpu,
403 .data_offset = data_offset,
404 .offset = event->auxtrace.offset,
405 .reference = event->auxtrace.reference,
406 .size = event->auxtrace.size,
407 };
408 unsigned int idx = event->auxtrace.idx;
e5027893 409
0d75f123
AH		 410 return auxtrace_queues__add_buffer(queues, session, idx, &buffer,
411 buffer_ptr);
e5027893
AH		 412}
413
99fa2984
AH		 414static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,
415 struct perf_session *session,
416 off_t file_offset, size_t sz)
417{
418 union perf_event *event;
419 int err;
420 char buf[PERF_SAMPLE_MAX_SIZE];
421
422 err = perf_session__peek_event(session, file_offset, buf,
423 PERF_SAMPLE_MAX_SIZE, &event, NULL);
424 if (err)
425 return err;
426
427 if (event->header.type == PERF_RECORD_AUXTRACE) {
72932371 428 if (event->header.size < sizeof(struct perf_record_auxtrace) ||
99fa2984
AH		 429 event->header.size != sz) {
430 err = -EINVAL;
431 goto out;
432 }
433 file_offset += event->header.size;
434 err = auxtrace_queues__add_event(queues, session, event,
435 file_offset, NULL);
436 }
437out:
438 return err;
439}
440
e5027893
AH		 441void auxtrace_queues__free(struct auxtrace_queues *queues)
442{
443 unsigned int i;
444
445 for (i = 0; i < queues->nr_queues; i++) {
446 while (!list_empty(&queues->queue_array[i].head)) {
447 struct auxtrace_buffer *buffer;
448
449 buffer = list_entry(queues->queue_array[i].head.next,
450 struct auxtrace_buffer, list);
e56fbc9d 451 list_del_init(&buffer->list);
e5027893
AH		 452 auxtrace_buffer__free(buffer);
453 }
454 }
455
456 zfree(&queues->queue_array);
457 queues->nr_queues = 0;
458}
459
f9397155
AH		 460static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,
461 unsigned int pos, unsigned int queue_nr,
462 u64 ordinal)
463{
464 unsigned int parent;
465
466 while (pos) {
467 parent = (pos - 1) >> 1;
468 if (heap_array[parent].ordinal <= ordinal)
469 break;
470 heap_array[pos] = heap_array[parent];
471 pos = parent;
472 }
473 heap_array[pos].queue_nr = queue_nr;
474 heap_array[pos].ordinal = ordinal;
475}
476
477int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
478 u64 ordinal)
479{
480 struct auxtrace_heap_item *heap_array;
481
482 if (queue_nr >= heap->heap_sz) {
483 unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;
484
485 while (heap_sz <= queue_nr)
486 heap_sz <<= 1;
487 heap_array = realloc(heap->heap_array,
488 heap_sz * sizeof(struct auxtrace_heap_item));
489 if (!heap_array)
490 return -ENOMEM;
491 heap->heap_array = heap_array;
492 heap->heap_sz = heap_sz;
493 }
494
495 auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);
496
497 return 0;
498}
499
500void auxtrace_heap__free(struct auxtrace_heap *heap)
501{
502 zfree(&heap->heap_array);
503 heap->heap_cnt = 0;
504 heap->heap_sz = 0;
505}
506
507void auxtrace_heap__pop(struct auxtrace_heap *heap)
508{
509 unsigned int pos, last, heap_cnt = heap->heap_cnt;
510 struct auxtrace_heap_item *heap_array;
511
512 if (!heap_cnt)
513 return;
514
515 heap->heap_cnt -= 1;
516
517 heap_array = heap->heap_array;
518
519 pos = 0;
520 while (1) {
521 unsigned int left, right;
522
523 left = (pos << 1) + 1;
524 if (left >= heap_cnt)
525 break;
526 right = left + 1;
527 if (right >= heap_cnt) {
528 heap_array[pos] = heap_array[left];
529 return;
530 }
531 if (heap_array[left].ordinal < heap_array[right].ordinal) {
532 heap_array[pos] = heap_array[left];
533 pos = left;
534 } else {
535 heap_array[pos] = heap_array[right];
536 pos = right;
537 }
538 }
539
540 last = heap_cnt - 1;
541 auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,
542 heap_array[last].ordinal);
543}
544
14a05e13 545size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
63503dba 546 struct evlist *evlist)
9e0cc4fe
AH		 547{
548 if (itr)
14a05e13 549 return itr->info_priv_size(itr, evlist);
9e0cc4fe
AH		 550 return 0;
551}
552
553static int auxtrace_not_supported(void)
554{
555 pr_err("AUX area tracing is not supported on this architecture\n");
556 return -EINVAL;
557}
558
559int auxtrace_record__info_fill(struct auxtrace_record *itr,
560 struct perf_session *session,
72932371 561 struct perf_record_auxtrace_info *auxtrace_info,
9e0cc4fe
AH		 562 size_t priv_size)
563{
564 if (itr)
565 return itr->info_fill(itr, session, auxtrace_info, priv_size);
566 return auxtrace_not_supported();
567}
568
569void auxtrace_record__free(struct auxtrace_record *itr)
570{
571 if (itr)
572 itr->free(itr);
573}
574
d20031bb
AH		 575int auxtrace_record__snapshot_start(struct auxtrace_record *itr)
576{
577 if (itr && itr->snapshot_start)
578 return itr->snapshot_start(itr);
579 return 0;
580}
581
ce7b0e42 582int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit)
d20031bb 583{
ce7b0e42 584 if (!on_exit && itr && itr->snapshot_finish)
d20031bb
AH		 585 return itr->snapshot_finish(itr);
586 return 0;
587}
588
589int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
590 struct auxtrace_mmap *mm,
591 unsigned char *data, u64 *head, u64 *old)
592{
593 if (itr && itr->find_snapshot)
594 return itr->find_snapshot(itr, idx, mm, data, head, old);
595 return 0;
596}
597
9e0cc4fe 598int auxtrace_record__options(struct auxtrace_record *itr,
63503dba 599 struct evlist *evlist,
9e0cc4fe
AH		 600 struct record_opts *opts)
601{
ad60ba0c
AH		 602 if (itr) {
603 itr->evlist = evlist;
9e0cc4fe 604 return itr->recording_options(itr, evlist, opts);
ad60ba0c 605 }
9e0cc4fe
AH		 606 return 0;
607}
608
609u64 auxtrace_record__reference(struct auxtrace_record *itr)
610{
611 if (itr)
612 return itr->reference(itr);
613 return 0;
614}
615
d20031bb
AH		 616int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
617 struct record_opts *opts, const char *str)
618{
619 if (!str)
620 return 0;
621
ce7b0e42
AS		 622 /* PMU-agnostic options */
623 switch (*str) {
624 case 'e':
625 opts->auxtrace_snapshot_on_exit = true;
626 str++;
627 break;
628 default:
629 break;
630 }
631
b14585d9 632 if (itr && itr->parse_snapshot_options)
d20031bb
AH		 633 return itr->parse_snapshot_options(itr, opts, str);
634
635 pr_err("No AUX area tracing to snapshot\n");
636 return -EINVAL;
637}
638
ad60ba0c
AH		 639int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx)
640{
641 struct evsel *evsel;
642
643 if (!itr->evlist || !itr->pmu)
644 return -EINVAL;
645
646 evlist__for_each_entry(itr->evlist, evsel) {
647 if (evsel->core.attr.type == itr->pmu->type) {
648 if (evsel->disabled)
649 return 0;
37b01abe 650 return evlist__enable_event_idx(itr->evlist, evsel, idx);
ad60ba0c
AH		 651 }
652 }
653 return -EINVAL;
654}
655
f0bb7ee8
AH		 656/*
657 * Event record size is 16-bit which results in a maximum size of about 64KiB.
658 * Allow about 4KiB for the rest of the sample record, to give a maximum
659 * AUX area sample size of 60KiB.
660 */
661#define MAX_AUX_SAMPLE_SIZE (60 * 1024)
662
663/* Arbitrary default size if no other default provided */
664#define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)
665
666static int auxtrace_validate_aux_sample_size(struct evlist *evlist,
667 struct record_opts *opts)
668{
669 struct evsel *evsel;
670 bool has_aux_leader = false;
671 u32 sz;
672
673 evlist__for_each_entry(evlist, evsel) {
674 sz = evsel->core.attr.aux_sample_size;
c754c382 675 if (evsel__is_group_leader(evsel)) {
39453ed5 676 has_aux_leader = evsel__is_aux_event(evsel);
f0bb7ee8
AH		 677 if (sz) {
678 if (has_aux_leader)
679 pr_err("Cannot add AUX area sampling to an AUX area event\n");
680 else
681 pr_err("Cannot add AUX area sampling to a group leader\n");
682 return -EINVAL;
683 }
684 }
685 if (sz > MAX_AUX_SAMPLE_SIZE) {
686 pr_err("AUX area sample size %u too big, max. %d\n",
687 sz, MAX_AUX_SAMPLE_SIZE);
688 return -EINVAL;
689 }
690 if (sz) {
691 if (!has_aux_leader) {
692 pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");
693 return -EINVAL;
694 }
862b2f8f 695 evsel__set_sample_bit(evsel, AUX);
f0bb7ee8
AH		 696 opts->auxtrace_sample_mode = true;
697 } else {
862b2f8f 698 evsel__reset_sample_bit(evsel, AUX);
f0bb7ee8
AH		 699 }
700 }
701
702 if (!opts->auxtrace_sample_mode) {
703 pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");
704 return -EINVAL;
705 }
706
707 if (!perf_can_aux_sample()) {
708 pr_err("AUX area sampling is not supported by kernel\n");
709 return -EINVAL;
710 }
711
712 return 0;
713}
714
715int auxtrace_parse_sample_options(struct auxtrace_record *itr,
716 struct evlist *evlist,
717 struct record_opts *opts, const char *str)
718{
35ac0cad 719 struct evsel_config_term *term;
eb7a52d4
AH		 720 struct evsel *aux_evsel;
721 bool has_aux_sample_size = false;
f0bb7ee8
AH		 722 bool has_aux_leader = false;
723 struct evsel *evsel;
724 char *endptr;
725 unsigned long sz;
726
727 if (!str)
eb7a52d4 728 goto no_opt;
f0bb7ee8
AH		 729
730 if (!itr) {
731 pr_err("No AUX area event to sample\n");
732 return -EINVAL;
733 }
734
735 sz = strtoul(str, &endptr, 0);
736 if (*endptr || sz > UINT_MAX) {
737 pr_err("Bad AUX area sampling option: '%s'\n", str);
738 return -EINVAL;
739 }
740
741 if (!sz)
742 sz = itr->default_aux_sample_size;
743
744 if (!sz)
745 sz = DEFAULT_AUX_SAMPLE_SIZE;
746
747 /* Set aux_sample_size based on --aux-sample option */
748 evlist__for_each_entry(evlist, evsel) {
c754c382 749 if (evsel__is_group_leader(evsel)) {
39453ed5 750 has_aux_leader = evsel__is_aux_event(evsel);
f0bb7ee8
AH		 751 } else if (has_aux_leader) {
752 evsel->core.attr.aux_sample_size = sz;
753 }
754 }
eb7a52d4
AH		 755no_opt:
756 aux_evsel = NULL;
757 /* Override with aux_sample_size from config term */
758 evlist__for_each_entry(evlist, evsel) {
39453ed5 759 if (evsel__is_aux_event(evsel))
eb7a52d4 760 aux_evsel = evsel;
35ac0cad 761 term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE);
eb7a52d4
AH		 762 if (term) {
763 has_aux_sample_size = true;
764 evsel->core.attr.aux_sample_size = term->val.aux_sample_size;
765 /* If possible, group with the AUX event */
766 if (aux_evsel && evsel->core.attr.aux_sample_size)
64b4778b 767 evlist__regroup(evlist, aux_evsel, evsel);
eb7a52d4
AH		 768 }
769 }
770
771 if (!str && !has_aux_sample_size)
772 return 0;
773
774 if (!itr) {
775 pr_err("No AUX area event to sample\n");
776 return -EINVAL;
777 }
f0bb7ee8
AH		 778
779 return auxtrace_validate_aux_sample_size(evlist, opts);
780}
781
d58b3f7e
AH		 782void auxtrace_regroup_aux_output(struct evlist *evlist)
783{
784 struct evsel *evsel, *aux_evsel = NULL;
785 struct evsel_config_term *term;
786
787 evlist__for_each_entry(evlist, evsel) {
788 if (evsel__is_aux_event(evsel))
789 aux_evsel = evsel;
790 term = evsel__get_config_term(evsel, AUX_OUTPUT);
791 /* If possible, group with the AUX event */
792 if (term && aux_evsel)
793 evlist__regroup(evlist, aux_evsel, evsel);
794 }
795}
796
9e0cc4fe 797struct auxtrace_record *__weak
63503dba 798auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err)
9e0cc4fe
AH		 799{
800 *err = 0;
801 return NULL;
802}
803
99fa2984
AH		 804static int auxtrace_index__alloc(struct list_head *head)
805{
806 struct auxtrace_index *auxtrace_index;
807
808 auxtrace_index = malloc(sizeof(struct auxtrace_index));
809 if (!auxtrace_index)
810 return -ENOMEM;
811
812 auxtrace_index->nr = 0;
813 INIT_LIST_HEAD(&auxtrace_index->list);
814
815 list_add_tail(&auxtrace_index->list, head);
816
817 return 0;
818}
819
820void auxtrace_index__free(struct list_head *head)
821{
822 struct auxtrace_index *auxtrace_index, *n;
823
824 list_for_each_entry_safe(auxtrace_index, n, head, list) {
e56fbc9d 825 list_del_init(&auxtrace_index->list);
99fa2984
AH		 826 free(auxtrace_index);
827 }
828}
829
830static struct auxtrace_index *auxtrace_index__last(struct list_head *head)
831{
832 struct auxtrace_index *auxtrace_index;
833 int err;
834
835 if (list_empty(head)) {
836 err = auxtrace_index__alloc(head);
837 if (err)
838 return NULL;
839 }
840
841 auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);
842
843 if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {
844 err = auxtrace_index__alloc(head);
845 if (err)
846 return NULL;
847 auxtrace_index = list_entry(head->prev, struct auxtrace_index,
848 list);
849 }
850
851 return auxtrace_index;
852}
853
854int auxtrace_index__auxtrace_event(struct list_head *head,
855 union perf_event *event, off_t file_offset)
856{
857 struct auxtrace_index *auxtrace_index;
858 size_t nr;
859
860 auxtrace_index = auxtrace_index__last(head);
861 if (!auxtrace_index)
862 return -ENOMEM;
863
864 nr = auxtrace_index->nr;
865 auxtrace_index->entries[nr].file_offset = file_offset;
866 auxtrace_index->entries[nr].sz = event->header.size;
867 auxtrace_index->nr += 1;
868
869 return 0;
870}
871
872static int auxtrace_index__do_write(int fd,
873 struct auxtrace_index *auxtrace_index)
874{
875 struct auxtrace_index_entry ent;
876 size_t i;
877
878 for (i = 0; i < auxtrace_index->nr; i++) {
879 ent.file_offset = auxtrace_index->entries[i].file_offset;
880 ent.sz = auxtrace_index->entries[i].sz;
881 if (writen(fd, &ent, sizeof(ent)) != sizeof(ent))
882 return -errno;
883 }
884 return 0;
885}
886
887int auxtrace_index__write(int fd, struct list_head *head)
888{
889 struct auxtrace_index *auxtrace_index;
890 u64 total = 0;
891 int err;
892
893 list_for_each_entry(auxtrace_index, head, list)
894 total += auxtrace_index->nr;
895
896 if (writen(fd, &total, sizeof(total)) != sizeof(total))
897 return -errno;
898
899 list_for_each_entry(auxtrace_index, head, list) {
900 err = auxtrace_index__do_write(fd, auxtrace_index);
901 if (err)
902 return err;
903 }
904
905 return 0;
906}
907
908static int auxtrace_index__process_entry(int fd, struct list_head *head,
909 bool needs_swap)
910{
911 struct auxtrace_index *auxtrace_index;
912 struct auxtrace_index_entry ent;
913 size_t nr;
914
915 if (readn(fd, &ent, sizeof(ent)) != sizeof(ent))
916 return -1;
917
918 auxtrace_index = auxtrace_index__last(head);
919 if (!auxtrace_index)
920 return -1;
921
922 nr = auxtrace_index->nr;
923 if (needs_swap) {
924 auxtrace_index->entries[nr].file_offset =
925 bswap_64(ent.file_offset);
926 auxtrace_index->entries[nr].sz = bswap_64(ent.sz);
927 } else {
928 auxtrace_index->entries[nr].file_offset = ent.file_offset;
929 auxtrace_index->entries[nr].sz = ent.sz;
930 }
931
932 auxtrace_index->nr = nr + 1;
933
934 return 0;
935}
936
937int auxtrace_index__process(int fd, u64 size, struct perf_session *session,
938 bool needs_swap)
939{
940 struct list_head *head = &session->auxtrace_index;
941 u64 nr;
942
943 if (readn(fd, &nr, sizeof(u64)) != sizeof(u64))
944 return -1;
945
946 if (needs_swap)
947 nr = bswap_64(nr);
948
949 if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size)
950 return -1;
951
952 while (nr--) {
953 int err;
954
955 err = auxtrace_index__process_entry(fd, head, needs_swap);
956 if (err)
957 return -1;
958 }
959
960 return 0;
961}
962
963static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues,
964 struct perf_session *session,
965 struct auxtrace_index_entry *ent)
966{
967 return auxtrace_queues__add_indexed_event(queues, session,
968 ent->file_offset, ent->sz);
969}
970
971int auxtrace_queues__process_index(struct auxtrace_queues *queues,
972 struct perf_session *session)
973{
974 struct auxtrace_index *auxtrace_index;
975 struct auxtrace_index_entry *ent;
976 size_t i;
977 int err;
978
2e2967f4
AH		 979 if (auxtrace__dont_decode(session))
980 return 0;
981
99fa2984
AH		 982 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
983 for (i = 0; i < auxtrace_index->nr; i++) {
984 ent = &auxtrace_index->entries[i];
985 err = auxtrace_queues__process_index_entry(queues,
986 session,
987 ent);
988 if (err)
989 return err;
990 }
991 }
992 return 0;
993}
994
e5027893
AH		 995struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,
996 struct auxtrace_buffer *buffer)
997{
998 if (buffer) {
999 if (list_is_last(&buffer->list, &queue->head))
1000 return NULL;
1001 return list_entry(buffer->list.next, struct auxtrace_buffer,
1002 list);
1003 } else {
1004 if (list_empty(&queue->head))
1005 return NULL;
1006 return list_entry(queue->head.next, struct auxtrace_buffer,
1007 list);
1008 }
1009}
1010
ac2f445f
AH		 1011struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues,
1012 struct perf_sample *sample,
1013 struct perf_session *session)
1014{
1015 struct perf_sample_id *sid;
1016 unsigned int idx;
1017 u64 id;
1018
1019 id = sample->id;
1020 if (!id)
1021 return NULL;
1022
3ccf8a7b 1023 sid = evlist__id2sid(session->evlist, id);
ac2f445f
AH		 1024 if (!sid)
1025 return NULL;
1026
1027 idx = sid->idx;
1028
1029 if (idx >= queues->nr_queues)
1030 return NULL;
1031
1032 return &queues->queue_array[idx];
1033}
1034
1035int auxtrace_queues__add_sample(struct auxtrace_queues *queues,
1036 struct perf_session *session,
1037 struct perf_sample *sample, u64 data_offset,
1038 u64 reference)
1039{
1040 struct auxtrace_buffer buffer = {
1041 .pid = -1,
1042 .data_offset = data_offset,
1043 .reference = reference,
1044 .size = sample->aux_sample.size,
1045 };
1046 struct perf_sample_id *sid;
1047 u64 id = sample->id;
1048 unsigned int idx;
1049
1050 if (!id)
1051 return -EINVAL;
1052
3ccf8a7b 1053 sid = evlist__id2sid(session->evlist, id);
ac2f445f
AH		 1054 if (!sid)
1055 return -ENOENT;
1056
1057 idx = sid->idx;
1058 buffer.tid = sid->tid;
1059 buffer.cpu = sid->cpu;
1060
1061 return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL);
1062}
1063
1064struct queue_data {
1065 bool samples;
1066 bool events;
1067};
1068
1069static int auxtrace_queue_data_cb(struct perf_session *session,
1070 union perf_event *event, u64 offset,
1071 void *data)
1072{
1073 struct queue_data *qd = data;
1074 struct perf_sample sample;
1075 int err;
1076
1077 if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) {
1078 if (event->header.size < sizeof(struct perf_record_auxtrace))
1079 return -EINVAL;
1080 offset += event->header.size;
1081 return session->auxtrace->queue_data(session, NULL, event,
1082 offset);
1083 }
1084
1085 if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE)
1086 return 0;
1087
2a6599cd 1088 err = evlist__parse_sample(session->evlist, event, &sample);
ac2f445f
AH		 1089 if (err)
1090 return err;
1091
1092 if (!sample.aux_sample.size)
1093 return 0;
1094
1095 offset += sample.aux_sample.data - (void *)event;
1096
1097 return session->auxtrace->queue_data(session, &sample, NULL, offset);
1098}
1099
1100int auxtrace_queue_data(struct perf_session *session, bool samples, bool events)
1101{
1102 struct queue_data qd = {
1103 .samples = samples,
1104 .events = events,
1105 };
1106
1107 if (auxtrace__dont_decode(session))
1108 return 0;
1109
1110 if (!session->auxtrace || !session->auxtrace->queue_data)
1111 return -EINVAL;
1112
1113 return perf_session__peek_events(session, session->header.data_offset,
1114 session->header.data_size,
1115 auxtrace_queue_data_cb, &qd);
1116}
1117
6aa3afc9 1118void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw)
e5027893 1119{
6aa3afc9 1120 int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ;
e5027893
AH		 1121 size_t adj = buffer->data_offset & (page_size - 1);
1122 size_t size = buffer->size + adj;
1123 off_t file_offset = buffer->data_offset - adj;
1124 void *addr;
1125
1126 if (buffer->data)
1127 return buffer->data;
1128
6aa3afc9 1129 addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset);
e5027893
AH		 1130 if (addr == MAP_FAILED)
1131 return NULL;
1132
1133 buffer->mmap_addr = addr;
1134 buffer->mmap_size = size;
1135
1136 buffer->data = addr + adj;
1137
1138 return buffer->data;
1139}
1140
1141void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer)
1142{
1143 if (!buffer->data || !buffer->mmap_addr)
1144 return;
1145 munmap(buffer->mmap_addr, buffer->mmap_size);
1146 buffer->mmap_addr = NULL;
1147 buffer->mmap_size = 0;
1148 buffer->data = NULL;
1149 buffer->use_data = NULL;
1150}
1151
1152void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer)
1153{
1154 auxtrace_buffer__put_data(buffer);
1155 if (buffer->data_needs_freeing) {
1156 buffer->data_needs_freeing = false;
1157 zfree(&buffer->data);
1158 buffer->use_data = NULL;
1159 buffer->size = 0;
1160 }
1161}
1162
1163void auxtrace_buffer__free(struct auxtrace_buffer *buffer)
1164{
1165 auxtrace_buffer__drop_data(buffer);
1166 free(buffer);
1167}
1168
72932371 1169void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,
85ed4729 1170 int code, int cpu, pid_t pid, pid_t tid, u64 ip,
16bd4321 1171 const char *msg, u64 timestamp)
85ed4729
AH		 1172{
1173 size_t size;
1174
72932371 1175 memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error));
85ed4729
AH		 1176
1177 auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR;
1178 auxtrace_error->type = type;
1179 auxtrace_error->code = code;
1180 auxtrace_error->cpu = cpu;
1181 auxtrace_error->pid = pid;
1182 auxtrace_error->tid = tid;
16bd4321 1183 auxtrace_error->fmt = 1;
85ed4729 1184 auxtrace_error->ip = ip;
16bd4321 1185 auxtrace_error->time = timestamp;
85ed4729
AH		 1186 strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG);
1187
1188 size = (void *)auxtrace_error->msg - (void *)auxtrace_error +
1189 strlen(auxtrace_error->msg) + 1;
1190 auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64));
1191}
1192
9e0cc4fe
AH		 1193int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr,
1194 struct perf_tool *tool,
1195 struct perf_session *session,
1196 perf_event__handler_t process)
1197{
1198 union perf_event *ev;
1199 size_t priv_size;
1200 int err;
1201
1202 pr_debug2("Synthesizing auxtrace information\n");
14a05e13 1203 priv_size = auxtrace_record__info_priv_size(itr, session->evlist);
72932371 1204 ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size);
9e0cc4fe
AH		 1205 if (!ev)
1206 return -ENOMEM;
1207
1208 ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO;
72932371 1209 ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) +
9e0cc4fe
AH		 1210 priv_size;
1211 err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info,
1212 priv_size);
1213 if (err)
1214 goto out_free;
1215
1216 err = process(tool, ev, NULL, NULL);
1217out_free:
1218 free(ev);
1219 return err;
1220}
1221
5c7bec0c
AH		 1222static void unleader_evsel(struct evlist *evlist, struct evsel *leader)
1223{
1224 struct evsel *new_leader = NULL;
1225 struct evsel *evsel;
1226
1227 /* Find new leader for the group */
1228 evlist__for_each_entry(evlist, evsel) {
fba7c866 1229 if (!evsel__has_leader(evsel, leader) || evsel == leader)
5c7bec0c
AH		 1230 continue;
1231 if (!new_leader)
1232 new_leader = evsel;
fba7c866 1233 evsel__set_leader(evsel, new_leader);
5c7bec0c
AH		 1234 }
1235
1236 /* Update group information */
1237 if (new_leader) {
1238 zfree(&new_leader->group_name);
1239 new_leader->group_name = leader->group_name;
1240 leader->group_name = NULL;
1241
1242 new_leader->core.nr_members = leader->core.nr_members - 1;
1243 leader->core.nr_members = 1;
1244 }
1245}
1246
1247static void unleader_auxtrace(struct perf_session *session)
1248{
1249 struct evsel *evsel;
1250
1251 evlist__for_each_entry(session->evlist, evsel) {
1252 if (auxtrace__evsel_is_auxtrace(session, evsel) &&
c754c382 1253 evsel__is_group_leader(evsel)) {
5c7bec0c
AH		 1254 unleader_evsel(session->evlist, evsel);
1255 }
1256 }
1257}
1258
89f1688a
JO		 1259int perf_event__process_auxtrace_info(struct perf_session *session,
1260 union perf_event *event)
73f75fb1
AH		 1261{
1262 enum auxtrace_type type = event->auxtrace_info.type;
5c7bec0c 1263 int err;
73f75fb1
AH		 1264
1265 if (dump_trace)
1266 fprintf(stdout, " type: %u\n", type);
1267
1268 switch (type) {
55ea4ab4 1269 case PERF_AUXTRACE_INTEL_PT:
5c7bec0c
AH		 1270 err = intel_pt_process_auxtrace_info(event, session);
1271 break;
d0170af7 1272 case PERF_AUXTRACE_INTEL_BTS:
5c7bec0c
AH		 1273 err = intel_bts_process_auxtrace_info(event, session);
1274 break;
ffd3d18c 1275 case PERF_AUXTRACE_ARM_SPE:
5c7bec0c
AH		 1276 err = arm_spe_process_auxtrace_info(event, session);
1277 break;
a818c563 1278 case PERF_AUXTRACE_CS_ETM:
5c7bec0c
AH		 1279 err = cs_etm__process_auxtrace_info(event, session);
1280 break;
b96e6615 1281 case PERF_AUXTRACE_S390_CPUMSF:
5c7bec0c
AH		 1282 err = s390_cpumsf_process_auxtrace_info(event, session);
1283 break;
73f75fb1
AH		 1284 case PERF_AUXTRACE_UNKNOWN:
1285 default:
1286 return -EINVAL;
1287 }
5c7bec0c
AH		 1288
1289 if (err)
1290 return err;
1291
1292 unleader_auxtrace(session);
1293
1294 return 0;
73f75fb1
AH		 1295}
1296
7336555a
JO		 1297s64 perf_event__process_auxtrace(struct perf_session *session,
1298 union perf_event *event)
73f75fb1
AH		 1299{
1300 s64 err;
1301
1302 if (dump_trace)
306c9d24 1303 fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n",
73f75fb1
AH		 1304 event->auxtrace.size, event->auxtrace.offset,
1305 event->auxtrace.reference, event->auxtrace.idx,
1306 event->auxtrace.tid, event->auxtrace.cpu);
1307
1308 if (auxtrace__dont_decode(session))
1309 return event->auxtrace.size;
1310
1311 if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE)
1312 return -EINVAL;
1313
7336555a 1314 err = session->auxtrace->process_auxtrace_event(session, event, session->tool);
73f75fb1
AH		 1315 if (err < 0)
1316 return err;
1317
1318 return event->auxtrace.size;
1319}
1320
f6986c95
AH		 1321#define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS
1322#define PERF_ITRACE_DEFAULT_PERIOD 100000
1323#define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16
1324#define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024
601897b5
AH		 1325#define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64
1326#define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024
f6986c95 1327
4eb06815
AK		 1328void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,
1329 bool no_sample)
f6986c95 1330{
f6986c95 1331 synth_opts->branches = true;
53c76b0e 1332 synth_opts->transactions = true;
3bdafdff 1333 synth_opts->ptwrites = true;
70d110d7 1334 synth_opts->pwr_events = true;
181ebb5e 1335 synth_opts->other_events = true;
f6986c95 1336 synth_opts->errors = true;
9f74d770
TX		 1337 synth_opts->flc = true;
1338 synth_opts->llc = true;
1339 synth_opts->tlb = true;
014a771c 1340 synth_opts->mem = true;
9f74d770
TX		 1341 synth_opts->remote_access = true;
1342
4eb06815
AK		 1343 if (no_sample) {
1344 synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS;
1345 synth_opts->period = 1;
1346 synth_opts->calls = true;
1347 } else {
1348 synth_opts->instructions = true;
1349 synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE;
1350 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1351 }
f6986c95 1352 synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
601897b5 1353 synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
d1706b39 1354 synth_opts->initial_skip = 0;
f6986c95
AH		 1355}
1356
cb971438
AH		 1357static int get_flag(const char **ptr, unsigned int *flags)
1358{
1359 while (1) {
1360 char c = **ptr;
1361
1362 if (c >= 'a' && c <= 'z') {
1363 *flags |= 1 << (c - 'a');
1364 ++*ptr;
1365 return 0;
1366 } else if (c == ' ') {
1367 ++*ptr;
1368 continue;
1369 } else {
1370 return -1;
1371 }
1372 }
1373}
1374
1375static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags)
1376{
1377 while (1) {
1378 switch (**ptr) {
1379 case '+':
1380 ++*ptr;
1381 if (get_flag(ptr, plus_flags))
1382 return -1;
1383 break;
1384 case '-':
1385 ++*ptr;
1386 if (get_flag(ptr, minus_flags))
1387 return -1;
1388 break;
1389 case ' ':
1390 ++*ptr;
1391 break;
1392 default:
1393 return 0;
1394 }
1395 }
1396}
1397
f6986c95
AH		 1398/*
1399 * Please check tools/perf/Documentation/perf-script.txt for information
1400 * about the options parsed here, which is introduced after this cset,
1401 * when support in 'perf script' for these options is introduced.
1402 */
e621b8ff
AH		 1403int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts,
1404 const char *str, int unset)
f6986c95 1405{
f6986c95
AH		 1406 const char *p;
1407 char *endptr;
f70cfa07 1408 bool period_type_set = false;
e1791347 1409 bool period_set = false;
f6986c95
AH		 1410
1411 synth_opts->set = true;
1412
1413 if (unset) {
1414 synth_opts->dont_decode = true;
1415 return 0;
1416 }
1417
1418 if (!str) {
355200e0
AH		 1419 itrace_synth_opts__set_default(synth_opts,
1420 synth_opts->default_no_sample);
f6986c95
AH		 1421 return 0;
1422 }
1423
1424 for (p = str; *p;) {
1425 switch (*p++) {
1426 case 'i':
1427 synth_opts->instructions = true;
1428 while (*p == ' ' || *p == ',')
1429 p += 1;
1430 if (isdigit(*p)) {
1431 synth_opts->period = strtoull(p, &endptr, 10);
e1791347 1432 period_set = true;
f6986c95
AH		 1433 p = endptr;
1434 while (*p == ' ' || *p == ',')
1435 p += 1;
1436 switch (*p++) {
1437 case 'i':
1438 synth_opts->period_type =
1439 PERF_ITRACE_PERIOD_INSTRUCTIONS;
f70cfa07 1440 period_type_set = true;
f6986c95
AH		 1441 break;
1442 case 't':
1443 synth_opts->period_type =
1444 PERF_ITRACE_PERIOD_TICKS;
f70cfa07 1445 period_type_set = true;
f6986c95
AH		 1446 break;
1447 case 'm':
1448 synth_opts->period *= 1000;
1449 /* Fall through */
1450 case 'u':
1451 synth_opts->period *= 1000;
1452 /* Fall through */
1453 case 'n':
1454 if (*p++ != 's')
1455 goto out_err;
1456 synth_opts->period_type =
1457 PERF_ITRACE_PERIOD_NANOSECS;
f70cfa07 1458 period_type_set = true;
f6986c95
AH		 1459 break;
1460 case '\0':
1461 goto out;
1462 default:
1463 goto out_err;
1464 }
1465 }
1466 break;
1467 case 'b':
1468 synth_opts->branches = true;
1469 break;
53c76b0e
AH		 1470 case 'x':
1471 synth_opts->transactions = true;
1472 break;
3bdafdff
AH		 1473 case 'w':
1474 synth_opts->ptwrites = true;
1475 break;
70d110d7
AH		 1476 case 'p':
1477 synth_opts->pwr_events = true;
1478 break;
181ebb5e
AH		 1479 case 'o':
1480 synth_opts->other_events = true;
1481 break;
f6986c95
AH		 1482 case 'e':
1483 synth_opts->errors = true;
cb971438
AH		 1484 if (get_flags(&p, &synth_opts->error_plus_flags,
1485 &synth_opts->error_minus_flags))
1486 goto out_err;
f6986c95
AH		 1487 break;
1488 case 'd':
1489 synth_opts->log = true;
935aac2d
AH		 1490 if (get_flags(&p, &synth_opts->log_plus_flags,
1491 &synth_opts->log_minus_flags))
1492 goto out_err;
f6986c95
AH		 1493 break;
1494 case 'c':
1495 synth_opts->branches = true;
1496 synth_opts->calls = true;
1497 break;
1498 case 'r':
1499 synth_opts->branches = true;
1500 synth_opts->returns = true;
1501 break;
1c5c25b3 1502 case 'G':
f6986c95 1503 case 'g':
1c5c25b3
AH		 1504 if (p[-1] == 'G')
1505 synth_opts->add_callchain = true;
1506 else
1507 synth_opts->callchain = true;
f6986c95
AH		 1508 synth_opts->callchain_sz =
1509 PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
1510 while (*p == ' ' || *p == ',')
1511 p += 1;
1512 if (isdigit(*p)) {
1513 unsigned int val;
1514
1515 val = strtoul(p, &endptr, 10);
1516 p = endptr;
1517 if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ)
1518 goto out_err;
1519 synth_opts->callchain_sz = val;
1520 }
1521 break;
ec90e42c 1522 case 'L':
601897b5 1523 case 'l':
ec90e42c
AH		 1524 if (p[-1] == 'L')
1525 synth_opts->add_last_branch = true;
1526 else
1527 synth_opts->last_branch = true;
601897b5
AH		 1528 synth_opts->last_branch_sz =
1529 PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
1530 while (*p == ' ' || *p == ',')
1531 p += 1;
1532 if (isdigit(*p)) {
1533 unsigned int val;
1534
1535 val = strtoul(p, &endptr, 10);
1536 p = endptr;
1537 if (!val ||
1538 val > PERF_ITRACE_MAX_LAST_BRANCH_SZ)
1539 goto out_err;
1540 synth_opts->last_branch_sz = val;
1541 }
1542 break;
d1706b39
AK		 1543 case 's':
1544 synth_opts->initial_skip = strtoul(p, &endptr, 10);
1545 if (p == endptr)
1546 goto out_err;
1547 p = endptr;
1548 break;
9f74d770
TX		 1549 case 'f':
1550 synth_opts->flc = true;
1551 break;
1552 case 'm':
1553 synth_opts->llc = true;
1554 break;
1555 case 't':
1556 synth_opts->tlb = true;
1557 break;
1558 case 'a':
1559 synth_opts->remote_access = true;
1560 break;
014a771c
LY		 1561 case 'M':
1562 synth_opts->mem = true;
1563 break;
51971536
AH		 1564 case 'q':
1565 synth_opts->quick += 1;
1566 break;
18f49494
AH		 1567 case 'Z':
1568 synth_opts->timeless_decoding = true;
1569 break;
f6986c95
AH		 1570 case ' ':
1571 case ',':
1572 break;
1573 default:
1574 goto out_err;
1575 }
1576 }
1577out:
1578 if (synth_opts->instructions) {
f70cfa07 1579 if (!period_type_set)
f6986c95
AH		 1580 synth_opts->period_type =
1581 PERF_ITRACE_DEFAULT_PERIOD_TYPE;
e1791347 1582 if (!period_set)
f6986c95
AH		 1583 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1584 }
1585
1586 return 0;
1587
1588out_err:
1589 pr_err("Bad Instruction Tracing options '%s'\n", str);
1590 return -EINVAL;
1591}
1592
e621b8ff
AH		 1593int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset)
1594{
1595 return itrace_do_parse_synth_opts(opt->value, str, unset);
1596}
1597
85ed4729
AH		 1598static const char * const auxtrace_error_type_name[] = {
1599 [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace",
1600};
1601
1602static const char *auxtrace_error_name(int type)
1603{
1604 const char *error_type_name = NULL;
1605
1606 if (type < PERF_AUXTRACE_ERROR_MAX)
1607 error_type_name = auxtrace_error_type_name[type];
1608 if (!error_type_name)
1609 error_type_name = "unknown AUX";
1610 return error_type_name;
1611}
1612
1613size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp)
1614{
72932371 1615 struct perf_record_auxtrace_error *e = &event->auxtrace_error;
16bd4321
AH		 1616 unsigned long long nsecs = e->time;
1617 const char *msg = e->msg;
85ed4729
AH		 1618 int ret;
1619
1620 ret = fprintf(fp, " %s error type %u",
1621 auxtrace_error_name(e->type), e->type);
16bd4321
AH		 1622
1623 if (e->fmt && nsecs) {
1624 unsigned long secs = nsecs / NSEC_PER_SEC;
1625
1626 nsecs -= secs * NSEC_PER_SEC;
1627 ret += fprintf(fp, " time %lu.%09llu", secs, nsecs);
1628 } else {
1629 ret += fprintf(fp, " time 0");
1630 }
1631
1632 if (!e->fmt)
1633 msg = (const char *)&e->time;
1634
3460efb2 1635 ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n",
16bd4321 1636 e->cpu, e->pid, e->tid, e->ip, e->code, msg);
85ed4729
AH		 1637 return ret;
1638}
1639
1640void perf_session__auxtrace_error_inc(struct perf_session *session,
1641 union perf_event *event)
1642{
72932371 1643 struct perf_record_auxtrace_error *e = &event->auxtrace_error;
85ed4729
AH		 1644
1645 if (e->type < PERF_AUXTRACE_ERROR_MAX)
1646 session->evlist->stats.nr_auxtrace_errors[e->type] += 1;
1647}
1648
1649void events_stats__auxtrace_error_warn(const struct events_stats *stats)
1650{
1651 int i;
1652
1653 for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) {
1654 if (!stats->nr_auxtrace_errors[i])
1655 continue;
1656 ui__warning("%u %s errors\n",
1657 stats->nr_auxtrace_errors[i],
1658 auxtrace_error_name(i));
1659 }
1660}
1661
89f1688a
JO		 1662int perf_event__process_auxtrace_error(struct perf_session *session,
1663 union perf_event *event)
85ed4729 1664{
73f75fb1
AH		 1665 if (auxtrace__dont_decode(session))
1666 return 0;
1667
85ed4729
AH		 1668 perf_event__fprintf_auxtrace_error(event, stdout);
1669 return 0;
1670}
1671
a5830532 1672static int __auxtrace_mmap__read(struct mmap *map,
d20031bb
AH		 1673 struct auxtrace_record *itr,
1674 struct perf_tool *tool, process_auxtrace_t fn,
1675 bool snapshot, size_t snapshot_size)
9e0cc4fe 1676{
e035f4ca 1677 struct auxtrace_mmap *mm = &map->auxtrace_mmap;
d20031bb 1678 u64 head, old = mm->prev, offset, ref;
9e0cc4fe
AH		 1679 unsigned char *data = mm->base;
1680 size_t size, head_off, old_off, len1, len2, padding;
1681 union perf_event ev;
1682 void *data1, *data2;
1683
9d645033
LY		 1684 head = auxtrace_mmap__read_head(mm);
1685
1686 if (snapshot &&
1687 auxtrace_record__find_snapshot(itr, mm->idx, mm, data, &head, &old))
1688 return -1;
d20031bb 1689
9e0cc4fe
AH		 1690 if (old == head)
1691 return 0;
1692
1693 pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n",
1694 mm->idx, old, head, head - old);
1695
1696 if (mm->mask) {
1697 head_off = head & mm->mask;
1698 old_off = old & mm->mask;
1699 } else {
1700 head_off = head % mm->len;
1701 old_off = old % mm->len;
1702 }
1703
1704 if (head_off > old_off)
1705 size = head_off - old_off;
1706 else
1707 size = mm->len - (old_off - head_off);
1708
d20031bb
AH		 1709 if (snapshot && size > snapshot_size)
1710 size = snapshot_size;
1711
9e0cc4fe
AH		 1712 ref = auxtrace_record__reference(itr);
1713
1714 if (head > old || size <= head || mm->mask) {
1715 offset = head - size;
1716 } else {
1717 /*
1718 * When the buffer size is not a power of 2, 'head' wraps at the
1719 * highest multiple of the buffer size, so we have to subtract
1720 * the remainder here.
1721 */
1722 u64 rem = (0ULL - mm->len) % mm->len;
1723
1724 offset = head - size - rem;
1725 }
1726
1727 if (size > head_off) {
1728 len1 = size - head_off;
1729 data1 = &data[mm->len - len1];
1730 len2 = head_off;
1731 data2 = &data[0];
1732 } else {
1733 len1 = size;
1734 data1 = &data[head_off - len1];
1735 len2 = 0;
1736 data2 = NULL;
1737 }
1738
83b2ea25
AH		 1739 if (itr->alignment) {
1740 unsigned int unwanted = len1 % itr->alignment;
1741
1742 len1 -= unwanted;
1743 size -= unwanted;
1744 }
1745
9e0cc4fe 1746 /* padding must be written by fn() e.g. record__process_auxtrace() */
c3fcadf0 1747 padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1);
9e0cc4fe 1748 if (padding)
c3fcadf0 1749 padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding;
9e0cc4fe
AH		 1750
1751 memset(&ev, 0, sizeof(ev));
1752 ev.auxtrace.header.type = PERF_RECORD_AUXTRACE;
1753 ev.auxtrace.header.size = sizeof(ev.auxtrace);
1754 ev.auxtrace.size = size + padding;
1755 ev.auxtrace.offset = offset;
1756 ev.auxtrace.reference = ref;
1757 ev.auxtrace.idx = mm->idx;
1758 ev.auxtrace.tid = mm->tid;
1759 ev.auxtrace.cpu = mm->cpu;
1760
ded2b8fe 1761 if (fn(tool, map, &ev, data1, len1, data2, len2))
9e0cc4fe
AH		 1762 return -1;
1763
1764 mm->prev = head;
1765
d20031bb
AH		 1766 if (!snapshot) {
1767 auxtrace_mmap__write_tail(mm, head);
1768 if (itr->read_finish) {
1769 int err;
9e0cc4fe 1770
d20031bb
AH		 1771 err = itr->read_finish(itr, mm->idx);
1772 if (err < 0)
1773 return err;
1774 }
9e0cc4fe
AH		 1775 }
1776
1777 return 1;
1778}
c3278f02 1779
a5830532 1780int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,
d20031bb
AH		 1781 struct perf_tool *tool, process_auxtrace_t fn)
1782{
e035f4ca 1783 return __auxtrace_mmap__read(map, itr, tool, fn, false, 0);
d20031bb
AH		 1784}
1785
a5830532 1786int auxtrace_mmap__read_snapshot(struct mmap *map,
d20031bb
AH		 1787 struct auxtrace_record *itr,
1788 struct perf_tool *tool, process_auxtrace_t fn,
1789 size_t snapshot_size)
1790{
e035f4ca 1791 return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size);
d20031bb
AH		 1792}
1793
c3278f02
AH		 1794/**
1795 * struct auxtrace_cache - hash table to implement a cache
1796 * @hashtable: the hashtable
1797 * @sz: hashtable size (number of hlists)
1798 * @entry_size: size of an entry
1799 * @limit: limit the number of entries to this maximum, when reached the cache
1800 * is dropped and caching begins again with an empty cache
1801 * @cnt: current number of entries
1802 * @bits: hashtable size (@sz = 2^@bits)
1803 */
1804struct auxtrace_cache {
1805 struct hlist_head *hashtable;
1806 size_t sz;
1807 size_t entry_size;
1808 size_t limit;
1809 size_t cnt;
1810 unsigned int bits;
1811};
1812
1813struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,
1814 unsigned int limit_percent)
1815{
1816 struct auxtrace_cache *c;
1817 struct hlist_head *ht;
1818 size_t sz, i;
1819
1820 c = zalloc(sizeof(struct auxtrace_cache));
1821 if (!c)
1822 return NULL;
1823
1824 sz = 1UL << bits;
1825
1826 ht = calloc(sz, sizeof(struct hlist_head));
1827 if (!ht)
1828 goto out_free;
1829
1830 for (i = 0; i < sz; i++)
1831 INIT_HLIST_HEAD(&ht[i]);
1832
1833 c->hashtable = ht;
1834 c->sz = sz;
1835 c->entry_size = entry_size;
1836 c->limit = (c->sz * limit_percent) / 100;
1837 c->bits = bits;
1838
1839 return c;
1840
1841out_free:
1842 free(c);
1843 return NULL;
1844}
1845
1846static void auxtrace_cache__drop(struct auxtrace_cache *c)
1847{
1848 struct auxtrace_cache_entry *entry;
1849 struct hlist_node *tmp;
1850 size_t i;
1851
1852 if (!c)
1853 return;
1854
1855 for (i = 0; i < c->sz; i++) {
1856 hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) {
1857 hlist_del(&entry->hash);
1858 auxtrace_cache__free_entry(c, entry);
1859 }
1860 }
1861
1862 c->cnt = 0;
1863}
1864
1865void auxtrace_cache__free(struct auxtrace_cache *c)
1866{
1867 if (!c)
1868 return;
1869
1870 auxtrace_cache__drop(c);
d8f9da24 1871 zfree(&c->hashtable);
c3278f02
AH		 1872 free(c);
1873}
1874
1875void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c)
1876{
1877 return malloc(c->entry_size);
1878}
1879
1880void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused,
1881 void *entry)
1882{
1883 free(entry);
1884}
1885
1886int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,
1887 struct auxtrace_cache_entry *entry)
1888{
1889 if (c->limit && ++c->cnt > c->limit)
1890 auxtrace_cache__drop(c);
1891
1892 entry->key = key;
1893 hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]);
1894
1895 return 0;
1896}
1897
fd62c109
AH		 1898static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c,
1899 u32 key)
1900{
1901 struct auxtrace_cache_entry *entry;
1902 struct hlist_head *hlist;
1903 struct hlist_node *n;
1904
1905 if (!c)
1906 return NULL;
1907
1908 hlist = &c->hashtable[hash_32(key, c->bits)];
1909 hlist_for_each_entry_safe(entry, n, hlist, hash) {
1910 if (entry->key == key) {
1911 hlist_del(&entry->hash);
1912 return entry;
1913 }
1914 }
1915
1916 return NULL;
1917}
1918
1919void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key)
1920{
1921 struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key);
1922
1923 auxtrace_cache__free_entry(c, entry);
1924}
1925
c3278f02
AH		 1926void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key)
1927{
1928 struct auxtrace_cache_entry *entry;
1929 struct hlist_head *hlist;
1930
1931 if (!c)
1932 return NULL;
1933
1934 hlist = &c->hashtable[hash_32(key, c->bits)];
1935 hlist_for_each_entry(entry, hlist, hash) {
1936 if (entry->key == key)
1937 return entry;
1938 }
1939
1940 return NULL;
1941}
1b36c03e
AH		 1942
1943static void addr_filter__free_str(struct addr_filter *filt)
1944{
d8f9da24 1945 zfree(&filt->str);
1b36c03e
AH		 1946 filt->action = NULL;
1947 filt->sym_from = NULL;
1948 filt->sym_to = NULL;
1949 filt->filename = NULL;
1b36c03e
AH		 1950}
1951
1952static struct addr_filter *addr_filter__new(void)
1953{
1954 struct addr_filter *filt = zalloc(sizeof(*filt));
1955
1956 if (filt)
1957 INIT_LIST_HEAD(&filt->list);
1958
1959 return filt;
1960}
1961
1962static void addr_filter__free(struct addr_filter *filt)
1963{
1964 if (filt)
1965 addr_filter__free_str(filt);
1966 free(filt);
1967}
1968
1969static void addr_filters__add(struct addr_filters *filts,
1970 struct addr_filter *filt)
1971{
1972 list_add_tail(&filt->list, &filts->head);
1973 filts->cnt += 1;
1974}
1975
1976static void addr_filters__del(struct addr_filters *filts,
1977 struct addr_filter *filt)
1978{
1979 list_del_init(&filt->list);
1980 filts->cnt -= 1;
1981}
1982
1983void addr_filters__init(struct addr_filters *filts)
1984{
1985 INIT_LIST_HEAD(&filts->head);
1986 filts->cnt = 0;
1987}
1988
1989void addr_filters__exit(struct addr_filters *filts)
1990{
1991 struct addr_filter *filt, *n;
1992
1993 list_for_each_entry_safe(filt, n, &filts->head, list) {
1994 addr_filters__del(filts, filt);
1995 addr_filter__free(filt);
1996 }
1997}
1998
1999static int parse_num_or_str(char **inp, u64 *num, const char **str,
2000 const char *str_delim)
2001{
2002 *inp += strspn(*inp, " ");
2003
2004 if (isdigit(**inp)) {
2005 char *endptr;
2006
2007 if (!num)
2008 return -EINVAL;
2009 errno = 0;
2010 *num = strtoull(*inp, &endptr, 0);
2011 if (errno)
2012 return -errno;
2013 if (endptr == *inp)
2014 return -EINVAL;
2015 *inp = endptr;
2016 } else {
2017 size_t n;
2018
2019 if (!str)
2020 return -EINVAL;
2021 *inp += strspn(*inp, " ");
2022 *str = *inp;
2023 n = strcspn(*inp, str_delim);
2024 if (!n)
2025 return -EINVAL;
2026 *inp += n;
2027 if (**inp) {
2028 **inp = '\0';
2029 *inp += 1;
2030 }
2031 }
2032 return 0;
2033}
2034
2035static int parse_action(struct addr_filter *filt)
2036{
2037 if (!strcmp(filt->action, "filter")) {
2038 filt->start = true;
2039 filt->range = true;
2040 } else if (!strcmp(filt->action, "start")) {
2041 filt->start = true;
2042 } else if (!strcmp(filt->action, "stop")) {
2043 filt->start = false;
2044 } else if (!strcmp(filt->action, "tracestop")) {
2045 filt->start = false;
2046 filt->range = true;
2047 filt->action += 5; /* Change 'tracestop' to 'stop' */
2048 } else {
2049 return -EINVAL;
2050 }
2051 return 0;
2052}
2053
2054static int parse_sym_idx(char **inp, int *idx)
2055{
2056 *idx = -1;
2057
2058 *inp += strspn(*inp, " ");
2059
2060 if (**inp != '#')
2061 return 0;
2062
2063 *inp += 1;
2064
2065 if (**inp == 'g' || **inp == 'G') {
2066 *inp += 1;
2067 *idx = 0;
2068 } else {
2069 unsigned long num;
2070 char *endptr;
2071
2072 errno = 0;
2073 num = strtoul(*inp, &endptr, 0);
2074 if (errno)
2075 return -errno;
2076 if (endptr == *inp || num > INT_MAX)
2077 return -EINVAL;
2078 *inp = endptr;
2079 *idx = num;
2080 }
2081
2082 return 0;
2083}
2084
2085static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx)
2086{
2087 int err = parse_num_or_str(inp, num, str, " ");
2088
2089 if (!err && *str)
2090 err = parse_sym_idx(inp, idx);
2091
2092 return err;
2093}
2094
2095static int parse_one_filter(struct addr_filter *filt, const char **filter_inp)
2096{
2097 char *fstr;
2098 int err;
2099
2100 filt->str = fstr = strdup(*filter_inp);
2101 if (!fstr)
2102 return -ENOMEM;
2103
2104 err = parse_num_or_str(&fstr, NULL, &filt->action, " ");
2105 if (err)
2106 goto out_err;
2107
2108 err = parse_action(filt);
2109 if (err)
2110 goto out_err;
2111
2112 err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from,
2113 &filt->sym_from_idx);
2114 if (err)
2115 goto out_err;
2116
2117 fstr += strspn(fstr, " ");
2118
2119 if (*fstr == '/') {
2120 fstr += 1;
2121 err = parse_addr_size(&fstr, &filt->size, &filt->sym_to,
2122 &filt->sym_to_idx);
2123 if (err)
2124 goto out_err;
2125 filt->range = true;
2126 }
2127
2128 fstr += strspn(fstr, " ");
2129
2130 if (*fstr == '@') {
2131 fstr += 1;
2132 err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,");
2133 if (err)
2134 goto out_err;
2135 }
2136
2137 fstr += strspn(fstr, " ,");
2138
2139 *filter_inp += fstr - filt->str;
2140
2141 return 0;
2142
2143out_err:
2144 addr_filter__free_str(filt);
2145
2146 return err;
2147}
2148
2149int addr_filters__parse_bare_filter(struct addr_filters *filts,
2150 const char *filter)
2151{
2152 struct addr_filter *filt;
2153 const char *fstr = filter;
2154 int err;
2155
2156 while (*fstr) {
2157 filt = addr_filter__new();
2158 err = parse_one_filter(filt, &fstr);
2159 if (err) {
2160 addr_filter__free(filt);
2161 addr_filters__exit(filts);
2162 return err;
2163 }
2164 addr_filters__add(filts, filt);
2165 }
2166
2167 return 0;
2168}
2169
2170struct sym_args {
2171 const char *name;
2172 u64 start;
2173 u64 size;
2174 int idx;
2175 int cnt;
2176 bool started;
2177 bool global;
2178 bool selected;
2179 bool duplicate;
2180 bool near;
2181};
2182
2183static bool kern_sym_match(struct sym_args *args, const char *name, char type)
2184{
2185 /* A function with the same name, and global or the n'th found or any */
e85e0e0c 2186 return kallsyms__is_function(type) &&
1b36c03e
AH		 2187 !strcmp(name, args->name) &&
2188 ((args->global && isupper(type)) ||
2189 (args->selected && ++(args->cnt) == args->idx) ||
2190 (!args->global && !args->selected));
2191}
2192
2193static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2194{
2195 struct sym_args *args = arg;
2196
2197 if (args->started) {
2198 if (!args->size)
2199 args->size = start - args->start;
2200 if (args->selected) {
2201 if (args->size)
2202 return 1;
2203 } else if (kern_sym_match(args, name, type)) {
2204 args->duplicate = true;
2205 return 1;
2206 }
2207 } else if (kern_sym_match(args, name, type)) {
2208 args->started = true;
2209 args->start = start;
2210 }
2211
2212 return 0;
2213}
2214
2215static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2216{
2217 struct sym_args *args = arg;
2218
2219 if (kern_sym_match(args, name, type)) {
2220 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2221 ++args->cnt, start, type, name);
2222 args->near = true;
2223 } else if (args->near) {
2224 args->near = false;
2225 pr_err("\t\twhich is near\t\t%s\n", name);
2226 }
2227
2228 return 0;
2229}
2230
2231static int sym_not_found_error(const char *sym_name, int idx)
2232{
2233 if (idx > 0) {
2234 pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",
2235 idx, sym_name);
2236 } else if (!idx) {
2237 pr_err("Global symbol '%s' not found.\n", sym_name);
2238 } else {
2239 pr_err("Symbol '%s' not found.\n", sym_name);
2240 }
2241 pr_err("Note that symbols must be functions.\n");
2242
2243 return -EINVAL;
2244}
2245
2246static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx)
2247{
2248 struct sym_args args = {
2249 .name = sym_name,
2250 .idx = idx,
2251 .global = !idx,
2252 .selected = idx > 0,
2253 };
2254 int err;
2255
2256 *start = 0;
2257 *size = 0;
2258
2259 err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb);
2260 if (err < 0) {
2261 pr_err("Failed to parse /proc/kallsyms\n");
2262 return err;
2263 }
2264
2265 if (args.duplicate) {
2266 pr_err("Multiple kernel symbols with name '%s'\n", sym_name);
2267 args.cnt = 0;
2268 kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb);
2269 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2270 sym_name);
2271 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2272 return -EINVAL;
2273 }
2274
2275 if (!args.started) {
2276 pr_err("Kernel symbol lookup: ");
2277 return sym_not_found_error(sym_name, idx);
2278 }
2279
2280 *start = args.start;
2281 *size = args.size;
2282
2283 return 0;
2284}
2285
2286static int find_entire_kern_cb(void *arg, const char *name __maybe_unused,
2287 char type, u64 start)
2288{
2289 struct sym_args *args = arg;
2290
e85e0e0c 2291 if (!kallsyms__is_function(type))
1b36c03e
AH		 2292 return 0;
2293
2294 if (!args->started) {
2295 args->started = true;
2296 args->start = start;
2297 }
2298 /* Don't know exactly where the kernel ends, so we add a page */
2299 args->size = round_up(start, page_size) + page_size - args->start;
2300
2301 return 0;
2302}
2303
2304static int addr_filter__entire_kernel(struct addr_filter *filt)
2305{
2306 struct sym_args args = { .started = false };
2307 int err;
2308
2309 err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb);
2310 if (err < 0 || !args.started) {
2311 pr_err("Failed to parse /proc/kallsyms\n");
2312 return err;
2313 }
2314
2315 filt->addr = args.start;
2316 filt->size = args.size;
2317
2318 return 0;
2319}
2320
2321static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size)
2322{
2323 if (start + size >= filt->addr)
2324 return 0;
2325
2326 if (filt->sym_from) {
2327 pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n",
2328 filt->sym_to, start, filt->sym_from, filt->addr);
2329 } else {
2330 pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n",
2331 filt->sym_to, start, filt->addr);
2332 }
2333
2334 return -EINVAL;
2335}
2336
2337static int addr_filter__resolve_kernel_syms(struct addr_filter *filt)
2338{
2339 bool no_size = false;
2340 u64 start, size;
2341 int err;
2342
2343 if (symbol_conf.kptr_restrict) {
2344 pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");
2345 return -EINVAL;
2346 }
2347
2348 if (filt->sym_from && !strcmp(filt->sym_from, "*"))
2349 return addr_filter__entire_kernel(filt);
2350
2351 if (filt->sym_from) {
2352 err = find_kern_sym(filt->sym_from, &start, &size,
2353 filt->sym_from_idx);
2354 if (err)
2355 return err;
2356 filt->addr = start;
2357 if (filt->range && !filt->size && !filt->sym_to) {
2358 filt->size = size;
c3a0bbc7 2359 no_size = !size;
1b36c03e
AH		 2360 }
2361 }
2362
2363 if (filt->sym_to) {
2364 err = find_kern_sym(filt->sym_to, &start, &size,
2365 filt->sym_to_idx);
2366 if (err)
2367 return err;
2368
2369 err = check_end_after_start(filt, start, size);
2370 if (err)
2371 return err;
2372 filt->size = start + size - filt->addr;
c3a0bbc7 2373 no_size = !size;
1b36c03e
AH		 2374 }
2375
2376 /* The very last symbol in kallsyms does not imply a particular size */
2377 if (no_size) {
2378 pr_err("Cannot determine size of symbol '%s'\n",
2379 filt->sym_to ? filt->sym_to : filt->sym_from);
2380 return -EINVAL;
2381 }
2382
2383 return 0;
2384}
2385
2386static struct dso *load_dso(const char *name)
2387{
2388 struct map *map;
2389 struct dso *dso;
2390
2391 map = dso__new_map(name);
2392 if (!map)
2393 return NULL;
2394
c1c49204
AH		 2395 if (map__load(map) < 0)
2396 pr_err("File '%s' not found or has no symbols.\n", name);
1b36c03e
AH		 2397
2398 dso = dso__get(map->dso);
2399
2400 map__put(map);
2401
2402 return dso;
2403}
2404
2405static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt,
2406 int idx)
2407{
2408 /* Same name, and global or the n'th found or any */
2409 return !arch__compare_symbol_names(name, sym->name) &&
2410 ((!idx && sym->binding == STB_GLOBAL) ||
2411 (idx > 0 && ++*cnt == idx) ||
2412 idx < 0);
2413}
2414
2415static void print_duplicate_syms(struct dso *dso, const char *sym_name)
2416{
2417 struct symbol *sym;
2418 bool near = false;
2419 int cnt = 0;
2420
2421 pr_err("Multiple symbols with name '%s'\n", sym_name);
2422
5cf88a63 2423 sym = dso__first_symbol(dso);
1b36c03e
AH		 2424 while (sym) {
2425 if (dso_sym_match(sym, sym_name, &cnt, -1)) {
2426 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2427 ++cnt, sym->start,
2428 sym->binding == STB_GLOBAL ? 'g' :
2429 sym->binding == STB_LOCAL ? 'l' : 'w',
2430 sym->name);
2431 near = true;
2432 } else if (near) {
2433 near = false;
2434 pr_err("\t\twhich is near\t\t%s\n", sym->name);
2435 }
2436 sym = dso__next_symbol(sym);
2437 }
2438
2439 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2440 sym_name);
2441 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2442}
2443
2444static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start,
2445 u64 *size, int idx)
2446{
2447 struct symbol *sym;
2448 int cnt = 0;
2449
2450 *start = 0;
2451 *size = 0;
2452
5cf88a63 2453 sym = dso__first_symbol(dso);
1b36c03e
AH		 2454 while (sym) {
2455 if (*start) {
2456 if (!*size)
2457 *size = sym->start - *start;
2458 if (idx > 0) {
2459 if (*size)
2460 return 1;
2461 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2462 print_duplicate_syms(dso, sym_name);
2463 return -EINVAL;
2464 }
2465 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2466 *start = sym->start;
2467 *size = sym->end - sym->start;
2468 }
2469 sym = dso__next_symbol(sym);
2470 }
2471
2472 if (!*start)
2473 return sym_not_found_error(sym_name, idx);
2474
2475 return 0;
2476}
2477
2478static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso)
2479{
57176601
AH		 2480 if (dso__data_file_size(dso, NULL)) {
2481 pr_err("Failed to determine filter for %s\nCannot determine file size.\n",
1b36c03e
AH		 2482 filt->filename);
2483 return -EINVAL;
2484 }
2485
57176601
AH		 2486 filt->addr = 0;
2487 filt->size = dso->data.file_size;
1b36c03e
AH		 2488
2489 return 0;
2490}
2491
2492static int addr_filter__resolve_syms(struct addr_filter *filt)
2493{
2494 u64 start, size;
2495 struct dso *dso;
2496 int err = 0;
2497
2498 if (!filt->sym_from && !filt->sym_to)
2499 return 0;
2500
2501 if (!filt->filename)
2502 return addr_filter__resolve_kernel_syms(filt);
2503
2504 dso = load_dso(filt->filename);
2505 if (!dso) {
2506 pr_err("Failed to load symbols from: %s\n", filt->filename);
2507 return -EINVAL;
2508 }
2509
2510 if (filt->sym_from && !strcmp(filt->sym_from, "*")) {
2511 err = addr_filter__entire_dso(filt, dso);
2512 goto put_dso;
2513 }
2514
2515 if (filt->sym_from) {
2516 err = find_dso_sym(dso, filt->sym_from, &start, &size,
2517 filt->sym_from_idx);
2518 if (err)
2519 goto put_dso;
2520 filt->addr = start;
2521 if (filt->range && !filt->size && !filt->sym_to)
2522 filt->size = size;
2523 }
2524
2525 if (filt->sym_to) {
2526 err = find_dso_sym(dso, filt->sym_to, &start, &size,
2527 filt->sym_to_idx);
2528 if (err)
2529 goto put_dso;
2530
2531 err = check_end_after_start(filt, start, size);
2532 if (err)
2533 return err;
2534
2535 filt->size = start + size - filt->addr;
2536 }
2537
2538put_dso:
2539 dso__put(dso);
2540
2541 return err;
2542}
2543
2544static char *addr_filter__to_str(struct addr_filter *filt)
2545{
2546 char filename_buf[PATH_MAX];
2547 const char *at = "";
2548 const char *fn = "";
2549 char *filter;
2550 int err;
2551
2552 if (filt->filename) {
2553 at = "@";
2554 fn = realpath(filt->filename, filename_buf);
2555 if (!fn)
2556 return NULL;
2557 }
2558
2559 if (filt->range) {
2560 err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s",
2561 filt->action, filt->addr, filt->size, at, fn);
2562 } else {
2563 err = asprintf(&filter, "%s 0x%"PRIx64"%s%s",
2564 filt->action, filt->addr, at, fn);
2565 }
2566
2567 return err < 0 ? NULL : filter;
2568}
2569
32dcd021 2570static int parse_addr_filter(struct evsel *evsel, const char *filter,
1b36c03e
AH		 2571 int max_nr)
2572{
2573 struct addr_filters filts;
2574 struct addr_filter *filt;
2575 int err;
2576
2577 addr_filters__init(&filts);
2578
2579 err = addr_filters__parse_bare_filter(&filts, filter);
2580 if (err)
2581 goto out_exit;
2582
2583 if (filts.cnt > max_nr) {
2584 pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",
2585 filts.cnt, max_nr);
2586 err = -EINVAL;
2587 goto out_exit;
2588 }
2589
2590 list_for_each_entry(filt, &filts.head, list) {
2591 char *new_filter;
2592
2593 err = addr_filter__resolve_syms(filt);
2594 if (err)
2595 goto out_exit;
2596
2597 new_filter = addr_filter__to_str(filt);
2598 if (!new_filter) {
2599 err = -ENOMEM;
2600 goto out_exit;
2601 }
2602
ad681adf 2603 if (evsel__append_addr_filter(evsel, new_filter)) {
1b36c03e
AH		 2604 err = -ENOMEM;
2605 goto out_exit;
2606 }
2607 }
2608
2609out_exit:
2610 addr_filters__exit(&filts);
2611
2612 if (err) {
2613 pr_err("Failed to parse address filter: '%s'\n", filter);
2614 pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");
2615 pr_err("Where multiple filters are separated by space or comma.\n");
2616 }
2617
2618 return err;
2619}
2620
ad681adf 2621static int evsel__nr_addr_filter(struct evsel *evsel)
1b36c03e 2622{
e76026bd 2623 struct perf_pmu *pmu = evsel__find_pmu(evsel);
1b36c03e
AH		 2624 int nr_addr_filters = 0;
2625
2626 if (!pmu)
2627 return 0;
2628
2629 perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters);
2630
2631 return nr_addr_filters;
2632}
2633
63503dba 2634int auxtrace_parse_filters(struct evlist *evlist)
1b36c03e 2635{
32dcd021 2636 struct evsel *evsel;
1b36c03e
AH		 2637 char *filter;
2638 int err, max_nr;
2639
2640 evlist__for_each_entry(evlist, evsel) {
2641 filter = evsel->filter;
ad681adf 2642 max_nr = evsel__nr_addr_filter(evsel);
1b36c03e
AH		 2643 if (!filter || !max_nr)
2644 continue;
2645 evsel->filter = NULL;
2646 err = parse_addr_filter(evsel, filter, max_nr);
2647 free(filter);
2648 if (err)
2649 return err;
2650 pr_debug("Address filter: %s\n", evsel->filter);
2651 }
2652
2653 return 0;
2654}
f2a39fe8
ACM		 2655
2656int auxtrace__process_event(struct perf_session *session, union perf_event *event,
2657 struct perf_sample *sample, struct perf_tool *tool)
2658{
2659 if (!session->auxtrace)
2660 return 0;
2661
2662 return session->auxtrace->process_event(session, event, sample, tool);
2663}
2664
b04b8dd1
AH		 2665void auxtrace__dump_auxtrace_sample(struct perf_session *session,
2666 struct perf_sample *sample)
2667{
2668 if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample ||
2669 auxtrace__dont_decode(session))
2670 return;
2671
2672 session->auxtrace->dump_auxtrace_sample(session, sample);
2673}
2674
f2a39fe8
ACM		 2675int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool)
2676{
2677 if (!session->auxtrace)
2678 return 0;
2679
2680 return session->auxtrace->flush_events(session, tool);
2681}
2682
2683void auxtrace__free_events(struct perf_session *session)
2684{
2685 if (!session->auxtrace)
2686 return;
2687
2688 return session->auxtrace->free_events(session);
2689}
2690
2691void auxtrace__free(struct perf_session *session)
2692{
2693 if (!session->auxtrace)
2694 return;
2695
2696 return session->auxtrace->free(session);
2697}
853f37d7
AH		 2698
2699bool auxtrace__evsel_is_auxtrace(struct perf_session *session,
2700 struct evsel *evsel)
2701{
2702 if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace)
2703 return false;
2704
2705 return session->auxtrace->evsel_is_auxtrace(session, evsel);
2706}
Ubuntu Jammy Linux kernel mirror