1 #include <sys/sysmacros.h>
13 #include <linux/stringify.h>
27 #include "../builtin.h"
29 #include "sane_ctype.h"
32 struct perf_data_file
*output
;
33 struct perf_session
*session
;
34 struct machine
*machine
;
35 union jr_entry
*entry
;
40 bool needs_bswap
; /* handles cross-endianess */
41 bool use_arch_timestamp
;
44 uint64_t unwinding_size
;
45 uint64_t unwinding_mapped_size
;
46 uint64_t eh_frame_hdr_size
;
47 size_t nr_debug_entries
;
48 uint32_t code_load_count
;
50 struct rb_root code_root
;
54 struct debug_line_info
{
57 /* The filename format is unspecified, absolute path, relative etc. */
58 char const filename
[0];
62 struct perf_tool tool
;
63 struct perf_data_file output
;
64 struct perf_data_file input
;
68 #define hmax(a, b) ((a) > (b) ? (a) : (b))
69 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
72 jit_emit_elf(char *filename
,
80 uint32_t unwinding_header_size
,
81 uint32_t unwinding_size
)
86 fprintf(stderr
, "write ELF image %s\n", filename
);
88 fd
= open(filename
, O_CREAT
|O_TRUNC
|O_WRONLY
, 0644);
90 pr_warning("cannot create jit ELF %s: %s\n", filename
, strerror(errno
));
94 ret
= jit_write_elf(fd
, code_addr
, sym
, (const void *)code
, csize
, debug
, nr_debug_entries
,
95 unwinding
, unwinding_header_size
, unwinding_size
);
106 jit_close(struct jit_buf_desc
*jd
)
116 jit_validate_events(struct perf_session
*session
)
118 struct perf_evsel
*evsel
;
121 * check that all events use CLOCK_MONOTONIC
123 evlist__for_each_entry(session
->evlist
, evsel
) {
124 if (evsel
->attr
.use_clockid
== 0 || evsel
->attr
.clockid
!= CLOCK_MONOTONIC
)
131 jit_open(struct jit_buf_desc
*jd
, const char *name
)
133 struct jitheader header
;
134 struct jr_prefix
*prefix
;
136 void *n
, *buf
= NULL
;
137 int ret
, retval
= -1;
139 jd
->in
= fopen(name
, "r");
143 bsz
= hmax(sizeof(header
), sizeof(*prefix
));
150 * protect from writer modifying the file while we are reading it
154 ret
= fread(buf
, sizeof(header
), 1, jd
->in
);
158 memcpy(&header
, buf
, sizeof(header
));
160 if (header
.magic
!= JITHEADER_MAGIC
) {
161 if (header
.magic
!= JITHEADER_MAGIC_SW
)
163 jd
->needs_bswap
= true;
166 if (jd
->needs_bswap
) {
167 header
.version
= bswap_32(header
.version
);
168 header
.total_size
= bswap_32(header
.total_size
);
169 header
.pid
= bswap_32(header
.pid
);
170 header
.elf_mach
= bswap_32(header
.elf_mach
);
171 header
.timestamp
= bswap_64(header
.timestamp
);
172 header
.flags
= bswap_64(header
.flags
);
175 jd
->use_arch_timestamp
= header
.flags
& JITDUMP_FLAGS_ARCH_TIMESTAMP
;
178 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
181 (unsigned long long)header
.timestamp
,
184 jd
->use_arch_timestamp
);
186 if (header
.version
> JITHEADER_VERSION
) {
187 pr_err("wrong jitdump version %u, expected " __stringify(JITHEADER_VERSION
),
192 if (header
.flags
& JITDUMP_FLAGS_RESERVED
) {
193 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
194 (unsigned long long)header
.flags
& JITDUMP_FLAGS_RESERVED
);
198 if (jd
->use_arch_timestamp
&& !jd
->session
->time_conv
.time_mult
) {
199 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
204 * validate event is using the correct clockid
206 if (!jd
->use_arch_timestamp
&& jit_validate_events(jd
->session
)) {
207 pr_err("error, jitted code must be sampled with perf record -k 1\n");
211 bs
= header
.total_size
- sizeof(header
);
214 n
= realloc(buf
, bs
);
219 /* read extra we do not know about */
220 ret
= fread(buf
, bs
- bsz
, 1, jd
->in
);
225 * keep dirname for generating files and mmap records
227 strcpy(jd
->dir
, name
);
237 static union jr_entry
*
238 jit_get_next_entry(struct jit_buf_desc
*jd
)
240 struct jr_prefix
*prefix
;
249 if (jd
->buf
== NULL
) {
250 size_t sz
= getpagesize();
251 if (sz
< sizeof(*prefix
))
252 sz
= sizeof(*prefix
);
254 jd
->buf
= malloc(sz
);
264 * file is still locked at this point
266 ret
= fread(prefix
, sizeof(*prefix
), 1, jd
->in
);
270 if (jd
->needs_bswap
) {
271 prefix
->id
= bswap_32(prefix
->id
);
272 prefix
->total_size
= bswap_32(prefix
->total_size
);
273 prefix
->timestamp
= bswap_64(prefix
->timestamp
);
276 size
= prefix
->total_size
;
279 if (bs
< sizeof(*prefix
))
282 if (id
>= JIT_CODE_MAX
) {
283 pr_warning("next_entry: unknown record type %d, skipping\n", id
);
285 if (bs
> jd
->bufsize
) {
287 n
= realloc(jd
->buf
, bs
);
294 addr
= ((void *)jd
->buf
) + sizeof(*prefix
);
296 ret
= fread(addr
, bs
- sizeof(*prefix
), 1, jd
->in
);
300 jr
= (union jr_entry
*)jd
->buf
;
303 case JIT_CODE_DEBUG_INFO
:
304 if (jd
->needs_bswap
) {
306 jr
->info
.code_addr
= bswap_64(jr
->info
.code_addr
);
307 jr
->info
.nr_entry
= bswap_64(jr
->info
.nr_entry
);
308 for (n
= 0 ; n
< jr
->info
.nr_entry
; n
++) {
309 jr
->info
.entries
[n
].addr
= bswap_64(jr
->info
.entries
[n
].addr
);
310 jr
->info
.entries
[n
].lineno
= bswap_32(jr
->info
.entries
[n
].lineno
);
311 jr
->info
.entries
[n
].discrim
= bswap_32(jr
->info
.entries
[n
].discrim
);
315 case JIT_CODE_UNWINDING_INFO
:
316 if (jd
->needs_bswap
) {
317 jr
->unwinding
.unwinding_size
= bswap_64(jr
->unwinding
.unwinding_size
);
318 jr
->unwinding
.eh_frame_hdr_size
= bswap_64(jr
->unwinding
.eh_frame_hdr_size
);
319 jr
->unwinding
.mapped_size
= bswap_64(jr
->unwinding
.mapped_size
);
325 if (jd
->needs_bswap
) {
326 jr
->load
.pid
= bswap_32(jr
->load
.pid
);
327 jr
->load
.tid
= bswap_32(jr
->load
.tid
);
328 jr
->load
.vma
= bswap_64(jr
->load
.vma
);
329 jr
->load
.code_addr
= bswap_64(jr
->load
.code_addr
);
330 jr
->load
.code_size
= bswap_64(jr
->load
.code_size
);
331 jr
->load
.code_index
= bswap_64(jr
->load
.code_index
);
333 jd
->code_load_count
++;
336 if (jd
->needs_bswap
) {
337 jr
->move
.pid
= bswap_32(jr
->move
.pid
);
338 jr
->move
.tid
= bswap_32(jr
->move
.tid
);
339 jr
->move
.vma
= bswap_64(jr
->move
.vma
);
340 jr
->move
.old_code_addr
= bswap_64(jr
->move
.old_code_addr
);
341 jr
->move
.new_code_addr
= bswap_64(jr
->move
.new_code_addr
);
342 jr
->move
.code_size
= bswap_64(jr
->move
.code_size
);
343 jr
->move
.code_index
= bswap_64(jr
->move
.code_index
);
348 /* skip unknown record (we have read them) */
355 jit_inject_event(struct jit_buf_desc
*jd
, union perf_event
*event
)
359 size
= perf_data_file__write(jd
->output
, event
, event
->header
.size
);
363 jd
->bytes_written
+= size
;
367 static uint64_t convert_timestamp(struct jit_buf_desc
*jd
, uint64_t timestamp
)
369 struct perf_tsc_conversion tc
;
371 if (!jd
->use_arch_timestamp
)
374 tc
.time_shift
= jd
->session
->time_conv
.time_shift
;
375 tc
.time_mult
= jd
->session
->time_conv
.time_mult
;
376 tc
.time_zero
= jd
->session
->time_conv
.time_zero
;
381 return tsc_to_perf_time(timestamp
, &tc
);
384 static int jit_repipe_code_load(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
386 struct perf_sample sample
;
387 union perf_event
*event
;
388 struct perf_tool
*tool
= jd
->session
->tool
;
397 int ret
, csize
, usize
;
406 csize
= jr
->load
.code_size
;
407 usize
= jd
->unwinding_mapped_size
;
408 addr
= jr
->load
.code_addr
;
409 sym
= (void *)((unsigned long)jr
+ sizeof(jr
->load
));
410 code
= (unsigned long)jr
+ jr
->load
.p
.total_size
- csize
;
411 count
= jr
->load
.code_index
;
412 idr_size
= jd
->machine
->id_hdr_size
;
414 event
= calloc(1, sizeof(*event
) + idr_size
);
418 filename
= event
->mmap2
.filename
;
419 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%u.so",
426 size
= PERF_ALIGN(size
, sizeof(u64
));
427 uaddr
= (uintptr_t)code
;
428 ret
= jit_emit_elf(filename
, sym
, addr
, (const void *)uaddr
, csize
, jd
->debug_data
, jd
->nr_debug_entries
,
429 jd
->unwinding_data
, jd
->eh_frame_hdr_size
, jd
->unwinding_size
);
431 if (jd
->debug_data
&& jd
->nr_debug_entries
) {
432 free(jd
->debug_data
);
433 jd
->debug_data
= NULL
;
434 jd
->nr_debug_entries
= 0;
437 if (jd
->unwinding_data
&& jd
->eh_frame_hdr_size
) {
438 free(jd
->unwinding_data
);
439 jd
->unwinding_data
= NULL
;
440 jd
->eh_frame_hdr_size
= 0;
441 jd
->unwinding_mapped_size
= 0;
442 jd
->unwinding_size
= 0;
449 if (stat(filename
, &st
))
450 memset(&st
, 0, sizeof(st
));
452 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
453 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
454 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
455 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
457 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
458 event
->mmap2
.start
= addr
;
459 event
->mmap2
.len
= usize
? ALIGN_8(csize
) + usize
: csize
;
460 event
->mmap2
.pid
= pid
;
461 event
->mmap2
.tid
= tid
;
462 event
->mmap2
.ino
= st
.st_ino
;
463 event
->mmap2
.maj
= major(st
.st_dev
);
464 event
->mmap2
.min
= minor(st
.st_dev
);
465 event
->mmap2
.prot
= st
.st_mode
;
466 event
->mmap2
.flags
= MAP_SHARED
;
467 event
->mmap2
.ino_generation
= 1;
469 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
470 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
474 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
475 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
478 * create pseudo sample to induce dso hit increment
479 * use first address as sample address
481 memset(&sample
, 0, sizeof(sample
));
482 sample
.cpumode
= PERF_RECORD_MISC_USER
;
485 sample
.time
= id
->time
;
488 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
492 ret
= jit_inject_event(jd
, event
);
494 * mark dso as use to generate buildid in the header
497 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
502 static int jit_repipe_code_move(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
504 struct perf_sample sample
;
505 union perf_event
*event
;
506 struct perf_tool
*tool
= jd
->session
->tool
;
521 usize
= jd
->unwinding_mapped_size
;
522 idr_size
= jd
->machine
->id_hdr_size
;
525 * +16 to account for sample_id_all (hack)
527 event
= calloc(1, sizeof(*event
) + 16);
531 filename
= event
->mmap2
.filename
;
532 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%"PRIu64
,
535 jr
->move
.code_index
);
539 if (stat(filename
, &st
))
540 memset(&st
, 0, sizeof(st
));
542 size
= PERF_ALIGN(size
, sizeof(u64
));
544 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
545 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
546 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
547 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
548 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
549 event
->mmap2
.start
= jr
->move
.new_code_addr
;
550 event
->mmap2
.len
= usize
? ALIGN_8(jr
->move
.code_size
) + usize
551 : jr
->move
.code_size
;
552 event
->mmap2
.pid
= pid
;
553 event
->mmap2
.tid
= tid
;
554 event
->mmap2
.ino
= st
.st_ino
;
555 event
->mmap2
.maj
= major(st
.st_dev
);
556 event
->mmap2
.min
= minor(st
.st_dev
);
557 event
->mmap2
.prot
= st
.st_mode
;
558 event
->mmap2
.flags
= MAP_SHARED
;
559 event
->mmap2
.ino_generation
= 1;
561 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
562 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
566 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
567 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
570 * create pseudo sample to induce dso hit increment
571 * use first address as sample address
573 memset(&sample
, 0, sizeof(sample
));
574 sample
.cpumode
= PERF_RECORD_MISC_USER
;
577 sample
.time
= id
->time
;
578 sample
.ip
= jr
->move
.new_code_addr
;
580 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
584 ret
= jit_inject_event(jd
, event
);
586 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
591 static int jit_repipe_debug_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
599 sz
= jr
->prefix
.total_size
- sizeof(jr
->info
);
604 memcpy(data
, &jr
->info
.entries
, sz
);
606 jd
->debug_data
= data
;
609 * we must use nr_entry instead of size here because
610 * we cannot distinguish actual entry from padding otherwise
612 jd
->nr_debug_entries
= jr
->info
.nr_entry
;
618 jit_repipe_unwinding_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
620 void *unwinding_data
;
621 uint32_t unwinding_data_size
;
626 unwinding_data_size
= jr
->prefix
.total_size
- sizeof(jr
->unwinding
);
627 unwinding_data
= malloc(unwinding_data_size
);
631 memcpy(unwinding_data
, &jr
->unwinding
.unwinding_data
,
632 unwinding_data_size
);
634 jd
->eh_frame_hdr_size
= jr
->unwinding
.eh_frame_hdr_size
;
635 jd
->unwinding_size
= jr
->unwinding
.unwinding_size
;
636 jd
->unwinding_mapped_size
= jr
->unwinding
.mapped_size
;
637 jd
->unwinding_data
= unwinding_data
;
643 jit_process_dump(struct jit_buf_desc
*jd
)
648 while ((jr
= jit_get_next_entry(jd
))) {
649 switch(jr
->prefix
.id
) {
651 ret
= jit_repipe_code_load(jd
, jr
);
654 ret
= jit_repipe_code_move(jd
, jr
);
656 case JIT_CODE_DEBUG_INFO
:
657 ret
= jit_repipe_debug_info(jd
, jr
);
659 case JIT_CODE_UNWINDING_INFO
:
660 ret
= jit_repipe_unwinding_info(jd
, jr
);
671 jit_inject(struct jit_buf_desc
*jd
, char *path
)
676 fprintf(stderr
, "injecting: %s\n", path
);
678 ret
= jit_open(jd
, path
);
682 ret
= jit_process_dump(jd
);
687 fprintf(stderr
, "injected: %s (%d)\n", path
, ret
);
693 * File must be with pattern .../jit-XXXX.dump
694 * where XXXX is the PID of the process which did the mmap()
695 * as captured in the RECORD_MMAP record
698 jit_detect(char *mmap_name
, pid_t pid
)
705 fprintf(stderr
, "jit marker trying : %s\n", mmap_name
);
709 p
= strrchr(mmap_name
, '/');
716 if (strncmp(p
, "/jit-", 5))
725 * must be followed by a pid
730 pid2
= (int)strtol(p
, &end
, 10);
735 * pid does not match mmap pid
736 * pid==0 in system-wide mode (synthesized)
738 if (pid
&& pid2
!= pid
)
743 if (strcmp(end
, ".dump"))
747 fprintf(stderr
, "jit marker found: %s\n", mmap_name
);
753 jit_process(struct perf_session
*session
,
754 struct perf_data_file
*output
,
755 struct machine
*machine
,
760 struct perf_evsel
*first
;
761 struct jit_buf_desc jd
;
765 * first, detect marker mmap (i.e., the jitdump mmap)
767 if (jit_detect(filename
, pid
))
770 memset(&jd
, 0, sizeof(jd
));
772 jd
.session
= session
;
774 jd
.machine
= machine
;
777 * track sample_type to compute id_all layout
778 * perf sets the same sample type to all events as of now
780 first
= perf_evlist__first(session
->evlist
);
781 jd
.sample_type
= first
->attr
.sample_type
;
785 ret
= jit_inject(&jd
, filename
);
787 *nbytes
= jd
.bytes_written
;