1 #include <sys/sysmacros.h>
26 #include "../builtin.h"
29 struct perf_data_file
*output
;
30 struct perf_session
*session
;
31 struct machine
*machine
;
32 union jr_entry
*entry
;
37 bool needs_bswap
; /* handles cross-endianess */
38 bool use_arch_timestamp
;
40 size_t nr_debug_entries
;
41 uint32_t code_load_count
;
43 struct rb_root code_root
;
47 struct debug_line_info
{
50 /* The filename format is unspecified, absolute path, relative etc. */
51 char const filename
[0];
55 struct perf_tool tool
;
56 struct perf_data_file output
;
57 struct perf_data_file input
;
61 #define hmax(a, b) ((a) > (b) ? (a) : (b))
62 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
65 jit_emit_elf(char *filename
,
76 fprintf(stderr
, "write ELF image %s\n", filename
);
78 fd
= open(filename
, O_CREAT
|O_TRUNC
|O_WRONLY
, 0644);
80 pr_warning("cannot create jit ELF %s: %s\n", filename
, strerror(errno
));
84 ret
= jit_write_elf(fd
, code_addr
, sym
, (const void *)code
, csize
, debug
, nr_debug_entries
);
95 jit_close(struct jit_buf_desc
*jd
)
105 jit_validate_events(struct perf_session
*session
)
107 struct perf_evsel
*evsel
;
110 * check that all events use CLOCK_MONOTONIC
112 evlist__for_each_entry(session
->evlist
, evsel
) {
113 if (evsel
->attr
.use_clockid
== 0 || evsel
->attr
.clockid
!= CLOCK_MONOTONIC
)
120 jit_open(struct jit_buf_desc
*jd
, const char *name
)
122 struct jitheader header
;
123 struct jr_prefix
*prefix
;
125 void *n
, *buf
= NULL
;
126 int ret
, retval
= -1;
128 jd
->in
= fopen(name
, "r");
132 bsz
= hmax(sizeof(header
), sizeof(*prefix
));
139 * protect from writer modifying the file while we are reading it
143 ret
= fread(buf
, sizeof(header
), 1, jd
->in
);
147 memcpy(&header
, buf
, sizeof(header
));
149 if (header
.magic
!= JITHEADER_MAGIC
) {
150 if (header
.magic
!= JITHEADER_MAGIC_SW
)
152 jd
->needs_bswap
= true;
155 if (jd
->needs_bswap
) {
156 header
.version
= bswap_32(header
.version
);
157 header
.total_size
= bswap_32(header
.total_size
);
158 header
.pid
= bswap_32(header
.pid
);
159 header
.elf_mach
= bswap_32(header
.elf_mach
);
160 header
.timestamp
= bswap_64(header
.timestamp
);
161 header
.flags
= bswap_64(header
.flags
);
164 jd
->use_arch_timestamp
= header
.flags
& JITDUMP_FLAGS_ARCH_TIMESTAMP
;
167 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
170 (unsigned long long)header
.timestamp
,
173 jd
->use_arch_timestamp
);
175 if (header
.flags
& JITDUMP_FLAGS_RESERVED
) {
176 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
177 (unsigned long long)header
.flags
& JITDUMP_FLAGS_RESERVED
);
181 if (jd
->use_arch_timestamp
&& !jd
->session
->time_conv
.time_mult
) {
182 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
187 * validate event is using the correct clockid
189 if (!jd
->use_arch_timestamp
&& jit_validate_events(jd
->session
)) {
190 pr_err("error, jitted code must be sampled with perf record -k 1\n");
194 bs
= header
.total_size
- sizeof(header
);
197 n
= realloc(buf
, bs
);
202 /* read extra we do not know about */
203 ret
= fread(buf
, bs
- bsz
, 1, jd
->in
);
208 * keep dirname for generating files and mmap records
210 strcpy(jd
->dir
, name
);
220 static union jr_entry
*
221 jit_get_next_entry(struct jit_buf_desc
*jd
)
223 struct jr_prefix
*prefix
;
232 if (jd
->buf
== NULL
) {
233 size_t sz
= getpagesize();
234 if (sz
< sizeof(*prefix
))
235 sz
= sizeof(*prefix
);
237 jd
->buf
= malloc(sz
);
247 * file is still locked at this point
249 ret
= fread(prefix
, sizeof(*prefix
), 1, jd
->in
);
253 if (jd
->needs_bswap
) {
254 prefix
->id
= bswap_32(prefix
->id
);
255 prefix
->total_size
= bswap_32(prefix
->total_size
);
256 prefix
->timestamp
= bswap_64(prefix
->timestamp
);
259 size
= prefix
->total_size
;
262 if (bs
< sizeof(*prefix
))
265 if (id
>= JIT_CODE_MAX
) {
266 pr_warning("next_entry: unknown prefix %d, skipping\n", id
);
269 if (bs
> jd
->bufsize
) {
271 n
= realloc(jd
->buf
, bs
);
278 addr
= ((void *)jd
->buf
) + sizeof(*prefix
);
280 ret
= fread(addr
, bs
- sizeof(*prefix
), 1, jd
->in
);
284 jr
= (union jr_entry
*)jd
->buf
;
287 case JIT_CODE_DEBUG_INFO
:
288 if (jd
->needs_bswap
) {
290 jr
->info
.code_addr
= bswap_64(jr
->info
.code_addr
);
291 jr
->info
.nr_entry
= bswap_64(jr
->info
.nr_entry
);
292 for (n
= 0 ; n
< jr
->info
.nr_entry
; n
++) {
293 jr
->info
.entries
[n
].addr
= bswap_64(jr
->info
.entries
[n
].addr
);
294 jr
->info
.entries
[n
].lineno
= bswap_32(jr
->info
.entries
[n
].lineno
);
295 jr
->info
.entries
[n
].discrim
= bswap_32(jr
->info
.entries
[n
].discrim
);
302 if (jd
->needs_bswap
) {
303 jr
->load
.pid
= bswap_32(jr
->load
.pid
);
304 jr
->load
.tid
= bswap_32(jr
->load
.tid
);
305 jr
->load
.vma
= bswap_64(jr
->load
.vma
);
306 jr
->load
.code_addr
= bswap_64(jr
->load
.code_addr
);
307 jr
->load
.code_size
= bswap_64(jr
->load
.code_size
);
308 jr
->load
.code_index
= bswap_64(jr
->load
.code_index
);
310 jd
->code_load_count
++;
313 if (jd
->needs_bswap
) {
314 jr
->move
.pid
= bswap_32(jr
->move
.pid
);
315 jr
->move
.tid
= bswap_32(jr
->move
.tid
);
316 jr
->move
.vma
= bswap_64(jr
->move
.vma
);
317 jr
->move
.old_code_addr
= bswap_64(jr
->move
.old_code_addr
);
318 jr
->move
.new_code_addr
= bswap_64(jr
->move
.new_code_addr
);
319 jr
->move
.code_size
= bswap_64(jr
->move
.code_size
);
320 jr
->move
.code_index
= bswap_64(jr
->move
.code_index
);
331 jit_inject_event(struct jit_buf_desc
*jd
, union perf_event
*event
)
335 size
= perf_data_file__write(jd
->output
, event
, event
->header
.size
);
339 jd
->bytes_written
+= size
;
343 static uint64_t convert_timestamp(struct jit_buf_desc
*jd
, uint64_t timestamp
)
345 struct perf_tsc_conversion tc
;
347 if (!jd
->use_arch_timestamp
)
350 tc
.time_shift
= jd
->session
->time_conv
.time_shift
;
351 tc
.time_mult
= jd
->session
->time_conv
.time_mult
;
352 tc
.time_zero
= jd
->session
->time_conv
.time_zero
;
357 return tsc_to_perf_time(timestamp
, &tc
);
360 static int jit_repipe_code_load(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
362 struct perf_sample sample
;
363 union perf_event
*event
;
364 struct perf_tool
*tool
= jd
->session
->tool
;
382 csize
= jr
->load
.code_size
;
383 addr
= jr
->load
.code_addr
;
384 sym
= (void *)((unsigned long)jr
+ sizeof(jr
->load
));
385 code
= (unsigned long)jr
+ jr
->load
.p
.total_size
- csize
;
386 count
= jr
->load
.code_index
;
387 idr_size
= jd
->machine
->id_hdr_size
;
389 event
= calloc(1, sizeof(*event
) + idr_size
);
393 filename
= event
->mmap2
.filename
;
394 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%u.so",
401 size
= PERF_ALIGN(size
, sizeof(u64
));
402 uaddr
= (uintptr_t)code
;
403 ret
= jit_emit_elf(filename
, sym
, addr
, (const void *)uaddr
, csize
, jd
->debug_data
, jd
->nr_debug_entries
);
405 if (jd
->debug_data
&& jd
->nr_debug_entries
) {
406 free(jd
->debug_data
);
407 jd
->debug_data
= NULL
;
408 jd
->nr_debug_entries
= 0;
415 if (stat(filename
, &st
))
416 memset(&st
, 0, sizeof(st
));
418 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
419 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
420 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
421 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
423 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
424 event
->mmap2
.start
= addr
;
425 event
->mmap2
.len
= csize
;
426 event
->mmap2
.pid
= pid
;
427 event
->mmap2
.tid
= tid
;
428 event
->mmap2
.ino
= st
.st_ino
;
429 event
->mmap2
.maj
= major(st
.st_dev
);
430 event
->mmap2
.min
= minor(st
.st_dev
);
431 event
->mmap2
.prot
= st
.st_mode
;
432 event
->mmap2
.flags
= MAP_SHARED
;
433 event
->mmap2
.ino_generation
= 1;
435 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
436 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
440 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
441 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
444 * create pseudo sample to induce dso hit increment
445 * use first address as sample address
447 memset(&sample
, 0, sizeof(sample
));
448 sample
.cpumode
= PERF_RECORD_MISC_USER
;
451 sample
.time
= id
->time
;
454 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
458 ret
= jit_inject_event(jd
, event
);
460 * mark dso as use to generate buildid in the header
463 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
468 static int jit_repipe_code_move(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
470 struct perf_sample sample
;
471 union perf_event
*event
;
472 struct perf_tool
*tool
= jd
->session
->tool
;
486 idr_size
= jd
->machine
->id_hdr_size
;
489 * +16 to account for sample_id_all (hack)
491 event
= calloc(1, sizeof(*event
) + 16);
495 filename
= event
->mmap2
.filename
;
496 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%"PRIu64
,
499 jr
->move
.code_index
);
503 if (stat(filename
, &st
))
504 memset(&st
, 0, sizeof(st
));
506 size
= PERF_ALIGN(size
, sizeof(u64
));
508 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
509 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
510 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
511 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
512 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
513 event
->mmap2
.start
= jr
->move
.new_code_addr
;
514 event
->mmap2
.len
= jr
->move
.code_size
;
515 event
->mmap2
.pid
= pid
;
516 event
->mmap2
.tid
= tid
;
517 event
->mmap2
.ino
= st
.st_ino
;
518 event
->mmap2
.maj
= major(st
.st_dev
);
519 event
->mmap2
.min
= minor(st
.st_dev
);
520 event
->mmap2
.prot
= st
.st_mode
;
521 event
->mmap2
.flags
= MAP_SHARED
;
522 event
->mmap2
.ino_generation
= 1;
524 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
525 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
529 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
530 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
533 * create pseudo sample to induce dso hit increment
534 * use first address as sample address
536 memset(&sample
, 0, sizeof(sample
));
537 sample
.cpumode
= PERF_RECORD_MISC_USER
;
540 sample
.time
= id
->time
;
541 sample
.ip
= jr
->move
.new_code_addr
;
543 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
547 ret
= jit_inject_event(jd
, event
);
549 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
554 static int jit_repipe_debug_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
562 sz
= jr
->prefix
.total_size
- sizeof(jr
->info
);
567 memcpy(data
, &jr
->info
.entries
, sz
);
569 jd
->debug_data
= data
;
572 * we must use nr_entry instead of size here because
573 * we cannot distinguish actual entry from padding otherwise
575 jd
->nr_debug_entries
= jr
->info
.nr_entry
;
581 jit_process_dump(struct jit_buf_desc
*jd
)
586 while ((jr
= jit_get_next_entry(jd
))) {
587 switch(jr
->prefix
.id
) {
589 ret
= jit_repipe_code_load(jd
, jr
);
592 ret
= jit_repipe_code_move(jd
, jr
);
594 case JIT_CODE_DEBUG_INFO
:
595 ret
= jit_repipe_debug_info(jd
, jr
);
606 jit_inject(struct jit_buf_desc
*jd
, char *path
)
611 fprintf(stderr
, "injecting: %s\n", path
);
613 ret
= jit_open(jd
, path
);
617 ret
= jit_process_dump(jd
);
622 fprintf(stderr
, "injected: %s (%d)\n", path
, ret
);
628 * File must be with pattern .../jit-XXXX.dump
629 * where XXXX is the PID of the process which did the mmap()
630 * as captured in the RECORD_MMAP record
633 jit_detect(char *mmap_name
, pid_t pid
)
640 fprintf(stderr
, "jit marker trying : %s\n", mmap_name
);
644 p
= strrchr(mmap_name
, '/');
651 if (strncmp(p
, "/jit-", 5))
660 * must be followed by a pid
665 pid2
= (int)strtol(p
, &end
, 10);
670 * pid does not match mmap pid
671 * pid==0 in system-wide mode (synthesized)
673 if (pid
&& pid2
!= pid
)
678 if (strcmp(end
, ".dump"))
682 fprintf(stderr
, "jit marker found: %s\n", mmap_name
);
688 jit_process(struct perf_session
*session
,
689 struct perf_data_file
*output
,
690 struct machine
*machine
,
695 struct perf_evsel
*first
;
696 struct jit_buf_desc jd
;
700 * first, detect marker mmap (i.e., the jitdump mmap)
702 if (jit_detect(filename
, pid
))
705 memset(&jd
, 0, sizeof(jd
));
707 jd
.session
= session
;
709 jd
.machine
= machine
;
712 * track sample_type to compute id_all layout
713 * perf sets the same sample type to all events as of now
715 first
= perf_evlist__first(session
->evlist
);
716 jd
.sample_type
= first
->attr
.sample_type
;
720 ret
= jit_inject(&jd
, filename
);
722 *nbytes
= jd
.bytes_written
;