[mirror_ubuntu-artful-kernel.git] / tools / perf / util / jitdump.c

#include <sys/sysmacros.h>
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <inttypes.h>
#include <byteswap.h>
#include <sys/stat.h>
#include <sys/mman.h>

#include "util.h"
#include "event.h"
#include "debug.h"
#include "evlist.h"
#include "symbol.h"
#include "strlist.h"
#include <elf.h>

#include "tsc.h"
#include "session.h"
#include "jit.h"
#include "jitdump.h"
#include "genelf.h"
#include "../builtin.h"

struct jit_buf_desc {
	struct perf_data_file *output;
	struct perf_session *session;
	struct machine *machine;
	union jr_entry   *entry;
	void             *buf;
	uint64_t	 sample_type;
	size_t           bufsize;
	FILE             *in;
	bool		 needs_bswap; /* handles cross-endianess */
	bool		 use_arch_timestamp;
	void		 *debug_data;
	void		 *unwinding_data;
	uint64_t	 unwinding_size;
	uint64_t	 unwinding_mapped_size;
	uint64_t         eh_frame_hdr_size;
	size_t		 nr_debug_entries;
	uint32_t         code_load_count;
	u64		 bytes_written;
	struct rb_root   code_root;
	char		 dir[PATH_MAX];
};

struct debug_line_info {
	unsigned long vma;
	unsigned int lineno;
	/* The filename format is unspecified, absolute path, relative etc. */
	char const filename[0];
};

struct jit_tool {
	struct perf_tool tool;
	struct perf_data_file	output;
	struct perf_data_file	input;
	u64 bytes_written;
};

#define hmax(a, b) ((a) > (b) ? (a) : (b))
#define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))

static int
jit_emit_elf(char *filename,
	     const char *sym,
	     uint64_t code_addr,
	     const void *code,
	     int csize,
	     void *debug,
	     int nr_debug_entries)
{
	int ret, fd;

	if (verbose > 0)
		fprintf(stderr, "write ELF image %s\n", filename);

	fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644);
	if (fd == -1) {
		pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno));
		return -1;
	}

        ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries);

        close(fd);

        if (ret)
                unlink(filename);

	return ret;
}

static void
jit_close(struct jit_buf_desc *jd)
{
	if (!(jd && jd->in))
		return;
	funlockfile(jd->in);
	fclose(jd->in);
	jd->in = NULL;
}

static int
jit_validate_events(struct perf_session *session)
{
	struct perf_evsel *evsel;

	/*
	 * check that all events use CLOCK_MONOTONIC
	 */
	evlist__for_each_entry(session->evlist, evsel) {
		if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC)
			return -1;
	}
	return 0;
}

static int
jit_open(struct jit_buf_desc *jd, const char *name)
{
	struct jitheader header;
	struct jr_prefix *prefix;
	ssize_t bs, bsz = 0;
	void *n, *buf = NULL;
	int ret, retval = -1;

	jd->in = fopen(name, "r");
	if (!jd->in)
		return -1;

	bsz = hmax(sizeof(header), sizeof(*prefix));

	buf = malloc(bsz);
	if (!buf)
		goto error;

	/*
	 * protect from writer modifying the file while we are reading it
	 */
	flockfile(jd->in);

	ret = fread(buf, sizeof(header), 1, jd->in);
	if (ret != 1)
		goto error;

	memcpy(&header, buf, sizeof(header));

	if (header.magic != JITHEADER_MAGIC) {
		if (header.magic != JITHEADER_MAGIC_SW)
			goto error;
		jd->needs_bswap = true;
	}

	if (jd->needs_bswap) {
		header.version    = bswap_32(header.version);
		header.total_size = bswap_32(header.total_size);
		header.pid	  = bswap_32(header.pid);
		header.elf_mach   = bswap_32(header.elf_mach);
		header.timestamp  = bswap_64(header.timestamp);
		header.flags      = bswap_64(header.flags);
	}

	jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP;

	if (verbose > 2)
		pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
			header.version,
			header.total_size,
			(unsigned long long)header.timestamp,
			header.pid,
			header.elf_mach,
			jd->use_arch_timestamp);

	if (header.flags & JITDUMP_FLAGS_RESERVED) {
		pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
		       (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED);
		goto error;
	}

	if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) {
		pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
		goto error;
	}

	/*
	 * validate event is using the correct clockid
	 */
	if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) {
		pr_err("error, jitted code must be sampled with perf record -k 1\n");
		goto error;
	}

	bs = header.total_size - sizeof(header);

	if (bs > bsz) {
		n = realloc(buf, bs);
		if (!n)
			goto error;
		bsz = bs;
		buf = n;
		/* read extra we do not know about */
		ret = fread(buf, bs - bsz, 1, jd->in);
		if (ret != 1)
			goto error;
	}
	/*
	 * keep dirname for generating files and mmap records
	 */
	strcpy(jd->dir, name);
	dirname(jd->dir);

	return 0;
error:
	funlockfile(jd->in);
	fclose(jd->in);
	return retval;
}

static union jr_entry *
jit_get_next_entry(struct jit_buf_desc *jd)
{
	struct jr_prefix *prefix;
	union jr_entry *jr;
	void *addr;
	size_t bs, size;
	int id, ret;

	if (!(jd && jd->in))
		return NULL;

	if (jd->buf == NULL) {
		size_t sz = getpagesize();
		if (sz < sizeof(*prefix))
			sz = sizeof(*prefix);

		jd->buf = malloc(sz);
		if (jd->buf == NULL)
			return NULL;

		jd->bufsize = sz;
	}

	prefix = jd->buf;

	/*
	 * file is still locked at this point
	 */
	ret = fread(prefix, sizeof(*prefix), 1, jd->in);
	if (ret  != 1)
		return NULL;

	if (jd->needs_bswap) {
		prefix->id   	   = bswap_32(prefix->id);
		prefix->total_size = bswap_32(prefix->total_size);
		prefix->timestamp  = bswap_64(prefix->timestamp);
	}
	id   = prefix->id;
	size = prefix->total_size;

	bs = (size_t)size;
	if (bs < sizeof(*prefix))
		return NULL;

	if (id >= JIT_CODE_MAX) {
		pr_warning("next_entry: unknown record type %d, skipping\n", id);
	}
	if (bs > jd->bufsize) {
		void *n;
		n = realloc(jd->buf, bs);
		if (!n)
			return NULL;
		jd->buf = n;
		jd->bufsize = bs;
	}

	addr = ((void *)jd->buf) + sizeof(*prefix);

	ret = fread(addr, bs - sizeof(*prefix), 1, jd->in);
	if (ret != 1)
		return NULL;

	jr = (union jr_entry *)jd->buf;

	switch(id) {
	case JIT_CODE_DEBUG_INFO:
		if (jd->needs_bswap) {
			uint64_t n;
			jr->info.code_addr = bswap_64(jr->info.code_addr);
			jr->info.nr_entry  = bswap_64(jr->info.nr_entry);
			for (n = 0 ; n < jr->info.nr_entry; n++) {
				jr->info.entries[n].addr    = bswap_64(jr->info.entries[n].addr);
				jr->info.entries[n].lineno  = bswap_32(jr->info.entries[n].lineno);
				jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim);
			}
		}
		break;
	case JIT_CODE_UNWINDING_INFO:
		if (jd->needs_bswap) {
			jr->unwinding.unwinding_size = bswap_64(jr->unwinding.unwinding_size);
			jr->unwinding.eh_frame_hdr_size = bswap_64(jr->unwinding.eh_frame_hdr_size);
			jr->unwinding.mapped_size = bswap_64(jr->unwinding.mapped_size);
		}
		break;
	case JIT_CODE_CLOSE:
		break;
	case JIT_CODE_LOAD:
		if (jd->needs_bswap) {
			jr->load.pid       = bswap_32(jr->load.pid);
			jr->load.tid       = bswap_32(jr->load.tid);
			jr->load.vma       = bswap_64(jr->load.vma);
			jr->load.code_addr = bswap_64(jr->load.code_addr);
			jr->load.code_size = bswap_64(jr->load.code_size);
			jr->load.code_index= bswap_64(jr->load.code_index);
		}
		jd->code_load_count++;
		break;
	case JIT_CODE_MOVE:
		if (jd->needs_bswap) {
			jr->move.pid           = bswap_32(jr->move.pid);
			jr->move.tid           = bswap_32(jr->move.tid);
			jr->move.vma           = bswap_64(jr->move.vma);
			jr->move.old_code_addr = bswap_64(jr->move.old_code_addr);
			jr->move.new_code_addr = bswap_64(jr->move.new_code_addr);
			jr->move.code_size     = bswap_64(jr->move.code_size);
			jr->move.code_index    = bswap_64(jr->move.code_index);
		}
		break;
	case JIT_CODE_MAX:
	default:
		/* skip unknown record (we have read them) */
		break;
	}
	return jr;
}

static int
jit_inject_event(struct jit_buf_desc *jd, union perf_event *event)
{
	ssize_t size;

	size = perf_data_file__write(jd->output, event, event->header.size);
	if (size < 0)
		return -1;

	jd->bytes_written += size;
	return 0;
}

static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp)
{
	struct perf_tsc_conversion tc;

	if (!jd->use_arch_timestamp)
		return timestamp;

	tc.time_shift = jd->session->time_conv.time_shift;
	tc.time_mult  = jd->session->time_conv.time_mult;
	tc.time_zero  = jd->session->time_conv.time_zero;

	if (!tc.time_mult)
		return 0;

	return tsc_to_perf_time(timestamp, &tc);
}

static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr)
{
	struct perf_sample sample;
	union perf_event *event;
	struct perf_tool *tool = jd->session->tool;
	uint64_t code, addr;
	uintptr_t uaddr;
	char *filename;
	struct stat st;
	size_t size;
	u16 idr_size;
	const char *sym;
	uint32_t count;
	int ret, csize, usize;
	pid_t pid, tid;
	struct {
		u32 pid, tid;
		u64 time;
	} *id;

	pid   = jr->load.pid;
	tid   = jr->load.tid;
	csize = jr->load.code_size;
	usize = jd->unwinding_mapped_size;
	addr  = jr->load.code_addr;
	sym   = (void *)((unsigned long)jr + sizeof(jr->load));
	code  = (unsigned long)jr + jr->load.p.total_size - csize;
	count = jr->load.code_index;
	idr_size = jd->machine->id_hdr_size;

	event = calloc(1, sizeof(*event) + idr_size);
	if (!event)
		return -1;

	filename = event->mmap2.filename;
	size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so",
			jd->dir,
			pid,
			count);

	size++; /* for \0 */

	size = PERF_ALIGN(size, sizeof(u64));
	uaddr = (uintptr_t)code;
	ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries);

	if (jd->debug_data && jd->nr_debug_entries) {
		free(jd->debug_data);
		jd->debug_data = NULL;
		jd->nr_debug_entries = 0;
	}

	if (jd->unwinding_data && jd->eh_frame_hdr_size) {
		free(jd->unwinding_data);
		jd->unwinding_data = NULL;
		jd->eh_frame_hdr_size = 0;
		jd->unwinding_mapped_size = 0;
		jd->unwinding_size = 0;
	}

	if (ret) {
		free(event);
		return -1;
	}
	if (stat(filename, &st))
		memset(&st, 0, sizeof(st));

	event->mmap2.header.type = PERF_RECORD_MMAP2;
	event->mmap2.header.misc = PERF_RECORD_MISC_USER;
	event->mmap2.header.size = (sizeof(event->mmap2) -
			(sizeof(event->mmap2.filename) - size) + idr_size);

	event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
	event->mmap2.start = addr;
	event->mmap2.len   = usize ? ALIGN_8(csize) + usize : csize;
	event->mmap2.pid   = pid;
	event->mmap2.tid   = tid;
	event->mmap2.ino   = st.st_ino;
	event->mmap2.maj   = major(st.st_dev);
	event->mmap2.min   = minor(st.st_dev);
	event->mmap2.prot  = st.st_mode;
	event->mmap2.flags = MAP_SHARED;
	event->mmap2.ino_generation = 1;

	id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
	if (jd->sample_type & PERF_SAMPLE_TID) {
		id->pid  = pid;
		id->tid  = tid;
	}
	if (jd->sample_type & PERF_SAMPLE_TIME)
		id->time = convert_timestamp(jd, jr->load.p.timestamp);

	/*
	 * create pseudo sample to induce dso hit increment
	 * use first address as sample address
	 */
	memset(&sample, 0, sizeof(sample));
	sample.cpumode = PERF_RECORD_MISC_USER;
	sample.pid  = pid;
	sample.tid  = tid;
	sample.time = id->time;
	sample.ip   = addr;

	ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
	if (ret)
		return ret;

	ret = jit_inject_event(jd, event);
	/*
	 * mark dso as use to generate buildid in the header
	 */
	if (!ret)
		build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);

	return ret;
}

static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr)
{
	struct perf_sample sample;
	union perf_event *event;
	struct perf_tool *tool = jd->session->tool;
	char *filename;
	size_t size;
	struct stat st;
	int usize;
	u16 idr_size;
	int ret;
	pid_t pid, tid;
	struct {
		u32 pid, tid;
		u64 time;
	} *id;

	pid = jr->move.pid;
	tid =  jr->move.tid;
	usize = jd->unwinding_mapped_size;
	idr_size = jd->machine->id_hdr_size;

	/*
	 * +16 to account for sample_id_all (hack)
	 */
	event = calloc(1, sizeof(*event) + 16);
	if (!event)
		return -1;

	filename = event->mmap2.filename;
	size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64,
	         jd->dir,
	         pid,
		 jr->move.code_index);

	size++; /* for \0 */

	if (stat(filename, &st))
		memset(&st, 0, sizeof(st));

	size = PERF_ALIGN(size, sizeof(u64));

	event->mmap2.header.type = PERF_RECORD_MMAP2;
	event->mmap2.header.misc = PERF_RECORD_MISC_USER;
	event->mmap2.header.size = (sizeof(event->mmap2) -
			(sizeof(event->mmap2.filename) - size) + idr_size);
	event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
	event->mmap2.start = jr->move.new_code_addr;
	event->mmap2.len   = usize ? ALIGN_8(jr->move.code_size) + usize
				   : jr->move.code_size;
	event->mmap2.pid   = pid;
	event->mmap2.tid   = tid;
	event->mmap2.ino   = st.st_ino;
	event->mmap2.maj   = major(st.st_dev);
	event->mmap2.min   = minor(st.st_dev);
	event->mmap2.prot  = st.st_mode;
	event->mmap2.flags = MAP_SHARED;
	event->mmap2.ino_generation = 1;

	id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
	if (jd->sample_type & PERF_SAMPLE_TID) {
		id->pid  = pid;
		id->tid  = tid;
	}
	if (jd->sample_type & PERF_SAMPLE_TIME)
		id->time = convert_timestamp(jd, jr->load.p.timestamp);

	/*
	 * create pseudo sample to induce dso hit increment
	 * use first address as sample address
	 */
	memset(&sample, 0, sizeof(sample));
	sample.cpumode = PERF_RECORD_MISC_USER;
	sample.pid  = pid;
	sample.tid  = tid;
	sample.time = id->time;
	sample.ip   = jr->move.new_code_addr;

	ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
	if (ret)
		return ret;

	ret = jit_inject_event(jd, event);
	if (!ret)
		build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);

	return ret;
}

static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr)
{
	void *data;
	size_t sz;

	if (!(jd && jr))
		return -1;

	sz  = jr->prefix.total_size - sizeof(jr->info);
	data = malloc(sz);
	if (!data)
		return -1;

	memcpy(data, &jr->info.entries, sz);

	jd->debug_data       = data;

	/*
	 * we must use nr_entry instead of size here because
	 * we cannot distinguish actual entry from padding otherwise
	 */
	jd->nr_debug_entries = jr->info.nr_entry;

	return 0;
}

static int
jit_repipe_unwinding_info(struct jit_buf_desc *jd, union jr_entry *jr)
{
	void *unwinding_data;
	uint32_t unwinding_data_size;

	if (!(jd && jr))
		return -1;

	unwinding_data_size  = jr->prefix.total_size - sizeof(jr->unwinding);
	unwinding_data = malloc(unwinding_data_size);
	if (!unwinding_data)
		return -1;

	memcpy(unwinding_data, &jr->unwinding.unwinding_data,
	       unwinding_data_size);

	jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size;
	jd->unwinding_size = jr->unwinding.unwinding_size;
	jd->unwinding_mapped_size = jr->unwinding.mapped_size;
	jd->unwinding_data = unwinding_data;

	return 0;
}

static int
jit_process_dump(struct jit_buf_desc *jd)
{
	union jr_entry *jr;
	int ret = 0;

	while ((jr = jit_get_next_entry(jd))) {
		switch(jr->prefix.id) {
		case JIT_CODE_LOAD:
			ret = jit_repipe_code_load(jd, jr);
			break;
		case JIT_CODE_MOVE:
			ret = jit_repipe_code_move(jd, jr);
			break;
		case JIT_CODE_DEBUG_INFO:
			ret = jit_repipe_debug_info(jd, jr);
			break;
		case JIT_CODE_UNWINDING_INFO:
			ret = jit_repipe_unwinding_info(jd, jr);
			break;
		default:
			ret = 0;
			continue;
		}
	}
	return ret;
}

static int
jit_inject(struct jit_buf_desc *jd, char *path)
{
	int ret;

	if (verbose > 0)
		fprintf(stderr, "injecting: %s\n", path);

	ret = jit_open(jd, path);
	if (ret)
		return -1;

	ret = jit_process_dump(jd);

	jit_close(jd);

	if (verbose > 0)
		fprintf(stderr, "injected: %s (%d)\n", path, ret);

	return 0;
}

/*
 * File must be with pattern .../jit-XXXX.dump
 * where XXXX is the PID of the process which did the mmap()
 * as captured in the RECORD_MMAP record
 */
static int
jit_detect(char *mmap_name, pid_t pid)
 {
	char *p;
	char *end = NULL;
	pid_t pid2;

	if (verbose > 2)
		fprintf(stderr, "jit marker trying : %s\n", mmap_name);
	/*
	 * get file name
	 */
	p = strrchr(mmap_name, '/');
	if (!p)
		return -1;

	/*
	 * match prefix
	 */
	if (strncmp(p, "/jit-", 5))
		return -1;

	/*
	 * skip prefix
	 */
	p += 5;

	/*
	 * must be followed by a pid
	 */
	if (!isdigit(*p))
		return -1;

	pid2 = (int)strtol(p, &end, 10);
	if (!end)
		return -1;

	/*
	 * pid does not match mmap pid
	 * pid==0 in system-wide mode (synthesized)
	 */
	if (pid && pid2 != pid)
		return -1;
	/*
	 * validate suffix
	 */
	if (strcmp(end, ".dump"))
		return -1;

	if (verbose > 0)
		fprintf(stderr, "jit marker found: %s\n", mmap_name);

	return 0;
}

int
jit_process(struct perf_session *session,
	    struct perf_data_file *output,
	    struct machine *machine,
	    char *filename,
	    pid_t pid,
	    u64 *nbytes)
{
	struct perf_evsel *first;
	struct jit_buf_desc jd;
	int ret;

	/*
	 * first, detect marker mmap (i.e., the jitdump mmap)
	 */
	if (jit_detect(filename, pid))
		return 0;

	memset(&jd, 0, sizeof(jd));

	jd.session = session;
	jd.output  = output;
	jd.machine = machine;

	/*
	 * track sample_type to compute id_all layout
	 * perf sets the same sample type to all events as of now
	 */
	first = perf_evlist__first(session->evlist);
	jd.sample_type = first->attr.sample_type;

	*nbytes = 0;

	ret = jit_inject(&jd, filename);
	if (!ret) {
		*nbytes = jd.bytes_written;
		ret = 1;
	}

	return ret;
}
Commit	Line	Data
49a7f010	1	#include <sys/sysmacros.h>
9b07e27f SE	2	#include <sys/types.h>
	3	#include <stdio.h>
	4	#include <stdlib.h>
	5	#include <string.h>
	6	#include <fcntl.h>
	7	#include <unistd.h>
	8	#include <inttypes.h>
	9	#include <byteswap.h>
	10	#include <sys/stat.h>
	11	#include <sys/mman.h>
	12
	13	#include "util.h"
	14	#include "event.h"
	15	#include "debug.h"
	16	#include "evlist.h"
	17	#include "symbol.h"
	18	#include "strlist.h"
	19	#include <elf.h>
	20
2a28e230	21	#include "tsc.h"
9b07e27f SE	22	#include "session.h"
	23	#include "jit.h"
	24	#include "jitdump.h"
	25	#include "genelf.h"
	26	#include "../builtin.h"
	27
	28	struct jit_buf_desc {
	29	struct perf_data_file *output;
	30	struct perf_session *session;
	31	struct machine *machine;
	32	union jr_entry *entry;
	33	void *buf;
	34	uint64_t sample_type;
	35	size_t bufsize;
	36	FILE *in;
	37	bool needs_bswap; /* handles cross-endianess */
2a28e230	38	bool use_arch_timestamp;
9b07e27f	39	void *debug_data;
0284fecd SS	40	void *unwinding_data;
	41	uint64_t unwinding_size;
	42	uint64_t unwinding_mapped_size;
	43	uint64_t eh_frame_hdr_size;
9b07e27f SE	44	size_t nr_debug_entries;
	45	uint32_t code_load_count;
	46	u64 bytes_written;
	47	struct rb_root code_root;
	48	char dir[PATH_MAX];
	49	};
	50
	51	struct debug_line_info {
	52	unsigned long vma;
	53	unsigned int lineno;
	54	/* The filename format is unspecified, absolute path, relative etc. */
	55	char const filename[0];
	56	};
	57
	58	struct jit_tool {
	59	struct perf_tool tool;
	60	struct perf_data_file output;
	61	struct perf_data_file input;
	62	u64 bytes_written;
	63	};
	64
	65	#define hmax(a, b) ((a) > (b) ? (a) : (b))
	66	#define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
	67
	68	static int
	69	jit_emit_elf(char *filename,
	70	const char *sym,
	71	uint64_t code_addr,
	72	const void *code,
598b7c69 SE	73	int csize,
	74	void *debug,
	75	int nr_debug_entries)
9b07e27f SE	76	{
	77	int ret, fd;
	78
	79	if (verbose > 0)
	80	fprintf(stderr, "write ELF image %s\n", filename);
	81
	82	fd = open(filename, O_CREAT\|O_TRUNC\|O_WRONLY, 0644);
	83	if (fd == -1) {
	84	pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno));
	85	return -1;
	86	}
	87
598b7c69	88	ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries);
9b07e27f SE	89
	90	close(fd);
	91
	92	if (ret)
	93	unlink(filename);
	94
	95	return ret;
	96	}
	97
	98	static void
	99	jit_close(struct jit_buf_desc *jd)
	100	{
	101	if (!(jd && jd->in))
	102	return;
	103	funlockfile(jd->in);
	104	fclose(jd->in);
	105	jd->in = NULL;
	106	}
	107
4a018cc4 AH	108	static int
	109	jit_validate_events(struct perf_session *session)
	110	{
	111	struct perf_evsel *evsel;
	112
	113	/*
	114	* check that all events use CLOCK_MONOTONIC
	115	*/
e5cadb93	116	evlist__for_each_entry(session->evlist, evsel) {
4a018cc4 AH	117	if (evsel->attr.use_clockid == 0 \|\| evsel->attr.clockid != CLOCK_MONOTONIC)
	118	return -1;
	119	}
	120	return 0;
	121	}
	122
9b07e27f SE	123	static int
	124	jit_open(struct jit_buf_desc jd, const char name)
	125	{
	126	struct jitheader header;
	127	struct jr_prefix *prefix;
	128	ssize_t bs, bsz = 0;
	129	void n, buf = NULL;
	130	int ret, retval = -1;
	131
	132	jd->in = fopen(name, "r");
	133	if (!jd->in)
	134	return -1;
	135
	136	bsz = hmax(sizeof(header), sizeof(*prefix));
	137
	138	buf = malloc(bsz);
	139	if (!buf)
	140	goto error;
	141
	142	/*
	143	* protect from writer modifying the file while we are reading it
	144	*/
	145	flockfile(jd->in);
	146
	147	ret = fread(buf, sizeof(header), 1, jd->in);
	148	if (ret != 1)
	149	goto error;
	150
	151	memcpy(&header, buf, sizeof(header));
	152
	153	if (header.magic != JITHEADER_MAGIC) {
	154	if (header.magic != JITHEADER_MAGIC_SW)
	155	goto error;
	156	jd->needs_bswap = true;
	157	}
	158
	159	if (jd->needs_bswap) {
	160	header.version = bswap_32(header.version);
	161	header.total_size = bswap_32(header.total_size);
	162	header.pid = bswap_32(header.pid);
	163	header.elf_mach = bswap_32(header.elf_mach);
	164	header.timestamp = bswap_64(header.timestamp);
	165	header.flags = bswap_64(header.flags);
	166	}
	167
2a28e230 AH	168	jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP;
2a28e230 AH	169
9b07e27f	170	if (verbose > 2)
2a28e230	171	pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
9b07e27f SE	172	header.version,
	173	header.total_size,
	174	(unsigned long long)header.timestamp,
	175	header.pid,
2a28e230 AH	176	header.elf_mach,
2a28e230 AH	177	jd->use_arch_timestamp);
9b07e27f SE	178
	179	if (header.flags & JITDUMP_FLAGS_RESERVED) {
	180	pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
	181	(unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED);
	182	goto error;
	183	}
	184
2a28e230 AH	185	if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) {
	186	pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
	187	goto error;
	188	}
	189
4a018cc4 AH	190	/*
	191	* validate event is using the correct clockid
	192	*/
2a28e230	193	if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) {
4a018cc4 AH	194	pr_err("error, jitted code must be sampled with perf record -k 1\n");
	195	goto error;
	196	}
	197
9b07e27f SE	198	bs = header.total_size - sizeof(header);
	199
	200	if (bs > bsz) {
	201	n = realloc(buf, bs);
	202	if (!n)
	203	goto error;
	204	bsz = bs;
	205	buf = n;
	206	/* read extra we do not know about */
	207	ret = fread(buf, bs - bsz, 1, jd->in);
	208	if (ret != 1)
	209	goto error;
	210	}
	211	/*
	212	* keep dirname for generating files and mmap records
	213	*/
	214	strcpy(jd->dir, name);
	215	dirname(jd->dir);
	216
	217	return 0;
	218	error:
	219	funlockfile(jd->in);
	220	fclose(jd->in);
	221	return retval;
	222	}
	223
	224	static union jr_entry *
	225	jit_get_next_entry(struct jit_buf_desc *jd)
	226	{
	227	struct jr_prefix *prefix;
	228	union jr_entry *jr;
	229	void *addr;
	230	size_t bs, size;
	231	int id, ret;
	232
	233	if (!(jd && jd->in))
	234	return NULL;
	235
	236	if (jd->buf == NULL) {
	237	size_t sz = getpagesize();
	238	if (sz < sizeof(*prefix))
	239	sz = sizeof(*prefix);
	240
	241	jd->buf = malloc(sz);
	242	if (jd->buf == NULL)
	243	return NULL;
	244
	245	jd->bufsize = sz;
	246	}
	247
	248	prefix = jd->buf;
	249
	250	/*
	251	* file is still locked at this point
	252	*/
	253	ret = fread(prefix, sizeof(*prefix), 1, jd->in);
	254	if (ret != 1)
	255	return NULL;
	256
	257	if (jd->needs_bswap) {
	258	prefix->id = bswap_32(prefix->id);
	259	prefix->total_size = bswap_32(prefix->total_size);
	260	prefix->timestamp = bswap_64(prefix->timestamp);
	261	}
262	id = prefix->id;
263	size = prefix->total_size;
264
265	bs = (size_t)size;
266	if (bs < sizeof(*prefix))
267	return NULL;
268
269	if (id >= JIT_CODE_MAX) {
7354ec7a	270	pr_warning("next_entry: unknown record type %d, skipping\n", id);
9b07e27f SE	271	}
	272	if (bs > jd->bufsize) {
	273	void *n;
	274	n = realloc(jd->buf, bs);
	275	if (!n)
	276	return NULL;
	277	jd->buf = n;
	278	jd->bufsize = bs;
	279	}
	280
	281	addr = ((void )jd->buf) + sizeof(prefix);
	282
	283	ret = fread(addr, bs - sizeof(*prefix), 1, jd->in);
	284	if (ret != 1)
	285	return NULL;
	286
	287	jr = (union jr_entry *)jd->buf;
	288
	289	switch(id) {
	290	case JIT_CODE_DEBUG_INFO:
	291	if (jd->needs_bswap) {
	292	uint64_t n;
	293	jr->info.code_addr = bswap_64(jr->info.code_addr);
	294	jr->info.nr_entry = bswap_64(jr->info.nr_entry);
	295	for (n = 0 ; n < jr->info.nr_entry; n++) {
	296	jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr);
	297	jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno);
	298	jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim);
	299	}
	300	}
	301	break;
0284fecd SS	302	case JIT_CODE_UNWINDING_INFO:
	303	if (jd->needs_bswap) {
	304	jr->unwinding.unwinding_size = bswap_64(jr->unwinding.unwinding_size);
	305	jr->unwinding.eh_frame_hdr_size = bswap_64(jr->unwinding.eh_frame_hdr_size);
	306	jr->unwinding.mapped_size = bswap_64(jr->unwinding.mapped_size);
	307	}
	308	break;
9b07e27f SE	309	case JIT_CODE_CLOSE:
	310	break;
	311	case JIT_CODE_LOAD:
	312	if (jd->needs_bswap) {
	313	jr->load.pid = bswap_32(jr->load.pid);
	314	jr->load.tid = bswap_32(jr->load.tid);
	315	jr->load.vma = bswap_64(jr->load.vma);
	316	jr->load.code_addr = bswap_64(jr->load.code_addr);
	317	jr->load.code_size = bswap_64(jr->load.code_size);
	318	jr->load.code_index= bswap_64(jr->load.code_index);
	319	}
	320	jd->code_load_count++;
	321	break;
	322	case JIT_CODE_MOVE:
	323	if (jd->needs_bswap) {
	324	jr->move.pid = bswap_32(jr->move.pid);
	325	jr->move.tid = bswap_32(jr->move.tid);
	326	jr->move.vma = bswap_64(jr->move.vma);
	327	jr->move.old_code_addr = bswap_64(jr->move.old_code_addr);
	328	jr->move.new_code_addr = bswap_64(jr->move.new_code_addr);
	329	jr->move.code_size = bswap_64(jr->move.code_size);
	330	jr->move.code_index = bswap_64(jr->move.code_index);
	331	}
	332	break;
	333	case JIT_CODE_MAX:
	334	default:
7354ec7a SS	335	/* skip unknown record (we have read them) */
7354ec7a SS	336	break;
9b07e27f SE	337	}
	338	return jr;
	339	}
	340
	341	static int
	342	jit_inject_event(struct jit_buf_desc jd, union perf_event event)
	343	{
	344	ssize_t size;
	345
	346	size = perf_data_file__write(jd->output, event, event->header.size);
	347	if (size < 0)
	348	return -1;
	349
	350	jd->bytes_written += size;
	351	return 0;
	352	}
	353
2a28e230 AH	354	static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp)
	355	{
	356	struct perf_tsc_conversion tc;
	357
	358	if (!jd->use_arch_timestamp)
	359	return timestamp;
	360
	361	tc.time_shift = jd->session->time_conv.time_shift;
	362	tc.time_mult = jd->session->time_conv.time_mult;
	363	tc.time_zero = jd->session->time_conv.time_zero;
	364
	365	if (!tc.time_mult)
	366	return 0;
	367
	368	return tsc_to_perf_time(timestamp, &tc);
	369	}
	370
9b07e27f SE	371	static int jit_repipe_code_load(struct jit_buf_desc jd, union jr_entry jr)
	372	{
	373	struct perf_sample sample;
	374	union perf_event *event;
	375	struct perf_tool *tool = jd->session->tool;
	376	uint64_t code, addr;
	377	uintptr_t uaddr;
	378	char *filename;
	379	struct stat st;
	380	size_t size;
	381	u16 idr_size;
	382	const char *sym;
	383	uint32_t count;
0284fecd	384	int ret, csize, usize;
9b07e27f SE	385	pid_t pid, tid;
	386	struct {
	387	u32 pid, tid;
	388	u64 time;
	389	} *id;
	390
	391	pid = jr->load.pid;
	392	tid = jr->load.tid;
	393	csize = jr->load.code_size;
0284fecd	394	usize = jd->unwinding_mapped_size;
9b07e27f SE	395	addr = jr->load.code_addr;
	396	sym = (void *)((unsigned long)jr + sizeof(jr->load));
	397	code = (unsigned long)jr + jr->load.p.total_size - csize;
	398	count = jr->load.code_index;
	399	idr_size = jd->machine->id_hdr_size;
	400
	401	event = calloc(1, sizeof(*event) + idr_size);
	402	if (!event)
	403	return -1;
	404
	405	filename = event->mmap2.filename;
	406	size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so",
	407	jd->dir,
	408	pid,
	409	count);
	410
	411	size++; /* for \0 */
	412
	413	size = PERF_ALIGN(size, sizeof(u64));
	414	uaddr = (uintptr_t)code;
598b7c69	415	ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries);
9b07e27f SE	416
	417	if (jd->debug_data && jd->nr_debug_entries) {
	418	free(jd->debug_data);
	419	jd->debug_data = NULL;
	420	jd->nr_debug_entries = 0;
	421	}
	422
0284fecd SS	423	if (jd->unwinding_data && jd->eh_frame_hdr_size) {
	424	free(jd->unwinding_data);
	425	jd->unwinding_data = NULL;
	426	jd->eh_frame_hdr_size = 0;
	427	jd->unwinding_mapped_size = 0;
	428	jd->unwinding_size = 0;
	429	}
	430
9b07e27f SE	431	if (ret) {
	432	free(event);
	433	return -1;
	434	}
	435	if (stat(filename, &st))
f56ebf20	436	memset(&st, 0, sizeof(st));
9b07e27f SE	437
	438	event->mmap2.header.type = PERF_RECORD_MMAP2;
	439	event->mmap2.header.misc = PERF_RECORD_MISC_USER;
	440	event->mmap2.header.size = (sizeof(event->mmap2) -
	441	(sizeof(event->mmap2.filename) - size) + idr_size);
	442
	443	event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
	444	event->mmap2.start = addr;
0284fecd	445	event->mmap2.len = usize ? ALIGN_8(csize) + usize : csize;
9b07e27f SE	446	event->mmap2.pid = pid;
	447	event->mmap2.tid = tid;
	448	event->mmap2.ino = st.st_ino;
	449	event->mmap2.maj = major(st.st_dev);
	450	event->mmap2.min = minor(st.st_dev);
	451	event->mmap2.prot = st.st_mode;
	452	event->mmap2.flags = MAP_SHARED;
	453	event->mmap2.ino_generation = 1;
	454
	455	id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
	456	if (jd->sample_type & PERF_SAMPLE_TID) {
	457	id->pid = pid;
	458	id->tid = tid;
	459	}
	460	if (jd->sample_type & PERF_SAMPLE_TIME)
2a28e230	461	id->time = convert_timestamp(jd, jr->load.p.timestamp);
9b07e27f SE	462
	463	/*
	464	* create pseudo sample to induce dso hit increment
	465	* use first address as sample address
	466	*/
	467	memset(&sample, 0, sizeof(sample));
3ea223ad	468	sample.cpumode = PERF_RECORD_MISC_USER;
9b07e27f SE	469	sample.pid = pid;
	470	sample.tid = tid;
	471	sample.time = id->time;
	472	sample.ip = addr;
	473
	474	ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
	475	if (ret)
	476	return ret;
	477
	478	ret = jit_inject_event(jd, event);
	479	/*
	480	* mark dso as use to generate buildid in the header
	481	*/
	482	if (!ret)
	483	build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
	484
	485	return ret;
	486	}
	487
	488	static int jit_repipe_code_move(struct jit_buf_desc jd, union jr_entry jr)
	489	{
	490	struct perf_sample sample;
	491	union perf_event *event;
	492	struct perf_tool *tool = jd->session->tool;
	493	char *filename;
	494	size_t size;
	495	struct stat st;
0284fecd	496	int usize;
9b07e27f SE	497	u16 idr_size;
	498	int ret;
	499	pid_t pid, tid;
	500	struct {
	501	u32 pid, tid;
	502	u64 time;
	503	} *id;
	504
	505	pid = jr->move.pid;
	506	tid = jr->move.tid;
0284fecd	507	usize = jd->unwinding_mapped_size;
9b07e27f SE	508	idr_size = jd->machine->id_hdr_size;
	509
	510	/*
	511	* +16 to account for sample_id_all (hack)
	512	*/
	513	event = calloc(1, sizeof(*event) + 16);
	514	if (!event)
	515	return -1;
	516
	517	filename = event->mmap2.filename;
	518	size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64,
	519	jd->dir,
	520	pid,
	521	jr->move.code_index);
	522
	523	size++; /* for \0 */
	524
	525	if (stat(filename, &st))
f56ebf20	526	memset(&st, 0, sizeof(st));
9b07e27f SE	527
	528	size = PERF_ALIGN(size, sizeof(u64));
	529
	530	event->mmap2.header.type = PERF_RECORD_MMAP2;
	531	event->mmap2.header.misc = PERF_RECORD_MISC_USER;
	532	event->mmap2.header.size = (sizeof(event->mmap2) -
	533	(sizeof(event->mmap2.filename) - size) + idr_size);
	534	event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
	535	event->mmap2.start = jr->move.new_code_addr;
0284fecd SS	536	event->mmap2.len = usize ? ALIGN_8(jr->move.code_size) + usize
0284fecd SS	537	: jr->move.code_size;
9b07e27f SE	538	event->mmap2.pid = pid;
	539	event->mmap2.tid = tid;
	540	event->mmap2.ino = st.st_ino;
	541	event->mmap2.maj = major(st.st_dev);
	542	event->mmap2.min = minor(st.st_dev);
	543	event->mmap2.prot = st.st_mode;
	544	event->mmap2.flags = MAP_SHARED;
	545	event->mmap2.ino_generation = 1;
	546
	547	id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
	548	if (jd->sample_type & PERF_SAMPLE_TID) {
	549	id->pid = pid;
	550	id->tid = tid;
	551	}
	552	if (jd->sample_type & PERF_SAMPLE_TIME)
2a28e230	553	id->time = convert_timestamp(jd, jr->load.p.timestamp);
9b07e27f SE	554
	555	/*
	556	* create pseudo sample to induce dso hit increment
	557	* use first address as sample address
	558	*/
	559	memset(&sample, 0, sizeof(sample));
3ea223ad	560	sample.cpumode = PERF_RECORD_MISC_USER;
9b07e27f SE	561	sample.pid = pid;
	562	sample.tid = tid;
	563	sample.time = id->time;
	564	sample.ip = jr->move.new_code_addr;
	565
	566	ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
	567	if (ret)
	568	return ret;
	569
	570	ret = jit_inject_event(jd, event);
	571	if (!ret)
	572	build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
	573
	574	return ret;
	575	}
	576
	577	static int jit_repipe_debug_info(struct jit_buf_desc jd, union jr_entry jr)
	578	{
	579	void *data;
	580	size_t sz;
	581
	582	if (!(jd && jr))
	583	return -1;
	584
	585	sz = jr->prefix.total_size - sizeof(jr->info);
	586	data = malloc(sz);
	587	if (!data)
	588	return -1;
	589
	590	memcpy(data, &jr->info.entries, sz);
	591
	592	jd->debug_data = data;
	593
	594	/*
	595	* we must use nr_entry instead of size here because
	596	* we cannot distinguish actual entry from padding otherwise
	597	*/
	598	jd->nr_debug_entries = jr->info.nr_entry;
	599
	600	return 0;
	601	}
	602
0284fecd SS	603	static int
	604	jit_repipe_unwinding_info(struct jit_buf_desc jd, union jr_entry jr)
	605	{
	606	void *unwinding_data;
	607	uint32_t unwinding_data_size;
	608
	609	if (!(jd && jr))
	610	return -1;
	611
	612	unwinding_data_size = jr->prefix.total_size - sizeof(jr->unwinding);
	613	unwinding_data = malloc(unwinding_data_size);
	614	if (!unwinding_data)
	615	return -1;
	616
	617	memcpy(unwinding_data, &jr->unwinding.unwinding_data,
	618	unwinding_data_size);
	619
	620	jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size;
	621	jd->unwinding_size = jr->unwinding.unwinding_size;
	622	jd->unwinding_mapped_size = jr->unwinding.mapped_size;
	623	jd->unwinding_data = unwinding_data;
	624
	625	return 0;
	626	}
	627
9b07e27f SE	628	static int
	629	jit_process_dump(struct jit_buf_desc *jd)
	630	{
	631	union jr_entry *jr;
ef2c3e76	632	int ret = 0;
9b07e27f SE	633
	634	while ((jr = jit_get_next_entry(jd))) {
	635	switch(jr->prefix.id) {
	636	case JIT_CODE_LOAD:
	637	ret = jit_repipe_code_load(jd, jr);
	638	break;
	639	case JIT_CODE_MOVE:
	640	ret = jit_repipe_code_move(jd, jr);
	641	break;
	642	case JIT_CODE_DEBUG_INFO:
	643	ret = jit_repipe_debug_info(jd, jr);
	644	break;
0284fecd SS	645	case JIT_CODE_UNWINDING_INFO:
	646	ret = jit_repipe_unwinding_info(jd, jr);
	647	break;
9b07e27f SE	648	default:
	649	ret = 0;
	650	continue;
	651	}
	652	}
	653	return ret;
	654	}
	655
	656	static int
	657	jit_inject(struct jit_buf_desc jd, char path)
	658	{
	659	int ret;
	660
	661	if (verbose > 0)
	662	fprintf(stderr, "injecting: %s\n", path);
	663
	664	ret = jit_open(jd, path);
	665	if (ret)
	666	return -1;
	667
	668	ret = jit_process_dump(jd);
	669
	670	jit_close(jd);
	671
	672	if (verbose > 0)
	673	fprintf(stderr, "injected: %s (%d)\n", path, ret);
	674
	675	return 0;
	676	}
	677
	678	/*
	679	* File must be with pattern .../jit-XXXX.dump
	680	* where XXXX is the PID of the process which did the mmap()
	681	* as captured in the RECORD_MMAP record
	682	*/
	683	static int
	684	jit_detect(char *mmap_name, pid_t pid)
	685	{
	686	char *p;
	687	char *end = NULL;
	688	pid_t pid2;
	689
	690	if (verbose > 2)
	691	fprintf(stderr, "jit marker trying : %s\n", mmap_name);
	692	/*
	693	* get file name
	694	*/
	695	p = strrchr(mmap_name, '/');
	696	if (!p)
	697	return -1;
	698
	699	/*
	700	* match prefix
	701	*/
	702	if (strncmp(p, "/jit-", 5))
	703	return -1;
	704
	705	/*
	706	* skip prefix
	707	*/
	708	p += 5;
	709
	710	/*
	711	* must be followed by a pid
712	*/
713	if (!isdigit(*p))
714	return -1;
715
716	pid2 = (int)strtol(p, &end, 10);
717	if (!end)
718	return -1;
719
720	/*
721	* pid does not match mmap pid
722	* pid==0 in system-wide mode (synthesized)
723	*/
724	if (pid && pid2 != pid)
725	return -1;
726	/*
727	* validate suffix
728	*/
729	if (strcmp(end, ".dump"))
730	return -1;
731
732	if (verbose > 0)
733	fprintf(stderr, "jit marker found: %s\n", mmap_name);
734
735	return 0;
736	}
737
738	int
739	jit_process(struct perf_session *session,
740	struct perf_data_file *output,
741	struct machine *machine,
742	char *filename,
743	pid_t pid,
744	u64 *nbytes)
745	{
746	struct perf_evsel *first;
747	struct jit_buf_desc jd;
748	int ret;
749
750	/*
751	* first, detect marker mmap (i.e., the jitdump mmap)
752	*/
753	if (jit_detect(filename, pid))
570735b3	754	return 0;
9b07e27f SE	755
	756	memset(&jd, 0, sizeof(jd));
	757
	758	jd.session = session;
	759	jd.output = output;
	760	jd.machine = machine;
	761
	762	/*
	763	* track sample_type to compute id_all layout
	764	* perf sets the same sample type to all events as of now
	765	*/
	766	first = perf_evlist__first(session->evlist);
	767	jd.sample_type = first->attr.sample_type;
	768
	769	*nbytes = 0;
	770
	771	ret = jit_inject(&jd, filename);
570735b3	772	if (!ret) {
9b07e27f	773	*nbytes = jd.bytes_written;
570735b3 AH	774	ret = 1;
570735b3 AH	775	}
9b07e27f SE	776
	777	return ret;
	778	}