[mirror_ubuntu-bionic-kernel.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type);
static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type);

static int syscall_enter_define_fields(struct ftrace_event_call *call);
static int syscall_exit_define_fields(struct ftrace_event_call *call);

static struct list_head *
syscall_get_enter_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->enter_fields;
}

static struct list_head *
syscall_get_exit_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->exit_fields;
}

struct ftrace_event_class event_class_syscall_enter = {
	.system			= "syscalls",
	.reg			= syscall_enter_register,
	.define_fields		= syscall_enter_define_fields,
	.get_fields		= syscall_get_enter_fields,
};

struct ftrace_event_class event_class_syscall_exit = {
	.system			= "syscalls",
	.reg			= syscall_exit_register,
	.define_fields		= syscall_exit_define_fields,
	.get_fields		= syscall_get_exit_fields,
};

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

static
int  __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
{
	int i;
	int pos = 0;

	/* When len=0, we just calculate the needed length */
#define LEN_OR_ZERO (len ? len - pos : 0)

	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
				entry->args[i], sizeof(unsigned long),
				i == entry->nb_args - 1 ? "" : ", ");
	}
	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");

	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO,
				", ((unsigned long)(REC->%s))", entry->args[i]);
	}

#undef LEN_OR_ZERO

	/* return the length of print_fmt */
	return pos;
}

static int set_syscall_print_fmt(struct ftrace_event_call *call)
{
	char *print_fmt;
	int len;
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event != call) {
		call->print_fmt = "\"0x%lx\", REC->ret";
		return 0;
	}

	/* First: called with 0 length to calculate the needed length */
	len = __set_enter_print_fmt(entry, NULL, 0);

	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	if (!print_fmt)
		return -ENOMEM;

	/* Second: actually write the @print_fmt */
	__set_enter_print_fmt(entry, print_fmt, len + 1);
	call->print_fmt = print_fmt;

	return 0;
}

static void free_syscall_print_fmt(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event == call)
		kfree(call->print_fmt);
}

static int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

static int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
	if (!ret) {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
	if (!ret) {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	if (set_syscall_print_fmt(call) < 0)
		return -ENOMEM;

	id = trace_event_raw_init(call);

	if (id < 0) {
		free_syscall_print_fmt(call);
		return id;
	}

	return id;
}

unsigned long __init arch_syscall_addr(int nr)
{
	return (unsigned long)sys_call_table[nr];
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_PERF_EVENTS

static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
static int sys_perf_refcount_enter;
static int sys_perf_refcount_exit;

static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		      "perf buffer not large enough"))
		return;

	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
				sys_data->enter_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_enter)
		ret = register_trace_sys_enter(perf_syscall_enter, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_perf_enter_syscalls);
		sys_perf_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_enter--;
	clear_bit(num, enabled_perf_enter_syscalls);
	if (!sys_perf_refcount_enter)
		unregister_trace_sys_enter(perf_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		"exit event has grown above perf buffer size"))
		return;

	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
				sys_data->exit_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_exit)
		ret = register_trace_sys_exit(perf_syscall_exit, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_perf_exit_syscalls);
		sys_perf_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_exit--;
	clear_bit(num, enabled_perf_exit_syscalls);
	if (!sys_perf_refcount_exit)
		unregister_trace_sys_exit(perf_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

#endif /* CONFIG_PERF_EVENTS */

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_enter(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_enter(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysenter_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysenter_disable(event);
		return 0;
#endif
	}
	return 0;
}

static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_exit(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_exit(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysexit_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysexit_disable(event);
		return 0;
#endif
	}
	return 0;
}
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
5a0e3ad6	3	#include <linux/slab.h>
ee08c6ec	4	#include <linux/kernel.h>
fb34a08c	5	#include <linux/ftrace.h>
cdd6c482	6	#include <linux/perf_event.h>
ee08c6ec FW	7	#include <asm/syscall.h>
	8
	9	#include "trace_output.h"
	10	#include "trace.h"
	11
5be71b61	12	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	13	static int sys_refcount_enter;
fb34a08c JB	14	static int sys_refcount_exit;
57421dbb JB	15	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	16	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	17
2239291a SR	18	static int syscall_enter_register(struct ftrace_event_call *event,
	19	enum trace_reg type);
	20	static int syscall_exit_register(struct ftrace_event_call *event,
	21	enum trace_reg type);
	22
2e33af02 SR	23	static int syscall_enter_define_fields(struct ftrace_event_call *call);
	24	static int syscall_exit_define_fields(struct ftrace_event_call *call);
	25
	26	static struct list_head *
	27	syscall_get_enter_fields(struct ftrace_event_call *call)
	28	{
	29	struct syscall_metadata *entry = call->data;
	30
	31	return &entry->enter_fields;
	32	}
	33
	34	static struct list_head *
	35	syscall_get_exit_fields(struct ftrace_event_call *call)
	36	{
	37	struct syscall_metadata *entry = call->data;
	38
	39	return &entry->exit_fields;
	40	}
	41
2239291a SR	42	struct ftrace_event_class event_class_syscall_enter = {
2239291a SR	43	.system = "syscalls",
2e33af02 SR	44	.reg = syscall_enter_register,
	45	.define_fields = syscall_enter_define_fields,
	46	.get_fields = syscall_get_enter_fields,
2239291a SR	47	};
	48
	49	struct ftrace_event_class event_class_syscall_exit = {
	50	.system = "syscalls",
2e33af02 SR	51	.reg = syscall_exit_register,
	52	.define_fields = syscall_exit_define_fields,
	53	.get_fields = syscall_get_exit_fields,
8f082018 SR	54	};
8f082018 SR	55
c44fc770 FW	56	extern unsigned long __start_syscalls_metadata[];
	57	extern unsigned long __stop_syscalls_metadata[];
	58
	59	static struct syscall_metadata **syscalls_metadata;
	60
	61	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	62	{
	63	struct syscall_metadata *start;
	64	struct syscall_metadata *stop;
	65	char str[KSYM_SYMBOL_LEN];
	66
	67
	68	start = (struct syscall_metadata *)__start_syscalls_metadata;
	69	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	70	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	71
	72	for ( ; start < stop; start++) {
	73	/*
	74	* Only compare after the "sys" prefix. Archs that use
	75	* syscall wrappers may have syscalls symbols aliases prefixed
	76	* with "SyS" instead of "sys", leading to an unwanted
	77	* mismatch.
	78	*/
	79	if (start->name && !strcmp(start->name + 3, str + 3))
	80	return start;
	81	}
	82	return NULL;
	83	}
	84
	85	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	86	{
	87	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	88	return NULL;
	89
	90	return syscalls_metadata[nr];
	91	}
	92
bed1ffca FW	93	enum print_line_t
	94	print_syscall_enter(struct trace_iterator *iter, int flags)
	95	{
	96	struct trace_seq *s = &iter->seq;
	97	struct trace_entry *ent = iter->ent;
	98	struct syscall_trace_enter *trace;
	99	struct syscall_metadata *entry;
	100	int i, ret, syscall;
	101
64c12e04	102	trace = (typeof(trace))ent;
bed1ffca	103	syscall = trace->nr;
bed1ffca	104	entry = syscall_nr_to_meta(syscall);
64c12e04	105
bed1ffca FW	106	if (!entry)
	107	goto end;
	108
fcc19438	109	if (entry->enter_event->id != ent->type) {
64c12e04 JB	110	WARN_ON_ONCE(1);
	111	goto end;
	112	}
	113
bed1ffca FW	114	ret = trace_seq_printf(s, "%s(", entry->name);
	115	if (!ret)
	116	return TRACE_TYPE_PARTIAL_LINE;
	117
	118	for (i = 0; i < entry->nb_args; i++) {
	119	/* parameter types */
ba8b3a40	120	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	121	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	122	if (!ret)
	123	return TRACE_TYPE_PARTIAL_LINE;
	124	}
	125	/* parameter values */
4539f077	126	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	127	trace->args[i],
4539f077	128	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	129	if (!ret)
	130	return TRACE_TYPE_PARTIAL_LINE;
	131	}
	132
4539f077 LZ	133	ret = trace_seq_putc(s, ')');
	134	if (!ret)
	135	return TRACE_TYPE_PARTIAL_LINE;
	136
bed1ffca	137	end:
4539f077 LZ	138	ret = trace_seq_putc(s, '\n');
	139	if (!ret)
	140	return TRACE_TYPE_PARTIAL_LINE;
	141
bed1ffca FW	142	return TRACE_TYPE_HANDLED;
	143	}
	144
	145	enum print_line_t
	146	print_syscall_exit(struct trace_iterator *iter, int flags)
	147	{
	148	struct trace_seq *s = &iter->seq;
	149	struct trace_entry *ent = iter->ent;
	150	struct syscall_trace_exit *trace;
	151	int syscall;
	152	struct syscall_metadata *entry;
	153	int ret;
	154
64c12e04	155	trace = (typeof(trace))ent;
bed1ffca	156	syscall = trace->nr;
bed1ffca	157	entry = syscall_nr_to_meta(syscall);
64c12e04	158
bed1ffca FW	159	if (!entry) {
	160	trace_seq_printf(s, "\n");
	161	return TRACE_TYPE_HANDLED;
	162	}
	163
fcc19438	164	if (entry->exit_event->id != ent->type) {
64c12e04 JB	165	WARN_ON_ONCE(1);
	166	return TRACE_TYPE_UNHANDLED;
	167	}
	168
bed1ffca FW	169	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	170	trace->ret);
	171	if (!ret)
	172	return TRACE_TYPE_PARTIAL_LINE;
	173
	174	return TRACE_TYPE_HANDLED;
	175	}
	176
e6971969 LZ	177	extern char *__bad_type_size(void);
	178
	179	#define SYSCALL_FIELD(type, name) \
	180	sizeof(type) != sizeof(trace.name) ? \
	181	__bad_type_size() : \
26a50744 TZ	182	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	183	sizeof(trace.name), is_signed_type(type)
e6971969	184
50307a45 LJ	185	static
	186	int __set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)
	187	{
	188	int i;
	189	int pos = 0;
	190
	191	/* When len=0, we just calculate the needed length */
	192	#define LEN_OR_ZERO (len ? len - pos : 0)
	193
	194	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	195	for (i = 0; i < entry->nb_args; i++) {
	196	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
	197	entry->args[i], sizeof(unsigned long),
	198	i == entry->nb_args - 1 ? "" : ", ");
	199	}
	200	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	201
	202	for (i = 0; i < entry->nb_args; i++) {
	203	pos += snprintf(buf + pos, LEN_OR_ZERO,
	204	", ((unsigned long)(REC->%s))", entry->args[i]);
	205	}
	206
	207	#undef LEN_OR_ZERO
	208
	209	/* return the length of print_fmt */
	210	return pos;
	211	}
	212
	213	static int set_syscall_print_fmt(struct ftrace_event_call *call)
	214	{
	215	char *print_fmt;
	216	int len;
	217	struct syscall_metadata *entry = call->data;
	218
	219	if (entry->enter_event != call) {
	220	call->print_fmt = "\"0x%lx\", REC->ret";
	221	return 0;
	222	}
	223
	224	/* First: called with 0 length to calculate the needed length */
	225	len = __set_enter_print_fmt(entry, NULL, 0);
	226
	227	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	228	if (!print_fmt)
	229	return -ENOMEM;
	230
	231	/* Second: actually write the @print_fmt */
	232	__set_enter_print_fmt(entry, print_fmt, len + 1);
	233	call->print_fmt = print_fmt;
	234
	235	return 0;
	236	}
	237
	238	static void free_syscall_print_fmt(struct ftrace_event_call *call)
	239	{
	240	struct syscall_metadata *entry = call->data;
	241
	242	if (entry->enter_event == call)
	243	kfree(call->print_fmt);
	244	}
	245
2e33af02	246	static int syscall_enter_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	247	{
540b7b8d LZ	248	struct syscall_trace_enter trace;
31c16b13	249	struct syscall_metadata *meta = call->data;
540b7b8d	250	int ret;
540b7b8d LZ	251	int i;
	252	int offset = offsetof(typeof(trace), args);
	253
0f1ef51d LJ	254	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	255	if (ret)
	256	return ret;
	257
540b7b8d	258	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	259	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	260	meta->args[i], offset,
43b51ead LZ	261	sizeof(unsigned long), 0,
43b51ead LZ	262	FILTER_OTHER);
540b7b8d LZ	263	offset += sizeof(unsigned long);
	264	}
	265
	266	return ret;
	267	}
	268
2e33af02	269	static int syscall_exit_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	270	{
	271	struct syscall_trace_exit trace;
	272	int ret;
	273
0f1ef51d LJ	274	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	275	if (ret)
	276	return ret;
	277
26a50744	278	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	279	FILTER_OTHER);
540b7b8d LZ	280
	281	return ret;
	282	}
	283
38516ab5	284	void ftrace_syscall_enter(void ignore, struct pt_regs regs, long id)
ee08c6ec	285	{
bed1ffca FW	286	struct syscall_trace_enter *entry;
	287	struct syscall_metadata *sys_data;
	288	struct ring_buffer_event *event;
e77405ad	289	struct ring_buffer *buffer;
bed1ffca	290	int size;
ee08c6ec FW	291	int syscall_nr;
	292
	293	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	294	if (syscall_nr < 0)
cd0980fc HB	295	return;
fb34a08c JB	296	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	297	return;
ee08c6ec	298
bed1ffca FW	299	sys_data = syscall_nr_to_meta(syscall_nr);
	300	if (!sys_data)
	301	return;
	302
	303	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	304
fcc19438 LJ	305	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	306	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	307	if (!event)
	308	return;
	309
	310	entry = ring_buffer_event_data(event);
	311	entry->nr = syscall_nr;
	312	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	313
e77405ad SR	314	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	315	entry, event))
	316	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	317	}
ee08c6ec FW	318
38516ab5	319	void ftrace_syscall_exit(void ignore, struct pt_regs regs, long ret)
ee08c6ec	320	{
bed1ffca FW	321	struct syscall_trace_exit *entry;
	322	struct syscall_metadata *sys_data;
	323	struct ring_buffer_event *event;
e77405ad	324	struct ring_buffer *buffer;
ee08c6ec FW	325	int syscall_nr;
	326
	327	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	328	if (syscall_nr < 0)
cd0980fc HB	329	return;
fb34a08c JB	330	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	331	return;
ee08c6ec	332
bed1ffca FW	333	sys_data = syscall_nr_to_meta(syscall_nr);
	334	if (!sys_data)
	335	return;
	336
fcc19438 LJ	337	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	338	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	339	if (!event)
	340	return;
	341
	342	entry = ring_buffer_event_data(event);
	343	entry->nr = syscall_nr;
	344	entry->ret = syscall_get_return_value(current, regs);
	345
e77405ad SR	346	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	347	entry, event))
	348	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	349	}
ee08c6ec FW	350
bd1a5c84	351	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	352	{
fb34a08c JB	353	int ret = 0;
fb34a08c JB	354	int num;
fb34a08c	355
c252f657	356	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	357	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	358	return -ENOSYS;
	359	mutex_lock(&syscall_trace_lock);
	360	if (!sys_refcount_enter)
38516ab5	361	ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
3b8e4273	362	if (!ret) {
fb34a08c JB	363	set_bit(num, enabled_enter_syscalls);
	364	sys_refcount_enter++;
	365	}
	366	mutex_unlock(&syscall_trace_lock);
	367	return ret;
ee08c6ec FW	368	}
ee08c6ec FW	369
bd1a5c84	370	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	371	{
fb34a08c	372	int num;
ee08c6ec	373
c252f657	374	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	375	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	376	return;
	377	mutex_lock(&syscall_trace_lock);
	378	sys_refcount_enter--;
	379	clear_bit(num, enabled_enter_syscalls);
	380	if (!sys_refcount_enter)
38516ab5	381	unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
fb34a08c JB	382	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	383	}
ee08c6ec	384
bd1a5c84	385	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	386	{
fb34a08c JB	387	int ret = 0;
fb34a08c JB	388	int num;
fb34a08c	389
c252f657	390	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	391	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	392	return -ENOSYS;
	393	mutex_lock(&syscall_trace_lock);
	394	if (!sys_refcount_exit)
38516ab5	395	ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
3b8e4273	396	if (!ret) {
fb34a08c JB	397	set_bit(num, enabled_exit_syscalls);
fb34a08c JB	398	sys_refcount_exit++;
ee08c6ec	399	}
fb34a08c JB	400	mutex_unlock(&syscall_trace_lock);
	401	return ret;
	402	}
ee08c6ec	403
bd1a5c84	404	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	405	{
fb34a08c JB	406	int num;
ee08c6ec	407
c252f657	408	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	409	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	410	return;
	411	mutex_lock(&syscall_trace_lock);
	412	sys_refcount_exit--;
	413	clear_bit(num, enabled_exit_syscalls);
	414	if (!sys_refcount_exit)
38516ab5	415	unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
fb34a08c	416	mutex_unlock(&syscall_trace_lock);
ee08c6ec	417	}
fb34a08c	418
a1301da0 LJ	419	int init_syscall_trace(struct ftrace_event_call *call)
	420	{
	421	int id;
	422
50307a45 LJ	423	if (set_syscall_print_fmt(call) < 0)
	424	return -ENOMEM;
	425
c7ef3a90 SR	426	id = trace_event_raw_init(call);
	427
	428	if (id < 0) {
50307a45	429	free_syscall_print_fmt(call);
c7ef3a90	430	return id;
50307a45	431	}
c7ef3a90 SR	432
c7ef3a90 SR	433	return id;
a1301da0 LJ	434	}
a1301da0 LJ	435
e7b8e675 MF	436	unsigned long __init arch_syscall_addr(int nr)
	437	{
	438	return (unsigned long)sys_call_table[nr];
	439	}
	440
c44fc770 FW	441	int __init init_ftrace_syscalls(void)
	442	{
	443	struct syscall_metadata *meta;
	444	unsigned long addr;
	445	int i;
	446
	447	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	448	NR_syscalls, GFP_KERNEL);
	449	if (!syscalls_metadata) {
	450	WARN_ON(1);
	451	return -ENOMEM;
	452	}
	453
	454	for (i = 0; i < NR_syscalls; i++) {
	455	addr = arch_syscall_addr(i);
	456	meta = find_syscall_meta(addr);
c252f657 LJ	457	if (!meta)
	458	continue;
	459
	460	meta->syscall_nr = i;
c44fc770 FW	461	syscalls_metadata[i] = meta;
	462	}
	463
	464	return 0;
	465	}
	466	core_initcall(init_ftrace_syscalls);
	467
07b139c8	468	#ifdef CONFIG_PERF_EVENTS
19007a67	469
97d5a220 FW	470	static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
	471	static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
	472	static int sys_perf_refcount_enter;
	473	static int sys_perf_refcount_exit;
f4b5ffcc	474
38516ab5	475	static void perf_syscall_enter(void ignore, struct pt_regs regs, long id)
f4b5ffcc JB	476	{
f4b5ffcc JB	477	struct syscall_metadata *sys_data;
20ab4425 FW	478	struct syscall_trace_enter *rec;
20ab4425 FW	479	unsigned long flags;
f4b5ffcc	480	int syscall_nr;
4ed7c92d	481	int rctx;
19007a67	482	int size;
f4b5ffcc JB	483
f4b5ffcc JB	484	syscall_nr = syscall_get_nr(current, regs);
97d5a220	485	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
f4b5ffcc JB	486	return;
	487
	488	sys_data = syscall_nr_to_meta(syscall_nr);
	489	if (!sys_data)
	490	return;
	491
19007a67 FW	492	/* get the size after alignment with the u32 buffer size field */
	493	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	494	size = ALIGN(size + sizeof(u32), sizeof(u64));
	495	size -= sizeof(u32);
	496
97d5a220 FW	497	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	498	"perf buffer not large enough"))
20ab4425 FW	499	return;
20ab4425 FW	500
97d5a220	501	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
430ad5a6 XG	502	sys_data->enter_event->id, &rctx, &flags);
	503	if (!rec)
	504	return;
20ab4425	505
20ab4425 FW	506	rec->nr = syscall_nr;
	507	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	508	(unsigned long *)&rec->args);
97d5a220	509	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	510	}
f4b5ffcc JB	511
97d5a220	512	int perf_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	513	{
	514	int ret = 0;
	515	int num;
	516
3bbe84e9	517	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	518
f4b5ffcc JB	519	mutex_lock(&syscall_trace_lock);
97d5a220	520	if (!sys_perf_refcount_enter)
38516ab5	521	ret = register_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	522	if (ret) {
	523	pr_info("event trace: Could not activate"
	524	"syscall entry trace point");
	525	} else {
97d5a220 FW	526	set_bit(num, enabled_perf_enter_syscalls);
97d5a220 FW	527	sys_perf_refcount_enter++;
f4b5ffcc JB	528	}
	529	mutex_unlock(&syscall_trace_lock);
	530	return ret;
	531	}
	532
97d5a220	533	void perf_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	534	{
	535	int num;
	536
3bbe84e9	537	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	538
f4b5ffcc JB	539	mutex_lock(&syscall_trace_lock);
97d5a220 FW	540	sys_perf_refcount_enter--;
	541	clear_bit(num, enabled_perf_enter_syscalls);
	542	if (!sys_perf_refcount_enter)
38516ab5	543	unregister_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	544	mutex_unlock(&syscall_trace_lock);
	545	}
	546
38516ab5	547	static void perf_syscall_exit(void ignore, struct pt_regs regs, long ret)
f4b5ffcc JB	548	{
f4b5ffcc JB	549	struct syscall_metadata *sys_data;
20ab4425 FW	550	struct syscall_trace_exit *rec;
20ab4425 FW	551	unsigned long flags;
f4b5ffcc	552	int syscall_nr;
4ed7c92d	553	int rctx;
20ab4425	554	int size;
f4b5ffcc JB	555
f4b5ffcc JB	556	syscall_nr = syscall_get_nr(current, regs);
97d5a220	557	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
f4b5ffcc JB	558	return;
	559
	560	sys_data = syscall_nr_to_meta(syscall_nr);
	561	if (!sys_data)
	562	return;
	563
20ab4425 FW	564	/* We can probably do that at build time */
	565	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	566	size -= sizeof(u32);
19007a67	567
20ab4425 FW	568	/*
	569	* Impossible, but be paranoid with the future
	570	* How to put this check outside runtime?
	571	*/
97d5a220 FW	572	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	573	"exit event has grown above perf buffer size"))
20ab4425 FW	574	return;
20ab4425 FW	575
97d5a220	576	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
430ad5a6 XG	577	sys_data->exit_event->id, &rctx, &flags);
	578	if (!rec)
	579	return;
20ab4425	580
20ab4425 FW	581	rec->nr = syscall_nr;
	582	rec->ret = syscall_get_return_value(current, regs);
	583
97d5a220	584	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	585	}
f4b5ffcc JB	586
97d5a220	587	int perf_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	588	{
	589	int ret = 0;
	590	int num;
	591
3bbe84e9	592	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	593
f4b5ffcc JB	594	mutex_lock(&syscall_trace_lock);
97d5a220	595	if (!sys_perf_refcount_exit)
38516ab5	596	ret = register_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	597	if (ret) {
f4b5ffcc JB	598	pr_info("event trace: Could not activate"
6574658b	599	"syscall exit trace point");
f4b5ffcc	600	} else {
97d5a220 FW	601	set_bit(num, enabled_perf_exit_syscalls);
97d5a220 FW	602	sys_perf_refcount_exit++;
f4b5ffcc JB	603	}
	604	mutex_unlock(&syscall_trace_lock);
	605	return ret;
	606	}
	607
97d5a220	608	void perf_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	609	{
	610	int num;
	611
3bbe84e9	612	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	613
f4b5ffcc JB	614	mutex_lock(&syscall_trace_lock);
97d5a220 FW	615	sys_perf_refcount_exit--;
	616	clear_bit(num, enabled_perf_exit_syscalls);
	617	if (!sys_perf_refcount_exit)
38516ab5	618	unregister_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	619	mutex_unlock(&syscall_trace_lock);
	620	}
	621
07b139c8	622	#endif /* CONFIG_PERF_EVENTS */
f4b5ffcc	623
2239291a SR	624	static int syscall_enter_register(struct ftrace_event_call *event,
	625	enum trace_reg type)
	626	{
	627	switch (type) {
	628	case TRACE_REG_REGISTER:
	629	return reg_event_syscall_enter(event);
	630	case TRACE_REG_UNREGISTER:
	631	unreg_event_syscall_enter(event);
	632	return 0;
	633
	634	#ifdef CONFIG_PERF_EVENTS
	635	case TRACE_REG_PERF_REGISTER:
	636	return perf_sysenter_enable(event);
	637	case TRACE_REG_PERF_UNREGISTER:
	638	perf_sysenter_disable(event);
	639	return 0;
	640	#endif
	641	}
	642	return 0;
	643	}
	644
	645	static int syscall_exit_register(struct ftrace_event_call *event,
	646	enum trace_reg type)
	647	{
	648	switch (type) {
	649	case TRACE_REG_REGISTER:
	650	return reg_event_syscall_exit(event);
	651	case TRACE_REG_UNREGISTER:
	652	unreg_event_syscall_exit(event);
	653	return 0;
	654
	655	#ifdef CONFIG_PERF_EVENTS
	656	case TRACE_REG_PERF_REGISTER:
	657	return perf_sysexit_enable(event);
	658	case TRACE_REG_PERF_UNREGISTER:
	659	perf_sysexit_disable(event);
	660	return 0;
	661	#endif
	662	}
	663	return 0;
	664	}