[mirror_ubuntu-bionic-kernel.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type);
static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type);

static int syscall_enter_define_fields(struct ftrace_event_call *call);
static int syscall_exit_define_fields(struct ftrace_event_call *call);

static struct list_head *
syscall_get_enter_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->enter_fields;
}

static struct list_head *
syscall_get_exit_fields(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	return &entry->exit_fields;
}

struct ftrace_event_class event_class_syscall_enter = {
	.system			= "syscalls",
	.reg			= syscall_enter_register,
	.define_fields		= syscall_enter_define_fields,
	.get_fields		= syscall_get_enter_fields,
	.raw_init		= init_syscall_trace,
};

struct ftrace_event_class event_class_syscall_exit = {
	.system			= "syscalls",
	.reg			= syscall_exit_register,
	.define_fields		= syscall_exit_define_fields,
	.get_fields		= syscall_get_exit_fields,
	.raw_init		= init_syscall_trace,
};

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags,
		    struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags,
		   struct trace_event *event)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

static
int  __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
{
	int i;
	int pos = 0;

	/* When len=0, we just calculate the needed length */
#define LEN_OR_ZERO (len ? len - pos : 0)

	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
				entry->args[i], sizeof(unsigned long),
				i == entry->nb_args - 1 ? "" : ", ");
	}
	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");

	for (i = 0; i < entry->nb_args; i++) {
		pos += snprintf(buf + pos, LEN_OR_ZERO,
				", ((unsigned long)(REC->%s))", entry->args[i]);
	}

#undef LEN_OR_ZERO

	/* return the length of print_fmt */
	return pos;
}

static int set_syscall_print_fmt(struct ftrace_event_call *call)
{
	char *print_fmt;
	int len;
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event != call) {
		call->print_fmt = "\"0x%lx\", REC->ret";
		return 0;
	}

	/* First: called with 0 length to calculate the needed length */
	len = __set_enter_print_fmt(entry, NULL, 0);

	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	if (!print_fmt)
		return -ENOMEM;

	/* Second: actually write the @print_fmt */
	__set_enter_print_fmt(entry, print_fmt, len + 1);
	call->print_fmt = print_fmt;

	return 0;
}

static void free_syscall_print_fmt(struct ftrace_event_call *call)
{
	struct syscall_metadata *entry = call->data;

	if (entry->enter_event == call)
		kfree(call->print_fmt);
}

static int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

static int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
	if (!ret) {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
	if (!ret) {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	if (set_syscall_print_fmt(call) < 0)
		return -ENOMEM;

	id = trace_event_raw_init(call);

	if (id < 0) {
		free_syscall_print_fmt(call);
		return id;
	}

	return id;
}

unsigned long __init arch_syscall_addr(int nr)
{
	return (unsigned long)sys_call_table[nr];
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_PERF_EVENTS

static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
static int sys_perf_refcount_enter;
static int sys_perf_refcount_exit;

static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		      "perf buffer not large enough"))
		return;

	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
				sys_data->enter_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_enter)
		ret = register_trace_sys_enter(perf_syscall_enter, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_perf_enter_syscalls);
		sys_perf_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_enter--;
	clear_bit(num, enabled_perf_enter_syscalls);
	if (!sys_perf_refcount_enter)
		unregister_trace_sys_enter(perf_syscall_enter, NULL);
	mutex_unlock(&syscall_trace_lock);
}

static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	int rctx;
	int size;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		"exit event has grown above perf buffer size"))
		return;

	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
				sys_data->exit_event->id, &rctx, &flags);
	if (!rec)
		return;

	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
}

int perf_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_perf_refcount_exit)
		ret = register_trace_sys_exit(perf_syscall_exit, NULL);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_perf_exit_syscalls);
		sys_perf_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void perf_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_perf_refcount_exit--;
	clear_bit(num, enabled_perf_exit_syscalls);
	if (!sys_perf_refcount_exit)
		unregister_trace_sys_exit(perf_syscall_exit, NULL);
	mutex_unlock(&syscall_trace_lock);
}

#endif /* CONFIG_PERF_EVENTS */

static int syscall_enter_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_enter(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_enter(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysenter_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysenter_disable(event);
		return 0;
#endif
	}
	return 0;
}

static int syscall_exit_register(struct ftrace_event_call *event,
				 enum trace_reg type)
{
	switch (type) {
	case TRACE_REG_REGISTER:
		return reg_event_syscall_exit(event);
	case TRACE_REG_UNREGISTER:
		unreg_event_syscall_exit(event);
		return 0;

#ifdef CONFIG_PERF_EVENTS
	case TRACE_REG_PERF_REGISTER:
		return perf_sysexit_enable(event);
	case TRACE_REG_PERF_UNREGISTER:
		perf_sysexit_disable(event);
		return 0;
#endif
	}
	return 0;
}
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
5a0e3ad6	3	#include <linux/slab.h>
ee08c6ec	4	#include <linux/kernel.h>
fb34a08c	5	#include <linux/ftrace.h>
cdd6c482	6	#include <linux/perf_event.h>
ee08c6ec FW	7	#include <asm/syscall.h>
	8
	9	#include "trace_output.h"
	10	#include "trace.h"
	11
5be71b61	12	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	13	static int sys_refcount_enter;
fb34a08c JB	14	static int sys_refcount_exit;
57421dbb JB	15	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	16	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	17
2239291a SR	18	static int syscall_enter_register(struct ftrace_event_call *event,
	19	enum trace_reg type);
	20	static int syscall_exit_register(struct ftrace_event_call *event,
	21	enum trace_reg type);
	22
2e33af02 SR	23	static int syscall_enter_define_fields(struct ftrace_event_call *call);
	24	static int syscall_exit_define_fields(struct ftrace_event_call *call);
	25
	26	static struct list_head *
	27	syscall_get_enter_fields(struct ftrace_event_call *call)
	28	{
	29	struct syscall_metadata *entry = call->data;
	30
	31	return &entry->enter_fields;
	32	}
	33
	34	static struct list_head *
	35	syscall_get_exit_fields(struct ftrace_event_call *call)
	36	{
	37	struct syscall_metadata *entry = call->data;
	38
	39	return &entry->exit_fields;
	40	}
	41
2239291a SR	42	struct ftrace_event_class event_class_syscall_enter = {
2239291a SR	43	.system = "syscalls",
2e33af02 SR	44	.reg = syscall_enter_register,
	45	.define_fields = syscall_enter_define_fields,
	46	.get_fields = syscall_get_enter_fields,
0405ab80	47	.raw_init = init_syscall_trace,
2239291a SR	48	};
	49
	50	struct ftrace_event_class event_class_syscall_exit = {
	51	.system = "syscalls",
2e33af02 SR	52	.reg = syscall_exit_register,
	53	.define_fields = syscall_exit_define_fields,
	54	.get_fields = syscall_get_exit_fields,
0405ab80	55	.raw_init = init_syscall_trace,
8f082018 SR	56	};
8f082018 SR	57
c44fc770 FW	58	extern unsigned long __start_syscalls_metadata[];
	59	extern unsigned long __stop_syscalls_metadata[];
	60
	61	static struct syscall_metadata **syscalls_metadata;
	62
	63	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	64	{
	65	struct syscall_metadata *start;
	66	struct syscall_metadata *stop;
	67	char str[KSYM_SYMBOL_LEN];
	68
	69
	70	start = (struct syscall_metadata *)__start_syscalls_metadata;
	71	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	72	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	73
	74	for ( ; start < stop; start++) {
	75	/*
	76	* Only compare after the "sys" prefix. Archs that use
	77	* syscall wrappers may have syscalls symbols aliases prefixed
	78	* with "SyS" instead of "sys", leading to an unwanted
	79	* mismatch.
	80	*/
	81	if (start->name && !strcmp(start->name + 3, str + 3))
	82	return start;
	83	}
	84	return NULL;
	85	}
	86
	87	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	88	{
	89	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	90	return NULL;
	91
	92	return syscalls_metadata[nr];
	93	}
	94
bed1ffca	95	enum print_line_t
a9a57763 SR	96	print_syscall_enter(struct trace_iterator *iter, int flags,
a9a57763 SR	97	struct trace_event *event)
bed1ffca FW	98	{
	99	struct trace_seq *s = &iter->seq;
	100	struct trace_entry *ent = iter->ent;
	101	struct syscall_trace_enter *trace;
	102	struct syscall_metadata *entry;
	103	int i, ret, syscall;
	104
64c12e04	105	trace = (typeof(trace))ent;
bed1ffca	106	syscall = trace->nr;
bed1ffca	107	entry = syscall_nr_to_meta(syscall);
64c12e04	108
bed1ffca FW	109	if (!entry)
	110	goto end;
	111
fcc19438	112	if (entry->enter_event->id != ent->type) {
64c12e04 JB	113	WARN_ON_ONCE(1);
	114	goto end;
	115	}
	116
bed1ffca FW	117	ret = trace_seq_printf(s, "%s(", entry->name);
	118	if (!ret)
	119	return TRACE_TYPE_PARTIAL_LINE;
	120
	121	for (i = 0; i < entry->nb_args; i++) {
	122	/* parameter types */
ba8b3a40	123	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	124	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	125	if (!ret)
	126	return TRACE_TYPE_PARTIAL_LINE;
	127	}
	128	/* parameter values */
4539f077	129	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	130	trace->args[i],
4539f077	131	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	132	if (!ret)
	133	return TRACE_TYPE_PARTIAL_LINE;
	134	}
	135
4539f077 LZ	136	ret = trace_seq_putc(s, ')');
	137	if (!ret)
	138	return TRACE_TYPE_PARTIAL_LINE;
	139
bed1ffca	140	end:
4539f077 LZ	141	ret = trace_seq_putc(s, '\n');
	142	if (!ret)
	143	return TRACE_TYPE_PARTIAL_LINE;
	144
bed1ffca FW	145	return TRACE_TYPE_HANDLED;
	146	}
	147
	148	enum print_line_t
a9a57763 SR	149	print_syscall_exit(struct trace_iterator *iter, int flags,
a9a57763 SR	150	struct trace_event *event)
bed1ffca FW	151	{
	152	struct trace_seq *s = &iter->seq;
	153	struct trace_entry *ent = iter->ent;
	154	struct syscall_trace_exit *trace;
	155	int syscall;
	156	struct syscall_metadata *entry;
	157	int ret;
	158
64c12e04	159	trace = (typeof(trace))ent;
bed1ffca	160	syscall = trace->nr;
bed1ffca	161	entry = syscall_nr_to_meta(syscall);
64c12e04	162
bed1ffca FW	163	if (!entry) {
	164	trace_seq_printf(s, "\n");
	165	return TRACE_TYPE_HANDLED;
	166	}
	167
fcc19438	168	if (entry->exit_event->id != ent->type) {
64c12e04 JB	169	WARN_ON_ONCE(1);
	170	return TRACE_TYPE_UNHANDLED;
	171	}
	172
bed1ffca FW	173	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	174	trace->ret);
	175	if (!ret)
	176	return TRACE_TYPE_PARTIAL_LINE;
	177
	178	return TRACE_TYPE_HANDLED;
	179	}
	180
e6971969 LZ	181	extern char *__bad_type_size(void);
	182
	183	#define SYSCALL_FIELD(type, name) \
	184	sizeof(type) != sizeof(trace.name) ? \
	185	__bad_type_size() : \
26a50744 TZ	186	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	187	sizeof(trace.name), is_signed_type(type)
e6971969	188
50307a45 LJ	189	static
	190	int __set_enter_print_fmt(struct syscall_metadata entry, char buf, int len)
	191	{
	192	int i;
	193	int pos = 0;
	194
	195	/* When len=0, we just calculate the needed length */
	196	#define LEN_OR_ZERO (len ? len - pos : 0)
	197
	198	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	199	for (i = 0; i < entry->nb_args; i++) {
	200	pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
	201	entry->args[i], sizeof(unsigned long),
	202	i == entry->nb_args - 1 ? "" : ", ");
	203	}
	204	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
	205
	206	for (i = 0; i < entry->nb_args; i++) {
	207	pos += snprintf(buf + pos, LEN_OR_ZERO,
	208	", ((unsigned long)(REC->%s))", entry->args[i]);
	209	}
	210
	211	#undef LEN_OR_ZERO
	212
	213	/* return the length of print_fmt */
	214	return pos;
	215	}
	216
	217	static int set_syscall_print_fmt(struct ftrace_event_call *call)
	218	{
	219	char *print_fmt;
	220	int len;
	221	struct syscall_metadata *entry = call->data;
	222
	223	if (entry->enter_event != call) {
	224	call->print_fmt = "\"0x%lx\", REC->ret";
	225	return 0;
	226	}
	227
	228	/* First: called with 0 length to calculate the needed length */
	229	len = __set_enter_print_fmt(entry, NULL, 0);
	230
	231	print_fmt = kmalloc(len + 1, GFP_KERNEL);
	232	if (!print_fmt)
	233	return -ENOMEM;
	234
	235	/* Second: actually write the @print_fmt */
	236	__set_enter_print_fmt(entry, print_fmt, len + 1);
	237	call->print_fmt = print_fmt;
	238
	239	return 0;
	240	}
	241
	242	static void free_syscall_print_fmt(struct ftrace_event_call *call)
	243	{
	244	struct syscall_metadata *entry = call->data;
	245
	246	if (entry->enter_event == call)
	247	kfree(call->print_fmt);
	248	}
	249
2e33af02	250	static int syscall_enter_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	251	{
540b7b8d LZ	252	struct syscall_trace_enter trace;
31c16b13	253	struct syscall_metadata *meta = call->data;
540b7b8d	254	int ret;
540b7b8d LZ	255	int i;
	256	int offset = offsetof(typeof(trace), args);
	257
0f1ef51d LJ	258	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	259	if (ret)
	260	return ret;
	261
540b7b8d	262	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	263	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	264	meta->args[i], offset,
43b51ead LZ	265	sizeof(unsigned long), 0,
43b51ead LZ	266	FILTER_OTHER);
540b7b8d LZ	267	offset += sizeof(unsigned long);
	268	}
	269
	270	return ret;
	271	}
	272
2e33af02	273	static int syscall_exit_define_fields(struct ftrace_event_call *call)
540b7b8d LZ	274	{
	275	struct syscall_trace_exit trace;
	276	int ret;
	277
0f1ef51d LJ	278	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	279	if (ret)
	280	return ret;
	281
26a50744	282	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	283	FILTER_OTHER);
540b7b8d LZ	284
	285	return ret;
	286	}
	287
38516ab5	288	void ftrace_syscall_enter(void ignore, struct pt_regs regs, long id)
ee08c6ec	289	{
bed1ffca FW	290	struct syscall_trace_enter *entry;
	291	struct syscall_metadata *sys_data;
	292	struct ring_buffer_event *event;
e77405ad	293	struct ring_buffer *buffer;
bed1ffca	294	int size;
ee08c6ec FW	295	int syscall_nr;
	296
	297	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	298	if (syscall_nr < 0)
cd0980fc HB	299	return;
fb34a08c JB	300	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	301	return;
ee08c6ec	302
bed1ffca FW	303	sys_data = syscall_nr_to_meta(syscall_nr);
	304	if (!sys_data)
	305	return;
	306
	307	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	308
fcc19438 LJ	309	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	310	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	311	if (!event)
	312	return;
	313
	314	entry = ring_buffer_event_data(event);
	315	entry->nr = syscall_nr;
	316	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	317
e77405ad SR	318	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	319	entry, event))
	320	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	321	}
ee08c6ec FW	322
38516ab5	323	void ftrace_syscall_exit(void ignore, struct pt_regs regs, long ret)
ee08c6ec	324	{
bed1ffca FW	325	struct syscall_trace_exit *entry;
	326	struct syscall_metadata *sys_data;
	327	struct ring_buffer_event *event;
e77405ad	328	struct ring_buffer *buffer;
ee08c6ec FW	329	int syscall_nr;
	330
	331	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	332	if (syscall_nr < 0)
cd0980fc HB	333	return;
fb34a08c JB	334	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	335	return;
ee08c6ec	336
bed1ffca FW	337	sys_data = syscall_nr_to_meta(syscall_nr);
	338	if (!sys_data)
	339	return;
	340
fcc19438 LJ	341	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	342	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	343	if (!event)
	344	return;
	345
	346	entry = ring_buffer_event_data(event);
	347	entry->nr = syscall_nr;
	348	entry->ret = syscall_get_return_value(current, regs);
	349
e77405ad SR	350	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	351	entry, event))
	352	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	353	}
ee08c6ec FW	354
bd1a5c84	355	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	356	{
fb34a08c JB	357	int ret = 0;
fb34a08c JB	358	int num;
fb34a08c	359
c252f657	360	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	361	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	362	return -ENOSYS;
	363	mutex_lock(&syscall_trace_lock);
	364	if (!sys_refcount_enter)
38516ab5	365	ret = register_trace_sys_enter(ftrace_syscall_enter, NULL);
3b8e4273	366	if (!ret) {
fb34a08c JB	367	set_bit(num, enabled_enter_syscalls);
	368	sys_refcount_enter++;
	369	}
	370	mutex_unlock(&syscall_trace_lock);
	371	return ret;
ee08c6ec FW	372	}
ee08c6ec FW	373
bd1a5c84	374	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	375	{
fb34a08c	376	int num;
ee08c6ec	377
c252f657	378	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	379	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	380	return;
	381	mutex_lock(&syscall_trace_lock);
	382	sys_refcount_enter--;
	383	clear_bit(num, enabled_enter_syscalls);
	384	if (!sys_refcount_enter)
38516ab5	385	unregister_trace_sys_enter(ftrace_syscall_enter, NULL);
fb34a08c JB	386	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	387	}
ee08c6ec	388
bd1a5c84	389	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	390	{
fb34a08c JB	391	int ret = 0;
fb34a08c JB	392	int num;
fb34a08c	393
c252f657	394	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	395	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	396	return -ENOSYS;
	397	mutex_lock(&syscall_trace_lock);
	398	if (!sys_refcount_exit)
38516ab5	399	ret = register_trace_sys_exit(ftrace_syscall_exit, NULL);
3b8e4273	400	if (!ret) {
fb34a08c JB	401	set_bit(num, enabled_exit_syscalls);
fb34a08c JB	402	sys_refcount_exit++;
ee08c6ec	403	}
fb34a08c JB	404	mutex_unlock(&syscall_trace_lock);
	405	return ret;
	406	}
ee08c6ec	407
bd1a5c84	408	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	409	{
fb34a08c JB	410	int num;
ee08c6ec	411
c252f657	412	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	413	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	414	return;
	415	mutex_lock(&syscall_trace_lock);
	416	sys_refcount_exit--;
	417	clear_bit(num, enabled_exit_syscalls);
	418	if (!sys_refcount_exit)
38516ab5	419	unregister_trace_sys_exit(ftrace_syscall_exit, NULL);
fb34a08c	420	mutex_unlock(&syscall_trace_lock);
ee08c6ec	421	}
fb34a08c	422
a1301da0 LJ	423	int init_syscall_trace(struct ftrace_event_call *call)
	424	{
	425	int id;
	426
50307a45 LJ	427	if (set_syscall_print_fmt(call) < 0)
	428	return -ENOMEM;
	429
c7ef3a90 SR	430	id = trace_event_raw_init(call);
	431
	432	if (id < 0) {
50307a45	433	free_syscall_print_fmt(call);
c7ef3a90	434	return id;
50307a45	435	}
c7ef3a90 SR	436
c7ef3a90 SR	437	return id;
a1301da0 LJ	438	}
a1301da0 LJ	439
e7b8e675 MF	440	unsigned long __init arch_syscall_addr(int nr)
	441	{
	442	return (unsigned long)sys_call_table[nr];
	443	}
	444
c44fc770 FW	445	int __init init_ftrace_syscalls(void)
	446	{
	447	struct syscall_metadata *meta;
	448	unsigned long addr;
	449	int i;
	450
	451	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	452	NR_syscalls, GFP_KERNEL);
	453	if (!syscalls_metadata) {
	454	WARN_ON(1);
	455	return -ENOMEM;
	456	}
	457
	458	for (i = 0; i < NR_syscalls; i++) {
	459	addr = arch_syscall_addr(i);
	460	meta = find_syscall_meta(addr);
c252f657 LJ	461	if (!meta)
	462	continue;
	463
	464	meta->syscall_nr = i;
c44fc770 FW	465	syscalls_metadata[i] = meta;
	466	}
	467
	468	return 0;
	469	}
	470	core_initcall(init_ftrace_syscalls);
	471
07b139c8	472	#ifdef CONFIG_PERF_EVENTS
19007a67	473
97d5a220 FW	474	static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
	475	static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
	476	static int sys_perf_refcount_enter;
	477	static int sys_perf_refcount_exit;
f4b5ffcc	478
38516ab5	479	static void perf_syscall_enter(void ignore, struct pt_regs regs, long id)
f4b5ffcc JB	480	{
f4b5ffcc JB	481	struct syscall_metadata *sys_data;
20ab4425 FW	482	struct syscall_trace_enter *rec;
20ab4425 FW	483	unsigned long flags;
f4b5ffcc	484	int syscall_nr;
4ed7c92d	485	int rctx;
19007a67	486	int size;
f4b5ffcc JB	487
f4b5ffcc JB	488	syscall_nr = syscall_get_nr(current, regs);
97d5a220	489	if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
f4b5ffcc JB	490	return;
	491
	492	sys_data = syscall_nr_to_meta(syscall_nr);
	493	if (!sys_data)
	494	return;
	495
19007a67 FW	496	/* get the size after alignment with the u32 buffer size field */
	497	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	498	size = ALIGN(size + sizeof(u32), sizeof(u64));
	499	size -= sizeof(u32);
	500
97d5a220 FW	501	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	502	"perf buffer not large enough"))
20ab4425 FW	503	return;
20ab4425 FW	504
97d5a220	505	rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
430ad5a6 XG	506	sys_data->enter_event->id, &rctx, &flags);
	507	if (!rec)
	508	return;
20ab4425	509
20ab4425 FW	510	rec->nr = syscall_nr;
	511	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	512	(unsigned long *)&rec->args);
97d5a220	513	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	514	}
f4b5ffcc JB	515
97d5a220	516	int perf_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	517	{
	518	int ret = 0;
	519	int num;
	520
3bbe84e9	521	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	522
f4b5ffcc JB	523	mutex_lock(&syscall_trace_lock);
97d5a220	524	if (!sys_perf_refcount_enter)
38516ab5	525	ret = register_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	526	if (ret) {
	527	pr_info("event trace: Could not activate"
	528	"syscall entry trace point");
	529	} else {
97d5a220 FW	530	set_bit(num, enabled_perf_enter_syscalls);
97d5a220 FW	531	sys_perf_refcount_enter++;
f4b5ffcc JB	532	}
	533	mutex_unlock(&syscall_trace_lock);
	534	return ret;
	535	}
	536
97d5a220	537	void perf_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	538	{
	539	int num;
	540
3bbe84e9	541	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	542
f4b5ffcc JB	543	mutex_lock(&syscall_trace_lock);
97d5a220 FW	544	sys_perf_refcount_enter--;
	545	clear_bit(num, enabled_perf_enter_syscalls);
	546	if (!sys_perf_refcount_enter)
38516ab5	547	unregister_trace_sys_enter(perf_syscall_enter, NULL);
f4b5ffcc JB	548	mutex_unlock(&syscall_trace_lock);
	549	}
	550
38516ab5	551	static void perf_syscall_exit(void ignore, struct pt_regs regs, long ret)
f4b5ffcc JB	552	{
f4b5ffcc JB	553	struct syscall_metadata *sys_data;
20ab4425 FW	554	struct syscall_trace_exit *rec;
20ab4425 FW	555	unsigned long flags;
f4b5ffcc	556	int syscall_nr;
4ed7c92d	557	int rctx;
20ab4425	558	int size;
f4b5ffcc JB	559
f4b5ffcc JB	560	syscall_nr = syscall_get_nr(current, regs);
97d5a220	561	if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
f4b5ffcc JB	562	return;
	563
	564	sys_data = syscall_nr_to_meta(syscall_nr);
	565	if (!sys_data)
	566	return;
	567
20ab4425 FW	568	/* We can probably do that at build time */
	569	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	570	size -= sizeof(u32);
19007a67	571
20ab4425 FW	572	/*
	573	* Impossible, but be paranoid with the future
	574	* How to put this check outside runtime?
	575	*/
97d5a220 FW	576	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
97d5a220 FW	577	"exit event has grown above perf buffer size"))
20ab4425 FW	578	return;
20ab4425 FW	579
97d5a220	580	rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
430ad5a6 XG	581	sys_data->exit_event->id, &rctx, &flags);
	582	if (!rec)
	583	return;
20ab4425	584
20ab4425 FW	585	rec->nr = syscall_nr;
	586	rec->ret = syscall_get_return_value(current, regs);
	587
97d5a220	588	perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs);
f4b5ffcc JB	589	}
f4b5ffcc JB	590
97d5a220	591	int perf_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	592	{
	593	int ret = 0;
	594	int num;
	595
3bbe84e9	596	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	597
f4b5ffcc JB	598	mutex_lock(&syscall_trace_lock);
97d5a220	599	if (!sys_perf_refcount_exit)
38516ab5	600	ret = register_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	601	if (ret) {
f4b5ffcc JB	602	pr_info("event trace: Could not activate"
6574658b	603	"syscall exit trace point");
f4b5ffcc	604	} else {
97d5a220 FW	605	set_bit(num, enabled_perf_exit_syscalls);
97d5a220 FW	606	sys_perf_refcount_exit++;
f4b5ffcc JB	607	}
	608	mutex_unlock(&syscall_trace_lock);
	609	return ret;
	610	}
	611
97d5a220	612	void perf_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	613	{
	614	int num;
	615
3bbe84e9	616	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	617
f4b5ffcc JB	618	mutex_lock(&syscall_trace_lock);
97d5a220 FW	619	sys_perf_refcount_exit--;
	620	clear_bit(num, enabled_perf_exit_syscalls);
	621	if (!sys_perf_refcount_exit)
38516ab5	622	unregister_trace_sys_exit(perf_syscall_exit, NULL);
f4b5ffcc JB	623	mutex_unlock(&syscall_trace_lock);
	624	}
	625
07b139c8	626	#endif /* CONFIG_PERF_EVENTS */
f4b5ffcc	627
2239291a SR	628	static int syscall_enter_register(struct ftrace_event_call *event,
	629	enum trace_reg type)
	630	{
	631	switch (type) {
	632	case TRACE_REG_REGISTER:
	633	return reg_event_syscall_enter(event);
	634	case TRACE_REG_UNREGISTER:
	635	unreg_event_syscall_enter(event);
	636	return 0;
	637
	638	#ifdef CONFIG_PERF_EVENTS
	639	case TRACE_REG_PERF_REGISTER:
	640	return perf_sysenter_enable(event);
	641	case TRACE_REG_PERF_UNREGISTER:
	642	perf_sysenter_disable(event);
	643	return 0;
	644	#endif
	645	}
	646	return 0;
	647	}
	648
	649	static int syscall_exit_register(struct ftrace_event_call *event,
	650	enum trace_reg type)
	651	{
	652	switch (type) {
	653	case TRACE_REG_REGISTER:
	654	return reg_event_syscall_exit(event);
	655	case TRACE_REG_UNREGISTER:
	656	unreg_event_syscall_exit(event);
	657	return 0;
	658
	659	#ifdef CONFIG_PERF_EVENTS
	660	case TRACE_REG_PERF_REGISTER:
	661	return perf_sysexit_enable(event);
	662	case TRACE_REG_PERF_UNREGISTER:
	663	perf_sysexit_disable(event);
	664	return 0;
	665	#endif
	666	}
	667	return 0;
	668	}