[mirror_ubuntu-bionic-kernel.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int i;
	int ret;
	struct syscall_metadata *entry = call->data;
	struct syscall_trace_enter trace;
	int offset = offsetof(struct syscall_trace_enter, args);

	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr));
	if (!ret)
		return 0;

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
				        entry->args[i]);
		if (!ret)
			return 0;
		ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
				       "\tsigned:%u;\n", offset,
				       sizeof(unsigned long),
				       is_signed_type(unsigned long));
		if (!ret)
			return 0;
		offset += sizeof(unsigned long);
	}

	trace_seq_puts(s, "\nprint fmt: \"");
	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
				        sizeof(unsigned long),
					i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return 0;
	}
	trace_seq_putc(s, '"');

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
				       entry->args[i]);
		if (!ret)
			return 0;
	}

	return trace_seq_putc(s, '\n');
}

int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int ret;
	struct syscall_trace_exit trace;

	ret = trace_seq_printf(s,
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n"
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr),
			       SYSCALL_FIELD(long, ret));
	if (!ret)
		return 0;

	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
}

int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter);
	if (!ret) {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit);
	if (!ret) {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	id = register_ftrace_event(call->event);
	if (!id)
		return -ENODEV;
	call->id = id;
	INIT_LIST_HEAD(&call->fields);
	return 0;
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_EVENT_PROFILE

static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
static int sys_prof_refcount_enter;
static int sys_prof_refcount_exit;

static void prof_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	char *trace_buf;
	char *raw_data;
	int syscall_nr;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		      "profile buffer not large enough"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_enter *) raw_data;
	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->enter_event->id;
	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_enter)
		ret = register_trace_sys_enter(prof_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_enter_syscalls);
		sys_prof_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_enter--;
	clear_bit(num, enabled_prof_enter_syscalls);
	if (!sys_prof_refcount_enter)
		unregister_trace_sys_enter(prof_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	char *trace_buf;
	char *raw_data;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		"exit event has grown above profile buffer size"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_exit *)raw_data;

	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->exit_event->id;
	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_exit)
		ret = register_trace_sys_exit(prof_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_exit_syscalls);
		sys_prof_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_exit--;
	clear_bit(num, enabled_prof_exit_syscalls);
	if (!sys_prof_refcount_exit)
		unregister_trace_sys_exit(prof_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

#endif
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
ee08c6ec	3	#include <linux/kernel.h>
fb34a08c	4	#include <linux/ftrace.h>
cdd6c482	5	#include <linux/perf_event.h>
ee08c6ec FW	6	#include <asm/syscall.h>
	7
	8	#include "trace_output.h"
	9	#include "trace.h"
	10
5be71b61	11	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	12	static int sys_refcount_enter;
fb34a08c JB	13	static int sys_refcount_exit;
57421dbb JB	14	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	15	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	16
c44fc770 FW	17	extern unsigned long __start_syscalls_metadata[];
	18	extern unsigned long __stop_syscalls_metadata[];
	19
	20	static struct syscall_metadata **syscalls_metadata;
	21
	22	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	23	{
	24	struct syscall_metadata *start;
	25	struct syscall_metadata *stop;
	26	char str[KSYM_SYMBOL_LEN];
	27
	28
	29	start = (struct syscall_metadata *)__start_syscalls_metadata;
	30	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	31	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	32
	33	for ( ; start < stop; start++) {
	34	/*
	35	* Only compare after the "sys" prefix. Archs that use
	36	* syscall wrappers may have syscalls symbols aliases prefixed
	37	* with "SyS" instead of "sys", leading to an unwanted
	38	* mismatch.
	39	*/
	40	if (start->name && !strcmp(start->name + 3, str + 3))
	41	return start;
	42	}
	43	return NULL;
	44	}
	45
	46	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	47	{
	48	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	49	return NULL;
	50
	51	return syscalls_metadata[nr];
	52	}
	53
bed1ffca FW	54	enum print_line_t
	55	print_syscall_enter(struct trace_iterator *iter, int flags)
	56	{
	57	struct trace_seq *s = &iter->seq;
	58	struct trace_entry *ent = iter->ent;
	59	struct syscall_trace_enter *trace;
	60	struct syscall_metadata *entry;
	61	int i, ret, syscall;
	62
64c12e04	63	trace = (typeof(trace))ent;
bed1ffca	64	syscall = trace->nr;
bed1ffca	65	entry = syscall_nr_to_meta(syscall);
64c12e04	66
bed1ffca FW	67	if (!entry)
	68	goto end;
	69
fcc19438	70	if (entry->enter_event->id != ent->type) {
64c12e04 JB	71	WARN_ON_ONCE(1);
	72	goto end;
	73	}
	74
bed1ffca FW	75	ret = trace_seq_printf(s, "%s(", entry->name);
	76	if (!ret)
	77	return TRACE_TYPE_PARTIAL_LINE;
	78
	79	for (i = 0; i < entry->nb_args; i++) {
	80	/* parameter types */
ba8b3a40	81	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	82	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	83	if (!ret)
	84	return TRACE_TYPE_PARTIAL_LINE;
	85	}
	86	/* parameter values */
4539f077	87	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	88	trace->args[i],
4539f077	89	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	90	if (!ret)
	91	return TRACE_TYPE_PARTIAL_LINE;
	92	}
	93
4539f077 LZ	94	ret = trace_seq_putc(s, ')');
	95	if (!ret)
	96	return TRACE_TYPE_PARTIAL_LINE;
	97
bed1ffca	98	end:
4539f077 LZ	99	ret = trace_seq_putc(s, '\n');
	100	if (!ret)
	101	return TRACE_TYPE_PARTIAL_LINE;
	102
bed1ffca FW	103	return TRACE_TYPE_HANDLED;
	104	}
	105
	106	enum print_line_t
	107	print_syscall_exit(struct trace_iterator *iter, int flags)
	108	{
	109	struct trace_seq *s = &iter->seq;
	110	struct trace_entry *ent = iter->ent;
	111	struct syscall_trace_exit *trace;
	112	int syscall;
	113	struct syscall_metadata *entry;
	114	int ret;
	115
64c12e04	116	trace = (typeof(trace))ent;
bed1ffca	117	syscall = trace->nr;
bed1ffca	118	entry = syscall_nr_to_meta(syscall);
64c12e04	119
bed1ffca FW	120	if (!entry) {
	121	trace_seq_printf(s, "\n");
	122	return TRACE_TYPE_HANDLED;
	123	}
	124
fcc19438	125	if (entry->exit_event->id != ent->type) {
64c12e04 JB	126	WARN_ON_ONCE(1);
	127	return TRACE_TYPE_UNHANDLED;
	128	}
	129
bed1ffca FW	130	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	131	trace->ret);
	132	if (!ret)
	133	return TRACE_TYPE_PARTIAL_LINE;
	134
	135	return TRACE_TYPE_HANDLED;
	136	}
	137
e6971969 LZ	138	extern char *__bad_type_size(void);
	139
	140	#define SYSCALL_FIELD(type, name) \
	141	sizeof(type) != sizeof(trace.name) ? \
	142	__bad_type_size() : \
26a50744 TZ	143	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	144	sizeof(trace.name), is_signed_type(type)
e6971969	145
10a5b66f	146	int syscall_enter_format(struct ftrace_event_call call, struct trace_seq s)
dc4ddb4c FW	147	{
dc4ddb4c FW	148	int i;
e6971969	149	int ret;
31c16b13	150	struct syscall_metadata *entry = call->data;
e6971969 LZ	151	struct syscall_trace_enter trace;
e6971969 LZ	152	int offset = offsetof(struct syscall_trace_enter, args);
dc4ddb4c	153
26a50744 TZ	154	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
26a50744 TZ	155	"\tsigned:%u;\n",
e6971969 LZ	156	SYSCALL_FIELD(int, nr));
	157	if (!ret)
	158	return 0;
dc4ddb4c FW	159
	160	for (i = 0; i < entry->nb_args; i++) {
	161	ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
	162	entry->args[i]);
	163	if (!ret)
	164	return 0;
26a50744 TZ	165	ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
	166	"\tsigned:%u;\n", offset,
	167	sizeof(unsigned long),
	168	is_signed_type(unsigned long));
dc4ddb4c FW	169	if (!ret)
	170	return 0;
	171	offset += sizeof(unsigned long);
	172	}
	173
4539f077	174	trace_seq_puts(s, "\nprint fmt: \"");
dc4ddb4c	175	for (i = 0; i < entry->nb_args; i++) {
e6971969	176	ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
dc4ddb4c	177	sizeof(unsigned long),
4539f077	178	i == entry->nb_args - 1 ? "" : ", ");
dc4ddb4c FW	179	if (!ret)
	180	return 0;
	181	}
4539f077	182	trace_seq_putc(s, '"');
dc4ddb4c FW	183
dc4ddb4c FW	184	for (i = 0; i < entry->nb_args; i++) {
4539f077 LZ	185	ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
4539f077 LZ	186	entry->args[i]);
dc4ddb4c FW	187	if (!ret)
	188	return 0;
	189	}
	190
4539f077	191	return trace_seq_putc(s, '\n');
dc4ddb4c FW	192	}
dc4ddb4c FW	193
10a5b66f LZ	194	int syscall_exit_format(struct ftrace_event_call call, struct trace_seq s)
	195	{
	196	int ret;
	197	struct syscall_trace_exit trace;
	198
	199	ret = trace_seq_printf(s,
26a50744 TZ	200	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	201	"\tsigned:%u;\n"
	202	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	203	"\tsigned:%u;\n",
10a5b66f	204	SYSCALL_FIELD(int, nr),
ee949a86	205	SYSCALL_FIELD(long, ret));
10a5b66f LZ	206	if (!ret)
	207	return 0;
	208
	209	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
	210	}
	211
540b7b8d LZ	212	int syscall_enter_define_fields(struct ftrace_event_call *call)
	213	{
	214	struct syscall_trace_enter trace;
31c16b13	215	struct syscall_metadata *meta = call->data;
540b7b8d	216	int ret;
540b7b8d LZ	217	int i;
	218	int offset = offsetof(typeof(trace), args);
	219
0f1ef51d LJ	220	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	221	if (ret)
	222	return ret;
	223
540b7b8d	224	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	225	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	226	meta->args[i], offset,
43b51ead LZ	227	sizeof(unsigned long), 0,
43b51ead LZ	228	FILTER_OTHER);
540b7b8d LZ	229	offset += sizeof(unsigned long);
	230	}
	231
	232	return ret;
	233	}
	234
	235	int syscall_exit_define_fields(struct ftrace_event_call *call)
	236	{
	237	struct syscall_trace_exit trace;
	238	int ret;
	239
0f1ef51d LJ	240	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	241	if (ret)
	242	return ret;
	243
26a50744	244	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	245	FILTER_OTHER);
540b7b8d LZ	246
	247	return ret;
	248	}
	249
fb34a08c	250	void ftrace_syscall_enter(struct pt_regs *regs, long id)
ee08c6ec	251	{
bed1ffca FW	252	struct syscall_trace_enter *entry;
	253	struct syscall_metadata *sys_data;
	254	struct ring_buffer_event *event;
e77405ad	255	struct ring_buffer *buffer;
bed1ffca	256	int size;
ee08c6ec FW	257	int syscall_nr;
	258
	259	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	260	if (syscall_nr < 0)
cd0980fc HB	261	return;
fb34a08c JB	262	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	263	return;
ee08c6ec	264
bed1ffca FW	265	sys_data = syscall_nr_to_meta(syscall_nr);
	266	if (!sys_data)
	267	return;
	268
	269	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	270
fcc19438 LJ	271	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	272	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	273	if (!event)
	274	return;
	275
	276	entry = ring_buffer_event_data(event);
	277	entry->nr = syscall_nr;
	278	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	279
e77405ad SR	280	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	281	entry, event))
	282	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	283	}
ee08c6ec FW	284
fb34a08c	285	void ftrace_syscall_exit(struct pt_regs *regs, long ret)
ee08c6ec	286	{
bed1ffca FW	287	struct syscall_trace_exit *entry;
	288	struct syscall_metadata *sys_data;
	289	struct ring_buffer_event *event;
e77405ad	290	struct ring_buffer *buffer;
ee08c6ec FW	291	int syscall_nr;
	292
	293	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	294	if (syscall_nr < 0)
cd0980fc HB	295	return;
fb34a08c JB	296	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	297	return;
ee08c6ec	298
bed1ffca FW	299	sys_data = syscall_nr_to_meta(syscall_nr);
	300	if (!sys_data)
	301	return;
	302
fcc19438 LJ	303	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	304	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	305	if (!event)
	306	return;
	307
	308	entry = ring_buffer_event_data(event);
	309	entry->nr = syscall_nr;
	310	entry->ret = syscall_get_return_value(current, regs);
	311
e77405ad SR	312	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	313	entry, event))
	314	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	315	}
ee08c6ec FW	316
bd1a5c84	317	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	318	{
fb34a08c JB	319	int ret = 0;
fb34a08c JB	320	int num;
fb34a08c	321
c252f657	322	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	323	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	324	return -ENOSYS;
	325	mutex_lock(&syscall_trace_lock);
	326	if (!sys_refcount_enter)
1c569f02	327	ret = register_trace_sys_enter(ftrace_syscall_enter);
3b8e4273	328	if (!ret) {
fb34a08c JB	329	set_bit(num, enabled_enter_syscalls);
	330	sys_refcount_enter++;
	331	}
	332	mutex_unlock(&syscall_trace_lock);
	333	return ret;
ee08c6ec FW	334	}
ee08c6ec FW	335
bd1a5c84	336	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	337	{
fb34a08c	338	int num;
ee08c6ec	339
c252f657	340	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	341	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	342	return;
	343	mutex_lock(&syscall_trace_lock);
	344	sys_refcount_enter--;
	345	clear_bit(num, enabled_enter_syscalls);
	346	if (!sys_refcount_enter)
1c569f02	347	unregister_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	348	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	349	}
ee08c6ec	350
bd1a5c84	351	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	352	{
fb34a08c JB	353	int ret = 0;
fb34a08c JB	354	int num;
fb34a08c	355
c252f657	356	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	357	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	358	return -ENOSYS;
	359	mutex_lock(&syscall_trace_lock);
	360	if (!sys_refcount_exit)
1c569f02	361	ret = register_trace_sys_exit(ftrace_syscall_exit);
3b8e4273	362	if (!ret) {
fb34a08c JB	363	set_bit(num, enabled_exit_syscalls);
fb34a08c JB	364	sys_refcount_exit++;
ee08c6ec	365	}
fb34a08c JB	366	mutex_unlock(&syscall_trace_lock);
	367	return ret;
	368	}
ee08c6ec	369
bd1a5c84	370	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	371	{
fb34a08c JB	372	int num;
ee08c6ec	373
c252f657	374	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	375	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	376	return;
	377	mutex_lock(&syscall_trace_lock);
	378	sys_refcount_exit--;
	379	clear_bit(num, enabled_exit_syscalls);
	380	if (!sys_refcount_exit)
1c569f02	381	unregister_trace_sys_exit(ftrace_syscall_exit);
fb34a08c	382	mutex_unlock(&syscall_trace_lock);
ee08c6ec	383	}
fb34a08c	384
a1301da0 LJ	385	int init_syscall_trace(struct ftrace_event_call *call)
	386	{
	387	int id;
	388
	389	id = register_ftrace_event(call->event);
	390	if (!id)
	391	return -ENODEV;
	392	call->id = id;
	393	INIT_LIST_HEAD(&call->fields);
	394	return 0;
	395	}
	396
c44fc770 FW	397	int __init init_ftrace_syscalls(void)
	398	{
	399	struct syscall_metadata *meta;
	400	unsigned long addr;
	401	int i;
	402
	403	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	404	NR_syscalls, GFP_KERNEL);
	405	if (!syscalls_metadata) {
	406	WARN_ON(1);
	407	return -ENOMEM;
	408	}
	409
	410	for (i = 0; i < NR_syscalls; i++) {
	411	addr = arch_syscall_addr(i);
	412	meta = find_syscall_meta(addr);
c252f657 LJ	413	if (!meta)
	414	continue;
	415
	416	meta->syscall_nr = i;
c44fc770 FW	417	syscalls_metadata[i] = meta;
	418	}
	419
	420	return 0;
	421	}
	422	core_initcall(init_ftrace_syscalls);
	423
f4b5ffcc	424	#ifdef CONFIG_EVENT_PROFILE
19007a67	425
57421dbb JB	426	static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
57421dbb JB	427	static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
f4b5ffcc JB	428	static int sys_prof_refcount_enter;
	429	static int sys_prof_refcount_exit;
	430
	431	static void prof_syscall_enter(struct pt_regs *regs, long id)
	432	{
	433	struct syscall_metadata *sys_data;
20ab4425 FW	434	struct syscall_trace_enter *rec;
20ab4425 FW	435	unsigned long flags;
ce71b9df	436	char *trace_buf;
20ab4425	437	char *raw_data;
f4b5ffcc	438	int syscall_nr;
4ed7c92d	439	int rctx;
19007a67	440	int size;
20ab4425	441	int cpu;
f4b5ffcc JB	442
	443	syscall_nr = syscall_get_nr(current, regs);
	444	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
	445	return;
	446
	447	sys_data = syscall_nr_to_meta(syscall_nr);
	448	if (!sys_data)
	449	return;
	450
19007a67 FW	451	/* get the size after alignment with the u32 buffer size field */
	452	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	453	size = ALIGN(size + sizeof(u32), sizeof(u64));
	454	size -= sizeof(u32);
	455
20ab4425 FW	456	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	457	"profile buffer not large enough"))
	458	return;
	459
	460	/* Protect the per cpu buffer, begin the rcu read side */
	461	local_irq_save(flags);
19007a67	462
4ed7c92d PZ	463	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	464	if (rctx < 0)
ce71b9df FW	465	goto end_recursion;
ce71b9df FW	466
20ab4425 FW	467	cpu = smp_processor_id();
20ab4425 FW	468
28889bf9	469	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	470
444a2a3b	471	if (!trace_buf)
20ab4425	472	goto end;
19007a67	473
ce71b9df	474	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	475
	476	/* zero the dead bytes from align to not leak stack to user */
	477	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	478
	479	rec = (struct syscall_trace_enter *) raw_data;
	480	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	481	rec->ent.type = sys_data->enter_event->id;
20ab4425 FW	482	rec->nr = syscall_nr;
	483	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	484	(unsigned long *)&rec->args);
fcc19438	485	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
20ab4425 FW	486
20ab4425 FW	487	end:
4ed7c92d	488	perf_swevent_put_recursion_context(rctx);
ce71b9df	489	end_recursion:
20ab4425	490	local_irq_restore(flags);
f4b5ffcc JB	491	}
f4b5ffcc JB	492
3bbe84e9	493	int prof_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	494	{
	495	int ret = 0;
	496	int num;
	497
3bbe84e9	498	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	499
	500	mutex_lock(&syscall_trace_lock);
	501	if (!sys_prof_refcount_enter)
1c569f02	502	ret = register_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	503	if (ret) {
	504	pr_info("event trace: Could not activate"
	505	"syscall entry trace point");
	506	} else {
	507	set_bit(num, enabled_prof_enter_syscalls);
	508	sys_prof_refcount_enter++;
	509	}
	510	mutex_unlock(&syscall_trace_lock);
	511	return ret;
	512	}
	513
3bbe84e9	514	void prof_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	515	{
	516	int num;
	517
3bbe84e9	518	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	519
	520	mutex_lock(&syscall_trace_lock);
	521	sys_prof_refcount_enter--;
	522	clear_bit(num, enabled_prof_enter_syscalls);
	523	if (!sys_prof_refcount_enter)
1c569f02	524	unregister_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	525	mutex_unlock(&syscall_trace_lock);
	526	}
	527
	528	static void prof_syscall_exit(struct pt_regs *regs, long ret)
	529	{
	530	struct syscall_metadata *sys_data;
20ab4425 FW	531	struct syscall_trace_exit *rec;
20ab4425 FW	532	unsigned long flags;
f4b5ffcc	533	int syscall_nr;
ce71b9df	534	char *trace_buf;
20ab4425	535	char *raw_data;
4ed7c92d	536	int rctx;
20ab4425 FW	537	int size;
20ab4425 FW	538	int cpu;
f4b5ffcc JB	539
	540	syscall_nr = syscall_get_nr(current, regs);
	541	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
	542	return;
	543
	544	sys_data = syscall_nr_to_meta(syscall_nr);
	545	if (!sys_data)
	546	return;
	547
20ab4425 FW	548	/* We can probably do that at build time */
	549	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	550	size -= sizeof(u32);
19007a67	551
20ab4425 FW	552	/*
	553	* Impossible, but be paranoid with the future
	554	* How to put this check outside runtime?
	555	*/
	556	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	557	"exit event has grown above profile buffer size"))
	558	return;
	559
	560	/* Protect the per cpu buffer, begin the rcu read side */
	561	local_irq_save(flags);
ce71b9df	562
4ed7c92d PZ	563	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	564	if (rctx < 0)
ce71b9df FW	565	goto end_recursion;
ce71b9df FW	566
20ab4425 FW	567	cpu = smp_processor_id();
20ab4425 FW	568
28889bf9	569	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	570
444a2a3b	571	if (!trace_buf)
20ab4425 FW	572	goto end;
20ab4425 FW	573
ce71b9df	574	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	575
	576	/* zero the dead bytes from align to not leak stack to user */
	577	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	578
	579	rec = (struct syscall_trace_exit *)raw_data;
	580
	581	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	582	rec->ent.type = sys_data->exit_event->id;
20ab4425 FW	583	rec->nr = syscall_nr;
	584	rec->ret = syscall_get_return_value(current, regs);
	585
fcc19438	586	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
20ab4425 FW	587
20ab4425 FW	588	end:
4ed7c92d	589	perf_swevent_put_recursion_context(rctx);
ce71b9df	590	end_recursion:
20ab4425	591	local_irq_restore(flags);
f4b5ffcc JB	592	}
f4b5ffcc JB	593
3bbe84e9	594	int prof_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	595	{
	596	int ret = 0;
	597	int num;
	598
3bbe84e9	599	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	600
	601	mutex_lock(&syscall_trace_lock);
	602	if (!sys_prof_refcount_exit)
1c569f02	603	ret = register_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	604	if (ret) {
	605	pr_info("event trace: Could not activate"
	606	"syscall entry trace point");
	607	} else {
	608	set_bit(num, enabled_prof_exit_syscalls);
	609	sys_prof_refcount_exit++;
	610	}
	611	mutex_unlock(&syscall_trace_lock);
	612	return ret;
	613	}
	614
3bbe84e9	615	void prof_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	616	{
	617	int num;
	618
3bbe84e9	619	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	620
	621	mutex_lock(&syscall_trace_lock);
	622	sys_prof_refcount_exit--;
	623	clear_bit(num, enabled_prof_exit_syscalls);
	624	if (!sys_prof_refcount_exit)
1c569f02	625	unregister_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	626	mutex_unlock(&syscall_trace_lock);
	627	}
	628
	629	#endif
	630
	631