[mirror_ubuntu-bionic-kernel.git] / include / trace / events / sched.h

/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM sched

#if !defined(_TRACE_SCHED_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SCHED_H

#include <linux/sched/numa_balancing.h>
#include <linux/tracepoint.h>
#include <linux/binfmts.h>

/*
 * Tracepoint for calling kthread_stop, performed to end a kthread:
 */
TRACE_EVENT(sched_kthread_stop,

	TP_PROTO(struct task_struct *t),

	TP_ARGS(t),

	TP_STRUCT__entry(
		__array(	char,	comm,	TASK_COMM_LEN	)
		__field(	pid_t,	pid			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, t->comm, TASK_COMM_LEN);
		__entry->pid	= t->pid;
	),

	TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
);

/*
 * Tracepoint for the return value of the kthread stopping:
 */
TRACE_EVENT(sched_kthread_stop_ret,

	TP_PROTO(int ret),

	TP_ARGS(ret),

	TP_STRUCT__entry(
		__field(	int,	ret	)
	),

	TP_fast_assign(
		__entry->ret	= ret;
	),

	TP_printk("ret=%d", __entry->ret)
);

/*
 * Tracepoint for waking up a task:
 */
DECLARE_EVENT_CLASS(sched_wakeup_template,

	TP_PROTO(struct task_struct *p),

	TP_ARGS(__perf_task(p)),

	TP_STRUCT__entry(
		__array(	char,	comm,	TASK_COMM_LEN	)
		__field(	pid_t,	pid			)
		__field(	int,	prio			)
		__field(	int,	success			)
		__field(	int,	target_cpu		)
	),

	TP_fast_assign(
		memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
		__entry->pid		= p->pid;
		__entry->prio		= p->prio; /* XXX SCHED_DEADLINE */
		__entry->success	= 1; /* rudiment, kill when possible */
		__entry->target_cpu	= task_cpu(p);
	),

	TP_printk("comm=%s pid=%d prio=%d target_cpu=%03d",
		  __entry->comm, __entry->pid, __entry->prio,
		  __entry->target_cpu)
);

/*
 * Tracepoint called when waking a task; this tracepoint is guaranteed to be
 * called from the waking context.
 */
DEFINE_EVENT(sched_wakeup_template, sched_waking,
	     TP_PROTO(struct task_struct *p),
	     TP_ARGS(p));

/*
 * Tracepoint called when the task is actually woken; p->state == TASK_RUNNNG.
 * It it not always called from the waking context.
 */
DEFINE_EVENT(sched_wakeup_template, sched_wakeup,
	     TP_PROTO(struct task_struct *p),
	     TP_ARGS(p));

/*
 * Tracepoint for waking up a new task:
 */
DEFINE_EVENT(sched_wakeup_template, sched_wakeup_new,
	     TP_PROTO(struct task_struct *p),
	     TP_ARGS(p));

#ifdef CREATE_TRACE_POINTS
static inline long __trace_sched_switch_state(bool preempt, struct task_struct *p)
{
	unsigned int state;

#ifdef CONFIG_SCHED_DEBUG
	BUG_ON(p != current);
#endif /* CONFIG_SCHED_DEBUG */

	/*
	 * Preemption ignores task state, therefore preempted tasks are always
	 * RUNNING (we will not have dequeued if state != RUNNING).
	 */
	if (preempt)
		return TASK_REPORT_MAX;

	/*
	 * task_state_index() uses fls() and returns a value from 0-8 range.
	 * Decrement it by 1 (except TASK_RUNNING state i.e 0) before using
	 * it for left shift operation to get the correct task->state
	 * mapping.
	 */
	state = task_state_index(p);

	return state ? (1 << (state - 1)) : state;
}
#endif /* CREATE_TRACE_POINTS */

/*
 * Tracepoint for task switches, performed by the scheduler:
 */
TRACE_EVENT(sched_switch,

	TP_PROTO(bool preempt,
		 struct task_struct *prev,
		 struct task_struct *next),

	TP_ARGS(preempt, prev, next),

	TP_STRUCT__entry(
		__array(	char,	prev_comm,	TASK_COMM_LEN	)
		__field(	pid_t,	prev_pid			)
		__field(	int,	prev_prio			)
		__field(	long,	prev_state			)
		__array(	char,	next_comm,	TASK_COMM_LEN	)
		__field(	pid_t,	next_pid			)
		__field(	int,	next_prio			)
	),

	TP_fast_assign(
		memcpy(__entry->next_comm, next->comm, TASK_COMM_LEN);
		__entry->prev_pid	= prev->pid;
		__entry->prev_prio	= prev->prio;
		__entry->prev_state	= __trace_sched_switch_state(preempt, prev);
		memcpy(__entry->prev_comm, prev->comm, TASK_COMM_LEN);
		__entry->next_pid	= next->pid;
		__entry->next_prio	= next->prio;
		/* XXX SCHED_DEADLINE */
	),

	TP_printk("prev_comm=%s prev_pid=%d prev_prio=%d prev_state=%s%s ==> next_comm=%s next_pid=%d next_prio=%d",
		__entry->prev_comm, __entry->prev_pid, __entry->prev_prio,

		(__entry->prev_state & (TASK_REPORT_MAX - 1)) ?
		  __print_flags(__entry->prev_state & (TASK_REPORT_MAX - 1), "|",
				{ TASK_INTERRUPTIBLE, "S" },
				{ TASK_UNINTERRUPTIBLE, "D" },
				{ __TASK_STOPPED, "T" },
				{ __TASK_TRACED, "t" },
				{ EXIT_DEAD, "X" },
				{ EXIT_ZOMBIE, "Z" },
				{ TASK_PARKED, "P" },
				{ TASK_DEAD, "I" }) :
		  "R",

		__entry->prev_state & TASK_REPORT_MAX ? "+" : "",
		__entry->next_comm, __entry->next_pid, __entry->next_prio)
);

/*
 * Tracepoint for a task being migrated:
 */
TRACE_EVENT(sched_migrate_task,

	TP_PROTO(struct task_struct *p, int dest_cpu),

	TP_ARGS(p, dest_cpu),

	TP_STRUCT__entry(
		__array(	char,	comm,	TASK_COMM_LEN	)
		__field(	pid_t,	pid			)
		__field(	int,	prio			)
		__field(	int,	orig_cpu		)
		__field(	int,	dest_cpu		)
	),

	TP_fast_assign(
		memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
		__entry->pid		= p->pid;
		__entry->prio		= p->prio; /* XXX SCHED_DEADLINE */
		__entry->orig_cpu	= task_cpu(p);
		__entry->dest_cpu	= dest_cpu;
	),

	TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d",
		  __entry->comm, __entry->pid, __entry->prio,
		  __entry->orig_cpu, __entry->dest_cpu)
);

DECLARE_EVENT_CLASS(sched_process_template,

	TP_PROTO(struct task_struct *p),

	TP_ARGS(p),

	TP_STRUCT__entry(
		__array(	char,	comm,	TASK_COMM_LEN	)
		__field(	pid_t,	pid			)
		__field(	int,	prio			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
		__entry->pid		= p->pid;
		__entry->prio		= p->prio; /* XXX SCHED_DEADLINE */
	),

	TP_printk("comm=%s pid=%d prio=%d",
		  __entry->comm, __entry->pid, __entry->prio)
);

/*
 * Tracepoint for freeing a task:
 */
DEFINE_EVENT(sched_process_template, sched_process_free,
	     TP_PROTO(struct task_struct *p),
	     TP_ARGS(p));
	     

/*
 * Tracepoint for a task exiting:
 */
DEFINE_EVENT(sched_process_template, sched_process_exit,
	     TP_PROTO(struct task_struct *p),
	     TP_ARGS(p));

/*
 * Tracepoint for waiting on task to unschedule:
 */
DEFINE_EVENT(sched_process_template, sched_wait_task,
	TP_PROTO(struct task_struct *p),
	TP_ARGS(p));

/*
 * Tracepoint for a waiting task:
 */
TRACE_EVENT(sched_process_wait,

	TP_PROTO(struct pid *pid),

	TP_ARGS(pid),

	TP_STRUCT__entry(
		__array(	char,	comm,	TASK_COMM_LEN	)
		__field(	pid_t,	pid			)
		__field(	int,	prio			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
		__entry->pid		= pid_nr(pid);
		__entry->prio		= current->prio; /* XXX SCHED_DEADLINE */
	),

	TP_printk("comm=%s pid=%d prio=%d",
		  __entry->comm, __entry->pid, __entry->prio)
);

/*
 * Tracepoint for do_fork:
 */
TRACE_EVENT(sched_process_fork,

	TP_PROTO(struct task_struct *parent, struct task_struct *child),

	TP_ARGS(parent, child),

	TP_STRUCT__entry(
		__array(	char,	parent_comm,	TASK_COMM_LEN	)
		__field(	pid_t,	parent_pid			)
		__array(	char,	child_comm,	TASK_COMM_LEN	)
		__field(	pid_t,	child_pid			)
	),

	TP_fast_assign(
		memcpy(__entry->parent_comm, parent->comm, TASK_COMM_LEN);
		__entry->parent_pid	= parent->pid;
		memcpy(__entry->child_comm, child->comm, TASK_COMM_LEN);
		__entry->child_pid	= child->pid;
	),

	TP_printk("comm=%s pid=%d child_comm=%s child_pid=%d",
		__entry->parent_comm, __entry->parent_pid,
		__entry->child_comm, __entry->child_pid)
);

/*
 * Tracepoint for exec:
 */
TRACE_EVENT(sched_process_exec,

	TP_PROTO(struct task_struct *p, pid_t old_pid,
		 struct linux_binprm *bprm),

	TP_ARGS(p, old_pid, bprm),

	TP_STRUCT__entry(
		__string(	filename,	bprm->filename	)
		__field(	pid_t,		pid		)
		__field(	pid_t,		old_pid		)
	),

	TP_fast_assign(
		__assign_str(filename, bprm->filename);
		__entry->pid		= p->pid;
		__entry->old_pid	= old_pid;
	),

	TP_printk("filename=%s pid=%d old_pid=%d", __get_str(filename),
		  __entry->pid, __entry->old_pid)
);

/*
 * XXX the below sched_stat tracepoints only apply to SCHED_OTHER/BATCH/IDLE
 *     adding sched_stat support to SCHED_FIFO/RR would be welcome.
 */
DECLARE_EVENT_CLASS(sched_stat_template,

	TP_PROTO(struct task_struct *tsk, u64 delay),

	TP_ARGS(__perf_task(tsk), __perf_count(delay)),

	TP_STRUCT__entry(
		__array( char,	comm,	TASK_COMM_LEN	)
		__field( pid_t,	pid			)
		__field( u64,	delay			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
		__entry->pid	= tsk->pid;
		__entry->delay	= delay;
	),

	TP_printk("comm=%s pid=%d delay=%Lu [ns]",
			__entry->comm, __entry->pid,
			(unsigned long long)__entry->delay)
);


/*
 * Tracepoint for accounting wait time (time the task is runnable
 * but not actually running due to scheduler contention).
 */
DEFINE_EVENT(sched_stat_template, sched_stat_wait,
	     TP_PROTO(struct task_struct *tsk, u64 delay),
	     TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting sleep time (time the task is not runnable,
 * including iowait, see below).
 */
DEFINE_EVENT(sched_stat_template, sched_stat_sleep,
	     TP_PROTO(struct task_struct *tsk, u64 delay),
	     TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting iowait time (time the task is not runnable
 * due to waiting on IO to complete).
 */
DEFINE_EVENT(sched_stat_template, sched_stat_iowait,
	     TP_PROTO(struct task_struct *tsk, u64 delay),
	     TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting blocked time (time the task is in uninterruptible).
 */
DEFINE_EVENT(sched_stat_template, sched_stat_blocked,
	     TP_PROTO(struct task_struct *tsk, u64 delay),
	     TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting runtime (time the task is executing
 * on a CPU).
 */
DECLARE_EVENT_CLASS(sched_stat_runtime,

	TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),

	TP_ARGS(tsk, __perf_count(runtime), vruntime),

	TP_STRUCT__entry(
		__array( char,	comm,	TASK_COMM_LEN	)
		__field( pid_t,	pid			)
		__field( u64,	runtime			)
		__field( u64,	vruntime			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
		__entry->pid		= tsk->pid;
		__entry->runtime	= runtime;
		__entry->vruntime	= vruntime;
	),

	TP_printk("comm=%s pid=%d runtime=%Lu [ns] vruntime=%Lu [ns]",
			__entry->comm, __entry->pid,
			(unsigned long long)__entry->runtime,
			(unsigned long long)__entry->vruntime)
);

DEFINE_EVENT(sched_stat_runtime, sched_stat_runtime,
	     TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),
	     TP_ARGS(tsk, runtime, vruntime));

/*
 * Tracepoint for showing priority inheritance modifying a tasks
 * priority.
 */
TRACE_EVENT(sched_pi_setprio,

	TP_PROTO(struct task_struct *tsk, struct task_struct *pi_task),

	TP_ARGS(tsk, pi_task),

	TP_STRUCT__entry(
		__array( char,	comm,	TASK_COMM_LEN	)
		__field( pid_t,	pid			)
		__field( int,	oldprio			)
		__field( int,	newprio			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
		__entry->pid		= tsk->pid;
		__entry->oldprio	= tsk->prio;
		__entry->newprio	= pi_task ?
				min(tsk->normal_prio, pi_task->prio) :
				tsk->normal_prio;
		/* XXX SCHED_DEADLINE bits missing */
	),

	TP_printk("comm=%s pid=%d oldprio=%d newprio=%d",
			__entry->comm, __entry->pid,
			__entry->oldprio, __entry->newprio)
);

#ifdef CONFIG_DETECT_HUNG_TASK
TRACE_EVENT(sched_process_hang,
	TP_PROTO(struct task_struct *tsk),
	TP_ARGS(tsk),

	TP_STRUCT__entry(
		__array( char,	comm,	TASK_COMM_LEN	)
		__field( pid_t,	pid			)
	),

	TP_fast_assign(
		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
		__entry->pid = tsk->pid;
	),

	TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
);
#endif /* CONFIG_DETECT_HUNG_TASK */

DECLARE_EVENT_CLASS(sched_move_task_template,

	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),

	TP_ARGS(tsk, src_cpu, dst_cpu),

	TP_STRUCT__entry(
		__field( pid_t,	pid			)
		__field( pid_t,	tgid			)
		__field( pid_t,	ngid			)
		__field( int,	src_cpu			)
		__field( int,	src_nid			)
		__field( int,	dst_cpu			)
		__field( int,	dst_nid			)
	),

	TP_fast_assign(
		__entry->pid		= task_pid_nr(tsk);
		__entry->tgid		= task_tgid_nr(tsk);
		__entry->ngid		= task_numa_group_id(tsk);
		__entry->src_cpu	= src_cpu;
		__entry->src_nid	= cpu_to_node(src_cpu);
		__entry->dst_cpu	= dst_cpu;
		__entry->dst_nid	= cpu_to_node(dst_cpu);
	),

	TP_printk("pid=%d tgid=%d ngid=%d src_cpu=%d src_nid=%d dst_cpu=%d dst_nid=%d",
			__entry->pid, __entry->tgid, __entry->ngid,
			__entry->src_cpu, __entry->src_nid,
			__entry->dst_cpu, __entry->dst_nid)
);

/*
 * Tracks migration of tasks from one runqueue to another. Can be used to
 * detect if automatic NUMA balancing is bouncing between nodes
 */
DEFINE_EVENT(sched_move_task_template, sched_move_numa,
	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),

	TP_ARGS(tsk, src_cpu, dst_cpu)
);

DEFINE_EVENT(sched_move_task_template, sched_stick_numa,
	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),

	TP_ARGS(tsk, src_cpu, dst_cpu)
);

TRACE_EVENT(sched_swap_numa,

	TP_PROTO(struct task_struct *src_tsk, int src_cpu,
		 struct task_struct *dst_tsk, int dst_cpu),

	TP_ARGS(src_tsk, src_cpu, dst_tsk, dst_cpu),

	TP_STRUCT__entry(
		__field( pid_t,	src_pid			)
		__field( pid_t,	src_tgid		)
		__field( pid_t,	src_ngid		)
		__field( int,	src_cpu			)
		__field( int,	src_nid			)
		__field( pid_t,	dst_pid			)
		__field( pid_t,	dst_tgid		)
		__field( pid_t,	dst_ngid		)
		__field( int,	dst_cpu			)
		__field( int,	dst_nid			)
	),

	TP_fast_assign(
		__entry->src_pid	= task_pid_nr(src_tsk);
		__entry->src_tgid	= task_tgid_nr(src_tsk);
		__entry->src_ngid	= task_numa_group_id(src_tsk);
		__entry->src_cpu	= src_cpu;
		__entry->src_nid	= cpu_to_node(src_cpu);
		__entry->dst_pid	= task_pid_nr(dst_tsk);
		__entry->dst_tgid	= task_tgid_nr(dst_tsk);
		__entry->dst_ngid	= task_numa_group_id(dst_tsk);
		__entry->dst_cpu	= dst_cpu;
		__entry->dst_nid	= cpu_to_node(dst_cpu);
	),

	TP_printk("src_pid=%d src_tgid=%d src_ngid=%d src_cpu=%d src_nid=%d dst_pid=%d dst_tgid=%d dst_ngid=%d dst_cpu=%d dst_nid=%d",
			__entry->src_pid, __entry->src_tgid, __entry->src_ngid,
			__entry->src_cpu, __entry->src_nid,
			__entry->dst_pid, __entry->dst_tgid, __entry->dst_ngid,
			__entry->dst_cpu, __entry->dst_nid)
);

/*
 * Tracepoint for waking a polling cpu without an IPI.
 */
TRACE_EVENT(sched_wake_idle_without_ipi,

	TP_PROTO(int cpu),

	TP_ARGS(cpu),

	TP_STRUCT__entry(
		__field(	int,	cpu	)
	),

	TP_fast_assign(
		__entry->cpu	= cpu;
	),

	TP_printk("cpu=%d", __entry->cpu)
);
#endif /* _TRACE_SCHED_H */

/* This part must be outside protection */
#include <trace/define_trace.h>
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
d0b6e04a LZ	2	#undef TRACE_SYSTEM
	3	#define TRACE_SYSTEM sched
	4
ea20d929	5	#if !defined(_TRACE_SCHED_H) \|\| defined(TRACE_HEADER_MULTI_READ)
0a16b607 MD	6	#define _TRACE_SCHED_H
0a16b607 MD	7
6a3827d7	8	#include <linux/sched/numa_balancing.h>
0a16b607	9	#include <linux/tracepoint.h>
4ff16c25	10	#include <linux/binfmts.h>
0a16b607	11
ea20d929 SR	12	/*
	13	* Tracepoint for calling kthread_stop, performed to end a kthread:
	14	*/
	15	TRACE_EVENT(sched_kthread_stop,
	16
	17	TP_PROTO(struct task_struct *t),
	18
	19	TP_ARGS(t),
	20
	21	TP_STRUCT__entry(
	22	__array( char, comm, TASK_COMM_LEN )
	23	__field( pid_t, pid )
	24	),
	25
	26	TP_fast_assign(
	27	memcpy(__entry->comm, t->comm, TASK_COMM_LEN);
	28	__entry->pid = t->pid;
	29	),
	30
434a83c3	31	TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
ea20d929 SR	32	);
	33
	34	/*
	35	* Tracepoint for the return value of the kthread stopping:
	36	*/
	37	TRACE_EVENT(sched_kthread_stop_ret,
	38
	39	TP_PROTO(int ret),
	40
	41	TP_ARGS(ret),
	42
	43	TP_STRUCT__entry(
	44	__field( int, ret )
	45	),
	46
	47	TP_fast_assign(
	48	__entry->ret = ret;
	49	),
	50
434a83c3	51	TP_printk("ret=%d", __entry->ret)
ea20d929 SR	52	);
ea20d929 SR	53
ea20d929 SR	54	/*
ea20d929 SR	55	* Tracepoint for waking up a task:
ea20d929	56	*/
091ad365	57	DECLARE_EVENT_CLASS(sched_wakeup_template,
ea20d929	58
fbd705a0	59	TP_PROTO(struct task_struct *p),
ea20d929	60
fbd705a0	61	TP_ARGS(__perf_task(p)),
ea20d929 SR	62
	63	TP_STRUCT__entry(
	64	__array( char, comm, TASK_COMM_LEN )
	65	__field( pid_t, pid )
	66	__field( int, prio )
	67	__field( int, success )
434a83c3	68	__field( int, target_cpu )
ea20d929 SR	69	),
	70
	71	TP_fast_assign(
	72	memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
	73	__entry->pid = p->pid;
b91473ff	74	__entry->prio = p->prio; /* XXX SCHED_DEADLINE */
fbd705a0	75	__entry->success = 1; /* rudiment, kill when possible */
434a83c3	76	__entry->target_cpu = task_cpu(p);
ea20d929 SR	77	),
ea20d929 SR	78
fbd705a0	79	TP_printk("comm=%s pid=%d prio=%d target_cpu=%03d",
ea20d929	80	__entry->comm, __entry->pid, __entry->prio,
fbd705a0	81	__entry->target_cpu)
ea20d929 SR	82	);
ea20d929 SR	83
fbd705a0 PZ	84	/*
	85	* Tracepoint called when waking a task; this tracepoint is guaranteed to be
	86	* called from the waking context.
	87	*/
	88	DEFINE_EVENT(sched_wakeup_template, sched_waking,
	89	TP_PROTO(struct task_struct *p),
	90	TP_ARGS(p));
	91
	92	/*
	93	* Tracepoint called when the task is actually woken; p->state == TASK_RUNNNG.
	94	* It it not always called from the waking context.
	95	*/
75ec29ab	96	DEFINE_EVENT(sched_wakeup_template, sched_wakeup,
fbd705a0 PZ	97	TP_PROTO(struct task_struct *p),
fbd705a0 PZ	98	TP_ARGS(p));
75ec29ab	99
ea20d929 SR	100	/*
ea20d929 SR	101	* Tracepoint for waking up a new task:
ea20d929	102	*/
75ec29ab	103	DEFINE_EVENT(sched_wakeup_template, sched_wakeup_new,
fbd705a0 PZ	104	TP_PROTO(struct task_struct *p),
fbd705a0 PZ	105	TP_ARGS(p));
ea20d929	106
02f72694	107	#ifdef CREATE_TRACE_POINTS
c73464b1	108	static inline long __trace_sched_switch_state(bool preempt, struct task_struct *p)
02f72694	109	{
fffc3d9c PK	110	unsigned int state;
fffc3d9c PK	111
8f9fbf09 ON	112	#ifdef CONFIG_SCHED_DEBUG
	113	BUG_ON(p != current);
	114	#endif /* CONFIG_SCHED_DEBUG */
c73464b1	115
02f72694	116	/*
c73464b1 PZ	117	* Preemption ignores task state, therefore preempted tasks are always
c73464b1 PZ	118	* RUNNING (we will not have dequeued if state != RUNNING).
02f72694	119	*/
efb40f58	120	if (preempt)
3f5fe9fe	121	return TASK_REPORT_MAX;
efb40f58	122
fffc3d9c PK	123	/*
	124	* task_state_index() uses fls() and returns a value from 0-8 range.
	125	* Decrement it by 1 (except TASK_RUNNING state i.e 0) before using
	126	* it for left shift operation to get the correct task->state
	127	* mapping.
	128	*/
	129	state = task_state_index(p);
	130
	131	return state ? (1 << (state - 1)) : state;
02f72694	132	}
8f9fbf09	133	#endif /* CREATE_TRACE_POINTS */
02f72694	134
ea20d929 SR	135	/*
ea20d929 SR	136	* Tracepoint for task switches, performed by the scheduler:
ea20d929 SR	137	*/
	138	TRACE_EVENT(sched_switch,
	139
c73464b1 PZ	140	TP_PROTO(bool preempt,
c73464b1 PZ	141	struct task_struct *prev,
ea20d929 SR	142	struct task_struct *next),
ea20d929 SR	143
c73464b1	144	TP_ARGS(preempt, prev, next),
ea20d929 SR	145
	146	TP_STRUCT__entry(
	147	__array( char, prev_comm, TASK_COMM_LEN )
	148	__field( pid_t, prev_pid )
	149	__field( int, prev_prio )
937cdb9d	150	__field( long, prev_state )
ea20d929 SR	151	__array( char, next_comm, TASK_COMM_LEN )
	152	__field( pid_t, next_pid )
	153	__field( int, next_prio )
	154	),
	155
	156	TP_fast_assign(
	157	memcpy(__entry->next_comm, next->comm, TASK_COMM_LEN);
	158	__entry->prev_pid = prev->pid;
	159	__entry->prev_prio = prev->prio;
c73464b1	160	__entry->prev_state = __trace_sched_switch_state(preempt, prev);
ea20d929 SR	161	memcpy(__entry->prev_comm, prev->comm, TASK_COMM_LEN);
	162	__entry->next_pid = next->pid;
	163	__entry->next_prio = next->prio;
b91473ff	164	/* XXX SCHED_DEADLINE */
ea20d929 SR	165	),
ea20d929 SR	166
557ab425	167	TP_printk("prev_comm=%s prev_pid=%d prev_prio=%d prev_state=%s%s ==> next_comm=%s next_pid=%d next_prio=%d",
ea20d929	168	__entry->prev_comm, __entry->prev_pid, __entry->prev_prio,
efb40f58	169
06eb6184 PZ	170	(__entry->prev_state & (TASK_REPORT_MAX - 1)) ?
06eb6184 PZ	171	__print_flags(__entry->prev_state & (TASK_REPORT_MAX - 1), "\|",
0d93c025 UKK	172	{ TASK_INTERRUPTIBLE, "S" },
	173	{ TASK_UNINTERRUPTIBLE, "D" },
	174	{ __TASK_STOPPED, "T" },
	175	{ __TASK_TRACED, "t" },
	176	{ EXIT_DEAD, "X" },
	177	{ EXIT_ZOMBIE, "Z" },
	178	{ TASK_PARKED, "P" },
	179	{ TASK_DEAD, "I" }) :
efb40f58 PZ	180	"R",
efb40f58 PZ	181
3f5fe9fe	182	__entry->prev_state & TASK_REPORT_MAX ? "+" : "",
ea20d929 SR	183	__entry->next_comm, __entry->next_pid, __entry->next_prio)
	184	);
	185
	186	/*
	187	* Tracepoint for a task being migrated:
	188	*/
	189	TRACE_EVENT(sched_migrate_task,
	190
de1d7286	191	TP_PROTO(struct task_struct *p, int dest_cpu),
ea20d929	192
de1d7286	193	TP_ARGS(p, dest_cpu),
ea20d929 SR	194
	195	TP_STRUCT__entry(
	196	__array( char, comm, TASK_COMM_LEN )
	197	__field( pid_t, pid )
	198	__field( int, prio )
	199	__field( int, orig_cpu )
	200	__field( int, dest_cpu )
	201	),
	202
	203	TP_fast_assign(
	204	memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
	205	__entry->pid = p->pid;
b91473ff	206	__entry->prio = p->prio; /* XXX SCHED_DEADLINE */
de1d7286	207	__entry->orig_cpu = task_cpu(p);
ea20d929 SR	208	__entry->dest_cpu = dest_cpu;
	209	),
	210
434a83c3	211	TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d",
ea20d929 SR	212	__entry->comm, __entry->pid, __entry->prio,
	213	__entry->orig_cpu, __entry->dest_cpu)
	214	);
	215
091ad365	216	DECLARE_EVENT_CLASS(sched_process_template,
ea20d929 SR	217
	218	TP_PROTO(struct task_struct *p),
	219
	220	TP_ARGS(p),
	221
	222	TP_STRUCT__entry(
	223	__array( char, comm, TASK_COMM_LEN )
	224	__field( pid_t, pid )
	225	__field( int, prio )
	226	),
	227
	228	TP_fast_assign(
	229	memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
	230	__entry->pid = p->pid;
b91473ff	231	__entry->prio = p->prio; /* XXX SCHED_DEADLINE */
ea20d929 SR	232	),
ea20d929 SR	233
434a83c3	234	TP_printk("comm=%s pid=%d prio=%d",
ea20d929 SR	235	__entry->comm, __entry->pid, __entry->prio)
	236	);
	237
	238	/*
75ec29ab	239	* Tracepoint for freeing a task:
ea20d929	240	*/
75ec29ab SR	241	DEFINE_EVENT(sched_process_template, sched_process_free,
	242	TP_PROTO(struct task_struct *p),
	243	TP_ARGS(p));
	244
ea20d929	245
75ec29ab SR	246	/*
	247	* Tracepoint for a task exiting:
	248	*/
	249	DEFINE_EVENT(sched_process_template, sched_process_exit,
	250	TP_PROTO(struct task_struct *p),
	251	TP_ARGS(p));
ea20d929	252
210f7669 LZ	253	/*
	254	* Tracepoint for waiting on task to unschedule:
	255	*/
	256	DEFINE_EVENT(sched_process_template, sched_wait_task,
	257	TP_PROTO(struct task_struct *p),
	258	TP_ARGS(p));
	259
ea20d929 SR	260	/*
	261	* Tracepoint for a waiting task:
	262	*/
	263	TRACE_EVENT(sched_process_wait,
	264
	265	TP_PROTO(struct pid *pid),
	266
	267	TP_ARGS(pid),
	268
	269	TP_STRUCT__entry(
	270	__array( char, comm, TASK_COMM_LEN )
	271	__field( pid_t, pid )
	272	__field( int, prio )
	273	),
	274
	275	TP_fast_assign(
	276	memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
	277	__entry->pid = pid_nr(pid);
b91473ff	278	__entry->prio = current->prio; /* XXX SCHED_DEADLINE */
ea20d929 SR	279	),
ea20d929 SR	280
434a83c3	281	TP_printk("comm=%s pid=%d prio=%d",
ea20d929 SR	282	__entry->comm, __entry->pid, __entry->prio)
	283	);
	284
	285	/*
	286	* Tracepoint for do_fork:
	287	*/
	288	TRACE_EVENT(sched_process_fork,
	289
	290	TP_PROTO(struct task_struct parent, struct task_struct child),
	291
	292	TP_ARGS(parent, child),
	293
	294	TP_STRUCT__entry(
	295	__array( char, parent_comm, TASK_COMM_LEN )
	296	__field( pid_t, parent_pid )
	297	__array( char, child_comm, TASK_COMM_LEN )
	298	__field( pid_t, child_pid )
	299	),
	300
	301	TP_fast_assign(
	302	memcpy(__entry->parent_comm, parent->comm, TASK_COMM_LEN);
	303	__entry->parent_pid = parent->pid;
	304	memcpy(__entry->child_comm, child->comm, TASK_COMM_LEN);
	305	__entry->child_pid = child->pid;
	306	),
	307
434a83c3	308	TP_printk("comm=%s pid=%d child_comm=%s child_pid=%d",
ea20d929 SR	309	__entry->parent_comm, __entry->parent_pid,
	310	__entry->child_comm, __entry->child_pid)
	311	);
	312
4ff16c25 DS	313	/*
	314	* Tracepoint for exec:
	315	*/
	316	TRACE_EVENT(sched_process_exec,
	317
	318	TP_PROTO(struct task_struct *p, pid_t old_pid,
	319	struct linux_binprm *bprm),
	320
	321	TP_ARGS(p, old_pid, bprm),
	322
	323	TP_STRUCT__entry(
	324	__string( filename, bprm->filename )
	325	__field( pid_t, pid )
	326	__field( pid_t, old_pid )
	327	),
	328
	329	TP_fast_assign(
	330	__assign_str(filename, bprm->filename);
	331	__entry->pid = p->pid;
6308191f	332	__entry->old_pid = old_pid;
4ff16c25 DS	333	),
	334
	335	TP_printk("filename=%s pid=%d old_pid=%d", __get_str(filename),
	336	__entry->pid, __entry->old_pid)
	337	);
	338
768d0c27 PZ	339	/*
	340	* XXX the below sched_stat tracepoints only apply to SCHED_OTHER/BATCH/IDLE
	341	* adding sched_stat support to SCHED_FIFO/RR would be welcome.
	342	*/
091ad365	343	DECLARE_EVENT_CLASS(sched_stat_template,
768d0c27 PZ	344
	345	TP_PROTO(struct task_struct *tsk, u64 delay),
	346
12473965	347	TP_ARGS(__perf_task(tsk), __perf_count(delay)),
768d0c27 PZ	348
	349	TP_STRUCT__entry(
	350	__array( char, comm, TASK_COMM_LEN )
	351	__field( pid_t, pid )
	352	__field( u64, delay )
	353	),
	354
	355	TP_fast_assign(
	356	memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
	357	__entry->pid = tsk->pid;
	358	__entry->delay = delay;
768d0c27 PZ	359	),
768d0c27 PZ	360
434a83c3	361	TP_printk("comm=%s pid=%d delay=%Lu [ns]",
768d0c27 PZ	362	__entry->comm, __entry->pid,
	363	(unsigned long long)__entry->delay)
	364	);
	365
75ec29ab SR	366
	367	/*
	368	* Tracepoint for accounting wait time (time the task is runnable
	369	* but not actually running due to scheduler contention).
	370	*/
	371	DEFINE_EVENT(sched_stat_template, sched_stat_wait,
	372	TP_PROTO(struct task_struct *tsk, u64 delay),
	373	TP_ARGS(tsk, delay));
	374
	375	/*
	376	* Tracepoint for accounting sleep time (time the task is not runnable,
	377	* including iowait, see below).
	378	*/
470dda74 LZ	379	DEFINE_EVENT(sched_stat_template, sched_stat_sleep,
	380	TP_PROTO(struct task_struct *tsk, u64 delay),
	381	TP_ARGS(tsk, delay));
75ec29ab SR	382
	383	/*
	384	* Tracepoint for accounting iowait time (time the task is not runnable
	385	* due to waiting on IO to complete).
	386	*/
470dda74 LZ	387	DEFINE_EVENT(sched_stat_template, sched_stat_iowait,
	388	TP_PROTO(struct task_struct *tsk, u64 delay),
	389	TP_ARGS(tsk, delay));
75ec29ab	390
b781a602 AV	391	/*
	392	* Tracepoint for accounting blocked time (time the task is in uninterruptible).
	393	*/
	394	DEFINE_EVENT(sched_stat_template, sched_stat_blocked,
	395	TP_PROTO(struct task_struct *tsk, u64 delay),
	396	TP_ARGS(tsk, delay));
	397
f977bb49 IM	398	/*
	399	* Tracepoint for accounting runtime (time the task is executing
	400	* on a CPU).
	401	*/
36009d07	402	DECLARE_EVENT_CLASS(sched_stat_runtime,
f977bb49 IM	403
	404	TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),
	405
12473965	406	TP_ARGS(tsk, __perf_count(runtime), vruntime),
f977bb49 IM	407
	408	TP_STRUCT__entry(
	409	__array( char, comm, TASK_COMM_LEN )
	410	__field( pid_t, pid )
	411	__field( u64, runtime )
	412	__field( u64, vruntime )
	413	),
	414
	415	TP_fast_assign(
	416	memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
	417	__entry->pid = tsk->pid;
	418	__entry->runtime = runtime;
	419	__entry->vruntime = vruntime;
f977bb49 IM	420	),
f977bb49 IM	421
434a83c3	422	TP_printk("comm=%s pid=%d runtime=%Lu [ns] vruntime=%Lu [ns]",
f977bb49 IM	423	__entry->comm, __entry->pid,
	424	(unsigned long long)__entry->runtime,
	425	(unsigned long long)__entry->vruntime)
	426	);
	427
36009d07 ON	428	DEFINE_EVENT(sched_stat_runtime, sched_stat_runtime,
	429	TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),
	430	TP_ARGS(tsk, runtime, vruntime));
	431
a8027073 SR	432	/*
	433	* Tracepoint for showing priority inheritance modifying a tasks
	434	* priority.
	435	*/
	436	TRACE_EVENT(sched_pi_setprio,
	437
b91473ff	438	TP_PROTO(struct task_struct tsk, struct task_struct pi_task),
a8027073	439
b91473ff	440	TP_ARGS(tsk, pi_task),
a8027073 SR	441
	442	TP_STRUCT__entry(
	443	__array( char, comm, TASK_COMM_LEN )
	444	__field( pid_t, pid )
	445	__field( int, oldprio )
	446	__field( int, newprio )
	447	),
	448
	449	TP_fast_assign(
	450	memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
	451	__entry->pid = tsk->pid;
	452	__entry->oldprio = tsk->prio;
e5ade162 SAS	453	__entry->newprio = pi_task ?
	454	min(tsk->normal_prio, pi_task->prio) :
	455	tsk->normal_prio;
b91473ff	456	/* XXX SCHED_DEADLINE bits missing */
a8027073 SR	457	),
	458
	459	TP_printk("comm=%s pid=%d oldprio=%d newprio=%d",
	460	__entry->comm, __entry->pid,
	461	__entry->oldprio, __entry->newprio)
	462	);
	463
6a716c90 ON	464	#ifdef CONFIG_DETECT_HUNG_TASK
	465	TRACE_EVENT(sched_process_hang,
	466	TP_PROTO(struct task_struct *tsk),
	467	TP_ARGS(tsk),
	468
	469	TP_STRUCT__entry(
	470	__array( char, comm, TASK_COMM_LEN )
	471	__field( pid_t, pid )
	472	),
	473
	474	TP_fast_assign(
	475	memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
	476	__entry->pid = tsk->pid;
	477	),
	478
	479	TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
	480	);
	481	#endif /* CONFIG_DETECT_HUNG_TASK */
	482
286549dc MG	483	DECLARE_EVENT_CLASS(sched_move_task_template,
	484
	485	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),
	486
	487	TP_ARGS(tsk, src_cpu, dst_cpu),
	488
	489	TP_STRUCT__entry(
	490	__field( pid_t, pid )
	491	__field( pid_t, tgid )
	492	__field( pid_t, ngid )
	493	__field( int, src_cpu )
	494	__field( int, src_nid )
	495	__field( int, dst_cpu )
	496	__field( int, dst_nid )
	497	),
	498
	499	TP_fast_assign(
	500	__entry->pid = task_pid_nr(tsk);
	501	__entry->tgid = task_tgid_nr(tsk);
	502	__entry->ngid = task_numa_group_id(tsk);
	503	__entry->src_cpu = src_cpu;
	504	__entry->src_nid = cpu_to_node(src_cpu);
	505	__entry->dst_cpu = dst_cpu;
	506	__entry->dst_nid = cpu_to_node(dst_cpu);
	507	),
	508
	509	TP_printk("pid=%d tgid=%d ngid=%d src_cpu=%d src_nid=%d dst_cpu=%d dst_nid=%d",
	510	__entry->pid, __entry->tgid, __entry->ngid,
	511	__entry->src_cpu, __entry->src_nid,
	512	__entry->dst_cpu, __entry->dst_nid)
	513	);
	514
	515	/*
	516	* Tracks migration of tasks from one runqueue to another. Can be used to
	517	* detect if automatic NUMA balancing is bouncing between nodes
	518	*/
	519	DEFINE_EVENT(sched_move_task_template, sched_move_numa,
	520	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),
	521
	522	TP_ARGS(tsk, src_cpu, dst_cpu)
	523	);
	524
	525	DEFINE_EVENT(sched_move_task_template, sched_stick_numa,
	526	TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),
	527
	528	TP_ARGS(tsk, src_cpu, dst_cpu)
	529	);
	530
	531	TRACE_EVENT(sched_swap_numa,
	532
	533	TP_PROTO(struct task_struct *src_tsk, int src_cpu,
	534	struct task_struct *dst_tsk, int dst_cpu),
	535
	536	TP_ARGS(src_tsk, src_cpu, dst_tsk, dst_cpu),
	537
	538	TP_STRUCT__entry(
	539	__field( pid_t, src_pid )
	540	__field( pid_t, src_tgid )
	541	__field( pid_t, src_ngid )
	542	__field( int, src_cpu )
	543	__field( int, src_nid )
	544	__field( pid_t, dst_pid )
	545	__field( pid_t, dst_tgid )
	546	__field( pid_t, dst_ngid )
547	__field( int, dst_cpu )
548	__field( int, dst_nid )
549	),
550
551	TP_fast_assign(
552	__entry->src_pid = task_pid_nr(src_tsk);
553	__entry->src_tgid = task_tgid_nr(src_tsk);
554	__entry->src_ngid = task_numa_group_id(src_tsk);
555	__entry->src_cpu = src_cpu;
556	__entry->src_nid = cpu_to_node(src_cpu);
557	__entry->dst_pid = task_pid_nr(dst_tsk);
558	__entry->dst_tgid = task_tgid_nr(dst_tsk);
559	__entry->dst_ngid = task_numa_group_id(dst_tsk);
560	__entry->dst_cpu = dst_cpu;
561	__entry->dst_nid = cpu_to_node(dst_cpu);
562	),
563
564	TP_printk("src_pid=%d src_tgid=%d src_ngid=%d src_cpu=%d src_nid=%d dst_pid=%d dst_tgid=%d dst_ngid=%d dst_cpu=%d dst_nid=%d",
565	__entry->src_pid, __entry->src_tgid, __entry->src_ngid,
566	__entry->src_cpu, __entry->src_nid,
567	__entry->dst_pid, __entry->dst_tgid, __entry->dst_ngid,
568	__entry->dst_cpu, __entry->dst_nid)
569	);
dfc68f29 AL	570
	571	/*
	572	* Tracepoint for waking a polling cpu without an IPI.
	573	*/
	574	TRACE_EVENT(sched_wake_idle_without_ipi,
	575
	576	TP_PROTO(int cpu),
	577
	578	TP_ARGS(cpu),
	579
	580	TP_STRUCT__entry(
	581	__field( int, cpu )
	582	),
	583
	584	TP_fast_assign(
	585	__entry->cpu = cpu;
	586	),
	587
	588	TP_printk("cpu=%d", __entry->cpu)
	589	);
ea20d929	590	#endif /* _TRACE_SCHED_H */
a8d154b0 SR	591
	592	/* This part must be outside protection */
	593	#include <trace/define_trace.h>