[mirror_ubuntu-jammy-kernel.git] / include / trace / events / rcu.h

#undef TRACE_SYSTEM
#define TRACE_SYSTEM rcu

#if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_RCU_H

#include <linux/tracepoint.h>

/*
 * Tracepoint for start/end markers used for utilization calculations.
 * By convention, the string is of the following forms:
 *
 * "Start <activity>" -- Mark the start of the specified activity,
 *			 such as "context switch".  Nesting is permitted.
 * "End <activity>" -- Mark the end of the specified activity.
 *
 * An "@" character within "<activity>" is a comment character: Data
 * reduction scripts will ignore the "@" and the remainder of the line.
 */
TRACE_EVENT(rcu_utilization,

	TP_PROTO(char *s),

	TP_ARGS(s),

	TP_STRUCT__entry(
		__field(char *, s)
	),

	TP_fast_assign(
		__entry->s = s;
	),

	TP_printk("%s", __entry->s)
);

#ifdef CONFIG_RCU_TRACE

#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)

/*
 * Tracepoint for grace-period events: starting and ending a grace
 * period ("start" and "end", respectively), a CPU noting the start
 * of a new grace period or the end of an old grace period ("cpustart"
 * and "cpuend", respectively), a CPU passing through a quiescent
 * state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
 * and "cpuofl", respectively), and a CPU being kicked for being too
 * long in dyntick-idle mode ("kick").
 */
TRACE_EVENT(rcu_grace_period,

	TP_PROTO(char *rcuname, unsigned long gpnum, char *gpevent),

	TP_ARGS(rcuname, gpnum, gpevent),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(char *, gpevent)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->gpevent = gpevent;
	),

	TP_printk("%s %lu %s",
		  __entry->rcuname, __entry->gpnum, __entry->gpevent)
);

/*
 * Tracepoint for grace-period-initialization events.  These are
 * distinguished by the type of RCU, the new grace-period number, the
 * rcu_node structure level, the starting and ending CPU covered by the
 * rcu_node structure, and the mask of CPUs that will be waited for.
 * All but the type of RCU are extracted from the rcu_node structure.
 */
TRACE_EVENT(rcu_grace_period_init,

	TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
		 int grplo, int grphi, unsigned long qsmask),

	TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(u8, level)
		__field(int, grplo)
		__field(int, grphi)
		__field(unsigned long, qsmask)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->level = level;
		__entry->grplo = grplo;
		__entry->grphi = grphi;
		__entry->qsmask = qsmask;
	),

	TP_printk("%s %lu %u %d %d %lx",
		  __entry->rcuname, __entry->gpnum, __entry->level,
		  __entry->grplo, __entry->grphi, __entry->qsmask)
);

/*
 * Tracepoint for tasks blocking within preemptible-RCU read-side
 * critical sections.  Track the type of RCU (which one day might
 * include SRCU), the grace-period number that the task is blocking
 * (the current or the next), and the task's PID.
 */
TRACE_EVENT(rcu_preempt_task,

	TP_PROTO(char *rcuname, int pid, unsigned long gpnum),

	TP_ARGS(rcuname, pid, gpnum),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(int, pid)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->pid = pid;
	),

	TP_printk("%s %lu %d",
		  __entry->rcuname, __entry->gpnum, __entry->pid)
);

/*
 * Tracepoint for tasks that blocked within a given preemptible-RCU
 * read-side critical section exiting that critical section.  Track the
 * type of RCU (which one day might include SRCU) and the task's PID.
 */
TRACE_EVENT(rcu_unlock_preempted_task,

	TP_PROTO(char *rcuname, unsigned long gpnum, int pid),

	TP_ARGS(rcuname, gpnum, pid),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(int, pid)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->pid = pid;
	),

	TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
);

/*
 * Tracepoint for quiescent-state-reporting events.  These are
 * distinguished by the type of RCU, the grace-period number, the
 * mask of quiescent lower-level entities, the rcu_node structure level,
 * the starting and ending CPU covered by the rcu_node structure, and
 * whether there are any blocked tasks blocking the current grace period.
 * All but the type of RCU are extracted from the rcu_node structure.
 */
TRACE_EVENT(rcu_quiescent_state_report,

	TP_PROTO(char *rcuname, unsigned long gpnum,
		 unsigned long mask, unsigned long qsmask,
		 u8 level, int grplo, int grphi, int gp_tasks),

	TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(unsigned long, mask)
		__field(unsigned long, qsmask)
		__field(u8, level)
		__field(int, grplo)
		__field(int, grphi)
		__field(u8, gp_tasks)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->mask = mask;
		__entry->qsmask = qsmask;
		__entry->level = level;
		__entry->grplo = grplo;
		__entry->grphi = grphi;
		__entry->gp_tasks = gp_tasks;
	),

	TP_printk("%s %lu %lx>%lx %u %d %d %u",
		  __entry->rcuname, __entry->gpnum,
		  __entry->mask, __entry->qsmask, __entry->level,
		  __entry->grplo, __entry->grphi, __entry->gp_tasks)
);

/*
 * Tracepoint for quiescent states detected by force_quiescent_state().
 * These trace events include the type of RCU, the grace-period number
 * that was blocked by the CPU, the CPU itself, and the type of quiescent
 * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
 * or "kick" when kicking a CPU that has been in dyntick-idle mode for
 * too long.
 */
TRACE_EVENT(rcu_fqs,

	TP_PROTO(char *rcuname, unsigned long gpnum, int cpu, char *qsevent),

	TP_ARGS(rcuname, gpnum, cpu, qsevent),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(unsigned long, gpnum)
		__field(int, cpu)
		__field(char *, qsevent)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->gpnum = gpnum;
		__entry->cpu = cpu;
		__entry->qsevent = qsevent;
	),

	TP_printk("%s %lu %d %s",
		  __entry->rcuname, __entry->gpnum,
		  __entry->cpu, __entry->qsevent)
);

#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) */

/*
 * Tracepoint for dyntick-idle entry/exit events.  These take a string
 * as argument: "Start" for entering dyntick-idle mode, "End" for
 * leaving it, "--=" for events moving towards idle, and "++=" for events
 * moving away from idle.  "Error on entry: not idle task" and "Error on
 * exit: not idle task" indicate that a non-idle task is erroneously
 * toying with the idle loop.
 *
 * These events also take a pair of numbers, which indicate the nesting
 * depth before and after the event of interest.  Note that task-related
 * events use the upper bits of each number, while interrupt-related
 * events use the lower bits.
 */
TRACE_EVENT(rcu_dyntick,

	TP_PROTO(char *polarity, long long oldnesting, long long newnesting),

	TP_ARGS(polarity, oldnesting, newnesting),

	TP_STRUCT__entry(
		__field(char *, polarity)
		__field(long long, oldnesting)
		__field(long long, newnesting)
	),

	TP_fast_assign(
		__entry->polarity = polarity;
		__entry->oldnesting = oldnesting;
		__entry->newnesting = newnesting;
	),

	TP_printk("%s %llx %llx", __entry->polarity,
		  __entry->oldnesting, __entry->newnesting)
);

/*
 * Tracepoint for RCU preparation for idle, the goal being to get RCU
 * processing done so that the current CPU can shut off its scheduling
 * clock and enter dyntick-idle mode.  One way to accomplish this is
 * to drain all RCU callbacks from this CPU, and the other is to have
 * done everything RCU requires for the current grace period.  In this
 * latter case, the CPU will be awakened at the end of the current grace
 * period in order to process the remainder of its callbacks.
 *
 * These tracepoints take a string as argument:
 *
 *	"No callbacks": Nothing to do, no callbacks on this CPU.
 *	"In holdoff": Nothing to do, holding off after unsuccessful attempt.
 *	"Begin holdoff": Attempt failed, don't retry until next jiffy.
 *	"Dyntick with callbacks": Entering dyntick-idle despite callbacks.
 *	"More callbacks": Still more callbacks, try again to clear them out.
 *	"Callbacks drained": All callbacks processed, off to dyntick idle!
 *	"Timer": Timer fired to cause CPU to continue processing callbacks.
 */
TRACE_EVENT(rcu_prep_idle,

	TP_PROTO(char *reason),

	TP_ARGS(reason),

	TP_STRUCT__entry(
		__field(char *, reason)
	),

	TP_fast_assign(
		__entry->reason = reason;
	),

	TP_printk("%s", __entry->reason)
);

/*
 * Tracepoint for the registration of a single RCU callback function.
 * The first argument is the type of RCU, the second argument is
 * a pointer to the RCU callback itself, and the third element is the
 * new RCU callback queue length for the current CPU.
 */
TRACE_EVENT(rcu_callback,

	TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),

	TP_ARGS(rcuname, rhp, qlen),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(void *, rhp)
		__field(void *, func)
		__field(long, qlen)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->rhp = rhp;
		__entry->func = rhp->func;
		__entry->qlen = qlen;
	),

	TP_printk("%s rhp=%p func=%pf %ld",
		  __entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)
);

/*
 * Tracepoint for the registration of a single RCU callback of the special
 * kfree() form.  The first argument is the RCU type, the second argument
 * is a pointer to the RCU callback, the third argument is the offset
 * of the callback within the enclosing RCU-protected data structure,
 * and the fourth argument is the new RCU callback queue length for the
 * current CPU.
 */
TRACE_EVENT(rcu_kfree_callback,

	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
		 long qlen),

	TP_ARGS(rcuname, rhp, offset, qlen),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(void *, rhp)
		__field(unsigned long, offset)
		__field(long, qlen)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->rhp = rhp;
		__entry->offset = offset;
		__entry->qlen = qlen;
	),

	TP_printk("%s rhp=%p func=%ld %ld",
		  __entry->rcuname, __entry->rhp, __entry->offset,
		  __entry->qlen)
);

/*
 * Tracepoint for marking the beginning rcu_do_batch, performed to start
 * RCU callback invocation.  The first argument is the RCU flavor,
 * the second is the total number of callbacks (including those that
 * are not yet ready to be invoked), and the third argument is the
 * current RCU-callback batch limit.
 */
TRACE_EVENT(rcu_batch_start,

	TP_PROTO(char *rcuname, long qlen, int blimit),

	TP_ARGS(rcuname, qlen, blimit),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(long, qlen)
		__field(int, blimit)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->qlen = qlen;
		__entry->blimit = blimit;
	),

	TP_printk("%s CBs=%ld bl=%d",
		  __entry->rcuname, __entry->qlen, __entry->blimit)
);

/*
 * Tracepoint for the invocation of a single RCU callback function.
 * The first argument is the type of RCU, and the second argument is
 * a pointer to the RCU callback itself.
 */
TRACE_EVENT(rcu_invoke_callback,

	TP_PROTO(char *rcuname, struct rcu_head *rhp),

	TP_ARGS(rcuname, rhp),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(void *, rhp)
		__field(void *, func)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->rhp = rhp;
		__entry->func = rhp->func;
	),

	TP_printk("%s rhp=%p func=%pf",
		  __entry->rcuname, __entry->rhp, __entry->func)
);

/*
 * Tracepoint for the invocation of a single RCU callback of the special
 * kfree() form.  The first argument is the RCU flavor, the second
 * argument is a pointer to the RCU callback, and the third argument
 * is the offset of the callback within the enclosing RCU-protected
 * data structure.
 */
TRACE_EVENT(rcu_invoke_kfree_callback,

	TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset),

	TP_ARGS(rcuname, rhp, offset),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(void *, rhp)
		__field(unsigned long, offset)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->rhp = rhp;
		__entry->offset	= offset;
	),

	TP_printk("%s rhp=%p func=%ld",
		  __entry->rcuname, __entry->rhp, __entry->offset)
);

/*
 * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
 * invoked.  The first argument is the name of the RCU flavor and
 * the second argument is number of callbacks actually invoked.
 */
TRACE_EVENT(rcu_batch_end,

	TP_PROTO(char *rcuname, int callbacks_invoked),

	TP_ARGS(rcuname, callbacks_invoked),

	TP_STRUCT__entry(
		__field(char *, rcuname)
		__field(int, callbacks_invoked)
	),

	TP_fast_assign(
		__entry->rcuname = rcuname;
		__entry->callbacks_invoked = callbacks_invoked;
	),

	TP_printk("%s CBs-invoked=%d",
		  __entry->rcuname, __entry->callbacks_invoked)
);

/*
 * Tracepoint for rcutorture readers.  The first argument is the name
 * of the RCU flavor from rcutorture's viewpoint and the second argument
 * is the callback address.
 */
TRACE_EVENT(rcu_torture_read,

	TP_PROTO(char *rcutorturename, struct rcu_head *rhp),

	TP_ARGS(rcutorturename, rhp),

	TP_STRUCT__entry(
		__field(char *, rcutorturename)
		__field(struct rcu_head *, rhp)
	),

	TP_fast_assign(
		__entry->rcutorturename = rcutorturename;
		__entry->rhp = rhp;
	),

	TP_printk("%s torture read %p",
		  __entry->rcutorturename, __entry->rhp)
);

#else /* #ifdef CONFIG_RCU_TRACE */

#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)
#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)
#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)
#define trace_rcu_prep_idle(reason) do { } while (0)
#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
#define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)

#endif /* #else #ifdef CONFIG_RCU_TRACE */

#endif /* _TRACE_RCU_H */

/* This part must be outside protection */
#include <trace/define_trace.h>
Commit	Line	Data
29c00b4a PM	1	#undef TRACE_SYSTEM
	2	#define TRACE_SYSTEM rcu
	3
	4	#if !defined(_TRACE_RCU_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	5	#define _TRACE_RCU_H
	6
	7	#include <linux/tracepoint.h>
	8
	9	/*
300df91c PM	10	* Tracepoint for start/end markers used for utilization calculations.
	11	* By convention, the string is of the following forms:
	12	*
	13	* "Start <activity>" -- Mark the start of the specified activity,
	14	* such as "context switch". Nesting is permitted.
	15	* "End <activity>" -- Mark the end of the specified activity.
385680a9 PM	16	*
	17	* An "@" character within "<activity>" is a comment character: Data
	18	* reduction scripts will ignore the "@" and the remainder of the line.
300df91c PM	19	*/
	20	TRACE_EVENT(rcu_utilization,
	21
	22	TP_PROTO(char *s),
	23
	24	TP_ARGS(s),
	25
	26	TP_STRUCT__entry(
d4c08f2a	27	__field(char *, s)
300df91c PM	28	),
	29
	30	TP_fast_assign(
	31	__entry->s = s;
	32	),
	33
	34	TP_printk("%s", __entry->s)
	35	);
	36
d4c08f2a PM	37	#ifdef CONFIG_RCU_TRACE
	38
	39	#if defined(CONFIG_TREE_RCU) \|\| defined(CONFIG_TREE_PREEMPT_RCU)
	40
	41	/*
	42	* Tracepoint for grace-period events: starting and ending a grace
	43	* period ("start" and "end", respectively), a CPU noting the start
	44	* of a new grace period or the end of an old grace period ("cpustart"
	45	* and "cpuend", respectively), a CPU passing through a quiescent
	46	* state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
	47	* and "cpuofl", respectively), and a CPU being kicked for being too
	48	* long in dyntick-idle mode ("kick").
	49	*/
	50	TRACE_EVENT(rcu_grace_period,
	51
	52	TP_PROTO(char rcuname, unsigned long gpnum, char gpevent),
	53
	54	TP_ARGS(rcuname, gpnum, gpevent),
	55
	56	TP_STRUCT__entry(
	57	__field(char *, rcuname)
	58	__field(unsigned long, gpnum)
	59	__field(char *, gpevent)
	60	),
	61
	62	TP_fast_assign(
	63	__entry->rcuname = rcuname;
	64	__entry->gpnum = gpnum;
	65	__entry->gpevent = gpevent;
	66	),
	67
	68	TP_printk("%s %lu %s",
	69	__entry->rcuname, __entry->gpnum, __entry->gpevent)
	70	);
	71
	72	/*
	73	* Tracepoint for grace-period-initialization events. These are
	74	* distinguished by the type of RCU, the new grace-period number, the
	75	* rcu_node structure level, the starting and ending CPU covered by the
	76	* rcu_node structure, and the mask of CPUs that will be waited for.
	77	* All but the type of RCU are extracted from the rcu_node structure.
	78	*/
	79	TRACE_EVENT(rcu_grace_period_init,
	80
	81	TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
	82	int grplo, int grphi, unsigned long qsmask),
	83
	84	TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
	85
	86	TP_STRUCT__entry(
	87	__field(char *, rcuname)
	88	__field(unsigned long, gpnum)
	89	__field(u8, level)
	90	__field(int, grplo)
	91	__field(int, grphi)
	92	__field(unsigned long, qsmask)
	93	),
	94
	95	TP_fast_assign(
	96	__entry->rcuname = rcuname;
	97	__entry->gpnum = gpnum;
	98	__entry->level = level;
	99	__entry->grplo = grplo;
	100	__entry->grphi = grphi;
101	__entry->qsmask = qsmask;
102	),
103
104	TP_printk("%s %lu %u %d %d %lx",
105	__entry->rcuname, __entry->gpnum, __entry->level,
106	__entry->grplo, __entry->grphi, __entry->qsmask)
107	);
108
109	/*
110	* Tracepoint for tasks blocking within preemptible-RCU read-side
111	* critical sections. Track the type of RCU (which one day might
112	* include SRCU), the grace-period number that the task is blocking
113	* (the current or the next), and the task's PID.
114	*/
115	TRACE_EVENT(rcu_preempt_task,
116
117	TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
118
119	TP_ARGS(rcuname, pid, gpnum),
120
121	TP_STRUCT__entry(
122	__field(char *, rcuname)
123	__field(unsigned long, gpnum)
124	__field(int, pid)
125	),
126
127	TP_fast_assign(
128	__entry->rcuname = rcuname;
129	__entry->gpnum = gpnum;
130	__entry->pid = pid;
131	),
132
133	TP_printk("%s %lu %d",
134	__entry->rcuname, __entry->gpnum, __entry->pid)
135	);
136
137	/*
138	* Tracepoint for tasks that blocked within a given preemptible-RCU
139	* read-side critical section exiting that critical section. Track the
140	* type of RCU (which one day might include SRCU) and the task's PID.
141	*/
142	TRACE_EVENT(rcu_unlock_preempted_task,
143
144	TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
145
146	TP_ARGS(rcuname, gpnum, pid),
147
148	TP_STRUCT__entry(
149	__field(char *, rcuname)
150	__field(unsigned long, gpnum)
151	__field(int, pid)
152	),
153
154	TP_fast_assign(
155	__entry->rcuname = rcuname;
156	__entry->gpnum = gpnum;
157	__entry->pid = pid;
158	),
159
160	TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
161	);
162
163	/*
164	* Tracepoint for quiescent-state-reporting events. These are
165	* distinguished by the type of RCU, the grace-period number, the
166	* mask of quiescent lower-level entities, the rcu_node structure level,
167	* the starting and ending CPU covered by the rcu_node structure, and
168	* whether there are any blocked tasks blocking the current grace period.
169	* All but the type of RCU are extracted from the rcu_node structure.
170	*/
171	TRACE_EVENT(rcu_quiescent_state_report,
172
173	TP_PROTO(char *rcuname, unsigned long gpnum,
174	unsigned long mask, unsigned long qsmask,
175	u8 level, int grplo, int grphi, int gp_tasks),
176
177	TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
178
179	TP_STRUCT__entry(
180	__field(char *, rcuname)
181	__field(unsigned long, gpnum)
182	__field(unsigned long, mask)
183	__field(unsigned long, qsmask)
184	__field(u8, level)
185	__field(int, grplo)
186	__field(int, grphi)
187	__field(u8, gp_tasks)
188	),
189
190	TP_fast_assign(
191	__entry->rcuname = rcuname;
192	__entry->gpnum = gpnum;
193	__entry->mask = mask;
194	__entry->qsmask = qsmask;
195	__entry->level = level;
196	__entry->grplo = grplo;
197	__entry->grphi = grphi;
198	__entry->gp_tasks = gp_tasks;
199	),
200
201	TP_printk("%s %lu %lx>%lx %u %d %d %u",
202	__entry->rcuname, __entry->gpnum,
203	__entry->mask, __entry->qsmask, __entry->level,
204	__entry->grplo, __entry->grphi, __entry->gp_tasks)
205	);
206
207	/*
208	* Tracepoint for quiescent states detected by force_quiescent_state().
209	* These trace events include the type of RCU, the grace-period number
210	* that was blocked by the CPU, the CPU itself, and the type of quiescent
211	* state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
212	* or "kick" when kicking a CPU that has been in dyntick-idle mode for
213	* too long.
214	*/
215	TRACE_EVENT(rcu_fqs,
216
217	TP_PROTO(char rcuname, unsigned long gpnum, int cpu, char qsevent),
218
219	TP_ARGS(rcuname, gpnum, cpu, qsevent),
220
221	TP_STRUCT__entry(
222	__field(char *, rcuname)
223	__field(unsigned long, gpnum)
224	__field(int, cpu)
225	__field(char *, qsevent)
226	),
227
228	TP_fast_assign(
229	__entry->rcuname = rcuname;
230	__entry->gpnum = gpnum;
231	__entry->cpu = cpu;
232	__entry->qsevent = qsevent;
233	),
234
235	TP_printk("%s %lu %d %s",
236	__entry->rcuname, __entry->gpnum,
237	__entry->cpu, __entry->qsevent)
238	);
239
240	#endif /* #if defined(CONFIG_TREE_RCU) \|\| defined(CONFIG_TREE_PREEMPT_RCU) */
241
242	/*
243	* Tracepoint for dyntick-idle entry/exit events. These take a string
045fb931 PM	244	* as argument: "Start" for entering dyntick-idle mode, "End" for
	245	* leaving it, "--=" for events moving towards idle, and "++=" for events
	246	* moving away from idle. "Error on entry: not idle task" and "Error on
	247	* exit: not idle task" indicate that a non-idle task is erroneously
	248	* toying with the idle loop.
	249	*
	250	* These events also take a pair of numbers, which indicate the nesting
	251	* depth before and after the event of interest. Note that task-related
	252	* events use the upper bits of each number, while interrupt-related
	253	* events use the lower bits.
d4c08f2a PM	254	*/
	255	TRACE_EVENT(rcu_dyntick,
	256
4145fa7f	257	TP_PROTO(char *polarity, long long oldnesting, long long newnesting),
d4c08f2a	258
4145fa7f	259	TP_ARGS(polarity, oldnesting, newnesting),
d4c08f2a PM	260
	261	TP_STRUCT__entry(
	262	__field(char *, polarity)
4145fa7f PM	263	__field(long long, oldnesting)
4145fa7f PM	264	__field(long long, newnesting)
d4c08f2a PM	265	),
	266
	267	TP_fast_assign(
	268	__entry->polarity = polarity;
4145fa7f PM	269	__entry->oldnesting = oldnesting;
4145fa7f PM	270	__entry->newnesting = newnesting;
d4c08f2a PM	271	),
d4c08f2a PM	272
4145fa7f PM	273	TP_printk("%s %llx %llx", __entry->polarity,
4145fa7f PM	274	__entry->oldnesting, __entry->newnesting)
d4c08f2a PM	275	);
d4c08f2a PM	276
433cdddc PM	277	/*
	278	* Tracepoint for RCU preparation for idle, the goal being to get RCU
	279	* processing done so that the current CPU can shut off its scheduling
	280	* clock and enter dyntick-idle mode. One way to accomplish this is
	281	* to drain all RCU callbacks from this CPU, and the other is to have
	282	* done everything RCU requires for the current grace period. In this
	283	* latter case, the CPU will be awakened at the end of the current grace
	284	* period in order to process the remainder of its callbacks.
	285	*
	286	* These tracepoints take a string as argument:
	287	*
	288	* "No callbacks": Nothing to do, no callbacks on this CPU.
	289	* "In holdoff": Nothing to do, holding off after unsuccessful attempt.
433cdddc	290	* "Begin holdoff": Attempt failed, don't retry until next jiffy.
7cb92499	291	* "Dyntick with callbacks": Entering dyntick-idle despite callbacks.
433cdddc PM	292	* "More callbacks": Still more callbacks, try again to clear them out.
433cdddc PM	293	* "Callbacks drained": All callbacks processed, off to dyntick idle!
7cb92499	294	* "Timer": Timer fired to cause CPU to continue processing callbacks.
433cdddc PM	295	*/
	296	TRACE_EVENT(rcu_prep_idle,
	297
	298	TP_PROTO(char *reason),
	299
	300	TP_ARGS(reason),
	301
	302	TP_STRUCT__entry(
	303	__field(char *, reason)
	304	),
	305
	306	TP_fast_assign(
	307	__entry->reason = reason;
	308	),
	309
	310	TP_printk("%s", __entry->reason)
	311	);
	312
d4c08f2a PM	313	/*
	314	* Tracepoint for the registration of a single RCU callback function.
	315	* The first argument is the type of RCU, the second argument is
	316	* a pointer to the RCU callback itself, and the third element is the
	317	* new RCU callback queue length for the current CPU.
	318	*/
	319	TRACE_EVENT(rcu_callback,
	320
	321	TP_PROTO(char rcuname, struct rcu_head rhp, long qlen),
	322
	323	TP_ARGS(rcuname, rhp, qlen),
	324
	325	TP_STRUCT__entry(
	326	__field(char *, rcuname)
	327	__field(void *, rhp)
	328	__field(void *, func)
	329	__field(long, qlen)
	330	),
	331
	332	TP_fast_assign(
	333	__entry->rcuname = rcuname;
	334	__entry->rhp = rhp;
	335	__entry->func = rhp->func;
	336	__entry->qlen = qlen;
	337	),
	338
	339	TP_printk("%s rhp=%p func=%pf %ld",
	340	__entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)
	341	);
	342
	343	/*
	344	* Tracepoint for the registration of a single RCU callback of the special
	345	* kfree() form. The first argument is the RCU type, the second argument
	346	* is a pointer to the RCU callback, the third argument is the offset
	347	* of the callback within the enclosing RCU-protected data structure,
	348	* and the fourth argument is the new RCU callback queue length for the
	349	* current CPU.
	350	*/
	351	TRACE_EVENT(rcu_kfree_callback,
	352
	353	TP_PROTO(char rcuname, struct rcu_head rhp, unsigned long offset,
	354	long qlen),
	355
	356	TP_ARGS(rcuname, rhp, offset, qlen),
	357
	358	TP_STRUCT__entry(
	359	__field(char *, rcuname)
	360	__field(void *, rhp)
	361	__field(unsigned long, offset)
	362	__field(long, qlen)
	363	),
	364
	365	TP_fast_assign(
	366	__entry->rcuname = rcuname;
	367	__entry->rhp = rhp;
	368	__entry->offset = offset;
	369	__entry->qlen = qlen;
	370	),
	371
	372	TP_printk("%s rhp=%p func=%ld %ld",
	373	__entry->rcuname, __entry->rhp, __entry->offset,
	374	__entry->qlen)
	375	);
	376
300df91c PM	377	/*
300df91c PM	378	* Tracepoint for marking the beginning rcu_do_batch, performed to start
72fe701b PM	379	* RCU callback invocation. The first argument is the RCU flavor,
	380	* the second is the total number of callbacks (including those that
	381	* are not yet ready to be invoked), and the third argument is the
	382	* current RCU-callback batch limit.
29c00b4a PM	383	*/
	384	TRACE_EVENT(rcu_batch_start,
	385
72fe701b	386	TP_PROTO(char *rcuname, long qlen, int blimit),
29c00b4a	387
72fe701b	388	TP_ARGS(rcuname, qlen, blimit),
29c00b4a PM	389
29c00b4a PM	390	TP_STRUCT__entry(
72fe701b	391	__field(char *, rcuname)
300df91c PM	392	__field(long, qlen)
300df91c PM	393	__field(int, blimit)
29c00b4a PM	394	),
	395
	396	TP_fast_assign(
72fe701b	397	__entry->rcuname = rcuname;
300df91c PM	398	__entry->qlen = qlen;
300df91c PM	399	__entry->blimit = blimit;
29c00b4a PM	400	),
29c00b4a PM	401
72fe701b PM	402	TP_printk("%s CBs=%ld bl=%d",
72fe701b PM	403	__entry->rcuname, __entry->qlen, __entry->blimit)
29c00b4a PM	404	);
	405
	406	/*
300df91c	407	* Tracepoint for the invocation of a single RCU callback function.
d4c08f2a PM	408	* The first argument is the type of RCU, and the second argument is
d4c08f2a PM	409	* a pointer to the RCU callback itself.
29c00b4a PM	410	*/
	411	TRACE_EVENT(rcu_invoke_callback,
	412
d4c08f2a	413	TP_PROTO(char rcuname, struct rcu_head rhp),
29c00b4a	414
d4c08f2a	415	TP_ARGS(rcuname, rhp),
29c00b4a PM	416
29c00b4a PM	417	TP_STRUCT__entry(
d4c08f2a PM	418	__field(char *, rcuname)
	419	__field(void *, rhp)
	420	__field(void *, func)
29c00b4a PM	421	),
	422
	423	TP_fast_assign(
d4c08f2a	424	__entry->rcuname = rcuname;
300df91c PM	425	__entry->rhp = rhp;
300df91c PM	426	__entry->func = rhp->func;
29c00b4a PM	427	),
29c00b4a PM	428
d4c08f2a PM	429	TP_printk("%s rhp=%p func=%pf",
d4c08f2a PM	430	__entry->rcuname, __entry->rhp, __entry->func)
29c00b4a PM	431	);
	432
	433	/*
300df91c	434	* Tracepoint for the invocation of a single RCU callback of the special
d4c08f2a PM	435	* kfree() form. The first argument is the RCU flavor, the second
	436	* argument is a pointer to the RCU callback, and the third argument
	437	* is the offset of the callback within the enclosing RCU-protected
	438	* data structure.
29c00b4a PM	439	*/
	440	TRACE_EVENT(rcu_invoke_kfree_callback,
	441
d4c08f2a	442	TP_PROTO(char rcuname, struct rcu_head rhp, unsigned long offset),
29c00b4a	443
d4c08f2a	444	TP_ARGS(rcuname, rhp, offset),
29c00b4a PM	445
29c00b4a PM	446	TP_STRUCT__entry(
d4c08f2a PM	447	__field(char *, rcuname)
d4c08f2a PM	448	__field(void *, rhp)
300df91c	449	__field(unsigned long, offset)
29c00b4a PM	450	),
	451
	452	TP_fast_assign(
d4c08f2a	453	__entry->rcuname = rcuname;
300df91c	454	__entry->rhp = rhp;
29c00b4a PM	455	__entry->offset = offset;
	456	),
	457
d4c08f2a PM	458	TP_printk("%s rhp=%p func=%ld",
d4c08f2a PM	459	__entry->rcuname, __entry->rhp, __entry->offset)
29c00b4a PM	460	);
	461
	462	/*
300df91c	463	* Tracepoint for exiting rcu_do_batch after RCU callbacks have been
72fe701b PM	464	* invoked. The first argument is the name of the RCU flavor and
72fe701b PM	465	* the second argument is number of callbacks actually invoked.
29c00b4a PM	466	*/
	467	TRACE_EVENT(rcu_batch_end,
	468
72fe701b	469	TP_PROTO(char *rcuname, int callbacks_invoked),
29c00b4a	470
72fe701b	471	TP_ARGS(rcuname, callbacks_invoked),
29c00b4a PM	472
29c00b4a PM	473	TP_STRUCT__entry(
72fe701b	474	__field(char *, rcuname)
300df91c	475	__field(int, callbacks_invoked)
29c00b4a PM	476	),
	477
	478	TP_fast_assign(
72fe701b	479	__entry->rcuname = rcuname;
300df91c	480	__entry->callbacks_invoked = callbacks_invoked;
29c00b4a PM	481	),
29c00b4a PM	482
72fe701b PM	483	TP_printk("%s CBs-invoked=%d",
72fe701b PM	484	__entry->rcuname, __entry->callbacks_invoked)
29c00b4a PM	485	);
29c00b4a PM	486
91afaf30 PM	487	/*
	488	* Tracepoint for rcutorture readers. The first argument is the name
	489	* of the RCU flavor from rcutorture's viewpoint and the second argument
	490	* is the callback address.
	491	*/
	492	TRACE_EVENT(rcu_torture_read,
	493
	494	TP_PROTO(char rcutorturename, struct rcu_head rhp),
	495
	496	TP_ARGS(rcutorturename, rhp),
	497
	498	TP_STRUCT__entry(
	499	__field(char *, rcutorturename)
	500	__field(struct rcu_head *, rhp)
	501	),
	502
	503	TP_fast_assign(
	504	__entry->rcutorturename = rcutorturename;
	505	__entry->rhp = rhp;
	506	),
	507
	508	TP_printk("%s torture read %p",
	509	__entry->rcutorturename, __entry->rhp)
	510	);
	511
d4c08f2a PM	512	#else /* #ifdef CONFIG_RCU_TRACE */
	513
	514	#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
	515	#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)
	516	#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
	517	#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
	518	#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)
	519	#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
4145fa7f	520	#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0)
433cdddc	521	#define trace_rcu_prep_idle(reason) do { } while (0)
d4c08f2a PM	522	#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
	523	#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
	524	#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
	525	#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
	526	#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
	527	#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
91afaf30	528	#define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
d4c08f2a PM	529
	530	#endif /* #else #ifdef CONFIG_RCU_TRACE */
	531
29c00b4a PM	532	#endif /* _TRACE_RCU_H */
	533
	534	/* This part must be outside protection */
	535	#include <trace/define_trace.h>