]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
29c00b4a PM |
2 | #undef TRACE_SYSTEM |
3 | #define TRACE_SYSTEM rcu | |
4 | ||
5 | #if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ) | |
6 | #define _TRACE_RCU_H | |
7 | ||
8 | #include <linux/tracepoint.h> | |
9 | ||
4f5fbd78 YS |
10 | #ifdef CONFIG_RCU_TRACE |
11 | #define TRACE_EVENT_RCU TRACE_EVENT | |
12 | #else | |
13 | #define TRACE_EVENT_RCU TRACE_EVENT_NOP | |
14 | #endif | |
15 | ||
29c00b4a | 16 | /* |
300df91c PM |
17 | * Tracepoint for start/end markers used for utilization calculations. |
18 | * By convention, the string is of the following forms: | |
19 | * | |
20 | * "Start <activity>" -- Mark the start of the specified activity, | |
21 | * such as "context switch". Nesting is permitted. | |
22 | * "End <activity>" -- Mark the end of the specified activity. | |
385680a9 PM |
23 | * |
24 | * An "@" character within "<activity>" is a comment character: Data | |
25 | * reduction scripts will ignore the "@" and the remainder of the line. | |
300df91c PM |
26 | */ |
27 | TRACE_EVENT(rcu_utilization, | |
28 | ||
e66c33d5 | 29 | TP_PROTO(const char *s), |
300df91c PM |
30 | |
31 | TP_ARGS(s), | |
32 | ||
33 | TP_STRUCT__entry( | |
e66c33d5 | 34 | __field(const char *, s) |
300df91c PM |
35 | ), |
36 | ||
37 | TP_fast_assign( | |
38 | __entry->s = s; | |
39 | ), | |
40 | ||
41 | TP_printk("%s", __entry->s) | |
42 | ); | |
43 | ||
b3e627d3 | 44 | #if defined(CONFIG_TREE_RCU) |
d4c08f2a PM |
45 | |
46 | /* | |
63c4db78 PM |
47 | * Tracepoint for grace-period events. Takes a string identifying the |
48 | * RCU flavor, the grace-period number, and a string identifying the | |
49 | * grace-period-related event as follows: | |
50 | * | |
f2cc020d | 51 | * "AccReadyCB": CPU accelerates new callbacks to RCU_NEXT_READY_TAIL. |
63c4db78 | 52 | * "AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL. |
bb311ecc | 53 | * "newreq": Request a new grace period. |
63c4db78 PM |
54 | * "start": Start a grace period. |
55 | * "cpustart": CPU first notices a grace-period start. | |
56 | * "cpuqs": CPU passes through a quiescent state. | |
57 | * "cpuonl": CPU comes online. | |
58 | * "cpuofl": CPU goes offline. | |
477351f7 | 59 | * "cpuofl-bgp": CPU goes offline while blocking a grace period. |
63c4db78 PM |
60 | * "reqwait": GP kthread sleeps waiting for grace-period request. |
61 | * "reqwaitsig": GP kthread awakened by signal from reqwait state. | |
62 | * "fqswait": GP kthread waiting until time to force quiescent states. | |
63 | * "fqsstart": GP kthread starts forcing quiescent states. | |
64 | * "fqsend": GP kthread done forcing quiescent states. | |
65 | * "fqswaitsig": GP kthread awakened by signal from fqswait state. | |
66 | * "end": End a grace period. | |
67 | * "cpuend": CPU first notices a grace-period end. | |
d4c08f2a | 68 | */ |
4f5fbd78 | 69 | TRACE_EVENT_RCU(rcu_grace_period, |
d4c08f2a | 70 | |
477351f7 | 71 | TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent), |
d4c08f2a | 72 | |
477351f7 | 73 | TP_ARGS(rcuname, gp_seq, gpevent), |
d4c08f2a PM |
74 | |
75 | TP_STRUCT__entry( | |
e66c33d5 | 76 | __field(const char *, rcuname) |
c30068f4 | 77 | __field(long, gp_seq) |
e66c33d5 | 78 | __field(const char *, gpevent) |
d4c08f2a PM |
79 | ), |
80 | ||
81 | TP_fast_assign( | |
82 | __entry->rcuname = rcuname; | |
c30068f4 | 83 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
84 | __entry->gpevent = gpevent; |
85 | ), | |
86 | ||
c30068f4 | 87 | TP_printk("%s %ld %s", |
477351f7 | 88 | __entry->rcuname, __entry->gp_seq, __entry->gpevent) |
d4c08f2a PM |
89 | ); |
90 | ||
09c7b890 | 91 | /* |
c21b8b3e PM |
92 | * Tracepoint for future grace-period events. The caller should pull |
93 | * the data from the rcu_node structure, other than rcuname, which comes | |
94 | * from the rcu_state structure, and event, which is one of the following: | |
09c7b890 | 95 | * |
7cc0fffd PM |
96 | * "Cleanup": Clean up rcu_node structure after previous GP. |
97 | * "CleanupMore": Clean up, and another GP is needed. | |
98 | * "EndWait": Complete wait. | |
99 | * "NoGPkthread": The RCU grace-period kthread has not yet started. | |
c21b8b3e | 100 | * "Prestarted": Someone beat us to the request |
a2165e41 PM |
101 | * "Startedleaf": Leaf node marked for future GP. |
102 | * "Startedleafroot": All nodes from leaf to root marked for future GP. | |
09c7b890 | 103 | * "Startedroot": Requested a nocb grace period based on root-node data. |
7cc0fffd | 104 | * "Startleaf": Request a grace period based on leaf-node data. |
09c7b890 | 105 | * "StartWait": Start waiting for the requested grace period. |
09c7b890 | 106 | */ |
4f5fbd78 | 107 | TRACE_EVENT_RCU(rcu_future_grace_period, |
09c7b890 | 108 | |
b73de91d JF |
109 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
110 | unsigned long gp_seq_req, u8 level, int grplo, int grphi, | |
111 | const char *gpevent), | |
09c7b890 | 112 | |
b73de91d | 113 | TP_ARGS(rcuname, gp_seq, gp_seq_req, level, grplo, grphi, gpevent), |
09c7b890 PM |
114 | |
115 | TP_STRUCT__entry( | |
e66c33d5 | 116 | __field(const char *, rcuname) |
c30068f4 JFG |
117 | __field(long, gp_seq) |
118 | __field(long, gp_seq_req) | |
09c7b890 PM |
119 | __field(u8, level) |
120 | __field(int, grplo) | |
121 | __field(int, grphi) | |
e66c33d5 | 122 | __field(const char *, gpevent) |
09c7b890 PM |
123 | ), |
124 | ||
125 | TP_fast_assign( | |
126 | __entry->rcuname = rcuname; | |
c30068f4 JFG |
127 | __entry->gp_seq = (long)gp_seq; |
128 | __entry->gp_seq_req = (long)gp_seq_req; | |
09c7b890 PM |
129 | __entry->level = level; |
130 | __entry->grplo = grplo; | |
131 | __entry->grphi = grphi; | |
132 | __entry->gpevent = gpevent; | |
133 | ), | |
134 | ||
c30068f4 JFG |
135 | TP_printk("%s %ld %ld %u %d %d %s", |
136 | __entry->rcuname, (long)__entry->gp_seq, (long)__entry->gp_seq_req, __entry->level, | |
abd13fdd | 137 | __entry->grplo, __entry->grphi, __entry->gpevent) |
09c7b890 PM |
138 | ); |
139 | ||
d4c08f2a PM |
140 | /* |
141 | * Tracepoint for grace-period-initialization events. These are | |
142 | * distinguished by the type of RCU, the new grace-period number, the | |
143 | * rcu_node structure level, the starting and ending CPU covered by the | |
144 | * rcu_node structure, and the mask of CPUs that will be waited for. | |
145 | * All but the type of RCU are extracted from the rcu_node structure. | |
146 | */ | |
4f5fbd78 | 147 | TRACE_EVENT_RCU(rcu_grace_period_init, |
d4c08f2a | 148 | |
63d86a7e | 149 | TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level, |
d4c08f2a PM |
150 | int grplo, int grphi, unsigned long qsmask), |
151 | ||
63d86a7e | 152 | TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask), |
d4c08f2a PM |
153 | |
154 | TP_STRUCT__entry( | |
e66c33d5 | 155 | __field(const char *, rcuname) |
c30068f4 | 156 | __field(long, gp_seq) |
d4c08f2a PM |
157 | __field(u8, level) |
158 | __field(int, grplo) | |
159 | __field(int, grphi) | |
160 | __field(unsigned long, qsmask) | |
161 | ), | |
162 | ||
163 | TP_fast_assign( | |
164 | __entry->rcuname = rcuname; | |
c30068f4 | 165 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
166 | __entry->level = level; |
167 | __entry->grplo = grplo; | |
168 | __entry->grphi = grphi; | |
169 | __entry->qsmask = qsmask; | |
170 | ), | |
171 | ||
c30068f4 | 172 | TP_printk("%s %ld %u %d %d %lx", |
63d86a7e | 173 | __entry->rcuname, __entry->gp_seq, __entry->level, |
d4c08f2a PM |
174 | __entry->grplo, __entry->grphi, __entry->qsmask) |
175 | ); | |
176 | ||
e087816d PM |
177 | /* |
178 | * Tracepoint for expedited grace-period events. Takes a string identifying | |
179 | * the RCU flavor, the expedited grace-period sequence number, and a string | |
180 | * identifying the grace-period-related event as follows: | |
181 | * | |
182 | * "snap": Captured snapshot of expedited grace period sequence number. | |
183 | * "start": Started a real expedited grace period. | |
9a414201 PM |
184 | * "reset": Started resetting the tree |
185 | * "select": Started selecting the CPUs to wait on. | |
7f5d42d0 | 186 | * "selectofl": Selected CPU partially offline. |
9a414201 | 187 | * "startwait": Started waiting on selected CPUs. |
e087816d | 188 | * "end": Ended a real expedited grace period. |
f6a12f34 | 189 | * "endwake": Woke piggybackers up. |
e087816d PM |
190 | * "done": Someone else did the expedited grace period for us. |
191 | */ | |
4f5fbd78 | 192 | TRACE_EVENT_RCU(rcu_exp_grace_period, |
e087816d PM |
193 | |
194 | TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent), | |
195 | ||
196 | TP_ARGS(rcuname, gpseq, gpevent), | |
197 | ||
198 | TP_STRUCT__entry( | |
199 | __field(const char *, rcuname) | |
c30068f4 | 200 | __field(long, gpseq) |
e087816d PM |
201 | __field(const char *, gpevent) |
202 | ), | |
203 | ||
204 | TP_fast_assign( | |
205 | __entry->rcuname = rcuname; | |
c30068f4 | 206 | __entry->gpseq = (long)gpseq; |
e087816d PM |
207 | __entry->gpevent = gpevent; |
208 | ), | |
209 | ||
c30068f4 | 210 | TP_printk("%s %ld %s", |
e087816d PM |
211 | __entry->rcuname, __entry->gpseq, __entry->gpevent) |
212 | ); | |
213 | ||
214 | /* | |
215 | * Tracepoint for expedited grace-period funnel-locking events. Takes a | |
216 | * string identifying the RCU flavor, an integer identifying the rcu_node | |
217 | * combining-tree level, another pair of integers identifying the lowest- | |
218 | * and highest-numbered CPU associated with the current rcu_node structure, | |
219 | * and a string. identifying the grace-period-related event as follows: | |
220 | * | |
f6a12f34 PM |
221 | * "nxtlvl": Advance to next level of rcu_node funnel |
222 | * "wait": Wait for someone else to do expedited GP | |
e087816d | 223 | */ |
4f5fbd78 | 224 | TRACE_EVENT_RCU(rcu_exp_funnel_lock, |
e087816d PM |
225 | |
226 | TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi, | |
227 | const char *gpevent), | |
228 | ||
229 | TP_ARGS(rcuname, level, grplo, grphi, gpevent), | |
230 | ||
231 | TP_STRUCT__entry( | |
232 | __field(const char *, rcuname) | |
233 | __field(u8, level) | |
234 | __field(int, grplo) | |
235 | __field(int, grphi) | |
236 | __field(const char *, gpevent) | |
237 | ), | |
238 | ||
239 | TP_fast_assign( | |
240 | __entry->rcuname = rcuname; | |
241 | __entry->level = level; | |
242 | __entry->grplo = grplo; | |
243 | __entry->grphi = grphi; | |
244 | __entry->gpevent = gpevent; | |
245 | ), | |
246 | ||
247 | TP_printk("%s %d %d %d %s", | |
248 | __entry->rcuname, __entry->level, __entry->grplo, | |
249 | __entry->grphi, __entry->gpevent) | |
250 | ); | |
251 | ||
9122caf9 | 252 | #ifdef CONFIG_RCU_NOCB_CPU |
9261dd0d PM |
253 | /* |
254 | * Tracepoint for RCU no-CBs CPU callback handoffs. This event is intended | |
255 | * to assist debugging of these handoffs. | |
256 | * | |
257 | * The first argument is the name of the RCU flavor, and the second is | |
258 | * the number of the offloaded CPU are extracted. The third and final | |
259 | * argument is a string as follows: | |
260 | * | |
d01f8620 PM |
261 | * "AlreadyAwake": The to-be-awakened rcuo kthread is already awake. |
262 | * "Bypass": rcuo GP kthread sees non-empty ->nocb_bypass. | |
263 | * "CBSleep": rcuo CB kthread sleeping waiting for CBs. | |
264 | * "Check": rcuo GP kthread checking specified CPU for work. | |
265 | * "DeferredWake": Timer expired or polled check, time to wake. | |
266 | * "DoWake": The to-be-awakened rcuo kthread needs to be awakened. | |
267 | * "EndSleep": Done waiting for GP for !rcu_nocb_poll. | |
268 | * "FirstBQ": New CB to empty ->nocb_bypass (->cblist maybe non-empty). | |
269 | * "FirstBQnoWake": FirstBQ plus rcuo kthread need not be awakened. | |
270 | * "FirstBQwake": FirstBQ plus rcuo kthread must be awakened. | |
271 | * "FirstQ": New CB to empty ->cblist (->nocb_bypass maybe non-empty). | |
272 | * "NeedWaitGP": rcuo GP kthread must wait on a grace period. | |
273 | * "Poll": Start of new polling cycle for rcu_nocb_poll. | |
274 | * "Sleep": Sleep waiting for GP for !rcu_nocb_poll. | |
275 | * "Timer": Deferred-wake timer expired. | |
276 | * "WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list. | |
277 | * "WakeEmpty": Wake rcuo kthread, first CB to empty list. | |
278 | * "WakeNot": Don't wake rcuo kthread. | |
279 | * "WakeNotPoll": Don't wake rcuo kthread because it is polling. | |
280 | * "WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge. | |
e75bcd48 | 281 | * "WakeBypassIsDeferred": Wake rcuo kthread later, bypass list is contended. |
d01f8620 | 282 | * "WokeEmpty": rcuo CB kthread woke to find empty list. |
9261dd0d | 283 | */ |
4f5fbd78 | 284 | TRACE_EVENT_RCU(rcu_nocb_wake, |
9261dd0d PM |
285 | |
286 | TP_PROTO(const char *rcuname, int cpu, const char *reason), | |
287 | ||
288 | TP_ARGS(rcuname, cpu, reason), | |
289 | ||
290 | TP_STRUCT__entry( | |
291 | __field(const char *, rcuname) | |
292 | __field(int, cpu) | |
293 | __field(const char *, reason) | |
294 | ), | |
295 | ||
296 | TP_fast_assign( | |
297 | __entry->rcuname = rcuname; | |
298 | __entry->cpu = cpu; | |
299 | __entry->reason = reason; | |
300 | ), | |
301 | ||
302 | TP_printk("%s %d %s", __entry->rcuname, __entry->cpu, __entry->reason) | |
303 | ); | |
9122caf9 | 304 | #endif |
9261dd0d | 305 | |
d4c08f2a PM |
306 | /* |
307 | * Tracepoint for tasks blocking within preemptible-RCU read-side | |
308 | * critical sections. Track the type of RCU (which one day might | |
309 | * include SRCU), the grace-period number that the task is blocking | |
310 | * (the current or the next), and the task's PID. | |
311 | */ | |
4f5fbd78 | 312 | TRACE_EVENT_RCU(rcu_preempt_task, |
d4c08f2a | 313 | |
598ce094 | 314 | TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq), |
d4c08f2a | 315 | |
598ce094 | 316 | TP_ARGS(rcuname, pid, gp_seq), |
d4c08f2a PM |
317 | |
318 | TP_STRUCT__entry( | |
e66c33d5 | 319 | __field(const char *, rcuname) |
c30068f4 | 320 | __field(long, gp_seq) |
d4c08f2a PM |
321 | __field(int, pid) |
322 | ), | |
323 | ||
324 | TP_fast_assign( | |
325 | __entry->rcuname = rcuname; | |
c30068f4 | 326 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
327 | __entry->pid = pid; |
328 | ), | |
329 | ||
c30068f4 | 330 | TP_printk("%s %ld %d", |
598ce094 | 331 | __entry->rcuname, __entry->gp_seq, __entry->pid) |
d4c08f2a PM |
332 | ); |
333 | ||
334 | /* | |
335 | * Tracepoint for tasks that blocked within a given preemptible-RCU | |
336 | * read-side critical section exiting that critical section. Track the | |
337 | * type of RCU (which one day might include SRCU) and the task's PID. | |
338 | */ | |
4f5fbd78 | 339 | TRACE_EVENT_RCU(rcu_unlock_preempted_task, |
d4c08f2a | 340 | |
865aa1e0 | 341 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid), |
d4c08f2a | 342 | |
865aa1e0 | 343 | TP_ARGS(rcuname, gp_seq, pid), |
d4c08f2a PM |
344 | |
345 | TP_STRUCT__entry( | |
e66c33d5 | 346 | __field(const char *, rcuname) |
c30068f4 | 347 | __field(long, gp_seq) |
d4c08f2a PM |
348 | __field(int, pid) |
349 | ), | |
350 | ||
351 | TP_fast_assign( | |
352 | __entry->rcuname = rcuname; | |
c30068f4 | 353 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
354 | __entry->pid = pid; |
355 | ), | |
356 | ||
c30068f4 | 357 | TP_printk("%s %ld %d", __entry->rcuname, __entry->gp_seq, __entry->pid) |
d4c08f2a PM |
358 | ); |
359 | ||
360 | /* | |
361 | * Tracepoint for quiescent-state-reporting events. These are | |
362 | * distinguished by the type of RCU, the grace-period number, the | |
363 | * mask of quiescent lower-level entities, the rcu_node structure level, | |
364 | * the starting and ending CPU covered by the rcu_node structure, and | |
365 | * whether there are any blocked tasks blocking the current grace period. | |
366 | * All but the type of RCU are extracted from the rcu_node structure. | |
367 | */ | |
4f5fbd78 | 368 | TRACE_EVENT_RCU(rcu_quiescent_state_report, |
d4c08f2a | 369 | |
db023296 | 370 | TP_PROTO(const char *rcuname, unsigned long gp_seq, |
d4c08f2a PM |
371 | unsigned long mask, unsigned long qsmask, |
372 | u8 level, int grplo, int grphi, int gp_tasks), | |
373 | ||
db023296 | 374 | TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks), |
d4c08f2a PM |
375 | |
376 | TP_STRUCT__entry( | |
e66c33d5 | 377 | __field(const char *, rcuname) |
c30068f4 | 378 | __field(long, gp_seq) |
d4c08f2a PM |
379 | __field(unsigned long, mask) |
380 | __field(unsigned long, qsmask) | |
381 | __field(u8, level) | |
382 | __field(int, grplo) | |
383 | __field(int, grphi) | |
384 | __field(u8, gp_tasks) | |
385 | ), | |
386 | ||
387 | TP_fast_assign( | |
388 | __entry->rcuname = rcuname; | |
c30068f4 | 389 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
390 | __entry->mask = mask; |
391 | __entry->qsmask = qsmask; | |
392 | __entry->level = level; | |
393 | __entry->grplo = grplo; | |
394 | __entry->grphi = grphi; | |
395 | __entry->gp_tasks = gp_tasks; | |
396 | ), | |
397 | ||
c30068f4 | 398 | TP_printk("%s %ld %lx>%lx %u %d %d %u", |
db023296 | 399 | __entry->rcuname, __entry->gp_seq, |
d4c08f2a PM |
400 | __entry->mask, __entry->qsmask, __entry->level, |
401 | __entry->grplo, __entry->grphi, __entry->gp_tasks) | |
402 | ); | |
403 | ||
404 | /* | |
405 | * Tracepoint for quiescent states detected by force_quiescent_state(). | |
e05121ba PM |
406 | * These trace events include the type of RCU, the grace-period number |
407 | * that was blocked by the CPU, the CPU itself, and the type of quiescent | |
7e28c5af PM |
408 | * state, which can be "dti" for dyntick-idle mode or "kick" when kicking |
409 | * a CPU that has been in dyntick-idle mode for too long. | |
d4c08f2a | 410 | */ |
4f5fbd78 | 411 | TRACE_EVENT_RCU(rcu_fqs, |
d4c08f2a | 412 | |
fee5997c | 413 | TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent), |
d4c08f2a | 414 | |
fee5997c | 415 | TP_ARGS(rcuname, gp_seq, cpu, qsevent), |
d4c08f2a PM |
416 | |
417 | TP_STRUCT__entry( | |
e66c33d5 | 418 | __field(const char *, rcuname) |
c30068f4 | 419 | __field(long, gp_seq) |
d4c08f2a | 420 | __field(int, cpu) |
e66c33d5 | 421 | __field(const char *, qsevent) |
d4c08f2a PM |
422 | ), |
423 | ||
424 | TP_fast_assign( | |
425 | __entry->rcuname = rcuname; | |
c30068f4 | 426 | __entry->gp_seq = (long)gp_seq; |
d4c08f2a PM |
427 | __entry->cpu = cpu; |
428 | __entry->qsevent = qsevent; | |
429 | ), | |
430 | ||
c30068f4 | 431 | TP_printk("%s %ld %d %s", |
fee5997c | 432 | __entry->rcuname, __entry->gp_seq, |
d4c08f2a PM |
433 | __entry->cpu, __entry->qsevent) |
434 | ); | |
435 | ||
565cfb9e SK |
436 | /* |
437 | * Tracepoint for RCU stall events. Takes a string identifying the RCU flavor | |
438 | * and a string identifying which function detected the RCU stall as follows: | |
439 | * | |
440 | * "StallDetected": Scheduler-tick detects other CPU's stalls. | |
441 | * "SelfDetected": Scheduler-tick detects a current CPU's stall. | |
442 | * "ExpeditedStall": Expedited grace period detects stalls. | |
443 | */ | |
444 | TRACE_EVENT(rcu_stall_warning, | |
445 | ||
446 | TP_PROTO(const char *rcuname, const char *msg), | |
447 | ||
448 | TP_ARGS(rcuname, msg), | |
449 | ||
450 | TP_STRUCT__entry( | |
451 | __field(const char *, rcuname) | |
452 | __field(const char *, msg) | |
453 | ), | |
454 | ||
455 | TP_fast_assign( | |
456 | __entry->rcuname = rcuname; | |
457 | __entry->msg = msg; | |
458 | ), | |
459 | ||
460 | TP_printk("%s %s", | |
461 | __entry->rcuname, __entry->msg) | |
462 | ); | |
463 | ||
b3e627d3 | 464 | #endif /* #if defined(CONFIG_TREE_RCU) */ |
d4c08f2a PM |
465 | |
466 | /* | |
88748e33 MB |
467 | * Tracepoint for dyntick-idle entry/exit events. These take 2 strings |
468 | * as argument: | |
469 | * polarity: "Start", "End", "StillNonIdle" for entering, exiting or still not | |
470 | * being in dyntick-idle mode. | |
471 | * context: "USER" or "IDLE" or "IRQ". | |
472 | * NMIs nested in IRQs are inferred with dynticks_nesting > 1 in IRQ context. | |
045fb931 PM |
473 | * |
474 | * These events also take a pair of numbers, which indicate the nesting | |
e68bbb26 PM |
475 | * depth before and after the event of interest, and a third number that is |
476 | * the ->dynticks counter. Note that task-related and interrupt-related | |
477 | * events use two separate counters, and that the "++=" and "--=" events | |
478 | * for irq/NMI will change the counter by two, otherwise by one. | |
d4c08f2a | 479 | */ |
4f5fbd78 | 480 | TRACE_EVENT_RCU(rcu_dyntick, |
d4c08f2a | 481 | |
6cf539a8 | 482 | TP_PROTO(const char *polarity, long oldnesting, long newnesting, int dynticks), |
d4c08f2a | 483 | |
dec98900 | 484 | TP_ARGS(polarity, oldnesting, newnesting, dynticks), |
d4c08f2a PM |
485 | |
486 | TP_STRUCT__entry( | |
e66c33d5 | 487 | __field(const char *, polarity) |
84585aa8 PM |
488 | __field(long, oldnesting) |
489 | __field(long, newnesting) | |
dec98900 | 490 | __field(int, dynticks) |
d4c08f2a PM |
491 | ), |
492 | ||
493 | TP_fast_assign( | |
494 | __entry->polarity = polarity; | |
4145fa7f PM |
495 | __entry->oldnesting = oldnesting; |
496 | __entry->newnesting = newnesting; | |
6cf539a8 | 497 | __entry->dynticks = dynticks; |
d4c08f2a PM |
498 | ), |
499 | ||
dec98900 PM |
500 | TP_printk("%s %lx %lx %#3x", __entry->polarity, |
501 | __entry->oldnesting, __entry->newnesting, | |
502 | __entry->dynticks & 0xfff) | |
433cdddc PM |
503 | ); |
504 | ||
d4c08f2a PM |
505 | /* |
506 | * Tracepoint for the registration of a single RCU callback function. | |
507 | * The first argument is the type of RCU, the second argument is | |
486e2593 PM |
508 | * a pointer to the RCU callback itself, the third element is the |
509 | * number of lazy callbacks queued, and the fourth element is the | |
510 | * total number of callbacks queued. | |
d4c08f2a | 511 | */ |
4f5fbd78 | 512 | TRACE_EVENT_RCU(rcu_callback, |
d4c08f2a | 513 | |
77a40f97 | 514 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen), |
d4c08f2a | 515 | |
77a40f97 | 516 | TP_ARGS(rcuname, rhp, qlen), |
d4c08f2a PM |
517 | |
518 | TP_STRUCT__entry( | |
e66c33d5 | 519 | __field(const char *, rcuname) |
d4c08f2a PM |
520 | __field(void *, rhp) |
521 | __field(void *, func) | |
522 | __field(long, qlen) | |
523 | ), | |
524 | ||
525 | TP_fast_assign( | |
526 | __entry->rcuname = rcuname; | |
527 | __entry->rhp = rhp; | |
528 | __entry->func = rhp->func; | |
529 | __entry->qlen = qlen; | |
530 | ), | |
531 | ||
77a40f97 | 532 | TP_printk("%s rhp=%p func=%ps %ld", |
486e2593 | 533 | __entry->rcuname, __entry->rhp, __entry->func, |
77a40f97 | 534 | __entry->qlen) |
d4c08f2a PM |
535 | ); |
536 | ||
3afe7fa5 JFG |
537 | TRACE_EVENT_RCU(rcu_segcb_stats, |
538 | ||
539 | TP_PROTO(struct rcu_segcblist *rs, const char *ctx), | |
540 | ||
541 | TP_ARGS(rs, ctx), | |
542 | ||
543 | TP_STRUCT__entry( | |
544 | __field(const char *, ctx) | |
545 | __array(unsigned long, gp_seq, RCU_CBLIST_NSEGS) | |
546 | __array(long, seglen, RCU_CBLIST_NSEGS) | |
547 | ), | |
548 | ||
549 | TP_fast_assign( | |
550 | __entry->ctx = ctx; | |
551 | memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); | |
552 | memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); | |
553 | ||
554 | ), | |
555 | ||
556 | TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " | |
557 | "gp_seq: (DONE=%lu, WAIT=%lu, NEXT_READY=%lu, NEXT=%lu)", __entry->ctx, | |
558 | __entry->seglen[0], __entry->seglen[1], __entry->seglen[2], __entry->seglen[3], | |
559 | __entry->gp_seq[0], __entry->gp_seq[1], __entry->gp_seq[2], __entry->gp_seq[3]) | |
560 | ||
561 | ); | |
562 | ||
d4c08f2a PM |
563 | /* |
564 | * Tracepoint for the registration of a single RCU callback of the special | |
c408b215 | 565 | * kvfree() form. The first argument is the RCU type, the second argument |
d4c08f2a PM |
566 | * is a pointer to the RCU callback, the third argument is the offset |
567 | * of the callback within the enclosing RCU-protected data structure, | |
486e2593 PM |
568 | * the fourth argument is the number of lazy callbacks queued, and the |
569 | * fifth argument is the total number of callbacks queued. | |
d4c08f2a | 570 | */ |
c408b215 | 571 | TRACE_EVENT_RCU(rcu_kvfree_callback, |
d4c08f2a | 572 | |
e66c33d5 | 573 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, |
77a40f97 | 574 | long qlen), |
d4c08f2a | 575 | |
77a40f97 | 576 | TP_ARGS(rcuname, rhp, offset, qlen), |
d4c08f2a PM |
577 | |
578 | TP_STRUCT__entry( | |
e66c33d5 | 579 | __field(const char *, rcuname) |
d4c08f2a PM |
580 | __field(void *, rhp) |
581 | __field(unsigned long, offset) | |
582 | __field(long, qlen) | |
583 | ), | |
584 | ||
585 | TP_fast_assign( | |
586 | __entry->rcuname = rcuname; | |
587 | __entry->rhp = rhp; | |
588 | __entry->offset = offset; | |
589 | __entry->qlen = qlen; | |
590 | ), | |
591 | ||
77a40f97 | 592 | TP_printk("%s rhp=%p func=%ld %ld", |
d4c08f2a | 593 | __entry->rcuname, __entry->rhp, __entry->offset, |
77a40f97 | 594 | __entry->qlen) |
d4c08f2a PM |
595 | ); |
596 | ||
300df91c PM |
597 | /* |
598 | * Tracepoint for marking the beginning rcu_do_batch, performed to start | |
72fe701b | 599 | * RCU callback invocation. The first argument is the RCU flavor, |
486e2593 PM |
600 | * the second is the number of lazy callbacks queued, the third is |
601 | * the total number of callbacks queued, and the fourth argument is | |
602 | * the current RCU-callback batch limit. | |
29c00b4a | 603 | */ |
4f5fbd78 | 604 | TRACE_EVENT_RCU(rcu_batch_start, |
29c00b4a | 605 | |
77a40f97 | 606 | TP_PROTO(const char *rcuname, long qlen, long blimit), |
29c00b4a | 607 | |
77a40f97 | 608 | TP_ARGS(rcuname, qlen, blimit), |
29c00b4a PM |
609 | |
610 | TP_STRUCT__entry( | |
e66c33d5 | 611 | __field(const char *, rcuname) |
300df91c | 612 | __field(long, qlen) |
3aac7a8d | 613 | __field(long, blimit) |
29c00b4a PM |
614 | ), |
615 | ||
616 | TP_fast_assign( | |
72fe701b | 617 | __entry->rcuname = rcuname; |
300df91c PM |
618 | __entry->qlen = qlen; |
619 | __entry->blimit = blimit; | |
29c00b4a PM |
620 | ), |
621 | ||
77a40f97 JFG |
622 | TP_printk("%s CBs=%ld bl=%ld", |
623 | __entry->rcuname, __entry->qlen, __entry->blimit) | |
29c00b4a PM |
624 | ); |
625 | ||
626 | /* | |
300df91c | 627 | * Tracepoint for the invocation of a single RCU callback function. |
d4c08f2a PM |
628 | * The first argument is the type of RCU, and the second argument is |
629 | * a pointer to the RCU callback itself. | |
29c00b4a | 630 | */ |
4f5fbd78 | 631 | TRACE_EVENT_RCU(rcu_invoke_callback, |
29c00b4a | 632 | |
e66c33d5 | 633 | TP_PROTO(const char *rcuname, struct rcu_head *rhp), |
29c00b4a | 634 | |
d4c08f2a | 635 | TP_ARGS(rcuname, rhp), |
29c00b4a PM |
636 | |
637 | TP_STRUCT__entry( | |
e66c33d5 | 638 | __field(const char *, rcuname) |
d4c08f2a PM |
639 | __field(void *, rhp) |
640 | __field(void *, func) | |
29c00b4a PM |
641 | ), |
642 | ||
643 | TP_fast_assign( | |
d4c08f2a | 644 | __entry->rcuname = rcuname; |
300df91c PM |
645 | __entry->rhp = rhp; |
646 | __entry->func = rhp->func; | |
29c00b4a PM |
647 | ), |
648 | ||
d75f773c | 649 | TP_printk("%s rhp=%p func=%ps", |
d4c08f2a | 650 | __entry->rcuname, __entry->rhp, __entry->func) |
29c00b4a PM |
651 | ); |
652 | ||
653 | /* | |
300df91c | 654 | * Tracepoint for the invocation of a single RCU callback of the special |
c408b215 | 655 | * kvfree() form. The first argument is the RCU flavor, the second |
d4c08f2a PM |
656 | * argument is a pointer to the RCU callback, and the third argument |
657 | * is the offset of the callback within the enclosing RCU-protected | |
658 | * data structure. | |
29c00b4a | 659 | */ |
c408b215 | 660 | TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, |
29c00b4a | 661 | |
e66c33d5 | 662 | TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), |
29c00b4a | 663 | |
d4c08f2a | 664 | TP_ARGS(rcuname, rhp, offset), |
29c00b4a PM |
665 | |
666 | TP_STRUCT__entry( | |
e66c33d5 | 667 | __field(const char *, rcuname) |
d4c08f2a | 668 | __field(void *, rhp) |
300df91c | 669 | __field(unsigned long, offset) |
29c00b4a PM |
670 | ), |
671 | ||
672 | TP_fast_assign( | |
d4c08f2a | 673 | __entry->rcuname = rcuname; |
300df91c | 674 | __entry->rhp = rhp; |
29c00b4a PM |
675 | __entry->offset = offset; |
676 | ), | |
677 | ||
d4c08f2a PM |
678 | TP_printk("%s rhp=%p func=%ld", |
679 | __entry->rcuname, __entry->rhp, __entry->offset) | |
29c00b4a PM |
680 | ); |
681 | ||
61370792 URS |
682 | /* |
683 | * Tracepoint for the invocation of a single RCU callback of the special | |
684 | * kfree_bulk() form. The first argument is the RCU flavor, the second | |
685 | * argument is a number of elements in array to free, the third is an | |
686 | * address of the array holding nr_records entries. | |
687 | */ | |
688 | TRACE_EVENT_RCU(rcu_invoke_kfree_bulk_callback, | |
689 | ||
690 | TP_PROTO(const char *rcuname, unsigned long nr_records, void **p), | |
691 | ||
692 | TP_ARGS(rcuname, nr_records, p), | |
693 | ||
694 | TP_STRUCT__entry( | |
695 | __field(const char *, rcuname) | |
696 | __field(unsigned long, nr_records) | |
697 | __field(void **, p) | |
698 | ), | |
699 | ||
700 | TP_fast_assign( | |
701 | __entry->rcuname = rcuname; | |
702 | __entry->nr_records = nr_records; | |
703 | __entry->p = p; | |
704 | ), | |
705 | ||
706 | TP_printk("%s bulk=0x%p nr_records=%lu", | |
707 | __entry->rcuname, __entry->p, __entry->nr_records) | |
708 | ); | |
709 | ||
29c00b4a | 710 | /* |
300df91c | 711 | * Tracepoint for exiting rcu_do_batch after RCU callbacks have been |
4968c300 PM |
712 | * invoked. The first argument is the name of the RCU flavor, |
713 | * the second argument is number of callbacks actually invoked, | |
714 | * the third argument (cb) is whether or not any of the callbacks that | |
715 | * were ready to invoke at the beginning of this batch are still | |
716 | * queued, the fourth argument (nr) is the return value of need_resched(), | |
717 | * the fifth argument (iit) is 1 if the current task is the idle task, | |
718 | * and the sixth argument (risk) is the return value from | |
719 | * rcu_is_callbacks_kthread(). | |
29c00b4a | 720 | */ |
4f5fbd78 | 721 | TRACE_EVENT_RCU(rcu_batch_end, |
29c00b4a | 722 | |
e66c33d5 | 723 | TP_PROTO(const char *rcuname, int callbacks_invoked, |
15f5191b | 724 | char cb, char nr, char iit, char risk), |
29c00b4a | 725 | |
4968c300 | 726 | TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk), |
29c00b4a PM |
727 | |
728 | TP_STRUCT__entry( | |
e66c33d5 | 729 | __field(const char *, rcuname) |
300df91c | 730 | __field(int, callbacks_invoked) |
15f5191b PM |
731 | __field(char, cb) |
732 | __field(char, nr) | |
733 | __field(char, iit) | |
734 | __field(char, risk) | |
29c00b4a PM |
735 | ), |
736 | ||
737 | TP_fast_assign( | |
72fe701b | 738 | __entry->rcuname = rcuname; |
300df91c | 739 | __entry->callbacks_invoked = callbacks_invoked; |
4968c300 PM |
740 | __entry->cb = cb; |
741 | __entry->nr = nr; | |
742 | __entry->iit = iit; | |
743 | __entry->risk = risk; | |
744 | ), | |
745 | ||
746 | TP_printk("%s CBs-invoked=%d idle=%c%c%c%c", | |
747 | __entry->rcuname, __entry->callbacks_invoked, | |
748 | __entry->cb ? 'C' : '.', | |
749 | __entry->nr ? 'S' : '.', | |
750 | __entry->iit ? 'I' : '.', | |
751 | __entry->risk ? 'R' : '.') | |
29c00b4a PM |
752 | ); |
753 | ||
91afaf30 PM |
754 | /* |
755 | * Tracepoint for rcutorture readers. The first argument is the name | |
756 | * of the RCU flavor from rcutorture's viewpoint and the second argument | |
d0af39e8 PM |
757 | * is the callback address. The third argument is the start time in |
758 | * seconds, and the last two arguments are the grace period numbers | |
759 | * at the beginning and end of the read, respectively. Note that the | |
760 | * callback address can be NULL. | |
91afaf30 | 761 | */ |
b3c98314 | 762 | #define RCUTORTURENAME_LEN 8 |
4f5fbd78 | 763 | TRACE_EVENT_RCU(rcu_torture_read, |
91afaf30 | 764 | |
e66c33d5 | 765 | TP_PROTO(const char *rcutorturename, struct rcu_head *rhp, |
52494535 | 766 | unsigned long secs, unsigned long c_old, unsigned long c), |
91afaf30 | 767 | |
52494535 | 768 | TP_ARGS(rcutorturename, rhp, secs, c_old, c), |
91afaf30 PM |
769 | |
770 | TP_STRUCT__entry( | |
b3c98314 | 771 | __field(char, rcutorturename[RCUTORTURENAME_LEN]) |
91afaf30 | 772 | __field(struct rcu_head *, rhp) |
52494535 PM |
773 | __field(unsigned long, secs) |
774 | __field(unsigned long, c_old) | |
775 | __field(unsigned long, c) | |
91afaf30 PM |
776 | ), |
777 | ||
778 | TP_fast_assign( | |
b3c98314 PM |
779 | strncpy(__entry->rcutorturename, rcutorturename, |
780 | RCUTORTURENAME_LEN); | |
781 | __entry->rcutorturename[RCUTORTURENAME_LEN - 1] = 0; | |
91afaf30 | 782 | __entry->rhp = rhp; |
52494535 PM |
783 | __entry->secs = secs; |
784 | __entry->c_old = c_old; | |
785 | __entry->c = c; | |
91afaf30 PM |
786 | ), |
787 | ||
52494535 PM |
788 | TP_printk("%s torture read %p %luus c: %lu %lu", |
789 | __entry->rcutorturename, __entry->rhp, | |
790 | __entry->secs, __entry->c_old, __entry->c) | |
91afaf30 PM |
791 | ); |
792 | ||
a83eff0a | 793 | /* |
dd46a788 PM |
794 | * Tracepoint for rcu_barrier() execution. The string "s" describes |
795 | * the rcu_barrier phase: | |
796 | * "Begin": rcu_barrier() started. | |
797 | * "EarlyExit": rcu_barrier() piggybacked, thus early exit. | |
798 | * "Inc1": rcu_barrier() piggyback check counter incremented. | |
127e2981 | 799 | * "OfflineNoCBQ": rcu_barrier() found offline no-CBs CPU with callbacks. |
dd46a788 PM |
800 | * "OnlineQ": rcu_barrier() found online CPU with callbacks. |
801 | * "OnlineNQ": rcu_barrier() found online CPU, no callbacks. | |
a83eff0a | 802 | * "IRQ": An rcu_barrier_callback() callback posted on remote CPU. |
f92c734f | 803 | * "IRQNQ": An rcu_barrier_callback() callback found no callbacks. |
a83eff0a PM |
804 | * "CB": An rcu_barrier_callback() invoked a callback, not the last. |
805 | * "LastCB": An rcu_barrier_callback() invoked the last callback. | |
dd46a788 | 806 | * "Inc2": rcu_barrier() piggyback check counter incremented. |
a83eff0a PM |
807 | * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument |
808 | * is the count of remaining callbacks, and "done" is the piggybacking count. | |
809 | */ | |
4f5fbd78 | 810 | TRACE_EVENT_RCU(rcu_barrier, |
a83eff0a | 811 | |
e66c33d5 | 812 | TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done), |
a83eff0a PM |
813 | |
814 | TP_ARGS(rcuname, s, cpu, cnt, done), | |
815 | ||
816 | TP_STRUCT__entry( | |
e66c33d5 SRRH |
817 | __field(const char *, rcuname) |
818 | __field(const char *, s) | |
a83eff0a PM |
819 | __field(int, cpu) |
820 | __field(int, cnt) | |
821 | __field(unsigned long, done) | |
822 | ), | |
823 | ||
824 | TP_fast_assign( | |
825 | __entry->rcuname = rcuname; | |
826 | __entry->s = s; | |
827 | __entry->cpu = cpu; | |
828 | __entry->cnt = cnt; | |
829 | __entry->done = done; | |
830 | ), | |
831 | ||
832 | TP_printk("%s %s cpu %d remaining %d # %lu", | |
833 | __entry->rcuname, __entry->s, __entry->cpu, __entry->cnt, | |
834 | __entry->done) | |
835 | ); | |
836 | ||
29c00b4a PM |
837 | #endif /* _TRACE_RCU_H */ |
838 | ||
839 | /* This part must be outside protection */ | |
840 | #include <trace/define_trace.h> |