]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
352ad25a SR |
2 | /* |
3 | * trace task wakeup timings | |
4 | * | |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Based on code from the latency_tracer, that is: | |
9 | * | |
10 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 11 | * Copyright (C) 2004 Nadia Yvette Chambers |
352ad25a SR |
12 | */ |
13 | #include <linux/module.h> | |
352ad25a SR |
14 | #include <linux/kallsyms.h> |
15 | #include <linux/uaccess.h> | |
16 | #include <linux/ftrace.h> | |
8bd75c77 | 17 | #include <linux/sched/rt.h> |
2d3d891d | 18 | #include <linux/sched/deadline.h> |
ad8d75ff | 19 | #include <trace/events/sched.h> |
352ad25a SR |
20 | #include "trace.h" |
21 | ||
22 | static struct trace_array *wakeup_trace; | |
23 | static int __read_mostly tracer_enabled; | |
24 | ||
25 | static struct task_struct *wakeup_task; | |
26 | static int wakeup_cpu; | |
478142c3 | 27 | static int wakeup_current_cpu; |
352ad25a | 28 | static unsigned wakeup_prio = -1; |
3244351c | 29 | static int wakeup_rt; |
af6ace76 DF |
30 | static int wakeup_dl; |
31 | static int tracing_dl = 0; | |
352ad25a | 32 | |
445c8951 | 33 | static arch_spinlock_t wakeup_lock = |
edc35bd7 | 34 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
352ad25a | 35 | |
7495a5be | 36 | static void wakeup_reset(struct trace_array *tr); |
e309b41d | 37 | static void __wakeup_reset(struct trace_array *tr); |
352ad25a | 38 | |
613f04a0 | 39 | static int save_flags; |
7495a5be | 40 | |
7495a5be | 41 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 42 | static int wakeup_display_graph(struct trace_array *tr, int set); |
983f938a | 43 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
03905582 SRRH |
44 | #else |
45 | static inline int wakeup_display_graph(struct trace_array *tr, int set) | |
46 | { | |
8179e8a1 | 47 | return 0; |
03905582 | 48 | } |
983f938a | 49 | # define is_graph(tr) false |
7495a5be | 50 | #endif |
7495a5be | 51 | |
7495a5be | 52 | |
606576ce | 53 | #ifdef CONFIG_FUNCTION_TRACER |
542181d3 | 54 | |
8179e8a1 SRRH |
55 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace); |
56 | static void wakeup_graph_return(struct ftrace_graph_ret *trace); | |
57 | ||
58 | static bool function_enabled; | |
59 | ||
7e18d8e7 | 60 | /* |
542181d3 SR |
61 | * Prologue for the wakeup function tracers. |
62 | * | |
63 | * Returns 1 if it is OK to continue, and preemption | |
64 | * is disabled and data->disabled is incremented. | |
65 | * 0 if the trace is to be ignored, and preemption | |
66 | * is not disabled and data->disabled is | |
67 | * kept the same. | |
68 | * | |
69 | * Note, this function is also used outside this ifdef but | |
70 | * inside the #ifdef of the function graph tracer below. | |
71 | * This is OK, since the function graph tracer is | |
72 | * dependent on the function tracer. | |
7e18d8e7 | 73 | */ |
542181d3 SR |
74 | static int |
75 | func_prolog_preempt_disable(struct trace_array *tr, | |
76 | struct trace_array_cpu **data, | |
77 | int *pc) | |
7e18d8e7 | 78 | { |
7e18d8e7 | 79 | long disabled; |
7e18d8e7 SR |
80 | int cpu; |
81 | ||
82 | if (likely(!wakeup_task)) | |
542181d3 | 83 | return 0; |
7e18d8e7 | 84 | |
542181d3 | 85 | *pc = preempt_count(); |
5168ae50 | 86 | preempt_disable_notrace(); |
7e18d8e7 SR |
87 | |
88 | cpu = raw_smp_processor_id(); | |
478142c3 SR |
89 | if (cpu != wakeup_current_cpu) |
90 | goto out_enable; | |
91 | ||
12883efb | 92 | *data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
542181d3 | 93 | disabled = atomic_inc_return(&(*data)->disabled); |
7e18d8e7 SR |
94 | if (unlikely(disabled != 1)) |
95 | goto out; | |
96 | ||
542181d3 | 97 | return 1; |
7e18d8e7 | 98 | |
542181d3 SR |
99 | out: |
100 | atomic_dec(&(*data)->disabled); | |
7e18d8e7 | 101 | |
542181d3 SR |
102 | out_enable: |
103 | preempt_enable_notrace(); | |
104 | return 0; | |
105 | } | |
106 | ||
107 | /* | |
108 | * wakeup uses its own tracer function to keep the overhead down: | |
109 | */ | |
110 | static void | |
a1e2e31d SR |
111 | wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, |
112 | struct ftrace_ops *op, struct pt_regs *pt_regs) | |
542181d3 SR |
113 | { |
114 | struct trace_array *tr = wakeup_trace; | |
115 | struct trace_array_cpu *data; | |
116 | unsigned long flags; | |
117 | int pc; | |
118 | ||
119 | if (!func_prolog_preempt_disable(tr, &data, &pc)) | |
120 | return; | |
121 | ||
122 | local_irq_save(flags); | |
123 | trace_function(tr, ip, parent_ip, flags, pc); | |
e59494f4 | 124 | local_irq_restore(flags); |
7e18d8e7 | 125 | |
7e18d8e7 | 126 | atomic_dec(&data->disabled); |
5168ae50 | 127 | preempt_enable_notrace(); |
7e18d8e7 | 128 | } |
7495a5be | 129 | |
4104d326 | 130 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) |
7495a5be JO |
131 | { |
132 | int ret; | |
133 | ||
328df475 | 134 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ |
983f938a | 135 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) |
328df475 SRRH |
136 | return 0; |
137 | ||
138 | if (graph) | |
7495a5be JO |
139 | ret = register_ftrace_graph(&wakeup_graph_return, |
140 | &wakeup_graph_entry); | |
328df475 | 141 | else |
4104d326 | 142 | ret = register_ftrace_function(tr->ops); |
328df475 SRRH |
143 | |
144 | if (!ret) | |
145 | function_enabled = true; | |
146 | ||
147 | return ret; | |
148 | } | |
149 | ||
4104d326 | 150 | static void unregister_wakeup_function(struct trace_array *tr, int graph) |
328df475 SRRH |
151 | { |
152 | if (!function_enabled) | |
153 | return; | |
154 | ||
155 | if (graph) | |
156 | unregister_ftrace_graph(); | |
157 | else | |
4104d326 | 158 | unregister_ftrace_function(tr->ops); |
328df475 SRRH |
159 | |
160 | function_enabled = false; | |
161 | } | |
162 | ||
8179e8a1 | 163 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) |
328df475 | 164 | { |
8179e8a1 SRRH |
165 | if (!(mask & TRACE_ITER_FUNCTION)) |
166 | return 0; | |
167 | ||
328df475 | 168 | if (set) |
983f938a | 169 | register_wakeup_function(tr, is_graph(tr), 1); |
328df475 | 170 | else |
983f938a | 171 | unregister_wakeup_function(tr, is_graph(tr)); |
8179e8a1 SRRH |
172 | return 1; |
173 | } | |
174 | #else | |
175 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | |
176 | { | |
03905582 | 177 | return 0; |
328df475 | 178 | } |
8179e8a1 SRRH |
179 | static void unregister_wakeup_function(struct trace_array *tr, int graph) { } |
180 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | |
181 | { | |
182 | return 0; | |
328df475 | 183 | } |
8179e8a1 | 184 | #endif /* CONFIG_FUNCTION_TRACER */ |
328df475 | 185 | |
bf6065b5 | 186 | static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) |
328df475 | 187 | { |
bf6065b5 SRRH |
188 | struct tracer *tracer = tr->current_trace; |
189 | ||
8179e8a1 SRRH |
190 | if (wakeup_function_set(tr, mask, set)) |
191 | return 0; | |
03905582 | 192 | |
729358da | 193 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 SRRH |
194 | if (mask & TRACE_ITER_DISPLAY_GRAPH) |
195 | return wakeup_display_graph(tr, set); | |
729358da | 196 | #endif |
328df475 SRRH |
197 | |
198 | return trace_keep_overwrite(tracer, mask, set); | |
199 | } | |
200 | ||
4104d326 | 201 | static int start_func_tracer(struct trace_array *tr, int graph) |
328df475 SRRH |
202 | { |
203 | int ret; | |
204 | ||
4104d326 | 205 | ret = register_wakeup_function(tr, graph, 0); |
7495a5be JO |
206 | |
207 | if (!ret && tracing_is_enabled()) | |
208 | tracer_enabled = 1; | |
209 | else | |
210 | tracer_enabled = 0; | |
211 | ||
212 | return ret; | |
213 | } | |
214 | ||
4104d326 | 215 | static void stop_func_tracer(struct trace_array *tr, int graph) |
7495a5be JO |
216 | { |
217 | tracer_enabled = 0; | |
218 | ||
4104d326 | 219 | unregister_wakeup_function(tr, graph); |
7495a5be JO |
220 | } |
221 | ||
7495a5be | 222 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 223 | static int wakeup_display_graph(struct trace_array *tr, int set) |
7495a5be | 224 | { |
983f938a | 225 | if (!(is_graph(tr) ^ set)) |
7495a5be JO |
226 | return 0; |
227 | ||
4104d326 | 228 | stop_func_tracer(tr, !set); |
7495a5be JO |
229 | |
230 | wakeup_reset(wakeup_trace); | |
6d9b3fa5 | 231 | tr->max_latency = 0; |
7495a5be | 232 | |
4104d326 | 233 | return start_func_tracer(tr, set); |
7495a5be JO |
234 | } |
235 | ||
236 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) | |
237 | { | |
238 | struct trace_array *tr = wakeup_trace; | |
239 | struct trace_array_cpu *data; | |
240 | unsigned long flags; | |
542181d3 | 241 | int pc, ret = 0; |
7495a5be | 242 | |
1a414428 SRRH |
243 | if (ftrace_graph_ignore_func(trace)) |
244 | return 0; | |
245 | /* | |
246 | * Do not trace a function if it's filtered by set_graph_notrace. | |
247 | * Make the index of ret stack negative to indicate that it should | |
248 | * ignore further functions. But it needs its own ret stack entry | |
249 | * to recover the original index in order to continue tracing after | |
250 | * returning from the function. | |
251 | */ | |
252 | if (ftrace_graph_notrace_addr(trace->func)) | |
253 | return 1; | |
254 | ||
542181d3 | 255 | if (!func_prolog_preempt_disable(tr, &data, &pc)) |
7495a5be JO |
256 | return 0; |
257 | ||
7495a5be JO |
258 | local_save_flags(flags); |
259 | ret = __trace_graph_entry(tr, trace, flags, pc); | |
7495a5be | 260 | atomic_dec(&data->disabled); |
7495a5be | 261 | preempt_enable_notrace(); |
542181d3 | 262 | |
7495a5be JO |
263 | return ret; |
264 | } | |
265 | ||
266 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) | |
267 | { | |
268 | struct trace_array *tr = wakeup_trace; | |
269 | struct trace_array_cpu *data; | |
270 | unsigned long flags; | |
542181d3 | 271 | int pc; |
7495a5be | 272 | |
542181d3 | 273 | if (!func_prolog_preempt_disable(tr, &data, &pc)) |
7495a5be JO |
274 | return; |
275 | ||
7495a5be JO |
276 | local_save_flags(flags); |
277 | __trace_graph_return(tr, trace, flags, pc); | |
7495a5be JO |
278 | atomic_dec(&data->disabled); |
279 | ||
7495a5be JO |
280 | preempt_enable_notrace(); |
281 | return; | |
282 | } | |
283 | ||
284 | static void wakeup_trace_open(struct trace_iterator *iter) | |
285 | { | |
983f938a | 286 | if (is_graph(iter->tr)) |
7495a5be JO |
287 | graph_trace_open(iter); |
288 | } | |
289 | ||
290 | static void wakeup_trace_close(struct trace_iterator *iter) | |
291 | { | |
292 | if (iter->private) | |
293 | graph_trace_close(iter); | |
294 | } | |
295 | ||
321e68b0 JO |
296 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \ |
297 | TRACE_GRAPH_PRINT_ABS_TIME | \ | |
298 | TRACE_GRAPH_PRINT_DURATION) | |
7495a5be JO |
299 | |
300 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | |
301 | { | |
302 | /* | |
303 | * In graph mode call the graph tracer output function, | |
304 | * otherwise go with the TRACE_FN event handler | |
305 | */ | |
983f938a | 306 | if (is_graph(iter->tr)) |
7495a5be JO |
307 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); |
308 | ||
309 | return TRACE_TYPE_UNHANDLED; | |
310 | } | |
311 | ||
312 | static void wakeup_print_header(struct seq_file *s) | |
313 | { | |
983f938a | 314 | if (is_graph(wakeup_trace)) |
7495a5be JO |
315 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); |
316 | else | |
317 | trace_default_header(s); | |
318 | } | |
319 | ||
320 | static void | |
321 | __trace_function(struct trace_array *tr, | |
322 | unsigned long ip, unsigned long parent_ip, | |
323 | unsigned long flags, int pc) | |
324 | { | |
983f938a | 325 | if (is_graph(tr)) |
7495a5be JO |
326 | trace_graph_function(tr, ip, parent_ip, flags, pc); |
327 | else | |
328 | trace_function(tr, ip, parent_ip, flags, pc); | |
329 | } | |
330 | #else | |
331 | #define __trace_function trace_function | |
332 | ||
7495a5be JO |
333 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) |
334 | { | |
335 | return TRACE_TYPE_UNHANDLED; | |
336 | } | |
337 | ||
7495a5be JO |
338 | static void wakeup_trace_open(struct trace_iterator *iter) { } |
339 | static void wakeup_trace_close(struct trace_iterator *iter) { } | |
7e9a49ef JO |
340 | |
341 | #ifdef CONFIG_FUNCTION_TRACER | |
8179e8a1 SRRH |
342 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) |
343 | { | |
344 | return -1; | |
345 | } | |
346 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) { } | |
7e9a49ef JO |
347 | static void wakeup_print_header(struct seq_file *s) |
348 | { | |
349 | trace_default_header(s); | |
350 | } | |
351 | #else | |
352 | static void wakeup_print_header(struct seq_file *s) | |
353 | { | |
354 | trace_latency_header(s); | |
355 | } | |
356 | #endif /* CONFIG_FUNCTION_TRACER */ | |
7495a5be JO |
357 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
358 | ||
352ad25a SR |
359 | /* |
360 | * Should this new latency be reported/recorded? | |
361 | */ | |
a5a1d1c2 | 362 | static bool report_latency(struct trace_array *tr, u64 delta) |
352ad25a SR |
363 | { |
364 | if (tracing_thresh) { | |
365 | if (delta < tracing_thresh) | |
26ab2ef4 | 366 | return false; |
352ad25a | 367 | } else { |
6d9b3fa5 | 368 | if (delta <= tr->max_latency) |
26ab2ef4 | 369 | return false; |
352ad25a | 370 | } |
26ab2ef4 | 371 | return true; |
352ad25a SR |
372 | } |
373 | ||
38516ab5 SR |
374 | static void |
375 | probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu) | |
478142c3 SR |
376 | { |
377 | if (task != wakeup_task) | |
378 | return; | |
379 | ||
380 | wakeup_current_cpu = cpu; | |
381 | } | |
382 | ||
243f7610 SRRH |
383 | static void |
384 | tracing_sched_switch_trace(struct trace_array *tr, | |
385 | struct task_struct *prev, | |
386 | struct task_struct *next, | |
387 | unsigned long flags, int pc) | |
388 | { | |
2425bcb9 | 389 | struct trace_event_call *call = &event_context_switch; |
243f7610 SRRH |
390 | struct ring_buffer *buffer = tr->trace_buffer.buffer; |
391 | struct ring_buffer_event *event; | |
392 | struct ctx_switch_entry *entry; | |
393 | ||
394 | event = trace_buffer_lock_reserve(buffer, TRACE_CTX, | |
395 | sizeof(*entry), flags, pc); | |
396 | if (!event) | |
397 | return; | |
398 | entry = ring_buffer_event_data(event); | |
399 | entry->prev_pid = prev->pid; | |
400 | entry->prev_prio = prev->prio; | |
5f6ad26e | 401 | entry->prev_state = __get_task_state(prev); |
243f7610 SRRH |
402 | entry->next_pid = next->pid; |
403 | entry->next_prio = next->prio; | |
5f6ad26e | 404 | entry->next_state = __get_task_state(next); |
243f7610 SRRH |
405 | entry->next_cpu = task_cpu(next); |
406 | ||
407 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
b7f0c959 | 408 | trace_buffer_unlock_commit(tr, buffer, event, flags, pc); |
243f7610 SRRH |
409 | } |
410 | ||
411 | static void | |
412 | tracing_sched_wakeup_trace(struct trace_array *tr, | |
413 | struct task_struct *wakee, | |
414 | struct task_struct *curr, | |
415 | unsigned long flags, int pc) | |
416 | { | |
2425bcb9 | 417 | struct trace_event_call *call = &event_wakeup; |
243f7610 SRRH |
418 | struct ring_buffer_event *event; |
419 | struct ctx_switch_entry *entry; | |
420 | struct ring_buffer *buffer = tr->trace_buffer.buffer; | |
421 | ||
422 | event = trace_buffer_lock_reserve(buffer, TRACE_WAKE, | |
423 | sizeof(*entry), flags, pc); | |
424 | if (!event) | |
425 | return; | |
426 | entry = ring_buffer_event_data(event); | |
427 | entry->prev_pid = curr->pid; | |
428 | entry->prev_prio = curr->prio; | |
5f6ad26e | 429 | entry->prev_state = __get_task_state(curr); |
243f7610 SRRH |
430 | entry->next_pid = wakee->pid; |
431 | entry->next_prio = wakee->prio; | |
5f6ad26e | 432 | entry->next_state = __get_task_state(wakee); |
243f7610 SRRH |
433 | entry->next_cpu = task_cpu(wakee); |
434 | ||
435 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
b7f0c959 | 436 | trace_buffer_unlock_commit(tr, buffer, event, flags, pc); |
243f7610 SRRH |
437 | } |
438 | ||
5b82a1b0 | 439 | static void notrace |
c73464b1 | 440 | probe_wakeup_sched_switch(void *ignore, bool preempt, |
38516ab5 | 441 | struct task_struct *prev, struct task_struct *next) |
352ad25a | 442 | { |
352ad25a | 443 | struct trace_array_cpu *data; |
a5a1d1c2 | 444 | u64 T0, T1, delta; |
352ad25a SR |
445 | unsigned long flags; |
446 | long disabled; | |
447 | int cpu; | |
38697053 | 448 | int pc; |
352ad25a | 449 | |
b07c3f19 MD |
450 | tracing_record_cmdline(prev); |
451 | ||
352ad25a SR |
452 | if (unlikely(!tracer_enabled)) |
453 | return; | |
454 | ||
455 | /* | |
456 | * When we start a new trace, we set wakeup_task to NULL | |
457 | * and then set tracer_enabled = 1. We want to make sure | |
458 | * that another CPU does not see the tracer_enabled = 1 | |
459 | * and the wakeup_task with an older task, that might | |
460 | * actually be the same as next. | |
461 | */ | |
462 | smp_rmb(); | |
463 | ||
464 | if (next != wakeup_task) | |
465 | return; | |
466 | ||
38697053 SR |
467 | pc = preempt_count(); |
468 | ||
352ad25a SR |
469 | /* disable local data, not wakeup_cpu data */ |
470 | cpu = raw_smp_processor_id(); | |
12883efb | 471 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
472 | if (likely(disabled != 1)) |
473 | goto out; | |
474 | ||
e59494f4 | 475 | local_irq_save(flags); |
0199c4e6 | 476 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
477 | |
478 | /* We could race with grabbing wakeup_lock */ | |
479 | if (unlikely(!tracer_enabled || next != wakeup_task)) | |
480 | goto out_unlock; | |
481 | ||
9be24414 | 482 | /* The task we are waiting for is waking up */ |
12883efb | 483 | data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu); |
9be24414 | 484 | |
7495a5be | 485 | __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc); |
7be42151 | 486 | tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc); |
352ad25a | 487 | |
352ad25a | 488 | T0 = data->preempt_timestamp; |
750ed1a4 | 489 | T1 = ftrace_now(cpu); |
352ad25a SR |
490 | delta = T1-T0; |
491 | ||
6d9b3fa5 | 492 | if (!report_latency(wakeup_trace, delta)) |
352ad25a SR |
493 | goto out_unlock; |
494 | ||
b5130b1e | 495 | if (likely(!is_tracing_stopped())) { |
6d9b3fa5 | 496 | wakeup_trace->max_latency = delta; |
b5130b1e CE |
497 | update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu); |
498 | } | |
352ad25a | 499 | |
352ad25a | 500 | out_unlock: |
b07c3f19 | 501 | __wakeup_reset(wakeup_trace); |
0199c4e6 | 502 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 503 | local_irq_restore(flags); |
352ad25a | 504 | out: |
12883efb | 505 | atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
5b82a1b0 MD |
506 | } |
507 | ||
e309b41d | 508 | static void __wakeup_reset(struct trace_array *tr) |
352ad25a | 509 | { |
352ad25a SR |
510 | wakeup_cpu = -1; |
511 | wakeup_prio = -1; | |
af6ace76 | 512 | tracing_dl = 0; |
352ad25a SR |
513 | |
514 | if (wakeup_task) | |
515 | put_task_struct(wakeup_task); | |
516 | ||
517 | wakeup_task = NULL; | |
518 | } | |
519 | ||
e309b41d | 520 | static void wakeup_reset(struct trace_array *tr) |
352ad25a SR |
521 | { |
522 | unsigned long flags; | |
523 | ||
12883efb | 524 | tracing_reset_online_cpus(&tr->trace_buffer); |
2f26ebd5 | 525 | |
e59494f4 | 526 | local_irq_save(flags); |
0199c4e6 | 527 | arch_spin_lock(&wakeup_lock); |
352ad25a | 528 | __wakeup_reset(tr); |
0199c4e6 | 529 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 530 | local_irq_restore(flags); |
352ad25a SR |
531 | } |
532 | ||
e309b41d | 533 | static void |
fbd705a0 | 534 | probe_wakeup(void *ignore, struct task_struct *p) |
352ad25a | 535 | { |
f8ec1062 | 536 | struct trace_array_cpu *data; |
352ad25a SR |
537 | int cpu = smp_processor_id(); |
538 | unsigned long flags; | |
539 | long disabled; | |
38697053 | 540 | int pc; |
352ad25a | 541 | |
b07c3f19 MD |
542 | if (likely(!tracer_enabled)) |
543 | return; | |
544 | ||
545 | tracing_record_cmdline(p); | |
546 | tracing_record_cmdline(current); | |
547 | ||
af6ace76 DF |
548 | /* |
549 | * Semantic is like this: | |
550 | * - wakeup tracer handles all tasks in the system, independently | |
551 | * from their scheduling class; | |
552 | * - wakeup_rt tracer handles tasks belonging to sched_dl and | |
553 | * sched_rt class; | |
554 | * - wakeup_dl handles tasks belonging to sched_dl class only. | |
555 | */ | |
556 | if (tracing_dl || (wakeup_dl && !dl_task(p)) || | |
557 | (wakeup_rt && !dl_task(p) && !rt_task(p)) || | |
558 | (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio))) | |
352ad25a SR |
559 | return; |
560 | ||
38697053 | 561 | pc = preempt_count(); |
12883efb | 562 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
563 | if (unlikely(disabled != 1)) |
564 | goto out; | |
565 | ||
566 | /* interrupts should be off from try_to_wake_up */ | |
0199c4e6 | 567 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
568 | |
569 | /* check for races. */ | |
af6ace76 DF |
570 | if (!tracer_enabled || tracing_dl || |
571 | (!dl_task(p) && p->prio >= wakeup_prio)) | |
352ad25a SR |
572 | goto out_locked; |
573 | ||
574 | /* reset the trace */ | |
b07c3f19 | 575 | __wakeup_reset(wakeup_trace); |
352ad25a SR |
576 | |
577 | wakeup_cpu = task_cpu(p); | |
478142c3 | 578 | wakeup_current_cpu = wakeup_cpu; |
352ad25a SR |
579 | wakeup_prio = p->prio; |
580 | ||
af6ace76 DF |
581 | /* |
582 | * Once you start tracing a -deadline task, don't bother tracing | |
583 | * another task until the first one wakes up. | |
584 | */ | |
585 | if (dl_task(p)) | |
586 | tracing_dl = 1; | |
587 | else | |
588 | tracing_dl = 0; | |
589 | ||
352ad25a SR |
590 | wakeup_task = p; |
591 | get_task_struct(wakeup_task); | |
592 | ||
593 | local_save_flags(flags); | |
594 | ||
12883efb | 595 | data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu); |
f8ec1062 | 596 | data->preempt_timestamp = ftrace_now(cpu); |
7be42151 | 597 | tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc); |
301fd748 SR |
598 | |
599 | /* | |
600 | * We must be careful in using CALLER_ADDR2. But since wake_up | |
601 | * is not called by an assembly function (where as schedule is) | |
602 | * it should be safe to use it here. | |
603 | */ | |
7495a5be | 604 | __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc); |
352ad25a SR |
605 | |
606 | out_locked: | |
0199c4e6 | 607 | arch_spin_unlock(&wakeup_lock); |
352ad25a | 608 | out: |
12883efb | 609 | atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled); |
352ad25a SR |
610 | } |
611 | ||
e309b41d | 612 | static void start_wakeup_tracer(struct trace_array *tr) |
352ad25a | 613 | { |
5b82a1b0 MD |
614 | int ret; |
615 | ||
38516ab5 | 616 | ret = register_trace_sched_wakeup(probe_wakeup, NULL); |
5b82a1b0 | 617 | if (ret) { |
b07c3f19 | 618 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
619 | " probe to kernel_sched_wakeup\n"); |
620 | return; | |
621 | } | |
622 | ||
38516ab5 | 623 | ret = register_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 624 | if (ret) { |
b07c3f19 | 625 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
626 | " probe to kernel_sched_wakeup_new\n"); |
627 | goto fail_deprobe; | |
628 | } | |
629 | ||
38516ab5 | 630 | ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
5b82a1b0 | 631 | if (ret) { |
b07c3f19 | 632 | pr_info("sched trace: Couldn't activate tracepoint" |
73d8b8bc | 633 | " probe to kernel_sched_switch\n"); |
5b82a1b0 MD |
634 | goto fail_deprobe_wake_new; |
635 | } | |
636 | ||
38516ab5 | 637 | ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); |
478142c3 SR |
638 | if (ret) { |
639 | pr_info("wakeup trace: Couldn't activate tracepoint" | |
640 | " probe to kernel_sched_migrate_task\n"); | |
641 | return; | |
642 | } | |
643 | ||
352ad25a SR |
644 | wakeup_reset(tr); |
645 | ||
646 | /* | |
647 | * Don't let the tracer_enabled = 1 show up before | |
648 | * the wakeup_task is reset. This may be overkill since | |
649 | * wakeup_reset does a spin_unlock after setting the | |
650 | * wakeup_task to NULL, but I want to be safe. | |
651 | * This is a slow path anyway. | |
652 | */ | |
653 | smp_wmb(); | |
654 | ||
983f938a | 655 | if (start_func_tracer(tr, is_graph(tr))) |
7495a5be | 656 | printk(KERN_ERR "failed to start wakeup tracer\n"); |
ad591240 | 657 | |
352ad25a | 658 | return; |
5b82a1b0 | 659 | fail_deprobe_wake_new: |
38516ab5 | 660 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 661 | fail_deprobe: |
38516ab5 | 662 | unregister_trace_sched_wakeup(probe_wakeup, NULL); |
352ad25a SR |
663 | } |
664 | ||
e309b41d | 665 | static void stop_wakeup_tracer(struct trace_array *tr) |
352ad25a SR |
666 | { |
667 | tracer_enabled = 0; | |
983f938a | 668 | stop_func_tracer(tr, is_graph(tr)); |
38516ab5 SR |
669 | unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
670 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); | |
671 | unregister_trace_sched_wakeup(probe_wakeup, NULL); | |
672 | unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); | |
352ad25a SR |
673 | } |
674 | ||
65daaca7 SRRH |
675 | static bool wakeup_busy; |
676 | ||
3244351c | 677 | static int __wakeup_tracer_init(struct trace_array *tr) |
352ad25a | 678 | { |
983f938a | 679 | save_flags = tr->trace_flags; |
613f04a0 SRRH |
680 | |
681 | /* non overwrite screws up the latency tracers */ | |
2b6080f2 SR |
682 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
683 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); | |
e9d25fe6 | 684 | |
6d9b3fa5 | 685 | tr->max_latency = 0; |
352ad25a | 686 | wakeup_trace = tr; |
4104d326 | 687 | ftrace_init_array_ops(tr, wakeup_tracer_call); |
c76f0694 | 688 | start_wakeup_tracer(tr); |
65daaca7 SRRH |
689 | |
690 | wakeup_busy = true; | |
1c80025a | 691 | return 0; |
352ad25a SR |
692 | } |
693 | ||
3244351c SR |
694 | static int wakeup_tracer_init(struct trace_array *tr) |
695 | { | |
65daaca7 SRRH |
696 | if (wakeup_busy) |
697 | return -EBUSY; | |
698 | ||
af6ace76 | 699 | wakeup_dl = 0; |
3244351c SR |
700 | wakeup_rt = 0; |
701 | return __wakeup_tracer_init(tr); | |
702 | } | |
703 | ||
704 | static int wakeup_rt_tracer_init(struct trace_array *tr) | |
705 | { | |
65daaca7 SRRH |
706 | if (wakeup_busy) |
707 | return -EBUSY; | |
708 | ||
af6ace76 | 709 | wakeup_dl = 0; |
3244351c SR |
710 | wakeup_rt = 1; |
711 | return __wakeup_tracer_init(tr); | |
712 | } | |
713 | ||
af6ace76 DF |
714 | static int wakeup_dl_tracer_init(struct trace_array *tr) |
715 | { | |
65daaca7 SRRH |
716 | if (wakeup_busy) |
717 | return -EBUSY; | |
718 | ||
af6ace76 DF |
719 | wakeup_dl = 1; |
720 | wakeup_rt = 0; | |
721 | return __wakeup_tracer_init(tr); | |
722 | } | |
723 | ||
e309b41d | 724 | static void wakeup_tracer_reset(struct trace_array *tr) |
352ad25a | 725 | { |
613f04a0 SRRH |
726 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
727 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; | |
728 | ||
c76f0694 SR |
729 | stop_wakeup_tracer(tr); |
730 | /* make sure we put back any tasks we are tracing */ | |
731 | wakeup_reset(tr); | |
e9d25fe6 | 732 | |
2b6080f2 SR |
733 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
734 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); | |
4104d326 | 735 | ftrace_reset_array_ops(tr); |
65daaca7 | 736 | wakeup_busy = false; |
352ad25a SR |
737 | } |
738 | ||
9036990d SR |
739 | static void wakeup_tracer_start(struct trace_array *tr) |
740 | { | |
741 | wakeup_reset(tr); | |
742 | tracer_enabled = 1; | |
9036990d SR |
743 | } |
744 | ||
745 | static void wakeup_tracer_stop(struct trace_array *tr) | |
746 | { | |
747 | tracer_enabled = 0; | |
352ad25a SR |
748 | } |
749 | ||
750 | static struct tracer wakeup_tracer __read_mostly = | |
751 | { | |
752 | .name = "wakeup", | |
753 | .init = wakeup_tracer_init, | |
754 | .reset = wakeup_tracer_reset, | |
9036990d SR |
755 | .start = wakeup_tracer_start, |
756 | .stop = wakeup_tracer_stop, | |
f43c738b | 757 | .print_max = true, |
7495a5be JO |
758 | .print_header = wakeup_print_header, |
759 | .print_line = wakeup_print_line, | |
328df475 | 760 | .flag_changed = wakeup_flag_changed, |
60a11774 SR |
761 | #ifdef CONFIG_FTRACE_SELFTEST |
762 | .selftest = trace_selftest_startup_wakeup, | |
763 | #endif | |
7495a5be JO |
764 | .open = wakeup_trace_open, |
765 | .close = wakeup_trace_close, | |
65daaca7 | 766 | .allow_instances = true, |
f43c738b | 767 | .use_max_tr = true, |
352ad25a SR |
768 | }; |
769 | ||
3244351c SR |
770 | static struct tracer wakeup_rt_tracer __read_mostly = |
771 | { | |
772 | .name = "wakeup_rt", | |
773 | .init = wakeup_rt_tracer_init, | |
774 | .reset = wakeup_tracer_reset, | |
775 | .start = wakeup_tracer_start, | |
776 | .stop = wakeup_tracer_stop, | |
f43c738b | 777 | .print_max = true, |
7495a5be JO |
778 | .print_header = wakeup_print_header, |
779 | .print_line = wakeup_print_line, | |
328df475 | 780 | .flag_changed = wakeup_flag_changed, |
3244351c SR |
781 | #ifdef CONFIG_FTRACE_SELFTEST |
782 | .selftest = trace_selftest_startup_wakeup, | |
783 | #endif | |
7495a5be JO |
784 | .open = wakeup_trace_open, |
785 | .close = wakeup_trace_close, | |
65daaca7 | 786 | .allow_instances = true, |
f43c738b | 787 | .use_max_tr = true, |
3244351c SR |
788 | }; |
789 | ||
af6ace76 DF |
790 | static struct tracer wakeup_dl_tracer __read_mostly = |
791 | { | |
792 | .name = "wakeup_dl", | |
793 | .init = wakeup_dl_tracer_init, | |
794 | .reset = wakeup_tracer_reset, | |
795 | .start = wakeup_tracer_start, | |
796 | .stop = wakeup_tracer_stop, | |
af6ace76 DF |
797 | .print_max = true, |
798 | .print_header = wakeup_print_header, | |
799 | .print_line = wakeup_print_line, | |
af6ace76 DF |
800 | .flag_changed = wakeup_flag_changed, |
801 | #ifdef CONFIG_FTRACE_SELFTEST | |
802 | .selftest = trace_selftest_startup_wakeup, | |
803 | #endif | |
804 | .open = wakeup_trace_open, | |
805 | .close = wakeup_trace_close, | |
8d414bd2 | 806 | .allow_instances = true, |
af6ace76 DF |
807 | .use_max_tr = true, |
808 | }; | |
809 | ||
352ad25a SR |
810 | __init static int init_wakeup_tracer(void) |
811 | { | |
812 | int ret; | |
813 | ||
814 | ret = register_tracer(&wakeup_tracer); | |
815 | if (ret) | |
816 | return ret; | |
817 | ||
3244351c SR |
818 | ret = register_tracer(&wakeup_rt_tracer); |
819 | if (ret) | |
820 | return ret; | |
821 | ||
af6ace76 DF |
822 | ret = register_tracer(&wakeup_dl_tracer); |
823 | if (ret) | |
824 | return ret; | |
825 | ||
352ad25a SR |
826 | return 0; |
827 | } | |
6f415672 | 828 | core_initcall(init_wakeup_tracer); |