kernel/trace/trace_sched_wakeup.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace task wakeup timings
   4  *
   5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   6  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   7  *
   8  * Based on code from the latency_tracer, that is:
   9  *
  10  *  Copyright (C) 2004-2006 Ingo Molnar
  11  *  Copyright (C) 2004 Nadia Yvette Chambers
  12  */
  13 #include <linux/module.h>
  14 #include <linux/kallsyms.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/ftrace.h>
  17 #include <linux/sched/rt.h>
  18 #include <linux/sched/deadline.h>
  19 #include <trace/events/sched.h>
  20 #include "trace.h"
  21
  22 static struct trace_array       *wakeup_trace;
  23 static int __read_mostly        tracer_enabled;
  24
  25 static struct task_struct       *wakeup_task;
  26 static int                      wakeup_cpu;
  27 static int                      wakeup_current_cpu;
  28 static unsigned                 wakeup_prio = -1;
  29 static int                      wakeup_rt;
  30 static int                      wakeup_dl;
  31 static int                      tracing_dl = 0;
  32
  33 static arch_spinlock_t wakeup_lock =
  34         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  35
  36 static void wakeup_reset(struct trace_array *tr);
  37 static void __wakeup_reset(struct trace_array *tr);
  38
  39 static int save_flags;
  40
  41 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  42 static int wakeup_display_graph(struct trace_array *tr, int set);
  43 # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
  44 #else
  45 static inline int wakeup_display_graph(struct trace_array *tr, int set)
  46 {
  47         return 0;
  48 }
  49 # define is_graph(tr) false
  50 #endif
  51
  52
  53 #ifdef CONFIG_FUNCTION_TRACER
  54
  55 static int wakeup_graph_entry(struct ftrace_graph_ent *trace);
  56 static void wakeup_graph_return(struct ftrace_graph_ret *trace);
  57
  58 static bool function_enabled;
  59
  60 /*
  61  * Prologue for the wakeup function tracers.
  62  *
  63  * Returns 1 if it is OK to continue, and preemption
  64  *            is disabled and data->disabled is incremented.
  65  *         0 if the trace is to be ignored, and preemption
  66  *            is not disabled and data->disabled is
  67  *            kept the same.
  68  *
  69  * Note, this function is also used outside this ifdef but
  70  *  inside the #ifdef of the function graph tracer below.
  71  *  This is OK, since the function graph tracer is
  72  *  dependent on the function tracer.
  73  */
  74 static int
  75 func_prolog_preempt_disable(struct trace_array *tr,
  76                             struct trace_array_cpu **data,
  77                             int *pc)
  78 {
  79         long disabled;
  80         int cpu;
  81
  82         if (likely(!wakeup_task))
  83                 return 0;
  84
  85         *pc = preempt_count();
  86         preempt_disable_notrace();
  87
  88         cpu = raw_smp_processor_id();
  89         if (cpu != wakeup_current_cpu)
  90                 goto out_enable;
  91
  92         *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
  93         disabled = atomic_inc_return(&(*data)->disabled);
  94         if (unlikely(disabled != 1))
  95                 goto out;
  96
  97         return 1;
  98
  99 out:
 100         atomic_dec(&(*data)->disabled);
 101
 102 out_enable:
 103         preempt_enable_notrace();
 104         return 0;
 105 }
 106
 107 /*
 108  * wakeup uses its own tracer function to keep the overhead down:
 109  */
 110 static void
 111 wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
 112                    struct ftrace_ops *op, struct pt_regs *pt_regs)
 113 {
 114         struct trace_array *tr = wakeup_trace;
 115         struct trace_array_cpu *data;
 116         unsigned long flags;
 117         int pc;
 118
 119         if (!func_prolog_preempt_disable(tr, &data, &pc))
 120                 return;
 121
 122         local_irq_save(flags);
 123         trace_function(tr, ip, parent_ip, flags, pc);
 124         local_irq_restore(flags);
 125
 126         atomic_dec(&data->disabled);
 127         preempt_enable_notrace();
 128 }
 129
 130 static int register_wakeup_function(struct trace_array *tr, int graph, int set)
 131 {
 132         int ret;
 133
 134         /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
 135         if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
 136                 return 0;
 137
 138         if (graph)
 139                 ret = register_ftrace_graph(&wakeup_graph_return,
 140                                             &wakeup_graph_entry);
 141         else
 142                 ret = register_ftrace_function(tr->ops);
 143
 144         if (!ret)
 145                 function_enabled = true;
 146
 147         return ret;
 148 }
 149
 150 static void unregister_wakeup_function(struct trace_array *tr, int graph)
 151 {
 152         if (!function_enabled)
 153                 return;
 154
 155         if (graph)
 156                 unregister_ftrace_graph();
 157         else
 158                 unregister_ftrace_function(tr->ops);
 159
 160         function_enabled = false;
 161 }
 162
 163 static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
 164 {
 165         if (!(mask & TRACE_ITER_FUNCTION))
 166                 return 0;
 167
 168         if (set)
 169                 register_wakeup_function(tr, is_graph(tr), 1);
 170         else
 171                 unregister_wakeup_function(tr, is_graph(tr));
 172         return 1;
 173 }
 174 #else
 175 static int register_wakeup_function(struct trace_array *tr, int graph, int set)
 176 {
 177         return 0;
 178 }
 179 static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
 180 static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
 181 {
 182         return 0;
 183 }
 184 #endif /* CONFIG_FUNCTION_TRACER */
 185
 186 static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
 187 {
 188         struct tracer *tracer = tr->current_trace;
 189
 190         if (wakeup_function_set(tr, mask, set))
 191                 return 0;
 192
 193 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 194         if (mask & TRACE_ITER_DISPLAY_GRAPH)
 195                 return wakeup_display_graph(tr, set);
 196 #endif
 197
 198         return trace_keep_overwrite(tracer, mask, set);
 199 }
 200
 201 static int start_func_tracer(struct trace_array *tr, int graph)
 202 {
 203         int ret;
 204
 205         ret = register_wakeup_function(tr, graph, 0);
 206
 207         if (!ret && tracing_is_enabled())
 208                 tracer_enabled = 1;
 209         else
 210                 tracer_enabled = 0;
 211
 212         return ret;
 213 }
 214
 215 static void stop_func_tracer(struct trace_array *tr, int graph)
 216 {
 217         tracer_enabled = 0;
 218
 219         unregister_wakeup_function(tr, graph);
 220 }
 221
 222 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 223 static int wakeup_display_graph(struct trace_array *tr, int set)
 224 {
 225         if (!(is_graph(tr) ^ set))
 226                 return 0;
 227
 228         stop_func_tracer(tr, !set);
 229
 230         wakeup_reset(wakeup_trace);
 231         tr->max_latency = 0;
 232
 233         return start_func_tracer(tr, set);
 234 }
 235
 236 static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
 237 {
 238         struct trace_array *tr = wakeup_trace;
 239         struct trace_array_cpu *data;
 240         unsigned long flags;
 241         int pc, ret = 0;
 242
 243         if (ftrace_graph_ignore_func(trace))
 244                 return 0;
 245         /*
 246          * Do not trace a function if it's filtered by set_graph_notrace.
 247          * Make the index of ret stack negative to indicate that it should
 248          * ignore further functions.  But it needs its own ret stack entry
 249          * to recover the original index in order to continue tracing after
 250          * returning from the function.
 251          */
 252         if (ftrace_graph_notrace_addr(trace->func))
 253                 return 1;
 254
 255         if (!func_prolog_preempt_disable(tr, &data, &pc))
 256                 return 0;
 257
 258         local_save_flags(flags);
 259         ret = __trace_graph_entry(tr, trace, flags, pc);
 260         atomic_dec(&data->disabled);
 261         preempt_enable_notrace();
 262
 263         return ret;
 264 }
 265
 266 static void wakeup_graph_return(struct ftrace_graph_ret *trace)
 267 {
 268         struct trace_array *tr = wakeup_trace;
 269         struct trace_array_cpu *data;
 270         unsigned long flags;
 271         int pc;
 272
 273         ftrace_graph_addr_finish(trace);
 274
 275         if (!func_prolog_preempt_disable(tr, &data, &pc))
 276                 return;
 277
 278         local_save_flags(flags);
 279         __trace_graph_return(tr, trace, flags, pc);
 280         atomic_dec(&data->disabled);
 281
 282         preempt_enable_notrace();
 283         return;
 284 }
 285
 286 static void wakeup_trace_open(struct trace_iterator *iter)
 287 {
 288         if (is_graph(iter->tr))
 289                 graph_trace_open(iter);
 290 }
 291
 292 static void wakeup_trace_close(struct trace_iterator *iter)
 293 {
 294         if (iter->private)
 295                 graph_trace_close(iter);
 296 }
 297
 298 #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
 299                             TRACE_GRAPH_PRINT_ABS_TIME | \
 300                             TRACE_GRAPH_PRINT_DURATION)
 301
 302 static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
 303 {
 304         /*
 305          * In graph mode call the graph tracer output function,
 306          * otherwise go with the TRACE_FN event handler
 307          */
 308         if (is_graph(iter->tr))
 309                 return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
 310
 311         return TRACE_TYPE_UNHANDLED;
 312 }
 313
 314 static void wakeup_print_header(struct seq_file *s)
 315 {
 316         if (is_graph(wakeup_trace))
 317                 print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
 318         else
 319                 trace_default_header(s);
 320 }
 321
 322 static void
 323 __trace_function(struct trace_array *tr,
 324                  unsigned long ip, unsigned long parent_ip,
 325                  unsigned long flags, int pc)
 326 {
 327         if (is_graph(tr))
 328                 trace_graph_function(tr, ip, parent_ip, flags, pc);
 329         else
 330                 trace_function(tr, ip, parent_ip, flags, pc);
 331 }
 332 #else
 333 #define __trace_function trace_function
 334
 335 static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
 336 {
 337         return TRACE_TYPE_UNHANDLED;
 338 }
 339
 340 static void wakeup_trace_open(struct trace_iterator *iter) { }
 341 static void wakeup_trace_close(struct trace_iterator *iter) { }
 342
 343 #ifdef CONFIG_FUNCTION_TRACER
 344 static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
 345 {
 346         return -1;
 347 }
 348 static void wakeup_graph_return(struct ftrace_graph_ret *trace) { }
 349 static void wakeup_print_header(struct seq_file *s)
 350 {
 351         trace_default_header(s);
 352 }
 353 #else
 354 static void wakeup_print_header(struct seq_file *s)
 355 {
 356         trace_latency_header(s);
 357 }
 358 #endif /* CONFIG_FUNCTION_TRACER */
 359 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 360
 361 /*
 362  * Should this new latency be reported/recorded?
 363  */
 364 static bool report_latency(struct trace_array *tr, u64 delta)
 365 {
 366         if (tracing_thresh) {
 367                 if (delta < tracing_thresh)
 368                         return false;
 369         } else {
 370                 if (delta <= tr->max_latency)
 371                         return false;
 372         }
 373         return true;
 374 }
 375
 376 static void
 377 probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
 378 {
 379         if (task != wakeup_task)
 380                 return;
 381
 382         wakeup_current_cpu = cpu;
 383 }
 384
 385 static void
 386 tracing_sched_switch_trace(struct trace_array *tr,
 387                            struct task_struct *prev,
 388                            struct task_struct *next,
 389                            unsigned long flags, int pc)
 390 {
 391         struct trace_event_call *call = &event_context_switch;
 392         struct ring_buffer *buffer = tr->trace_buffer.buffer;
 393         struct ring_buffer_event *event;
 394         struct ctx_switch_entry *entry;
 395
 396         event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
 397                                           sizeof(*entry), flags, pc);
 398         if (!event)
 399                 return;
 400         entry   = ring_buffer_event_data(event);
 401         entry->prev_pid                 = prev->pid;
 402         entry->prev_prio                = prev->prio;
 403         entry->prev_state               = task_state_index(prev);
 404         entry->next_pid                 = next->pid;
 405         entry->next_prio                = next->prio;
 406         entry->next_state               = task_state_index(next);
 407         entry->next_cpu = task_cpu(next);
 408
 409         if (!call_filter_check_discard(call, entry, buffer, event))
 410                 trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
 411 }
 412
 413 static void
 414 tracing_sched_wakeup_trace(struct trace_array *tr,
 415                            struct task_struct *wakee,
 416                            struct task_struct *curr,
 417                            unsigned long flags, int pc)
 418 {
 419         struct trace_event_call *call = &event_wakeup;
 420         struct ring_buffer_event *event;
 421         struct ctx_switch_entry *entry;
 422         struct ring_buffer *buffer = tr->trace_buffer.buffer;
 423
 424         event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
 425                                           sizeof(*entry), flags, pc);
 426         if (!event)
 427                 return;
 428         entry   = ring_buffer_event_data(event);
 429         entry->prev_pid                 = curr->pid;
 430         entry->prev_prio                = curr->prio;
 431         entry->prev_state               = task_state_index(curr);
 432         entry->next_pid                 = wakee->pid;
 433         entry->next_prio                = wakee->prio;
 434         entry->next_state               = task_state_index(wakee);
 435         entry->next_cpu                 = task_cpu(wakee);
 436
 437         if (!call_filter_check_discard(call, entry, buffer, event))
 438                 trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
 439 }
 440
 441 static void notrace
 442 probe_wakeup_sched_switch(void *ignore, bool preempt,
 443                           struct task_struct *prev, struct task_struct *next)
 444 {
 445         struct trace_array_cpu *data;
 446         u64 T0, T1, delta;
 447         unsigned long flags;
 448         long disabled;
 449         int cpu;
 450         int pc;
 451
 452         tracing_record_cmdline(prev);
 453
 454         if (unlikely(!tracer_enabled))
 455                 return;
 456
 457         /*
 458          * When we start a new trace, we set wakeup_task to NULL
 459          * and then set tracer_enabled = 1. We want to make sure
 460          * that another CPU does not see the tracer_enabled = 1
 461          * and the wakeup_task with an older task, that might
 462          * actually be the same as next.
 463          */
 464         smp_rmb();
 465
 466         if (next != wakeup_task)
 467                 return;
 468
 469         pc = preempt_count();
 470
 471         /* disable local data, not wakeup_cpu data */
 472         cpu = raw_smp_processor_id();
 473         disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 474         if (likely(disabled != 1))
 475                 goto out;
 476
 477         local_irq_save(flags);
 478         arch_spin_lock(&wakeup_lock);
 479
 480         /* We could race with grabbing wakeup_lock */
 481         if (unlikely(!tracer_enabled || next != wakeup_task))
 482                 goto out_unlock;
 483
 484         /* The task we are waiting for is waking up */
 485         data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
 486
 487         __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
 488         tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
 489
 490         T0 = data->preempt_timestamp;
 491         T1 = ftrace_now(cpu);
 492         delta = T1-T0;
 493
 494         if (!report_latency(wakeup_trace, delta))
 495                 goto out_unlock;
 496
 497         if (likely(!is_tracing_stopped())) {
 498                 wakeup_trace->max_latency = delta;
 499                 update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
 500         }
 501
 502 out_unlock:
 503         __wakeup_reset(wakeup_trace);
 504         arch_spin_unlock(&wakeup_lock);
 505         local_irq_restore(flags);
 506 out:
 507         atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 508 }
 509
 510 static void __wakeup_reset(struct trace_array *tr)
 511 {
 512         wakeup_cpu = -1;
 513         wakeup_prio = -1;
 514         tracing_dl = 0;
 515
 516         if (wakeup_task)
 517                 put_task_struct(wakeup_task);
 518
 519         wakeup_task = NULL;
 520 }
 521
 522 static void wakeup_reset(struct trace_array *tr)
 523 {
 524         unsigned long flags;
 525
 526         tracing_reset_online_cpus(&tr->trace_buffer);
 527
 528         local_irq_save(flags);
 529         arch_spin_lock(&wakeup_lock);
 530         __wakeup_reset(tr);
 531         arch_spin_unlock(&wakeup_lock);
 532         local_irq_restore(flags);
 533 }
 534
 535 static void
 536 probe_wakeup(void *ignore, struct task_struct *p)
 537 {
 538         struct trace_array_cpu *data;
 539         int cpu = smp_processor_id();
 540         unsigned long flags;
 541         long disabled;
 542         int pc;
 543
 544         if (likely(!tracer_enabled))
 545                 return;
 546
 547         tracing_record_cmdline(p);
 548         tracing_record_cmdline(current);
 549
 550         /*
 551          * Semantic is like this:
 552          *  - wakeup tracer handles all tasks in the system, independently
 553          *    from their scheduling class;
 554          *  - wakeup_rt tracer handles tasks belonging to sched_dl and
 555          *    sched_rt class;
 556          *  - wakeup_dl handles tasks belonging to sched_dl class only.
 557          */
 558         if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
 559             (wakeup_rt && !dl_task(p) && !rt_task(p)) ||
 560             (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
 561                 return;
 562
 563         pc = preempt_count();
 564         disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 565         if (unlikely(disabled != 1))
 566                 goto out;
 567
 568         /* interrupts should be off from try_to_wake_up */
 569         arch_spin_lock(&wakeup_lock);
 570
 571         /* check for races. */
 572         if (!tracer_enabled || tracing_dl ||
 573             (!dl_task(p) && p->prio >= wakeup_prio))
 574                 goto out_locked;
 575
 576         /* reset the trace */
 577         __wakeup_reset(wakeup_trace);
 578
 579         wakeup_cpu = task_cpu(p);
 580         wakeup_current_cpu = wakeup_cpu;
 581         wakeup_prio = p->prio;
 582
 583         /*
 584          * Once you start tracing a -deadline task, don't bother tracing
 585          * another task until the first one wakes up.
 586          */
 587         if (dl_task(p))
 588                 tracing_dl = 1;
 589         else
 590                 tracing_dl = 0;
 591
 592         wakeup_task = p;
 593         get_task_struct(wakeup_task);
 594
 595         local_save_flags(flags);
 596
 597         data = per_cpu_ptr(wakeup_trace->trace_buffer.data, wakeup_cpu);
 598         data->preempt_timestamp = ftrace_now(cpu);
 599         tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
 600
 601         /*
 602          * We must be careful in using CALLER_ADDR2. But since wake_up
 603          * is not called by an assembly function  (where as schedule is)
 604          * it should be safe to use it here.
 605          */
 606         __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
 607
 608 out_locked:
 609         arch_spin_unlock(&wakeup_lock);
 610 out:
 611         atomic_dec(&per_cpu_ptr(wakeup_trace->trace_buffer.data, cpu)->disabled);
 612 }
 613
 614 static void start_wakeup_tracer(struct trace_array *tr)
 615 {
 616         int ret;
 617
 618         ret = register_trace_sched_wakeup(probe_wakeup, NULL);
 619         if (ret) {
 620                 pr_info("wakeup trace: Couldn't activate tracepoint"
 621                         " probe to kernel_sched_wakeup\n");
 622                 return;
 623         }
 624
 625         ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
 626         if (ret) {
 627                 pr_info("wakeup trace: Couldn't activate tracepoint"
 628                         " probe to kernel_sched_wakeup_new\n");
 629                 goto fail_deprobe;
 630         }
 631
 632         ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
 633         if (ret) {
 634                 pr_info("sched trace: Couldn't activate tracepoint"
 635                         " probe to kernel_sched_switch\n");
 636                 goto fail_deprobe_wake_new;
 637         }
 638
 639         ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
 640         if (ret) {
 641                 pr_info("wakeup trace: Couldn't activate tracepoint"
 642                         " probe to kernel_sched_migrate_task\n");
 643                 goto fail_deprobe_sched_switch;
 644         }
 645
 646         wakeup_reset(tr);
 647
 648         /*
 649          * Don't let the tracer_enabled = 1 show up before
 650          * the wakeup_task is reset. This may be overkill since
 651          * wakeup_reset does a spin_unlock after setting the
 652          * wakeup_task to NULL, but I want to be safe.
 653          * This is a slow path anyway.
 654          */
 655         smp_wmb();
 656
 657         if (start_func_tracer(tr, is_graph(tr)))
 658                 printk(KERN_ERR "failed to start wakeup tracer\n");
 659
 660         return;
 661 fail_deprobe_sched_switch:
 662         unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
 663 fail_deprobe_wake_new:
 664         unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
 665 fail_deprobe:
 666         unregister_trace_sched_wakeup(probe_wakeup, NULL);
 667 }
 668
 669 static void stop_wakeup_tracer(struct trace_array *tr)
 670 {
 671         tracer_enabled = 0;
 672         stop_func_tracer(tr, is_graph(tr));
 673         unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
 674         unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
 675         unregister_trace_sched_wakeup(probe_wakeup, NULL);
 676         unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
 677 }
 678
 679 static bool wakeup_busy;
 680
 681 static int __wakeup_tracer_init(struct trace_array *tr)
 682 {
 683         save_flags = tr->trace_flags;
 684
 685         /* non overwrite screws up the latency tracers */
 686         set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
 687         set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
 688
 689         tr->max_latency = 0;
 690         wakeup_trace = tr;
 691         ftrace_init_array_ops(tr, wakeup_tracer_call);
 692         start_wakeup_tracer(tr);
 693
 694         wakeup_busy = true;
 695         return 0;
 696 }
 697
 698 static int wakeup_tracer_init(struct trace_array *tr)
 699 {
 700         if (wakeup_busy)
 701                 return -EBUSY;
 702
 703         wakeup_dl = 0;
 704         wakeup_rt = 0;
 705         return __wakeup_tracer_init(tr);
 706 }
 707
 708 static int wakeup_rt_tracer_init(struct trace_array *tr)
 709 {
 710         if (wakeup_busy)
 711                 return -EBUSY;
 712
 713         wakeup_dl = 0;
 714         wakeup_rt = 1;
 715         return __wakeup_tracer_init(tr);
 716 }
 717
 718 static int wakeup_dl_tracer_init(struct trace_array *tr)
 719 {
 720         if (wakeup_busy)
 721                 return -EBUSY;
 722
 723         wakeup_dl = 1;
 724         wakeup_rt = 0;
 725         return __wakeup_tracer_init(tr);
 726 }
 727
 728 static void wakeup_tracer_reset(struct trace_array *tr)
 729 {
 730         int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
 731         int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
 732
 733         stop_wakeup_tracer(tr);
 734         /* make sure we put back any tasks we are tracing */
 735         wakeup_reset(tr);
 736
 737         set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
 738         set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
 739         ftrace_reset_array_ops(tr);
 740         wakeup_busy = false;
 741 }
 742
 743 static void wakeup_tracer_start(struct trace_array *tr)
 744 {
 745         wakeup_reset(tr);
 746         tracer_enabled = 1;
 747 }
 748
 749 static void wakeup_tracer_stop(struct trace_array *tr)
 750 {
 751         tracer_enabled = 0;
 752 }
 753
 754 static struct tracer wakeup_tracer __read_mostly =
 755 {
 756         .name           = "wakeup",
 757         .init           = wakeup_tracer_init,
 758         .reset          = wakeup_tracer_reset,
 759         .start          = wakeup_tracer_start,
 760         .stop           = wakeup_tracer_stop,
 761         .print_max      = true,
 762         .print_header   = wakeup_print_header,
 763         .print_line     = wakeup_print_line,
 764         .flag_changed   = wakeup_flag_changed,
 765 #ifdef CONFIG_FTRACE_SELFTEST
 766         .selftest    = trace_selftest_startup_wakeup,
 767 #endif
 768         .open           = wakeup_trace_open,
 769         .close          = wakeup_trace_close,
 770         .allow_instances = true,
 771         .use_max_tr     = true,
 772 };
 773
 774 static struct tracer wakeup_rt_tracer __read_mostly =
 775 {
 776         .name           = "wakeup_rt",
 777         .init           = wakeup_rt_tracer_init,
 778         .reset          = wakeup_tracer_reset,
 779         .start          = wakeup_tracer_start,
 780         .stop           = wakeup_tracer_stop,
 781         .print_max      = true,
 782         .print_header   = wakeup_print_header,
 783         .print_line     = wakeup_print_line,
 784         .flag_changed   = wakeup_flag_changed,
 785 #ifdef CONFIG_FTRACE_SELFTEST
 786         .selftest    = trace_selftest_startup_wakeup,
 787 #endif
 788         .open           = wakeup_trace_open,
 789         .close          = wakeup_trace_close,
 790         .allow_instances = true,
 791         .use_max_tr     = true,
 792 };
 793
 794 static struct tracer wakeup_dl_tracer __read_mostly =
 795 {
 796         .name           = "wakeup_dl",
 797         .init           = wakeup_dl_tracer_init,
 798         .reset          = wakeup_tracer_reset,
 799         .start          = wakeup_tracer_start,
 800         .stop           = wakeup_tracer_stop,
 801         .print_max      = true,
 802         .print_header   = wakeup_print_header,
 803         .print_line     = wakeup_print_line,
 804         .flag_changed   = wakeup_flag_changed,
 805 #ifdef CONFIG_FTRACE_SELFTEST
 806         .selftest    = trace_selftest_startup_wakeup,
 807 #endif
 808         .open           = wakeup_trace_open,
 809         .close          = wakeup_trace_close,
 810         .allow_instances = true,
 811         .use_max_tr     = true,
 812 };
 813
 814 __init static int init_wakeup_tracer(void)
 815 {
 816         int ret;
 817
 818         ret = register_tracer(&wakeup_tracer);
 819         if (ret)
 820                 return ret;
 821
 822         ret = register_tracer(&wakeup_rt_tracer);
 823         if (ret)
 824                 return ret;
 825
 826         ret = register_tracer(&wakeup_dl_tracer);
 827         if (ret)
 828                 return ret;
 829
 830         return 0;
 831 }
 832 core_initcall(init_wakeup_tracer);