1 // SPDX-License-Identifier: GPL-2.0
3 * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4 * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
6 * Based on "hwlat_detector" tracer by:
7 * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8 * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9 * With feedback from Clark Williams <williams@redhat.com>
11 * And also based on the rtsl tracer presented on:
12 * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13 * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14 * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
16 * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
19 #include <linux/kthread.h>
20 #include <linux/tracefs.h>
21 #include <linux/uaccess.h>
22 #include <linux/cpumask.h>
23 #include <linux/delay.h>
24 #include <linux/sched/clock.h>
25 #include <uapi/linux/sched/types.h>
26 #include <linux/sched.h>
29 #ifdef CONFIG_X86_LOCAL_APIC
30 #include <asm/trace/irq_vectors.h>
31 #undef TRACE_INCLUDE_PATH
32 #undef TRACE_INCLUDE_FILE
33 #endif /* CONFIG_X86_LOCAL_APIC */
35 #include <trace/events/irq.h>
36 #include <trace/events/sched.h>
38 #define CREATE_TRACE_POINTS
39 #include <trace/events/osnoise.h>
41 static struct trace_array
*osnoise_trace
;
46 #define BANNER "osnoise: "
47 #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
48 #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
50 #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
51 #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
71 #define THREAD_CONTEXT 1
73 * sofirq runtime info.
82 * thread runtime info.
91 * Runtime information: this structure saves the runtime information used by
92 * one sampling thread.
94 struct osnoise_variables
{
95 struct task_struct
*kthread
;
100 struct osn_softirq softirq
;
101 struct osn_thread thread
;
106 * Per-cpu runtime information.
108 DEFINE_PER_CPU(struct osnoise_variables
, per_cpu_osnoise_var
);
111 * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
113 static inline struct osnoise_variables
*this_cpu_osn_var(void)
115 return this_cpu_ptr(&per_cpu_osnoise_var
);
118 #ifdef CONFIG_TIMERLAT_TRACER
120 * Runtime information for the timer mode.
122 struct timerlat_variables
{
123 struct task_struct
*kthread
;
124 struct hrtimer timer
;
131 DEFINE_PER_CPU(struct timerlat_variables
, per_cpu_timerlat_var
);
134 * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
136 static inline struct timerlat_variables
*this_cpu_tmr_var(void)
138 return this_cpu_ptr(&per_cpu_timerlat_var
);
142 * tlat_var_reset - Reset the values of the given timerlat_variables
144 static inline void tlat_var_reset(void)
146 struct timerlat_variables
*tlat_var
;
149 * So far, all the values are initialized as 0, so
150 * zeroing the structure is perfect.
152 for_each_cpu(cpu
, cpu_online_mask
) {
153 tlat_var
= per_cpu_ptr(&per_cpu_timerlat_var
, cpu
);
154 memset(tlat_var
, 0, sizeof(*tlat_var
));
157 #else /* CONFIG_TIMERLAT_TRACER */
158 #define tlat_var_reset() do {} while (0)
159 #endif /* CONFIG_TIMERLAT_TRACER */
162 * osn_var_reset - Reset the values of the given osnoise_variables
164 static inline void osn_var_reset(void)
166 struct osnoise_variables
*osn_var
;
170 * So far, all the values are initialized as 0, so
171 * zeroing the structure is perfect.
173 for_each_cpu(cpu
, cpu_online_mask
) {
174 osn_var
= per_cpu_ptr(&per_cpu_osnoise_var
, cpu
);
175 memset(osn_var
, 0, sizeof(*osn_var
));
180 * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
182 static inline void osn_var_reset_all(void)
189 * Tells NMIs to call back to the osnoise tracer to record timestamps.
191 bool trace_osnoise_callback_enabled
;
194 * osnoise sample structure definition. Used to store the statistics of a
197 struct osnoise_sample
{
198 u64 runtime
; /* runtime */
199 u64 noise
; /* noise */
200 u64 max_sample
; /* max single noise sample */
201 int hw_count
; /* # HW (incl. hypervisor) interference */
202 int nmi_count
; /* # NMIs during this sample */
203 int irq_count
; /* # IRQs during this sample */
204 int softirq_count
; /* # softirqs during this sample */
205 int thread_count
; /* # threads during this sample */
208 #ifdef CONFIG_TIMERLAT_TRACER
210 * timerlat sample structure definition. Used to store the statistics of
213 struct timerlat_sample
{
214 u64 timer_latency
; /* timer_latency */
215 unsigned int seqnum
; /* unique sequence */
216 int context
; /* timer context */
221 * Protect the interface.
223 struct mutex interface_lock
;
228 static struct osnoise_data
{
229 u64 sample_period
; /* total sampling period */
230 u64 sample_runtime
; /* active sampling portion of period */
231 u64 stop_tracing
; /* stop trace in the internal operation (loop/irq) */
232 u64 stop_tracing_total
; /* stop trace in the final operation (report/thread) */
233 #ifdef CONFIG_TIMERLAT_TRACER
234 u64 timerlat_period
; /* timerlat period */
235 u64 print_stack
; /* print IRQ stack if total > */
236 int timerlat_tracer
; /* timerlat tracer */
238 bool tainted
; /* infor users and developers about a problem */
240 .sample_period
= DEFAULT_SAMPLE_PERIOD
,
241 .sample_runtime
= DEFAULT_SAMPLE_RUNTIME
,
243 .stop_tracing_total
= 0,
244 #ifdef CONFIG_TIMERLAT_TRACER
246 .timerlat_period
= DEFAULT_TIMERLAT_PERIOD
,
247 .timerlat_tracer
= 0,
252 * Boolean variable used to inform that the tracer is currently sampling.
254 static bool osnoise_busy
;
256 #ifdef CONFIG_PREEMPT_RT
258 * Print the osnoise header info.
260 static void print_osnoise_headers(struct seq_file
*s
)
262 if (osnoise_data
.tainted
)
263 seq_puts(s
, "# osnoise is tainted!\n");
265 seq_puts(s
, "# _-------=> irqs-off\n");
266 seq_puts(s
, "# / _------=> need-resched\n");
267 seq_puts(s
, "# | / _-----=> need-resched-lazy\n");
268 seq_puts(s
, "# || / _----=> hardirq/softirq\n");
269 seq_puts(s
, "# ||| / _---=> preempt-depth\n");
270 seq_puts(s
, "# |||| / _--=> preempt-lazy-depth\n");
271 seq_puts(s
, "# ||||| / _-=> migrate-disable\n");
273 seq_puts(s
, "# |||||| / ");
274 seq_puts(s
, " MAX\n");
276 seq_puts(s
, "# ||||| / ");
277 seq_puts(s
, " SINGLE Interference counters:\n");
279 seq_puts(s
, "# ||||||| RUNTIME ");
280 seq_puts(s
, " NOISE %% OF CPU NOISE +-----------------------------+\n");
282 seq_puts(s
, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
283 seq_puts(s
, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
285 seq_puts(s
, "# | | | ||||||| | | ");
286 seq_puts(s
, " | | | | | | | |\n");
288 #else /* CONFIG_PREEMPT_RT */
289 static void print_osnoise_headers(struct seq_file
*s
)
291 if (osnoise_data
.tainted
)
292 seq_puts(s
, "# osnoise is tainted!\n");
294 seq_puts(s
, "# _-----=> irqs-off\n");
295 seq_puts(s
, "# / _----=> need-resched\n");
296 seq_puts(s
, "# | / _---=> hardirq/softirq\n");
297 seq_puts(s
, "# || / _--=> preempt-depth ");
298 seq_puts(s
, " MAX\n");
300 seq_puts(s
, "# || / ");
301 seq_puts(s
, " SINGLE Interference counters:\n");
303 seq_puts(s
, "# |||| RUNTIME ");
304 seq_puts(s
, " NOISE %% OF CPU NOISE +-----------------------------+\n");
306 seq_puts(s
, "# TASK-PID CPU# |||| TIMESTAMP IN US ");
307 seq_puts(s
, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
309 seq_puts(s
, "# | | | |||| | | ");
310 seq_puts(s
, " | | | | | | | |\n");
312 #endif /* CONFIG_PREEMPT_RT */
315 * osnoise_taint - report an osnoise error.
317 #define osnoise_taint(msg) ({ \
318 struct trace_array *tr = osnoise_trace; \
320 trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg); \
321 osnoise_data.tainted = true; \
325 * Record an osnoise_sample into the tracer buffer.
327 static void trace_osnoise_sample(struct osnoise_sample
*sample
)
329 struct trace_array
*tr
= osnoise_trace
;
330 struct trace_buffer
*buffer
= tr
->array_buffer
.buffer
;
331 struct trace_event_call
*call
= &event_osnoise
;
332 struct ring_buffer_event
*event
;
333 struct osnoise_entry
*entry
;
335 event
= trace_buffer_lock_reserve(buffer
, TRACE_OSNOISE
, sizeof(*entry
),
339 entry
= ring_buffer_event_data(event
);
340 entry
->runtime
= sample
->runtime
;
341 entry
->noise
= sample
->noise
;
342 entry
->max_sample
= sample
->max_sample
;
343 entry
->hw_count
= sample
->hw_count
;
344 entry
->nmi_count
= sample
->nmi_count
;
345 entry
->irq_count
= sample
->irq_count
;
346 entry
->softirq_count
= sample
->softirq_count
;
347 entry
->thread_count
= sample
->thread_count
;
349 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
350 trace_buffer_unlock_commit_nostack(buffer
, event
);
353 #ifdef CONFIG_TIMERLAT_TRACER
355 * Print the timerlat header info.
357 #ifdef CONFIG_PREEMPT_RT
358 static void print_timerlat_headers(struct seq_file
*s
)
360 seq_puts(s
, "# _-------=> irqs-off\n");
361 seq_puts(s
, "# / _------=> need-resched\n");
362 seq_puts(s
, "# | / _-----=> need-resched-lazy\n");
363 seq_puts(s
, "# || / _----=> hardirq/softirq\n");
364 seq_puts(s
, "# ||| / _---=> preempt-depth\n");
365 seq_puts(s
, "# |||| / _--=> preempt-lazy-depth\n");
366 seq_puts(s
, "# ||||| / _-=> migrate-disable\n");
367 seq_puts(s
, "# |||||| /\n");
368 seq_puts(s
, "# ||||||| ACTIVATION\n");
369 seq_puts(s
, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
370 seq_puts(s
, " CONTEXT LATENCY\n");
371 seq_puts(s
, "# | | | ||||||| | | ");
372 seq_puts(s
, " | |\n");
374 #else /* CONFIG_PREEMPT_RT */
375 static void print_timerlat_headers(struct seq_file
*s
)
377 seq_puts(s
, "# _-----=> irqs-off\n");
378 seq_puts(s
, "# / _----=> need-resched\n");
379 seq_puts(s
, "# | / _---=> hardirq/softirq\n");
380 seq_puts(s
, "# || / _--=> preempt-depth\n");
381 seq_puts(s
, "# || /\n");
382 seq_puts(s
, "# |||| ACTIVATION\n");
383 seq_puts(s
, "# TASK-PID CPU# |||| TIMESTAMP ID ");
384 seq_puts(s
, " CONTEXT LATENCY\n");
385 seq_puts(s
, "# | | | |||| | | ");
386 seq_puts(s
, " | |\n");
388 #endif /* CONFIG_PREEMPT_RT */
391 * Record an timerlat_sample into the tracer buffer.
393 static void trace_timerlat_sample(struct timerlat_sample
*sample
)
395 struct trace_array
*tr
= osnoise_trace
;
396 struct trace_event_call
*call
= &event_osnoise
;
397 struct trace_buffer
*buffer
= tr
->array_buffer
.buffer
;
398 struct ring_buffer_event
*event
;
399 struct timerlat_entry
*entry
;
401 event
= trace_buffer_lock_reserve(buffer
, TRACE_TIMERLAT
, sizeof(*entry
),
405 entry
= ring_buffer_event_data(event
);
406 entry
->seqnum
= sample
->seqnum
;
407 entry
->context
= sample
->context
;
408 entry
->timer_latency
= sample
->timer_latency
;
410 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
411 trace_buffer_unlock_commit_nostack(buffer
, event
);
414 #ifdef CONFIG_STACKTRACE
416 #define MAX_CALLS 256
419 * Stack trace will take place only at IRQ level, so, no need
420 * to control nesting here.
425 unsigned long calls
[MAX_CALLS
];
428 static DEFINE_PER_CPU(struct trace_stack
, trace_stack
);
431 * timerlat_save_stack - save a stack trace without printing
433 * Save the current stack trace without printing. The
434 * stack will be printed later, after the end of the measurement.
436 static void timerlat_save_stack(int skip
)
438 unsigned int size
, nr_entries
;
439 struct trace_stack
*fstack
;
441 fstack
= this_cpu_ptr(&trace_stack
);
443 size
= ARRAY_SIZE(fstack
->calls
);
445 nr_entries
= stack_trace_save(fstack
->calls
, size
, skip
);
447 fstack
->stack_size
= nr_entries
* sizeof(unsigned long);
448 fstack
->nr_entries
= nr_entries
;
454 * timerlat_dump_stack - dump a stack trace previously saved
456 * Dump a saved stack trace into the trace buffer.
458 static void timerlat_dump_stack(void)
460 struct trace_event_call
*call
= &event_osnoise
;
461 struct trace_array
*tr
= osnoise_trace
;
462 struct trace_buffer
*buffer
= tr
->array_buffer
.buffer
;
463 struct ring_buffer_event
*event
;
464 struct trace_stack
*fstack
;
465 struct stack_entry
*entry
;
468 preempt_disable_notrace();
469 fstack
= this_cpu_ptr(&trace_stack
);
470 size
= fstack
->stack_size
;
472 event
= trace_buffer_lock_reserve(buffer
, TRACE_STACK
, sizeof(*entry
) + size
,
477 entry
= ring_buffer_event_data(event
);
479 memcpy(&entry
->caller
, fstack
->calls
, size
);
480 entry
->size
= fstack
->nr_entries
;
482 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
483 trace_buffer_unlock_commit_nostack(buffer
, event
);
486 preempt_enable_notrace();
489 #define timerlat_dump_stack() do {} while (0)
490 #define timerlat_save_stack(a) do {} while (0)
491 #endif /* CONFIG_STACKTRACE */
492 #endif /* CONFIG_TIMERLAT_TRACER */
495 * Macros to encapsulate the time capturing infrastructure.
497 #define time_get() trace_clock_local()
498 #define time_to_us(x) div_u64(x, 1000)
499 #define time_sub(a, b) ((a) - (b))
502 * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
504 * If an IRQ is preempted by an NMI, its delta_start is pushed forward
505 * to discount the NMI interference.
507 * See get_int_safe_duration().
510 cond_move_irq_delta_start(struct osnoise_variables
*osn_var
, u64 duration
)
512 if (osn_var
->irq
.delta_start
)
513 osn_var
->irq
.delta_start
+= duration
;
516 #ifndef CONFIG_PREEMPT_RT
518 * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
520 * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
521 * forward to discount the interference.
523 * See get_int_safe_duration().
526 cond_move_softirq_delta_start(struct osnoise_variables
*osn_var
, u64 duration
)
528 if (osn_var
->softirq
.delta_start
)
529 osn_var
->softirq
.delta_start
+= duration
;
531 #else /* CONFIG_PREEMPT_RT */
532 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
536 * cond_move_thread_delta_start - Forward the delta_start of a running thread
538 * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
539 * is pushed forward to discount the interference.
541 * See get_int_safe_duration().
544 cond_move_thread_delta_start(struct osnoise_variables
*osn_var
, u64 duration
)
546 if (osn_var
->thread
.delta_start
)
547 osn_var
->thread
.delta_start
+= duration
;
551 * get_int_safe_duration - Get the duration of a window
553 * The irq, softirq and thread varaibles need to have its duration without
554 * the interference from higher priority interrupts. Instead of keeping a
555 * variable to discount the interrupt interference from these variables, the
556 * starting time of these variables are pushed forward with the interrupt's
557 * duration. In this way, a single variable is used to:
559 * - Know if a given window is being measured.
560 * - Account its duration.
561 * - Discount the interference.
563 * To avoid getting inconsistent values, e.g.,:
567 * delta_start -= int duration;
569 * duration = now - delta_start;
571 * result: negative duration if the variable duration before the
572 * interrupt was smaller than the interrupt execution.
574 * A counter of interrupts is used. If the counter increased, try
575 * to capture an interference safe duration.
578 get_int_safe_duration(struct osnoise_variables
*osn_var
, u64
*delta_start
)
580 u64 int_counter
, now
;
584 int_counter
= local_read(&osn_var
->int_counter
);
585 /* synchronize with interrupts */
589 duration
= (now
- *delta_start
);
591 /* synchronize with interrupts */
593 } while (int_counter
!= local_read(&osn_var
->int_counter
));
596 * This is an evidence of race conditions that cause
597 * a value to be "discounted" too much.
600 osnoise_taint("Negative duration!\n");
609 * set_int_safe_time - Save the current time on *time, aware of interference
611 * Get the time, taking into consideration a possible interference from
612 * higher priority interrupts.
614 * See get_int_safe_duration() for an explanation.
617 set_int_safe_time(struct osnoise_variables
*osn_var
, u64
*time
)
622 int_counter
= local_read(&osn_var
->int_counter
);
623 /* synchronize with interrupts */
628 /* synchronize with interrupts */
630 } while (int_counter
!= local_read(&osn_var
->int_counter
));
635 #ifdef CONFIG_TIMERLAT_TRACER
637 * copy_int_safe_time - Copy *src into *desc aware of interference
640 copy_int_safe_time(struct osnoise_variables
*osn_var
, u64
*dst
, u64
*src
)
645 int_counter
= local_read(&osn_var
->int_counter
);
646 /* synchronize with interrupts */
651 /* synchronize with interrupts */
653 } while (int_counter
!= local_read(&osn_var
->int_counter
));
657 #endif /* CONFIG_TIMERLAT_TRACER */
660 * trace_osnoise_callback - NMI entry/exit callback
662 * This function is called at the entry and exit NMI code. The bool enter
663 * distinguishes between either case. This function is used to note a NMI
664 * occurrence, compute the noise caused by the NMI, and to remove the noise
665 * it is potentially causing on other interference variables.
667 void trace_osnoise_callback(bool enter
)
669 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
672 if (!osn_var
->sampling
)
676 * Currently trace_clock_local() calls sched_clock() and the
677 * generic version is not NMI safe.
679 if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK
)) {
681 osn_var
->nmi
.delta_start
= time_get();
682 local_inc(&osn_var
->int_counter
);
684 duration
= time_get() - osn_var
->nmi
.delta_start
;
686 trace_nmi_noise(osn_var
->nmi
.delta_start
, duration
);
688 cond_move_irq_delta_start(osn_var
, duration
);
689 cond_move_softirq_delta_start(osn_var
, duration
);
690 cond_move_thread_delta_start(osn_var
, duration
);
695 osn_var
->nmi
.count
++;
699 * osnoise_trace_irq_entry - Note the starting of an IRQ
701 * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
702 * it is safe to use a single variable (ons_var->irq) to save the statistics.
703 * The arrival_time is used to report... the arrival time. The delta_start
704 * is used to compute the duration at the IRQ exit handler. See
705 * cond_move_irq_delta_start().
707 void osnoise_trace_irq_entry(int id
)
709 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
711 if (!osn_var
->sampling
)
714 * This value will be used in the report, but not to compute
715 * the execution time, so it is safe to get it unsafe.
717 osn_var
->irq
.arrival_time
= time_get();
718 set_int_safe_time(osn_var
, &osn_var
->irq
.delta_start
);
719 osn_var
->irq
.count
++;
721 local_inc(&osn_var
->int_counter
);
725 * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
727 * Computes the duration of the IRQ noise, and trace it. Also discounts the
728 * interference from other sources of noise could be currently being accounted.
730 void osnoise_trace_irq_exit(int id
, const char *desc
)
732 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
735 if (!osn_var
->sampling
)
738 duration
= get_int_safe_duration(osn_var
, &osn_var
->irq
.delta_start
);
739 trace_irq_noise(id
, desc
, osn_var
->irq
.arrival_time
, duration
);
740 osn_var
->irq
.arrival_time
= 0;
741 cond_move_softirq_delta_start(osn_var
, duration
);
742 cond_move_thread_delta_start(osn_var
, duration
);
746 * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
748 * Used to note the starting of an IRQ occurece.
750 static void trace_irqentry_callback(void *data
, int irq
,
751 struct irqaction
*action
)
753 osnoise_trace_irq_entry(irq
);
757 * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
759 * Used to note the end of an IRQ occurece.
761 static void trace_irqexit_callback(void *data
, int irq
,
762 struct irqaction
*action
, int ret
)
764 osnoise_trace_irq_exit(irq
, action
->name
);
768 * arch specific register function.
770 int __weak
osnoise_arch_register(void)
776 * arch specific unregister function.
778 void __weak
osnoise_arch_unregister(void)
784 * hook_irq_events - Hook IRQ handling events
786 * This function hooks the IRQ related callbacks to the respective trace
789 static int hook_irq_events(void)
793 ret
= register_trace_irq_handler_entry(trace_irqentry_callback
, NULL
);
797 ret
= register_trace_irq_handler_exit(trace_irqexit_callback
, NULL
);
799 goto out_unregister_entry
;
801 ret
= osnoise_arch_register();
808 unregister_trace_irq_handler_exit(trace_irqexit_callback
, NULL
);
809 out_unregister_entry
:
810 unregister_trace_irq_handler_entry(trace_irqentry_callback
, NULL
);
816 * unhook_irq_events - Unhook IRQ handling events
818 * This function unhooks the IRQ related callbacks to the respective trace
821 static void unhook_irq_events(void)
823 osnoise_arch_unregister();
824 unregister_trace_irq_handler_exit(trace_irqexit_callback
, NULL
);
825 unregister_trace_irq_handler_entry(trace_irqentry_callback
, NULL
);
828 #ifndef CONFIG_PREEMPT_RT
830 * trace_softirq_entry_callback - Note the starting of a softirq
832 * Save the starting time of a softirq. As softirqs are non-preemptive to
833 * other softirqs, it is safe to use a single variable (ons_var->softirq)
834 * to save the statistics. The arrival_time is used to report... the
835 * arrival time. The delta_start is used to compute the duration at the
836 * softirq exit handler. See cond_move_softirq_delta_start().
838 static void trace_softirq_entry_callback(void *data
, unsigned int vec_nr
)
840 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
842 if (!osn_var
->sampling
)
845 * This value will be used in the report, but not to compute
846 * the execution time, so it is safe to get it unsafe.
848 osn_var
->softirq
.arrival_time
= time_get();
849 set_int_safe_time(osn_var
, &osn_var
->softirq
.delta_start
);
850 osn_var
->softirq
.count
++;
852 local_inc(&osn_var
->int_counter
);
856 * trace_softirq_exit_callback - Note the end of an softirq
858 * Computes the duration of the softirq noise, and trace it. Also discounts the
859 * interference from other sources of noise could be currently being accounted.
861 static void trace_softirq_exit_callback(void *data
, unsigned int vec_nr
)
863 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
866 if (!osn_var
->sampling
)
869 #ifdef CONFIG_TIMERLAT_TRACER
871 * If the timerlat is enabled, but the irq handler did
872 * not run yet enabling timerlat_tracer, do not trace.
874 if (unlikely(osnoise_data
.timerlat_tracer
)) {
875 struct timerlat_variables
*tlat_var
;
876 tlat_var
= this_cpu_tmr_var();
877 if (!tlat_var
->tracing_thread
) {
878 osn_var
->softirq
.arrival_time
= 0;
879 osn_var
->softirq
.delta_start
= 0;
885 duration
= get_int_safe_duration(osn_var
, &osn_var
->softirq
.delta_start
);
886 trace_softirq_noise(vec_nr
, osn_var
->softirq
.arrival_time
, duration
);
887 cond_move_thread_delta_start(osn_var
, duration
);
888 osn_var
->softirq
.arrival_time
= 0;
892 * hook_softirq_events - Hook softirq handling events
894 * This function hooks the softirq related callbacks to the respective trace
897 static int hook_softirq_events(void)
901 ret
= register_trace_softirq_entry(trace_softirq_entry_callback
, NULL
);
905 ret
= register_trace_softirq_exit(trace_softirq_exit_callback
, NULL
);
907 goto out_unreg_entry
;
912 unregister_trace_softirq_entry(trace_softirq_entry_callback
, NULL
);
918 * unhook_softirq_events - Unhook softirq handling events
920 * This function hooks the softirq related callbacks to the respective trace
923 static void unhook_softirq_events(void)
925 unregister_trace_softirq_entry(trace_softirq_entry_callback
, NULL
);
926 unregister_trace_softirq_exit(trace_softirq_exit_callback
, NULL
);
928 #else /* CONFIG_PREEMPT_RT */
930 * softirq are threads on the PREEMPT_RT mode.
932 static int hook_softirq_events(void)
936 static void unhook_softirq_events(void)
942 * thread_entry - Record the starting of a thread noise window
944 * It saves the context switch time for a noisy thread, and increments
945 * the interference counters.
948 thread_entry(struct osnoise_variables
*osn_var
, struct task_struct
*t
)
950 if (!osn_var
->sampling
)
953 * The arrival time will be used in the report, but not to compute
954 * the execution time, so it is safe to get it unsafe.
956 osn_var
->thread
.arrival_time
= time_get();
958 set_int_safe_time(osn_var
, &osn_var
->thread
.delta_start
);
960 osn_var
->thread
.count
++;
961 local_inc(&osn_var
->int_counter
);
965 * thread_exit - Report the end of a thread noise window
967 * It computes the total noise from a thread, tracing if needed.
970 thread_exit(struct osnoise_variables
*osn_var
, struct task_struct
*t
)
974 if (!osn_var
->sampling
)
977 #ifdef CONFIG_TIMERLAT_TRACER
978 if (osnoise_data
.timerlat_tracer
) {
979 struct timerlat_variables
*tlat_var
;
980 tlat_var
= this_cpu_tmr_var();
981 if (!tlat_var
->tracing_thread
) {
982 osn_var
->thread
.delta_start
= 0;
983 osn_var
->thread
.arrival_time
= 0;
989 duration
= get_int_safe_duration(osn_var
, &osn_var
->thread
.delta_start
);
991 trace_thread_noise(t
, osn_var
->thread
.arrival_time
, duration
);
993 osn_var
->thread
.arrival_time
= 0;
997 * trace_sched_switch - sched:sched_switch trace event handler
999 * This function is hooked to the sched:sched_switch trace event, and it is
1000 * used to record the beginning and to report the end of a thread noise window.
1003 trace_sched_switch_callback(void *data
, bool preempt
, struct task_struct
*p
,
1004 struct task_struct
*n
)
1006 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
1008 if (p
->pid
!= osn_var
->pid
)
1009 thread_exit(osn_var
, p
);
1011 if (n
->pid
!= osn_var
->pid
)
1012 thread_entry(osn_var
, n
);
1016 * hook_thread_events - Hook the insturmentation for thread noise
1018 * Hook the osnoise tracer callbacks to handle the noise from other
1019 * threads on the necessary kernel events.
1021 static int hook_thread_events(void)
1025 ret
= register_trace_sched_switch(trace_sched_switch_callback
, NULL
);
1033 * unhook_thread_events - *nhook the insturmentation for thread noise
1035 * Unook the osnoise tracer callbacks to handle the noise from other
1036 * threads on the necessary kernel events.
1038 static void unhook_thread_events(void)
1040 unregister_trace_sched_switch(trace_sched_switch_callback
, NULL
);
1044 * save_osn_sample_stats - Save the osnoise_sample statistics
1046 * Save the osnoise_sample statistics before the sampling phase. These
1047 * values will be used later to compute the diff betwneen the statistics
1048 * before and after the osnoise sampling.
1051 save_osn_sample_stats(struct osnoise_variables
*osn_var
, struct osnoise_sample
*s
)
1053 s
->nmi_count
= osn_var
->nmi
.count
;
1054 s
->irq_count
= osn_var
->irq
.count
;
1055 s
->softirq_count
= osn_var
->softirq
.count
;
1056 s
->thread_count
= osn_var
->thread
.count
;
1060 * diff_osn_sample_stats - Compute the osnoise_sample statistics
1062 * After a sample period, compute the difference on the osnoise_sample
1063 * statistics. The struct osnoise_sample *s contains the statistics saved via
1064 * save_osn_sample_stats() before the osnoise sampling.
1067 diff_osn_sample_stats(struct osnoise_variables
*osn_var
, struct osnoise_sample
*s
)
1069 s
->nmi_count
= osn_var
->nmi
.count
- s
->nmi_count
;
1070 s
->irq_count
= osn_var
->irq
.count
- s
->irq_count
;
1071 s
->softirq_count
= osn_var
->softirq
.count
- s
->softirq_count
;
1072 s
->thread_count
= osn_var
->thread
.count
- s
->thread_count
;
1076 * osnoise_stop_tracing - Stop tracing and the tracer.
1078 static __always_inline
void osnoise_stop_tracing(void)
1080 struct trace_array
*tr
= osnoise_trace
;
1082 trace_array_printk_buf(tr
->array_buffer
.buffer
, _THIS_IP_
,
1083 "stop tracing hit on cpu %d\n", smp_processor_id());
1085 tracer_tracing_off(tr
);
1089 * run_osnoise - Sample the time and look for osnoise
1091 * Used to capture the time, looking for potential osnoise latency repeatedly.
1092 * Different from hwlat_detector, it is called with preemption and interrupts
1093 * enabled. This allows irqs, softirqs and threads to run, interfering on the
1094 * osnoise sampling thread, as they would do with a regular thread.
1096 static int run_osnoise(void)
1098 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
1099 struct trace_array
*tr
= osnoise_trace
;
1100 u64 start
, sample
, last_sample
;
1101 u64 last_int_count
, int_count
;
1102 s64 noise
= 0, max_noise
= 0;
1103 s64 total
, last_total
= 0;
1104 struct osnoise_sample s
;
1105 unsigned int threshold
;
1106 u64 runtime
, stop_in
;
1112 * Considers the current thread as the workload.
1114 osn_var
->pid
= current
->pid
;
1117 * Save the current stats for the diff
1119 save_osn_sample_stats(osn_var
, &s
);
1122 * if threshold is 0, use the default value of 5 us.
1124 threshold
= tracing_thresh
? : 5000;
1127 * Make sure NMIs see sampling first
1129 osn_var
->sampling
= true;
1133 * Transform the *_us config to nanoseconds to avoid the
1134 * division on the main loop.
1136 runtime
= osnoise_data
.sample_runtime
* NSEC_PER_USEC
;
1137 stop_in
= osnoise_data
.stop_tracing
* NSEC_PER_USEC
;
1147 last_int_count
= set_int_safe_time(osn_var
, &last_sample
);
1153 int_count
= set_int_safe_time(osn_var
, &sample
);
1155 noise
= time_sub(sample
, last_sample
);
1158 * This shouldn't happen.
1161 osnoise_taint("negative noise!");
1168 total
= time_sub(sample
, start
);
1171 * Check for possible overflows.
1173 if (total
< last_total
) {
1174 osnoise_taint("total overflow!");
1180 if (noise
>= threshold
) {
1181 int interference
= int_count
- last_int_count
;
1183 if (noise
> max_noise
)
1191 trace_sample_threshold(last_sample
, noise
, interference
);
1193 if (osnoise_data
.stop_tracing
)
1194 if (noise
> stop_in
)
1195 osnoise_stop_tracing();
1199 * For the non-preemptive kernel config: let threads runs, if
1204 last_sample
= sample
;
1205 last_int_count
= int_count
;
1207 } while (total
< runtime
&& !kthread_should_stop());
1210 * Finish the above in the view for interrupts.
1214 osn_var
->sampling
= false;
1217 * Make sure sampling data is no longer updated.
1224 s
.noise
= time_to_us(sum_noise
);
1225 s
.runtime
= time_to_us(total
);
1226 s
.max_sample
= time_to_us(max_noise
);
1227 s
.hw_count
= hw_count
;
1229 /* Save interference stats info */
1230 diff_osn_sample_stats(osn_var
, &s
);
1232 trace_osnoise_sample(&s
);
1234 /* Keep a running maximum ever recorded osnoise "latency" */
1235 if (max_noise
> tr
->max_latency
) {
1236 tr
->max_latency
= max_noise
;
1237 latency_fsnotify(tr
);
1240 if (osnoise_data
.stop_tracing_total
)
1241 if (s
.noise
> osnoise_data
.stop_tracing_total
)
1242 osnoise_stop_tracing();
1249 static struct cpumask osnoise_cpumask
;
1250 static struct cpumask save_cpumask
;
1253 * osnoise_main - The osnoise detection kernel thread
1255 * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1258 static int osnoise_main(void *data
)
1262 while (!kthread_should_stop()) {
1266 mutex_lock(&interface_lock
);
1267 interval
= osnoise_data
.sample_period
- osnoise_data
.sample_runtime
;
1268 mutex_unlock(&interface_lock
);
1270 do_div(interval
, USEC_PER_MSEC
);
1273 * differently from hwlat_detector, the osnoise tracer can run
1274 * without a pause because preemption is on.
1277 /* Let synchronize_rcu_tasks() make progress */
1278 cond_resched_tasks_rcu_qs();
1282 if (msleep_interruptible(interval
))
1289 #ifdef CONFIG_TIMERLAT_TRACER
1291 * timerlat_irq - hrtimer handler for timerlat.
1293 static enum hrtimer_restart
timerlat_irq(struct hrtimer
*timer
)
1295 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
1296 struct trace_array
*tr
= osnoise_trace
;
1297 struct timerlat_variables
*tlat
;
1298 struct timerlat_sample s
;
1303 * I am not sure if the timer was armed for this CPU. So, get
1304 * the timerlat struct from the timer itself, not from this
1307 tlat
= container_of(timer
, struct timerlat_variables
, timer
);
1309 now
= ktime_to_ns(hrtimer_cb_get_time(&tlat
->timer
));
1312 * Enable the osnoise: events for thread an softirq.
1314 tlat
->tracing_thread
= true;
1316 osn_var
->thread
.arrival_time
= time_get();
1319 * A hardirq is running: the timer IRQ. It is for sure preempting
1320 * a thread, and potentially preempting a softirq.
1322 * At this point, it is not interesting to know the duration of the
1323 * preempted thread (and maybe softirq), but how much time they will
1324 * delay the beginning of the execution of the timer thread.
1326 * To get the correct (net) delay added by the softirq, its delta_start
1327 * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1328 * start of the sofitrq will be zeroed, accounting then only the time
1331 * The thread follows the same principle. However, if a softirq is
1332 * running, the thread needs to receive the softirq delta_start. The
1333 * reason being is that the softirq will be the last to be unfolded,
1334 * resseting the thread delay to zero.
1336 #ifndef CONFIG_PREEMPT_RT
1337 if (osn_var
->softirq
.delta_start
) {
1338 copy_int_safe_time(osn_var
, &osn_var
->thread
.delta_start
,
1339 &osn_var
->softirq
.delta_start
);
1341 copy_int_safe_time(osn_var
, &osn_var
->softirq
.delta_start
,
1342 &osn_var
->irq
.delta_start
);
1344 copy_int_safe_time(osn_var
, &osn_var
->thread
.delta_start
,
1345 &osn_var
->irq
.delta_start
);
1347 #else /* CONFIG_PREEMPT_RT */
1349 * The sofirqs run as threads on RT, so there is not need
1350 * to keep track of it.
1352 copy_int_safe_time(osn_var
, &osn_var
->thread
.delta_start
, &osn_var
->irq
.delta_start
);
1353 #endif /* CONFIG_PREEMPT_RT */
1356 * Compute the current time with the expected time.
1358 diff
= now
- tlat
->abs_period
;
1361 s
.seqnum
= tlat
->count
;
1362 s
.timer_latency
= diff
;
1363 s
.context
= IRQ_CONTEXT
;
1365 trace_timerlat_sample(&s
);
1367 /* Keep a running maximum ever recorded os noise "latency" */
1368 if (diff
> tr
->max_latency
) {
1369 tr
->max_latency
= diff
;
1370 latency_fsnotify(tr
);
1373 if (osnoise_data
.stop_tracing
)
1374 if (time_to_us(diff
) >= osnoise_data
.stop_tracing
)
1375 osnoise_stop_tracing();
1377 wake_up_process(tlat
->kthread
);
1379 if (osnoise_data
.print_stack
)
1380 timerlat_save_stack(0);
1382 return HRTIMER_NORESTART
;
1386 * wait_next_period - Wait for the next period for timerlat
1388 static int wait_next_period(struct timerlat_variables
*tlat
)
1390 ktime_t next_abs_period
, now
;
1391 u64 rel_period
= osnoise_data
.timerlat_period
* 1000;
1393 now
= hrtimer_cb_get_time(&tlat
->timer
);
1394 next_abs_period
= ns_to_ktime(tlat
->abs_period
+ rel_period
);
1397 * Save the next abs_period.
1399 tlat
->abs_period
= (u64
) ktime_to_ns(next_abs_period
);
1402 * If the new abs_period is in the past, skip the activation.
1404 while (ktime_compare(now
, next_abs_period
) > 0) {
1405 next_abs_period
= ns_to_ktime(tlat
->abs_period
+ rel_period
);
1406 tlat
->abs_period
= (u64
) ktime_to_ns(next_abs_period
);
1409 set_current_state(TASK_INTERRUPTIBLE
);
1411 hrtimer_start(&tlat
->timer
, next_abs_period
, HRTIMER_MODE_ABS_PINNED_HARD
);
1417 * timerlat_main- Timerlat main
1419 static int timerlat_main(void *data
)
1421 struct osnoise_variables
*osn_var
= this_cpu_osn_var();
1422 struct timerlat_variables
*tlat
= this_cpu_tmr_var();
1423 struct timerlat_sample s
;
1424 struct sched_param sp
;
1428 * Make the thread RT, that is how cyclictest is usually used.
1430 sp
.sched_priority
= DEFAULT_TIMERLAT_PRIO
;
1431 sched_setscheduler_nocheck(current
, SCHED_FIFO
, &sp
);
1434 tlat
->tracing_thread
= false;
1436 hrtimer_init(&tlat
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_ABS_PINNED_HARD
);
1437 tlat
->timer
.function
= timerlat_irq
;
1438 tlat
->kthread
= current
;
1439 osn_var
->pid
= current
->pid
;
1441 * Anotate the arrival time.
1443 tlat
->abs_period
= hrtimer_cb_get_time(&tlat
->timer
);
1445 wait_next_period(tlat
);
1447 osn_var
->sampling
= 1;
1449 while (!kthread_should_stop()) {
1450 now
= ktime_to_ns(hrtimer_cb_get_time(&tlat
->timer
));
1451 diff
= now
- tlat
->abs_period
;
1453 s
.seqnum
= tlat
->count
;
1454 s
.timer_latency
= diff
;
1455 s
.context
= THREAD_CONTEXT
;
1457 trace_timerlat_sample(&s
);
1459 #ifdef CONFIG_STACKTRACE
1460 if (osnoise_data
.print_stack
)
1461 if (osnoise_data
.print_stack
<= time_to_us(diff
))
1462 timerlat_dump_stack();
1463 #endif /* CONFIG_STACKTRACE */
1465 tlat
->tracing_thread
= false;
1466 if (osnoise_data
.stop_tracing_total
)
1467 if (time_to_us(diff
) >= osnoise_data
.stop_tracing_total
)
1468 osnoise_stop_tracing();
1470 wait_next_period(tlat
);
1473 hrtimer_cancel(&tlat
->timer
);
1476 #endif /* CONFIG_TIMERLAT_TRACER */
1479 * stop_kthread - stop a workload thread
1481 static void stop_kthread(unsigned int cpu
)
1483 struct task_struct
*kthread
;
1485 kthread
= per_cpu(per_cpu_osnoise_var
, cpu
).kthread
;
1487 kthread_stop(kthread
);
1488 per_cpu(per_cpu_osnoise_var
, cpu
).kthread
= NULL
;
1492 * stop_per_cpu_kthread - Stop per-cpu threads
1494 * Stop the osnoise sampling htread. Use this on unload and at system
1497 static void stop_per_cpu_kthreads(void)
1503 for_each_online_cpu(cpu
)
1510 * start_kthread - Start a workload tread
1512 static int start_kthread(unsigned int cpu
)
1514 struct task_struct
*kthread
;
1515 void *main
= osnoise_main
;
1518 #ifdef CONFIG_TIMERLAT_TRACER
1519 if (osnoise_data
.timerlat_tracer
) {
1520 snprintf(comm
, 24, "timerlat/%d", cpu
);
1521 main
= timerlat_main
;
1523 snprintf(comm
, 24, "osnoise/%d", cpu
);
1526 snprintf(comm
, 24, "osnoise/%d", cpu
);
1528 kthread
= kthread_create_on_cpu(main
, NULL
, cpu
, comm
);
1530 if (IS_ERR(kthread
)) {
1531 pr_err(BANNER
"could not start sampling thread\n");
1532 stop_per_cpu_kthreads();
1536 per_cpu(per_cpu_osnoise_var
, cpu
).kthread
= kthread
;
1537 wake_up_process(kthread
);
1543 * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
1545 * This starts the kernel thread that will look for osnoise on many
1548 static int start_per_cpu_kthreads(struct trace_array
*tr
)
1550 struct cpumask
*current_mask
= &save_cpumask
;
1556 * Run only on CPUs in which trace and osnoise are allowed to run.
1558 cpumask_and(current_mask
, tr
->tracing_cpumask
, &osnoise_cpumask
);
1560 * And the CPU is online.
1562 cpumask_and(current_mask
, cpu_online_mask
, current_mask
);
1564 for_each_possible_cpu(cpu
)
1565 per_cpu(per_cpu_osnoise_var
, cpu
).kthread
= NULL
;
1567 for_each_cpu(cpu
, current_mask
) {
1568 retval
= start_kthread(cpu
);
1570 stop_per_cpu_kthreads();
1580 #ifdef CONFIG_HOTPLUG_CPU
1581 static void osnoise_hotplug_workfn(struct work_struct
*dummy
)
1583 struct trace_array
*tr
= osnoise_trace
;
1584 unsigned int cpu
= smp_processor_id();
1587 mutex_lock(&trace_types_lock
);
1590 goto out_unlock_trace
;
1592 mutex_lock(&interface_lock
);
1595 if (!cpumask_test_cpu(cpu
, &osnoise_cpumask
))
1598 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
))
1605 mutex_unlock(&interface_lock
);
1607 mutex_unlock(&trace_types_lock
);
1610 static DECLARE_WORK(osnoise_hotplug_work
, osnoise_hotplug_workfn
);
1613 * osnoise_cpu_init - CPU hotplug online callback function
1615 static int osnoise_cpu_init(unsigned int cpu
)
1617 schedule_work_on(cpu
, &osnoise_hotplug_work
);
1622 * osnoise_cpu_die - CPU hotplug offline callback function
1624 static int osnoise_cpu_die(unsigned int cpu
)
1630 static void osnoise_init_hotplug_support(void)
1634 ret
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "trace/osnoise:online",
1635 osnoise_cpu_init
, osnoise_cpu_die
);
1637 pr_warn(BANNER
"Error to init cpu hotplug support\n");
1641 #else /* CONFIG_HOTPLUG_CPU */
1642 static void osnoise_init_hotplug_support(void)
1646 #endif /* CONFIG_HOTPLUG_CPU */
1649 * osnoise_cpus_read - Read function for reading the "cpus" file
1650 * @filp: The active open file structure
1651 * @ubuf: The userspace provided buffer to read value into
1652 * @cnt: The maximum number of bytes to read
1653 * @ppos: The current "file" position
1655 * Prints the "cpus" output into the user-provided buffer.
1658 osnoise_cpus_read(struct file
*filp
, char __user
*ubuf
, size_t count
,
1664 mutex_lock(&interface_lock
);
1666 len
= snprintf(NULL
, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask
)) + 1;
1667 mask_str
= kmalloc(len
, GFP_KERNEL
);
1673 len
= snprintf(mask_str
, len
, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask
));
1679 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, len
);
1684 mutex_unlock(&interface_lock
);
1689 static void osnoise_tracer_start(struct trace_array
*tr
);
1690 static void osnoise_tracer_stop(struct trace_array
*tr
);
1693 * osnoise_cpus_write - Write function for "cpus" entry
1694 * @filp: The active open file structure
1695 * @ubuf: The user buffer that contains the value to write
1696 * @cnt: The maximum number of bytes to write to "file"
1697 * @ppos: The current position in @file
1699 * This function provides a write implementation for the "cpus"
1700 * interface to the osnoise trace. By default, it lists all CPUs,
1701 * in this way, allowing osnoise threads to run on any online CPU
1702 * of the system. It serves to restrict the execution of osnoise to the
1703 * set of CPUs writing via this interface. Note that osnoise also
1704 * respects the "tracing_cpumask." Hence, osnoise threads will run only
1705 * on the set of CPUs allowed here AND on "tracing_cpumask." Why not
1706 * have just "tracing_cpumask?" Because the user might be interested
1707 * in tracing what is running on other CPUs. For instance, one might
1708 * run osnoise in one HT CPU while observing what is running on the
1712 osnoise_cpus_write(struct file
*filp
, const char __user
*ubuf
, size_t count
,
1715 struct trace_array
*tr
= osnoise_trace
;
1716 cpumask_var_t osnoise_cpumask_new
;
1723 if (copy_from_user(buf
, ubuf
, count
))
1726 if (!zalloc_cpumask_var(&osnoise_cpumask_new
, GFP_KERNEL
))
1729 err
= cpulist_parse(buf
, osnoise_cpumask_new
);
1734 * trace_types_lock is taken to avoid concurrency on start/stop
1737 mutex_lock(&trace_types_lock
);
1738 running
= osnoise_busy
;
1740 osnoise_tracer_stop(tr
);
1742 mutex_lock(&interface_lock
);
1744 * osnoise_cpumask is read by CPU hotplug operations.
1748 cpumask_copy(&osnoise_cpumask
, osnoise_cpumask_new
);
1751 mutex_unlock(&interface_lock
);
1754 osnoise_tracer_start(tr
);
1755 mutex_unlock(&trace_types_lock
);
1757 free_cpumask_var(osnoise_cpumask_new
);
1761 free_cpumask_var(osnoise_cpumask_new
);
1767 * osnoise/runtime_us: cannot be greater than the period.
1769 static struct trace_min_max_param osnoise_runtime
= {
1770 .lock
= &interface_lock
,
1771 .val
= &osnoise_data
.sample_runtime
,
1772 .max
= &osnoise_data
.sample_period
,
1777 * osnoise/period_us: cannot be smaller than the runtime.
1779 static struct trace_min_max_param osnoise_period
= {
1780 .lock
= &interface_lock
,
1781 .val
= &osnoise_data
.sample_period
,
1783 .min
= &osnoise_data
.sample_runtime
,
1787 * osnoise/stop_tracing_us: no limit.
1789 static struct trace_min_max_param osnoise_stop_tracing_in
= {
1790 .lock
= &interface_lock
,
1791 .val
= &osnoise_data
.stop_tracing
,
1797 * osnoise/stop_tracing_total_us: no limit.
1799 static struct trace_min_max_param osnoise_stop_tracing_total
= {
1800 .lock
= &interface_lock
,
1801 .val
= &osnoise_data
.stop_tracing_total
,
1806 #ifdef CONFIG_TIMERLAT_TRACER
1808 * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
1809 * latency is higher than val.
1811 static struct trace_min_max_param osnoise_print_stack
= {
1812 .lock
= &interface_lock
,
1813 .val
= &osnoise_data
.print_stack
,
1819 * osnoise/timerlat_period: min 100 us, max 1 s
1821 u64 timerlat_min_period
= 100;
1822 u64 timerlat_max_period
= 1000000;
1823 static struct trace_min_max_param timerlat_period
= {
1824 .lock
= &interface_lock
,
1825 .val
= &osnoise_data
.timerlat_period
,
1826 .max
= &timerlat_max_period
,
1827 .min
= &timerlat_min_period
,
1831 static const struct file_operations cpus_fops
= {
1832 .open
= tracing_open_generic
,
1833 .read
= osnoise_cpus_read
,
1834 .write
= osnoise_cpus_write
,
1835 .llseek
= generic_file_llseek
,
1839 * init_tracefs - A function to initialize the tracefs interface files
1841 * This function creates entries in tracefs for "osnoise" and "timerlat".
1842 * It creates these directories in the tracing directory, and within that
1843 * directory the use can change and view the configs.
1845 static int init_tracefs(void)
1847 struct dentry
*top_dir
;
1851 ret
= tracing_init_dentry();
1855 top_dir
= tracefs_create_dir("osnoise", NULL
);
1859 tmp
= tracefs_create_file("period_us", 0640, top_dir
,
1860 &osnoise_period
, &trace_min_max_fops
);
1864 tmp
= tracefs_create_file("runtime_us", 0644, top_dir
,
1865 &osnoise_runtime
, &trace_min_max_fops
);
1869 tmp
= tracefs_create_file("stop_tracing_us", 0640, top_dir
,
1870 &osnoise_stop_tracing_in
, &trace_min_max_fops
);
1874 tmp
= tracefs_create_file("stop_tracing_total_us", 0640, top_dir
,
1875 &osnoise_stop_tracing_total
, &trace_min_max_fops
);
1879 tmp
= trace_create_file("cpus", 0644, top_dir
, NULL
, &cpus_fops
);
1882 #ifdef CONFIG_TIMERLAT_TRACER
1883 #ifdef CONFIG_STACKTRACE
1884 tmp
= tracefs_create_file("print_stack", 0640, top_dir
,
1885 &osnoise_print_stack
, &trace_min_max_fops
);
1890 tmp
= tracefs_create_file("timerlat_period_us", 0640, top_dir
,
1891 &timerlat_period
, &trace_min_max_fops
);
1899 tracefs_remove(top_dir
);
1903 static int osnoise_hook_events(void)
1908 * Trace is already hooked, we are re-enabling from
1911 if (trace_osnoise_callback_enabled
)
1914 retval
= hook_irq_events();
1918 retval
= hook_softirq_events();
1920 goto out_unhook_irq
;
1922 retval
= hook_thread_events();
1929 unhook_softirq_events();
1931 unhook_irq_events();
1935 static int __osnoise_tracer_start(struct trace_array
*tr
)
1939 osn_var_reset_all();
1941 retval
= osnoise_hook_events();
1945 * Make sure NMIs see reseted values.
1948 trace_osnoise_callback_enabled
= true;
1950 retval
= start_per_cpu_kthreads(tr
);
1952 unhook_irq_events();
1956 osnoise_busy
= true;
1961 static void osnoise_tracer_start(struct trace_array
*tr
)
1968 retval
= __osnoise_tracer_start(tr
);
1970 pr_err(BANNER
"Error starting osnoise tracer\n");
1974 static void osnoise_tracer_stop(struct trace_array
*tr
)
1979 trace_osnoise_callback_enabled
= false;
1982 stop_per_cpu_kthreads();
1984 unhook_irq_events();
1985 unhook_softirq_events();
1986 unhook_thread_events();
1988 osnoise_busy
= false;
1991 static int osnoise_tracer_init(struct trace_array
*tr
)
1994 /* Only allow one instance to enable this */
1999 tr
->max_latency
= 0;
2001 osnoise_tracer_start(tr
);
2006 static void osnoise_tracer_reset(struct trace_array
*tr
)
2008 osnoise_tracer_stop(tr
);
2011 static struct tracer osnoise_tracer __read_mostly
= {
2013 .init
= osnoise_tracer_init
,
2014 .reset
= osnoise_tracer_reset
,
2015 .start
= osnoise_tracer_start
,
2016 .stop
= osnoise_tracer_stop
,
2017 .print_header
= print_osnoise_headers
,
2018 .allow_instances
= true,
2021 #ifdef CONFIG_TIMERLAT_TRACER
2022 static void timerlat_tracer_start(struct trace_array
*tr
)
2029 osnoise_data
.timerlat_tracer
= 1;
2031 retval
= __osnoise_tracer_start(tr
);
2037 pr_err(BANNER
"Error starting timerlat tracer\n");
2040 static void timerlat_tracer_stop(struct trace_array
*tr
)
2047 for_each_online_cpu(cpu
)
2048 per_cpu(per_cpu_osnoise_var
, cpu
).sampling
= 0;
2050 osnoise_tracer_stop(tr
);
2052 osnoise_data
.timerlat_tracer
= 0;
2055 static int timerlat_tracer_init(struct trace_array
*tr
)
2057 /* Only allow one instance to enable this */
2063 tr
->max_latency
= 0;
2065 timerlat_tracer_start(tr
);
2070 static void timerlat_tracer_reset(struct trace_array
*tr
)
2072 timerlat_tracer_stop(tr
);
2075 static struct tracer timerlat_tracer __read_mostly
= {
2077 .init
= timerlat_tracer_init
,
2078 .reset
= timerlat_tracer_reset
,
2079 .start
= timerlat_tracer_start
,
2080 .stop
= timerlat_tracer_stop
,
2081 .print_header
= print_timerlat_headers
,
2082 .allow_instances
= true,
2084 #endif /* CONFIG_TIMERLAT_TRACER */
2086 __init
static int init_osnoise_tracer(void)
2090 mutex_init(&interface_lock
);
2092 cpumask_copy(&osnoise_cpumask
, cpu_all_mask
);
2094 ret
= register_tracer(&osnoise_tracer
);
2096 pr_err(BANNER
"Error registering osnoise!\n");
2100 #ifdef CONFIG_TIMERLAT_TRACER
2101 ret
= register_tracer(&timerlat_tracer
);
2103 pr_err(BANNER
"Error registering timerlat\n");
2107 osnoise_init_hotplug_support();
2113 late_initcall(init_osnoise_tracer
);