]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
73ce0511 BM |
2 | /* |
3 | * Detect hard lockups on a system | |
4 | * | |
5 | * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc. | |
6 | * | |
7 | * Note: Most of this code is borrowed heavily from the original softlockup | |
8 | * detector, so thanks to Ingo for the initial implementation. | |
9 | * Some chunks also taken from the old x86-specific nmi watchdog code, thanks | |
10 | * to those contributors as well. | |
11 | */ | |
12 | ||
13 | #define pr_fmt(fmt) "NMI watchdog: " fmt | |
14 | ||
15 | #include <linux/nmi.h> | |
16 | #include <linux/module.h> | |
b17b0153 IM |
17 | #include <linux/sched/debug.h> |
18 | ||
73ce0511 BM |
19 | #include <asm/irq_regs.h> |
20 | #include <linux/perf_event.h> | |
21 | ||
22 | static DEFINE_PER_CPU(bool, hard_watchdog_warn); | |
23 | static DEFINE_PER_CPU(bool, watchdog_nmi_touch); | |
24 | static DEFINE_PER_CPU(struct perf_event *, watchdog_ev); | |
941154bd | 25 | static struct cpumask dead_events_mask; |
73ce0511 | 26 | |
73ce0511 | 27 | static unsigned long hardlockup_allcpu_dumped; |
146c9d0e | 28 | static unsigned int watchdog_cpus; |
73ce0511 | 29 | |
f2e0cff8 | 30 | void arch_touch_nmi_watchdog(void) |
73ce0511 BM |
31 | { |
32 | /* | |
33 | * Using __raw here because some code paths have | |
34 | * preemption enabled. If preemption is enabled | |
35 | * then interrupts should be enabled too, in which | |
36 | * case we shouldn't have to worry about the watchdog | |
37 | * going off. | |
38 | */ | |
39 | raw_cpu_write(watchdog_nmi_touch, true); | |
73ce0511 | 40 | } |
f2e0cff8 | 41 | EXPORT_SYMBOL(arch_touch_nmi_watchdog); |
73ce0511 | 42 | |
7edaeb68 TG |
43 | #ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP |
44 | static DEFINE_PER_CPU(ktime_t, last_timestamp); | |
45 | static DEFINE_PER_CPU(unsigned int, nmi_rearmed); | |
46 | static ktime_t watchdog_hrtimer_sample_threshold __read_mostly; | |
47 | ||
48 | void watchdog_update_hrtimer_threshold(u64 period) | |
49 | { | |
50 | /* | |
51 | * The hrtimer runs with a period of (watchdog_threshold * 2) / 5 | |
52 | * | |
53 | * So it runs effectively with 2.5 times the rate of the NMI | |
54 | * watchdog. That means the hrtimer should fire 2-3 times before | |
55 | * the NMI watchdog expires. The NMI watchdog on x86 is based on | |
56 | * unhalted CPU cycles, so if Turbo-Mode is enabled the CPU cycles | |
57 | * might run way faster than expected and the NMI fires in a | |
58 | * smaller period than the one deduced from the nominal CPU | |
59 | * frequency. Depending on the Turbo-Mode factor this might be fast | |
60 | * enough to get the NMI period smaller than the hrtimer watchdog | |
61 | * period and trigger false positives. | |
62 | * | |
63 | * The sample threshold is used to check in the NMI handler whether | |
64 | * the minimum time between two NMI samples has elapsed. That | |
65 | * prevents false positives. | |
66 | * | |
67 | * Set this to 4/5 of the actual watchdog threshold period so the | |
68 | * hrtimer is guaranteed to fire at least once within the real | |
69 | * watchdog threshold. | |
70 | */ | |
71 | watchdog_hrtimer_sample_threshold = period * 2; | |
72 | } | |
73 | ||
74 | static bool watchdog_check_timestamp(void) | |
75 | { | |
76 | ktime_t delta, now = ktime_get_mono_fast_ns(); | |
77 | ||
78 | delta = now - __this_cpu_read(last_timestamp); | |
79 | if (delta < watchdog_hrtimer_sample_threshold) { | |
80 | /* | |
81 | * If ktime is jiffies based, a stalled timer would prevent | |
82 | * jiffies from being incremented and the filter would look | |
83 | * at a stale timestamp and never trigger. | |
84 | */ | |
85 | if (__this_cpu_inc_return(nmi_rearmed) < 10) | |
86 | return false; | |
87 | } | |
88 | __this_cpu_write(nmi_rearmed, 0); | |
89 | __this_cpu_write(last_timestamp, now); | |
90 | return true; | |
91 | } | |
92 | #else | |
93 | static inline bool watchdog_check_timestamp(void) | |
94 | { | |
95 | return true; | |
96 | } | |
97 | #endif | |
98 | ||
73ce0511 BM |
99 | static struct perf_event_attr wd_hw_attr = { |
100 | .type = PERF_TYPE_HARDWARE, | |
101 | .config = PERF_COUNT_HW_CPU_CYCLES, | |
102 | .size = sizeof(struct perf_event_attr), | |
103 | .pinned = 1, | |
104 | .disabled = 1, | |
105 | }; | |
106 | ||
107 | /* Callback function for perf event subsystem */ | |
108 | static void watchdog_overflow_callback(struct perf_event *event, | |
01f0a027 TG |
109 | struct perf_sample_data *data, |
110 | struct pt_regs *regs) | |
73ce0511 BM |
111 | { |
112 | /* Ensure the watchdog never gets throttled */ | |
113 | event->hw.interrupts = 0; | |
114 | ||
115 | if (__this_cpu_read(watchdog_nmi_touch) == true) { | |
116 | __this_cpu_write(watchdog_nmi_touch, false); | |
117 | return; | |
118 | } | |
119 | ||
7edaeb68 TG |
120 | if (!watchdog_check_timestamp()) |
121 | return; | |
122 | ||
73ce0511 BM |
123 | /* check for a hardlockup |
124 | * This is done by making sure our timer interrupt | |
125 | * is incrementing. The timer interrupt should have | |
126 | * fired multiple times before we overflow'd. If it hasn't | |
127 | * then this is a good indication the cpu is stuck | |
128 | */ | |
129 | if (is_hardlockup()) { | |
130 | int this_cpu = smp_processor_id(); | |
131 | ||
132 | /* only print hardlockups once */ | |
133 | if (__this_cpu_read(hard_watchdog_warn) == true) | |
134 | return; | |
135 | ||
136 | pr_emerg("Watchdog detected hard LOCKUP on cpu %d", this_cpu); | |
137 | print_modules(); | |
138 | print_irqtrace_events(current); | |
139 | if (regs) | |
140 | show_regs(regs); | |
141 | else | |
142 | dump_stack(); | |
143 | ||
144 | /* | |
145 | * Perform all-CPU dump only once to avoid multiple hardlockups | |
146 | * generating interleaving traces | |
147 | */ | |
148 | if (sysctl_hardlockup_all_cpu_backtrace && | |
149 | !test_and_set_bit(0, &hardlockup_allcpu_dumped)) | |
150 | trigger_allbutself_cpu_backtrace(); | |
151 | ||
152 | if (hardlockup_panic) | |
153 | nmi_panic(regs, "Hard LOCKUP"); | |
154 | ||
155 | __this_cpu_write(hard_watchdog_warn, true); | |
156 | return; | |
157 | } | |
158 | ||
159 | __this_cpu_write(hard_watchdog_warn, false); | |
160 | return; | |
161 | } | |
162 | ||
178b9f7a TG |
163 | static int hardlockup_detector_event_create(void) |
164 | { | |
165 | unsigned int cpu = smp_processor_id(); | |
166 | struct perf_event_attr *wd_attr; | |
167 | struct perf_event *evt; | |
168 | ||
169 | wd_attr = &wd_hw_attr; | |
170 | wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); | |
171 | ||
172 | /* Try to register using hardware perf events */ | |
173 | evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL, | |
174 | watchdog_overflow_callback, NULL); | |
175 | if (IS_ERR(evt)) { | |
176 | pr_info("Perf event create on CPU %d failed with %ld\n", cpu, | |
177 | PTR_ERR(evt)); | |
178 | return PTR_ERR(evt); | |
179 | } | |
180 | this_cpu_write(watchdog_ev, evt); | |
181 | return 0; | |
182 | } | |
183 | ||
2a1b8ee4 TG |
184 | /** |
185 | * hardlockup_detector_perf_enable - Enable the local event | |
186 | */ | |
187 | void hardlockup_detector_perf_enable(void) | |
188 | { | |
189 | if (hardlockup_detector_event_create()) | |
190 | return; | |
191 | ||
146c9d0e TG |
192 | if (!watchdog_cpus++) |
193 | pr_info("Enabled. Permanently consumes one hw-PMU counter.\n"); | |
194 | ||
2a1b8ee4 TG |
195 | perf_event_enable(this_cpu_read(watchdog_ev)); |
196 | } | |
197 | ||
941154bd TG |
198 | /** |
199 | * hardlockup_detector_perf_disable - Disable the local event | |
200 | */ | |
201 | void hardlockup_detector_perf_disable(void) | |
73ce0511 | 202 | { |
941154bd | 203 | struct perf_event *event = this_cpu_read(watchdog_ev); |
73ce0511 BM |
204 | |
205 | if (event) { | |
206 | perf_event_disable(event); | |
941154bd | 207 | cpumask_set_cpu(smp_processor_id(), &dead_events_mask); |
146c9d0e | 208 | watchdog_cpus--; |
73ce0511 BM |
209 | } |
210 | } | |
d0b6e0a8 | 211 | |
941154bd TG |
212 | /** |
213 | * hardlockup_detector_perf_cleanup - Cleanup disabled events and destroy them | |
214 | * | |
215 | * Called from lockup_detector_cleanup(). Serialized by the caller. | |
216 | */ | |
217 | void hardlockup_detector_perf_cleanup(void) | |
218 | { | |
219 | int cpu; | |
220 | ||
221 | for_each_cpu(cpu, &dead_events_mask) { | |
a33d4484 | 222 | struct perf_event *event = per_cpu(watchdog_ev, cpu); |
941154bd | 223 | |
115ef3b7 TG |
224 | /* |
225 | * Required because for_each_cpu() reports unconditionally | |
226 | * CPU0 as set on UP kernels. Sigh. | |
227 | */ | |
228 | if (event) | |
229 | perf_event_release_kernel(event); | |
a33d4484 | 230 | per_cpu(watchdog_ev, cpu) = NULL; |
941154bd TG |
231 | } |
232 | cpumask_clear(&dead_events_mask); | |
233 | } | |
234 | ||
d0b6e0a8 PZ |
235 | /** |
236 | * hardlockup_detector_perf_stop - Globally stop watchdog events | |
237 | * | |
238 | * Special interface for x86 to handle the perf HT bug. | |
239 | */ | |
240 | void __init hardlockup_detector_perf_stop(void) | |
241 | { | |
242 | int cpu; | |
243 | ||
244 | lockdep_assert_cpus_held(); | |
245 | ||
246 | for_each_online_cpu(cpu) { | |
247 | struct perf_event *event = per_cpu(watchdog_ev, cpu); | |
248 | ||
249 | if (event) | |
250 | perf_event_disable(event); | |
251 | } | |
252 | } | |
253 | ||
254 | /** | |
255 | * hardlockup_detector_perf_restart - Globally restart watchdog events | |
256 | * | |
257 | * Special interface for x86 to handle the perf HT bug. | |
258 | */ | |
259 | void __init hardlockup_detector_perf_restart(void) | |
260 | { | |
261 | int cpu; | |
262 | ||
263 | lockdep_assert_cpus_held(); | |
264 | ||
265 | if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) | |
266 | return; | |
267 | ||
268 | for_each_online_cpu(cpu) { | |
269 | struct perf_event *event = per_cpu(watchdog_ev, cpu); | |
270 | ||
271 | if (event) | |
272 | perf_event_enable(event); | |
273 | } | |
274 | } | |
178b9f7a TG |
275 | |
276 | /** | |
277 | * hardlockup_detector_perf_init - Probe whether NMI event is available at all | |
278 | */ | |
279 | int __init hardlockup_detector_perf_init(void) | |
280 | { | |
281 | int ret = hardlockup_detector_event_create(); | |
282 | ||
283 | if (ret) { | |
77c01d11 | 284 | pr_info("Perf NMI watchdog permanently disabled\n"); |
178b9f7a TG |
285 | } else { |
286 | perf_event_release_kernel(this_cpu_read(watchdog_ev)); | |
287 | this_cpu_write(watchdog_ev, NULL); | |
288 | } | |
289 | return ret; | |
290 | } |