]>
Commit | Line | Data |
---|---|---|
2104180a NP |
1 | /* |
2 | * Watchdog support on powerpc systems. | |
3 | * | |
4 | * Copyright 2017, IBM Corporation. | |
5 | * | |
6 | * This uses code from arch/sparc/kernel/nmi.c and kernel/watchdog.c | |
7 | */ | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/param.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/percpu.h> | |
12 | #include <linux/cpu.h> | |
13 | #include <linux/nmi.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/export.h> | |
16 | #include <linux/kprobes.h> | |
17 | #include <linux/hardirq.h> | |
18 | #include <linux/reboot.h> | |
19 | #include <linux/slab.h> | |
20 | #include <linux/kdebug.h> | |
21 | #include <linux/sched/debug.h> | |
22 | #include <linux/delay.h> | |
23 | #include <linux/smp.h> | |
24 | ||
25 | #include <asm/paca.h> | |
26 | ||
27 | /* | |
28 | * The watchdog has a simple timer that runs on each CPU, once per timer | |
29 | * period. This is the heartbeat. | |
30 | * | |
31 | * Then there are checks to see if the heartbeat has not triggered on a CPU | |
32 | * for the panic timeout period. Currently the watchdog only supports an | |
33 | * SMP check, so the heartbeat only turns on when we have 2 or more CPUs. | |
34 | * | |
35 | * This is not an NMI watchdog, but Linux uses that name for a generic | |
36 | * watchdog in some cases, so NMI gets used in some places. | |
37 | */ | |
38 | ||
39 | static cpumask_t wd_cpus_enabled __read_mostly; | |
40 | ||
41 | static u64 wd_panic_timeout_tb __read_mostly; /* timebase ticks until panic */ | |
42 | static u64 wd_smp_panic_timeout_tb __read_mostly; /* panic other CPUs */ | |
43 | ||
44 | static u64 wd_timer_period_ms __read_mostly; /* interval between heartbeat */ | |
45 | ||
46 | static DEFINE_PER_CPU(struct timer_list, wd_timer); | |
47 | static DEFINE_PER_CPU(u64, wd_timer_tb); | |
48 | ||
49 | /* | |
50 | * These are for the SMP checker. CPUs clear their pending bit in their | |
51 | * heartbeat. If the bitmask becomes empty, the time is noted and the | |
52 | * bitmask is refilled. | |
53 | * | |
54 | * All CPUs clear their bit in the pending mask every timer period. | |
55 | * Once all have cleared, the time is noted and the bits are reset. | |
56 | * If the time since all clear was greater than the panic timeout, | |
57 | * we can panic with the list of stuck CPUs. | |
58 | * | |
59 | * This will work best with NMI IPIs for crash code so the stuck CPUs | |
60 | * can be pulled out to get their backtraces. | |
61 | */ | |
62 | static unsigned long __wd_smp_lock; | |
63 | static cpumask_t wd_smp_cpus_pending; | |
64 | static cpumask_t wd_smp_cpus_stuck; | |
65 | static u64 wd_smp_last_reset_tb; | |
66 | ||
67 | static inline void wd_smp_lock(unsigned long *flags) | |
68 | { | |
69 | /* | |
70 | * Avoid locking layers if possible. | |
71 | * This may be called from low level interrupt handlers at some | |
72 | * point in future. | |
73 | */ | |
d8e2a405 NP |
74 | raw_local_irq_save(*flags); |
75 | hard_irq_disable(); /* Make it soft-NMI safe */ | |
76 | while (unlikely(test_and_set_bit_lock(0, &__wd_smp_lock))) { | |
77 | raw_local_irq_restore(*flags); | |
78 | spin_until_cond(!test_bit(0, &__wd_smp_lock)); | |
79 | raw_local_irq_save(*flags); | |
80 | hard_irq_disable(); | |
81 | } | |
2104180a NP |
82 | } |
83 | ||
84 | static inline void wd_smp_unlock(unsigned long *flags) | |
85 | { | |
86 | clear_bit_unlock(0, &__wd_smp_lock); | |
d8e2a405 | 87 | raw_local_irq_restore(*flags); |
2104180a NP |
88 | } |
89 | ||
90 | static void wd_lockup_ipi(struct pt_regs *regs) | |
91 | { | |
92 | pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", raw_smp_processor_id()); | |
93 | print_modules(); | |
94 | print_irqtrace_events(current); | |
95 | if (regs) | |
96 | show_regs(regs); | |
97 | else | |
98 | dump_stack(); | |
99 | ||
100 | if (hardlockup_panic) | |
101 | nmi_panic(regs, "Hard LOCKUP"); | |
102 | } | |
103 | ||
87607a30 | 104 | static void set_cpumask_stuck(const struct cpumask *cpumask, u64 tb) |
2104180a | 105 | { |
87607a30 NP |
106 | cpumask_or(&wd_smp_cpus_stuck, &wd_smp_cpus_stuck, cpumask); |
107 | cpumask_andnot(&wd_smp_cpus_pending, &wd_smp_cpus_pending, cpumask); | |
2104180a NP |
108 | if (cpumask_empty(&wd_smp_cpus_pending)) { |
109 | wd_smp_last_reset_tb = tb; | |
110 | cpumask_andnot(&wd_smp_cpus_pending, | |
111 | &wd_cpus_enabled, | |
112 | &wd_smp_cpus_stuck); | |
113 | } | |
114 | } | |
87607a30 NP |
115 | static void set_cpu_stuck(int cpu, u64 tb) |
116 | { | |
117 | set_cpumask_stuck(cpumask_of(cpu), tb); | |
118 | } | |
2104180a NP |
119 | |
120 | static void watchdog_smp_panic(int cpu, u64 tb) | |
121 | { | |
122 | unsigned long flags; | |
123 | int c; | |
124 | ||
125 | wd_smp_lock(&flags); | |
126 | /* Double check some things under lock */ | |
127 | if ((s64)(tb - wd_smp_last_reset_tb) < (s64)wd_smp_panic_timeout_tb) | |
128 | goto out; | |
129 | if (cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) | |
130 | goto out; | |
131 | if (cpumask_weight(&wd_smp_cpus_pending) == 0) | |
132 | goto out; | |
133 | ||
134 | pr_emerg("Watchdog CPU:%d detected Hard LOCKUP other CPUS:%*pbl\n", | |
135 | cpu, cpumask_pr_args(&wd_smp_cpus_pending)); | |
136 | ||
137 | /* | |
138 | * Try to trigger the stuck CPUs. | |
139 | */ | |
140 | for_each_cpu(c, &wd_smp_cpus_pending) { | |
141 | if (c == cpu) | |
142 | continue; | |
143 | smp_send_nmi_ipi(c, wd_lockup_ipi, 1000000); | |
144 | } | |
145 | smp_flush_nmi_ipi(1000000); | |
146 | ||
87607a30 NP |
147 | /* Take the stuck CPUs out of the watch group */ |
148 | set_cpumask_stuck(&wd_smp_cpus_pending, tb); | |
2104180a | 149 | |
2104180a NP |
150 | wd_smp_unlock(&flags); |
151 | ||
152 | printk_safe_flush(); | |
153 | /* | |
154 | * printk_safe_flush() seems to require another print | |
155 | * before anything actually goes out to console. | |
156 | */ | |
157 | if (sysctl_hardlockup_all_cpu_backtrace) | |
158 | trigger_allbutself_cpu_backtrace(); | |
159 | ||
160 | if (hardlockup_panic) | |
161 | nmi_panic(NULL, "Hard LOCKUP"); | |
8e236921 NP |
162 | |
163 | return; | |
164 | ||
165 | out: | |
166 | wd_smp_unlock(&flags); | |
2104180a NP |
167 | } |
168 | ||
169 | static void wd_smp_clear_cpu_pending(int cpu, u64 tb) | |
170 | { | |
171 | if (!cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) { | |
172 | if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) { | |
173 | unsigned long flags; | |
174 | ||
175 | pr_emerg("Watchdog CPU:%d became unstuck\n", cpu); | |
176 | wd_smp_lock(&flags); | |
177 | cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck); | |
178 | wd_smp_unlock(&flags); | |
179 | } | |
180 | return; | |
181 | } | |
182 | cpumask_clear_cpu(cpu, &wd_smp_cpus_pending); | |
183 | if (cpumask_empty(&wd_smp_cpus_pending)) { | |
184 | unsigned long flags; | |
185 | ||
186 | wd_smp_lock(&flags); | |
187 | if (cpumask_empty(&wd_smp_cpus_pending)) { | |
188 | wd_smp_last_reset_tb = tb; | |
189 | cpumask_andnot(&wd_smp_cpus_pending, | |
190 | &wd_cpus_enabled, | |
191 | &wd_smp_cpus_stuck); | |
192 | } | |
193 | wd_smp_unlock(&flags); | |
194 | } | |
195 | } | |
196 | ||
197 | static void watchdog_timer_interrupt(int cpu) | |
198 | { | |
199 | u64 tb = get_tb(); | |
200 | ||
201 | per_cpu(wd_timer_tb, cpu) = tb; | |
202 | ||
203 | wd_smp_clear_cpu_pending(cpu, tb); | |
204 | ||
205 | if ((s64)(tb - wd_smp_last_reset_tb) >= (s64)wd_smp_panic_timeout_tb) | |
206 | watchdog_smp_panic(cpu, tb); | |
207 | } | |
208 | ||
209 | void soft_nmi_interrupt(struct pt_regs *regs) | |
210 | { | |
211 | unsigned long flags; | |
212 | int cpu = raw_smp_processor_id(); | |
213 | u64 tb; | |
214 | ||
215 | if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) | |
216 | return; | |
217 | ||
218 | nmi_enter(); | |
04019bf8 NP |
219 | |
220 | __this_cpu_inc(irq_stat.soft_nmi_irqs); | |
221 | ||
2104180a NP |
222 | tb = get_tb(); |
223 | if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) { | |
224 | per_cpu(wd_timer_tb, cpu) = tb; | |
225 | ||
226 | wd_smp_lock(&flags); | |
227 | if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) { | |
228 | wd_smp_unlock(&flags); | |
229 | goto out; | |
230 | } | |
231 | set_cpu_stuck(cpu, tb); | |
232 | ||
233 | pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", cpu); | |
234 | print_modules(); | |
235 | print_irqtrace_events(current); | |
236 | if (regs) | |
237 | show_regs(regs); | |
238 | else | |
239 | dump_stack(); | |
240 | ||
241 | wd_smp_unlock(&flags); | |
242 | ||
243 | if (sysctl_hardlockup_all_cpu_backtrace) | |
244 | trigger_allbutself_cpu_backtrace(); | |
245 | ||
246 | if (hardlockup_panic) | |
247 | nmi_panic(regs, "Hard LOCKUP"); | |
248 | } | |
249 | if (wd_panic_timeout_tb < 0x7fffffff) | |
250 | mtspr(SPRN_DEC, wd_panic_timeout_tb); | |
251 | ||
252 | out: | |
253 | nmi_exit(); | |
254 | } | |
255 | ||
256 | static void wd_timer_reset(unsigned int cpu, struct timer_list *t) | |
257 | { | |
258 | t->expires = jiffies + msecs_to_jiffies(wd_timer_period_ms); | |
259 | if (wd_timer_period_ms > 1000) | |
260 | t->expires = __round_jiffies_up(t->expires, cpu); | |
261 | add_timer_on(t, cpu); | |
262 | } | |
263 | ||
264 | static void wd_timer_fn(unsigned long data) | |
265 | { | |
266 | struct timer_list *t = this_cpu_ptr(&wd_timer); | |
267 | int cpu = smp_processor_id(); | |
268 | ||
269 | watchdog_timer_interrupt(cpu); | |
270 | ||
271 | wd_timer_reset(cpu, t); | |
272 | } | |
273 | ||
274 | void arch_touch_nmi_watchdog(void) | |
275 | { | |
26c5c6e1 | 276 | unsigned long ticks = tb_ticks_per_usec * wd_timer_period_ms * 1000; |
2104180a NP |
277 | int cpu = smp_processor_id(); |
278 | ||
26c5c6e1 NP |
279 | if (get_tb() - per_cpu(wd_timer_tb, cpu) >= ticks) |
280 | watchdog_timer_interrupt(cpu); | |
2104180a NP |
281 | } |
282 | EXPORT_SYMBOL(arch_touch_nmi_watchdog); | |
283 | ||
284 | static void start_watchdog_timer_on(unsigned int cpu) | |
285 | { | |
286 | struct timer_list *t = per_cpu_ptr(&wd_timer, cpu); | |
287 | ||
288 | per_cpu(wd_timer_tb, cpu) = get_tb(); | |
289 | ||
290 | setup_pinned_timer(t, wd_timer_fn, 0); | |
291 | wd_timer_reset(cpu, t); | |
292 | } | |
293 | ||
294 | static void stop_watchdog_timer_on(unsigned int cpu) | |
295 | { | |
296 | struct timer_list *t = per_cpu_ptr(&wd_timer, cpu); | |
297 | ||
298 | del_timer_sync(t); | |
299 | } | |
300 | ||
301 | static int start_wd_on_cpu(unsigned int cpu) | |
302 | { | |
96ea91e7 NP |
303 | unsigned long flags; |
304 | ||
2104180a NP |
305 | if (cpumask_test_cpu(cpu, &wd_cpus_enabled)) { |
306 | WARN_ON(1); | |
307 | return 0; | |
308 | } | |
309 | ||
310 | if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED)) | |
311 | return 0; | |
312 | ||
2104180a NP |
313 | if (!cpumask_test_cpu(cpu, &watchdog_cpumask)) |
314 | return 0; | |
315 | ||
96ea91e7 | 316 | wd_smp_lock(&flags); |
2104180a NP |
317 | cpumask_set_cpu(cpu, &wd_cpus_enabled); |
318 | if (cpumask_weight(&wd_cpus_enabled) == 1) { | |
319 | cpumask_set_cpu(cpu, &wd_smp_cpus_pending); | |
320 | wd_smp_last_reset_tb = get_tb(); | |
321 | } | |
96ea91e7 NP |
322 | wd_smp_unlock(&flags); |
323 | ||
2104180a NP |
324 | start_watchdog_timer_on(cpu); |
325 | ||
326 | return 0; | |
327 | } | |
328 | ||
329 | static int stop_wd_on_cpu(unsigned int cpu) | |
330 | { | |
96ea91e7 NP |
331 | unsigned long flags; |
332 | ||
2104180a NP |
333 | if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) |
334 | return 0; /* Can happen in CPU unplug case */ | |
335 | ||
336 | stop_watchdog_timer_on(cpu); | |
337 | ||
96ea91e7 | 338 | wd_smp_lock(&flags); |
2104180a | 339 | cpumask_clear_cpu(cpu, &wd_cpus_enabled); |
96ea91e7 NP |
340 | wd_smp_unlock(&flags); |
341 | ||
2104180a NP |
342 | wd_smp_clear_cpu_pending(cpu, get_tb()); |
343 | ||
344 | return 0; | |
345 | } | |
346 | ||
347 | static void watchdog_calc_timeouts(void) | |
348 | { | |
349 | wd_panic_timeout_tb = watchdog_thresh * ppc_tb_freq; | |
350 | ||
351 | /* Have the SMP detector trigger a bit later */ | |
352 | wd_smp_panic_timeout_tb = wd_panic_timeout_tb * 3 / 2; | |
353 | ||
354 | /* 2/5 is the factor that the perf based detector uses */ | |
355 | wd_timer_period_ms = watchdog_thresh * 1000 * 2 / 5; | |
356 | } | |
357 | ||
6b9dc480 | 358 | void watchdog_nmi_stop(void) |
2104180a NP |
359 | { |
360 | int cpu; | |
361 | ||
6b9dc480 TG |
362 | for_each_cpu(cpu, &wd_cpus_enabled) |
363 | stop_wd_on_cpu(cpu); | |
6b9dc480 TG |
364 | } |
365 | ||
366 | void watchdog_nmi_start(void) | |
367 | { | |
368 | int cpu; | |
369 | ||
6b9dc480 TG |
370 | watchdog_calc_timeouts(); |
371 | for_each_cpu_and(cpu, cpu_online_mask, &watchdog_cpumask) | |
372 | start_wd_on_cpu(cpu); | |
2104180a NP |
373 | } |
374 | ||
375 | /* | |
376 | * This runs after lockup_detector_init() which sets up watchdog_cpumask. | |
377 | */ | |
378 | static int __init powerpc_watchdog_init(void) | |
379 | { | |
380 | int err; | |
381 | ||
382 | watchdog_calc_timeouts(); | |
383 | ||
384 | err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/watchdog:online", | |
385 | start_wd_on_cpu, stop_wd_on_cpu); | |
386 | if (err < 0) | |
387 | pr_warn("Watchdog could not be initialized"); | |
388 | ||
389 | return 0; | |
390 | } | |
391 | arch_initcall(powerpc_watchdog_init); | |
392 | ||
393 | static void handle_backtrace_ipi(struct pt_regs *regs) | |
394 | { | |
395 | nmi_cpu_backtrace(regs); | |
396 | } | |
397 | ||
398 | static void raise_backtrace_ipi(cpumask_t *mask) | |
399 | { | |
400 | unsigned int cpu; | |
401 | ||
402 | for_each_cpu(cpu, mask) { | |
403 | if (cpu == smp_processor_id()) | |
404 | handle_backtrace_ipi(NULL); | |
405 | else | |
406 | smp_send_nmi_ipi(cpu, handle_backtrace_ipi, 1000000); | |
407 | } | |
408 | } | |
409 | ||
410 | void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) | |
411 | { | |
412 | nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); | |
413 | } |