]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Common interrupt code for 32 and 64 bit | |
3 | */ | |
4 | #include <linux/cpu.h> | |
5 | #include <linux/interrupt.h> | |
6 | #include <linux/kernel_stat.h> | |
7 | #include <linux/of.h> | |
8 | #include <linux/seq_file.h> | |
9 | #include <linux/smp.h> | |
10 | #include <linux/ftrace.h> | |
11 | #include <linux/delay.h> | |
12 | #include <linux/export.h> | |
13 | ||
14 | #include <asm/apic.h> | |
15 | #include <asm/io_apic.h> | |
16 | #include <asm/irq.h> | |
17 | #include <asm/mce.h> | |
18 | #include <asm/hw_irq.h> | |
19 | #include <asm/desc.h> | |
20 | ||
21 | #define CREATE_TRACE_POINTS | |
22 | #include <asm/trace/irq_vectors.h> | |
23 | ||
24 | DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); | |
25 | EXPORT_PER_CPU_SYMBOL(irq_stat); | |
26 | ||
27 | DEFINE_PER_CPU(struct pt_regs *, irq_regs); | |
28 | EXPORT_PER_CPU_SYMBOL(irq_regs); | |
29 | ||
30 | atomic_t irq_err_count; | |
31 | ||
32 | /* | |
33 | * 'what should we do if we get a hw irq event on an illegal vector'. | |
34 | * each architecture has to answer this themselves. | |
35 | */ | |
36 | void ack_bad_irq(unsigned int irq) | |
37 | { | |
38 | if (printk_ratelimit()) | |
39 | pr_err("unexpected IRQ trap at vector %02x\n", irq); | |
40 | ||
41 | /* | |
42 | * Currently unexpected vectors happen only on SMP and APIC. | |
43 | * We _must_ ack these because every local APIC has only N | |
44 | * irq slots per priority level, and a 'hanging, unacked' IRQ | |
45 | * holds up an irq slot - in excessive cases (when multiple | |
46 | * unexpected vectors occur) that might lock up the APIC | |
47 | * completely. | |
48 | * But only ack when the APIC is enabled -AK | |
49 | */ | |
50 | ack_APIC_irq(); | |
51 | } | |
52 | ||
53 | #define irq_stats(x) (&per_cpu(irq_stat, x)) | |
54 | /* | |
55 | * /proc/interrupts printing for arch specific interrupts | |
56 | */ | |
57 | int arch_show_interrupts(struct seq_file *p, int prec) | |
58 | { | |
59 | int j; | |
60 | ||
61 | seq_printf(p, "%*s: ", prec, "NMI"); | |
62 | for_each_online_cpu(j) | |
63 | seq_printf(p, "%10u ", irq_stats(j)->__nmi_count); | |
64 | seq_puts(p, " Non-maskable interrupts\n"); | |
65 | #ifdef CONFIG_X86_LOCAL_APIC | |
66 | seq_printf(p, "%*s: ", prec, "LOC"); | |
67 | for_each_online_cpu(j) | |
68 | seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs); | |
69 | seq_puts(p, " Local timer interrupts\n"); | |
70 | ||
71 | seq_printf(p, "%*s: ", prec, "SPU"); | |
72 | for_each_online_cpu(j) | |
73 | seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count); | |
74 | seq_puts(p, " Spurious interrupts\n"); | |
75 | seq_printf(p, "%*s: ", prec, "PMI"); | |
76 | for_each_online_cpu(j) | |
77 | seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs); | |
78 | seq_puts(p, " Performance monitoring interrupts\n"); | |
79 | seq_printf(p, "%*s: ", prec, "IWI"); | |
80 | for_each_online_cpu(j) | |
81 | seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs); | |
82 | seq_puts(p, " IRQ work interrupts\n"); | |
83 | seq_printf(p, "%*s: ", prec, "RTR"); | |
84 | for_each_online_cpu(j) | |
85 | seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count); | |
86 | seq_puts(p, " APIC ICR read retries\n"); | |
87 | if (x86_platform_ipi_callback) { | |
88 | seq_printf(p, "%*s: ", prec, "PLT"); | |
89 | for_each_online_cpu(j) | |
90 | seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis); | |
91 | seq_puts(p, " Platform interrupts\n"); | |
92 | } | |
93 | #endif | |
94 | #ifdef CONFIG_SMP | |
95 | seq_printf(p, "%*s: ", prec, "RES"); | |
96 | for_each_online_cpu(j) | |
97 | seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count); | |
98 | seq_puts(p, " Rescheduling interrupts\n"); | |
99 | seq_printf(p, "%*s: ", prec, "CAL"); | |
100 | for_each_online_cpu(j) | |
101 | seq_printf(p, "%10u ", irq_stats(j)->irq_call_count); | |
102 | seq_puts(p, " Function call interrupts\n"); | |
103 | seq_printf(p, "%*s: ", prec, "TLB"); | |
104 | for_each_online_cpu(j) | |
105 | seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count); | |
106 | seq_puts(p, " TLB shootdowns\n"); | |
107 | #endif | |
108 | #ifdef CONFIG_X86_THERMAL_VECTOR | |
109 | seq_printf(p, "%*s: ", prec, "TRM"); | |
110 | for_each_online_cpu(j) | |
111 | seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count); | |
112 | seq_puts(p, " Thermal event interrupts\n"); | |
113 | #endif | |
114 | #ifdef CONFIG_X86_MCE_THRESHOLD | |
115 | seq_printf(p, "%*s: ", prec, "THR"); | |
116 | for_each_online_cpu(j) | |
117 | seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count); | |
118 | seq_puts(p, " Threshold APIC interrupts\n"); | |
119 | #endif | |
120 | #ifdef CONFIG_X86_MCE_AMD | |
121 | seq_printf(p, "%*s: ", prec, "DFR"); | |
122 | for_each_online_cpu(j) | |
123 | seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count); | |
124 | seq_puts(p, " Deferred Error APIC interrupts\n"); | |
125 | #endif | |
126 | #ifdef CONFIG_X86_MCE | |
127 | seq_printf(p, "%*s: ", prec, "MCE"); | |
128 | for_each_online_cpu(j) | |
129 | seq_printf(p, "%10u ", per_cpu(mce_exception_count, j)); | |
130 | seq_puts(p, " Machine check exceptions\n"); | |
131 | seq_printf(p, "%*s: ", prec, "MCP"); | |
132 | for_each_online_cpu(j) | |
133 | seq_printf(p, "%10u ", per_cpu(mce_poll_count, j)); | |
134 | seq_puts(p, " Machine check polls\n"); | |
135 | #endif | |
136 | #if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN) | |
137 | if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) { | |
138 | seq_printf(p, "%*s: ", prec, "HYP"); | |
139 | for_each_online_cpu(j) | |
140 | seq_printf(p, "%10u ", | |
141 | irq_stats(j)->irq_hv_callback_count); | |
142 | seq_puts(p, " Hypervisor callback interrupts\n"); | |
143 | } | |
144 | #endif | |
145 | seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); | |
146 | #if defined(CONFIG_X86_IO_APIC) | |
147 | seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count)); | |
148 | #endif | |
149 | #ifdef CONFIG_HAVE_KVM | |
150 | seq_printf(p, "%*s: ", prec, "PIN"); | |
151 | for_each_online_cpu(j) | |
152 | seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis); | |
153 | seq_puts(p, " Posted-interrupt notification event\n"); | |
154 | ||
155 | seq_printf(p, "%*s: ", prec, "NPI"); | |
156 | for_each_online_cpu(j) | |
157 | seq_printf(p, "%10u ", | |
158 | irq_stats(j)->kvm_posted_intr_nested_ipis); | |
159 | seq_puts(p, " Nested posted-interrupt event\n"); | |
160 | ||
161 | seq_printf(p, "%*s: ", prec, "PIW"); | |
162 | for_each_online_cpu(j) | |
163 | seq_printf(p, "%10u ", | |
164 | irq_stats(j)->kvm_posted_intr_wakeup_ipis); | |
165 | seq_puts(p, " Posted-interrupt wakeup event\n"); | |
166 | #endif | |
167 | return 0; | |
168 | } | |
169 | ||
170 | /* | |
171 | * /proc/stat helpers | |
172 | */ | |
173 | u64 arch_irq_stat_cpu(unsigned int cpu) | |
174 | { | |
175 | u64 sum = irq_stats(cpu)->__nmi_count; | |
176 | ||
177 | #ifdef CONFIG_X86_LOCAL_APIC | |
178 | sum += irq_stats(cpu)->apic_timer_irqs; | |
179 | sum += irq_stats(cpu)->irq_spurious_count; | |
180 | sum += irq_stats(cpu)->apic_perf_irqs; | |
181 | sum += irq_stats(cpu)->apic_irq_work_irqs; | |
182 | sum += irq_stats(cpu)->icr_read_retry_count; | |
183 | if (x86_platform_ipi_callback) | |
184 | sum += irq_stats(cpu)->x86_platform_ipis; | |
185 | #endif | |
186 | #ifdef CONFIG_SMP | |
187 | sum += irq_stats(cpu)->irq_resched_count; | |
188 | sum += irq_stats(cpu)->irq_call_count; | |
189 | #endif | |
190 | #ifdef CONFIG_X86_THERMAL_VECTOR | |
191 | sum += irq_stats(cpu)->irq_thermal_count; | |
192 | #endif | |
193 | #ifdef CONFIG_X86_MCE_THRESHOLD | |
194 | sum += irq_stats(cpu)->irq_threshold_count; | |
195 | #endif | |
196 | #ifdef CONFIG_X86_MCE | |
197 | sum += per_cpu(mce_exception_count, cpu); | |
198 | sum += per_cpu(mce_poll_count, cpu); | |
199 | #endif | |
200 | return sum; | |
201 | } | |
202 | ||
203 | u64 arch_irq_stat(void) | |
204 | { | |
205 | u64 sum = atomic_read(&irq_err_count); | |
206 | return sum; | |
207 | } | |
208 | ||
209 | ||
210 | /* | |
211 | * do_IRQ handles all normal device IRQ's (the special | |
212 | * SMP cross-CPU interrupts have their own specific | |
213 | * handlers). | |
214 | */ | |
215 | __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) | |
216 | { | |
217 | struct pt_regs *old_regs = set_irq_regs(regs); | |
218 | struct irq_desc * desc; | |
219 | /* high bit used in ret_from_ code */ | |
220 | unsigned vector = ~regs->orig_ax; | |
221 | ||
222 | /* | |
223 | * NB: Unlike exception entries, IRQ entries do not reliably | |
224 | * handle context tracking in the low-level entry code. This is | |
225 | * because syscall entries execute briefly with IRQs on before | |
226 | * updating context tracking state, so we can take an IRQ from | |
227 | * kernel mode with CONTEXT_USER. The low-level entry code only | |
228 | * updates the context if we came from user mode, so we won't | |
229 | * switch to CONTEXT_KERNEL. We'll fix that once the syscall | |
230 | * code is cleaned up enough that we can cleanly defer enabling | |
231 | * IRQs. | |
232 | */ | |
233 | ||
234 | entering_irq(); | |
235 | ||
236 | /* entering_irq() tells RCU that we're not quiescent. Check it. */ | |
237 | RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU"); | |
238 | ||
239 | desc = __this_cpu_read(vector_irq[vector]); | |
240 | ||
241 | if (!handle_irq(desc, regs)) { | |
242 | ack_APIC_irq(); | |
243 | ||
244 | if (desc != VECTOR_RETRIGGERED) { | |
245 | pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n", | |
246 | __func__, smp_processor_id(), | |
247 | vector); | |
248 | } else { | |
249 | __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); | |
250 | } | |
251 | } | |
252 | ||
253 | exiting_irq(); | |
254 | ||
255 | set_irq_regs(old_regs); | |
256 | return 1; | |
257 | } | |
258 | ||
259 | #ifdef CONFIG_X86_LOCAL_APIC | |
260 | /* Function pointer for generic interrupt vector handling */ | |
261 | void (*x86_platform_ipi_callback)(void) = NULL; | |
262 | /* | |
263 | * Handler for X86_PLATFORM_IPI_VECTOR. | |
264 | */ | |
265 | __visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs) | |
266 | { | |
267 | struct pt_regs *old_regs = set_irq_regs(regs); | |
268 | ||
269 | entering_ack_irq(); | |
270 | trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); | |
271 | inc_irq_stat(x86_platform_ipis); | |
272 | if (x86_platform_ipi_callback) | |
273 | x86_platform_ipi_callback(); | |
274 | trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); | |
275 | exiting_irq(); | |
276 | set_irq_regs(old_regs); | |
277 | } | |
278 | #endif | |
279 | ||
280 | #ifdef CONFIG_HAVE_KVM | |
281 | static void dummy_handler(void) {} | |
282 | static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler; | |
283 | ||
284 | void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)) | |
285 | { | |
286 | if (handler) | |
287 | kvm_posted_intr_wakeup_handler = handler; | |
288 | else | |
289 | kvm_posted_intr_wakeup_handler = dummy_handler; | |
290 | } | |
291 | EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler); | |
292 | ||
293 | /* | |
294 | * Handler for POSTED_INTERRUPT_VECTOR. | |
295 | */ | |
296 | __visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs) | |
297 | { | |
298 | struct pt_regs *old_regs = set_irq_regs(regs); | |
299 | ||
300 | entering_ack_irq(); | |
301 | inc_irq_stat(kvm_posted_intr_ipis); | |
302 | exiting_irq(); | |
303 | set_irq_regs(old_regs); | |
304 | } | |
305 | ||
306 | /* | |
307 | * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. | |
308 | */ | |
309 | __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs) | |
310 | { | |
311 | struct pt_regs *old_regs = set_irq_regs(regs); | |
312 | ||
313 | entering_ack_irq(); | |
314 | inc_irq_stat(kvm_posted_intr_wakeup_ipis); | |
315 | kvm_posted_intr_wakeup_handler(); | |
316 | exiting_irq(); | |
317 | set_irq_regs(old_regs); | |
318 | } | |
319 | ||
320 | /* | |
321 | * Handler for POSTED_INTERRUPT_NESTED_VECTOR. | |
322 | */ | |
323 | __visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs) | |
324 | { | |
325 | struct pt_regs *old_regs = set_irq_regs(regs); | |
326 | ||
327 | entering_ack_irq(); | |
328 | inc_irq_stat(kvm_posted_intr_nested_ipis); | |
329 | exiting_irq(); | |
330 | set_irq_regs(old_regs); | |
331 | } | |
332 | #endif | |
333 | ||
334 | ||
335 | #ifdef CONFIG_HOTPLUG_CPU | |
336 | /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ | |
337 | void fixup_irqs(void) | |
338 | { | |
339 | unsigned int irr, vector; | |
340 | struct irq_desc *desc; | |
341 | struct irq_data *data; | |
342 | struct irq_chip *chip; | |
343 | ||
344 | irq_migrate_all_off_this_cpu(); | |
345 | ||
346 | /* | |
347 | * We can remove mdelay() and then send spuriuous interrupts to | |
348 | * new cpu targets for all the irqs that were handled previously by | |
349 | * this cpu. While it works, I have seen spurious interrupt messages | |
350 | * (nothing wrong but still...). | |
351 | * | |
352 | * So for now, retain mdelay(1) and check the IRR and then send those | |
353 | * interrupts to new targets as this cpu is already offlined... | |
354 | */ | |
355 | mdelay(1); | |
356 | ||
357 | /* | |
358 | * We can walk the vector array of this cpu without holding | |
359 | * vector_lock because the cpu is already marked !online, so | |
360 | * nothing else will touch it. | |
361 | */ | |
362 | for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { | |
363 | if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector]))) | |
364 | continue; | |
365 | ||
366 | irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); | |
367 | if (irr & (1 << (vector % 32))) { | |
368 | desc = __this_cpu_read(vector_irq[vector]); | |
369 | ||
370 | raw_spin_lock(&desc->lock); | |
371 | data = irq_desc_get_irq_data(desc); | |
372 | chip = irq_data_get_irq_chip(data); | |
373 | if (chip->irq_retrigger) { | |
374 | chip->irq_retrigger(data); | |
375 | __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED); | |
376 | } | |
377 | raw_spin_unlock(&desc->lock); | |
378 | } | |
379 | if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED) | |
380 | __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); | |
381 | } | |
382 | } | |
383 | #endif |