[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / irq.c

/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>

#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>
#include <asm/desc.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>

DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
EXPORT_PER_CPU_SYMBOL(irq_stat);

DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);

atomic_t irq_err_count;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	ack_APIC_irq();
}

#define irq_stats(x)		(&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
	int j;

	seq_printf(p, "%*s: ", prec, "NMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	seq_puts(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
	seq_printf(p, "%*s: ", prec, "LOC");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	seq_puts(p, "  Local timer interrupts\n");

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	seq_puts(p, "  Spurious interrupts\n");
	seq_printf(p, "%*s: ", prec, "PMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
	seq_puts(p, "  Performance monitoring interrupts\n");
	seq_printf(p, "%*s: ", prec, "IWI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
	seq_puts(p, "  IRQ work interrupts\n");
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
	seq_puts(p, "  APIC ICR read retries\n");
	if (x86_platform_ipi_callback) {
		seq_printf(p, "%*s: ", prec, "PLT");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
		seq_puts(p, "  Platform interrupts\n");
	}
#endif
#ifdef CONFIG_SMP
	seq_printf(p, "%*s: ", prec, "RES");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	seq_puts(p, "  Rescheduling interrupts\n");
	seq_printf(p, "%*s: ", prec, "CAL");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
	seq_puts(p, "  Function call interrupts\n");
	seq_printf(p, "%*s: ", prec, "TLB");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	seq_puts(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	seq_printf(p, "%*s: ", prec, "TRM");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	seq_puts(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	seq_puts(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_AMD
	seq_printf(p, "%*s: ", prec, "DFR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	seq_puts(p, "  Deferred Error APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	seq_puts(p, "  Machine check exceptions\n");
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	seq_puts(p, "  Machine check polls\n");
#endif
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
		seq_printf(p, "%*s: ", prec, "HYP");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->irq_hv_callback_count);
		seq_puts(p, "  Hypervisor callback interrupts\n");
	}
#endif
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
#ifdef CONFIG_HAVE_KVM
	seq_printf(p, "%*s: ", prec, "PIN");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	seq_puts(p, "  Posted-interrupt notification event\n");

	seq_printf(p, "%*s: ", prec, "NPI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_nested_ipis);
	seq_puts(p, "  Nested posted-interrupt event\n");

	seq_printf(p, "%*s: ", prec, "PIW");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	seq_puts(p, "  Posted-interrupt wakeup event\n");
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
	sum += irq_stats(cpu)->irq_spurious_count;
	sum += irq_stats(cpu)->apic_perf_irqs;
	sum += irq_stats(cpu)->apic_irq_work_irqs;
	sum += irq_stats(cpu)->icr_read_retry_count;
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
#endif
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}


/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);
	struct irq_desc * desc;
	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;

	entering_irq();

	/* entering_irq() tells RCU that we're not quiescent.  Check it. */
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");

	desc = __this_cpu_read(vector_irq[vector]);

	if (!handle_irq(desc, regs)) {
		ack_APIC_irq();

		if (desc != VECTOR_RETRIGGERED) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
					     __func__, smp_processor_id(),
					     vector);
		} else {
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
		}
	}

	exiting_irq();

	set_irq_regs(old_regs);
	return 1;
}

#ifdef CONFIG_X86_LOCAL_APIC
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	inc_irq_stat(x86_platform_ipis);
	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif

#ifdef CONFIG_HAVE_KVM
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;

void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
{
	if (handler)
		kvm_posted_intr_wakeup_handler = handler;
	else
		kvm_posted_intr_wakeup_handler = dummy_handler;
}
EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);

/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	kvm_posted_intr_wakeup_handler();
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
 */
__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_nested_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif


#ifdef CONFIG_HOTPLUG_CPU
/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
	unsigned int irr, vector;
	struct irq_desc *desc;
	struct irq_data *data;
	struct irq_chip *chip;

	irq_migrate_all_off_this_cpu();

	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
	mdelay(1);

	/*
	 * We can walk the vector array of this cpu without holding
	 * vector_lock because the cpu is already marked !online, so
	 * nothing else will touch it.
	 */
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
			desc = __this_cpu_read(vector_irq[vector]);

			raw_spin_lock(&desc->lock);
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
			if (chip->irq_retrigger) {
				chip->irq_retrigger(data);
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
			raw_spin_unlock(&desc->lock);
		}
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
	}
}
#endif
Commit	Line	Data
6b39ba77 TG	1	/*
	2	* Common interrupt code for 32 and 64 bit
	3	*/
	4	#include <linux/cpu.h>
	5	#include <linux/interrupt.h>
	6	#include <linux/kernel_stat.h>
4722d194	7	#include <linux/of.h>
6b39ba77	8	#include <linux/seq_file.h>
6a02e710	9	#include <linux/smp.h>
7c1d7cdc	10	#include <linux/ftrace.h>
ca444564	11	#include <linux/delay.h>
69c60c88	12	#include <linux/export.h>
6b39ba77	13
7b6aa335	14	#include <asm/apic.h>
6b39ba77	15	#include <asm/io_apic.h>
c3d80000	16	#include <asm/irq.h>
01ca79f1	17	#include <asm/mce.h>
2c1b284e	18	#include <asm/hw_irq.h>
ac2a5539	19	#include <asm/desc.h>
83ab8514 SRRH	20
83ab8514 SRRH	21	#define CREATE_TRACE_POINTS
cf910e83	22	#include <asm/trace/irq_vectors.h>
6b39ba77	23
c5bde906 BG	24	DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
	25	EXPORT_PER_CPU_SYMBOL(irq_stat);
	26
	27	DEFINE_PER_CPU(struct pt_regs *, irq_regs);
	28	EXPORT_PER_CPU_SYMBOL(irq_regs);
	29
6b39ba77 TG	30	atomic_t irq_err_count;
6b39ba77 TG	31
249f6d9e TG	32	/*
	33	* 'what should we do if we get a hw irq event on an illegal vector'.
	34	* each architecture has to answer this themselves.
	35	*/
	36	void ack_bad_irq(unsigned int irq)
	37	{
edea7148 CG	38	if (printk_ratelimit())
edea7148 CG	39	pr_err("unexpected IRQ trap at vector %02x\n", irq);
249f6d9e	40
249f6d9e TG	41	/*
	42	* Currently unexpected vectors happen only on SMP and APIC.
	43	* We _must_ ack these because every local APIC has only N
	44	* irq slots per priority level, and a 'hanging, unacked' IRQ
	45	* holds up an irq slot - in excessive cases (when multiple
	46	* unexpected vectors occur) that might lock up the APIC
	47	* completely.
	48	* But only ack when the APIC is enabled -AK
	49	*/
08306ce6	50	ack_APIC_irq();
249f6d9e TG	51	}
249f6d9e TG	52
1b437c8c	53	#define irq_stats(x) (&per_cpu(irq_stat, x))
6b39ba77	54	/*
517e4981	55	* /proc/interrupts printing for arch specific interrupts
6b39ba77	56	*/
517e4981	57	int arch_show_interrupts(struct seq_file *p, int prec)
6b39ba77 TG	58	{
	59	int j;
	60
7a81d9a7	61	seq_printf(p, "%*s: ", prec, "NMI");
6b39ba77 TG	62	for_each_online_cpu(j)
6b39ba77 TG	63	seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
3736708f	64	seq_puts(p, " Non-maskable interrupts\n");
6b39ba77	65	#ifdef CONFIG_X86_LOCAL_APIC
7a81d9a7	66	seq_printf(p, "%*s: ", prec, "LOC");
6b39ba77 TG	67	for_each_online_cpu(j)
6b39ba77 TG	68	seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
3736708f	69	seq_puts(p, " Local timer interrupts\n");
474e56b8 JSR	70
	71	seq_printf(p, "%*s: ", prec, "SPU");
	72	for_each_online_cpu(j)
	73	seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
3736708f	74	seq_puts(p, " Spurious interrupts\n");
89ccf465	75	seq_printf(p, "%*s: ", prec, "PMI");
241771ef IM	76	for_each_online_cpu(j)
241771ef IM	77	seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
3736708f	78	seq_puts(p, " Performance monitoring interrupts\n");
e360adbe	79	seq_printf(p, "%*s: ", prec, "IWI");
b6276f35	80	for_each_online_cpu(j)
e360adbe	81	seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
3736708f	82	seq_puts(p, " IRQ work interrupts\n");
346b46be FLVC	83	seq_printf(p, "%*s: ", prec, "RTR");
346b46be FLVC	84	for_each_online_cpu(j)
b49d7d87	85	seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
3736708f	86	seq_puts(p, " APIC ICR read retries\n");
4a4de9c7	87	if (x86_platform_ipi_callback) {
59d13812	88	seq_printf(p, "%*s: ", prec, "PLT");
acaabe79	89	for_each_online_cpu(j)
4a4de9c7	90	seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
3736708f	91	seq_puts(p, " Platform interrupts\n");
acaabe79	92	}
0428e01a	93	#endif
6b39ba77	94	#ifdef CONFIG_SMP
7a81d9a7	95	seq_printf(p, "%*s: ", prec, "RES");
6b39ba77 TG	96	for_each_online_cpu(j)
6b39ba77 TG	97	seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
3736708f	98	seq_puts(p, " Rescheduling interrupts\n");
7a81d9a7	99	seq_printf(p, "%*s: ", prec, "CAL");
6b39ba77	100	for_each_online_cpu(j)
82ba4fac	101	seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
3736708f	102	seq_puts(p, " Function call interrupts\n");
7a81d9a7	103	seq_printf(p, "%*s: ", prec, "TLB");
6b39ba77 TG	104	for_each_online_cpu(j)
6b39ba77 TG	105	seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
3736708f	106	seq_puts(p, " TLB shootdowns\n");
6b39ba77	107	#endif
0444c9bd	108	#ifdef CONFIG_X86_THERMAL_VECTOR
7a81d9a7	109	seq_printf(p, "%*s: ", prec, "TRM");
6b39ba77 TG	110	for_each_online_cpu(j)
6b39ba77 TG	111	seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
3736708f	112	seq_puts(p, " Thermal event interrupts\n");
0444c9bd JB	113	#endif
0444c9bd JB	114	#ifdef CONFIG_X86_MCE_THRESHOLD
7a81d9a7	115	seq_printf(p, "%*s: ", prec, "THR");
6b39ba77 TG	116	for_each_online_cpu(j)
6b39ba77 TG	117	seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
3736708f	118	seq_puts(p, " Threshold APIC interrupts\n");
01ca79f1	119	#endif
24fd78a8 AG	120	#ifdef CONFIG_X86_MCE_AMD
	121	seq_printf(p, "%*s: ", prec, "DFR");
	122	for_each_online_cpu(j)
	123	seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	124	seq_puts(p, " Deferred Error APIC interrupts\n");
	125	#endif
c1ebf835	126	#ifdef CONFIG_X86_MCE
01ca79f1 AK	127	seq_printf(p, "%*s: ", prec, "MCE");
	128	for_each_online_cpu(j)
	129	seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
3736708f	130	seq_puts(p, " Machine check exceptions\n");
ca84f696 AK	131	seq_printf(p, "%*s: ", prec, "MCP");
	132	for_each_online_cpu(j)
	133	seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
3736708f	134	seq_puts(p, " Machine check polls\n");
6b39ba77	135	#endif
f704a7d7	136	#if IS_ENABLED(CONFIG_HYPERV) \|\| defined(CONFIG_XEN)
7854f822	137	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
9d87cd61 VK	138	seq_printf(p, "%*s: ", prec, "HYP");
	139	for_each_online_cpu(j)
	140	seq_printf(p, "%10u ",
	141	irq_stats(j)->irq_hv_callback_count);
	142	seq_puts(p, " Hypervisor callback interrupts\n");
	143	}
929320e4	144	#endif
7a81d9a7	145	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
6b39ba77	146	#if defined(CONFIG_X86_IO_APIC)
7a81d9a7	147	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
501b3265 FW	148	#endif
	149	#ifdef CONFIG_HAVE_KVM
	150	seq_printf(p, "%*s: ", prec, "PIN");
	151	for_each_online_cpu(j)
	152	seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	153	seq_puts(p, " Posted-interrupt notification event\n");
	154
210f84b0 WV	155	seq_printf(p, "%*s: ", prec, "NPI");
	156	for_each_online_cpu(j)
	157	seq_printf(p, "%10u ",
	158	irq_stats(j)->kvm_posted_intr_nested_ipis);
	159	seq_puts(p, " Nested posted-interrupt event\n");
	160
501b3265 FW	161	seq_printf(p, "%*s: ", prec, "PIW");
	162	for_each_online_cpu(j)
	163	seq_printf(p, "%10u ",
	164	irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	165	seq_puts(p, " Posted-interrupt wakeup event\n");
6b39ba77 TG	166	#endif
	167	return 0;
	168	}
	169
6b39ba77 TG	170	/*
	171	* /proc/stat helpers
	172	*/
	173	u64 arch_irq_stat_cpu(unsigned int cpu)
	174	{
	175	u64 sum = irq_stats(cpu)->__nmi_count;
	176
	177	#ifdef CONFIG_X86_LOCAL_APIC
	178	sum += irq_stats(cpu)->apic_timer_irqs;
474e56b8	179	sum += irq_stats(cpu)->irq_spurious_count;
241771ef	180	sum += irq_stats(cpu)->apic_perf_irqs;
e360adbe	181	sum += irq_stats(cpu)->apic_irq_work_irqs;
b49d7d87	182	sum += irq_stats(cpu)->icr_read_retry_count;
4a4de9c7 DS	183	if (x86_platform_ipi_callback)
4a4de9c7 DS	184	sum += irq_stats(cpu)->x86_platform_ipis;
0428e01a	185	#endif
6b39ba77 TG	186	#ifdef CONFIG_SMP
	187	sum += irq_stats(cpu)->irq_resched_count;
	188	sum += irq_stats(cpu)->irq_call_count;
6b39ba77	189	#endif
0444c9bd	190	#ifdef CONFIG_X86_THERMAL_VECTOR
6b39ba77	191	sum += irq_stats(cpu)->irq_thermal_count;
0444c9bd JB	192	#endif
0444c9bd JB	193	#ifdef CONFIG_X86_MCE_THRESHOLD
6b39ba77	194	sum += irq_stats(cpu)->irq_threshold_count;
8051dbd2	195	#endif
c1ebf835	196	#ifdef CONFIG_X86_MCE
8051dbd2 HS	197	sum += per_cpu(mce_exception_count, cpu);
8051dbd2 HS	198	sum += per_cpu(mce_poll_count, cpu);
6b39ba77 TG	199	#endif
	200	return sum;
	201	}
	202
	203	u64 arch_irq_stat(void)
	204	{
	205	u64 sum = atomic_read(&irq_err_count);
6b39ba77 TG	206	return sum;
6b39ba77 TG	207	}
c3d80000	208
7c1d7cdc JF	209
	210	/*
	211	* do_IRQ handles all normal device IRQ's (the special
	212	* SMP cross-CPU interrupts have their own specific
	213	* handlers).
	214	*/
1d9090e2	215	__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
7c1d7cdc JF	216	{
7c1d7cdc JF	217	struct pt_regs *old_regs = set_irq_regs(regs);
a782a7e4	218	struct irq_desc * desc;
7c1d7cdc JF	219	/* high bit used in ret_from_ code */
7c1d7cdc JF	220	unsigned vector = ~regs->orig_ax;
7c1d7cdc	221
6af7faf6	222	entering_irq();
7c1d7cdc	223
0333a209	224	/* entering_irq() tells RCU that we're not quiescent. Check it. */
5778077d	225	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
0333a209	226
a782a7e4	227	desc = __this_cpu_read(vector_irq[vector]);
7c1d7cdc	228
a782a7e4	229	if (!handle_irq(desc, regs)) {
08306ce6	230	ack_APIC_irq();
7c1d7cdc	231
a782a7e4 TG	232	if (desc != VECTOR_RETRIGGERED) {
a782a7e4 TG	233	pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
9345005f	234	__func__, smp_processor_id(),
a782a7e4	235	vector);
9345005f	236	} else {
7276c6a2	237	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
9345005f	238	}
7c1d7cdc JF	239	}
7c1d7cdc JF	240
6af7faf6	241	exiting_irq();
7c1d7cdc JF	242
	243	set_irq_regs(old_regs);
	244	return 1;
	245	}
	246
0428e01a TG	247	#ifdef CONFIG_X86_LOCAL_APIC
	248	/* Function pointer for generic interrupt vector handling */
	249	void (*x86_platform_ipi_callback)(void) = NULL;
acaabe79	250	/*
4a4de9c7	251	* Handler for X86_PLATFORM_IPI_VECTOR.
acaabe79	252	*/
c4158ff5	253	__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
eddc0e92 SA	254	{
eddc0e92 SA	255	struct pt_regs *old_regs = set_irq_regs(regs);
acaabe79	256
eddc0e92	257	entering_ack_irq();
8a17116b TG	258	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	259	inc_irq_stat(x86_platform_ipis);
	260	if (x86_platform_ipi_callback)
	261	x86_platform_ipi_callback();
	262	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
eddc0e92	263	exiting_irq();
acaabe79 DS	264	set_irq_regs(old_regs);
acaabe79 DS	265	}
0428e01a	266	#endif
acaabe79	267
d78f2664	268	#ifdef CONFIG_HAVE_KVM
f6b3c72c FW	269	static void dummy_handler(void) {}
	270	static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
	271
	272	void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
	273	{
	274	if (handler)
	275	kvm_posted_intr_wakeup_handler = handler;
	276	else
	277	kvm_posted_intr_wakeup_handler = dummy_handler;
	278	}
	279	EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
	280
d78f2664 YZ	281	/*
	282	* Handler for POSTED_INTERRUPT_VECTOR.
	283	*/
1d9090e2	284	__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
d78f2664 YZ	285	{
	286	struct pt_regs *old_regs = set_irq_regs(regs);
	287
6af7faf6	288	entering_ack_irq();
d78f2664	289	inc_irq_stat(kvm_posted_intr_ipis);
f6b3c72c FW	290	exiting_irq();
	291	set_irq_regs(old_regs);
	292	}
	293
	294	/*
	295	* Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
	296	*/
	297	__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
	298	{
	299	struct pt_regs *old_regs = set_irq_regs(regs);
	300
	301	entering_ack_irq();
	302	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	303	kvm_posted_intr_wakeup_handler();
6af7faf6	304	exiting_irq();
d78f2664 YZ	305	set_irq_regs(old_regs);
d78f2664 YZ	306	}
210f84b0 WV	307
	308	/*
	309	* Handler for POSTED_INTERRUPT_NESTED_VECTOR.
	310	*/
	311	__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
	312	{
	313	struct pt_regs *old_regs = set_irq_regs(regs);
	314
	315	entering_ack_irq();
	316	inc_irq_stat(kvm_posted_intr_nested_ipis);
	317	exiting_irq();
	318	set_irq_regs(old_regs);
	319	}
d78f2664 YZ	320	#endif
d78f2664 YZ	321
cf910e83	322
7a7732bc SS	323	#ifdef CONFIG_HOTPLUG_CPU
	324	/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
	325	void fixup_irqs(void)
	326	{
ad7a929f	327	unsigned int irr, vector;
7a7732bc	328	struct irq_desc *desc;
a3c08e5d	329	struct irq_data *data;
51c43ac6	330	struct irq_chip *chip;
7a7732bc	331
ad7a929f	332	irq_migrate_all_off_this_cpu();
7a7732bc	333
5231a686 SS	334	/*
	335	* We can remove mdelay() and then send spuriuous interrupts to
	336	* new cpu targets for all the irqs that were handled previously by
	337	* this cpu. While it works, I have seen spurious interrupt messages
	338	* (nothing wrong but still...).
	339	*
	340	* So for now, retain mdelay(1) and check the IRR and then send those
	341	* interrupts to new targets as this cpu is already offlined...
	342	*/
7a7732bc	343	mdelay(1);
5231a686	344
09cf92b7 TG	345	/*
	346	* We can walk the vector array of this cpu without holding
	347	* vector_lock because the cpu is already marked !online, so
	348	* nothing else will touch it.
	349	*/
5231a686	350	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
a782a7e4	351	if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
5231a686 SS	352	continue;
	353
	354	irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
	355	if (irr & (1 << (vector % 32))) {
a782a7e4	356	desc = __this_cpu_read(vector_irq[vector]);
5231a686	357
09cf92b7	358	raw_spin_lock(&desc->lock);
51c43ac6 TG	359	data = irq_desc_get_irq_data(desc);
51c43ac6 TG	360	chip = irq_data_get_irq_chip(data);
9345005f	361	if (chip->irq_retrigger) {
51c43ac6	362	chip->irq_retrigger(data);
9345005f PB	363	__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
9345005f PB	364	}
239007b8	365	raw_spin_unlock(&desc->lock);
5231a686	366	}
9345005f	367	if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
7276c6a2	368	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
5231a686	369	}
7a7732bc SS	370	}
7a7732bc SS	371	#endif