[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / irq.c

/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/irq.h>

#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>
#include <asm/desc.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>

DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
EXPORT_PER_CPU_SYMBOL(irq_stat);

DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);

atomic_t irq_err_count;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	ack_APIC_irq();
}

#define irq_stats(x)		(&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
	int j;

	seq_printf(p, "%*s: ", prec, "NMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	seq_puts(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
	seq_printf(p, "%*s: ", prec, "LOC");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	seq_puts(p, "  Local timer interrupts\n");

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	seq_puts(p, "  Spurious interrupts\n");
	seq_printf(p, "%*s: ", prec, "PMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
	seq_puts(p, "  Performance monitoring interrupts\n");
	seq_printf(p, "%*s: ", prec, "IWI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
	seq_puts(p, "  IRQ work interrupts\n");
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
	seq_puts(p, "  APIC ICR read retries\n");
	if (x86_platform_ipi_callback) {
		seq_printf(p, "%*s: ", prec, "PLT");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
		seq_puts(p, "  Platform interrupts\n");
	}
#endif
#ifdef CONFIG_SMP
	seq_printf(p, "%*s: ", prec, "RES");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	seq_puts(p, "  Rescheduling interrupts\n");
	seq_printf(p, "%*s: ", prec, "CAL");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
	seq_puts(p, "  Function call interrupts\n");
	seq_printf(p, "%*s: ", prec, "TLB");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	seq_puts(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	seq_printf(p, "%*s: ", prec, "TRM");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	seq_puts(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	seq_puts(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_AMD
	seq_printf(p, "%*s: ", prec, "DFR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	seq_puts(p, "  Deferred Error APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	seq_puts(p, "  Machine check exceptions\n");
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	seq_puts(p, "  Machine check polls\n");
#endif
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
		seq_printf(p, "%*s: ", prec, "HYP");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->irq_hv_callback_count);
		seq_puts(p, "  Hypervisor callback interrupts\n");
	}
#endif
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
#ifdef CONFIG_HAVE_KVM
	seq_printf(p, "%*s: ", prec, "PIN");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	seq_puts(p, "  Posted-interrupt notification event\n");

	seq_printf(p, "%*s: ", prec, "NPI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_nested_ipis);
	seq_puts(p, "  Nested posted-interrupt event\n");

	seq_printf(p, "%*s: ", prec, "PIW");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	seq_puts(p, "  Posted-interrupt wakeup event\n");
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
	sum += irq_stats(cpu)->irq_spurious_count;
	sum += irq_stats(cpu)->apic_perf_irqs;
	sum += irq_stats(cpu)->apic_irq_work_irqs;
	sum += irq_stats(cpu)->icr_read_retry_count;
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
#endif
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}


/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);
	struct irq_desc * desc;
	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;

	entering_irq();

	/* entering_irq() tells RCU that we're not quiescent.  Check it. */
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");

	desc = __this_cpu_read(vector_irq[vector]);

	if (!handle_irq(desc, regs)) {
		ack_APIC_irq();

		if (desc != VECTOR_RETRIGGERED && desc != VECTOR_SHUTDOWN) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
					     __func__, smp_processor_id(),
					     vector);
		} else {
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
		}
	}

	exiting_irq();

	set_irq_regs(old_regs);
	return 1;
}

#ifdef CONFIG_X86_LOCAL_APIC
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	inc_irq_stat(x86_platform_ipis);
	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif

#ifdef CONFIG_HAVE_KVM
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;

void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
{
	if (handler)
		kvm_posted_intr_wakeup_handler = handler;
	else
		kvm_posted_intr_wakeup_handler = dummy_handler;
}
EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);

/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	kvm_posted_intr_wakeup_handler();
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
 */
__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_nested_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif


#ifdef CONFIG_HOTPLUG_CPU
/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
	unsigned int irr, vector;
	struct irq_desc *desc;
	struct irq_data *data;
	struct irq_chip *chip;

	irq_migrate_all_off_this_cpu();

	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
	mdelay(1);

	/*
	 * We can walk the vector array of this cpu without holding
	 * vector_lock because the cpu is already marked !online, so
	 * nothing else will touch it.
	 */
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
			desc = __this_cpu_read(vector_irq[vector]);

			raw_spin_lock(&desc->lock);
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
			if (chip->irq_retrigger) {
				chip->irq_retrigger(data);
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
			raw_spin_unlock(&desc->lock);
		}
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
	}
}
#endif
Commit	Line	Data
6b39ba77 TG	1	/*
	2	* Common interrupt code for 32 and 64 bit
	3	*/
	4	#include <linux/cpu.h>
	5	#include <linux/interrupt.h>
	6	#include <linux/kernel_stat.h>
4722d194	7	#include <linux/of.h>
6b39ba77	8	#include <linux/seq_file.h>
6a02e710	9	#include <linux/smp.h>
7c1d7cdc	10	#include <linux/ftrace.h>
ca444564	11	#include <linux/delay.h>
69c60c88	12	#include <linux/export.h>
f901f138	13	#include <linux/irq.h>
6b39ba77	14
7b6aa335	15	#include <asm/apic.h>
6b39ba77	16	#include <asm/io_apic.h>
c3d80000	17	#include <asm/irq.h>
01ca79f1	18	#include <asm/mce.h>
2c1b284e	19	#include <asm/hw_irq.h>
ac2a5539	20	#include <asm/desc.h>
83ab8514 SRRH	21
83ab8514 SRRH	22	#define CREATE_TRACE_POINTS
cf910e83	23	#include <asm/trace/irq_vectors.h>
6b39ba77	24
c5bde906 BG	25	DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
	26	EXPORT_PER_CPU_SYMBOL(irq_stat);
	27
	28	DEFINE_PER_CPU(struct pt_regs *, irq_regs);
	29	EXPORT_PER_CPU_SYMBOL(irq_regs);
	30
6b39ba77 TG	31	atomic_t irq_err_count;
6b39ba77 TG	32
249f6d9e TG	33	/*
	34	* 'what should we do if we get a hw irq event on an illegal vector'.
	35	* each architecture has to answer this themselves.
	36	*/
	37	void ack_bad_irq(unsigned int irq)
	38	{
edea7148 CG	39	if (printk_ratelimit())
edea7148 CG	40	pr_err("unexpected IRQ trap at vector %02x\n", irq);
249f6d9e	41
249f6d9e TG	42	/*
	43	* Currently unexpected vectors happen only on SMP and APIC.
	44	* We _must_ ack these because every local APIC has only N
	45	* irq slots per priority level, and a 'hanging, unacked' IRQ
	46	* holds up an irq slot - in excessive cases (when multiple
	47	* unexpected vectors occur) that might lock up the APIC
	48	* completely.
	49	* But only ack when the APIC is enabled -AK
	50	*/
08306ce6	51	ack_APIC_irq();
249f6d9e TG	52	}
249f6d9e TG	53
1b437c8c	54	#define irq_stats(x) (&per_cpu(irq_stat, x))
6b39ba77	55	/*
517e4981	56	* /proc/interrupts printing for arch specific interrupts
6b39ba77	57	*/
517e4981	58	int arch_show_interrupts(struct seq_file *p, int prec)
6b39ba77 TG	59	{
	60	int j;
	61
7a81d9a7	62	seq_printf(p, "%*s: ", prec, "NMI");
6b39ba77 TG	63	for_each_online_cpu(j)
6b39ba77 TG	64	seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
3736708f	65	seq_puts(p, " Non-maskable interrupts\n");
6b39ba77	66	#ifdef CONFIG_X86_LOCAL_APIC
7a81d9a7	67	seq_printf(p, "%*s: ", prec, "LOC");
6b39ba77 TG	68	for_each_online_cpu(j)
6b39ba77 TG	69	seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
3736708f	70	seq_puts(p, " Local timer interrupts\n");
474e56b8 JSR	71
	72	seq_printf(p, "%*s: ", prec, "SPU");
	73	for_each_online_cpu(j)
	74	seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
3736708f	75	seq_puts(p, " Spurious interrupts\n");
89ccf465	76	seq_printf(p, "%*s: ", prec, "PMI");
241771ef IM	77	for_each_online_cpu(j)
241771ef IM	78	seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
3736708f	79	seq_puts(p, " Performance monitoring interrupts\n");
e360adbe	80	seq_printf(p, "%*s: ", prec, "IWI");
b6276f35	81	for_each_online_cpu(j)
e360adbe	82	seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
3736708f	83	seq_puts(p, " IRQ work interrupts\n");
346b46be FLVC	84	seq_printf(p, "%*s: ", prec, "RTR");
346b46be FLVC	85	for_each_online_cpu(j)
b49d7d87	86	seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
3736708f	87	seq_puts(p, " APIC ICR read retries\n");
4a4de9c7	88	if (x86_platform_ipi_callback) {
59d13812	89	seq_printf(p, "%*s: ", prec, "PLT");
acaabe79	90	for_each_online_cpu(j)
4a4de9c7	91	seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
3736708f	92	seq_puts(p, " Platform interrupts\n");
acaabe79	93	}
0428e01a	94	#endif
6b39ba77	95	#ifdef CONFIG_SMP
7a81d9a7	96	seq_printf(p, "%*s: ", prec, "RES");
6b39ba77 TG	97	for_each_online_cpu(j)
6b39ba77 TG	98	seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
3736708f	99	seq_puts(p, " Rescheduling interrupts\n");
7a81d9a7	100	seq_printf(p, "%*s: ", prec, "CAL");
6b39ba77	101	for_each_online_cpu(j)
82ba4fac	102	seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
3736708f	103	seq_puts(p, " Function call interrupts\n");
7a81d9a7	104	seq_printf(p, "%*s: ", prec, "TLB");
6b39ba77 TG	105	for_each_online_cpu(j)
6b39ba77 TG	106	seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
3736708f	107	seq_puts(p, " TLB shootdowns\n");
6b39ba77	108	#endif
0444c9bd	109	#ifdef CONFIG_X86_THERMAL_VECTOR
7a81d9a7	110	seq_printf(p, "%*s: ", prec, "TRM");
6b39ba77 TG	111	for_each_online_cpu(j)
6b39ba77 TG	112	seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
3736708f	113	seq_puts(p, " Thermal event interrupts\n");
0444c9bd JB	114	#endif
0444c9bd JB	115	#ifdef CONFIG_X86_MCE_THRESHOLD
7a81d9a7	116	seq_printf(p, "%*s: ", prec, "THR");
6b39ba77 TG	117	for_each_online_cpu(j)
6b39ba77 TG	118	seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
3736708f	119	seq_puts(p, " Threshold APIC interrupts\n");
01ca79f1	120	#endif
24fd78a8 AG	121	#ifdef CONFIG_X86_MCE_AMD
	122	seq_printf(p, "%*s: ", prec, "DFR");
	123	for_each_online_cpu(j)
	124	seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	125	seq_puts(p, " Deferred Error APIC interrupts\n");
	126	#endif
c1ebf835	127	#ifdef CONFIG_X86_MCE
01ca79f1 AK	128	seq_printf(p, "%*s: ", prec, "MCE");
	129	for_each_online_cpu(j)
	130	seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
3736708f	131	seq_puts(p, " Machine check exceptions\n");
ca84f696 AK	132	seq_printf(p, "%*s: ", prec, "MCP");
	133	for_each_online_cpu(j)
	134	seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
3736708f	135	seq_puts(p, " Machine check polls\n");
6b39ba77	136	#endif
f704a7d7	137	#if IS_ENABLED(CONFIG_HYPERV) \|\| defined(CONFIG_XEN)
7854f822	138	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
9d87cd61 VK	139	seq_printf(p, "%*s: ", prec, "HYP");
	140	for_each_online_cpu(j)
	141	seq_printf(p, "%10u ",
	142	irq_stats(j)->irq_hv_callback_count);
	143	seq_puts(p, " Hypervisor callback interrupts\n");
	144	}
929320e4	145	#endif
7a81d9a7	146	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
6b39ba77	147	#if defined(CONFIG_X86_IO_APIC)
7a81d9a7	148	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
501b3265 FW	149	#endif
	150	#ifdef CONFIG_HAVE_KVM
	151	seq_printf(p, "%*s: ", prec, "PIN");
	152	for_each_online_cpu(j)
	153	seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	154	seq_puts(p, " Posted-interrupt notification event\n");
	155
210f84b0 WV	156	seq_printf(p, "%*s: ", prec, "NPI");
	157	for_each_online_cpu(j)
	158	seq_printf(p, "%10u ",
	159	irq_stats(j)->kvm_posted_intr_nested_ipis);
	160	seq_puts(p, " Nested posted-interrupt event\n");
	161
501b3265 FW	162	seq_printf(p, "%*s: ", prec, "PIW");
	163	for_each_online_cpu(j)
	164	seq_printf(p, "%10u ",
	165	irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	166	seq_puts(p, " Posted-interrupt wakeup event\n");
6b39ba77 TG	167	#endif
	168	return 0;
	169	}
	170
6b39ba77 TG	171	/*
	172	* /proc/stat helpers
	173	*/
	174	u64 arch_irq_stat_cpu(unsigned int cpu)
	175	{
	176	u64 sum = irq_stats(cpu)->__nmi_count;
	177
	178	#ifdef CONFIG_X86_LOCAL_APIC
	179	sum += irq_stats(cpu)->apic_timer_irqs;
474e56b8	180	sum += irq_stats(cpu)->irq_spurious_count;
241771ef	181	sum += irq_stats(cpu)->apic_perf_irqs;
e360adbe	182	sum += irq_stats(cpu)->apic_irq_work_irqs;
b49d7d87	183	sum += irq_stats(cpu)->icr_read_retry_count;
4a4de9c7 DS	184	if (x86_platform_ipi_callback)
4a4de9c7 DS	185	sum += irq_stats(cpu)->x86_platform_ipis;
0428e01a	186	#endif
6b39ba77 TG	187	#ifdef CONFIG_SMP
	188	sum += irq_stats(cpu)->irq_resched_count;
	189	sum += irq_stats(cpu)->irq_call_count;
6b39ba77	190	#endif
0444c9bd	191	#ifdef CONFIG_X86_THERMAL_VECTOR
6b39ba77	192	sum += irq_stats(cpu)->irq_thermal_count;
0444c9bd JB	193	#endif
0444c9bd JB	194	#ifdef CONFIG_X86_MCE_THRESHOLD
6b39ba77	195	sum += irq_stats(cpu)->irq_threshold_count;
8051dbd2	196	#endif
c1ebf835	197	#ifdef CONFIG_X86_MCE
8051dbd2 HS	198	sum += per_cpu(mce_exception_count, cpu);
8051dbd2 HS	199	sum += per_cpu(mce_poll_count, cpu);
6b39ba77 TG	200	#endif
	201	return sum;
	202	}
	203
	204	u64 arch_irq_stat(void)
	205	{
	206	u64 sum = atomic_read(&irq_err_count);
6b39ba77 TG	207	return sum;
6b39ba77 TG	208	}
c3d80000	209
7c1d7cdc JF	210
	211	/*
	212	* do_IRQ handles all normal device IRQ's (the special
	213	* SMP cross-CPU interrupts have their own specific
	214	* handlers).
	215	*/
1d9090e2	216	__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
7c1d7cdc JF	217	{
7c1d7cdc JF	218	struct pt_regs *old_regs = set_irq_regs(regs);
a782a7e4	219	struct irq_desc * desc;
7c1d7cdc JF	220	/* high bit used in ret_from_ code */
7c1d7cdc JF	221	unsigned vector = ~regs->orig_ax;
7c1d7cdc	222
6af7faf6	223	entering_irq();
7c1d7cdc	224
0333a209	225	/* entering_irq() tells RCU that we're not quiescent. Check it. */
5778077d	226	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
0333a209	227
a782a7e4	228	desc = __this_cpu_read(vector_irq[vector]);
7c1d7cdc	229
a782a7e4	230	if (!handle_irq(desc, regs)) {
08306ce6	231	ack_APIC_irq();
7c1d7cdc	232
3c8b2e14	233	if (desc != VECTOR_RETRIGGERED && desc != VECTOR_SHUTDOWN) {
a782a7e4	234	pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
9345005f	235	__func__, smp_processor_id(),
a782a7e4	236	vector);
9345005f	237	} else {
7276c6a2	238	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
9345005f	239	}
7c1d7cdc JF	240	}
7c1d7cdc JF	241
6af7faf6	242	exiting_irq();
7c1d7cdc JF	243
	244	set_irq_regs(old_regs);
	245	return 1;
	246	}
	247
0428e01a TG	248	#ifdef CONFIG_X86_LOCAL_APIC
	249	/* Function pointer for generic interrupt vector handling */
	250	void (*x86_platform_ipi_callback)(void) = NULL;
acaabe79	251	/*
4a4de9c7	252	* Handler for X86_PLATFORM_IPI_VECTOR.
acaabe79	253	*/
c4158ff5	254	__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
eddc0e92 SA	255	{
eddc0e92 SA	256	struct pt_regs *old_regs = set_irq_regs(regs);
acaabe79	257
eddc0e92	258	entering_ack_irq();
8a17116b TG	259	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	260	inc_irq_stat(x86_platform_ipis);
	261	if (x86_platform_ipi_callback)
	262	x86_platform_ipi_callback();
	263	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
eddc0e92	264	exiting_irq();
acaabe79 DS	265	set_irq_regs(old_regs);
acaabe79 DS	266	}
0428e01a	267	#endif
acaabe79	268
d78f2664	269	#ifdef CONFIG_HAVE_KVM
f6b3c72c FW	270	static void dummy_handler(void) {}
	271	static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
	272
	273	void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
	274	{
	275	if (handler)
	276	kvm_posted_intr_wakeup_handler = handler;
	277	else
	278	kvm_posted_intr_wakeup_handler = dummy_handler;
	279	}
	280	EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
	281
d78f2664 YZ	282	/*
	283	* Handler for POSTED_INTERRUPT_VECTOR.
	284	*/
1d9090e2	285	__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
d78f2664 YZ	286	{
	287	struct pt_regs *old_regs = set_irq_regs(regs);
	288
6af7faf6	289	entering_ack_irq();
d78f2664	290	inc_irq_stat(kvm_posted_intr_ipis);
f6b3c72c FW	291	exiting_irq();
	292	set_irq_regs(old_regs);
	293	}
	294
	295	/*
	296	* Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
	297	*/
	298	__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
	299	{
	300	struct pt_regs *old_regs = set_irq_regs(regs);
	301
	302	entering_ack_irq();
	303	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	304	kvm_posted_intr_wakeup_handler();
6af7faf6	305	exiting_irq();
d78f2664 YZ	306	set_irq_regs(old_regs);
d78f2664 YZ	307	}
210f84b0 WV	308
	309	/*
	310	* Handler for POSTED_INTERRUPT_NESTED_VECTOR.
	311	*/
	312	__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
	313	{
	314	struct pt_regs *old_regs = set_irq_regs(regs);
	315
	316	entering_ack_irq();
	317	inc_irq_stat(kvm_posted_intr_nested_ipis);
	318	exiting_irq();
	319	set_irq_regs(old_regs);
	320	}
d78f2664 YZ	321	#endif
d78f2664 YZ	322
cf910e83	323
7a7732bc SS	324	#ifdef CONFIG_HOTPLUG_CPU
	325	/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
	326	void fixup_irqs(void)
	327	{
ad7a929f	328	unsigned int irr, vector;
7a7732bc	329	struct irq_desc *desc;
a3c08e5d	330	struct irq_data *data;
51c43ac6	331	struct irq_chip *chip;
7a7732bc	332
ad7a929f	333	irq_migrate_all_off_this_cpu();
7a7732bc	334
5231a686 SS	335	/*
	336	* We can remove mdelay() and then send spuriuous interrupts to
	337	* new cpu targets for all the irqs that were handled previously by
	338	* this cpu. While it works, I have seen spurious interrupt messages
	339	* (nothing wrong but still...).
	340	*
	341	* So for now, retain mdelay(1) and check the IRR and then send those
	342	* interrupts to new targets as this cpu is already offlined...
	343	*/
7a7732bc	344	mdelay(1);
5231a686	345
09cf92b7 TG	346	/*
	347	* We can walk the vector array of this cpu without holding
	348	* vector_lock because the cpu is already marked !online, so
	349	* nothing else will touch it.
	350	*/
5231a686	351	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
a782a7e4	352	if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
5231a686 SS	353	continue;
	354
	355	irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
	356	if (irr & (1 << (vector % 32))) {
a782a7e4	357	desc = __this_cpu_read(vector_irq[vector]);
5231a686	358
09cf92b7	359	raw_spin_lock(&desc->lock);
51c43ac6 TG	360	data = irq_desc_get_irq_data(desc);
51c43ac6 TG	361	chip = irq_data_get_irq_chip(data);
9345005f	362	if (chip->irq_retrigger) {
51c43ac6	363	chip->irq_retrigger(data);
9345005f PB	364	__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
9345005f PB	365	}
239007b8	366	raw_spin_unlock(&desc->lock);
5231a686	367	}
9345005f	368	if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
7276c6a2	369	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
5231a686	370	}
7a7732bc SS	371	}
7a7732bc SS	372	#endif