[mirror_ubuntu-focal-kernel.git] / arch / x86 / kernel / irq.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/irq.h>

#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>
#include <asm/desc.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>

DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
EXPORT_PER_CPU_SYMBOL(irq_stat);

DEFINE_PER_CPU(struct pt_regs *, irq_regs);
EXPORT_PER_CPU_SYMBOL(irq_regs);

atomic_t irq_err_count;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	ack_APIC_irq();
}

#define irq_stats(x)		(&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
	int j;

	seq_printf(p, "%*s: ", prec, "NMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	seq_puts(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
	seq_printf(p, "%*s: ", prec, "LOC");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	seq_puts(p, "  Local timer interrupts\n");

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	seq_puts(p, "  Spurious interrupts\n");
	seq_printf(p, "%*s: ", prec, "PMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
	seq_puts(p, "  Performance monitoring interrupts\n");
	seq_printf(p, "%*s: ", prec, "IWI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
	seq_puts(p, "  IRQ work interrupts\n");
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
	seq_puts(p, "  APIC ICR read retries\n");
	if (x86_platform_ipi_callback) {
		seq_printf(p, "%*s: ", prec, "PLT");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
		seq_puts(p, "  Platform interrupts\n");
	}
#endif
#ifdef CONFIG_SMP
	seq_printf(p, "%*s: ", prec, "RES");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	seq_puts(p, "  Rescheduling interrupts\n");
	seq_printf(p, "%*s: ", prec, "CAL");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
	seq_puts(p, "  Function call interrupts\n");
	seq_printf(p, "%*s: ", prec, "TLB");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	seq_puts(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	seq_printf(p, "%*s: ", prec, "TRM");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	seq_puts(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	seq_puts(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_AMD
	seq_printf(p, "%*s: ", prec, "DFR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	seq_puts(p, "  Deferred Error APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	seq_puts(p, "  Machine check exceptions\n");
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	seq_puts(p, "  Machine check polls\n");
#endif
#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
		seq_printf(p, "%*s: ", prec, "HYP");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->irq_hv_callback_count);
		seq_puts(p, "  Hypervisor callback interrupts\n");
	}
#endif
#if IS_ENABLED(CONFIG_HYPERV)
	if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
		seq_printf(p, "%*s: ", prec, "HRE");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->irq_hv_reenlightenment_count);
		seq_puts(p, "  Hyper-V reenlightenment interrupts\n");
	}
	if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
		seq_printf(p, "%*s: ", prec, "HVS");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				   irq_stats(j)->hyperv_stimer0_count);
		seq_puts(p, "  Hyper-V stimer0 interrupts\n");
	}
#endif
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
#ifdef CONFIG_HAVE_KVM
	seq_printf(p, "%*s: ", prec, "PIN");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	seq_puts(p, "  Posted-interrupt notification event\n");

	seq_printf(p, "%*s: ", prec, "NPI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_nested_ipis);
	seq_puts(p, "  Nested posted-interrupt event\n");

	seq_printf(p, "%*s: ", prec, "PIW");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ",
			   irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	seq_puts(p, "  Posted-interrupt wakeup event\n");
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
	sum += irq_stats(cpu)->irq_spurious_count;
	sum += irq_stats(cpu)->apic_perf_irqs;
	sum += irq_stats(cpu)->apic_irq_work_irqs;
	sum += irq_stats(cpu)->icr_read_retry_count;
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
#endif
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}


/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);
	struct irq_desc * desc;
	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;

	entering_irq();

	/* entering_irq() tells RCU that we're not quiescent.  Check it. */
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");

	desc = __this_cpu_read(vector_irq[vector]);
	if (likely(!IS_ERR_OR_NULL(desc))) {
		if (IS_ENABLED(CONFIG_X86_32))
			handle_irq(desc, regs);
		else
			generic_handle_irq_desc(desc);
	} else {
		ack_APIC_irq();

		if (desc == VECTOR_UNUSED) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
					     __func__, smp_processor_id(),
					     vector);
		} else {
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
		}
	}

	exiting_irq();

	set_irq_regs(old_regs);
	return 1;
}

#ifdef CONFIG_X86_LOCAL_APIC
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	inc_irq_stat(x86_platform_ipis);
	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif

#ifdef CONFIG_HAVE_KVM
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;

void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
{
	if (handler)
		kvm_posted_intr_wakeup_handler = handler;
	else
		kvm_posted_intr_wakeup_handler = dummy_handler;
}
EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);

/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	kvm_posted_intr_wakeup_handler();
	exiting_irq();
	set_irq_regs(old_regs);
}

/*
 * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
 */
__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	inc_irq_stat(kvm_posted_intr_nested_ipis);
	exiting_irq();
	set_irq_regs(old_regs);
}
#endif


#ifdef CONFIG_HOTPLUG_CPU
/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
	unsigned int irr, vector;
	struct irq_desc *desc;
	struct irq_data *data;
	struct irq_chip *chip;

	irq_migrate_all_off_this_cpu();

	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
	mdelay(1);

	/*
	 * We can walk the vector array of this cpu without holding
	 * vector_lock because the cpu is already marked !online, so
	 * nothing else will touch it.
	 */
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
			desc = __this_cpu_read(vector_irq[vector]);

			raw_spin_lock(&desc->lock);
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
			if (chip->irq_retrigger) {
				chip->irq_retrigger(data);
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
			raw_spin_unlock(&desc->lock);
		}
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
			__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
	}
}
#endif
Commit	Line	Data
457c8996	1	// SPDX-License-Identifier: GPL-2.0-only
6b39ba77 TG	2	/*
	3	* Common interrupt code for 32 and 64 bit
	4	*/
	5	#include <linux/cpu.h>
	6	#include <linux/interrupt.h>
	7	#include <linux/kernel_stat.h>
4722d194	8	#include <linux/of.h>
6b39ba77	9	#include <linux/seq_file.h>
6a02e710	10	#include <linux/smp.h>
7c1d7cdc	11	#include <linux/ftrace.h>
ca444564	12	#include <linux/delay.h>
69c60c88	13	#include <linux/export.h>
447ae316	14	#include <linux/irq.h>
6b39ba77	15
7b6aa335	16	#include <asm/apic.h>
6b39ba77	17	#include <asm/io_apic.h>
c3d80000	18	#include <asm/irq.h>
01ca79f1	19	#include <asm/mce.h>
2c1b284e	20	#include <asm/hw_irq.h>
ac2a5539	21	#include <asm/desc.h>
83ab8514 SRRH	22
83ab8514 SRRH	23	#define CREATE_TRACE_POINTS
cf910e83	24	#include <asm/trace/irq_vectors.h>
6b39ba77	25
c5bde906 BG	26	DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
	27	EXPORT_PER_CPU_SYMBOL(irq_stat);
	28
	29	DEFINE_PER_CPU(struct pt_regs *, irq_regs);
	30	EXPORT_PER_CPU_SYMBOL(irq_regs);
	31
6b39ba77 TG	32	atomic_t irq_err_count;
6b39ba77 TG	33
249f6d9e TG	34	/*
	35	* 'what should we do if we get a hw irq event on an illegal vector'.
	36	* each architecture has to answer this themselves.
	37	*/
	38	void ack_bad_irq(unsigned int irq)
	39	{
edea7148 CG	40	if (printk_ratelimit())
edea7148 CG	41	pr_err("unexpected IRQ trap at vector %02x\n", irq);
249f6d9e	42
249f6d9e TG	43	/*
	44	* Currently unexpected vectors happen only on SMP and APIC.
	45	* We _must_ ack these because every local APIC has only N
	46	* irq slots per priority level, and a 'hanging, unacked' IRQ
	47	* holds up an irq slot - in excessive cases (when multiple
	48	* unexpected vectors occur) that might lock up the APIC
	49	* completely.
	50	* But only ack when the APIC is enabled -AK
	51	*/
08306ce6	52	ack_APIC_irq();
249f6d9e TG	53	}
249f6d9e TG	54
1b437c8c	55	#define irq_stats(x) (&per_cpu(irq_stat, x))
6b39ba77	56	/*
517e4981	57	* /proc/interrupts printing for arch specific interrupts
6b39ba77	58	*/
517e4981	59	int arch_show_interrupts(struct seq_file *p, int prec)
6b39ba77 TG	60	{
	61	int j;
	62
7a81d9a7	63	seq_printf(p, "%*s: ", prec, "NMI");
6b39ba77 TG	64	for_each_online_cpu(j)
6b39ba77 TG	65	seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
3736708f	66	seq_puts(p, " Non-maskable interrupts\n");
6b39ba77	67	#ifdef CONFIG_X86_LOCAL_APIC
7a81d9a7	68	seq_printf(p, "%*s: ", prec, "LOC");
6b39ba77 TG	69	for_each_online_cpu(j)
6b39ba77 TG	70	seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
3736708f	71	seq_puts(p, " Local timer interrupts\n");
474e56b8 JSR	72
	73	seq_printf(p, "%*s: ", prec, "SPU");
	74	for_each_online_cpu(j)
	75	seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
3736708f	76	seq_puts(p, " Spurious interrupts\n");
89ccf465	77	seq_printf(p, "%*s: ", prec, "PMI");
241771ef IM	78	for_each_online_cpu(j)
241771ef IM	79	seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
3736708f	80	seq_puts(p, " Performance monitoring interrupts\n");
e360adbe	81	seq_printf(p, "%*s: ", prec, "IWI");
b6276f35	82	for_each_online_cpu(j)
e360adbe	83	seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
3736708f	84	seq_puts(p, " IRQ work interrupts\n");
346b46be FLVC	85	seq_printf(p, "%*s: ", prec, "RTR");
346b46be FLVC	86	for_each_online_cpu(j)
b49d7d87	87	seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
3736708f	88	seq_puts(p, " APIC ICR read retries\n");
4a4de9c7	89	if (x86_platform_ipi_callback) {
59d13812	90	seq_printf(p, "%*s: ", prec, "PLT");
acaabe79	91	for_each_online_cpu(j)
4a4de9c7	92	seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
3736708f	93	seq_puts(p, " Platform interrupts\n");
acaabe79	94	}
0428e01a	95	#endif
6b39ba77	96	#ifdef CONFIG_SMP
7a81d9a7	97	seq_printf(p, "%*s: ", prec, "RES");
6b39ba77 TG	98	for_each_online_cpu(j)
6b39ba77 TG	99	seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
3736708f	100	seq_puts(p, " Rescheduling interrupts\n");
7a81d9a7	101	seq_printf(p, "%*s: ", prec, "CAL");
6b39ba77	102	for_each_online_cpu(j)
82ba4fac	103	seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
3736708f	104	seq_puts(p, " Function call interrupts\n");
7a81d9a7	105	seq_printf(p, "%*s: ", prec, "TLB");
6b39ba77 TG	106	for_each_online_cpu(j)
6b39ba77 TG	107	seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
3736708f	108	seq_puts(p, " TLB shootdowns\n");
6b39ba77	109	#endif
0444c9bd	110	#ifdef CONFIG_X86_THERMAL_VECTOR
7a81d9a7	111	seq_printf(p, "%*s: ", prec, "TRM");
6b39ba77 TG	112	for_each_online_cpu(j)
6b39ba77 TG	113	seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
3736708f	114	seq_puts(p, " Thermal event interrupts\n");
0444c9bd JB	115	#endif
0444c9bd JB	116	#ifdef CONFIG_X86_MCE_THRESHOLD
7a81d9a7	117	seq_printf(p, "%*s: ", prec, "THR");
6b39ba77 TG	118	for_each_online_cpu(j)
6b39ba77 TG	119	seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
3736708f	120	seq_puts(p, " Threshold APIC interrupts\n");
01ca79f1	121	#endif
24fd78a8 AG	122	#ifdef CONFIG_X86_MCE_AMD
	123	seq_printf(p, "%*s: ", prec, "DFR");
	124	for_each_online_cpu(j)
	125	seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
	126	seq_puts(p, " Deferred Error APIC interrupts\n");
	127	#endif
c1ebf835	128	#ifdef CONFIG_X86_MCE
01ca79f1 AK	129	seq_printf(p, "%*s: ", prec, "MCE");
	130	for_each_online_cpu(j)
	131	seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
3736708f	132	seq_puts(p, " Machine check exceptions\n");
ca84f696 AK	133	seq_printf(p, "%*s: ", prec, "MCP");
	134	for_each_online_cpu(j)
	135	seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
3736708f	136	seq_puts(p, " Machine check polls\n");
6b39ba77	137	#endif
ecca2502	138	#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
7854f822	139	if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
9d87cd61 VK	140	seq_printf(p, "%*s: ", prec, "HYP");
	141	for_each_online_cpu(j)
	142	seq_printf(p, "%10u ",
	143	irq_stats(j)->irq_hv_callback_count);
	144	seq_puts(p, " Hypervisor callback interrupts\n");
	145	}
51d4e5da VK	146	#endif
	147	#if IS_ENABLED(CONFIG_HYPERV)
	148	if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
	149	seq_printf(p, "%*s: ", prec, "HRE");
	150	for_each_online_cpu(j)
	151	seq_printf(p, "%10u ",
	152	irq_stats(j)->irq_hv_reenlightenment_count);
	153	seq_puts(p, " Hyper-V reenlightenment interrupts\n");
	154	}
248e742a MK	155	if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
	156	seq_printf(p, "%*s: ", prec, "HVS");
	157	for_each_online_cpu(j)
	158	seq_printf(p, "%10u ",
	159	irq_stats(j)->hyperv_stimer0_count);
	160	seq_puts(p, " Hyper-V stimer0 interrupts\n");
	161	}
929320e4	162	#endif
7a81d9a7	163	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
6b39ba77	164	#if defined(CONFIG_X86_IO_APIC)
7a81d9a7	165	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
501b3265 FW	166	#endif
	167	#ifdef CONFIG_HAVE_KVM
	168	seq_printf(p, "%*s: ", prec, "PIN");
	169	for_each_online_cpu(j)
	170	seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
	171	seq_puts(p, " Posted-interrupt notification event\n");
	172
210f84b0 WV	173	seq_printf(p, "%*s: ", prec, "NPI");
	174	for_each_online_cpu(j)
	175	seq_printf(p, "%10u ",
	176	irq_stats(j)->kvm_posted_intr_nested_ipis);
	177	seq_puts(p, " Nested posted-interrupt event\n");
	178
501b3265 FW	179	seq_printf(p, "%*s: ", prec, "PIW");
	180	for_each_online_cpu(j)
	181	seq_printf(p, "%10u ",
	182	irq_stats(j)->kvm_posted_intr_wakeup_ipis);
	183	seq_puts(p, " Posted-interrupt wakeup event\n");
6b39ba77 TG	184	#endif
	185	return 0;
	186	}
	187
6b39ba77 TG	188	/*
	189	* /proc/stat helpers
	190	*/
	191	u64 arch_irq_stat_cpu(unsigned int cpu)
	192	{
	193	u64 sum = irq_stats(cpu)->__nmi_count;
	194
	195	#ifdef CONFIG_X86_LOCAL_APIC
	196	sum += irq_stats(cpu)->apic_timer_irqs;
474e56b8	197	sum += irq_stats(cpu)->irq_spurious_count;
241771ef	198	sum += irq_stats(cpu)->apic_perf_irqs;
e360adbe	199	sum += irq_stats(cpu)->apic_irq_work_irqs;
b49d7d87	200	sum += irq_stats(cpu)->icr_read_retry_count;
4a4de9c7 DS	201	if (x86_platform_ipi_callback)
4a4de9c7 DS	202	sum += irq_stats(cpu)->x86_platform_ipis;
0428e01a	203	#endif
6b39ba77 TG	204	#ifdef CONFIG_SMP
	205	sum += irq_stats(cpu)->irq_resched_count;
	206	sum += irq_stats(cpu)->irq_call_count;
6b39ba77	207	#endif
0444c9bd	208	#ifdef CONFIG_X86_THERMAL_VECTOR
6b39ba77	209	sum += irq_stats(cpu)->irq_thermal_count;
0444c9bd JB	210	#endif
0444c9bd JB	211	#ifdef CONFIG_X86_MCE_THRESHOLD
6b39ba77	212	sum += irq_stats(cpu)->irq_threshold_count;
8051dbd2	213	#endif
c1ebf835	214	#ifdef CONFIG_X86_MCE
8051dbd2 HS	215	sum += per_cpu(mce_exception_count, cpu);
8051dbd2 HS	216	sum += per_cpu(mce_poll_count, cpu);
6b39ba77 TG	217	#endif
	218	return sum;
	219	}
	220
	221	u64 arch_irq_stat(void)
	222	{
	223	u64 sum = atomic_read(&irq_err_count);
6b39ba77 TG	224	return sum;
6b39ba77 TG	225	}
c3d80000	226
7c1d7cdc JF	227
	228	/*
	229	* do_IRQ handles all normal device IRQ's (the special
	230	* SMP cross-CPU interrupts have their own specific
	231	* handlers).
	232	*/
1d9090e2	233	__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
7c1d7cdc JF	234	{
7c1d7cdc JF	235	struct pt_regs *old_regs = set_irq_regs(regs);
a782a7e4	236	struct irq_desc * desc;
7c1d7cdc JF	237	/* high bit used in ret_from_ code */
7c1d7cdc JF	238	unsigned vector = ~regs->orig_ax;
7c1d7cdc	239
6af7faf6	240	entering_irq();
7c1d7cdc	241
0333a209	242	/* entering_irq() tells RCU that we're not quiescent. Check it. */
5778077d	243	RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
0333a209	244
a782a7e4	245	desc = __this_cpu_read(vector_irq[vector]);
d6f83427 HK	246	if (likely(!IS_ERR_OR_NULL(desc))) {
	247	if (IS_ENABLED(CONFIG_X86_32))
	248	handle_irq(desc, regs);
	249	else
	250	generic_handle_irq_desc(desc);
	251	} else {
08306ce6	252	ack_APIC_irq();
7c1d7cdc	253
8725fcd9	254	if (desc == VECTOR_UNUSED) {
a782a7e4	255	pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
9345005f	256	__func__, smp_processor_id(),
a782a7e4	257	vector);
9345005f	258	} else {
7276c6a2	259	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
9345005f	260	}
7c1d7cdc JF	261	}
7c1d7cdc JF	262
6af7faf6	263	exiting_irq();
7c1d7cdc JF	264
	265	set_irq_regs(old_regs);
	266	return 1;
	267	}
	268
0428e01a TG	269	#ifdef CONFIG_X86_LOCAL_APIC
	270	/* Function pointer for generic interrupt vector handling */
	271	void (*x86_platform_ipi_callback)(void) = NULL;
acaabe79	272	/*
4a4de9c7	273	* Handler for X86_PLATFORM_IPI_VECTOR.
acaabe79	274	*/
c4158ff5	275	__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs)
eddc0e92 SA	276	{
eddc0e92 SA	277	struct pt_regs *old_regs = set_irq_regs(regs);
acaabe79	278
eddc0e92	279	entering_ack_irq();
8a17116b TG	280	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	281	inc_irq_stat(x86_platform_ipis);
	282	if (x86_platform_ipi_callback)
	283	x86_platform_ipi_callback();
	284	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
eddc0e92	285	exiting_irq();
acaabe79 DS	286	set_irq_regs(old_regs);
acaabe79 DS	287	}
0428e01a	288	#endif
acaabe79	289
d78f2664	290	#ifdef CONFIG_HAVE_KVM
f6b3c72c FW	291	static void dummy_handler(void) {}
	292	static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
	293
	294	void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
	295	{
	296	if (handler)
	297	kvm_posted_intr_wakeup_handler = handler;
	298	else
	299	kvm_posted_intr_wakeup_handler = dummy_handler;
	300	}
	301	EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
	302
d78f2664 YZ	303	/*
	304	* Handler for POSTED_INTERRUPT_VECTOR.
	305	*/
1d9090e2	306	__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
d78f2664 YZ	307	{
	308	struct pt_regs *old_regs = set_irq_regs(regs);
	309
6af7faf6	310	entering_ack_irq();
d78f2664	311	inc_irq_stat(kvm_posted_intr_ipis);
f6b3c72c FW	312	exiting_irq();
	313	set_irq_regs(old_regs);
	314	}
	315
	316	/*
	317	* Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
	318	*/
	319	__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
	320	{
	321	struct pt_regs *old_regs = set_irq_regs(regs);
	322
	323	entering_ack_irq();
	324	inc_irq_stat(kvm_posted_intr_wakeup_ipis);
	325	kvm_posted_intr_wakeup_handler();
6af7faf6	326	exiting_irq();
d78f2664 YZ	327	set_irq_regs(old_regs);
d78f2664 YZ	328	}
210f84b0 WV	329
	330	/*
	331	* Handler for POSTED_INTERRUPT_NESTED_VECTOR.
	332	*/
	333	__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
	334	{
	335	struct pt_regs *old_regs = set_irq_regs(regs);
	336
	337	entering_ack_irq();
	338	inc_irq_stat(kvm_posted_intr_nested_ipis);
	339	exiting_irq();
	340	set_irq_regs(old_regs);
	341	}
d78f2664 YZ	342	#endif
d78f2664 YZ	343
cf910e83	344
7a7732bc SS	345	#ifdef CONFIG_HOTPLUG_CPU
	346	/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
	347	void fixup_irqs(void)
	348	{
ad7a929f	349	unsigned int irr, vector;
7a7732bc	350	struct irq_desc *desc;
a3c08e5d	351	struct irq_data *data;
51c43ac6	352	struct irq_chip *chip;
7a7732bc	353
ad7a929f	354	irq_migrate_all_off_this_cpu();
7a7732bc	355
5231a686 SS	356	/*
	357	* We can remove mdelay() and then send spuriuous interrupts to
	358	* new cpu targets for all the irqs that were handled previously by
	359	* this cpu. While it works, I have seen spurious interrupt messages
	360	* (nothing wrong but still...).
	361	*
	362	* So for now, retain mdelay(1) and check the IRR and then send those
	363	* interrupts to new targets as this cpu is already offlined...
	364	*/
7a7732bc	365	mdelay(1);
5231a686	366
09cf92b7 TG	367	/*
	368	* We can walk the vector array of this cpu without holding
	369	* vector_lock because the cpu is already marked !online, so
	370	* nothing else will touch it.
	371	*/
5231a686	372	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
a782a7e4	373	if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
5231a686 SS	374	continue;
	375
	376	irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
	377	if (irr & (1 << (vector % 32))) {
a782a7e4	378	desc = __this_cpu_read(vector_irq[vector]);
5231a686	379
09cf92b7	380	raw_spin_lock(&desc->lock);
51c43ac6 TG	381	data = irq_desc_get_irq_data(desc);
51c43ac6 TG	382	chip = irq_data_get_irq_chip(data);
9345005f	383	if (chip->irq_retrigger) {
51c43ac6	384	chip->irq_retrigger(data);
9345005f PB	385	__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
9345005f PB	386	}
239007b8	387	raw_spin_unlock(&desc->lock);
5231a686	388	}
9345005f	389	if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
7276c6a2	390	__this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
5231a686	391	}
7a7732bc SS	392	}
7a7732bc SS	393	#endif