[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / irq.c

/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>

#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/idle.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>
#include <asm/desc.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>

atomic_t irq_err_count;

/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
	if (printk_ratelimit())
		pr_err("unexpected IRQ trap at vector %02x\n", irq);

	/*
	 * Currently unexpected vectors happen only on SMP and APIC.
	 * We _must_ ack these because every local APIC has only N
	 * irq slots per priority level, and a 'hanging, unacked' IRQ
	 * holds up an irq slot - in excessive cases (when multiple
	 * unexpected vectors occur) that might lock up the APIC
	 * completely.
	 * But only ack when the APIC is enabled -AK
	 */
	ack_APIC_irq();
}

#define irq_stats(x)		(&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
	int j;

	seq_printf(p, "%*s: ", prec, "NMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
	seq_puts(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
	seq_printf(p, "%*s: ", prec, "LOC");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
	seq_puts(p, "  Local timer interrupts\n");

	seq_printf(p, "%*s: ", prec, "SPU");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
	seq_puts(p, "  Spurious interrupts\n");
	seq_printf(p, "%*s: ", prec, "PMI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
	seq_puts(p, "  Performance monitoring interrupts\n");
	seq_printf(p, "%*s: ", prec, "IWI");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
	seq_puts(p, "  IRQ work interrupts\n");
	seq_printf(p, "%*s: ", prec, "RTR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
	seq_puts(p, "  APIC ICR read retries\n");
#endif
	if (x86_platform_ipi_callback) {
		seq_printf(p, "%*s: ", prec, "PLT");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
		seq_puts(p, "  Platform interrupts\n");
	}
#ifdef CONFIG_SMP
	seq_printf(p, "%*s: ", prec, "RES");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
	seq_puts(p, "  Rescheduling interrupts\n");
	seq_printf(p, "%*s: ", prec, "CAL");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
					irq_stats(j)->irq_tlb_count);
	seq_puts(p, "  Function call interrupts\n");
	seq_printf(p, "%*s: ", prec, "TLB");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
	seq_puts(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	seq_printf(p, "%*s: ", prec, "TRM");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
	seq_puts(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	seq_printf(p, "%*s: ", prec, "THR");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
	seq_puts(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
	seq_printf(p, "%*s: ", prec, "MCE");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
	seq_puts(p, "  Machine check exceptions\n");
	seq_printf(p, "%*s: ", prec, "MCP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
	seq_puts(p, "  Machine check polls\n");
#endif
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
	seq_printf(p, "%*s: ", prec, "HYP");
	for_each_online_cpu(j)
		seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
	seq_puts(p, "  Hypervisor callback interrupts\n");
#endif
	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
	return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
	u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
	sum += irq_stats(cpu)->apic_timer_irqs;
	sum += irq_stats(cpu)->irq_spurious_count;
	sum += irq_stats(cpu)->apic_perf_irqs;
	sum += irq_stats(cpu)->apic_irq_work_irqs;
	sum += irq_stats(cpu)->icr_read_retry_count;
#endif
	if (x86_platform_ipi_callback)
		sum += irq_stats(cpu)->x86_platform_ipis;
#ifdef CONFIG_SMP
	sum += irq_stats(cpu)->irq_resched_count;
	sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
	sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
	sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_MCE
	sum += per_cpu(mce_exception_count, cpu);
	sum += per_cpu(mce_poll_count, cpu);
#endif
	return sum;
}

u64 arch_irq_stat(void)
{
	u64 sum = atomic_read(&irq_err_count);
	return sum;
}


/*
 * do_IRQ handles all normal device IRQ's (the special
 * SMP cross-CPU interrupts have their own specific
 * handlers).
 */
__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	/* high bit used in ret_from_ code  */
	unsigned vector = ~regs->orig_ax;
	unsigned irq;

	irq_enter();
	exit_idle();

	irq = __this_cpu_read(vector_irq[vector]);

	if (!handle_irq(irq, regs)) {
		ack_APIC_irq();

		if (irq != VECTOR_RETRIGGERED) {
			pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n",
					     __func__, smp_processor_id(),
					     vector, irq);
		} else {
			__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
		}
	}

	irq_exit();

	set_irq_regs(old_regs);
	return 1;
}

/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
void __smp_x86_platform_ipi(void)
{
	inc_irq_stat(x86_platform_ipis);

	if (x86_platform_ipi_callback)
		x86_platform_ipi_callback();
}

__visible void smp_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	__smp_x86_platform_ipi();
	exiting_irq();
	set_irq_regs(old_regs);
}

#ifdef CONFIG_HAVE_KVM
/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	ack_APIC_irq();

	irq_enter();

	exit_idle();

	inc_irq_stat(kvm_posted_intr_ipis);

	irq_exit();

	set_irq_regs(old_regs);
}
#endif

__visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
{
	struct pt_regs *old_regs = set_irq_regs(regs);

	entering_ack_irq();
	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	__smp_x86_platform_ipi();
	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	exiting_irq();
	set_irq_regs(old_regs);
}

EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);

#ifdef CONFIG_HOTPLUG_CPU

/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
 * below, which is protected by stop_machine().  Putting them on the stack
 * results in a stack frame overflow.  Dynamically allocating could result in a
 * failure so declare these two cpumasks as global.
 */
static struct cpumask affinity_new, online_new;

/*
 * This cpu is going to be removed and its vectors migrated to the remaining
 * online cpus.  Check to see if there are enough vectors in the remaining cpus.
 * This function is protected by stop_machine().
 */
int check_irq_vectors_for_cpu_disable(void)
{
	int irq, cpu;
	unsigned int this_cpu, vector, this_count, count;
	struct irq_desc *desc;
	struct irq_data *data;

	this_cpu = smp_processor_id();
	cpumask_copy(&online_new, cpu_online_mask);
	cpu_clear(this_cpu, online_new);

	this_count = 0;
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		irq = __this_cpu_read(vector_irq[vector]);
		if (irq >= 0) {
			desc = irq_to_desc(irq);
			if (!desc)
				continue;

			data = irq_desc_get_irq_data(desc);
			cpumask_copy(&affinity_new, data->affinity);
			cpu_clear(this_cpu, affinity_new);

			/* Do not count inactive or per-cpu irqs. */
			if (!irq_has_action(irq) || irqd_is_per_cpu(data))
				continue;

			/*
			 * A single irq may be mapped to multiple
			 * cpu's vector_irq[] (for example IOAPIC cluster
			 * mode).  In this case we have two
			 * possibilities:
			 *
			 * 1) the resulting affinity mask is empty; that is
			 * this the down'd cpu is the last cpu in the irq's
			 * affinity mask, or
			 *
			 * 2) the resulting affinity mask is no longer
			 * a subset of the online cpus but the affinity
			 * mask is not zero; that is the down'd cpu is the
			 * last online cpu in a user set affinity mask.
			 */
			if (cpumask_empty(&affinity_new) ||
			    !cpumask_subset(&affinity_new, &online_new))
				this_count++;
		}
	}

	count = 0;
	for_each_online_cpu(cpu) {
		if (cpu == this_cpu)
			continue;
		/*
		 * We scan from FIRST_EXTERNAL_VECTOR to first system
		 * vector. If the vector is marked in the used vectors
		 * bitmap or an irq is assigned to it, we don't count
		 * it as available.
		 */
		for (vector = FIRST_EXTERNAL_VECTOR;
		     vector < first_system_vector; vector++) {
			if (!test_bit(vector, used_vectors) &&
			    per_cpu(vector_irq, cpu)[vector] < 0)
					count++;
		}
	}

	if (count < this_count) {
		pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
			this_cpu, this_count, count);
		return -ERANGE;
	}
	return 0;
}

/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
	unsigned int irq, vector;
	static int warned;
	struct irq_desc *desc;
	struct irq_data *data;
	struct irq_chip *chip;
	int ret;

	for_each_irq_desc(irq, desc) {
		int break_affinity = 0;
		int set_affinity = 1;
		const struct cpumask *affinity;

		if (!desc)
			continue;
		if (irq == 2)
			continue;

		/* interrupt's are disabled at this point */
		raw_spin_lock(&desc->lock);

		data = irq_desc_get_irq_data(desc);
		affinity = data->affinity;
		if (!irq_has_action(irq) || irqd_is_per_cpu(data) ||
		    cpumask_subset(affinity, cpu_online_mask)) {
			raw_spin_unlock(&desc->lock);
			continue;
		}

		/*
		 * Complete the irq move. This cpu is going down and for
		 * non intr-remapping case, we can't wait till this interrupt
		 * arrives at this cpu before completing the irq move.
		 */
		irq_force_complete_move(irq);

		if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
			break_affinity = 1;
			affinity = cpu_online_mask;
		}

		chip = irq_data_get_irq_chip(data);
		if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
			chip->irq_mask(data);

		if (chip->irq_set_affinity) {
			ret = chip->irq_set_affinity(data, affinity, true);
			if (ret == -ENOSPC)
				pr_crit("IRQ %d set affinity failed because there are no available vectors.  The device assigned to this IRQ is unstable.\n", irq);
		} else {
			if (!(warned++))
				set_affinity = 0;
		}

		/*
		 * We unmask if the irq was not marked masked by the
		 * core code. That respects the lazy irq disable
		 * behaviour.
		 */
		if (!irqd_can_move_in_process_context(data) &&
		    !irqd_irq_masked(data) && chip->irq_unmask)
			chip->irq_unmask(data);

		raw_spin_unlock(&desc->lock);

		if (break_affinity && set_affinity)
			pr_notice("Broke affinity for irq %i\n", irq);
		else if (!set_affinity)
			pr_notice("Cannot set affinity for irq %i\n", irq);
	}

	/*
	 * We can remove mdelay() and then send spuriuous interrupts to
	 * new cpu targets for all the irqs that were handled previously by
	 * this cpu. While it works, I have seen spurious interrupt messages
	 * (nothing wrong but still...).
	 *
	 * So for now, retain mdelay(1) and check the IRR and then send those
	 * interrupts to new targets as this cpu is already offlined...
	 */
	mdelay(1);

	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irr;

		if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNDEFINED)
			continue;

		irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
		if (irr  & (1 << (vector % 32))) {
			irq = __this_cpu_read(vector_irq[vector]);

			desc = irq_to_desc(irq);
			data = irq_desc_get_irq_data(desc);
			chip = irq_data_get_irq_chip(data);
			raw_spin_lock(&desc->lock);
			if (chip->irq_retrigger) {
				chip->irq_retrigger(data);
				__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
			}
			raw_spin_unlock(&desc->lock);
		}
		if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
			__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
	}
}
#endif
Commit	Line	Data
6b39ba77 TG	1	/*
	2	* Common interrupt code for 32 and 64 bit
	3	*/
	4	#include <linux/cpu.h>
	5	#include <linux/interrupt.h>
	6	#include <linux/kernel_stat.h>
4722d194	7	#include <linux/of.h>
6b39ba77	8	#include <linux/seq_file.h>
6a02e710	9	#include <linux/smp.h>
7c1d7cdc	10	#include <linux/ftrace.h>
ca444564	11	#include <linux/delay.h>
69c60c88	12	#include <linux/export.h>
6b39ba77	13
7b6aa335	14	#include <asm/apic.h>
6b39ba77	15	#include <asm/io_apic.h>
c3d80000	16	#include <asm/irq.h>
7c1d7cdc	17	#include <asm/idle.h>
01ca79f1	18	#include <asm/mce.h>
2c1b284e	19	#include <asm/hw_irq.h>
ac2a5539	20	#include <asm/desc.h>
83ab8514 SRRH	21
83ab8514 SRRH	22	#define CREATE_TRACE_POINTS
cf910e83	23	#include <asm/trace/irq_vectors.h>
6b39ba77 TG	24
	25	atomic_t irq_err_count;
	26
acaabe79	27	/* Function pointer for generic interrupt vector handling */
4a4de9c7	28	void (*x86_platform_ipi_callback)(void) = NULL;
acaabe79	29
249f6d9e TG	30	/*
	31	* 'what should we do if we get a hw irq event on an illegal vector'.
	32	* each architecture has to answer this themselves.
	33	*/
	34	void ack_bad_irq(unsigned int irq)
	35	{
edea7148 CG	36	if (printk_ratelimit())
edea7148 CG	37	pr_err("unexpected IRQ trap at vector %02x\n", irq);
249f6d9e	38
249f6d9e TG	39	/*
	40	* Currently unexpected vectors happen only on SMP and APIC.
	41	* We _must_ ack these because every local APIC has only N
	42	* irq slots per priority level, and a 'hanging, unacked' IRQ
	43	* holds up an irq slot - in excessive cases (when multiple
	44	* unexpected vectors occur) that might lock up the APIC
	45	* completely.
	46	* But only ack when the APIC is enabled -AK
	47	*/
08306ce6	48	ack_APIC_irq();
249f6d9e TG	49	}
249f6d9e TG	50
1b437c8c	51	#define irq_stats(x) (&per_cpu(irq_stat, x))
6b39ba77	52	/*
517e4981	53	* /proc/interrupts printing for arch specific interrupts
6b39ba77	54	*/
517e4981	55	int arch_show_interrupts(struct seq_file *p, int prec)
6b39ba77 TG	56	{
	57	int j;
	58
7a81d9a7	59	seq_printf(p, "%*s: ", prec, "NMI");
6b39ba77 TG	60	for_each_online_cpu(j)
6b39ba77 TG	61	seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
3736708f	62	seq_puts(p, " Non-maskable interrupts\n");
6b39ba77	63	#ifdef CONFIG_X86_LOCAL_APIC
7a81d9a7	64	seq_printf(p, "%*s: ", prec, "LOC");
6b39ba77 TG	65	for_each_online_cpu(j)
6b39ba77 TG	66	seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
3736708f	67	seq_puts(p, " Local timer interrupts\n");
474e56b8 JSR	68
	69	seq_printf(p, "%*s: ", prec, "SPU");
	70	for_each_online_cpu(j)
	71	seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
3736708f	72	seq_puts(p, " Spurious interrupts\n");
89ccf465	73	seq_printf(p, "%*s: ", prec, "PMI");
241771ef IM	74	for_each_online_cpu(j)
241771ef IM	75	seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
3736708f	76	seq_puts(p, " Performance monitoring interrupts\n");
e360adbe	77	seq_printf(p, "%*s: ", prec, "IWI");
b6276f35	78	for_each_online_cpu(j)
e360adbe	79	seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
3736708f	80	seq_puts(p, " IRQ work interrupts\n");
346b46be FLVC	81	seq_printf(p, "%*s: ", prec, "RTR");
346b46be FLVC	82	for_each_online_cpu(j)
b49d7d87	83	seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
3736708f	84	seq_puts(p, " APIC ICR read retries\n");
6b39ba77	85	#endif
4a4de9c7	86	if (x86_platform_ipi_callback) {
59d13812	87	seq_printf(p, "%*s: ", prec, "PLT");
acaabe79	88	for_each_online_cpu(j)
4a4de9c7	89	seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
3736708f	90	seq_puts(p, " Platform interrupts\n");
acaabe79	91	}
6b39ba77	92	#ifdef CONFIG_SMP
7a81d9a7	93	seq_printf(p, "%*s: ", prec, "RES");
6b39ba77 TG	94	for_each_online_cpu(j)
6b39ba77 TG	95	seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
3736708f	96	seq_puts(p, " Rescheduling interrupts\n");
7a81d9a7	97	seq_printf(p, "%*s: ", prec, "CAL");
6b39ba77	98	for_each_online_cpu(j)
fd0f5869 TS	99	seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
fd0f5869 TS	100	irq_stats(j)->irq_tlb_count);
3736708f	101	seq_puts(p, " Function call interrupts\n");
7a81d9a7	102	seq_printf(p, "%*s: ", prec, "TLB");
6b39ba77 TG	103	for_each_online_cpu(j)
6b39ba77 TG	104	seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
3736708f	105	seq_puts(p, " TLB shootdowns\n");
6b39ba77	106	#endif
0444c9bd	107	#ifdef CONFIG_X86_THERMAL_VECTOR
7a81d9a7	108	seq_printf(p, "%*s: ", prec, "TRM");
6b39ba77 TG	109	for_each_online_cpu(j)
6b39ba77 TG	110	seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
3736708f	111	seq_puts(p, " Thermal event interrupts\n");
0444c9bd JB	112	#endif
0444c9bd JB	113	#ifdef CONFIG_X86_MCE_THRESHOLD
7a81d9a7	114	seq_printf(p, "%*s: ", prec, "THR");
6b39ba77 TG	115	for_each_online_cpu(j)
6b39ba77 TG	116	seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
3736708f	117	seq_puts(p, " Threshold APIC interrupts\n");
01ca79f1	118	#endif
c1ebf835	119	#ifdef CONFIG_X86_MCE
01ca79f1 AK	120	seq_printf(p, "%*s: ", prec, "MCE");
	121	for_each_online_cpu(j)
	122	seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
3736708f	123	seq_puts(p, " Machine check exceptions\n");
ca84f696 AK	124	seq_printf(p, "%*s: ", prec, "MCP");
	125	for_each_online_cpu(j)
	126	seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
3736708f	127	seq_puts(p, " Machine check polls\n");
6b39ba77	128	#endif
f704a7d7	129	#if IS_ENABLED(CONFIG_HYPERV) \|\| defined(CONFIG_XEN)
4a0d3107	130	seq_printf(p, "%*s: ", prec, "HYP");
929320e4 TG	131	for_each_online_cpu(j)
929320e4 TG	132	seq_printf(p, "%10u ", irq_stats(j)->irq_hv_callback_count);
3736708f	133	seq_puts(p, " Hypervisor callback interrupts\n");
929320e4	134	#endif
7a81d9a7	135	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
6b39ba77	136	#if defined(CONFIG_X86_IO_APIC)
7a81d9a7	137	seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
6b39ba77 TG	138	#endif
	139	return 0;
	140	}
	141
6b39ba77 TG	142	/*
	143	* /proc/stat helpers
	144	*/
	145	u64 arch_irq_stat_cpu(unsigned int cpu)
	146	{
	147	u64 sum = irq_stats(cpu)->__nmi_count;
	148
	149	#ifdef CONFIG_X86_LOCAL_APIC
	150	sum += irq_stats(cpu)->apic_timer_irqs;
474e56b8	151	sum += irq_stats(cpu)->irq_spurious_count;
241771ef	152	sum += irq_stats(cpu)->apic_perf_irqs;
e360adbe	153	sum += irq_stats(cpu)->apic_irq_work_irqs;
b49d7d87	154	sum += irq_stats(cpu)->icr_read_retry_count;
6b39ba77	155	#endif
4a4de9c7 DS	156	if (x86_platform_ipi_callback)
4a4de9c7 DS	157	sum += irq_stats(cpu)->x86_platform_ipis;
6b39ba77 TG	158	#ifdef CONFIG_SMP
	159	sum += irq_stats(cpu)->irq_resched_count;
	160	sum += irq_stats(cpu)->irq_call_count;
6b39ba77	161	#endif
0444c9bd	162	#ifdef CONFIG_X86_THERMAL_VECTOR
6b39ba77	163	sum += irq_stats(cpu)->irq_thermal_count;
0444c9bd JB	164	#endif
0444c9bd JB	165	#ifdef CONFIG_X86_MCE_THRESHOLD
6b39ba77	166	sum += irq_stats(cpu)->irq_threshold_count;
8051dbd2	167	#endif
c1ebf835	168	#ifdef CONFIG_X86_MCE
8051dbd2 HS	169	sum += per_cpu(mce_exception_count, cpu);
8051dbd2 HS	170	sum += per_cpu(mce_poll_count, cpu);
6b39ba77 TG	171	#endif
	172	return sum;
	173	}
	174
	175	u64 arch_irq_stat(void)
	176	{
	177	u64 sum = atomic_read(&irq_err_count);
6b39ba77 TG	178	return sum;
6b39ba77 TG	179	}
c3d80000	180
7c1d7cdc JF	181
	182	/*
	183	* do_IRQ handles all normal device IRQ's (the special
	184	* SMP cross-CPU interrupts have their own specific
	185	* handlers).
	186	*/
1d9090e2	187	__visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
7c1d7cdc JF	188	{
	189	struct pt_regs *old_regs = set_irq_regs(regs);
	190
	191	/* high bit used in ret_from_ code */
	192	unsigned vector = ~regs->orig_ax;
	193	unsigned irq;
	194
7c1d7cdc	195	irq_enter();
98ad1cc1	196	exit_idle();
7c1d7cdc	197
0a3aee0d	198	irq = __this_cpu_read(vector_irq[vector]);
7c1d7cdc JF	199
7c1d7cdc JF	200	if (!handle_irq(irq, regs)) {
08306ce6	201	ack_APIC_irq();
7c1d7cdc	202
9345005f PB	203	if (irq != VECTOR_RETRIGGERED) {
	204	pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n",
	205	__func__, smp_processor_id(),
	206	vector, irq);
	207	} else {
	208	__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
	209	}
7c1d7cdc JF	210	}
	211
	212	irq_exit();
	213
	214	set_irq_regs(old_regs);
	215	return 1;
	216	}
	217
acaabe79	218	/*
4a4de9c7	219	* Handler for X86_PLATFORM_IPI_VECTOR.
acaabe79	220	*/
eddc0e92	221	void __smp_x86_platform_ipi(void)
acaabe79	222	{
4a4de9c7	223	inc_irq_stat(x86_platform_ipis);
acaabe79	224
4a4de9c7 DS	225	if (x86_platform_ipi_callback)
4a4de9c7 DS	226	x86_platform_ipi_callback();
eddc0e92	227	}
acaabe79	228
1d9090e2	229	__visible void smp_x86_platform_ipi(struct pt_regs *regs)
eddc0e92 SA	230	{
eddc0e92 SA	231	struct pt_regs *old_regs = set_irq_regs(regs);
acaabe79	232
eddc0e92 SA	233	entering_ack_irq();
	234	__smp_x86_platform_ipi();
	235	exiting_irq();
acaabe79 DS	236	set_irq_regs(old_regs);
	237	}
	238
d78f2664 YZ	239	#ifdef CONFIG_HAVE_KVM
	240	/*
	241	* Handler for POSTED_INTERRUPT_VECTOR.
	242	*/
1d9090e2	243	__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs)
d78f2664 YZ	244	{
	245	struct pt_regs *old_regs = set_irq_regs(regs);
	246
	247	ack_APIC_irq();
	248
	249	irq_enter();
	250
	251	exit_idle();
	252
	253	inc_irq_stat(kvm_posted_intr_ipis);
	254
	255	irq_exit();
	256
	257	set_irq_regs(old_regs);
	258	}
	259	#endif
	260
1d9090e2	261	__visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
cf910e83 SA	262	{
	263	struct pt_regs *old_regs = set_irq_regs(regs);
	264
	265	entering_ack_irq();
	266	trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
	267	__smp_x86_platform_ipi();
	268	trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
	269	exiting_irq();
	270	set_irq_regs(old_regs);
	271	}
	272
c3d80000	273	EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
7a7732bc SS	274
7a7732bc SS	275	#ifdef CONFIG_HOTPLUG_CPU
39424e89 PB	276
	277	/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
	278	* below, which is protected by stop_machine(). Putting them on the stack
	279	* results in a stack frame overflow. Dynamically allocating could result in a
	280	* failure so declare these two cpumasks as global.
	281	*/
	282	static struct cpumask affinity_new, online_new;
	283
da6139e4 PB	284	/*
	285	* This cpu is going to be removed and its vectors migrated to the remaining
	286	* online cpus. Check to see if there are enough vectors in the remaining cpus.
	287	* This function is protected by stop_machine().
	288	*/
	289	int check_irq_vectors_for_cpu_disable(void)
	290	{
	291	int irq, cpu;
	292	unsigned int this_cpu, vector, this_count, count;
	293	struct irq_desc *desc;
	294	struct irq_data *data;
da6139e4 PB	295
	296	this_cpu = smp_processor_id();
	297	cpumask_copy(&online_new, cpu_online_mask);
	298	cpu_clear(this_cpu, online_new);
	299
	300	this_count = 0;
	301	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
	302	irq = __this_cpu_read(vector_irq[vector]);
	303	if (irq >= 0) {
	304	desc = irq_to_desc(irq);
d97eb896 JR	305	if (!desc)
	306	continue;
	307
da6139e4 PB	308	data = irq_desc_get_irq_data(desc);
	309	cpumask_copy(&affinity_new, data->affinity);
	310	cpu_clear(this_cpu, affinity_new);
	311
	312	/* Do not count inactive or per-cpu irqs. */
	313	if (!irq_has_action(irq) \|\| irqd_is_per_cpu(data))
	314	continue;
	315
	316	/*
	317	* A single irq may be mapped to multiple
	318	* cpu's vector_irq[] (for example IOAPIC cluster
	319	* mode). In this case we have two
	320	* possibilities:
	321	*
	322	* 1) the resulting affinity mask is empty; that is
	323	* this the down'd cpu is the last cpu in the irq's
	324	* affinity mask, or
	325	*
	326	* 2) the resulting affinity mask is no longer
	327	* a subset of the online cpus but the affinity
	328	* mask is not zero; that is the down'd cpu is the
	329	* last online cpu in a user set affinity mask.
	330	*/
	331	if (cpumask_empty(&affinity_new) \|\|
	332	!cpumask_subset(&affinity_new, &online_new))
	333	this_count++;
	334	}
	335	}
	336
	337	count = 0;
	338	for_each_online_cpu(cpu) {
	339	if (cpu == this_cpu)
	340	continue;
ac2a5539 YL	341	/*
	342	* We scan from FIRST_EXTERNAL_VECTOR to first system
	343	* vector. If the vector is marked in the used vectors
	344	* bitmap or an irq is assigned to it, we don't count
	345	* it as available.
	346	*/
	347	for (vector = FIRST_EXTERNAL_VECTOR;
	348	vector < first_system_vector; vector++) {
	349	if (!test_bit(vector, used_vectors) &&
	350	per_cpu(vector_irq, cpu)[vector] < 0)
	351	count++;
da6139e4 PB	352	}
	353	}
	354
	355	if (count < this_count) {
	356	pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
	357	this_cpu, this_count, count);
	358	return -ERANGE;
	359	}
	360	return 0;
	361	}
	362
7a7732bc SS	363	/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
	364	void fixup_irqs(void)
	365	{
5231a686	366	unsigned int irq, vector;
7a7732bc SS	367	static int warned;
7a7732bc SS	368	struct irq_desc *desc;
a3c08e5d	369	struct irq_data *data;
51c43ac6	370	struct irq_chip *chip;
fb24da80	371	int ret;
7a7732bc SS	372
	373	for_each_irq_desc(irq, desc) {
	374	int break_affinity = 0;
	375	int set_affinity = 1;
	376	const struct cpumask *affinity;
	377
	378	if (!desc)
	379	continue;
	380	if (irq == 2)
	381	continue;
	382
	383	/* interrupt's are disabled at this point */
239007b8	384	raw_spin_lock(&desc->lock);
7a7732bc	385
51c43ac6	386	data = irq_desc_get_irq_data(desc);
a3c08e5d	387	affinity = data->affinity;
b87ba87c	388	if (!irq_has_action(irq) \|\| irqd_is_per_cpu(data) \|\|
58bff947	389	cpumask_subset(affinity, cpu_online_mask)) {
239007b8	390	raw_spin_unlock(&desc->lock);
7a7732bc SS	391	continue;
	392	}
	393
a5e74b84 SS	394	/*
	395	* Complete the irq move. This cpu is going down and for
	396	* non intr-remapping case, we can't wait till this interrupt
	397	* arrives at this cpu before completing the irq move.
	398	*/
	399	irq_force_complete_move(irq);
	400
7a7732bc SS	401	if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
7a7732bc SS	402	break_affinity = 1;
2530cd4f	403	affinity = cpu_online_mask;
7a7732bc SS	404	}
7a7732bc SS	405
51c43ac6 TG	406	chip = irq_data_get_irq_chip(data);
	407	if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
	408	chip->irq_mask(data);
7a7732bc	409
fb24da80 PB	410	if (chip->irq_set_affinity) {
	411	ret = chip->irq_set_affinity(data, affinity, true);
	412	if (ret == -ENOSPC)
	413	pr_crit("IRQ %d set affinity failed because there are no available vectors. The device assigned to this IRQ is unstable.\n", irq);
	414	} else {
	415	if (!(warned++))
	416	set_affinity = 0;
	417	}
7a7732bc	418
99dd5497 LC	419	/*
	420	* We unmask if the irq was not marked masked by the
	421	* core code. That respects the lazy irq disable
	422	* behaviour.
	423	*/
983bbf1a	424	if (!irqd_can_move_in_process_context(data) &&
99dd5497	425	!irqd_irq_masked(data) && chip->irq_unmask)
51c43ac6	426	chip->irq_unmask(data);
7a7732bc	427
239007b8	428	raw_spin_unlock(&desc->lock);
7a7732bc SS	429
7a7732bc SS	430	if (break_affinity && set_affinity)
c767a54b	431	pr_notice("Broke affinity for irq %i\n", irq);
7a7732bc	432	else if (!set_affinity)
c767a54b	433	pr_notice("Cannot set affinity for irq %i\n", irq);
7a7732bc SS	434	}
7a7732bc SS	435
5231a686 SS	436	/*
	437	* We can remove mdelay() and then send spuriuous interrupts to
	438	* new cpu targets for all the irqs that were handled previously by
	439	* this cpu. While it works, I have seen spurious interrupt messages
	440	* (nothing wrong but still...).
	441	*
	442	* So for now, retain mdelay(1) and check the IRR and then send those
	443	* interrupts to new targets as this cpu is already offlined...
	444	*/
7a7732bc	445	mdelay(1);
5231a686 SS	446
	447	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
	448	unsigned int irr;
	449
9345005f	450	if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNDEFINED)
5231a686 SS	451	continue;
	452
	453	irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
	454	if (irr & (1 << (vector % 32))) {
0a3aee0d	455	irq = __this_cpu_read(vector_irq[vector]);
5231a686	456
5117348d	457	desc = irq_to_desc(irq);
51c43ac6 TG	458	data = irq_desc_get_irq_data(desc);
51c43ac6 TG	459	chip = irq_data_get_irq_chip(data);
239007b8	460	raw_spin_lock(&desc->lock);
9345005f	461	if (chip->irq_retrigger) {
51c43ac6	462	chip->irq_retrigger(data);
9345005f PB	463	__this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
9345005f PB	464	}
239007b8	465	raw_spin_unlock(&desc->lock);
5231a686	466	}
9345005f PB	467	if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
9345005f PB	468	__this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED);
5231a686	469	}
7a7732bc SS	470	}
7a7732bc SS	471	#endif