[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / apic / msi.c

/*
 * Support of MSI, HPET and DMAR interrupts.
 *
 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
 *	Moved from arch/x86/kernel/apic/io_apic.c.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/dmar.h>
#include <linux/hpet.h>
#include <linux/msi.h>
#include <linux/irqdomain.h>
#include <asm/msidef.h>
#include <asm/hpet.h>
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/irq_remapping.h>

void native_compose_msi_msg(struct pci_dev *pdev,
			    unsigned int irq, unsigned int dest,
			    struct msi_msg *msg, u8 hpet_id)
{
	struct irq_cfg *cfg = irq_cfg(irq);

	msg->address_hi = MSI_ADDR_BASE_HI;

	if (x2apic_enabled())
		msg->address_hi |= MSI_ADDR_EXT_DEST_ID(dest);

	msg->address_lo =
		MSI_ADDR_BASE_LO |
		((apic->irq_dest_mode == 0) ?
			MSI_ADDR_DEST_MODE_PHYSICAL :
			MSI_ADDR_DEST_MODE_LOGICAL) |
		((apic->irq_delivery_mode != dest_LowestPrio) ?
			MSI_ADDR_REDIRECTION_CPU :
			MSI_ADDR_REDIRECTION_LOWPRI) |
		MSI_ADDR_DEST_ID(dest);

	msg->data =
		MSI_DATA_TRIGGER_EDGE |
		MSI_DATA_LEVEL_ASSERT |
		((apic->irq_delivery_mode != dest_LowestPrio) ?
			MSI_DATA_DELIVERY_FIXED :
			MSI_DATA_DELIVERY_LOWPRI) |
		MSI_DATA_VECTOR(cfg->vector);
}

static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
			   struct msi_msg *msg, u8 hpet_id)
{
	struct irq_cfg *cfg;
	int err;
	unsigned dest;

	if (disable_apic)
		return -ENXIO;

	cfg = irq_cfg(irq);
	err = assign_irq_vector(irq, cfg, apic->target_cpus());
	if (err)
		return err;

	err = apic->cpu_mask_to_apicid_and(cfg->domain,
					   apic->target_cpus(), &dest);
	if (err)
		return err;

	x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id);

	return 0;
}

static int
msi_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force)
{
	struct irq_cfg *cfg = irqd_cfg(data);
	struct msi_msg msg;
	unsigned int dest;
	int ret;

	ret = apic_set_affinity(data, mask, &dest);
	if (ret)
		return ret;

	__get_cached_msi_msg(data->msi_desc, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	__pci_write_msi_msg(data->msi_desc, &msg);

	return IRQ_SET_MASK_OK_NOCOPY;
}

/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
	.name			= "PCI-MSI",
	.irq_unmask		= pci_msi_unmask_irq,
	.irq_mask		= pci_msi_mask_irq,
	.irq_ack		= apic_ack_edge,
	.irq_set_affinity	= msi_set_affinity,
	.irq_retrigger		= apic_retrigger_irq,
	.flags			= IRQCHIP_SKIP_SET_WAKE,
};

int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc,
		  unsigned int irq_base, unsigned int irq_offset)
{
	struct irq_chip *chip = &msi_chip;
	struct msi_msg msg;
	unsigned int irq = irq_base + irq_offset;
	int ret;

	ret = msi_compose_msg(dev, irq, &msg, -1);
	if (ret < 0)
		return ret;

	irq_set_msi_desc_off(irq_base, irq_offset, msidesc);

	/*
	 * MSI-X message is written per-IRQ, the offset is always 0.
	 * MSI message denotes a contiguous group of IRQs, written for 0th IRQ.
	 */
	if (!irq_offset)
		pci_write_msi_msg(irq, &msg);

	setup_remapped_irq(irq, irq_cfg(irq), chip);

	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");

	dev_dbg(&dev->dev, "irq %d for MSI/MSI-X\n", irq);

	return 0;
}

int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
	struct msi_desc *msidesc;
	int irq, ret;

	/* Multiple MSI vectors only supported with interrupt remapping */
	if (type == PCI_CAP_ID_MSI && nvec > 1)
		return 1;

	list_for_each_entry(msidesc, &dev->msi_list, list) {
		irq = irq_domain_alloc_irqs(NULL, 1, NUMA_NO_NODE, NULL);
		if (irq <= 0)
			return -ENOSPC;

		ret = setup_msi_irq(dev, msidesc, irq, 0);
		if (ret < 0) {
			irq_domain_free_irqs(irq, 1);
			return ret;
		}

	}
	return 0;
}

void native_teardown_msi_irq(unsigned int irq)
{
	irq_domain_free_irqs(irq, 1);
}

#ifdef CONFIG_DMAR_TABLE
static int
dmar_msi_set_affinity(struct irq_data *data, const struct cpumask *mask,
		      bool force)
{
	struct irq_cfg *cfg = irqd_cfg(data);
	unsigned int dest, irq = data->irq;
	struct msi_msg msg;
	int ret;

	ret = apic_set_affinity(data, mask, &dest);
	if (ret)
		return ret;

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);
	msg.address_hi = MSI_ADDR_BASE_HI | MSI_ADDR_EXT_DEST_ID(dest);

	dmar_msi_write(irq, &msg);

	return IRQ_SET_MASK_OK_NOCOPY;
}

static struct irq_chip dmar_msi_type = {
	.name			= "DMAR_MSI",
	.irq_unmask		= dmar_msi_unmask,
	.irq_mask		= dmar_msi_mask,
	.irq_ack		= apic_ack_edge,
	.irq_set_affinity	= dmar_msi_set_affinity,
	.irq_retrigger		= apic_retrigger_irq,
	.flags			= IRQCHIP_SKIP_SET_WAKE,
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(NULL, irq, &msg, -1);
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
	irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
				      "edge");
	return 0;
}

int dmar_alloc_hwirq(void)
{
	return irq_domain_alloc_irqs(NULL, 1, NUMA_NO_NODE, NULL);
}

void dmar_free_hwirq(int irq)
{
	irq_domain_free_irqs(irq, 1);
}
#endif

/*
 * MSI message composition
 */
#ifdef CONFIG_HPET_TIMER

static int hpet_msi_set_affinity(struct irq_data *data,
				 const struct cpumask *mask, bool force)
{
	struct irq_cfg *cfg = irqd_cfg(data);
	struct msi_msg msg;
	unsigned int dest;
	int ret;

	ret = apic_set_affinity(data, mask, &dest);
	if (ret)
		return ret;

	hpet_msi_read(data->handler_data, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	hpet_msi_write(data->handler_data, &msg);

	return IRQ_SET_MASK_OK_NOCOPY;
}

static struct irq_chip hpet_msi_type = {
	.name = "HPET_MSI",
	.irq_unmask = hpet_msi_unmask,
	.irq_mask = hpet_msi_mask,
	.irq_ack = apic_ack_edge,
	.irq_set_affinity = hpet_msi_set_affinity,
	.irq_retrigger = apic_retrigger_irq,
	.flags = IRQCHIP_SKIP_SET_WAKE,
};

int default_setup_hpet_msi(unsigned int irq, unsigned int id)
{
	struct irq_chip *chip = &hpet_msi_type;
	struct msi_msg msg;
	int ret;

	ret = msi_compose_msg(NULL, irq, &msg, id);
	if (ret < 0)
		return ret;

	hpet_msi_write(irq_get_handler_data(irq), &msg);
	irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
	setup_remapped_irq(irq, irq_cfg(irq), chip);

	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
	return 0;
}
#endif
Commit	Line	Data
44380982 JL	1	/*
	2	* Support of MSI, HPET and DMAR interrupts.
	3	*
	4	* Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
	5	* Moved from arch/x86/kernel/apic/io_apic.c.
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*/
	11	#include <linux/mm.h>
	12	#include <linux/interrupt.h>
	13	#include <linux/pci.h>
	14	#include <linux/dmar.h>
	15	#include <linux/hpet.h>
	16	#include <linux/msi.h>
4c8f9960	17	#include <linux/irqdomain.h>
44380982 JL	18	#include <asm/msidef.h>
	19	#include <asm/hpet.h>
	20	#include <asm/hw_irq.h>
	21	#include <asm/apic.h>
	22	#include <asm/irq_remapping.h>
	23
	24	void native_compose_msi_msg(struct pci_dev *pdev,
	25	unsigned int irq, unsigned int dest,
	26	struct msi_msg *msg, u8 hpet_id)
	27	{
	28	struct irq_cfg *cfg = irq_cfg(irq);
	29
	30	msg->address_hi = MSI_ADDR_BASE_HI;
	31
	32	if (x2apic_enabled())
	33	msg->address_hi \|= MSI_ADDR_EXT_DEST_ID(dest);
	34
	35	msg->address_lo =
	36	MSI_ADDR_BASE_LO \|
	37	((apic->irq_dest_mode == 0) ?
	38	MSI_ADDR_DEST_MODE_PHYSICAL :
	39	MSI_ADDR_DEST_MODE_LOGICAL) \|
	40	((apic->irq_delivery_mode != dest_LowestPrio) ?
	41	MSI_ADDR_REDIRECTION_CPU :
	42	MSI_ADDR_REDIRECTION_LOWPRI) \|
	43	MSI_ADDR_DEST_ID(dest);
	44
	45	msg->data =
	46	MSI_DATA_TRIGGER_EDGE \|
	47	MSI_DATA_LEVEL_ASSERT \|
	48	((apic->irq_delivery_mode != dest_LowestPrio) ?
	49	MSI_DATA_DELIVERY_FIXED :
	50	MSI_DATA_DELIVERY_LOWPRI) \|
	51	MSI_DATA_VECTOR(cfg->vector);
	52	}
	53
	54	static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
	55	struct msi_msg *msg, u8 hpet_id)
	56	{
	57	struct irq_cfg *cfg;
	58	int err;
	59	unsigned dest;
	60
	61	if (disable_apic)
	62	return -ENXIO;
	63
	64	cfg = irq_cfg(irq);
	65	err = assign_irq_vector(irq, cfg, apic->target_cpus());
	66	if (err)
	67	return err;
	68
	69	err = apic->cpu_mask_to_apicid_and(cfg->domain,
	70	apic->target_cpus(), &dest);
	71	if (err)
	72	return err;
	73
	74	x86_msi.compose_msi_msg(pdev, irq, dest, msg, hpet_id);
	75
	76	return 0;
	77	}
	78
	79	static int
	80	msi_set_affinity(struct irq_data data, const struct cpumask mask, bool force)
	81	{
a9786091	82	struct irq_cfg *cfg = irqd_cfg(data);
44380982 JL	83	struct msi_msg msg;
	84	unsigned int dest;
	85	int ret;
	86
	87	ret = apic_set_affinity(data, mask, &dest);
	88	if (ret)
	89	return ret;
	90
	91	__get_cached_msi_msg(data->msi_desc, &msg);
	92
	93	msg.data &= ~MSI_DATA_VECTOR_MASK;
	94	msg.data \|= MSI_DATA_VECTOR(cfg->vector);
	95	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	96	msg.address_lo \|= MSI_ADDR_DEST_ID(dest);
	97
	98	__pci_write_msi_msg(data->msi_desc, &msg);
	99
	100	return IRQ_SET_MASK_OK_NOCOPY;
	101	}
	102
	103	/*
	104	* IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
	105	* which implement the MSI or MSI-X Capability Structure.
	106	*/
	107	static struct irq_chip msi_chip = {
	108	.name = "PCI-MSI",
	109	.irq_unmask = pci_msi_unmask_irq,
	110	.irq_mask = pci_msi_mask_irq,
	111	.irq_ack = apic_ack_edge,
	112	.irq_set_affinity = msi_set_affinity,
	113	.irq_retrigger = apic_retrigger_irq,
	114	.flags = IRQCHIP_SKIP_SET_WAKE,
	115	};
	116
	117	int setup_msi_irq(struct pci_dev dev, struct msi_desc msidesc,
	118	unsigned int irq_base, unsigned int irq_offset)
	119	{
	120	struct irq_chip *chip = &msi_chip;
	121	struct msi_msg msg;
	122	unsigned int irq = irq_base + irq_offset;
	123	int ret;
	124
	125	ret = msi_compose_msg(dev, irq, &msg, -1);
	126	if (ret < 0)
	127	return ret;
	128
	129	irq_set_msi_desc_off(irq_base, irq_offset, msidesc);
	130
	131	/*
	132	* MSI-X message is written per-IRQ, the offset is always 0.
	133	* MSI message denotes a contiguous group of IRQs, written for 0th IRQ.
	134	*/
	135	if (!irq_offset)
	136	pci_write_msi_msg(irq, &msg);
	137
	138	setup_remapped_irq(irq, irq_cfg(irq), chip);
	139
	140	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
	141
	142	dev_dbg(&dev->dev, "irq %d for MSI/MSI-X\n", irq);
	143
	144	return 0;
	145	}
	146
147	int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
148	{
149	struct msi_desc *msidesc;
4c8f9960	150	int irq, ret;
44380982 JL	151
	152	/* Multiple MSI vectors only supported with interrupt remapping */
	153	if (type == PCI_CAP_ID_MSI && nvec > 1)
	154	return 1;
	155
44380982	156	list_for_each_entry(msidesc, &dev->msi_list, list) {
4c8f9960 JL	157	irq = irq_domain_alloc_irqs(NULL, 1, NUMA_NO_NODE, NULL);
4c8f9960 JL	158	if (irq <= 0)
44380982 JL	159	return -ENOSPC;
	160
	161	ret = setup_msi_irq(dev, msidesc, irq, 0);
	162	if (ret < 0) {
4c8f9960	163	irq_domain_free_irqs(irq, 1);
44380982 JL	164	return ret;
	165	}
	166
	167	}
	168	return 0;
	169	}
	170
	171	void native_teardown_msi_irq(unsigned int irq)
	172	{
4c8f9960	173	irq_domain_free_irqs(irq, 1);
44380982 JL	174	}
	175
	176	#ifdef CONFIG_DMAR_TABLE
	177	static int
	178	dmar_msi_set_affinity(struct irq_data data, const struct cpumask mask,
	179	bool force)
	180	{
a9786091	181	struct irq_cfg *cfg = irqd_cfg(data);
44380982 JL	182	unsigned int dest, irq = data->irq;
	183	struct msi_msg msg;
	184	int ret;
	185
	186	ret = apic_set_affinity(data, mask, &dest);
	187	if (ret)
	188	return ret;
	189
	190	dmar_msi_read(irq, &msg);
	191
	192	msg.data &= ~MSI_DATA_VECTOR_MASK;
	193	msg.data \|= MSI_DATA_VECTOR(cfg->vector);
	194	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	195	msg.address_lo \|= MSI_ADDR_DEST_ID(dest);
	196	msg.address_hi = MSI_ADDR_BASE_HI \| MSI_ADDR_EXT_DEST_ID(dest);
	197
	198	dmar_msi_write(irq, &msg);
	199
	200	return IRQ_SET_MASK_OK_NOCOPY;
	201	}
	202
	203	static struct irq_chip dmar_msi_type = {
	204	.name = "DMAR_MSI",
	205	.irq_unmask = dmar_msi_unmask,
	206	.irq_mask = dmar_msi_mask,
	207	.irq_ack = apic_ack_edge,
	208	.irq_set_affinity = dmar_msi_set_affinity,
	209	.irq_retrigger = apic_retrigger_irq,
	210	.flags = IRQCHIP_SKIP_SET_WAKE,
	211	};
	212
	213	int arch_setup_dmar_msi(unsigned int irq)
	214	{
	215	int ret;
	216	struct msi_msg msg;
	217
	218	ret = msi_compose_msg(NULL, irq, &msg, -1);
	219	if (ret < 0)
	220	return ret;
	221	dmar_msi_write(irq, &msg);
	222	irq_set_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
	223	"edge");
	224	return 0;
	225	}
a62b32cd JL	226
	227	int dmar_alloc_hwirq(void)
	228	{
	229	return irq_domain_alloc_irqs(NULL, 1, NUMA_NO_NODE, NULL);
	230	}
	231
	232	void dmar_free_hwirq(int irq)
	233	{
	234	irq_domain_free_irqs(irq, 1);
	235	}
44380982 JL	236	#endif
	237
	238	/*
	239	* MSI message composition
	240	*/
	241	#ifdef CONFIG_HPET_TIMER
	242
	243	static int hpet_msi_set_affinity(struct irq_data *data,
	244	const struct cpumask *mask, bool force)
	245	{
a9786091	246	struct irq_cfg *cfg = irqd_cfg(data);
44380982 JL	247	struct msi_msg msg;
	248	unsigned int dest;
	249	int ret;
	250
	251	ret = apic_set_affinity(data, mask, &dest);
	252	if (ret)
	253	return ret;
	254
	255	hpet_msi_read(data->handler_data, &msg);
	256
	257	msg.data &= ~MSI_DATA_VECTOR_MASK;
	258	msg.data \|= MSI_DATA_VECTOR(cfg->vector);
	259	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	260	msg.address_lo \|= MSI_ADDR_DEST_ID(dest);
	261
	262	hpet_msi_write(data->handler_data, &msg);
	263
	264	return IRQ_SET_MASK_OK_NOCOPY;
	265	}
	266
	267	static struct irq_chip hpet_msi_type = {
	268	.name = "HPET_MSI",
	269	.irq_unmask = hpet_msi_unmask,
	270	.irq_mask = hpet_msi_mask,
	271	.irq_ack = apic_ack_edge,
	272	.irq_set_affinity = hpet_msi_set_affinity,
	273	.irq_retrigger = apic_retrigger_irq,
	274	.flags = IRQCHIP_SKIP_SET_WAKE,
	275	};
	276
	277	int default_setup_hpet_msi(unsigned int irq, unsigned int id)
	278	{
	279	struct irq_chip *chip = &hpet_msi_type;
	280	struct msi_msg msg;
	281	int ret;
	282
	283	ret = msi_compose_msg(NULL, irq, &msg, id);
	284	if (ret < 0)
	285	return ret;
	286
	287	hpet_msi_write(irq_get_handler_data(irq), &msg);
	288	irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
	289	setup_remapped_irq(irq, irq_cfg(irq), chip);
	290
	291	irq_set_chip_and_handler_name(irq, chip, handle_edge_irq, "edge");
	292	return 0;
	293	}
	294	#endif