2 * Support of MSI, HPET and DMAR interrupts.
4 * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
5 * Moved from arch/x86/kernel/apic/io_apic.c.
6 * Jiang Liu <jiang.liu@linux.intel.com>
7 * Convert to hierarchical irqdomain
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/interrupt.h>
15 #include <linux/irq.h>
16 #include <linux/pci.h>
17 #include <linux/dmar.h>
18 #include <linux/hpet.h>
19 #include <linux/msi.h>
20 #include <asm/irqdomain.h>
21 #include <asm/msidef.h>
23 #include <asm/hw_irq.h>
25 #include <asm/irq_remapping.h>
27 static struct irq_domain
*msi_default_domain
;
29 static void __irq_msi_compose_msg(struct irq_cfg
*cfg
, struct msi_msg
*msg
)
31 msg
->address_hi
= MSI_ADDR_BASE_HI
;
34 msg
->address_hi
|= MSI_ADDR_EXT_DEST_ID(cfg
->dest_apicid
);
38 ((apic
->irq_dest_mode
== 0) ?
39 MSI_ADDR_DEST_MODE_PHYSICAL
:
40 MSI_ADDR_DEST_MODE_LOGICAL
) |
41 MSI_ADDR_REDIRECTION_CPU
|
42 MSI_ADDR_DEST_ID(cfg
->dest_apicid
);
45 MSI_DATA_TRIGGER_EDGE
|
46 MSI_DATA_LEVEL_ASSERT
|
47 MSI_DATA_DELIVERY_FIXED
|
48 MSI_DATA_VECTOR(cfg
->vector
);
51 static void irq_msi_compose_msg(struct irq_data
*data
, struct msi_msg
*msg
)
53 __irq_msi_compose_msg(irqd_cfg(data
), msg
);
56 static void irq_msi_update_msg(struct irq_data
*irqd
, struct irq_cfg
*cfg
)
58 struct msi_msg msg
[2] = { [1] = { }, };
60 __irq_msi_compose_msg(cfg
, msg
);
61 irq_data_get_irq_chip(irqd
)->irq_write_msi_msg(irqd
, msg
);
65 msi_set_affinity(struct irq_data
*irqd
, const struct cpumask
*mask
, bool force
)
67 struct irq_cfg old_cfg
, *cfg
= irqd_cfg(irqd
);
68 struct irq_data
*parent
= irqd
->parent_data
;
72 /* Save the current configuration */
73 cpu
= cpumask_first(irq_data_get_effective_affinity_mask(irqd
));
76 /* Allocate a new target vector */
77 ret
= parent
->chip
->irq_set_affinity(parent
, mask
, force
);
78 if (ret
< 0 || ret
== IRQ_SET_MASK_OK_DONE
)
82 * For non-maskable and non-remapped MSI interrupts the migration
83 * to a different destination CPU and a different vector has to be
84 * done careful to handle the possible stray interrupt which can be
85 * caused by the non-atomic update of the address/data pair.
87 * Direct update is possible when:
88 * - The MSI is maskable (remapped MSI does not use this code path)).
89 * The quirk bit is not set in this case.
90 * - The new vector is the same as the old vector
91 * - The old vector is MANAGED_IRQ_SHUTDOWN_VECTOR (interrupt starts up)
92 * - The new destination CPU is the same as the old destination CPU
94 if (!irqd_msi_nomask_quirk(irqd
) ||
95 cfg
->vector
== old_cfg
.vector
||
96 old_cfg
.vector
== MANAGED_IRQ_SHUTDOWN_VECTOR
||
97 cfg
->dest_apicid
== old_cfg
.dest_apicid
) {
98 irq_msi_update_msg(irqd
, cfg
);
103 * Paranoia: Validate that the interrupt target is the local
106 if (WARN_ON_ONCE(cpu
!= smp_processor_id())) {
107 irq_msi_update_msg(irqd
, cfg
);
112 * Redirect the interrupt to the new vector on the current CPU
113 * first. This might cause a spurious interrupt on this vector if
114 * the device raises an interrupt right between this update and the
115 * update to the final destination CPU.
117 * If the vector is in use then the installed device handler will
118 * denote it as spurious which is no harm as this is a rare event
119 * and interrupt handlers have to cope with spurious interrupts
120 * anyway. If the vector is unused, then it is marked so it won't
121 * trigger the 'No irq handler for vector' warning in do_IRQ().
123 * This requires to hold vector lock to prevent concurrent updates to
124 * the affected vector.
129 * Mark the new target vector on the local CPU if it is currently
130 * unused. Reuse the VECTOR_RETRIGGERED state which is also used in
131 * the CPU hotplug path for a similar purpose. This cannot be
132 * undone here as the current CPU has interrupts disabled and
133 * cannot handle the interrupt before the whole set_affinity()
134 * section is done. In the CPU unplug case, the current CPU is
135 * about to vanish and will not handle any interrupts anymore. The
136 * vector is cleaned up when the CPU comes online again.
138 if (IS_ERR_OR_NULL(this_cpu_read(vector_irq
[cfg
->vector
])))
139 this_cpu_write(vector_irq
[cfg
->vector
], VECTOR_RETRIGGERED
);
141 /* Redirect it to the new vector on the local CPU temporarily */
142 old_cfg
.vector
= cfg
->vector
;
143 irq_msi_update_msg(irqd
, &old_cfg
);
145 /* Now transition it to the target CPU */
146 irq_msi_update_msg(irqd
, cfg
);
149 * All interrupts after this point are now targeted at the new
152 * Drop vector lock before testing whether the temporary assignment
153 * to the local CPU was hit by an interrupt raised in the device,
154 * because the retrigger function acquires vector lock again.
156 unlock_vector_lock();
159 * Check whether the transition raced with a device interrupt and
160 * is pending in the local APICs IRR. It is safe to do this outside
161 * of vector lock as the irq_desc::lock of this interrupt is still
162 * held and interrupts are disabled: The check is not accessing the
163 * underlying vector store. It's just checking the local APIC's
166 if (lapic_vector_set_in_irr(cfg
->vector
))
167 irq_data_get_irq_chip(irqd
)->irq_retrigger(irqd
);
173 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
174 * which implement the MSI or MSI-X Capability Structure.
176 static struct irq_chip pci_msi_controller
= {
178 .irq_unmask
= pci_msi_unmask_irq
,
179 .irq_mask
= pci_msi_mask_irq
,
180 .irq_ack
= irq_chip_ack_parent
,
181 .irq_retrigger
= irq_chip_retrigger_hierarchy
,
182 .irq_compose_msi_msg
= irq_msi_compose_msg
,
183 .irq_set_affinity
= msi_set_affinity
,
184 .flags
= IRQCHIP_SKIP_SET_WAKE
,
187 int native_setup_msi_irqs(struct pci_dev
*dev
, int nvec
, int type
)
189 struct irq_domain
*domain
;
190 struct irq_alloc_info info
;
192 init_irq_alloc_info(&info
, NULL
);
193 info
.type
= X86_IRQ_ALLOC_TYPE_MSI
;
196 domain
= irq_remapping_get_irq_domain(&info
);
198 domain
= msi_default_domain
;
202 return msi_domain_alloc_irqs(domain
, &dev
->dev
, nvec
);
205 void native_teardown_msi_irq(unsigned int irq
)
207 irq_domain_free_irqs(irq
, 1);
210 static irq_hw_number_t
pci_msi_get_hwirq(struct msi_domain_info
*info
,
211 msi_alloc_info_t
*arg
)
213 return arg
->msi_hwirq
;
216 int pci_msi_prepare(struct irq_domain
*domain
, struct device
*dev
, int nvec
,
217 msi_alloc_info_t
*arg
)
219 struct pci_dev
*pdev
= to_pci_dev(dev
);
220 struct msi_desc
*desc
= first_pci_msi_entry(pdev
);
222 init_irq_alloc_info(arg
, NULL
);
224 if (desc
->msi_attrib
.is_msix
) {
225 arg
->type
= X86_IRQ_ALLOC_TYPE_MSIX
;
227 arg
->type
= X86_IRQ_ALLOC_TYPE_MSI
;
228 arg
->flags
|= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
;
233 EXPORT_SYMBOL_GPL(pci_msi_prepare
);
235 void pci_msi_set_desc(msi_alloc_info_t
*arg
, struct msi_desc
*desc
)
237 arg
->msi_hwirq
= pci_msi_domain_calc_hwirq(arg
->msi_dev
, desc
);
239 EXPORT_SYMBOL_GPL(pci_msi_set_desc
);
241 static struct msi_domain_ops pci_msi_domain_ops
= {
242 .get_hwirq
= pci_msi_get_hwirq
,
243 .msi_prepare
= pci_msi_prepare
,
244 .set_desc
= pci_msi_set_desc
,
247 static struct msi_domain_info pci_msi_domain_info
= {
248 .flags
= MSI_FLAG_USE_DEF_DOM_OPS
| MSI_FLAG_USE_DEF_CHIP_OPS
|
250 .ops
= &pci_msi_domain_ops
,
251 .chip
= &pci_msi_controller
,
252 .handler
= handle_edge_irq
,
253 .handler_name
= "edge",
256 void __init
arch_init_msi_domain(struct irq_domain
*parent
)
258 struct fwnode_handle
*fn
;
263 fn
= irq_domain_alloc_named_fwnode("PCI-MSI");
266 pci_msi_create_irq_domain(fn
, &pci_msi_domain_info
,
268 irq_domain_free_fwnode(fn
);
270 if (!msi_default_domain
)
271 pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n");
273 msi_default_domain
->flags
|= IRQ_DOMAIN_MSI_NOMASK_QUIRK
;
276 #ifdef CONFIG_IRQ_REMAP
277 static struct irq_chip pci_msi_ir_controller
= {
278 .name
= "IR-PCI-MSI",
279 .irq_unmask
= pci_msi_unmask_irq
,
280 .irq_mask
= pci_msi_mask_irq
,
281 .irq_ack
= irq_chip_ack_parent
,
282 .irq_retrigger
= irq_chip_retrigger_hierarchy
,
283 .irq_set_vcpu_affinity
= irq_chip_set_vcpu_affinity_parent
,
284 .flags
= IRQCHIP_SKIP_SET_WAKE
,
287 static struct msi_domain_info pci_msi_ir_domain_info
= {
288 .flags
= MSI_FLAG_USE_DEF_DOM_OPS
| MSI_FLAG_USE_DEF_CHIP_OPS
|
289 MSI_FLAG_MULTI_PCI_MSI
| MSI_FLAG_PCI_MSIX
,
290 .ops
= &pci_msi_domain_ops
,
291 .chip
= &pci_msi_ir_controller
,
292 .handler
= handle_edge_irq
,
293 .handler_name
= "edge",
296 struct irq_domain
*arch_create_remap_msi_irq_domain(struct irq_domain
*parent
,
297 const char *name
, int id
)
299 struct fwnode_handle
*fn
;
300 struct irq_domain
*d
;
302 fn
= irq_domain_alloc_named_id_fwnode(name
, id
);
305 d
= pci_msi_create_irq_domain(fn
, &pci_msi_ir_domain_info
, parent
);
306 irq_domain_free_fwnode(fn
);
311 #ifdef CONFIG_DMAR_TABLE
312 static void dmar_msi_write_msg(struct irq_data
*data
, struct msi_msg
*msg
)
314 dmar_msi_write(data
->irq
, msg
);
317 static struct irq_chip dmar_msi_controller
= {
319 .irq_unmask
= dmar_msi_unmask
,
320 .irq_mask
= dmar_msi_mask
,
321 .irq_ack
= irq_chip_ack_parent
,
322 .irq_set_affinity
= msi_domain_set_affinity
,
323 .irq_retrigger
= irq_chip_retrigger_hierarchy
,
324 .irq_compose_msi_msg
= irq_msi_compose_msg
,
325 .irq_write_msi_msg
= dmar_msi_write_msg
,
326 .flags
= IRQCHIP_SKIP_SET_WAKE
,
329 static irq_hw_number_t
dmar_msi_get_hwirq(struct msi_domain_info
*info
,
330 msi_alloc_info_t
*arg
)
335 static int dmar_msi_init(struct irq_domain
*domain
,
336 struct msi_domain_info
*info
, unsigned int virq
,
337 irq_hw_number_t hwirq
, msi_alloc_info_t
*arg
)
339 irq_domain_set_info(domain
, virq
, arg
->dmar_id
, info
->chip
, NULL
,
340 handle_edge_irq
, arg
->dmar_data
, "edge");
345 static struct msi_domain_ops dmar_msi_domain_ops
= {
346 .get_hwirq
= dmar_msi_get_hwirq
,
347 .msi_init
= dmar_msi_init
,
350 static struct msi_domain_info dmar_msi_domain_info
= {
351 .ops
= &dmar_msi_domain_ops
,
352 .chip
= &dmar_msi_controller
,
355 static struct irq_domain
*dmar_get_irq_domain(void)
357 static struct irq_domain
*dmar_domain
;
358 static DEFINE_MUTEX(dmar_lock
);
359 struct fwnode_handle
*fn
;
361 mutex_lock(&dmar_lock
);
365 fn
= irq_domain_alloc_named_fwnode("DMAR-MSI");
367 dmar_domain
= msi_create_irq_domain(fn
, &dmar_msi_domain_info
,
369 irq_domain_free_fwnode(fn
);
372 mutex_unlock(&dmar_lock
);
376 int dmar_alloc_hwirq(int id
, int node
, void *arg
)
378 struct irq_domain
*domain
= dmar_get_irq_domain();
379 struct irq_alloc_info info
;
384 init_irq_alloc_info(&info
, NULL
);
385 info
.type
= X86_IRQ_ALLOC_TYPE_DMAR
;
387 info
.dmar_data
= arg
;
389 return irq_domain_alloc_irqs(domain
, 1, node
, &info
);
392 void dmar_free_hwirq(int irq
)
394 irq_domain_free_irqs(irq
, 1);
399 * MSI message composition
401 #ifdef CONFIG_HPET_TIMER
402 static inline int hpet_dev_id(struct irq_domain
*domain
)
404 struct msi_domain_info
*info
= msi_get_domain_info(domain
);
406 return (int)(long)info
->data
;
409 static void hpet_msi_write_msg(struct irq_data
*data
, struct msi_msg
*msg
)
411 hpet_msi_write(irq_data_get_irq_handler_data(data
), msg
);
414 static struct irq_chip hpet_msi_controller __ro_after_init
= {
416 .irq_unmask
= hpet_msi_unmask
,
417 .irq_mask
= hpet_msi_mask
,
418 .irq_ack
= irq_chip_ack_parent
,
419 .irq_set_affinity
= msi_domain_set_affinity
,
420 .irq_retrigger
= irq_chip_retrigger_hierarchy
,
421 .irq_compose_msi_msg
= irq_msi_compose_msg
,
422 .irq_write_msi_msg
= hpet_msi_write_msg
,
423 .flags
= IRQCHIP_SKIP_SET_WAKE
,
426 static irq_hw_number_t
hpet_msi_get_hwirq(struct msi_domain_info
*info
,
427 msi_alloc_info_t
*arg
)
429 return arg
->hpet_index
;
432 static int hpet_msi_init(struct irq_domain
*domain
,
433 struct msi_domain_info
*info
, unsigned int virq
,
434 irq_hw_number_t hwirq
, msi_alloc_info_t
*arg
)
436 irq_set_status_flags(virq
, IRQ_MOVE_PCNTXT
);
437 irq_domain_set_info(domain
, virq
, arg
->hpet_index
, info
->chip
, NULL
,
438 handle_edge_irq
, arg
->hpet_data
, "edge");
443 static void hpet_msi_free(struct irq_domain
*domain
,
444 struct msi_domain_info
*info
, unsigned int virq
)
446 irq_clear_status_flags(virq
, IRQ_MOVE_PCNTXT
);
449 static struct msi_domain_ops hpet_msi_domain_ops
= {
450 .get_hwirq
= hpet_msi_get_hwirq
,
451 .msi_init
= hpet_msi_init
,
452 .msi_free
= hpet_msi_free
,
455 static struct msi_domain_info hpet_msi_domain_info
= {
456 .ops
= &hpet_msi_domain_ops
,
457 .chip
= &hpet_msi_controller
,
460 struct irq_domain
*hpet_create_irq_domain(int hpet_id
)
462 struct msi_domain_info
*domain_info
;
463 struct irq_domain
*parent
, *d
;
464 struct irq_alloc_info info
;
465 struct fwnode_handle
*fn
;
467 if (x86_vector_domain
== NULL
)
470 domain_info
= kzalloc(sizeof(*domain_info
), GFP_KERNEL
);
474 *domain_info
= hpet_msi_domain_info
;
475 domain_info
->data
= (void *)(long)hpet_id
;
477 init_irq_alloc_info(&info
, NULL
);
478 info
.type
= X86_IRQ_ALLOC_TYPE_HPET
;
479 info
.hpet_id
= hpet_id
;
480 parent
= irq_remapping_get_ir_irq_domain(&info
);
482 parent
= x86_vector_domain
;
484 hpet_msi_controller
.name
= "IR-HPET-MSI";
486 fn
= irq_domain_alloc_named_id_fwnode(hpet_msi_controller
.name
,
493 d
= msi_create_irq_domain(fn
, domain_info
, parent
);
494 irq_domain_free_fwnode(fn
);
498 int hpet_assign_irq(struct irq_domain
*domain
, struct hpet_dev
*dev
,
501 struct irq_alloc_info info
;
503 init_irq_alloc_info(&info
, NULL
);
504 info
.type
= X86_IRQ_ALLOC_TYPE_HPET
;
505 info
.hpet_data
= dev
;
506 info
.hpet_id
= hpet_dev_id(domain
);
507 info
.hpet_index
= dev_num
;
509 return irq_domain_alloc_irqs(domain
, 1, NUMA_NO_NODE
, &info
);