2 #define pr_fmt(fmt) "DMAR-IR: " fmt
4 #include <linux/interrupt.h>
5 #include <linux/dmar.h>
6 #include <linux/spinlock.h>
7 #include <linux/slab.h>
8 #include <linux/jiffies.h>
9 #include <linux/hpet.h>
10 #include <linux/pci.h>
11 #include <linux/irq.h>
12 #include <linux/intel-iommu.h>
13 #include <linux/acpi.h>
14 #include <linux/irqdomain.h>
15 #include <linux/crash_dump.h>
16 #include <asm/io_apic.h>
19 #include <asm/irq_remapping.h>
20 #include <asm/pci-direct.h>
21 #include <asm/msidef.h>
23 #include "irq_remapping.h"
31 struct intel_iommu
*iommu
;
33 unsigned int bus
; /* PCI bus number */
34 unsigned int devfn
; /* PCI devfn number */
38 struct intel_iommu
*iommu
;
45 struct intel_iommu
*iommu
;
52 struct intel_ir_data
{
53 struct irq_2_iommu irq_2_iommu
;
54 struct irte irte_entry
;
56 struct msi_msg msi_entry
;
60 #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
61 #define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8)
63 static int __read_mostly eim_mode
;
64 static struct ioapic_scope ir_ioapic
[MAX_IO_APICS
];
65 static struct hpet_scope ir_hpet
[MAX_HPET_TBS
];
72 * ->iommu->register_lock
74 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
75 * in single-threaded environment with interrupt disabled, so no need to tabke
76 * the dmar_global_lock.
78 static DEFINE_RAW_SPINLOCK(irq_2_ir_lock
);
79 static struct irq_domain_ops intel_ir_domain_ops
;
81 static void iommu_disable_irq_remapping(struct intel_iommu
*iommu
);
82 static int __init
parse_ioapics_under_ir(void);
84 static bool ir_pre_enabled(struct intel_iommu
*iommu
)
86 return (iommu
->flags
& VTD_FLAG_IRQ_REMAP_PRE_ENABLED
);
89 static void clear_ir_pre_enabled(struct intel_iommu
*iommu
)
91 iommu
->flags
&= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED
;
94 static void init_ir_status(struct intel_iommu
*iommu
)
98 gsts
= readl(iommu
->reg
+ DMAR_GSTS_REG
);
99 if (gsts
& DMA_GSTS_IRES
)
100 iommu
->flags
|= VTD_FLAG_IRQ_REMAP_PRE_ENABLED
;
103 static int alloc_irte(struct intel_iommu
*iommu
, int irq
,
104 struct irq_2_iommu
*irq_iommu
, u16 count
)
106 struct ir_table
*table
= iommu
->ir_table
;
107 unsigned int mask
= 0;
111 if (!count
|| !irq_iommu
)
115 count
= __roundup_pow_of_two(count
);
119 if (mask
> ecap_max_handle_mask(iommu
->ecap
)) {
120 pr_err("Requested mask %x exceeds the max invalidation handle"
121 " mask value %Lx\n", mask
,
122 ecap_max_handle_mask(iommu
->ecap
));
126 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
127 index
= bitmap_find_free_region(table
->bitmap
,
128 INTR_REMAP_TABLE_ENTRIES
, mask
);
130 pr_warn("IR%d: can't allocate an IRTE\n", iommu
->seq_id
);
132 irq_iommu
->iommu
= iommu
;
133 irq_iommu
->irte_index
= index
;
134 irq_iommu
->sub_handle
= 0;
135 irq_iommu
->irte_mask
= mask
;
136 irq_iommu
->mode
= IRQ_REMAPPING
;
138 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
143 static int qi_flush_iec(struct intel_iommu
*iommu
, int index
, int mask
)
147 desc
.low
= QI_IEC_IIDEX(index
) | QI_IEC_TYPE
| QI_IEC_IM(mask
)
151 return qi_submit_sync(&desc
, iommu
);
154 static int modify_irte(struct irq_2_iommu
*irq_iommu
,
155 struct irte
*irte_modified
)
157 struct intel_iommu
*iommu
;
165 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
167 iommu
= irq_iommu
->iommu
;
169 index
= irq_iommu
->irte_index
+ irq_iommu
->sub_handle
;
170 irte
= &iommu
->ir_table
->base
[index
];
172 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE)
173 if ((irte
->pst
== 1) || (irte_modified
->pst
== 1)) {
176 ret
= cmpxchg_double(&irte
->low
, &irte
->high
,
177 irte
->low
, irte
->high
,
178 irte_modified
->low
, irte_modified
->high
);
180 * We use cmpxchg16 to atomically update the 128-bit IRTE,
181 * and it cannot be updated by the hardware or other processors
182 * behind us, so the return value of cmpxchg16 should be the
183 * same as the old value.
189 set_64bit(&irte
->low
, irte_modified
->low
);
190 set_64bit(&irte
->high
, irte_modified
->high
);
192 __iommu_flush_cache(iommu
, irte
, sizeof(*irte
));
194 rc
= qi_flush_iec(iommu
, index
, 0);
196 /* Update iommu mode according to the IRTE mode */
197 irq_iommu
->mode
= irte
->pst
? IRQ_POSTING
: IRQ_REMAPPING
;
198 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
203 static struct intel_iommu
*map_hpet_to_ir(u8 hpet_id
)
207 for (i
= 0; i
< MAX_HPET_TBS
; i
++)
208 if (ir_hpet
[i
].id
== hpet_id
&& ir_hpet
[i
].iommu
)
209 return ir_hpet
[i
].iommu
;
213 static struct intel_iommu
*map_ioapic_to_ir(int apic
)
217 for (i
= 0; i
< MAX_IO_APICS
; i
++)
218 if (ir_ioapic
[i
].id
== apic
&& ir_ioapic
[i
].iommu
)
219 return ir_ioapic
[i
].iommu
;
223 static struct intel_iommu
*map_dev_to_ir(struct pci_dev
*dev
)
225 struct dmar_drhd_unit
*drhd
;
227 drhd
= dmar_find_matched_drhd_unit(dev
);
234 static int clear_entries(struct irq_2_iommu
*irq_iommu
)
236 struct irte
*start
, *entry
, *end
;
237 struct intel_iommu
*iommu
;
240 if (irq_iommu
->sub_handle
)
243 iommu
= irq_iommu
->iommu
;
244 index
= irq_iommu
->irte_index
;
246 start
= iommu
->ir_table
->base
+ index
;
247 end
= start
+ (1 << irq_iommu
->irte_mask
);
249 for (entry
= start
; entry
< end
; entry
++) {
250 set_64bit(&entry
->low
, 0);
251 set_64bit(&entry
->high
, 0);
253 bitmap_release_region(iommu
->ir_table
->bitmap
, index
,
254 irq_iommu
->irte_mask
);
256 return qi_flush_iec(iommu
, index
, irq_iommu
->irte_mask
);
260 * source validation type
262 #define SVT_NO_VERIFY 0x0 /* no verification is required */
263 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
264 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
267 * source-id qualifier
269 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
270 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
271 * the third least significant bit
273 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
274 * the second and third least significant bits
276 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
277 * the least three significant bits
281 * set SVT, SQ and SID fields of irte to verify
282 * source ids of interrupt requests
284 static void set_irte_sid(struct irte
*irte
, unsigned int svt
,
285 unsigned int sq
, unsigned int sid
)
287 if (disable_sourceid_checking
)
294 static int set_ioapic_sid(struct irte
*irte
, int apic
)
302 down_read(&dmar_global_lock
);
303 for (i
= 0; i
< MAX_IO_APICS
; i
++) {
304 if (ir_ioapic
[i
].iommu
&& ir_ioapic
[i
].id
== apic
) {
305 sid
= (ir_ioapic
[i
].bus
<< 8) | ir_ioapic
[i
].devfn
;
309 up_read(&dmar_global_lock
);
312 pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic
);
316 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
, sid
);
321 static int set_hpet_sid(struct irte
*irte
, u8 id
)
329 down_read(&dmar_global_lock
);
330 for (i
= 0; i
< MAX_HPET_TBS
; i
++) {
331 if (ir_hpet
[i
].iommu
&& ir_hpet
[i
].id
== id
) {
332 sid
= (ir_hpet
[i
].bus
<< 8) | ir_hpet
[i
].devfn
;
336 up_read(&dmar_global_lock
);
339 pr_warn("Failed to set source-id of HPET block (%d)\n", id
);
344 * Should really use SQ_ALL_16. Some platforms are broken.
345 * While we figure out the right quirks for these broken platforms, use
346 * SQ_13_IGNORE_3 for now.
348 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_13_IGNORE_3
, sid
);
353 struct set_msi_sid_data
{
354 struct pci_dev
*pdev
;
358 static int set_msi_sid_cb(struct pci_dev
*pdev
, u16 alias
, void *opaque
)
360 struct set_msi_sid_data
*data
= opaque
;
368 static int set_msi_sid(struct irte
*irte
, struct pci_dev
*dev
)
370 struct set_msi_sid_data data
;
375 pci_for_each_dma_alias(dev
, set_msi_sid_cb
, &data
);
378 * DMA alias provides us with a PCI device and alias. The only case
379 * where the it will return an alias on a different bus than the
380 * device is the case of a PCIe-to-PCI bridge, where the alias is for
381 * the subordinate bus. In this case we can only verify the bus.
383 * If the alias device is on a different bus than our source device
384 * then we have a topology based alias, use it.
386 * Otherwise, the alias is for a device DMA quirk and we cannot
387 * assume that MSI uses the same requester ID. Therefore use the
390 if (PCI_BUS_NUM(data
.alias
) != data
.pdev
->bus
->number
)
391 set_irte_sid(irte
, SVT_VERIFY_BUS
, SQ_ALL_16
,
392 PCI_DEVID(PCI_BUS_NUM(data
.alias
),
394 else if (data
.pdev
->bus
->number
!= dev
->bus
->number
)
395 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
, data
.alias
);
397 set_irte_sid(irte
, SVT_VERIFY_SID_SQ
, SQ_ALL_16
,
398 PCI_DEVID(dev
->bus
->number
, dev
->devfn
));
403 static int iommu_load_old_irte(struct intel_iommu
*iommu
)
405 struct irte
*old_ir_table
;
406 phys_addr_t irt_phys
;
411 /* Check whether the old ir-table has the same size as ours */
412 irta
= dmar_readq(iommu
->reg
+ DMAR_IRTA_REG
);
413 if ((irta
& INTR_REMAP_TABLE_REG_SIZE_MASK
)
414 != INTR_REMAP_TABLE_REG_SIZE
)
417 irt_phys
= irta
& VTD_PAGE_MASK
;
418 size
= INTR_REMAP_TABLE_ENTRIES
*sizeof(struct irte
);
420 /* Map the old IR table */
421 old_ir_table
= memremap(irt_phys
, size
, MEMREMAP_WB
);
426 memcpy(iommu
->ir_table
->base
, old_ir_table
, size
);
428 __iommu_flush_cache(iommu
, iommu
->ir_table
->base
, size
);
431 * Now check the table for used entries and mark those as
432 * allocated in the bitmap
434 for (i
= 0; i
< INTR_REMAP_TABLE_ENTRIES
; i
++) {
435 if (iommu
->ir_table
->base
[i
].present
)
436 bitmap_set(iommu
->ir_table
->bitmap
, i
, 1);
439 memunmap(old_ir_table
);
445 static void iommu_set_irq_remapping(struct intel_iommu
*iommu
, int mode
)
451 addr
= virt_to_phys((void *)iommu
->ir_table
->base
);
453 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
455 dmar_writeq(iommu
->reg
+ DMAR_IRTA_REG
,
456 (addr
) | IR_X2APIC_MODE(mode
) | INTR_REMAP_TABLE_REG_SIZE
);
458 /* Set interrupt-remapping table pointer */
459 writel(iommu
->gcmd
| DMA_GCMD_SIRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
461 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
462 readl
, (sts
& DMA_GSTS_IRTPS
), sts
);
463 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
466 * Global invalidation of interrupt entry cache to make sure the
467 * hardware uses the new irq remapping table.
469 qi_global_iec(iommu
);
472 static void iommu_enable_irq_remapping(struct intel_iommu
*iommu
)
477 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
479 /* Enable interrupt-remapping */
480 iommu
->gcmd
|= DMA_GCMD_IRE
;
481 iommu
->gcmd
&= ~DMA_GCMD_CFI
; /* Block compatibility-format MSIs */
482 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
484 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
485 readl
, (sts
& DMA_GSTS_IRES
), sts
);
488 * With CFI clear in the Global Command register, we should be
489 * protected from dangerous (i.e. compatibility) interrupts
490 * regardless of x2apic status. Check just to be sure.
492 if (sts
& DMA_GSTS_CFIS
)
494 "Compatibility-format IRQs enabled despite intr remapping;\n"
495 "you are vulnerable to IRQ injection.\n");
497 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
500 static int intel_setup_irq_remapping(struct intel_iommu
*iommu
)
502 struct ir_table
*ir_table
;
504 unsigned long *bitmap
;
509 ir_table
= kzalloc(sizeof(struct ir_table
), GFP_KERNEL
);
513 pages
= alloc_pages_node(iommu
->node
, GFP_KERNEL
| __GFP_ZERO
,
514 INTR_REMAP_PAGE_ORDER
);
516 pr_err("IR%d: failed to allocate pages of order %d\n",
517 iommu
->seq_id
, INTR_REMAP_PAGE_ORDER
);
521 bitmap
= kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES
),
522 sizeof(long), GFP_ATOMIC
);
523 if (bitmap
== NULL
) {
524 pr_err("IR%d: failed to allocate bitmap\n", iommu
->seq_id
);
528 iommu
->ir_domain
= irq_domain_add_hierarchy(arch_get_ir_parent_domain(),
529 0, INTR_REMAP_TABLE_ENTRIES
,
530 NULL
, &intel_ir_domain_ops
,
532 if (!iommu
->ir_domain
) {
533 pr_err("IR%d: failed to allocate irqdomain\n", iommu
->seq_id
);
534 goto out_free_bitmap
;
536 iommu
->ir_msi_domain
= arch_create_msi_irq_domain(iommu
->ir_domain
);
538 ir_table
->base
= page_address(pages
);
539 ir_table
->bitmap
= bitmap
;
540 iommu
->ir_table
= ir_table
;
543 * If the queued invalidation is already initialized,
544 * shouldn't disable it.
548 * Clear previous faults.
550 dmar_fault(-1, iommu
);
551 dmar_disable_qi(iommu
);
553 if (dmar_enable_qi(iommu
)) {
554 pr_err("Failed to enable queued invalidation\n");
555 goto out_free_bitmap
;
559 init_ir_status(iommu
);
561 if (ir_pre_enabled(iommu
)) {
562 if (!is_kdump_kernel()) {
563 pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
565 clear_ir_pre_enabled(iommu
);
566 iommu_disable_irq_remapping(iommu
);
567 } else if (iommu_load_old_irte(iommu
))
568 pr_err("Failed to copy IR table for %s from previous kernel\n",
571 pr_info("Copied IR table for %s from previous kernel\n",
575 iommu_set_irq_remapping(iommu
, eim_mode
);
582 __free_pages(pages
, INTR_REMAP_PAGE_ORDER
);
586 iommu
->ir_table
= NULL
;
591 static void intel_teardown_irq_remapping(struct intel_iommu
*iommu
)
593 if (iommu
&& iommu
->ir_table
) {
594 if (iommu
->ir_msi_domain
) {
595 irq_domain_remove(iommu
->ir_msi_domain
);
596 iommu
->ir_msi_domain
= NULL
;
598 if (iommu
->ir_domain
) {
599 irq_domain_remove(iommu
->ir_domain
);
600 iommu
->ir_domain
= NULL
;
602 free_pages((unsigned long)iommu
->ir_table
->base
,
603 INTR_REMAP_PAGE_ORDER
);
604 kfree(iommu
->ir_table
->bitmap
);
605 kfree(iommu
->ir_table
);
606 iommu
->ir_table
= NULL
;
611 * Disable Interrupt Remapping.
613 static void iommu_disable_irq_remapping(struct intel_iommu
*iommu
)
618 if (!ecap_ir_support(iommu
->ecap
))
622 * global invalidation of interrupt entry cache before disabling
623 * interrupt-remapping.
625 qi_global_iec(iommu
);
627 raw_spin_lock_irqsave(&iommu
->register_lock
, flags
);
629 sts
= readl(iommu
->reg
+ DMAR_GSTS_REG
);
630 if (!(sts
& DMA_GSTS_IRES
))
633 iommu
->gcmd
&= ~DMA_GCMD_IRE
;
634 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
636 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
637 readl
, !(sts
& DMA_GSTS_IRES
), sts
);
640 raw_spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
643 static int __init
dmar_x2apic_optout(void)
645 struct acpi_table_dmar
*dmar
;
646 dmar
= (struct acpi_table_dmar
*)dmar_tbl
;
647 if (!dmar
|| no_x2apic_optout
)
649 return dmar
->flags
& DMAR_X2APIC_OPT_OUT
;
652 static void __init
intel_cleanup_irq_remapping(void)
654 struct dmar_drhd_unit
*drhd
;
655 struct intel_iommu
*iommu
;
657 for_each_iommu(iommu
, drhd
) {
658 if (ecap_ir_support(iommu
->ecap
)) {
659 iommu_disable_irq_remapping(iommu
);
660 intel_teardown_irq_remapping(iommu
);
664 if (x2apic_supported())
665 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
668 static int __init
intel_prepare_irq_remapping(void)
670 struct dmar_drhd_unit
*drhd
;
671 struct intel_iommu
*iommu
;
674 if (irq_remap_broken
) {
675 pr_warn("This system BIOS has enabled interrupt remapping\n"
676 "on a chipset that contains an erratum making that\n"
677 "feature unstable. To maintain system stability\n"
678 "interrupt remapping is being disabled. Please\n"
679 "contact your BIOS vendor for an update\n");
680 add_taint(TAINT_FIRMWARE_WORKAROUND
, LOCKDEP_STILL_OK
);
684 if (dmar_table_init() < 0)
687 if (!dmar_ir_support())
690 if (parse_ioapics_under_ir()) {
691 pr_info("Not enabling interrupt remapping\n");
695 /* First make sure all IOMMUs support IRQ remapping */
696 for_each_iommu(iommu
, drhd
)
697 if (!ecap_ir_support(iommu
->ecap
))
700 /* Detect remapping mode: lapic or x2apic */
701 if (x2apic_supported()) {
702 eim
= !dmar_x2apic_optout();
704 pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
705 pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
709 for_each_iommu(iommu
, drhd
) {
710 if (eim
&& !ecap_eim_support(iommu
->ecap
)) {
711 pr_info("%s does not support EIM\n", iommu
->name
);
718 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
720 /* Do the initializations early */
721 for_each_iommu(iommu
, drhd
) {
722 if (intel_setup_irq_remapping(iommu
)) {
723 pr_err("Failed to setup irq remapping for %s\n",
732 intel_cleanup_irq_remapping();
737 * Set Posted-Interrupts capability.
739 static inline void set_irq_posting_cap(void)
741 struct dmar_drhd_unit
*drhd
;
742 struct intel_iommu
*iommu
;
744 if (!disable_irq_post
) {
746 * If IRTE is in posted format, the 'pda' field goes across the
747 * 64-bit boundary, we need use cmpxchg16b to atomically update
748 * it. We only expose posted-interrupt when X86_FEATURE_CX16
749 * is supported. Actually, hardware platforms supporting PI
750 * should have X86_FEATURE_CX16 support, this has been confirmed
751 * with Intel hardware guys.
753 if (boot_cpu_has(X86_FEATURE_CX16
))
754 intel_irq_remap_ops
.capability
|= 1 << IRQ_POSTING_CAP
;
756 for_each_iommu(iommu
, drhd
)
757 if (!cap_pi_support(iommu
->cap
)) {
758 intel_irq_remap_ops
.capability
&=
759 ~(1 << IRQ_POSTING_CAP
);
765 static int __init
intel_enable_irq_remapping(void)
767 struct dmar_drhd_unit
*drhd
;
768 struct intel_iommu
*iommu
;
772 * Setup Interrupt-remapping for all the DRHD's now.
774 for_each_iommu(iommu
, drhd
) {
775 if (!ir_pre_enabled(iommu
))
776 iommu_enable_irq_remapping(iommu
);
783 irq_remapping_enabled
= 1;
785 set_irq_posting_cap();
787 pr_info("Enabled IRQ remapping in %s mode\n", eim_mode
? "x2apic" : "xapic");
789 return eim_mode
? IRQ_REMAP_X2APIC_MODE
: IRQ_REMAP_XAPIC_MODE
;
792 intel_cleanup_irq_remapping();
796 static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope
*scope
,
797 struct intel_iommu
*iommu
,
798 struct acpi_dmar_hardware_unit
*drhd
)
800 struct acpi_dmar_pci_path
*path
;
802 int count
, free
= -1;
805 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
806 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
807 / sizeof(struct acpi_dmar_pci_path
);
809 while (--count
> 0) {
811 * Access PCI directly due to the PCI
812 * subsystem isn't initialized yet.
814 bus
= read_pci_config_byte(bus
, path
->device
, path
->function
,
819 for (count
= 0; count
< MAX_HPET_TBS
; count
++) {
820 if (ir_hpet
[count
].iommu
== iommu
&&
821 ir_hpet
[count
].id
== scope
->enumeration_id
)
823 else if (ir_hpet
[count
].iommu
== NULL
&& free
== -1)
827 pr_warn("Exceeded Max HPET blocks\n");
831 ir_hpet
[free
].iommu
= iommu
;
832 ir_hpet
[free
].id
= scope
->enumeration_id
;
833 ir_hpet
[free
].bus
= bus
;
834 ir_hpet
[free
].devfn
= PCI_DEVFN(path
->device
, path
->function
);
835 pr_info("HPET id %d under DRHD base 0x%Lx\n",
836 scope
->enumeration_id
, drhd
->address
);
841 static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope
*scope
,
842 struct intel_iommu
*iommu
,
843 struct acpi_dmar_hardware_unit
*drhd
)
845 struct acpi_dmar_pci_path
*path
;
847 int count
, free
= -1;
850 path
= (struct acpi_dmar_pci_path
*)(scope
+ 1);
851 count
= (scope
->length
- sizeof(struct acpi_dmar_device_scope
))
852 / sizeof(struct acpi_dmar_pci_path
);
854 while (--count
> 0) {
856 * Access PCI directly due to the PCI
857 * subsystem isn't initialized yet.
859 bus
= read_pci_config_byte(bus
, path
->device
, path
->function
,
864 for (count
= 0; count
< MAX_IO_APICS
; count
++) {
865 if (ir_ioapic
[count
].iommu
== iommu
&&
866 ir_ioapic
[count
].id
== scope
->enumeration_id
)
868 else if (ir_ioapic
[count
].iommu
== NULL
&& free
== -1)
872 pr_warn("Exceeded Max IO APICS\n");
876 ir_ioapic
[free
].bus
= bus
;
877 ir_ioapic
[free
].devfn
= PCI_DEVFN(path
->device
, path
->function
);
878 ir_ioapic
[free
].iommu
= iommu
;
879 ir_ioapic
[free
].id
= scope
->enumeration_id
;
880 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n",
881 scope
->enumeration_id
, drhd
->address
, iommu
->seq_id
);
886 static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header
*header
,
887 struct intel_iommu
*iommu
)
890 struct acpi_dmar_hardware_unit
*drhd
;
891 struct acpi_dmar_device_scope
*scope
;
894 drhd
= (struct acpi_dmar_hardware_unit
*)header
;
895 start
= (void *)(drhd
+ 1);
896 end
= ((void *)drhd
) + header
->length
;
898 while (start
< end
&& ret
== 0) {
900 if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_IOAPIC
)
901 ret
= ir_parse_one_ioapic_scope(scope
, iommu
, drhd
);
902 else if (scope
->entry_type
== ACPI_DMAR_SCOPE_TYPE_HPET
)
903 ret
= ir_parse_one_hpet_scope(scope
, iommu
, drhd
);
904 start
+= scope
->length
;
910 static void ir_remove_ioapic_hpet_scope(struct intel_iommu
*iommu
)
914 for (i
= 0; i
< MAX_HPET_TBS
; i
++)
915 if (ir_hpet
[i
].iommu
== iommu
)
916 ir_hpet
[i
].iommu
= NULL
;
918 for (i
= 0; i
< MAX_IO_APICS
; i
++)
919 if (ir_ioapic
[i
].iommu
== iommu
)
920 ir_ioapic
[i
].iommu
= NULL
;
924 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
927 static int __init
parse_ioapics_under_ir(void)
929 struct dmar_drhd_unit
*drhd
;
930 struct intel_iommu
*iommu
;
931 bool ir_supported
= false;
934 for_each_iommu(iommu
, drhd
) {
937 if (!ecap_ir_support(iommu
->ecap
))
940 ret
= ir_parse_ioapic_hpet_scope(drhd
->hdr
, iommu
);
950 for (ioapic_idx
= 0; ioapic_idx
< nr_ioapics
; ioapic_idx
++) {
951 int ioapic_id
= mpc_ioapic_id(ioapic_idx
);
952 if (!map_ioapic_to_ir(ioapic_id
)) {
953 pr_err(FW_BUG
"ioapic %d has no mapping iommu, "
954 "interrupt remapping will be disabled\n",
963 static int __init
ir_dev_scope_init(void)
967 if (!irq_remapping_enabled
)
970 down_write(&dmar_global_lock
);
971 ret
= dmar_dev_scope_init();
972 up_write(&dmar_global_lock
);
976 rootfs_initcall(ir_dev_scope_init
);
978 static void disable_irq_remapping(void)
980 struct dmar_drhd_unit
*drhd
;
981 struct intel_iommu
*iommu
= NULL
;
984 * Disable Interrupt-remapping for all the DRHD's now.
986 for_each_iommu(iommu
, drhd
) {
987 if (!ecap_ir_support(iommu
->ecap
))
990 iommu_disable_irq_remapping(iommu
);
994 * Clear Posted-Interrupts capability.
996 if (!disable_irq_post
)
997 intel_irq_remap_ops
.capability
&= ~(1 << IRQ_POSTING_CAP
);
1000 static int reenable_irq_remapping(int eim
)
1002 struct dmar_drhd_unit
*drhd
;
1004 struct intel_iommu
*iommu
= NULL
;
1006 for_each_iommu(iommu
, drhd
)
1008 dmar_reenable_qi(iommu
);
1011 * Setup Interrupt-remapping for all the DRHD's now.
1013 for_each_iommu(iommu
, drhd
) {
1014 if (!ecap_ir_support(iommu
->ecap
))
1017 /* Set up interrupt remapping for iommu.*/
1018 iommu_set_irq_remapping(iommu
, eim
);
1019 iommu_enable_irq_remapping(iommu
);
1026 set_irq_posting_cap();
1032 * handle error condition gracefully here!
1037 static void prepare_irte(struct irte
*irte
, int vector
, unsigned int dest
)
1039 memset(irte
, 0, sizeof(*irte
));
1042 irte
->dst_mode
= apic
->irq_dest_mode
;
1044 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
1045 * actual level or edge trigger will be setup in the IO-APIC
1046 * RTE. This will help simplify level triggered irq migration.
1047 * For more details, see the comments (in io_apic.c) explainig IO-APIC
1048 * irq migration in the presence of interrupt-remapping.
1050 irte
->trigger_mode
= 0;
1051 irte
->dlvry_mode
= apic
->irq_delivery_mode
;
1052 irte
->vector
= vector
;
1053 irte
->dest_id
= IRTE_DEST(dest
);
1054 irte
->redir_hint
= 1;
1057 static struct irq_domain
*intel_get_ir_irq_domain(struct irq_alloc_info
*info
)
1059 struct intel_iommu
*iommu
= NULL
;
1064 switch (info
->type
) {
1065 case X86_IRQ_ALLOC_TYPE_IOAPIC
:
1066 iommu
= map_ioapic_to_ir(info
->ioapic_id
);
1068 case X86_IRQ_ALLOC_TYPE_HPET
:
1069 iommu
= map_hpet_to_ir(info
->hpet_id
);
1071 case X86_IRQ_ALLOC_TYPE_MSI
:
1072 case X86_IRQ_ALLOC_TYPE_MSIX
:
1073 iommu
= map_dev_to_ir(info
->msi_dev
);
1080 return iommu
? iommu
->ir_domain
: NULL
;
1083 static struct irq_domain
*intel_get_irq_domain(struct irq_alloc_info
*info
)
1085 struct intel_iommu
*iommu
;
1090 switch (info
->type
) {
1091 case X86_IRQ_ALLOC_TYPE_MSI
:
1092 case X86_IRQ_ALLOC_TYPE_MSIX
:
1093 iommu
= map_dev_to_ir(info
->msi_dev
);
1095 return iommu
->ir_msi_domain
;
1104 struct irq_remap_ops intel_irq_remap_ops
= {
1105 .prepare
= intel_prepare_irq_remapping
,
1106 .enable
= intel_enable_irq_remapping
,
1107 .disable
= disable_irq_remapping
,
1108 .reenable
= reenable_irq_remapping
,
1109 .enable_faulting
= enable_drhd_fault_handling
,
1110 .get_ir_irq_domain
= intel_get_ir_irq_domain
,
1111 .get_irq_domain
= intel_get_irq_domain
,
1115 * Migrate the IO-APIC irq in the presence of intr-remapping.
1117 * For both level and edge triggered, irq migration is a simple atomic
1118 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
1120 * For level triggered, we eliminate the io-apic RTE modification (with the
1121 * updated vector information), by using a virtual vector (io-apic pin number).
1122 * Real vector that is used for interrupting cpu will be coming from
1123 * the interrupt-remapping table entry.
1125 * As the migration is a simple atomic update of IRTE, the same mechanism
1126 * is used to migrate MSI irq's in the presence of interrupt-remapping.
1129 intel_ir_set_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
1132 struct intel_ir_data
*ir_data
= data
->chip_data
;
1133 struct irte
*irte
= &ir_data
->irte_entry
;
1134 struct irq_cfg
*cfg
= irqd_cfg(data
);
1135 struct irq_data
*parent
= data
->parent_data
;
1138 ret
= parent
->chip
->irq_set_affinity(parent
, mask
, force
);
1139 if (ret
< 0 || ret
== IRQ_SET_MASK_OK_DONE
)
1143 * Atomically updates the IRTE with the new destination, vector
1144 * and flushes the interrupt entry cache.
1146 irte
->vector
= cfg
->vector
;
1147 irte
->dest_id
= IRTE_DEST(cfg
->dest_apicid
);
1149 /* Update the hardware only if the interrupt is in remapped mode. */
1150 if (ir_data
->irq_2_iommu
.mode
== IRQ_REMAPPING
)
1151 modify_irte(&ir_data
->irq_2_iommu
, irte
);
1154 * After this point, all the interrupts will start arriving
1155 * at the new destination. So, time to cleanup the previous
1156 * vector allocation.
1158 send_cleanup_vector(cfg
);
1160 return IRQ_SET_MASK_OK_DONE
;
1163 static void intel_ir_compose_msi_msg(struct irq_data
*irq_data
,
1164 struct msi_msg
*msg
)
1166 struct intel_ir_data
*ir_data
= irq_data
->chip_data
;
1168 *msg
= ir_data
->msi_entry
;
1171 static int intel_ir_set_vcpu_affinity(struct irq_data
*data
, void *info
)
1173 struct intel_ir_data
*ir_data
= data
->chip_data
;
1174 struct vcpu_data
*vcpu_pi_info
= info
;
1176 /* stop posting interrupts, back to remapping mode */
1177 if (!vcpu_pi_info
) {
1178 modify_irte(&ir_data
->irq_2_iommu
, &ir_data
->irte_entry
);
1180 struct irte irte_pi
;
1183 * We are not caching the posted interrupt entry. We
1184 * copy the data from the remapped entry and modify
1185 * the fields which are relevant for posted mode. The
1186 * cached remapped entry is used for switching back to
1189 memset(&irte_pi
, 0, sizeof(irte_pi
));
1190 dmar_copy_shared_irte(&irte_pi
, &ir_data
->irte_entry
);
1192 /* Update the posted mode fields */
1194 irte_pi
.p_urgent
= 0;
1195 irte_pi
.p_vector
= vcpu_pi_info
->vector
;
1196 irte_pi
.pda_l
= (vcpu_pi_info
->pi_desc_addr
>>
1197 (32 - PDA_LOW_BIT
)) & ~(-1UL << PDA_LOW_BIT
);
1198 irte_pi
.pda_h
= (vcpu_pi_info
->pi_desc_addr
>> 32) &
1199 ~(-1UL << PDA_HIGH_BIT
);
1201 modify_irte(&ir_data
->irq_2_iommu
, &irte_pi
);
1207 static struct irq_chip intel_ir_chip
= {
1208 .irq_ack
= ir_ack_apic_edge
,
1209 .irq_set_affinity
= intel_ir_set_affinity
,
1210 .irq_compose_msi_msg
= intel_ir_compose_msi_msg
,
1211 .irq_set_vcpu_affinity
= intel_ir_set_vcpu_affinity
,
1214 static void intel_irq_remapping_prepare_irte(struct intel_ir_data
*data
,
1215 struct irq_cfg
*irq_cfg
,
1216 struct irq_alloc_info
*info
,
1217 int index
, int sub_handle
)
1219 struct IR_IO_APIC_route_entry
*entry
;
1220 struct irte
*irte
= &data
->irte_entry
;
1221 struct msi_msg
*msg
= &data
->msi_entry
;
1223 prepare_irte(irte
, irq_cfg
->vector
, irq_cfg
->dest_apicid
);
1224 switch (info
->type
) {
1225 case X86_IRQ_ALLOC_TYPE_IOAPIC
:
1226 /* Set source-id of interrupt request */
1227 set_ioapic_sid(irte
, info
->ioapic_id
);
1228 apic_printk(APIC_VERBOSE
, KERN_DEBUG
"IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n",
1229 info
->ioapic_id
, irte
->present
, irte
->fpd
,
1230 irte
->dst_mode
, irte
->redir_hint
,
1231 irte
->trigger_mode
, irte
->dlvry_mode
,
1232 irte
->avail
, irte
->vector
, irte
->dest_id
,
1233 irte
->sid
, irte
->sq
, irte
->svt
);
1235 entry
= (struct IR_IO_APIC_route_entry
*)info
->ioapic_entry
;
1236 info
->ioapic_entry
= NULL
;
1237 memset(entry
, 0, sizeof(*entry
));
1238 entry
->index2
= (index
>> 15) & 0x1;
1241 entry
->index
= (index
& 0x7fff);
1243 * IO-APIC RTE will be configured with virtual vector.
1244 * irq handler will do the explicit EOI to the io-apic.
1246 entry
->vector
= info
->ioapic_pin
;
1247 entry
->mask
= 0; /* enable IRQ */
1248 entry
->trigger
= info
->ioapic_trigger
;
1249 entry
->polarity
= info
->ioapic_polarity
;
1250 if (info
->ioapic_trigger
)
1251 entry
->mask
= 1; /* Mask level triggered irqs. */
1254 case X86_IRQ_ALLOC_TYPE_HPET
:
1255 case X86_IRQ_ALLOC_TYPE_MSI
:
1256 case X86_IRQ_ALLOC_TYPE_MSIX
:
1257 if (info
->type
== X86_IRQ_ALLOC_TYPE_HPET
)
1258 set_hpet_sid(irte
, info
->hpet_id
);
1260 set_msi_sid(irte
, info
->msi_dev
);
1262 msg
->address_hi
= MSI_ADDR_BASE_HI
;
1263 msg
->data
= sub_handle
;
1264 msg
->address_lo
= MSI_ADDR_BASE_LO
| MSI_ADDR_IR_EXT_INT
|
1266 MSI_ADDR_IR_INDEX1(index
) |
1267 MSI_ADDR_IR_INDEX2(index
);
1276 static void intel_free_irq_resources(struct irq_domain
*domain
,
1277 unsigned int virq
, unsigned int nr_irqs
)
1279 struct irq_data
*irq_data
;
1280 struct intel_ir_data
*data
;
1281 struct irq_2_iommu
*irq_iommu
;
1282 unsigned long flags
;
1284 for (i
= 0; i
< nr_irqs
; i
++) {
1285 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
1286 if (irq_data
&& irq_data
->chip_data
) {
1287 data
= irq_data
->chip_data
;
1288 irq_iommu
= &data
->irq_2_iommu
;
1289 raw_spin_lock_irqsave(&irq_2_ir_lock
, flags
);
1290 clear_entries(irq_iommu
);
1291 raw_spin_unlock_irqrestore(&irq_2_ir_lock
, flags
);
1292 irq_domain_reset_irq_data(irq_data
);
1298 static int intel_irq_remapping_alloc(struct irq_domain
*domain
,
1299 unsigned int virq
, unsigned int nr_irqs
,
1302 struct intel_iommu
*iommu
= domain
->host_data
;
1303 struct irq_alloc_info
*info
= arg
;
1304 struct intel_ir_data
*data
, *ird
;
1305 struct irq_data
*irq_data
;
1306 struct irq_cfg
*irq_cfg
;
1309 if (!info
|| !iommu
)
1311 if (nr_irqs
> 1 && info
->type
!= X86_IRQ_ALLOC_TYPE_MSI
&&
1312 info
->type
!= X86_IRQ_ALLOC_TYPE_MSIX
)
1316 * With IRQ remapping enabled, don't need contiguous CPU vectors
1317 * to support multiple MSI interrupts.
1319 if (info
->type
== X86_IRQ_ALLOC_TYPE_MSI
)
1320 info
->flags
&= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS
;
1322 ret
= irq_domain_alloc_irqs_parent(domain
, virq
, nr_irqs
, arg
);
1327 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
1329 goto out_free_parent
;
1331 down_read(&dmar_global_lock
);
1332 index
= alloc_irte(iommu
, virq
, &data
->irq_2_iommu
, nr_irqs
);
1333 up_read(&dmar_global_lock
);
1335 pr_warn("Failed to allocate IRTE\n");
1337 goto out_free_parent
;
1340 for (i
= 0; i
< nr_irqs
; i
++) {
1341 irq_data
= irq_domain_get_irq_data(domain
, virq
+ i
);
1342 irq_cfg
= irqd_cfg(irq_data
);
1343 if (!irq_data
|| !irq_cfg
) {
1349 ird
= kzalloc(sizeof(*ird
), GFP_KERNEL
);
1352 /* Initialize the common data */
1353 ird
->irq_2_iommu
= data
->irq_2_iommu
;
1354 ird
->irq_2_iommu
.sub_handle
= i
;
1359 irq_data
->hwirq
= (index
<< 16) + i
;
1360 irq_data
->chip_data
= ird
;
1361 irq_data
->chip
= &intel_ir_chip
;
1362 intel_irq_remapping_prepare_irte(ird
, irq_cfg
, info
, index
, i
);
1363 irq_set_status_flags(virq
+ i
, IRQ_MOVE_PCNTXT
);
1368 intel_free_irq_resources(domain
, virq
, i
);
1370 irq_domain_free_irqs_common(domain
, virq
, nr_irqs
);
1374 static void intel_irq_remapping_free(struct irq_domain
*domain
,
1375 unsigned int virq
, unsigned int nr_irqs
)
1377 intel_free_irq_resources(domain
, virq
, nr_irqs
);
1378 irq_domain_free_irqs_common(domain
, virq
, nr_irqs
);
1381 static void intel_irq_remapping_activate(struct irq_domain
*domain
,
1382 struct irq_data
*irq_data
)
1384 struct intel_ir_data
*data
= irq_data
->chip_data
;
1386 modify_irte(&data
->irq_2_iommu
, &data
->irte_entry
);
1389 static void intel_irq_remapping_deactivate(struct irq_domain
*domain
,
1390 struct irq_data
*irq_data
)
1392 struct intel_ir_data
*data
= irq_data
->chip_data
;
1395 memset(&entry
, 0, sizeof(entry
));
1396 modify_irte(&data
->irq_2_iommu
, &entry
);
1399 static struct irq_domain_ops intel_ir_domain_ops
= {
1400 .alloc
= intel_irq_remapping_alloc
,
1401 .free
= intel_irq_remapping_free
,
1402 .activate
= intel_irq_remapping_activate
,
1403 .deactivate
= intel_irq_remapping_deactivate
,
1407 * Support of Interrupt Remapping Unit Hotplug
1409 static int dmar_ir_add(struct dmar_drhd_unit
*dmaru
, struct intel_iommu
*iommu
)
1412 int eim
= x2apic_enabled();
1414 if (eim
&& !ecap_eim_support(iommu
->ecap
)) {
1415 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n",
1416 iommu
->reg_phys
, iommu
->ecap
);
1420 if (ir_parse_ioapic_hpet_scope(dmaru
->hdr
, iommu
)) {
1421 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n",
1426 /* TODO: check all IOAPICs are covered by IOMMU */
1428 /* Setup Interrupt-remapping now. */
1429 ret
= intel_setup_irq_remapping(iommu
);
1431 pr_err("Failed to setup irq remapping for %s\n",
1433 intel_teardown_irq_remapping(iommu
);
1434 ir_remove_ioapic_hpet_scope(iommu
);
1436 iommu_enable_irq_remapping(iommu
);
1442 int dmar_ir_hotplug(struct dmar_drhd_unit
*dmaru
, bool insert
)
1445 struct intel_iommu
*iommu
= dmaru
->iommu
;
1447 if (!irq_remapping_enabled
)
1451 if (!ecap_ir_support(iommu
->ecap
))
1453 if (irq_remapping_cap(IRQ_POSTING_CAP
) &&
1454 !cap_pi_support(iommu
->cap
))
1458 if (!iommu
->ir_table
)
1459 ret
= dmar_ir_add(dmaru
, iommu
);
1461 if (iommu
->ir_table
) {
1462 if (!bitmap_empty(iommu
->ir_table
->bitmap
,
1463 INTR_REMAP_TABLE_ENTRIES
)) {
1466 iommu_disable_irq_remapping(iommu
);
1467 intel_teardown_irq_remapping(iommu
);
1468 ir_remove_ioapic_hpet_scope(iommu
);