3 * Purpose: PCI Message Signaled Interrupt (MSI)
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
11 #include <linux/irq.h>
12 #include <linux/interrupt.h>
13 #include <linux/init.h>
14 #include <linux/ioport.h>
15 #include <linux/smp_lock.h>
16 #include <linux/pci.h>
17 #include <linux/proc_fs.h>
19 #include <asm/errno.h>
26 static DEFINE_SPINLOCK(msi_lock
);
27 static struct msi_desc
* msi_desc
[NR_IRQS
] = { [0 ... NR_IRQS
-1] = NULL
};
28 static kmem_cache_t
* msi_cachep
;
30 static int pci_msi_enable
= 1;
32 static struct msi_ops
*msi_ops
;
35 msi_register(struct msi_ops
*ops
)
41 static int msi_cache_init(void)
43 msi_cachep
= kmem_cache_create("msi_cache", sizeof(struct msi_desc
),
44 0, SLAB_HWCACHE_ALIGN
, NULL
, NULL
);
51 static void msi_set_mask_bit(unsigned int irq
, int flag
)
53 struct msi_desc
*entry
;
55 entry
= msi_desc
[irq
];
56 BUG_ON(!entry
|| !entry
->dev
);
57 switch (entry
->msi_attrib
.type
) {
59 if (entry
->msi_attrib
.maskbit
) {
63 pos
= (long)entry
->mask_base
;
64 pci_read_config_dword(entry
->dev
, pos
, &mask_bits
);
67 pci_write_config_dword(entry
->dev
, pos
, mask_bits
);
72 int offset
= entry
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
+
73 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET
;
74 writel(flag
, entry
->mask_base
+ offset
);
83 static void read_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
85 switch(entry
->msi_attrib
.type
) {
88 struct pci_dev
*dev
= entry
->dev
;
89 int pos
= entry
->msi_attrib
.pos
;
92 pci_read_config_dword(dev
, msi_lower_address_reg(pos
),
94 if (entry
->msi_attrib
.is_64
) {
95 pci_read_config_dword(dev
, msi_upper_address_reg(pos
),
97 pci_read_config_word(dev
, msi_data_reg(pos
, 1), &data
);
100 pci_read_config_word(dev
, msi_data_reg(pos
, 1), &data
);
105 case PCI_CAP_ID_MSIX
:
108 base
= entry
->mask_base
+
109 entry
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
;
111 msg
->address_lo
= readl(base
+ PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET
);
112 msg
->address_hi
= readl(base
+ PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET
);
113 msg
->data
= readl(base
+ PCI_MSIX_ENTRY_DATA_OFFSET
);
121 static void write_msi_msg(struct msi_desc
*entry
, struct msi_msg
*msg
)
123 switch (entry
->msi_attrib
.type
) {
126 struct pci_dev
*dev
= entry
->dev
;
127 int pos
= entry
->msi_attrib
.pos
;
129 pci_write_config_dword(dev
, msi_lower_address_reg(pos
),
131 if (entry
->msi_attrib
.is_64
) {
132 pci_write_config_dword(dev
, msi_upper_address_reg(pos
),
134 pci_write_config_word(dev
, msi_data_reg(pos
, 1),
137 pci_write_config_word(dev
, msi_data_reg(pos
, 0),
142 case PCI_CAP_ID_MSIX
:
145 base
= entry
->mask_base
+
146 entry
->msi_attrib
.entry_nr
* PCI_MSIX_ENTRY_SIZE
;
148 writel(msg
->address_lo
,
149 base
+ PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET
);
150 writel(msg
->address_hi
,
151 base
+ PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET
);
152 writel(msg
->data
, base
+ PCI_MSIX_ENTRY_DATA_OFFSET
);
161 static void set_msi_affinity(unsigned int irq
, cpumask_t cpu_mask
)
163 struct msi_desc
*entry
;
166 entry
= msi_desc
[irq
];
167 if (!entry
|| !entry
->dev
)
170 read_msi_msg(entry
, &msg
);
171 msi_ops
->target(irq
, cpu_mask
, &msg
);
172 write_msi_msg(entry
, &msg
);
173 set_native_irq_info(irq
, cpu_mask
);
176 #define set_msi_affinity NULL
177 #endif /* CONFIG_SMP */
179 static void mask_MSI_irq(unsigned int irq
)
181 msi_set_mask_bit(irq
, 1);
184 static void unmask_MSI_irq(unsigned int irq
)
186 msi_set_mask_bit(irq
, 0);
189 static void ack_msi_irq(unsigned int irq
)
191 move_native_irq(irq
);
196 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
197 * which implement the MSI or MSI-X Capability Structure.
199 static struct irq_chip msi_chip
= {
201 .unmask
= unmask_MSI_irq
,
202 .mask
= mask_MSI_irq
,
204 .set_affinity
= set_msi_affinity
207 static int msi_free_irq(struct pci_dev
* dev
, int irq
);
208 static int msi_init(void)
210 static int status
= -ENOMEM
;
217 printk(KERN_WARNING
"PCI: MSI quirk detected. MSI disabled.\n");
222 status
= msi_arch_init();
226 "PCI: MSI arch init failed. MSI disabled.\n");
233 "PCI: MSI ops not registered. MSI disabled.\n");
238 status
= msi_cache_init();
241 printk(KERN_WARNING
"PCI: MSI cache init failed\n");
248 static struct msi_desc
* alloc_msi_entry(void)
250 struct msi_desc
*entry
;
252 entry
= kmem_cache_zalloc(msi_cachep
, GFP_KERNEL
);
256 entry
->link
.tail
= entry
->link
.head
= 0; /* single message */
262 static void attach_msi_entry(struct msi_desc
*entry
, int irq
)
266 spin_lock_irqsave(&msi_lock
, flags
);
267 msi_desc
[irq
] = entry
;
268 spin_unlock_irqrestore(&msi_lock
, flags
);
271 static int create_msi_irq(struct irq_chip
*chip
)
273 struct msi_desc
*entry
;
276 entry
= alloc_msi_entry();
282 kmem_cache_free(msi_cachep
, entry
);
286 set_irq_chip_and_handler(irq
, chip
, handle_edge_irq
);
287 set_irq_data(irq
, entry
);
292 static void destroy_msi_irq(unsigned int irq
)
294 struct msi_desc
*entry
;
296 entry
= get_irq_data(irq
);
297 set_irq_chip(irq
, NULL
);
298 set_irq_data(irq
, NULL
);
300 kmem_cache_free(msi_cachep
, entry
);
303 static void enable_msi_mode(struct pci_dev
*dev
, int pos
, int type
)
307 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
308 if (type
== PCI_CAP_ID_MSI
) {
309 /* Set enabled bits to single MSI & enable MSI_enable bit */
310 msi_enable(control
, 1);
311 pci_write_config_word(dev
, msi_control_reg(pos
), control
);
312 dev
->msi_enabled
= 1;
314 msix_enable(control
);
315 pci_write_config_word(dev
, msi_control_reg(pos
), control
);
316 dev
->msix_enabled
= 1;
318 if (pci_find_capability(dev
, PCI_CAP_ID_EXP
)) {
319 /* PCI Express Endpoint device detected */
320 pci_intx(dev
, 0); /* disable intx */
324 void disable_msi_mode(struct pci_dev
*dev
, int pos
, int type
)
328 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
329 if (type
== PCI_CAP_ID_MSI
) {
330 /* Set enabled bits to single MSI & enable MSI_enable bit */
331 msi_disable(control
);
332 pci_write_config_word(dev
, msi_control_reg(pos
), control
);
333 dev
->msi_enabled
= 0;
335 msix_disable(control
);
336 pci_write_config_word(dev
, msi_control_reg(pos
), control
);
337 dev
->msix_enabled
= 0;
339 if (pci_find_capability(dev
, PCI_CAP_ID_EXP
)) {
340 /* PCI Express Endpoint device detected */
341 pci_intx(dev
, 1); /* enable intx */
345 static int msi_lookup_irq(struct pci_dev
*dev
, int type
)
350 spin_lock_irqsave(&msi_lock
, flags
);
351 for (irq
= 0; irq
< NR_IRQS
; irq
++) {
352 if (!msi_desc
[irq
] || msi_desc
[irq
]->dev
!= dev
||
353 msi_desc
[irq
]->msi_attrib
.type
!= type
||
354 msi_desc
[irq
]->msi_attrib
.default_irq
!= dev
->irq
)
356 spin_unlock_irqrestore(&msi_lock
, flags
);
357 /* This pre-assigned MSI irq for this device
358 already exits. Override dev->irq with this irq */
362 spin_unlock_irqrestore(&msi_lock
, flags
);
367 void pci_scan_msi_device(struct pci_dev
*dev
)
374 int pci_save_msi_state(struct pci_dev
*dev
)
378 struct pci_cap_saved_state
*save_state
;
381 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
382 if (pos
<= 0 || dev
->no_msi
)
385 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
386 if (!(control
& PCI_MSI_FLAGS_ENABLE
))
389 save_state
= kzalloc(sizeof(struct pci_cap_saved_state
) + sizeof(u32
) * 5,
392 printk(KERN_ERR
"Out of memory in pci_save_msi_state\n");
395 cap
= &save_state
->data
[0];
397 pci_read_config_dword(dev
, pos
, &cap
[i
++]);
398 control
= cap
[0] >> 16;
399 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
, &cap
[i
++]);
400 if (control
& PCI_MSI_FLAGS_64BIT
) {
401 pci_read_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
, &cap
[i
++]);
402 pci_read_config_dword(dev
, pos
+ PCI_MSI_DATA_64
, &cap
[i
++]);
404 pci_read_config_dword(dev
, pos
+ PCI_MSI_DATA_32
, &cap
[i
++]);
405 if (control
& PCI_MSI_FLAGS_MASKBIT
)
406 pci_read_config_dword(dev
, pos
+ PCI_MSI_MASK_BIT
, &cap
[i
++]);
407 save_state
->cap_nr
= PCI_CAP_ID_MSI
;
408 pci_add_saved_cap(dev
, save_state
);
412 void pci_restore_msi_state(struct pci_dev
*dev
)
416 struct pci_cap_saved_state
*save_state
;
419 save_state
= pci_find_saved_cap(dev
, PCI_CAP_ID_MSI
);
420 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
421 if (!save_state
|| pos
<= 0)
423 cap
= &save_state
->data
[0];
425 control
= cap
[i
++] >> 16;
426 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_LO
, cap
[i
++]);
427 if (control
& PCI_MSI_FLAGS_64BIT
) {
428 pci_write_config_dword(dev
, pos
+ PCI_MSI_ADDRESS_HI
, cap
[i
++]);
429 pci_write_config_dword(dev
, pos
+ PCI_MSI_DATA_64
, cap
[i
++]);
431 pci_write_config_dword(dev
, pos
+ PCI_MSI_DATA_32
, cap
[i
++]);
432 if (control
& PCI_MSI_FLAGS_MASKBIT
)
433 pci_write_config_dword(dev
, pos
+ PCI_MSI_MASK_BIT
, cap
[i
++]);
434 pci_write_config_word(dev
, pos
+ PCI_MSI_FLAGS
, control
);
435 enable_msi_mode(dev
, pos
, PCI_CAP_ID_MSI
);
436 pci_remove_saved_cap(save_state
);
440 int pci_save_msix_state(struct pci_dev
*dev
)
444 int irq
, head
, tail
= 0;
446 struct pci_cap_saved_state
*save_state
;
448 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
449 if (pos
<= 0 || dev
->no_msi
)
452 /* save the capability */
453 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
454 if (!(control
& PCI_MSIX_FLAGS_ENABLE
))
456 save_state
= kzalloc(sizeof(struct pci_cap_saved_state
) + sizeof(u16
),
459 printk(KERN_ERR
"Out of memory in pci_save_msix_state\n");
462 *((u16
*)&save_state
->data
[0]) = control
;
466 if (msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
)) {
471 irq
= head
= dev
->irq
;
472 while (head
!= tail
) {
473 struct msi_desc
*entry
;
475 entry
= msi_desc
[irq
];
476 read_msi_msg(entry
, &entry
->msg_save
);
478 tail
= msi_desc
[irq
]->link
.tail
;
483 save_state
->cap_nr
= PCI_CAP_ID_MSIX
;
484 pci_add_saved_cap(dev
, save_state
);
488 void pci_restore_msix_state(struct pci_dev
*dev
)
492 int irq
, head
, tail
= 0;
493 struct msi_desc
*entry
;
495 struct pci_cap_saved_state
*save_state
;
497 save_state
= pci_find_saved_cap(dev
, PCI_CAP_ID_MSIX
);
500 save
= *((u16
*)&save_state
->data
[0]);
501 pci_remove_saved_cap(save_state
);
504 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
508 /* route the table */
510 if (msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
))
512 irq
= head
= dev
->irq
;
513 while (head
!= tail
) {
514 entry
= msi_desc
[irq
];
515 write_msi_msg(entry
, &entry
->msg_save
);
517 tail
= msi_desc
[irq
]->link
.tail
;
522 pci_write_config_word(dev
, msi_control_reg(pos
), save
);
523 enable_msi_mode(dev
, pos
, PCI_CAP_ID_MSIX
);
527 static int msi_register_init(struct pci_dev
*dev
, struct msi_desc
*entry
)
534 pos
= entry
->msi_attrib
.pos
;
535 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
537 /* Configure MSI capability structure */
538 status
= msi_ops
->setup(dev
, dev
->irq
, &msg
);
542 write_msi_msg(entry
, &msg
);
543 if (entry
->msi_attrib
.maskbit
) {
544 unsigned int maskbits
, temp
;
545 /* All MSIs are unmasked by default, Mask them all */
546 pci_read_config_dword(dev
,
547 msi_mask_bits_reg(pos
, is_64bit_address(control
)),
549 temp
= (1 << multi_msi_capable(control
));
550 temp
= ((temp
- 1) & ~temp
);
552 pci_write_config_dword(dev
,
553 msi_mask_bits_reg(pos
, is_64bit_address(control
)),
561 * msi_capability_init - configure device's MSI capability structure
562 * @dev: pointer to the pci_dev data structure of MSI device function
564 * Setup the MSI capability structure of device function with a single
565 * MSI irq, regardless of device function is capable of handling
566 * multiple messages. A return of zero indicates the successful setup
567 * of an entry zero with the new MSI irq or non-zero for otherwise.
569 static int msi_capability_init(struct pci_dev
*dev
)
572 struct msi_desc
*entry
;
576 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
577 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
578 /* MSI Entry Initialization */
579 irq
= create_msi_irq(&msi_chip
);
583 entry
= get_irq_data(irq
);
584 entry
->link
.head
= irq
;
585 entry
->link
.tail
= irq
;
586 entry
->msi_attrib
.type
= PCI_CAP_ID_MSI
;
587 entry
->msi_attrib
.is_64
= is_64bit_address(control
);
588 entry
->msi_attrib
.entry_nr
= 0;
589 entry
->msi_attrib
.maskbit
= is_mask_bit_support(control
);
590 entry
->msi_attrib
.default_irq
= dev
->irq
; /* Save IOAPIC IRQ */
591 entry
->msi_attrib
.pos
= pos
;
594 if (is_mask_bit_support(control
)) {
595 entry
->mask_base
= (void __iomem
*)(long)msi_mask_bits_reg(pos
,
596 is_64bit_address(control
));
598 /* Configure MSI capability structure */
599 status
= msi_register_init(dev
, entry
);
601 dev
->irq
= entry
->msi_attrib
.default_irq
;
602 destroy_msi_irq(irq
);
606 attach_msi_entry(entry
, irq
);
607 /* Set MSI enabled bits */
608 enable_msi_mode(dev
, pos
, PCI_CAP_ID_MSI
);
614 * msix_capability_init - configure device's MSI-X capability
615 * @dev: pointer to the pci_dev data structure of MSI-X device function
616 * @entries: pointer to an array of struct msix_entry entries
617 * @nvec: number of @entries
619 * Setup the MSI-X capability structure of device function with a
620 * single MSI-X irq. A return of zero indicates the successful setup of
621 * requested MSI-X entries with allocated irqs or non-zero for otherwise.
623 static int msix_capability_init(struct pci_dev
*dev
,
624 struct msix_entry
*entries
, int nvec
)
626 struct msi_desc
*head
= NULL
, *tail
= NULL
, *entry
= NULL
;
629 int irq
, pos
, i
, j
, nr_entries
, temp
= 0;
630 unsigned long phys_addr
;
636 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
637 /* Request & Map MSI-X table region */
638 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
639 nr_entries
= multi_msix_capable(control
);
641 pci_read_config_dword(dev
, msix_table_offset_reg(pos
), &table_offset
);
642 bir
= (u8
)(table_offset
& PCI_MSIX_FLAGS_BIRMASK
);
643 table_offset
&= ~PCI_MSIX_FLAGS_BIRMASK
;
644 phys_addr
= pci_resource_start (dev
, bir
) + table_offset
;
645 base
= ioremap_nocache(phys_addr
, nr_entries
* PCI_MSIX_ENTRY_SIZE
);
649 /* MSI-X Table Initialization */
650 for (i
= 0; i
< nvec
; i
++) {
651 irq
= create_msi_irq(&msi_chip
);
655 entry
= get_irq_data(irq
);
656 j
= entries
[i
].entry
;
657 entries
[i
].vector
= irq
;
658 entry
->msi_attrib
.type
= PCI_CAP_ID_MSIX
;
659 entry
->msi_attrib
.is_64
= 1;
660 entry
->msi_attrib
.entry_nr
= j
;
661 entry
->msi_attrib
.maskbit
= 1;
662 entry
->msi_attrib
.default_irq
= dev
->irq
;
663 entry
->msi_attrib
.pos
= pos
;
665 entry
->mask_base
= base
;
667 entry
->link
.head
= irq
;
668 entry
->link
.tail
= irq
;
671 entry
->link
.head
= temp
;
672 entry
->link
.tail
= tail
->link
.tail
;
673 tail
->link
.tail
= irq
;
674 head
->link
.head
= irq
;
678 /* Configure MSI-X capability structure */
679 status
= msi_ops
->setup(dev
, irq
, &msg
);
681 destroy_msi_irq(irq
);
685 write_msi_msg(entry
, &msg
);
686 attach_msi_entry(entry
, irq
);
691 for (; i
>= 0; i
--) {
692 irq
= (entries
+ i
)->vector
;
693 msi_free_irq(dev
, irq
);
694 (entries
+ i
)->vector
= 0;
696 /* If we had some success report the number of irqs
697 * we succeeded in setting up.
703 /* Set MSI-X enabled bits */
704 enable_msi_mode(dev
, pos
, PCI_CAP_ID_MSIX
);
710 * pci_msi_supported - check whether MSI may be enabled on device
711 * @dev: pointer to the pci_dev data structure of MSI device function
713 * MSI must be globally enabled and supported by the device and its root
714 * bus. But, the root bus is not easy to find since some architectures
715 * have virtual busses on top of the PCI hierarchy (for instance the
716 * hypertransport bus), while the actual bus where MSI must be supported
717 * is below. So we test the MSI flag on all parent busses and assume
718 * that no quirk will ever set the NO_MSI flag on a non-root bus.
721 int pci_msi_supported(struct pci_dev
* dev
)
725 if (!pci_msi_enable
|| !dev
|| dev
->no_msi
)
728 /* check MSI flags of all parent busses */
729 for (bus
= dev
->bus
; bus
; bus
= bus
->parent
)
730 if (bus
->bus_flags
& PCI_BUS_FLAGS_NO_MSI
)
737 * pci_enable_msi - configure device's MSI capability structure
738 * @dev: pointer to the pci_dev data structure of MSI device function
740 * Setup the MSI capability structure of device function with
741 * a single MSI irq upon its software driver call to request for
742 * MSI mode enabled on its hardware device function. A return of zero
743 * indicates the successful setup of an entry zero with the new MSI
744 * irq or non-zero for otherwise.
746 int pci_enable_msi(struct pci_dev
* dev
)
748 int pos
, temp
, status
;
751 if (pci_msi_supported(dev
) < 0)
760 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
764 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
765 if (!is_64bit_address(control
) && msi_ops
->needs_64bit_address
)
768 WARN_ON(!msi_lookup_irq(dev
, PCI_CAP_ID_MSI
));
770 /* Check whether driver already requested for MSI-X irqs */
771 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
772 if (pos
> 0 && !msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
)) {
773 printk(KERN_INFO
"PCI: %s: Can't enable MSI. "
774 "Device already has MSI-X irq assigned\n",
779 status
= msi_capability_init(dev
);
783 void pci_disable_msi(struct pci_dev
* dev
)
785 struct msi_desc
*entry
;
786 int pos
, default_irq
;
795 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
799 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
800 if (!(control
& PCI_MSI_FLAGS_ENABLE
))
803 disable_msi_mode(dev
, pos
, PCI_CAP_ID_MSI
);
805 spin_lock_irqsave(&msi_lock
, flags
);
806 entry
= msi_desc
[dev
->irq
];
807 if (!entry
|| !entry
->dev
|| entry
->msi_attrib
.type
!= PCI_CAP_ID_MSI
) {
808 spin_unlock_irqrestore(&msi_lock
, flags
);
811 if (irq_has_action(dev
->irq
)) {
812 spin_unlock_irqrestore(&msi_lock
, flags
);
813 printk(KERN_WARNING
"PCI: %s: pci_disable_msi() called without "
814 "free_irq() on MSI irq %d\n",
815 pci_name(dev
), dev
->irq
);
816 BUG_ON(irq_has_action(dev
->irq
));
818 default_irq
= entry
->msi_attrib
.default_irq
;
819 spin_unlock_irqrestore(&msi_lock
, flags
);
820 msi_free_irq(dev
, dev
->irq
);
822 /* Restore dev->irq to its default pin-assertion irq */
823 dev
->irq
= default_irq
;
827 static int msi_free_irq(struct pci_dev
* dev
, int irq
)
829 struct msi_desc
*entry
;
830 int head
, entry_nr
, type
;
834 msi_ops
->teardown(irq
);
836 spin_lock_irqsave(&msi_lock
, flags
);
837 entry
= msi_desc
[irq
];
838 if (!entry
|| entry
->dev
!= dev
) {
839 spin_unlock_irqrestore(&msi_lock
, flags
);
842 type
= entry
->msi_attrib
.type
;
843 entry_nr
= entry
->msi_attrib
.entry_nr
;
844 head
= entry
->link
.head
;
845 base
= entry
->mask_base
;
846 msi_desc
[entry
->link
.head
]->link
.tail
= entry
->link
.tail
;
847 msi_desc
[entry
->link
.tail
]->link
.head
= entry
->link
.head
;
849 msi_desc
[irq
] = NULL
;
850 spin_unlock_irqrestore(&msi_lock
, flags
);
852 destroy_msi_irq(irq
);
854 if (type
== PCI_CAP_ID_MSIX
) {
855 writel(1, base
+ entry_nr
* PCI_MSIX_ENTRY_SIZE
+
856 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET
);
866 * pci_enable_msix - configure device's MSI-X capability structure
867 * @dev: pointer to the pci_dev data structure of MSI-X device function
868 * @entries: pointer to an array of MSI-X entries
869 * @nvec: number of MSI-X irqs requested for allocation by device driver
871 * Setup the MSI-X capability structure of device function with the number
872 * of requested irqs upon its software driver call to request for
873 * MSI-X mode enabled on its hardware device function. A return of zero
874 * indicates the successful configuration of MSI-X capability structure
875 * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
876 * Or a return of > 0 indicates that driver request is exceeding the number
877 * of irqs available. Driver should use the returned value to re-send
880 int pci_enable_msix(struct pci_dev
* dev
, struct msix_entry
*entries
, int nvec
)
882 int status
, pos
, nr_entries
;
886 if (!entries
|| pci_msi_supported(dev
) < 0)
893 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
897 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
898 nr_entries
= multi_msix_capable(control
);
899 if (nvec
> nr_entries
)
902 /* Check for any invalid entries */
903 for (i
= 0; i
< nvec
; i
++) {
904 if (entries
[i
].entry
>= nr_entries
)
905 return -EINVAL
; /* invalid entry */
906 for (j
= i
+ 1; j
< nvec
; j
++) {
907 if (entries
[i
].entry
== entries
[j
].entry
)
908 return -EINVAL
; /* duplicate entry */
912 WARN_ON(!msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
));
914 /* Check whether driver already requested for MSI irq */
915 if (pci_find_capability(dev
, PCI_CAP_ID_MSI
) > 0 &&
916 !msi_lookup_irq(dev
, PCI_CAP_ID_MSI
)) {
917 printk(KERN_INFO
"PCI: %s: Can't enable MSI-X. "
918 "Device already has an MSI irq assigned\n",
923 status
= msix_capability_init(dev
, entries
, nvec
);
927 void pci_disable_msix(struct pci_dev
* dev
)
937 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
941 pci_read_config_word(dev
, msi_control_reg(pos
), &control
);
942 if (!(control
& PCI_MSIX_FLAGS_ENABLE
))
945 disable_msi_mode(dev
, pos
, PCI_CAP_ID_MSIX
);
948 if (!msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
)) {
949 int irq
, head
, tail
= 0, warning
= 0;
952 irq
= head
= dev
->irq
;
953 dev
->irq
= temp
; /* Restore pin IRQ */
954 while (head
!= tail
) {
955 spin_lock_irqsave(&msi_lock
, flags
);
956 tail
= msi_desc
[irq
]->link
.tail
;
957 spin_unlock_irqrestore(&msi_lock
, flags
);
958 if (irq_has_action(irq
))
960 else if (irq
!= head
) /* Release MSI-X irq */
961 msi_free_irq(dev
, irq
);
964 msi_free_irq(dev
, irq
);
966 printk(KERN_WARNING
"PCI: %s: pci_disable_msix() called without "
967 "free_irq() on all MSI-X irqs\n",
975 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state
976 * @dev: pointer to the pci_dev data structure of MSI(X) device function
978 * Being called during hotplug remove, from which the device function
979 * is hot-removed. All previous assigned MSI/MSI-X irqs, if
980 * allocated for this device function, are reclaimed to unused state,
981 * which may be used later on.
983 void msi_remove_pci_irq_vectors(struct pci_dev
* dev
)
988 if (!pci_msi_enable
|| !dev
)
991 temp
= dev
->irq
; /* Save IOAPIC IRQ */
992 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSI
);
993 if (pos
> 0 && !msi_lookup_irq(dev
, PCI_CAP_ID_MSI
)) {
994 if (irq_has_action(dev
->irq
)) {
995 printk(KERN_WARNING
"PCI: %s: msi_remove_pci_irq_vectors() "
996 "called without free_irq() on MSI irq %d\n",
997 pci_name(dev
), dev
->irq
);
998 BUG_ON(irq_has_action(dev
->irq
));
999 } else /* Release MSI irq assigned to this device */
1000 msi_free_irq(dev
, dev
->irq
);
1001 dev
->irq
= temp
; /* Restore IOAPIC IRQ */
1003 pos
= pci_find_capability(dev
, PCI_CAP_ID_MSIX
);
1004 if (pos
> 0 && !msi_lookup_irq(dev
, PCI_CAP_ID_MSIX
)) {
1005 int irq
, head
, tail
= 0, warning
= 0;
1006 void __iomem
*base
= NULL
;
1008 irq
= head
= dev
->irq
;
1009 while (head
!= tail
) {
1010 spin_lock_irqsave(&msi_lock
, flags
);
1011 tail
= msi_desc
[irq
]->link
.tail
;
1012 base
= msi_desc
[irq
]->mask_base
;
1013 spin_unlock_irqrestore(&msi_lock
, flags
);
1014 if (irq_has_action(irq
))
1016 else if (irq
!= head
) /* Release MSI-X irq */
1017 msi_free_irq(dev
, irq
);
1020 msi_free_irq(dev
, irq
);
1023 printk(KERN_WARNING
"PCI: %s: msi_remove_pci_irq_vectors() "
1024 "called without free_irq() on all MSI-X irqs\n",
1026 BUG_ON(warning
> 0);
1028 dev
->irq
= temp
; /* Restore IOAPIC IRQ */
1032 void pci_no_msi(void)
1037 EXPORT_SYMBOL(pci_enable_msi
);
1038 EXPORT_SYMBOL(pci_disable_msi
);
1039 EXPORT_SYMBOL(pci_enable_msix
);
1040 EXPORT_SYMBOL(pci_disable_msix
);