#include <linux/vfio.h>
#include "config.h"
-#include "event_notifier.h"
-#include "exec-memory.h"
-#include "kvm.h"
-#include "memory.h"
-#include "msi.h"
-#include "msix.h"
-#include "qemu-error.h"
-#include "range.h"
-#include "vfio_pci_int.h"
+#include "qemu/event_notifier.h"
+#include "exec/address-spaces.h"
+#include "sysemu/kvm.h"
+#include "exec/memory.h"
+#include "pci/msi.h"
+#include "pci/msix.h"
+#include "pci/pci.h"
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/queue.h"
+#include "qemu/range.h"
/* #define DEBUG_VFIO */
#ifdef DEBUG_VFIO
do { } while (0)
#endif
+typedef struct VFIOBAR {
+ off_t fd_offset; /* offset of BAR within device fd */
+ int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
+ MemoryRegion mem; /* slow, read/write access */
+ MemoryRegion mmap_mem; /* direct mapped access */
+ void *mmap;
+ size_t size;
+ uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
+ uint8_t nr; /* cache the BAR number for debug */
+} VFIOBAR;
+
+typedef struct VFIOINTx {
+ bool pending; /* interrupt pending */
+ bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+ uint8_t pin; /* which pin to pull for qemu_set_irq */
+ EventNotifier interrupt; /* eventfd triggered on interrupt */
+ EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+ PCIINTxRoute route; /* routing info for QEMU bypass */
+ uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+ QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+struct VFIODevice;
+
+typedef struct VFIOMSIVector {
+ EventNotifier interrupt; /* eventfd triggered on interrupt */
+ struct VFIODevice *vdev; /* back pointer to device */
+ int virq; /* KVM irqchip route for QEMU bypass */
+ bool use;
+} VFIOMSIVector;
+
+enum {
+ VFIO_INT_NONE = 0,
+ VFIO_INT_INTx = 1,
+ VFIO_INT_MSI = 2,
+ VFIO_INT_MSIX = 3,
+};
+
+struct VFIOGroup;
+
+typedef struct VFIOContainer {
+ int fd; /* /dev/vfio/vfio, empowered by the attached groups */
+ struct {
+ /* enable abstraction to support various iommu backends */
+ union {
+ MemoryListener listener; /* Used by type1 iommu */
+ };
+ void (*release)(struct VFIOContainer *);
+ } iommu_data;
+ QLIST_HEAD(, VFIOGroup) group_list;
+ QLIST_ENTRY(VFIOContainer) next;
+} VFIOContainer;
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+ uint8_t table_bar;
+ uint8_t pba_bar;
+ uint16_t entries;
+ uint32_t table_offset;
+ uint32_t pba_offset;
+ MemoryRegion mmap_mem;
+ void *mmap;
+} VFIOMSIXInfo;
+
+typedef struct VFIODevice {
+ PCIDevice pdev;
+ int fd;
+ VFIOINTx intx;
+ unsigned int config_size;
+ off_t config_offset; /* Offset of config space region within device fd */
+ unsigned int rom_size;
+ off_t rom_offset; /* Offset of ROM region within device fd */
+ int msi_cap_size;
+ VFIOMSIVector *msi_vectors;
+ VFIOMSIXInfo *msix;
+ int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+ int interrupt; /* Current interrupt type */
+ VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+ PCIHostDeviceAddress host;
+ QLIST_ENTRY(VFIODevice) next;
+ struct VFIOGroup *group;
+ bool reset_works;
+} VFIODevice;
+
+typedef struct VFIOGroup {
+ int fd;
+ int groupid;
+ VFIOContainer *container;
+ QLIST_HEAD(, VFIODevice) device_list;
+ QLIST_ENTRY(VFIOGroup) next;
+ QLIST_ENTRY(VFIOGroup) container_next;
+} VFIOGroup;
+
#define MSIX_CAP_LENGTH 12
static QLIST_HEAD(, VFIOContainer)
ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
}
+#ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
+static void vfio_mask_intx(VFIODevice *vdev)
+{
+ struct vfio_irq_set irq_set = {
+ .argsz = sizeof(irq_set),
+ .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
+ .index = VFIO_PCI_INTX_IRQ_INDEX,
+ .start = 0,
+ .count = 1,
+ };
+
+ ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+#endif
+
/*
* Disabling BAR mmaping can be slow, but toggling it around INTx can
* also be a huge overhead. We try to get the best of both worlds by
vfio_unmask_intx(vdev);
}
-typedef struct QEMU_PACKED VFIOIRQSetFD {
- struct vfio_irq_set irq_set;
- int32_t fd;
-} VFIOIRQSetFD;
+static void vfio_enable_intx_kvm(VFIODevice *vdev)
+{
+#ifdef CONFIG_KVM
+ struct kvm_irqfd irqfd = {
+ .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+ .gsi = vdev->intx.route.irq,
+ .flags = KVM_IRQFD_FLAG_RESAMPLE,
+ };
+ struct vfio_irq_set *irq_set;
+ int ret, argsz;
+ int32_t *pfd;
-static int vfio_enable_intx(VFIODevice *vdev)
+ if (!kvm_irqfds_enabled() ||
+ vdev->intx.route.mode != PCI_INTX_ENABLED ||
+ !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+ return;
+ }
+
+ /* Get to a known interrupt state */
+ qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
+ vfio_mask_intx(vdev);
+ vdev->intx.pending = false;
+ qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+ /* Get an eventfd for resample/unmask */
+ if (event_notifier_init(&vdev->intx.unmask, 0)) {
+ error_report("vfio: Error: event_notifier_init failed eoi\n");
+ goto fail;
+ }
+
+ /* KVM triggers it, VFIO listens for it */
+ irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
+
+ if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+ error_report("vfio: Error: Failed to setup resample irqfd: %m\n");
+ goto fail_irqfd;
+ }
+
+ argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+ irq_set = g_malloc0(argsz);
+ irq_set->argsz = argsz;
+ irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
+ irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+ irq_set->start = 0;
+ irq_set->count = 1;
+ pfd = (int32_t *)&irq_set->data;
+
+ *pfd = irqfd.resamplefd;
+
+ ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+ g_free(irq_set);
+ if (ret) {
+ error_report("vfio: Error: Failed to setup INTx unmask fd: %m\n");
+ goto fail_vfio;
+ }
+
+ /* Let'em rip */
+ vfio_unmask_intx(vdev);
+
+ vdev->intx.kvm_accel = true;
+
+ DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
+ __func__, vdev->host.domain, vdev->host.bus,
+ vdev->host.slot, vdev->host.function);
+
+ return;
+
+fail_vfio:
+ irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
+ kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
+fail_irqfd:
+ event_notifier_cleanup(&vdev->intx.unmask);
+fail:
+ qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+ vfio_unmask_intx(vdev);
+#endif
+}
+
+static void vfio_disable_intx_kvm(VFIODevice *vdev)
{
- VFIOIRQSetFD irq_set_fd = {
- .irq_set = {
- .argsz = sizeof(irq_set_fd),
- .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER,
- .index = VFIO_PCI_INTX_IRQ_INDEX,
- .start = 0,
- .count = 1,
- },
+#ifdef CONFIG_KVM
+ struct kvm_irqfd irqfd = {
+ .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+ .gsi = vdev->intx.route.irq,
+ .flags = KVM_IRQFD_FLAG_DEASSIGN,
};
+
+ if (!vdev->intx.kvm_accel) {
+ return;
+ }
+
+ /*
+ * Get to a known state, hardware masked, QEMU ready to accept new
+ * interrupts, QEMU IRQ de-asserted.
+ */
+ vfio_mask_intx(vdev);
+ vdev->intx.pending = false;
+ qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+ /* Tell KVM to stop listening for an INTx irqfd */
+ if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+ error_report("vfio: Error: Failed to disable INTx irqfd: %m\n");
+ }
+
+ /* We only need to close the eventfd for VFIO to cleanup the kernel side */
+ event_notifier_cleanup(&vdev->intx.unmask);
+
+ /* QEMU starts listening for interrupt events. */
+ qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+
+ vdev->intx.kvm_accel = false;
+
+ /* If we've missed an event, let it re-fire through QEMU */
+ vfio_unmask_intx(vdev);
+
+ DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
+ __func__, vdev->host.domain, vdev->host.bus,
+ vdev->host.slot, vdev->host.function);
+#endif
+}
+
+static void vfio_update_irq(PCIDevice *pdev)
+{
+ VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+ PCIINTxRoute route;
+
+ if (vdev->interrupt != VFIO_INT_INTx) {
+ return;
+ }
+
+ route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
+
+ if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
+ return; /* Nothing changed */
+ }
+
+ DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
+ vdev->host.domain, vdev->host.bus, vdev->host.slot,
+ vdev->host.function, vdev->intx.route.irq, route.irq);
+
+ vfio_disable_intx_kvm(vdev);
+
+ vdev->intx.route = route;
+
+ if (route.mode != PCI_INTX_ENABLED) {
+ return;
+ }
+
+ vfio_enable_intx_kvm(vdev);
+
+ /* Re-enable the interrupt in cased we missed an EOI */
+ vfio_eoi(vdev);
+}
+
+static int vfio_enable_intx(VFIODevice *vdev)
+{
uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
- int ret;
+ int ret, argsz;
+ struct vfio_irq_set *irq_set;
+ int32_t *pfd;
if (!pin) {
return 0;
vfio_disable_interrupts(vdev);
vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
+
+#ifdef CONFIG_KVM
+ /*
+ * Only conditional to avoid generating error messages on platforms
+ * where we won't actually use the result anyway.
+ */
+ if (kvm_irqfds_enabled() &&
+ kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+ vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
+ vdev->intx.pin);
+ }
+#endif
+
ret = event_notifier_init(&vdev->intx.interrupt, 0);
if (ret) {
error_report("vfio: Error: event_notifier_init failed\n");
return ret;
}
- irq_set_fd.fd = event_notifier_get_fd(&vdev->intx.interrupt);
- qemu_set_fd_handler(irq_set_fd.fd, vfio_intx_interrupt, NULL, vdev);
+ argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+ irq_set = g_malloc0(argsz);
+ irq_set->argsz = argsz;
+ irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+ irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+ irq_set->start = 0;
+ irq_set->count = 1;
+ pfd = (int32_t *)&irq_set->data;
+
+ *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
+ qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
- if (ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set_fd)) {
+ ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+ g_free(irq_set);
+ if (ret) {
error_report("vfio: Error: Failed to setup INTx fd: %m\n");
+ qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+ event_notifier_cleanup(&vdev->intx.interrupt);
return -errno;
}
+ vfio_enable_intx_kvm(vdev);
+
vdev->interrupt = VFIO_INT_INTx;
DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
int fd;
qemu_del_timer(vdev->intx.mmap_timer);
+ vfio_disable_intx_kvm(vdev);
vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
vdev->intx.pending = false;
qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
return ret;
}
-static int vfio_msix_vector_use(PCIDevice *pdev,
- unsigned int nr, MSIMessage msg)
+static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
+ MSIMessage *msg, IOHandler *handler)
{
VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
VFIOMSIVector *vector;
* Attempt to enable route through KVM irqchip,
* default to userspace handling if unavailable.
*/
- vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+ vector->virq = msg ? kvm_irqchip_add_msi_route(kvm_state, *msg) : -1;
if (vector->virq < 0 ||
kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
vector->virq) < 0) {
vector->virq = -1;
}
qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
- vfio_msi_interrupt, NULL, vector);
+ handler, NULL, vector);
}
/*
error_report("vfio: failed to enable vectors, %d\n", ret);
}
} else {
- VFIOIRQSetFD irq_set_fd = {
- .irq_set = {
- .argsz = sizeof(irq_set_fd),
- .flags = VFIO_IRQ_SET_DATA_EVENTFD |
- VFIO_IRQ_SET_ACTION_TRIGGER,
- .index = VFIO_PCI_MSIX_IRQ_INDEX,
- .start = nr,
- .count = 1,
- },
- .fd = event_notifier_get_fd(&vector->interrupt),
- };
- ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set_fd);
+ int argsz;
+ struct vfio_irq_set *irq_set;
+ int32_t *pfd;
+
+ argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+ irq_set = g_malloc0(argsz);
+ irq_set->argsz = argsz;
+ irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+ VFIO_IRQ_SET_ACTION_TRIGGER;
+ irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+ irq_set->start = nr;
+ irq_set->count = 1;
+ pfd = (int32_t *)&irq_set->data;
+
+ *pfd = event_notifier_get_fd(&vector->interrupt);
+
+ ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+ g_free(irq_set);
if (ret) {
error_report("vfio: failed to modify vector, %d\n", ret);
}
return 0;
}
+static int vfio_msix_vector_use(PCIDevice *pdev,
+ unsigned int nr, MSIMessage msg)
+{
+ return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
+}
+
static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
{
VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
VFIOMSIVector *vector = &vdev->msi_vectors[nr];
- VFIOIRQSetFD irq_set_fd = {
- .irq_set = {
- .argsz = sizeof(irq_set_fd),
- .flags = VFIO_IRQ_SET_DATA_EVENTFD |
- VFIO_IRQ_SET_ACTION_TRIGGER,
- .index = VFIO_PCI_MSIX_IRQ_INDEX,
- .start = nr,
- .count = 1,
- },
- .fd = -1,
- };
+ int argsz;
+ struct vfio_irq_set *irq_set;
+ int32_t *pfd;
DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
vdev->host.domain, vdev->host.bus, vdev->host.slot,
* bouncing through userspace and let msix.c drop it? Not sure.
*/
msix_vector_unuse(pdev, nr);
- ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set_fd);
+
+ argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+ irq_set = g_malloc0(argsz);
+ irq_set->argsz = argsz;
+ irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+ VFIO_IRQ_SET_ACTION_TRIGGER;
+ irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+ irq_set->start = nr;
+ irq_set->count = 1;
+ pfd = (int32_t *)&irq_set->data;
+
+ *pfd = -1;
+
+ ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+ g_free(irq_set);
if (vector->virq < 0) {
qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
vector->use = false;
}
-/* TODO This should move to msi.c */
-static MSIMessage msi_get_msg(PCIDevice *pdev, unsigned int vector)
-{
- uint16_t flags = pci_get_word(pdev->config + pdev->msi_cap + PCI_MSI_FLAGS);
- bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
- MSIMessage msg;
-
- if (msi64bit) {
- msg.address = pci_get_quad(pdev->config +
- pdev->msi_cap + PCI_MSI_ADDRESS_LO);
- } else {
- msg.address = pci_get_long(pdev->config +
- pdev->msi_cap + PCI_MSI_ADDRESS_LO);
- }
-
- msg.data = pci_get_word(pdev->config + pdev->msi_cap +
- (msi64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA_32));
- msg.data += vector;
-
- return msg;
-}
-
-/* So should this */
-static void msi_set_qsize(PCIDevice *pdev, uint8_t size)
-{
- uint8_t *config = pdev->config + pdev->msi_cap;
- uint16_t flags;
-
- flags = pci_get_word(config + PCI_MSI_FLAGS);
- flags = le16_to_cpu(flags);
- flags &= ~PCI_MSI_FLAGS_QSIZE;
- flags |= (size & 0x7) << 4;
- flags = cpu_to_le16(flags);
- pci_set_word(config + PCI_MSI_FLAGS, flags);
-}
-
static void vfio_enable_msix(VFIODevice *vdev)
{
vfio_disable_interrupts(vdev);
vdev->interrupt = VFIO_INT_MSIX;
+ /*
+ * Some communication channels between VF & PF or PF & fw rely on the
+ * physical state of the device and expect that enabling MSI-X from the
+ * guest enables the same on the host. When our guest is Linux, the
+ * guest driver call to pci_enable_msix() sets the enabling bit in the
+ * MSI-X capability, but leaves the vector table masked. We therefore
+ * can't rely on a vector_use callback (from request_irq() in the guest)
+ * to switch the physical device into MSI-X mode because that may come a
+ * long time after pci_enable_msix(). This code enables vector 0 with
+ * triggering to userspace, then immediately release the vector, leaving
+ * the physical device with no vectors enabled, but MSI-X enabled, just
+ * like the guest view.
+ */
+ vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
+ vfio_msix_vector_release(&vdev->pdev, 0);
+
if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
- vfio_msix_vector_release)) {
+ vfio_msix_vector_release, NULL)) {
error_report("vfio: msix_set_vector_notifiers failed\n");
}
error_report("vfio: Error: event_notifier_init failed\n");
}
- msg = msi_get_msg(&vdev->pdev, i);
+ msg = msi_get_message(&vdev->pdev, i);
/*
* Attempt to enable route through KVM irqchip,
return;
}
- msi_set_qsize(&vdev->pdev, vdev->nr_vectors);
-
vdev->interrupt = VFIO_INT_MSI;
DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
vfio_disable_msi_common(vdev);
- DPRINTF("%s(%04x:%02x:%02x.%x, msi%s)\n", __func__,
- vdev->host.domain, vdev->host.bus, vdev->host.slot,
- vdev->host.function, msix ? "x" : "");
+ DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+ vdev->host.bus, vdev->host.slot, vdev->host.function);
}
static void vfio_disable_msi(VFIODevice *vdev)
vfio_disable_msi_common(vdev);
- msi_set_qsize(&vdev->pdev, 0); /* Actually still means 1 vector */
-
DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
vdev->host.bus, vdev->host.slot, vdev->host.function);
}
/*
* IO Port/MMIO - Beware of the endians, VFIO is always little endian
*/
-static void vfio_bar_write(void *opaque, target_phys_addr_t addr,
+static void vfio_bar_write(void *opaque, hwaddr addr,
uint64_t data, unsigned size)
{
VFIOBAR *bar = opaque;
}
if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
- error_report("%s(,0x%"TARGET_PRIxPHYS", 0x%"PRIx64", %d) failed: %m\n",
+ error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m\n",
__func__, addr, data, size);
}
- DPRINTF("%s(BAR%d+0x%"TARGET_PRIxPHYS", 0x%"PRIx64", %d)\n",
+ DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
__func__, bar->nr, addr, data, size);
/*
* which access will service the interrupt, so we're potentially
* getting quite a few host interrupts per guest interrupt.
*/
- vfio_eoi(DO_UPCAST(VFIODevice, bars[bar->nr], bar));
+ vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
}
static uint64_t vfio_bar_read(void *opaque,
- target_phys_addr_t addr, unsigned size)
+ hwaddr addr, unsigned size)
{
VFIOBAR *bar = opaque;
union {
uint64_t data = 0;
if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
- error_report("%s(,0x%"TARGET_PRIxPHYS", %d) failed: %m\n",
+ error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m\n",
__func__, addr, size);
return (uint64_t)-1;
}
break;
}
- DPRINTF("%s(BAR%d+0x%"TARGET_PRIxPHYS", %d) = 0x%"PRIx64"\n",
+ DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
__func__, bar->nr, addr, size, data);
/* Same as write above */
- vfio_eoi(DO_UPCAST(VFIODevice, bars[bar->nr], bar));
+ vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
return data;
}
* DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
*/
static int vfio_dma_unmap(VFIOContainer *container,
- target_phys_addr_t iova, ram_addr_t size)
+ hwaddr iova, ram_addr_t size)
{
struct vfio_iommu_type1_dma_unmap unmap = {
.argsz = sizeof(unmap),
return 0;
}
-static int vfio_dma_map(VFIOContainer *container, target_phys_addr_t iova,
+static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
ram_addr_t size, void *vaddr, bool readonly)
{
struct vfio_iommu_type1_dma_map map = {
.argsz = sizeof(map),
.flags = VFIO_DMA_MAP_FLAG_READ,
- .vaddr = (__u64)(intptr_t)vaddr,
+ .vaddr = (__u64)(uintptr_t)vaddr,
.iova = iova,
.size = size,
};
return -errno;
}
-static void vfio_listener_dummy1(MemoryListener *listener)
-{
- /* We don't do batching (begin/commit) or care about logging */
-}
-
-static void vfio_listener_dummy2(MemoryListener *listener,
- MemoryRegionSection *section)
-{
- /* We don't do logging or care about nops */
-}
-
-static void vfio_listener_dummy3(MemoryListener *listener,
- MemoryRegionSection *section,
- bool match_data, uint64_t data,
- EventNotifier *e)
-{
- /* We don't care about eventfds */
-}
-
static bool vfio_listener_skipped_section(MemoryRegionSection *section)
{
return !memory_region_is_ram(section->mr);
{
VFIOContainer *container = container_of(listener, VFIOContainer,
iommu_data.listener);
- target_phys_addr_t iova, end;
+ hwaddr iova, end;
void *vaddr;
int ret;
if (vfio_listener_skipped_section(section)) {
- DPRINTF("vfio: SKIPPING region_add %"TARGET_PRIxPHYS" - %"PRIx64"\n",
+ DPRINTF("vfio: SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
section->offset_within_address_space,
section->offset_within_address_space + section->size - 1);
return;
section->offset_within_region +
(iova - section->offset_within_address_space);
- DPRINTF("vfio: region_add %"TARGET_PRIxPHYS" - %"TARGET_PRIxPHYS" [%p]\n",
+ DPRINTF("vfio: region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
iova, end - 1, vaddr);
ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
if (ret) {
- error_report("vfio_dma_map(%p, 0x%"TARGET_PRIxPHYS", "
- "0x%"TARGET_PRIxPHYS", %p) = %d (%m)\n",
+ error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+ "0x%"HWADDR_PRIx", %p) = %d (%m)\n",
container, iova, end - iova, vaddr, ret);
}
}
{
VFIOContainer *container = container_of(listener, VFIOContainer,
iommu_data.listener);
- target_phys_addr_t iova, end;
+ hwaddr iova, end;
int ret;
if (vfio_listener_skipped_section(section)) {
- DPRINTF("vfio: SKIPPING region_del %"TARGET_PRIxPHYS" - %"PRIx64"\n",
+ DPRINTF("vfio: SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
section->offset_within_address_space,
section->offset_within_address_space + section->size - 1);
return;
return;
}
- DPRINTF("vfio: region_del %"TARGET_PRIxPHYS" - %"TARGET_PRIxPHYS"\n",
+ DPRINTF("vfio: region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
iova, end - 1);
ret = vfio_dma_unmap(container, iova, end - iova);
if (ret) {
- error_report("vfio_dma_unmap(%p, 0x%"TARGET_PRIxPHYS", "
- "0x%"TARGET_PRIxPHYS") = %d (%m)\n",
+ error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+ "0x%"HWADDR_PRIx") = %d (%m)\n",
container, iova, end - iova, ret);
}
}
static MemoryListener vfio_memory_listener = {
- .begin = vfio_listener_dummy1,
- .commit = vfio_listener_dummy1,
.region_add = vfio_listener_region_add,
.region_del = vfio_listener_region_del,
- .region_nop = vfio_listener_dummy2,
- .log_start = vfio_listener_dummy2,
- .log_stop = vfio_listener_dummy2,
- .log_sync = vfio_listener_dummy2,
- .log_global_start = vfio_listener_dummy1,
- .log_global_stop = vfio_listener_dummy1,
- .eventfd_add = vfio_listener_dummy3,
- .eventfd_del = vfio_listener_dummy3,
};
static void vfio_listener_release(VFIOContainer *container)
bool msi_64bit, msi_maskbit;
int ret, entries;
- /*
- * TODO: don't peek into msi_supported, let msi_init fail and
- * check for ENOTSUP
- */
- if (!msi_supported) {
- return 0;
- }
-
if (pread(vdev->fd, &ctrl, sizeof(ctrl),
vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
return -errno;
ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
if (ret < 0) {
+ if (ret == -ENOTSUP) {
+ return 0;
+ }
error_report("vfio: msi_init failed\n");
return ret;
}
{
int ret;
- /*
- * TODO: don't peek into msi_supported, let msix_init fail and
- * check for ENOTSUP
- */
- if (!msi_supported) {
- return 0;
- }
-
ret = msix_init(&vdev->pdev, vdev->msix->entries,
&vdev->bars[vdev->msix->table_bar].mem,
vdev->msix->table_bar, vdev->msix->table_offset,
&vdev->bars[vdev->msix->pba_bar].mem,
vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
if (ret < 0) {
+ if (ret == -ENOTSUP) {
+ return 0;
+ }
error_report("vfio: msix_init failed\n");
return ret;
}
container->iommu_data.listener = vfio_memory_listener;
container->iommu_data.release = vfio_listener_release;
- memory_listener_register(&container->iommu_data.listener,
- get_system_memory());
+ memory_listener_register(&container->iommu_data.listener, &address_space_memory);
} else {
error_report("vfio: No available IOMMU models\n");
g_free(container);
ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
if (ret < 0) {
- error_report("vfio: error getting device %s from group %d: %m\n",
+ error_report("vfio: error getting device %s from group %d: %m",
name, group->groupid);
- error_report("Verify all devices in group %d are bound to vfio-pci "
+ error_printf("Verify all devices in group %d are bound to vfio-pci "
"or pci-stub and not already in use\n", group->groupid);
return ret;
}
error_report("Warning, device %s does not support reset\n", name);
}
- if (dev_info.num_regions != VFIO_PCI_NUM_REGIONS) {
+ if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
error_report("vfio: unexpected number of io regions %u\n",
dev_info.num_regions);
goto error;
}
- if (dev_info.num_irqs != VFIO_PCI_NUM_IRQS) {
+ if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
error_report("vfio: unexpected number of irqs %u\n", dev_info.num_irqs);
goto error;
}
(unsigned long)reg_info.flags);
vdev->config_size = reg_info.size;
+ if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
+ vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+ }
vdev->config_offset = reg_info.offset;
error:
if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
vdev->intx.mmap_timer = qemu_new_timer_ms(vm_clock,
vfio_intx_mmap_enable, vdev);
+ pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
ret = vfio_enable_intx(vdev);
if (ret) {
goto out_teardown;
{
PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+ uint16_t cmd;
- if (!vdev->reset_works) {
- return;
- }
+ DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+ vdev->host.bus, vdev->host.slot, vdev->host.function);
- if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
- error_report("vfio: Error unable to reset physical device "
- "(%04x:%02x:%02x.%x): %m\n", vdev->host.domain,
- vdev->host.bus, vdev->host.slot, vdev->host.function);
+ vfio_disable_interrupts(vdev);
+
+ /*
+ * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
+ * Also put INTx Disable in known state.
+ */
+ cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
+ cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+ PCI_COMMAND_INTX_DISABLE);
+ vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+
+ if (vdev->reset_works) {
+ if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
+ error_report("vfio: Error unable to reset physical device "
+ "(%04x:%02x:%02x.%x): %m\n", vdev->host.domain,
+ vdev->host.bus, vdev->host.slot, vdev->host.function);
+ }
}
+
+ vfio_enable_intx(vdev);
}
static Property vfio_pci_dev_properties[] = {
DEFINE_PROP_END_OF_LIST(),
};
+static const VMStateDescription vfio_pci_vmstate = {
+ .name = "vfio-pci",
+ .unmigratable = 1,
+};
static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
{
dc->reset = vfio_pci_reset;
dc->props = vfio_pci_dev_properties;
+ dc->vmsd = &vfio_pci_vmstate;
+ dc->desc = "VFIO-based PCI device assignment";
pdc->init = vfio_initfn;
pdc->exit = vfio_exitfn;
pdc->config_read = vfio_pci_read_config;
pdc->config_write = vfio_pci_write_config;
+ pdc->is_express = 1; /* We might be */
}
static const TypeInfo vfio_pci_dev_info = {
projects / qemu.git / blobdiff