2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
23 #include <sys/ioctl.h>
26 #include <sys/types.h>
27 #include <linux/vfio.h>
30 #include "event_notifier.h"
31 #include "exec-memory.h"
37 #include "qemu-common.h"
38 #include "qemu-error.h"
39 #include "qemu-queue.h"
42 /* #define DEBUG_VFIO */
44 #define DPRINTF(fmt, ...) \
45 do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
47 #define DPRINTF(fmt, ...) \
51 typedef struct VFIOBAR
{
52 off_t fd_offset
; /* offset of BAR within device fd */
53 int fd
; /* device fd, allows us to pass VFIOBAR as opaque data */
54 MemoryRegion mem
; /* slow, read/write access */
55 MemoryRegion mmap_mem
; /* direct mapped access */
58 uint32_t flags
; /* VFIO region flags (rd/wr/mmap) */
59 uint8_t nr
; /* cache the BAR number for debug */
62 typedef struct VFIOINTx
{
63 bool pending
; /* interrupt pending */
64 bool kvm_accel
; /* set when QEMU bypass through KVM enabled */
65 uint8_t pin
; /* which pin to pull for qemu_set_irq */
66 EventNotifier interrupt
; /* eventfd triggered on interrupt */
67 EventNotifier unmask
; /* eventfd for unmask on QEMU bypass */
68 PCIINTxRoute route
; /* routing info for QEMU bypass */
69 uint32_t mmap_timeout
; /* delay to re-enable mmaps after interrupt */
70 QEMUTimer
*mmap_timer
; /* enable mmaps after periods w/o interrupts */
75 typedef struct VFIOMSIVector
{
76 EventNotifier interrupt
; /* eventfd triggered on interrupt */
77 struct VFIODevice
*vdev
; /* back pointer to device */
78 int virq
; /* KVM irqchip route for QEMU bypass */
91 typedef struct VFIOContainer
{
92 int fd
; /* /dev/vfio/vfio, empowered by the attached groups */
94 /* enable abstraction to support various iommu backends */
96 MemoryListener listener
; /* Used by type1 iommu */
98 void (*release
)(struct VFIOContainer
*);
100 QLIST_HEAD(, VFIOGroup
) group_list
;
101 QLIST_ENTRY(VFIOContainer
) next
;
104 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
105 typedef struct VFIOMSIXInfo
{
109 uint32_t table_offset
;
111 MemoryRegion mmap_mem
;
115 typedef struct VFIODevice
{
119 unsigned int config_size
;
120 off_t config_offset
; /* Offset of config space region within device fd */
121 unsigned int rom_size
;
122 off_t rom_offset
; /* Offset of ROM region within device fd */
124 VFIOMSIVector
*msi_vectors
;
126 int nr_vectors
; /* Number of MSI/MSIX vectors currently in use */
127 int interrupt
; /* Current interrupt type */
128 VFIOBAR bars
[PCI_NUM_REGIONS
- 1]; /* No ROM */
129 PCIHostDeviceAddress host
;
130 QLIST_ENTRY(VFIODevice
) next
;
131 struct VFIOGroup
*group
;
135 typedef struct VFIOGroup
{
138 VFIOContainer
*container
;
139 QLIST_HEAD(, VFIODevice
) device_list
;
140 QLIST_ENTRY(VFIOGroup
) next
;
141 QLIST_ENTRY(VFIOGroup
) container_next
;
144 #define MSIX_CAP_LENGTH 12
146 static QLIST_HEAD(, VFIOContainer
)
147 container_list
= QLIST_HEAD_INITIALIZER(container_list
);
149 static QLIST_HEAD(, VFIOGroup
)
150 group_list
= QLIST_HEAD_INITIALIZER(group_list
);
152 static void vfio_disable_interrupts(VFIODevice
*vdev
);
153 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
);
154 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
);
157 * Common VFIO interrupt disable
159 static void vfio_disable_irqindex(VFIODevice
*vdev
, int index
)
161 struct vfio_irq_set irq_set
= {
162 .argsz
= sizeof(irq_set
),
163 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_TRIGGER
,
169 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
175 static void vfio_unmask_intx(VFIODevice
*vdev
)
177 struct vfio_irq_set irq_set
= {
178 .argsz
= sizeof(irq_set
),
179 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_UNMASK
,
180 .index
= VFIO_PCI_INTX_IRQ_INDEX
,
185 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
189 * Disabling BAR mmaping can be slow, but toggling it around INTx can
190 * also be a huge overhead. We try to get the best of both worlds by
191 * waiting until an interrupt to disable mmaps (subsequent transitions
192 * to the same state are effectively no overhead). If the interrupt has
193 * been serviced and the time gap is long enough, we re-enable mmaps for
194 * performance. This works well for things like graphics cards, which
195 * may not use their interrupt at all and are penalized to an unusable
196 * level by read/write BAR traps. Other devices, like NICs, have more
197 * regular interrupts and see much better latency by staying in non-mmap
198 * mode. We therefore set the default mmap_timeout such that a ping
199 * is just enough to keep the mmap disabled. Users can experiment with
200 * other options with the x-intx-mmap-timeout-ms parameter (a value of
201 * zero disables the timer).
203 static void vfio_intx_mmap_enable(void *opaque
)
205 VFIODevice
*vdev
= opaque
;
207 if (vdev
->intx
.pending
) {
208 qemu_mod_timer(vdev
->intx
.mmap_timer
,
209 qemu_get_clock_ms(vm_clock
) + vdev
->intx
.mmap_timeout
);
213 vfio_mmap_set_enabled(vdev
, true);
216 static void vfio_intx_interrupt(void *opaque
)
218 VFIODevice
*vdev
= opaque
;
220 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
224 DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__
, vdev
->host
.domain
,
225 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
226 'A' + vdev
->intx
.pin
);
228 vdev
->intx
.pending
= true;
229 qemu_set_irq(vdev
->pdev
.irq
[vdev
->intx
.pin
], 1);
230 vfio_mmap_set_enabled(vdev
, false);
231 if (vdev
->intx
.mmap_timeout
) {
232 qemu_mod_timer(vdev
->intx
.mmap_timer
,
233 qemu_get_clock_ms(vm_clock
) + vdev
->intx
.mmap_timeout
);
237 static void vfio_eoi(VFIODevice
*vdev
)
239 if (!vdev
->intx
.pending
) {
243 DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__
, vdev
->host
.domain
,
244 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
246 vdev
->intx
.pending
= false;
247 qemu_set_irq(vdev
->pdev
.irq
[vdev
->intx
.pin
], 0);
248 vfio_unmask_intx(vdev
);
251 static int vfio_enable_intx(VFIODevice
*vdev
)
253 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
255 struct vfio_irq_set
*irq_set
;
262 vfio_disable_interrupts(vdev
);
264 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
265 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
267 error_report("vfio: Error: event_notifier_init failed\n");
271 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
273 irq_set
= g_malloc0(argsz
);
274 irq_set
->argsz
= argsz
;
275 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
276 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
279 pfd
= (int32_t *)&irq_set
->data
;
281 *pfd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
282 qemu_set_fd_handler(*pfd
, vfio_intx_interrupt
, NULL
, vdev
);
284 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
287 error_report("vfio: Error: Failed to setup INTx fd: %m\n");
288 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
289 event_notifier_cleanup(&vdev
->intx
.interrupt
);
293 vdev
->interrupt
= VFIO_INT_INTx
;
295 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
296 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
301 static void vfio_disable_intx(VFIODevice
*vdev
)
305 qemu_del_timer(vdev
->intx
.mmap_timer
);
306 vfio_disable_irqindex(vdev
, VFIO_PCI_INTX_IRQ_INDEX
);
307 vdev
->intx
.pending
= false;
308 qemu_set_irq(vdev
->pdev
.irq
[vdev
->intx
.pin
], 0);
309 vfio_mmap_set_enabled(vdev
, true);
311 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
312 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
313 event_notifier_cleanup(&vdev
->intx
.interrupt
);
315 vdev
->interrupt
= VFIO_INT_NONE
;
317 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
318 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
324 static void vfio_msi_interrupt(void *opaque
)
326 VFIOMSIVector
*vector
= opaque
;
327 VFIODevice
*vdev
= vector
->vdev
;
328 int nr
= vector
- vdev
->msi_vectors
;
330 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
334 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d\n", __func__
,
335 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
336 vdev
->host
.function
, nr
);
338 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
339 msix_notify(&vdev
->pdev
, nr
);
340 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
341 msi_notify(&vdev
->pdev
, nr
);
343 error_report("vfio: MSI interrupt receieved, but not enabled?\n");
347 static int vfio_enable_vectors(VFIODevice
*vdev
, bool msix
)
349 struct vfio_irq_set
*irq_set
;
350 int ret
= 0, i
, argsz
;
353 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
355 irq_set
= g_malloc0(argsz
);
356 irq_set
->argsz
= argsz
;
357 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
358 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
360 irq_set
->count
= vdev
->nr_vectors
;
361 fds
= (int32_t *)&irq_set
->data
;
363 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
364 if (!vdev
->msi_vectors
[i
].use
) {
369 fds
[i
] = event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
372 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
379 static int vfio_msix_vector_use(PCIDevice
*pdev
,
380 unsigned int nr
, MSIMessage msg
)
382 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
383 VFIOMSIVector
*vector
;
386 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__
,
387 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
388 vdev
->host
.function
, nr
);
390 vector
= &vdev
->msi_vectors
[nr
];
394 msix_vector_use(pdev
, nr
);
396 if (event_notifier_init(&vector
->interrupt
, 0)) {
397 error_report("vfio: Error: event_notifier_init failed\n");
401 * Attempt to enable route through KVM irqchip,
402 * default to userspace handling if unavailable.
404 vector
->virq
= kvm_irqchip_add_msi_route(kvm_state
, msg
);
405 if (vector
->virq
< 0 ||
406 kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->interrupt
,
408 if (vector
->virq
>= 0) {
409 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
412 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
413 vfio_msi_interrupt
, NULL
, vector
);
417 * We don't want to have the host allocate all possible MSI vectors
418 * for a device if they're not in use, so we shutdown and incrementally
419 * increase them as needed.
421 if (vdev
->nr_vectors
< nr
+ 1) {
422 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
423 vdev
->nr_vectors
= nr
+ 1;
424 ret
= vfio_enable_vectors(vdev
, true);
426 error_report("vfio: failed to enable vectors, %d\n", ret
);
430 struct vfio_irq_set
*irq_set
;
433 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
435 irq_set
= g_malloc0(argsz
);
436 irq_set
->argsz
= argsz
;
437 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
438 VFIO_IRQ_SET_ACTION_TRIGGER
;
439 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
442 pfd
= (int32_t *)&irq_set
->data
;
444 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
446 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
449 error_report("vfio: failed to modify vector, %d\n", ret
);
456 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
458 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
459 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
461 struct vfio_irq_set
*irq_set
;
464 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__
,
465 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
466 vdev
->host
.function
, nr
);
469 * XXX What's the right thing to do here? This turns off the interrupt
470 * completely, but do we really just want to switch the interrupt to
471 * bouncing through userspace and let msix.c drop it? Not sure.
473 msix_vector_unuse(pdev
, nr
);
475 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
477 irq_set
= g_malloc0(argsz
);
478 irq_set
->argsz
= argsz
;
479 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
480 VFIO_IRQ_SET_ACTION_TRIGGER
;
481 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
484 pfd
= (int32_t *)&irq_set
->data
;
488 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
492 if (vector
->virq
< 0) {
493 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
496 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->interrupt
,
498 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
502 event_notifier_cleanup(&vector
->interrupt
);
506 /* TODO This should move to msi.c */
507 static MSIMessage
msi_get_msg(PCIDevice
*pdev
, unsigned int vector
)
509 uint16_t flags
= pci_get_word(pdev
->config
+ pdev
->msi_cap
+ PCI_MSI_FLAGS
);
510 bool msi64bit
= flags
& PCI_MSI_FLAGS_64BIT
;
514 msg
.address
= pci_get_quad(pdev
->config
+
515 pdev
->msi_cap
+ PCI_MSI_ADDRESS_LO
);
517 msg
.address
= pci_get_long(pdev
->config
+
518 pdev
->msi_cap
+ PCI_MSI_ADDRESS_LO
);
521 msg
.data
= pci_get_word(pdev
->config
+ pdev
->msi_cap
+
522 (msi64bit
? PCI_MSI_DATA_64
: PCI_MSI_DATA_32
));
528 static void vfio_enable_msix(VFIODevice
*vdev
)
530 vfio_disable_interrupts(vdev
);
532 vdev
->msi_vectors
= g_malloc0(vdev
->msix
->entries
* sizeof(VFIOMSIVector
));
534 vdev
->interrupt
= VFIO_INT_MSIX
;
536 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
537 vfio_msix_vector_release
)) {
538 error_report("vfio: msix_set_vector_notifiers failed\n");
541 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
542 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
545 static void vfio_enable_msi(VFIODevice
*vdev
)
549 vfio_disable_interrupts(vdev
);
551 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
553 vdev
->msi_vectors
= g_malloc0(vdev
->nr_vectors
* sizeof(VFIOMSIVector
));
555 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
557 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
562 if (event_notifier_init(&vector
->interrupt
, 0)) {
563 error_report("vfio: Error: event_notifier_init failed\n");
566 msg
= msi_get_msg(&vdev
->pdev
, i
);
569 * Attempt to enable route through KVM irqchip,
570 * default to userspace handling if unavailable.
572 vector
->virq
= kvm_irqchip_add_msi_route(kvm_state
, msg
);
573 if (vector
->virq
< 0 ||
574 kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->interrupt
,
576 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
577 vfio_msi_interrupt
, NULL
, vector
);
581 ret
= vfio_enable_vectors(vdev
, false);
584 error_report("vfio: Error: Failed to setup MSI fds: %m\n");
585 } else if (ret
!= vdev
->nr_vectors
) {
586 error_report("vfio: Error: Failed to enable %d "
587 "MSI vectors, retry with %d\n", vdev
->nr_vectors
, ret
);
590 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
591 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
592 if (vector
->virq
>= 0) {
593 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->interrupt
,
595 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
598 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
601 event_notifier_cleanup(&vector
->interrupt
);
604 g_free(vdev
->msi_vectors
);
606 if (ret
> 0 && ret
!= vdev
->nr_vectors
) {
607 vdev
->nr_vectors
= ret
;
610 vdev
->nr_vectors
= 0;
615 vdev
->interrupt
= VFIO_INT_MSI
;
617 DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__
,
618 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
619 vdev
->host
.function
, vdev
->nr_vectors
);
622 static void vfio_disable_msi_common(VFIODevice
*vdev
)
624 g_free(vdev
->msi_vectors
);
625 vdev
->msi_vectors
= NULL
;
626 vdev
->nr_vectors
= 0;
627 vdev
->interrupt
= VFIO_INT_NONE
;
629 vfio_enable_intx(vdev
);
632 static void vfio_disable_msix(VFIODevice
*vdev
)
634 msix_unset_vector_notifiers(&vdev
->pdev
);
636 if (vdev
->nr_vectors
) {
637 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
640 vfio_disable_msi_common(vdev
);
642 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
643 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
646 static void vfio_disable_msi(VFIODevice
*vdev
)
650 vfio_disable_irqindex(vdev
, VFIO_PCI_MSI_IRQ_INDEX
);
652 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
653 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
659 if (vector
->virq
>= 0) {
660 kvm_irqchip_remove_irqfd_notifier(kvm_state
,
661 &vector
->interrupt
, vector
->virq
);
662 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
665 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
669 event_notifier_cleanup(&vector
->interrupt
);
672 vfio_disable_msi_common(vdev
);
674 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
675 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
679 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
681 static void vfio_bar_write(void *opaque
, target_phys_addr_t addr
,
682 uint64_t data
, unsigned size
)
684 VFIOBAR
*bar
= opaque
;
697 buf
.word
= cpu_to_le16(data
);
700 buf
.dword
= cpu_to_le32(data
);
703 hw_error("vfio: unsupported write size, %d bytes\n", size
);
707 if (pwrite(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
708 error_report("%s(,0x%"TARGET_PRIxPHYS
", 0x%"PRIx64
", %d) failed: %m\n",
709 __func__
, addr
, data
, size
);
712 DPRINTF("%s(BAR%d+0x%"TARGET_PRIxPHYS
", 0x%"PRIx64
", %d)\n",
713 __func__
, bar
->nr
, addr
, data
, size
);
716 * A read or write to a BAR always signals an INTx EOI. This will
717 * do nothing if not pending (including not in INTx mode). We assume
718 * that a BAR access is in response to an interrupt and that BAR
719 * accesses will service the interrupt. Unfortunately, we don't know
720 * which access will service the interrupt, so we're potentially
721 * getting quite a few host interrupts per guest interrupt.
723 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
726 static uint64_t vfio_bar_read(void *opaque
,
727 target_phys_addr_t addr
, unsigned size
)
729 VFIOBAR
*bar
= opaque
;
738 if (pread(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
739 error_report("%s(,0x%"TARGET_PRIxPHYS
", %d) failed: %m\n",
740 __func__
, addr
, size
);
749 data
= le16_to_cpu(buf
.word
);
752 data
= le32_to_cpu(buf
.dword
);
755 hw_error("vfio: unsupported read size, %d bytes\n", size
);
759 DPRINTF("%s(BAR%d+0x%"TARGET_PRIxPHYS
", %d) = 0x%"PRIx64
"\n",
760 __func__
, bar
->nr
, addr
, size
, data
);
762 /* Same as write above */
763 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
768 static const MemoryRegionOps vfio_bar_ops
= {
769 .read
= vfio_bar_read
,
770 .write
= vfio_bar_write
,
771 .endianness
= DEVICE_LITTLE_ENDIAN
,
777 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
779 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
783 * We only need QEMU PCI config support for the ROM BAR, the MSI and MSIX
784 * capabilities, and the multifunction bit below. We let VFIO handle
785 * virtualizing everything else. Performance is not a concern here.
787 if (ranges_overlap(addr
, len
, PCI_ROM_ADDRESS
, 4) ||
788 (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
789 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) ||
790 (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
791 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
))) {
793 val
= pci_default_read_config(pdev
, addr
, len
);
795 if (pread(vdev
->fd
, &val
, len
, vdev
->config_offset
+ addr
) != len
) {
796 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m\n",
797 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
798 vdev
->host
.slot
, vdev
->host
.function
, addr
, len
);
801 val
= le32_to_cpu(val
);
804 /* Multifunction bit is virualized in QEMU */
805 if (unlikely(ranges_overlap(addr
, len
, PCI_HEADER_TYPE
, 1))) {
806 uint32_t mask
= PCI_HEADER_TYPE_MULTI_FUNCTION
;
812 if (pdev
->cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
819 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__
,
820 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
821 vdev
->host
.function
, addr
, len
, val
);
826 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
827 uint32_t val
, int len
)
829 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
830 uint32_t val_le
= cpu_to_le32(val
);
832 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__
,
833 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
834 vdev
->host
.function
, addr
, val
, len
);
836 /* Write everything to VFIO, let it filter out what we can't write */
837 if (pwrite(vdev
->fd
, &val_le
, len
, vdev
->config_offset
+ addr
) != len
) {
838 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m\n",
839 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
840 vdev
->host
.slot
, vdev
->host
.function
, addr
, val
, len
);
843 /* Write standard header bits to emulation */
844 if (addr
< PCI_CONFIG_HEADER_SIZE
) {
845 pci_default_write_config(pdev
, addr
, val
, len
);
849 /* MSI/MSI-X Enabling/Disabling */
850 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
851 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
852 int is_enabled
, was_enabled
= msi_enabled(pdev
);
854 pci_default_write_config(pdev
, addr
, val
, len
);
856 is_enabled
= msi_enabled(pdev
);
858 if (!was_enabled
&& is_enabled
) {
859 vfio_enable_msi(vdev
);
860 } else if (was_enabled
&& !is_enabled
) {
861 vfio_disable_msi(vdev
);
865 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
866 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
867 int is_enabled
, was_enabled
= msix_enabled(pdev
);
869 pci_default_write_config(pdev
, addr
, val
, len
);
871 is_enabled
= msix_enabled(pdev
);
873 if (!was_enabled
&& is_enabled
) {
874 vfio_enable_msix(vdev
);
875 } else if (was_enabled
&& !is_enabled
) {
876 vfio_disable_msix(vdev
);
882 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
884 static int vfio_dma_unmap(VFIOContainer
*container
,
885 target_phys_addr_t iova
, ram_addr_t size
)
887 struct vfio_iommu_type1_dma_unmap unmap
= {
888 .argsz
= sizeof(unmap
),
894 if (ioctl(container
->fd
, VFIO_IOMMU_UNMAP_DMA
, &unmap
)) {
895 DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno
);
902 static int vfio_dma_map(VFIOContainer
*container
, target_phys_addr_t iova
,
903 ram_addr_t size
, void *vaddr
, bool readonly
)
905 struct vfio_iommu_type1_dma_map map
= {
906 .argsz
= sizeof(map
),
907 .flags
= VFIO_DMA_MAP_FLAG_READ
,
908 .vaddr
= (__u64
)(uintptr_t)vaddr
,
914 map
.flags
|= VFIO_DMA_MAP_FLAG_WRITE
;
918 * Try the mapping, if it fails with EBUSY, unmap the region and try
919 * again. This shouldn't be necessary, but we sometimes see it in
920 * the the VGA ROM space.
922 if (ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0 ||
923 (errno
== EBUSY
&& vfio_dma_unmap(container
, iova
, size
) == 0 &&
924 ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0)) {
928 DPRINTF("VFIO_MAP_DMA: %d\n", -errno
);
932 static void vfio_listener_dummy1(MemoryListener
*listener
)
934 /* We don't do batching (begin/commit) or care about logging */
937 static void vfio_listener_dummy2(MemoryListener
*listener
,
938 MemoryRegionSection
*section
)
940 /* We don't do logging or care about nops */
943 static void vfio_listener_dummy3(MemoryListener
*listener
,
944 MemoryRegionSection
*section
,
945 bool match_data
, uint64_t data
,
948 /* We don't care about eventfds */
951 static bool vfio_listener_skipped_section(MemoryRegionSection
*section
)
953 return !memory_region_is_ram(section
->mr
);
956 static void vfio_listener_region_add(MemoryListener
*listener
,
957 MemoryRegionSection
*section
)
959 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
960 iommu_data
.listener
);
961 target_phys_addr_t iova
, end
;
965 if (vfio_listener_skipped_section(section
)) {
966 DPRINTF("vfio: SKIPPING region_add %"TARGET_PRIxPHYS
" - %"PRIx64
"\n",
967 section
->offset_within_address_space
,
968 section
->offset_within_address_space
+ section
->size
- 1);
972 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
973 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
974 error_report("%s received unaligned region\n", __func__
);
978 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
979 end
= (section
->offset_within_address_space
+ section
->size
) &
986 vaddr
= memory_region_get_ram_ptr(section
->mr
) +
987 section
->offset_within_region
+
988 (iova
- section
->offset_within_address_space
);
990 DPRINTF("vfio: region_add %"TARGET_PRIxPHYS
" - %"TARGET_PRIxPHYS
" [%p]\n",
991 iova
, end
- 1, vaddr
);
993 ret
= vfio_dma_map(container
, iova
, end
- iova
, vaddr
, section
->readonly
);
995 error_report("vfio_dma_map(%p, 0x%"TARGET_PRIxPHYS
", "
996 "0x%"TARGET_PRIxPHYS
", %p) = %d (%m)\n",
997 container
, iova
, end
- iova
, vaddr
, ret
);
1001 static void vfio_listener_region_del(MemoryListener
*listener
,
1002 MemoryRegionSection
*section
)
1004 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
1005 iommu_data
.listener
);
1006 target_phys_addr_t iova
, end
;
1009 if (vfio_listener_skipped_section(section
)) {
1010 DPRINTF("vfio: SKIPPING region_del %"TARGET_PRIxPHYS
" - %"PRIx64
"\n",
1011 section
->offset_within_address_space
,
1012 section
->offset_within_address_space
+ section
->size
- 1);
1016 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
1017 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
1018 error_report("%s received unaligned region\n", __func__
);
1022 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
1023 end
= (section
->offset_within_address_space
+ section
->size
) &
1030 DPRINTF("vfio: region_del %"TARGET_PRIxPHYS
" - %"TARGET_PRIxPHYS
"\n",
1033 ret
= vfio_dma_unmap(container
, iova
, end
- iova
);
1035 error_report("vfio_dma_unmap(%p, 0x%"TARGET_PRIxPHYS
", "
1036 "0x%"TARGET_PRIxPHYS
") = %d (%m)\n",
1037 container
, iova
, end
- iova
, ret
);
1041 static MemoryListener vfio_memory_listener
= {
1042 .begin
= vfio_listener_dummy1
,
1043 .commit
= vfio_listener_dummy1
,
1044 .region_add
= vfio_listener_region_add
,
1045 .region_del
= vfio_listener_region_del
,
1046 .region_nop
= vfio_listener_dummy2
,
1047 .log_start
= vfio_listener_dummy2
,
1048 .log_stop
= vfio_listener_dummy2
,
1049 .log_sync
= vfio_listener_dummy2
,
1050 .log_global_start
= vfio_listener_dummy1
,
1051 .log_global_stop
= vfio_listener_dummy1
,
1052 .eventfd_add
= vfio_listener_dummy3
,
1053 .eventfd_del
= vfio_listener_dummy3
,
1056 static void vfio_listener_release(VFIOContainer
*container
)
1058 memory_listener_unregister(&container
->iommu_data
.listener
);
1064 static void vfio_disable_interrupts(VFIODevice
*vdev
)
1066 switch (vdev
->interrupt
) {
1068 vfio_disable_intx(vdev
);
1071 vfio_disable_msi(vdev
);
1074 vfio_disable_msix(vdev
);
1079 static int vfio_setup_msi(VFIODevice
*vdev
, int pos
)
1082 bool msi_64bit
, msi_maskbit
;
1085 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
1086 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
1089 ctrl
= le16_to_cpu(ctrl
);
1091 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
1092 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
1093 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
1095 DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev
->host
.domain
,
1096 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, pos
);
1098 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
);
1100 if (ret
== -ENOTSUP
) {
1103 error_report("vfio: msi_init failed\n");
1106 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
1112 * We don't have any control over how pci_add_capability() inserts
1113 * capabilities into the chain. In order to setup MSI-X we need a
1114 * MemoryRegion for the BAR. In order to setup the BAR and not
1115 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1116 * need to first look for where the MSI-X table lives. So we
1117 * unfortunately split MSI-X setup across two functions.
1119 static int vfio_early_setup_msix(VFIODevice
*vdev
)
1123 uint32_t table
, pba
;
1125 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
1130 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
1131 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
1135 if (pread(vdev
->fd
, &table
, sizeof(table
),
1136 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
1140 if (pread(vdev
->fd
, &pba
, sizeof(pba
),
1141 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
1145 ctrl
= le16_to_cpu(ctrl
);
1146 table
= le32_to_cpu(table
);
1147 pba
= le32_to_cpu(pba
);
1149 vdev
->msix
= g_malloc0(sizeof(*(vdev
->msix
)));
1150 vdev
->msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
1151 vdev
->msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
1152 vdev
->msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
1153 vdev
->msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
1154 vdev
->msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
1156 DPRINTF("%04x:%02x:%02x.%x "
1157 "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
1158 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1159 vdev
->host
.function
, pos
, vdev
->msix
->table_bar
,
1160 vdev
->msix
->table_offset
, vdev
->msix
->entries
);
1165 static int vfio_setup_msix(VFIODevice
*vdev
, int pos
)
1169 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
1170 &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
1171 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
1172 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
,
1173 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
);
1175 if (ret
== -ENOTSUP
) {
1178 error_report("vfio: msix_init failed\n");
1185 static void vfio_teardown_msi(VFIODevice
*vdev
)
1187 msi_uninit(&vdev
->pdev
);
1190 msix_uninit(&vdev
->pdev
, &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
1191 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
);
1198 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
)
1202 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1203 VFIOBAR
*bar
= &vdev
->bars
[i
];
1209 memory_region_set_enabled(&bar
->mmap_mem
, enabled
);
1210 if (vdev
->msix
&& vdev
->msix
->table_bar
== i
) {
1211 memory_region_set_enabled(&vdev
->msix
->mmap_mem
, enabled
);
1216 static void vfio_unmap_bar(VFIODevice
*vdev
, int nr
)
1218 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1224 memory_region_del_subregion(&bar
->mem
, &bar
->mmap_mem
);
1225 munmap(bar
->mmap
, memory_region_size(&bar
->mmap_mem
));
1227 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
1228 memory_region_del_subregion(&bar
->mem
, &vdev
->msix
->mmap_mem
);
1229 munmap(vdev
->msix
->mmap
, memory_region_size(&vdev
->msix
->mmap_mem
));
1232 memory_region_destroy(&bar
->mem
);
1235 static int vfio_mmap_bar(VFIOBAR
*bar
, MemoryRegion
*mem
, MemoryRegion
*submem
,
1236 void **map
, size_t size
, off_t offset
,
1241 if (size
&& bar
->flags
& VFIO_REGION_INFO_FLAG_MMAP
) {
1244 if (bar
->flags
& VFIO_REGION_INFO_FLAG_READ
) {
1248 if (bar
->flags
& VFIO_REGION_INFO_FLAG_WRITE
) {
1252 *map
= mmap(NULL
, size
, prot
, MAP_SHARED
,
1253 bar
->fd
, bar
->fd_offset
+ offset
);
1254 if (*map
== MAP_FAILED
) {
1260 memory_region_init_ram_ptr(submem
, name
, size
, *map
);
1263 /* Create a zero sized sub-region to make cleanup easy. */
1264 memory_region_init(submem
, name
, 0);
1267 memory_region_add_subregion(mem
, offset
, submem
);
1272 static void vfio_map_bar(VFIODevice
*vdev
, int nr
)
1274 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1275 unsigned size
= bar
->size
;
1281 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1286 snprintf(name
, sizeof(name
), "VFIO %04x:%02x:%02x.%x BAR %d",
1287 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1288 vdev
->host
.function
, nr
);
1290 /* Determine what type of BAR this is for registration */
1291 ret
= pread(vdev
->fd
, &pci_bar
, sizeof(pci_bar
),
1292 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
1293 if (ret
!= sizeof(pci_bar
)) {
1294 error_report("vfio: Failed to read BAR %d (%m)\n", nr
);
1298 pci_bar
= le32_to_cpu(pci_bar
);
1299 type
= pci_bar
& (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
?
1300 ~PCI_BASE_ADDRESS_IO_MASK
: ~PCI_BASE_ADDRESS_MEM_MASK
);
1302 /* A "slow" read/write mapping underlies all BARs */
1303 memory_region_init_io(&bar
->mem
, &vfio_bar_ops
, bar
, name
, size
);
1304 pci_register_bar(&vdev
->pdev
, nr
, type
, &bar
->mem
);
1307 * We can't mmap areas overlapping the MSIX vector table, so we
1308 * potentially insert a direct-mapped subregion before and after it.
1310 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
1311 size
= vdev
->msix
->table_offset
& TARGET_PAGE_MASK
;
1314 strncat(name
, " mmap", sizeof(name
) - strlen(name
) - 1);
1315 if (vfio_mmap_bar(bar
, &bar
->mem
,
1316 &bar
->mmap_mem
, &bar
->mmap
, size
, 0, name
)) {
1317 error_report("%s unsupported. Performance may be slow\n", name
);
1320 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
1323 start
= TARGET_PAGE_ALIGN(vdev
->msix
->table_offset
+
1324 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
1326 size
= start
< bar
->size
? bar
->size
- start
: 0;
1327 strncat(name
, " msix-hi", sizeof(name
) - strlen(name
) - 1);
1328 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
1329 if (vfio_mmap_bar(bar
, &bar
->mem
, &vdev
->msix
->mmap_mem
,
1330 &vdev
->msix
->mmap
, size
, start
, name
)) {
1331 error_report("%s unsupported. Performance may be slow\n", name
);
1336 static void vfio_map_bars(VFIODevice
*vdev
)
1340 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1341 vfio_map_bar(vdev
, i
);
1345 static void vfio_unmap_bars(VFIODevice
*vdev
)
1349 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1350 vfio_unmap_bar(vdev
, i
);
1357 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
1359 uint8_t tmp
, next
= 0xff;
1361 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
1362 tmp
= pdev
->config
[tmp
+ 1]) {
1363 if (tmp
> pos
&& tmp
< next
) {
1371 static int vfio_add_std_cap(VFIODevice
*vdev
, uint8_t pos
)
1373 PCIDevice
*pdev
= &vdev
->pdev
;
1374 uint8_t cap_id
, next
, size
;
1377 cap_id
= pdev
->config
[pos
];
1378 next
= pdev
->config
[pos
+ 1];
1381 * If it becomes important to configure capabilities to their actual
1382 * size, use this as the default when it's something we don't recognize.
1383 * Since QEMU doesn't actually handle many of the config accesses,
1384 * exact size doesn't seem worthwhile.
1386 size
= vfio_std_cap_max_size(pdev
, pos
);
1389 * pci_add_capability always inserts the new capability at the head
1390 * of the chain. Therefore to end up with a chain that matches the
1391 * physical device, we insert from the end by making this recursive.
1392 * This is also why we pre-caclulate size above as cached config space
1393 * will be changed as we unwind the stack.
1396 ret
= vfio_add_std_cap(vdev
, next
);
1401 pdev
->config
[PCI_CAPABILITY_LIST
] = 0; /* Begin the rebuild */
1405 case PCI_CAP_ID_MSI
:
1406 ret
= vfio_setup_msi(vdev
, pos
);
1408 case PCI_CAP_ID_MSIX
:
1409 ret
= vfio_setup_msix(vdev
, pos
);
1412 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
1417 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
1418 "0x%x[0x%x]@0x%x: %d\n", vdev
->host
.domain
,
1419 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1420 cap_id
, size
, pos
, ret
);
1427 static int vfio_add_capabilities(VFIODevice
*vdev
)
1429 PCIDevice
*pdev
= &vdev
->pdev
;
1431 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
1432 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
1433 return 0; /* Nothing to add */
1436 return vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
]);
1439 static int vfio_load_rom(VFIODevice
*vdev
)
1441 uint64_t size
= vdev
->rom_size
;
1443 off_t off
= 0, voff
= vdev
->rom_offset
;
1447 /* If loading ROM from file, pci handles it */
1448 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
|| !size
) {
1452 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
1453 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1455 snprintf(name
, sizeof(name
), "vfio[%04x:%02x:%02x.%x].rom",
1456 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1457 vdev
->host
.function
);
1458 memory_region_init_ram(&vdev
->pdev
.rom
, name
, size
);
1459 ptr
= memory_region_get_ram_ptr(&vdev
->pdev
.rom
);
1460 memset(ptr
, 0xff, size
);
1463 bytes
= pread(vdev
->fd
, ptr
+ off
, size
, voff
+ off
);
1465 break; /* expect that we could get back less than the ROM BAR */
1466 } else if (bytes
> 0) {
1470 if (errno
== EINTR
|| errno
== EAGAIN
) {
1473 error_report("vfio: Error reading device ROM: %m\n");
1474 memory_region_destroy(&vdev
->pdev
.rom
);
1479 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
, 0, &vdev
->pdev
.rom
);
1480 vdev
->pdev
.has_rom
= true;
1484 static int vfio_connect_container(VFIOGroup
*group
)
1486 VFIOContainer
*container
;
1489 if (group
->container
) {
1493 QLIST_FOREACH(container
, &container_list
, next
) {
1494 if (!ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &container
->fd
)) {
1495 group
->container
= container
;
1496 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
1501 fd
= qemu_open("/dev/vfio/vfio", O_RDWR
);
1503 error_report("vfio: failed to open /dev/vfio/vfio: %m\n");
1507 ret
= ioctl(fd
, VFIO_GET_API_VERSION
);
1508 if (ret
!= VFIO_API_VERSION
) {
1509 error_report("vfio: supported vfio version: %d, "
1510 "reported version: %d\n", VFIO_API_VERSION
, ret
);
1515 container
= g_malloc0(sizeof(*container
));
1518 if (ioctl(fd
, VFIO_CHECK_EXTENSION
, VFIO_TYPE1_IOMMU
)) {
1519 ret
= ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &fd
);
1521 error_report("vfio: failed to set group container: %m\n");
1527 ret
= ioctl(fd
, VFIO_SET_IOMMU
, VFIO_TYPE1_IOMMU
);
1529 error_report("vfio: failed to set iommu for container: %m\n");
1535 container
->iommu_data
.listener
= vfio_memory_listener
;
1536 container
->iommu_data
.release
= vfio_listener_release
;
1538 memory_listener_register(&container
->iommu_data
.listener
,
1539 get_system_memory());
1541 error_report("vfio: No available IOMMU models\n");
1547 QLIST_INIT(&container
->group_list
);
1548 QLIST_INSERT_HEAD(&container_list
, container
, next
);
1550 group
->container
= container
;
1551 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
1556 static void vfio_disconnect_container(VFIOGroup
*group
)
1558 VFIOContainer
*container
= group
->container
;
1560 if (ioctl(group
->fd
, VFIO_GROUP_UNSET_CONTAINER
, &container
->fd
)) {
1561 error_report("vfio: error disconnecting group %d from container\n",
1565 QLIST_REMOVE(group
, container_next
);
1566 group
->container
= NULL
;
1568 if (QLIST_EMPTY(&container
->group_list
)) {
1569 if (container
->iommu_data
.release
) {
1570 container
->iommu_data
.release(container
);
1572 QLIST_REMOVE(container
, next
);
1573 DPRINTF("vfio_disconnect_container: close container->fd\n");
1574 close(container
->fd
);
1579 static VFIOGroup
*vfio_get_group(int groupid
)
1583 struct vfio_group_status status
= { .argsz
= sizeof(status
) };
1585 QLIST_FOREACH(group
, &group_list
, next
) {
1586 if (group
->groupid
== groupid
) {
1591 group
= g_malloc0(sizeof(*group
));
1593 snprintf(path
, sizeof(path
), "/dev/vfio/%d", groupid
);
1594 group
->fd
= qemu_open(path
, O_RDWR
);
1595 if (group
->fd
< 0) {
1596 error_report("vfio: error opening %s: %m\n", path
);
1601 if (ioctl(group
->fd
, VFIO_GROUP_GET_STATUS
, &status
)) {
1602 error_report("vfio: error getting group status: %m\n");
1608 if (!(status
.flags
& VFIO_GROUP_FLAGS_VIABLE
)) {
1609 error_report("vfio: error, group %d is not viable, please ensure "
1610 "all devices within the iommu_group are bound to their "
1611 "vfio bus driver.\n", groupid
);
1617 group
->groupid
= groupid
;
1618 QLIST_INIT(&group
->device_list
);
1620 if (vfio_connect_container(group
)) {
1621 error_report("vfio: failed to setup container for group %d\n", groupid
);
1627 QLIST_INSERT_HEAD(&group_list
, group
, next
);
1632 static void vfio_put_group(VFIOGroup
*group
)
1634 if (!QLIST_EMPTY(&group
->device_list
)) {
1638 vfio_disconnect_container(group
);
1639 QLIST_REMOVE(group
, next
);
1640 DPRINTF("vfio_put_group: close group->fd\n");
1645 static int vfio_get_device(VFIOGroup
*group
, const char *name
, VFIODevice
*vdev
)
1647 struct vfio_device_info dev_info
= { .argsz
= sizeof(dev_info
) };
1648 struct vfio_region_info reg_info
= { .argsz
= sizeof(reg_info
) };
1651 ret
= ioctl(group
->fd
, VFIO_GROUP_GET_DEVICE_FD
, name
);
1653 error_report("vfio: error getting device %s from group %d: %m\n",
1654 name
, group
->groupid
);
1655 error_report("Verify all devices in group %d are bound to vfio-pci "
1656 "or pci-stub and not already in use\n", group
->groupid
);
1661 vdev
->group
= group
;
1662 QLIST_INSERT_HEAD(&group
->device_list
, vdev
, next
);
1664 /* Sanity check device */
1665 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_INFO
, &dev_info
);
1667 error_report("vfio: error getting device info: %m\n");
1671 DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name
,
1672 dev_info
.flags
, dev_info
.num_regions
, dev_info
.num_irqs
);
1674 if (!(dev_info
.flags
& VFIO_DEVICE_FLAGS_PCI
)) {
1675 error_report("vfio: Um, this isn't a PCI device\n");
1679 vdev
->reset_works
= !!(dev_info
.flags
& VFIO_DEVICE_FLAGS_RESET
);
1680 if (!vdev
->reset_works
) {
1681 error_report("Warning, device %s does not support reset\n", name
);
1684 if (dev_info
.num_regions
!= VFIO_PCI_NUM_REGIONS
) {
1685 error_report("vfio: unexpected number of io regions %u\n",
1686 dev_info
.num_regions
);
1690 if (dev_info
.num_irqs
!= VFIO_PCI_NUM_IRQS
) {
1691 error_report("vfio: unexpected number of irqs %u\n", dev_info
.num_irqs
);
1695 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
1698 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
1700 error_report("vfio: Error getting region %d info: %m\n", i
);
1704 DPRINTF("Device %s region %d:\n", name
, i
);
1705 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1706 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
1707 (unsigned long)reg_info
.flags
);
1709 vdev
->bars
[i
].flags
= reg_info
.flags
;
1710 vdev
->bars
[i
].size
= reg_info
.size
;
1711 vdev
->bars
[i
].fd_offset
= reg_info
.offset
;
1712 vdev
->bars
[i
].fd
= vdev
->fd
;
1713 vdev
->bars
[i
].nr
= i
;
1716 reg_info
.index
= VFIO_PCI_ROM_REGION_INDEX
;
1718 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
1720 error_report("vfio: Error getting ROM info: %m\n");
1724 DPRINTF("Device %s ROM:\n", name
);
1725 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1726 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
1727 (unsigned long)reg_info
.flags
);
1729 vdev
->rom_size
= reg_info
.size
;
1730 vdev
->rom_offset
= reg_info
.offset
;
1732 reg_info
.index
= VFIO_PCI_CONFIG_REGION_INDEX
;
1734 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
1736 error_report("vfio: Error getting config info: %m\n");
1740 DPRINTF("Device %s config:\n", name
);
1741 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1742 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
1743 (unsigned long)reg_info
.flags
);
1745 vdev
->config_size
= reg_info
.size
;
1746 vdev
->config_offset
= reg_info
.offset
;
1750 QLIST_REMOVE(vdev
, next
);
1757 static void vfio_put_device(VFIODevice
*vdev
)
1759 QLIST_REMOVE(vdev
, next
);
1761 DPRINTF("vfio_put_device: close vdev->fd\n");
1769 static int vfio_initfn(PCIDevice
*pdev
)
1771 VFIODevice
*pvdev
, *vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
1773 char path
[PATH_MAX
], iommu_group_path
[PATH_MAX
], *group_name
;
1779 /* Check that the host device exists */
1780 snprintf(path
, sizeof(path
),
1781 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
1782 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1783 vdev
->host
.function
);
1784 if (stat(path
, &st
) < 0) {
1785 error_report("vfio: error: no such host device: %s\n", path
);
1789 strncat(path
, "iommu_group", sizeof(path
) - strlen(path
) - 1);
1791 len
= readlink(path
, iommu_group_path
, PATH_MAX
);
1793 error_report("vfio: error no iommu_group for device\n");
1797 iommu_group_path
[len
] = 0;
1798 group_name
= basename(iommu_group_path
);
1800 if (sscanf(group_name
, "%d", &groupid
) != 1) {
1801 error_report("vfio: error reading %s: %m\n", path
);
1805 DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__
, vdev
->host
.domain
,
1806 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, groupid
);
1808 group
= vfio_get_group(groupid
);
1810 error_report("vfio: failed to get group %d\n", groupid
);
1814 snprintf(path
, sizeof(path
), "%04x:%02x:%02x.%01x",
1815 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1816 vdev
->host
.function
);
1818 QLIST_FOREACH(pvdev
, &group
->device_list
, next
) {
1819 if (pvdev
->host
.domain
== vdev
->host
.domain
&&
1820 pvdev
->host
.bus
== vdev
->host
.bus
&&
1821 pvdev
->host
.slot
== vdev
->host
.slot
&&
1822 pvdev
->host
.function
== vdev
->host
.function
) {
1824 error_report("vfio: error: device %s is already attached\n", path
);
1825 vfio_put_group(group
);
1830 ret
= vfio_get_device(group
, path
, vdev
);
1832 error_report("vfio: failed to get device %s\n", path
);
1833 vfio_put_group(group
);
1837 /* Get a copy of config space */
1838 ret
= pread(vdev
->fd
, vdev
->pdev
.config
,
1839 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
1840 vdev
->config_offset
);
1841 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
1842 ret
= ret
< 0 ? -errno
: -EFAULT
;
1843 error_report("vfio: Failed to read device config space\n");
1848 * Clear host resource mapping info. If we choose not to register a
1849 * BAR, such as might be the case with the option ROM, we can get
1850 * confusing, unwritable, residual addresses from the host here.
1852 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
1853 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
1855 vfio_load_rom(vdev
);
1857 ret
= vfio_early_setup_msix(vdev
);
1862 vfio_map_bars(vdev
);
1864 ret
= vfio_add_capabilities(vdev
);
1869 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
1870 vdev
->intx
.mmap_timer
= qemu_new_timer_ms(vm_clock
,
1871 vfio_intx_mmap_enable
, vdev
);
1872 ret
= vfio_enable_intx(vdev
);
1881 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
1882 vfio_teardown_msi(vdev
);
1883 vfio_unmap_bars(vdev
);
1885 vfio_put_device(vdev
);
1886 vfio_put_group(group
);
1890 static void vfio_exitfn(PCIDevice
*pdev
)
1892 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
1893 VFIOGroup
*group
= vdev
->group
;
1895 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
1896 vfio_disable_interrupts(vdev
);
1897 if (vdev
->intx
.mmap_timer
) {
1898 qemu_free_timer(vdev
->intx
.mmap_timer
);
1900 vfio_teardown_msi(vdev
);
1901 vfio_unmap_bars(vdev
);
1902 vfio_put_device(vdev
);
1903 vfio_put_group(group
);
1906 static void vfio_pci_reset(DeviceState
*dev
)
1908 PCIDevice
*pdev
= DO_UPCAST(PCIDevice
, qdev
, dev
);
1909 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
1912 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
1913 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1915 vfio_disable_interrupts(vdev
);
1918 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
1919 * Also put INTx Disable in known state.
1921 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
1922 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
1923 PCI_COMMAND_INTX_DISABLE
);
1924 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
1926 if (vdev
->reset_works
) {
1927 if (ioctl(vdev
->fd
, VFIO_DEVICE_RESET
)) {
1928 error_report("vfio: Error unable to reset physical device "
1929 "(%04x:%02x:%02x.%x): %m\n", vdev
->host
.domain
,
1930 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1934 vfio_enable_intx(vdev
);
1937 static Property vfio_pci_dev_properties
[] = {
1938 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice
, host
),
1939 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice
,
1940 intx
.mmap_timeout
, 1100),
1942 * TODO - support passed fds... is this necessary?
1943 * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
1944 * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
1946 DEFINE_PROP_END_OF_LIST(),
1949 static const VMStateDescription vfio_pci_vmstate
= {
1954 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
1956 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1957 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
1959 dc
->reset
= vfio_pci_reset
;
1960 dc
->props
= vfio_pci_dev_properties
;
1961 dc
->vmsd
= &vfio_pci_vmstate
;
1962 dc
->desc
= "VFIO-based PCI device assignment";
1963 pdc
->init
= vfio_initfn
;
1964 pdc
->exit
= vfio_exitfn
;
1965 pdc
->config_read
= vfio_pci_read_config
;
1966 pdc
->config_write
= vfio_pci_write_config
;
1969 static const TypeInfo vfio_pci_dev_info
= {
1971 .parent
= TYPE_PCI_DEVICE
,
1972 .instance_size
= sizeof(VFIODevice
),
1973 .class_init
= vfio_pci_dev_class_init
,
1976 static void register_vfio_pci_dev_type(void)
1978 type_register_static(&vfio_pci_dev_info
);
1981 type_init(register_vfio_pci_dev_type
)