2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
26 #include <sys/types.h>
30 #include "exec/address-spaces.h"
31 #include "exec/memory.h"
32 #include "hw/pci/msi.h"
33 #include "hw/pci/msix.h"
34 #include "hw/pci/pci.h"
35 #include "qemu-common.h"
36 #include "qemu/error-report.h"
37 #include "qemu/event_notifier.h"
38 #include "qemu/queue.h"
39 #include "qemu/range.h"
40 #include "sysemu/kvm.h"
41 #include "sysemu/sysemu.h"
43 /* #define DEBUG_VFIO */
45 #define DPRINTF(fmt, ...) \
46 do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
48 #define DPRINTF(fmt, ...) \
52 /* Extra debugging, trap acceleration paths for more logging */
53 #define VFIO_ALLOW_MMAP 1
54 #define VFIO_ALLOW_KVM_INTX 1
55 #define VFIO_ALLOW_KVM_MSI 1
56 #define VFIO_ALLOW_KVM_MSIX 1
60 typedef struct VFIOQuirk
{
62 struct VFIODevice
*vdev
;
63 QLIST_ENTRY(VFIOQuirk
) next
;
65 uint32_t base_offset
:TARGET_PAGE_BITS
;
66 uint32_t address_offset
:TARGET_PAGE_BITS
;
67 uint32_t address_size
:3;
70 uint32_t address_match
;
71 uint32_t address_mask
;
73 uint32_t address_val
:TARGET_PAGE_BITS
;
74 uint32_t data_offset
:TARGET_PAGE_BITS
;
83 typedef struct VFIOBAR
{
84 off_t fd_offset
; /* offset of BAR within device fd */
85 int fd
; /* device fd, allows us to pass VFIOBAR as opaque data */
86 MemoryRegion mem
; /* slow, read/write access */
87 MemoryRegion mmap_mem
; /* direct mapped access */
90 uint32_t flags
; /* VFIO region flags (rd/wr/mmap) */
91 uint8_t nr
; /* cache the BAR number for debug */
94 QLIST_HEAD(, VFIOQuirk
) quirks
;
97 typedef struct VFIOVGARegion
{
101 QLIST_HEAD(, VFIOQuirk
) quirks
;
104 typedef struct VFIOVGA
{
107 VFIOVGARegion region
[QEMU_PCI_VGA_NUM_REGIONS
];
110 typedef struct VFIOINTx
{
111 bool pending
; /* interrupt pending */
112 bool kvm_accel
; /* set when QEMU bypass through KVM enabled */
113 uint8_t pin
; /* which pin to pull for qemu_set_irq */
114 EventNotifier interrupt
; /* eventfd triggered on interrupt */
115 EventNotifier unmask
; /* eventfd for unmask on QEMU bypass */
116 PCIINTxRoute route
; /* routing info for QEMU bypass */
117 uint32_t mmap_timeout
; /* delay to re-enable mmaps after interrupt */
118 QEMUTimer
*mmap_timer
; /* enable mmaps after periods w/o interrupts */
121 typedef struct VFIOMSIVector
{
122 EventNotifier interrupt
; /* eventfd triggered on interrupt */
123 struct VFIODevice
*vdev
; /* back pointer to device */
124 MSIMessage msg
; /* cache the MSI message so we know when it changes */
125 int virq
; /* KVM irqchip route for QEMU bypass */
138 typedef struct VFIOContainer
{
139 int fd
; /* /dev/vfio/vfio, empowered by the attached groups */
141 /* enable abstraction to support various iommu backends */
143 MemoryListener listener
; /* Used by type1 iommu */
145 void (*release
)(struct VFIOContainer
*);
147 QLIST_HEAD(, VFIOGroup
) group_list
;
148 QLIST_ENTRY(VFIOContainer
) next
;
151 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
152 typedef struct VFIOMSIXInfo
{
156 uint32_t table_offset
;
158 MemoryRegion mmap_mem
;
162 typedef struct VFIODevice
{
166 unsigned int config_size
;
167 uint8_t *emulated_config_bits
; /* QEMU emulated bits, little-endian */
168 off_t config_offset
; /* Offset of config space region within device fd */
169 unsigned int rom_size
;
170 off_t rom_offset
; /* Offset of ROM region within device fd */
173 VFIOMSIVector
*msi_vectors
;
175 int nr_vectors
; /* Number of MSI/MSIX vectors currently in use */
176 int interrupt
; /* Current interrupt type */
177 VFIOBAR bars
[PCI_NUM_REGIONS
- 1]; /* No ROM */
178 VFIOVGA vga
; /* 0xa0000, 0x3b0, 0x3c0 */
179 PCIHostDeviceAddress host
;
180 QLIST_ENTRY(VFIODevice
) next
;
181 struct VFIOGroup
*group
;
182 EventNotifier err_notifier
;
184 #define VFIO_FEATURE_ENABLE_VGA_BIT 0
185 #define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
196 typedef struct VFIOGroup
{
199 VFIOContainer
*container
;
200 QLIST_HEAD(, VFIODevice
) device_list
;
201 QLIST_ENTRY(VFIOGroup
) next
;
202 QLIST_ENTRY(VFIOGroup
) container_next
;
205 #define MSIX_CAP_LENGTH 12
207 static QLIST_HEAD(, VFIOContainer
)
208 container_list
= QLIST_HEAD_INITIALIZER(container_list
);
210 static QLIST_HEAD(, VFIOGroup
)
211 group_list
= QLIST_HEAD_INITIALIZER(group_list
);
215 * We have a single VFIO pseudo device per KVM VM. Once created it lives
216 * for the life of the VM. Closing the file descriptor only drops our
217 * reference to it and the device's reference to kvm. Therefore once
218 * initialized, this file descriptor is only released on QEMU exit and
219 * we'll re-use it should another vfio device be attached before then.
221 static int vfio_kvm_device_fd
= -1;
224 static void vfio_disable_interrupts(VFIODevice
*vdev
);
225 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
);
226 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
227 uint32_t val
, int len
);
228 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
);
231 * Common VFIO interrupt disable
233 static void vfio_disable_irqindex(VFIODevice
*vdev
, int index
)
235 struct vfio_irq_set irq_set
= {
236 .argsz
= sizeof(irq_set
),
237 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_TRIGGER
,
243 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
249 static void vfio_unmask_intx(VFIODevice
*vdev
)
251 struct vfio_irq_set irq_set
= {
252 .argsz
= sizeof(irq_set
),
253 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_UNMASK
,
254 .index
= VFIO_PCI_INTX_IRQ_INDEX
,
259 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
262 #ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
263 static void vfio_mask_intx(VFIODevice
*vdev
)
265 struct vfio_irq_set irq_set
= {
266 .argsz
= sizeof(irq_set
),
267 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_MASK
,
268 .index
= VFIO_PCI_INTX_IRQ_INDEX
,
273 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
278 * Disabling BAR mmaping can be slow, but toggling it around INTx can
279 * also be a huge overhead. We try to get the best of both worlds by
280 * waiting until an interrupt to disable mmaps (subsequent transitions
281 * to the same state are effectively no overhead). If the interrupt has
282 * been serviced and the time gap is long enough, we re-enable mmaps for
283 * performance. This works well for things like graphics cards, which
284 * may not use their interrupt at all and are penalized to an unusable
285 * level by read/write BAR traps. Other devices, like NICs, have more
286 * regular interrupts and see much better latency by staying in non-mmap
287 * mode. We therefore set the default mmap_timeout such that a ping
288 * is just enough to keep the mmap disabled. Users can experiment with
289 * other options with the x-intx-mmap-timeout-ms parameter (a value of
290 * zero disables the timer).
292 static void vfio_intx_mmap_enable(void *opaque
)
294 VFIODevice
*vdev
= opaque
;
296 if (vdev
->intx
.pending
) {
297 timer_mod(vdev
->intx
.mmap_timer
,
298 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
302 vfio_mmap_set_enabled(vdev
, true);
305 static void vfio_intx_interrupt(void *opaque
)
307 VFIODevice
*vdev
= opaque
;
309 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
313 DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__
, vdev
->host
.domain
,
314 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
315 'A' + vdev
->intx
.pin
);
317 vdev
->intx
.pending
= true;
318 pci_irq_assert(&vdev
->pdev
);
319 vfio_mmap_set_enabled(vdev
, false);
320 if (vdev
->intx
.mmap_timeout
) {
321 timer_mod(vdev
->intx
.mmap_timer
,
322 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
326 static void vfio_eoi(VFIODevice
*vdev
)
328 if (!vdev
->intx
.pending
) {
332 DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__
, vdev
->host
.domain
,
333 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
335 vdev
->intx
.pending
= false;
336 pci_irq_deassert(&vdev
->pdev
);
337 vfio_unmask_intx(vdev
);
340 static void vfio_enable_intx_kvm(VFIODevice
*vdev
)
343 struct kvm_irqfd irqfd
= {
344 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
345 .gsi
= vdev
->intx
.route
.irq
,
346 .flags
= KVM_IRQFD_FLAG_RESAMPLE
,
348 struct vfio_irq_set
*irq_set
;
352 if (!VFIO_ALLOW_KVM_INTX
|| !kvm_irqfds_enabled() ||
353 vdev
->intx
.route
.mode
!= PCI_INTX_ENABLED
||
354 !kvm_check_extension(kvm_state
, KVM_CAP_IRQFD_RESAMPLE
)) {
358 /* Get to a known interrupt state */
359 qemu_set_fd_handler(irqfd
.fd
, NULL
, NULL
, vdev
);
360 vfio_mask_intx(vdev
);
361 vdev
->intx
.pending
= false;
362 pci_irq_deassert(&vdev
->pdev
);
364 /* Get an eventfd for resample/unmask */
365 if (event_notifier_init(&vdev
->intx
.unmask
, 0)) {
366 error_report("vfio: Error: event_notifier_init failed eoi");
370 /* KVM triggers it, VFIO listens for it */
371 irqfd
.resamplefd
= event_notifier_get_fd(&vdev
->intx
.unmask
);
373 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
374 error_report("vfio: Error: Failed to setup resample irqfd: %m");
378 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
380 irq_set
= g_malloc0(argsz
);
381 irq_set
->argsz
= argsz
;
382 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_UNMASK
;
383 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
386 pfd
= (int32_t *)&irq_set
->data
;
388 *pfd
= irqfd
.resamplefd
;
390 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
393 error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
398 vfio_unmask_intx(vdev
);
400 vdev
->intx
.kvm_accel
= true;
402 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
403 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
404 vdev
->host
.slot
, vdev
->host
.function
);
409 irqfd
.flags
= KVM_IRQFD_FLAG_DEASSIGN
;
410 kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
);
412 event_notifier_cleanup(&vdev
->intx
.unmask
);
414 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
415 vfio_unmask_intx(vdev
);
419 static void vfio_disable_intx_kvm(VFIODevice
*vdev
)
422 struct kvm_irqfd irqfd
= {
423 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
424 .gsi
= vdev
->intx
.route
.irq
,
425 .flags
= KVM_IRQFD_FLAG_DEASSIGN
,
428 if (!vdev
->intx
.kvm_accel
) {
433 * Get to a known state, hardware masked, QEMU ready to accept new
434 * interrupts, QEMU IRQ de-asserted.
436 vfio_mask_intx(vdev
);
437 vdev
->intx
.pending
= false;
438 pci_irq_deassert(&vdev
->pdev
);
440 /* Tell KVM to stop listening for an INTx irqfd */
441 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
442 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
445 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
446 event_notifier_cleanup(&vdev
->intx
.unmask
);
448 /* QEMU starts listening for interrupt events. */
449 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
451 vdev
->intx
.kvm_accel
= false;
453 /* If we've missed an event, let it re-fire through QEMU */
454 vfio_unmask_intx(vdev
);
456 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
457 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
458 vdev
->host
.slot
, vdev
->host
.function
);
462 static void vfio_update_irq(PCIDevice
*pdev
)
464 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
467 if (vdev
->interrupt
!= VFIO_INT_INTx
) {
471 route
= pci_device_route_intx_to_irq(&vdev
->pdev
, vdev
->intx
.pin
);
473 if (!pci_intx_route_changed(&vdev
->intx
.route
, &route
)) {
474 return; /* Nothing changed */
477 DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__
,
478 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
479 vdev
->host
.function
, vdev
->intx
.route
.irq
, route
.irq
);
481 vfio_disable_intx_kvm(vdev
);
483 vdev
->intx
.route
= route
;
485 if (route
.mode
!= PCI_INTX_ENABLED
) {
489 vfio_enable_intx_kvm(vdev
);
491 /* Re-enable the interrupt in cased we missed an EOI */
495 static int vfio_enable_intx(VFIODevice
*vdev
)
497 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
499 struct vfio_irq_set
*irq_set
;
506 vfio_disable_interrupts(vdev
);
508 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
509 pci_config_set_interrupt_pin(vdev
->pdev
.config
, pin
);
513 * Only conditional to avoid generating error messages on platforms
514 * where we won't actually use the result anyway.
516 if (kvm_irqfds_enabled() &&
517 kvm_check_extension(kvm_state
, KVM_CAP_IRQFD_RESAMPLE
)) {
518 vdev
->intx
.route
= pci_device_route_intx_to_irq(&vdev
->pdev
,
523 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
525 error_report("vfio: Error: event_notifier_init failed");
529 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
531 irq_set
= g_malloc0(argsz
);
532 irq_set
->argsz
= argsz
;
533 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
534 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
537 pfd
= (int32_t *)&irq_set
->data
;
539 *pfd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
540 qemu_set_fd_handler(*pfd
, vfio_intx_interrupt
, NULL
, vdev
);
542 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
545 error_report("vfio: Error: Failed to setup INTx fd: %m");
546 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
547 event_notifier_cleanup(&vdev
->intx
.interrupt
);
551 vfio_enable_intx_kvm(vdev
);
553 vdev
->interrupt
= VFIO_INT_INTx
;
555 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
556 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
561 static void vfio_disable_intx(VFIODevice
*vdev
)
565 timer_del(vdev
->intx
.mmap_timer
);
566 vfio_disable_intx_kvm(vdev
);
567 vfio_disable_irqindex(vdev
, VFIO_PCI_INTX_IRQ_INDEX
);
568 vdev
->intx
.pending
= false;
569 pci_irq_deassert(&vdev
->pdev
);
570 vfio_mmap_set_enabled(vdev
, true);
572 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
573 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
574 event_notifier_cleanup(&vdev
->intx
.interrupt
);
576 vdev
->interrupt
= VFIO_INT_NONE
;
578 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
579 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
585 static void vfio_msi_interrupt(void *opaque
)
587 VFIOMSIVector
*vector
= opaque
;
588 VFIODevice
*vdev
= vector
->vdev
;
589 int nr
= vector
- vdev
->msi_vectors
;
591 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
598 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
599 msg
= msi_get_message(&vdev
->pdev
, nr
);
600 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
601 msg
= msix_get_message(&vdev
->pdev
, nr
);
606 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d 0x%"PRIx64
"/0x%x\n", __func__
,
607 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
608 vdev
->host
.function
, nr
, msg
.address
, msg
.data
);
611 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
612 msix_notify(&vdev
->pdev
, nr
);
613 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
614 msi_notify(&vdev
->pdev
, nr
);
616 error_report("vfio: MSI interrupt receieved, but not enabled?");
620 static int vfio_enable_vectors(VFIODevice
*vdev
, bool msix
)
622 struct vfio_irq_set
*irq_set
;
623 int ret
= 0, i
, argsz
;
626 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
628 irq_set
= g_malloc0(argsz
);
629 irq_set
->argsz
= argsz
;
630 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
631 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
633 irq_set
->count
= vdev
->nr_vectors
;
634 fds
= (int32_t *)&irq_set
->data
;
636 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
637 if (!vdev
->msi_vectors
[i
].use
) {
642 fds
[i
] = event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
645 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
652 static int vfio_msix_vector_do_use(PCIDevice
*pdev
, unsigned int nr
,
653 MSIMessage
*msg
, IOHandler
*handler
)
655 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
656 VFIOMSIVector
*vector
;
659 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__
,
660 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
661 vdev
->host
.function
, nr
);
663 vector
= &vdev
->msi_vectors
[nr
];
667 msix_vector_use(pdev
, nr
);
669 if (event_notifier_init(&vector
->interrupt
, 0)) {
670 error_report("vfio: Error: event_notifier_init failed");
674 * Attempt to enable route through KVM irqchip,
675 * default to userspace handling if unavailable.
677 vector
->virq
= msg
&& VFIO_ALLOW_KVM_MSIX
?
678 kvm_irqchip_add_msi_route(kvm_state
, *msg
) : -1;
679 if (vector
->virq
< 0 ||
680 kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->interrupt
,
681 NULL
, vector
->virq
) < 0) {
682 if (vector
->virq
>= 0) {
683 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
686 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
687 handler
, NULL
, vector
);
691 * We don't want to have the host allocate all possible MSI vectors
692 * for a device if they're not in use, so we shutdown and incrementally
693 * increase them as needed.
695 if (vdev
->nr_vectors
< nr
+ 1) {
696 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
697 vdev
->nr_vectors
= nr
+ 1;
698 ret
= vfio_enable_vectors(vdev
, true);
700 error_report("vfio: failed to enable vectors, %d", ret
);
704 struct vfio_irq_set
*irq_set
;
707 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
709 irq_set
= g_malloc0(argsz
);
710 irq_set
->argsz
= argsz
;
711 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
712 VFIO_IRQ_SET_ACTION_TRIGGER
;
713 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
716 pfd
= (int32_t *)&irq_set
->data
;
718 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
720 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
723 error_report("vfio: failed to modify vector, %d", ret
);
730 static int vfio_msix_vector_use(PCIDevice
*pdev
,
731 unsigned int nr
, MSIMessage msg
)
733 return vfio_msix_vector_do_use(pdev
, nr
, &msg
, vfio_msi_interrupt
);
736 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
738 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
739 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
741 struct vfio_irq_set
*irq_set
;
744 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__
,
745 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
746 vdev
->host
.function
, nr
);
749 * XXX What's the right thing to do here? This turns off the interrupt
750 * completely, but do we really just want to switch the interrupt to
751 * bouncing through userspace and let msix.c drop it? Not sure.
753 msix_vector_unuse(pdev
, nr
);
755 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
757 irq_set
= g_malloc0(argsz
);
758 irq_set
->argsz
= argsz
;
759 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
760 VFIO_IRQ_SET_ACTION_TRIGGER
;
761 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
764 pfd
= (int32_t *)&irq_set
->data
;
768 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
772 if (vector
->virq
< 0) {
773 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
776 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->interrupt
,
778 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
782 event_notifier_cleanup(&vector
->interrupt
);
786 static void vfio_enable_msix(VFIODevice
*vdev
)
788 vfio_disable_interrupts(vdev
);
790 vdev
->msi_vectors
= g_malloc0(vdev
->msix
->entries
* sizeof(VFIOMSIVector
));
792 vdev
->interrupt
= VFIO_INT_MSIX
;
795 * Some communication channels between VF & PF or PF & fw rely on the
796 * physical state of the device and expect that enabling MSI-X from the
797 * guest enables the same on the host. When our guest is Linux, the
798 * guest driver call to pci_enable_msix() sets the enabling bit in the
799 * MSI-X capability, but leaves the vector table masked. We therefore
800 * can't rely on a vector_use callback (from request_irq() in the guest)
801 * to switch the physical device into MSI-X mode because that may come a
802 * long time after pci_enable_msix(). This code enables vector 0 with
803 * triggering to userspace, then immediately release the vector, leaving
804 * the physical device with no vectors enabled, but MSI-X enabled, just
805 * like the guest view.
807 vfio_msix_vector_do_use(&vdev
->pdev
, 0, NULL
, NULL
);
808 vfio_msix_vector_release(&vdev
->pdev
, 0);
810 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
811 vfio_msix_vector_release
, NULL
)) {
812 error_report("vfio: msix_set_vector_notifiers failed");
815 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
816 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
819 static void vfio_enable_msi(VFIODevice
*vdev
)
823 vfio_disable_interrupts(vdev
);
825 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
827 vdev
->msi_vectors
= g_malloc0(vdev
->nr_vectors
* sizeof(VFIOMSIVector
));
829 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
830 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
835 if (event_notifier_init(&vector
->interrupt
, 0)) {
836 error_report("vfio: Error: event_notifier_init failed");
839 vector
->msg
= msi_get_message(&vdev
->pdev
, i
);
842 * Attempt to enable route through KVM irqchip,
843 * default to userspace handling if unavailable.
845 vector
->virq
= VFIO_ALLOW_KVM_MSI
?
846 kvm_irqchip_add_msi_route(kvm_state
, vector
->msg
) : -1;
847 if (vector
->virq
< 0 ||
848 kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->interrupt
,
849 NULL
, vector
->virq
) < 0) {
850 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
851 vfio_msi_interrupt
, NULL
, vector
);
855 ret
= vfio_enable_vectors(vdev
, false);
858 error_report("vfio: Error: Failed to setup MSI fds: %m");
859 } else if (ret
!= vdev
->nr_vectors
) {
860 error_report("vfio: Error: Failed to enable %d "
861 "MSI vectors, retry with %d", vdev
->nr_vectors
, ret
);
864 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
865 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
866 if (vector
->virq
>= 0) {
867 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->interrupt
,
869 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
872 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
875 event_notifier_cleanup(&vector
->interrupt
);
878 g_free(vdev
->msi_vectors
);
880 if (ret
> 0 && ret
!= vdev
->nr_vectors
) {
881 vdev
->nr_vectors
= ret
;
884 vdev
->nr_vectors
= 0;
889 vdev
->interrupt
= VFIO_INT_MSI
;
891 DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__
,
892 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
893 vdev
->host
.function
, vdev
->nr_vectors
);
896 static void vfio_disable_msi_common(VFIODevice
*vdev
)
898 g_free(vdev
->msi_vectors
);
899 vdev
->msi_vectors
= NULL
;
900 vdev
->nr_vectors
= 0;
901 vdev
->interrupt
= VFIO_INT_NONE
;
903 vfio_enable_intx(vdev
);
906 static void vfio_disable_msix(VFIODevice
*vdev
)
910 msix_unset_vector_notifiers(&vdev
->pdev
);
913 * MSI-X will only release vectors if MSI-X is still enabled on the
914 * device, check through the rest and release it ourselves if necessary.
916 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
917 if (vdev
->msi_vectors
[i
].use
) {
918 vfio_msix_vector_release(&vdev
->pdev
, i
);
922 if (vdev
->nr_vectors
) {
923 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
926 vfio_disable_msi_common(vdev
);
928 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
929 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
932 static void vfio_disable_msi(VFIODevice
*vdev
)
936 vfio_disable_irqindex(vdev
, VFIO_PCI_MSI_IRQ_INDEX
);
938 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
939 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
945 if (vector
->virq
>= 0) {
946 kvm_irqchip_remove_irqfd_notifier(kvm_state
,
947 &vector
->interrupt
, vector
->virq
);
948 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
951 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
955 event_notifier_cleanup(&vector
->interrupt
);
958 vfio_disable_msi_common(vdev
);
960 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
961 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
964 static void vfio_update_msi(VFIODevice
*vdev
)
968 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
969 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
972 if (!vector
->use
|| vector
->virq
< 0) {
976 msg
= msi_get_message(&vdev
->pdev
, i
);
978 if (msg
.address
!= vector
->msg
.address
||
979 msg
.data
!= vector
->msg
.data
) {
981 DPRINTF("%s(%04x:%02x:%02x.%x) MSI vector %d changed\n",
982 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
983 vdev
->host
.slot
, vdev
->host
.function
, i
);
985 kvm_irqchip_update_msi_route(kvm_state
, vector
->virq
, msg
);
992 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
994 static void vfio_bar_write(void *opaque
, hwaddr addr
,
995 uint64_t data
, unsigned size
)
997 VFIOBAR
*bar
= opaque
;
1010 buf
.word
= cpu_to_le16(data
);
1013 buf
.dword
= cpu_to_le32(data
);
1016 hw_error("vfio: unsupported write size, %d bytes\n", size
);
1020 if (pwrite(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
1021 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1022 __func__
, addr
, data
, size
);
1027 VFIODevice
*vdev
= container_of(bar
, VFIODevice
, bars
[bar
->nr
]);
1029 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"PRIx64
1030 ", %d)\n", __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
1031 vdev
->host
.slot
, vdev
->host
.function
, bar
->nr
, addr
,
1037 * A read or write to a BAR always signals an INTx EOI. This will
1038 * do nothing if not pending (including not in INTx mode). We assume
1039 * that a BAR access is in response to an interrupt and that BAR
1040 * accesses will service the interrupt. Unfortunately, we don't know
1041 * which access will service the interrupt, so we're potentially
1042 * getting quite a few host interrupts per guest interrupt.
1044 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
1047 static uint64_t vfio_bar_read(void *opaque
,
1048 hwaddr addr
, unsigned size
)
1050 VFIOBAR
*bar
= opaque
;
1059 if (pread(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
1060 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1061 __func__
, addr
, size
);
1062 return (uint64_t)-1;
1070 data
= le16_to_cpu(buf
.word
);
1073 data
= le32_to_cpu(buf
.dword
);
1076 hw_error("vfio: unsupported read size, %d bytes\n", size
);
1082 VFIODevice
*vdev
= container_of(bar
, VFIODevice
, bars
[bar
->nr
]);
1084 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
1085 ", %d) = 0x%"PRIx64
"\n", __func__
, vdev
->host
.domain
,
1086 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1087 bar
->nr
, addr
, size
, data
);
1091 /* Same as write above */
1092 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
1097 static const MemoryRegionOps vfio_bar_ops
= {
1098 .read
= vfio_bar_read
,
1099 .write
= vfio_bar_write
,
1100 .endianness
= DEVICE_LITTLE_ENDIAN
,
1103 static void vfio_pci_load_rom(VFIODevice
*vdev
)
1105 struct vfio_region_info reg_info
= {
1106 .argsz
= sizeof(reg_info
),
1107 .index
= VFIO_PCI_ROM_REGION_INDEX
1113 if (ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
)) {
1114 error_report("vfio: Error getting ROM info: %m");
1118 DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev
->host
.domain
,
1119 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1120 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1121 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
1122 (unsigned long)reg_info
.flags
);
1124 vdev
->rom_size
= size
= reg_info
.size
;
1125 vdev
->rom_offset
= reg_info
.offset
;
1127 if (!vdev
->rom_size
) {
1131 vdev
->rom
= g_malloc(size
);
1132 memset(vdev
->rom
, 0xff, size
);
1135 bytes
= pread(vdev
->fd
, vdev
->rom
+ off
, size
, vdev
->rom_offset
+ off
);
1138 } else if (bytes
> 0) {
1142 if (errno
== EINTR
|| errno
== EAGAIN
) {
1145 error_report("vfio: Error reading device ROM: %m");
1151 static uint64_t vfio_rom_read(void *opaque
, hwaddr addr
, unsigned size
)
1153 VFIODevice
*vdev
= opaque
;
1154 uint64_t val
= ((uint64_t)1 << (size
* 8)) - 1;
1156 /* Load the ROM lazily when the guest tries to read it */
1157 if (unlikely(!vdev
->rom
)) {
1158 vfio_pci_load_rom(vdev
);
1161 memcpy(&val
, vdev
->rom
+ addr
,
1162 (addr
< vdev
->rom_size
) ? MIN(size
, vdev
->rom_size
- addr
) : 0);
1164 DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx
", 0x%x) = 0x%"PRIx64
"\n",
1165 __func__
, vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1166 vdev
->host
.function
, addr
, size
, val
);
1171 static void vfio_rom_write(void *opaque
, hwaddr addr
,
1172 uint64_t data
, unsigned size
)
1176 static const MemoryRegionOps vfio_rom_ops
= {
1177 .read
= vfio_rom_read
,
1178 .write
= vfio_rom_write
,
1179 .endianness
= DEVICE_LITTLE_ENDIAN
,
1182 static void vfio_pci_size_rom(VFIODevice
*vdev
)
1184 uint32_t orig
, size
= cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK
);
1185 off_t offset
= vdev
->config_offset
+ PCI_ROM_ADDRESS
;
1188 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
) {
1193 * Use the same size ROM BAR as the physical device. The contents
1194 * will get filled in later when the guest tries to read it.
1196 if (pread(vdev
->fd
, &orig
, 4, offset
) != 4 ||
1197 pwrite(vdev
->fd
, &size
, 4, offset
) != 4 ||
1198 pread(vdev
->fd
, &size
, 4, offset
) != 4 ||
1199 pwrite(vdev
->fd
, &orig
, 4, offset
) != 4) {
1200 error_report("%s(%04x:%02x:%02x.%x) failed: %m",
1201 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
1202 vdev
->host
.slot
, vdev
->host
.function
);
1206 size
= ~(le32_to_cpu(size
) & PCI_ROM_ADDRESS_MASK
) + 1;
1212 DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev
->host
.domain
,
1213 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, size
);
1215 snprintf(name
, sizeof(name
), "vfio[%04x:%02x:%02x.%x].rom",
1216 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1217 vdev
->host
.function
);
1219 memory_region_init_io(&vdev
->pdev
.rom
, OBJECT(vdev
),
1220 &vfio_rom_ops
, vdev
, name
, size
);
1222 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
,
1223 PCI_BASE_ADDRESS_SPACE_MEMORY
, &vdev
->pdev
.rom
);
1225 vdev
->pdev
.has_rom
= true;
1228 static void vfio_vga_write(void *opaque
, hwaddr addr
,
1229 uint64_t data
, unsigned size
)
1231 VFIOVGARegion
*region
= opaque
;
1232 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1239 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1246 buf
.word
= cpu_to_le16(data
);
1249 buf
.dword
= cpu_to_le32(data
);
1252 hw_error("vfio: unsupported write size, %d bytes\n", size
);
1256 if (pwrite(vga
->fd
, &buf
, size
, offset
) != size
) {
1257 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1258 __func__
, region
->offset
+ addr
, data
, size
);
1261 DPRINTF("%s(0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d)\n",
1262 __func__
, region
->offset
+ addr
, data
, size
);
1265 static uint64_t vfio_vga_read(void *opaque
, hwaddr addr
, unsigned size
)
1267 VFIOVGARegion
*region
= opaque
;
1268 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1276 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1278 if (pread(vga
->fd
, &buf
, size
, offset
) != size
) {
1279 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1280 __func__
, region
->offset
+ addr
, size
);
1281 return (uint64_t)-1;
1289 data
= le16_to_cpu(buf
.word
);
1292 data
= le32_to_cpu(buf
.dword
);
1295 hw_error("vfio: unsupported read size, %d bytes\n", size
);
1299 DPRINTF("%s(0x%"HWADDR_PRIx
", %d) = 0x%"PRIx64
"\n",
1300 __func__
, region
->offset
+ addr
, size
, data
);
1305 static const MemoryRegionOps vfio_vga_ops
= {
1306 .read
= vfio_vga_read
,
1307 .write
= vfio_vga_write
,
1308 .endianness
= DEVICE_LITTLE_ENDIAN
,
1312 * Device specific quirks
1315 /* Is range1 fully contained within range2? */
1316 static bool vfio_range_contained(uint64_t first1
, uint64_t len1
,
1317 uint64_t first2
, uint64_t len2
) {
1318 return (first1
>= first2
&& first1
+ len1
<= first2
+ len2
);
1321 static bool vfio_flags_enabled(uint8_t flags
, uint8_t mask
)
1323 return (mask
&& (flags
& mask
) == mask
);
1326 static uint64_t vfio_generic_window_quirk_read(void *opaque
,
1327 hwaddr addr
, unsigned size
)
1329 VFIOQuirk
*quirk
= opaque
;
1330 VFIODevice
*vdev
= quirk
->vdev
;
1333 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1334 ranges_overlap(addr
, size
,
1335 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1336 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1338 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1339 quirk
->data
.data_size
)) {
1340 hw_error("%s: window data read not fully contained: %s\n",
1341 __func__
, memory_region_name(&quirk
->mem
));
1344 data
= vfio_pci_read_config(&vdev
->pdev
,
1345 quirk
->data
.address_val
+ offset
, size
);
1347 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", %d) = 0x%"
1348 PRIx64
"\n", memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1349 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1350 quirk
->data
.bar
, addr
, size
, data
);
1352 data
= vfio_bar_read(&vdev
->bars
[quirk
->data
.bar
],
1353 addr
+ quirk
->data
.base_offset
, size
);
1359 static void vfio_generic_window_quirk_write(void *opaque
, hwaddr addr
,
1360 uint64_t data
, unsigned size
)
1362 VFIOQuirk
*quirk
= opaque
;
1363 VFIODevice
*vdev
= quirk
->vdev
;
1365 if (ranges_overlap(addr
, size
,
1366 quirk
->data
.address_offset
, quirk
->data
.address_size
)) {
1368 if (addr
!= quirk
->data
.address_offset
) {
1369 hw_error("%s: offset write into address window: %s\n",
1370 __func__
, memory_region_name(&quirk
->mem
));
1373 if ((data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1374 quirk
->data
.flags
|= quirk
->data
.write_flags
|
1375 quirk
->data
.read_flags
;
1376 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1378 quirk
->data
.flags
&= ~(quirk
->data
.write_flags
|
1379 quirk
->data
.read_flags
);
1383 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1384 ranges_overlap(addr
, size
,
1385 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1386 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1388 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1389 quirk
->data
.data_size
)) {
1390 hw_error("%s: window data write not fully contained: %s\n",
1391 __func__
, memory_region_name(&quirk
->mem
));
1394 vfio_pci_write_config(&vdev
->pdev
,
1395 quirk
->data
.address_val
+ offset
, data
, size
);
1396 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"
1397 PRIx64
", %d)\n", memory_region_name(&quirk
->mem
),
1398 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1399 vdev
->host
.function
, quirk
->data
.bar
, addr
, data
, size
);
1403 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
],
1404 addr
+ quirk
->data
.base_offset
, data
, size
);
1407 static const MemoryRegionOps vfio_generic_window_quirk
= {
1408 .read
= vfio_generic_window_quirk_read
,
1409 .write
= vfio_generic_window_quirk_write
,
1410 .endianness
= DEVICE_LITTLE_ENDIAN
,
1413 static uint64_t vfio_generic_quirk_read(void *opaque
,
1414 hwaddr addr
, unsigned size
)
1416 VFIOQuirk
*quirk
= opaque
;
1417 VFIODevice
*vdev
= quirk
->vdev
;
1418 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1419 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1422 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1423 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1424 if (!vfio_range_contained(addr
, size
, offset
,
1425 quirk
->data
.address_mask
+ 1)) {
1426 hw_error("%s: read not fully contained: %s\n",
1427 __func__
, memory_region_name(&quirk
->mem
));
1430 data
= vfio_pci_read_config(&vdev
->pdev
, addr
- offset
, size
);
1432 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", %d) = 0x%"
1433 PRIx64
"\n", memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1434 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1435 quirk
->data
.bar
, addr
+ base
, size
, data
);
1437 data
= vfio_bar_read(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, size
);
1443 static void vfio_generic_quirk_write(void *opaque
, hwaddr addr
,
1444 uint64_t data
, unsigned size
)
1446 VFIOQuirk
*quirk
= opaque
;
1447 VFIODevice
*vdev
= quirk
->vdev
;
1448 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1449 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1451 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1452 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1453 if (!vfio_range_contained(addr
, size
, offset
,
1454 quirk
->data
.address_mask
+ 1)) {
1455 hw_error("%s: write not fully contained: %s\n",
1456 __func__
, memory_region_name(&quirk
->mem
));
1459 vfio_pci_write_config(&vdev
->pdev
, addr
- offset
, data
, size
);
1461 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"
1462 PRIx64
", %d)\n", memory_region_name(&quirk
->mem
),
1463 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1464 vdev
->host
.function
, quirk
->data
.bar
, addr
+ base
, data
, size
);
1466 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, data
, size
);
1470 static const MemoryRegionOps vfio_generic_quirk
= {
1471 .read
= vfio_generic_quirk_read
,
1472 .write
= vfio_generic_quirk_write
,
1473 .endianness
= DEVICE_LITTLE_ENDIAN
,
1476 #define PCI_VENDOR_ID_ATI 0x1002
1479 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
1480 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always
1481 * BAR4 (older cards like the X550 used BAR1, but we don't care to support
1482 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the
1483 * I/O port BAR address. Originally this was coded to return the virtual BAR
1484 * address only if the physical register read returns the actual BAR address,
1485 * but users have reported greater success if we return the virtual address
1488 static uint64_t vfio_ati_3c3_quirk_read(void *opaque
,
1489 hwaddr addr
, unsigned size
)
1491 VFIOQuirk
*quirk
= opaque
;
1492 VFIODevice
*vdev
= quirk
->vdev
;
1493 uint64_t data
= vfio_pci_read_config(&vdev
->pdev
,
1494 PCI_BASE_ADDRESS_0
+ (4 * 4) + 1,
1496 DPRINTF("%s(0x3c3, 1) = 0x%"PRIx64
"\n", __func__
, data
);
1501 static const MemoryRegionOps vfio_ati_3c3_quirk
= {
1502 .read
= vfio_ati_3c3_quirk_read
,
1503 .endianness
= DEVICE_LITTLE_ENDIAN
,
1506 static void vfio_vga_probe_ati_3c3_quirk(VFIODevice
*vdev
)
1508 PCIDevice
*pdev
= &vdev
->pdev
;
1511 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1516 * As long as the BAR is >= 256 bytes it will be aligned such that the
1517 * lower byte is always zero. Filter out anything else, if it exists.
1519 if (!vdev
->bars
[4].ioport
|| vdev
->bars
[4].size
< 256) {
1523 quirk
= g_malloc0(sizeof(*quirk
));
1526 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_ati_3c3_quirk
, quirk
,
1527 "vfio-ati-3c3-quirk", 1);
1528 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
1529 3 /* offset 3 bytes from 0x3c0 */, &quirk
->mem
);
1531 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
1534 DPRINTF("Enabled ATI/AMD quirk 0x3c3 BAR4for device %04x:%02x:%02x.%x\n",
1535 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1536 vdev
->host
.function
);
1540 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
1541 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access
1542 * the MMIO space directly, but a window to this space is provided through
1543 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the
1544 * data register. When the address is programmed to a range of 0x4000-0x4fff
1545 * PCI configuration space is available. Experimentation seems to indicate
1546 * that only read-only access is provided, but we drop writes when the window
1547 * is enabled to config space nonetheless.
1549 static void vfio_probe_ati_bar4_window_quirk(VFIODevice
*vdev
, int nr
)
1551 PCIDevice
*pdev
= &vdev
->pdev
;
1554 if (!vdev
->has_vga
|| nr
!= 4 ||
1555 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1559 quirk
= g_malloc0(sizeof(*quirk
));
1561 quirk
->data
.address_size
= 4;
1562 quirk
->data
.data_offset
= 4;
1563 quirk
->data
.data_size
= 4;
1564 quirk
->data
.address_match
= 0x4000;
1565 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1566 quirk
->data
.bar
= nr
;
1567 quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1569 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
1570 &vfio_generic_window_quirk
, quirk
,
1571 "vfio-ati-bar4-window-quirk", 8);
1572 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1573 quirk
->data
.base_offset
, &quirk
->mem
, 1);
1575 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1577 DPRINTF("Enabled ATI/AMD BAR4 window quirk for device %04x:%02x:%02x.%x\n",
1578 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1579 vdev
->host
.function
);
1583 * Trap the BAR2 MMIO window to config space as well.
1585 static void vfio_probe_ati_bar2_4000_quirk(VFIODevice
*vdev
, int nr
)
1587 PCIDevice
*pdev
= &vdev
->pdev
;
1590 /* Only enable on newer devices where BAR2 is 64bit */
1591 if (!vdev
->has_vga
|| nr
!= 2 || !vdev
->bars
[2].mem64
||
1592 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1596 quirk
= g_malloc0(sizeof(*quirk
));
1598 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1599 quirk
->data
.address_match
= 0x4000;
1600 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1601 quirk
->data
.bar
= nr
;
1603 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
1604 "vfio-ati-bar2-4000-quirk",
1605 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1606 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1607 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1610 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1612 DPRINTF("Enabled ATI/AMD BAR2 0x4000 quirk for device %04x:%02x:%02x.%x\n",
1613 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1614 vdev
->host
.function
);
1618 * Older ATI/AMD cards like the X550 have a similar window to that above.
1619 * I/O port BAR1 provides a window to a mirror of PCI config space located
1620 * in BAR2 at offset 0xf00. We don't care to support such older cards, but
1621 * note it for future reference.
1624 #define PCI_VENDOR_ID_NVIDIA 0x10de
1627 * Nvidia has several different methods to get to config space, the
1628 * nouveu project has several of these documented here:
1629 * https://github.com/pathscale/envytools/tree/master/hwdocs
1631 * The first quirk is actually not documented in envytools and is found
1632 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an
1633 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access
1634 * the mirror of PCI config space found at BAR0 offset 0x1800. The access
1635 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is
1636 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738
1637 * is written for a write to 0x3d4. The BAR0 offset is then accessible
1638 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards
1639 * that use the I/O port BAR5 window but it doesn't hurt to leave it.
1649 static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque
,
1650 hwaddr addr
, unsigned size
)
1652 VFIOQuirk
*quirk
= opaque
;
1653 VFIODevice
*vdev
= quirk
->vdev
;
1654 PCIDevice
*pdev
= &vdev
->pdev
;
1655 uint64_t data
= vfio_vga_read(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1656 addr
+ quirk
->data
.base_offset
, size
);
1658 if (quirk
->data
.flags
== NV_3D0_READ
&& addr
== quirk
->data
.data_offset
) {
1659 data
= vfio_pci_read_config(pdev
, quirk
->data
.address_val
, size
);
1660 DPRINTF("%s(0x3d0, %d) = 0x%"PRIx64
"\n", __func__
, size
, data
);
1663 quirk
->data
.flags
= NV_3D0_NONE
;
1668 static void vfio_nvidia_3d0_quirk_write(void *opaque
, hwaddr addr
,
1669 uint64_t data
, unsigned size
)
1671 VFIOQuirk
*quirk
= opaque
;
1672 VFIODevice
*vdev
= quirk
->vdev
;
1673 PCIDevice
*pdev
= &vdev
->pdev
;
1675 switch (quirk
->data
.flags
) {
1677 if (addr
== quirk
->data
.address_offset
&& data
== 0x338) {
1678 quirk
->data
.flags
= NV_3D0_SELECT
;
1682 quirk
->data
.flags
= NV_3D0_NONE
;
1683 if (addr
== quirk
->data
.data_offset
&&
1684 (data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1685 quirk
->data
.flags
= NV_3D0_WINDOW
;
1686 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1690 quirk
->data
.flags
= NV_3D0_NONE
;
1691 if (addr
== quirk
->data
.address_offset
) {
1692 if (data
== 0x538) {
1693 quirk
->data
.flags
= NV_3D0_READ
;
1694 } else if (data
== 0x738) {
1695 quirk
->data
.flags
= NV_3D0_WRITE
;
1700 quirk
->data
.flags
= NV_3D0_NONE
;
1701 if (addr
== quirk
->data
.data_offset
) {
1702 vfio_pci_write_config(pdev
, quirk
->data
.address_val
, data
, size
);
1703 DPRINTF("%s(0x3d0, 0x%"PRIx64
", %d)\n", __func__
, data
, size
);
1709 vfio_vga_write(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1710 addr
+ quirk
->data
.base_offset
, data
, size
);
1713 static const MemoryRegionOps vfio_nvidia_3d0_quirk
= {
1714 .read
= vfio_nvidia_3d0_quirk_read
,
1715 .write
= vfio_nvidia_3d0_quirk_write
,
1716 .endianness
= DEVICE_LITTLE_ENDIAN
,
1719 static void vfio_vga_probe_nvidia_3d0_quirk(VFIODevice
*vdev
)
1721 PCIDevice
*pdev
= &vdev
->pdev
;
1724 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
||
1725 !vdev
->bars
[1].size
) {
1729 quirk
= g_malloc0(sizeof(*quirk
));
1731 quirk
->data
.base_offset
= 0x10;
1732 quirk
->data
.address_offset
= 4;
1733 quirk
->data
.address_size
= 2;
1734 quirk
->data
.address_match
= 0x1800;
1735 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
1736 quirk
->data
.data_offset
= 0;
1737 quirk
->data
.data_size
= 4;
1739 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_3d0_quirk
,
1740 quirk
, "vfio-nvidia-3d0-quirk", 6);
1741 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
1742 quirk
->data
.base_offset
, &quirk
->mem
);
1744 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
1747 DPRINTF("Enabled NVIDIA VGA 0x3d0 quirk for device %04x:%02x:%02x.%x\n",
1748 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1749 vdev
->host
.function
);
1753 * The second quirk is documented in envytools. The I/O port BAR5 is just
1754 * a set of address/data ports to the MMIO BARs. The BAR we care about is
1755 * again BAR0. This backdoor is apparently a bit newer than the one above
1756 * so we need to not only trap 256 bytes @0x1800, but all of PCI config
1757 * space, including extended space is available at the 4k @0x88000.
1760 NV_BAR5_ADDRESS
= 0x1,
1761 NV_BAR5_ENABLE
= 0x2,
1762 NV_BAR5_MASTER
= 0x4,
1763 NV_BAR5_VALID
= 0x7,
1766 static void vfio_nvidia_bar5_window_quirk_write(void *opaque
, hwaddr addr
,
1767 uint64_t data
, unsigned size
)
1769 VFIOQuirk
*quirk
= opaque
;
1774 quirk
->data
.flags
|= NV_BAR5_MASTER
;
1776 quirk
->data
.flags
&= ~NV_BAR5_MASTER
;
1781 quirk
->data
.flags
|= NV_BAR5_ENABLE
;
1783 quirk
->data
.flags
&= ~NV_BAR5_ENABLE
;
1787 if (quirk
->data
.flags
& NV_BAR5_MASTER
) {
1788 if ((data
& ~0xfff) == 0x88000) {
1789 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
1790 quirk
->data
.address_val
= data
& 0xfff;
1791 } else if ((data
& ~0xff) == 0x1800) {
1792 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
1793 quirk
->data
.address_val
= data
& 0xff;
1795 quirk
->data
.flags
&= ~NV_BAR5_ADDRESS
;
1801 vfio_generic_window_quirk_write(opaque
, addr
, data
, size
);
1804 static const MemoryRegionOps vfio_nvidia_bar5_window_quirk
= {
1805 .read
= vfio_generic_window_quirk_read
,
1806 .write
= vfio_nvidia_bar5_window_quirk_write
,
1807 .valid
.min_access_size
= 4,
1808 .endianness
= DEVICE_LITTLE_ENDIAN
,
1811 static void vfio_probe_nvidia_bar5_window_quirk(VFIODevice
*vdev
, int nr
)
1813 PCIDevice
*pdev
= &vdev
->pdev
;
1816 if (!vdev
->has_vga
|| nr
!= 5 ||
1817 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
1821 quirk
= g_malloc0(sizeof(*quirk
));
1823 quirk
->data
.read_flags
= quirk
->data
.write_flags
= NV_BAR5_VALID
;
1824 quirk
->data
.address_offset
= 0x8;
1825 quirk
->data
.address_size
= 0; /* actually 4, but avoids generic code */
1826 quirk
->data
.data_offset
= 0xc;
1827 quirk
->data
.data_size
= 4;
1828 quirk
->data
.bar
= nr
;
1830 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
1831 &vfio_nvidia_bar5_window_quirk
, quirk
,
1832 "vfio-nvidia-bar5-window-quirk", 16);
1833 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
, 0, &quirk
->mem
, 1);
1835 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1837 DPRINTF("Enabled NVIDIA BAR5 window quirk for device %04x:%02x:%02x.%x\n",
1838 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1839 vdev
->host
.function
);
1842 static void vfio_nvidia_88000_quirk_write(void *opaque
, hwaddr addr
,
1843 uint64_t data
, unsigned size
)
1845 VFIOQuirk
*quirk
= opaque
;
1846 VFIODevice
*vdev
= quirk
->vdev
;
1847 PCIDevice
*pdev
= &vdev
->pdev
;
1848 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1850 vfio_generic_quirk_write(opaque
, addr
, data
, size
);
1853 * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
1854 * MSI capability ID register. Both the ID and next register are
1855 * read-only, so we allow writes covering either of those to real hw.
1856 * NB - only fixed for the 0x88000 MMIO window.
1858 if ((pdev
->cap_present
& QEMU_PCI_CAP_MSI
) &&
1859 vfio_range_contained(addr
, size
, pdev
->msi_cap
, PCI_MSI_FLAGS
)) {
1860 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, data
, size
);
1864 static const MemoryRegionOps vfio_nvidia_88000_quirk
= {
1865 .read
= vfio_generic_quirk_read
,
1866 .write
= vfio_nvidia_88000_quirk_write
,
1867 .endianness
= DEVICE_LITTLE_ENDIAN
,
1871 * Finally, BAR0 itself. We want to redirect any accesses to either
1872 * 0x1800 or 0x88000 through the PCI config space access functions.
1874 * NB - quirk at a page granularity or else they don't seem to work when
1877 * Here's offset 0x88000...
1879 static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice
*vdev
, int nr
)
1881 PCIDevice
*pdev
= &vdev
->pdev
;
1884 if (!vdev
->has_vga
|| nr
!= 0 ||
1885 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
1889 quirk
= g_malloc0(sizeof(*quirk
));
1891 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1892 quirk
->data
.address_match
= 0x88000;
1893 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1894 quirk
->data
.bar
= nr
;
1896 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_88000_quirk
,
1897 quirk
, "vfio-nvidia-bar0-88000-quirk",
1898 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1899 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1900 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1903 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1905 DPRINTF("Enabled NVIDIA BAR0 0x88000 quirk for device %04x:%02x:%02x.%x\n",
1906 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1907 vdev
->host
.function
);
1911 * And here's the same for BAR0 offset 0x1800...
1913 static void vfio_probe_nvidia_bar0_1800_quirk(VFIODevice
*vdev
, int nr
)
1915 PCIDevice
*pdev
= &vdev
->pdev
;
1918 if (!vdev
->has_vga
|| nr
!= 0 ||
1919 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
1923 /* Log the chipset ID */
1924 DPRINTF("Nvidia NV%02x\n",
1925 (unsigned int)(vfio_bar_read(&vdev
->bars
[0], 0, 4) >> 20) & 0xff);
1927 quirk
= g_malloc0(sizeof(*quirk
));
1929 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1930 quirk
->data
.address_match
= 0x1800;
1931 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
1932 quirk
->data
.bar
= nr
;
1934 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
1935 "vfio-nvidia-bar0-1800-quirk",
1936 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1937 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1938 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1941 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1943 DPRINTF("Enabled NVIDIA BAR0 0x1800 quirk for device %04x:%02x:%02x.%x\n",
1944 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1945 vdev
->host
.function
);
1949 * TODO - Some Nvidia devices provide config access to their companion HDA
1950 * device and even to their parent bridge via these config space mirrors.
1951 * Add quirks for those regions.
1955 * Common quirk probe entry points.
1957 static void vfio_vga_quirk_setup(VFIODevice
*vdev
)
1959 vfio_vga_probe_ati_3c3_quirk(vdev
);
1960 vfio_vga_probe_nvidia_3d0_quirk(vdev
);
1963 static void vfio_vga_quirk_teardown(VFIODevice
*vdev
)
1967 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
.region
); i
++) {
1968 while (!QLIST_EMPTY(&vdev
->vga
.region
[i
].quirks
)) {
1969 VFIOQuirk
*quirk
= QLIST_FIRST(&vdev
->vga
.region
[i
].quirks
);
1970 memory_region_del_subregion(&vdev
->vga
.region
[i
].mem
, &quirk
->mem
);
1971 memory_region_destroy(&quirk
->mem
);
1972 QLIST_REMOVE(quirk
, next
);
1978 static void vfio_bar_quirk_setup(VFIODevice
*vdev
, int nr
)
1980 vfio_probe_ati_bar4_window_quirk(vdev
, nr
);
1981 vfio_probe_ati_bar2_4000_quirk(vdev
, nr
);
1982 vfio_probe_nvidia_bar5_window_quirk(vdev
, nr
);
1983 vfio_probe_nvidia_bar0_88000_quirk(vdev
, nr
);
1984 vfio_probe_nvidia_bar0_1800_quirk(vdev
, nr
);
1987 static void vfio_bar_quirk_teardown(VFIODevice
*vdev
, int nr
)
1989 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1991 while (!QLIST_EMPTY(&bar
->quirks
)) {
1992 VFIOQuirk
*quirk
= QLIST_FIRST(&bar
->quirks
);
1993 memory_region_del_subregion(&bar
->mem
, &quirk
->mem
);
1994 memory_region_destroy(&quirk
->mem
);
1995 QLIST_REMOVE(quirk
, next
);
2003 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
2005 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
2006 uint32_t emu_bits
= 0, emu_val
= 0, phys_val
= 0, val
;
2008 memcpy(&emu_bits
, vdev
->emulated_config_bits
+ addr
, len
);
2009 emu_bits
= le32_to_cpu(emu_bits
);
2012 emu_val
= pci_default_read_config(pdev
, addr
, len
);
2015 if (~emu_bits
& (0xffffffffU
>> (32 - len
* 8))) {
2018 ret
= pread(vdev
->fd
, &phys_val
, len
, vdev
->config_offset
+ addr
);
2020 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
2021 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2022 vdev
->host
.slot
, vdev
->host
.function
, addr
, len
);
2025 phys_val
= le32_to_cpu(phys_val
);
2028 val
= (emu_val
& emu_bits
) | (phys_val
& ~emu_bits
);
2030 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__
,
2031 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2032 vdev
->host
.function
, addr
, len
, val
);
2037 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
2038 uint32_t val
, int len
)
2040 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
2041 uint32_t val_le
= cpu_to_le32(val
);
2043 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__
,
2044 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2045 vdev
->host
.function
, addr
, val
, len
);
2047 /* Write everything to VFIO, let it filter out what we can't write */
2048 if (pwrite(vdev
->fd
, &val_le
, len
, vdev
->config_offset
+ addr
) != len
) {
2049 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
2050 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2051 vdev
->host
.slot
, vdev
->host
.function
, addr
, val
, len
);
2054 /* MSI/MSI-X Enabling/Disabling */
2055 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
2056 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
2057 int is_enabled
, was_enabled
= msi_enabled(pdev
);
2059 pci_default_write_config(pdev
, addr
, val
, len
);
2061 is_enabled
= msi_enabled(pdev
);
2065 vfio_enable_msi(vdev
);
2069 vfio_disable_msi(vdev
);
2071 vfio_update_msi(vdev
);
2074 } else if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
2075 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
2076 int is_enabled
, was_enabled
= msix_enabled(pdev
);
2078 pci_default_write_config(pdev
, addr
, val
, len
);
2080 is_enabled
= msix_enabled(pdev
);
2082 if (!was_enabled
&& is_enabled
) {
2083 vfio_enable_msix(vdev
);
2084 } else if (was_enabled
&& !is_enabled
) {
2085 vfio_disable_msix(vdev
);
2088 /* Write everything to QEMU to keep emulated bits correct */
2089 pci_default_write_config(pdev
, addr
, val
, len
);
2094 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
2096 static int vfio_dma_unmap(VFIOContainer
*container
,
2097 hwaddr iova
, ram_addr_t size
)
2099 struct vfio_iommu_type1_dma_unmap unmap
= {
2100 .argsz
= sizeof(unmap
),
2106 if (ioctl(container
->fd
, VFIO_IOMMU_UNMAP_DMA
, &unmap
)) {
2107 DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno
);
2114 static int vfio_dma_map(VFIOContainer
*container
, hwaddr iova
,
2115 ram_addr_t size
, void *vaddr
, bool readonly
)
2117 struct vfio_iommu_type1_dma_map map
= {
2118 .argsz
= sizeof(map
),
2119 .flags
= VFIO_DMA_MAP_FLAG_READ
,
2120 .vaddr
= (__u64
)(uintptr_t)vaddr
,
2126 map
.flags
|= VFIO_DMA_MAP_FLAG_WRITE
;
2130 * Try the mapping, if it fails with EBUSY, unmap the region and try
2131 * again. This shouldn't be necessary, but we sometimes see it in
2132 * the the VGA ROM space.
2134 if (ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0 ||
2135 (errno
== EBUSY
&& vfio_dma_unmap(container
, iova
, size
) == 0 &&
2136 ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0)) {
2140 DPRINTF("VFIO_MAP_DMA: %d\n", -errno
);
2144 static bool vfio_listener_skipped_section(MemoryRegionSection
*section
)
2146 return !memory_region_is_ram(section
->mr
);
2149 static void vfio_listener_region_add(MemoryListener
*listener
,
2150 MemoryRegionSection
*section
)
2152 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
2153 iommu_data
.listener
);
2158 assert(!memory_region_is_iommu(section
->mr
));
2160 if (vfio_listener_skipped_section(section
)) {
2161 DPRINTF("SKIPPING region_add %"HWADDR_PRIx
" - %"PRIx64
"\n",
2162 section
->offset_within_address_space
,
2163 section
->offset_within_address_space
+
2164 int128_get64(int128_sub(section
->size
, int128_one())));
2168 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
2169 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
2170 error_report("%s received unaligned region", __func__
);
2174 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
2175 end
= (section
->offset_within_address_space
+ int128_get64(section
->size
)) &
2182 vaddr
= memory_region_get_ram_ptr(section
->mr
) +
2183 section
->offset_within_region
+
2184 (iova
- section
->offset_within_address_space
);
2186 DPRINTF("region_add %"HWADDR_PRIx
" - %"HWADDR_PRIx
" [%p]\n",
2187 iova
, end
- 1, vaddr
);
2189 memory_region_ref(section
->mr
);
2190 ret
= vfio_dma_map(container
, iova
, end
- iova
, vaddr
, section
->readonly
);
2192 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx
", "
2193 "0x%"HWADDR_PRIx
", %p) = %d (%m)",
2194 container
, iova
, end
- iova
, vaddr
, ret
);
2198 static void vfio_listener_region_del(MemoryListener
*listener
,
2199 MemoryRegionSection
*section
)
2201 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
2202 iommu_data
.listener
);
2206 if (vfio_listener_skipped_section(section
)) {
2207 DPRINTF("SKIPPING region_del %"HWADDR_PRIx
" - %"PRIx64
"\n",
2208 section
->offset_within_address_space
,
2209 section
->offset_within_address_space
+
2210 int128_get64(int128_sub(section
->size
, int128_one())));
2214 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
2215 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
2216 error_report("%s received unaligned region", __func__
);
2220 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
2221 end
= (section
->offset_within_address_space
+ int128_get64(section
->size
)) &
2228 DPRINTF("region_del %"HWADDR_PRIx
" - %"HWADDR_PRIx
"\n",
2231 ret
= vfio_dma_unmap(container
, iova
, end
- iova
);
2232 memory_region_unref(section
->mr
);
2234 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx
", "
2235 "0x%"HWADDR_PRIx
") = %d (%m)",
2236 container
, iova
, end
- iova
, ret
);
2240 static MemoryListener vfio_memory_listener
= {
2241 .region_add
= vfio_listener_region_add
,
2242 .region_del
= vfio_listener_region_del
,
2245 static void vfio_listener_release(VFIOContainer
*container
)
2247 memory_listener_unregister(&container
->iommu_data
.listener
);
2253 static void vfio_disable_interrupts(VFIODevice
*vdev
)
2255 switch (vdev
->interrupt
) {
2257 vfio_disable_intx(vdev
);
2260 vfio_disable_msi(vdev
);
2263 vfio_disable_msix(vdev
);
2268 static int vfio_setup_msi(VFIODevice
*vdev
, int pos
)
2271 bool msi_64bit
, msi_maskbit
;
2274 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
2275 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2278 ctrl
= le16_to_cpu(ctrl
);
2280 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
2281 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
2282 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
2284 DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev
->host
.domain
,
2285 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, pos
);
2287 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
);
2289 if (ret
== -ENOTSUP
) {
2292 error_report("vfio: msi_init failed");
2295 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
2301 * We don't have any control over how pci_add_capability() inserts
2302 * capabilities into the chain. In order to setup MSI-X we need a
2303 * MemoryRegion for the BAR. In order to setup the BAR and not
2304 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
2305 * need to first look for where the MSI-X table lives. So we
2306 * unfortunately split MSI-X setup across two functions.
2308 static int vfio_early_setup_msix(VFIODevice
*vdev
)
2312 uint32_t table
, pba
;
2314 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
2319 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
2320 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2324 if (pread(vdev
->fd
, &table
, sizeof(table
),
2325 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
2329 if (pread(vdev
->fd
, &pba
, sizeof(pba
),
2330 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
2334 ctrl
= le16_to_cpu(ctrl
);
2335 table
= le32_to_cpu(table
);
2336 pba
= le32_to_cpu(pba
);
2338 vdev
->msix
= g_malloc0(sizeof(*(vdev
->msix
)));
2339 vdev
->msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
2340 vdev
->msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
2341 vdev
->msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
2342 vdev
->msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
2343 vdev
->msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
2345 DPRINTF("%04x:%02x:%02x.%x "
2346 "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
2347 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2348 vdev
->host
.function
, pos
, vdev
->msix
->table_bar
,
2349 vdev
->msix
->table_offset
, vdev
->msix
->entries
);
2354 static int vfio_setup_msix(VFIODevice
*vdev
, int pos
)
2358 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
2359 &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
2360 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
2361 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
,
2362 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
);
2364 if (ret
== -ENOTSUP
) {
2367 error_report("vfio: msix_init failed");
2374 static void vfio_teardown_msi(VFIODevice
*vdev
)
2376 msi_uninit(&vdev
->pdev
);
2379 msix_uninit(&vdev
->pdev
, &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
2380 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
);
2387 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
)
2391 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2392 VFIOBAR
*bar
= &vdev
->bars
[i
];
2398 memory_region_set_enabled(&bar
->mmap_mem
, enabled
);
2399 if (vdev
->msix
&& vdev
->msix
->table_bar
== i
) {
2400 memory_region_set_enabled(&vdev
->msix
->mmap_mem
, enabled
);
2405 static void vfio_unmap_bar(VFIODevice
*vdev
, int nr
)
2407 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2413 vfio_bar_quirk_teardown(vdev
, nr
);
2415 memory_region_del_subregion(&bar
->mem
, &bar
->mmap_mem
);
2416 munmap(bar
->mmap
, memory_region_size(&bar
->mmap_mem
));
2417 memory_region_destroy(&bar
->mmap_mem
);
2419 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2420 memory_region_del_subregion(&bar
->mem
, &vdev
->msix
->mmap_mem
);
2421 munmap(vdev
->msix
->mmap
, memory_region_size(&vdev
->msix
->mmap_mem
));
2422 memory_region_destroy(&vdev
->msix
->mmap_mem
);
2425 memory_region_destroy(&bar
->mem
);
2428 static int vfio_mmap_bar(VFIODevice
*vdev
, VFIOBAR
*bar
,
2429 MemoryRegion
*mem
, MemoryRegion
*submem
,
2430 void **map
, size_t size
, off_t offset
,
2435 if (VFIO_ALLOW_MMAP
&& size
&& bar
->flags
& VFIO_REGION_INFO_FLAG_MMAP
) {
2438 if (bar
->flags
& VFIO_REGION_INFO_FLAG_READ
) {
2442 if (bar
->flags
& VFIO_REGION_INFO_FLAG_WRITE
) {
2446 *map
= mmap(NULL
, size
, prot
, MAP_SHARED
,
2447 bar
->fd
, bar
->fd_offset
+ offset
);
2448 if (*map
== MAP_FAILED
) {
2454 memory_region_init_ram_ptr(submem
, OBJECT(vdev
), name
, size
, *map
);
2457 /* Create a zero sized sub-region to make cleanup easy. */
2458 memory_region_init(submem
, OBJECT(vdev
), name
, 0);
2461 memory_region_add_subregion(mem
, offset
, submem
);
2466 static void vfio_map_bar(VFIODevice
*vdev
, int nr
)
2468 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2469 unsigned size
= bar
->size
;
2475 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
2480 snprintf(name
, sizeof(name
), "VFIO %04x:%02x:%02x.%x BAR %d",
2481 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2482 vdev
->host
.function
, nr
);
2484 /* Determine what type of BAR this is for registration */
2485 ret
= pread(vdev
->fd
, &pci_bar
, sizeof(pci_bar
),
2486 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
2487 if (ret
!= sizeof(pci_bar
)) {
2488 error_report("vfio: Failed to read BAR %d (%m)", nr
);
2492 pci_bar
= le32_to_cpu(pci_bar
);
2493 bar
->ioport
= (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
);
2494 bar
->mem64
= bar
->ioport
? 0 : (pci_bar
& PCI_BASE_ADDRESS_MEM_TYPE_64
);
2495 type
= pci_bar
& (bar
->ioport
? ~PCI_BASE_ADDRESS_IO_MASK
:
2496 ~PCI_BASE_ADDRESS_MEM_MASK
);
2498 /* A "slow" read/write mapping underlies all BARs */
2499 memory_region_init_io(&bar
->mem
, OBJECT(vdev
), &vfio_bar_ops
,
2501 pci_register_bar(&vdev
->pdev
, nr
, type
, &bar
->mem
);
2504 * We can't mmap areas overlapping the MSIX vector table, so we
2505 * potentially insert a direct-mapped subregion before and after it.
2507 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2508 size
= vdev
->msix
->table_offset
& TARGET_PAGE_MASK
;
2511 strncat(name
, " mmap", sizeof(name
) - strlen(name
) - 1);
2512 if (vfio_mmap_bar(vdev
, bar
, &bar
->mem
,
2513 &bar
->mmap_mem
, &bar
->mmap
, size
, 0, name
)) {
2514 error_report("%s unsupported. Performance may be slow", name
);
2517 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2520 start
= TARGET_PAGE_ALIGN(vdev
->msix
->table_offset
+
2521 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
2523 size
= start
< bar
->size
? bar
->size
- start
: 0;
2524 strncat(name
, " msix-hi", sizeof(name
) - strlen(name
) - 1);
2525 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
2526 if (vfio_mmap_bar(vdev
, bar
, &bar
->mem
, &vdev
->msix
->mmap_mem
,
2527 &vdev
->msix
->mmap
, size
, start
, name
)) {
2528 error_report("%s unsupported. Performance may be slow", name
);
2532 vfio_bar_quirk_setup(vdev
, nr
);
2535 static void vfio_map_bars(VFIODevice
*vdev
)
2539 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2540 vfio_map_bar(vdev
, i
);
2543 if (vdev
->has_vga
) {
2544 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2545 OBJECT(vdev
), &vfio_vga_ops
,
2546 &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
],
2547 "vfio-vga-mmio@0xa0000",
2548 QEMU_PCI_VGA_MEM_SIZE
);
2549 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2550 OBJECT(vdev
), &vfio_vga_ops
,
2551 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
],
2552 "vfio-vga-io@0x3b0",
2553 QEMU_PCI_VGA_IO_LO_SIZE
);
2554 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
2555 OBJECT(vdev
), &vfio_vga_ops
,
2556 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
2557 "vfio-vga-io@0x3c0",
2558 QEMU_PCI_VGA_IO_HI_SIZE
);
2560 pci_register_vga(&vdev
->pdev
, &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2561 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2562 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
);
2563 vfio_vga_quirk_setup(vdev
);
2567 static void vfio_unmap_bars(VFIODevice
*vdev
)
2571 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2572 vfio_unmap_bar(vdev
, i
);
2575 if (vdev
->has_vga
) {
2576 vfio_vga_quirk_teardown(vdev
);
2577 pci_unregister_vga(&vdev
->pdev
);
2578 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
);
2579 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
);
2580 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
);
2587 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
2589 uint8_t tmp
, next
= 0xff;
2591 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
2592 tmp
= pdev
->config
[tmp
+ 1]) {
2593 if (tmp
> pos
&& tmp
< next
) {
2601 static void vfio_set_word_bits(uint8_t *buf
, uint16_t val
, uint16_t mask
)
2603 pci_set_word(buf
, (pci_get_word(buf
) & ~mask
) | val
);
2606 static void vfio_add_emulated_word(VFIODevice
*vdev
, int pos
,
2607 uint16_t val
, uint16_t mask
)
2609 vfio_set_word_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
2610 vfio_set_word_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
2611 vfio_set_word_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
2614 static void vfio_set_long_bits(uint8_t *buf
, uint32_t val
, uint32_t mask
)
2616 pci_set_long(buf
, (pci_get_long(buf
) & ~mask
) | val
);
2619 static void vfio_add_emulated_long(VFIODevice
*vdev
, int pos
,
2620 uint32_t val
, uint32_t mask
)
2622 vfio_set_long_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
2623 vfio_set_long_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
2624 vfio_set_long_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
2627 static int vfio_setup_pcie_cap(VFIODevice
*vdev
, int pos
, uint8_t size
)
2632 flags
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_CAP_FLAGS
);
2633 type
= (flags
& PCI_EXP_FLAGS_TYPE
) >> 4;
2635 if (type
!= PCI_EXP_TYPE_ENDPOINT
&&
2636 type
!= PCI_EXP_TYPE_LEG_END
&&
2637 type
!= PCI_EXP_TYPE_RC_END
) {
2639 error_report("vfio: Assignment of PCIe type 0x%x "
2640 "devices is not currently supported", type
);
2644 if (!pci_bus_is_express(vdev
->pdev
.bus
)) {
2646 * Use express capability as-is on PCI bus. It doesn't make much
2647 * sense to even expose, but some drivers (ex. tg3) depend on it
2648 * and guests don't seem to be particular about it. We'll need
2649 * to revist this or force express devices to express buses if we
2650 * ever expose an IOMMU to the guest.
2652 } else if (pci_bus_is_root(vdev
->pdev
.bus
)) {
2654 * On a Root Complex bus Endpoints become Root Complex Integrated
2655 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2657 if (type
== PCI_EXP_TYPE_ENDPOINT
) {
2658 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2659 PCI_EXP_TYPE_RC_END
<< 4,
2660 PCI_EXP_FLAGS_TYPE
);
2662 /* Link Capabilities, Status, and Control goes away */
2663 if (size
> PCI_EXP_LNKCTL
) {
2664 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
, 0, ~0);
2665 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2666 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
, 0, ~0);
2668 #ifndef PCI_EXP_LNKCAP2
2669 #define PCI_EXP_LNKCAP2 44
2671 #ifndef PCI_EXP_LNKSTA2
2672 #define PCI_EXP_LNKSTA2 50
2674 /* Link 2 Capabilities, Status, and Control goes away */
2675 if (size
> PCI_EXP_LNKCAP2
) {
2676 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP2
, 0, ~0);
2677 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL2
, 0, ~0);
2678 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA2
, 0, ~0);
2682 } else if (type
== PCI_EXP_TYPE_LEG_END
) {
2684 * Legacy endpoints don't belong on the root complex. Windows
2685 * seems to be happier with devices if we skip the capability.
2692 * Convert Root Complex Integrated Endpoints to regular endpoints.
2693 * These devices don't support LNK/LNK2 capabilities, so make them up.
2695 if (type
== PCI_EXP_TYPE_RC_END
) {
2696 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2697 PCI_EXP_TYPE_ENDPOINT
<< 4,
2698 PCI_EXP_FLAGS_TYPE
);
2699 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
,
2700 PCI_EXP_LNK_MLW_1
| PCI_EXP_LNK_LS_25
, ~0);
2701 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2704 /* Mark the Link Status bits as emulated to allow virtual negotiation */
2705 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
,
2706 pci_get_word(vdev
->pdev
.config
+ pos
+
2708 PCI_EXP_LNKCAP_MLW
| PCI_EXP_LNKCAP_SLS
);
2711 pos
= pci_add_capability(&vdev
->pdev
, PCI_CAP_ID_EXP
, pos
, size
);
2713 vdev
->pdev
.exp
.exp_cap
= pos
;
2719 static void vfio_check_pcie_flr(VFIODevice
*vdev
, uint8_t pos
)
2721 uint32_t cap
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_EXP_DEVCAP
);
2723 if (cap
& PCI_EXP_DEVCAP_FLR
) {
2724 DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
2725 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2726 vdev
->host
.function
);
2727 vdev
->has_flr
= true;
2731 static void vfio_check_pm_reset(VFIODevice
*vdev
, uint8_t pos
)
2733 uint16_t csr
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_PM_CTRL
);
2735 if (!(csr
& PCI_PM_CTRL_NO_SOFT_RESET
)) {
2736 DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
2737 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2738 vdev
->host
.function
);
2739 vdev
->has_pm_reset
= true;
2743 static void vfio_check_af_flr(VFIODevice
*vdev
, uint8_t pos
)
2745 uint8_t cap
= pci_get_byte(vdev
->pdev
.config
+ pos
+ PCI_AF_CAP
);
2747 if ((cap
& PCI_AF_CAP_TP
) && (cap
& PCI_AF_CAP_FLR
)) {
2748 DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
2749 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2750 vdev
->host
.function
);
2751 vdev
->has_flr
= true;
2755 static int vfio_add_std_cap(VFIODevice
*vdev
, uint8_t pos
)
2757 PCIDevice
*pdev
= &vdev
->pdev
;
2758 uint8_t cap_id
, next
, size
;
2761 cap_id
= pdev
->config
[pos
];
2762 next
= pdev
->config
[pos
+ 1];
2765 * If it becomes important to configure capabilities to their actual
2766 * size, use this as the default when it's something we don't recognize.
2767 * Since QEMU doesn't actually handle many of the config accesses,
2768 * exact size doesn't seem worthwhile.
2770 size
= vfio_std_cap_max_size(pdev
, pos
);
2773 * pci_add_capability always inserts the new capability at the head
2774 * of the chain. Therefore to end up with a chain that matches the
2775 * physical device, we insert from the end by making this recursive.
2776 * This is also why we pre-caclulate size above as cached config space
2777 * will be changed as we unwind the stack.
2780 ret
= vfio_add_std_cap(vdev
, next
);
2785 /* Begin the rebuild, use QEMU emulated list bits */
2786 pdev
->config
[PCI_CAPABILITY_LIST
] = 0;
2787 vdev
->emulated_config_bits
[PCI_CAPABILITY_LIST
] = 0xff;
2788 vdev
->emulated_config_bits
[PCI_STATUS
] |= PCI_STATUS_CAP_LIST
;
2791 /* Use emulated next pointer to allow dropping caps */
2792 pci_set_byte(vdev
->emulated_config_bits
+ pos
+ 1, 0xff);
2795 case PCI_CAP_ID_MSI
:
2796 ret
= vfio_setup_msi(vdev
, pos
);
2798 case PCI_CAP_ID_EXP
:
2799 vfio_check_pcie_flr(vdev
, pos
);
2800 ret
= vfio_setup_pcie_cap(vdev
, pos
, size
);
2802 case PCI_CAP_ID_MSIX
:
2803 ret
= vfio_setup_msix(vdev
, pos
);
2806 vfio_check_pm_reset(vdev
, pos
);
2808 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2811 vfio_check_af_flr(vdev
, pos
);
2812 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2815 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2820 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
2821 "0x%x[0x%x]@0x%x: %d", vdev
->host
.domain
,
2822 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
2823 cap_id
, size
, pos
, ret
);
2830 static int vfio_add_capabilities(VFIODevice
*vdev
)
2832 PCIDevice
*pdev
= &vdev
->pdev
;
2834 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
2835 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
2836 return 0; /* Nothing to add */
2839 return vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
]);
2842 static void vfio_pci_pre_reset(VFIODevice
*vdev
)
2844 PCIDevice
*pdev
= &vdev
->pdev
;
2847 vfio_disable_interrupts(vdev
);
2849 /* Make sure the device is in D0 */
2854 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2855 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2857 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
2858 vfio_pci_write_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, pmcsr
, 2);
2859 /* vfio handles the necessary delay here */
2860 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2861 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2863 error_report("vfio: Unable to power on device, stuck in D%d\n",
2870 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2871 * Also put INTx Disable in known state.
2873 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
2874 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
2875 PCI_COMMAND_INTX_DISABLE
);
2876 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
2879 static void vfio_pci_post_reset(VFIODevice
*vdev
)
2881 vfio_enable_intx(vdev
);
2884 static bool vfio_pci_host_match(PCIHostDeviceAddress
*host1
,
2885 PCIHostDeviceAddress
*host2
)
2887 return (host1
->domain
== host2
->domain
&& host1
->bus
== host2
->bus
&&
2888 host1
->slot
== host2
->slot
&& host1
->function
== host2
->function
);
2891 static int vfio_pci_hot_reset(VFIODevice
*vdev
, bool single
)
2894 struct vfio_pci_hot_reset_info
*info
;
2895 struct vfio_pci_dependent_device
*devices
;
2896 struct vfio_pci_hot_reset
*reset
;
2901 DPRINTF("%s(%04x:%02x:%02x.%x) %s\n", __func__
, vdev
->host
.domain
,
2902 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
2903 single
? "one" : "multi");
2905 vfio_pci_pre_reset(vdev
);
2906 vdev
->needs_reset
= false;
2908 info
= g_malloc0(sizeof(*info
));
2909 info
->argsz
= sizeof(*info
);
2911 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2912 if (ret
&& errno
!= ENOSPC
) {
2914 if (!vdev
->has_pm_reset
) {
2915 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
2916 "no available reset mechanism.", vdev
->host
.domain
,
2917 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
2922 count
= info
->count
;
2923 info
= g_realloc(info
, sizeof(*info
) + (count
* sizeof(*devices
)));
2924 info
->argsz
= sizeof(*info
) + (count
* sizeof(*devices
));
2925 devices
= &info
->devices
[0];
2927 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2930 error_report("vfio: hot reset info failed: %m");
2934 DPRINTF("%04x:%02x:%02x.%x: hot reset dependent devices:\n",
2935 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2936 vdev
->host
.function
);
2938 /* Verify that we have all the groups required */
2939 for (i
= 0; i
< info
->count
; i
++) {
2940 PCIHostDeviceAddress host
;
2943 host
.domain
= devices
[i
].segment
;
2944 host
.bus
= devices
[i
].bus
;
2945 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
2946 host
.function
= PCI_FUNC(devices
[i
].devfn
);
2948 DPRINTF("\t%04x:%02x:%02x.%x group %d\n", host
.domain
,
2949 host
.bus
, host
.slot
, host
.function
, devices
[i
].group_id
);
2951 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
2955 QLIST_FOREACH(group
, &group_list
, next
) {
2956 if (group
->groupid
== devices
[i
].group_id
) {
2962 if (!vdev
->has_pm_reset
) {
2963 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
2964 "depends on group %d which is not owned.",
2965 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2966 vdev
->host
.function
, devices
[i
].group_id
);
2972 /* Prep dependent devices for reset and clear our marker. */
2973 QLIST_FOREACH(tmp
, &group
->device_list
, next
) {
2974 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
2976 DPRINTF("vfio: found another in-use device "
2977 "%04x:%02x:%02x.%x\n", host
.domain
, host
.bus
,
2978 host
.slot
, host
.function
);
2982 vfio_pci_pre_reset(tmp
);
2983 tmp
->needs_reset
= false;
2990 if (!single
&& !multi
) {
2991 DPRINTF("vfio: No other in-use devices for multi hot reset\n");
2996 /* Determine how many group fds need to be passed */
2998 QLIST_FOREACH(group
, &group_list
, next
) {
2999 for (i
= 0; i
< info
->count
; i
++) {
3000 if (group
->groupid
== devices
[i
].group_id
) {
3007 reset
= g_malloc0(sizeof(*reset
) + (count
* sizeof(*fds
)));
3008 reset
->argsz
= sizeof(*reset
) + (count
* sizeof(*fds
));
3009 fds
= &reset
->group_fds
[0];
3011 /* Fill in group fds */
3012 QLIST_FOREACH(group
, &group_list
, next
) {
3013 for (i
= 0; i
< info
->count
; i
++) {
3014 if (group
->groupid
== devices
[i
].group_id
) {
3015 fds
[reset
->count
++] = group
->fd
;
3022 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_PCI_HOT_RESET
, reset
);
3025 DPRINTF("%04x:%02x:%02x.%x hot reset: %s\n", vdev
->host
.domain
,
3026 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
3027 ret
? "%m" : "Success");
3030 /* Re-enable INTx on affected devices */
3031 for (i
= 0; i
< info
->count
; i
++) {
3032 PCIHostDeviceAddress host
;
3035 host
.domain
= devices
[i
].segment
;
3036 host
.bus
= devices
[i
].bus
;
3037 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
3038 host
.function
= PCI_FUNC(devices
[i
].devfn
);
3040 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
3044 QLIST_FOREACH(group
, &group_list
, next
) {
3045 if (group
->groupid
== devices
[i
].group_id
) {
3054 QLIST_FOREACH(tmp
, &group
->device_list
, next
) {
3055 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
3056 vfio_pci_post_reset(tmp
);
3062 vfio_pci_post_reset(vdev
);
3069 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
3070 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
3071 * of doing hot resets when there is only a single device per bus. The in-use
3072 * here refers to how many VFIODevices are affected. A hot reset that affects
3073 * multiple devices, but only a single in-use device, means that we can call
3074 * it from our bus ->reset() callback since the extent is effectively a single
3075 * device. This allows us to make use of it in the hotplug path. When there
3076 * are multiple in-use devices, we can only trigger the hot reset during a
3077 * system reset and thus from our reset handler. We separate _one vs _multi
3078 * here so that we don't overlap and do a double reset on the system reset
3079 * path where both our reset handler and ->reset() callback are used. Calling
3080 * _one() will only do a hot reset for the one in-use devices case, calling
3081 * _multi() will do nothing if a _one() would have been sufficient.
3083 static int vfio_pci_hot_reset_one(VFIODevice
*vdev
)
3085 return vfio_pci_hot_reset(vdev
, true);
3088 static int vfio_pci_hot_reset_multi(VFIODevice
*vdev
)
3090 return vfio_pci_hot_reset(vdev
, false);
3093 static void vfio_pci_reset_handler(void *opaque
)
3098 QLIST_FOREACH(group
, &group_list
, next
) {
3099 QLIST_FOREACH(vdev
, &group
->device_list
, next
) {
3100 if (!vdev
->reset_works
|| (!vdev
->has_flr
&& vdev
->has_pm_reset
)) {
3101 vdev
->needs_reset
= true;
3106 QLIST_FOREACH(group
, &group_list
, next
) {
3107 QLIST_FOREACH(vdev
, &group
->device_list
, next
) {
3108 if (vdev
->needs_reset
) {
3109 vfio_pci_hot_reset_multi(vdev
);
3115 static void vfio_kvm_device_add_group(VFIOGroup
*group
)
3118 struct kvm_device_attr attr
= {
3119 .group
= KVM_DEV_VFIO_GROUP
,
3120 .attr
= KVM_DEV_VFIO_GROUP_ADD
,
3121 .addr
= (uint64_t)(unsigned long)&group
->fd
,
3124 if (!kvm_enabled()) {
3128 if (vfio_kvm_device_fd
< 0) {
3129 struct kvm_create_device cd
= {
3130 .type
= KVM_DEV_TYPE_VFIO
,
3133 if (kvm_vm_ioctl(kvm_state
, KVM_CREATE_DEVICE
, &cd
)) {
3134 DPRINTF("KVM_CREATE_DEVICE: %m\n");
3138 vfio_kvm_device_fd
= cd
.fd
;
3141 if (ioctl(vfio_kvm_device_fd
, KVM_SET_DEVICE_ATTR
, &attr
)) {
3142 error_report("Failed to add group %d to KVM VFIO device: %m",
3148 static void vfio_kvm_device_del_group(VFIOGroup
*group
)
3151 struct kvm_device_attr attr
= {
3152 .group
= KVM_DEV_VFIO_GROUP
,
3153 .attr
= KVM_DEV_VFIO_GROUP_DEL
,
3154 .addr
= (uint64_t)(unsigned long)&group
->fd
,
3157 if (vfio_kvm_device_fd
< 0) {
3161 if (ioctl(vfio_kvm_device_fd
, KVM_SET_DEVICE_ATTR
, &attr
)) {
3162 error_report("Failed to remove group %d to KVM VFIO device: %m",
3168 static int vfio_connect_container(VFIOGroup
*group
)
3170 VFIOContainer
*container
;
3173 if (group
->container
) {
3177 QLIST_FOREACH(container
, &container_list
, next
) {
3178 if (!ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &container
->fd
)) {
3179 group
->container
= container
;
3180 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
3185 fd
= qemu_open("/dev/vfio/vfio", O_RDWR
);
3187 error_report("vfio: failed to open /dev/vfio/vfio: %m");
3191 ret
= ioctl(fd
, VFIO_GET_API_VERSION
);
3192 if (ret
!= VFIO_API_VERSION
) {
3193 error_report("vfio: supported vfio version: %d, "
3194 "reported version: %d", VFIO_API_VERSION
, ret
);
3199 container
= g_malloc0(sizeof(*container
));
3202 if (ioctl(fd
, VFIO_CHECK_EXTENSION
, VFIO_TYPE1_IOMMU
)) {
3203 ret
= ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &fd
);
3205 error_report("vfio: failed to set group container: %m");
3211 ret
= ioctl(fd
, VFIO_SET_IOMMU
, VFIO_TYPE1_IOMMU
);
3213 error_report("vfio: failed to set iommu for container: %m");
3219 container
->iommu_data
.listener
= vfio_memory_listener
;
3220 container
->iommu_data
.release
= vfio_listener_release
;
3222 memory_listener_register(&container
->iommu_data
.listener
, &address_space_memory
);
3224 error_report("vfio: No available IOMMU models");
3230 QLIST_INIT(&container
->group_list
);
3231 QLIST_INSERT_HEAD(&container_list
, container
, next
);
3233 group
->container
= container
;
3234 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
3239 static void vfio_disconnect_container(VFIOGroup
*group
)
3241 VFIOContainer
*container
= group
->container
;
3243 if (ioctl(group
->fd
, VFIO_GROUP_UNSET_CONTAINER
, &container
->fd
)) {
3244 error_report("vfio: error disconnecting group %d from container",
3248 QLIST_REMOVE(group
, container_next
);
3249 group
->container
= NULL
;
3251 if (QLIST_EMPTY(&container
->group_list
)) {
3252 if (container
->iommu_data
.release
) {
3253 container
->iommu_data
.release(container
);
3255 QLIST_REMOVE(container
, next
);
3256 DPRINTF("vfio_disconnect_container: close container->fd\n");
3257 close(container
->fd
);
3262 static VFIOGroup
*vfio_get_group(int groupid
)
3266 struct vfio_group_status status
= { .argsz
= sizeof(status
) };
3268 QLIST_FOREACH(group
, &group_list
, next
) {
3269 if (group
->groupid
== groupid
) {
3274 group
= g_malloc0(sizeof(*group
));
3276 snprintf(path
, sizeof(path
), "/dev/vfio/%d", groupid
);
3277 group
->fd
= qemu_open(path
, O_RDWR
);
3278 if (group
->fd
< 0) {
3279 error_report("vfio: error opening %s: %m", path
);
3284 if (ioctl(group
->fd
, VFIO_GROUP_GET_STATUS
, &status
)) {
3285 error_report("vfio: error getting group status: %m");
3291 if (!(status
.flags
& VFIO_GROUP_FLAGS_VIABLE
)) {
3292 error_report("vfio: error, group %d is not viable, please ensure "
3293 "all devices within the iommu_group are bound to their "
3294 "vfio bus driver.", groupid
);
3300 group
->groupid
= groupid
;
3301 QLIST_INIT(&group
->device_list
);
3303 if (vfio_connect_container(group
)) {
3304 error_report("vfio: failed to setup container for group %d", groupid
);
3310 if (QLIST_EMPTY(&group_list
)) {
3311 qemu_register_reset(vfio_pci_reset_handler
, NULL
);
3314 QLIST_INSERT_HEAD(&group_list
, group
, next
);
3316 vfio_kvm_device_add_group(group
);
3321 static void vfio_put_group(VFIOGroup
*group
)
3323 if (!QLIST_EMPTY(&group
->device_list
)) {
3327 vfio_kvm_device_del_group(group
);
3328 vfio_disconnect_container(group
);
3329 QLIST_REMOVE(group
, next
);
3330 DPRINTF("vfio_put_group: close group->fd\n");
3334 if (QLIST_EMPTY(&group_list
)) {
3335 qemu_unregister_reset(vfio_pci_reset_handler
, NULL
);
3339 static int vfio_get_device(VFIOGroup
*group
, const char *name
, VFIODevice
*vdev
)
3341 struct vfio_device_info dev_info
= { .argsz
= sizeof(dev_info
) };
3342 struct vfio_region_info reg_info
= { .argsz
= sizeof(reg_info
) };
3343 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
) };
3346 ret
= ioctl(group
->fd
, VFIO_GROUP_GET_DEVICE_FD
, name
);
3348 error_report("vfio: error getting device %s from group %d: %m",
3349 name
, group
->groupid
);
3350 error_printf("Verify all devices in group %d are bound to vfio-pci "
3351 "or pci-stub and not already in use\n", group
->groupid
);
3356 vdev
->group
= group
;
3357 QLIST_INSERT_HEAD(&group
->device_list
, vdev
, next
);
3359 /* Sanity check device */
3360 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_INFO
, &dev_info
);
3362 error_report("vfio: error getting device info: %m");
3366 DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name
,
3367 dev_info
.flags
, dev_info
.num_regions
, dev_info
.num_irqs
);
3369 if (!(dev_info
.flags
& VFIO_DEVICE_FLAGS_PCI
)) {
3370 error_report("vfio: Um, this isn't a PCI device");
3374 vdev
->reset_works
= !!(dev_info
.flags
& VFIO_DEVICE_FLAGS_RESET
);
3376 if (dev_info
.num_regions
< VFIO_PCI_CONFIG_REGION_INDEX
+ 1) {
3377 error_report("vfio: unexpected number of io regions %u",
3378 dev_info
.num_regions
);
3382 if (dev_info
.num_irqs
< VFIO_PCI_MSIX_IRQ_INDEX
+ 1) {
3383 error_report("vfio: unexpected number of irqs %u", dev_info
.num_irqs
);
3387 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
3390 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3392 error_report("vfio: Error getting region %d info: %m", i
);
3396 DPRINTF("Device %s region %d:\n", name
, i
);
3397 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3398 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
3399 (unsigned long)reg_info
.flags
);
3401 vdev
->bars
[i
].flags
= reg_info
.flags
;
3402 vdev
->bars
[i
].size
= reg_info
.size
;
3403 vdev
->bars
[i
].fd_offset
= reg_info
.offset
;
3404 vdev
->bars
[i
].fd
= vdev
->fd
;
3405 vdev
->bars
[i
].nr
= i
;
3406 QLIST_INIT(&vdev
->bars
[i
].quirks
);
3409 reg_info
.index
= VFIO_PCI_CONFIG_REGION_INDEX
;
3411 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3413 error_report("vfio: Error getting config info: %m");
3417 DPRINTF("Device %s config:\n", name
);
3418 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3419 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
3420 (unsigned long)reg_info
.flags
);
3422 vdev
->config_size
= reg_info
.size
;
3423 if (vdev
->config_size
== PCI_CONFIG_SPACE_SIZE
) {
3424 vdev
->pdev
.cap_present
&= ~QEMU_PCI_CAP_EXPRESS
;
3426 vdev
->config_offset
= reg_info
.offset
;
3428 if ((vdev
->features
& VFIO_FEATURE_ENABLE_VGA
) &&
3429 dev_info
.num_regions
> VFIO_PCI_VGA_REGION_INDEX
) {
3430 struct vfio_region_info vga_info
= {
3431 .argsz
= sizeof(vga_info
),
3432 .index
= VFIO_PCI_VGA_REGION_INDEX
,
3435 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, &vga_info
);
3438 "vfio: Device does not support requested feature x-vga");
3442 if (!(vga_info
.flags
& VFIO_REGION_INFO_FLAG_READ
) ||
3443 !(vga_info
.flags
& VFIO_REGION_INFO_FLAG_WRITE
) ||
3444 vga_info
.size
< 0xbffff + 1) {
3445 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
3446 (unsigned long)vga_info
.flags
,
3447 (unsigned long)vga_info
.size
);
3451 vdev
->vga
.fd_offset
= vga_info
.offset
;
3452 vdev
->vga
.fd
= vdev
->fd
;
3454 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].offset
= QEMU_PCI_VGA_MEM_BASE
;
3455 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].nr
= QEMU_PCI_VGA_MEM
;
3456 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].quirks
);
3458 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].offset
= QEMU_PCI_VGA_IO_LO_BASE
;
3459 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].nr
= QEMU_PCI_VGA_IO_LO
;
3460 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].quirks
);
3462 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].offset
= QEMU_PCI_VGA_IO_HI_BASE
;
3463 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].nr
= QEMU_PCI_VGA_IO_HI
;
3464 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
);
3466 vdev
->has_vga
= true;
3468 irq_info
.index
= VFIO_PCI_ERR_IRQ_INDEX
;
3470 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
3472 /* This can fail for an old kernel or legacy PCI dev */
3473 DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure: %m\n");
3475 } else if (irq_info
.count
== 1) {
3476 vdev
->pci_aer
= true;
3478 error_report("vfio: %04x:%02x:%02x.%x "
3479 "Could not enable error recovery for the device",
3480 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3481 vdev
->host
.function
);
3486 QLIST_REMOVE(vdev
, next
);
3493 static void vfio_put_device(VFIODevice
*vdev
)
3495 QLIST_REMOVE(vdev
, next
);
3497 DPRINTF("vfio_put_device: close vdev->fd\n");
3505 static void vfio_err_notifier_handler(void *opaque
)
3507 VFIODevice
*vdev
= opaque
;
3509 if (!event_notifier_test_and_clear(&vdev
->err_notifier
)) {
3514 * TBD. Retrieve the error details and decide what action
3515 * needs to be taken. One of the actions could be to pass
3516 * the error to the guest and have the guest driver recover
3517 * from the error. This requires that PCIe capabilities be
3518 * exposed to the guest. For now, we just terminate the
3519 * guest to contain the error.
3522 error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. "
3523 "Please collect any data possible and then kill the guest",
3524 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
3525 vdev
->host
.slot
, vdev
->host
.function
);
3527 vm_stop(RUN_STATE_IO_ERROR
);
3531 * Registers error notifier for devices supporting error recovery.
3532 * If we encounter a failure in this function, we report an error
3533 * and continue after disabling error recovery support for the
3536 static void vfio_register_err_notifier(VFIODevice
*vdev
)
3540 struct vfio_irq_set
*irq_set
;
3543 if (!vdev
->pci_aer
) {
3547 if (event_notifier_init(&vdev
->err_notifier
, 0)) {
3548 error_report("vfio: Unable to init event notifier for error detection");
3549 vdev
->pci_aer
= false;
3553 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
3555 irq_set
= g_malloc0(argsz
);
3556 irq_set
->argsz
= argsz
;
3557 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
3558 VFIO_IRQ_SET_ACTION_TRIGGER
;
3559 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
3562 pfd
= (int32_t *)&irq_set
->data
;
3564 *pfd
= event_notifier_get_fd(&vdev
->err_notifier
);
3565 qemu_set_fd_handler(*pfd
, vfio_err_notifier_handler
, NULL
, vdev
);
3567 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
3569 error_report("vfio: Failed to set up error notification");
3570 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
3571 event_notifier_cleanup(&vdev
->err_notifier
);
3572 vdev
->pci_aer
= false;
3577 static void vfio_unregister_err_notifier(VFIODevice
*vdev
)
3580 struct vfio_irq_set
*irq_set
;
3584 if (!vdev
->pci_aer
) {
3588 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
3590 irq_set
= g_malloc0(argsz
);
3591 irq_set
->argsz
= argsz
;
3592 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
3593 VFIO_IRQ_SET_ACTION_TRIGGER
;
3594 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
3597 pfd
= (int32_t *)&irq_set
->data
;
3600 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
3602 error_report("vfio: Failed to de-assign error fd: %m");
3605 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->err_notifier
),
3607 event_notifier_cleanup(&vdev
->err_notifier
);
3610 static int vfio_initfn(PCIDevice
*pdev
)
3612 VFIODevice
*pvdev
, *vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
3614 char path
[PATH_MAX
], iommu_group_path
[PATH_MAX
], *group_name
;
3620 /* Check that the host device exists */
3621 snprintf(path
, sizeof(path
),
3622 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
3623 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3624 vdev
->host
.function
);
3625 if (stat(path
, &st
) < 0) {
3626 error_report("vfio: error: no such host device: %s", path
);
3630 strncat(path
, "iommu_group", sizeof(path
) - strlen(path
) - 1);
3632 len
= readlink(path
, iommu_group_path
, PATH_MAX
);
3634 error_report("vfio: error no iommu_group for device");
3638 iommu_group_path
[len
] = 0;
3639 group_name
= basename(iommu_group_path
);
3641 if (sscanf(group_name
, "%d", &groupid
) != 1) {
3642 error_report("vfio: error reading %s: %m", path
);
3646 DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__
, vdev
->host
.domain
,
3647 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, groupid
);
3649 group
= vfio_get_group(groupid
);
3651 error_report("vfio: failed to get group %d", groupid
);
3655 snprintf(path
, sizeof(path
), "%04x:%02x:%02x.%01x",
3656 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3657 vdev
->host
.function
);
3659 QLIST_FOREACH(pvdev
, &group
->device_list
, next
) {
3660 if (pvdev
->host
.domain
== vdev
->host
.domain
&&
3661 pvdev
->host
.bus
== vdev
->host
.bus
&&
3662 pvdev
->host
.slot
== vdev
->host
.slot
&&
3663 pvdev
->host
.function
== vdev
->host
.function
) {
3665 error_report("vfio: error: device %s is already attached", path
);
3666 vfio_put_group(group
);
3671 ret
= vfio_get_device(group
, path
, vdev
);
3673 error_report("vfio: failed to get device %s", path
);
3674 vfio_put_group(group
);
3678 /* Get a copy of config space */
3679 ret
= pread(vdev
->fd
, vdev
->pdev
.config
,
3680 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
3681 vdev
->config_offset
);
3682 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
3683 ret
= ret
< 0 ? -errno
: -EFAULT
;
3684 error_report("vfio: Failed to read device config space");
3688 /* vfio emulates a lot for us, but some bits need extra love */
3689 vdev
->emulated_config_bits
= g_malloc0(vdev
->config_size
);
3691 /* QEMU can choose to expose the ROM or not */
3692 memset(vdev
->emulated_config_bits
+ PCI_ROM_ADDRESS
, 0xff, 4);
3694 /* QEMU can change multi-function devices to single function, or reverse */
3695 vdev
->emulated_config_bits
[PCI_HEADER_TYPE
] =
3696 PCI_HEADER_TYPE_MULTI_FUNCTION
;
3698 /* Restore or clear multifunction, this is always controlled by QEMU */
3699 if (vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
3700 vdev
->pdev
.config
[PCI_HEADER_TYPE
] |= PCI_HEADER_TYPE_MULTI_FUNCTION
;
3702 vdev
->pdev
.config
[PCI_HEADER_TYPE
] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION
;
3706 * Clear host resource mapping info. If we choose not to register a
3707 * BAR, such as might be the case with the option ROM, we can get
3708 * confusing, unwritable, residual addresses from the host here.
3710 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
3711 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
3713 vfio_pci_size_rom(vdev
);
3715 ret
= vfio_early_setup_msix(vdev
);
3720 vfio_map_bars(vdev
);
3722 ret
= vfio_add_capabilities(vdev
);
3727 /* QEMU emulates all of MSI & MSIX */
3728 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
) {
3729 memset(vdev
->emulated_config_bits
+ pdev
->msix_cap
, 0xff,
3733 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
) {
3734 memset(vdev
->emulated_config_bits
+ pdev
->msi_cap
, 0xff,
3735 vdev
->msi_cap_size
);
3738 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
3739 vdev
->intx
.mmap_timer
= timer_new_ms(QEMU_CLOCK_VIRTUAL
,
3740 vfio_intx_mmap_enable
, vdev
);
3741 pci_device_set_intx_routing_notifier(&vdev
->pdev
, vfio_update_irq
);
3742 ret
= vfio_enable_intx(vdev
);
3748 add_boot_device_path(vdev
->bootindex
, &pdev
->qdev
, NULL
);
3749 vfio_register_err_notifier(vdev
);
3754 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3755 vfio_teardown_msi(vdev
);
3756 vfio_unmap_bars(vdev
);
3758 g_free(vdev
->emulated_config_bits
);
3759 vfio_put_device(vdev
);
3760 vfio_put_group(group
);
3764 static void vfio_exitfn(PCIDevice
*pdev
)
3766 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
3767 VFIOGroup
*group
= vdev
->group
;
3769 vfio_unregister_err_notifier(vdev
);
3770 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3771 vfio_disable_interrupts(vdev
);
3772 if (vdev
->intx
.mmap_timer
) {
3773 timer_free(vdev
->intx
.mmap_timer
);
3775 vfio_teardown_msi(vdev
);
3776 vfio_unmap_bars(vdev
);
3777 g_free(vdev
->emulated_config_bits
);
3779 vfio_put_device(vdev
);
3780 vfio_put_group(group
);
3783 static void vfio_pci_reset(DeviceState
*dev
)
3785 PCIDevice
*pdev
= DO_UPCAST(PCIDevice
, qdev
, dev
);
3786 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
3788 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
3789 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
3791 vfio_pci_pre_reset(vdev
);
3793 if (vdev
->reset_works
&& (vdev
->has_flr
|| !vdev
->has_pm_reset
) &&
3794 !ioctl(vdev
->fd
, VFIO_DEVICE_RESET
)) {
3795 DPRINTF("%04x:%02x:%02x.%x FLR/VFIO_DEVICE_RESET\n", vdev
->host
.domain
,
3796 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
3800 /* See if we can do our own bus reset */
3801 if (!vfio_pci_hot_reset_one(vdev
)) {
3805 /* If nothing else works and the device supports PM reset, use it */
3806 if (vdev
->reset_works
&& vdev
->has_pm_reset
&&
3807 !ioctl(vdev
->fd
, VFIO_DEVICE_RESET
)) {
3808 DPRINTF("%04x:%02x:%02x.%x PCI PM Reset\n", vdev
->host
.domain
,
3809 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
3814 vfio_pci_post_reset(vdev
);
3817 static Property vfio_pci_dev_properties
[] = {
3818 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice
, host
),
3819 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice
,
3820 intx
.mmap_timeout
, 1100),
3821 DEFINE_PROP_BIT("x-vga", VFIODevice
, features
,
3822 VFIO_FEATURE_ENABLE_VGA_BIT
, false),
3823 DEFINE_PROP_INT32("bootindex", VFIODevice
, bootindex
, -1),
3825 * TODO - support passed fds... is this necessary?
3826 * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
3827 * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
3829 DEFINE_PROP_END_OF_LIST(),
3832 static const VMStateDescription vfio_pci_vmstate
= {
3837 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
3839 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3840 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
3842 dc
->reset
= vfio_pci_reset
;
3843 dc
->props
= vfio_pci_dev_properties
;
3844 dc
->vmsd
= &vfio_pci_vmstate
;
3845 dc
->desc
= "VFIO-based PCI device assignment";
3846 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
3847 pdc
->init
= vfio_initfn
;
3848 pdc
->exit
= vfio_exitfn
;
3849 pdc
->config_read
= vfio_pci_read_config
;
3850 pdc
->config_write
= vfio_pci_write_config
;
3851 pdc
->is_express
= 1; /* We might be */
3854 static const TypeInfo vfio_pci_dev_info
= {
3856 .parent
= TYPE_PCI_DEVICE
,
3857 .instance_size
= sizeof(VFIODevice
),
3858 .class_init
= vfio_pci_dev_class_init
,
3861 static void register_vfio_pci_dev_type(void)
3863 type_register_static(&vfio_pci_dev_info
);
3866 type_init(register_vfio_pci_dev_type
)