2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
26 #include <sys/types.h>
30 #include "exec/address-spaces.h"
31 #include "exec/memory.h"
32 #include "hw/pci/msi.h"
33 #include "hw/pci/msix.h"
34 #include "hw/pci/pci.h"
35 #include "qemu-common.h"
36 #include "qemu/error-report.h"
37 #include "qemu/event_notifier.h"
38 #include "qemu/queue.h"
39 #include "qemu/range.h"
40 #include "sysemu/kvm.h"
41 #include "sysemu/sysemu.h"
43 #include "hw/vfio/vfio.h"
44 #include "hw/vfio/vfio-common.h"
48 typedef struct VFIOQuirk
{
50 struct VFIOPCIDevice
*vdev
;
51 QLIST_ENTRY(VFIOQuirk
) next
;
53 uint32_t base_offset
:TARGET_PAGE_BITS
;
54 uint32_t address_offset
:TARGET_PAGE_BITS
;
55 uint32_t address_size
:3;
58 uint32_t address_match
;
59 uint32_t address_mask
;
61 uint32_t address_val
:TARGET_PAGE_BITS
;
62 uint32_t data_offset
:TARGET_PAGE_BITS
;
71 typedef struct VFIOBAR
{
75 QLIST_HEAD(, VFIOQuirk
) quirks
;
78 typedef struct VFIOVGARegion
{
82 QLIST_HEAD(, VFIOQuirk
) quirks
;
85 typedef struct VFIOVGA
{
88 VFIOVGARegion region
[QEMU_PCI_VGA_NUM_REGIONS
];
91 typedef struct VFIOINTx
{
92 bool pending
; /* interrupt pending */
93 bool kvm_accel
; /* set when QEMU bypass through KVM enabled */
94 uint8_t pin
; /* which pin to pull for qemu_set_irq */
95 EventNotifier interrupt
; /* eventfd triggered on interrupt */
96 EventNotifier unmask
; /* eventfd for unmask on QEMU bypass */
97 PCIINTxRoute route
; /* routing info for QEMU bypass */
98 uint32_t mmap_timeout
; /* delay to re-enable mmaps after interrupt */
99 QEMUTimer
*mmap_timer
; /* enable mmaps after periods w/o interrupts */
102 typedef struct VFIOMSIVector
{
104 * Two interrupt paths are configured per vector. The first, is only used
105 * for interrupts injected via QEMU. This is typically the non-accel path,
106 * but may also be used when we want QEMU to handle masking and pending
107 * bits. The KVM path bypasses QEMU and is therefore higher performance,
108 * but requires masking at the device. virq is used to track the MSI route
109 * through KVM, thus kvm_interrupt is only available when virq is set to a
110 * valid (>= 0) value.
112 EventNotifier interrupt
;
113 EventNotifier kvm_interrupt
;
114 struct VFIOPCIDevice
*vdev
; /* back pointer to device */
126 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
127 typedef struct VFIOMSIXInfo
{
131 uint32_t table_offset
;
133 MemoryRegion mmap_mem
;
137 typedef struct VFIOPCIDevice
{
141 unsigned int config_size
;
142 uint8_t *emulated_config_bits
; /* QEMU emulated bits, little-endian */
143 off_t config_offset
; /* Offset of config space region within device fd */
144 unsigned int rom_size
;
145 off_t rom_offset
; /* Offset of ROM region within device fd */
148 VFIOMSIVector
*msi_vectors
;
150 int nr_vectors
; /* Number of MSI/MSIX vectors currently in use */
151 int interrupt
; /* Current interrupt type */
152 VFIOBAR bars
[PCI_NUM_REGIONS
- 1]; /* No ROM */
153 VFIOVGA vga
; /* 0xa0000, 0x3b0, 0x3c0 */
154 PCIHostDeviceAddress host
;
155 EventNotifier err_notifier
;
157 #define VFIO_FEATURE_ENABLE_VGA_BIT 0
158 #define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
165 bool rom_read_failed
;
168 typedef struct VFIORomBlacklistEntry
{
171 } VFIORomBlacklistEntry
;
174 * List of device ids/vendor ids for which to disable
175 * option rom loading. This avoids the guest hangs during rom
176 * execution as noticed with the BCM 57810 card for lack of a
177 * more better way to handle such issues.
178 * The user can still override by specifying a romfile or
180 * Please see https://bugs.launchpad.net/qemu/+bug/1284874
181 * for an analysis of the 57810 card hang. When adding
182 * a new vendor id/device id combination below, please also add
183 * your card/environment details and information that could
184 * help in debugging to the bug tracking this issue
186 static const VFIORomBlacklistEntry romblacklist
[] = {
187 /* Broadcom BCM 57810 */
191 #define MSIX_CAP_LENGTH 12
193 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
);
194 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
);
195 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
196 uint32_t val
, int len
);
197 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
);
200 * Disabling BAR mmaping can be slow, but toggling it around INTx can
201 * also be a huge overhead. We try to get the best of both worlds by
202 * waiting until an interrupt to disable mmaps (subsequent transitions
203 * to the same state are effectively no overhead). If the interrupt has
204 * been serviced and the time gap is long enough, we re-enable mmaps for
205 * performance. This works well for things like graphics cards, which
206 * may not use their interrupt at all and are penalized to an unusable
207 * level by read/write BAR traps. Other devices, like NICs, have more
208 * regular interrupts and see much better latency by staying in non-mmap
209 * mode. We therefore set the default mmap_timeout such that a ping
210 * is just enough to keep the mmap disabled. Users can experiment with
211 * other options with the x-intx-mmap-timeout-ms parameter (a value of
212 * zero disables the timer).
214 static void vfio_intx_mmap_enable(void *opaque
)
216 VFIOPCIDevice
*vdev
= opaque
;
218 if (vdev
->intx
.pending
) {
219 timer_mod(vdev
->intx
.mmap_timer
,
220 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
224 vfio_mmap_set_enabled(vdev
, true);
227 static void vfio_intx_interrupt(void *opaque
)
229 VFIOPCIDevice
*vdev
= opaque
;
231 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
235 trace_vfio_intx_interrupt(vdev
->vbasedev
.name
, 'A' + vdev
->intx
.pin
);
237 vdev
->intx
.pending
= true;
238 pci_irq_assert(&vdev
->pdev
);
239 vfio_mmap_set_enabled(vdev
, false);
240 if (vdev
->intx
.mmap_timeout
) {
241 timer_mod(vdev
->intx
.mmap_timer
,
242 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
246 static void vfio_eoi(VFIODevice
*vbasedev
)
248 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
250 if (!vdev
->intx
.pending
) {
254 trace_vfio_eoi(vbasedev
->name
);
256 vdev
->intx
.pending
= false;
257 pci_irq_deassert(&vdev
->pdev
);
258 vfio_unmask_single_irqindex(vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
261 static void vfio_enable_intx_kvm(VFIOPCIDevice
*vdev
)
264 struct kvm_irqfd irqfd
= {
265 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
266 .gsi
= vdev
->intx
.route
.irq
,
267 .flags
= KVM_IRQFD_FLAG_RESAMPLE
,
269 struct vfio_irq_set
*irq_set
;
273 if (!VFIO_ALLOW_KVM_INTX
|| !kvm_irqfds_enabled() ||
274 vdev
->intx
.route
.mode
!= PCI_INTX_ENABLED
||
275 !kvm_resamplefds_enabled()) {
279 /* Get to a known interrupt state */
280 qemu_set_fd_handler(irqfd
.fd
, NULL
, NULL
, vdev
);
281 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
282 vdev
->intx
.pending
= false;
283 pci_irq_deassert(&vdev
->pdev
);
285 /* Get an eventfd for resample/unmask */
286 if (event_notifier_init(&vdev
->intx
.unmask
, 0)) {
287 error_report("vfio: Error: event_notifier_init failed eoi");
291 /* KVM triggers it, VFIO listens for it */
292 irqfd
.resamplefd
= event_notifier_get_fd(&vdev
->intx
.unmask
);
294 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
295 error_report("vfio: Error: Failed to setup resample irqfd: %m");
299 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
301 irq_set
= g_malloc0(argsz
);
302 irq_set
->argsz
= argsz
;
303 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_UNMASK
;
304 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
307 pfd
= (int32_t *)&irq_set
->data
;
309 *pfd
= irqfd
.resamplefd
;
311 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
314 error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
319 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
321 vdev
->intx
.kvm_accel
= true;
323 trace_vfio_enable_intx_kvm(vdev
->vbasedev
.name
);
328 irqfd
.flags
= KVM_IRQFD_FLAG_DEASSIGN
;
329 kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
);
331 event_notifier_cleanup(&vdev
->intx
.unmask
);
333 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
334 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
338 static void vfio_disable_intx_kvm(VFIOPCIDevice
*vdev
)
341 struct kvm_irqfd irqfd
= {
342 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
343 .gsi
= vdev
->intx
.route
.irq
,
344 .flags
= KVM_IRQFD_FLAG_DEASSIGN
,
347 if (!vdev
->intx
.kvm_accel
) {
352 * Get to a known state, hardware masked, QEMU ready to accept new
353 * interrupts, QEMU IRQ de-asserted.
355 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
356 vdev
->intx
.pending
= false;
357 pci_irq_deassert(&vdev
->pdev
);
359 /* Tell KVM to stop listening for an INTx irqfd */
360 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
361 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
364 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
365 event_notifier_cleanup(&vdev
->intx
.unmask
);
367 /* QEMU starts listening for interrupt events. */
368 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
370 vdev
->intx
.kvm_accel
= false;
372 /* If we've missed an event, let it re-fire through QEMU */
373 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
375 trace_vfio_disable_intx_kvm(vdev
->vbasedev
.name
);
379 static void vfio_update_irq(PCIDevice
*pdev
)
381 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
384 if (vdev
->interrupt
!= VFIO_INT_INTx
) {
388 route
= pci_device_route_intx_to_irq(&vdev
->pdev
, vdev
->intx
.pin
);
390 if (!pci_intx_route_changed(&vdev
->intx
.route
, &route
)) {
391 return; /* Nothing changed */
394 trace_vfio_update_irq(vdev
->vbasedev
.name
,
395 vdev
->intx
.route
.irq
, route
.irq
);
397 vfio_disable_intx_kvm(vdev
);
399 vdev
->intx
.route
= route
;
401 if (route
.mode
!= PCI_INTX_ENABLED
) {
405 vfio_enable_intx_kvm(vdev
);
407 /* Re-enable the interrupt in cased we missed an EOI */
408 vfio_eoi(&vdev
->vbasedev
);
411 static int vfio_enable_intx(VFIOPCIDevice
*vdev
)
413 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
415 struct vfio_irq_set
*irq_set
;
422 vfio_disable_interrupts(vdev
);
424 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
425 pci_config_set_interrupt_pin(vdev
->pdev
.config
, pin
);
429 * Only conditional to avoid generating error messages on platforms
430 * where we won't actually use the result anyway.
432 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
433 vdev
->intx
.route
= pci_device_route_intx_to_irq(&vdev
->pdev
,
438 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
440 error_report("vfio: Error: event_notifier_init failed");
444 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
446 irq_set
= g_malloc0(argsz
);
447 irq_set
->argsz
= argsz
;
448 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
449 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
452 pfd
= (int32_t *)&irq_set
->data
;
454 *pfd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
455 qemu_set_fd_handler(*pfd
, vfio_intx_interrupt
, NULL
, vdev
);
457 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
460 error_report("vfio: Error: Failed to setup INTx fd: %m");
461 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
462 event_notifier_cleanup(&vdev
->intx
.interrupt
);
466 vfio_enable_intx_kvm(vdev
);
468 vdev
->interrupt
= VFIO_INT_INTx
;
470 trace_vfio_enable_intx(vdev
->vbasedev
.name
);
475 static void vfio_disable_intx(VFIOPCIDevice
*vdev
)
479 timer_del(vdev
->intx
.mmap_timer
);
480 vfio_disable_intx_kvm(vdev
);
481 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
482 vdev
->intx
.pending
= false;
483 pci_irq_deassert(&vdev
->pdev
);
484 vfio_mmap_set_enabled(vdev
, true);
486 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
487 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
488 event_notifier_cleanup(&vdev
->intx
.interrupt
);
490 vdev
->interrupt
= VFIO_INT_NONE
;
492 trace_vfio_disable_intx(vdev
->vbasedev
.name
);
498 static void vfio_msi_interrupt(void *opaque
)
500 VFIOMSIVector
*vector
= opaque
;
501 VFIOPCIDevice
*vdev
= vector
->vdev
;
502 int nr
= vector
- vdev
->msi_vectors
;
504 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
511 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
512 msg
= msix_get_message(&vdev
->pdev
, nr
);
513 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
514 msg
= msi_get_message(&vdev
->pdev
, nr
);
519 trace_vfio_msi_interrupt(vdev
->vbasedev
.name
, nr
, msg
.address
, msg
.data
);
522 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
523 msix_notify(&vdev
->pdev
, nr
);
524 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
525 msi_notify(&vdev
->pdev
, nr
);
527 error_report("vfio: MSI interrupt receieved, but not enabled?");
531 static int vfio_enable_vectors(VFIOPCIDevice
*vdev
, bool msix
)
533 struct vfio_irq_set
*irq_set
;
534 int ret
= 0, i
, argsz
;
537 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
539 irq_set
= g_malloc0(argsz
);
540 irq_set
->argsz
= argsz
;
541 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
542 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
544 irq_set
->count
= vdev
->nr_vectors
;
545 fds
= (int32_t *)&irq_set
->data
;
547 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
551 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
552 * bits, therefore we always use the KVM signaling path when setup.
553 * MSI-X mask and pending bits are emulated, so we want to use the
554 * KVM signaling path only when configured and unmasked.
556 if (vdev
->msi_vectors
[i
].use
) {
557 if (vdev
->msi_vectors
[i
].virq
< 0 ||
558 (msix
&& msix_is_masked(&vdev
->pdev
, i
))) {
559 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
561 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].kvm_interrupt
);
568 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
575 static void vfio_add_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage
*msg
,
580 if ((msix
&& !VFIO_ALLOW_KVM_MSIX
) ||
581 (!msix
&& !VFIO_ALLOW_KVM_MSI
) || !msg
) {
585 if (event_notifier_init(&vector
->kvm_interrupt
, 0)) {
589 virq
= kvm_irqchip_add_msi_route(kvm_state
, *msg
);
591 event_notifier_cleanup(&vector
->kvm_interrupt
);
595 if (kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->kvm_interrupt
,
597 kvm_irqchip_release_virq(kvm_state
, virq
);
598 event_notifier_cleanup(&vector
->kvm_interrupt
);
605 static void vfio_remove_kvm_msi_virq(VFIOMSIVector
*vector
)
607 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->kvm_interrupt
,
609 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
611 event_notifier_cleanup(&vector
->kvm_interrupt
);
614 static void vfio_update_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage msg
)
616 kvm_irqchip_update_msi_route(kvm_state
, vector
->virq
, msg
);
619 static int vfio_msix_vector_do_use(PCIDevice
*pdev
, unsigned int nr
,
620 MSIMessage
*msg
, IOHandler
*handler
)
622 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
623 VFIOMSIVector
*vector
;
626 trace_vfio_msix_vector_do_use(vdev
->vbasedev
.name
, nr
);
628 vector
= &vdev
->msi_vectors
[nr
];
633 if (event_notifier_init(&vector
->interrupt
, 0)) {
634 error_report("vfio: Error: event_notifier_init failed");
637 msix_vector_use(pdev
, nr
);
640 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
641 handler
, NULL
, vector
);
644 * Attempt to enable route through KVM irqchip,
645 * default to userspace handling if unavailable.
647 if (vector
->virq
>= 0) {
649 vfio_remove_kvm_msi_virq(vector
);
651 vfio_update_kvm_msi_virq(vector
, *msg
);
654 vfio_add_kvm_msi_virq(vector
, msg
, true);
658 * We don't want to have the host allocate all possible MSI vectors
659 * for a device if they're not in use, so we shutdown and incrementally
660 * increase them as needed.
662 if (vdev
->nr_vectors
< nr
+ 1) {
663 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
664 vdev
->nr_vectors
= nr
+ 1;
665 ret
= vfio_enable_vectors(vdev
, true);
667 error_report("vfio: failed to enable vectors, %d", ret
);
671 struct vfio_irq_set
*irq_set
;
674 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
676 irq_set
= g_malloc0(argsz
);
677 irq_set
->argsz
= argsz
;
678 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
679 VFIO_IRQ_SET_ACTION_TRIGGER
;
680 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
683 pfd
= (int32_t *)&irq_set
->data
;
685 if (vector
->virq
>= 0) {
686 *pfd
= event_notifier_get_fd(&vector
->kvm_interrupt
);
688 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
691 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
694 error_report("vfio: failed to modify vector, %d", ret
);
701 static int vfio_msix_vector_use(PCIDevice
*pdev
,
702 unsigned int nr
, MSIMessage msg
)
704 return vfio_msix_vector_do_use(pdev
, nr
, &msg
, vfio_msi_interrupt
);
707 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
709 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
710 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
712 trace_vfio_msix_vector_release(vdev
->vbasedev
.name
, nr
);
715 * There are still old guests that mask and unmask vectors on every
716 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
717 * the KVM setup in place, simply switch VFIO to use the non-bypass
718 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
719 * core will mask the interrupt and set pending bits, allowing it to
720 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
722 if (vector
->virq
>= 0) {
724 struct vfio_irq_set
*irq_set
;
727 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
729 irq_set
= g_malloc0(argsz
);
730 irq_set
->argsz
= argsz
;
731 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
732 VFIO_IRQ_SET_ACTION_TRIGGER
;
733 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
736 pfd
= (int32_t *)&irq_set
->data
;
738 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
740 ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
746 static void vfio_enable_msix(VFIOPCIDevice
*vdev
)
748 vfio_disable_interrupts(vdev
);
750 vdev
->msi_vectors
= g_malloc0(vdev
->msix
->entries
* sizeof(VFIOMSIVector
));
752 vdev
->interrupt
= VFIO_INT_MSIX
;
755 * Some communication channels between VF & PF or PF & fw rely on the
756 * physical state of the device and expect that enabling MSI-X from the
757 * guest enables the same on the host. When our guest is Linux, the
758 * guest driver call to pci_enable_msix() sets the enabling bit in the
759 * MSI-X capability, but leaves the vector table masked. We therefore
760 * can't rely on a vector_use callback (from request_irq() in the guest)
761 * to switch the physical device into MSI-X mode because that may come a
762 * long time after pci_enable_msix(). This code enables vector 0 with
763 * triggering to userspace, then immediately release the vector, leaving
764 * the physical device with no vectors enabled, but MSI-X enabled, just
765 * like the guest view.
767 vfio_msix_vector_do_use(&vdev
->pdev
, 0, NULL
, NULL
);
768 vfio_msix_vector_release(&vdev
->pdev
, 0);
770 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
771 vfio_msix_vector_release
, NULL
)) {
772 error_report("vfio: msix_set_vector_notifiers failed");
775 trace_vfio_enable_msix(vdev
->vbasedev
.name
);
778 static void vfio_enable_msi(VFIOPCIDevice
*vdev
)
782 vfio_disable_interrupts(vdev
);
784 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
786 vdev
->msi_vectors
= g_malloc0(vdev
->nr_vectors
* sizeof(VFIOMSIVector
));
788 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
789 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
790 MSIMessage msg
= msi_get_message(&vdev
->pdev
, i
);
796 if (event_notifier_init(&vector
->interrupt
, 0)) {
797 error_report("vfio: Error: event_notifier_init failed");
800 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
801 vfio_msi_interrupt
, NULL
, vector
);
804 * Attempt to enable route through KVM irqchip,
805 * default to userspace handling if unavailable.
807 vfio_add_kvm_msi_virq(vector
, &msg
, false);
810 /* Set interrupt type prior to possible interrupts */
811 vdev
->interrupt
= VFIO_INT_MSI
;
813 ret
= vfio_enable_vectors(vdev
, false);
816 error_report("vfio: Error: Failed to setup MSI fds: %m");
817 } else if (ret
!= vdev
->nr_vectors
) {
818 error_report("vfio: Error: Failed to enable %d "
819 "MSI vectors, retry with %d", vdev
->nr_vectors
, ret
);
822 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
823 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
824 if (vector
->virq
>= 0) {
825 vfio_remove_kvm_msi_virq(vector
);
827 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
829 event_notifier_cleanup(&vector
->interrupt
);
832 g_free(vdev
->msi_vectors
);
834 if (ret
> 0 && ret
!= vdev
->nr_vectors
) {
835 vdev
->nr_vectors
= ret
;
838 vdev
->nr_vectors
= 0;
841 * Failing to setup MSI doesn't really fall within any specification.
842 * Let's try leaving interrupts disabled and hope the guest figures
843 * out to fall back to INTx for this device.
845 error_report("vfio: Error: Failed to enable MSI");
846 vdev
->interrupt
= VFIO_INT_NONE
;
851 trace_vfio_enable_msi(vdev
->vbasedev
.name
, vdev
->nr_vectors
);
854 static void vfio_disable_msi_common(VFIOPCIDevice
*vdev
)
858 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
859 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
860 if (vdev
->msi_vectors
[i
].use
) {
861 if (vector
->virq
>= 0) {
862 vfio_remove_kvm_msi_virq(vector
);
864 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
866 event_notifier_cleanup(&vector
->interrupt
);
870 g_free(vdev
->msi_vectors
);
871 vdev
->msi_vectors
= NULL
;
872 vdev
->nr_vectors
= 0;
873 vdev
->interrupt
= VFIO_INT_NONE
;
875 vfio_enable_intx(vdev
);
878 static void vfio_disable_msix(VFIOPCIDevice
*vdev
)
882 msix_unset_vector_notifiers(&vdev
->pdev
);
885 * MSI-X will only release vectors if MSI-X is still enabled on the
886 * device, check through the rest and release it ourselves if necessary.
888 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
889 if (vdev
->msi_vectors
[i
].use
) {
890 vfio_msix_vector_release(&vdev
->pdev
, i
);
891 msix_vector_unuse(&vdev
->pdev
, i
);
895 if (vdev
->nr_vectors
) {
896 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
899 vfio_disable_msi_common(vdev
);
901 trace_vfio_disable_msix(vdev
->vbasedev
.name
);
904 static void vfio_disable_msi(VFIOPCIDevice
*vdev
)
906 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSI_IRQ_INDEX
);
907 vfio_disable_msi_common(vdev
);
909 trace_vfio_disable_msi(vdev
->vbasedev
.name
);
912 static void vfio_update_msi(VFIOPCIDevice
*vdev
)
916 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
917 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
920 if (!vector
->use
|| vector
->virq
< 0) {
924 msg
= msi_get_message(&vdev
->pdev
, i
);
925 vfio_update_kvm_msi_virq(vector
, msg
);
929 static void vfio_pci_load_rom(VFIOPCIDevice
*vdev
)
931 struct vfio_region_info reg_info
= {
932 .argsz
= sizeof(reg_info
),
933 .index
= VFIO_PCI_ROM_REGION_INDEX
939 if (ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
)) {
940 error_report("vfio: Error getting ROM info: %m");
944 trace_vfio_pci_load_rom(vdev
->vbasedev
.name
, (unsigned long)reg_info
.size
,
945 (unsigned long)reg_info
.offset
,
946 (unsigned long)reg_info
.flags
);
948 vdev
->rom_size
= size
= reg_info
.size
;
949 vdev
->rom_offset
= reg_info
.offset
;
951 if (!vdev
->rom_size
) {
952 vdev
->rom_read_failed
= true;
953 error_report("vfio-pci: Cannot read device rom at "
954 "%s", vdev
->vbasedev
.name
);
955 error_printf("Device option ROM contents are probably invalid "
956 "(check dmesg).\nSkip option ROM probe with rombar=0, "
957 "or load from file with romfile=\n");
961 vdev
->rom
= g_malloc(size
);
962 memset(vdev
->rom
, 0xff, size
);
965 bytes
= pread(vdev
->vbasedev
.fd
, vdev
->rom
+ off
,
966 size
, vdev
->rom_offset
+ off
);
969 } else if (bytes
> 0) {
973 if (errno
== EINTR
|| errno
== EAGAIN
) {
976 error_report("vfio: Error reading device ROM: %m");
982 static uint64_t vfio_rom_read(void *opaque
, hwaddr addr
, unsigned size
)
984 VFIOPCIDevice
*vdev
= opaque
;
993 /* Load the ROM lazily when the guest tries to read it */
994 if (unlikely(!vdev
->rom
&& !vdev
->rom_read_failed
)) {
995 vfio_pci_load_rom(vdev
);
998 memcpy(&val
, vdev
->rom
+ addr
,
999 (addr
< vdev
->rom_size
) ? MIN(size
, vdev
->rom_size
- addr
) : 0);
1006 data
= le16_to_cpu(val
.word
);
1009 data
= le32_to_cpu(val
.dword
);
1012 hw_error("vfio: unsupported read size, %d bytes\n", size
);
1016 trace_vfio_rom_read(vdev
->vbasedev
.name
, addr
, size
, data
);
1021 static void vfio_rom_write(void *opaque
, hwaddr addr
,
1022 uint64_t data
, unsigned size
)
1026 static const MemoryRegionOps vfio_rom_ops
= {
1027 .read
= vfio_rom_read
,
1028 .write
= vfio_rom_write
,
1029 .endianness
= DEVICE_LITTLE_ENDIAN
,
1032 static bool vfio_blacklist_opt_rom(VFIOPCIDevice
*vdev
)
1034 PCIDevice
*pdev
= &vdev
->pdev
;
1035 uint16_t vendor_id
, device_id
;
1038 vendor_id
= pci_get_word(pdev
->config
+ PCI_VENDOR_ID
);
1039 device_id
= pci_get_word(pdev
->config
+ PCI_DEVICE_ID
);
1041 while (count
< ARRAY_SIZE(romblacklist
)) {
1042 if (romblacklist
[count
].vendor_id
== vendor_id
&&
1043 romblacklist
[count
].device_id
== device_id
) {
1052 static void vfio_pci_size_rom(VFIOPCIDevice
*vdev
)
1054 uint32_t orig
, size
= cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK
);
1055 off_t offset
= vdev
->config_offset
+ PCI_ROM_ADDRESS
;
1056 DeviceState
*dev
= DEVICE(vdev
);
1058 int fd
= vdev
->vbasedev
.fd
;
1060 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
) {
1061 /* Since pci handles romfile, just print a message and return */
1062 if (vfio_blacklist_opt_rom(vdev
) && vdev
->pdev
.romfile
) {
1063 error_printf("Warning : Device at %04x:%02x:%02x.%x "
1064 "is known to cause system instability issues during "
1065 "option rom execution. "
1066 "Proceeding anyway since user specified romfile\n",
1067 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1068 vdev
->host
.function
);
1074 * Use the same size ROM BAR as the physical device. The contents
1075 * will get filled in later when the guest tries to read it.
1077 if (pread(fd
, &orig
, 4, offset
) != 4 ||
1078 pwrite(fd
, &size
, 4, offset
) != 4 ||
1079 pread(fd
, &size
, 4, offset
) != 4 ||
1080 pwrite(fd
, &orig
, 4, offset
) != 4) {
1081 error_report("%s(%04x:%02x:%02x.%x) failed: %m",
1082 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
1083 vdev
->host
.slot
, vdev
->host
.function
);
1087 size
= ~(le32_to_cpu(size
) & PCI_ROM_ADDRESS_MASK
) + 1;
1093 if (vfio_blacklist_opt_rom(vdev
)) {
1094 if (dev
->opts
&& qemu_opt_get(dev
->opts
, "rombar")) {
1095 error_printf("Warning : Device at %04x:%02x:%02x.%x "
1096 "is known to cause system instability issues during "
1097 "option rom execution. "
1098 "Proceeding anyway since user specified non zero value for "
1100 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1101 vdev
->host
.function
);
1103 error_printf("Warning : Rom loading for device at "
1104 "%04x:%02x:%02x.%x has been disabled due to "
1105 "system instability issues. "
1106 "Specify rombar=1 or romfile to force\n",
1107 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1108 vdev
->host
.function
);
1113 trace_vfio_pci_size_rom(vdev
->vbasedev
.name
, size
);
1115 snprintf(name
, sizeof(name
), "vfio[%04x:%02x:%02x.%x].rom",
1116 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1117 vdev
->host
.function
);
1119 memory_region_init_io(&vdev
->pdev
.rom
, OBJECT(vdev
),
1120 &vfio_rom_ops
, vdev
, name
, size
);
1122 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
,
1123 PCI_BASE_ADDRESS_SPACE_MEMORY
, &vdev
->pdev
.rom
);
1125 vdev
->pdev
.has_rom
= true;
1126 vdev
->rom_read_failed
= false;
1129 static void vfio_vga_write(void *opaque
, hwaddr addr
,
1130 uint64_t data
, unsigned size
)
1132 VFIOVGARegion
*region
= opaque
;
1133 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1140 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1147 buf
.word
= cpu_to_le16(data
);
1150 buf
.dword
= cpu_to_le32(data
);
1153 hw_error("vfio: unsupported write size, %d bytes", size
);
1157 if (pwrite(vga
->fd
, &buf
, size
, offset
) != size
) {
1158 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1159 __func__
, region
->offset
+ addr
, data
, size
);
1162 trace_vfio_vga_write(region
->offset
+ addr
, data
, size
);
1165 static uint64_t vfio_vga_read(void *opaque
, hwaddr addr
, unsigned size
)
1167 VFIOVGARegion
*region
= opaque
;
1168 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1176 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1178 if (pread(vga
->fd
, &buf
, size
, offset
) != size
) {
1179 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1180 __func__
, region
->offset
+ addr
, size
);
1181 return (uint64_t)-1;
1189 data
= le16_to_cpu(buf
.word
);
1192 data
= le32_to_cpu(buf
.dword
);
1195 hw_error("vfio: unsupported read size, %d bytes", size
);
1199 trace_vfio_vga_read(region
->offset
+ addr
, size
, data
);
1204 static const MemoryRegionOps vfio_vga_ops
= {
1205 .read
= vfio_vga_read
,
1206 .write
= vfio_vga_write
,
1207 .endianness
= DEVICE_LITTLE_ENDIAN
,
1211 * Device specific quirks
1214 /* Is range1 fully contained within range2? */
1215 static bool vfio_range_contained(uint64_t first1
, uint64_t len1
,
1216 uint64_t first2
, uint64_t len2
) {
1217 return (first1
>= first2
&& first1
+ len1
<= first2
+ len2
);
1220 static bool vfio_flags_enabled(uint8_t flags
, uint8_t mask
)
1222 return (mask
&& (flags
& mask
) == mask
);
1225 static uint64_t vfio_generic_window_quirk_read(void *opaque
,
1226 hwaddr addr
, unsigned size
)
1228 VFIOQuirk
*quirk
= opaque
;
1229 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1232 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1233 ranges_overlap(addr
, size
,
1234 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1235 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1237 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1238 quirk
->data
.data_size
)) {
1239 hw_error("%s: window data read not fully contained: %s",
1240 __func__
, memory_region_name(&quirk
->mem
));
1243 data
= vfio_pci_read_config(&vdev
->pdev
,
1244 quirk
->data
.address_val
+ offset
, size
);
1246 trace_vfio_generic_window_quirk_read(memory_region_name(&quirk
->mem
),
1247 vdev
->vbasedev
.name
,
1251 data
= vfio_region_read(&vdev
->bars
[quirk
->data
.bar
].region
,
1252 addr
+ quirk
->data
.base_offset
, size
);
1258 static void vfio_generic_window_quirk_write(void *opaque
, hwaddr addr
,
1259 uint64_t data
, unsigned size
)
1261 VFIOQuirk
*quirk
= opaque
;
1262 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1264 if (ranges_overlap(addr
, size
,
1265 quirk
->data
.address_offset
, quirk
->data
.address_size
)) {
1267 if (addr
!= quirk
->data
.address_offset
) {
1268 hw_error("%s: offset write into address window: %s",
1269 __func__
, memory_region_name(&quirk
->mem
));
1272 if ((data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1273 quirk
->data
.flags
|= quirk
->data
.write_flags
|
1274 quirk
->data
.read_flags
;
1275 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1277 quirk
->data
.flags
&= ~(quirk
->data
.write_flags
|
1278 quirk
->data
.read_flags
);
1282 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1283 ranges_overlap(addr
, size
,
1284 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1285 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1287 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1288 quirk
->data
.data_size
)) {
1289 hw_error("%s: window data write not fully contained: %s",
1290 __func__
, memory_region_name(&quirk
->mem
));
1293 vfio_pci_write_config(&vdev
->pdev
,
1294 quirk
->data
.address_val
+ offset
, data
, size
);
1295 trace_vfio_generic_window_quirk_write(memory_region_name(&quirk
->mem
),
1296 vdev
->vbasedev
.name
,
1302 vfio_region_write(&vdev
->bars
[quirk
->data
.bar
].region
,
1303 addr
+ quirk
->data
.base_offset
, data
, size
);
1306 static const MemoryRegionOps vfio_generic_window_quirk
= {
1307 .read
= vfio_generic_window_quirk_read
,
1308 .write
= vfio_generic_window_quirk_write
,
1309 .endianness
= DEVICE_LITTLE_ENDIAN
,
1312 static uint64_t vfio_generic_quirk_read(void *opaque
,
1313 hwaddr addr
, unsigned size
)
1315 VFIOQuirk
*quirk
= opaque
;
1316 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1317 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1318 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1321 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1322 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1323 if (!vfio_range_contained(addr
, size
, offset
,
1324 quirk
->data
.address_mask
+ 1)) {
1325 hw_error("%s: read not fully contained: %s",
1326 __func__
, memory_region_name(&quirk
->mem
));
1329 data
= vfio_pci_read_config(&vdev
->pdev
, addr
- offset
, size
);
1331 trace_vfio_generic_quirk_read(memory_region_name(&quirk
->mem
),
1332 vdev
->vbasedev
.name
, quirk
->data
.bar
,
1333 addr
+ base
, size
, data
);
1335 data
= vfio_region_read(&vdev
->bars
[quirk
->data
.bar
].region
,
1342 static void vfio_generic_quirk_write(void *opaque
, hwaddr addr
,
1343 uint64_t data
, unsigned size
)
1345 VFIOQuirk
*quirk
= opaque
;
1346 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1347 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1348 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1350 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1351 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1352 if (!vfio_range_contained(addr
, size
, offset
,
1353 quirk
->data
.address_mask
+ 1)) {
1354 hw_error("%s: write not fully contained: %s",
1355 __func__
, memory_region_name(&quirk
->mem
));
1358 vfio_pci_write_config(&vdev
->pdev
, addr
- offset
, data
, size
);
1360 trace_vfio_generic_quirk_write(memory_region_name(&quirk
->mem
),
1361 vdev
->vbasedev
.name
, quirk
->data
.bar
,
1362 addr
+ base
, data
, size
);
1364 vfio_region_write(&vdev
->bars
[quirk
->data
.bar
].region
,
1365 addr
+ base
, data
, size
);
1369 static const MemoryRegionOps vfio_generic_quirk
= {
1370 .read
= vfio_generic_quirk_read
,
1371 .write
= vfio_generic_quirk_write
,
1372 .endianness
= DEVICE_LITTLE_ENDIAN
,
1375 #define PCI_VENDOR_ID_ATI 0x1002
1378 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
1379 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always
1380 * BAR4 (older cards like the X550 used BAR1, but we don't care to support
1381 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the
1382 * I/O port BAR address. Originally this was coded to return the virtual BAR
1383 * address only if the physical register read returns the actual BAR address,
1384 * but users have reported greater success if we return the virtual address
1387 static uint64_t vfio_ati_3c3_quirk_read(void *opaque
,
1388 hwaddr addr
, unsigned size
)
1390 VFIOQuirk
*quirk
= opaque
;
1391 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1392 uint64_t data
= vfio_pci_read_config(&vdev
->pdev
,
1393 PCI_BASE_ADDRESS_0
+ (4 * 4) + 1,
1395 trace_vfio_ati_3c3_quirk_read(data
);
1400 static const MemoryRegionOps vfio_ati_3c3_quirk
= {
1401 .read
= vfio_ati_3c3_quirk_read
,
1402 .endianness
= DEVICE_LITTLE_ENDIAN
,
1405 static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice
*vdev
)
1407 PCIDevice
*pdev
= &vdev
->pdev
;
1410 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1415 * As long as the BAR is >= 256 bytes it will be aligned such that the
1416 * lower byte is always zero. Filter out anything else, if it exists.
1418 if (!vdev
->bars
[4].ioport
|| vdev
->bars
[4].region
.size
< 256) {
1422 quirk
= g_malloc0(sizeof(*quirk
));
1425 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_ati_3c3_quirk
, quirk
,
1426 "vfio-ati-3c3-quirk", 1);
1427 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
1428 3 /* offset 3 bytes from 0x3c0 */, &quirk
->mem
);
1430 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
1433 trace_vfio_vga_probe_ati_3c3_quirk(vdev
->vbasedev
.name
);
1437 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
1438 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access
1439 * the MMIO space directly, but a window to this space is provided through
1440 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the
1441 * data register. When the address is programmed to a range of 0x4000-0x4fff
1442 * PCI configuration space is available. Experimentation seems to indicate
1443 * that only read-only access is provided, but we drop writes when the window
1444 * is enabled to config space nonetheless.
1446 static void vfio_probe_ati_bar4_window_quirk(VFIOPCIDevice
*vdev
, int nr
)
1448 PCIDevice
*pdev
= &vdev
->pdev
;
1451 if (!vdev
->has_vga
|| nr
!= 4 ||
1452 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1456 quirk
= g_malloc0(sizeof(*quirk
));
1458 quirk
->data
.address_size
= 4;
1459 quirk
->data
.data_offset
= 4;
1460 quirk
->data
.data_size
= 4;
1461 quirk
->data
.address_match
= 0x4000;
1462 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1463 quirk
->data
.bar
= nr
;
1464 quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1466 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
1467 &vfio_generic_window_quirk
, quirk
,
1468 "vfio-ati-bar4-window-quirk", 8);
1469 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1470 quirk
->data
.base_offset
, &quirk
->mem
, 1);
1472 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1474 trace_vfio_probe_ati_bar4_window_quirk(vdev
->vbasedev
.name
);
1477 #define PCI_VENDOR_ID_REALTEK 0x10ec
1480 * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2
1481 * offset 0x70 there is a dword data register, offset 0x74 is a dword address
1482 * register. According to the Linux r8169 driver, the MSI-X table is addressed
1483 * when the "type" portion of the address register is set to 0x1. This appears
1484 * to be bits 16:30. Bit 31 is both a write indicator and some sort of
1485 * "address latched" indicator. Bits 12:15 are a mask field, which we can
1486 * ignore because the MSI-X table should always be accessed as a dword (full
1487 * mask). Bits 0:11 is offset within the type.
1491 * Read from MSI-X table offset 0
1492 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
1493 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
1494 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
1496 * Write 0xfee00000 to MSI-X table offset 0
1497 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
1498 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
1499 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
1502 static uint64_t vfio_rtl8168_window_quirk_read(void *opaque
,
1503 hwaddr addr
, unsigned size
)
1505 VFIOQuirk
*quirk
= opaque
;
1506 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1509 case 4: /* address */
1510 if (quirk
->data
.flags
) {
1511 trace_vfio_rtl8168_window_quirk_read_fake(
1512 memory_region_name(&quirk
->mem
),
1513 vdev
->vbasedev
.name
);
1515 return quirk
->data
.address_match
^ 0x10000000U
;
1519 if (quirk
->data
.flags
) {
1522 trace_vfio_rtl8168_window_quirk_read_table(
1523 memory_region_name(&quirk
->mem
),
1524 vdev
->vbasedev
.name
);
1526 if (!(vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MSIX
)) {
1530 io_mem_read(&vdev
->pdev
.msix_table_mmio
,
1531 (hwaddr
)(quirk
->data
.address_match
& 0xfff),
1537 trace_vfio_rtl8168_window_quirk_read_direct(memory_region_name(&quirk
->mem
),
1538 vdev
->vbasedev
.name
);
1540 return vfio_region_read(&vdev
->bars
[quirk
->data
.bar
].region
,
1544 static void vfio_rtl8168_window_quirk_write(void *opaque
, hwaddr addr
,
1545 uint64_t data
, unsigned size
)
1547 VFIOQuirk
*quirk
= opaque
;
1548 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1551 case 4: /* address */
1552 if ((data
& 0x7fff0000) == 0x10000) {
1553 if (data
& 0x10000000U
&&
1554 vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MSIX
) {
1556 trace_vfio_rtl8168_window_quirk_write_table(
1557 memory_region_name(&quirk
->mem
),
1558 vdev
->vbasedev
.name
);
1560 io_mem_write(&vdev
->pdev
.msix_table_mmio
,
1561 (hwaddr
)(quirk
->data
.address_match
& 0xfff),
1565 quirk
->data
.flags
= 1;
1566 quirk
->data
.address_match
= data
;
1570 quirk
->data
.flags
= 0;
1573 quirk
->data
.address_mask
= data
;
1577 trace_vfio_rtl8168_window_quirk_write_direct(
1578 memory_region_name(&quirk
->mem
),
1579 vdev
->vbasedev
.name
);
1581 vfio_region_write(&vdev
->bars
[quirk
->data
.bar
].region
,
1582 addr
+ 0x70, data
, size
);
1585 static const MemoryRegionOps vfio_rtl8168_window_quirk
= {
1586 .read
= vfio_rtl8168_window_quirk_read
,
1587 .write
= vfio_rtl8168_window_quirk_write
,
1589 .min_access_size
= 4,
1590 .max_access_size
= 4,
1593 .endianness
= DEVICE_LITTLE_ENDIAN
,
1596 static void vfio_probe_rtl8168_bar2_window_quirk(VFIOPCIDevice
*vdev
, int nr
)
1598 PCIDevice
*pdev
= &vdev
->pdev
;
1601 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_REALTEK
||
1602 pci_get_word(pdev
->config
+ PCI_DEVICE_ID
) != 0x8168 || nr
!= 2) {
1606 quirk
= g_malloc0(sizeof(*quirk
));
1608 quirk
->data
.bar
= nr
;
1610 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_rtl8168_window_quirk
,
1611 quirk
, "vfio-rtl8168-window-quirk", 8);
1612 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1613 0x70, &quirk
->mem
, 1);
1615 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1617 trace_vfio_probe_rtl8168_bar2_window_quirk(vdev
->vbasedev
.name
);
1620 * Trap the BAR2 MMIO window to config space as well.
1622 static void vfio_probe_ati_bar2_4000_quirk(VFIOPCIDevice
*vdev
, int nr
)
1624 PCIDevice
*pdev
= &vdev
->pdev
;
1627 /* Only enable on newer devices where BAR2 is 64bit */
1628 if (!vdev
->has_vga
|| nr
!= 2 || !vdev
->bars
[2].mem64
||
1629 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1633 quirk
= g_malloc0(sizeof(*quirk
));
1635 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1636 quirk
->data
.address_match
= 0x4000;
1637 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1638 quirk
->data
.bar
= nr
;
1640 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
1641 "vfio-ati-bar2-4000-quirk",
1642 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1643 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1644 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1647 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1649 trace_vfio_probe_ati_bar2_4000_quirk(vdev
->vbasedev
.name
);
1653 * Older ATI/AMD cards like the X550 have a similar window to that above.
1654 * I/O port BAR1 provides a window to a mirror of PCI config space located
1655 * in BAR2 at offset 0xf00. We don't care to support such older cards, but
1656 * note it for future reference.
1659 #define PCI_VENDOR_ID_NVIDIA 0x10de
1662 * Nvidia has several different methods to get to config space, the
1663 * nouveu project has several of these documented here:
1664 * https://github.com/pathscale/envytools/tree/master/hwdocs
1666 * The first quirk is actually not documented in envytools and is found
1667 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an
1668 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access
1669 * the mirror of PCI config space found at BAR0 offset 0x1800. The access
1670 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is
1671 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738
1672 * is written for a write to 0x3d4. The BAR0 offset is then accessible
1673 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards
1674 * that use the I/O port BAR5 window but it doesn't hurt to leave it.
1684 static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque
,
1685 hwaddr addr
, unsigned size
)
1687 VFIOQuirk
*quirk
= opaque
;
1688 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1689 PCIDevice
*pdev
= &vdev
->pdev
;
1690 uint64_t data
= vfio_vga_read(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1691 addr
+ quirk
->data
.base_offset
, size
);
1693 if (quirk
->data
.flags
== NV_3D0_READ
&& addr
== quirk
->data
.data_offset
) {
1694 data
= vfio_pci_read_config(pdev
, quirk
->data
.address_val
, size
);
1695 trace_vfio_nvidia_3d0_quirk_read(size
, data
);
1698 quirk
->data
.flags
= NV_3D0_NONE
;
1703 static void vfio_nvidia_3d0_quirk_write(void *opaque
, hwaddr addr
,
1704 uint64_t data
, unsigned size
)
1706 VFIOQuirk
*quirk
= opaque
;
1707 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1708 PCIDevice
*pdev
= &vdev
->pdev
;
1710 switch (quirk
->data
.flags
) {
1712 if (addr
== quirk
->data
.address_offset
&& data
== 0x338) {
1713 quirk
->data
.flags
= NV_3D0_SELECT
;
1717 quirk
->data
.flags
= NV_3D0_NONE
;
1718 if (addr
== quirk
->data
.data_offset
&&
1719 (data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1720 quirk
->data
.flags
= NV_3D0_WINDOW
;
1721 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1725 quirk
->data
.flags
= NV_3D0_NONE
;
1726 if (addr
== quirk
->data
.address_offset
) {
1727 if (data
== 0x538) {
1728 quirk
->data
.flags
= NV_3D0_READ
;
1729 } else if (data
== 0x738) {
1730 quirk
->data
.flags
= NV_3D0_WRITE
;
1735 quirk
->data
.flags
= NV_3D0_NONE
;
1736 if (addr
== quirk
->data
.data_offset
) {
1737 vfio_pci_write_config(pdev
, quirk
->data
.address_val
, data
, size
);
1738 trace_vfio_nvidia_3d0_quirk_write(data
, size
);
1744 vfio_vga_write(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1745 addr
+ quirk
->data
.base_offset
, data
, size
);
1748 static const MemoryRegionOps vfio_nvidia_3d0_quirk
= {
1749 .read
= vfio_nvidia_3d0_quirk_read
,
1750 .write
= vfio_nvidia_3d0_quirk_write
,
1751 .endianness
= DEVICE_LITTLE_ENDIAN
,
1754 static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice
*vdev
)
1756 PCIDevice
*pdev
= &vdev
->pdev
;
1759 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
||
1760 !vdev
->bars
[1].region
.size
) {
1764 quirk
= g_malloc0(sizeof(*quirk
));
1766 quirk
->data
.base_offset
= 0x10;
1767 quirk
->data
.address_offset
= 4;
1768 quirk
->data
.address_size
= 2;
1769 quirk
->data
.address_match
= 0x1800;
1770 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
1771 quirk
->data
.data_offset
= 0;
1772 quirk
->data
.data_size
= 4;
1774 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_3d0_quirk
,
1775 quirk
, "vfio-nvidia-3d0-quirk", 6);
1776 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
1777 quirk
->data
.base_offset
, &quirk
->mem
);
1779 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
1782 trace_vfio_vga_probe_nvidia_3d0_quirk(vdev
->vbasedev
.name
);
1786 * The second quirk is documented in envytools. The I/O port BAR5 is just
1787 * a set of address/data ports to the MMIO BARs. The BAR we care about is
1788 * again BAR0. This backdoor is apparently a bit newer than the one above
1789 * so we need to not only trap 256 bytes @0x1800, but all of PCI config
1790 * space, including extended space is available at the 4k @0x88000.
1793 NV_BAR5_ADDRESS
= 0x1,
1794 NV_BAR5_ENABLE
= 0x2,
1795 NV_BAR5_MASTER
= 0x4,
1796 NV_BAR5_VALID
= 0x7,
1799 static void vfio_nvidia_bar5_window_quirk_write(void *opaque
, hwaddr addr
,
1800 uint64_t data
, unsigned size
)
1802 VFIOQuirk
*quirk
= opaque
;
1807 quirk
->data
.flags
|= NV_BAR5_MASTER
;
1809 quirk
->data
.flags
&= ~NV_BAR5_MASTER
;
1814 quirk
->data
.flags
|= NV_BAR5_ENABLE
;
1816 quirk
->data
.flags
&= ~NV_BAR5_ENABLE
;
1820 if (quirk
->data
.flags
& NV_BAR5_MASTER
) {
1821 if ((data
& ~0xfff) == 0x88000) {
1822 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
1823 quirk
->data
.address_val
= data
& 0xfff;
1824 } else if ((data
& ~0xff) == 0x1800) {
1825 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
1826 quirk
->data
.address_val
= data
& 0xff;
1828 quirk
->data
.flags
&= ~NV_BAR5_ADDRESS
;
1834 vfio_generic_window_quirk_write(opaque
, addr
, data
, size
);
1837 static const MemoryRegionOps vfio_nvidia_bar5_window_quirk
= {
1838 .read
= vfio_generic_window_quirk_read
,
1839 .write
= vfio_nvidia_bar5_window_quirk_write
,
1840 .valid
.min_access_size
= 4,
1841 .endianness
= DEVICE_LITTLE_ENDIAN
,
1844 static void vfio_probe_nvidia_bar5_window_quirk(VFIOPCIDevice
*vdev
, int nr
)
1846 PCIDevice
*pdev
= &vdev
->pdev
;
1849 if (!vdev
->has_vga
|| nr
!= 5 ||
1850 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
1854 quirk
= g_malloc0(sizeof(*quirk
));
1856 quirk
->data
.read_flags
= quirk
->data
.write_flags
= NV_BAR5_VALID
;
1857 quirk
->data
.address_offset
= 0x8;
1858 quirk
->data
.address_size
= 0; /* actually 4, but avoids generic code */
1859 quirk
->data
.data_offset
= 0xc;
1860 quirk
->data
.data_size
= 4;
1861 quirk
->data
.bar
= nr
;
1863 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
1864 &vfio_nvidia_bar5_window_quirk
, quirk
,
1865 "vfio-nvidia-bar5-window-quirk", 16);
1866 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1869 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1871 trace_vfio_probe_nvidia_bar5_window_quirk(vdev
->vbasedev
.name
);
1874 static void vfio_nvidia_88000_quirk_write(void *opaque
, hwaddr addr
,
1875 uint64_t data
, unsigned size
)
1877 VFIOQuirk
*quirk
= opaque
;
1878 VFIOPCIDevice
*vdev
= quirk
->vdev
;
1879 PCIDevice
*pdev
= &vdev
->pdev
;
1880 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1882 vfio_generic_quirk_write(opaque
, addr
, data
, size
);
1885 * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
1886 * MSI capability ID register. Both the ID and next register are
1887 * read-only, so we allow writes covering either of those to real hw.
1888 * NB - only fixed for the 0x88000 MMIO window.
1890 if ((pdev
->cap_present
& QEMU_PCI_CAP_MSI
) &&
1891 vfio_range_contained(addr
, size
, pdev
->msi_cap
, PCI_MSI_FLAGS
)) {
1892 vfio_region_write(&vdev
->bars
[quirk
->data
.bar
].region
,
1893 addr
+ base
, data
, size
);
1897 static const MemoryRegionOps vfio_nvidia_88000_quirk
= {
1898 .read
= vfio_generic_quirk_read
,
1899 .write
= vfio_nvidia_88000_quirk_write
,
1900 .endianness
= DEVICE_LITTLE_ENDIAN
,
1904 * Finally, BAR0 itself. We want to redirect any accesses to either
1905 * 0x1800 or 0x88000 through the PCI config space access functions.
1907 * NB - quirk at a page granularity or else they don't seem to work when
1910 * Here's offset 0x88000...
1912 static void vfio_probe_nvidia_bar0_88000_quirk(VFIOPCIDevice
*vdev
, int nr
)
1914 PCIDevice
*pdev
= &vdev
->pdev
;
1916 uint16_t vendor
, class;
1918 vendor
= pci_get_word(pdev
->config
+ PCI_VENDOR_ID
);
1919 class = pci_get_word(pdev
->config
+ PCI_CLASS_DEVICE
);
1921 if (nr
!= 0 || vendor
!= PCI_VENDOR_ID_NVIDIA
||
1922 class != PCI_CLASS_DISPLAY_VGA
) {
1926 quirk
= g_malloc0(sizeof(*quirk
));
1928 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1929 quirk
->data
.address_match
= 0x88000;
1930 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1931 quirk
->data
.bar
= nr
;
1933 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_88000_quirk
,
1934 quirk
, "vfio-nvidia-bar0-88000-quirk",
1935 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1936 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1937 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1940 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1942 trace_vfio_probe_nvidia_bar0_88000_quirk(vdev
->vbasedev
.name
);
1946 * And here's the same for BAR0 offset 0x1800...
1948 static void vfio_probe_nvidia_bar0_1800_quirk(VFIOPCIDevice
*vdev
, int nr
)
1950 PCIDevice
*pdev
= &vdev
->pdev
;
1953 if (!vdev
->has_vga
|| nr
!= 0 ||
1954 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
1958 /* Log the chipset ID */
1959 trace_vfio_probe_nvidia_bar0_1800_quirk_id(
1960 (unsigned int)(vfio_region_read(&vdev
->bars
[0].region
, 0, 4) >> 20)
1963 quirk
= g_malloc0(sizeof(*quirk
));
1965 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1966 quirk
->data
.address_match
= 0x1800;
1967 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
1968 quirk
->data
.bar
= nr
;
1970 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
1971 "vfio-nvidia-bar0-1800-quirk",
1972 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1973 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].region
.mem
,
1974 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1977 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1979 trace_vfio_probe_nvidia_bar0_1800_quirk(vdev
->vbasedev
.name
);
1983 * TODO - Some Nvidia devices provide config access to their companion HDA
1984 * device and even to their parent bridge via these config space mirrors.
1985 * Add quirks for those regions.
1989 * Common quirk probe entry points.
1991 static void vfio_vga_quirk_setup(VFIOPCIDevice
*vdev
)
1993 vfio_vga_probe_ati_3c3_quirk(vdev
);
1994 vfio_vga_probe_nvidia_3d0_quirk(vdev
);
1997 static void vfio_vga_quirk_teardown(VFIOPCIDevice
*vdev
)
2002 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
.region
); i
++) {
2003 QLIST_FOREACH(quirk
, &vdev
->vga
.region
[i
].quirks
, next
) {
2004 memory_region_del_subregion(&vdev
->vga
.region
[i
].mem
, &quirk
->mem
);
2009 static void vfio_vga_quirk_free(VFIOPCIDevice
*vdev
)
2013 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
.region
); i
++) {
2014 while (!QLIST_EMPTY(&vdev
->vga
.region
[i
].quirks
)) {
2015 VFIOQuirk
*quirk
= QLIST_FIRST(&vdev
->vga
.region
[i
].quirks
);
2016 object_unparent(OBJECT(&quirk
->mem
));
2017 QLIST_REMOVE(quirk
, next
);
2023 static void vfio_bar_quirk_setup(VFIOPCIDevice
*vdev
, int nr
)
2025 vfio_probe_ati_bar4_window_quirk(vdev
, nr
);
2026 vfio_probe_ati_bar2_4000_quirk(vdev
, nr
);
2027 vfio_probe_nvidia_bar5_window_quirk(vdev
, nr
);
2028 vfio_probe_nvidia_bar0_88000_quirk(vdev
, nr
);
2029 vfio_probe_nvidia_bar0_1800_quirk(vdev
, nr
);
2030 vfio_probe_rtl8168_bar2_window_quirk(vdev
, nr
);
2033 static void vfio_bar_quirk_teardown(VFIOPCIDevice
*vdev
, int nr
)
2035 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2038 QLIST_FOREACH(quirk
, &bar
->quirks
, next
) {
2039 memory_region_del_subregion(&bar
->region
.mem
, &quirk
->mem
);
2043 static void vfio_bar_quirk_free(VFIOPCIDevice
*vdev
, int nr
)
2045 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2047 while (!QLIST_EMPTY(&bar
->quirks
)) {
2048 VFIOQuirk
*quirk
= QLIST_FIRST(&bar
->quirks
);
2049 object_unparent(OBJECT(&quirk
->mem
));
2050 QLIST_REMOVE(quirk
, next
);
2058 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
2060 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
2061 uint32_t emu_bits
= 0, emu_val
= 0, phys_val
= 0, val
;
2063 memcpy(&emu_bits
, vdev
->emulated_config_bits
+ addr
, len
);
2064 emu_bits
= le32_to_cpu(emu_bits
);
2067 emu_val
= pci_default_read_config(pdev
, addr
, len
);
2070 if (~emu_bits
& (0xffffffffU
>> (32 - len
* 8))) {
2073 ret
= pread(vdev
->vbasedev
.fd
, &phys_val
, len
,
2074 vdev
->config_offset
+ addr
);
2076 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
2077 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2078 vdev
->host
.slot
, vdev
->host
.function
, addr
, len
);
2081 phys_val
= le32_to_cpu(phys_val
);
2084 val
= (emu_val
& emu_bits
) | (phys_val
& ~emu_bits
);
2086 trace_vfio_pci_read_config(vdev
->vbasedev
.name
, addr
, len
, val
);
2091 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
2092 uint32_t val
, int len
)
2094 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
2095 uint32_t val_le
= cpu_to_le32(val
);
2097 trace_vfio_pci_write_config(vdev
->vbasedev
.name
, addr
, val
, len
);
2099 /* Write everything to VFIO, let it filter out what we can't write */
2100 if (pwrite(vdev
->vbasedev
.fd
, &val_le
, len
, vdev
->config_offset
+ addr
)
2102 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
2103 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2104 vdev
->host
.slot
, vdev
->host
.function
, addr
, val
, len
);
2107 /* MSI/MSI-X Enabling/Disabling */
2108 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
2109 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
2110 int is_enabled
, was_enabled
= msi_enabled(pdev
);
2112 pci_default_write_config(pdev
, addr
, val
, len
);
2114 is_enabled
= msi_enabled(pdev
);
2118 vfio_enable_msi(vdev
);
2122 vfio_disable_msi(vdev
);
2124 vfio_update_msi(vdev
);
2127 } else if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
2128 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
2129 int is_enabled
, was_enabled
= msix_enabled(pdev
);
2131 pci_default_write_config(pdev
, addr
, val
, len
);
2133 is_enabled
= msix_enabled(pdev
);
2135 if (!was_enabled
&& is_enabled
) {
2136 vfio_enable_msix(vdev
);
2137 } else if (was_enabled
&& !is_enabled
) {
2138 vfio_disable_msix(vdev
);
2141 /* Write everything to QEMU to keep emulated bits correct */
2142 pci_default_write_config(pdev
, addr
, val
, len
);
2149 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
)
2152 * More complicated than it looks. Disabling MSI/X transitions the
2153 * device to INTx mode (if supported). Therefore we need to first
2154 * disable MSI/X and then cleanup by disabling INTx.
2156 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
2157 vfio_disable_msix(vdev
);
2158 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
2159 vfio_disable_msi(vdev
);
2162 if (vdev
->interrupt
== VFIO_INT_INTx
) {
2163 vfio_disable_intx(vdev
);
2167 static int vfio_setup_msi(VFIOPCIDevice
*vdev
, int pos
)
2170 bool msi_64bit
, msi_maskbit
;
2173 if (pread(vdev
->vbasedev
.fd
, &ctrl
, sizeof(ctrl
),
2174 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2177 ctrl
= le16_to_cpu(ctrl
);
2179 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
2180 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
2181 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
2183 trace_vfio_setup_msi(vdev
->vbasedev
.name
, pos
);
2185 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
);
2187 if (ret
== -ENOTSUP
) {
2190 error_report("vfio: msi_init failed");
2193 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
2199 * We don't have any control over how pci_add_capability() inserts
2200 * capabilities into the chain. In order to setup MSI-X we need a
2201 * MemoryRegion for the BAR. In order to setup the BAR and not
2202 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
2203 * need to first look for where the MSI-X table lives. So we
2204 * unfortunately split MSI-X setup across two functions.
2206 static int vfio_early_setup_msix(VFIOPCIDevice
*vdev
)
2210 uint32_t table
, pba
;
2211 int fd
= vdev
->vbasedev
.fd
;
2213 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
2218 if (pread(fd
, &ctrl
, sizeof(ctrl
),
2219 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2223 if (pread(fd
, &table
, sizeof(table
),
2224 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
2228 if (pread(fd
, &pba
, sizeof(pba
),
2229 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
2233 ctrl
= le16_to_cpu(ctrl
);
2234 table
= le32_to_cpu(table
);
2235 pba
= le32_to_cpu(pba
);
2237 vdev
->msix
= g_malloc0(sizeof(*(vdev
->msix
)));
2238 vdev
->msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
2239 vdev
->msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
2240 vdev
->msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
2241 vdev
->msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
2242 vdev
->msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
2244 trace_vfio_early_setup_msix(vdev
->vbasedev
.name
, pos
,
2245 vdev
->msix
->table_bar
,
2246 vdev
->msix
->table_offset
,
2247 vdev
->msix
->entries
);
2252 static int vfio_setup_msix(VFIOPCIDevice
*vdev
, int pos
)
2256 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
2257 &vdev
->bars
[vdev
->msix
->table_bar
].region
.mem
,
2258 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
2259 &vdev
->bars
[vdev
->msix
->pba_bar
].region
.mem
,
2260 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
);
2262 if (ret
== -ENOTSUP
) {
2265 error_report("vfio: msix_init failed");
2272 static void vfio_teardown_msi(VFIOPCIDevice
*vdev
)
2274 msi_uninit(&vdev
->pdev
);
2277 msix_uninit(&vdev
->pdev
,
2278 &vdev
->bars
[vdev
->msix
->table_bar
].region
.mem
,
2279 &vdev
->bars
[vdev
->msix
->pba_bar
].region
.mem
);
2286 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
)
2290 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2291 VFIOBAR
*bar
= &vdev
->bars
[i
];
2293 if (!bar
->region
.size
) {
2297 memory_region_set_enabled(&bar
->region
.mmap_mem
, enabled
);
2298 if (vdev
->msix
&& vdev
->msix
->table_bar
== i
) {
2299 memory_region_set_enabled(&vdev
->msix
->mmap_mem
, enabled
);
2304 static void vfio_unregister_bar(VFIOPCIDevice
*vdev
, int nr
)
2306 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2308 if (!bar
->region
.size
) {
2312 vfio_bar_quirk_teardown(vdev
, nr
);
2314 memory_region_del_subregion(&bar
->region
.mem
, &bar
->region
.mmap_mem
);
2316 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2317 memory_region_del_subregion(&bar
->region
.mem
, &vdev
->msix
->mmap_mem
);
2321 static void vfio_unmap_bar(VFIOPCIDevice
*vdev
, int nr
)
2323 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2325 if (!bar
->region
.size
) {
2329 vfio_bar_quirk_free(vdev
, nr
);
2331 munmap(bar
->region
.mmap
, memory_region_size(&bar
->region
.mmap_mem
));
2333 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2334 munmap(vdev
->msix
->mmap
, memory_region_size(&vdev
->msix
->mmap_mem
));
2338 static void vfio_map_bar(VFIOPCIDevice
*vdev
, int nr
)
2340 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2341 uint64_t size
= bar
->region
.size
;
2347 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
2352 snprintf(name
, sizeof(name
), "VFIO %04x:%02x:%02x.%x BAR %d",
2353 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2354 vdev
->host
.function
, nr
);
2356 /* Determine what type of BAR this is for registration */
2357 ret
= pread(vdev
->vbasedev
.fd
, &pci_bar
, sizeof(pci_bar
),
2358 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
2359 if (ret
!= sizeof(pci_bar
)) {
2360 error_report("vfio: Failed to read BAR %d (%m)", nr
);
2364 pci_bar
= le32_to_cpu(pci_bar
);
2365 bar
->ioport
= (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
);
2366 bar
->mem64
= bar
->ioport
? 0 : (pci_bar
& PCI_BASE_ADDRESS_MEM_TYPE_64
);
2367 type
= pci_bar
& (bar
->ioport
? ~PCI_BASE_ADDRESS_IO_MASK
:
2368 ~PCI_BASE_ADDRESS_MEM_MASK
);
2370 /* A "slow" read/write mapping underlies all BARs */
2371 memory_region_init_io(&bar
->region
.mem
, OBJECT(vdev
), &vfio_region_ops
,
2373 pci_register_bar(&vdev
->pdev
, nr
, type
, &bar
->region
.mem
);
2376 * We can't mmap areas overlapping the MSIX vector table, so we
2377 * potentially insert a direct-mapped subregion before and after it.
2379 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2380 size
= vdev
->msix
->table_offset
& qemu_host_page_mask
;
2383 strncat(name
, " mmap", sizeof(name
) - strlen(name
) - 1);
2384 if (vfio_mmap_region(OBJECT(vdev
), &bar
->region
, &bar
->region
.mem
,
2385 &bar
->region
.mmap_mem
, &bar
->region
.mmap
,
2387 error_report("%s unsupported. Performance may be slow", name
);
2390 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2393 start
= HOST_PAGE_ALIGN(vdev
->msix
->table_offset
+
2394 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
2396 size
= start
< bar
->region
.size
? bar
->region
.size
- start
: 0;
2397 strncat(name
, " msix-hi", sizeof(name
) - strlen(name
) - 1);
2398 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
2399 if (vfio_mmap_region(OBJECT(vdev
), &bar
->region
, &bar
->region
.mem
,
2400 &vdev
->msix
->mmap_mem
,
2401 &vdev
->msix
->mmap
, size
, start
, name
)) {
2402 error_report("%s unsupported. Performance may be slow", name
);
2406 vfio_bar_quirk_setup(vdev
, nr
);
2409 static void vfio_map_bars(VFIOPCIDevice
*vdev
)
2413 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2414 vfio_map_bar(vdev
, i
);
2417 if (vdev
->has_vga
) {
2418 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2419 OBJECT(vdev
), &vfio_vga_ops
,
2420 &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
],
2421 "vfio-vga-mmio@0xa0000",
2422 QEMU_PCI_VGA_MEM_SIZE
);
2423 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2424 OBJECT(vdev
), &vfio_vga_ops
,
2425 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
],
2426 "vfio-vga-io@0x3b0",
2427 QEMU_PCI_VGA_IO_LO_SIZE
);
2428 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
2429 OBJECT(vdev
), &vfio_vga_ops
,
2430 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
2431 "vfio-vga-io@0x3c0",
2432 QEMU_PCI_VGA_IO_HI_SIZE
);
2434 pci_register_vga(&vdev
->pdev
, &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2435 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2436 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
);
2437 vfio_vga_quirk_setup(vdev
);
2441 static void vfio_unregister_bars(VFIOPCIDevice
*vdev
)
2445 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2446 vfio_unregister_bar(vdev
, i
);
2449 if (vdev
->has_vga
) {
2450 vfio_vga_quirk_teardown(vdev
);
2451 pci_unregister_vga(&vdev
->pdev
);
2455 static void vfio_unmap_bars(VFIOPCIDevice
*vdev
)
2459 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2460 vfio_unmap_bar(vdev
, i
);
2463 if (vdev
->has_vga
) {
2464 vfio_vga_quirk_free(vdev
);
2471 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
2473 uint8_t tmp
, next
= 0xff;
2475 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
2476 tmp
= pdev
->config
[tmp
+ 1]) {
2477 if (tmp
> pos
&& tmp
< next
) {
2485 static void vfio_set_word_bits(uint8_t *buf
, uint16_t val
, uint16_t mask
)
2487 pci_set_word(buf
, (pci_get_word(buf
) & ~mask
) | val
);
2490 static void vfio_add_emulated_word(VFIOPCIDevice
*vdev
, int pos
,
2491 uint16_t val
, uint16_t mask
)
2493 vfio_set_word_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
2494 vfio_set_word_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
2495 vfio_set_word_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
2498 static void vfio_set_long_bits(uint8_t *buf
, uint32_t val
, uint32_t mask
)
2500 pci_set_long(buf
, (pci_get_long(buf
) & ~mask
) | val
);
2503 static void vfio_add_emulated_long(VFIOPCIDevice
*vdev
, int pos
,
2504 uint32_t val
, uint32_t mask
)
2506 vfio_set_long_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
2507 vfio_set_long_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
2508 vfio_set_long_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
2511 static int vfio_setup_pcie_cap(VFIOPCIDevice
*vdev
, int pos
, uint8_t size
)
2516 flags
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_CAP_FLAGS
);
2517 type
= (flags
& PCI_EXP_FLAGS_TYPE
) >> 4;
2519 if (type
!= PCI_EXP_TYPE_ENDPOINT
&&
2520 type
!= PCI_EXP_TYPE_LEG_END
&&
2521 type
!= PCI_EXP_TYPE_RC_END
) {
2523 error_report("vfio: Assignment of PCIe type 0x%x "
2524 "devices is not currently supported", type
);
2528 if (!pci_bus_is_express(vdev
->pdev
.bus
)) {
2530 * Use express capability as-is on PCI bus. It doesn't make much
2531 * sense to even expose, but some drivers (ex. tg3) depend on it
2532 * and guests don't seem to be particular about it. We'll need
2533 * to revist this or force express devices to express buses if we
2534 * ever expose an IOMMU to the guest.
2536 } else if (pci_bus_is_root(vdev
->pdev
.bus
)) {
2538 * On a Root Complex bus Endpoints become Root Complex Integrated
2539 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2541 if (type
== PCI_EXP_TYPE_ENDPOINT
) {
2542 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2543 PCI_EXP_TYPE_RC_END
<< 4,
2544 PCI_EXP_FLAGS_TYPE
);
2546 /* Link Capabilities, Status, and Control goes away */
2547 if (size
> PCI_EXP_LNKCTL
) {
2548 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
, 0, ~0);
2549 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2550 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
, 0, ~0);
2552 #ifndef PCI_EXP_LNKCAP2
2553 #define PCI_EXP_LNKCAP2 44
2555 #ifndef PCI_EXP_LNKSTA2
2556 #define PCI_EXP_LNKSTA2 50
2558 /* Link 2 Capabilities, Status, and Control goes away */
2559 if (size
> PCI_EXP_LNKCAP2
) {
2560 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP2
, 0, ~0);
2561 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL2
, 0, ~0);
2562 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA2
, 0, ~0);
2566 } else if (type
== PCI_EXP_TYPE_LEG_END
) {
2568 * Legacy endpoints don't belong on the root complex. Windows
2569 * seems to be happier with devices if we skip the capability.
2576 * Convert Root Complex Integrated Endpoints to regular endpoints.
2577 * These devices don't support LNK/LNK2 capabilities, so make them up.
2579 if (type
== PCI_EXP_TYPE_RC_END
) {
2580 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2581 PCI_EXP_TYPE_ENDPOINT
<< 4,
2582 PCI_EXP_FLAGS_TYPE
);
2583 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
,
2584 PCI_EXP_LNK_MLW_1
| PCI_EXP_LNK_LS_25
, ~0);
2585 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2588 /* Mark the Link Status bits as emulated to allow virtual negotiation */
2589 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
,
2590 pci_get_word(vdev
->pdev
.config
+ pos
+
2592 PCI_EXP_LNKCAP_MLW
| PCI_EXP_LNKCAP_SLS
);
2595 pos
= pci_add_capability(&vdev
->pdev
, PCI_CAP_ID_EXP
, pos
, size
);
2597 vdev
->pdev
.exp
.exp_cap
= pos
;
2603 static void vfio_check_pcie_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
2605 uint32_t cap
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_EXP_DEVCAP
);
2607 if (cap
& PCI_EXP_DEVCAP_FLR
) {
2608 trace_vfio_check_pcie_flr(vdev
->vbasedev
.name
);
2609 vdev
->has_flr
= true;
2613 static void vfio_check_pm_reset(VFIOPCIDevice
*vdev
, uint8_t pos
)
2615 uint16_t csr
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_PM_CTRL
);
2617 if (!(csr
& PCI_PM_CTRL_NO_SOFT_RESET
)) {
2618 trace_vfio_check_pm_reset(vdev
->vbasedev
.name
);
2619 vdev
->has_pm_reset
= true;
2623 static void vfio_check_af_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
2625 uint8_t cap
= pci_get_byte(vdev
->pdev
.config
+ pos
+ PCI_AF_CAP
);
2627 if ((cap
& PCI_AF_CAP_TP
) && (cap
& PCI_AF_CAP_FLR
)) {
2628 trace_vfio_check_af_flr(vdev
->vbasedev
.name
);
2629 vdev
->has_flr
= true;
2633 static int vfio_add_std_cap(VFIOPCIDevice
*vdev
, uint8_t pos
)
2635 PCIDevice
*pdev
= &vdev
->pdev
;
2636 uint8_t cap_id
, next
, size
;
2639 cap_id
= pdev
->config
[pos
];
2640 next
= pdev
->config
[pos
+ 1];
2643 * If it becomes important to configure capabilities to their actual
2644 * size, use this as the default when it's something we don't recognize.
2645 * Since QEMU doesn't actually handle many of the config accesses,
2646 * exact size doesn't seem worthwhile.
2648 size
= vfio_std_cap_max_size(pdev
, pos
);
2651 * pci_add_capability always inserts the new capability at the head
2652 * of the chain. Therefore to end up with a chain that matches the
2653 * physical device, we insert from the end by making this recursive.
2654 * This is also why we pre-caclulate size above as cached config space
2655 * will be changed as we unwind the stack.
2658 ret
= vfio_add_std_cap(vdev
, next
);
2663 /* Begin the rebuild, use QEMU emulated list bits */
2664 pdev
->config
[PCI_CAPABILITY_LIST
] = 0;
2665 vdev
->emulated_config_bits
[PCI_CAPABILITY_LIST
] = 0xff;
2666 vdev
->emulated_config_bits
[PCI_STATUS
] |= PCI_STATUS_CAP_LIST
;
2669 /* Use emulated next pointer to allow dropping caps */
2670 pci_set_byte(vdev
->emulated_config_bits
+ pos
+ 1, 0xff);
2673 case PCI_CAP_ID_MSI
:
2674 ret
= vfio_setup_msi(vdev
, pos
);
2676 case PCI_CAP_ID_EXP
:
2677 vfio_check_pcie_flr(vdev
, pos
);
2678 ret
= vfio_setup_pcie_cap(vdev
, pos
, size
);
2680 case PCI_CAP_ID_MSIX
:
2681 ret
= vfio_setup_msix(vdev
, pos
);
2684 vfio_check_pm_reset(vdev
, pos
);
2686 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2689 vfio_check_af_flr(vdev
, pos
);
2690 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2693 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
2698 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
2699 "0x%x[0x%x]@0x%x: %d", vdev
->host
.domain
,
2700 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
2701 cap_id
, size
, pos
, ret
);
2708 static int vfio_add_capabilities(VFIOPCIDevice
*vdev
)
2710 PCIDevice
*pdev
= &vdev
->pdev
;
2712 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
2713 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
2714 return 0; /* Nothing to add */
2717 return vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
]);
2720 static void vfio_pci_pre_reset(VFIOPCIDevice
*vdev
)
2722 PCIDevice
*pdev
= &vdev
->pdev
;
2725 vfio_disable_interrupts(vdev
);
2727 /* Make sure the device is in D0 */
2732 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2733 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2735 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
2736 vfio_pci_write_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, pmcsr
, 2);
2737 /* vfio handles the necessary delay here */
2738 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2739 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2741 error_report("vfio: Unable to power on device, stuck in D%d",
2748 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2749 * Also put INTx Disable in known state.
2751 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
2752 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
2753 PCI_COMMAND_INTX_DISABLE
);
2754 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
2757 static void vfio_pci_post_reset(VFIOPCIDevice
*vdev
)
2759 vfio_enable_intx(vdev
);
2762 static bool vfio_pci_host_match(PCIHostDeviceAddress
*host1
,
2763 PCIHostDeviceAddress
*host2
)
2765 return (host1
->domain
== host2
->domain
&& host1
->bus
== host2
->bus
&&
2766 host1
->slot
== host2
->slot
&& host1
->function
== host2
->function
);
2769 static int vfio_pci_hot_reset(VFIOPCIDevice
*vdev
, bool single
)
2772 struct vfio_pci_hot_reset_info
*info
;
2773 struct vfio_pci_dependent_device
*devices
;
2774 struct vfio_pci_hot_reset
*reset
;
2779 trace_vfio_pci_hot_reset(vdev
->vbasedev
.name
, single
? "one" : "multi");
2781 vfio_pci_pre_reset(vdev
);
2782 vdev
->vbasedev
.needs_reset
= false;
2784 info
= g_malloc0(sizeof(*info
));
2785 info
->argsz
= sizeof(*info
);
2787 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2788 if (ret
&& errno
!= ENOSPC
) {
2790 if (!vdev
->has_pm_reset
) {
2791 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
2792 "no available reset mechanism.", vdev
->host
.domain
,
2793 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
2798 count
= info
->count
;
2799 info
= g_realloc(info
, sizeof(*info
) + (count
* sizeof(*devices
)));
2800 info
->argsz
= sizeof(*info
) + (count
* sizeof(*devices
));
2801 devices
= &info
->devices
[0];
2803 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2806 error_report("vfio: hot reset info failed: %m");
2810 trace_vfio_pci_hot_reset_has_dep_devices(vdev
->vbasedev
.name
);
2812 /* Verify that we have all the groups required */
2813 for (i
= 0; i
< info
->count
; i
++) {
2814 PCIHostDeviceAddress host
;
2816 VFIODevice
*vbasedev_iter
;
2818 host
.domain
= devices
[i
].segment
;
2819 host
.bus
= devices
[i
].bus
;
2820 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
2821 host
.function
= PCI_FUNC(devices
[i
].devfn
);
2823 trace_vfio_pci_hot_reset_dep_devices(host
.domain
,
2824 host
.bus
, host
.slot
, host
.function
, devices
[i
].group_id
);
2826 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
2830 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2831 if (group
->groupid
== devices
[i
].group_id
) {
2837 if (!vdev
->has_pm_reset
) {
2838 error_report("vfio: Cannot reset device %s, "
2839 "depends on group %d which is not owned.",
2840 vdev
->vbasedev
.name
, devices
[i
].group_id
);
2846 /* Prep dependent devices for reset and clear our marker. */
2847 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
2848 if (vbasedev_iter
->type
!= VFIO_DEVICE_TYPE_PCI
) {
2851 tmp
= container_of(vbasedev_iter
, VFIOPCIDevice
, vbasedev
);
2852 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
2857 vfio_pci_pre_reset(tmp
);
2858 tmp
->vbasedev
.needs_reset
= false;
2865 if (!single
&& !multi
) {
2870 /* Determine how many group fds need to be passed */
2872 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2873 for (i
= 0; i
< info
->count
; i
++) {
2874 if (group
->groupid
== devices
[i
].group_id
) {
2881 reset
= g_malloc0(sizeof(*reset
) + (count
* sizeof(*fds
)));
2882 reset
->argsz
= sizeof(*reset
) + (count
* sizeof(*fds
));
2883 fds
= &reset
->group_fds
[0];
2885 /* Fill in group fds */
2886 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2887 for (i
= 0; i
< info
->count
; i
++) {
2888 if (group
->groupid
== devices
[i
].group_id
) {
2889 fds
[reset
->count
++] = group
->fd
;
2896 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_PCI_HOT_RESET
, reset
);
2899 trace_vfio_pci_hot_reset_result(vdev
->vbasedev
.name
,
2900 ret
? "%m" : "Success");
2903 /* Re-enable INTx on affected devices */
2904 for (i
= 0; i
< info
->count
; i
++) {
2905 PCIHostDeviceAddress host
;
2907 VFIODevice
*vbasedev_iter
;
2909 host
.domain
= devices
[i
].segment
;
2910 host
.bus
= devices
[i
].bus
;
2911 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
2912 host
.function
= PCI_FUNC(devices
[i
].devfn
);
2914 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
2918 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2919 if (group
->groupid
== devices
[i
].group_id
) {
2928 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
2929 if (vbasedev_iter
->type
!= VFIO_DEVICE_TYPE_PCI
) {
2932 tmp
= container_of(vbasedev_iter
, VFIOPCIDevice
, vbasedev
);
2933 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
2934 vfio_pci_post_reset(tmp
);
2940 vfio_pci_post_reset(vdev
);
2947 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2948 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2949 * of doing hot resets when there is only a single device per bus. The in-use
2950 * here refers to how many VFIODevices are affected. A hot reset that affects
2951 * multiple devices, but only a single in-use device, means that we can call
2952 * it from our bus ->reset() callback since the extent is effectively a single
2953 * device. This allows us to make use of it in the hotplug path. When there
2954 * are multiple in-use devices, we can only trigger the hot reset during a
2955 * system reset and thus from our reset handler. We separate _one vs _multi
2956 * here so that we don't overlap and do a double reset on the system reset
2957 * path where both our reset handler and ->reset() callback are used. Calling
2958 * _one() will only do a hot reset for the one in-use devices case, calling
2959 * _multi() will do nothing if a _one() would have been sufficient.
2961 static int vfio_pci_hot_reset_one(VFIOPCIDevice
*vdev
)
2963 return vfio_pci_hot_reset(vdev
, true);
2966 static int vfio_pci_hot_reset_multi(VFIODevice
*vbasedev
)
2968 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2969 return vfio_pci_hot_reset(vdev
, false);
2972 static void vfio_pci_compute_needs_reset(VFIODevice
*vbasedev
)
2974 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2975 if (!vbasedev
->reset_works
|| (!vdev
->has_flr
&& vdev
->has_pm_reset
)) {
2976 vbasedev
->needs_reset
= true;
2980 static VFIODeviceOps vfio_pci_ops
= {
2981 .vfio_compute_needs_reset
= vfio_pci_compute_needs_reset
,
2982 .vfio_hot_reset_multi
= vfio_pci_hot_reset_multi
,
2983 .vfio_eoi
= vfio_eoi
,
2986 static int vfio_populate_device(VFIOPCIDevice
*vdev
)
2988 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2989 struct vfio_region_info reg_info
= { .argsz
= sizeof(reg_info
) };
2990 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
) };
2993 /* Sanity check device */
2994 if (!(vbasedev
->flags
& VFIO_DEVICE_FLAGS_PCI
)) {
2995 error_report("vfio: Um, this isn't a PCI device");
2999 if (vbasedev
->num_regions
< VFIO_PCI_CONFIG_REGION_INDEX
+ 1) {
3000 error_report("vfio: unexpected number of io regions %u",
3001 vbasedev
->num_regions
);
3005 if (vbasedev
->num_irqs
< VFIO_PCI_MSIX_IRQ_INDEX
+ 1) {
3006 error_report("vfio: unexpected number of irqs %u", vbasedev
->num_irqs
);
3010 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
3013 ret
= ioctl(vbasedev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3015 error_report("vfio: Error getting region %d info: %m", i
);
3019 trace_vfio_populate_device_region(vbasedev
->name
, i
,
3020 (unsigned long)reg_info
.size
,
3021 (unsigned long)reg_info
.offset
,
3022 (unsigned long)reg_info
.flags
);
3024 vdev
->bars
[i
].region
.vbasedev
= vbasedev
;
3025 vdev
->bars
[i
].region
.flags
= reg_info
.flags
;
3026 vdev
->bars
[i
].region
.size
= reg_info
.size
;
3027 vdev
->bars
[i
].region
.fd_offset
= reg_info
.offset
;
3028 vdev
->bars
[i
].region
.nr
= i
;
3029 QLIST_INIT(&vdev
->bars
[i
].quirks
);
3032 reg_info
.index
= VFIO_PCI_CONFIG_REGION_INDEX
;
3034 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3036 error_report("vfio: Error getting config info: %m");
3040 trace_vfio_populate_device_config(vdev
->vbasedev
.name
,
3041 (unsigned long)reg_info
.size
,
3042 (unsigned long)reg_info
.offset
,
3043 (unsigned long)reg_info
.flags
);
3045 vdev
->config_size
= reg_info
.size
;
3046 if (vdev
->config_size
== PCI_CONFIG_SPACE_SIZE
) {
3047 vdev
->pdev
.cap_present
&= ~QEMU_PCI_CAP_EXPRESS
;
3049 vdev
->config_offset
= reg_info
.offset
;
3051 if ((vdev
->features
& VFIO_FEATURE_ENABLE_VGA
) &&
3052 vbasedev
->num_regions
> VFIO_PCI_VGA_REGION_INDEX
) {
3053 struct vfio_region_info vga_info
= {
3054 .argsz
= sizeof(vga_info
),
3055 .index
= VFIO_PCI_VGA_REGION_INDEX
,
3058 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_REGION_INFO
, &vga_info
);
3061 "vfio: Device does not support requested feature x-vga");
3065 if (!(vga_info
.flags
& VFIO_REGION_INFO_FLAG_READ
) ||
3066 !(vga_info
.flags
& VFIO_REGION_INFO_FLAG_WRITE
) ||
3067 vga_info
.size
< 0xbffff + 1) {
3068 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
3069 (unsigned long)vga_info
.flags
,
3070 (unsigned long)vga_info
.size
);
3074 vdev
->vga
.fd_offset
= vga_info
.offset
;
3075 vdev
->vga
.fd
= vdev
->vbasedev
.fd
;
3077 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].offset
= QEMU_PCI_VGA_MEM_BASE
;
3078 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].nr
= QEMU_PCI_VGA_MEM
;
3079 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].quirks
);
3081 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].offset
= QEMU_PCI_VGA_IO_LO_BASE
;
3082 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].nr
= QEMU_PCI_VGA_IO_LO
;
3083 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].quirks
);
3085 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].offset
= QEMU_PCI_VGA_IO_HI_BASE
;
3086 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].nr
= QEMU_PCI_VGA_IO_HI
;
3087 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
);
3089 vdev
->has_vga
= true;
3091 irq_info
.index
= VFIO_PCI_ERR_IRQ_INDEX
;
3093 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
3095 /* This can fail for an old kernel or legacy PCI dev */
3096 trace_vfio_populate_device_get_irq_info_failure();
3098 } else if (irq_info
.count
== 1) {
3099 vdev
->pci_aer
= true;
3101 error_report("vfio: %s "
3102 "Could not enable error recovery for the device",
3110 static void vfio_put_device(VFIOPCIDevice
*vdev
)
3112 g_free(vdev
->vbasedev
.name
);
3114 object_unparent(OBJECT(&vdev
->msix
->mmap_mem
));
3118 vfio_put_base_device(&vdev
->vbasedev
);
3121 static void vfio_err_notifier_handler(void *opaque
)
3123 VFIOPCIDevice
*vdev
= opaque
;
3125 if (!event_notifier_test_and_clear(&vdev
->err_notifier
)) {
3130 * TBD. Retrieve the error details and decide what action
3131 * needs to be taken. One of the actions could be to pass
3132 * the error to the guest and have the guest driver recover
3133 * from the error. This requires that PCIe capabilities be
3134 * exposed to the guest. For now, we just terminate the
3135 * guest to contain the error.
3138 error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. "
3139 "Please collect any data possible and then kill the guest",
3140 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
3141 vdev
->host
.slot
, vdev
->host
.function
);
3143 vm_stop(RUN_STATE_INTERNAL_ERROR
);
3147 * Registers error notifier for devices supporting error recovery.
3148 * If we encounter a failure in this function, we report an error
3149 * and continue after disabling error recovery support for the
3152 static void vfio_register_err_notifier(VFIOPCIDevice
*vdev
)
3156 struct vfio_irq_set
*irq_set
;
3159 if (!vdev
->pci_aer
) {
3163 if (event_notifier_init(&vdev
->err_notifier
, 0)) {
3164 error_report("vfio: Unable to init event notifier for error detection");
3165 vdev
->pci_aer
= false;
3169 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
3171 irq_set
= g_malloc0(argsz
);
3172 irq_set
->argsz
= argsz
;
3173 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
3174 VFIO_IRQ_SET_ACTION_TRIGGER
;
3175 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
3178 pfd
= (int32_t *)&irq_set
->data
;
3180 *pfd
= event_notifier_get_fd(&vdev
->err_notifier
);
3181 qemu_set_fd_handler(*pfd
, vfio_err_notifier_handler
, NULL
, vdev
);
3183 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
3185 error_report("vfio: Failed to set up error notification");
3186 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
3187 event_notifier_cleanup(&vdev
->err_notifier
);
3188 vdev
->pci_aer
= false;
3193 static void vfio_unregister_err_notifier(VFIOPCIDevice
*vdev
)
3196 struct vfio_irq_set
*irq_set
;
3200 if (!vdev
->pci_aer
) {
3204 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
3206 irq_set
= g_malloc0(argsz
);
3207 irq_set
->argsz
= argsz
;
3208 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
3209 VFIO_IRQ_SET_ACTION_TRIGGER
;
3210 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
3213 pfd
= (int32_t *)&irq_set
->data
;
3216 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
3218 error_report("vfio: Failed to de-assign error fd: %m");
3221 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->err_notifier
),
3223 event_notifier_cleanup(&vdev
->err_notifier
);
3226 static int vfio_initfn(PCIDevice
*pdev
)
3228 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
3229 VFIODevice
*vbasedev_iter
;
3231 char path
[PATH_MAX
], iommu_group_path
[PATH_MAX
], *group_name
;
3237 /* Check that the host device exists */
3238 snprintf(path
, sizeof(path
),
3239 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
3240 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3241 vdev
->host
.function
);
3242 if (stat(path
, &st
) < 0) {
3243 error_report("vfio: error: no such host device: %s", path
);
3247 vdev
->vbasedev
.ops
= &vfio_pci_ops
;
3249 vdev
->vbasedev
.type
= VFIO_DEVICE_TYPE_PCI
;
3250 vdev
->vbasedev
.name
= g_strdup_printf("%04x:%02x:%02x.%01x",
3251 vdev
->host
.domain
, vdev
->host
.bus
,
3252 vdev
->host
.slot
, vdev
->host
.function
);
3254 strncat(path
, "iommu_group", sizeof(path
) - strlen(path
) - 1);
3256 len
= readlink(path
, iommu_group_path
, sizeof(path
));
3257 if (len
<= 0 || len
>= sizeof(path
)) {
3258 error_report("vfio: error no iommu_group for device");
3259 return len
< 0 ? -errno
: ENAMETOOLONG
;
3262 iommu_group_path
[len
] = 0;
3263 group_name
= basename(iommu_group_path
);
3265 if (sscanf(group_name
, "%d", &groupid
) != 1) {
3266 error_report("vfio: error reading %s: %m", path
);
3270 trace_vfio_initfn(vdev
->vbasedev
.name
, groupid
);
3272 group
= vfio_get_group(groupid
, pci_device_iommu_address_space(pdev
));
3274 error_report("vfio: failed to get group %d", groupid
);
3278 snprintf(path
, sizeof(path
), "%04x:%02x:%02x.%01x",
3279 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3280 vdev
->host
.function
);
3282 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
3283 if (strcmp(vbasedev_iter
->name
, vdev
->vbasedev
.name
) == 0) {
3284 error_report("vfio: error: device %s is already attached", path
);
3285 vfio_put_group(group
);
3290 ret
= vfio_get_device(group
, path
, &vdev
->vbasedev
);
3292 error_report("vfio: failed to get device %s", path
);
3293 vfio_put_group(group
);
3297 ret
= vfio_populate_device(vdev
);
3302 /* Get a copy of config space */
3303 ret
= pread(vdev
->vbasedev
.fd
, vdev
->pdev
.config
,
3304 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
3305 vdev
->config_offset
);
3306 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
3307 ret
= ret
< 0 ? -errno
: -EFAULT
;
3308 error_report("vfio: Failed to read device config space");
3312 /* vfio emulates a lot for us, but some bits need extra love */
3313 vdev
->emulated_config_bits
= g_malloc0(vdev
->config_size
);
3315 /* QEMU can choose to expose the ROM or not */
3316 memset(vdev
->emulated_config_bits
+ PCI_ROM_ADDRESS
, 0xff, 4);
3318 /* QEMU can change multi-function devices to single function, or reverse */
3319 vdev
->emulated_config_bits
[PCI_HEADER_TYPE
] =
3320 PCI_HEADER_TYPE_MULTI_FUNCTION
;
3322 /* Restore or clear multifunction, this is always controlled by QEMU */
3323 if (vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
3324 vdev
->pdev
.config
[PCI_HEADER_TYPE
] |= PCI_HEADER_TYPE_MULTI_FUNCTION
;
3326 vdev
->pdev
.config
[PCI_HEADER_TYPE
] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION
;
3330 * Clear host resource mapping info. If we choose not to register a
3331 * BAR, such as might be the case with the option ROM, we can get
3332 * confusing, unwritable, residual addresses from the host here.
3334 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
3335 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
3337 vfio_pci_size_rom(vdev
);
3339 ret
= vfio_early_setup_msix(vdev
);
3344 vfio_map_bars(vdev
);
3346 ret
= vfio_add_capabilities(vdev
);
3351 /* QEMU emulates all of MSI & MSIX */
3352 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
) {
3353 memset(vdev
->emulated_config_bits
+ pdev
->msix_cap
, 0xff,
3357 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
) {
3358 memset(vdev
->emulated_config_bits
+ pdev
->msi_cap
, 0xff,
3359 vdev
->msi_cap_size
);
3362 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
3363 vdev
->intx
.mmap_timer
= timer_new_ms(QEMU_CLOCK_VIRTUAL
,
3364 vfio_intx_mmap_enable
, vdev
);
3365 pci_device_set_intx_routing_notifier(&vdev
->pdev
, vfio_update_irq
);
3366 ret
= vfio_enable_intx(vdev
);
3372 vfio_register_err_notifier(vdev
);
3377 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3378 vfio_teardown_msi(vdev
);
3379 vfio_unregister_bars(vdev
);
3383 static void vfio_instance_finalize(Object
*obj
)
3385 PCIDevice
*pci_dev
= PCI_DEVICE(obj
);
3386 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pci_dev
);
3387 VFIOGroup
*group
= vdev
->vbasedev
.group
;
3389 vfio_unmap_bars(vdev
);
3390 g_free(vdev
->emulated_config_bits
);
3392 vfio_put_device(vdev
);
3393 vfio_put_group(group
);
3396 static void vfio_exitfn(PCIDevice
*pdev
)
3398 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
3400 vfio_unregister_err_notifier(vdev
);
3401 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3402 vfio_disable_interrupts(vdev
);
3403 if (vdev
->intx
.mmap_timer
) {
3404 timer_free(vdev
->intx
.mmap_timer
);
3406 vfio_teardown_msi(vdev
);
3407 vfio_unregister_bars(vdev
);
3410 static void vfio_pci_reset(DeviceState
*dev
)
3412 PCIDevice
*pdev
= DO_UPCAST(PCIDevice
, qdev
, dev
);
3413 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
3415 trace_vfio_pci_reset(vdev
->vbasedev
.name
);
3417 vfio_pci_pre_reset(vdev
);
3419 if (vdev
->vbasedev
.reset_works
&&
3420 (vdev
->has_flr
|| !vdev
->has_pm_reset
) &&
3421 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
3422 trace_vfio_pci_reset_flr(vdev
->vbasedev
.name
);
3426 /* See if we can do our own bus reset */
3427 if (!vfio_pci_hot_reset_one(vdev
)) {
3431 /* If nothing else works and the device supports PM reset, use it */
3432 if (vdev
->vbasedev
.reset_works
&& vdev
->has_pm_reset
&&
3433 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
3434 trace_vfio_pci_reset_pm(vdev
->vbasedev
.name
);
3439 vfio_pci_post_reset(vdev
);
3442 static void vfio_instance_init(Object
*obj
)
3444 PCIDevice
*pci_dev
= PCI_DEVICE(obj
);
3445 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, PCI_DEVICE(obj
));
3447 device_add_bootindex_property(obj
, &vdev
->bootindex
,
3449 &pci_dev
->qdev
, NULL
);
3452 static Property vfio_pci_dev_properties
[] = {
3453 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice
, host
),
3454 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice
,
3455 intx
.mmap_timeout
, 1100),
3456 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice
, features
,
3457 VFIO_FEATURE_ENABLE_VGA_BIT
, false),
3458 DEFINE_PROP_INT32("bootindex", VFIOPCIDevice
, bootindex
, -1),
3459 DEFINE_PROP_BOOL("x-mmap", VFIOPCIDevice
, vbasedev
.allow_mmap
, true),
3461 * TODO - support passed fds... is this necessary?
3462 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3463 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3465 DEFINE_PROP_END_OF_LIST(),
3468 static const VMStateDescription vfio_pci_vmstate
= {
3473 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
3475 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3476 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
3478 dc
->reset
= vfio_pci_reset
;
3479 dc
->props
= vfio_pci_dev_properties
;
3480 dc
->vmsd
= &vfio_pci_vmstate
;
3481 dc
->desc
= "VFIO-based PCI device assignment";
3482 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
3483 pdc
->init
= vfio_initfn
;
3484 pdc
->exit
= vfio_exitfn
;
3485 pdc
->config_read
= vfio_pci_read_config
;
3486 pdc
->config_write
= vfio_pci_write_config
;
3487 pdc
->is_express
= 1; /* We might be */
3490 static const TypeInfo vfio_pci_dev_info
= {
3492 .parent
= TYPE_PCI_DEVICE
,
3493 .instance_size
= sizeof(VFIOPCIDevice
),
3494 .class_init
= vfio_pci_dev_class_init
,
3495 .instance_init
= vfio_instance_init
,
3496 .instance_finalize
= vfio_instance_finalize
,
3499 static void register_vfio_pci_dev_type(void)
3501 type_register_static(&vfio_pci_dev_info
);
3504 type_init(register_vfio_pci_dev_type
)