]> git.proxmox.com Git - qemu.git/blame - hw/misc/vfio.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
vfio-pci: Lazy PCI option ROM loading
[qemu.git] / hw / misc / vfio.c
CommitLineData
65501a74
AW		 1/*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21#include <dirent.h>
6dcfdbad 22#include <linux/vfio.h>
65501a74
AW		 23#include <sys/ioctl.h>
24#include <sys/mman.h>
25#include <sys/stat.h>
26#include <sys/types.h>
6dcfdbad 27#include <unistd.h>
65501a74
AW		 28
29#include "config.h"
022c62cb 30#include "exec/address-spaces.h"
022c62cb 31#include "exec/memory.h"
83c9f4ca
PB		 32#include "hw/pci/msi.h"
33#include "hw/pci/msix.h"
34#include "hw/pci/pci.h"
5c97e5eb 35#include "qemu-common.h"
1de7afc9 36#include "qemu/error-report.h"
6dcfdbad 37#include "qemu/event_notifier.h"
1de7afc9
PB		 38#include "qemu/queue.h"
39#include "qemu/range.h"
6dcfdbad
AW		 40#include "sysemu/kvm.h"
41#include "sysemu/sysemu.h"
65501a74
AW		 42
43/* #define DEBUG_VFIO */
44#ifdef DEBUG_VFIO
45#define DPRINTF(fmt, ...) \
46 do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
47#else
48#define DPRINTF(fmt, ...) \
49 do { } while (0)
50#endif
51
82ca8912
AW		 52/* Extra debugging, trap acceleration paths for more logging */
53#define VFIO_ALLOW_MMAP 1
54#define VFIO_ALLOW_KVM_INTX 1
55
7076eabc
AW		 56struct VFIODevice;
57
58typedef struct VFIOQuirk {
59 MemoryRegion mem;
60 struct VFIODevice *vdev;
61 QLIST_ENTRY(VFIOQuirk) next;
39360f0b
AW		 62 struct {
63 uint32_t base_offset:TARGET_PAGE_BITS;
64 uint32_t address_offset:TARGET_PAGE_BITS;
65 uint32_t address_size:3;
66 uint32_t bar:3;
67
68 uint32_t address_match;
69 uint32_t address_mask;
70
71 uint32_t address_val:TARGET_PAGE_BITS;
72 uint32_t data_offset:TARGET_PAGE_BITS;
73 uint32_t data_size:3;
74
75 uint8_t flags;
76 uint8_t read_flags;
77 uint8_t write_flags;
78 } data;
7076eabc
AW		 79} VFIOQuirk;
80
5c97e5eb
AW		 81typedef struct VFIOBAR {
82 off_t fd_offset; /* offset of BAR within device fd */
83 int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
84 MemoryRegion mem; /* slow, read/write access */
85 MemoryRegion mmap_mem; /* direct mapped access */
86 void *mmap;
87 size_t size;
88 uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
89 uint8_t nr; /* cache the BAR number for debug */
39360f0b
AW		 90 bool ioport;
91 bool mem64;
7076eabc 92 QLIST_HEAD(, VFIOQuirk) quirks;
5c97e5eb
AW		 93} VFIOBAR;
94
f15689c7
AW		 95typedef struct VFIOVGARegion {
96 MemoryRegion mem;
97 off_t offset;
98 int nr;
7076eabc 99 QLIST_HEAD(, VFIOQuirk) quirks;
f15689c7
AW		 100} VFIOVGARegion;
101
102typedef struct VFIOVGA {
103 off_t fd_offset;
104 int fd;
105 VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
106} VFIOVGA;
107
5c97e5eb
AW		 108typedef struct VFIOINTx {
109 bool pending; /* interrupt pending */
110 bool kvm_accel; /* set when QEMU bypass through KVM enabled */
111 uint8_t pin; /* which pin to pull for qemu_set_irq */
112 EventNotifier interrupt; /* eventfd triggered on interrupt */
113 EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
114 PCIINTxRoute route; /* routing info for QEMU bypass */
115 uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
116 QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
117} VFIOINTx;
118
5c97e5eb
AW		 119typedef struct VFIOMSIVector {
120 EventNotifier interrupt; /* eventfd triggered on interrupt */
121 struct VFIODevice *vdev; /* back pointer to device */
c7679d45 122 MSIMessage msg; /* cache the MSI message so we know when it changes */
5c97e5eb
AW		 123 int virq; /* KVM irqchip route for QEMU bypass */
124 bool use;
125} VFIOMSIVector;
126
127enum {
128 VFIO_INT_NONE = 0,
129 VFIO_INT_INTx = 1,
130 VFIO_INT_MSI = 2,
131 VFIO_INT_MSIX = 3,
132};
133
134struct VFIOGroup;
135
136typedef struct VFIOContainer {
137 int fd; /* /dev/vfio/vfio, empowered by the attached groups */
138 struct {
139 /* enable abstraction to support various iommu backends */
140 union {
141 MemoryListener listener; /* Used by type1 iommu */
142 };
143 void (*release)(struct VFIOContainer *);
144 } iommu_data;
145 QLIST_HEAD(, VFIOGroup) group_list;
146 QLIST_ENTRY(VFIOContainer) next;
147} VFIOContainer;
148
149/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
150typedef struct VFIOMSIXInfo {
151 uint8_t table_bar;
152 uint8_t pba_bar;
153 uint16_t entries;
154 uint32_t table_offset;
155 uint32_t pba_offset;
156 MemoryRegion mmap_mem;
157 void *mmap;
158} VFIOMSIXInfo;
159
160typedef struct VFIODevice {
161 PCIDevice pdev;
162 int fd;
163 VFIOINTx intx;
164 unsigned int config_size;
4b5d5e87 165 uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
5c97e5eb
AW		 166 off_t config_offset; /* Offset of config space region within device fd */
167 unsigned int rom_size;
168 off_t rom_offset; /* Offset of ROM region within device fd */
6f864e6e 169 void *rom;
5c97e5eb
AW		 170 int msi_cap_size;
171 VFIOMSIVector *msi_vectors;
172 VFIOMSIXInfo *msix;
173 int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
174 int interrupt; /* Current interrupt type */
175 VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
f15689c7 176 VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
5c97e5eb
AW		 177 PCIHostDeviceAddress host;
178 QLIST_ENTRY(VFIODevice) next;
179 struct VFIOGroup *group;
7b4b0e9e 180 EventNotifier err_notifier;
f15689c7
AW		 181 uint32_t features;
182#define VFIO_FEATURE_ENABLE_VGA_BIT 0
183#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
c29029dd 184 int32_t bootindex;
ba661818 185 uint8_t pm_cap;
5c97e5eb 186 bool reset_works;
f15689c7 187 bool has_vga;
7b4b0e9e 188 bool pci_aer;
befe5176
AW		 189 bool has_flr;
190 bool has_pm_reset;
5c97e5eb
AW		 191} VFIODevice;
192
193typedef struct VFIOGroup {
194 int fd;
195 int groupid;
196 VFIOContainer *container;
197 QLIST_HEAD(, VFIODevice) device_list;
198 QLIST_ENTRY(VFIOGroup) next;
199 QLIST_ENTRY(VFIOGroup) container_next;
200} VFIOGroup;
201
65501a74
AW		 202#define MSIX_CAP_LENGTH 12
203
204static QLIST_HEAD(, VFIOContainer)
205 container_list = QLIST_HEAD_INITIALIZER(container_list);
206
207static QLIST_HEAD(, VFIOGroup)
208 group_list = QLIST_HEAD_INITIALIZER(group_list);
209
210static void vfio_disable_interrupts(VFIODevice *vdev);
211static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
7076eabc
AW		 212static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
213 uint32_t val, int len);
65501a74
AW		 214static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
215
216/*
217 * Common VFIO interrupt disable
218 */
219static void vfio_disable_irqindex(VFIODevice *vdev, int index)
220{
221 struct vfio_irq_set irq_set = {
222 .argsz = sizeof(irq_set),
223 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
224 .index = index,
225 .start = 0,
226 .count = 0,
227 };
228
229 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
65501a74
AW		 230}
231
232/*
233 * INTx
234 */
235static void vfio_unmask_intx(VFIODevice *vdev)
236{
237 struct vfio_irq_set irq_set = {
238 .argsz = sizeof(irq_set),
239 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
240 .index = VFIO_PCI_INTX_IRQ_INDEX,
241 .start = 0,
242 .count = 1,
243 };
244
245 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
246}
247
e1d1e586
AW		 248#ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
249static void vfio_mask_intx(VFIODevice *vdev)
250{
251 struct vfio_irq_set irq_set = {
252 .argsz = sizeof(irq_set),
253 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
254 .index = VFIO_PCI_INTX_IRQ_INDEX,
255 .start = 0,
256 .count = 1,
257 };
258
259 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
260}
261#endif
262
ea486926
AW		 263/*
264 * Disabling BAR mmaping can be slow, but toggling it around INTx can
265 * also be a huge overhead. We try to get the best of both worlds by
266 * waiting until an interrupt to disable mmaps (subsequent transitions
267 * to the same state are effectively no overhead). If the interrupt has
268 * been serviced and the time gap is long enough, we re-enable mmaps for
269 * performance. This works well for things like graphics cards, which
270 * may not use their interrupt at all and are penalized to an unusable
271 * level by read/write BAR traps. Other devices, like NICs, have more
272 * regular interrupts and see much better latency by staying in non-mmap
273 * mode. We therefore set the default mmap_timeout such that a ping
274 * is just enough to keep the mmap disabled. Users can experiment with
275 * other options with the x-intx-mmap-timeout-ms parameter (a value of
276 * zero disables the timer).
277 */
278static void vfio_intx_mmap_enable(void *opaque)
279{
280 VFIODevice *vdev = opaque;
281
282 if (vdev->intx.pending) {
bc72ad67
AB		 283 timer_mod(vdev->intx.mmap_timer,
284 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
ea486926
AW		 285 return;
286 }
287
288 vfio_mmap_set_enabled(vdev, true);
289}
290
65501a74
AW		 291static void vfio_intx_interrupt(void *opaque)
292{
293 VFIODevice *vdev = opaque;
294
295 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
296 return;
297 }
298
299 DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__, vdev->host.domain,
300 vdev->host.bus, vdev->host.slot, vdev->host.function,
301 'A' + vdev->intx.pin);
302
303 vdev->intx.pending = true;
304 qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 1);
ea486926
AW		 305 vfio_mmap_set_enabled(vdev, false);
306 if (vdev->intx.mmap_timeout) {
bc72ad67
AB		 307 timer_mod(vdev->intx.mmap_timer,
308 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
ea486926 309 }
65501a74
AW		 310}
311
312static void vfio_eoi(VFIODevice *vdev)
313{
314 if (!vdev->intx.pending) {
315 return;
316 }
317
318 DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__, vdev->host.domain,
319 vdev->host.bus, vdev->host.slot, vdev->host.function);
320
321 vdev->intx.pending = false;
322 qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
323 vfio_unmask_intx(vdev);
324}
325
e1d1e586
AW		 326static void vfio_enable_intx_kvm(VFIODevice *vdev)
327{
328#ifdef CONFIG_KVM
329 struct kvm_irqfd irqfd = {
330 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
331 .gsi = vdev->intx.route.irq,
332 .flags = KVM_IRQFD_FLAG_RESAMPLE,
333 };
334 struct vfio_irq_set *irq_set;
335 int ret, argsz;
336 int32_t *pfd;
337
82ca8912 338 if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() ||
e1d1e586
AW		 339 vdev->intx.route.mode != PCI_INTX_ENABLED ||
340 !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
341 return;
342 }
343
344 /* Get to a known interrupt state */
345 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
346 vfio_mask_intx(vdev);
347 vdev->intx.pending = false;
348 qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
349
350 /* Get an eventfd for resample/unmask */
351 if (event_notifier_init(&vdev->intx.unmask, 0)) {
312fd5f2 352 error_report("vfio: Error: event_notifier_init failed eoi");
e1d1e586
AW		 353 goto fail;
354 }
355
356 /* KVM triggers it, VFIO listens for it */
357 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
358
359 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
312fd5f2 360 error_report("vfio: Error: Failed to setup resample irqfd: %m");
e1d1e586
AW		 361 goto fail_irqfd;
362 }
363
364 argsz = sizeof(*irq_set) + sizeof(*pfd);
365
366 irq_set = g_malloc0(argsz);
367 irq_set->argsz = argsz;
368 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
369 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
370 irq_set->start = 0;
371 irq_set->count = 1;
372 pfd = (int32_t *)&irq_set->data;
373
374 *pfd = irqfd.resamplefd;
375
376 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
377 g_free(irq_set);
378 if (ret) {
312fd5f2 379 error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
e1d1e586
AW		 380 goto fail_vfio;
381 }
382
383 /* Let'em rip */
384 vfio_unmask_intx(vdev);
385
386 vdev->intx.kvm_accel = true;
387
388 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
389 __func__, vdev->host.domain, vdev->host.bus,
390 vdev->host.slot, vdev->host.function);
391
392 return;
393
394fail_vfio:
395 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
396 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
397fail_irqfd:
398 event_notifier_cleanup(&vdev->intx.unmask);
399fail:
400 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
401 vfio_unmask_intx(vdev);
402#endif
403}
404
405static void vfio_disable_intx_kvm(VFIODevice *vdev)
406{
407#ifdef CONFIG_KVM
408 struct kvm_irqfd irqfd = {
409 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
410 .gsi = vdev->intx.route.irq,
411 .flags = KVM_IRQFD_FLAG_DEASSIGN,
412 };
413
414 if (!vdev->intx.kvm_accel) {
415 return;
416 }
417
418 /*
419 * Get to a known state, hardware masked, QEMU ready to accept new
420 * interrupts, QEMU IRQ de-asserted.
421 */
422 vfio_mask_intx(vdev);
423 vdev->intx.pending = false;
424 qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
425
426 /* Tell KVM to stop listening for an INTx irqfd */
427 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
312fd5f2 428 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
e1d1e586
AW		 429 }
430
431 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
432 event_notifier_cleanup(&vdev->intx.unmask);
433
434 /* QEMU starts listening for interrupt events. */
435 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
436
437 vdev->intx.kvm_accel = false;
438
439 /* If we've missed an event, let it re-fire through QEMU */
440 vfio_unmask_intx(vdev);
441
442 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
443 __func__, vdev->host.domain, vdev->host.bus,
444 vdev->host.slot, vdev->host.function);
445#endif
446}
447
448static void vfio_update_irq(PCIDevice *pdev)
449{
450 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
451 PCIINTxRoute route;
452
453 if (vdev->interrupt != VFIO_INT_INTx) {
454 return;
455 }
456
457 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
458
459 if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
460 return; /* Nothing changed */
461 }
462
463 DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
464 vdev->host.domain, vdev->host.bus, vdev->host.slot,
465 vdev->host.function, vdev->intx.route.irq, route.irq);
466
467 vfio_disable_intx_kvm(vdev);
468
469 vdev->intx.route = route;
470
471 if (route.mode != PCI_INTX_ENABLED) {
472 return;
473 }
474
475 vfio_enable_intx_kvm(vdev);
476
477 /* Re-enable the interrupt in cased we missed an EOI */
478 vfio_eoi(vdev);
479}
480
65501a74
AW		 481static int vfio_enable_intx(VFIODevice *vdev)
482{
65501a74 483 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
1a403133
AW		 484 int ret, argsz;
485 struct vfio_irq_set *irq_set;
486 int32_t *pfd;
65501a74 487
ea486926 488 if (!pin) {
65501a74
AW		 489 return 0;
490 }
491
492 vfio_disable_interrupts(vdev);
493
494 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
e1d1e586
AW		 495
496#ifdef CONFIG_KVM
497 /*
498 * Only conditional to avoid generating error messages on platforms
499 * where we won't actually use the result anyway.
500 */
d281084d
AW		 501 if (kvm_irqfds_enabled() &&
502 kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
e1d1e586
AW		 503 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
504 vdev->intx.pin);
505 }
506#endif
507
65501a74
AW		 508 ret = event_notifier_init(&vdev->intx.interrupt, 0);
509 if (ret) {
312fd5f2 510 error_report("vfio: Error: event_notifier_init failed");
65501a74
AW		 511 return ret;
512 }
513
1a403133
AW		 514 argsz = sizeof(*irq_set) + sizeof(*pfd);
515
516 irq_set = g_malloc0(argsz);
517 irq_set->argsz = argsz;
518 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
519 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
520 irq_set->start = 0;
521 irq_set->count = 1;
522 pfd = (int32_t *)&irq_set->data;
523
524 *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
525 qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
65501a74 526
1a403133
AW		 527 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
528 g_free(irq_set);
529 if (ret) {
312fd5f2 530 error_report("vfio: Error: Failed to setup INTx fd: %m");
1a403133 531 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
ce59af2d 532 event_notifier_cleanup(&vdev->intx.interrupt);
65501a74
AW		 533 return -errno;
534 }
535
e1d1e586
AW		 536 vfio_enable_intx_kvm(vdev);
537
65501a74
AW		 538 vdev->interrupt = VFIO_INT_INTx;
539
540 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
541 vdev->host.bus, vdev->host.slot, vdev->host.function);
542
543 return 0;
544}
545
546static void vfio_disable_intx(VFIODevice *vdev)
547{
548 int fd;
549
bc72ad67 550 timer_del(vdev->intx.mmap_timer);
e1d1e586 551 vfio_disable_intx_kvm(vdev);
65501a74
AW		 552 vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
553 vdev->intx.pending = false;
554 qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
555 vfio_mmap_set_enabled(vdev, true);
556
557 fd = event_notifier_get_fd(&vdev->intx.interrupt);
558 qemu_set_fd_handler(fd, NULL, NULL, vdev);
559 event_notifier_cleanup(&vdev->intx.interrupt);
560
561 vdev->interrupt = VFIO_INT_NONE;
562
563 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
564 vdev->host.bus, vdev->host.slot, vdev->host.function);
565}
566
567/*
568 * MSI/X
569 */
570static void vfio_msi_interrupt(void *opaque)
571{
572 VFIOMSIVector *vector = opaque;
573 VFIODevice *vdev = vector->vdev;
574 int nr = vector - vdev->msi_vectors;
575
576 if (!event_notifier_test_and_clear(&vector->interrupt)) {
577 return;
578 }
579
580 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d\n", __func__,
581 vdev->host.domain, vdev->host.bus, vdev->host.slot,
582 vdev->host.function, nr);
583
584 if (vdev->interrupt == VFIO_INT_MSIX) {
585 msix_notify(&vdev->pdev, nr);
586 } else if (vdev->interrupt == VFIO_INT_MSI) {
587 msi_notify(&vdev->pdev, nr);
588 } else {
312fd5f2 589 error_report("vfio: MSI interrupt receieved, but not enabled?");
65501a74
AW		 590 }
591}
592
593static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
594{
595 struct vfio_irq_set *irq_set;
596 int ret = 0, i, argsz;
597 int32_t *fds;
598
599 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
600
601 irq_set = g_malloc0(argsz);
602 irq_set->argsz = argsz;
603 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
604 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
605 irq_set->start = 0;
606 irq_set->count = vdev->nr_vectors;
607 fds = (int32_t *)&irq_set->data;
608
609 for (i = 0; i < vdev->nr_vectors; i++) {
610 if (!vdev->msi_vectors[i].use) {
611 fds[i] = -1;
612 continue;
613 }
614
615 fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
616 }
617
618 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
619
620 g_free(irq_set);
621
65501a74
AW		 622 return ret;
623}
624
b0223e29
AW		 625static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
626 MSIMessage *msg, IOHandler *handler)
65501a74
AW		 627{
628 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
629 VFIOMSIVector *vector;
630 int ret;
631
632 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
633 vdev->host.domain, vdev->host.bus, vdev->host.slot,
634 vdev->host.function, nr);
635
65501a74
AW		 636 vector = &vdev->msi_vectors[nr];
637 vector->vdev = vdev;
638 vector->use = true;
639
640 msix_vector_use(pdev, nr);
641
642 if (event_notifier_init(&vector->interrupt, 0)) {
312fd5f2 643 error_report("vfio: Error: event_notifier_init failed");
65501a74
AW		 644 }
645
646 /*
647 * Attempt to enable route through KVM irqchip,
648 * default to userspace handling if unavailable.
649 */
b0223e29 650 vector->virq = msg ? kvm_irqchip_add_msi_route(kvm_state, *msg) : -1;
65501a74
AW		 651 if (vector->virq < 0 ||
652 kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
ca916d37 653 NULL, vector->virq) < 0) {
65501a74
AW		 654 if (vector->virq >= 0) {
655 kvm_irqchip_release_virq(kvm_state, vector->virq);
656 vector->virq = -1;
657 }
658 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
b0223e29 659 handler, NULL, vector);
65501a74
AW		 660 }
661
662 /*
663 * We don't want to have the host allocate all possible MSI vectors
664 * for a device if they're not in use, so we shutdown and incrementally
665 * increase them as needed.
666 */
667 if (vdev->nr_vectors < nr + 1) {
65501a74
AW		 668 vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
669 vdev->nr_vectors = nr + 1;
670 ret = vfio_enable_vectors(vdev, true);
671 if (ret) {
312fd5f2 672 error_report("vfio: failed to enable vectors, %d", ret);
65501a74 673 }
65501a74 674 } else {
1a403133
AW		 675 int argsz;
676 struct vfio_irq_set *irq_set;
677 int32_t *pfd;
678
679 argsz = sizeof(*irq_set) + sizeof(*pfd);
680
681 irq_set = g_malloc0(argsz);
682 irq_set->argsz = argsz;
683 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
684 VFIO_IRQ_SET_ACTION_TRIGGER;
685 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
686 irq_set->start = nr;
687 irq_set->count = 1;
688 pfd = (int32_t *)&irq_set->data;
689
690 *pfd = event_notifier_get_fd(&vector->interrupt);
691
692 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
693 g_free(irq_set);
65501a74 694 if (ret) {
312fd5f2 695 error_report("vfio: failed to modify vector, %d", ret);
65501a74 696 }
65501a74
AW		 697 }
698
699 return 0;
700}
701
b0223e29
AW		 702static int vfio_msix_vector_use(PCIDevice *pdev,
703 unsigned int nr, MSIMessage msg)
704{
705 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
706}
707
65501a74
AW		 708static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
709{
710 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
711 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
1a403133
AW		 712 int argsz;
713 struct vfio_irq_set *irq_set;
714 int32_t *pfd;
65501a74
AW		 715
716 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
717 vdev->host.domain, vdev->host.bus, vdev->host.slot,
718 vdev->host.function, nr);
719
720 /*
721 * XXX What's the right thing to do here? This turns off the interrupt
722 * completely, but do we really just want to switch the interrupt to
723 * bouncing through userspace and let msix.c drop it? Not sure.
724 */
725 msix_vector_unuse(pdev, nr);
1a403133
AW		 726
727 argsz = sizeof(*irq_set) + sizeof(*pfd);
728
729 irq_set = g_malloc0(argsz);
730 irq_set->argsz = argsz;
731 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
732 VFIO_IRQ_SET_ACTION_TRIGGER;
733 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
734 irq_set->start = nr;
735 irq_set->count = 1;
736 pfd = (int32_t *)&irq_set->data;
737
738 *pfd = -1;
739
740 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
741
742 g_free(irq_set);
65501a74
AW		 743
744 if (vector->virq < 0) {
745 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
746 NULL, NULL, NULL);
747 } else {
748 kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
749 vector->virq);
750 kvm_irqchip_release_virq(kvm_state, vector->virq);
751 vector->virq = -1;
752 }
753
754 event_notifier_cleanup(&vector->interrupt);
755 vector->use = false;
756}
757
fd704adc
AW		 758static void vfio_enable_msix(VFIODevice *vdev)
759{
760 vfio_disable_interrupts(vdev);
761
762 vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
763
764 vdev->interrupt = VFIO_INT_MSIX;
765
b0223e29
AW		 766 /*
767 * Some communication channels between VF & PF or PF & fw rely on the
768 * physical state of the device and expect that enabling MSI-X from the
769 * guest enables the same on the host. When our guest is Linux, the
770 * guest driver call to pci_enable_msix() sets the enabling bit in the
771 * MSI-X capability, but leaves the vector table masked. We therefore
772 * can't rely on a vector_use callback (from request_irq() in the guest)
773 * to switch the physical device into MSI-X mode because that may come a
774 * long time after pci_enable_msix(). This code enables vector 0 with
775 * triggering to userspace, then immediately release the vector, leaving
776 * the physical device with no vectors enabled, but MSI-X enabled, just
777 * like the guest view.
778 */
779 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
780 vfio_msix_vector_release(&vdev->pdev, 0);
781
fd704adc 782 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
bbef882c 783 vfio_msix_vector_release, NULL)) {
312fd5f2 784 error_report("vfio: msix_set_vector_notifiers failed");
fd704adc
AW		 785 }
786
787 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
788 vdev->host.bus, vdev->host.slot, vdev->host.function);
789}
790
65501a74
AW		 791static void vfio_enable_msi(VFIODevice *vdev)
792{
793 int ret, i;
794
795 vfio_disable_interrupts(vdev);
796
797 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
798retry:
799 vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
800
801 for (i = 0; i < vdev->nr_vectors; i++) {
65501a74
AW		 802 VFIOMSIVector *vector = &vdev->msi_vectors[i];
803
804 vector->vdev = vdev;
805 vector->use = true;
806
807 if (event_notifier_init(&vector->interrupt, 0)) {
312fd5f2 808 error_report("vfio: Error: event_notifier_init failed");
65501a74
AW		 809 }
810
c7679d45 811 vector->msg = msi_get_message(&vdev->pdev, i);
65501a74
AW		 812
813 /*
814 * Attempt to enable route through KVM irqchip,
815 * default to userspace handling if unavailable.
816 */
c7679d45 817 vector->virq = kvm_irqchip_add_msi_route(kvm_state, vector->msg);
65501a74
AW		 818 if (vector->virq < 0 ||
819 kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
ca916d37 820 NULL, vector->virq) < 0) {
65501a74
AW		 821 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
822 vfio_msi_interrupt, NULL, vector);
823 }
824 }
825
826 ret = vfio_enable_vectors(vdev, false);
827 if (ret) {
828 if (ret < 0) {
312fd5f2 829 error_report("vfio: Error: Failed to setup MSI fds: %m");
65501a74
AW		 830 } else if (ret != vdev->nr_vectors) {
831 error_report("vfio: Error: Failed to enable %d "
312fd5f2 832 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
65501a74
AW		 833 }
834
835 for (i = 0; i < vdev->nr_vectors; i++) {
836 VFIOMSIVector *vector = &vdev->msi_vectors[i];
837 if (vector->virq >= 0) {
838 kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
839 vector->virq);
840 kvm_irqchip_release_virq(kvm_state, vector->virq);
841 vector->virq = -1;
842 } else {
843 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
844 NULL, NULL, NULL);
845 }
846 event_notifier_cleanup(&vector->interrupt);
847 }
848
849 g_free(vdev->msi_vectors);
850
851 if (ret > 0 && ret != vdev->nr_vectors) {
852 vdev->nr_vectors = ret;
853 goto retry;
854 }
855 vdev->nr_vectors = 0;
856
857 return;
858 }
859
fd704adc
AW		 860 vdev->interrupt = VFIO_INT_MSI;
861
65501a74
AW		 862 DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
863 vdev->host.domain, vdev->host.bus, vdev->host.slot,
864 vdev->host.function, vdev->nr_vectors);
865}
866
fd704adc
AW		 867static void vfio_disable_msi_common(VFIODevice *vdev)
868{
869 g_free(vdev->msi_vectors);
870 vdev->msi_vectors = NULL;
871 vdev->nr_vectors = 0;
872 vdev->interrupt = VFIO_INT_NONE;
873
874 vfio_enable_intx(vdev);
875}
876
877static void vfio_disable_msix(VFIODevice *vdev)
878{
879 msix_unset_vector_notifiers(&vdev->pdev);
880
881 if (vdev->nr_vectors) {
882 vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
883 }
884
885 vfio_disable_msi_common(vdev);
886
a011b10e
AW		 887 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
888 vdev->host.bus, vdev->host.slot, vdev->host.function);
fd704adc
AW		 889}
890
891static void vfio_disable_msi(VFIODevice *vdev)
65501a74
AW		 892{
893 int i;
894
fd704adc 895 vfio_disable_irqindex(vdev, VFIO_PCI_MSI_IRQ_INDEX);
65501a74
AW		 896
897 for (i = 0; i < vdev->nr_vectors; i++) {
898 VFIOMSIVector *vector = &vdev->msi_vectors[i];
899
900 if (!vector->use) {
901 continue;
902 }
903
904 if (vector->virq >= 0) {
905 kvm_irqchip_remove_irqfd_notifier(kvm_state,
906 &vector->interrupt, vector->virq);
907 kvm_irqchip_release_virq(kvm_state, vector->virq);
908 vector->virq = -1;
909 } else {
910 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
911 NULL, NULL, NULL);
912 }
913
65501a74
AW		 914 event_notifier_cleanup(&vector->interrupt);
915 }
916
fd704adc 917 vfio_disable_msi_common(vdev);
65501a74 918
fd704adc
AW		 919 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
920 vdev->host.bus, vdev->host.slot, vdev->host.function);
65501a74
AW		 921}
922
c7679d45
AW		 923static void vfio_update_msi(VFIODevice *vdev)
924{
925 int i;
926
927 for (i = 0; i < vdev->nr_vectors; i++) {
928 VFIOMSIVector *vector = &vdev->msi_vectors[i];
929 MSIMessage msg;
930
931 if (!vector->use || vector->virq < 0) {
932 continue;
933 }
934
935 msg = msi_get_message(&vdev->pdev, i);
936
937 if (msg.address != vector->msg.address ||
938 msg.data != vector->msg.data) {
939
940 DPRINTF("%s(%04x:%02x:%02x.%x) MSI vector %d changed\n",
941 __func__, vdev->host.domain, vdev->host.bus,
942 vdev->host.slot, vdev->host.function, i);
943
944 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
945 vector->msg = msg;
946 }
947 }
948}
949
65501a74
AW		 950/*
951 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
952 */
a8170e5e 953static void vfio_bar_write(void *opaque, hwaddr addr,
65501a74
AW		 954 uint64_t data, unsigned size)
955{
956 VFIOBAR *bar = opaque;
957 union {
958 uint8_t byte;
959 uint16_t word;
960 uint32_t dword;
961 uint64_t qword;
962 } buf;
963
964 switch (size) {
965 case 1:
966 buf.byte = data;
967 break;
968 case 2:
969 buf.word = cpu_to_le16(data);
970 break;
971 case 4:
972 buf.dword = cpu_to_le32(data);
973 break;
974 default:
975 hw_error("vfio: unsupported write size, %d bytes\n", size);
976 break;
977 }
978
979 if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
312fd5f2 980 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
65501a74
AW		 981 __func__, addr, data, size);
982 }
983
82ca8912
AW		 984#ifdef DEBUG_VFIO
985 {
986 VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
987
988 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64
989 ", %d)\n", __func__, vdev->host.domain, vdev->host.bus,
990 vdev->host.slot, vdev->host.function, bar->nr, addr,
991 data, size);
992 }
993#endif
65501a74
AW		 994
995 /*
996 * A read or write to a BAR always signals an INTx EOI. This will
997 * do nothing if not pending (including not in INTx mode). We assume
998 * that a BAR access is in response to an interrupt and that BAR
999 * accesses will service the interrupt. Unfortunately, we don't know
1000 * which access will service the interrupt, so we're potentially
1001 * getting quite a few host interrupts per guest interrupt.
1002 */
3a4f2816 1003 vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
65501a74
AW		 1004}
1005
1006static uint64_t vfio_bar_read(void *opaque,
a8170e5e 1007 hwaddr addr, unsigned size)
65501a74
AW		 1008{
1009 VFIOBAR *bar = opaque;
1010 union {
1011 uint8_t byte;
1012 uint16_t word;
1013 uint32_t dword;
1014 uint64_t qword;
1015 } buf;
1016 uint64_t data = 0;
1017
1018 if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
312fd5f2 1019 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
65501a74
AW		 1020 __func__, addr, size);
1021 return (uint64_t)-1;
1022 }
1023
1024 switch (size) {
1025 case 1:
1026 data = buf.byte;
1027 break;
1028 case 2:
1029 data = le16_to_cpu(buf.word);
1030 break;
1031 case 4:
1032 data = le32_to_cpu(buf.dword);
1033 break;
1034 default:
1035 hw_error("vfio: unsupported read size, %d bytes\n", size);
1036 break;
1037 }
1038
82ca8912
AW		 1039#ifdef DEBUG_VFIO
1040 {
1041 VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
1042
1043 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
1044 ", %d) = 0x%"PRIx64"\n", __func__, vdev->host.domain,
1045 vdev->host.bus, vdev->host.slot, vdev->host.function,
1046 bar->nr, addr, size, data);
1047 }
1048#endif
65501a74
AW		 1049
1050 /* Same as write above */
3a4f2816 1051 vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
65501a74
AW		 1052
1053 return data;
1054}
1055
1056static const MemoryRegionOps vfio_bar_ops = {
1057 .read = vfio_bar_read,
1058 .write = vfio_bar_write,
1059 .endianness = DEVICE_LITTLE_ENDIAN,
1060};
1061
6f864e6e
AW		 1062static void vfio_pci_load_rom(VFIODevice *vdev)
1063{
1064 struct vfio_region_info reg_info = {
1065 .argsz = sizeof(reg_info),
1066 .index = VFIO_PCI_ROM_REGION_INDEX
1067 };
1068 uint64_t size;
1069 off_t off = 0;
1070 size_t bytes;
1071
1072 if (ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
1073 error_report("vfio: Error getting ROM info: %m");
1074 return;
1075 }
1076
1077 DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev->host.domain,
1078 vdev->host.bus, vdev->host.slot, vdev->host.function);
1079 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1080 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
1081 (unsigned long)reg_info.flags);
1082
1083 vdev->rom_size = size = reg_info.size;
1084 vdev->rom_offset = reg_info.offset;
1085
1086 if (!vdev->rom_size) {
1087 return;
1088 }
1089
1090 vdev->rom = g_malloc(size);
1091 memset(vdev->rom, 0xff, size);
1092
1093 while (size) {
1094 bytes = pread(vdev->fd, vdev->rom + off, size, vdev->rom_offset + off);
1095 if (bytes == 0) {
1096 break;
1097 } else if (bytes > 0) {
1098 off += bytes;
1099 size -= bytes;
1100 } else {
1101 if (errno == EINTR || errno == EAGAIN) {
1102 continue;
1103 }
1104 error_report("vfio: Error reading device ROM: %m");
1105 break;
1106 }
1107 }
1108}
1109
1110static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
1111{
1112 VFIODevice *vdev = opaque;
1113 uint64_t val = ((uint64_t)1 << (size * 8)) - 1;
1114
1115 /* Load the ROM lazily when the guest tries to read it */
1116 if (unlikely(!vdev->rom)) {
1117 vfio_pci_load_rom(vdev);
1118 }
1119
1120 memcpy(&val, vdev->rom + addr,
1121 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
1122
1123 DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx", 0x%x) = 0x%"PRIx64"\n",
1124 __func__, vdev->host.domain, vdev->host.bus, vdev->host.slot,
1125 vdev->host.function, addr, size, val);
1126
1127 return val;
1128}
1129
1130static const MemoryRegionOps vfio_rom_ops = {
1131 .read = vfio_rom_read,
1132 .endianness = DEVICE_LITTLE_ENDIAN,
1133};
1134
1135static void vfio_pci_size_rom(VFIODevice *vdev)
1136{
1137 uint32_t orig, size = (uint32_t)PCI_ROM_ADDRESS_MASK;
1138 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
1139 char name[32];
1140
1141 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
1142 return;
1143 }
1144
1145 /*
1146 * Use the same size ROM BAR as the physical device. The contents
1147 * will get filled in later when the guest tries to read it.
1148 */
1149 if (pread(vdev->fd, &orig, 4, offset) != 4 ||
1150 pwrite(vdev->fd, &size, 4, offset) != 4 ||
1151 pread(vdev->fd, &size, 4, offset) != 4 ||
1152 pwrite(vdev->fd, &orig, 4, offset) != 4) {
1153 error_report("%s(%04x:%02x:%02x.%x) failed: %m",
1154 __func__, vdev->host.domain, vdev->host.bus,
1155 vdev->host.slot, vdev->host.function);
1156 return;
1157 }
1158
1159 size = ~(size & PCI_ROM_ADDRESS_MASK) + 1;
1160
1161 if (!size) {
1162 return;
1163 }
1164
1165 DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev->host.domain,
1166 vdev->host.bus, vdev->host.slot, vdev->host.function, size);
1167
1168 snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
1169 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1170 vdev->host.function);
1171
1172 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
1173 &vfio_rom_ops, vdev, name, size);
1174
1175 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
1176 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
1177
1178 vdev->pdev.has_rom = true;
1179}
1180
f15689c7
AW		 1181static void vfio_vga_write(void *opaque, hwaddr addr,
1182 uint64_t data, unsigned size)
1183{
1184 VFIOVGARegion *region = opaque;
1185 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1186 union {
1187 uint8_t byte;
1188 uint16_t word;
1189 uint32_t dword;
1190 uint64_t qword;
1191 } buf;
1192 off_t offset = vga->fd_offset + region->offset + addr;
1193
1194 switch (size) {
1195 case 1:
1196 buf.byte = data;
1197 break;
1198 case 2:
1199 buf.word = cpu_to_le16(data);
1200 break;
1201 case 4:
1202 buf.dword = cpu_to_le32(data);
1203 break;
1204 default:
1205 hw_error("vfio: unsupported write size, %d bytes\n", size);
1206 break;
1207 }
1208
1209 if (pwrite(vga->fd, &buf, size, offset) != size) {
1210 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1211 __func__, region->offset + addr, data, size);
1212 }
1213
1214 DPRINTF("%s(0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
1215 __func__, region->offset + addr, data, size);
1216}
1217
1218static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1219{
1220 VFIOVGARegion *region = opaque;
1221 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1222 union {
1223 uint8_t byte;
1224 uint16_t word;
1225 uint32_t dword;
1226 uint64_t qword;
1227 } buf;
1228 uint64_t data = 0;
1229 off_t offset = vga->fd_offset + region->offset + addr;
1230
1231 if (pread(vga->fd, &buf, size, offset) != size) {
1232 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1233 __func__, region->offset + addr, size);
1234 return (uint64_t)-1;
1235 }
1236
1237 switch (size) {
1238 case 1:
1239 data = buf.byte;
1240 break;
1241 case 2:
1242 data = le16_to_cpu(buf.word);
1243 break;
1244 case 4:
1245 data = le32_to_cpu(buf.dword);
1246 break;
1247 default:
1248 hw_error("vfio: unsupported read size, %d bytes\n", size);
1249 break;
1250 }
1251
1252 DPRINTF("%s(0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
1253 __func__, region->offset + addr, size, data);
1254
1255 return data;
1256}
1257
1258static const MemoryRegionOps vfio_vga_ops = {
1259 .read = vfio_vga_read,
1260 .write = vfio_vga_write,
1261 .endianness = DEVICE_LITTLE_ENDIAN,
1262};
1263
7076eabc
AW		 1264/*
1265 * Device specific quirks
1266 */
1267
39360f0b
AW		 1268/* Is range1 fully contained within range2? */
1269static bool vfio_range_contained(uint64_t first1, uint64_t len1,
1270 uint64_t first2, uint64_t len2) {
1271 return (first1 >= first2 && first1 + len1 <= first2 + len2);
1272}
7076eabc 1273
39360f0b
AW		 1274static bool vfio_flags_enabled(uint8_t flags, uint8_t mask)
1275{
1276 return (mask && (flags & mask) == mask);
1277}
1278
1279static uint64_t vfio_generic_window_quirk_read(void *opaque,
1280 hwaddr addr, unsigned size)
7076eabc
AW		 1281{
1282 VFIOQuirk *quirk = opaque;
1283 VFIODevice *vdev = quirk->vdev;
39360f0b 1284 uint64_t data;
7076eabc 1285
39360f0b
AW		 1286 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
1287 ranges_overlap(addr, size,
1288 quirk->data.data_offset, quirk->data.data_size)) {
1289 hwaddr offset = addr - quirk->data.data_offset;
1290
1291 if (!vfio_range_contained(addr, size, quirk->data.data_offset,
1292 quirk->data.data_size)) {
1293 hw_error("%s: window data read not fully contained: %s\n",
1294 __func__, memory_region_name(&quirk->mem));
1295 }
1296
1297 data = vfio_pci_read_config(&vdev->pdev,
1298 quirk->data.address_val + offset, size);
1299
1300 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
1301 PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
1302 vdev->host.bus, vdev->host.slot, vdev->host.function,
1303 quirk->data.bar, addr, size, data);
1304 } else {
1305 data = vfio_bar_read(&vdev->bars[quirk->data.bar],
1306 addr + quirk->data.base_offset, size);
7076eabc
AW		 1307 }
1308
1309 return data;
1310}
1311
39360f0b
AW		 1312static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr,
1313 uint64_t data, unsigned size)
7076eabc 1314{
39360f0b
AW		 1315 VFIOQuirk *quirk = opaque;
1316 VFIODevice *vdev = quirk->vdev;
7076eabc 1317
39360f0b
AW		 1318 if (ranges_overlap(addr, size,
1319 quirk->data.address_offset, quirk->data.address_size)) {
7076eabc 1320
39360f0b
AW		 1321 if (addr != quirk->data.address_offset) {
1322 hw_error("%s: offset write into address window: %s\n",
1323 __func__, memory_region_name(&quirk->mem));
1324 }
1325
1326 if ((data & ~quirk->data.address_mask) == quirk->data.address_match) {
1327 quirk->data.flags |= quirk->data.write_flags |
1328 quirk->data.read_flags;
1329 quirk->data.address_val = data & quirk->data.address_mask;
1330 } else {
1331 quirk->data.flags &= ~(quirk->data.write_flags |
1332 quirk->data.read_flags);
1333 }
7076eabc
AW		 1334 }
1335
39360f0b
AW		 1336 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
1337 ranges_overlap(addr, size,
1338 quirk->data.data_offset, quirk->data.data_size)) {
1339 hwaddr offset = addr - quirk->data.data_offset;
7076eabc 1340
39360f0b
AW		 1341 if (!vfio_range_contained(addr, size, quirk->data.data_offset,
1342 quirk->data.data_size)) {
1343 hw_error("%s: window data write not fully contained: %s\n",
1344 __func__, memory_region_name(&quirk->mem));
1345 }
7076eabc 1346
39360f0b
AW		 1347 vfio_pci_write_config(&vdev->pdev,
1348 quirk->data.address_val + offset, data, size);
1349 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
1350 PRIx64", %d)\n", memory_region_name(&quirk->mem),
1351 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1352 vdev->host.function, quirk->data.bar, addr, data, size);
1353 return;
1354 }
7076eabc 1355
39360f0b
AW		 1356 vfio_bar_write(&vdev->bars[quirk->data.bar],
1357 addr + quirk->data.base_offset, data, size);
7076eabc
AW		 1358}
1359
39360f0b
AW		 1360static const MemoryRegionOps vfio_generic_window_quirk = {
1361 .read = vfio_generic_window_quirk_read,
1362 .write = vfio_generic_window_quirk_write,
1363 .endianness = DEVICE_LITTLE_ENDIAN,
1364};
1365
1366static uint64_t vfio_generic_quirk_read(void *opaque,
1367 hwaddr addr, unsigned size)
7076eabc
AW		 1368{
1369 VFIOQuirk *quirk = opaque;
1370 VFIODevice *vdev = quirk->vdev;
39360f0b
AW		 1371 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
1372 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
1373 uint64_t data;
7076eabc 1374
39360f0b
AW		 1375 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
1376 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
1377 if (!vfio_range_contained(addr, size, offset,
1378 quirk->data.address_mask + 1)) {
1379 hw_error("%s: read not fully contained: %s\n",
1380 __func__, memory_region_name(&quirk->mem));
1381 }
7076eabc 1382
39360f0b
AW		 1383 data = vfio_pci_read_config(&vdev->pdev, addr - offset, size);
1384
1385 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
1386 PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
1387 vdev->host.bus, vdev->host.slot, vdev->host.function,
1388 quirk->data.bar, addr + base, size, data);
1389 } else {
1390 data = vfio_bar_read(&vdev->bars[quirk->data.bar], addr + base, size);
1391 }
7076eabc
AW		 1392
1393 return data;
1394}
1395
39360f0b
AW		 1396static void vfio_generic_quirk_write(void *opaque, hwaddr addr,
1397 uint64_t data, unsigned size)
7076eabc
AW		 1398{
1399 VFIOQuirk *quirk = opaque;
1400 VFIODevice *vdev = quirk->vdev;
39360f0b
AW		 1401 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
1402 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
1403
1404 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
1405 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
1406 if (!vfio_range_contained(addr, size, offset,
1407 quirk->data.address_mask + 1)) {
1408 hw_error("%s: write not fully contained: %s\n",
1409 __func__, memory_region_name(&quirk->mem));
1410 }
7076eabc 1411
39360f0b 1412 vfio_pci_write_config(&vdev->pdev, addr - offset, data, size);
7076eabc 1413
39360f0b
AW		 1414 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
1415 PRIx64", %d)\n", memory_region_name(&quirk->mem),
1416 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1417 vdev->host.function, quirk->data.bar, addr + base, data, size);
1418 } else {
1419 vfio_bar_write(&vdev->bars[quirk->data.bar], addr + base, data, size);
1420 }
7076eabc
AW		 1421}
1422
39360f0b
AW		 1423static const MemoryRegionOps vfio_generic_quirk = {
1424 .read = vfio_generic_quirk_read,
1425 .write = vfio_generic_quirk_write,
7076eabc
AW		 1426 .endianness = DEVICE_LITTLE_ENDIAN,
1427};
1428
39360f0b
AW		 1429#define PCI_VENDOR_ID_ATI 0x1002
1430
1431/*
1432 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
1433 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always
1434 * BAR4 (older cards like the X550 used BAR1, but we don't care to support
1435 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the
1436 * I/O port BAR address. Originally this was coded to return the virtual BAR
1437 * address only if the physical register read returns the actual BAR address,
1438 * but users have reported greater success if we return the virtual address
1439 * unconditionally.
1440 */
1441static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
1442 hwaddr addr, unsigned size)
1443{
1444 VFIOQuirk *quirk = opaque;
1445 VFIODevice *vdev = quirk->vdev;
1446 uint64_t data = vfio_pci_read_config(&vdev->pdev,
1447 PCI_BASE_ADDRESS_0 + (4 * 4) + 1,
1448 size);
1449 DPRINTF("%s(0x3c3, 1) = 0x%"PRIx64"\n", __func__, data);
1450
1451 return data;
1452}
1453
1454static const MemoryRegionOps vfio_ati_3c3_quirk = {
1455 .read = vfio_ati_3c3_quirk_read,
1456 .endianness = DEVICE_LITTLE_ENDIAN,
1457};
1458
1459static void vfio_vga_probe_ati_3c3_quirk(VFIODevice *vdev)
7076eabc
AW		 1460{
1461 PCIDevice *pdev = &vdev->pdev;
7076eabc
AW		 1462 VFIOQuirk *quirk;
1463
39360f0b 1464 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
7076eabc
AW		 1465 return;
1466 }
1467
39360f0b
AW		 1468 /*
1469 * As long as the BAR is >= 256 bytes it will be aligned such that the
1470 * lower byte is always zero. Filter out anything else, if it exists.
1471 */
1472 if (!vdev->bars[4].ioport || vdev->bars[4].size < 256) {
7076eabc
AW		 1473 return;
1474 }
1475
1476 quirk = g_malloc0(sizeof(*quirk));
1477 quirk->vdev = vdev;
1478
39360f0b
AW		 1479 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk,
1480 "vfio-ati-3c3-quirk", 1);
1481 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
1482 3 /* offset 3 bytes from 0x3c0 */, &quirk->mem);
7076eabc 1483
39360f0b
AW		 1484 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
1485 quirk, next);
7076eabc 1486
39360f0b 1487 DPRINTF("Enabled ATI/AMD quirk 0x3c3 BAR4for device %04x:%02x:%02x.%x\n",
7076eabc
AW		 1488 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1489 vdev->host.function);
1490}
1491
1492/*
39360f0b
AW		 1493 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
1494 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access
1495 * the MMIO space directly, but a window to this space is provided through
1496 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the
1497 * data register. When the address is programmed to a range of 0x4000-0x4fff
1498 * PCI configuration space is available. Experimentation seems to indicate
1499 * that only read-only access is provided, but we drop writes when the window
1500 * is enabled to config space nonetheless.
7076eabc 1501 */
39360f0b 1502static void vfio_probe_ati_bar4_window_quirk(VFIODevice *vdev, int nr)
7076eabc 1503{
7076eabc 1504 PCIDevice *pdev = &vdev->pdev;
39360f0b 1505 VFIOQuirk *quirk;
7076eabc 1506
39360f0b
AW		 1507 if (!vdev->has_vga || nr != 4 ||
1508 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
1509 return;
7076eabc
AW		 1510 }
1511
39360f0b
AW		 1512 quirk = g_malloc0(sizeof(*quirk));
1513 quirk->vdev = vdev;
1514 quirk->data.address_size = 4;
1515 quirk->data.data_offset = 4;
1516 quirk->data.data_size = 4;
1517 quirk->data.address_match = 0x4000;
1518 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1519 quirk->data.bar = nr;
1520 quirk->data.read_flags = quirk->data.write_flags = 1;
1521
1522 memory_region_init_io(&quirk->mem, OBJECT(vdev),
1523 &vfio_generic_window_quirk, quirk,
1524 "vfio-ati-bar4-window-quirk", 8);
1525 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1526 quirk->data.base_offset, &quirk->mem, 1);
7076eabc 1527
39360f0b 1528 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
7076eabc 1529
39360f0b
AW		 1530 DPRINTF("Enabled ATI/AMD BAR4 window quirk for device %04x:%02x:%02x.%x\n",
1531 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1532 vdev->host.function);
7076eabc
AW		 1533}
1534
39360f0b
AW		 1535/*
1536 * Trap the BAR2 MMIO window to config space as well.
1537 */
1538static void vfio_probe_ati_bar2_4000_quirk(VFIODevice *vdev, int nr)
7076eabc
AW		 1539{
1540 PCIDevice *pdev = &vdev->pdev;
7076eabc
AW		 1541 VFIOQuirk *quirk;
1542
39360f0b
AW		 1543 /* Only enable on newer devices where BAR2 is 64bit */
1544 if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 ||
7076eabc
AW		 1545 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
1546 return;
1547 }
1548
7076eabc
AW		 1549 quirk = g_malloc0(sizeof(*quirk));
1550 quirk->vdev = vdev;
39360f0b
AW		 1551 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1552 quirk->data.address_match = 0x4000;
1553 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1554 quirk->data.bar = nr;
1555
1556 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
1557 "vfio-ati-bar2-4000-quirk",
1558 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
1559 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1560 quirk->data.address_match & TARGET_PAGE_MASK,
1561 &quirk->mem, 1);
7076eabc
AW		 1562
1563 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1564
39360f0b 1565 DPRINTF("Enabled ATI/AMD BAR2 0x4000 quirk for device %04x:%02x:%02x.%x\n",
7076eabc
AW		 1566 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1567 vdev->host.function);
1568}
1569
39360f0b
AW		 1570/*
1571 * Older ATI/AMD cards like the X550 have a similar window to that above.
1572 * I/O port BAR1 provides a window to a mirror of PCI config space located
1573 * in BAR2 at offset 0xf00. We don't care to support such older cards, but
1574 * note it for future reference.
1575 */
1576
7076eabc
AW		 1577#define PCI_VENDOR_ID_NVIDIA 0x10de
1578
1579/*
1580 * Nvidia has several different methods to get to config space, the
1581 * nouveu project has several of these documented here:
1582 * https://github.com/pathscale/envytools/tree/master/hwdocs
1583 *
1584 * The first quirk is actually not documented in envytools and is found
1585 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an
1586 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access
1587 * the mirror of PCI config space found at BAR0 offset 0x1800. The access
1588 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is
1589 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738
1590 * is written for a write to 0x3d4. The BAR0 offset is then accessible
1591 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards
1592 * that use the I/O port BAR5 window but it doesn't hurt to leave it.
1593 */
1594enum {
39360f0b 1595 NV_3D0_NONE = 0,
7076eabc
AW		 1596 NV_3D0_SELECT,
1597 NV_3D0_WINDOW,
1598 NV_3D0_READ,
1599 NV_3D0_WRITE,
1600};
1601
1602static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
1603 hwaddr addr, unsigned size)
1604{
1605 VFIOQuirk *quirk = opaque;
1606 VFIODevice *vdev = quirk->vdev;
1607 PCIDevice *pdev = &vdev->pdev;
1608 uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
39360f0b 1609 addr + quirk->data.base_offset, size);
7076eabc 1610
39360f0b
AW		 1611 if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) {
1612 data = vfio_pci_read_config(pdev, quirk->data.address_val, size);
7076eabc
AW		 1613 DPRINTF("%s(0x3d0, %d) = 0x%"PRIx64"\n", __func__, size, data);
1614 }
1615
39360f0b 1616 quirk->data.flags = NV_3D0_NONE;
7076eabc
AW		 1617
1618 return data;
1619}
1620
1621static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
1622 uint64_t data, unsigned size)
1623{
1624 VFIOQuirk *quirk = opaque;
1625 VFIODevice *vdev = quirk->vdev;
1626 PCIDevice *pdev = &vdev->pdev;
1627
39360f0b 1628 switch (quirk->data.flags) {
7076eabc 1629 case NV_3D0_NONE:
39360f0b
AW		 1630 if (addr == quirk->data.address_offset && data == 0x338) {
1631 quirk->data.flags = NV_3D0_SELECT;
7076eabc
AW		 1632 }
1633 break;
1634 case NV_3D0_SELECT:
39360f0b
AW		 1635 quirk->data.flags = NV_3D0_NONE;
1636 if (addr == quirk->data.data_offset &&
1637 (data & ~quirk->data.address_mask) == quirk->data.address_match) {
1638 quirk->data.flags = NV_3D0_WINDOW;
1639 quirk->data.address_val = data & quirk->data.address_mask;
7076eabc
AW		 1640 }
1641 break;
1642 case NV_3D0_WINDOW:
39360f0b
AW		 1643 quirk->data.flags = NV_3D0_NONE;
1644 if (addr == quirk->data.address_offset) {
7076eabc 1645 if (data == 0x538) {
39360f0b 1646 quirk->data.flags = NV_3D0_READ;
7076eabc 1647 } else if (data == 0x738) {
39360f0b 1648 quirk->data.flags = NV_3D0_WRITE;
7076eabc
AW		 1649 }
1650 }
1651 break;
1652 case NV_3D0_WRITE:
39360f0b
AW		 1653 quirk->data.flags = NV_3D0_NONE;
1654 if (addr == quirk->data.data_offset) {
1655 vfio_pci_write_config(pdev, quirk->data.address_val, data, size);
7076eabc
AW		 1656 DPRINTF("%s(0x3d0, 0x%"PRIx64", %d)\n", __func__, data, size);
1657 return;
1658 }
1659 break;
7076eabc
AW		 1660 }
1661
1662 vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
39360f0b 1663 addr + quirk->data.base_offset, data, size);
7076eabc
AW		 1664}
1665
1666static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
1667 .read = vfio_nvidia_3d0_quirk_read,
1668 .write = vfio_nvidia_3d0_quirk_write,
1669 .endianness = DEVICE_LITTLE_ENDIAN,
1670};
1671
1672static void vfio_vga_probe_nvidia_3d0_quirk(VFIODevice *vdev)
1673{
1674 PCIDevice *pdev = &vdev->pdev;
1675 VFIOQuirk *quirk;
1676
1677 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA ||
1678 !vdev->bars[1].size) {
1679 return;
1680 }
1681
1682 quirk = g_malloc0(sizeof(*quirk));
1683 quirk->vdev = vdev;
39360f0b
AW		 1684 quirk->data.base_offset = 0x10;
1685 quirk->data.address_offset = 4;
1686 quirk->data.address_size = 2;
1687 quirk->data.address_match = 0x1800;
1688 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
1689 quirk->data.data_offset = 0;
1690 quirk->data.data_size = 4;
1691
1692 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk,
1693 quirk, "vfio-nvidia-3d0-quirk", 6);
7076eabc 1694 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
39360f0b 1695 quirk->data.base_offset, &quirk->mem);
7076eabc
AW		 1696
1697 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
1698 quirk, next);
1699
1700 DPRINTF("Enabled NVIDIA VGA 0x3d0 quirk for device %04x:%02x:%02x.%x\n",
1701 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1702 vdev->host.function);
1703}
1704
1705/*
1706 * The second quirk is documented in envytools. The I/O port BAR5 is just
1707 * a set of address/data ports to the MMIO BARs. The BAR we care about is
1708 * again BAR0. This backdoor is apparently a bit newer than the one above
1709 * so we need to not only trap 256 bytes @0x1800, but all of PCI config
1710 * space, including extended space is available at the 4k @0x88000.
1711 */
1712enum {
1713 NV_BAR5_ADDRESS = 0x1,
1714 NV_BAR5_ENABLE = 0x2,
1715 NV_BAR5_MASTER = 0x4,
1716 NV_BAR5_VALID = 0x7,
1717};
1718
7076eabc
AW		 1719static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr,
1720 uint64_t data, unsigned size)
1721{
1722 VFIOQuirk *quirk = opaque;
7076eabc 1723
7076eabc
AW		 1724 switch (addr) {
1725 case 0x0:
1726 if (data & 0x1) {
39360f0b 1727 quirk->data.flags |= NV_BAR5_MASTER;
7076eabc 1728 } else {
39360f0b 1729 quirk->data.flags &= ~NV_BAR5_MASTER;
7076eabc
AW		 1730 }
1731 break;
1732 case 0x4:
1733 if (data & 0x1) {
39360f0b 1734 quirk->data.flags |= NV_BAR5_ENABLE;
7076eabc 1735 } else {
39360f0b 1736 quirk->data.flags &= ~NV_BAR5_ENABLE;
7076eabc
AW		 1737 }
1738 break;
1739 case 0x8:
39360f0b 1740 if (quirk->data.flags & NV_BAR5_MASTER) {
7076eabc 1741 if ((data & ~0xfff) == 0x88000) {
39360f0b
AW		 1742 quirk->data.flags |= NV_BAR5_ADDRESS;
1743 quirk->data.address_val = data & 0xfff;
7076eabc 1744 } else if ((data & ~0xff) == 0x1800) {
39360f0b
AW		 1745 quirk->data.flags |= NV_BAR5_ADDRESS;
1746 quirk->data.address_val = data & 0xff;
7076eabc 1747 } else {
39360f0b 1748 quirk->data.flags &= ~NV_BAR5_ADDRESS;
7076eabc
AW		 1749 }
1750 }
1751 break;
7076eabc
AW		 1752 }
1753
39360f0b 1754 vfio_generic_window_quirk_write(opaque, addr, data, size);
7076eabc
AW		 1755}
1756
1757static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = {
39360f0b 1758 .read = vfio_generic_window_quirk_read,
7076eabc
AW		 1759 .write = vfio_nvidia_bar5_window_quirk_write,
1760 .valid.min_access_size = 4,
1761 .endianness = DEVICE_LITTLE_ENDIAN,
1762};
1763
1764static void vfio_probe_nvidia_bar5_window_quirk(VFIODevice *vdev, int nr)
1765{
1766 PCIDevice *pdev = &vdev->pdev;
1767 VFIOQuirk *quirk;
1768
1769 if (!vdev->has_vga || nr != 5 ||
1770 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1771 return;
1772 }
1773
1774 quirk = g_malloc0(sizeof(*quirk));
1775 quirk->vdev = vdev;
39360f0b
AW		 1776 quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID;
1777 quirk->data.address_offset = 0x8;
1778 quirk->data.address_size = 0; /* actually 4, but avoids generic code */
1779 quirk->data.data_offset = 0xc;
1780 quirk->data.data_size = 4;
1781 quirk->data.bar = nr;
1782
1783 memory_region_init_io(&quirk->mem, OBJECT(vdev),
1784 &vfio_nvidia_bar5_window_quirk, quirk,
7076eabc
AW		 1785 "vfio-nvidia-bar5-window-quirk", 16);
1786 memory_region_add_subregion_overlap(&vdev->bars[nr].mem, 0, &quirk->mem, 1);
1787
1788 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1789
1790 DPRINTF("Enabled NVIDIA BAR5 window quirk for device %04x:%02x:%02x.%x\n",
1791 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1792 vdev->host.function);
1793}
1794
1795/*
1796 * Finally, BAR0 itself. We want to redirect any accesses to either
1797 * 0x1800 or 0x88000 through the PCI config space access functions.
1798 *
1799 * NB - quirk at a page granularity or else they don't seem to work when
1800 * BARs are mmap'd
1801 *
1802 * Here's offset 0x88000...
1803 */
7076eabc
AW		 1804static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice *vdev, int nr)
1805{
1806 PCIDevice *pdev = &vdev->pdev;
1807 VFIOQuirk *quirk;
1808
1809 if (!vdev->has_vga || nr != 0 ||
1810 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1811 return;
1812 }
1813
1814 quirk = g_malloc0(sizeof(*quirk));
1815 quirk->vdev = vdev;
39360f0b
AW		 1816 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1817 quirk->data.address_match = 0x88000;
1818 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1819 quirk->data.bar = nr;
1820
1821 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk,
1822 quirk, "vfio-nvidia-bar0-88000-quirk",
1823 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
7076eabc 1824 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
39360f0b
AW		 1825 quirk->data.address_match & TARGET_PAGE_MASK,
1826 &quirk->mem, 1);
7076eabc
AW		 1827
1828 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1829
1830 DPRINTF("Enabled NVIDIA BAR0 0x88000 quirk for device %04x:%02x:%02x.%x\n",
1831 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1832 vdev->host.function);
1833}
1834
1835/*
1836 * And here's the same for BAR0 offset 0x1800...
1837 */
7076eabc
AW		 1838static void vfio_probe_nvidia_bar0_1800_quirk(VFIODevice *vdev, int nr)
1839{
1840 PCIDevice *pdev = &vdev->pdev;
1841 VFIOQuirk *quirk;
1842
1843 if (!vdev->has_vga || nr != 0 ||
1844 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1845 return;
1846 }
1847
1848 /* Log the chipset ID */
1849 DPRINTF("Nvidia NV%02x\n",
1850 (unsigned int)(vfio_bar_read(&vdev->bars[0], 0, 4) >> 20) & 0xff);
1851
1852 quirk = g_malloc0(sizeof(*quirk));
1853 quirk->vdev = vdev;
39360f0b
AW		 1854 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1855 quirk->data.address_match = 0x1800;
1856 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
1857 quirk->data.bar = nr;
7076eabc 1858
39360f0b 1859 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
7076eabc 1860 "vfio-nvidia-bar0-1800-quirk",
39360f0b 1861 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
7076eabc 1862 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
39360f0b
AW		 1863 quirk->data.address_match & TARGET_PAGE_MASK,
1864 &quirk->mem, 1);
7076eabc
AW		 1865
1866 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1867
1868 DPRINTF("Enabled NVIDIA BAR0 0x1800 quirk for device %04x:%02x:%02x.%x\n",
1869 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1870 vdev->host.function);
1871}
1872
1873/*
1874 * TODO - Some Nvidia devices provide config access to their companion HDA
1875 * device and even to their parent bridge via these config space mirrors.
1876 * Add quirks for those regions.
1877 */
1878
1879/*
1880 * Common quirk probe entry points.
1881 */
1882static void vfio_vga_quirk_setup(VFIODevice *vdev)
1883{
1884 vfio_vga_probe_ati_3c3_quirk(vdev);
1885 vfio_vga_probe_nvidia_3d0_quirk(vdev);
1886}
1887
1888static void vfio_vga_quirk_teardown(VFIODevice *vdev)
1889{
1890 int i;
1891
1892 for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
1893 while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
1894 VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
1895 memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
1896 QLIST_REMOVE(quirk, next);
1897 g_free(quirk);
1898 }
1899 }
1900}
1901
1902static void vfio_bar_quirk_setup(VFIODevice *vdev, int nr)
1903{
39360f0b
AW		 1904 vfio_probe_ati_bar4_window_quirk(vdev, nr);
1905 vfio_probe_ati_bar2_4000_quirk(vdev, nr);
7076eabc
AW		 1906 vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
1907 vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
1908 vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
1909}
1910
1911static void vfio_bar_quirk_teardown(VFIODevice *vdev, int nr)
1912{
1913 VFIOBAR *bar = &vdev->bars[nr];
1914
1915 while (!QLIST_EMPTY(&bar->quirks)) {
1916 VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
1917 memory_region_del_subregion(&bar->mem, &quirk->mem);
1918 QLIST_REMOVE(quirk, next);
1919 g_free(quirk);
1920 }
1921}
1922
65501a74
AW		 1923/*
1924 * PCI config space
1925 */
1926static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1927{
1928 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
4b5d5e87 1929 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
65501a74 1930
4b5d5e87
AW		 1931 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1932 emu_bits = le32_to_cpu(emu_bits);
65501a74 1933
4b5d5e87
AW		 1934 if (emu_bits) {
1935 emu_val = pci_default_read_config(pdev, addr, len);
1936 }
1937
1938 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1939 ssize_t ret;
1940
1941 ret = pread(vdev->fd, &phys_val, len, vdev->config_offset + addr);
1942 if (ret != len) {
312fd5f2 1943 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
65501a74
AW		 1944 __func__, vdev->host.domain, vdev->host.bus,
1945 vdev->host.slot, vdev->host.function, addr, len);
1946 return -errno;
1947 }
4b5d5e87 1948 phys_val = le32_to_cpu(phys_val);
65501a74
AW		 1949 }
1950
4b5d5e87 1951 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
65501a74
AW		 1952
1953 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__,
1954 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1955 vdev->host.function, addr, len, val);
1956
1957 return val;
1958}
1959
1960static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
1961 uint32_t val, int len)
1962{
1963 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
1964 uint32_t val_le = cpu_to_le32(val);
1965
1966 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__,
1967 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1968 vdev->host.function, addr, val, len);
1969
1970 /* Write everything to VFIO, let it filter out what we can't write */
1971 if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
312fd5f2 1972 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
65501a74
AW		 1973 __func__, vdev->host.domain, vdev->host.bus,
1974 vdev->host.slot, vdev->host.function, addr, val, len);
1975 }
1976
65501a74
AW		 1977 /* MSI/MSI-X Enabling/Disabling */
1978 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1979 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1980 int is_enabled, was_enabled = msi_enabled(pdev);
1981
1982 pci_default_write_config(pdev, addr, val, len);
1983
1984 is_enabled = msi_enabled(pdev);
1985
c7679d45
AW		 1986 if (!was_enabled) {
1987 if (is_enabled) {
1988 vfio_enable_msi(vdev);
1989 }
1990 } else {
1991 if (!is_enabled) {
1992 vfio_disable_msi(vdev);
1993 } else {
1994 vfio_update_msi(vdev);
1995 }
65501a74 1996 }
4b5d5e87 1997 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
65501a74
AW		 1998 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1999 int is_enabled, was_enabled = msix_enabled(pdev);
2000
2001 pci_default_write_config(pdev, addr, val, len);
2002
2003 is_enabled = msix_enabled(pdev);
2004
2005 if (!was_enabled && is_enabled) {
fd704adc 2006 vfio_enable_msix(vdev);
65501a74 2007 } else if (was_enabled && !is_enabled) {
fd704adc 2008 vfio_disable_msix(vdev);
65501a74 2009 }
4b5d5e87
AW		 2010 } else {
2011 /* Write everything to QEMU to keep emulated bits correct */
2012 pci_default_write_config(pdev, addr, val, len);
65501a74
AW		 2013 }
2014}
2015
2016/*
2017 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
2018 */
af6bc27e 2019static int vfio_dma_unmap(VFIOContainer *container,
a8170e5e 2020 hwaddr iova, ram_addr_t size)
af6bc27e
AW		 2021{
2022 struct vfio_iommu_type1_dma_unmap unmap = {
2023 .argsz = sizeof(unmap),
2024 .flags = 0,
2025 .iova = iova,
2026 .size = size,
2027 };
2028
2029 if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
2030 DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno);
2031 return -errno;
2032 }
2033
2034 return 0;
2035}
2036
a8170e5e 2037static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
65501a74
AW		 2038 ram_addr_t size, void *vaddr, bool readonly)
2039{
2040 struct vfio_iommu_type1_dma_map map = {
2041 .argsz = sizeof(map),
2042 .flags = VFIO_DMA_MAP_FLAG_READ,
5976cdd5 2043 .vaddr = (__u64)(uintptr_t)vaddr,
65501a74
AW		 2044 .iova = iova,
2045 .size = size,
2046 };
2047
2048 if (!readonly) {
2049 map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
2050 }
2051
12af1344
AW		 2052 /*
2053 * Try the mapping, if it fails with EBUSY, unmap the region and try
2054 * again. This shouldn't be necessary, but we sometimes see it in
2055 * the the VGA ROM space.
2056 */
2057 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
2058 (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
2059 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
2060 return 0;
65501a74
AW		 2061 }
2062
12af1344
AW		 2063 DPRINTF("VFIO_MAP_DMA: %d\n", -errno);
2064 return -errno;
65501a74
AW		 2065}
2066
65501a74
AW		 2067static bool vfio_listener_skipped_section(MemoryRegionSection *section)
2068{
2069 return !memory_region_is_ram(section->mr);
2070}
2071
2072static void vfio_listener_region_add(MemoryListener *listener,
2073 MemoryRegionSection *section)
2074{
2075 VFIOContainer *container = container_of(listener, VFIOContainer,
2076 iommu_data.listener);
a8170e5e 2077 hwaddr iova, end;
65501a74
AW		 2078 void *vaddr;
2079 int ret;
2080
06d985f5
AK		 2081 assert(!memory_region_is_iommu(section->mr));
2082
65501a74 2083 if (vfio_listener_skipped_section(section)) {
82ca8912 2084 DPRINTF("SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
65501a74
AW		 2085 section->offset_within_address_space,
2086 section->offset_within_address_space + section->size - 1);
2087 return;
2088 }
2089
2090 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
2091 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
312fd5f2 2092 error_report("%s received unaligned region", __func__);
65501a74
AW		 2093 return;
2094 }
2095
2096 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
052e87b0 2097 end = (section->offset_within_address_space + int128_get64(section->size)) &
65501a74
AW		 2098 TARGET_PAGE_MASK;
2099
2100 if (iova >= end) {
2101 return;
2102 }
2103
2104 vaddr = memory_region_get_ram_ptr(section->mr) +
2105 section->offset_within_region +
2106 (iova - section->offset_within_address_space);
2107
82ca8912 2108 DPRINTF("region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
65501a74
AW		 2109 iova, end - 1, vaddr);
2110
dfde4e6e 2111 memory_region_ref(section->mr);
65501a74
AW		 2112 ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
2113 if (ret) {
a8170e5e 2114 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
312fd5f2 2115 "0x%"HWADDR_PRIx", %p) = %d (%m)",
65501a74
AW		 2116 container, iova, end - iova, vaddr, ret);
2117 }
2118}
2119
2120static void vfio_listener_region_del(MemoryListener *listener,
2121 MemoryRegionSection *section)
2122{
2123 VFIOContainer *container = container_of(listener, VFIOContainer,
2124 iommu_data.listener);
a8170e5e 2125 hwaddr iova, end;
65501a74
AW		 2126 int ret;
2127
2128 if (vfio_listener_skipped_section(section)) {
82ca8912 2129 DPRINTF("SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
65501a74
AW		 2130 section->offset_within_address_space,
2131 section->offset_within_address_space + section->size - 1);
2132 return;
2133 }
2134
2135 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
2136 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
312fd5f2 2137 error_report("%s received unaligned region", __func__);
65501a74
AW		 2138 return;
2139 }
2140
2141 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
052e87b0 2142 end = (section->offset_within_address_space + int128_get64(section->size)) &
65501a74
AW		 2143 TARGET_PAGE_MASK;
2144
2145 if (iova >= end) {
2146 return;
2147 }
2148
82ca8912 2149 DPRINTF("region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
65501a74
AW		 2150 iova, end - 1);
2151
2152 ret = vfio_dma_unmap(container, iova, end - iova);
dfde4e6e 2153 memory_region_unref(section->mr);
65501a74 2154 if (ret) {
a8170e5e 2155 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
312fd5f2 2156 "0x%"HWADDR_PRIx") = %d (%m)",
65501a74
AW		 2157 container, iova, end - iova, ret);
2158 }
2159}
2160
2161static MemoryListener vfio_memory_listener = {
65501a74
AW		 2162 .region_add = vfio_listener_region_add,
2163 .region_del = vfio_listener_region_del,
65501a74
AW		 2164};
2165
2166static void vfio_listener_release(VFIOContainer *container)
2167{
2168 memory_listener_unregister(&container->iommu_data.listener);
2169}
2170
2171/*
2172 * Interrupt setup
2173 */
2174static void vfio_disable_interrupts(VFIODevice *vdev)
2175{
2176 switch (vdev->interrupt) {
2177 case VFIO_INT_INTx:
2178 vfio_disable_intx(vdev);
2179 break;
2180 case VFIO_INT_MSI:
fd704adc 2181 vfio_disable_msi(vdev);
65501a74
AW		 2182 break;
2183 case VFIO_INT_MSIX:
fd704adc 2184 vfio_disable_msix(vdev);
65501a74
AW		 2185 break;
2186 }
2187}
2188
2189static int vfio_setup_msi(VFIODevice *vdev, int pos)
2190{
2191 uint16_t ctrl;
2192 bool msi_64bit, msi_maskbit;
2193 int ret, entries;
2194
65501a74
AW		 2195 if (pread(vdev->fd, &ctrl, sizeof(ctrl),
2196 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
2197 return -errno;
2198 }
2199 ctrl = le16_to_cpu(ctrl);
2200
2201 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
2202 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
2203 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
2204
2205 DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev->host.domain,
2206 vdev->host.bus, vdev->host.slot, vdev->host.function, pos);
2207
2208 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
2209 if (ret < 0) {
e43b9a5a
AW		 2210 if (ret == -ENOTSUP) {
2211 return 0;
2212 }
312fd5f2 2213 error_report("vfio: msi_init failed");
65501a74
AW		 2214 return ret;
2215 }
2216 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
2217
2218 return 0;
2219}
2220
2221/*
2222 * We don't have any control over how pci_add_capability() inserts
2223 * capabilities into the chain. In order to setup MSI-X we need a
2224 * MemoryRegion for the BAR. In order to setup the BAR and not
2225 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
2226 * need to first look for where the MSI-X table lives. So we
2227 * unfortunately split MSI-X setup across two functions.
2228 */
2229static int vfio_early_setup_msix(VFIODevice *vdev)
2230{
2231 uint8_t pos;
2232 uint16_t ctrl;
2233 uint32_t table, pba;
2234
2235 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
2236 if (!pos) {
2237 return 0;
2238 }
2239
2240 if (pread(vdev->fd, &ctrl, sizeof(ctrl),
2241 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
2242 return -errno;
2243 }
2244
2245 if (pread(vdev->fd, &table, sizeof(table),
2246 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
2247 return -errno;
2248 }
2249
2250 if (pread(vdev->fd, &pba, sizeof(pba),
2251 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
2252 return -errno;
2253 }
2254
2255 ctrl = le16_to_cpu(ctrl);
2256 table = le32_to_cpu(table);
2257 pba = le32_to_cpu(pba);
2258
2259 vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
2260 vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
2261 vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
2262 vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
2263 vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
2264 vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
2265
2266 DPRINTF("%04x:%02x:%02x.%x "
2267 "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
2268 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2269 vdev->host.function, pos, vdev->msix->table_bar,
2270 vdev->msix->table_offset, vdev->msix->entries);
2271
2272 return 0;
2273}
2274
2275static int vfio_setup_msix(VFIODevice *vdev, int pos)
2276{
2277 int ret;
2278
65501a74
AW		 2279 ret = msix_init(&vdev->pdev, vdev->msix->entries,
2280 &vdev->bars[vdev->msix->table_bar].mem,
2281 vdev->msix->table_bar, vdev->msix->table_offset,
2282 &vdev->bars[vdev->msix->pba_bar].mem,
2283 vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
2284 if (ret < 0) {
e43b9a5a
AW		 2285 if (ret == -ENOTSUP) {
2286 return 0;
2287 }
312fd5f2 2288 error_report("vfio: msix_init failed");
65501a74
AW		 2289 return ret;
2290 }
2291
65501a74
AW		 2292 return 0;
2293}
2294
2295static void vfio_teardown_msi(VFIODevice *vdev)
2296{
2297 msi_uninit(&vdev->pdev);
2298
2299 if (vdev->msix) {
65501a74
AW		 2300 msix_uninit(&vdev->pdev, &vdev->bars[vdev->msix->table_bar].mem,
2301 &vdev->bars[vdev->msix->pba_bar].mem);
2302 }
2303}
2304
2305/*
2306 * Resource setup
2307 */
2308static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled)
2309{
2310 int i;
2311
2312 for (i = 0; i < PCI_ROM_SLOT; i++) {
2313 VFIOBAR *bar = &vdev->bars[i];
2314
2315 if (!bar->size) {
2316 continue;
2317 }
2318
2319 memory_region_set_enabled(&bar->mmap_mem, enabled);
2320 if (vdev->msix && vdev->msix->table_bar == i) {
2321 memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
2322 }
2323 }
2324}
2325
2326static void vfio_unmap_bar(VFIODevice *vdev, int nr)
2327{
2328 VFIOBAR *bar = &vdev->bars[nr];
2329
2330 if (!bar->size) {
2331 return;
2332 }
2333
7076eabc
AW		 2334 vfio_bar_quirk_teardown(vdev, nr);
2335
65501a74
AW		 2336 memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
2337 munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
2338
2339 if (vdev->msix && vdev->msix->table_bar == nr) {
2340 memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
2341 munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
2342 }
2343
2344 memory_region_destroy(&bar->mem);
2345}
2346
5cb022a1
PB		 2347static int vfio_mmap_bar(VFIODevice *vdev, VFIOBAR *bar,
2348 MemoryRegion *mem, MemoryRegion *submem,
65501a74
AW		 2349 void **map, size_t size, off_t offset,
2350 const char *name)
2351{
2352 int ret = 0;
2353
82ca8912 2354 if (VFIO_ALLOW_MMAP && size && bar->flags & VFIO_REGION_INFO_FLAG_MMAP) {
65501a74
AW		 2355 int prot = 0;
2356
2357 if (bar->flags & VFIO_REGION_INFO_FLAG_READ) {
2358 prot |= PROT_READ;
2359 }
2360
2361 if (bar->flags & VFIO_REGION_INFO_FLAG_WRITE) {
2362 prot |= PROT_WRITE;
2363 }
2364
2365 *map = mmap(NULL, size, prot, MAP_SHARED,
2366 bar->fd, bar->fd_offset + offset);
2367 if (*map == MAP_FAILED) {
2368 *map = NULL;
2369 ret = -errno;
2370 goto empty_region;
2371 }
2372
5cb022a1 2373 memory_region_init_ram_ptr(submem, OBJECT(vdev), name, size, *map);
65501a74
AW		 2374 } else {
2375empty_region:
2376 /* Create a zero sized sub-region to make cleanup easy. */
5cb022a1 2377 memory_region_init(submem, OBJECT(vdev), name, 0);
65501a74
AW		 2378 }
2379
2380 memory_region_add_subregion(mem, offset, submem);
2381
2382 return ret;
2383}
2384
2385static void vfio_map_bar(VFIODevice *vdev, int nr)
2386{
2387 VFIOBAR *bar = &vdev->bars[nr];
2388 unsigned size = bar->size;
2389 char name[64];
2390 uint32_t pci_bar;
2391 uint8_t type;
2392 int ret;
2393
2394 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
2395 if (!size) {
2396 return;
2397 }
2398
2399 snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
2400 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2401 vdev->host.function, nr);
2402
2403 /* Determine what type of BAR this is for registration */
2404 ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
2405 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
2406 if (ret != sizeof(pci_bar)) {
312fd5f2 2407 error_report("vfio: Failed to read BAR %d (%m)", nr);
65501a74
AW		 2408 return;
2409 }
2410
2411 pci_bar = le32_to_cpu(pci_bar);
39360f0b
AW		 2412 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
2413 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
2414 type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
2415 ~PCI_BASE_ADDRESS_MEM_MASK);
65501a74
AW		 2416
2417 /* A "slow" read/write mapping underlies all BARs */
39360f0b
AW		 2418 memory_region_init_io(&bar->mem, OBJECT(vdev), &vfio_bar_ops,
2419 bar, name, size);
65501a74
AW		 2420 pci_register_bar(&vdev->pdev, nr, type, &bar->mem);
2421
2422 /*
2423 * We can't mmap areas overlapping the MSIX vector table, so we
2424 * potentially insert a direct-mapped subregion before and after it.
2425 */
2426 if (vdev->msix && vdev->msix->table_bar == nr) {
2427 size = vdev->msix->table_offset & TARGET_PAGE_MASK;
2428 }
2429
2430 strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
5cb022a1 2431 if (vfio_mmap_bar(vdev, bar, &bar->mem,
65501a74 2432 &bar->mmap_mem, &bar->mmap, size, 0, name)) {
312fd5f2 2433 error_report("%s unsupported. Performance may be slow", name);
65501a74
AW		 2434 }
2435
2436 if (vdev->msix && vdev->msix->table_bar == nr) {
2437 unsigned start;
2438
2439 start = TARGET_PAGE_ALIGN(vdev->msix->table_offset +
2440 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
2441
2442 size = start < bar->size ? bar->size - start : 0;
2443 strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
2444 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
5cb022a1 2445 if (vfio_mmap_bar(vdev, bar, &bar->mem, &vdev->msix->mmap_mem,
65501a74 2446 &vdev->msix->mmap, size, start, name)) {
312fd5f2 2447 error_report("%s unsupported. Performance may be slow", name);
65501a74
AW		 2448 }
2449 }
7076eabc
AW		 2450
2451 vfio_bar_quirk_setup(vdev, nr);
65501a74
AW		 2452}
2453
2454static void vfio_map_bars(VFIODevice *vdev)
2455{
2456 int i;
2457
2458 for (i = 0; i < PCI_ROM_SLOT; i++) {
2459 vfio_map_bar(vdev, i);
2460 }
f15689c7
AW		 2461
2462 if (vdev->has_vga) {
3c161542
PB		 2463 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
2464 OBJECT(vdev), &vfio_vga_ops,
f15689c7
AW		 2465 &vdev->vga.region[QEMU_PCI_VGA_MEM],
2466 "vfio-vga-mmio@0xa0000",
2467 QEMU_PCI_VGA_MEM_SIZE);
3c161542
PB		 2468 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
2469 OBJECT(vdev), &vfio_vga_ops,
f15689c7
AW		 2470 &vdev->vga.region[QEMU_PCI_VGA_IO_LO],
2471 "vfio-vga-io@0x3b0",
2472 QEMU_PCI_VGA_IO_LO_SIZE);
3c161542
PB		 2473 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
2474 OBJECT(vdev), &vfio_vga_ops,
f15689c7
AW		 2475 &vdev->vga.region[QEMU_PCI_VGA_IO_HI],
2476 "vfio-vga-io@0x3c0",
2477 QEMU_PCI_VGA_IO_HI_SIZE);
2478
2479 pci_register_vga(&vdev->pdev, &vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
2480 &vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
2481 &vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
7076eabc 2482 vfio_vga_quirk_setup(vdev);
f15689c7 2483 }
65501a74
AW		 2484}
2485
2486static void vfio_unmap_bars(VFIODevice *vdev)
2487{
2488 int i;
2489
2490 for (i = 0; i < PCI_ROM_SLOT; i++) {
2491 vfio_unmap_bar(vdev, i);
2492 }
f15689c7
AW		 2493
2494 if (vdev->has_vga) {
7076eabc 2495 vfio_vga_quirk_teardown(vdev);
f15689c7
AW		 2496 pci_unregister_vga(&vdev->pdev);
2497 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem);
2498 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem);
2499 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
2500 }
65501a74
AW		 2501}
2502
2503/*
2504 * General setup
2505 */
2506static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
2507{
2508 uint8_t tmp, next = 0xff;
2509
2510 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
2511 tmp = pdev->config[tmp + 1]) {
2512 if (tmp > pos && tmp < next) {
2513 next = tmp;
2514 }
2515 }
2516
2517 return next - pos;
2518}
2519
96adc5c7
AW		 2520static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
2521{
2522 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
2523}
2524
2525static void vfio_add_emulated_word(VFIODevice *vdev, int pos,
2526 uint16_t val, uint16_t mask)
2527{
2528 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
2529 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
2530 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
2531}
2532
2533static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
2534{
2535 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
2536}
2537
2538static void vfio_add_emulated_long(VFIODevice *vdev, int pos,
2539 uint32_t val, uint32_t mask)
2540{
2541 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
2542 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
2543 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
2544}
2545
2546static int vfio_setup_pcie_cap(VFIODevice *vdev, int pos, uint8_t size)
2547{
2548 uint16_t flags;
2549 uint8_t type;
2550
2551 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
2552 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
2553
2554 if (type != PCI_EXP_TYPE_ENDPOINT &&
2555 type != PCI_EXP_TYPE_LEG_END &&
2556 type != PCI_EXP_TYPE_RC_END) {
2557
2558 error_report("vfio: Assignment of PCIe type 0x%x "
2559 "devices is not currently supported", type);
2560 return -EINVAL;
2561 }
2562
2563 if (!pci_bus_is_express(vdev->pdev.bus)) {
2564 /*
2565 * Use express capability as-is on PCI bus. It doesn't make much
2566 * sense to even expose, but some drivers (ex. tg3) depend on it
2567 * and guests don't seem to be particular about it. We'll need
2568 * to revist this or force express devices to express buses if we
2569 * ever expose an IOMMU to the guest.
2570 */
2571 } else if (pci_bus_is_root(vdev->pdev.bus)) {
2572 /*
2573 * On a Root Complex bus Endpoints become Root Complex Integrated
2574 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2575 */
2576 if (type == PCI_EXP_TYPE_ENDPOINT) {
2577 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2578 PCI_EXP_TYPE_RC_END << 4,
2579 PCI_EXP_FLAGS_TYPE);
2580
2581 /* Link Capabilities, Status, and Control goes away */
2582 if (size > PCI_EXP_LNKCTL) {
2583 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
2584 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2585 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
2586
2587#ifndef PCI_EXP_LNKCAP2
2588#define PCI_EXP_LNKCAP2 44
2589#endif
2590#ifndef PCI_EXP_LNKSTA2
2591#define PCI_EXP_LNKSTA2 50
2592#endif
2593 /* Link 2 Capabilities, Status, and Control goes away */
2594 if (size > PCI_EXP_LNKCAP2) {
2595 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
2596 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
2597 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
2598 }
2599 }
2600
2601 } else if (type == PCI_EXP_TYPE_LEG_END) {
2602 /*
2603 * Legacy endpoints don't belong on the root complex. Windows
2604 * seems to be happier with devices if we skip the capability.
2605 */
2606 return 0;
2607 }
2608
2609 } else {
2610 /*
2611 * Convert Root Complex Integrated Endpoints to regular endpoints.
2612 * These devices don't support LNK/LNK2 capabilities, so make them up.
2613 */
2614 if (type == PCI_EXP_TYPE_RC_END) {
2615 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2616 PCI_EXP_TYPE_ENDPOINT << 4,
2617 PCI_EXP_FLAGS_TYPE);
2618 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
2619 PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
2620 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2621 }
2622
2623 /* Mark the Link Status bits as emulated to allow virtual negotiation */
2624 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
2625 pci_get_word(vdev->pdev.config + pos +
2626 PCI_EXP_LNKSTA),
2627 PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
2628 }
2629
2630 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
2631 if (pos >= 0) {
2632 vdev->pdev.exp.exp_cap = pos;
2633 }
2634
2635 return pos;
2636}
2637
befe5176
AW		 2638static void vfio_check_pcie_flr(VFIODevice *vdev, uint8_t pos)
2639{
2640 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
2641
2642 if (cap & PCI_EXP_DEVCAP_FLR) {
2643 DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
2644 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2645 vdev->host.function);
2646 vdev->has_flr = true;
2647 }
2648}
2649
2650static void vfio_check_pm_reset(VFIODevice *vdev, uint8_t pos)
2651{
2652 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
2653
2654 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
2655 DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
2656 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2657 vdev->host.function);
2658 vdev->has_pm_reset = true;
2659 }
2660}
2661
2662static void vfio_check_af_flr(VFIODevice *vdev, uint8_t pos)
2663{
2664 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
2665
2666 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
2667 DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
2668 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2669 vdev->host.function);
2670 vdev->has_flr = true;
2671 }
2672}
2673
65501a74
AW		 2674static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
2675{
2676 PCIDevice *pdev = &vdev->pdev;
2677 uint8_t cap_id, next, size;
2678 int ret;
2679
2680 cap_id = pdev->config[pos];
2681 next = pdev->config[pos + 1];
2682
2683 /*
2684 * If it becomes important to configure capabilities to their actual
2685 * size, use this as the default when it's something we don't recognize.
2686 * Since QEMU doesn't actually handle many of the config accesses,
2687 * exact size doesn't seem worthwhile.
2688 */
2689 size = vfio_std_cap_max_size(pdev, pos);
2690
2691 /*
2692 * pci_add_capability always inserts the new capability at the head
2693 * of the chain. Therefore to end up with a chain that matches the
2694 * physical device, we insert from the end by making this recursive.
2695 * This is also why we pre-caclulate size above as cached config space
2696 * will be changed as we unwind the stack.
2697 */
2698 if (next) {
2699 ret = vfio_add_std_cap(vdev, next);
2700 if (ret) {
2701 return ret;
2702 }
2703 } else {
96adc5c7
AW		 2704 /* Begin the rebuild, use QEMU emulated list bits */
2705 pdev->config[PCI_CAPABILITY_LIST] = 0;
2706 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
2707 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
65501a74
AW		 2708 }
2709
96adc5c7
AW		 2710 /* Use emulated next pointer to allow dropping caps */
2711 pci_set_byte(vdev->emulated_config_bits + pos + 1, 0xff);
2712
65501a74
AW		 2713 switch (cap_id) {
2714 case PCI_CAP_ID_MSI:
2715 ret = vfio_setup_msi(vdev, pos);
2716 break;
96adc5c7 2717 case PCI_CAP_ID_EXP:
befe5176 2718 vfio_check_pcie_flr(vdev, pos);
96adc5c7
AW		 2719 ret = vfio_setup_pcie_cap(vdev, pos, size);
2720 break;
65501a74
AW		 2721 case PCI_CAP_ID_MSIX:
2722 ret = vfio_setup_msix(vdev, pos);
2723 break;
ba661818 2724 case PCI_CAP_ID_PM:
befe5176 2725 vfio_check_pm_reset(vdev, pos);
ba661818 2726 vdev->pm_cap = pos;
befe5176
AW		 2727 ret = pci_add_capability(pdev, cap_id, pos, size);
2728 break;
2729 case PCI_CAP_ID_AF:
2730 vfio_check_af_flr(vdev, pos);
2731 ret = pci_add_capability(pdev, cap_id, pos, size);
2732 break;
65501a74
AW		 2733 default:
2734 ret = pci_add_capability(pdev, cap_id, pos, size);
2735 break;
2736 }
2737
2738 if (ret < 0) {
2739 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
312fd5f2 2740 "0x%x[0x%x]@0x%x: %d", vdev->host.domain,
65501a74
AW		 2741 vdev->host.bus, vdev->host.slot, vdev->host.function,
2742 cap_id, size, pos, ret);
2743 return ret;
2744 }
2745
2746 return 0;
2747}
2748
2749static int vfio_add_capabilities(VFIODevice *vdev)
2750{
2751 PCIDevice *pdev = &vdev->pdev;
2752
2753 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2754 !pdev->config[PCI_CAPABILITY_LIST]) {
2755 return 0; /* Nothing to add */
2756 }
2757
2758 return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
2759}
2760
65501a74
AW		 2761static int vfio_connect_container(VFIOGroup *group)
2762{
2763 VFIOContainer *container;
2764 int ret, fd;
2765
2766 if (group->container) {
2767 return 0;
2768 }
2769
2770 QLIST_FOREACH(container, &container_list, next) {
2771 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
2772 group->container = container;
2773 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
2774 return 0;
2775 }
2776 }
2777
2778 fd = qemu_open("/dev/vfio/vfio", O_RDWR);
2779 if (fd < 0) {
312fd5f2 2780 error_report("vfio: failed to open /dev/vfio/vfio: %m");
65501a74
AW		 2781 return -errno;
2782 }
2783
2784 ret = ioctl(fd, VFIO_GET_API_VERSION);
2785 if (ret != VFIO_API_VERSION) {
2786 error_report("vfio: supported vfio version: %d, "
312fd5f2 2787 "reported version: %d", VFIO_API_VERSION, ret);
65501a74
AW		 2788 close(fd);
2789 return -EINVAL;
2790 }
2791
2792 container = g_malloc0(sizeof(*container));
2793 container->fd = fd;
2794
2795 if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
2796 ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
2797 if (ret) {
312fd5f2 2798 error_report("vfio: failed to set group container: %m");
65501a74
AW		 2799 g_free(container);
2800 close(fd);
2801 return -errno;
2802 }
2803
2804 ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
2805 if (ret) {
312fd5f2 2806 error_report("vfio: failed to set iommu for container: %m");
65501a74
AW		 2807 g_free(container);
2808 close(fd);
2809 return -errno;
2810 }
2811
2812 container->iommu_data.listener = vfio_memory_listener;
2813 container->iommu_data.release = vfio_listener_release;
2814
f6790af6 2815 memory_listener_register(&container->iommu_data.listener, &address_space_memory);
65501a74 2816 } else {
312fd5f2 2817 error_report("vfio: No available IOMMU models");
65501a74
AW		 2818 g_free(container);
2819 close(fd);
2820 return -EINVAL;
2821 }
2822
2823 QLIST_INIT(&container->group_list);
2824 QLIST_INSERT_HEAD(&container_list, container, next);
2825
2826 group->container = container;
2827 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
2828
2829 return 0;
2830}
2831
2832static void vfio_disconnect_container(VFIOGroup *group)
2833{
2834 VFIOContainer *container = group->container;
2835
2836 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
312fd5f2 2837 error_report("vfio: error disconnecting group %d from container",
65501a74
AW		 2838 group->groupid);
2839 }
2840
2841 QLIST_REMOVE(group, container_next);
2842 group->container = NULL;
2843
2844 if (QLIST_EMPTY(&container->group_list)) {
2845 if (container->iommu_data.release) {
2846 container->iommu_data.release(container);
2847 }
2848 QLIST_REMOVE(container, next);
2849 DPRINTF("vfio_disconnect_container: close container->fd\n");
2850 close(container->fd);
2851 g_free(container);
2852 }
2853}
2854
2855static VFIOGroup *vfio_get_group(int groupid)
2856{
2857 VFIOGroup *group;
2858 char path[32];
2859 struct vfio_group_status status = { .argsz = sizeof(status) };
2860
2861 QLIST_FOREACH(group, &group_list, next) {
2862 if (group->groupid == groupid) {
2863 return group;
2864 }
2865 }
2866
2867 group = g_malloc0(sizeof(*group));
2868
2869 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
2870 group->fd = qemu_open(path, O_RDWR);
2871 if (group->fd < 0) {
312fd5f2 2872 error_report("vfio: error opening %s: %m", path);
65501a74
AW		 2873 g_free(group);
2874 return NULL;
2875 }
2876
2877 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
312fd5f2 2878 error_report("vfio: error getting group status: %m");
65501a74
AW		 2879 close(group->fd);
2880 g_free(group);
2881 return NULL;
2882 }
2883
2884 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
2885 error_report("vfio: error, group %d is not viable, please ensure "
2886 "all devices within the iommu_group are bound to their "
312fd5f2 2887 "vfio bus driver.", groupid);
65501a74
AW		 2888 close(group->fd);
2889 g_free(group);
2890 return NULL;
2891 }
2892
2893 group->groupid = groupid;
2894 QLIST_INIT(&group->device_list);
2895
2896 if (vfio_connect_container(group)) {
312fd5f2 2897 error_report("vfio: failed to setup container for group %d", groupid);
65501a74
AW		 2898 close(group->fd);
2899 g_free(group);
2900 return NULL;
2901 }
2902
2903 QLIST_INSERT_HEAD(&group_list, group, next);
2904
2905 return group;
2906}
2907
2908static void vfio_put_group(VFIOGroup *group)
2909{
2910 if (!QLIST_EMPTY(&group->device_list)) {
2911 return;
2912 }
2913
2914 vfio_disconnect_container(group);
2915 QLIST_REMOVE(group, next);
2916 DPRINTF("vfio_put_group: close group->fd\n");
2917 close(group->fd);
2918 g_free(group);
2919}
2920
2921static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
2922{
2923 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
2924 struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
7b4b0e9e 2925 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
65501a74
AW		 2926 int ret, i;
2927
2928 ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
2929 if (ret < 0) {
1a9522cc 2930 error_report("vfio: error getting device %s from group %d: %m",
65501a74 2931 name, group->groupid);
1a9522cc 2932 error_printf("Verify all devices in group %d are bound to vfio-pci "
65501a74
AW		 2933 "or pci-stub and not already in use\n", group->groupid);
2934 return ret;
2935 }
2936
2937 vdev->fd = ret;
2938 vdev->group = group;
2939 QLIST_INSERT_HEAD(&group->device_list, vdev, next);
2940
2941 /* Sanity check device */
2942 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
2943 if (ret) {
312fd5f2 2944 error_report("vfio: error getting device info: %m");
65501a74
AW		 2945 goto error;
2946 }
2947
2948 DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name,
2949 dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
2950
2951 if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
312fd5f2 2952 error_report("vfio: Um, this isn't a PCI device");
65501a74
AW		 2953 goto error;
2954 }
2955
2956 vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
2957 if (!vdev->reset_works) {
312fd5f2 2958 error_report("Warning, device %s does not support reset", name);
65501a74
AW		 2959 }
2960
8fc94e5a 2961 if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
312fd5f2 2962 error_report("vfio: unexpected number of io regions %u",
65501a74
AW		 2963 dev_info.num_regions);
2964 goto error;
2965 }
2966
8fc94e5a 2967 if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
312fd5f2 2968 error_report("vfio: unexpected number of irqs %u", dev_info.num_irqs);
65501a74
AW		 2969 goto error;
2970 }
2971
2972 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2973 reg_info.index = i;
2974
2975 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
2976 if (ret) {
312fd5f2 2977 error_report("vfio: Error getting region %d info: %m", i);
65501a74
AW		 2978 goto error;
2979 }
2980
2981 DPRINTF("Device %s region %d:\n", name, i);
2982 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
2983 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
2984 (unsigned long)reg_info.flags);
2985
2986 vdev->bars[i].flags = reg_info.flags;
2987 vdev->bars[i].size = reg_info.size;
2988 vdev->bars[i].fd_offset = reg_info.offset;
2989 vdev->bars[i].fd = vdev->fd;
2990 vdev->bars[i].nr = i;
7076eabc 2991 QLIST_INIT(&vdev->bars[i].quirks);
65501a74
AW		 2992 }
2993
65501a74
AW		 2994 reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
2995
2996 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
2997 if (ret) {
312fd5f2 2998 error_report("vfio: Error getting config info: %m");
65501a74
AW		 2999 goto error;
3000 }
3001
3002 DPRINTF("Device %s config:\n", name);
3003 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3004 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
3005 (unsigned long)reg_info.flags);
3006
3007 vdev->config_size = reg_info.size;
6a659bbf
AW		 3008 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
3009 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
3010 }
65501a74
AW		 3011 vdev->config_offset = reg_info.offset;
3012
f15689c7
AW		 3013 if ((vdev->features & VFIO_FEATURE_ENABLE_VGA) &&
3014 dev_info.num_regions > VFIO_PCI_VGA_REGION_INDEX) {
3015 struct vfio_region_info vga_info = {
3016 .argsz = sizeof(vga_info),
3017 .index = VFIO_PCI_VGA_REGION_INDEX,
3018 };
3019
3020 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &vga_info);
3021 if (ret) {
3022 error_report(
3023 "vfio: Device does not support requested feature x-vga");
3024 goto error;
3025 }
3026
3027 if (!(vga_info.flags & VFIO_REGION_INFO_FLAG_READ) ||
3028 !(vga_info.flags & VFIO_REGION_INFO_FLAG_WRITE) ||
3029 vga_info.size < 0xbffff + 1) {
3030 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
3031 (unsigned long)vga_info.flags,
3032 (unsigned long)vga_info.size);
3033 goto error;
3034 }
3035
3036 vdev->vga.fd_offset = vga_info.offset;
3037 vdev->vga.fd = vdev->fd;
3038
3039 vdev->vga.region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
3040 vdev->vga.region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
7076eabc 3041 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_MEM].quirks);
f15689c7
AW		 3042
3043 vdev->vga.region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
3044 vdev->vga.region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
7076eabc 3045 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].quirks);
f15689c7
AW		 3046
3047 vdev->vga.region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
3048 vdev->vga.region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
7076eabc 3049 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks);
f15689c7
AW		 3050
3051 vdev->has_vga = true;
3052 }
7b4b0e9e
VMP		 3053 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
3054
3055 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
3056 if (ret) {
3057 /* This can fail for an old kernel or legacy PCI dev */
3058 DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure ret=%d\n", ret);
3059 ret = 0;
3060 } else if (irq_info.count == 1) {
3061 vdev->pci_aer = true;
3062 } else {
3063 error_report("vfio: Warning: "
3064 "Could not enable error recovery for the device\n");
3065 }
f15689c7 3066
65501a74
AW		 3067error:
3068 if (ret) {
3069 QLIST_REMOVE(vdev, next);
3070 vdev->group = NULL;
3071 close(vdev->fd);
3072 }
3073 return ret;
3074}
3075
3076static void vfio_put_device(VFIODevice *vdev)
3077{
3078 QLIST_REMOVE(vdev, next);
3079 vdev->group = NULL;
3080 DPRINTF("vfio_put_device: close vdev->fd\n");
3081 close(vdev->fd);
3082 if (vdev->msix) {
3083 g_free(vdev->msix);
3084 vdev->msix = NULL;
3085 }
3086}
3087
7b4b0e9e
VMP		 3088static void vfio_err_notifier_handler(void *opaque)
3089{
3090 VFIODevice *vdev = opaque;
3091
3092 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
3093 return;
3094 }
3095
3096 /*
3097 * TBD. Retrieve the error details and decide what action
3098 * needs to be taken. One of the actions could be to pass
3099 * the error to the guest and have the guest driver recover
3100 * from the error. This requires that PCIe capabilities be
3101 * exposed to the guest. For now, we just terminate the
3102 * guest to contain the error.
3103 */
3104
3105 error_report("%s (%04x:%02x:%02x.%x)"
3106 "Unrecoverable error detected...\n"
3107 "Please collect any data possible and then kill the guest",
3108 __func__, vdev->host.domain, vdev->host.bus,
3109 vdev->host.slot, vdev->host.function);
3110
3111 vm_stop(RUN_STATE_IO_ERROR);
3112}
3113
3114/*
3115 * Registers error notifier for devices supporting error recovery.
3116 * If we encounter a failure in this function, we report an error
3117 * and continue after disabling error recovery support for the
3118 * device.
3119 */
3120static void vfio_register_err_notifier(VFIODevice *vdev)
3121{
3122 int ret;
3123 int argsz;
3124 struct vfio_irq_set *irq_set;
3125 int32_t *pfd;
3126
3127 if (!vdev->pci_aer) {
3128 return;
3129 }
3130
3131 if (event_notifier_init(&vdev->err_notifier, 0)) {
3132 error_report("vfio: Warning: "
3133 "Unable to init event notifier for error detection\n");
3134 vdev->pci_aer = false;
3135 return;
3136 }
3137
3138 argsz = sizeof(*irq_set) + sizeof(*pfd);
3139
3140 irq_set = g_malloc0(argsz);
3141 irq_set->argsz = argsz;
3142 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
3143 VFIO_IRQ_SET_ACTION_TRIGGER;
3144 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
3145 irq_set->start = 0;
3146 irq_set->count = 1;
3147 pfd = (int32_t *)&irq_set->data;
3148
3149 *pfd = event_notifier_get_fd(&vdev->err_notifier);
3150 qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
3151
3152 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
3153 if (ret) {
3154 error_report("vfio: Failed to set up error notification\n");
3155 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
3156 event_notifier_cleanup(&vdev->err_notifier);
3157 vdev->pci_aer = false;
3158 }
3159 g_free(irq_set);
3160}
3161
3162static void vfio_unregister_err_notifier(VFIODevice *vdev)
3163{
3164 int argsz;
3165 struct vfio_irq_set *irq_set;
3166 int32_t *pfd;
3167 int ret;
3168
3169 if (!vdev->pci_aer) {
3170 return;
3171 }
3172
3173 argsz = sizeof(*irq_set) + sizeof(*pfd);
3174
3175 irq_set = g_malloc0(argsz);
3176 irq_set->argsz = argsz;
3177 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
3178 VFIO_IRQ_SET_ACTION_TRIGGER;
3179 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
3180 irq_set->start = 0;
3181 irq_set->count = 1;
3182 pfd = (int32_t *)&irq_set->data;
3183 *pfd = -1;
3184
3185 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
3186 if (ret) {
3187 error_report("vfio: Failed to de-assign error fd: %d\n", ret);
3188 }
3189 g_free(irq_set);
3190 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
3191 NULL, NULL, vdev);
3192 event_notifier_cleanup(&vdev->err_notifier);
3193}
3194
65501a74
AW		 3195static int vfio_initfn(PCIDevice *pdev)
3196{
3197 VFIODevice *pvdev, *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
3198 VFIOGroup *group;
3199 char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
3200 ssize_t len;
3201 struct stat st;
3202 int groupid;
3203 int ret;
3204
3205 /* Check that the host device exists */
3206 snprintf(path, sizeof(path),
3207 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
3208 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3209 vdev->host.function);
3210 if (stat(path, &st) < 0) {
312fd5f2 3211 error_report("vfio: error: no such host device: %s", path);
65501a74
AW		 3212 return -errno;
3213 }
3214
3215 strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
3216
3217 len = readlink(path, iommu_group_path, PATH_MAX);
3218 if (len <= 0) {
312fd5f2 3219 error_report("vfio: error no iommu_group for device");
65501a74
AW		 3220 return -errno;
3221 }
3222
3223 iommu_group_path[len] = 0;
3224 group_name = basename(iommu_group_path);
3225
3226 if (sscanf(group_name, "%d", &groupid) != 1) {
312fd5f2 3227 error_report("vfio: error reading %s: %m", path);
65501a74
AW		 3228 return -errno;
3229 }
3230
3231 DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
3232 vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
3233
3234 group = vfio_get_group(groupid);
3235 if (!group) {
312fd5f2 3236 error_report("vfio: failed to get group %d", groupid);
65501a74
AW		 3237 return -ENOENT;
3238 }
3239
3240 snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
3241 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3242 vdev->host.function);
3243
3244 QLIST_FOREACH(pvdev, &group->device_list, next) {
3245 if (pvdev->host.domain == vdev->host.domain &&
3246 pvdev->host.bus == vdev->host.bus &&
3247 pvdev->host.slot == vdev->host.slot &&
3248 pvdev->host.function == vdev->host.function) {
3249
312fd5f2 3250 error_report("vfio: error: device %s is already attached", path);
65501a74
AW		 3251 vfio_put_group(group);
3252 return -EBUSY;
3253 }
3254 }
3255
3256 ret = vfio_get_device(group, path, vdev);
3257 if (ret) {
312fd5f2 3258 error_report("vfio: failed to get device %s", path);
65501a74
AW		 3259 vfio_put_group(group);
3260 return ret;
3261 }
3262
3263 /* Get a copy of config space */
3264 ret = pread(vdev->fd, vdev->pdev.config,
3265 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
3266 vdev->config_offset);
3267 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
3268 ret = ret < 0 ? -errno : -EFAULT;
312fd5f2 3269 error_report("vfio: Failed to read device config space");
65501a74
AW		 3270 goto out_put;
3271 }
3272
4b5d5e87
AW		 3273 /* vfio emulates a lot for us, but some bits need extra love */
3274 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
3275
3276 /* QEMU can choose to expose the ROM or not */
3277 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
3278
3279 /* QEMU can change multi-function devices to single function, or reverse */
3280 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
3281 PCI_HEADER_TYPE_MULTI_FUNCTION;
3282
65501a74
AW		 3283 /*
3284 * Clear host resource mapping info. If we choose not to register a
3285 * BAR, such as might be the case with the option ROM, we can get
3286 * confusing, unwritable, residual addresses from the host here.
3287 */
3288 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
3289 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
3290
6f864e6e 3291 vfio_pci_size_rom(vdev);
65501a74
AW		 3292
3293 ret = vfio_early_setup_msix(vdev);
3294 if (ret) {
3295 goto out_put;
3296 }
3297
3298 vfio_map_bars(vdev);
3299
3300 ret = vfio_add_capabilities(vdev);
3301 if (ret) {
3302 goto out_teardown;
3303 }
3304
4b5d5e87
AW		 3305 /* QEMU emulates all of MSI & MSIX */
3306 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
3307 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
3308 MSIX_CAP_LENGTH);
3309 }
3310
3311 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
3312 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
3313 vdev->msi_cap_size);
3314 }
3315
65501a74 3316 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
bc72ad67 3317 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
ea486926 3318 vfio_intx_mmap_enable, vdev);
e1d1e586 3319 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
65501a74
AW		 3320 ret = vfio_enable_intx(vdev);
3321 if (ret) {
3322 goto out_teardown;
3323 }
3324 }
3325
c29029dd 3326 add_boot_device_path(vdev->bootindex, &pdev->qdev, NULL);
7b4b0e9e 3327 vfio_register_err_notifier(vdev);
c29029dd 3328
65501a74
AW		 3329 return 0;
3330
3331out_teardown:
3332 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3333 vfio_teardown_msi(vdev);
3334 vfio_unmap_bars(vdev);
3335out_put:
4b5d5e87 3336 g_free(vdev->emulated_config_bits);
65501a74
AW		 3337 vfio_put_device(vdev);
3338 vfio_put_group(group);
3339 return ret;
3340}
3341
3342static void vfio_exitfn(PCIDevice *pdev)
3343{
3344 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
3345 VFIOGroup *group = vdev->group;
3346
7b4b0e9e 3347 vfio_unregister_err_notifier(vdev);
65501a74
AW		 3348 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3349 vfio_disable_interrupts(vdev);
ea486926 3350 if (vdev->intx.mmap_timer) {
bc72ad67 3351 timer_free(vdev->intx.mmap_timer);
ea486926 3352 }
65501a74
AW		 3353 vfio_teardown_msi(vdev);
3354 vfio_unmap_bars(vdev);
4b5d5e87 3355 g_free(vdev->emulated_config_bits);
6f864e6e 3356 g_free(vdev->rom);
65501a74
AW		 3357 vfio_put_device(vdev);
3358 vfio_put_group(group);
3359}
3360
3361static void vfio_pci_reset(DeviceState *dev)
3362{
3363 PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
3364 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
5834a83f 3365 uint16_t cmd;
65501a74 3366
5834a83f
AW		 3367 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
3368 vdev->host.bus, vdev->host.slot, vdev->host.function);
3369
3370 vfio_disable_interrupts(vdev);
65501a74 3371
ba661818
AW		 3372 /* Make sure the device is in D0 */
3373 if (vdev->pm_cap) {
3374 uint16_t pmcsr;
3375 uint8_t state;
3376
3377 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
3378 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
3379 if (state) {
3380 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3381 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
3382 /* vfio handles the necessary delay here */
3383 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
3384 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
3385 if (state) {
3386 error_report("vfio: Unable to power on device, stuck in D%d\n",
3387 state);
3388 }
3389 }
3390 }
3391
5834a83f
AW		 3392 /*
3393 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
3394 * Also put INTx Disable in known state.
3395 */
3396 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
3397 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
3398 PCI_COMMAND_INTX_DISABLE);
3399 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
3400
3401 if (vdev->reset_works) {
3402 if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
3403 error_report("vfio: Error unable to reset physical device "
312fd5f2 3404 "(%04x:%02x:%02x.%x): %m", vdev->host.domain,
5834a83f
AW		 3405 vdev->host.bus, vdev->host.slot, vdev->host.function);
3406 }
65501a74 3407 }
5834a83f
AW		 3408
3409 vfio_enable_intx(vdev);
65501a74
AW		 3410}
3411
3412static Property vfio_pci_dev_properties[] = {
3413 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
ea486926
AW		 3414 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
3415 intx.mmap_timeout, 1100),
f15689c7
AW		 3416 DEFINE_PROP_BIT("x-vga", VFIODevice, features,
3417 VFIO_FEATURE_ENABLE_VGA_BIT, false),
c29029dd 3418 DEFINE_PROP_INT32("bootindex", VFIODevice, bootindex, -1),
65501a74
AW		 3419 /*
3420 * TODO - support passed fds... is this necessary?
3421 * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
3422 * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
3423 */
3424 DEFINE_PROP_END_OF_LIST(),
3425};
3426
d9f0e638
AW		 3427static const VMStateDescription vfio_pci_vmstate = {
3428 .name = "vfio-pci",
3429 .unmigratable = 1,
3430};
65501a74
AW		 3431
3432static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3433{
3434 DeviceClass *dc = DEVICE_CLASS(klass);
3435 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3436
3437 dc->reset = vfio_pci_reset;
3438 dc->props = vfio_pci_dev_properties;
d9f0e638
AW		 3439 dc->vmsd = &vfio_pci_vmstate;
3440 dc->desc = "VFIO-based PCI device assignment";
125ee0ed 3441 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
65501a74
AW		 3442 pdc->init = vfio_initfn;
3443 pdc->exit = vfio_exitfn;
3444 pdc->config_read = vfio_pci_read_config;
3445 pdc->config_write = vfio_pci_write_config;
6a659bbf 3446 pdc->is_express = 1; /* We might be */
65501a74
AW		 3447}
3448
3449static const TypeInfo vfio_pci_dev_info = {
3450 .name = "vfio-pci",
3451 .parent = TYPE_PCI_DEVICE,
3452 .instance_size = sizeof(VFIODevice),
3453 .class_init = vfio_pci_dev_class_init,
3454};
3455
3456static void register_vfio_pci_dev_type(void)
3457{
3458 type_register_static(&vfio_pci_dev_info);
3459}
3460
3461type_init(register_vfio_pci_dev_type)
Qemu copy for testing