hw/virtio/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  * Contributions after 2012-01-13 are licensed under the terms of the
  15  * GNU GPL, version 2 or (at your option) any later version.
  16  */
  17
  18 #include "qemu/osdep.h"
  19
  20 #include "exec/memop.h"
  21 #include "standard-headers/linux/virtio_pci.h"
  22 #include "hw/boards.h"
  23 #include "hw/virtio/virtio.h"
  24 #include "migration/qemu-file-types.h"
  25 #include "hw/pci/pci.h"
  26 #include "hw/pci/pci_bus.h"
  27 #include "hw/qdev-properties.h"
  28 #include "qapi/error.h"
  29 #include "qemu/error-report.h"
  30 #include "qemu/log.h"
  31 #include "qemu/module.h"
  32 #include "hw/pci/msi.h"
  33 #include "hw/pci/msix.h"
  34 #include "hw/loader.h"
  35 #include "sysemu/kvm.h"
  36 #include "virtio-pci.h"
  37 #include "qemu/range.h"
  38 #include "hw/virtio/virtio-bus.h"
  39 #include "qapi/visitor.h"
  40 #include "sysemu/replay.h"
  41
  42 #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
  43
  44 #undef VIRTIO_PCI_CONFIG
  45
  46 /* The remaining space is defined by each driver as the per-driver
  47  * configuration space */
  48 #define VIRTIO_PCI_CONFIG_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev))
  49
  50 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
  51                                VirtIOPCIProxy *dev);
  52 static void virtio_pci_reset(DeviceState *qdev);
  53
  54 /* virtio device */
  55 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
  56 static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d)
  57 {
  58     return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
  59 }
  60
  61 /* DeviceState to VirtIOPCIProxy. Note: used on datapath,
  62  * be careful and test performance if you change this.
  63  */
  64 static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d)
  65 {
  66     return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
  67 }
  68
  69 static void virtio_pci_notify(DeviceState *d, uint16_t vector)
  70 {
  71     VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
  72
  73     if (msix_enabled(&proxy->pci_dev))
  74         msix_notify(&proxy->pci_dev, vector);
  75     else {
  76         VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
  77         pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1);
  78     }
  79 }
  80
  81 static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
  82 {
  83     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
  84     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
  85
  86     pci_device_save(&proxy->pci_dev, f);
  87     msix_save(&proxy->pci_dev, f);
  88     if (msix_present(&proxy->pci_dev))
  89         qemu_put_be16(f, vdev->config_vector);
  90 }
  91
  92 static const VMStateDescription vmstate_virtio_pci_modern_queue_state = {
  93     .name = "virtio_pci/modern_queue_state",
  94     .version_id = 1,
  95     .minimum_version_id = 1,
  96     .fields = (VMStateField[]) {
  97         VMSTATE_UINT16(num, VirtIOPCIQueue),
  98         VMSTATE_UNUSED(1), /* enabled was stored as be16 */
  99         VMSTATE_BOOL(enabled, VirtIOPCIQueue),
 100         VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2),
 101         VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2),
 102         VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2),
 103         VMSTATE_END_OF_LIST()
 104     }
 105 };
 106
 107 static bool virtio_pci_modern_state_needed(void *opaque)
 108 {
 109     VirtIOPCIProxy *proxy = opaque;
 110
 111     return virtio_pci_modern(proxy);
 112 }
 113
 114 static const VMStateDescription vmstate_virtio_pci_modern_state_sub = {
 115     .name = "virtio_pci/modern_state",
 116     .version_id = 1,
 117     .minimum_version_id = 1,
 118     .needed = &virtio_pci_modern_state_needed,
 119     .fields = (VMStateField[]) {
 120         VMSTATE_UINT32(dfselect, VirtIOPCIProxy),
 121         VMSTATE_UINT32(gfselect, VirtIOPCIProxy),
 122         VMSTATE_UINT32_ARRAY(guest_features, VirtIOPCIProxy, 2),
 123         VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0,
 124                              vmstate_virtio_pci_modern_queue_state,
 125                              VirtIOPCIQueue),
 126         VMSTATE_END_OF_LIST()
 127     }
 128 };
 129
 130 static const VMStateDescription vmstate_virtio_pci = {
 131     .name = "virtio_pci",
 132     .version_id = 1,
 133     .minimum_version_id = 1,
 134     .minimum_version_id_old = 1,
 135     .fields = (VMStateField[]) {
 136         VMSTATE_END_OF_LIST()
 137     },
 138     .subsections = (const VMStateDescription*[]) {
 139         &vmstate_virtio_pci_modern_state_sub,
 140         NULL
 141     }
 142 };
 143
 144 static bool virtio_pci_has_extra_state(DeviceState *d)
 145 {
 146     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 147
 148     return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
 149 }
 150
 151 static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
 152 {
 153     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 154
 155     vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
 156 }
 157
 158 static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
 159 {
 160     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 161
 162     return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
 163 }
 164
 165 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 166 {
 167     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 168     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 169
 170     if (msix_present(&proxy->pci_dev))
 171         qemu_put_be16(f, virtio_queue_vector(vdev, n));
 172 }
 173
 174 static int virtio_pci_load_config(DeviceState *d, QEMUFile *f)
 175 {
 176     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 177     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 178
 179     int ret;
 180     ret = pci_device_load(&proxy->pci_dev, f);
 181     if (ret) {
 182         return ret;
 183     }
 184     msix_unuse_all_vectors(&proxy->pci_dev);
 185     msix_load(&proxy->pci_dev, f);
 186     if (msix_present(&proxy->pci_dev)) {
 187         qemu_get_be16s(f, &vdev->config_vector);
 188     } else {
 189         vdev->config_vector = VIRTIO_NO_VECTOR;
 190     }
 191     if (vdev->config_vector != VIRTIO_NO_VECTOR) {
 192         return msix_vector_use(&proxy->pci_dev, vdev->config_vector);
 193     }
 194     return 0;
 195 }
 196
 197 static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
 198 {
 199     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 200     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 201
 202     uint16_t vector;
 203     if (msix_present(&proxy->pci_dev)) {
 204         qemu_get_be16s(f, &vector);
 205     } else {
 206         vector = VIRTIO_NO_VECTOR;
 207     }
 208     virtio_queue_set_vector(vdev, n, vector);
 209     if (vector != VIRTIO_NO_VECTOR) {
 210         return msix_vector_use(&proxy->pci_dev, vector);
 211     }
 212
 213     return 0;
 214 }
 215
 216 static bool virtio_pci_ioeventfd_enabled(DeviceState *d)
 217 {
 218     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 219
 220     return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0;
 221 }
 222
 223 #define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000
 224
 225 static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy)
 226 {
 227     return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ?
 228         QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4;
 229 }
 230
 231 static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier,
 232                                        int n, bool assign)
 233 {
 234     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 235     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 236     VirtQueue *vq = virtio_get_queue(vdev, n);
 237     bool legacy = virtio_pci_legacy(proxy);
 238     bool modern = virtio_pci_modern(proxy);
 239     bool fast_mmio = kvm_ioeventfd_any_length_enabled();
 240     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 241     MemoryRegion *modern_mr = &proxy->notify.mr;
 242     MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
 243     MemoryRegion *legacy_mr = &proxy->bar;
 244     hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) *
 245                          virtio_get_queue_index(vq);
 246     hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY;
 247
 248     if (assign) {
 249         if (modern) {
 250             if (fast_mmio) {
 251                 memory_region_add_eventfd(modern_mr, modern_addr, 0,
 252                                           false, n, notifier);
 253             } else {
 254                 memory_region_add_eventfd(modern_mr, modern_addr, 2,
 255                                           false, n, notifier);
 256             }
 257             if (modern_pio) {
 258                 memory_region_add_eventfd(modern_notify_mr, 0, 2,
 259                                               true, n, notifier);
 260             }
 261         }
 262         if (legacy) {
 263             memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
 264                                       true, n, notifier);
 265         }
 266     } else {
 267         if (modern) {
 268             if (fast_mmio) {
 269                 memory_region_del_eventfd(modern_mr, modern_addr, 0,
 270                                           false, n, notifier);
 271             } else {
 272                 memory_region_del_eventfd(modern_mr, modern_addr, 2,
 273                                           false, n, notifier);
 274             }
 275             if (modern_pio) {
 276                 memory_region_del_eventfd(modern_notify_mr, 0, 2,
 277                                           true, n, notifier);
 278             }
 279         }
 280         if (legacy) {
 281             memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
 282                                       true, n, notifier);
 283         }
 284     }
 285     return 0;
 286 }
 287
 288 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 289 {
 290     virtio_bus_start_ioeventfd(&proxy->bus);
 291 }
 292
 293 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 294 {
 295     virtio_bus_stop_ioeventfd(&proxy->bus);
 296 }
 297
 298 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 299 {
 300     VirtIOPCIProxy *proxy = opaque;
 301     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 302     hwaddr pa;
 303
 304     switch (addr) {
 305     case VIRTIO_PCI_GUEST_FEATURES:
 306         /* Guest does not negotiate properly?  We have to assume nothing. */
 307         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 308             val = virtio_bus_get_vdev_bad_features(&proxy->bus);
 309         }
 310         virtio_set_features(vdev, val);
 311         break;
 312     case VIRTIO_PCI_QUEUE_PFN:
 313         pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 314         if (pa == 0) {
 315             virtio_pci_reset(DEVICE(proxy));
 316         }
 317         else
 318             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 319         break;
 320     case VIRTIO_PCI_QUEUE_SEL:
 321         if (val < VIRTIO_QUEUE_MAX)
 322             vdev->queue_sel = val;
 323         break;
 324     case VIRTIO_PCI_QUEUE_NOTIFY:
 325         if (val < VIRTIO_QUEUE_MAX) {
 326             virtio_queue_notify(vdev, val);
 327         }
 328         break;
 329     case VIRTIO_PCI_STATUS:
 330         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 331             virtio_pci_stop_ioeventfd(proxy);
 332         }
 333
 334         virtio_set_status(vdev, val & 0xFF);
 335
 336         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 337             virtio_pci_start_ioeventfd(proxy);
 338         }
 339
 340         if (vdev->status == 0) {
 341             virtio_pci_reset(DEVICE(proxy));
 342         }
 343
 344         /* Linux before 2.6.34 drives the device without enabling
 345            the PCI device bus master bit. Enable it automatically
 346            for the guest. This is a PCI spec violation but so is
 347            initiating DMA with bus master bit clear. */
 348         if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) {
 349             pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
 350                                      proxy->pci_dev.config[PCI_COMMAND] |
 351                                      PCI_COMMAND_MASTER, 1);
 352         }
 353         break;
 354     case VIRTIO_MSI_CONFIG_VECTOR:
 355         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 356         /* Make it possible for guest to discover an error took place. */
 357         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 358             val = VIRTIO_NO_VECTOR;
 359         vdev->config_vector = val;
 360         break;
 361     case VIRTIO_MSI_QUEUE_VECTOR:
 362         msix_vector_unuse(&proxy->pci_dev,
 363                           virtio_queue_vector(vdev, vdev->queue_sel));
 364         /* Make it possible for guest to discover an error took place. */
 365         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 366             val = VIRTIO_NO_VECTOR;
 367         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 368         break;
 369     default:
 370         qemu_log_mask(LOG_GUEST_ERROR,
 371                       "%s: unexpected address 0x%x value 0x%x\n",
 372                       __func__, addr, val);
 373         break;
 374     }
 375 }
 376
 377 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 378 {
 379     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 380     uint32_t ret = 0xFFFFFFFF;
 381
 382     switch (addr) {
 383     case VIRTIO_PCI_HOST_FEATURES:
 384         ret = vdev->host_features;
 385         break;
 386     case VIRTIO_PCI_GUEST_FEATURES:
 387         ret = vdev->guest_features;
 388         break;
 389     case VIRTIO_PCI_QUEUE_PFN:
 390         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 391               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 392         break;
 393     case VIRTIO_PCI_QUEUE_NUM:
 394         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 395         break;
 396     case VIRTIO_PCI_QUEUE_SEL:
 397         ret = vdev->queue_sel;
 398         break;
 399     case VIRTIO_PCI_STATUS:
 400         ret = vdev->status;
 401         break;
 402     case VIRTIO_PCI_ISR:
 403         /* reading from the ISR also clears it. */
 404         ret = qatomic_xchg(&vdev->isr, 0);
 405         pci_irq_deassert(&proxy->pci_dev);
 406         break;
 407     case VIRTIO_MSI_CONFIG_VECTOR:
 408         ret = vdev->config_vector;
 409         break;
 410     case VIRTIO_MSI_QUEUE_VECTOR:
 411         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 412         break;
 413     default:
 414         break;
 415     }
 416
 417     return ret;
 418 }
 419
 420 static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
 421                                        unsigned size)
 422 {
 423     VirtIOPCIProxy *proxy = opaque;
 424     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 425     uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
 426     uint64_t val = 0;
 427
 428     if (vdev == NULL) {
 429         return UINT64_MAX;
 430     }
 431
 432     if (addr < config) {
 433         return virtio_ioport_read(proxy, addr);
 434     }
 435     addr -= config;
 436
 437     switch (size) {
 438     case 1:
 439         val = virtio_config_readb(vdev, addr);
 440         break;
 441     case 2:
 442         val = virtio_config_readw(vdev, addr);
 443         if (virtio_is_big_endian(vdev)) {
 444             val = bswap16(val);
 445         }
 446         break;
 447     case 4:
 448         val = virtio_config_readl(vdev, addr);
 449         if (virtio_is_big_endian(vdev)) {
 450             val = bswap32(val);
 451         }
 452         break;
 453     }
 454     return val;
 455 }
 456
 457 static void virtio_pci_config_write(void *opaque, hwaddr addr,
 458                                     uint64_t val, unsigned size)
 459 {
 460     VirtIOPCIProxy *proxy = opaque;
 461     uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
 462     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 463
 464     if (vdev == NULL) {
 465         return;
 466     }
 467
 468     if (addr < config) {
 469         virtio_ioport_write(proxy, addr, val);
 470         return;
 471     }
 472     addr -= config;
 473     /*
 474      * Virtio-PCI is odd. Ioports are LE but config space is target native
 475      * endian.
 476      */
 477     switch (size) {
 478     case 1:
 479         virtio_config_writeb(vdev, addr, val);
 480         break;
 481     case 2:
 482         if (virtio_is_big_endian(vdev)) {
 483             val = bswap16(val);
 484         }
 485         virtio_config_writew(vdev, addr, val);
 486         break;
 487     case 4:
 488         if (virtio_is_big_endian(vdev)) {
 489             val = bswap32(val);
 490         }
 491         virtio_config_writel(vdev, addr, val);
 492         break;
 493     }
 494 }
 495
 496 static const MemoryRegionOps virtio_pci_config_ops = {
 497     .read = virtio_pci_config_read,
 498     .write = virtio_pci_config_write,
 499     .impl = {
 500         .min_access_size = 1,
 501         .max_access_size = 4,
 502     },
 503     .endianness = DEVICE_LITTLE_ENDIAN,
 504 };
 505
 506 static MemoryRegion *virtio_address_space_lookup(VirtIOPCIProxy *proxy,
 507                                                  hwaddr *off, int len)
 508 {
 509     int i;
 510     VirtIOPCIRegion *reg;
 511
 512     for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) {
 513         reg = &proxy->regs[i];
 514         if (*off >= reg->offset &&
 515             *off + len <= reg->offset + reg->size) {
 516             *off -= reg->offset;
 517             return &reg->mr;
 518         }
 519     }
 520
 521     return NULL;
 522 }
 523
 524 /* Below are generic functions to do memcpy from/to an address space,
 525  * without byteswaps, with input validation.
 526  *
 527  * As regular address_space_* APIs all do some kind of byteswap at least for
 528  * some host/target combinations, we are forced to explicitly convert to a
 529  * known-endianness integer value.
 530  * It doesn't really matter which endian format to go through, so the code
 531  * below selects the endian that causes the least amount of work on the given
 532  * host.
 533  *
 534  * Note: host pointer must be aligned.
 535  */
 536 static
 537 void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr,
 538                                 const uint8_t *buf, int len)
 539 {
 540     uint64_t val;
 541     MemoryRegion *mr;
 542
 543     /* address_space_* APIs assume an aligned address.
 544      * As address is under guest control, handle illegal values.
 545      */
 546     addr &= ~(len - 1);
 547
 548     mr = virtio_address_space_lookup(proxy, &addr, len);
 549     if (!mr) {
 550         return;
 551     }
 552
 553     /* Make sure caller aligned buf properly */
 554     assert(!(((uintptr_t)buf) & (len - 1)));
 555
 556     switch (len) {
 557     case 1:
 558         val = pci_get_byte(buf);
 559         break;
 560     case 2:
 561         val = pci_get_word(buf);
 562         break;
 563     case 4:
 564         val = pci_get_long(buf);
 565         break;
 566     default:
 567         /* As length is under guest control, handle illegal values. */
 568         return;
 569     }
 570     memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE,
 571                                  MEMTXATTRS_UNSPECIFIED);
 572 }
 573
 574 static void
 575 virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr,
 576                           uint8_t *buf, int len)
 577 {
 578     uint64_t val;
 579     MemoryRegion *mr;
 580
 581     /* address_space_* APIs assume an aligned address.
 582      * As address is under guest control, handle illegal values.
 583      */
 584     addr &= ~(len - 1);
 585
 586     mr = virtio_address_space_lookup(proxy, &addr, len);
 587     if (!mr) {
 588         return;
 589     }
 590
 591     /* Make sure caller aligned buf properly */
 592     assert(!(((uintptr_t)buf) & (len - 1)));
 593
 594     memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE,
 595                                 MEMTXATTRS_UNSPECIFIED);
 596     switch (len) {
 597     case 1:
 598         pci_set_byte(buf, val);
 599         break;
 600     case 2:
 601         pci_set_word(buf, val);
 602         break;
 603     case 4:
 604         pci_set_long(buf, val);
 605         break;
 606     default:
 607         /* As length is under guest control, handle illegal values. */
 608         break;
 609     }
 610 }
 611
 612 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 613                                 uint32_t val, int len)
 614 {
 615     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
 616     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 617     struct virtio_pci_cfg_cap *cfg;
 618
 619     pci_default_write_config(pci_dev, address, val, len);
 620
 621     if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
 622         pcie_cap_flr_write_config(pci_dev, address, val, len);
 623     }
 624
 625     if (range_covers_byte(address, len, PCI_COMMAND)) {
 626         if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 627             virtio_set_disabled(vdev, true);
 628             virtio_pci_stop_ioeventfd(proxy);
 629             virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 630         } else {
 631             virtio_set_disabled(vdev, false);
 632         }
 633     }
 634
 635     if (proxy->config_cap &&
 636         ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
 637                                                                   pci_cfg_data),
 638                        sizeof cfg->pci_cfg_data)) {
 639         uint32_t off;
 640         uint32_t len;
 641
 642         cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
 643         off = le32_to_cpu(cfg->cap.offset);
 644         len = le32_to_cpu(cfg->cap.length);
 645
 646         if (len == 1 || len == 2 || len == 4) {
 647             assert(len <= sizeof cfg->pci_cfg_data);
 648             virtio_address_space_write(proxy, off, cfg->pci_cfg_data, len);
 649         }
 650     }
 651 }
 652
 653 static uint32_t virtio_read_config(PCIDevice *pci_dev,
 654                                    uint32_t address, int len)
 655 {
 656     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
 657     struct virtio_pci_cfg_cap *cfg;
 658
 659     if (proxy->config_cap &&
 660         ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
 661                                                                   pci_cfg_data),
 662                        sizeof cfg->pci_cfg_data)) {
 663         uint32_t off;
 664         uint32_t len;
 665
 666         cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
 667         off = le32_to_cpu(cfg->cap.offset);
 668         len = le32_to_cpu(cfg->cap.length);
 669
 670         if (len == 1 || len == 2 || len == 4) {
 671             assert(len <= sizeof cfg->pci_cfg_data);
 672             virtio_address_space_read(proxy, off, cfg->pci_cfg_data, len);
 673         }
 674     }
 675
 676     return pci_default_read_config(pci_dev, address, len);
 677 }
 678
 679 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
 680                                         unsigned int vector)
 681 {
 682     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 683     int ret;
 684
 685     if (irqfd->users == 0) {
 686         ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev);
 687         if (ret < 0) {
 688             return ret;
 689         }
 690         irqfd->virq = ret;
 691     }
 692     irqfd->users++;
 693     return 0;
 694 }
 695
 696 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 697                                              unsigned int vector)
 698 {
 699     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 700     if (--irqfd->users == 0) {
 701         kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 702     }
 703 }
 704
 705 static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
 706                                  EventNotifier *n,
 707                                  unsigned int vector)
 708 {
 709     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 710     return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
 711 }
 712
 713 static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
 714                                       EventNotifier *n ,
 715                                       unsigned int vector)
 716 {
 717     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 718     int ret;
 719
 720     ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
 721     assert(ret == 0);
 722 }
 723 static int virtio_pci_get_notifier(VirtIOPCIProxy *proxy, int queue_no,
 724                                       EventNotifier **n, unsigned int *vector)
 725 {
 726     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 727     VirtQueue *vq;
 728
 729     if (queue_no == VIRTIO_CONFIG_IRQ_IDX) {
 730         *n = virtio_config_get_guest_notifier(vdev);
 731         *vector = vdev->config_vector;
 732     } else {
 733         if (!virtio_queue_get_num(vdev, queue_no)) {
 734             return -1;
 735         }
 736         *vector = virtio_queue_vector(vdev, queue_no);
 737         vq = virtio_get_queue(vdev, queue_no);
 738         *n = virtio_queue_get_guest_notifier(vq);
 739     }
 740     return 0;
 741 }
 742
 743 static int kvm_virtio_pci_vector_use_one(VirtIOPCIProxy *proxy, int queue_no)
 744 {
 745     unsigned int vector;
 746     int ret;
 747     EventNotifier *n;
 748     PCIDevice *dev = &proxy->pci_dev;
 749     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 750     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 751
 752     ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector);
 753     if (ret < 0) {
 754         return ret;
 755     }
 756     if (vector >= msix_nr_vectors_allocated(dev)) {
 757         return 0;
 758     }
 759     ret = kvm_virtio_pci_vq_vector_use(proxy, vector);
 760     if (ret < 0) {
 761         goto undo;
 762     }
 763     /*
 764      * If guest supports masking, set up irqfd now.
 765      * Otherwise, delay until unmasked in the frontend.
 766      */
 767     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 768         ret = kvm_virtio_pci_irqfd_use(proxy, n, vector);
 769         if (ret < 0) {
 770             kvm_virtio_pci_vq_vector_release(proxy, vector);
 771             goto undo;
 772         }
 773     }
 774
 775     return 0;
 776 undo:
 777
 778     vector = virtio_queue_vector(vdev, queue_no);
 779     if (vector >= msix_nr_vectors_allocated(dev)) {
 780         return ret;
 781     }
 782     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 783         ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector);
 784         if (ret < 0) {
 785             return ret;
 786         }
 787         kvm_virtio_pci_irqfd_release(proxy, n, vector);
 788     }
 789     return ret;
 790 }
 791 static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
 792 {
 793     int queue_no;
 794     int ret = 0;
 795     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 796
 797     for (queue_no = 0; queue_no < nvqs; queue_no++) {
 798         if (!virtio_queue_get_num(vdev, queue_no)) {
 799             return -1;
 800         }
 801         ret = kvm_virtio_pci_vector_use_one(proxy, queue_no);
 802     }
 803     return ret;
 804 }
 805
 806 static int kvm_virtio_pci_vector_config_use(VirtIOPCIProxy *proxy)
 807 {
 808     return kvm_virtio_pci_vector_use_one(proxy, VIRTIO_CONFIG_IRQ_IDX);
 809 }
 810
 811 static void kvm_virtio_pci_vector_release_one(VirtIOPCIProxy *proxy,
 812                                               int queue_no)
 813 {
 814     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 815     unsigned int vector;
 816     EventNotifier *n;
 817     int ret;
 818     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 819     PCIDevice *dev = &proxy->pci_dev;
 820
 821     ret = virtio_pci_get_notifier(proxy, queue_no, &n, &vector);
 822     if (ret < 0) {
 823         return;
 824     }
 825     if (vector >= msix_nr_vectors_allocated(dev)) {
 826         return;
 827     }
 828     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 829         kvm_virtio_pci_irqfd_release(proxy, n, vector);
 830     }
 831     kvm_virtio_pci_vq_vector_release(proxy, vector);
 832 }
 833
 834 static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
 835 {
 836     int queue_no;
 837     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 838
 839     for (queue_no = 0; queue_no < nvqs; queue_no++) {
 840         if (!virtio_queue_get_num(vdev, queue_no)) {
 841             break;
 842         }
 843         kvm_virtio_pci_vector_release_one(proxy, queue_no);
 844     }
 845 }
 846
 847 static void kvm_virtio_pci_vector_config_release(VirtIOPCIProxy *proxy)
 848 {
 849     kvm_virtio_pci_vector_release_one(proxy, VIRTIO_CONFIG_IRQ_IDX);
 850 }
 851
 852 static int virtio_pci_one_vector_unmask(VirtIOPCIProxy *proxy,
 853                                        unsigned int queue_no,
 854                                        unsigned int vector,
 855                                        MSIMessage msg,
 856                                        EventNotifier *n)
 857 {
 858     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 859     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 860     VirtIOIRQFD *irqfd;
 861     int ret = 0;
 862
 863     if (proxy->vector_irqfd) {
 864         irqfd = &proxy->vector_irqfd[vector];
 865         if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
 866             ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
 867                                                &proxy->pci_dev);
 868             if (ret < 0) {
 869                 return ret;
 870             }
 871             kvm_irqchip_commit_routes(kvm_state);
 872         }
 873     }
 874
 875     /* If guest supports masking, irqfd is already setup, unmask it.
 876      * Otherwise, set it up now.
 877      */
 878     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 879         k->guest_notifier_mask(vdev, queue_no, false);
 880         /* Test after unmasking to avoid losing events. */
 881         if (k->guest_notifier_pending &&
 882             k->guest_notifier_pending(vdev, queue_no)) {
 883             event_notifier_set(n);
 884         }
 885     } else {
 886         ret = kvm_virtio_pci_irqfd_use(proxy, n, vector);
 887     }
 888     return ret;
 889 }
 890
 891 static void virtio_pci_one_vector_mask(VirtIOPCIProxy *proxy,
 892                                              unsigned int queue_no,
 893                                              unsigned int vector,
 894                                              EventNotifier *n)
 895 {
 896     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 897     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 898
 899     /* If guest supports masking, keep irqfd but mask it.
 900      * Otherwise, clean it up now.
 901      */
 902     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 903         k->guest_notifier_mask(vdev, queue_no, true);
 904     } else {
 905         kvm_virtio_pci_irqfd_release(proxy, n, vector);
 906     }
 907 }
 908
 909 static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
 910                                     MSIMessage msg)
 911 {
 912     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 913     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 914     VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
 915     EventNotifier *n;
 916     int ret, index, unmasked = 0;
 917
 918     while (vq) {
 919         index = virtio_get_queue_index(vq);
 920         if (!virtio_queue_get_num(vdev, index)) {
 921             break;
 922         }
 923         if (index < proxy->nvqs_with_notifiers) {
 924             n = virtio_queue_get_guest_notifier(vq);
 925             ret = virtio_pci_one_vector_unmask(proxy, index, vector, msg, n);
 926             if (ret < 0) {
 927                 goto undo;
 928             }
 929             ++unmasked;
 930         }
 931         vq = virtio_vector_next_queue(vq);
 932     }
 933     /* unmask config intr */
 934     n = virtio_config_get_guest_notifier(vdev);
 935     ret = virtio_pci_one_vector_unmask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector,
 936                                        msg, n);
 937     if (ret < 0) {
 938         goto undo_config;
 939     }
 940     return 0;
 941 undo_config:
 942     n = virtio_config_get_guest_notifier(vdev);
 943     virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n);
 944 undo:
 945     vq = virtio_vector_first_queue(vdev, vector);
 946     while (vq && unmasked >= 0) {
 947         index = virtio_get_queue_index(vq);
 948         if (index < proxy->nvqs_with_notifiers) {
 949             n = virtio_queue_get_guest_notifier(vq);
 950             virtio_pci_one_vector_mask(proxy, index, vector, n);
 951             --unmasked;
 952         }
 953         vq = virtio_vector_next_queue(vq);
 954     }
 955     return ret;
 956 }
 957
 958 static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
 959 {
 960     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 961     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 962     VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
 963     EventNotifier *n;
 964     int index;
 965
 966     while (vq) {
 967         index = virtio_get_queue_index(vq);
 968         n = virtio_queue_get_guest_notifier(vq);
 969         if (!virtio_queue_get_num(vdev, index)) {
 970             break;
 971         }
 972         if (index < proxy->nvqs_with_notifiers) {
 973             virtio_pci_one_vector_mask(proxy, index, vector, n);
 974         }
 975         vq = virtio_vector_next_queue(vq);
 976     }
 977     n = virtio_config_get_guest_notifier(vdev);
 978     virtio_pci_one_vector_mask(proxy, VIRTIO_CONFIG_IRQ_IDX, vector, n);
 979 }
 980
 981 static void virtio_pci_vector_poll(PCIDevice *dev,
 982                                    unsigned int vector_start,
 983                                    unsigned int vector_end)
 984 {
 985     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 986     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 987     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 988     int queue_no;
 989     unsigned int vector;
 990     EventNotifier *notifier;
 991     int ret;
 992
 993     for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
 994         ret = virtio_pci_get_notifier(proxy, queue_no, &notifier, &vector);
 995         if (ret < 0) {
 996             break;
 997         }
 998         if (vector < vector_start || vector >= vector_end ||
 999             !msix_is_masked(dev, vector)) {
1000             continue;
1001         }
1002         if (k->guest_notifier_pending) {
1003             if (k->guest_notifier_pending(vdev, queue_no)) {
1004                 msix_set_pending(dev, vector);
1005             }
1006         } else if (event_notifier_test_and_clear(notifier)) {
1007             msix_set_pending(dev, vector);
1008         }
1009     }
1010     /* poll the config intr */
1011     ret = virtio_pci_get_notifier(proxy, VIRTIO_CONFIG_IRQ_IDX, &notifier,
1012                                   &vector);
1013     if (ret < 0) {
1014         return;
1015     }
1016     if (vector < vector_start || vector >= vector_end ||
1017         !msix_is_masked(dev, vector)) {
1018         return;
1019     }
1020     if (k->guest_notifier_pending) {
1021         if (k->guest_notifier_pending(vdev, VIRTIO_CONFIG_IRQ_IDX)) {
1022             msix_set_pending(dev, vector);
1023         }
1024     } else if (event_notifier_test_and_clear(notifier)) {
1025         msix_set_pending(dev, vector);
1026     }
1027 }
1028
1029 void virtio_pci_set_guest_notifier_fd_handler(VirtIODevice *vdev, VirtQueue *vq,
1030                                               int n, bool assign,
1031                                               bool with_irqfd)
1032 {
1033     if (n == VIRTIO_CONFIG_IRQ_IDX) {
1034         virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd);
1035     } else {
1036         virtio_queue_set_guest_notifier_fd_handler(vq, assign, with_irqfd);
1037     }
1038 }
1039
1040 static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
1041                                          bool with_irqfd)
1042 {
1043     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1044     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1045     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1046     VirtQueue *vq = NULL;
1047     EventNotifier *notifier = NULL;
1048
1049     if (n == VIRTIO_CONFIG_IRQ_IDX) {
1050         notifier = virtio_config_get_guest_notifier(vdev);
1051     } else {
1052         vq = virtio_get_queue(vdev, n);
1053         notifier = virtio_queue_get_guest_notifier(vq);
1054     }
1055
1056     if (assign) {
1057         int r = event_notifier_init(notifier, 0);
1058         if (r < 0) {
1059             return r;
1060         }
1061         virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, true, with_irqfd);
1062     } else {
1063         virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, false,
1064                                                  with_irqfd);
1065         event_notifier_cleanup(notifier);
1066     }
1067
1068     if (!msix_enabled(&proxy->pci_dev) &&
1069         vdev->use_guest_notifier_mask &&
1070         vdc->guest_notifier_mask) {
1071         vdc->guest_notifier_mask(vdev, n, !assign);
1072     }
1073
1074     return 0;
1075 }
1076
1077 static bool virtio_pci_query_guest_notifiers(DeviceState *d)
1078 {
1079     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1080     return msix_enabled(&proxy->pci_dev);
1081 }
1082
1083 static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
1084 {
1085     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1086     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1087     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1088     int r, n;
1089     bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
1090         kvm_msi_via_irqfd_enabled();
1091
1092     nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
1093
1094     /* When deassigning, pass a consistent nvqs value
1095      * to avoid leaking notifiers.
1096      */
1097     assert(assign || nvqs == proxy->nvqs_with_notifiers);
1098
1099     proxy->nvqs_with_notifiers = nvqs;
1100
1101     /* Must unset vector notifier while guest notifier is still assigned */
1102     if ((proxy->vector_irqfd || k->guest_notifier_mask) && !assign) {
1103         msix_unset_vector_notifiers(&proxy->pci_dev);
1104         if (proxy->vector_irqfd) {
1105             kvm_virtio_pci_vector_release(proxy, nvqs);
1106             kvm_virtio_pci_vector_config_release(proxy);
1107             g_free(proxy->vector_irqfd);
1108             proxy->vector_irqfd = NULL;
1109         }
1110     }
1111
1112     for (n = 0; n < nvqs; n++) {
1113         if (!virtio_queue_get_num(vdev, n)) {
1114             break;
1115         }
1116
1117         r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd);
1118         if (r < 0) {
1119             goto assign_error;
1120         }
1121     }
1122     r = virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, assign,
1123                                       with_irqfd);
1124     if (r < 0) {
1125         goto config_assign_error;
1126     }
1127     /* Must set vector notifier after guest notifier has been assigned */
1128     if ((with_irqfd || k->guest_notifier_mask) && assign) {
1129         if (with_irqfd) {
1130             proxy->vector_irqfd =
1131                 g_malloc0(sizeof(*proxy->vector_irqfd) *
1132                           msix_nr_vectors_allocated(&proxy->pci_dev));
1133             r = kvm_virtio_pci_vector_use(proxy, nvqs);
1134             if (r < 0) {
1135                 goto config_assign_error;
1136             }
1137         }
1138         r = kvm_virtio_pci_vector_config_use(proxy);
1139         if (r < 0) {
1140             goto config_error;
1141         }
1142         r = msix_set_vector_notifiers(&proxy->pci_dev, virtio_pci_vector_unmask,
1143                                       virtio_pci_vector_mask,
1144                                       virtio_pci_vector_poll);
1145         if (r < 0) {
1146             goto notifiers_error;
1147         }
1148     }
1149
1150     return 0;
1151
1152 notifiers_error:
1153     if (with_irqfd) {
1154         assert(assign);
1155         kvm_virtio_pci_vector_release(proxy, nvqs);
1156     }
1157 config_error:
1158     kvm_virtio_pci_vector_config_release(proxy);
1159 config_assign_error:
1160     virtio_pci_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, !assign,
1161                                   with_irqfd);
1162 assign_error:
1163     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
1164     assert(assign);
1165     while (--n >= 0) {
1166         virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
1167     }
1168     return r;
1169 }
1170
1171 static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n,
1172                                            MemoryRegion *mr, bool assign)
1173 {
1174     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1175     int offset;
1176
1177     if (n >= VIRTIO_QUEUE_MAX || !virtio_pci_modern(proxy) ||
1178         virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) {
1179         return -1;
1180     }
1181
1182     if (assign) {
1183         offset = virtio_pci_queue_mem_mult(proxy) * n;
1184         memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1);
1185     } else {
1186         memory_region_del_subregion(&proxy->notify.mr, mr);
1187     }
1188
1189     return 0;
1190 }
1191
1192 static void virtio_pci_vmstate_change(DeviceState *d, bool running)
1193 {
1194     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1195     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1196
1197     if (running) {
1198         /* Old QEMU versions did not set bus master enable on status write.
1199          * Detect DRIVER set and enable it.
1200          */
1201         if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
1202             (vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
1203             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
1204             pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
1205                                      proxy->pci_dev.config[PCI_COMMAND] |
1206                                      PCI_COMMAND_MASTER, 1);
1207         }
1208         virtio_pci_start_ioeventfd(proxy);
1209     } else {
1210         virtio_pci_stop_ioeventfd(proxy);
1211     }
1212 }
1213
1214 /*
1215  * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
1216  */
1217
1218 static int virtio_pci_query_nvectors(DeviceState *d)
1219 {
1220     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1221
1222     return proxy->nvectors;
1223 }
1224
1225 static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
1226 {
1227     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1228     PCIDevice *dev = &proxy->pci_dev;
1229
1230     return pci_get_address_space(dev);
1231 }
1232
1233 static bool virtio_pci_iommu_enabled(DeviceState *d)
1234 {
1235     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1236     PCIDevice *dev = &proxy->pci_dev;
1237     AddressSpace *dma_as = pci_device_iommu_address_space(dev);
1238
1239     if (dma_as == &address_space_memory) {
1240         return false;
1241     }
1242
1243     return true;
1244 }
1245
1246 static bool virtio_pci_queue_enabled(DeviceState *d, int n)
1247 {
1248     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1249     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1250
1251     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1252         return proxy->vqs[n].enabled;
1253     }
1254
1255     return virtio_queue_enabled_legacy(vdev, n);
1256 }
1257
1258 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
1259                                    struct virtio_pci_cap *cap)
1260 {
1261     PCIDevice *dev = &proxy->pci_dev;
1262     int offset;
1263
1264     offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0,
1265                                 cap->cap_len, &error_abort);
1266
1267     assert(cap->cap_len >= sizeof *cap);
1268     memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len,
1269            cap->cap_len - PCI_CAP_FLAGS);
1270
1271     return offset;
1272 }
1273
1274 static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr,
1275                                        unsigned size)
1276 {
1277     VirtIOPCIProxy *proxy = opaque;
1278     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1279     uint32_t val = 0;
1280     int i;
1281
1282     if (vdev == NULL) {
1283         return UINT64_MAX;
1284     }
1285
1286     switch (addr) {
1287     case VIRTIO_PCI_COMMON_DFSELECT:
1288         val = proxy->dfselect;
1289         break;
1290     case VIRTIO_PCI_COMMON_DF:
1291         if (proxy->dfselect <= 1) {
1292             VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1293
1294             val = (vdev->host_features & ~vdc->legacy_features) >>
1295                 (32 * proxy->dfselect);
1296         }
1297         break;
1298     case VIRTIO_PCI_COMMON_GFSELECT:
1299         val = proxy->gfselect;
1300         break;
1301     case VIRTIO_PCI_COMMON_GF:
1302         if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1303             val = proxy->guest_features[proxy->gfselect];
1304         }
1305         break;
1306     case VIRTIO_PCI_COMMON_MSIX:
1307         val = vdev->config_vector;
1308         break;
1309     case VIRTIO_PCI_COMMON_NUMQ:
1310         for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) {
1311             if (virtio_queue_get_num(vdev, i)) {
1312                 val = i + 1;
1313             }
1314         }
1315         break;
1316     case VIRTIO_PCI_COMMON_STATUS:
1317         val = vdev->status;
1318         break;
1319     case VIRTIO_PCI_COMMON_CFGGENERATION:
1320         val = vdev->generation;
1321         break;
1322     case VIRTIO_PCI_COMMON_Q_SELECT:
1323         val = vdev->queue_sel;
1324         break;
1325     case VIRTIO_PCI_COMMON_Q_SIZE:
1326         val = virtio_queue_get_num(vdev, vdev->queue_sel);
1327         break;
1328     case VIRTIO_PCI_COMMON_Q_MSIX:
1329         val = virtio_queue_vector(vdev, vdev->queue_sel);
1330         break;
1331     case VIRTIO_PCI_COMMON_Q_ENABLE:
1332         val = proxy->vqs[vdev->queue_sel].enabled;
1333         break;
1334     case VIRTIO_PCI_COMMON_Q_NOFF:
1335         /* Simply map queues in order */
1336         val = vdev->queue_sel;
1337         break;
1338     case VIRTIO_PCI_COMMON_Q_DESCLO:
1339         val = proxy->vqs[vdev->queue_sel].desc[0];
1340         break;
1341     case VIRTIO_PCI_COMMON_Q_DESCHI:
1342         val = proxy->vqs[vdev->queue_sel].desc[1];
1343         break;
1344     case VIRTIO_PCI_COMMON_Q_AVAILLO:
1345         val = proxy->vqs[vdev->queue_sel].avail[0];
1346         break;
1347     case VIRTIO_PCI_COMMON_Q_AVAILHI:
1348         val = proxy->vqs[vdev->queue_sel].avail[1];
1349         break;
1350     case VIRTIO_PCI_COMMON_Q_USEDLO:
1351         val = proxy->vqs[vdev->queue_sel].used[0];
1352         break;
1353     case VIRTIO_PCI_COMMON_Q_USEDHI:
1354         val = proxy->vqs[vdev->queue_sel].used[1];
1355         break;
1356     default:
1357         val = 0;
1358     }
1359
1360     return val;
1361 }
1362
1363 static void virtio_pci_common_write(void *opaque, hwaddr addr,
1364                                     uint64_t val, unsigned size)
1365 {
1366     VirtIOPCIProxy *proxy = opaque;
1367     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1368
1369     if (vdev == NULL) {
1370         return;
1371     }
1372
1373     switch (addr) {
1374     case VIRTIO_PCI_COMMON_DFSELECT:
1375         proxy->dfselect = val;
1376         break;
1377     case VIRTIO_PCI_COMMON_GFSELECT:
1378         proxy->gfselect = val;
1379         break;
1380     case VIRTIO_PCI_COMMON_GF:
1381         if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1382             proxy->guest_features[proxy->gfselect] = val;
1383             virtio_set_features(vdev,
1384                                 (((uint64_t)proxy->guest_features[1]) << 32) |
1385                                 proxy->guest_features[0]);
1386         }
1387         break;
1388     case VIRTIO_PCI_COMMON_MSIX:
1389         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
1390         /* Make it possible for guest to discover an error took place. */
1391         if (msix_vector_use(&proxy->pci_dev, val) < 0) {
1392             val = VIRTIO_NO_VECTOR;
1393         }
1394         vdev->config_vector = val;
1395         break;
1396     case VIRTIO_PCI_COMMON_STATUS:
1397         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1398             virtio_pci_stop_ioeventfd(proxy);
1399         }
1400
1401         virtio_set_status(vdev, val & 0xFF);
1402
1403         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
1404             virtio_pci_start_ioeventfd(proxy);
1405         }
1406
1407         if (vdev->status == 0) {
1408             virtio_pci_reset(DEVICE(proxy));
1409         }
1410
1411         break;
1412     case VIRTIO_PCI_COMMON_Q_SELECT:
1413         if (val < VIRTIO_QUEUE_MAX) {
1414             vdev->queue_sel = val;
1415         }
1416         break;
1417     case VIRTIO_PCI_COMMON_Q_SIZE:
1418         proxy->vqs[vdev->queue_sel].num = val;
1419         virtio_queue_set_num(vdev, vdev->queue_sel,
1420                              proxy->vqs[vdev->queue_sel].num);
1421         break;
1422     case VIRTIO_PCI_COMMON_Q_MSIX:
1423         msix_vector_unuse(&proxy->pci_dev,
1424                           virtio_queue_vector(vdev, vdev->queue_sel));
1425         /* Make it possible for guest to discover an error took place. */
1426         if (msix_vector_use(&proxy->pci_dev, val) < 0) {
1427             val = VIRTIO_NO_VECTOR;
1428         }
1429         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
1430         break;
1431     case VIRTIO_PCI_COMMON_Q_ENABLE:
1432         if (val == 1) {
1433             virtio_queue_set_num(vdev, vdev->queue_sel,
1434                                  proxy->vqs[vdev->queue_sel].num);
1435             virtio_queue_set_rings(vdev, vdev->queue_sel,
1436                        ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 |
1437                        proxy->vqs[vdev->queue_sel].desc[0],
1438                        ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 |
1439                        proxy->vqs[vdev->queue_sel].avail[0],
1440                        ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
1441                        proxy->vqs[vdev->queue_sel].used[0]);
1442             proxy->vqs[vdev->queue_sel].enabled = 1;
1443         } else {
1444             virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val);
1445         }
1446         break;
1447     case VIRTIO_PCI_COMMON_Q_DESCLO:
1448         proxy->vqs[vdev->queue_sel].desc[0] = val;
1449         break;
1450     case VIRTIO_PCI_COMMON_Q_DESCHI:
1451         proxy->vqs[vdev->queue_sel].desc[1] = val;
1452         break;
1453     case VIRTIO_PCI_COMMON_Q_AVAILLO:
1454         proxy->vqs[vdev->queue_sel].avail[0] = val;
1455         break;
1456     case VIRTIO_PCI_COMMON_Q_AVAILHI:
1457         proxy->vqs[vdev->queue_sel].avail[1] = val;
1458         break;
1459     case VIRTIO_PCI_COMMON_Q_USEDLO:
1460         proxy->vqs[vdev->queue_sel].used[0] = val;
1461         break;
1462     case VIRTIO_PCI_COMMON_Q_USEDHI:
1463         proxy->vqs[vdev->queue_sel].used[1] = val;
1464         break;
1465     default:
1466         break;
1467     }
1468 }
1469
1470
1471 static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr,
1472                                        unsigned size)
1473 {
1474     VirtIOPCIProxy *proxy = opaque;
1475     if (virtio_bus_get_device(&proxy->bus) == NULL) {
1476         return UINT64_MAX;
1477     }
1478
1479     return 0;
1480 }
1481
1482 static void virtio_pci_notify_write(void *opaque, hwaddr addr,
1483                                     uint64_t val, unsigned size)
1484 {
1485     VirtIOPCIProxy *proxy = opaque;
1486     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1487
1488     unsigned queue = addr / virtio_pci_queue_mem_mult(proxy);
1489
1490     if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1491         virtio_queue_notify(vdev, queue);
1492     }
1493 }
1494
1495 static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
1496                                         uint64_t val, unsigned size)
1497 {
1498     VirtIOPCIProxy *proxy = opaque;
1499     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1500
1501     unsigned queue = val;
1502
1503     if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1504         virtio_queue_notify(vdev, queue);
1505     }
1506 }
1507
1508 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
1509                                     unsigned size)
1510 {
1511     VirtIOPCIProxy *proxy = opaque;
1512     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1513     uint64_t val;
1514
1515     if (vdev == NULL) {
1516         return UINT64_MAX;
1517     }
1518
1519     val = qatomic_xchg(&vdev->isr, 0);
1520     pci_irq_deassert(&proxy->pci_dev);
1521     return val;
1522 }
1523
1524 static void virtio_pci_isr_write(void *opaque, hwaddr addr,
1525                                  uint64_t val, unsigned size)
1526 {
1527 }
1528
1529 static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr,
1530                                        unsigned size)
1531 {
1532     VirtIOPCIProxy *proxy = opaque;
1533     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1534     uint64_t val;
1535
1536     if (vdev == NULL) {
1537         return UINT64_MAX;
1538     }
1539
1540     switch (size) {
1541     case 1:
1542         val = virtio_config_modern_readb(vdev, addr);
1543         break;
1544     case 2:
1545         val = virtio_config_modern_readw(vdev, addr);
1546         break;
1547     case 4:
1548         val = virtio_config_modern_readl(vdev, addr);
1549         break;
1550     default:
1551         val = 0;
1552         break;
1553     }
1554     return val;
1555 }
1556
1557 static void virtio_pci_device_write(void *opaque, hwaddr addr,
1558                                     uint64_t val, unsigned size)
1559 {
1560     VirtIOPCIProxy *proxy = opaque;
1561     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1562
1563     if (vdev == NULL) {
1564         return;
1565     }
1566
1567     switch (size) {
1568     case 1:
1569         virtio_config_modern_writeb(vdev, addr, val);
1570         break;
1571     case 2:
1572         virtio_config_modern_writew(vdev, addr, val);
1573         break;
1574     case 4:
1575         virtio_config_modern_writel(vdev, addr, val);
1576         break;
1577     }
1578 }
1579
1580 static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy,
1581                                            const char *vdev_name)
1582 {
1583     static const MemoryRegionOps common_ops = {
1584         .read = virtio_pci_common_read,
1585         .write = virtio_pci_common_write,
1586         .impl = {
1587             .min_access_size = 1,
1588             .max_access_size = 4,
1589         },
1590         .endianness = DEVICE_LITTLE_ENDIAN,
1591     };
1592     static const MemoryRegionOps isr_ops = {
1593         .read = virtio_pci_isr_read,
1594         .write = virtio_pci_isr_write,
1595         .impl = {
1596             .min_access_size = 1,
1597             .max_access_size = 4,
1598         },
1599         .endianness = DEVICE_LITTLE_ENDIAN,
1600     };
1601     static const MemoryRegionOps device_ops = {
1602         .read = virtio_pci_device_read,
1603         .write = virtio_pci_device_write,
1604         .impl = {
1605             .min_access_size = 1,
1606             .max_access_size = 4,
1607         },
1608         .endianness = DEVICE_LITTLE_ENDIAN,
1609     };
1610     static const MemoryRegionOps notify_ops = {
1611         .read = virtio_pci_notify_read,
1612         .write = virtio_pci_notify_write,
1613         .impl = {
1614             .min_access_size = 1,
1615             .max_access_size = 4,
1616         },
1617         .endianness = DEVICE_LITTLE_ENDIAN,
1618     };
1619     static const MemoryRegionOps notify_pio_ops = {
1620         .read = virtio_pci_notify_read,
1621         .write = virtio_pci_notify_write_pio,
1622         .impl = {
1623             .min_access_size = 1,
1624             .max_access_size = 4,
1625         },
1626         .endianness = DEVICE_LITTLE_ENDIAN,
1627     };
1628     g_autoptr(GString) name = g_string_new(NULL);
1629
1630     g_string_printf(name, "virtio-pci-common-%s", vdev_name);
1631     memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
1632                           &common_ops,
1633                           proxy,
1634                           name->str,
1635                           proxy->common.size);
1636
1637     g_string_printf(name, "virtio-pci-isr-%s", vdev_name);
1638     memory_region_init_io(&proxy->isr.mr, OBJECT(proxy),
1639                           &isr_ops,
1640                           proxy,
1641                           name->str,
1642                           proxy->isr.size);
1643
1644     g_string_printf(name, "virtio-pci-device-%s", vdev_name);
1645     memory_region_init_io(&proxy->device.mr, OBJECT(proxy),
1646                           &device_ops,
1647                           proxy,
1648                           name->str,
1649                           proxy->device.size);
1650
1651     g_string_printf(name, "virtio-pci-notify-%s", vdev_name);
1652     memory_region_init_io(&proxy->notify.mr, OBJECT(proxy),
1653                           &notify_ops,
1654                           proxy,
1655                           name->str,
1656                           proxy->notify.size);
1657
1658     g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name);
1659     memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
1660                           &notify_pio_ops,
1661                           proxy,
1662                           name->str,
1663                           proxy->notify_pio.size);
1664 }
1665
1666 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
1667                                          VirtIOPCIRegion *region,
1668                                          struct virtio_pci_cap *cap,
1669                                          MemoryRegion *mr,
1670                                          uint8_t bar)
1671 {
1672     memory_region_add_subregion(mr, region->offset, &region->mr);
1673
1674     cap->cfg_type = region->type;
1675     cap->bar = bar;
1676     cap->offset = cpu_to_le32(region->offset);
1677     cap->length = cpu_to_le32(region->size);
1678     virtio_pci_add_mem_cap(proxy, cap);
1679
1680 }
1681
1682 static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
1683                                              VirtIOPCIRegion *region,
1684                                              struct virtio_pci_cap *cap)
1685 {
1686     virtio_pci_modern_region_map(proxy, region, cap,
1687                                  &proxy->modern_bar, proxy->modern_mem_bar_idx);
1688 }
1689
1690 static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
1691                                             VirtIOPCIRegion *region,
1692                                             struct virtio_pci_cap *cap)
1693 {
1694     virtio_pci_modern_region_map(proxy, region, cap,
1695                                  &proxy->io_bar, proxy->modern_io_bar_idx);
1696 }
1697
1698 static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
1699                                                VirtIOPCIRegion *region)
1700 {
1701     memory_region_del_subregion(&proxy->modern_bar,
1702                                 &region->mr);
1703 }
1704
1705 static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
1706                                               VirtIOPCIRegion *region)
1707 {
1708     memory_region_del_subregion(&proxy->io_bar,
1709                                 &region->mr);
1710 }
1711
1712 static void virtio_pci_pre_plugged(DeviceState *d, Error **errp)
1713 {
1714     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1715     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1716
1717     if (virtio_pci_modern(proxy)) {
1718         virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
1719     }
1720
1721     virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE);
1722 }
1723
1724 /* This is called by virtio-bus just after the device is plugged. */
1725 static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
1726 {
1727     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1728     VirtioBusState *bus = &proxy->bus;
1729     bool legacy = virtio_pci_legacy(proxy);
1730     bool modern;
1731     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1732     uint8_t *config;
1733     uint32_t size;
1734     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1735
1736     /*
1737      * Virtio capabilities present without
1738      * VIRTIO_F_VERSION_1 confuses guests
1739      */
1740     if (!proxy->ignore_backend_features &&
1741             !virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) {
1742         virtio_pci_disable_modern(proxy);
1743
1744         if (!legacy) {
1745             error_setg(errp, "Device doesn't support modern mode, and legacy"
1746                              " mode is disabled");
1747             error_append_hint(errp, "Set disable-legacy to off\n");
1748
1749             return;
1750         }
1751     }
1752
1753     modern = virtio_pci_modern(proxy);
1754
1755     config = proxy->pci_dev.config;
1756     if (proxy->class_code) {
1757         pci_config_set_class(config, proxy->class_code);
1758     }
1759
1760     if (legacy) {
1761         if (!virtio_legacy_allowed(vdev)) {
1762             /*
1763              * To avoid migration issues, we allow legacy mode when legacy
1764              * check is disabled in the old machine types (< 5.1).
1765              */
1766             if (virtio_legacy_check_disabled(vdev)) {
1767                 warn_report("device is modern-only, but for backward "
1768                             "compatibility legacy is allowed");
1769             } else {
1770                 error_setg(errp,
1771                            "device is modern-only, use disable-legacy=on");
1772                 return;
1773             }
1774         }
1775         if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1776             error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by"
1777                        " neither legacy nor transitional device");
1778             return ;
1779         }
1780         /*
1781          * Legacy and transitional devices use specific subsystem IDs.
1782          * Note that the subsystem vendor ID (config + PCI_SUBSYSTEM_VENDOR_ID)
1783          * is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default.
1784          */
1785         pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
1786     } else {
1787         /* pure virtio-1.0 */
1788         pci_set_word(config + PCI_VENDOR_ID,
1789                      PCI_VENDOR_ID_REDHAT_QUMRANET);
1790         pci_set_word(config + PCI_DEVICE_ID,
1791                      0x1040 + virtio_bus_get_vdev_id(bus));
1792         pci_config_set_revision(config, 1);
1793     }
1794     config[PCI_INTERRUPT_PIN] = 1;
1795
1796
1797     if (modern) {
1798         struct virtio_pci_cap cap = {
1799             .cap_len = sizeof cap,
1800         };
1801         struct virtio_pci_notify_cap notify = {
1802             .cap.cap_len = sizeof notify,
1803             .notify_off_multiplier =
1804                 cpu_to_le32(virtio_pci_queue_mem_mult(proxy)),
1805         };
1806         struct virtio_pci_cfg_cap cfg = {
1807             .cap.cap_len = sizeof cfg,
1808             .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
1809         };
1810         struct virtio_pci_notify_cap notify_pio = {
1811             .cap.cap_len = sizeof notify,
1812             .notify_off_multiplier = cpu_to_le32(0x0),
1813         };
1814
1815         struct virtio_pci_cfg_cap *cfg_mask;
1816
1817         virtio_pci_modern_regions_init(proxy, vdev->name);
1818
1819         virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
1820         virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
1821         virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
1822         virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
1823
1824         if (modern_pio) {
1825             memory_region_init(&proxy->io_bar, OBJECT(proxy),
1826                                "virtio-pci-io", 0x4);
1827
1828             pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx,
1829                              PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
1830
1831             virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
1832                                             &notify_pio.cap);
1833         }
1834
1835         pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx,
1836                          PCI_BASE_ADDRESS_SPACE_MEMORY |
1837                          PCI_BASE_ADDRESS_MEM_PREFETCH |
1838                          PCI_BASE_ADDRESS_MEM_TYPE_64,
1839                          &proxy->modern_bar);
1840
1841         proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap);
1842         cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap);
1843         pci_set_byte(&cfg_mask->cap.bar, ~0x0);
1844         pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0);
1845         pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0);
1846         pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
1847     }
1848
1849     if (proxy->nvectors) {
1850         int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
1851                                           proxy->msix_bar_idx, NULL);
1852         if (err) {
1853             /* Notice when a system that supports MSIx can't initialize it */
1854             if (err != -ENOTSUP) {
1855                 warn_report("unable to init msix vectors to %" PRIu32,
1856                             proxy->nvectors);
1857             }
1858             proxy->nvectors = 0;
1859         }
1860     }
1861
1862     proxy->pci_dev.config_write = virtio_write_config;
1863     proxy->pci_dev.config_read = virtio_read_config;
1864
1865     if (legacy) {
1866         size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
1867             + virtio_bus_get_vdev_config_len(bus);
1868         size = pow2ceil(size);
1869
1870         memory_region_init_io(&proxy->bar, OBJECT(proxy),
1871                               &virtio_pci_config_ops,
1872                               proxy, "virtio-pci", size);
1873
1874         pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
1875                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
1876     }
1877 }
1878
1879 static void virtio_pci_device_unplugged(DeviceState *d)
1880 {
1881     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1882     bool modern = virtio_pci_modern(proxy);
1883     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1884
1885     virtio_pci_stop_ioeventfd(proxy);
1886
1887     if (modern) {
1888         virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
1889         virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
1890         virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
1891         virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
1892         if (modern_pio) {
1893             virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
1894         }
1895     }
1896 }
1897
1898 static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
1899 {
1900     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
1901     VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
1902     bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
1903                      !pci_bus_is_root(pci_get_bus(pci_dev));
1904
1905     if (kvm_enabled() && !kvm_has_many_ioeventfds()) {
1906         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1907     }
1908
1909     /* fd-based ioevents can't be synchronized in record/replay */
1910     if (replay_mode != REPLAY_MODE_NONE) {
1911         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1912     }
1913
1914     /*
1915      * virtio pci bar layout used by default.
1916      * subclasses can re-arrange things if needed.
1917      *
1918      *   region 0   --  virtio legacy io bar
1919      *   region 1   --  msi-x bar
1920      *   region 2   --  virtio modern io bar (off by default)
1921      *   region 4+5 --  virtio modern memory (64bit) bar
1922      *
1923      */
1924     proxy->legacy_io_bar_idx  = 0;
1925     proxy->msix_bar_idx       = 1;
1926     proxy->modern_io_bar_idx  = 2;
1927     proxy->modern_mem_bar_idx = 4;
1928
1929     proxy->common.offset = 0x0;
1930     proxy->common.size = 0x1000;
1931     proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG;
1932
1933     proxy->isr.offset = 0x1000;
1934     proxy->isr.size = 0x1000;
1935     proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG;
1936
1937     proxy->device.offset = 0x2000;
1938     proxy->device.size = 0x1000;
1939     proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG;
1940
1941     proxy->notify.offset = 0x3000;
1942     proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX;
1943     proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1944
1945     proxy->notify_pio.offset = 0x0;
1946     proxy->notify_pio.size = 0x4;
1947     proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1948
1949     /* subclasses can enforce modern, so do this unconditionally */
1950     memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
1951                        /* PCI BAR regions must be powers of 2 */
1952                        pow2ceil(proxy->notify.offset + proxy->notify.size));
1953
1954     if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) {
1955         proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
1956     }
1957
1958     if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) {
1959         error_setg(errp, "device cannot work as neither modern nor legacy mode"
1960                    " is enabled");
1961         error_append_hint(errp, "Set either disable-modern or disable-legacy"
1962                           " to off\n");
1963         return;
1964     }
1965
1966     if (pcie_port && pci_is_express(pci_dev)) {
1967         int pos;
1968         uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
1969
1970         pos = pcie_endpoint_cap_init(pci_dev, 0);
1971         assert(pos > 0);
1972
1973         pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0,
1974                                  PCI_PM_SIZEOF, errp);
1975         if (pos < 0) {
1976             return;
1977         }
1978
1979         pci_dev->exp.pm_cap = pos;
1980
1981         /*
1982          * Indicates that this function complies with revision 1.2 of the
1983          * PCI Power Management Interface Specification.
1984          */
1985         pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
1986
1987         if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
1988             pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset,
1989                           PCI_ERR_SIZEOF, NULL);
1990             last_pcie_cap_offset += PCI_ERR_SIZEOF;
1991         }
1992
1993         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) {
1994             /* Init error enabling flags */
1995             pcie_cap_deverr_init(pci_dev);
1996         }
1997
1998         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_LNKCTL) {
1999             /* Init Link Control Register */
2000             pcie_cap_lnkctl_init(pci_dev);
2001         }
2002
2003         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) {
2004             /* Init Power Management Control Register */
2005             pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL,
2006                          PCI_PM_CTRL_STATE_MASK);
2007         }
2008
2009         if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
2010             pcie_ats_init(pci_dev, last_pcie_cap_offset,
2011                           proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
2012             last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
2013         }
2014
2015         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
2016             /* Set Function Level Reset capability bit */
2017             pcie_cap_flr_init(pci_dev);
2018         }
2019     } else {
2020         /*
2021          * make future invocations of pci_is_express() return false
2022          * and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
2023          */
2024         pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2025     }
2026
2027     virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
2028     if (k->realize) {
2029         k->realize(proxy, errp);
2030     }
2031 }
2032
2033 static void virtio_pci_exit(PCIDevice *pci_dev)
2034 {
2035     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
2036     bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
2037                      !pci_bus_is_root(pci_get_bus(pci_dev));
2038
2039     msix_uninit_exclusive_bar(pci_dev);
2040     if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
2041         pci_is_express(pci_dev)) {
2042         pcie_aer_exit(pci_dev);
2043     }
2044 }
2045
2046 static void virtio_pci_reset(DeviceState *qdev)
2047 {
2048     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2049     VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
2050     PCIDevice *dev = PCI_DEVICE(qdev);
2051     int i;
2052
2053     virtio_pci_stop_ioeventfd(proxy);
2054     virtio_bus_reset(bus);
2055     msix_unuse_all_vectors(&proxy->pci_dev);
2056
2057     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2058         proxy->vqs[i].enabled = 0;
2059         proxy->vqs[i].num = 0;
2060         proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0;
2061         proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0;
2062         proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0;
2063     }
2064
2065     if (pci_is_express(dev)) {
2066         pcie_cap_deverr_reset(dev);
2067         pcie_cap_lnkctl_reset(dev);
2068
2069         pci_set_word(dev->config + dev->exp.pm_cap + PCI_PM_CTRL, 0);
2070     }
2071 }
2072
2073 static Property virtio_pci_properties[] = {
2074     DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
2075                     VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
2076     DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
2077                     VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
2078     DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
2079                     VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
2080     DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
2081                     VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
2082     DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags,
2083                     VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false),
2084     DEFINE_PROP_BOOL("x-ignore-backend-features", VirtIOPCIProxy,
2085                      ignore_backend_features, false),
2086     DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags,
2087                     VIRTIO_PCI_FLAG_ATS_BIT, false),
2088     DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags,
2089                     VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true),
2090     DEFINE_PROP_BIT("x-pcie-deverr-init", VirtIOPCIProxy, flags,
2091                     VIRTIO_PCI_FLAG_INIT_DEVERR_BIT, true),
2092     DEFINE_PROP_BIT("x-pcie-lnkctl-init", VirtIOPCIProxy, flags,
2093                     VIRTIO_PCI_FLAG_INIT_LNKCTL_BIT, true),
2094     DEFINE_PROP_BIT("x-pcie-pm-init", VirtIOPCIProxy, flags,
2095                     VIRTIO_PCI_FLAG_INIT_PM_BIT, true),
2096     DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags,
2097                     VIRTIO_PCI_FLAG_INIT_FLR_BIT, true),
2098     DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags,
2099                     VIRTIO_PCI_FLAG_AER_BIT, false),
2100     DEFINE_PROP_END_OF_LIST(),
2101 };
2102
2103 static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
2104 {
2105     VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
2106     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2107     PCIDevice *pci_dev = &proxy->pci_dev;
2108
2109     if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
2110         virtio_pci_modern(proxy)) {
2111         pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
2112     }
2113
2114     vpciklass->parent_dc_realize(qdev, errp);
2115 }
2116
2117 static void virtio_pci_class_init(ObjectClass *klass, void *data)
2118 {
2119     DeviceClass *dc = DEVICE_CLASS(klass);
2120     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
2121     VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
2122
2123     device_class_set_props(dc, virtio_pci_properties);
2124     k->realize = virtio_pci_realize;
2125     k->exit = virtio_pci_exit;
2126     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
2127     k->revision = VIRTIO_PCI_ABI_VERSION;
2128     k->class_id = PCI_CLASS_OTHERS;
2129     device_class_set_parent_realize(dc, virtio_pci_dc_realize,
2130                                     &vpciklass->parent_dc_realize);
2131     dc->reset = virtio_pci_reset;
2132 }
2133
2134 static const TypeInfo virtio_pci_info = {
2135     .name          = TYPE_VIRTIO_PCI,
2136     .parent        = TYPE_PCI_DEVICE,
2137     .instance_size = sizeof(VirtIOPCIProxy),
2138     .class_init    = virtio_pci_class_init,
2139     .class_size    = sizeof(VirtioPCIClass),
2140     .abstract      = true,
2141 };
2142
2143 static Property virtio_pci_generic_properties[] = {
2144     DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy,
2145                             ON_OFF_AUTO_AUTO),
2146     DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false),
2147     DEFINE_PROP_END_OF_LIST(),
2148 };
2149
2150 static void virtio_pci_base_class_init(ObjectClass *klass, void *data)
2151 {
2152     const VirtioPCIDeviceTypeInfo *t = data;
2153     if (t->class_init) {
2154         t->class_init(klass, NULL);
2155     }
2156 }
2157
2158 static void virtio_pci_generic_class_init(ObjectClass *klass, void *data)
2159 {
2160     DeviceClass *dc = DEVICE_CLASS(klass);
2161
2162     device_class_set_props(dc, virtio_pci_generic_properties);
2163 }
2164
2165 static void virtio_pci_transitional_instance_init(Object *obj)
2166 {
2167     VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2168
2169     proxy->disable_legacy = ON_OFF_AUTO_OFF;
2170     proxy->disable_modern = false;
2171 }
2172
2173 static void virtio_pci_non_transitional_instance_init(Object *obj)
2174 {
2175     VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2176
2177     proxy->disable_legacy = ON_OFF_AUTO_ON;
2178     proxy->disable_modern = false;
2179 }
2180
2181 void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t)
2182 {
2183     char *base_name = NULL;
2184     TypeInfo base_type_info = {
2185         .name          = t->base_name,
2186         .parent        = t->parent ? t->parent : TYPE_VIRTIO_PCI,
2187         .instance_size = t->instance_size,
2188         .instance_init = t->instance_init,
2189         .class_size    = t->class_size,
2190         .abstract      = true,
2191         .interfaces    = t->interfaces,
2192     };
2193     TypeInfo generic_type_info = {
2194         .name = t->generic_name,
2195         .parent = base_type_info.name,
2196         .class_init = virtio_pci_generic_class_init,
2197         .interfaces = (InterfaceInfo[]) {
2198             { INTERFACE_PCIE_DEVICE },
2199             { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2200             { }
2201         },
2202     };
2203
2204     if (!base_type_info.name) {
2205         /* No base type -> register a single generic device type */
2206         /* use intermediate %s-base-type to add generic device props */
2207         base_name = g_strdup_printf("%s-base-type", t->generic_name);
2208         base_type_info.name = base_name;
2209         base_type_info.class_init = virtio_pci_generic_class_init;
2210
2211         generic_type_info.parent = base_name;
2212         generic_type_info.class_init = virtio_pci_base_class_init;
2213         generic_type_info.class_data = (void *)t;
2214
2215         assert(!t->non_transitional_name);
2216         assert(!t->transitional_name);
2217     } else {
2218         base_type_info.class_init = virtio_pci_base_class_init;
2219         base_type_info.class_data = (void *)t;
2220     }
2221
2222     type_register(&base_type_info);
2223     if (generic_type_info.name) {
2224         type_register(&generic_type_info);
2225     }
2226
2227     if (t->non_transitional_name) {
2228         const TypeInfo non_transitional_type_info = {
2229             .name          = t->non_transitional_name,
2230             .parent        = base_type_info.name,
2231             .instance_init = virtio_pci_non_transitional_instance_init,
2232             .interfaces = (InterfaceInfo[]) {
2233                 { INTERFACE_PCIE_DEVICE },
2234                 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2235                 { }
2236             },
2237         };
2238         type_register(&non_transitional_type_info);
2239     }
2240
2241     if (t->transitional_name) {
2242         const TypeInfo transitional_type_info = {
2243             .name          = t->transitional_name,
2244             .parent        = base_type_info.name,
2245             .instance_init = virtio_pci_transitional_instance_init,
2246             .interfaces = (InterfaceInfo[]) {
2247                 /*
2248                  * Transitional virtio devices work only as Conventional PCI
2249                  * devices because they require PIO ports.
2250                  */
2251                 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2252                 { }
2253             },
2254         };
2255         type_register(&transitional_type_info);
2256     }
2257     g_free(base_name);
2258 }
2259
2260 unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues)
2261 {
2262     /*
2263      * 1:1 vq to vCPU mapping is ideal because the same vCPU that submitted
2264      * virtqueue buffers can handle their completion. When a different vCPU
2265      * handles completion it may need to IPI the vCPU that submitted the
2266      * request and this adds overhead.
2267      *
2268      * Virtqueues consume guest RAM and MSI-X vectors. This is wasteful in
2269      * guests with very many vCPUs and a device that is only used by a few
2270      * vCPUs. Unfortunately optimizing that case requires manual pinning inside
2271      * the guest, so those users might as well manually set the number of
2272      * queues. There is no upper limit that can be applied automatically and
2273      * doing so arbitrarily would result in a sudden performance drop once the
2274      * threshold number of vCPUs is exceeded.
2275      */
2276     unsigned num_queues = current_machine->smp.cpus;
2277
2278     /*
2279      * The maximum number of MSI-X vectors is PCI_MSIX_FLAGS_QSIZE + 1, but the
2280      * config change interrupt and the fixed virtqueues must be taken into
2281      * account too.
2282      */
2283     num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues);
2284
2285     /*
2286      * There is a limit to how many virtqueues a device can have.
2287      */
2288     return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues);
2289 }
2290
2291 /* virtio-pci-bus */
2292
2293 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
2294                                VirtIOPCIProxy *dev)
2295 {
2296     DeviceState *qdev = DEVICE(dev);
2297     char virtio_bus_name[] = "virtio-bus";
2298
2299     qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name);
2300 }
2301
2302 static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
2303 {
2304     BusClass *bus_class = BUS_CLASS(klass);
2305     VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
2306     bus_class->max_dev = 1;
2307     k->notify = virtio_pci_notify;
2308     k->save_config = virtio_pci_save_config;
2309     k->load_config = virtio_pci_load_config;
2310     k->save_queue = virtio_pci_save_queue;
2311     k->load_queue = virtio_pci_load_queue;
2312     k->save_extra_state = virtio_pci_save_extra_state;
2313     k->load_extra_state = virtio_pci_load_extra_state;
2314     k->has_extra_state = virtio_pci_has_extra_state;
2315     k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
2316     k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
2317     k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr;
2318     k->vmstate_change = virtio_pci_vmstate_change;
2319     k->pre_plugged = virtio_pci_pre_plugged;
2320     k->device_plugged = virtio_pci_device_plugged;
2321     k->device_unplugged = virtio_pci_device_unplugged;
2322     k->query_nvectors = virtio_pci_query_nvectors;
2323     k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
2324     k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
2325     k->get_dma_as = virtio_pci_get_dma_as;
2326     k->iommu_enabled = virtio_pci_iommu_enabled;
2327     k->queue_enabled = virtio_pci_queue_enabled;
2328 }
2329
2330 static const TypeInfo virtio_pci_bus_info = {
2331     .name          = TYPE_VIRTIO_PCI_BUS,
2332     .parent        = TYPE_VIRTIO_BUS,
2333     .instance_size = sizeof(VirtioPCIBusState),
2334     .class_size    = sizeof(VirtioPCIBusClass),
2335     .class_init    = virtio_pci_bus_class_init,
2336 };
2337
2338 static void virtio_pci_register_types(void)
2339 {
2340     /* Base types: */
2341     type_register_static(&virtio_pci_bus_info);
2342     type_register_static(&virtio_pci_info);
2343 }
2344
2345 type_init(virtio_pci_register_types)
2346