hw/virtio/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  * Contributions after 2012-01-13 are licensed under the terms of the
  15  * GNU GPL, version 2 or (at your option) any later version.
  16  */
  17
  18 #include "qemu/osdep.h"
  19
  20 #include "exec/memop.h"
  21 #include "standard-headers/linux/virtio_pci.h"
  22 #include "hw/boards.h"
  23 #include "hw/virtio/virtio.h"
  24 #include "migration/qemu-file-types.h"
  25 #include "hw/pci/pci.h"
  26 #include "hw/pci/pci_bus.h"
  27 #include "hw/qdev-properties.h"
  28 #include "qapi/error.h"
  29 #include "qemu/error-report.h"
  30 #include "qemu/log.h"
  31 #include "qemu/module.h"
  32 #include "hw/pci/msi.h"
  33 #include "hw/pci/msix.h"
  34 #include "hw/loader.h"
  35 #include "sysemu/kvm.h"
  36 #include "hw/virtio/virtio-pci.h"
  37 #include "qemu/range.h"
  38 #include "hw/virtio/virtio-bus.h"
  39 #include "qapi/visitor.h"
  40 #include "sysemu/replay.h"
  41 #include "trace.h"
  42
  43 #define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
  44
  45 #undef VIRTIO_PCI_CONFIG
  46
  47 /* The remaining space is defined by each driver as the per-driver
  48  * configuration space */
  49 #define VIRTIO_PCI_CONFIG_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev))
  50
  51 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
  52                                VirtIOPCIProxy *dev);
  53 static void virtio_pci_reset(DeviceState *qdev);
  54
  55 /* virtio device */
  56 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
  57 static inline VirtIOPCIProxy *to_virtio_pci_proxy(DeviceState *d)
  58 {
  59     return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
  60 }
  61
  62 /* DeviceState to VirtIOPCIProxy. Note: used on datapath,
  63  * be careful and test performance if you change this.
  64  */
  65 static inline VirtIOPCIProxy *to_virtio_pci_proxy_fast(DeviceState *d)
  66 {
  67     return container_of(d, VirtIOPCIProxy, pci_dev.qdev);
  68 }
  69
  70 static void virtio_pci_notify(DeviceState *d, uint16_t vector)
  71 {
  72     VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d);
  73
  74     if (msix_enabled(&proxy->pci_dev)) {
  75         if (vector != VIRTIO_NO_VECTOR) {
  76             msix_notify(&proxy->pci_dev, vector);
  77         }
  78     } else {
  79         VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
  80         pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1);
  81     }
  82 }
  83
  84 static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
  85 {
  86     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
  87     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
  88
  89     pci_device_save(&proxy->pci_dev, f);
  90     msix_save(&proxy->pci_dev, f);
  91     if (msix_present(&proxy->pci_dev))
  92         qemu_put_be16(f, vdev->config_vector);
  93 }
  94
  95 static const VMStateDescription vmstate_virtio_pci_modern_queue_state = {
  96     .name = "virtio_pci/modern_queue_state",
  97     .version_id = 1,
  98     .minimum_version_id = 1,
  99     .fields = (VMStateField[]) {
 100         VMSTATE_UINT16(num, VirtIOPCIQueue),
 101         VMSTATE_UNUSED(1), /* enabled was stored as be16 */
 102         VMSTATE_BOOL(enabled, VirtIOPCIQueue),
 103         VMSTATE_UINT32_ARRAY(desc, VirtIOPCIQueue, 2),
 104         VMSTATE_UINT32_ARRAY(avail, VirtIOPCIQueue, 2),
 105         VMSTATE_UINT32_ARRAY(used, VirtIOPCIQueue, 2),
 106         VMSTATE_END_OF_LIST()
 107     }
 108 };
 109
 110 static bool virtio_pci_modern_state_needed(void *opaque)
 111 {
 112     VirtIOPCIProxy *proxy = opaque;
 113
 114     return virtio_pci_modern(proxy);
 115 }
 116
 117 static const VMStateDescription vmstate_virtio_pci_modern_state_sub = {
 118     .name = "virtio_pci/modern_state",
 119     .version_id = 1,
 120     .minimum_version_id = 1,
 121     .needed = &virtio_pci_modern_state_needed,
 122     .fields = (VMStateField[]) {
 123         VMSTATE_UINT32(dfselect, VirtIOPCIProxy),
 124         VMSTATE_UINT32(gfselect, VirtIOPCIProxy),
 125         VMSTATE_UINT32_ARRAY(guest_features, VirtIOPCIProxy, 2),
 126         VMSTATE_STRUCT_ARRAY(vqs, VirtIOPCIProxy, VIRTIO_QUEUE_MAX, 0,
 127                              vmstate_virtio_pci_modern_queue_state,
 128                              VirtIOPCIQueue),
 129         VMSTATE_END_OF_LIST()
 130     }
 131 };
 132
 133 static const VMStateDescription vmstate_virtio_pci = {
 134     .name = "virtio_pci",
 135     .version_id = 1,
 136     .minimum_version_id = 1,
 137     .fields = (VMStateField[]) {
 138         VMSTATE_END_OF_LIST()
 139     },
 140     .subsections = (const VMStateDescription*[]) {
 141         &vmstate_virtio_pci_modern_state_sub,
 142         NULL
 143     }
 144 };
 145
 146 static bool virtio_pci_has_extra_state(DeviceState *d)
 147 {
 148     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 149
 150     return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
 151 }
 152
 153 static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
 154 {
 155     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 156
 157     vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
 158 }
 159
 160 static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
 161 {
 162     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 163
 164     return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
 165 }
 166
 167 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 168 {
 169     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 170     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 171
 172     if (msix_present(&proxy->pci_dev))
 173         qemu_put_be16(f, virtio_queue_vector(vdev, n));
 174 }
 175
 176 static int virtio_pci_load_config(DeviceState *d, QEMUFile *f)
 177 {
 178     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 179     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 180     uint16_t vector;
 181
 182     int ret;
 183     ret = pci_device_load(&proxy->pci_dev, f);
 184     if (ret) {
 185         return ret;
 186     }
 187     msix_unuse_all_vectors(&proxy->pci_dev);
 188     msix_load(&proxy->pci_dev, f);
 189     if (msix_present(&proxy->pci_dev)) {
 190         qemu_get_be16s(f, &vector);
 191
 192         if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
 193             return -EINVAL;
 194         }
 195     } else {
 196         vector = VIRTIO_NO_VECTOR;
 197     }
 198     vdev->config_vector = vector;
 199     if (vector != VIRTIO_NO_VECTOR) {
 200         msix_vector_use(&proxy->pci_dev, vector);
 201     }
 202     return 0;
 203 }
 204
 205 static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
 206 {
 207     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 208     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 209
 210     uint16_t vector;
 211     if (msix_present(&proxy->pci_dev)) {
 212         qemu_get_be16s(f, &vector);
 213         if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) {
 214             return -EINVAL;
 215         }
 216     } else {
 217         vector = VIRTIO_NO_VECTOR;
 218     }
 219     virtio_queue_set_vector(vdev, n, vector);
 220     if (vector != VIRTIO_NO_VECTOR) {
 221         msix_vector_use(&proxy->pci_dev, vector);
 222     }
 223
 224     return 0;
 225 }
 226
 227 static bool virtio_pci_ioeventfd_enabled(DeviceState *d)
 228 {
 229     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 230
 231     return (proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) != 0;
 232 }
 233
 234 #define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000
 235
 236 static inline int virtio_pci_queue_mem_mult(struct VirtIOPCIProxy *proxy)
 237 {
 238     return (proxy->flags & VIRTIO_PCI_FLAG_PAGE_PER_VQ) ?
 239         QEMU_VIRTIO_PCI_QUEUE_MEM_MULT : 4;
 240 }
 241
 242 static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier,
 243                                        int n, bool assign)
 244 {
 245     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 246     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 247     VirtQueue *vq = virtio_get_queue(vdev, n);
 248     bool legacy = virtio_pci_legacy(proxy);
 249     bool modern = virtio_pci_modern(proxy);
 250     bool fast_mmio = kvm_ioeventfd_any_length_enabled();
 251     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 252     MemoryRegion *modern_mr = &proxy->notify.mr;
 253     MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
 254     MemoryRegion *legacy_mr = &proxy->bar;
 255     hwaddr modern_addr = virtio_pci_queue_mem_mult(proxy) *
 256                          virtio_get_queue_index(vq);
 257     hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY;
 258
 259     if (assign) {
 260         if (modern) {
 261             if (fast_mmio) {
 262                 memory_region_add_eventfd(modern_mr, modern_addr, 0,
 263                                           false, n, notifier);
 264             } else {
 265                 memory_region_add_eventfd(modern_mr, modern_addr, 2,
 266                                           false, n, notifier);
 267             }
 268             if (modern_pio) {
 269                 memory_region_add_eventfd(modern_notify_mr, 0, 2,
 270                                               true, n, notifier);
 271             }
 272         }
 273         if (legacy) {
 274             memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
 275                                       true, n, notifier);
 276         }
 277     } else {
 278         if (modern) {
 279             if (fast_mmio) {
 280                 memory_region_del_eventfd(modern_mr, modern_addr, 0,
 281                                           false, n, notifier);
 282             } else {
 283                 memory_region_del_eventfd(modern_mr, modern_addr, 2,
 284                                           false, n, notifier);
 285             }
 286             if (modern_pio) {
 287                 memory_region_del_eventfd(modern_notify_mr, 0, 2,
 288                                           true, n, notifier);
 289             }
 290         }
 291         if (legacy) {
 292             memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
 293                                       true, n, notifier);
 294         }
 295     }
 296     return 0;
 297 }
 298
 299 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 300 {
 301     virtio_bus_start_ioeventfd(&proxy->bus);
 302 }
 303
 304 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 305 {
 306     virtio_bus_stop_ioeventfd(&proxy->bus);
 307 }
 308
 309 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 310 {
 311     VirtIOPCIProxy *proxy = opaque;
 312     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 313     uint16_t vector;
 314     hwaddr pa;
 315
 316     switch (addr) {
 317     case VIRTIO_PCI_GUEST_FEATURES:
 318         /* Guest does not negotiate properly?  We have to assume nothing. */
 319         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 320             val = virtio_bus_get_vdev_bad_features(&proxy->bus);
 321         }
 322         virtio_set_features(vdev, val);
 323         break;
 324     case VIRTIO_PCI_QUEUE_PFN:
 325         pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 326         if (pa == 0) {
 327             virtio_pci_reset(DEVICE(proxy));
 328         }
 329         else
 330             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 331         break;
 332     case VIRTIO_PCI_QUEUE_SEL:
 333         if (val < VIRTIO_QUEUE_MAX)
 334             vdev->queue_sel = val;
 335         break;
 336     case VIRTIO_PCI_QUEUE_NOTIFY:
 337         if (val < VIRTIO_QUEUE_MAX) {
 338             virtio_queue_notify(vdev, val);
 339         }
 340         break;
 341     case VIRTIO_PCI_STATUS:
 342         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 343             virtio_pci_stop_ioeventfd(proxy);
 344         }
 345
 346         virtio_set_status(vdev, val & 0xFF);
 347
 348         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 349             virtio_pci_start_ioeventfd(proxy);
 350         }
 351
 352         if (vdev->status == 0) {
 353             virtio_pci_reset(DEVICE(proxy));
 354         }
 355
 356         /* Linux before 2.6.34 drives the device without enabling
 357            the PCI device bus master bit. Enable it automatically
 358            for the guest. This is a PCI spec violation but so is
 359            initiating DMA with bus master bit clear. */
 360         if (val == (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER)) {
 361             pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
 362                                      proxy->pci_dev.config[PCI_COMMAND] |
 363                                      PCI_COMMAND_MASTER, 1);
 364         }
 365         break;
 366     case VIRTIO_MSI_CONFIG_VECTOR:
 367         if (vdev->config_vector != VIRTIO_NO_VECTOR) {
 368             msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 369         }
 370         /* Make it possible for guest to discover an error took place. */
 371         if (val < proxy->nvectors) {
 372             msix_vector_use(&proxy->pci_dev, val);
 373         } else {
 374             val = VIRTIO_NO_VECTOR;
 375         }
 376         vdev->config_vector = val;
 377         break;
 378     case VIRTIO_MSI_QUEUE_VECTOR:
 379         vector = virtio_queue_vector(vdev, vdev->queue_sel);
 380         if (vector != VIRTIO_NO_VECTOR) {
 381             msix_vector_unuse(&proxy->pci_dev, vector);
 382         }
 383         /* Make it possible for guest to discover an error took place. */
 384         if (val < proxy->nvectors) {
 385             msix_vector_use(&proxy->pci_dev, val);
 386         } else {
 387             val = VIRTIO_NO_VECTOR;
 388         }
 389         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 390         break;
 391     default:
 392         qemu_log_mask(LOG_GUEST_ERROR,
 393                       "%s: unexpected address 0x%x value 0x%x\n",
 394                       __func__, addr, val);
 395         break;
 396     }
 397 }
 398
 399 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 400 {
 401     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 402     uint32_t ret = 0xFFFFFFFF;
 403
 404     switch (addr) {
 405     case VIRTIO_PCI_HOST_FEATURES:
 406         ret = vdev->host_features;
 407         break;
 408     case VIRTIO_PCI_GUEST_FEATURES:
 409         ret = vdev->guest_features;
 410         break;
 411     case VIRTIO_PCI_QUEUE_PFN:
 412         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 413               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 414         break;
 415     case VIRTIO_PCI_QUEUE_NUM:
 416         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 417         break;
 418     case VIRTIO_PCI_QUEUE_SEL:
 419         ret = vdev->queue_sel;
 420         break;
 421     case VIRTIO_PCI_STATUS:
 422         ret = vdev->status;
 423         break;
 424     case VIRTIO_PCI_ISR:
 425         /* reading from the ISR also clears it. */
 426         ret = qatomic_xchg(&vdev->isr, 0);
 427         pci_irq_deassert(&proxy->pci_dev);
 428         break;
 429     case VIRTIO_MSI_CONFIG_VECTOR:
 430         ret = vdev->config_vector;
 431         break;
 432     case VIRTIO_MSI_QUEUE_VECTOR:
 433         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 434         break;
 435     default:
 436         break;
 437     }
 438
 439     return ret;
 440 }
 441
 442 static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
 443                                        unsigned size)
 444 {
 445     VirtIOPCIProxy *proxy = opaque;
 446     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 447     uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
 448     uint64_t val = 0;
 449
 450     if (vdev == NULL) {
 451         return UINT64_MAX;
 452     }
 453
 454     if (addr < config) {
 455         return virtio_ioport_read(proxy, addr);
 456     }
 457     addr -= config;
 458
 459     switch (size) {
 460     case 1:
 461         val = virtio_config_readb(vdev, addr);
 462         break;
 463     case 2:
 464         val = virtio_config_readw(vdev, addr);
 465         if (virtio_is_big_endian(vdev)) {
 466             val = bswap16(val);
 467         }
 468         break;
 469     case 4:
 470         val = virtio_config_readl(vdev, addr);
 471         if (virtio_is_big_endian(vdev)) {
 472             val = bswap32(val);
 473         }
 474         break;
 475     }
 476     return val;
 477 }
 478
 479 static void virtio_pci_config_write(void *opaque, hwaddr addr,
 480                                     uint64_t val, unsigned size)
 481 {
 482     VirtIOPCIProxy *proxy = opaque;
 483     uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
 484     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 485
 486     if (vdev == NULL) {
 487         return;
 488     }
 489
 490     if (addr < config) {
 491         virtio_ioport_write(proxy, addr, val);
 492         return;
 493     }
 494     addr -= config;
 495     /*
 496      * Virtio-PCI is odd. Ioports are LE but config space is target native
 497      * endian.
 498      */
 499     switch (size) {
 500     case 1:
 501         virtio_config_writeb(vdev, addr, val);
 502         break;
 503     case 2:
 504         if (virtio_is_big_endian(vdev)) {
 505             val = bswap16(val);
 506         }
 507         virtio_config_writew(vdev, addr, val);
 508         break;
 509     case 4:
 510         if (virtio_is_big_endian(vdev)) {
 511             val = bswap32(val);
 512         }
 513         virtio_config_writel(vdev, addr, val);
 514         break;
 515     }
 516 }
 517
 518 static const MemoryRegionOps virtio_pci_config_ops = {
 519     .read = virtio_pci_config_read,
 520     .write = virtio_pci_config_write,
 521     .impl = {
 522         .min_access_size = 1,
 523         .max_access_size = 4,
 524     },
 525     .endianness = DEVICE_LITTLE_ENDIAN,
 526 };
 527
 528 static MemoryRegion *virtio_address_space_lookup(VirtIOPCIProxy *proxy,
 529                                                  hwaddr *off, int len)
 530 {
 531     int i;
 532     VirtIOPCIRegion *reg;
 533
 534     for (i = 0; i < ARRAY_SIZE(proxy->regs); ++i) {
 535         reg = &proxy->regs[i];
 536         if (*off >= reg->offset &&
 537             *off + len <= reg->offset + reg->size) {
 538             *off -= reg->offset;
 539             return &reg->mr;
 540         }
 541     }
 542
 543     return NULL;
 544 }
 545
 546 /* Below are generic functions to do memcpy from/to an address space,
 547  * without byteswaps, with input validation.
 548  *
 549  * As regular address_space_* APIs all do some kind of byteswap at least for
 550  * some host/target combinations, we are forced to explicitly convert to a
 551  * known-endianness integer value.
 552  * It doesn't really matter which endian format to go through, so the code
 553  * below selects the endian that causes the least amount of work on the given
 554  * host.
 555  *
 556  * Note: host pointer must be aligned.
 557  */
 558 static
 559 void virtio_address_space_write(VirtIOPCIProxy *proxy, hwaddr addr,
 560                                 const uint8_t *buf, int len)
 561 {
 562     uint64_t val;
 563     MemoryRegion *mr;
 564
 565     /* address_space_* APIs assume an aligned address.
 566      * As address is under guest control, handle illegal values.
 567      */
 568     addr &= ~(len - 1);
 569
 570     mr = virtio_address_space_lookup(proxy, &addr, len);
 571     if (!mr) {
 572         return;
 573     }
 574
 575     /* Make sure caller aligned buf properly */
 576     assert(!(((uintptr_t)buf) & (len - 1)));
 577
 578     switch (len) {
 579     case 1:
 580         val = pci_get_byte(buf);
 581         break;
 582     case 2:
 583         val = pci_get_word(buf);
 584         break;
 585     case 4:
 586         val = pci_get_long(buf);
 587         break;
 588     default:
 589         /* As length is under guest control, handle illegal values. */
 590         return;
 591     }
 592     memory_region_dispatch_write(mr, addr, val, size_memop(len) | MO_LE,
 593                                  MEMTXATTRS_UNSPECIFIED);
 594 }
 595
 596 static void
 597 virtio_address_space_read(VirtIOPCIProxy *proxy, hwaddr addr,
 598                           uint8_t *buf, int len)
 599 {
 600     uint64_t val;
 601     MemoryRegion *mr;
 602
 603     /* address_space_* APIs assume an aligned address.
 604      * As address is under guest control, handle illegal values.
 605      */
 606     addr &= ~(len - 1);
 607
 608     mr = virtio_address_space_lookup(proxy, &addr, len);
 609     if (!mr) {
 610         return;
 611     }
 612
 613     /* Make sure caller aligned buf properly */
 614     assert(!(((uintptr_t)buf) & (len - 1)));
 615
 616     memory_region_dispatch_read(mr, addr, &val, size_memop(len) | MO_LE,
 617                                 MEMTXATTRS_UNSPECIFIED);
 618     switch (len) {
 619     case 1:
 620         pci_set_byte(buf, val);
 621         break;
 622     case 2:
 623         pci_set_word(buf, val);
 624         break;
 625     case 4:
 626         pci_set_long(buf, val);
 627         break;
 628     default:
 629         /* As length is under guest control, handle illegal values. */
 630         break;
 631     }
 632 }
 633
 634 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 635                                 uint32_t val, int len)
 636 {
 637     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
 638     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 639     struct virtio_pci_cfg_cap *cfg;
 640
 641     pci_default_write_config(pci_dev, address, val, len);
 642
 643     if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
 644         pcie_cap_flr_write_config(pci_dev, address, val, len);
 645     }
 646
 647     if (range_covers_byte(address, len, PCI_COMMAND)) {
 648         if (!(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 649             virtio_set_disabled(vdev, true);
 650             virtio_pci_stop_ioeventfd(proxy);
 651             virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 652         } else {
 653             virtio_set_disabled(vdev, false);
 654         }
 655     }
 656
 657     if (proxy->config_cap &&
 658         ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
 659                                                                   pci_cfg_data),
 660                        sizeof cfg->pci_cfg_data)) {
 661         uint32_t off;
 662         uint32_t len;
 663
 664         cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
 665         off = le32_to_cpu(cfg->cap.offset);
 666         len = le32_to_cpu(cfg->cap.length);
 667
 668         if (len == 1 || len == 2 || len == 4) {
 669             assert(len <= sizeof cfg->pci_cfg_data);
 670             virtio_address_space_write(proxy, off, cfg->pci_cfg_data, len);
 671         }
 672     }
 673 }
 674
 675 static uint32_t virtio_read_config(PCIDevice *pci_dev,
 676                                    uint32_t address, int len)
 677 {
 678     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
 679     struct virtio_pci_cfg_cap *cfg;
 680
 681     if (proxy->config_cap &&
 682         ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
 683                                                                   pci_cfg_data),
 684                        sizeof cfg->pci_cfg_data)) {
 685         uint32_t off;
 686         uint32_t len;
 687
 688         cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
 689         off = le32_to_cpu(cfg->cap.offset);
 690         len = le32_to_cpu(cfg->cap.length);
 691
 692         if (len == 1 || len == 2 || len == 4) {
 693             assert(len <= sizeof cfg->pci_cfg_data);
 694             virtio_address_space_read(proxy, off, cfg->pci_cfg_data, len);
 695         }
 696     }
 697
 698     return pci_default_read_config(pci_dev, address, len);
 699 }
 700
 701 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
 702                                         unsigned int queue_no,
 703                                         unsigned int vector)
 704 {
 705     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 706     int ret;
 707
 708     if (irqfd->users == 0) {
 709         KVMRouteChange c = kvm_irqchip_begin_route_changes(kvm_state);
 710         ret = kvm_irqchip_add_msi_route(&c, vector, &proxy->pci_dev);
 711         if (ret < 0) {
 712             return ret;
 713         }
 714         kvm_irqchip_commit_route_changes(&c);
 715         irqfd->virq = ret;
 716     }
 717     irqfd->users++;
 718     return 0;
 719 }
 720
 721 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 722                                              unsigned int vector)
 723 {
 724     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 725     if (--irqfd->users == 0) {
 726         kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 727     }
 728 }
 729
 730 static int kvm_virtio_pci_irqfd_use(VirtIOPCIProxy *proxy,
 731                                  unsigned int queue_no,
 732                                  unsigned int vector)
 733 {
 734     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 735     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 736     VirtQueue *vq = virtio_get_queue(vdev, queue_no);
 737     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 738     return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
 739 }
 740
 741 static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
 742                                       unsigned int queue_no,
 743                                       unsigned int vector)
 744 {
 745     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 746     VirtQueue *vq = virtio_get_queue(vdev, queue_no);
 747     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 748     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 749     int ret;
 750
 751     ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
 752     assert(ret == 0);
 753 }
 754
 755 static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
 756 {
 757     PCIDevice *dev = &proxy->pci_dev;
 758     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 759     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 760     unsigned int vector;
 761     int ret, queue_no;
 762
 763     for (queue_no = 0; queue_no < nvqs; queue_no++) {
 764         if (!virtio_queue_get_num(vdev, queue_no)) {
 765             break;
 766         }
 767         vector = virtio_queue_vector(vdev, queue_no);
 768         if (vector >= msix_nr_vectors_allocated(dev)) {
 769             continue;
 770         }
 771         ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector);
 772         if (ret < 0) {
 773             goto undo;
 774         }
 775         /* If guest supports masking, set up irqfd now.
 776          * Otherwise, delay until unmasked in the frontend.
 777          */
 778         if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 779             ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
 780             if (ret < 0) {
 781                 kvm_virtio_pci_vq_vector_release(proxy, vector);
 782                 goto undo;
 783             }
 784         }
 785     }
 786     return 0;
 787
 788 undo:
 789     while (--queue_no >= 0) {
 790         vector = virtio_queue_vector(vdev, queue_no);
 791         if (vector >= msix_nr_vectors_allocated(dev)) {
 792             continue;
 793         }
 794         if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 795             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
 796         }
 797         kvm_virtio_pci_vq_vector_release(proxy, vector);
 798     }
 799     return ret;
 800 }
 801
 802 static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
 803 {
 804     PCIDevice *dev = &proxy->pci_dev;
 805     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 806     unsigned int vector;
 807     int queue_no;
 808     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 809
 810     for (queue_no = 0; queue_no < nvqs; queue_no++) {
 811         if (!virtio_queue_get_num(vdev, queue_no)) {
 812             break;
 813         }
 814         vector = virtio_queue_vector(vdev, queue_no);
 815         if (vector >= msix_nr_vectors_allocated(dev)) {
 816             continue;
 817         }
 818         /* If guest supports masking, clean up irqfd now.
 819          * Otherwise, it was cleaned when masked in the frontend.
 820          */
 821         if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 822             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
 823         }
 824         kvm_virtio_pci_vq_vector_release(proxy, vector);
 825     }
 826 }
 827
 828 static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
 829                                        unsigned int queue_no,
 830                                        unsigned int vector,
 831                                        MSIMessage msg)
 832 {
 833     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 834     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 835     VirtQueue *vq = virtio_get_queue(vdev, queue_no);
 836     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 837     VirtIOIRQFD *irqfd;
 838     int ret = 0;
 839
 840     if (proxy->vector_irqfd) {
 841         irqfd = &proxy->vector_irqfd[vector];
 842         if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
 843             ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
 844                                                &proxy->pci_dev);
 845             if (ret < 0) {
 846                 return ret;
 847             }
 848             kvm_irqchip_commit_routes(kvm_state);
 849         }
 850     }
 851
 852     /* If guest supports masking, irqfd is already setup, unmask it.
 853      * Otherwise, set it up now.
 854      */
 855     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 856         k->guest_notifier_mask(vdev, queue_no, false);
 857         /* Test after unmasking to avoid losing events. */
 858         if (k->guest_notifier_pending &&
 859             k->guest_notifier_pending(vdev, queue_no)) {
 860             event_notifier_set(n);
 861         }
 862     } else {
 863         ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
 864     }
 865     return ret;
 866 }
 867
 868 static void virtio_pci_vq_vector_mask(VirtIOPCIProxy *proxy,
 869                                              unsigned int queue_no,
 870                                              unsigned int vector)
 871 {
 872     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 873     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 874
 875     /* If guest supports masking, keep irqfd but mask it.
 876      * Otherwise, clean it up now.
 877      */
 878     if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
 879         k->guest_notifier_mask(vdev, queue_no, true);
 880     } else {
 881         kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
 882     }
 883 }
 884
 885 static int virtio_pci_vector_unmask(PCIDevice *dev, unsigned vector,
 886                                     MSIMessage msg)
 887 {
 888     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 889     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 890     VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
 891     int ret, index, unmasked = 0;
 892
 893     while (vq) {
 894         index = virtio_get_queue_index(vq);
 895         if (!virtio_queue_get_num(vdev, index)) {
 896             break;
 897         }
 898         if (index < proxy->nvqs_with_notifiers) {
 899             ret = virtio_pci_vq_vector_unmask(proxy, index, vector, msg);
 900             if (ret < 0) {
 901                 goto undo;
 902             }
 903             ++unmasked;
 904         }
 905         vq = virtio_vector_next_queue(vq);
 906     }
 907
 908     return 0;
 909
 910 undo:
 911     vq = virtio_vector_first_queue(vdev, vector);
 912     while (vq && unmasked >= 0) {
 913         index = virtio_get_queue_index(vq);
 914         if (index < proxy->nvqs_with_notifiers) {
 915             virtio_pci_vq_vector_mask(proxy, index, vector);
 916             --unmasked;
 917         }
 918         vq = virtio_vector_next_queue(vq);
 919     }
 920     return ret;
 921 }
 922
 923 static void virtio_pci_vector_mask(PCIDevice *dev, unsigned vector)
 924 {
 925     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 926     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 927     VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
 928     int index;
 929
 930     while (vq) {
 931         index = virtio_get_queue_index(vq);
 932         if (!virtio_queue_get_num(vdev, index)) {
 933             break;
 934         }
 935         if (index < proxy->nvqs_with_notifiers) {
 936             virtio_pci_vq_vector_mask(proxy, index, vector);
 937         }
 938         vq = virtio_vector_next_queue(vq);
 939     }
 940 }
 941
 942 static void virtio_pci_vector_poll(PCIDevice *dev,
 943                                    unsigned int vector_start,
 944                                    unsigned int vector_end)
 945 {
 946     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 947     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 948     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
 949     int queue_no;
 950     unsigned int vector;
 951     EventNotifier *notifier;
 952     VirtQueue *vq;
 953
 954     for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
 955         if (!virtio_queue_get_num(vdev, queue_no)) {
 956             break;
 957         }
 958         vector = virtio_queue_vector(vdev, queue_no);
 959         if (vector < vector_start || vector >= vector_end ||
 960             !msix_is_masked(dev, vector)) {
 961             continue;
 962         }
 963         vq = virtio_get_queue(vdev, queue_no);
 964         notifier = virtio_queue_get_guest_notifier(vq);
 965         if (k->guest_notifier_pending) {
 966             if (k->guest_notifier_pending(vdev, queue_no)) {
 967                 msix_set_pending(dev, vector);
 968             }
 969         } else if (event_notifier_test_and_clear(notifier)) {
 970             msix_set_pending(dev, vector);
 971         }
 972     }
 973 }
 974
 975 static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
 976                                          bool with_irqfd)
 977 {
 978     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
 979     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 980     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
 981     VirtQueue *vq = virtio_get_queue(vdev, n);
 982     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 983
 984     if (assign) {
 985         int r = event_notifier_init(notifier, 0);
 986         if (r < 0) {
 987             return r;
 988         }
 989         virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
 990     } else {
 991         virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
 992         event_notifier_cleanup(notifier);
 993     }
 994
 995     if (!msix_enabled(&proxy->pci_dev) &&
 996         vdev->use_guest_notifier_mask &&
 997         vdc->guest_notifier_mask) {
 998         vdc->guest_notifier_mask(vdev, n, !assign);
 999     }
1000
1001     return 0;
1002 }
1003
1004 static bool virtio_pci_query_guest_notifiers(DeviceState *d)
1005 {
1006     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1007     return msix_enabled(&proxy->pci_dev);
1008 }
1009
1010 static int virtio_pci_set_guest_notifiers(DeviceState *d, int nvqs, bool assign)
1011 {
1012     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1013     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1014     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1015     int r, n;
1016     bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
1017         kvm_msi_via_irqfd_enabled();
1018
1019     nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
1020
1021     /*
1022      * When deassigning, pass a consistent nvqs value to avoid leaking
1023      * notifiers. But first check we've actually been configured, exit
1024      * early if we haven't.
1025      */
1026     if (!assign && !proxy->nvqs_with_notifiers) {
1027         return 0;
1028     }
1029     assert(assign || nvqs == proxy->nvqs_with_notifiers);
1030
1031     proxy->nvqs_with_notifiers = nvqs;
1032
1033     /* Must unset vector notifier while guest notifier is still assigned */
1034     if ((proxy->vector_irqfd || k->guest_notifier_mask) && !assign) {
1035         msix_unset_vector_notifiers(&proxy->pci_dev);
1036         if (proxy->vector_irqfd) {
1037             kvm_virtio_pci_vector_release(proxy, nvqs);
1038             g_free(proxy->vector_irqfd);
1039             proxy->vector_irqfd = NULL;
1040         }
1041     }
1042
1043     for (n = 0; n < nvqs; n++) {
1044         if (!virtio_queue_get_num(vdev, n)) {
1045             break;
1046         }
1047
1048         r = virtio_pci_set_guest_notifier(d, n, assign, with_irqfd);
1049         if (r < 0) {
1050             goto assign_error;
1051         }
1052     }
1053
1054     /* Must set vector notifier after guest notifier has been assigned */
1055     if ((with_irqfd || k->guest_notifier_mask) && assign) {
1056         if (with_irqfd) {
1057             proxy->vector_irqfd =
1058                 g_malloc0(sizeof(*proxy->vector_irqfd) *
1059                           msix_nr_vectors_allocated(&proxy->pci_dev));
1060             r = kvm_virtio_pci_vector_use(proxy, nvqs);
1061             if (r < 0) {
1062                 goto assign_error;
1063             }
1064         }
1065         r = msix_set_vector_notifiers(&proxy->pci_dev,
1066                                       virtio_pci_vector_unmask,
1067                                       virtio_pci_vector_mask,
1068                                       virtio_pci_vector_poll);
1069         if (r < 0) {
1070             goto notifiers_error;
1071         }
1072     }
1073
1074     return 0;
1075
1076 notifiers_error:
1077     if (with_irqfd) {
1078         assert(assign);
1079         kvm_virtio_pci_vector_release(proxy, nvqs);
1080     }
1081
1082 assign_error:
1083     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
1084     assert(assign);
1085     while (--n >= 0) {
1086         virtio_pci_set_guest_notifier(d, n, !assign, with_irqfd);
1087     }
1088     return r;
1089 }
1090
1091 static int virtio_pci_set_host_notifier_mr(DeviceState *d, int n,
1092                                            MemoryRegion *mr, bool assign)
1093 {
1094     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1095     int offset;
1096
1097     if (n >= VIRTIO_QUEUE_MAX || !virtio_pci_modern(proxy) ||
1098         virtio_pci_queue_mem_mult(proxy) != memory_region_size(mr)) {
1099         return -1;
1100     }
1101
1102     if (assign) {
1103         offset = virtio_pci_queue_mem_mult(proxy) * n;
1104         memory_region_add_subregion_overlap(&proxy->notify.mr, offset, mr, 1);
1105     } else {
1106         memory_region_del_subregion(&proxy->notify.mr, mr);
1107     }
1108
1109     return 0;
1110 }
1111
1112 static void virtio_pci_vmstate_change(DeviceState *d, bool running)
1113 {
1114     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
1115     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1116
1117     if (running) {
1118         /* Old QEMU versions did not set bus master enable on status write.
1119          * Detect DRIVER set and enable it.
1120          */
1121         if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
1122             (vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
1123             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
1124             pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
1125                                      proxy->pci_dev.config[PCI_COMMAND] |
1126                                      PCI_COMMAND_MASTER, 1);
1127         }
1128         virtio_pci_start_ioeventfd(proxy);
1129     } else {
1130         virtio_pci_stop_ioeventfd(proxy);
1131     }
1132 }
1133
1134 /*
1135  * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
1136  */
1137
1138 static int virtio_pci_query_nvectors(DeviceState *d)
1139 {
1140     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1141
1142     return proxy->nvectors;
1143 }
1144
1145 static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
1146 {
1147     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1148     PCIDevice *dev = &proxy->pci_dev;
1149
1150     return pci_get_address_space(dev);
1151 }
1152
1153 static bool virtio_pci_iommu_enabled(DeviceState *d)
1154 {
1155     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1156     PCIDevice *dev = &proxy->pci_dev;
1157     AddressSpace *dma_as = pci_device_iommu_address_space(dev);
1158
1159     if (dma_as == &address_space_memory) {
1160         return false;
1161     }
1162
1163     return true;
1164 }
1165
1166 static bool virtio_pci_queue_enabled(DeviceState *d, int n)
1167 {
1168     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1169     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1170
1171     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1172         return proxy->vqs[n].enabled;
1173     }
1174
1175     return virtio_queue_enabled_legacy(vdev, n);
1176 }
1177
1178 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
1179                                    struct virtio_pci_cap *cap)
1180 {
1181     PCIDevice *dev = &proxy->pci_dev;
1182     int offset;
1183
1184     offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0,
1185                                 cap->cap_len, &error_abort);
1186
1187     assert(cap->cap_len >= sizeof *cap);
1188     memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len,
1189            cap->cap_len - PCI_CAP_FLAGS);
1190
1191     return offset;
1192 }
1193
1194 static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr,
1195                                        unsigned size)
1196 {
1197     VirtIOPCIProxy *proxy = opaque;
1198     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1199     uint32_t val = 0;
1200     int i;
1201
1202     if (vdev == NULL) {
1203         return UINT64_MAX;
1204     }
1205
1206     switch (addr) {
1207     case VIRTIO_PCI_COMMON_DFSELECT:
1208         val = proxy->dfselect;
1209         break;
1210     case VIRTIO_PCI_COMMON_DF:
1211         if (proxy->dfselect <= 1) {
1212             VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1213
1214             val = (vdev->host_features & ~vdc->legacy_features) >>
1215                 (32 * proxy->dfselect);
1216         }
1217         break;
1218     case VIRTIO_PCI_COMMON_GFSELECT:
1219         val = proxy->gfselect;
1220         break;
1221     case VIRTIO_PCI_COMMON_GF:
1222         if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1223             val = proxy->guest_features[proxy->gfselect];
1224         }
1225         break;
1226     case VIRTIO_PCI_COMMON_MSIX:
1227         val = vdev->config_vector;
1228         break;
1229     case VIRTIO_PCI_COMMON_NUMQ:
1230         for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) {
1231             if (virtio_queue_get_num(vdev, i)) {
1232                 val = i + 1;
1233             }
1234         }
1235         break;
1236     case VIRTIO_PCI_COMMON_STATUS:
1237         val = vdev->status;
1238         break;
1239     case VIRTIO_PCI_COMMON_CFGGENERATION:
1240         val = vdev->generation;
1241         break;
1242     case VIRTIO_PCI_COMMON_Q_SELECT:
1243         val = vdev->queue_sel;
1244         break;
1245     case VIRTIO_PCI_COMMON_Q_SIZE:
1246         val = virtio_queue_get_num(vdev, vdev->queue_sel);
1247         break;
1248     case VIRTIO_PCI_COMMON_Q_MSIX:
1249         val = virtio_queue_vector(vdev, vdev->queue_sel);
1250         break;
1251     case VIRTIO_PCI_COMMON_Q_ENABLE:
1252         val = proxy->vqs[vdev->queue_sel].enabled;
1253         break;
1254     case VIRTIO_PCI_COMMON_Q_NOFF:
1255         /* Simply map queues in order */
1256         val = vdev->queue_sel;
1257         break;
1258     case VIRTIO_PCI_COMMON_Q_DESCLO:
1259         val = proxy->vqs[vdev->queue_sel].desc[0];
1260         break;
1261     case VIRTIO_PCI_COMMON_Q_DESCHI:
1262         val = proxy->vqs[vdev->queue_sel].desc[1];
1263         break;
1264     case VIRTIO_PCI_COMMON_Q_AVAILLO:
1265         val = proxy->vqs[vdev->queue_sel].avail[0];
1266         break;
1267     case VIRTIO_PCI_COMMON_Q_AVAILHI:
1268         val = proxy->vqs[vdev->queue_sel].avail[1];
1269         break;
1270     case VIRTIO_PCI_COMMON_Q_USEDLO:
1271         val = proxy->vqs[vdev->queue_sel].used[0];
1272         break;
1273     case VIRTIO_PCI_COMMON_Q_USEDHI:
1274         val = proxy->vqs[vdev->queue_sel].used[1];
1275         break;
1276     case VIRTIO_PCI_COMMON_Q_RESET:
1277         val = proxy->vqs[vdev->queue_sel].reset;
1278         break;
1279     default:
1280         val = 0;
1281     }
1282
1283     return val;
1284 }
1285
1286 static void virtio_pci_common_write(void *opaque, hwaddr addr,
1287                                     uint64_t val, unsigned size)
1288 {
1289     VirtIOPCIProxy *proxy = opaque;
1290     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1291     uint16_t vector;
1292
1293     if (vdev == NULL) {
1294         return;
1295     }
1296
1297     switch (addr) {
1298     case VIRTIO_PCI_COMMON_DFSELECT:
1299         proxy->dfselect = val;
1300         break;
1301     case VIRTIO_PCI_COMMON_GFSELECT:
1302         proxy->gfselect = val;
1303         break;
1304     case VIRTIO_PCI_COMMON_GF:
1305         if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
1306             proxy->guest_features[proxy->gfselect] = val;
1307             virtio_set_features(vdev,
1308                                 (((uint64_t)proxy->guest_features[1]) << 32) |
1309                                 proxy->guest_features[0]);
1310         }
1311         break;
1312     case VIRTIO_PCI_COMMON_MSIX:
1313         if (vdev->config_vector != VIRTIO_NO_VECTOR) {
1314             msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
1315         }
1316         /* Make it possible for guest to discover an error took place. */
1317         if (val < proxy->nvectors) {
1318             msix_vector_use(&proxy->pci_dev, val);
1319         } else {
1320             val = VIRTIO_NO_VECTOR;
1321         }
1322         vdev->config_vector = val;
1323         break;
1324     case VIRTIO_PCI_COMMON_STATUS:
1325         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1326             virtio_pci_stop_ioeventfd(proxy);
1327         }
1328
1329         virtio_set_status(vdev, val & 0xFF);
1330
1331         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
1332             virtio_pci_start_ioeventfd(proxy);
1333         }
1334
1335         if (vdev->status == 0) {
1336             virtio_pci_reset(DEVICE(proxy));
1337         }
1338
1339         break;
1340     case VIRTIO_PCI_COMMON_Q_SELECT:
1341         if (val < VIRTIO_QUEUE_MAX) {
1342             vdev->queue_sel = val;
1343         }
1344         break;
1345     case VIRTIO_PCI_COMMON_Q_SIZE:
1346         proxy->vqs[vdev->queue_sel].num = val;
1347         virtio_queue_set_num(vdev, vdev->queue_sel,
1348                              proxy->vqs[vdev->queue_sel].num);
1349         break;
1350     case VIRTIO_PCI_COMMON_Q_MSIX:
1351         vector = virtio_queue_vector(vdev, vdev->queue_sel);
1352         if (vector != VIRTIO_NO_VECTOR) {
1353             msix_vector_unuse(&proxy->pci_dev, vector);
1354         }
1355         /* Make it possible for guest to discover an error took place. */
1356         if (val < proxy->nvectors) {
1357             msix_vector_use(&proxy->pci_dev, val);
1358         } else {
1359             val = VIRTIO_NO_VECTOR;
1360         }
1361         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
1362         break;
1363     case VIRTIO_PCI_COMMON_Q_ENABLE:
1364         if (val == 1) {
1365             virtio_queue_set_num(vdev, vdev->queue_sel,
1366                                  proxy->vqs[vdev->queue_sel].num);
1367             virtio_queue_set_rings(vdev, vdev->queue_sel,
1368                        ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 |
1369                        proxy->vqs[vdev->queue_sel].desc[0],
1370                        ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 |
1371                        proxy->vqs[vdev->queue_sel].avail[0],
1372                        ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
1373                        proxy->vqs[vdev->queue_sel].used[0]);
1374             proxy->vqs[vdev->queue_sel].enabled = 1;
1375             proxy->vqs[vdev->queue_sel].reset = 0;
1376             virtio_queue_enable(vdev, vdev->queue_sel);
1377         } else {
1378             virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val);
1379         }
1380         break;
1381     case VIRTIO_PCI_COMMON_Q_DESCLO:
1382         proxy->vqs[vdev->queue_sel].desc[0] = val;
1383         break;
1384     case VIRTIO_PCI_COMMON_Q_DESCHI:
1385         proxy->vqs[vdev->queue_sel].desc[1] = val;
1386         break;
1387     case VIRTIO_PCI_COMMON_Q_AVAILLO:
1388         proxy->vqs[vdev->queue_sel].avail[0] = val;
1389         break;
1390     case VIRTIO_PCI_COMMON_Q_AVAILHI:
1391         proxy->vqs[vdev->queue_sel].avail[1] = val;
1392         break;
1393     case VIRTIO_PCI_COMMON_Q_USEDLO:
1394         proxy->vqs[vdev->queue_sel].used[0] = val;
1395         break;
1396     case VIRTIO_PCI_COMMON_Q_USEDHI:
1397         proxy->vqs[vdev->queue_sel].used[1] = val;
1398         break;
1399     case VIRTIO_PCI_COMMON_Q_RESET:
1400         if (val == 1) {
1401             proxy->vqs[vdev->queue_sel].reset = 1;
1402
1403             virtio_queue_reset(vdev, vdev->queue_sel);
1404
1405             proxy->vqs[vdev->queue_sel].reset = 0;
1406             proxy->vqs[vdev->queue_sel].enabled = 0;
1407         }
1408         break;
1409     default:
1410         break;
1411     }
1412 }
1413
1414
1415 static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr,
1416                                        unsigned size)
1417 {
1418     VirtIOPCIProxy *proxy = opaque;
1419     if (virtio_bus_get_device(&proxy->bus) == NULL) {
1420         return UINT64_MAX;
1421     }
1422
1423     return 0;
1424 }
1425
1426 static void virtio_pci_notify_write(void *opaque, hwaddr addr,
1427                                     uint64_t val, unsigned size)
1428 {
1429     VirtIOPCIProxy *proxy = opaque;
1430     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1431
1432     unsigned queue = addr / virtio_pci_queue_mem_mult(proxy);
1433
1434     if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1435         trace_virtio_pci_notify_write(addr, val, size);
1436         virtio_queue_notify(vdev, queue);
1437     }
1438 }
1439
1440 static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
1441                                         uint64_t val, unsigned size)
1442 {
1443     VirtIOPCIProxy *proxy = opaque;
1444     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1445
1446     unsigned queue = val;
1447
1448     if (vdev != NULL && queue < VIRTIO_QUEUE_MAX) {
1449         trace_virtio_pci_notify_write_pio(addr, val, size);
1450         virtio_queue_notify(vdev, queue);
1451     }
1452 }
1453
1454 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
1455                                     unsigned size)
1456 {
1457     VirtIOPCIProxy *proxy = opaque;
1458     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1459     uint64_t val;
1460
1461     if (vdev == NULL) {
1462         return UINT64_MAX;
1463     }
1464
1465     val = qatomic_xchg(&vdev->isr, 0);
1466     pci_irq_deassert(&proxy->pci_dev);
1467     return val;
1468 }
1469
1470 static void virtio_pci_isr_write(void *opaque, hwaddr addr,
1471                                  uint64_t val, unsigned size)
1472 {
1473 }
1474
1475 static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr,
1476                                        unsigned size)
1477 {
1478     VirtIOPCIProxy *proxy = opaque;
1479     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1480     uint64_t val;
1481
1482     if (vdev == NULL) {
1483         return UINT64_MAX;
1484     }
1485
1486     switch (size) {
1487     case 1:
1488         val = virtio_config_modern_readb(vdev, addr);
1489         break;
1490     case 2:
1491         val = virtio_config_modern_readw(vdev, addr);
1492         break;
1493     case 4:
1494         val = virtio_config_modern_readl(vdev, addr);
1495         break;
1496     default:
1497         val = 0;
1498         break;
1499     }
1500     return val;
1501 }
1502
1503 static void virtio_pci_device_write(void *opaque, hwaddr addr,
1504                                     uint64_t val, unsigned size)
1505 {
1506     VirtIOPCIProxy *proxy = opaque;
1507     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1508
1509     if (vdev == NULL) {
1510         return;
1511     }
1512
1513     switch (size) {
1514     case 1:
1515         virtio_config_modern_writeb(vdev, addr, val);
1516         break;
1517     case 2:
1518         virtio_config_modern_writew(vdev, addr, val);
1519         break;
1520     case 4:
1521         virtio_config_modern_writel(vdev, addr, val);
1522         break;
1523     }
1524 }
1525
1526 static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy,
1527                                            const char *vdev_name)
1528 {
1529     static const MemoryRegionOps common_ops = {
1530         .read = virtio_pci_common_read,
1531         .write = virtio_pci_common_write,
1532         .impl = {
1533             .min_access_size = 1,
1534             .max_access_size = 4,
1535         },
1536         .endianness = DEVICE_LITTLE_ENDIAN,
1537     };
1538     static const MemoryRegionOps isr_ops = {
1539         .read = virtio_pci_isr_read,
1540         .write = virtio_pci_isr_write,
1541         .impl = {
1542             .min_access_size = 1,
1543             .max_access_size = 4,
1544         },
1545         .endianness = DEVICE_LITTLE_ENDIAN,
1546     };
1547     static const MemoryRegionOps device_ops = {
1548         .read = virtio_pci_device_read,
1549         .write = virtio_pci_device_write,
1550         .impl = {
1551             .min_access_size = 1,
1552             .max_access_size = 4,
1553         },
1554         .endianness = DEVICE_LITTLE_ENDIAN,
1555     };
1556     static const MemoryRegionOps notify_ops = {
1557         .read = virtio_pci_notify_read,
1558         .write = virtio_pci_notify_write,
1559         .impl = {
1560             .min_access_size = 1,
1561             .max_access_size = 4,
1562         },
1563         .endianness = DEVICE_LITTLE_ENDIAN,
1564     };
1565     static const MemoryRegionOps notify_pio_ops = {
1566         .read = virtio_pci_notify_read,
1567         .write = virtio_pci_notify_write_pio,
1568         .impl = {
1569             .min_access_size = 1,
1570             .max_access_size = 4,
1571         },
1572         .endianness = DEVICE_LITTLE_ENDIAN,
1573     };
1574     g_autoptr(GString) name = g_string_new(NULL);
1575
1576     g_string_printf(name, "virtio-pci-common-%s", vdev_name);
1577     memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
1578                           &common_ops,
1579                           proxy,
1580                           name->str,
1581                           proxy->common.size);
1582
1583     g_string_printf(name, "virtio-pci-isr-%s", vdev_name);
1584     memory_region_init_io(&proxy->isr.mr, OBJECT(proxy),
1585                           &isr_ops,
1586                           proxy,
1587                           name->str,
1588                           proxy->isr.size);
1589
1590     g_string_printf(name, "virtio-pci-device-%s", vdev_name);
1591     memory_region_init_io(&proxy->device.mr, OBJECT(proxy),
1592                           &device_ops,
1593                           proxy,
1594                           name->str,
1595                           proxy->device.size);
1596
1597     g_string_printf(name, "virtio-pci-notify-%s", vdev_name);
1598     memory_region_init_io(&proxy->notify.mr, OBJECT(proxy),
1599                           &notify_ops,
1600                           proxy,
1601                           name->str,
1602                           proxy->notify.size);
1603
1604     g_string_printf(name, "virtio-pci-notify-pio-%s", vdev_name);
1605     memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
1606                           &notify_pio_ops,
1607                           proxy,
1608                           name->str,
1609                           proxy->notify_pio.size);
1610 }
1611
1612 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
1613                                          VirtIOPCIRegion *region,
1614                                          struct virtio_pci_cap *cap,
1615                                          MemoryRegion *mr,
1616                                          uint8_t bar)
1617 {
1618     memory_region_add_subregion(mr, region->offset, &region->mr);
1619
1620     cap->cfg_type = region->type;
1621     cap->bar = bar;
1622     cap->offset = cpu_to_le32(region->offset);
1623     cap->length = cpu_to_le32(region->size);
1624     virtio_pci_add_mem_cap(proxy, cap);
1625
1626 }
1627
1628 static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
1629                                              VirtIOPCIRegion *region,
1630                                              struct virtio_pci_cap *cap)
1631 {
1632     virtio_pci_modern_region_map(proxy, region, cap,
1633                                  &proxy->modern_bar, proxy->modern_mem_bar_idx);
1634 }
1635
1636 static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
1637                                             VirtIOPCIRegion *region,
1638                                             struct virtio_pci_cap *cap)
1639 {
1640     virtio_pci_modern_region_map(proxy, region, cap,
1641                                  &proxy->io_bar, proxy->modern_io_bar_idx);
1642 }
1643
1644 static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
1645                                                VirtIOPCIRegion *region)
1646 {
1647     memory_region_del_subregion(&proxy->modern_bar,
1648                                 &region->mr);
1649 }
1650
1651 static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
1652                                               VirtIOPCIRegion *region)
1653 {
1654     memory_region_del_subregion(&proxy->io_bar,
1655                                 &region->mr);
1656 }
1657
1658 static void virtio_pci_pre_plugged(DeviceState *d, Error **errp)
1659 {
1660     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1661     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1662
1663     if (virtio_pci_modern(proxy)) {
1664         virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
1665     }
1666
1667     virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE);
1668 }
1669
1670 /* This is called by virtio-bus just after the device is plugged. */
1671 static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
1672 {
1673     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1674     VirtioBusState *bus = &proxy->bus;
1675     bool legacy = virtio_pci_legacy(proxy);
1676     bool modern;
1677     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1678     uint8_t *config;
1679     uint32_t size;
1680     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
1681
1682     /*
1683      * Virtio capabilities present without
1684      * VIRTIO_F_VERSION_1 confuses guests
1685      */
1686     if (!proxy->ignore_backend_features &&
1687             !virtio_has_feature(vdev->host_features, VIRTIO_F_VERSION_1)) {
1688         virtio_pci_disable_modern(proxy);
1689
1690         if (!legacy) {
1691             error_setg(errp, "Device doesn't support modern mode, and legacy"
1692                              " mode is disabled");
1693             error_append_hint(errp, "Set disable-legacy to off\n");
1694
1695             return;
1696         }
1697     }
1698
1699     modern = virtio_pci_modern(proxy);
1700
1701     config = proxy->pci_dev.config;
1702     if (proxy->class_code) {
1703         pci_config_set_class(config, proxy->class_code);
1704     }
1705
1706     if (legacy) {
1707         if (!virtio_legacy_allowed(vdev)) {
1708             /*
1709              * To avoid migration issues, we allow legacy mode when legacy
1710              * check is disabled in the old machine types (< 5.1).
1711              */
1712             if (virtio_legacy_check_disabled(vdev)) {
1713                 warn_report("device is modern-only, but for backward "
1714                             "compatibility legacy is allowed");
1715             } else {
1716                 error_setg(errp,
1717                            "device is modern-only, use disable-legacy=on");
1718                 return;
1719             }
1720         }
1721         if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1722             error_setg(errp, "VIRTIO_F_IOMMU_PLATFORM was supported by"
1723                        " neither legacy nor transitional device");
1724             return;
1725         }
1726         /*
1727          * Legacy and transitional devices use specific subsystem IDs.
1728          * Note that the subsystem vendor ID (config + PCI_SUBSYSTEM_VENDOR_ID)
1729          * is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default.
1730          */
1731         pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
1732     } else {
1733         /* pure virtio-1.0 */
1734         pci_set_word(config + PCI_VENDOR_ID,
1735                      PCI_VENDOR_ID_REDHAT_QUMRANET);
1736         pci_set_word(config + PCI_DEVICE_ID,
1737                      PCI_DEVICE_ID_VIRTIO_10_BASE + virtio_bus_get_vdev_id(bus));
1738         pci_config_set_revision(config, 1);
1739     }
1740     config[PCI_INTERRUPT_PIN] = 1;
1741
1742
1743     if (modern) {
1744         struct virtio_pci_cap cap = {
1745             .cap_len = sizeof cap,
1746         };
1747         struct virtio_pci_notify_cap notify = {
1748             .cap.cap_len = sizeof notify,
1749             .notify_off_multiplier =
1750                 cpu_to_le32(virtio_pci_queue_mem_mult(proxy)),
1751         };
1752         struct virtio_pci_cfg_cap cfg = {
1753             .cap.cap_len = sizeof cfg,
1754             .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
1755         };
1756         struct virtio_pci_notify_cap notify_pio = {
1757             .cap.cap_len = sizeof notify,
1758             .notify_off_multiplier = cpu_to_le32(0x0),
1759         };
1760
1761         struct virtio_pci_cfg_cap *cfg_mask;
1762
1763         virtio_pci_modern_regions_init(proxy, vdev->name);
1764
1765         virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
1766         virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
1767         virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
1768         virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
1769
1770         if (modern_pio) {
1771             memory_region_init(&proxy->io_bar, OBJECT(proxy),
1772                                "virtio-pci-io", 0x4);
1773
1774             pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar_idx,
1775                              PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
1776
1777             virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
1778                                             &notify_pio.cap);
1779         }
1780
1781         pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar_idx,
1782                          PCI_BASE_ADDRESS_SPACE_MEMORY |
1783                          PCI_BASE_ADDRESS_MEM_PREFETCH |
1784                          PCI_BASE_ADDRESS_MEM_TYPE_64,
1785                          &proxy->modern_bar);
1786
1787         proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap);
1788         cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap);
1789         pci_set_byte(&cfg_mask->cap.bar, ~0x0);
1790         pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0);
1791         pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0);
1792         pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
1793     }
1794
1795     if (proxy->nvectors) {
1796         int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
1797                                           proxy->msix_bar_idx, NULL);
1798         if (err) {
1799             /* Notice when a system that supports MSIx can't initialize it */
1800             if (err != -ENOTSUP) {
1801                 warn_report("unable to init msix vectors to %" PRIu32,
1802                             proxy->nvectors);
1803             }
1804             proxy->nvectors = 0;
1805         }
1806     }
1807
1808     proxy->pci_dev.config_write = virtio_write_config;
1809     proxy->pci_dev.config_read = virtio_read_config;
1810
1811     if (legacy) {
1812         size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
1813             + virtio_bus_get_vdev_config_len(bus);
1814         size = pow2ceil(size);
1815
1816         memory_region_init_io(&proxy->bar, OBJECT(proxy),
1817                               &virtio_pci_config_ops,
1818                               proxy, "virtio-pci", size);
1819
1820         pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar_idx,
1821                          PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
1822     }
1823 }
1824
1825 static void virtio_pci_device_unplugged(DeviceState *d)
1826 {
1827     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
1828     bool modern = virtio_pci_modern(proxy);
1829     bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
1830
1831     virtio_pci_stop_ioeventfd(proxy);
1832
1833     if (modern) {
1834         virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
1835         virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
1836         virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
1837         virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
1838         if (modern_pio) {
1839             virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
1840         }
1841     }
1842 }
1843
1844 static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
1845 {
1846     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
1847     VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
1848     bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
1849                      !pci_bus_is_root(pci_get_bus(pci_dev));
1850
1851     if (kvm_enabled() && !kvm_has_many_ioeventfds()) {
1852         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1853     }
1854
1855     /* fd-based ioevents can't be synchronized in record/replay */
1856     if (replay_mode != REPLAY_MODE_NONE) {
1857         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
1858     }
1859
1860     /*
1861      * virtio pci bar layout used by default.
1862      * subclasses can re-arrange things if needed.
1863      *
1864      *   region 0   --  virtio legacy io bar
1865      *   region 1   --  msi-x bar
1866      *   region 2   --  virtio modern io bar (off by default)
1867      *   region 4+5 --  virtio modern memory (64bit) bar
1868      *
1869      */
1870     proxy->legacy_io_bar_idx  = 0;
1871     proxy->msix_bar_idx       = 1;
1872     proxy->modern_io_bar_idx  = 2;
1873     proxy->modern_mem_bar_idx = 4;
1874
1875     proxy->common.offset = 0x0;
1876     proxy->common.size = 0x1000;
1877     proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG;
1878
1879     proxy->isr.offset = 0x1000;
1880     proxy->isr.size = 0x1000;
1881     proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG;
1882
1883     proxy->device.offset = 0x2000;
1884     proxy->device.size = 0x1000;
1885     proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG;
1886
1887     proxy->notify.offset = 0x3000;
1888     proxy->notify.size = virtio_pci_queue_mem_mult(proxy) * VIRTIO_QUEUE_MAX;
1889     proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1890
1891     proxy->notify_pio.offset = 0x0;
1892     proxy->notify_pio.size = 0x4;
1893     proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
1894
1895     /* subclasses can enforce modern, so do this unconditionally */
1896     memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
1897                        /* PCI BAR regions must be powers of 2 */
1898                        pow2ceil(proxy->notify.offset + proxy->notify.size));
1899
1900     if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) {
1901         proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
1902     }
1903
1904     if (!virtio_pci_modern(proxy) && !virtio_pci_legacy(proxy)) {
1905         error_setg(errp, "device cannot work as neither modern nor legacy mode"
1906                    " is enabled");
1907         error_append_hint(errp, "Set either disable-modern or disable-legacy"
1908                           " to off\n");
1909         return;
1910     }
1911
1912     if (pcie_port && pci_is_express(pci_dev)) {
1913         int pos;
1914         uint16_t last_pcie_cap_offset = PCI_CONFIG_SPACE_SIZE;
1915
1916         pos = pcie_endpoint_cap_init(pci_dev, 0);
1917         assert(pos > 0);
1918
1919         pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0,
1920                                  PCI_PM_SIZEOF, errp);
1921         if (pos < 0) {
1922             return;
1923         }
1924
1925         pci_dev->exp.pm_cap = pos;
1926
1927         /*
1928          * Indicates that this function complies with revision 1.2 of the
1929          * PCI Power Management Interface Specification.
1930          */
1931         pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
1932
1933         if (proxy->flags & VIRTIO_PCI_FLAG_AER) {
1934             pcie_aer_init(pci_dev, PCI_ERR_VER, last_pcie_cap_offset,
1935                           PCI_ERR_SIZEOF, NULL);
1936             last_pcie_cap_offset += PCI_ERR_SIZEOF;
1937         }
1938
1939         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_DEVERR) {
1940             /* Init error enabling flags */
1941             pcie_cap_deverr_init(pci_dev);
1942         }
1943
1944         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_LNKCTL) {
1945             /* Init Link Control Register */
1946             pcie_cap_lnkctl_init(pci_dev);
1947         }
1948
1949         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_PM) {
1950             /* Init Power Management Control Register */
1951             pci_set_word(pci_dev->wmask + pos + PCI_PM_CTRL,
1952                          PCI_PM_CTRL_STATE_MASK);
1953         }
1954
1955         if (proxy->flags & VIRTIO_PCI_FLAG_ATS) {
1956             pcie_ats_init(pci_dev, last_pcie_cap_offset,
1957                           proxy->flags & VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED);
1958             last_pcie_cap_offset += PCI_EXT_CAP_ATS_SIZEOF;
1959         }
1960
1961         if (proxy->flags & VIRTIO_PCI_FLAG_INIT_FLR) {
1962             /* Set Function Level Reset capability bit */
1963             pcie_cap_flr_init(pci_dev);
1964         }
1965     } else {
1966         /*
1967          * make future invocations of pci_is_express() return false
1968          * and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
1969          */
1970         pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
1971     }
1972
1973     virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
1974     if (k->realize) {
1975         k->realize(proxy, errp);
1976     }
1977 }
1978
1979 static void virtio_pci_exit(PCIDevice *pci_dev)
1980 {
1981     VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
1982     bool pcie_port = pci_bus_is_express(pci_get_bus(pci_dev)) &&
1983                      !pci_bus_is_root(pci_get_bus(pci_dev));
1984
1985     msix_uninit_exclusive_bar(pci_dev);
1986     if (proxy->flags & VIRTIO_PCI_FLAG_AER && pcie_port &&
1987         pci_is_express(pci_dev)) {
1988         pcie_aer_exit(pci_dev);
1989     }
1990 }
1991
1992 static void virtio_pci_reset(DeviceState *qdev)
1993 {
1994     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
1995     VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
1996     int i;
1997
1998     virtio_bus_reset(bus);
1999     msix_unuse_all_vectors(&proxy->pci_dev);
2000
2001     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2002         proxy->vqs[i].enabled = 0;
2003         proxy->vqs[i].reset = 0;
2004         proxy->vqs[i].num = 0;
2005         proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0;
2006         proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0;
2007         proxy->vqs[i].used[0] = proxy->vqs[i].used[1] = 0;
2008     }
2009 }
2010
2011 static void virtio_pci_bus_reset_hold(Object *obj)
2012 {
2013     PCIDevice *dev = PCI_DEVICE(obj);
2014     DeviceState *qdev = DEVICE(obj);
2015
2016     virtio_pci_reset(qdev);
2017
2018     if (pci_is_express(dev)) {
2019         pcie_cap_deverr_reset(dev);
2020         pcie_cap_lnkctl_reset(dev);
2021
2022         pci_set_word(dev->config + dev->exp.pm_cap + PCI_PM_CTRL, 0);
2023     }
2024 }
2025
2026 static Property virtio_pci_properties[] = {
2027     DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
2028                     VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
2029     DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
2030                     VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
2031     DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
2032                     VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
2033     DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
2034                     VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
2035     DEFINE_PROP_BIT("page-per-vq", VirtIOPCIProxy, flags,
2036                     VIRTIO_PCI_FLAG_PAGE_PER_VQ_BIT, false),
2037     DEFINE_PROP_BOOL("x-ignore-backend-features", VirtIOPCIProxy,
2038                      ignore_backend_features, false),
2039     DEFINE_PROP_BIT("ats", VirtIOPCIProxy, flags,
2040                     VIRTIO_PCI_FLAG_ATS_BIT, false),
2041     DEFINE_PROP_BIT("x-ats-page-aligned", VirtIOPCIProxy, flags,
2042                     VIRTIO_PCI_FLAG_ATS_PAGE_ALIGNED_BIT, true),
2043     DEFINE_PROP_BIT("x-pcie-deverr-init", VirtIOPCIProxy, flags,
2044                     VIRTIO_PCI_FLAG_INIT_DEVERR_BIT, true),
2045     DEFINE_PROP_BIT("x-pcie-lnkctl-init", VirtIOPCIProxy, flags,
2046                     VIRTIO_PCI_FLAG_INIT_LNKCTL_BIT, true),
2047     DEFINE_PROP_BIT("x-pcie-pm-init", VirtIOPCIProxy, flags,
2048                     VIRTIO_PCI_FLAG_INIT_PM_BIT, true),
2049     DEFINE_PROP_BIT("x-pcie-flr-init", VirtIOPCIProxy, flags,
2050                     VIRTIO_PCI_FLAG_INIT_FLR_BIT, true),
2051     DEFINE_PROP_BIT("aer", VirtIOPCIProxy, flags,
2052                     VIRTIO_PCI_FLAG_AER_BIT, false),
2053     DEFINE_PROP_END_OF_LIST(),
2054 };
2055
2056 static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
2057 {
2058     VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
2059     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
2060     PCIDevice *pci_dev = &proxy->pci_dev;
2061
2062     if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
2063         virtio_pci_modern(proxy)) {
2064         pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
2065     }
2066
2067     vpciklass->parent_dc_realize(qdev, errp);
2068 }
2069
2070 static void virtio_pci_class_init(ObjectClass *klass, void *data)
2071 {
2072     DeviceClass *dc = DEVICE_CLASS(klass);
2073     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
2074     VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
2075     ResettableClass *rc = RESETTABLE_CLASS(klass);
2076
2077     device_class_set_props(dc, virtio_pci_properties);
2078     k->realize = virtio_pci_realize;
2079     k->exit = virtio_pci_exit;
2080     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
2081     k->revision = VIRTIO_PCI_ABI_VERSION;
2082     k->class_id = PCI_CLASS_OTHERS;
2083     device_class_set_parent_realize(dc, virtio_pci_dc_realize,
2084                                     &vpciklass->parent_dc_realize);
2085     rc->phases.hold = virtio_pci_bus_reset_hold;
2086 }
2087
2088 static const TypeInfo virtio_pci_info = {
2089     .name          = TYPE_VIRTIO_PCI,
2090     .parent        = TYPE_PCI_DEVICE,
2091     .instance_size = sizeof(VirtIOPCIProxy),
2092     .class_init    = virtio_pci_class_init,
2093     .class_size    = sizeof(VirtioPCIClass),
2094     .abstract      = true,
2095 };
2096
2097 static Property virtio_pci_generic_properties[] = {
2098     DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy,
2099                             ON_OFF_AUTO_AUTO),
2100     DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false),
2101     DEFINE_PROP_END_OF_LIST(),
2102 };
2103
2104 static void virtio_pci_base_class_init(ObjectClass *klass, void *data)
2105 {
2106     const VirtioPCIDeviceTypeInfo *t = data;
2107     if (t->class_init) {
2108         t->class_init(klass, NULL);
2109     }
2110 }
2111
2112 static void virtio_pci_generic_class_init(ObjectClass *klass, void *data)
2113 {
2114     DeviceClass *dc = DEVICE_CLASS(klass);
2115
2116     device_class_set_props(dc, virtio_pci_generic_properties);
2117 }
2118
2119 static void virtio_pci_transitional_instance_init(Object *obj)
2120 {
2121     VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2122
2123     proxy->disable_legacy = ON_OFF_AUTO_OFF;
2124     proxy->disable_modern = false;
2125 }
2126
2127 static void virtio_pci_non_transitional_instance_init(Object *obj)
2128 {
2129     VirtIOPCIProxy *proxy = VIRTIO_PCI(obj);
2130
2131     proxy->disable_legacy = ON_OFF_AUTO_ON;
2132     proxy->disable_modern = false;
2133 }
2134
2135 void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t)
2136 {
2137     char *base_name = NULL;
2138     TypeInfo base_type_info = {
2139         .name          = t->base_name,
2140         .parent        = t->parent ? t->parent : TYPE_VIRTIO_PCI,
2141         .instance_size = t->instance_size,
2142         .instance_init = t->instance_init,
2143         .class_size    = t->class_size,
2144         .abstract      = true,
2145         .interfaces    = t->interfaces,
2146     };
2147     TypeInfo generic_type_info = {
2148         .name = t->generic_name,
2149         .parent = base_type_info.name,
2150         .class_init = virtio_pci_generic_class_init,
2151         .interfaces = (InterfaceInfo[]) {
2152             { INTERFACE_PCIE_DEVICE },
2153             { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2154             { }
2155         },
2156     };
2157
2158     if (!base_type_info.name) {
2159         /* No base type -> register a single generic device type */
2160         /* use intermediate %s-base-type to add generic device props */
2161         base_name = g_strdup_printf("%s-base-type", t->generic_name);
2162         base_type_info.name = base_name;
2163         base_type_info.class_init = virtio_pci_generic_class_init;
2164
2165         generic_type_info.parent = base_name;
2166         generic_type_info.class_init = virtio_pci_base_class_init;
2167         generic_type_info.class_data = (void *)t;
2168
2169         assert(!t->non_transitional_name);
2170         assert(!t->transitional_name);
2171     } else {
2172         base_type_info.class_init = virtio_pci_base_class_init;
2173         base_type_info.class_data = (void *)t;
2174     }
2175
2176     type_register(&base_type_info);
2177     if (generic_type_info.name) {
2178         type_register(&generic_type_info);
2179     }
2180
2181     if (t->non_transitional_name) {
2182         const TypeInfo non_transitional_type_info = {
2183             .name          = t->non_transitional_name,
2184             .parent        = base_type_info.name,
2185             .instance_init = virtio_pci_non_transitional_instance_init,
2186             .interfaces = (InterfaceInfo[]) {
2187                 { INTERFACE_PCIE_DEVICE },
2188                 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2189                 { }
2190             },
2191         };
2192         type_register(&non_transitional_type_info);
2193     }
2194
2195     if (t->transitional_name) {
2196         const TypeInfo transitional_type_info = {
2197             .name          = t->transitional_name,
2198             .parent        = base_type_info.name,
2199             .instance_init = virtio_pci_transitional_instance_init,
2200             .interfaces = (InterfaceInfo[]) {
2201                 /*
2202                  * Transitional virtio devices work only as Conventional PCI
2203                  * devices because they require PIO ports.
2204                  */
2205                 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
2206                 { }
2207             },
2208         };
2209         type_register(&transitional_type_info);
2210     }
2211     g_free(base_name);
2212 }
2213
2214 unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues)
2215 {
2216     /*
2217      * 1:1 vq to vCPU mapping is ideal because the same vCPU that submitted
2218      * virtqueue buffers can handle their completion. When a different vCPU
2219      * handles completion it may need to IPI the vCPU that submitted the
2220      * request and this adds overhead.
2221      *
2222      * Virtqueues consume guest RAM and MSI-X vectors. This is wasteful in
2223      * guests with very many vCPUs and a device that is only used by a few
2224      * vCPUs. Unfortunately optimizing that case requires manual pinning inside
2225      * the guest, so those users might as well manually set the number of
2226      * queues. There is no upper limit that can be applied automatically and
2227      * doing so arbitrarily would result in a sudden performance drop once the
2228      * threshold number of vCPUs is exceeded.
2229      */
2230     unsigned num_queues = current_machine->smp.cpus;
2231
2232     /*
2233      * The maximum number of MSI-X vectors is PCI_MSIX_FLAGS_QSIZE + 1, but the
2234      * config change interrupt and the fixed virtqueues must be taken into
2235      * account too.
2236      */
2237     num_queues = MIN(num_queues, PCI_MSIX_FLAGS_QSIZE - fixed_queues);
2238
2239     /*
2240      * There is a limit to how many virtqueues a device can have.
2241      */
2242     return MIN(num_queues, VIRTIO_QUEUE_MAX - fixed_queues);
2243 }
2244
2245 /* virtio-pci-bus */
2246
2247 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
2248                                VirtIOPCIProxy *dev)
2249 {
2250     DeviceState *qdev = DEVICE(dev);
2251     char virtio_bus_name[] = "virtio-bus";
2252
2253     qbus_init(bus, bus_size, TYPE_VIRTIO_PCI_BUS, qdev, virtio_bus_name);
2254 }
2255
2256 static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
2257 {
2258     BusClass *bus_class = BUS_CLASS(klass);
2259     VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
2260     bus_class->max_dev = 1;
2261     k->notify = virtio_pci_notify;
2262     k->save_config = virtio_pci_save_config;
2263     k->load_config = virtio_pci_load_config;
2264     k->save_queue = virtio_pci_save_queue;
2265     k->load_queue = virtio_pci_load_queue;
2266     k->save_extra_state = virtio_pci_save_extra_state;
2267     k->load_extra_state = virtio_pci_load_extra_state;
2268     k->has_extra_state = virtio_pci_has_extra_state;
2269     k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
2270     k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
2271     k->set_host_notifier_mr = virtio_pci_set_host_notifier_mr;
2272     k->vmstate_change = virtio_pci_vmstate_change;
2273     k->pre_plugged = virtio_pci_pre_plugged;
2274     k->device_plugged = virtio_pci_device_plugged;
2275     k->device_unplugged = virtio_pci_device_unplugged;
2276     k->query_nvectors = virtio_pci_query_nvectors;
2277     k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
2278     k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
2279     k->get_dma_as = virtio_pci_get_dma_as;
2280     k->iommu_enabled = virtio_pci_iommu_enabled;
2281     k->queue_enabled = virtio_pci_queue_enabled;
2282 }
2283
2284 static const TypeInfo virtio_pci_bus_info = {
2285     .name          = TYPE_VIRTIO_PCI_BUS,
2286     .parent        = TYPE_VIRTIO_BUS,
2287     .instance_size = sizeof(VirtioPCIBusState),
2288     .class_size    = sizeof(VirtioPCIBusClass),
2289     .class_init    = virtio_pci_bus_class_init,
2290 };
2291
2292 static void virtio_pci_register_types(void)
2293 {
2294     /* Base types: */
2295     type_register_static(&virtio_pci_bus_info);
2296     type_register_static(&virtio_pci_info);
2297 }
2298
2299 type_init(virtio_pci_register_types)
2300