hw/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  * Contributions after 2012-01-13 are licensed under the terms of the
  15  * GNU GPL, version 2 or (at your option) any later version.
  16  */
  17
  18 #include <inttypes.h>
  19
  20 #include "virtio.h"
  21 #include "virtio-blk.h"
  22 #include "virtio-net.h"
  23 #include "virtio-serial.h"
  24 #include "virtio-scsi.h"
  25 #include "pci.h"
  26 #include "qemu-error.h"
  27 #include "msi.h"
  28 #include "msix.h"
  29 #include "net.h"
  30 #include "loader.h"
  31 #include "kvm.h"
  32 #include "blockdev.h"
  33 #include "virtio-pci.h"
  34 #include "range.h"
  35
  36 /* from Linux's linux/virtio_pci.h */
  37
  38 /* A 32-bit r/o bitmask of the features supported by the host */
  39 #define VIRTIO_PCI_HOST_FEATURES        0
  40
  41 /* A 32-bit r/w bitmask of features activated by the guest */
  42 #define VIRTIO_PCI_GUEST_FEATURES       4
  43
  44 /* A 32-bit r/w PFN for the currently selected queue */
  45 #define VIRTIO_PCI_QUEUE_PFN            8
  46
  47 /* A 16-bit r/o queue size for the currently selected queue */
  48 #define VIRTIO_PCI_QUEUE_NUM            12
  49
  50 /* A 16-bit r/w queue selector */
  51 #define VIRTIO_PCI_QUEUE_SEL            14
  52
  53 /* A 16-bit r/w queue notifier */
  54 #define VIRTIO_PCI_QUEUE_NOTIFY         16
  55
  56 /* An 8-bit device status register.  */
  57 #define VIRTIO_PCI_STATUS               18
  58
  59 /* An 8-bit r/o interrupt status register.  Reading the value will return the
  60  * current contents of the ISR and will also clear it.  This is effectively
  61  * a read-and-acknowledge. */
  62 #define VIRTIO_PCI_ISR                  19
  63
  64 /* MSI-X registers: only enabled if MSI-X is enabled. */
  65 /* A 16-bit vector for configuration changes. */
  66 #define VIRTIO_MSI_CONFIG_VECTOR        20
  67 /* A 16-bit vector for selected queue notifications. */
  68 #define VIRTIO_MSI_QUEUE_VECTOR         22
  69
  70 /* Config space size */
  71 #define VIRTIO_PCI_CONFIG_NOMSI         20
  72 #define VIRTIO_PCI_CONFIG_MSI           24
  73 #define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
  74                                          VIRTIO_PCI_CONFIG_MSI : \
  75                                          VIRTIO_PCI_CONFIG_NOMSI)
  76
  77 /* The remaining space is defined by each driver as the per-driver
  78  * configuration space */
  79 #define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
  80                                          VIRTIO_PCI_CONFIG_MSI : \
  81                                          VIRTIO_PCI_CONFIG_NOMSI)
  82
  83 /* How many bits to shift physical queue address written to QUEUE_PFN.
  84  * 12 is historical, and due to x86 page size. */
  85 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
  86
  87 /* Flags track per-device state like workarounds for quirks in older guests. */
  88 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
  89
  90 /* QEMU doesn't strictly need write barriers since everything runs in
  91  * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
  92  * KVM or if kqemu gets SMP support.
  93  */
  94 #define wmb() do { } while (0)
  95
  96 /* HACK for virtio to determine if it's running a big endian guest */
  97 bool virtio_is_big_endian(void);
  98
  99 /* virtio device */
 100
 101 static void virtio_pci_notify(void *opaque, uint16_t vector)
 102 {
 103     VirtIOPCIProxy *proxy = opaque;
 104     if (msix_enabled(&proxy->pci_dev))
 105         msix_notify(&proxy->pci_dev, vector);
 106     else
 107         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 108 }
 109
 110 static void virtio_pci_save_config(void * opaque, QEMUFile *f)
 111 {
 112     VirtIOPCIProxy *proxy = opaque;
 113     pci_device_save(&proxy->pci_dev, f);
 114     msix_save(&proxy->pci_dev, f);
 115     if (msix_present(&proxy->pci_dev))
 116         qemu_put_be16(f, proxy->vdev->config_vector);
 117 }
 118
 119 static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
 120 {
 121     VirtIOPCIProxy *proxy = opaque;
 122     if (msix_present(&proxy->pci_dev))
 123         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 124 }
 125
 126 static int virtio_pci_load_config(void * opaque, QEMUFile *f)
 127 {
 128     VirtIOPCIProxy *proxy = opaque;
 129     int ret;
 130     ret = pci_device_load(&proxy->pci_dev, f);
 131     if (ret) {
 132         return ret;
 133     }
 134     msix_unuse_all_vectors(&proxy->pci_dev);
 135     msix_load(&proxy->pci_dev, f);
 136     if (msix_present(&proxy->pci_dev)) {
 137         qemu_get_be16s(f, &proxy->vdev->config_vector);
 138     } else {
 139         proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
 140     }
 141     if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
 142         return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
 143     }
 144     return 0;
 145 }
 146
 147 static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 148 {
 149     VirtIOPCIProxy *proxy = opaque;
 150     uint16_t vector;
 151     if (msix_present(&proxy->pci_dev)) {
 152         qemu_get_be16s(f, &vector);
 153     } else {
 154         vector = VIRTIO_NO_VECTOR;
 155     }
 156     virtio_queue_set_vector(proxy->vdev, n, vector);
 157     if (vector != VIRTIO_NO_VECTOR) {
 158         return msix_vector_use(&proxy->pci_dev, vector);
 159     }
 160     return 0;
 161 }
 162
 163 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
 164                                                  int n, bool assign, bool set_handler)
 165 {
 166     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 167     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 168     int r = 0;
 169
 170     if (assign) {
 171         r = event_notifier_init(notifier, 1);
 172         if (r < 0) {
 173             error_report("%s: unable to init event notifier: %d",
 174                          __func__, r);
 175             return r;
 176         }
 177         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
 178         memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 179                                   true, n, notifier);
 180     } else {
 181         memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 182                                   true, n, notifier);
 183         virtio_queue_set_host_notifier_fd_handler(vq, false, false);
 184         event_notifier_cleanup(notifier);
 185     }
 186     return r;
 187 }
 188
 189 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 190 {
 191     int n, r;
 192
 193     if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
 194         proxy->ioeventfd_disabled ||
 195         proxy->ioeventfd_started) {
 196         return;
 197     }
 198
 199     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 200         if (!virtio_queue_get_num(proxy->vdev, n)) {
 201             continue;
 202         }
 203
 204         r = virtio_pci_set_host_notifier_internal(proxy, n, true, true);
 205         if (r < 0) {
 206             goto assign_error;
 207         }
 208     }
 209     proxy->ioeventfd_started = true;
 210     return;
 211
 212 assign_error:
 213     while (--n >= 0) {
 214         if (!virtio_queue_get_num(proxy->vdev, n)) {
 215             continue;
 216         }
 217
 218         r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
 219         assert(r >= 0);
 220     }
 221     proxy->ioeventfd_started = false;
 222     error_report("%s: failed. Fallback to a userspace (slower).", __func__);
 223 }
 224
 225 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 226 {
 227     int r;
 228     int n;
 229
 230     if (!proxy->ioeventfd_started) {
 231         return;
 232     }
 233
 234     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 235         if (!virtio_queue_get_num(proxy->vdev, n)) {
 236             continue;
 237         }
 238
 239         r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
 240         assert(r >= 0);
 241     }
 242     proxy->ioeventfd_started = false;
 243 }
 244
 245 void virtio_pci_reset(DeviceState *d)
 246 {
 247     VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
 248     virtio_pci_stop_ioeventfd(proxy);
 249     virtio_reset(proxy->vdev);
 250     msix_unuse_all_vectors(&proxy->pci_dev);
 251     proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 252 }
 253
 254 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 255 {
 256     VirtIOPCIProxy *proxy = opaque;
 257     VirtIODevice *vdev = proxy->vdev;
 258     hwaddr pa;
 259
 260     switch (addr) {
 261     case VIRTIO_PCI_GUEST_FEATURES:
 262         /* Guest does not negotiate properly?  We have to assume nothing. */
 263         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 264             val = vdev->bad_features ? vdev->bad_features(vdev) : 0;
 265         }
 266         virtio_set_features(vdev, val);
 267         break;
 268     case VIRTIO_PCI_QUEUE_PFN:
 269         pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 270         if (pa == 0) {
 271             virtio_pci_stop_ioeventfd(proxy);
 272             virtio_reset(proxy->vdev);
 273             msix_unuse_all_vectors(&proxy->pci_dev);
 274         }
 275         else
 276             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 277         break;
 278     case VIRTIO_PCI_QUEUE_SEL:
 279         if (val < VIRTIO_PCI_QUEUE_MAX)
 280             vdev->queue_sel = val;
 281         break;
 282     case VIRTIO_PCI_QUEUE_NOTIFY:
 283         if (val < VIRTIO_PCI_QUEUE_MAX) {
 284             virtio_queue_notify(vdev, val);
 285         }
 286         break;
 287     case VIRTIO_PCI_STATUS:
 288         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 289             virtio_pci_stop_ioeventfd(proxy);
 290         }
 291
 292         virtio_set_status(vdev, val & 0xFF);
 293
 294         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 295             virtio_pci_start_ioeventfd(proxy);
 296         }
 297
 298         if (vdev->status == 0) {
 299             virtio_reset(proxy->vdev);
 300             msix_unuse_all_vectors(&proxy->pci_dev);
 301         }
 302
 303         /* Linux before 2.6.34 sets the device as OK without enabling
 304            the PCI device bus master bit. In this case we need to disable
 305            some safety checks. */
 306         if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
 307             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 308             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 309         }
 310         break;
 311     case VIRTIO_MSI_CONFIG_VECTOR:
 312         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 313         /* Make it possible for guest to discover an error took place. */
 314         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 315             val = VIRTIO_NO_VECTOR;
 316         vdev->config_vector = val;
 317         break;
 318     case VIRTIO_MSI_QUEUE_VECTOR:
 319         msix_vector_unuse(&proxy->pci_dev,
 320                           virtio_queue_vector(vdev, vdev->queue_sel));
 321         /* Make it possible for guest to discover an error took place. */
 322         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 323             val = VIRTIO_NO_VECTOR;
 324         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 325         break;
 326     default:
 327         error_report("%s: unexpected address 0x%x value 0x%x",
 328                      __func__, addr, val);
 329         break;
 330     }
 331 }
 332
 333 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 334 {
 335     VirtIODevice *vdev = proxy->vdev;
 336     uint32_t ret = 0xFFFFFFFF;
 337
 338     switch (addr) {
 339     case VIRTIO_PCI_HOST_FEATURES:
 340         ret = proxy->host_features;
 341         break;
 342     case VIRTIO_PCI_GUEST_FEATURES:
 343         ret = vdev->guest_features;
 344         break;
 345     case VIRTIO_PCI_QUEUE_PFN:
 346         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 347               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 348         break;
 349     case VIRTIO_PCI_QUEUE_NUM:
 350         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 351         break;
 352     case VIRTIO_PCI_QUEUE_SEL:
 353         ret = vdev->queue_sel;
 354         break;
 355     case VIRTIO_PCI_STATUS:
 356         ret = vdev->status;
 357         break;
 358     case VIRTIO_PCI_ISR:
 359         /* reading from the ISR also clears it. */
 360         ret = vdev->isr;
 361         vdev->isr = 0;
 362         qemu_set_irq(proxy->pci_dev.irq[0], 0);
 363         break;
 364     case VIRTIO_MSI_CONFIG_VECTOR:
 365         ret = vdev->config_vector;
 366         break;
 367     case VIRTIO_MSI_QUEUE_VECTOR:
 368         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 369         break;
 370     default:
 371         break;
 372     }
 373
 374     return ret;
 375 }
 376
 377 static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
 378 {
 379     VirtIOPCIProxy *proxy = opaque;
 380     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 381     if (addr < config)
 382         return virtio_ioport_read(proxy, addr);
 383     addr -= config;
 384     return virtio_config_readb(proxy->vdev, addr);
 385 }
 386
 387 static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
 388 {
 389     VirtIOPCIProxy *proxy = opaque;
 390     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 391     uint16_t val;
 392     if (addr < config)
 393         return virtio_ioport_read(proxy, addr);
 394     addr -= config;
 395     val = virtio_config_readw(proxy->vdev, addr);
 396     if (virtio_is_big_endian()) {
 397         /*
 398          * virtio is odd, ioports are LE but config space is target native
 399          * endian. However, in qemu, all PIO is LE, so we need to re-swap
 400          * on BE targets
 401          */
 402         val = bswap16(val);
 403     }
 404     return val;
 405 }
 406
 407 static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
 408 {
 409     VirtIOPCIProxy *proxy = opaque;
 410     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 411     uint32_t val;
 412     if (addr < config)
 413         return virtio_ioport_read(proxy, addr);
 414     addr -= config;
 415     val = virtio_config_readl(proxy->vdev, addr);
 416     if (virtio_is_big_endian()) {
 417         val = bswap32(val);
 418     }
 419     return val;
 420 }
 421
 422 static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
 423 {
 424     VirtIOPCIProxy *proxy = opaque;
 425     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 426     if (addr < config) {
 427         virtio_ioport_write(proxy, addr, val);
 428         return;
 429     }
 430     addr -= config;
 431     virtio_config_writeb(proxy->vdev, addr, val);
 432 }
 433
 434 static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
 435 {
 436     VirtIOPCIProxy *proxy = opaque;
 437     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 438     if (addr < config) {
 439         virtio_ioport_write(proxy, addr, val);
 440         return;
 441     }
 442     addr -= config;
 443     if (virtio_is_big_endian()) {
 444         val = bswap16(val);
 445     }
 446     virtio_config_writew(proxy->vdev, addr, val);
 447 }
 448
 449 static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
 450 {
 451     VirtIOPCIProxy *proxy = opaque;
 452     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 453     if (addr < config) {
 454         virtio_ioport_write(proxy, addr, val);
 455         return;
 456     }
 457     addr -= config;
 458     if (virtio_is_big_endian()) {
 459         val = bswap32(val);
 460     }
 461     virtio_config_writel(proxy->vdev, addr, val);
 462 }
 463
 464 static const MemoryRegionPortio virtio_portio[] = {
 465     { 0, 0x10000, 1, .write = virtio_pci_config_writeb, },
 466     { 0, 0x10000, 2, .write = virtio_pci_config_writew, },
 467     { 0, 0x10000, 4, .write = virtio_pci_config_writel, },
 468     { 0, 0x10000, 1, .read = virtio_pci_config_readb, },
 469     { 0, 0x10000, 2, .read = virtio_pci_config_readw, },
 470     { 0, 0x10000, 4, .read = virtio_pci_config_readl, },
 471     PORTIO_END_OF_LIST()
 472 };
 473
 474 static const MemoryRegionOps virtio_pci_config_ops = {
 475     .old_portio = virtio_portio,
 476     .endianness = DEVICE_LITTLE_ENDIAN,
 477 };
 478
 479 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 480                                 uint32_t val, int len)
 481 {
 482     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 483
 484     pci_default_write_config(pci_dev, address, val, len);
 485
 486     if (range_covers_byte(address, len, PCI_COMMAND) &&
 487         !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
 488         !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
 489         virtio_pci_stop_ioeventfd(proxy);
 490         virtio_set_status(proxy->vdev,
 491                           proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 492     }
 493 }
 494
 495 static unsigned virtio_pci_get_features(void *opaque)
 496 {
 497     VirtIOPCIProxy *proxy = opaque;
 498     return proxy->host_features;
 499 }
 500
 501 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
 502                                         unsigned int queue_no,
 503                                         unsigned int vector,
 504                                         MSIMessage msg)
 505 {
 506     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 507     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 508     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 509     int ret;
 510
 511     if (irqfd->users == 0) {
 512         ret = kvm_irqchip_add_msi_route(kvm_state, msg);
 513         if (ret < 0) {
 514             return ret;
 515         }
 516         irqfd->virq = ret;
 517     }
 518     irqfd->users++;
 519
 520     ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
 521     if (ret < 0) {
 522         if (--irqfd->users == 0) {
 523             kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 524         }
 525         return ret;
 526     }
 527
 528     virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
 529     return 0;
 530 }
 531
 532 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 533                                              unsigned int queue_no,
 534                                              unsigned int vector)
 535 {
 536     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 537     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 538     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 539     int ret;
 540
 541     ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
 542     assert(ret == 0);
 543
 544     if (--irqfd->users == 0) {
 545         kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 546     }
 547
 548     virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 549 }
 550
 551 static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
 552                                      MSIMessage msg)
 553 {
 554     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 555     VirtIODevice *vdev = proxy->vdev;
 556     int ret, queue_no;
 557
 558     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 559         if (!virtio_queue_get_num(vdev, queue_no)) {
 560             break;
 561         }
 562         if (virtio_queue_vector(vdev, queue_no) != vector) {
 563             continue;
 564         }
 565         ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
 566         if (ret < 0) {
 567             goto undo;
 568         }
 569     }
 570     return 0;
 571
 572 undo:
 573     while (--queue_no >= 0) {
 574         if (virtio_queue_vector(vdev, queue_no) != vector) {
 575             continue;
 576         }
 577         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 578     }
 579     return ret;
 580 }
 581
 582 static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
 583 {
 584     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 585     VirtIODevice *vdev = proxy->vdev;
 586     int queue_no;
 587
 588     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 589         if (!virtio_queue_get_num(vdev, queue_no)) {
 590             break;
 591         }
 592         if (virtio_queue_vector(vdev, queue_no) != vector) {
 593             continue;
 594         }
 595         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 596     }
 597 }
 598
 599 static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
 600 {
 601     VirtIOPCIProxy *proxy = opaque;
 602     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 603     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 604
 605     if (assign) {
 606         int r = event_notifier_init(notifier, 0);
 607         if (r < 0) {
 608             return r;
 609         }
 610         virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 611     } else {
 612         virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
 613         event_notifier_cleanup(notifier);
 614     }
 615
 616     return 0;
 617 }
 618
 619 static bool virtio_pci_query_guest_notifiers(void *opaque)
 620 {
 621     VirtIOPCIProxy *proxy = opaque;
 622     return msix_enabled(&proxy->pci_dev);
 623 }
 624
 625 static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
 626 {
 627     VirtIOPCIProxy *proxy = opaque;
 628     VirtIODevice *vdev = proxy->vdev;
 629     int r, n;
 630
 631     /* Must unset vector notifier while guest notifier is still assigned */
 632     if (kvm_msi_via_irqfd_enabled() && !assign) {
 633         msix_unset_vector_notifiers(&proxy->pci_dev);
 634         g_free(proxy->vector_irqfd);
 635         proxy->vector_irqfd = NULL;
 636     }
 637
 638     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 639         if (!virtio_queue_get_num(vdev, n)) {
 640             break;
 641         }
 642
 643         r = virtio_pci_set_guest_notifier(opaque, n, assign);
 644         if (r < 0) {
 645             goto assign_error;
 646         }
 647     }
 648
 649     /* Must set vector notifier after guest notifier has been assigned */
 650     if (kvm_msi_via_irqfd_enabled() && assign) {
 651         proxy->vector_irqfd =
 652             g_malloc0(sizeof(*proxy->vector_irqfd) *
 653                       msix_nr_vectors_allocated(&proxy->pci_dev));
 654         r = msix_set_vector_notifiers(&proxy->pci_dev,
 655                                       kvm_virtio_pci_vector_use,
 656                                       kvm_virtio_pci_vector_release);
 657         if (r < 0) {
 658             goto assign_error;
 659         }
 660     }
 661
 662     return 0;
 663
 664 assign_error:
 665     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
 666     assert(assign);
 667     while (--n >= 0) {
 668         virtio_pci_set_guest_notifier(opaque, n, !assign);
 669     }
 670     return r;
 671 }
 672
 673 static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
 674 {
 675     VirtIOPCIProxy *proxy = opaque;
 676
 677     /* Stop using ioeventfd for virtqueue kick if the device starts using host
 678      * notifiers.  This makes it easy to avoid stepping on each others' toes.
 679      */
 680     proxy->ioeventfd_disabled = assign;
 681     if (assign) {
 682         virtio_pci_stop_ioeventfd(proxy);
 683     }
 684     /* We don't need to start here: it's not needed because backend
 685      * currently only stops on status change away from ok,
 686      * reset, vmstop and such. If we do add code to start here,
 687      * need to check vmstate, device state etc. */
 688     return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
 689 }
 690
 691 static void virtio_pci_vmstate_change(void *opaque, bool running)
 692 {
 693     VirtIOPCIProxy *proxy = opaque;
 694
 695     if (running) {
 696         /* Try to find out if the guest has bus master disabled, but is
 697            in ready state. Then we have a buggy guest OS. */
 698         if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 699             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 700             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 701         }
 702         virtio_pci_start_ioeventfd(proxy);
 703     } else {
 704         virtio_pci_stop_ioeventfd(proxy);
 705     }
 706 }
 707
 708 static const VirtIOBindings virtio_pci_bindings = {
 709     .notify = virtio_pci_notify,
 710     .save_config = virtio_pci_save_config,
 711     .load_config = virtio_pci_load_config,
 712     .save_queue = virtio_pci_save_queue,
 713     .load_queue = virtio_pci_load_queue,
 714     .get_features = virtio_pci_get_features,
 715     .query_guest_notifiers = virtio_pci_query_guest_notifiers,
 716     .set_host_notifier = virtio_pci_set_host_notifier,
 717     .set_guest_notifiers = virtio_pci_set_guest_notifiers,
 718     .vmstate_change = virtio_pci_vmstate_change,
 719 };
 720
 721 void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
 722 {
 723     uint8_t *config;
 724     uint32_t size;
 725
 726     proxy->vdev = vdev;
 727
 728     config = proxy->pci_dev.config;
 729
 730     if (proxy->class_code) {
 731         pci_config_set_class(config, proxy->class_code);
 732     }
 733     pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
 734                  pci_get_word(config + PCI_VENDOR_ID));
 735     pci_set_word(config + PCI_SUBSYSTEM_ID, vdev->device_id);
 736     config[PCI_INTERRUPT_PIN] = 1;
 737
 738     if (vdev->nvectors &&
 739         msix_init_exclusive_bar(&proxy->pci_dev, vdev->nvectors, 1)) {
 740         vdev->nvectors = 0;
 741     }
 742
 743     proxy->pci_dev.config_write = virtio_write_config;
 744
 745     size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
 746     if (size & (size-1))
 747         size = 1 << qemu_fls(size);
 748
 749     memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
 750                           "virtio-pci", size);
 751     pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
 752                      &proxy->bar);
 753
 754     if (!kvm_has_many_ioeventfds()) {
 755         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
 756     }
 757
 758     virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
 759     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
 760     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
 761     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
 762 }
 763
 764 static int virtio_blk_init_pci(PCIDevice *pci_dev)
 765 {
 766     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 767     VirtIODevice *vdev;
 768
 769     if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
 770         proxy->class_code != PCI_CLASS_STORAGE_OTHER)
 771         proxy->class_code = PCI_CLASS_STORAGE_SCSI;
 772
 773     vdev = virtio_blk_init(&pci_dev->qdev, &proxy->blk);
 774     if (!vdev) {
 775         return -1;
 776     }
 777     vdev->nvectors = proxy->nvectors;
 778     virtio_init_pci(proxy, vdev);
 779     /* make the actual value visible */
 780     proxy->nvectors = vdev->nvectors;
 781     return 0;
 782 }
 783
 784 static void virtio_exit_pci(PCIDevice *pci_dev)
 785 {
 786     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 787
 788     memory_region_destroy(&proxy->bar);
 789     msix_uninit_exclusive_bar(pci_dev);
 790 }
 791
 792 static void virtio_blk_exit_pci(PCIDevice *pci_dev)
 793 {
 794     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 795
 796     virtio_pci_stop_ioeventfd(proxy);
 797     virtio_blk_exit(proxy->vdev);
 798     virtio_exit_pci(pci_dev);
 799 }
 800
 801 static int virtio_serial_init_pci(PCIDevice *pci_dev)
 802 {
 803     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 804     VirtIODevice *vdev;
 805
 806     if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
 807         proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
 808         proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
 809         proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
 810
 811     vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial);
 812     if (!vdev) {
 813         return -1;
 814     }
 815     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
 816                                         ? proxy->serial.max_virtserial_ports + 1
 817                                         : proxy->nvectors;
 818     virtio_init_pci(proxy, vdev);
 819     proxy->nvectors = vdev->nvectors;
 820     return 0;
 821 }
 822
 823 static void virtio_serial_exit_pci(PCIDevice *pci_dev)
 824 {
 825     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 826
 827     virtio_pci_stop_ioeventfd(proxy);
 828     virtio_serial_exit(proxy->vdev);
 829     virtio_exit_pci(pci_dev);
 830 }
 831
 832 static int virtio_net_init_pci(PCIDevice *pci_dev)
 833 {
 834     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 835     VirtIODevice *vdev;
 836
 837     vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
 838
 839     vdev->nvectors = proxy->nvectors;
 840     virtio_init_pci(proxy, vdev);
 841
 842     /* make the actual value visible */
 843     proxy->nvectors = vdev->nvectors;
 844     return 0;
 845 }
 846
 847 static void virtio_net_exit_pci(PCIDevice *pci_dev)
 848 {
 849     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 850
 851     virtio_pci_stop_ioeventfd(proxy);
 852     virtio_net_exit(proxy->vdev);
 853     virtio_exit_pci(pci_dev);
 854 }
 855
 856 static int virtio_balloon_init_pci(PCIDevice *pci_dev)
 857 {
 858     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 859     VirtIODevice *vdev;
 860
 861     if (proxy->class_code != PCI_CLASS_OTHERS &&
 862         proxy->class_code != PCI_CLASS_MEMORY_RAM) { /* qemu < 1.1 */
 863         proxy->class_code = PCI_CLASS_OTHERS;
 864     }
 865
 866     vdev = virtio_balloon_init(&pci_dev->qdev);
 867     if (!vdev) {
 868         return -1;
 869     }
 870     virtio_init_pci(proxy, vdev);
 871     return 0;
 872 }
 873
 874 static void virtio_balloon_exit_pci(PCIDevice *pci_dev)
 875 {
 876     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 877
 878     virtio_pci_stop_ioeventfd(proxy);
 879     virtio_balloon_exit(proxy->vdev);
 880     virtio_exit_pci(pci_dev);
 881 }
 882
 883 static Property virtio_blk_properties[] = {
 884     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 885     DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
 886     DEFINE_BLOCK_CHS_PROPERTIES(VirtIOPCIProxy, blk.conf),
 887     DEFINE_PROP_STRING("serial", VirtIOPCIProxy, blk.serial),
 888 #ifdef __linux__
 889     DEFINE_PROP_BIT("scsi", VirtIOPCIProxy, blk.scsi, 0, true),
 890 #endif
 891     DEFINE_PROP_BIT("config-wce", VirtIOPCIProxy, blk.config_wce, 0, true),
 892     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 893     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 894     DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
 895     DEFINE_PROP_END_OF_LIST(),
 896 };
 897
 898 static void virtio_blk_class_init(ObjectClass *klass, void *data)
 899 {
 900     DeviceClass *dc = DEVICE_CLASS(klass);
 901     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 902
 903     k->init = virtio_blk_init_pci;
 904     k->exit = virtio_blk_exit_pci;
 905     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 906     k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
 907     k->revision = VIRTIO_PCI_ABI_VERSION;
 908     k->class_id = PCI_CLASS_STORAGE_SCSI;
 909     dc->reset = virtio_pci_reset;
 910     dc->props = virtio_blk_properties;
 911 }
 912
 913 static TypeInfo virtio_blk_info = {
 914     .name          = "virtio-blk-pci",
 915     .parent        = TYPE_PCI_DEVICE,
 916     .instance_size = sizeof(VirtIOPCIProxy),
 917     .class_init    = virtio_blk_class_init,
 918 };
 919
 920 static Property virtio_net_properties[] = {
 921     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
 922     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
 923     DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
 924     DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
 925     DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy, net.txtimer, TX_TIMER_INTERVAL),
 926     DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy, net.txburst, TX_BURST),
 927     DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
 928     DEFINE_PROP_END_OF_LIST(),
 929 };
 930
 931 static void virtio_net_class_init(ObjectClass *klass, void *data)
 932 {
 933     DeviceClass *dc = DEVICE_CLASS(klass);
 934     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 935
 936     k->init = virtio_net_init_pci;
 937     k->exit = virtio_net_exit_pci;
 938     k->romfile = "pxe-virtio.rom";
 939     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 940     k->device_id = PCI_DEVICE_ID_VIRTIO_NET;
 941     k->revision = VIRTIO_PCI_ABI_VERSION;
 942     k->class_id = PCI_CLASS_NETWORK_ETHERNET;
 943     dc->reset = virtio_pci_reset;
 944     dc->props = virtio_net_properties;
 945 }
 946
 947 static TypeInfo virtio_net_info = {
 948     .name          = "virtio-net-pci",
 949     .parent        = TYPE_PCI_DEVICE,
 950     .instance_size = sizeof(VirtIOPCIProxy),
 951     .class_init    = virtio_net_class_init,
 952 };
 953
 954 static Property virtio_serial_properties[] = {
 955     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 956     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
 957     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 958     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 959     DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, serial.max_virtserial_ports, 31),
 960     DEFINE_PROP_END_OF_LIST(),
 961 };
 962
 963 static void virtio_serial_class_init(ObjectClass *klass, void *data)
 964 {
 965     DeviceClass *dc = DEVICE_CLASS(klass);
 966     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 967
 968     k->init = virtio_serial_init_pci;
 969     k->exit = virtio_serial_exit_pci;
 970     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 971     k->device_id = PCI_DEVICE_ID_VIRTIO_CONSOLE;
 972     k->revision = VIRTIO_PCI_ABI_VERSION;
 973     k->class_id = PCI_CLASS_COMMUNICATION_OTHER;
 974     dc->reset = virtio_pci_reset;
 975     dc->props = virtio_serial_properties;
 976 }
 977
 978 static TypeInfo virtio_serial_info = {
 979     .name          = "virtio-serial-pci",
 980     .parent        = TYPE_PCI_DEVICE,
 981     .instance_size = sizeof(VirtIOPCIProxy),
 982     .class_init    = virtio_serial_class_init,
 983 };
 984
 985 static Property virtio_balloon_properties[] = {
 986     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 987     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 988     DEFINE_PROP_END_OF_LIST(),
 989 };
 990
 991 static void virtio_balloon_class_init(ObjectClass *klass, void *data)
 992 {
 993     DeviceClass *dc = DEVICE_CLASS(klass);
 994     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 995
 996     k->init = virtio_balloon_init_pci;
 997     k->exit = virtio_balloon_exit_pci;
 998     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 999     k->device_id = PCI_DEVICE_ID_VIRTIO_BALLOON;
1000     k->revision = VIRTIO_PCI_ABI_VERSION;
1001     k->class_id = PCI_CLASS_OTHERS;
1002     dc->reset = virtio_pci_reset;
1003     dc->props = virtio_balloon_properties;
1004 }
1005
1006 static TypeInfo virtio_balloon_info = {
1007     .name          = "virtio-balloon-pci",
1008     .parent        = TYPE_PCI_DEVICE,
1009     .instance_size = sizeof(VirtIOPCIProxy),
1010     .class_init    = virtio_balloon_class_init,
1011 };
1012
1013 static int virtio_scsi_init_pci(PCIDevice *pci_dev)
1014 {
1015     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1016     VirtIODevice *vdev;
1017
1018     vdev = virtio_scsi_init(&pci_dev->qdev, &proxy->scsi);
1019     if (!vdev) {
1020         return -EINVAL;
1021     }
1022
1023     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
1024                                         ? proxy->scsi.num_queues + 3
1025                                         : proxy->nvectors;
1026     virtio_init_pci(proxy, vdev);
1027
1028     /* make the actual value visible */
1029     proxy->nvectors = vdev->nvectors;
1030     return 0;
1031 }
1032
1033 static void virtio_scsi_exit_pci(PCIDevice *pci_dev)
1034 {
1035     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1036
1037     virtio_scsi_exit(proxy->vdev);
1038     virtio_exit_pci(pci_dev);
1039 }
1040
1041 static Property virtio_scsi_properties[] = {
1042     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
1043     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
1044     DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
1045     DEFINE_PROP_END_OF_LIST(),
1046 };
1047
1048 static void virtio_scsi_class_init(ObjectClass *klass, void *data)
1049 {
1050     DeviceClass *dc = DEVICE_CLASS(klass);
1051     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1052
1053     k->init = virtio_scsi_init_pci;
1054     k->exit = virtio_scsi_exit_pci;
1055     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
1056     k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
1057     k->revision = 0x00;
1058     k->class_id = PCI_CLASS_STORAGE_SCSI;
1059     dc->reset = virtio_pci_reset;
1060     dc->props = virtio_scsi_properties;
1061 }
1062
1063 static TypeInfo virtio_scsi_info = {
1064     .name          = "virtio-scsi-pci",
1065     .parent        = TYPE_PCI_DEVICE,
1066     .instance_size = sizeof(VirtIOPCIProxy),
1067     .class_init    = virtio_scsi_class_init,
1068 };
1069
1070 static void virtio_pci_register_types(void)
1071 {
1072     type_register_static(&virtio_blk_info);
1073     type_register_static(&virtio_net_info);
1074     type_register_static(&virtio_serial_info);
1075     type_register_static(&virtio_balloon_info);
1076     type_register_static(&virtio_scsi_info);
1077 }
1078
1079 type_init(virtio_pci_register_types)