hw/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <inttypes.h>
  17
  18 #include "virtio.h"
  19 #include "virtio-blk.h"
  20 #include "virtio-net.h"
  21 #include "virtio-serial.h"
  22 #include "pci.h"
  23 #include "qemu-error.h"
  24 #include "msix.h"
  25 #include "net.h"
  26 #include "loader.h"
  27 #include "kvm.h"
  28 #include "blockdev.h"
  29
  30 /* from Linux's linux/virtio_pci.h */
  31
  32 /* A 32-bit r/o bitmask of the features supported by the host */
  33 #define VIRTIO_PCI_HOST_FEATURES        0
  34
  35 /* A 32-bit r/w bitmask of features activated by the guest */
  36 #define VIRTIO_PCI_GUEST_FEATURES       4
  37
  38 /* A 32-bit r/w PFN for the currently selected queue */
  39 #define VIRTIO_PCI_QUEUE_PFN            8
  40
  41 /* A 16-bit r/o queue size for the currently selected queue */
  42 #define VIRTIO_PCI_QUEUE_NUM            12
  43
  44 /* A 16-bit r/w queue selector */
  45 #define VIRTIO_PCI_QUEUE_SEL            14
  46
  47 /* A 16-bit r/w queue notifier */
  48 #define VIRTIO_PCI_QUEUE_NOTIFY         16
  49
  50 /* An 8-bit device status register.  */
  51 #define VIRTIO_PCI_STATUS               18
  52
  53 /* An 8-bit r/o interrupt status register.  Reading the value will return the
  54  * current contents of the ISR and will also clear it.  This is effectively
  55  * a read-and-acknowledge. */
  56 #define VIRTIO_PCI_ISR                  19
  57
  58 /* MSI-X registers: only enabled if MSI-X is enabled. */
  59 /* A 16-bit vector for configuration changes. */
  60 #define VIRTIO_MSI_CONFIG_VECTOR        20
  61 /* A 16-bit vector for selected queue notifications. */
  62 #define VIRTIO_MSI_QUEUE_VECTOR         22
  63
  64 /* Config space size */
  65 #define VIRTIO_PCI_CONFIG_NOMSI         20
  66 #define VIRTIO_PCI_CONFIG_MSI           24
  67 #define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
  68                                          VIRTIO_PCI_CONFIG_MSI : \
  69                                          VIRTIO_PCI_CONFIG_NOMSI)
  70
  71 /* The remaining space is defined by each driver as the per-driver
  72  * configuration space */
  73 #define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
  74                                          VIRTIO_PCI_CONFIG_MSI : \
  75                                          VIRTIO_PCI_CONFIG_NOMSI)
  76
  77 /* Virtio ABI version, if we increment this, we break the guest driver. */
  78 #define VIRTIO_PCI_ABI_VERSION          0
  79
  80 /* How many bits to shift physical queue address written to QUEUE_PFN.
  81  * 12 is historical, and due to x86 page size. */
  82 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
  83
  84 /* Flags track per-device state like workarounds for quirks in older guests. */
  85 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
  86
  87 /* Performance improves when virtqueue kick processing is decoupled from the
  88  * vcpu thread using ioeventfd for some devices. */
  89 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
  90 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
  91
  92 /* QEMU doesn't strictly need write barriers since everything runs in
  93  * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
  94  * KVM or if kqemu gets SMP support.
  95  */
  96 #define wmb() do { } while (0)
  97
  98 /* PCI bindings.  */
  99
 100 typedef struct {
 101     PCIDevice pci_dev;
 102     VirtIODevice *vdev;
 103     uint32_t flags;
 104     uint32_t addr;
 105     uint32_t class_code;
 106     uint32_t nvectors;
 107     BlockConf block;
 108     NICConf nic;
 109     uint32_t host_features;
 110 #ifdef CONFIG_LINUX
 111     V9fsConf fsconf;
 112 #endif
 113     virtio_serial_conf serial;
 114     virtio_net_conf net;
 115     bool ioeventfd_disabled;
 116     bool ioeventfd_started;
 117 } VirtIOPCIProxy;
 118
 119 /* virtio device */
 120
 121 static void virtio_pci_notify(void *opaque, uint16_t vector)
 122 {
 123     VirtIOPCIProxy *proxy = opaque;
 124     if (msix_enabled(&proxy->pci_dev))
 125         msix_notify(&proxy->pci_dev, vector);
 126     else
 127         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 128 }
 129
 130 static void virtio_pci_save_config(void * opaque, QEMUFile *f)
 131 {
 132     VirtIOPCIProxy *proxy = opaque;
 133     pci_device_save(&proxy->pci_dev, f);
 134     msix_save(&proxy->pci_dev, f);
 135     if (msix_present(&proxy->pci_dev))
 136         qemu_put_be16(f, proxy->vdev->config_vector);
 137 }
 138
 139 static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
 140 {
 141     VirtIOPCIProxy *proxy = opaque;
 142     if (msix_present(&proxy->pci_dev))
 143         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 144 }
 145
 146 static int virtio_pci_load_config(void * opaque, QEMUFile *f)
 147 {
 148     VirtIOPCIProxy *proxy = opaque;
 149     int ret;
 150     ret = pci_device_load(&proxy->pci_dev, f);
 151     if (ret) {
 152         return ret;
 153     }
 154     msix_load(&proxy->pci_dev, f);
 155     if (msix_present(&proxy->pci_dev)) {
 156         qemu_get_be16s(f, &proxy->vdev->config_vector);
 157     } else {
 158         proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
 159     }
 160     if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
 161         return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
 162     }
 163
 164     /* Try to find out if the guest has bus master disabled, but is
 165        in ready state. Then we have a buggy guest OS. */
 166     if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 167         !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 168         proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 169     }
 170     return 0;
 171 }
 172
 173 static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 174 {
 175     VirtIOPCIProxy *proxy = opaque;
 176     uint16_t vector;
 177     if (msix_present(&proxy->pci_dev)) {
 178         qemu_get_be16s(f, &vector);
 179     } else {
 180         vector = VIRTIO_NO_VECTOR;
 181     }
 182     virtio_queue_set_vector(proxy->vdev, n, vector);
 183     if (vector != VIRTIO_NO_VECTOR) {
 184         return msix_vector_use(&proxy->pci_dev, vector);
 185     }
 186     return 0;
 187 }
 188
 189 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
 190                                                  int n, bool assign)
 191 {
 192     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 193     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 194     int r;
 195     if (assign) {
 196         r = event_notifier_init(notifier, 1);
 197         if (r < 0) {
 198             error_report("%s: unable to init event notifier: %d",
 199                          __func__, r);
 200             return r;
 201         }
 202         r = kvm_set_ioeventfd_pio_word(event_notifier_get_fd(notifier),
 203                                        proxy->addr + VIRTIO_PCI_QUEUE_NOTIFY,
 204                                        n, assign);
 205         if (r < 0) {
 206             error_report("%s: unable to map ioeventfd: %d",
 207                          __func__, r);
 208             event_notifier_cleanup(notifier);
 209         }
 210     } else {
 211         r = kvm_set_ioeventfd_pio_word(event_notifier_get_fd(notifier),
 212                                        proxy->addr + VIRTIO_PCI_QUEUE_NOTIFY,
 213                                        n, assign);
 214         if (r < 0) {
 215             error_report("%s: unable to unmap ioeventfd: %d",
 216                          __func__, r);
 217             return r;
 218         }
 219
 220         /* Handle the race condition where the guest kicked and we deassigned
 221          * before we got around to handling the kick.
 222          */
 223         if (event_notifier_test_and_clear(notifier)) {
 224             virtio_queue_notify_vq(vq);
 225         }
 226
 227         event_notifier_cleanup(notifier);
 228     }
 229     return r;
 230 }
 231
 232 static void virtio_pci_host_notifier_read(void *opaque)
 233 {
 234     VirtQueue *vq = opaque;
 235     EventNotifier *n = virtio_queue_get_host_notifier(vq);
 236     if (event_notifier_test_and_clear(n)) {
 237         virtio_queue_notify_vq(vq);
 238     }
 239 }
 240
 241 static void virtio_pci_set_host_notifier_fd_handler(VirtIOPCIProxy *proxy,
 242                                                     int n, bool assign)
 243 {
 244     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 245     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 246     if (assign) {
 247         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 248                             virtio_pci_host_notifier_read, NULL, vq);
 249     } else {
 250         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 251                             NULL, NULL, NULL);
 252     }
 253 }
 254
 255 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 256 {
 257     int n, r;
 258
 259     if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
 260         proxy->ioeventfd_disabled ||
 261         proxy->ioeventfd_started) {
 262         return;
 263     }
 264
 265     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 266         if (!virtio_queue_get_num(proxy->vdev, n)) {
 267             continue;
 268         }
 269
 270         r = virtio_pci_set_host_notifier_internal(proxy, n, true);
 271         if (r < 0) {
 272             goto assign_error;
 273         }
 274
 275         virtio_pci_set_host_notifier_fd_handler(proxy, n, true);
 276     }
 277     proxy->ioeventfd_started = true;
 278     return;
 279
 280 assign_error:
 281     while (--n >= 0) {
 282         if (!virtio_queue_get_num(proxy->vdev, n)) {
 283             continue;
 284         }
 285
 286         virtio_pci_set_host_notifier_fd_handler(proxy, n, false);
 287         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 288         assert(r >= 0);
 289     }
 290     proxy->ioeventfd_started = false;
 291     error_report("%s: failed. Fallback to a userspace (slower).", __func__);
 292 }
 293
 294 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 295 {
 296     int r;
 297     int n;
 298
 299     if (!proxy->ioeventfd_started) {
 300         return;
 301     }
 302
 303     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 304         if (!virtio_queue_get_num(proxy->vdev, n)) {
 305             continue;
 306         }
 307
 308         virtio_pci_set_host_notifier_fd_handler(proxy, n, false);
 309         r = virtio_pci_set_host_notifier_internal(proxy, n, false);
 310         assert(r >= 0);
 311     }
 312     proxy->ioeventfd_started = false;
 313 }
 314
 315 static void virtio_pci_reset(DeviceState *d)
 316 {
 317     VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
 318     virtio_pci_stop_ioeventfd(proxy);
 319     virtio_reset(proxy->vdev);
 320     msix_reset(&proxy->pci_dev);
 321     proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 322 }
 323
 324 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 325 {
 326     VirtIOPCIProxy *proxy = opaque;
 327     VirtIODevice *vdev = proxy->vdev;
 328     target_phys_addr_t pa;
 329
 330     switch (addr) {
 331     case VIRTIO_PCI_GUEST_FEATURES:
 332         /* Guest does not negotiate properly?  We have to assume nothing. */
 333         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 334             if (vdev->bad_features)
 335                 val = proxy->host_features & vdev->bad_features(vdev);
 336             else
 337                 val = 0;
 338         }
 339         if (vdev->set_features)
 340             vdev->set_features(vdev, val);
 341         vdev->guest_features = val;
 342         break;
 343     case VIRTIO_PCI_QUEUE_PFN:
 344         pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 345         if (pa == 0) {
 346             virtio_pci_stop_ioeventfd(proxy);
 347             virtio_reset(proxy->vdev);
 348             msix_unuse_all_vectors(&proxy->pci_dev);
 349         }
 350         else
 351             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 352         break;
 353     case VIRTIO_PCI_QUEUE_SEL:
 354         if (val < VIRTIO_PCI_QUEUE_MAX)
 355             vdev->queue_sel = val;
 356         break;
 357     case VIRTIO_PCI_QUEUE_NOTIFY:
 358         virtio_queue_notify(vdev, val);
 359         break;
 360     case VIRTIO_PCI_STATUS:
 361         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 362             virtio_pci_stop_ioeventfd(proxy);
 363         }
 364
 365         virtio_set_status(vdev, val & 0xFF);
 366
 367         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 368             virtio_pci_start_ioeventfd(proxy);
 369         }
 370
 371         if (vdev->status == 0) {
 372             virtio_reset(proxy->vdev);
 373             msix_unuse_all_vectors(&proxy->pci_dev);
 374         }
 375
 376         /* Linux before 2.6.34 sets the device as OK without enabling
 377            the PCI device bus master bit. In this case we need to disable
 378            some safety checks. */
 379         if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
 380             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 381             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 382         }
 383         break;
 384     case VIRTIO_MSI_CONFIG_VECTOR:
 385         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 386         /* Make it possible for guest to discover an error took place. */
 387         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 388             val = VIRTIO_NO_VECTOR;
 389         vdev->config_vector = val;
 390         break;
 391     case VIRTIO_MSI_QUEUE_VECTOR:
 392         msix_vector_unuse(&proxy->pci_dev,
 393                           virtio_queue_vector(vdev, vdev->queue_sel));
 394         /* Make it possible for guest to discover an error took place. */
 395         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 396             val = VIRTIO_NO_VECTOR;
 397         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 398         break;
 399     default:
 400         error_report("%s: unexpected address 0x%x value 0x%x",
 401                      __func__, addr, val);
 402         break;
 403     }
 404 }
 405
 406 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 407 {
 408     VirtIODevice *vdev = proxy->vdev;
 409     uint32_t ret = 0xFFFFFFFF;
 410
 411     switch (addr) {
 412     case VIRTIO_PCI_HOST_FEATURES:
 413         ret = proxy->host_features;
 414         break;
 415     case VIRTIO_PCI_GUEST_FEATURES:
 416         ret = vdev->guest_features;
 417         break;
 418     case VIRTIO_PCI_QUEUE_PFN:
 419         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 420               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 421         break;
 422     case VIRTIO_PCI_QUEUE_NUM:
 423         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 424         break;
 425     case VIRTIO_PCI_QUEUE_SEL:
 426         ret = vdev->queue_sel;
 427         break;
 428     case VIRTIO_PCI_STATUS:
 429         ret = vdev->status;
 430         break;
 431     case VIRTIO_PCI_ISR:
 432         /* reading from the ISR also clears it. */
 433         ret = vdev->isr;
 434         vdev->isr = 0;
 435         qemu_set_irq(proxy->pci_dev.irq[0], 0);
 436         break;
 437     case VIRTIO_MSI_CONFIG_VECTOR:
 438         ret = vdev->config_vector;
 439         break;
 440     case VIRTIO_MSI_QUEUE_VECTOR:
 441         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 442         break;
 443     default:
 444         break;
 445     }
 446
 447     return ret;
 448 }
 449
 450 static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
 451 {
 452     VirtIOPCIProxy *proxy = opaque;
 453     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 454     addr -= proxy->addr;
 455     if (addr < config)
 456         return virtio_ioport_read(proxy, addr);
 457     addr -= config;
 458     return virtio_config_readb(proxy->vdev, addr);
 459 }
 460
 461 static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
 462 {
 463     VirtIOPCIProxy *proxy = opaque;
 464     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 465     addr -= proxy->addr;
 466     if (addr < config)
 467         return virtio_ioport_read(proxy, addr);
 468     addr -= config;
 469     return virtio_config_readw(proxy->vdev, addr);
 470 }
 471
 472 static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
 473 {
 474     VirtIOPCIProxy *proxy = opaque;
 475     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 476     addr -= proxy->addr;
 477     if (addr < config)
 478         return virtio_ioport_read(proxy, addr);
 479     addr -= config;
 480     return virtio_config_readl(proxy->vdev, addr);
 481 }
 482
 483 static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
 484 {
 485     VirtIOPCIProxy *proxy = opaque;
 486     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 487     addr -= proxy->addr;
 488     if (addr < config) {
 489         virtio_ioport_write(proxy, addr, val);
 490         return;
 491     }
 492     addr -= config;
 493     virtio_config_writeb(proxy->vdev, addr, val);
 494 }
 495
 496 static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
 497 {
 498     VirtIOPCIProxy *proxy = opaque;
 499     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 500     addr -= proxy->addr;
 501     if (addr < config) {
 502         virtio_ioport_write(proxy, addr, val);
 503         return;
 504     }
 505     addr -= config;
 506     virtio_config_writew(proxy->vdev, addr, val);
 507 }
 508
 509 static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
 510 {
 511     VirtIOPCIProxy *proxy = opaque;
 512     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 513     addr -= proxy->addr;
 514     if (addr < config) {
 515         virtio_ioport_write(proxy, addr, val);
 516         return;
 517     }
 518     addr -= config;
 519     virtio_config_writel(proxy->vdev, addr, val);
 520 }
 521
 522 static void virtio_map(PCIDevice *pci_dev, int region_num,
 523                        pcibus_t addr, pcibus_t size, int type)
 524 {
 525     VirtIOPCIProxy *proxy = container_of(pci_dev, VirtIOPCIProxy, pci_dev);
 526     VirtIODevice *vdev = proxy->vdev;
 527     unsigned config_len = VIRTIO_PCI_REGION_SIZE(pci_dev) + vdev->config_len;
 528
 529     proxy->addr = addr;
 530
 531     register_ioport_write(addr, config_len, 1, virtio_pci_config_writeb, proxy);
 532     register_ioport_write(addr, config_len, 2, virtio_pci_config_writew, proxy);
 533     register_ioport_write(addr, config_len, 4, virtio_pci_config_writel, proxy);
 534     register_ioport_read(addr, config_len, 1, virtio_pci_config_readb, proxy);
 535     register_ioport_read(addr, config_len, 2, virtio_pci_config_readw, proxy);
 536     register_ioport_read(addr, config_len, 4, virtio_pci_config_readl, proxy);
 537
 538     if (vdev->config_len)
 539         vdev->get_config(vdev, vdev->config);
 540 }
 541
 542 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 543                                 uint32_t val, int len)
 544 {
 545     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 546
 547     if (PCI_COMMAND == address) {
 548         if (!(val & PCI_COMMAND_MASTER)) {
 549             if (!(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
 550                 virtio_pci_stop_ioeventfd(proxy);
 551                 virtio_set_status(proxy->vdev,
 552                                   proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 553             }
 554         }
 555     }
 556
 557     pci_default_write_config(pci_dev, address, val, len);
 558     msix_write_config(pci_dev, address, val, len);
 559 }
 560
 561 static unsigned virtio_pci_get_features(void *opaque)
 562 {
 563     VirtIOPCIProxy *proxy = opaque;
 564     return proxy->host_features;
 565 }
 566
 567 static void virtio_pci_guest_notifier_read(void *opaque)
 568 {
 569     VirtQueue *vq = opaque;
 570     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 571     if (event_notifier_test_and_clear(n)) {
 572         virtio_irq(vq);
 573     }
 574 }
 575
 576 static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
 577 {
 578     VirtIOPCIProxy *proxy = opaque;
 579     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 580     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 581
 582     if (assign) {
 583         int r = event_notifier_init(notifier, 0);
 584         if (r < 0) {
 585             return r;
 586         }
 587         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 588                             virtio_pci_guest_notifier_read, NULL, vq);
 589     } else {
 590         qemu_set_fd_handler(event_notifier_get_fd(notifier),
 591                             NULL, NULL, NULL);
 592         event_notifier_cleanup(notifier);
 593     }
 594
 595     return 0;
 596 }
 597
 598 static bool virtio_pci_query_guest_notifiers(void *opaque)
 599 {
 600     VirtIOPCIProxy *proxy = opaque;
 601     return msix_enabled(&proxy->pci_dev);
 602 }
 603
 604 static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
 605 {
 606     VirtIOPCIProxy *proxy = opaque;
 607     VirtIODevice *vdev = proxy->vdev;
 608     int r, n;
 609
 610     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 611         if (!virtio_queue_get_num(vdev, n)) {
 612             break;
 613         }
 614
 615         r = virtio_pci_set_guest_notifier(opaque, n, assign);
 616         if (r < 0) {
 617             goto assign_error;
 618         }
 619     }
 620
 621     return 0;
 622
 623 assign_error:
 624     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
 625     while (--n >= 0) {
 626         virtio_pci_set_guest_notifier(opaque, n, !assign);
 627     }
 628     return r;
 629 }
 630
 631 static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
 632 {
 633     VirtIOPCIProxy *proxy = opaque;
 634
 635     /* Stop using ioeventfd for virtqueue kick if the device starts using host
 636      * notifiers.  This makes it easy to avoid stepping on each others' toes.
 637      */
 638     proxy->ioeventfd_disabled = assign;
 639     if (assign) {
 640         virtio_pci_stop_ioeventfd(proxy);
 641     }
 642     /* We don't need to start here: it's not needed because backend
 643      * currently only stops on status change away from ok,
 644      * reset, vmstop and such. If we do add code to start here,
 645      * need to check vmstate, device state etc. */
 646     return virtio_pci_set_host_notifier_internal(proxy, n, assign);
 647 }
 648
 649 static void virtio_pci_vmstate_change(void *opaque, bool running)
 650 {
 651     VirtIOPCIProxy *proxy = opaque;
 652
 653     if (running) {
 654         virtio_pci_start_ioeventfd(proxy);
 655     } else {
 656         virtio_pci_stop_ioeventfd(proxy);
 657     }
 658 }
 659
 660 static const VirtIOBindings virtio_pci_bindings = {
 661     .notify = virtio_pci_notify,
 662     .save_config = virtio_pci_save_config,
 663     .load_config = virtio_pci_load_config,
 664     .save_queue = virtio_pci_save_queue,
 665     .load_queue = virtio_pci_load_queue,
 666     .get_features = virtio_pci_get_features,
 667     .query_guest_notifiers = virtio_pci_query_guest_notifiers,
 668     .set_host_notifier = virtio_pci_set_host_notifier,
 669     .set_guest_notifiers = virtio_pci_set_guest_notifiers,
 670     .vmstate_change = virtio_pci_vmstate_change,
 671 };
 672
 673 static void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev,
 674                             uint16_t vendor, uint16_t device,
 675                             uint16_t class_code, uint8_t pif)
 676 {
 677     uint8_t *config;
 678     uint32_t size;
 679
 680     proxy->vdev = vdev;
 681
 682     config = proxy->pci_dev.config;
 683     pci_config_set_vendor_id(config, vendor);
 684     pci_config_set_device_id(config, device);
 685
 686     config[0x08] = VIRTIO_PCI_ABI_VERSION;
 687
 688     config[0x09] = pif;
 689     pci_config_set_class(config, class_code);
 690
 691     config[0x2c] = vendor & 0xFF;
 692     config[0x2d] = (vendor >> 8) & 0xFF;
 693     config[0x2e] = vdev->device_id & 0xFF;
 694     config[0x2f] = (vdev->device_id >> 8) & 0xFF;
 695
 696     config[0x3d] = 1;
 697
 698     if (vdev->nvectors && !msix_init(&proxy->pci_dev, vdev->nvectors, 1, 0)) {
 699         pci_register_bar(&proxy->pci_dev, 1,
 700                          msix_bar_size(&proxy->pci_dev),
 701                          PCI_BASE_ADDRESS_SPACE_MEMORY,
 702                          msix_mmio_map);
 703     } else
 704         vdev->nvectors = 0;
 705
 706     proxy->pci_dev.config_write = virtio_write_config;
 707
 708     size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
 709     if (size & (size-1))
 710         size = 1 << qemu_fls(size);
 711
 712     pci_register_bar(&proxy->pci_dev, 0, size, PCI_BASE_ADDRESS_SPACE_IO,
 713                            virtio_map);
 714
 715     if (!kvm_has_many_ioeventfds()) {
 716         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
 717     }
 718
 719     virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
 720     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
 721     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
 722     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
 723 }
 724
 725 static int virtio_blk_init_pci(PCIDevice *pci_dev)
 726 {
 727     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 728     VirtIODevice *vdev;
 729
 730     if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
 731         proxy->class_code != PCI_CLASS_STORAGE_OTHER)
 732         proxy->class_code = PCI_CLASS_STORAGE_SCSI;
 733
 734     vdev = virtio_blk_init(&pci_dev->qdev, &proxy->block);
 735     if (!vdev) {
 736         return -1;
 737     }
 738     vdev->nvectors = proxy->nvectors;
 739     virtio_init_pci(proxy, vdev,
 740                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 741                     PCI_DEVICE_ID_VIRTIO_BLOCK,
 742                     proxy->class_code, 0x00);
 743     /* make the actual value visible */
 744     proxy->nvectors = vdev->nvectors;
 745     return 0;
 746 }
 747
 748 static int virtio_exit_pci(PCIDevice *pci_dev)
 749 {
 750     return msix_uninit(pci_dev);
 751 }
 752
 753 static int virtio_blk_exit_pci(PCIDevice *pci_dev)
 754 {
 755     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 756
 757     virtio_pci_stop_ioeventfd(proxy);
 758     virtio_blk_exit(proxy->vdev);
 759     blockdev_mark_auto_del(proxy->block.bs);
 760     return virtio_exit_pci(pci_dev);
 761 }
 762
 763 static int virtio_serial_init_pci(PCIDevice *pci_dev)
 764 {
 765     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 766     VirtIODevice *vdev;
 767
 768     if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
 769         proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
 770         proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
 771         proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
 772
 773     vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial);
 774     if (!vdev) {
 775         return -1;
 776     }
 777     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
 778                                         ? proxy->serial.max_virtserial_ports + 1
 779                                         : proxy->nvectors;
 780     virtio_init_pci(proxy, vdev,
 781                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 782                     PCI_DEVICE_ID_VIRTIO_CONSOLE,
 783                     proxy->class_code, 0x00);
 784     proxy->nvectors = vdev->nvectors;
 785     return 0;
 786 }
 787
 788 static int virtio_serial_exit_pci(PCIDevice *pci_dev)
 789 {
 790     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 791
 792     virtio_pci_stop_ioeventfd(proxy);
 793     virtio_serial_exit(proxy->vdev);
 794     return virtio_exit_pci(pci_dev);
 795 }
 796
 797 static int virtio_net_init_pci(PCIDevice *pci_dev)
 798 {
 799     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 800     VirtIODevice *vdev;
 801
 802     vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
 803
 804     vdev->nvectors = proxy->nvectors;
 805     virtio_init_pci(proxy, vdev,
 806                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 807                     PCI_DEVICE_ID_VIRTIO_NET,
 808                     PCI_CLASS_NETWORK_ETHERNET,
 809                     0x00);
 810
 811     /* make the actual value visible */
 812     proxy->nvectors = vdev->nvectors;
 813     return 0;
 814 }
 815
 816 static int virtio_net_exit_pci(PCIDevice *pci_dev)
 817 {
 818     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 819
 820     virtio_pci_stop_ioeventfd(proxy);
 821     virtio_net_exit(proxy->vdev);
 822     return virtio_exit_pci(pci_dev);
 823 }
 824
 825 static int virtio_balloon_init_pci(PCIDevice *pci_dev)
 826 {
 827     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 828     VirtIODevice *vdev;
 829
 830     vdev = virtio_balloon_init(&pci_dev->qdev);
 831     virtio_init_pci(proxy, vdev,
 832                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 833                     PCI_DEVICE_ID_VIRTIO_BALLOON,
 834                     PCI_CLASS_MEMORY_RAM,
 835                     0x00);
 836     return 0;
 837 }
 838
 839 #ifdef CONFIG_VIRTFS
 840 static int virtio_9p_init_pci(PCIDevice *pci_dev)
 841 {
 842     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 843     VirtIODevice *vdev;
 844
 845     vdev = virtio_9p_init(&pci_dev->qdev, &proxy->fsconf);
 846     vdev->nvectors = proxy->nvectors;
 847     virtio_init_pci(proxy, vdev,
 848                     PCI_VENDOR_ID_REDHAT_QUMRANET,
 849                     0x1009,
 850                     0x2,
 851                     0x00);
 852     /* make the actual value visible */
 853     proxy->nvectors = vdev->nvectors;
 854     return 0;
 855 }
 856 #endif
 857
 858 static PCIDeviceInfo virtio_info[] = {
 859     {
 860         .qdev.name = "virtio-blk-pci",
 861         .qdev.alias = "virtio-blk",
 862         .qdev.size = sizeof(VirtIOPCIProxy),
 863         .init      = virtio_blk_init_pci,
 864         .exit      = virtio_blk_exit_pci,
 865         .qdev.props = (Property[]) {
 866             DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 867             DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, block),
 868             DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
 869                             VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 870             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 871             DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
 872             DEFINE_PROP_END_OF_LIST(),
 873         },
 874         .qdev.reset = virtio_pci_reset,
 875     },{
 876         .qdev.name  = "virtio-net-pci",
 877         .qdev.size  = sizeof(VirtIOPCIProxy),
 878         .init       = virtio_net_init_pci,
 879         .exit       = virtio_net_exit_pci,
 880         .romfile    = "pxe-virtio.bin",
 881         .qdev.props = (Property[]) {
 882             DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
 883                             VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
 884             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
 885             DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
 886             DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
 887             DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy,
 888                                net.txtimer, TX_TIMER_INTERVAL),
 889             DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy,
 890                               net.txburst, TX_BURST),
 891             DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
 892             DEFINE_PROP_END_OF_LIST(),
 893         },
 894         .qdev.reset = virtio_pci_reset,
 895     },{
 896         .qdev.name = "virtio-serial-pci",
 897         .qdev.alias = "virtio-serial",
 898         .qdev.size = sizeof(VirtIOPCIProxy),
 899         .init      = virtio_serial_init_pci,
 900         .exit      = virtio_serial_exit_pci,
 901         .qdev.props = (Property[]) {
 902             DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
 903                             VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 904             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
 905                                DEV_NVECTORS_UNSPECIFIED),
 906             DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 907             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 908             DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy,
 909                                serial.max_virtserial_ports, 31),
 910             DEFINE_PROP_END_OF_LIST(),
 911         },
 912         .qdev.reset = virtio_pci_reset,
 913     },{
 914         .qdev.name = "virtio-balloon-pci",
 915         .qdev.size = sizeof(VirtIOPCIProxy),
 916         .init      = virtio_balloon_init_pci,
 917         .exit      = virtio_exit_pci,
 918         .qdev.props = (Property[]) {
 919             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 920             DEFINE_PROP_END_OF_LIST(),
 921         },
 922         .qdev.reset = virtio_pci_reset,
 923     },{
 924 #ifdef CONFIG_VIRTFS
 925         .qdev.name = "virtio-9p-pci",
 926         .qdev.size = sizeof(VirtIOPCIProxy),
 927         .init      = virtio_9p_init_pci,
 928         .qdev.props = (Property[]) {
 929             DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 930             DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 931             DEFINE_PROP_STRING("mount_tag", VirtIOPCIProxy, fsconf.tag),
 932             DEFINE_PROP_STRING("fsdev", VirtIOPCIProxy, fsconf.fsdev_id),
 933             DEFINE_PROP_END_OF_LIST(),
 934         },
 935     }, {
 936 #endif
 937         /* end of list */
 938     }
 939 };
 940
 941 static void virtio_pci_register_devices(void)
 942 {
 943     pci_qdev_register_many(virtio_info);
 944 }
 945
 946 device_init(virtio_pci_register_devices)