hw/virtio-pci.c

   1 /*
   2  * Virtio PCI Bindings
   3  *
   4  * Copyright IBM, Corp. 2007
   5  * Copyright (c) 2009 CodeSourcery
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Paul Brook        <paul@codesourcery.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2.  See
  12  * the COPYING file in the top-level directory.
  13  *
  14  * Contributions after 2012-01-13 are licensed under the terms of the
  15  * GNU GPL, version 2 or (at your option) any later version.
  16  */
  17
  18 #include <inttypes.h>
  19
  20 #include "virtio.h"
  21 #include "virtio-blk.h"
  22 #include "virtio-net.h"
  23 #include "virtio-serial.h"
  24 #include "virtio-scsi.h"
  25 #include "pci.h"
  26 #include "qemu-error.h"
  27 #include "msi.h"
  28 #include "msix.h"
  29 #include "net.h"
  30 #include "loader.h"
  31 #include "kvm.h"
  32 #include "blockdev.h"
  33 #include "virtio-pci.h"
  34 #include "range.h"
  35
  36 /* from Linux's linux/virtio_pci.h */
  37
  38 /* A 32-bit r/o bitmask of the features supported by the host */
  39 #define VIRTIO_PCI_HOST_FEATURES        0
  40
  41 /* A 32-bit r/w bitmask of features activated by the guest */
  42 #define VIRTIO_PCI_GUEST_FEATURES       4
  43
  44 /* A 32-bit r/w PFN for the currently selected queue */
  45 #define VIRTIO_PCI_QUEUE_PFN            8
  46
  47 /* A 16-bit r/o queue size for the currently selected queue */
  48 #define VIRTIO_PCI_QUEUE_NUM            12
  49
  50 /* A 16-bit r/w queue selector */
  51 #define VIRTIO_PCI_QUEUE_SEL            14
  52
  53 /* A 16-bit r/w queue notifier */
  54 #define VIRTIO_PCI_QUEUE_NOTIFY         16
  55
  56 /* An 8-bit device status register.  */
  57 #define VIRTIO_PCI_STATUS               18
  58
  59 /* An 8-bit r/o interrupt status register.  Reading the value will return the
  60  * current contents of the ISR and will also clear it.  This is effectively
  61  * a read-and-acknowledge. */
  62 #define VIRTIO_PCI_ISR                  19
  63
  64 /* MSI-X registers: only enabled if MSI-X is enabled. */
  65 /* A 16-bit vector for configuration changes. */
  66 #define VIRTIO_MSI_CONFIG_VECTOR        20
  67 /* A 16-bit vector for selected queue notifications. */
  68 #define VIRTIO_MSI_QUEUE_VECTOR         22
  69
  70 /* Config space size */
  71 #define VIRTIO_PCI_CONFIG_NOMSI         20
  72 #define VIRTIO_PCI_CONFIG_MSI           24
  73 #define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
  74                                          VIRTIO_PCI_CONFIG_MSI : \
  75                                          VIRTIO_PCI_CONFIG_NOMSI)
  76
  77 /* The remaining space is defined by each driver as the per-driver
  78  * configuration space */
  79 #define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
  80                                          VIRTIO_PCI_CONFIG_MSI : \
  81                                          VIRTIO_PCI_CONFIG_NOMSI)
  82
  83 /* How many bits to shift physical queue address written to QUEUE_PFN.
  84  * 12 is historical, and due to x86 page size. */
  85 #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
  86
  87 /* Flags track per-device state like workarounds for quirks in older guests. */
  88 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
  89
  90 /* QEMU doesn't strictly need write barriers since everything runs in
  91  * lock-step.  We'll leave the calls to wmb() in though to make it obvious for
  92  * KVM or if kqemu gets SMP support.
  93  */
  94 #define wmb() do { } while (0)
  95
  96 /* HACK for virtio to determine if it's running a big endian guest */
  97 bool virtio_is_big_endian(void);
  98
  99 /* virtio device */
 100
 101 static void virtio_pci_notify(void *opaque, uint16_t vector)
 102 {
 103     VirtIOPCIProxy *proxy = opaque;
 104     if (msix_enabled(&proxy->pci_dev))
 105         msix_notify(&proxy->pci_dev, vector);
 106     else
 107         qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1);
 108 }
 109
 110 static void virtio_pci_save_config(void * opaque, QEMUFile *f)
 111 {
 112     VirtIOPCIProxy *proxy = opaque;
 113     pci_device_save(&proxy->pci_dev, f);
 114     msix_save(&proxy->pci_dev, f);
 115     if (msix_present(&proxy->pci_dev))
 116         qemu_put_be16(f, proxy->vdev->config_vector);
 117 }
 118
 119 static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f)
 120 {
 121     VirtIOPCIProxy *proxy = opaque;
 122     if (msix_present(&proxy->pci_dev))
 123         qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n));
 124 }
 125
 126 static int virtio_pci_load_config(void * opaque, QEMUFile *f)
 127 {
 128     VirtIOPCIProxy *proxy = opaque;
 129     int ret;
 130     ret = pci_device_load(&proxy->pci_dev, f);
 131     if (ret) {
 132         return ret;
 133     }
 134     msix_load(&proxy->pci_dev, f);
 135     if (msix_present(&proxy->pci_dev)) {
 136         qemu_get_be16s(f, &proxy->vdev->config_vector);
 137     } else {
 138         proxy->vdev->config_vector = VIRTIO_NO_VECTOR;
 139     }
 140     if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) {
 141         return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector);
 142     }
 143     return 0;
 144 }
 145
 146 static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 147 {
 148     VirtIOPCIProxy *proxy = opaque;
 149     uint16_t vector;
 150     if (msix_present(&proxy->pci_dev)) {
 151         qemu_get_be16s(f, &vector);
 152     } else {
 153         vector = VIRTIO_NO_VECTOR;
 154     }
 155     virtio_queue_set_vector(proxy->vdev, n, vector);
 156     if (vector != VIRTIO_NO_VECTOR) {
 157         return msix_vector_use(&proxy->pci_dev, vector);
 158     }
 159     return 0;
 160 }
 161
 162 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
 163                                                  int n, bool assign, bool set_handler)
 164 {
 165     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 166     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
 167     int r = 0;
 168
 169     if (assign) {
 170         r = event_notifier_init(notifier, 1);
 171         if (r < 0) {
 172             error_report("%s: unable to init event notifier: %d",
 173                          __func__, r);
 174             return r;
 175         }
 176         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
 177         memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 178                                   true, n, notifier);
 179     } else {
 180         memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
 181                                   true, n, notifier);
 182         virtio_queue_set_host_notifier_fd_handler(vq, false, false);
 183         event_notifier_cleanup(notifier);
 184     }
 185     return r;
 186 }
 187
 188 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 189 {
 190     int n, r;
 191
 192     if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
 193         proxy->ioeventfd_disabled ||
 194         proxy->ioeventfd_started) {
 195         return;
 196     }
 197
 198     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 199         if (!virtio_queue_get_num(proxy->vdev, n)) {
 200             continue;
 201         }
 202
 203         r = virtio_pci_set_host_notifier_internal(proxy, n, true, true);
 204         if (r < 0) {
 205             goto assign_error;
 206         }
 207     }
 208     proxy->ioeventfd_started = true;
 209     return;
 210
 211 assign_error:
 212     while (--n >= 0) {
 213         if (!virtio_queue_get_num(proxy->vdev, n)) {
 214             continue;
 215         }
 216
 217         r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
 218         assert(r >= 0);
 219     }
 220     proxy->ioeventfd_started = false;
 221     error_report("%s: failed. Fallback to a userspace (slower).", __func__);
 222 }
 223
 224 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 225 {
 226     int r;
 227     int n;
 228
 229     if (!proxy->ioeventfd_started) {
 230         return;
 231     }
 232
 233     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 234         if (!virtio_queue_get_num(proxy->vdev, n)) {
 235             continue;
 236         }
 237
 238         r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
 239         assert(r >= 0);
 240     }
 241     proxy->ioeventfd_started = false;
 242 }
 243
 244 void virtio_pci_reset(DeviceState *d)
 245 {
 246     VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev);
 247     virtio_pci_stop_ioeventfd(proxy);
 248     virtio_reset(proxy->vdev);
 249     proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 250 }
 251
 252 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 253 {
 254     VirtIOPCIProxy *proxy = opaque;
 255     VirtIODevice *vdev = proxy->vdev;
 256     target_phys_addr_t pa;
 257
 258     switch (addr) {
 259     case VIRTIO_PCI_GUEST_FEATURES:
 260         /* Guest does not negotiate properly?  We have to assume nothing. */
 261         if (val & (1 << VIRTIO_F_BAD_FEATURE)) {
 262             val = vdev->bad_features ? vdev->bad_features(vdev) : 0;
 263         }
 264         virtio_set_features(vdev, val);
 265         break;
 266     case VIRTIO_PCI_QUEUE_PFN:
 267         pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 268         if (pa == 0) {
 269             virtio_pci_stop_ioeventfd(proxy);
 270             virtio_reset(proxy->vdev);
 271             msix_unuse_all_vectors(&proxy->pci_dev);
 272         }
 273         else
 274             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
 275         break;
 276     case VIRTIO_PCI_QUEUE_SEL:
 277         if (val < VIRTIO_PCI_QUEUE_MAX)
 278             vdev->queue_sel = val;
 279         break;
 280     case VIRTIO_PCI_QUEUE_NOTIFY:
 281         if (val < VIRTIO_PCI_QUEUE_MAX) {
 282             virtio_queue_notify(vdev, val);
 283         }
 284         break;
 285     case VIRTIO_PCI_STATUS:
 286         if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
 287             virtio_pci_stop_ioeventfd(proxy);
 288         }
 289
 290         virtio_set_status(vdev, val & 0xFF);
 291
 292         if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
 293             virtio_pci_start_ioeventfd(proxy);
 294         }
 295
 296         if (vdev->status == 0) {
 297             virtio_reset(proxy->vdev);
 298             msix_unuse_all_vectors(&proxy->pci_dev);
 299         }
 300
 301         /* Linux before 2.6.34 sets the device as OK without enabling
 302            the PCI device bus master bit. In this case we need to disable
 303            some safety checks. */
 304         if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
 305             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 306             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 307         }
 308         break;
 309     case VIRTIO_MSI_CONFIG_VECTOR:
 310         msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
 311         /* Make it possible for guest to discover an error took place. */
 312         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 313             val = VIRTIO_NO_VECTOR;
 314         vdev->config_vector = val;
 315         break;
 316     case VIRTIO_MSI_QUEUE_VECTOR:
 317         msix_vector_unuse(&proxy->pci_dev,
 318                           virtio_queue_vector(vdev, vdev->queue_sel));
 319         /* Make it possible for guest to discover an error took place. */
 320         if (msix_vector_use(&proxy->pci_dev, val) < 0)
 321             val = VIRTIO_NO_VECTOR;
 322         virtio_queue_set_vector(vdev, vdev->queue_sel, val);
 323         break;
 324     default:
 325         error_report("%s: unexpected address 0x%x value 0x%x",
 326                      __func__, addr, val);
 327         break;
 328     }
 329 }
 330
 331 static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
 332 {
 333     VirtIODevice *vdev = proxy->vdev;
 334     uint32_t ret = 0xFFFFFFFF;
 335
 336     switch (addr) {
 337     case VIRTIO_PCI_HOST_FEATURES:
 338         ret = proxy->host_features;
 339         break;
 340     case VIRTIO_PCI_GUEST_FEATURES:
 341         ret = vdev->guest_features;
 342         break;
 343     case VIRTIO_PCI_QUEUE_PFN:
 344         ret = virtio_queue_get_addr(vdev, vdev->queue_sel)
 345               >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
 346         break;
 347     case VIRTIO_PCI_QUEUE_NUM:
 348         ret = virtio_queue_get_num(vdev, vdev->queue_sel);
 349         break;
 350     case VIRTIO_PCI_QUEUE_SEL:
 351         ret = vdev->queue_sel;
 352         break;
 353     case VIRTIO_PCI_STATUS:
 354         ret = vdev->status;
 355         break;
 356     case VIRTIO_PCI_ISR:
 357         /* reading from the ISR also clears it. */
 358         ret = vdev->isr;
 359         vdev->isr = 0;
 360         qemu_set_irq(proxy->pci_dev.irq[0], 0);
 361         break;
 362     case VIRTIO_MSI_CONFIG_VECTOR:
 363         ret = vdev->config_vector;
 364         break;
 365     case VIRTIO_MSI_QUEUE_VECTOR:
 366         ret = virtio_queue_vector(vdev, vdev->queue_sel);
 367         break;
 368     default:
 369         break;
 370     }
 371
 372     return ret;
 373 }
 374
 375 static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
 376 {
 377     VirtIOPCIProxy *proxy = opaque;
 378     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 379     if (addr < config)
 380         return virtio_ioport_read(proxy, addr);
 381     addr -= config;
 382     return virtio_config_readb(proxy->vdev, addr);
 383 }
 384
 385 static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
 386 {
 387     VirtIOPCIProxy *proxy = opaque;
 388     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 389     uint16_t val;
 390     if (addr < config)
 391         return virtio_ioport_read(proxy, addr);
 392     addr -= config;
 393     val = virtio_config_readw(proxy->vdev, addr);
 394     if (virtio_is_big_endian()) {
 395         /*
 396          * virtio is odd, ioports are LE but config space is target native
 397          * endian. However, in qemu, all PIO is LE, so we need to re-swap
 398          * on BE targets
 399          */
 400         val = bswap16(val);
 401     }
 402     return val;
 403 }
 404
 405 static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
 406 {
 407     VirtIOPCIProxy *proxy = opaque;
 408     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 409     uint32_t val;
 410     if (addr < config)
 411         return virtio_ioport_read(proxy, addr);
 412     addr -= config;
 413     val = virtio_config_readl(proxy->vdev, addr);
 414     if (virtio_is_big_endian()) {
 415         val = bswap32(val);
 416     }
 417     return val;
 418 }
 419
 420 static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
 421 {
 422     VirtIOPCIProxy *proxy = opaque;
 423     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 424     if (addr < config) {
 425         virtio_ioport_write(proxy, addr, val);
 426         return;
 427     }
 428     addr -= config;
 429     virtio_config_writeb(proxy->vdev, addr, val);
 430 }
 431
 432 static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
 433 {
 434     VirtIOPCIProxy *proxy = opaque;
 435     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 436     if (addr < config) {
 437         virtio_ioport_write(proxy, addr, val);
 438         return;
 439     }
 440     addr -= config;
 441     if (virtio_is_big_endian()) {
 442         val = bswap16(val);
 443     }
 444     virtio_config_writew(proxy->vdev, addr, val);
 445 }
 446
 447 static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
 448 {
 449     VirtIOPCIProxy *proxy = opaque;
 450     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
 451     if (addr < config) {
 452         virtio_ioport_write(proxy, addr, val);
 453         return;
 454     }
 455     addr -= config;
 456     if (virtio_is_big_endian()) {
 457         val = bswap32(val);
 458     }
 459     virtio_config_writel(proxy->vdev, addr, val);
 460 }
 461
 462 static const MemoryRegionPortio virtio_portio[] = {
 463     { 0, 0x10000, 1, .write = virtio_pci_config_writeb, },
 464     { 0, 0x10000, 2, .write = virtio_pci_config_writew, },
 465     { 0, 0x10000, 4, .write = virtio_pci_config_writel, },
 466     { 0, 0x10000, 1, .read = virtio_pci_config_readb, },
 467     { 0, 0x10000, 2, .read = virtio_pci_config_readw, },
 468     { 0, 0x10000, 4, .read = virtio_pci_config_readl, },
 469     PORTIO_END_OF_LIST()
 470 };
 471
 472 static const MemoryRegionOps virtio_pci_config_ops = {
 473     .old_portio = virtio_portio,
 474     .endianness = DEVICE_LITTLE_ENDIAN,
 475 };
 476
 477 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
 478                                 uint32_t val, int len)
 479 {
 480     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 481
 482     pci_default_write_config(pci_dev, address, val, len);
 483
 484     if (range_covers_byte(address, len, PCI_COMMAND) &&
 485         !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
 486         !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
 487         virtio_pci_stop_ioeventfd(proxy);
 488         virtio_set_status(proxy->vdev,
 489                           proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
 490     }
 491 }
 492
 493 static unsigned virtio_pci_get_features(void *opaque)
 494 {
 495     VirtIOPCIProxy *proxy = opaque;
 496     return proxy->host_features;
 497 }
 498
 499 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
 500                                         unsigned int queue_no,
 501                                         unsigned int vector,
 502                                         MSIMessage msg)
 503 {
 504     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 505     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 506     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 507     int ret;
 508
 509     if (irqfd->users == 0) {
 510         ret = kvm_irqchip_add_msi_route(kvm_state, msg);
 511         if (ret < 0) {
 512             return ret;
 513         }
 514         irqfd->virq = ret;
 515     }
 516     irqfd->users++;
 517
 518     ret = kvm_irqchip_add_irq_notifier(kvm_state, n, irqfd->virq);
 519     if (ret < 0) {
 520         if (--irqfd->users == 0) {
 521             kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 522         }
 523         return ret;
 524     }
 525
 526     virtio_queue_set_guest_notifier_fd_handler(vq, true, true);
 527     return 0;
 528 }
 529
 530 static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
 531                                              unsigned int queue_no,
 532                                              unsigned int vector)
 533 {
 534     VirtQueue *vq = virtio_get_queue(proxy->vdev, queue_no);
 535     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
 536     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
 537     int ret;
 538
 539     ret = kvm_irqchip_remove_irq_notifier(kvm_state, n, irqfd->virq);
 540     assert(ret == 0);
 541
 542     if (--irqfd->users == 0) {
 543         kvm_irqchip_release_virq(kvm_state, irqfd->virq);
 544     }
 545
 546     virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 547 }
 548
 549 static int kvm_virtio_pci_vector_use(PCIDevice *dev, unsigned vector,
 550                                      MSIMessage msg)
 551 {
 552     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 553     VirtIODevice *vdev = proxy->vdev;
 554     int ret, queue_no;
 555
 556     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 557         if (!virtio_queue_get_num(vdev, queue_no)) {
 558             break;
 559         }
 560         if (virtio_queue_vector(vdev, queue_no) != vector) {
 561             continue;
 562         }
 563         ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
 564         if (ret < 0) {
 565             goto undo;
 566         }
 567     }
 568     return 0;
 569
 570 undo:
 571     while (--queue_no >= 0) {
 572         if (virtio_queue_vector(vdev, queue_no) != vector) {
 573             continue;
 574         }
 575         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 576     }
 577     return ret;
 578 }
 579
 580 static void kvm_virtio_pci_vector_release(PCIDevice *dev, unsigned vector)
 581 {
 582     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
 583     VirtIODevice *vdev = proxy->vdev;
 584     int queue_no;
 585
 586     for (queue_no = 0; queue_no < VIRTIO_PCI_QUEUE_MAX; queue_no++) {
 587         if (!virtio_queue_get_num(vdev, queue_no)) {
 588             break;
 589         }
 590         if (virtio_queue_vector(vdev, queue_no) != vector) {
 591             continue;
 592         }
 593         kvm_virtio_pci_vq_vector_release(proxy, queue_no, vector);
 594     }
 595 }
 596
 597 static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign)
 598 {
 599     VirtIOPCIProxy *proxy = opaque;
 600     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
 601     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 602
 603     if (assign) {
 604         int r = event_notifier_init(notifier, 0);
 605         if (r < 0) {
 606             return r;
 607         }
 608         virtio_queue_set_guest_notifier_fd_handler(vq, true, false);
 609     } else {
 610         virtio_queue_set_guest_notifier_fd_handler(vq, false, false);
 611         event_notifier_cleanup(notifier);
 612     }
 613
 614     return 0;
 615 }
 616
 617 static bool virtio_pci_query_guest_notifiers(void *opaque)
 618 {
 619     VirtIOPCIProxy *proxy = opaque;
 620     return msix_enabled(&proxy->pci_dev);
 621 }
 622
 623 static int virtio_pci_set_guest_notifiers(void *opaque, bool assign)
 624 {
 625     VirtIOPCIProxy *proxy = opaque;
 626     VirtIODevice *vdev = proxy->vdev;
 627     int r, n;
 628
 629     /* Must unset vector notifier while guest notifier is still assigned */
 630     if (kvm_msi_via_irqfd_enabled() && !assign) {
 631         msix_unset_vector_notifiers(&proxy->pci_dev);
 632         g_free(proxy->vector_irqfd);
 633         proxy->vector_irqfd = NULL;
 634     }
 635
 636     for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
 637         if (!virtio_queue_get_num(vdev, n)) {
 638             break;
 639         }
 640
 641         r = virtio_pci_set_guest_notifier(opaque, n, assign);
 642         if (r < 0) {
 643             goto assign_error;
 644         }
 645     }
 646
 647     /* Must set vector notifier after guest notifier has been assigned */
 648     if (kvm_msi_via_irqfd_enabled() && assign) {
 649         proxy->vector_irqfd =
 650             g_malloc0(sizeof(*proxy->vector_irqfd) *
 651                       msix_nr_vectors_allocated(&proxy->pci_dev));
 652         r = msix_set_vector_notifiers(&proxy->pci_dev,
 653                                       kvm_virtio_pci_vector_use,
 654                                       kvm_virtio_pci_vector_release);
 655         if (r < 0) {
 656             goto assign_error;
 657         }
 658     }
 659
 660     return 0;
 661
 662 assign_error:
 663     /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
 664     assert(assign);
 665     while (--n >= 0) {
 666         virtio_pci_set_guest_notifier(opaque, n, !assign);
 667     }
 668     return r;
 669 }
 670
 671 static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
 672 {
 673     VirtIOPCIProxy *proxy = opaque;
 674
 675     /* Stop using ioeventfd for virtqueue kick if the device starts using host
 676      * notifiers.  This makes it easy to avoid stepping on each others' toes.
 677      */
 678     proxy->ioeventfd_disabled = assign;
 679     if (assign) {
 680         virtio_pci_stop_ioeventfd(proxy);
 681     }
 682     /* We don't need to start here: it's not needed because backend
 683      * currently only stops on status change away from ok,
 684      * reset, vmstop and such. If we do add code to start here,
 685      * need to check vmstate, device state etc. */
 686     return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
 687 }
 688
 689 static void virtio_pci_vmstate_change(void *opaque, bool running)
 690 {
 691     VirtIOPCIProxy *proxy = opaque;
 692
 693     if (running) {
 694         /* Try to find out if the guest has bus master disabled, but is
 695            in ready state. Then we have a buggy guest OS. */
 696         if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 697             !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
 698             proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
 699         }
 700         virtio_pci_start_ioeventfd(proxy);
 701     } else {
 702         virtio_pci_stop_ioeventfd(proxy);
 703     }
 704 }
 705
 706 static const VirtIOBindings virtio_pci_bindings = {
 707     .notify = virtio_pci_notify,
 708     .save_config = virtio_pci_save_config,
 709     .load_config = virtio_pci_load_config,
 710     .save_queue = virtio_pci_save_queue,
 711     .load_queue = virtio_pci_load_queue,
 712     .get_features = virtio_pci_get_features,
 713     .query_guest_notifiers = virtio_pci_query_guest_notifiers,
 714     .set_host_notifier = virtio_pci_set_host_notifier,
 715     .set_guest_notifiers = virtio_pci_set_guest_notifiers,
 716     .vmstate_change = virtio_pci_vmstate_change,
 717 };
 718
 719 void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev)
 720 {
 721     uint8_t *config;
 722     uint32_t size;
 723
 724     proxy->vdev = vdev;
 725
 726     config = proxy->pci_dev.config;
 727
 728     if (proxy->class_code) {
 729         pci_config_set_class(config, proxy->class_code);
 730     }
 731     pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
 732                  pci_get_word(config + PCI_VENDOR_ID));
 733     pci_set_word(config + PCI_SUBSYSTEM_ID, vdev->device_id);
 734     config[PCI_INTERRUPT_PIN] = 1;
 735
 736     if (vdev->nvectors &&
 737         msix_init_exclusive_bar(&proxy->pci_dev, vdev->nvectors, 1)) {
 738         vdev->nvectors = 0;
 739     }
 740
 741     proxy->pci_dev.config_write = virtio_write_config;
 742
 743     size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len;
 744     if (size & (size-1))
 745         size = 1 << qemu_fls(size);
 746
 747     memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy,
 748                           "virtio-pci", size);
 749     pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
 750                      &proxy->bar);
 751
 752     if (!kvm_has_many_ioeventfds()) {
 753         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
 754     }
 755
 756     virtio_bind_device(vdev, &virtio_pci_bindings, proxy);
 757     proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
 758     proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
 759     proxy->host_features = vdev->get_features(vdev, proxy->host_features);
 760 }
 761
 762 static int virtio_blk_init_pci(PCIDevice *pci_dev)
 763 {
 764     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 765     VirtIODevice *vdev;
 766
 767     if (proxy->class_code != PCI_CLASS_STORAGE_SCSI &&
 768         proxy->class_code != PCI_CLASS_STORAGE_OTHER)
 769         proxy->class_code = PCI_CLASS_STORAGE_SCSI;
 770
 771     vdev = virtio_blk_init(&pci_dev->qdev, &proxy->blk);
 772     if (!vdev) {
 773         return -1;
 774     }
 775     vdev->nvectors = proxy->nvectors;
 776     virtio_init_pci(proxy, vdev);
 777     /* make the actual value visible */
 778     proxy->nvectors = vdev->nvectors;
 779     return 0;
 780 }
 781
 782 static void virtio_exit_pci(PCIDevice *pci_dev)
 783 {
 784     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 785
 786     memory_region_destroy(&proxy->bar);
 787     msix_uninit_exclusive_bar(pci_dev);
 788 }
 789
 790 static void virtio_blk_exit_pci(PCIDevice *pci_dev)
 791 {
 792     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 793
 794     virtio_pci_stop_ioeventfd(proxy);
 795     virtio_blk_exit(proxy->vdev);
 796     virtio_exit_pci(pci_dev);
 797 }
 798
 799 static int virtio_serial_init_pci(PCIDevice *pci_dev)
 800 {
 801     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 802     VirtIODevice *vdev;
 803
 804     if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER &&
 805         proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */
 806         proxy->class_code != PCI_CLASS_OTHERS)          /* qemu-kvm  */
 807         proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER;
 808
 809     vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial);
 810     if (!vdev) {
 811         return -1;
 812     }
 813     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
 814                                         ? proxy->serial.max_virtserial_ports + 1
 815                                         : proxy->nvectors;
 816     virtio_init_pci(proxy, vdev);
 817     proxy->nvectors = vdev->nvectors;
 818     return 0;
 819 }
 820
 821 static void virtio_serial_exit_pci(PCIDevice *pci_dev)
 822 {
 823     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 824
 825     virtio_pci_stop_ioeventfd(proxy);
 826     virtio_serial_exit(proxy->vdev);
 827     virtio_exit_pci(pci_dev);
 828 }
 829
 830 static int virtio_net_init_pci(PCIDevice *pci_dev)
 831 {
 832     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 833     VirtIODevice *vdev;
 834
 835     vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net);
 836
 837     vdev->nvectors = proxy->nvectors;
 838     virtio_init_pci(proxy, vdev);
 839
 840     /* make the actual value visible */
 841     proxy->nvectors = vdev->nvectors;
 842     return 0;
 843 }
 844
 845 static void virtio_net_exit_pci(PCIDevice *pci_dev)
 846 {
 847     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 848
 849     virtio_pci_stop_ioeventfd(proxy);
 850     virtio_net_exit(proxy->vdev);
 851     virtio_exit_pci(pci_dev);
 852 }
 853
 854 static int virtio_balloon_init_pci(PCIDevice *pci_dev)
 855 {
 856     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 857     VirtIODevice *vdev;
 858
 859     if (proxy->class_code != PCI_CLASS_OTHERS &&
 860         proxy->class_code != PCI_CLASS_MEMORY_RAM) { /* qemu < 1.1 */
 861         proxy->class_code = PCI_CLASS_OTHERS;
 862     }
 863
 864     vdev = virtio_balloon_init(&pci_dev->qdev);
 865     if (!vdev) {
 866         return -1;
 867     }
 868     virtio_init_pci(proxy, vdev);
 869     return 0;
 870 }
 871
 872 static void virtio_balloon_exit_pci(PCIDevice *pci_dev)
 873 {
 874     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
 875
 876     virtio_pci_stop_ioeventfd(proxy);
 877     virtio_balloon_exit(proxy->vdev);
 878     virtio_exit_pci(pci_dev);
 879 }
 880
 881 static Property virtio_blk_properties[] = {
 882     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 883     DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
 884     DEFINE_BLOCK_CHS_PROPERTIES(VirtIOPCIProxy, blk.conf),
 885     DEFINE_PROP_STRING("serial", VirtIOPCIProxy, blk.serial),
 886 #ifdef __linux__
 887     DEFINE_PROP_BIT("scsi", VirtIOPCIProxy, blk.scsi, 0, true),
 888 #endif
 889     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 890     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
 891     DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features),
 892     DEFINE_PROP_END_OF_LIST(),
 893 };
 894
 895 static void virtio_blk_class_init(ObjectClass *klass, void *data)
 896 {
 897     DeviceClass *dc = DEVICE_CLASS(klass);
 898     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 899
 900     k->init = virtio_blk_init_pci;
 901     k->exit = virtio_blk_exit_pci;
 902     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 903     k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
 904     k->revision = VIRTIO_PCI_ABI_VERSION;
 905     k->class_id = PCI_CLASS_STORAGE_SCSI;
 906     dc->reset = virtio_pci_reset;
 907     dc->props = virtio_blk_properties;
 908 }
 909
 910 static TypeInfo virtio_blk_info = {
 911     .name          = "virtio-blk-pci",
 912     .parent        = TYPE_PCI_DEVICE,
 913     .instance_size = sizeof(VirtIOPCIProxy),
 914     .class_init    = virtio_blk_class_init,
 915 };
 916
 917 static Property virtio_net_properties[] = {
 918     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
 919     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
 920     DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
 921     DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic),
 922     DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy, net.txtimer, TX_TIMER_INTERVAL),
 923     DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy, net.txburst, TX_BURST),
 924     DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx),
 925     DEFINE_PROP_END_OF_LIST(),
 926 };
 927
 928 static void virtio_net_class_init(ObjectClass *klass, void *data)
 929 {
 930     DeviceClass *dc = DEVICE_CLASS(klass);
 931     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 932
 933     k->init = virtio_net_init_pci;
 934     k->exit = virtio_net_exit_pci;
 935     k->romfile = "pxe-virtio.rom";
 936     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 937     k->device_id = PCI_DEVICE_ID_VIRTIO_NET;
 938     k->revision = VIRTIO_PCI_ABI_VERSION;
 939     k->class_id = PCI_CLASS_NETWORK_ETHERNET;
 940     dc->reset = virtio_pci_reset;
 941     dc->props = virtio_net_properties;
 942 }
 943
 944 static TypeInfo virtio_net_info = {
 945     .name          = "virtio-net-pci",
 946     .parent        = TYPE_PCI_DEVICE,
 947     .instance_size = sizeof(VirtIOPCIProxy),
 948     .class_init    = virtio_net_class_init,
 949 };
 950
 951 static Property virtio_serial_properties[] = {
 952     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
 953     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
 954     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 955     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 956     DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, serial.max_virtserial_ports, 31),
 957     DEFINE_PROP_END_OF_LIST(),
 958 };
 959
 960 static void virtio_serial_class_init(ObjectClass *klass, void *data)
 961 {
 962     DeviceClass *dc = DEVICE_CLASS(klass);
 963     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 964
 965     k->init = virtio_serial_init_pci;
 966     k->exit = virtio_serial_exit_pci;
 967     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 968     k->device_id = PCI_DEVICE_ID_VIRTIO_CONSOLE;
 969     k->revision = VIRTIO_PCI_ABI_VERSION;
 970     k->class_id = PCI_CLASS_COMMUNICATION_OTHER;
 971     dc->reset = virtio_pci_reset;
 972     dc->props = virtio_serial_properties;
 973 }
 974
 975 static TypeInfo virtio_serial_info = {
 976     .name          = "virtio-serial-pci",
 977     .parent        = TYPE_PCI_DEVICE,
 978     .instance_size = sizeof(VirtIOPCIProxy),
 979     .class_init    = virtio_serial_class_init,
 980 };
 981
 982 static Property virtio_balloon_properties[] = {
 983     DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
 984     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
 985     DEFINE_PROP_END_OF_LIST(),
 986 };
 987
 988 static void virtio_balloon_class_init(ObjectClass *klass, void *data)
 989 {
 990     DeviceClass *dc = DEVICE_CLASS(klass);
 991     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 992
 993     k->init = virtio_balloon_init_pci;
 994     k->exit = virtio_balloon_exit_pci;
 995     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
 996     k->device_id = PCI_DEVICE_ID_VIRTIO_BALLOON;
 997     k->revision = VIRTIO_PCI_ABI_VERSION;
 998     k->class_id = PCI_CLASS_OTHERS;
 999     dc->reset = virtio_pci_reset;
1000     dc->props = virtio_balloon_properties;
1001 }
1002
1003 static TypeInfo virtio_balloon_info = {
1004     .name          = "virtio-balloon-pci",
1005     .parent        = TYPE_PCI_DEVICE,
1006     .instance_size = sizeof(VirtIOPCIProxy),
1007     .class_init    = virtio_balloon_class_init,
1008 };
1009
1010 static int virtio_scsi_init_pci(PCIDevice *pci_dev)
1011 {
1012     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1013     VirtIODevice *vdev;
1014
1015     vdev = virtio_scsi_init(&pci_dev->qdev, &proxy->scsi);
1016     if (!vdev) {
1017         return -EINVAL;
1018     }
1019
1020     vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED
1021                                         ? proxy->scsi.num_queues + 3
1022                                         : proxy->nvectors;
1023     virtio_init_pci(proxy, vdev);
1024
1025     /* make the actual value visible */
1026     proxy->nvectors = vdev->nvectors;
1027     return 0;
1028 }
1029
1030 static void virtio_scsi_exit_pci(PCIDevice *pci_dev)
1031 {
1032     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
1033
1034     virtio_scsi_exit(proxy->vdev);
1035     virtio_exit_pci(pci_dev);
1036 }
1037
1038 static Property virtio_scsi_properties[] = {
1039     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
1040     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, DEV_NVECTORS_UNSPECIFIED),
1041     DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOPCIProxy, host_features, scsi),
1042     DEFINE_PROP_END_OF_LIST(),
1043 };
1044
1045 static void virtio_scsi_class_init(ObjectClass *klass, void *data)
1046 {
1047     DeviceClass *dc = DEVICE_CLASS(klass);
1048     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1049
1050     k->init = virtio_scsi_init_pci;
1051     k->exit = virtio_scsi_exit_pci;
1052     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
1053     k->device_id = PCI_DEVICE_ID_VIRTIO_SCSI;
1054     k->revision = 0x00;
1055     k->class_id = PCI_CLASS_STORAGE_SCSI;
1056     dc->reset = virtio_pci_reset;
1057     dc->props = virtio_scsi_properties;
1058 }
1059
1060 static TypeInfo virtio_scsi_info = {
1061     .name          = "virtio-scsi-pci",
1062     .parent        = TYPE_PCI_DEVICE,
1063     .instance_size = sizeof(VirtIOPCIProxy),
1064     .class_init    = virtio_scsi_class_init,
1065 };
1066
1067 static void virtio_pci_register_types(void)
1068 {
1069     type_register_static(&virtio_blk_info);
1070     type_register_static(&virtio_net_info);
1071     type_register_static(&virtio_serial_info);
1072     type_register_static(&virtio_balloon_info);
1073     type_register_static(&virtio_scsi_info);
1074 }
1075
1076 type_init(virtio_pci_register_types)