hw/virtio/vhost.c

   1 /*
   2  * vhost support
   3  *
   4  * Copyright Red Hat, Inc. 2010
   5  *
   6  * Authors:
   7  *  Michael S. Tsirkin <mst@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include <sys/ioctl.h>
  17 #include "hw/virtio/vhost.h"
  18 #include "hw/hw.h"
  19 #include "qemu/range.h"
  20 #include <linux/vhost.h>
  21 #include "exec/address-spaces.h"
  22 #include "hw/virtio/virtio-bus.h"
  23
  24 static void vhost_dev_sync_region(struct vhost_dev *dev,
  25                                   MemoryRegionSection *section,
  26                                   uint64_t mfirst, uint64_t mlast,
  27                                   uint64_t rfirst, uint64_t rlast)
  28 {
  29     uint64_t start = MAX(mfirst, rfirst);
  30     uint64_t end = MIN(mlast, rlast);
  31     vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK;
  32     vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1;
  33     uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
  34
  35     if (end < start) {
  36         return;
  37     }
  38     assert(end / VHOST_LOG_CHUNK < dev->log_size);
  39     assert(start / VHOST_LOG_CHUNK < dev->log_size);
  40
  41     for (;from < to; ++from) {
  42         vhost_log_chunk_t log;
  43         int bit;
  44         /* We first check with non-atomic: much cheaper,
  45          * and we expect non-dirty to be the common case. */
  46         if (!*from) {
  47             addr += VHOST_LOG_CHUNK;
  48             continue;
  49         }
  50         /* Data must be read atomically. We don't really
  51          * need the barrier semantics of __sync
  52          * builtins, but it's easier to use them than
  53          * roll our own. */
  54         log = __sync_fetch_and_and(from, 0);
  55         while ((bit = sizeof(log) > sizeof(int) ?
  56                 ffsll(log) : ffs(log))) {
  57             hwaddr page_addr;
  58             hwaddr section_offset;
  59             hwaddr mr_offset;
  60             bit -= 1;
  61             page_addr = addr + bit * VHOST_LOG_PAGE;
  62             section_offset = page_addr - section->offset_within_address_space;
  63             mr_offset = section_offset + section->offset_within_region;
  64             memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE);
  65             log &= ~(0x1ull << bit);
  66         }
  67         addr += VHOST_LOG_CHUNK;
  68     }
  69 }
  70
  71 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
  72                                    MemoryRegionSection *section,
  73                                    hwaddr first,
  74                                    hwaddr last)
  75 {
  76     int i;
  77     hwaddr start_addr;
  78     hwaddr end_addr;
  79
  80     if (!dev->log_enabled || !dev->started) {
  81         return 0;
  82     }
  83     start_addr = section->offset_within_address_space;
  84     end_addr = range_get_last(start_addr, int128_get64(section->size));
  85     start_addr = MAX(first, start_addr);
  86     end_addr = MIN(last, end_addr);
  87
  88     for (i = 0; i < dev->mem->nregions; ++i) {
  89         struct vhost_memory_region *reg = dev->mem->regions + i;
  90         vhost_dev_sync_region(dev, section, start_addr, end_addr,
  91                               reg->guest_phys_addr,
  92                               range_get_last(reg->guest_phys_addr,
  93                                              reg->memory_size));
  94     }
  95     for (i = 0; i < dev->nvqs; ++i) {
  96         struct vhost_virtqueue *vq = dev->vqs + i;
  97         vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
  98                               range_get_last(vq->used_phys, vq->used_size));
  99     }
 100     return 0;
 101 }
 102
 103 static void vhost_log_sync(MemoryListener *listener,
 104                           MemoryRegionSection *section)
 105 {
 106     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 107                                          memory_listener);
 108     vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL);
 109 }
 110
 111 static void vhost_log_sync_range(struct vhost_dev *dev,
 112                                  hwaddr first, hwaddr last)
 113 {
 114     int i;
 115     /* FIXME: this is N^2 in number of sections */
 116     for (i = 0; i < dev->n_mem_sections; ++i) {
 117         MemoryRegionSection *section = &dev->mem_sections[i];
 118         vhost_sync_dirty_bitmap(dev, section, first, last);
 119     }
 120 }
 121
 122 /* Assign/unassign. Keep an unsorted array of non-overlapping
 123  * memory regions in dev->mem. */
 124 static void vhost_dev_unassign_memory(struct vhost_dev *dev,
 125                                       uint64_t start_addr,
 126                                       uint64_t size)
 127 {
 128     int from, to, n = dev->mem->nregions;
 129     /* Track overlapping/split regions for sanity checking. */
 130     int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0;
 131
 132     for (from = 0, to = 0; from < n; ++from, ++to) {
 133         struct vhost_memory_region *reg = dev->mem->regions + to;
 134         uint64_t reglast;
 135         uint64_t memlast;
 136         uint64_t change;
 137
 138         /* clone old region */
 139         if (to != from) {
 140             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 141         }
 142
 143         /* No overlap is simple */
 144         if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 145                             start_addr, size)) {
 146             continue;
 147         }
 148
 149         /* Split only happens if supplied region
 150          * is in the middle of an existing one. Thus it can not
 151          * overlap with any other existing region. */
 152         assert(!split);
 153
 154         reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 155         memlast = range_get_last(start_addr, size);
 156
 157         /* Remove whole region */
 158         if (start_addr <= reg->guest_phys_addr && memlast >= reglast) {
 159             --dev->mem->nregions;
 160             --to;
 161             ++overlap_middle;
 162             continue;
 163         }
 164
 165         /* Shrink region */
 166         if (memlast >= reglast) {
 167             reg->memory_size = start_addr - reg->guest_phys_addr;
 168             assert(reg->memory_size);
 169             assert(!overlap_end);
 170             ++overlap_end;
 171             continue;
 172         }
 173
 174         /* Shift region */
 175         if (start_addr <= reg->guest_phys_addr) {
 176             change = memlast + 1 - reg->guest_phys_addr;
 177             reg->memory_size -= change;
 178             reg->guest_phys_addr += change;
 179             reg->userspace_addr += change;
 180             assert(reg->memory_size);
 181             assert(!overlap_start);
 182             ++overlap_start;
 183             continue;
 184         }
 185
 186         /* This only happens if supplied region
 187          * is in the middle of an existing one. Thus it can not
 188          * overlap with any other existing region. */
 189         assert(!overlap_start);
 190         assert(!overlap_end);
 191         assert(!overlap_middle);
 192         /* Split region: shrink first part, shift second part. */
 193         memcpy(dev->mem->regions + n, reg, sizeof *reg);
 194         reg->memory_size = start_addr - reg->guest_phys_addr;
 195         assert(reg->memory_size);
 196         change = memlast + 1 - reg->guest_phys_addr;
 197         reg = dev->mem->regions + n;
 198         reg->memory_size -= change;
 199         assert(reg->memory_size);
 200         reg->guest_phys_addr += change;
 201         reg->userspace_addr += change;
 202         /* Never add more than 1 region */
 203         assert(dev->mem->nregions == n);
 204         ++dev->mem->nregions;
 205         ++split;
 206     }
 207 }
 208
 209 /* Called after unassign, so no regions overlap the given range. */
 210 static void vhost_dev_assign_memory(struct vhost_dev *dev,
 211                                     uint64_t start_addr,
 212                                     uint64_t size,
 213                                     uint64_t uaddr)
 214 {
 215     int from, to;
 216     struct vhost_memory_region *merged = NULL;
 217     for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) {
 218         struct vhost_memory_region *reg = dev->mem->regions + to;
 219         uint64_t prlast, urlast;
 220         uint64_t pmlast, umlast;
 221         uint64_t s, e, u;
 222
 223         /* clone old region */
 224         if (to != from) {
 225             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 226         }
 227         prlast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 228         pmlast = range_get_last(start_addr, size);
 229         urlast = range_get_last(reg->userspace_addr, reg->memory_size);
 230         umlast = range_get_last(uaddr, size);
 231
 232         /* check for overlapping regions: should never happen. */
 233         assert(prlast < start_addr || pmlast < reg->guest_phys_addr);
 234         /* Not an adjacent or overlapping region - do not merge. */
 235         if ((prlast + 1 != start_addr || urlast + 1 != uaddr) &&
 236             (pmlast + 1 != reg->guest_phys_addr ||
 237              umlast + 1 != reg->userspace_addr)) {
 238             continue;
 239         }
 240
 241         if (merged) {
 242             --to;
 243             assert(to >= 0);
 244         } else {
 245             merged = reg;
 246         }
 247         u = MIN(uaddr, reg->userspace_addr);
 248         s = MIN(start_addr, reg->guest_phys_addr);
 249         e = MAX(pmlast, prlast);
 250         uaddr = merged->userspace_addr = u;
 251         start_addr = merged->guest_phys_addr = s;
 252         size = merged->memory_size = e - s + 1;
 253         assert(merged->memory_size);
 254     }
 255
 256     if (!merged) {
 257         struct vhost_memory_region *reg = dev->mem->regions + to;
 258         memset(reg, 0, sizeof *reg);
 259         reg->memory_size = size;
 260         assert(reg->memory_size);
 261         reg->guest_phys_addr = start_addr;
 262         reg->userspace_addr = uaddr;
 263         ++to;
 264     }
 265     assert(to <= dev->mem->nregions + 1);
 266     dev->mem->nregions = to;
 267 }
 268
 269 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
 270 {
 271     uint64_t log_size = 0;
 272     int i;
 273     for (i = 0; i < dev->mem->nregions; ++i) {
 274         struct vhost_memory_region *reg = dev->mem->regions + i;
 275         uint64_t last = range_get_last(reg->guest_phys_addr,
 276                                        reg->memory_size);
 277         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 278     }
 279     for (i = 0; i < dev->nvqs; ++i) {
 280         struct vhost_virtqueue *vq = dev->vqs + i;
 281         uint64_t last = vq->used_phys + vq->used_size - 1;
 282         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 283     }
 284     return log_size;
 285 }
 286
 287 static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
 288 {
 289     vhost_log_chunk_t *log;
 290     uint64_t log_base;
 291     int r;
 292
 293     log = g_malloc0(size * sizeof *log);
 294     log_base = (uint64_t)(unsigned long)log;
 295     r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base);
 296     assert(r >= 0);
 297     /* Sync only the range covered by the old log */
 298     if (dev->log_size) {
 299         vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
 300     }
 301     if (dev->log) {
 302         g_free(dev->log);
 303     }
 304     dev->log = log;
 305     dev->log_size = size;
 306 }
 307
 308 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
 309                                       uint64_t start_addr,
 310                                       uint64_t size)
 311 {
 312     int i;
 313     for (i = 0; i < dev->nvqs; ++i) {
 314         struct vhost_virtqueue *vq = dev->vqs + i;
 315         hwaddr l;
 316         void *p;
 317
 318         if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
 319             continue;
 320         }
 321         l = vq->ring_size;
 322         p = cpu_physical_memory_map(vq->ring_phys, &l, 1);
 323         if (!p || l != vq->ring_size) {
 324             fprintf(stderr, "Unable to map ring buffer for ring %d\n", i);
 325             return -ENOMEM;
 326         }
 327         if (p != vq->ring) {
 328             fprintf(stderr, "Ring buffer relocated for ring %d\n", i);
 329             return -EBUSY;
 330         }
 331         cpu_physical_memory_unmap(p, l, 0, 0);
 332     }
 333     return 0;
 334 }
 335
 336 static struct vhost_memory_region *vhost_dev_find_reg(struct vhost_dev *dev,
 337                                                       uint64_t start_addr,
 338                                                       uint64_t size)
 339 {
 340     int i, n = dev->mem->nregions;
 341     for (i = 0; i < n; ++i) {
 342         struct vhost_memory_region *reg = dev->mem->regions + i;
 343         if (ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 344                            start_addr, size)) {
 345             return reg;
 346         }
 347     }
 348     return NULL;
 349 }
 350
 351 static bool vhost_dev_cmp_memory(struct vhost_dev *dev,
 352                                  uint64_t start_addr,
 353                                  uint64_t size,
 354                                  uint64_t uaddr)
 355 {
 356     struct vhost_memory_region *reg = vhost_dev_find_reg(dev, start_addr, size);
 357     uint64_t reglast;
 358     uint64_t memlast;
 359
 360     if (!reg) {
 361         return true;
 362     }
 363
 364     reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 365     memlast = range_get_last(start_addr, size);
 366
 367     /* Need to extend region? */
 368     if (start_addr < reg->guest_phys_addr || memlast > reglast) {
 369         return true;
 370     }
 371     /* userspace_addr changed? */
 372     return uaddr != reg->userspace_addr + start_addr - reg->guest_phys_addr;
 373 }
 374
 375 static void vhost_set_memory(MemoryListener *listener,
 376                              MemoryRegionSection *section,
 377                              bool add)
 378 {
 379     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 380                                          memory_listener);
 381     hwaddr start_addr = section->offset_within_address_space;
 382     ram_addr_t size = int128_get64(section->size);
 383     bool log_dirty = memory_region_is_logging(section->mr);
 384     int s = offsetof(struct vhost_memory, regions) +
 385         (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
 386     void *ram;
 387
 388     dev->mem = g_realloc(dev->mem, s);
 389
 390     if (log_dirty) {
 391         add = false;
 392     }
 393
 394     assert(size);
 395
 396     /* Optimize no-change case. At least cirrus_vga does this a lot at this time. */
 397     ram = memory_region_get_ram_ptr(section->mr) + section->offset_within_region;
 398     if (add) {
 399         if (!vhost_dev_cmp_memory(dev, start_addr, size, (uintptr_t)ram)) {
 400             /* Region exists with same address. Nothing to do. */
 401             return;
 402         }
 403     } else {
 404         if (!vhost_dev_find_reg(dev, start_addr, size)) {
 405             /* Removing region that we don't access. Nothing to do. */
 406             return;
 407         }
 408     }
 409
 410     vhost_dev_unassign_memory(dev, start_addr, size);
 411     if (add) {
 412         /* Add given mapping, merging adjacent regions if any */
 413         vhost_dev_assign_memory(dev, start_addr, size, (uintptr_t)ram);
 414     } else {
 415         /* Remove old mapping for this memory, if any. */
 416         vhost_dev_unassign_memory(dev, start_addr, size);
 417     }
 418     dev->mem_changed_start_addr = MIN(dev->mem_changed_start_addr, start_addr);
 419     dev->mem_changed_end_addr = MAX(dev->mem_changed_end_addr, start_addr + size - 1);
 420     dev->memory_changed = true;
 421 }
 422
 423 static bool vhost_section(MemoryRegionSection *section)
 424 {
 425     return memory_region_is_ram(section->mr);
 426 }
 427
 428 static void vhost_begin(MemoryListener *listener)
 429 {
 430     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 431                                          memory_listener);
 432     dev->mem_changed_end_addr = 0;
 433     dev->mem_changed_start_addr = -1;
 434 }
 435
 436 static void vhost_commit(MemoryListener *listener)
 437 {
 438     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 439                                          memory_listener);
 440     hwaddr start_addr = 0;
 441     ram_addr_t size = 0;
 442     uint64_t log_size;
 443     int r;
 444
 445     if (!dev->memory_changed) {
 446         return;
 447     }
 448     if (!dev->started) {
 449         return;
 450     }
 451     if (dev->mem_changed_start_addr > dev->mem_changed_end_addr) {
 452         return;
 453     }
 454
 455     if (dev->started) {
 456         start_addr = dev->mem_changed_start_addr;
 457         size = dev->mem_changed_end_addr - dev->mem_changed_start_addr + 1;
 458
 459         r = vhost_verify_ring_mappings(dev, start_addr, size);
 460         assert(r >= 0);
 461     }
 462
 463     if (!dev->log_enabled) {
 464         r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 465         assert(r >= 0);
 466         dev->memory_changed = false;
 467         return;
 468     }
 469     log_size = vhost_get_log_size(dev);
 470     /* We allocate an extra 4K bytes to log,
 471      * to reduce the * number of reallocations. */
 472 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
 473     /* To log more, must increase log size before table update. */
 474     if (dev->log_size < log_size) {
 475         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
 476     }
 477     r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 478     assert(r >= 0);
 479     /* To log less, can only decrease log size after table update. */
 480     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
 481         vhost_dev_log_resize(dev, log_size);
 482     }
 483     dev->memory_changed = false;
 484 }
 485
 486 static void vhost_region_add(MemoryListener *listener,
 487                              MemoryRegionSection *section)
 488 {
 489     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 490                                          memory_listener);
 491
 492     if (!vhost_section(section)) {
 493         return;
 494     }
 495
 496     ++dev->n_mem_sections;
 497     dev->mem_sections = g_renew(MemoryRegionSection, dev->mem_sections,
 498                                 dev->n_mem_sections);
 499     dev->mem_sections[dev->n_mem_sections - 1] = *section;
 500     memory_region_ref(section->mr);
 501     vhost_set_memory(listener, section, true);
 502 }
 503
 504 static void vhost_region_del(MemoryListener *listener,
 505                              MemoryRegionSection *section)
 506 {
 507     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 508                                          memory_listener);
 509     int i;
 510
 511     if (!vhost_section(section)) {
 512         return;
 513     }
 514
 515     vhost_set_memory(listener, section, false);
 516     memory_region_unref(section->mr);
 517     for (i = 0; i < dev->n_mem_sections; ++i) {
 518         if (dev->mem_sections[i].offset_within_address_space
 519             == section->offset_within_address_space) {
 520             --dev->n_mem_sections;
 521             memmove(&dev->mem_sections[i], &dev->mem_sections[i+1],
 522                     (dev->n_mem_sections - i) * sizeof(*dev->mem_sections));
 523             break;
 524         }
 525     }
 526 }
 527
 528 static void vhost_region_nop(MemoryListener *listener,
 529                              MemoryRegionSection *section)
 530 {
 531 }
 532
 533 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
 534                                     struct vhost_virtqueue *vq,
 535                                     unsigned idx, bool enable_log)
 536 {
 537     struct vhost_vring_addr addr = {
 538         .index = idx,
 539         .desc_user_addr = (uint64_t)(unsigned long)vq->desc,
 540         .avail_user_addr = (uint64_t)(unsigned long)vq->avail,
 541         .used_user_addr = (uint64_t)(unsigned long)vq->used,
 542         .log_guest_addr = vq->used_phys,
 543         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
 544     };
 545     int r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
 546     if (r < 0) {
 547         return -errno;
 548     }
 549     return 0;
 550 }
 551
 552 static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
 553 {
 554     uint64_t features = dev->acked_features;
 555     int r;
 556     if (enable_log) {
 557         features |= 0x1 << VHOST_F_LOG_ALL;
 558     }
 559     r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
 560     return r < 0 ? -errno : 0;
 561 }
 562
 563 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 564 {
 565     int r, t, i;
 566     r = vhost_dev_set_features(dev, enable_log);
 567     if (r < 0) {
 568         goto err_features;
 569     }
 570     for (i = 0; i < dev->nvqs; ++i) {
 571         r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 572                                      enable_log);
 573         if (r < 0) {
 574             goto err_vq;
 575         }
 576     }
 577     return 0;
 578 err_vq:
 579     for (; i >= 0; --i) {
 580         t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 581                                      dev->log_enabled);
 582         assert(t >= 0);
 583     }
 584     t = vhost_dev_set_features(dev, dev->log_enabled);
 585     assert(t >= 0);
 586 err_features:
 587     return r;
 588 }
 589
 590 static int vhost_migration_log(MemoryListener *listener, int enable)
 591 {
 592     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 593                                          memory_listener);
 594     int r;
 595     if (!!enable == dev->log_enabled) {
 596         return 0;
 597     }
 598     if (!dev->started) {
 599         dev->log_enabled = enable;
 600         return 0;
 601     }
 602     if (!enable) {
 603         r = vhost_dev_set_log(dev, false);
 604         if (r < 0) {
 605             return r;
 606         }
 607         if (dev->log) {
 608             g_free(dev->log);
 609         }
 610         dev->log = NULL;
 611         dev->log_size = 0;
 612     } else {
 613         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
 614         r = vhost_dev_set_log(dev, true);
 615         if (r < 0) {
 616             return r;
 617         }
 618     }
 619     dev->log_enabled = enable;
 620     return 0;
 621 }
 622
 623 static void vhost_log_global_start(MemoryListener *listener)
 624 {
 625     int r;
 626
 627     r = vhost_migration_log(listener, true);
 628     if (r < 0) {
 629         abort();
 630     }
 631 }
 632
 633 static void vhost_log_global_stop(MemoryListener *listener)
 634 {
 635     int r;
 636
 637     r = vhost_migration_log(listener, false);
 638     if (r < 0) {
 639         abort();
 640     }
 641 }
 642
 643 static void vhost_log_start(MemoryListener *listener,
 644                             MemoryRegionSection *section)
 645 {
 646     /* FIXME: implement */
 647 }
 648
 649 static void vhost_log_stop(MemoryListener *listener,
 650                            MemoryRegionSection *section)
 651 {
 652     /* FIXME: implement */
 653 }
 654
 655 static int vhost_virtqueue_start(struct vhost_dev *dev,
 656                                 struct VirtIODevice *vdev,
 657                                 struct vhost_virtqueue *vq,
 658                                 unsigned idx)
 659 {
 660     hwaddr s, l, a;
 661     int r;
 662     int vhost_vq_index = idx - dev->vq_index;
 663     struct vhost_vring_file file = {
 664         .index = vhost_vq_index
 665     };
 666     struct vhost_vring_state state = {
 667         .index = vhost_vq_index
 668     };
 669     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 670
 671     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 672
 673     vq->num = state.num = virtio_queue_get_num(vdev, idx);
 674     r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
 675     if (r) {
 676         return -errno;
 677     }
 678
 679     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
 680     r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
 681     if (r) {
 682         return -errno;
 683     }
 684
 685     s = l = virtio_queue_get_desc_size(vdev, idx);
 686     a = virtio_queue_get_desc_addr(vdev, idx);
 687     vq->desc = cpu_physical_memory_map(a, &l, 0);
 688     if (!vq->desc || l != s) {
 689         r = -ENOMEM;
 690         goto fail_alloc_desc;
 691     }
 692     s = l = virtio_queue_get_avail_size(vdev, idx);
 693     a = virtio_queue_get_avail_addr(vdev, idx);
 694     vq->avail = cpu_physical_memory_map(a, &l, 0);
 695     if (!vq->avail || l != s) {
 696         r = -ENOMEM;
 697         goto fail_alloc_avail;
 698     }
 699     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
 700     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
 701     vq->used = cpu_physical_memory_map(a, &l, 1);
 702     if (!vq->used || l != s) {
 703         r = -ENOMEM;
 704         goto fail_alloc_used;
 705     }
 706
 707     vq->ring_size = s = l = virtio_queue_get_ring_size(vdev, idx);
 708     vq->ring_phys = a = virtio_queue_get_ring_addr(vdev, idx);
 709     vq->ring = cpu_physical_memory_map(a, &l, 1);
 710     if (!vq->ring || l != s) {
 711         r = -ENOMEM;
 712         goto fail_alloc_ring;
 713     }
 714
 715     r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
 716     if (r < 0) {
 717         r = -errno;
 718         goto fail_alloc;
 719     }
 720
 721     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
 722     r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
 723     if (r) {
 724         r = -errno;
 725         goto fail_kick;
 726     }
 727
 728     /* Clear and discard previous events if any. */
 729     event_notifier_test_and_clear(&vq->masked_notifier);
 730
 731     return 0;
 732
 733 fail_kick:
 734 fail_alloc:
 735     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 736                               0, 0);
 737 fail_alloc_ring:
 738     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 739                               0, 0);
 740 fail_alloc_used:
 741     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 742                               0, 0);
 743 fail_alloc_avail:
 744     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 745                               0, 0);
 746 fail_alloc_desc:
 747     return r;
 748 }
 749
 750 static void vhost_virtqueue_stop(struct vhost_dev *dev,
 751                                     struct VirtIODevice *vdev,
 752                                     struct vhost_virtqueue *vq,
 753                                     unsigned idx)
 754 {
 755     struct vhost_vring_state state = {
 756         .index = idx - dev->vq_index
 757     };
 758     int r;
 759     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 760     r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state);
 761     if (r < 0) {
 762         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
 763         fflush(stderr);
 764     }
 765     virtio_queue_set_last_avail_idx(vdev, idx, state.num);
 766     assert (r >= 0);
 767     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 768                               0, virtio_queue_get_ring_size(vdev, idx));
 769     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 770                               1, virtio_queue_get_used_size(vdev, idx));
 771     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 772                               0, virtio_queue_get_avail_size(vdev, idx));
 773     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 774                               0, virtio_queue_get_desc_size(vdev, idx));
 775 }
 776
 777 static void vhost_eventfd_add(MemoryListener *listener,
 778                               MemoryRegionSection *section,
 779                               bool match_data, uint64_t data, EventNotifier *e)
 780 {
 781 }
 782
 783 static void vhost_eventfd_del(MemoryListener *listener,
 784                               MemoryRegionSection *section,
 785                               bool match_data, uint64_t data, EventNotifier *e)
 786 {
 787 }
 788
 789 static int vhost_virtqueue_init(struct vhost_dev *dev,
 790                                 struct vhost_virtqueue *vq, int n)
 791 {
 792     struct vhost_vring_file file = {
 793         .index = n,
 794     };
 795     int r = event_notifier_init(&vq->masked_notifier, 0);
 796     if (r < 0) {
 797         return r;
 798     }
 799
 800     file.fd = event_notifier_get_fd(&vq->masked_notifier);
 801     r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
 802     if (r) {
 803         r = -errno;
 804         goto fail_call;
 805     }
 806     return 0;
 807 fail_call:
 808     event_notifier_cleanup(&vq->masked_notifier);
 809     return r;
 810 }
 811
 812 static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
 813 {
 814     event_notifier_cleanup(&vq->masked_notifier);
 815 }
 816
 817 int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
 818                    bool force)
 819 {
 820     uint64_t features;
 821     int i, r;
 822     if (devfd >= 0) {
 823         hdev->control = devfd;
 824     } else {
 825         hdev->control = open(devpath, O_RDWR);
 826         if (hdev->control < 0) {
 827             return -errno;
 828         }
 829     }
 830     r = ioctl(hdev->control, VHOST_SET_OWNER, NULL);
 831     if (r < 0) {
 832         goto fail;
 833     }
 834
 835     r = ioctl(hdev->control, VHOST_GET_FEATURES, &features);
 836     if (r < 0) {
 837         goto fail;
 838     }
 839
 840     for (i = 0; i < hdev->nvqs; ++i) {
 841         r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
 842         if (r < 0) {
 843             goto fail_vq;
 844         }
 845     }
 846     hdev->features = features;
 847
 848     hdev->memory_listener = (MemoryListener) {
 849         .begin = vhost_begin,
 850         .commit = vhost_commit,
 851         .region_add = vhost_region_add,
 852         .region_del = vhost_region_del,
 853         .region_nop = vhost_region_nop,
 854         .log_start = vhost_log_start,
 855         .log_stop = vhost_log_stop,
 856         .log_sync = vhost_log_sync,
 857         .log_global_start = vhost_log_global_start,
 858         .log_global_stop = vhost_log_global_stop,
 859         .eventfd_add = vhost_eventfd_add,
 860         .eventfd_del = vhost_eventfd_del,
 861         .priority = 10
 862     };
 863     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
 864     hdev->n_mem_sections = 0;
 865     hdev->mem_sections = NULL;
 866     hdev->log = NULL;
 867     hdev->log_size = 0;
 868     hdev->log_enabled = false;
 869     hdev->started = false;
 870     hdev->memory_changed = false;
 871     memory_listener_register(&hdev->memory_listener, &address_space_memory);
 872     hdev->force = force;
 873     return 0;
 874 fail_vq:
 875     while (--i >= 0) {
 876         vhost_virtqueue_cleanup(hdev->vqs + i);
 877     }
 878 fail:
 879     r = -errno;
 880     close(hdev->control);
 881     return r;
 882 }
 883
 884 void vhost_dev_cleanup(struct vhost_dev *hdev)
 885 {
 886     int i;
 887     for (i = 0; i < hdev->nvqs; ++i) {
 888         vhost_virtqueue_cleanup(hdev->vqs + i);
 889     }
 890     memory_listener_unregister(&hdev->memory_listener);
 891     g_free(hdev->mem);
 892     g_free(hdev->mem_sections);
 893     close(hdev->control);
 894 }
 895
 896 bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev)
 897 {
 898     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
 899     VirtioBusState *vbus = VIRTIO_BUS(qbus);
 900     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
 901
 902     return !k->query_guest_notifiers ||
 903            k->query_guest_notifiers(qbus->parent) ||
 904            hdev->force;
 905 }
 906
 907 /* Stop processing guest IO notifications in qemu.
 908  * Start processing them in vhost in kernel.
 909  */
 910 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 911 {
 912     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
 913     VirtioBusState *vbus = VIRTIO_BUS(qbus);
 914     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
 915     int i, r;
 916     if (!k->set_host_notifier) {
 917         fprintf(stderr, "binding does not support host notifiers\n");
 918         r = -ENOSYS;
 919         goto fail;
 920     }
 921
 922     for (i = 0; i < hdev->nvqs; ++i) {
 923         r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, true);
 924         if (r < 0) {
 925             fprintf(stderr, "vhost VQ %d notifier binding failed: %d\n", i, -r);
 926             goto fail_vq;
 927         }
 928     }
 929
 930     return 0;
 931 fail_vq:
 932     while (--i >= 0) {
 933         r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, false);
 934         if (r < 0) {
 935             fprintf(stderr, "vhost VQ %d notifier cleanup error: %d\n", i, -r);
 936             fflush(stderr);
 937         }
 938         assert (r >= 0);
 939     }
 940 fail:
 941     return r;
 942 }
 943
 944 /* Stop processing guest IO notifications in vhost.
 945  * Start processing them in qemu.
 946  * This might actually run the qemu handlers right away,
 947  * so virtio in qemu must be completely setup when this is called.
 948  */
 949 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 950 {
 951     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
 952     VirtioBusState *vbus = VIRTIO_BUS(qbus);
 953     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
 954     int i, r;
 955
 956     for (i = 0; i < hdev->nvqs; ++i) {
 957         r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, false);
 958         if (r < 0) {
 959             fprintf(stderr, "vhost VQ %d notifier cleanup failed: %d\n", i, -r);
 960             fflush(stderr);
 961         }
 962         assert (r >= 0);
 963     }
 964 }
 965
 966 /* Test and clear event pending status.
 967  * Should be called after unmask to avoid losing events.
 968  */
 969 bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
 970 {
 971     struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
 972     assert(hdev->started);
 973     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
 974     return event_notifier_test_and_clear(&vq->masked_notifier);
 975 }
 976
 977 /* Mask/unmask events from this vq. */
 978 void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
 979                          bool mask)
 980 {
 981     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
 982     int r, index = n - hdev->vq_index;
 983
 984     assert(hdev->started);
 985     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
 986
 987     struct vhost_vring_file file = {
 988         .index = index
 989     };
 990     if (mask) {
 991         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
 992     } else {
 993         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
 994     }
 995     r = ioctl(hdev->control, VHOST_SET_VRING_CALL, &file);
 996     assert(r >= 0);
 997 }
 998
 999 /* Host notifiers must be enabled at this point. */
1000 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
1001 {
1002     int i, r;
1003
1004     hdev->started = true;
1005
1006     r = vhost_dev_set_features(hdev, hdev->log_enabled);
1007     if (r < 0) {
1008         goto fail_features;
1009     }
1010     r = ioctl(hdev->control, VHOST_SET_MEM_TABLE, hdev->mem);
1011     if (r < 0) {
1012         r = -errno;
1013         goto fail_mem;
1014     }
1015     for (i = 0; i < hdev->nvqs; ++i) {
1016         r = vhost_virtqueue_start(hdev,
1017                                   vdev,
1018                                   hdev->vqs + i,
1019                                   hdev->vq_index + i);
1020         if (r < 0) {
1021             goto fail_vq;
1022         }
1023     }
1024
1025     if (hdev->log_enabled) {
1026         hdev->log_size = vhost_get_log_size(hdev);
1027         hdev->log = hdev->log_size ?
1028             g_malloc0(hdev->log_size * sizeof *hdev->log) : NULL;
1029         r = ioctl(hdev->control, VHOST_SET_LOG_BASE,
1030                   (uint64_t)(unsigned long)hdev->log);
1031         if (r < 0) {
1032             r = -errno;
1033             goto fail_log;
1034         }
1035     }
1036
1037     return 0;
1038 fail_log:
1039 fail_vq:
1040     while (--i >= 0) {
1041         vhost_virtqueue_stop(hdev,
1042                              vdev,
1043                              hdev->vqs + i,
1044                              hdev->vq_index + i);
1045     }
1046     i = hdev->nvqs;
1047 fail_mem:
1048 fail_features:
1049
1050     hdev->started = false;
1051     return r;
1052 }
1053
1054 /* Host notifiers must be enabled at this point. */
1055 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
1056 {
1057     int i;
1058
1059     for (i = 0; i < hdev->nvqs; ++i) {
1060         vhost_virtqueue_stop(hdev,
1061                              vdev,
1062                              hdev->vqs + i,
1063                              hdev->vq_index + i);
1064     }
1065     vhost_log_sync_range(hdev, 0, ~0x0ull);
1066
1067     hdev->started = false;
1068     g_free(hdev->log);
1069     hdev->log = NULL;
1070     hdev->log_size = 0;
1071 }
1072