hw/vhost.c

   1 /*
   2  * vhost support
   3  *
   4  * Copyright Red Hat, Inc. 2010
   5  *
   6  * Authors:
   7  *  Michael S. Tsirkin <mst@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  */
  12
  13 #include <linux/vhost.h>
  14 #include <sys/ioctl.h>
  15 #include <sys/eventfd.h>
  16 #include "vhost.h"
  17 #include "hw/hw.h"
  18 /* For range_get_last */
  19 #include "pci.h"
  20
  21 static void vhost_dev_sync_region(struct vhost_dev *dev,
  22                                   uint64_t mfirst, uint64_t mlast,
  23                                   uint64_t rfirst, uint64_t rlast)
  24 {
  25     uint64_t start = MAX(mfirst, rfirst);
  26     uint64_t end = MIN(mlast, rlast);
  27     vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK;
  28     vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1;
  29     uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
  30
  31     assert(end / VHOST_LOG_CHUNK < dev->log_size);
  32     assert(start / VHOST_LOG_CHUNK < dev->log_size);
  33     if (end < start) {
  34         return;
  35     }
  36     for (;from < to; ++from) {
  37         vhost_log_chunk_t log;
  38         int bit;
  39         /* We first check with non-atomic: much cheaper,
  40          * and we expect non-dirty to be the common case. */
  41         if (!*from) {
  42             continue;
  43         }
  44         /* Data must be read atomically. We don't really
  45          * need the barrier semantics of __sync
  46          * builtins, but it's easier to use them than
  47          * roll our own. */
  48         log = __sync_fetch_and_and(from, 0);
  49         while ((bit = sizeof(log) > sizeof(int) ?
  50                 ffsll(log) : ffs(log))) {
  51             bit -= 1;
  52             cpu_physical_memory_set_dirty(addr + bit * VHOST_LOG_PAGE);
  53             log &= ~(0x1ull << bit);
  54         }
  55         addr += VHOST_LOG_CHUNK;
  56     }
  57 }
  58
  59 static int vhost_client_sync_dirty_bitmap(CPUPhysMemoryClient *client,
  60                                           target_phys_addr_t start_addr,
  61                                           target_phys_addr_t end_addr)
  62 {
  63     struct vhost_dev *dev = container_of(client, struct vhost_dev, client);
  64     int i;
  65     if (!dev->log_enabled || !dev->started) {
  66         return 0;
  67     }
  68     for (i = 0; i < dev->mem->nregions; ++i) {
  69         struct vhost_memory_region *reg = dev->mem->regions + i;
  70         vhost_dev_sync_region(dev, start_addr, end_addr,
  71                               reg->guest_phys_addr,
  72                               range_get_last(reg->guest_phys_addr,
  73                                              reg->memory_size));
  74     }
  75     for (i = 0; i < dev->nvqs; ++i) {
  76         struct vhost_virtqueue *vq = dev->vqs + i;
  77         vhost_dev_sync_region(dev, start_addr, end_addr, vq->used_phys,
  78                               range_get_last(vq->used_phys, vq->used_size));
  79     }
  80     return 0;
  81 }
  82
  83 /* Assign/unassign. Keep an unsorted array of non-overlapping
  84  * memory regions in dev->mem. */
  85 static void vhost_dev_unassign_memory(struct vhost_dev *dev,
  86                                       uint64_t start_addr,
  87                                       uint64_t size)
  88 {
  89     int from, to, n = dev->mem->nregions;
  90     /* Track overlapping/split regions for sanity checking. */
  91     int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0;
  92
  93     for (from = 0, to = 0; from < n; ++from, ++to) {
  94         struct vhost_memory_region *reg = dev->mem->regions + to;
  95         uint64_t reglast;
  96         uint64_t memlast;
  97         uint64_t change;
  98
  99         /* clone old region */
 100         if (to != from) {
 101             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 102         }
 103
 104         /* No overlap is simple */
 105         if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 106                             start_addr, size)) {
 107             continue;
 108         }
 109
 110         /* Split only happens if supplied region
 111          * is in the middle of an existing one. Thus it can not
 112          * overlap with any other existing region. */
 113         assert(!split);
 114
 115         reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 116         memlast = range_get_last(start_addr, size);
 117
 118         /* Remove whole region */
 119         if (start_addr <= reg->guest_phys_addr && memlast >= reglast) {
 120             --dev->mem->nregions;
 121             --to;
 122             assert(to >= 0);
 123             ++overlap_middle;
 124             continue;
 125         }
 126
 127         /* Shrink region */
 128         if (memlast >= reglast) {
 129             reg->memory_size = start_addr - reg->guest_phys_addr;
 130             assert(reg->memory_size);
 131             assert(!overlap_end);
 132             ++overlap_end;
 133             continue;
 134         }
 135
 136         /* Shift region */
 137         if (start_addr <= reg->guest_phys_addr) {
 138             change = memlast + 1 - reg->guest_phys_addr;
 139             reg->memory_size -= change;
 140             reg->guest_phys_addr += change;
 141             reg->userspace_addr += change;
 142             assert(reg->memory_size);
 143             assert(!overlap_start);
 144             ++overlap_start;
 145             continue;
 146         }
 147
 148         /* This only happens if supplied region
 149          * is in the middle of an existing one. Thus it can not
 150          * overlap with any other existing region. */
 151         assert(!overlap_start);
 152         assert(!overlap_end);
 153         assert(!overlap_middle);
 154         /* Split region: shrink first part, shift second part. */
 155         memcpy(dev->mem->regions + n, reg, sizeof *reg);
 156         reg->memory_size = start_addr - reg->guest_phys_addr;
 157         assert(reg->memory_size);
 158         change = memlast + 1 - reg->guest_phys_addr;
 159         reg = dev->mem->regions + n;
 160         reg->memory_size -= change;
 161         assert(reg->memory_size);
 162         reg->guest_phys_addr += change;
 163         reg->userspace_addr += change;
 164         /* Never add more than 1 region */
 165         assert(dev->mem->nregions == n);
 166         ++dev->mem->nregions;
 167         ++split;
 168     }
 169 }
 170
 171 /* Called after unassign, so no regions overlap the given range. */
 172 static void vhost_dev_assign_memory(struct vhost_dev *dev,
 173                                     uint64_t start_addr,
 174                                     uint64_t size,
 175                                     uint64_t uaddr)
 176 {
 177     int from, to;
 178     struct vhost_memory_region *merged = NULL;
 179     for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) {
 180         struct vhost_memory_region *reg = dev->mem->regions + to;
 181         uint64_t prlast, urlast;
 182         uint64_t pmlast, umlast;
 183         uint64_t s, e, u;
 184
 185         /* clone old region */
 186         if (to != from) {
 187             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 188         }
 189         prlast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 190         pmlast = range_get_last(start_addr, size);
 191         urlast = range_get_last(reg->userspace_addr, reg->memory_size);
 192         umlast = range_get_last(uaddr, size);
 193
 194         /* check for overlapping regions: should never happen. */
 195         assert(prlast < start_addr || pmlast < reg->guest_phys_addr);
 196         /* Not an adjacent or overlapping region - do not merge. */
 197         if ((prlast + 1 != start_addr || urlast + 1 != uaddr) &&
 198             (pmlast + 1 != reg->guest_phys_addr ||
 199              umlast + 1 != reg->userspace_addr)) {
 200             continue;
 201         }
 202
 203         if (merged) {
 204             --to;
 205             assert(to >= 0);
 206         } else {
 207             merged = reg;
 208         }
 209         u = MIN(uaddr, reg->userspace_addr);
 210         s = MIN(start_addr, reg->guest_phys_addr);
 211         e = MAX(pmlast, prlast);
 212         uaddr = merged->userspace_addr = u;
 213         start_addr = merged->guest_phys_addr = s;
 214         size = merged->memory_size = e - s + 1;
 215         assert(merged->memory_size);
 216     }
 217
 218     if (!merged) {
 219         struct vhost_memory_region *reg = dev->mem->regions + to;
 220         memset(reg, 0, sizeof *reg);
 221         reg->memory_size = size;
 222         assert(reg->memory_size);
 223         reg->guest_phys_addr = start_addr;
 224         reg->userspace_addr = uaddr;
 225         ++to;
 226     }
 227     assert(to <= dev->mem->nregions + 1);
 228     dev->mem->nregions = to;
 229 }
 230
 231 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
 232 {
 233     uint64_t log_size = 0;
 234     int i;
 235     for (i = 0; i < dev->mem->nregions; ++i) {
 236         struct vhost_memory_region *reg = dev->mem->regions + i;
 237         uint64_t last = range_get_last(reg->guest_phys_addr,
 238                                        reg->memory_size);
 239         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 240     }
 241     for (i = 0; i < dev->nvqs; ++i) {
 242         struct vhost_virtqueue *vq = dev->vqs + i;
 243         uint64_t last = vq->used_phys + vq->used_size - 1;
 244         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 245     }
 246     return log_size;
 247 }
 248
 249 static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
 250 {
 251     vhost_log_chunk_t *log;
 252     uint64_t log_base;
 253     int r;
 254     if (size) {
 255         log = qemu_mallocz(size * sizeof *log);
 256     } else {
 257         log = NULL;
 258     }
 259     log_base = (uint64_t)(unsigned long)log;
 260     r = ioctl(dev->control, VHOST_SET_LOG_BASE, &log_base);
 261     assert(r >= 0);
 262     vhost_client_sync_dirty_bitmap(&dev->client, 0,
 263                                    (target_phys_addr_t)~0x0ull);
 264     if (dev->log) {
 265         qemu_free(dev->log);
 266     }
 267     dev->log = log;
 268     dev->log_size = size;
 269 }
 270
 271 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
 272                                       uint64_t start_addr,
 273                                       uint64_t size)
 274 {
 275     int i;
 276     for (i = 0; i < dev->nvqs; ++i) {
 277         struct vhost_virtqueue *vq = dev->vqs + i;
 278         target_phys_addr_t l;
 279         void *p;
 280
 281         if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
 282             continue;
 283         }
 284         l = vq->ring_size;
 285         p = cpu_physical_memory_map(vq->ring_phys, &l, 1);
 286         if (!p || l != vq->ring_size) {
 287             fprintf(stderr, "Unable to map ring buffer for ring %d\n", i);
 288             return -ENOMEM;
 289         }
 290         if (p != vq->ring) {
 291             fprintf(stderr, "Ring buffer relocated for ring %d\n", i);
 292             return -EBUSY;
 293         }
 294         cpu_physical_memory_unmap(p, l, 0, 0);
 295     }
 296     return 0;
 297 }
 298
 299 static void vhost_client_set_memory(CPUPhysMemoryClient *client,
 300                                     target_phys_addr_t start_addr,
 301                                     ram_addr_t size,
 302                                     ram_addr_t phys_offset)
 303 {
 304     struct vhost_dev *dev = container_of(client, struct vhost_dev, client);
 305     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 306     int s = offsetof(struct vhost_memory, regions) +
 307         (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
 308     uint64_t log_size;
 309     int r;
 310     dev->mem = qemu_realloc(dev->mem, s);
 311
 312     assert(size);
 313
 314     vhost_dev_unassign_memory(dev, start_addr, size);
 315     if (flags == IO_MEM_RAM) {
 316         /* Add given mapping, merging adjacent regions if any */
 317         vhost_dev_assign_memory(dev, start_addr, size,
 318                                 (uintptr_t)qemu_get_ram_ptr(phys_offset));
 319     } else {
 320         /* Remove old mapping for this memory, if any. */
 321         vhost_dev_unassign_memory(dev, start_addr, size);
 322     }
 323
 324     if (!dev->started) {
 325         return;
 326     }
 327
 328     if (dev->started) {
 329         r = vhost_verify_ring_mappings(dev, start_addr, size);
 330         assert(r >= 0);
 331     }
 332
 333     if (!dev->log_enabled) {
 334         r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 335         assert(r >= 0);
 336         return;
 337     }
 338     log_size = vhost_get_log_size(dev);
 339     /* We allocate an extra 4K bytes to log,
 340      * to reduce the * number of reallocations. */
 341 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
 342     /* To log more, must increase log size before table update. */
 343     if (dev->log_size < log_size) {
 344         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
 345     }
 346     r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
 347     assert(r >= 0);
 348     /* To log less, can only decrease log size after table update. */
 349     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
 350         vhost_dev_log_resize(dev, log_size);
 351     }
 352 }
 353
 354 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
 355                                     struct vhost_virtqueue *vq,
 356                                     unsigned idx, bool enable_log)
 357 {
 358     struct vhost_vring_addr addr = {
 359         .index = idx,
 360         .desc_user_addr = (uint64_t)(unsigned long)vq->desc,
 361         .avail_user_addr = (uint64_t)(unsigned long)vq->avail,
 362         .used_user_addr = (uint64_t)(unsigned long)vq->used,
 363         .log_guest_addr = vq->used_phys,
 364         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
 365     };
 366     int r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
 367     if (r < 0) {
 368         return -errno;
 369     }
 370     return 0;
 371 }
 372
 373 static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
 374 {
 375     uint64_t features = dev->acked_features;
 376     int r;
 377     if (enable_log) {
 378         features |= 0x1 << VHOST_F_LOG_ALL;
 379     }
 380     r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
 381     return r < 0 ? -errno : 0;
 382 }
 383
 384 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 385 {
 386     int r, t, i;
 387     r = vhost_dev_set_features(dev, enable_log);
 388     if (r < 0) {
 389         goto err_features;
 390     }
 391     for (i = 0; i < dev->nvqs; ++i) {
 392         r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 393                                      enable_log);
 394         if (r < 0) {
 395             goto err_vq;
 396         }
 397     }
 398     return 0;
 399 err_vq:
 400     for (; i >= 0; --i) {
 401         t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
 402                                      dev->log_enabled);
 403         assert(t >= 0);
 404     }
 405     t = vhost_dev_set_features(dev, dev->log_enabled);
 406     assert(t >= 0);
 407 err_features:
 408     return r;
 409 }
 410
 411 static int vhost_client_migration_log(CPUPhysMemoryClient *client,
 412                                       int enable)
 413 {
 414     struct vhost_dev *dev = container_of(client, struct vhost_dev, client);
 415     int r;
 416     if (!!enable == dev->log_enabled) {
 417         return 0;
 418     }
 419     if (!dev->started) {
 420         dev->log_enabled = enable;
 421         return 0;
 422     }
 423     if (!enable) {
 424         r = vhost_dev_set_log(dev, false);
 425         if (r < 0) {
 426             return r;
 427         }
 428         if (dev->log) {
 429             qemu_free(dev->log);
 430         }
 431         dev->log = NULL;
 432         dev->log_size = 0;
 433     } else {
 434         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
 435         r = vhost_dev_set_log(dev, true);
 436         if (r < 0) {
 437             return r;
 438         }
 439     }
 440     dev->log_enabled = enable;
 441     return 0;
 442 }
 443
 444 static int vhost_virtqueue_init(struct vhost_dev *dev,
 445                                 struct VirtIODevice *vdev,
 446                                 struct vhost_virtqueue *vq,
 447                                 unsigned idx)
 448 {
 449     target_phys_addr_t s, l, a;
 450     int r;
 451     struct vhost_vring_file file = {
 452         .index = idx,
 453     };
 454     struct vhost_vring_state state = {
 455         .index = idx,
 456     };
 457     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 458
 459     if (!vdev->binding->set_guest_notifier) {
 460         fprintf(stderr, "binding does not support guest notifiers\n");
 461         return -ENOSYS;
 462     }
 463
 464     if (!vdev->binding->set_host_notifier) {
 465         fprintf(stderr, "binding does not support host notifiers\n");
 466         return -ENOSYS;
 467     }
 468
 469     vq->num = state.num = virtio_queue_get_num(vdev, idx);
 470     r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
 471     if (r) {
 472         return -errno;
 473     }
 474
 475     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
 476     r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
 477     if (r) {
 478         return -errno;
 479     }
 480
 481     s = l = virtio_queue_get_desc_size(vdev, idx);
 482     a = virtio_queue_get_desc_addr(vdev, idx);
 483     vq->desc = cpu_physical_memory_map(a, &l, 0);
 484     if (!vq->desc || l != s) {
 485         r = -ENOMEM;
 486         goto fail_alloc_desc;
 487     }
 488     s = l = virtio_queue_get_avail_size(vdev, idx);
 489     a = virtio_queue_get_avail_addr(vdev, idx);
 490     vq->avail = cpu_physical_memory_map(a, &l, 0);
 491     if (!vq->avail || l != s) {
 492         r = -ENOMEM;
 493         goto fail_alloc_avail;
 494     }
 495     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
 496     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
 497     vq->used = cpu_physical_memory_map(a, &l, 1);
 498     if (!vq->used || l != s) {
 499         r = -ENOMEM;
 500         goto fail_alloc_used;
 501     }
 502
 503     vq->ring_size = s = l = virtio_queue_get_ring_size(vdev, idx);
 504     vq->ring_phys = a = virtio_queue_get_ring_addr(vdev, idx);
 505     vq->ring = cpu_physical_memory_map(a, &l, 1);
 506     if (!vq->ring || l != s) {
 507         r = -ENOMEM;
 508         goto fail_alloc_ring;
 509     }
 510
 511     r = vhost_virtqueue_set_addr(dev, vq, idx, dev->log_enabled);
 512     if (r < 0) {
 513         r = -errno;
 514         goto fail_alloc;
 515     }
 516     r = vdev->binding->set_guest_notifier(vdev->binding_opaque, idx, true);
 517     if (r < 0) {
 518         fprintf(stderr, "Error binding guest notifier: %d\n", -r);
 519         goto fail_guest_notifier;
 520     }
 521
 522     r = vdev->binding->set_host_notifier(vdev->binding_opaque, idx, true);
 523     if (r < 0) {
 524         fprintf(stderr, "Error binding host notifier: %d\n", -r);
 525         goto fail_host_notifier;
 526     }
 527
 528     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
 529     r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
 530     if (r) {
 531         goto fail_kick;
 532     }
 533
 534     file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
 535     r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
 536     if (r) {
 537         goto fail_call;
 538     }
 539
 540     return 0;
 541
 542 fail_call:
 543 fail_kick:
 544     vdev->binding->set_host_notifier(vdev->binding_opaque, idx, false);
 545 fail_host_notifier:
 546     vdev->binding->set_guest_notifier(vdev->binding_opaque, idx, false);
 547 fail_guest_notifier:
 548 fail_alloc:
 549     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 550                               0, 0);
 551 fail_alloc_ring:
 552     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 553                               0, 0);
 554 fail_alloc_used:
 555     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 556                               0, 0);
 557 fail_alloc_avail:
 558     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 559                               0, 0);
 560 fail_alloc_desc:
 561     return r;
 562 }
 563
 564 static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
 565                                     struct VirtIODevice *vdev,
 566                                     struct vhost_virtqueue *vq,
 567                                     unsigned idx)
 568 {
 569     struct vhost_vring_state state = {
 570         .index = idx,
 571     };
 572     int r;
 573     r = vdev->binding->set_guest_notifier(vdev->binding_opaque, idx, false);
 574     if (r < 0) {
 575         fprintf(stderr, "vhost VQ %d guest cleanup failed: %d\n", idx, r);
 576         fflush(stderr);
 577     }
 578     assert (r >= 0);
 579
 580     r = vdev->binding->set_host_notifier(vdev->binding_opaque, idx, false);
 581     if (r < 0) {
 582         fprintf(stderr, "vhost VQ %d host cleanup failed: %d\n", idx, r);
 583         fflush(stderr);
 584     }
 585     assert (r >= 0);
 586     r = ioctl(dev->control, VHOST_GET_VRING_BASE, &state);
 587     if (r < 0) {
 588         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
 589         fflush(stderr);
 590     }
 591     virtio_queue_set_last_avail_idx(vdev, idx, state.num);
 592     assert (r >= 0);
 593     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
 594                               0, virtio_queue_get_ring_size(vdev, idx));
 595     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
 596                               1, virtio_queue_get_used_size(vdev, idx));
 597     cpu_physical_memory_unmap(vq->avail, virtio_queue_get_avail_size(vdev, idx),
 598                               0, virtio_queue_get_avail_size(vdev, idx));
 599     cpu_physical_memory_unmap(vq->desc, virtio_queue_get_desc_size(vdev, idx),
 600                               0, virtio_queue_get_desc_size(vdev, idx));
 601 }
 602
 603 int vhost_dev_init(struct vhost_dev *hdev, int devfd)
 604 {
 605     uint64_t features;
 606     int r;
 607     if (devfd >= 0) {
 608         hdev->control = devfd;
 609     } else {
 610         hdev->control = open("/dev/vhost-net", O_RDWR);
 611         if (hdev->control < 0) {
 612             return -errno;
 613         }
 614     }
 615     r = ioctl(hdev->control, VHOST_SET_OWNER, NULL);
 616     if (r < 0) {
 617         goto fail;
 618     }
 619
 620     r = ioctl(hdev->control, VHOST_GET_FEATURES, &features);
 621     if (r < 0) {
 622         goto fail;
 623     }
 624     hdev->features = features;
 625
 626     hdev->client.set_memory = vhost_client_set_memory;
 627     hdev->client.sync_dirty_bitmap = vhost_client_sync_dirty_bitmap;
 628     hdev->client.migration_log = vhost_client_migration_log;
 629     hdev->mem = qemu_mallocz(offsetof(struct vhost_memory, regions));
 630     hdev->log = NULL;
 631     hdev->log_size = 0;
 632     hdev->log_enabled = false;
 633     hdev->started = false;
 634     cpu_register_phys_memory_client(&hdev->client);
 635     return 0;
 636 fail:
 637     r = -errno;
 638     close(hdev->control);
 639     return r;
 640 }
 641
 642 void vhost_dev_cleanup(struct vhost_dev *hdev)
 643 {
 644     cpu_unregister_phys_memory_client(&hdev->client);
 645     qemu_free(hdev->mem);
 646     close(hdev->control);
 647 }
 648
 649 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
 650 {
 651     int i, r;
 652
 653     r = vhost_dev_set_features(hdev, hdev->log_enabled);
 654     if (r < 0) {
 655         goto fail;
 656     }
 657     r = ioctl(hdev->control, VHOST_SET_MEM_TABLE, hdev->mem);
 658     if (r < 0) {
 659         r = -errno;
 660         goto fail;
 661     }
 662     if (hdev->log_enabled) {
 663         hdev->log_size = vhost_get_log_size(hdev);
 664         hdev->log = hdev->log_size ?
 665             qemu_mallocz(hdev->log_size * sizeof *hdev->log) : NULL;
 666         r = ioctl(hdev->control, VHOST_SET_LOG_BASE,
 667                   (uint64_t)(unsigned long)hdev->log);
 668         if (r < 0) {
 669             r = -errno;
 670             goto fail;
 671         }
 672     }
 673
 674     for (i = 0; i < hdev->nvqs; ++i) {
 675         r = vhost_virtqueue_init(hdev,
 676                                  vdev,
 677                                  hdev->vqs + i,
 678                                  i);
 679         if (r < 0) {
 680             goto fail_vq;
 681         }
 682     }
 683     hdev->started = true;
 684
 685     return 0;
 686 fail_vq:
 687     while (--i >= 0) {
 688         vhost_virtqueue_cleanup(hdev,
 689                                 vdev,
 690                                 hdev->vqs + i,
 691                                 i);
 692     }
 693 fail:
 694     return r;
 695 }
 696
 697 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
 698 {
 699     int i;
 700     for (i = 0; i < hdev->nvqs; ++i) {
 701         vhost_virtqueue_cleanup(hdev,
 702                                 vdev,
 703                                 hdev->vqs + i,
 704                                 i);
 705     }
 706     vhost_client_sync_dirty_bitmap(&hdev->client, 0,
 707                                    (target_phys_addr_t)~0x0ull);
 708     hdev->started = false;
 709     qemu_free(hdev->log);
 710     hdev->log_size = 0;
 711 }