#include "qemu/error-report.h"
#include "qemu/memfd.h"
#include "standard-headers/linux/vhost_types.h"
-#include "exec/address-spaces.h"
#include "hw/virtio/virtio-bus.h"
#include "hw/virtio/virtio-access.h"
#include "migration/blocker.h"
#define _VHOST_DEBUG 1
#ifdef _VHOST_DEBUG
-#define VHOST_OPS_DEBUG(fmt, ...) \
- do { error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
- strerror(errno), errno); } while (0)
+#define VHOST_OPS_DEBUG(retval, fmt, ...) \
+ do { \
+ error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
+ strerror(-retval), -retval); \
+ } while (0)
#else
-#define VHOST_OPS_DEBUG(fmt, ...) \
+#define VHOST_OPS_DEBUG(retval, fmt, ...) \
do { } while (0)
#endif
}
/* Data must be read atomically. We don't really need barrier semantics
* but it's easier to use atomic_* than roll our own. */
- log = atomic_xchg(from, 0);
+ log = qatomic_xchg(from, 0);
while (log) {
int bit = ctzl(log);
hwaddr page_addr;
reg->memory_size);
log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
}
- for (i = 0; i < dev->nvqs; ++i) {
- struct vhost_virtqueue *vq = dev->vqs + i;
+ return log_size;
+}
- if (!vq->used_phys && !vq->used_size) {
- continue;
- }
+static int vhost_set_backend_type(struct vhost_dev *dev,
+ VhostBackendType backend_type)
+{
+ int r = 0;
- uint64_t last = vq->used_phys + vq->used_size - 1;
- log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
+ switch (backend_type) {
+#ifdef CONFIG_VHOST_KERNEL
+ case VHOST_BACKEND_TYPE_KERNEL:
+ dev->vhost_ops = &kernel_ops;
+ break;
+#endif
+#ifdef CONFIG_VHOST_USER
+ case VHOST_BACKEND_TYPE_USER:
+ dev->vhost_ops = &user_ops;
+ break;
+#endif
+#ifdef CONFIG_VHOST_VDPA
+ case VHOST_BACKEND_TYPE_VDPA:
+ dev->vhost_ops = &vdpa_ops;
+ break;
+#endif
+ default:
+ error_report("Unknown vhost backend type");
+ r = -1;
}
- return log_size;
+
+ return r;
}
static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
releasing the current log, to ensure no logging is lost */
r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_log_base failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
}
vhost_log_put(dev, true);
* For vhost, VIRTIO_F_IOMMU_PLATFORM means the backend support
* incremental memory mapping API via IOTLB API. For platform that
* does not have IOMMU, there's no need to enable this feature
- * which may cause unnecessary IOTLB miss/update trnasactions.
+ * which may cause unnecessary IOTLB miss/update transactions.
*/
- return vdev->dma_as != &address_space_memory &&
+ return virtio_bus_device_iommu_enabled(vdev) &&
virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM);
}
if (!dev->log_enabled) {
r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_mem_table failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
}
goto out;
}
}
r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_mem_table failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
}
/* To log less, can only decrease log size after table update. */
if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
iommu_listener);
struct vhost_iommu *iommu;
Int128 end;
- int iommu_idx, ret;
+ int iommu_idx;
IOMMUMemoryRegion *iommu_mr;
- Error *err = NULL;
+ int ret;
if (!memory_region_is_iommu(section->mr)) {
return;
iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
MEMTXATTRS_UNSPECIFIED);
iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
- IOMMU_NOTIFIER_UNMAP,
+ IOMMU_NOTIFIER_DEVIOTLB_UNMAP,
section->offset_within_region,
int128_get64(end),
iommu_idx);
iommu->iommu_offset = section->offset_within_address_space -
section->offset_within_region;
iommu->hdev = dev;
- ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, &err);
+ ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
if (ret) {
- error_report_err(err);
- exit(1);
+ /*
+ * Some vIOMMUs do not support dev-iotlb yet. If so, try to use the
+ * UNMAP legacy message
+ */
+ iommu->n.notifier_flags = IOMMU_NOTIFIER_UNMAP;
+ memory_region_register_iommu_notifier(section->mr, &iommu->n,
+ &error_fatal);
}
QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next);
/* TODO: can replay help performance here? */
if (dev->vhost_ops->vhost_vq_get_addr) {
r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_vq_get_addr failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_vq_get_addr failed");
+ return r;
}
} else {
addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc;
addr.flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0;
r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_addr failed");
}
- return 0;
+ return r;
}
static int vhost_dev_set_features(struct vhost_dev *dev,
}
r = dev->vhost_ops->vhost_set_features(dev, features);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_features failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_features failed");
+ goto out;
+ }
+ if (dev->vhost_ops->vhost_set_backend_cap) {
+ r = dev->vhost_ops->vhost_set_backend_cap(dev);
+ if (r < 0) {
+ VHOST_OPS_DEBUG(r, "vhost_set_backend_cap failed");
+ goto out;
+ }
}
- return r < 0 ? -errno : 0;
+
+out:
+ return r;
}
static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
{
int r, i, idx;
+ hwaddr addr;
+
r = vhost_dev_set_features(dev, enable_log);
if (r < 0) {
goto err_features;
}
for (i = 0; i < dev->nvqs; ++i) {
idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+ addr = virtio_queue_get_desc_addr(dev->vdev, idx);
+ if (!addr) {
+ /*
+ * The queue might not be ready for start. If this
+ * is the case there is no reason to continue the process.
+ * The similar logic is used by the vhost_virtqueue_start()
+ * routine.
+ */
+ continue;
+ }
r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
enable_log);
if (r < 0) {
dev->log_enabled = enable;
return 0;
}
+
+ r = 0;
if (!enable) {
r = vhost_dev_set_log(dev, false);
if (r < 0) {
- return r;
+ goto check_dev_state;
}
vhost_log_put(dev, false);
} else {
vhost_dev_log_resize(dev, vhost_get_log_size(dev));
r = vhost_dev_set_log(dev, true);
if (r < 0) {
- return r;
+ goto check_dev_state;
}
}
+
+check_dev_state:
dev->log_enabled = enable;
- return 0;
+ /*
+ * vhost-user-* devices could change their state during log
+ * initialization due to disconnect. So check dev state after
+ * vhost communication.
+ */
+ if (!dev->started) {
+ /*
+ * Since device is in the stopped state, it is okay for
+ * migration. Return success.
+ */
+ r = 0;
+ }
+ if (r) {
+ /* An error occurred. */
+ dev->log_enabled = false;
+ }
+
+ return r;
}
static void vhost_log_global_start(MemoryListener *listener)
if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
return false;
}
-#ifdef HOST_WORDS_BIGENDIAN
+#if HOST_BIG_ENDIAN
return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
#else
return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
bool is_big_endian,
int vhost_vq_index)
{
+ int r;
struct vhost_vring_state s = {
.index = vhost_vq_index,
.num = is_big_endian
};
- if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
- return 0;
- }
-
- VHOST_OPS_DEBUG("vhost_set_vring_endian failed");
- if (errno == ENOTTY) {
- error_report("vhost does not support cross-endian");
- return -ENOSYS;
+ r = dev->vhost_ops->vhost_set_vring_endian(dev, &s);
+ if (r < 0) {
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_endian failed");
}
-
- return -errno;
+ return r;
}
static int vhost_memory_region_lookup(struct vhost_dev *hdev,
vq->num = state.num = virtio_queue_get_num(vdev, idx);
r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_vring_num failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_num failed");
+ return r;
}
state.num = virtio_queue_get_last_avail_idx(vdev, idx);
r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_vring_base failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_base failed");
+ return r;
}
if (vhost_needs_vring_endian(vdev)) {
virtio_is_big_endian(vdev),
vhost_vq_index);
if (r) {
- return -errno;
+ return r;
}
}
r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
if (r < 0) {
- r = -errno;
goto fail_alloc;
}
file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_vring_kick failed");
- r = -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_kick failed");
goto fail_kick;
}
r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost VQ %u ring restore failed: %d", idx, r);
+ VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
/* Connection to the backend is broken, so let's sync internal
* last avail idx to the device used idx.
*/
r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_vring_busyloop_timeout failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_busyloop_timeout failed");
return r;
}
return r;
}
- file.fd = event_notifier_get_fd(&vq->masked_notifier);
+ file.fd = event_notifier_get_wfd(&vq->masked_notifier);
r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_vring_call failed");
- r = -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_call failed");
goto fail_call;
}
}
int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
- VhostBackendType backend_type, uint32_t busyloop_timeout)
+ VhostBackendType backend_type, uint32_t busyloop_timeout,
+ Error **errp)
{
uint64_t features;
int i, r, n_initialized_vqs = 0;
- Error *local_err = NULL;
hdev->vdev = NULL;
hdev->migration_blocker = NULL;
r = vhost_set_backend_type(hdev, backend_type);
assert(r >= 0);
- r = hdev->vhost_ops->vhost_backend_init(hdev, opaque);
+ r = hdev->vhost_ops->vhost_backend_init(hdev, opaque, errp);
if (r < 0) {
goto fail;
}
r = hdev->vhost_ops->vhost_set_owner(hdev);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_owner failed");
+ error_setg_errno(errp, -r, "vhost_set_owner failed");
goto fail;
}
r = hdev->vhost_ops->vhost_get_features(hdev, &features);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_get_features failed");
+ error_setg_errno(errp, -r, "vhost_get_features failed");
goto fail;
}
for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
if (r < 0) {
+ error_setg_errno(errp, -r, "Failed to initialize virtqueue %d", i);
goto fail;
}
}
r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i,
busyloop_timeout);
if (r < 0) {
+ error_setg_errno(errp, -r, "Failed to set busyloop timeout");
goto fail_busyloop;
}
}
hdev->features = features;
hdev->memory_listener = (MemoryListener) {
+ .name = "vhost",
.begin = vhost_begin,
.commit = vhost_commit,
.region_add = vhost_region_addnop,
};
hdev->iommu_listener = (MemoryListener) {
+ .name = "vhost-iommu",
.region_add = vhost_iommu_region_add,
.region_del = vhost_iommu_region_del,
};
}
if (hdev->migration_blocker != NULL) {
- r = migrate_add_blocker(hdev->migration_blocker, &local_err);
- if (local_err) {
- error_report_err(local_err);
+ r = migrate_add_blocker(hdev->migration_blocker, errp);
+ if (r < 0) {
error_free(hdev->migration_blocker);
goto fail_busyloop;
}
QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
- error_report("vhost backend memory slots limit is less"
- " than current number of present memory slots");
- r = -1;
- if (busyloop_timeout) {
- goto fail_busyloop;
- } else {
- goto fail;
- }
+ error_setg(errp, "vhost backend memory slots limit is less"
+ " than current number of present memory slots");
+ r = -EINVAL;
+ goto fail_busyloop;
}
return 0;
fail_busyloop:
- while (--i >= 0) {
- vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
+ if (busyloop_timeout) {
+ while (--i >= 0) {
+ vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
+ }
}
fail:
hdev->nvqs = n_initialized_vqs;
if (mask) {
assert(vdev->use_guest_notifier_mask);
- file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
+ file.fd = event_notifier_get_wfd(&hdev->vqs[index].masked_notifier);
} else {
- file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
+ file.fd = event_notifier_get_wfd(virtio_queue_get_guest_notifier(vvq));
}
file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_vring_call failed");
+ VHOST_OPS_DEBUG(r, "vhost_set_vring_call failed");
}
}
}
int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
- uint32_t config_len)
+ uint32_t config_len, Error **errp)
{
assert(hdev->vhost_ops);
if (hdev->vhost_ops->vhost_get_config) {
- return hdev->vhost_ops->vhost_get_config(hdev, config, config_len);
+ return hdev->vhost_ops->vhost_get_config(hdev, config, config_len,
+ errp);
}
- return -1;
+ error_setg(errp, "vhost_get_config not implemented");
+ return -ENOSYS;
}
int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
size, flags);
}
- return -1;
+ return -ENOSYS;
}
void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
if (err) {
error_report_err(err);
- return -1;
+ return -ENOMEM;
}
vhost_dev_free_inflight(inflight);
}
if (inflight->size != size) {
- if (vhost_dev_resize_inflight(inflight, size)) {
- return -1;
+ int ret = vhost_dev_resize_inflight(inflight, size);
+ if (ret < 0) {
+ return ret;
}
}
inflight->queue_size = qemu_get_be16(f);
return 0;
}
+int vhost_dev_prepare_inflight(struct vhost_dev *hdev, VirtIODevice *vdev)
+{
+ int r;
+
+ if (hdev->vhost_ops->vhost_get_inflight_fd == NULL ||
+ hdev->vhost_ops->vhost_set_inflight_fd == NULL) {
+ return 0;
+ }
+
+ hdev->vdev = vdev;
+
+ r = vhost_dev_set_features(hdev, hdev->log_enabled);
+ if (r < 0) {
+ VHOST_OPS_DEBUG(r, "vhost_dev_prepare_inflight failed");
+ return r;
+ }
+
+ return 0;
+}
+
int vhost_dev_set_inflight(struct vhost_dev *dev,
struct vhost_inflight *inflight)
{
if (dev->vhost_ops->vhost_set_inflight_fd && inflight->addr) {
r = dev->vhost_ops->vhost_set_inflight_fd(dev, inflight);
if (r) {
- VHOST_OPS_DEBUG("vhost_set_inflight_fd failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_inflight_fd failed");
+ return r;
}
}
if (dev->vhost_ops->vhost_get_inflight_fd) {
r = dev->vhost_ops->vhost_get_inflight_fd(dev, queue_size, inflight);
if (r) {
- VHOST_OPS_DEBUG("vhost_get_inflight_fd failed");
- return -errno;
+ VHOST_OPS_DEBUG(r, "vhost_get_inflight_fd failed");
+ return r;
}
}
r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_mem_table failed");
- r = -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
goto fail_mem;
}
for (i = 0; i < hdev->nvqs; ++i) {
hdev->log_size ? log_base : 0,
hdev->log);
if (r < 0) {
- VHOST_OPS_DEBUG("vhost_set_log_base failed");
- r = -errno;
+ VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
goto fail_log;
}
}
return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
}
- return -1;
+ return -ENOSYS;
}
projects / mirror_qemu.git / blobdiff