hw/virtio/vhost-vdpa.c

   1 /*
   2  * vhost-vdpa
   3  *
   4  *  Copyright(c) 2017-2018 Intel Corporation.
   5  *  Copyright(c) 2020 Red Hat, Inc.
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   8  * See the COPYING file in the top-level directory.
   9  *
  10  */
  11
  12 #include "qemu/osdep.h"
  13 #include <linux/vhost.h>
  14 #include <linux/vfio.h>
  15 #include <sys/eventfd.h>
  16 #include <sys/ioctl.h>
  17 #include "hw/virtio/vhost.h"
  18 #include "hw/virtio/vhost-backend.h"
  19 #include "hw/virtio/virtio-net.h"
  20 #include "hw/virtio/vhost-vdpa.h"
  21 #include "qemu/main-loop.h"
  22 #include "cpu.h"
  23
  24 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
  25 {
  26     return (!memory_region_is_ram(section->mr) &&
  27             !memory_region_is_iommu(section->mr)) ||
  28            /*
  29             * Sizing an enabled 64-bit BAR can cause spurious mappings to
  30             * addresses in the upper part of the 64-bit address space.  These
  31             * are never accessed by the CPU and beyond the address width of
  32             * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU width.
  33             */
  34            section->offset_within_address_space & (1ULL << 63);
  35 }
  36
  37 static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
  38                               void *vaddr, bool readonly)
  39 {
  40     struct vhost_msg_v2 msg = {};
  41     int fd = v->device_fd;
  42     int ret = 0;
  43
  44     msg.type = v->msg_type;
  45     msg.iotlb.iova = iova;
  46     msg.iotlb.size = size;
  47     msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
  48     msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
  49     msg.iotlb.type = VHOST_IOTLB_UPDATE;
  50
  51     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
  52         error_report("failed to write, fd=%d, errno=%d (%s)",
  53             fd, errno, strerror(errno));
  54         return -EIO ;
  55     }
  56
  57     return ret;
  58 }
  59
  60 static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
  61                                 hwaddr size)
  62 {
  63     struct vhost_msg_v2 msg = {};
  64     int fd = v->device_fd;
  65     int ret = 0;
  66
  67     msg.type = v->msg_type;
  68     msg.iotlb.iova = iova;
  69     msg.iotlb.size = size;
  70     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
  71
  72     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
  73         error_report("failed to write, fd=%d, errno=%d (%s)",
  74             fd, errno, strerror(errno));
  75         return -EIO ;
  76     }
  77
  78     return ret;
  79 }
  80
  81 static void vhost_vdpa_listener_region_add(MemoryListener *listener,
  82                                            MemoryRegionSection *section)
  83 {
  84     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
  85     hwaddr iova;
  86     Int128 llend, llsize;
  87     void *vaddr;
  88     int ret;
  89
  90     if (vhost_vdpa_listener_skipped_section(section)) {
  91         return;
  92     }
  93
  94     if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
  95                  (section->offset_within_region & ~TARGET_PAGE_MASK))) {
  96         error_report("%s received unaligned region", __func__);
  97         return;
  98     }
  99
 100     iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
 101     llend = int128_make64(section->offset_within_address_space);
 102     llend = int128_add(llend, section->size);
 103     llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
 104
 105     if (int128_ge(int128_make64(iova), llend)) {
 106         return;
 107     }
 108
 109     memory_region_ref(section->mr);
 110
 111     /* Here we assume that memory_region_is_ram(section->mr)==true */
 112
 113     vaddr = memory_region_get_ram_ptr(section->mr) +
 114             section->offset_within_region +
 115             (iova - section->offset_within_address_space);
 116
 117     llsize = int128_sub(llend, int128_make64(iova));
 118
 119     ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
 120                              vaddr, section->readonly);
 121     if (ret) {
 122         error_report("vhost vdpa map fail!");
 123         if (memory_region_is_ram_device(section->mr)) {
 124             /* Allow unexpected mappings not to be fatal for RAM devices */
 125             error_report("map ram fail!");
 126           return ;
 127         }
 128         goto fail;
 129     }
 130
 131     return;
 132
 133 fail:
 134     if (memory_region_is_ram_device(section->mr)) {
 135         error_report("failed to vdpa_dma_map. pci p2p may not work");
 136         return;
 137
 138     }
 139     /*
 140      * On the initfn path, store the first error in the container so we
 141      * can gracefully fail.  Runtime, there's not much we can do other
 142      * than throw a hardware error.
 143      */
 144     error_report("vhost-vdpa: DMA mapping failed, unable to continue");
 145     return;
 146
 147 }
 148
 149 static void vhost_vdpa_listener_region_del(MemoryListener *listener,
 150                                            MemoryRegionSection *section)
 151 {
 152     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
 153     hwaddr iova;
 154     Int128 llend, llsize;
 155     int ret;
 156     bool try_unmap = true;
 157
 158     if (vhost_vdpa_listener_skipped_section(section)) {
 159         return;
 160     }
 161
 162     if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
 163                  (section->offset_within_region & ~TARGET_PAGE_MASK))) {
 164         error_report("%s received unaligned region", __func__);
 165         return;
 166     }
 167
 168     iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
 169     llend = int128_make64(section->offset_within_address_space);
 170     llend = int128_add(llend, section->size);
 171     llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
 172
 173     if (int128_ge(int128_make64(iova), llend)) {
 174         return;
 175     }
 176
 177     llsize = int128_sub(llend, int128_make64(iova));
 178
 179     if (try_unmap) {
 180         ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
 181         if (ret) {
 182             error_report("vhost_vdpa dma unmap error!");
 183         }
 184     }
 185
 186     memory_region_unref(section->mr);
 187 }
 188 /*
 189  * IOTLB API is used by vhost-vpda which requires incremental updating
 190  * of the mapping. So we can not use generic vhost memory listener which
 191  * depends on the addnop().
 192  */
 193 static const MemoryListener vhost_vdpa_memory_listener = {
 194     .region_add = vhost_vdpa_listener_region_add,
 195     .region_del = vhost_vdpa_listener_region_del,
 196 };
 197
 198 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
 199                              void *arg)
 200 {
 201     struct vhost_vdpa *v = dev->opaque;
 202     int fd = v->device_fd;
 203
 204     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
 205
 206     return ioctl(fd, request, arg);
 207 }
 208
 209 static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
 210 {
 211     uint8_t s;
 212
 213     if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) {
 214         return;
 215     }
 216
 217     s |= status;
 218
 219     vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
 220 }
 221
 222 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
 223 {
 224     struct vhost_vdpa *v;
 225     uint64_t features;
 226     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
 227
 228     v = opaque;
 229     dev->opaque =  opaque ;
 230     vhost_vdpa_call(dev, VHOST_GET_FEATURES, &features);
 231     dev->backend_features = features;
 232     v->listener = vhost_vdpa_memory_listener;
 233     v->msg_type = VHOST_IOTLB_MSG_V2;
 234
 235     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
 236                                VIRTIO_CONFIG_S_DRIVER);
 237
 238     return 0;
 239 }
 240
 241 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
 242 {
 243     struct vhost_vdpa *v;
 244     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
 245     v = dev->opaque;
 246     memory_listener_unregister(&v->listener);
 247
 248     dev->opaque = NULL;
 249     return 0;
 250 }
 251
 252 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
 253 {
 254     return INT_MAX;
 255 }
 256
 257 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
 258                                     struct vhost_memory *mem)
 259 {
 260
 261     if (mem->padding) {
 262         return -1;
 263     }
 264
 265     return 0;
 266 }
 267
 268 static int vhost_vdpa_set_features(struct vhost_dev *dev,
 269                                    uint64_t features)
 270 {
 271     int ret;
 272     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
 273     uint8_t status = 0;
 274     if (ret) {
 275         return ret;
 276     }
 277     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
 278     vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
 279
 280     return !(status & VIRTIO_CONFIG_S_FEATURES_OK);
 281 }
 282
 283 int vhost_vdpa_get_device_id(struct vhost_dev *dev,
 284                                    uint32_t *device_id)
 285 {
 286     return vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
 287 }
 288
 289 static int vhost_vdpa_reset_device(struct vhost_dev *dev)
 290 {
 291     uint8_t status = 0;
 292
 293     return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
 294 }
 295
 296 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
 297 {
 298     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 299
 300     return idx - dev->vq_index;
 301 }
 302
 303 static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
 304 {
 305     int i;
 306     for (i = 0; i < dev->nvqs; ++i) {
 307         struct vhost_vring_state state = {
 308             .index = dev->vq_index + i,
 309             .num = 1,
 310         };
 311         vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
 312     }
 313     return 0;
 314 }
 315
 316 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
 317                                    uint32_t offset, uint32_t size,
 318                                    uint32_t flags)
 319 {
 320     struct vhost_vdpa_config *config;
 321     int ret;
 322     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
 323     config = g_malloc(size + config_size);
 324     if (config == NULL) {
 325         return -1;
 326     }
 327     config->off = offset;
 328     config->len = size;
 329     memcpy(config->buf, data, size);
 330     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
 331     g_free(config);
 332     return ret;
 333 }
 334
 335 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
 336                                    uint32_t config_len)
 337 {
 338     struct vhost_vdpa_config *v_config;
 339     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
 340     int ret;
 341
 342     v_config = g_malloc(config_len + config_size);
 343     if (v_config == NULL) {
 344         return -1;
 345     }
 346     v_config->len = config_len;
 347     v_config->off = 0;
 348     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
 349     memcpy(config, v_config->buf, config_len);
 350     g_free(v_config);
 351     return ret;
 352  }
 353
 354 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
 355 {
 356     struct vhost_vdpa *v = dev->opaque;
 357     if (started) {
 358         uint8_t status = 0;
 359         memory_listener_register(&v->listener, &address_space_memory);
 360         vhost_vdpa_set_vring_ready(dev);
 361         vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
 362         vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
 363
 364         return !(status & VIRTIO_CONFIG_S_DRIVER_OK);
 365     } else {
 366         vhost_vdpa_reset_device(dev);
 367         vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
 368                                    VIRTIO_CONFIG_S_DRIVER);
 369         memory_listener_unregister(&v->listener);
 370
 371         return 0;
 372     }
 373 }
 374
 375 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
 376                                      struct vhost_log *log)
 377 {
 378     return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
 379 }
 380
 381 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
 382                                        struct vhost_vring_addr *addr)
 383 {
 384     return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
 385 }
 386
 387 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
 388                                       struct vhost_vring_state *ring)
 389 {
 390     return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
 391 }
 392
 393 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
 394                                        struct vhost_vring_state *ring)
 395 {
 396     return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
 397 }
 398
 399 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
 400                                        struct vhost_vring_state *ring)
 401 {
 402     return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
 403 }
 404
 405 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
 406                                        struct vhost_vring_file *file)
 407 {
 408     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
 409 }
 410
 411 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
 412                                        struct vhost_vring_file *file)
 413 {
 414     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
 415 }
 416
 417 static int vhost_vdpa_get_features(struct vhost_dev *dev,
 418                                      uint64_t *features)
 419 {
 420     return vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
 421 }
 422
 423 static int vhost_vdpa_set_owner(struct vhost_dev *dev)
 424 {
 425     return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
 426 }
 427
 428 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
 429                     struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
 430 {
 431     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
 432     addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
 433     addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
 434     addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
 435     return 0;
 436 }
 437
 438 static bool  vhost_vdpa_force_iommu(struct vhost_dev *dev)
 439 {
 440     return true;
 441 }
 442
 443 const VhostOps vdpa_ops = {
 444         .backend_type = VHOST_BACKEND_TYPE_VDPA,
 445         .vhost_backend_init = vhost_vdpa_init,
 446         .vhost_backend_cleanup = vhost_vdpa_cleanup,
 447         .vhost_set_log_base = vhost_vdpa_set_log_base,
 448         .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
 449         .vhost_set_vring_num = vhost_vdpa_set_vring_num,
 450         .vhost_set_vring_base = vhost_vdpa_set_vring_base,
 451         .vhost_get_vring_base = vhost_vdpa_get_vring_base,
 452         .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
 453         .vhost_set_vring_call = vhost_vdpa_set_vring_call,
 454         .vhost_get_features = vhost_vdpa_get_features,
 455         .vhost_set_owner = vhost_vdpa_set_owner,
 456         .vhost_set_vring_endian = NULL,
 457         .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
 458         .vhost_set_mem_table = vhost_vdpa_set_mem_table,
 459         .vhost_set_features = vhost_vdpa_set_features,
 460         .vhost_reset_device = vhost_vdpa_reset_device,
 461         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
 462         .vhost_get_config  = vhost_vdpa_get_config,
 463         .vhost_set_config = vhost_vdpa_set_config,
 464         .vhost_requires_shm_log = NULL,
 465         .vhost_migration_done = NULL,
 466         .vhost_backend_can_merge = NULL,
 467         .vhost_net_set_mtu = NULL,
 468         .vhost_set_iotlb_callback = NULL,
 469         .vhost_send_device_iotlb_msg = NULL,
 470         .vhost_dev_start = vhost_vdpa_dev_start,
 471         .vhost_get_device_id = vhost_vdpa_get_device_id,
 472          .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
 473          .vhost_force_iommu = vhost_vdpa_force_iommu,
 474 };