4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
13 #include <linux/vhost.h>
14 #include <linux/vfio.h>
15 #include <sys/eventfd.h>
16 #include <sys/ioctl.h>
17 #include "exec/target_page.h"
18 #include "hw/virtio/vhost.h"
19 #include "hw/virtio/vhost-backend.h"
20 #include "hw/virtio/virtio-net.h"
21 #include "hw/virtio/vhost-shadow-virtqueue.h"
22 #include "hw/virtio/vhost-vdpa.h"
23 #include "exec/address-spaces.h"
24 #include "migration/blocker.h"
25 #include "qemu/cutils.h"
26 #include "qemu/main-loop.h"
28 #include "qapi/error.h"
31 * Return one past the end of the end of section. Be careful with uint64_t
34 static Int128
vhost_vdpa_section_end(const MemoryRegionSection
*section
,
37 Int128 llend
= int128_make64(section
->offset_within_address_space
);
38 llend
= int128_add(llend
, section
->size
);
39 llend
= int128_and(llend
, int128_exts64(page_mask
));
44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection
*section
,
50 bool is_ram
= memory_region_is_ram(section
->mr
);
51 bool is_iommu
= memory_region_is_iommu(section
->mr
);
52 bool is_protected
= memory_region_is_protected(section
->mr
);
54 /* vhost-vDPA doesn't allow MMIO to be mapped */
55 bool is_ram_device
= memory_region_is_ram_device(section
->mr
);
57 if ((!is_ram
&& !is_iommu
) || is_protected
|| is_ram_device
) {
61 if (section
->offset_within_address_space
< iova_min
) {
62 error_report("RAM section out of device range (min=0x%" PRIx64
63 ", addr=0x%" HWADDR_PRIx
")",
64 iova_min
, section
->offset_within_address_space
);
68 * While using vIOMMU, sometimes the section will be larger than iova_max,
69 * but the memory that actually maps is smaller, so move the check to
70 * function vhost_vdpa_iommu_map_notify(). That function will use the actual
71 * size that maps to the kernel
75 llend
= vhost_vdpa_section_end(section
, page_mask
);
76 if (int128_gt(llend
, int128_make64(iova_max
))) {
77 error_report("RAM section out of device range (max=0x%" PRIx64
78 ", end addr=0x%" PRIx64
")",
79 iova_max
, int128_get64(llend
));
88 * The caller must set asid = 0 if the device does not support asid.
89 * This is not an ABI break since it is set to 0 by the initializer anyway.
91 int vhost_vdpa_dma_map(VhostVDPAShared
*s
, uint32_t asid
, hwaddr iova
,
92 hwaddr size
, void *vaddr
, bool readonly
)
94 struct vhost_msg_v2 msg
= {};
95 int fd
= s
->device_fd
;
98 msg
.type
= VHOST_IOTLB_MSG_V2
;
100 msg
.iotlb
.iova
= iova
;
101 msg
.iotlb
.size
= size
;
102 msg
.iotlb
.uaddr
= (uint64_t)(uintptr_t)vaddr
;
103 msg
.iotlb
.perm
= readonly
? VHOST_ACCESS_RO
: VHOST_ACCESS_RW
;
104 msg
.iotlb
.type
= VHOST_IOTLB_UPDATE
;
106 trace_vhost_vdpa_dma_map(s
, fd
, msg
.type
, msg
.asid
, msg
.iotlb
.iova
,
107 msg
.iotlb
.size
, msg
.iotlb
.uaddr
, msg
.iotlb
.perm
,
110 if (write(fd
, &msg
, sizeof(msg
)) != sizeof(msg
)) {
111 error_report("failed to write, fd=%d, errno=%d (%s)",
112 fd
, errno
, strerror(errno
));
120 * The caller must set asid = 0 if the device does not support asid.
121 * This is not an ABI break since it is set to 0 by the initializer anyway.
123 int vhost_vdpa_dma_unmap(VhostVDPAShared
*s
, uint32_t asid
, hwaddr iova
,
126 struct vhost_msg_v2 msg
= {};
127 int fd
= s
->device_fd
;
130 msg
.type
= VHOST_IOTLB_MSG_V2
;
132 msg
.iotlb
.iova
= iova
;
133 msg
.iotlb
.size
= size
;
134 msg
.iotlb
.type
= VHOST_IOTLB_INVALIDATE
;
136 trace_vhost_vdpa_dma_unmap(s
, fd
, msg
.type
, msg
.asid
, msg
.iotlb
.iova
,
137 msg
.iotlb
.size
, msg
.iotlb
.type
);
139 if (write(fd
, &msg
, sizeof(msg
)) != sizeof(msg
)) {
140 error_report("failed to write, fd=%d, errno=%d (%s)",
141 fd
, errno
, strerror(errno
));
148 static void vhost_vdpa_listener_begin_batch(VhostVDPAShared
*s
)
150 int fd
= s
->device_fd
;
151 struct vhost_msg_v2 msg
= {
152 .type
= VHOST_IOTLB_MSG_V2
,
153 .iotlb
.type
= VHOST_IOTLB_BATCH_BEGIN
,
156 trace_vhost_vdpa_listener_begin_batch(s
, fd
, msg
.type
, msg
.iotlb
.type
);
157 if (write(fd
, &msg
, sizeof(msg
)) != sizeof(msg
)) {
158 error_report("failed to write, fd=%d, errno=%d (%s)",
159 fd
, errno
, strerror(errno
));
163 static void vhost_vdpa_iotlb_batch_begin_once(VhostVDPAShared
*s
)
165 if (s
->backend_cap
& (0x1ULL
<< VHOST_BACKEND_F_IOTLB_BATCH
) &&
166 !s
->iotlb_batch_begin_sent
) {
167 vhost_vdpa_listener_begin_batch(s
);
170 s
->iotlb_batch_begin_sent
= true;
173 static void vhost_vdpa_listener_commit(MemoryListener
*listener
)
175 VhostVDPAShared
*s
= container_of(listener
, VhostVDPAShared
, listener
);
176 struct vhost_msg_v2 msg
= {};
177 int fd
= s
->device_fd
;
179 if (!(s
->backend_cap
& (0x1ULL
<< VHOST_BACKEND_F_IOTLB_BATCH
))) {
183 if (!s
->iotlb_batch_begin_sent
) {
187 msg
.type
= VHOST_IOTLB_MSG_V2
;
188 msg
.iotlb
.type
= VHOST_IOTLB_BATCH_END
;
190 trace_vhost_vdpa_listener_commit(s
, fd
, msg
.type
, msg
.iotlb
.type
);
191 if (write(fd
, &msg
, sizeof(msg
)) != sizeof(msg
)) {
192 error_report("failed to write, fd=%d, errno=%d (%s)",
193 fd
, errno
, strerror(errno
));
196 s
->iotlb_batch_begin_sent
= false;
199 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier
*n
, IOMMUTLBEntry
*iotlb
)
201 struct vdpa_iommu
*iommu
= container_of(n
, struct vdpa_iommu
, n
);
203 hwaddr iova
= iotlb
->iova
+ iommu
->iommu_offset
;
204 VhostVDPAShared
*s
= iommu
->dev_shared
;
209 if (iotlb
->target_as
!= &address_space_memory
) {
210 error_report("Wrong target AS \"%s\", only system memory is allowed",
211 iotlb
->target_as
->name
? iotlb
->target_as
->name
: "none");
214 RCU_READ_LOCK_GUARD();
215 /* check if RAM section out of device range */
216 llend
= int128_add(int128_makes64(iotlb
->addr_mask
), int128_makes64(iova
));
217 if (int128_gt(llend
, int128_make64(s
->iova_range
.last
))) {
218 error_report("RAM section out of device range (max=0x%" PRIx64
219 ", end addr=0x%" PRIx64
")",
220 s
->iova_range
.last
, int128_get64(llend
));
224 if ((iotlb
->perm
& IOMMU_RW
) != IOMMU_NONE
) {
227 if (!memory_get_xlat_addr(iotlb
, &vaddr
, NULL
, &read_only
, NULL
)) {
230 ret
= vhost_vdpa_dma_map(s
, VHOST_VDPA_GUEST_PA_ASID
, iova
,
231 iotlb
->addr_mask
+ 1, vaddr
, read_only
);
233 error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx
", "
234 "0x%" HWADDR_PRIx
", %p) = %d (%m)",
235 s
, iova
, iotlb
->addr_mask
+ 1, vaddr
, ret
);
238 ret
= vhost_vdpa_dma_unmap(s
, VHOST_VDPA_GUEST_PA_ASID
, iova
,
239 iotlb
->addr_mask
+ 1);
241 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx
", "
242 "0x%" HWADDR_PRIx
") = %d (%m)",
243 s
, iova
, iotlb
->addr_mask
+ 1, ret
);
248 static void vhost_vdpa_iommu_region_add(MemoryListener
*listener
,
249 MemoryRegionSection
*section
)
251 VhostVDPAShared
*s
= container_of(listener
, VhostVDPAShared
, listener
);
253 struct vdpa_iommu
*iommu
;
256 IOMMUMemoryRegion
*iommu_mr
;
259 iommu_mr
= IOMMU_MEMORY_REGION(section
->mr
);
261 iommu
= g_malloc0(sizeof(*iommu
));
262 end
= int128_add(int128_make64(section
->offset_within_region
),
264 end
= int128_sub(end
, int128_one());
265 iommu_idx
= memory_region_iommu_attrs_to_index(iommu_mr
,
266 MEMTXATTRS_UNSPECIFIED
);
267 iommu
->iommu_mr
= iommu_mr
;
268 iommu_notifier_init(&iommu
->n
, vhost_vdpa_iommu_map_notify
,
269 IOMMU_NOTIFIER_IOTLB_EVENTS
,
270 section
->offset_within_region
,
273 iommu
->iommu_offset
= section
->offset_within_address_space
-
274 section
->offset_within_region
;
275 iommu
->dev_shared
= s
;
277 ret
= memory_region_register_iommu_notifier(section
->mr
, &iommu
->n
, NULL
);
283 QLIST_INSERT_HEAD(&s
->iommu_list
, iommu
, iommu_next
);
284 memory_region_iommu_replay(iommu
->iommu_mr
, &iommu
->n
);
289 static void vhost_vdpa_iommu_region_del(MemoryListener
*listener
,
290 MemoryRegionSection
*section
)
292 VhostVDPAShared
*s
= container_of(listener
, VhostVDPAShared
, listener
);
294 struct vdpa_iommu
*iommu
;
296 QLIST_FOREACH(iommu
, &s
->iommu_list
, iommu_next
)
298 if (MEMORY_REGION(iommu
->iommu_mr
) == section
->mr
&&
299 iommu
->n
.start
== section
->offset_within_region
) {
300 memory_region_unregister_iommu_notifier(section
->mr
, &iommu
->n
);
301 QLIST_REMOVE(iommu
, iommu_next
);
308 static void vhost_vdpa_listener_region_add(MemoryListener
*listener
,
309 MemoryRegionSection
*section
)
311 DMAMap mem_region
= {};
312 VhostVDPAShared
*s
= container_of(listener
, VhostVDPAShared
, listener
);
314 Int128 llend
, llsize
;
317 int page_size
= qemu_target_page_size();
318 int page_mask
= -page_size
;
320 if (vhost_vdpa_listener_skipped_section(section
, s
->iova_range
.first
,
321 s
->iova_range
.last
, page_mask
)) {
324 if (memory_region_is_iommu(section
->mr
)) {
325 vhost_vdpa_iommu_region_add(listener
, section
);
329 if (unlikely((section
->offset_within_address_space
& ~page_mask
) !=
330 (section
->offset_within_region
& ~page_mask
))) {
331 trace_vhost_vdpa_listener_region_add_unaligned(s
, section
->mr
->name
,
332 section
->offset_within_address_space
& ~page_mask
,
333 section
->offset_within_region
& ~page_mask
);
337 iova
= ROUND_UP(section
->offset_within_address_space
, page_size
);
338 llend
= vhost_vdpa_section_end(section
, page_mask
);
339 if (int128_ge(int128_make64(iova
), llend
)) {
343 memory_region_ref(section
->mr
);
345 /* Here we assume that memory_region_is_ram(section->mr)==true */
347 vaddr
= memory_region_get_ram_ptr(section
->mr
) +
348 section
->offset_within_region
+
349 (iova
- section
->offset_within_address_space
);
351 trace_vhost_vdpa_listener_region_add(s
, iova
, int128_get64(llend
),
352 vaddr
, section
->readonly
);
354 llsize
= int128_sub(llend
, int128_make64(iova
));
355 if (s
->shadow_data
) {
358 mem_region
.translated_addr
= (hwaddr
)(uintptr_t)vaddr
,
359 mem_region
.size
= int128_get64(llsize
) - 1,
360 mem_region
.perm
= IOMMU_ACCESS_FLAG(true, section
->readonly
),
362 r
= vhost_iova_tree_map_alloc(s
->iova_tree
, &mem_region
);
363 if (unlikely(r
!= IOVA_OK
)) {
364 error_report("Can't allocate a mapping (%d)", r
);
368 iova
= mem_region
.iova
;
371 vhost_vdpa_iotlb_batch_begin_once(s
);
372 ret
= vhost_vdpa_dma_map(s
, VHOST_VDPA_GUEST_PA_ASID
, iova
,
373 int128_get64(llsize
), vaddr
, section
->readonly
);
375 error_report("vhost vdpa map fail!");
382 if (s
->shadow_data
) {
383 vhost_iova_tree_remove(s
->iova_tree
, mem_region
);
388 * On the initfn path, store the first error in the container so we
389 * can gracefully fail. Runtime, there's not much we can do other
390 * than throw a hardware error.
392 error_report("vhost-vdpa: DMA mapping failed, unable to continue");
397 static void vhost_vdpa_listener_region_del(MemoryListener
*listener
,
398 MemoryRegionSection
*section
)
400 VhostVDPAShared
*s
= container_of(listener
, VhostVDPAShared
, listener
);
402 Int128 llend
, llsize
;
404 int page_size
= qemu_target_page_size();
405 int page_mask
= -page_size
;
407 if (vhost_vdpa_listener_skipped_section(section
, s
->iova_range
.first
,
408 s
->iova_range
.last
, page_mask
)) {
411 if (memory_region_is_iommu(section
->mr
)) {
412 vhost_vdpa_iommu_region_del(listener
, section
);
415 if (unlikely((section
->offset_within_address_space
& ~page_mask
) !=
416 (section
->offset_within_region
& ~page_mask
))) {
417 trace_vhost_vdpa_listener_region_del_unaligned(s
, section
->mr
->name
,
418 section
->offset_within_address_space
& ~page_mask
,
419 section
->offset_within_region
& ~page_mask
);
423 iova
= ROUND_UP(section
->offset_within_address_space
, page_size
);
424 llend
= vhost_vdpa_section_end(section
, page_mask
);
426 trace_vhost_vdpa_listener_region_del(s
, iova
,
427 int128_get64(int128_sub(llend
, int128_one())));
429 if (int128_ge(int128_make64(iova
), llend
)) {
433 llsize
= int128_sub(llend
, int128_make64(iova
));
435 if (s
->shadow_data
) {
436 const DMAMap
*result
;
437 const void *vaddr
= memory_region_get_ram_ptr(section
->mr
) +
438 section
->offset_within_region
+
439 (iova
- section
->offset_within_address_space
);
440 DMAMap mem_region
= {
441 .translated_addr
= (hwaddr
)(uintptr_t)vaddr
,
442 .size
= int128_get64(llsize
) - 1,
445 result
= vhost_iova_tree_find_iova(s
->iova_tree
, &mem_region
);
447 /* The memory listener map wasn't mapped */
451 vhost_iova_tree_remove(s
->iova_tree
, *result
);
453 vhost_vdpa_iotlb_batch_begin_once(s
);
455 * The unmap ioctl doesn't accept a full 64-bit. need to check it
457 if (int128_eq(llsize
, int128_2_64())) {
458 llsize
= int128_rshift(llsize
, 1);
459 ret
= vhost_vdpa_dma_unmap(s
, VHOST_VDPA_GUEST_PA_ASID
, iova
,
460 int128_get64(llsize
));
463 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx
", "
464 "0x%" HWADDR_PRIx
") = %d (%m)",
465 s
, iova
, int128_get64(llsize
), ret
);
467 iova
+= int128_get64(llsize
);
469 ret
= vhost_vdpa_dma_unmap(s
, VHOST_VDPA_GUEST_PA_ASID
, iova
,
470 int128_get64(llsize
));
473 error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx
", "
474 "0x%" HWADDR_PRIx
") = %d (%m)",
475 s
, iova
, int128_get64(llsize
), ret
);
478 memory_region_unref(section
->mr
);
481 * IOTLB API is used by vhost-vdpa which requires incremental updating
482 * of the mapping. So we can not use generic vhost memory listener which
483 * depends on the addnop().
485 static const MemoryListener vhost_vdpa_memory_listener
= {
486 .name
= "vhost-vdpa",
487 .commit
= vhost_vdpa_listener_commit
,
488 .region_add
= vhost_vdpa_listener_region_add
,
489 .region_del
= vhost_vdpa_listener_region_del
,
492 static int vhost_vdpa_call(struct vhost_dev
*dev
, unsigned long int request
,
495 struct vhost_vdpa
*v
= dev
->opaque
;
496 int fd
= v
->shared
->device_fd
;
499 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_VDPA
);
501 ret
= ioctl(fd
, request
, arg
);
502 return ret
< 0 ? -errno
: ret
;
505 static int vhost_vdpa_add_status(struct vhost_dev
*dev
, uint8_t status
)
510 trace_vhost_vdpa_add_status(dev
, status
);
511 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_GET_STATUS
, &s
);
515 if ((s
& status
) == status
) {
516 /* Don't set bits already set */
522 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_SET_STATUS
, &s
);
527 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_GET_STATUS
, &s
);
539 int vhost_vdpa_get_iova_range(int fd
, struct vhost_vdpa_iova_range
*iova_range
)
541 int ret
= ioctl(fd
, VHOST_VDPA_GET_IOVA_RANGE
, iova_range
);
543 return ret
< 0 ? -errno
: 0;
547 * The use of this function is for requests that only need to be
548 * applied once. Typically such request occurs at the beginning
549 * of operation, and before setting up queues. It should not be
550 * used for request that performs operation until all queues are
551 * set, which would need to check dev->vq_index_end instead.
553 static bool vhost_vdpa_first_dev(struct vhost_dev
*dev
)
555 struct vhost_vdpa
*v
= dev
->opaque
;
557 return v
->index
== 0;
560 static bool vhost_vdpa_last_dev(struct vhost_dev
*dev
)
562 return dev
->vq_index
+ dev
->nvqs
== dev
->vq_index_end
;
565 static int vhost_vdpa_get_dev_features(struct vhost_dev
*dev
,
570 ret
= vhost_vdpa_call(dev
, VHOST_GET_FEATURES
, features
);
571 trace_vhost_vdpa_get_features(dev
, *features
);
575 static void vhost_vdpa_init_svq(struct vhost_dev
*hdev
, struct vhost_vdpa
*v
)
577 g_autoptr(GPtrArray
) shadow_vqs
= NULL
;
579 shadow_vqs
= g_ptr_array_new_full(hdev
->nvqs
, vhost_svq_free
);
580 for (unsigned n
= 0; n
< hdev
->nvqs
; ++n
) {
581 VhostShadowVirtqueue
*svq
;
583 svq
= vhost_svq_new(v
->shadow_vq_ops
, v
->shadow_vq_ops_opaque
);
584 g_ptr_array_add(shadow_vqs
, svq
);
587 v
->shadow_vqs
= g_steal_pointer(&shadow_vqs
);
590 static int vhost_vdpa_init(struct vhost_dev
*dev
, void *opaque
, Error
**errp
)
592 struct vhost_vdpa
*v
= opaque
;
593 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_VDPA
);
594 trace_vhost_vdpa_init(dev
, v
->shared
, opaque
);
598 dev
->opaque
= opaque
;
599 v
->shared
->listener
= vhost_vdpa_memory_listener
;
600 vhost_vdpa_init_svq(dev
, v
);
602 error_propagate(&dev
->migration_blocker
, v
->migration_blocker
);
603 if (!vhost_vdpa_first_dev(dev
)) {
608 * If dev->shadow_vqs_enabled at initialization that means the device has
609 * been started with x-svq=on, so don't block migration
611 if (dev
->migration_blocker
== NULL
&& !v
->shadow_vqs_enabled
) {
612 /* We don't have dev->features yet */
614 ret
= vhost_vdpa_get_dev_features(dev
, &features
);
616 error_setg_errno(errp
, -ret
, "Could not get device features");
619 vhost_svq_valid_features(features
, &dev
->migration_blocker
);
623 * Similar to VFIO, we end up pinning all guest memory and have to
624 * disable discarding of RAM.
626 ret
= ram_block_discard_disable(true);
628 error_report("Cannot set discarding of RAM broken");
632 vhost_vdpa_add_status(dev
, VIRTIO_CONFIG_S_ACKNOWLEDGE
|
633 VIRTIO_CONFIG_S_DRIVER
);
638 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev
*dev
,
641 size_t page_size
= qemu_real_host_page_size();
642 struct vhost_vdpa
*v
= dev
->opaque
;
643 VirtIODevice
*vdev
= dev
->vdev
;
644 VhostVDPAHostNotifier
*n
;
646 n
= &v
->notifier
[queue_index
];
649 virtio_queue_set_host_notifier_mr(vdev
, queue_index
, &n
->mr
, false);
650 object_unparent(OBJECT(&n
->mr
));
651 munmap(n
->addr
, page_size
);
656 static int vhost_vdpa_host_notifier_init(struct vhost_dev
*dev
, int queue_index
)
658 size_t page_size
= qemu_real_host_page_size();
659 struct vhost_vdpa
*v
= dev
->opaque
;
660 VirtIODevice
*vdev
= dev
->vdev
;
661 VhostVDPAHostNotifier
*n
;
662 int fd
= v
->shared
->device_fd
;
666 vhost_vdpa_host_notifier_uninit(dev
, queue_index
);
668 n
= &v
->notifier
[queue_index
];
670 addr
= mmap(NULL
, page_size
, PROT_WRITE
, MAP_SHARED
, fd
,
671 queue_index
* page_size
);
672 if (addr
== MAP_FAILED
) {
676 name
= g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
678 memory_region_init_ram_device_ptr(&n
->mr
, OBJECT(vdev
), name
,
682 if (virtio_queue_set_host_notifier_mr(vdev
, queue_index
, &n
->mr
, true)) {
683 object_unparent(OBJECT(&n
->mr
));
684 munmap(addr
, page_size
);
695 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev
*dev
, int n
)
700 * Pack all the changes to the memory regions in a single
701 * transaction to avoid a few updating of the address space
704 memory_region_transaction_begin();
706 for (i
= dev
->vq_index
; i
< dev
->vq_index
+ n
; i
++) {
707 vhost_vdpa_host_notifier_uninit(dev
, i
);
710 memory_region_transaction_commit();
713 static void vhost_vdpa_host_notifiers_init(struct vhost_dev
*dev
)
715 struct vhost_vdpa
*v
= dev
->opaque
;
718 if (v
->shadow_vqs_enabled
) {
719 /* FIXME SVQ is not compatible with host notifiers mr */
724 * Pack all the changes to the memory regions in a single
725 * transaction to avoid a few updating of the address space
728 memory_region_transaction_begin();
730 for (i
= dev
->vq_index
; i
< dev
->vq_index
+ dev
->nvqs
; i
++) {
731 if (vhost_vdpa_host_notifier_init(dev
, i
)) {
732 vhost_vdpa_host_notifiers_uninit(dev
, i
- dev
->vq_index
);
737 memory_region_transaction_commit();
740 static void vhost_vdpa_svq_cleanup(struct vhost_dev
*dev
)
742 struct vhost_vdpa
*v
= dev
->opaque
;
745 for (idx
= 0; idx
< v
->shadow_vqs
->len
; ++idx
) {
746 vhost_svq_stop(g_ptr_array_index(v
->shadow_vqs
, idx
));
748 g_ptr_array_free(v
->shadow_vqs
, true);
751 static int vhost_vdpa_cleanup(struct vhost_dev
*dev
)
753 struct vhost_vdpa
*v
;
754 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_VDPA
);
756 trace_vhost_vdpa_cleanup(dev
, v
);
757 if (vhost_vdpa_first_dev(dev
)) {
758 ram_block_discard_disable(false);
759 memory_listener_unregister(&v
->shared
->listener
);
762 vhost_vdpa_host_notifiers_uninit(dev
, dev
->nvqs
);
763 vhost_vdpa_svq_cleanup(dev
);
770 static int vhost_vdpa_memslots_limit(struct vhost_dev
*dev
)
772 trace_vhost_vdpa_memslots_limit(dev
, INT_MAX
);
776 static int vhost_vdpa_set_mem_table(struct vhost_dev
*dev
,
777 struct vhost_memory
*mem
)
779 if (!vhost_vdpa_first_dev(dev
)) {
783 trace_vhost_vdpa_set_mem_table(dev
, mem
->nregions
, mem
->padding
);
784 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE
) &&
785 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS
)) {
787 for (i
= 0; i
< mem
->nregions
; i
++) {
788 trace_vhost_vdpa_dump_regions(dev
, i
,
789 mem
->regions
[i
].guest_phys_addr
,
790 mem
->regions
[i
].memory_size
,
791 mem
->regions
[i
].userspace_addr
,
792 mem
->regions
[i
].flags_padding
);
802 static int vhost_vdpa_set_features(struct vhost_dev
*dev
,
805 struct vhost_vdpa
*v
= dev
->opaque
;
808 if (!vhost_vdpa_first_dev(dev
)) {
812 if (v
->shadow_vqs_enabled
) {
813 if ((v
->acked_features
^ features
) == BIT_ULL(VHOST_F_LOG_ALL
)) {
815 * QEMU is just trying to enable or disable logging. SVQ handles
816 * this sepparately, so no need to forward this.
818 v
->acked_features
= features
;
822 v
->acked_features
= features
;
824 /* We must not ack _F_LOG if SVQ is enabled */
825 features
&= ~BIT_ULL(VHOST_F_LOG_ALL
);
828 trace_vhost_vdpa_set_features(dev
, features
);
829 ret
= vhost_vdpa_call(dev
, VHOST_SET_FEATURES
, &features
);
834 return vhost_vdpa_add_status(dev
, VIRTIO_CONFIG_S_FEATURES_OK
);
837 static int vhost_vdpa_set_backend_cap(struct vhost_dev
*dev
)
839 struct vhost_vdpa
*v
= dev
->opaque
;
842 uint64_t f
= 0x1ULL
<< VHOST_BACKEND_F_IOTLB_MSG_V2
|
843 0x1ULL
<< VHOST_BACKEND_F_IOTLB_BATCH
|
844 0x1ULL
<< VHOST_BACKEND_F_IOTLB_ASID
|
845 0x1ULL
<< VHOST_BACKEND_F_SUSPEND
;
848 if (vhost_vdpa_call(dev
, VHOST_GET_BACKEND_FEATURES
, &features
)) {
854 if (vhost_vdpa_first_dev(dev
)) {
855 r
= vhost_vdpa_call(dev
, VHOST_SET_BACKEND_FEATURES
, &features
);
861 dev
->backend_cap
= features
;
862 v
->shared
->backend_cap
= features
;
867 static int vhost_vdpa_get_device_id(struct vhost_dev
*dev
,
871 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_GET_DEVICE_ID
, device_id
);
872 trace_vhost_vdpa_get_device_id(dev
, *device_id
);
876 static int vhost_vdpa_reset_device(struct vhost_dev
*dev
)
878 struct vhost_vdpa
*v
= dev
->opaque
;
882 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_SET_STATUS
, &status
);
883 trace_vhost_vdpa_reset_device(dev
);
884 v
->suspended
= false;
888 static int vhost_vdpa_get_vq_index(struct vhost_dev
*dev
, int idx
)
890 assert(idx
>= dev
->vq_index
&& idx
< dev
->vq_index
+ dev
->nvqs
);
892 trace_vhost_vdpa_get_vq_index(dev
, idx
, idx
);
896 int vhost_vdpa_set_vring_ready(struct vhost_vdpa
*v
, unsigned idx
)
898 struct vhost_dev
*dev
= v
->dev
;
899 struct vhost_vring_state state
= {
903 int r
= vhost_vdpa_call(dev
, VHOST_VDPA_SET_VRING_ENABLE
, &state
);
905 trace_vhost_vdpa_set_vring_ready(dev
, idx
, r
);
909 static int vhost_vdpa_set_config_call(struct vhost_dev
*dev
,
912 trace_vhost_vdpa_set_config_call(dev
, fd
);
913 return vhost_vdpa_call(dev
, VHOST_VDPA_SET_CONFIG_CALL
, &fd
);
916 static void vhost_vdpa_dump_config(struct vhost_dev
*dev
, const uint8_t *config
,
920 char line
[QEMU_HEXDUMP_LINE_LEN
];
922 for (b
= 0; b
< config_len
; b
+= 16) {
923 len
= config_len
- b
;
924 qemu_hexdump_line(line
, b
, config
, len
, false);
925 trace_vhost_vdpa_dump_config(dev
, line
);
929 static int vhost_vdpa_set_config(struct vhost_dev
*dev
, const uint8_t *data
,
930 uint32_t offset
, uint32_t size
,
933 struct vhost_vdpa_config
*config
;
935 unsigned long config_size
= offsetof(struct vhost_vdpa_config
, buf
);
937 trace_vhost_vdpa_set_config(dev
, offset
, size
, flags
);
938 config
= g_malloc(size
+ config_size
);
939 config
->off
= offset
;
941 memcpy(config
->buf
, data
, size
);
942 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG
) &&
943 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG
)) {
944 vhost_vdpa_dump_config(dev
, data
, size
);
946 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_SET_CONFIG
, config
);
951 static int vhost_vdpa_get_config(struct vhost_dev
*dev
, uint8_t *config
,
952 uint32_t config_len
, Error
**errp
)
954 struct vhost_vdpa_config
*v_config
;
955 unsigned long config_size
= offsetof(struct vhost_vdpa_config
, buf
);
958 trace_vhost_vdpa_get_config(dev
, config
, config_len
);
959 v_config
= g_malloc(config_len
+ config_size
);
960 v_config
->len
= config_len
;
962 ret
= vhost_vdpa_call(dev
, VHOST_VDPA_GET_CONFIG
, v_config
);
963 memcpy(config
, v_config
->buf
, config_len
);
965 if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG
) &&
966 trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG
)) {
967 vhost_vdpa_dump_config(dev
, config
, config_len
);
972 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev
*dev
,
973 struct vhost_vring_state
*ring
)
975 struct vhost_vdpa
*v
= dev
->opaque
;
977 trace_vhost_vdpa_set_dev_vring_base(dev
, ring
->index
, ring
->num
,
978 v
->shadow_vqs_enabled
);
979 return vhost_vdpa_call(dev
, VHOST_SET_VRING_BASE
, ring
);
982 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev
*dev
,
983 struct vhost_vring_file
*file
)
985 trace_vhost_vdpa_set_vring_kick(dev
, file
->index
, file
->fd
);
986 return vhost_vdpa_call(dev
, VHOST_SET_VRING_KICK
, file
);
989 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev
*dev
,
990 struct vhost_vring_file
*file
)
992 trace_vhost_vdpa_set_vring_call(dev
, file
->index
, file
->fd
);
993 return vhost_vdpa_call(dev
, VHOST_SET_VRING_CALL
, file
);
996 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev
*dev
,
997 struct vhost_vring_addr
*addr
)
999 trace_vhost_vdpa_set_vring_addr(dev
, addr
->index
, addr
->flags
,
1000 addr
->desc_user_addr
, addr
->used_user_addr
,
1001 addr
->avail_user_addr
,
1002 addr
->log_guest_addr
);
1004 return vhost_vdpa_call(dev
, VHOST_SET_VRING_ADDR
, addr
);
1009 * Set the shadow virtqueue descriptors to the device
1011 * @dev: The vhost device model
1012 * @svq: The shadow virtqueue
1013 * @idx: The index of the virtqueue in the vhost device
1016 * Note that this function does not rewind kick file descriptor if cannot set
1019 static int vhost_vdpa_svq_set_fds(struct vhost_dev
*dev
,
1020 VhostShadowVirtqueue
*svq
, unsigned idx
,
1023 struct vhost_vring_file file
= {
1024 .index
= dev
->vq_index
+ idx
,
1026 const EventNotifier
*event_notifier
= &svq
->hdev_kick
;
1029 r
= event_notifier_init(&svq
->hdev_kick
, 0);
1031 error_setg_errno(errp
, -r
, "Couldn't create kick event notifier");
1032 goto err_init_hdev_kick
;
1035 r
= event_notifier_init(&svq
->hdev_call
, 0);
1037 error_setg_errno(errp
, -r
, "Couldn't create call event notifier");
1038 goto err_init_hdev_call
;
1041 file
.fd
= event_notifier_get_fd(event_notifier
);
1042 r
= vhost_vdpa_set_vring_dev_kick(dev
, &file
);
1043 if (unlikely(r
!= 0)) {
1044 error_setg_errno(errp
, -r
, "Can't set device kick fd");
1045 goto err_init_set_dev_fd
;
1048 event_notifier
= &svq
->hdev_call
;
1049 file
.fd
= event_notifier_get_fd(event_notifier
);
1050 r
= vhost_vdpa_set_vring_dev_call(dev
, &file
);
1051 if (unlikely(r
!= 0)) {
1052 error_setg_errno(errp
, -r
, "Can't set device call fd");
1053 goto err_init_set_dev_fd
;
1058 err_init_set_dev_fd
:
1059 event_notifier_set_handler(&svq
->hdev_call
, NULL
);
1062 event_notifier_cleanup(&svq
->hdev_kick
);
1069 * Unmap a SVQ area in the device
1071 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa
*v
, hwaddr addr
)
1073 const DMAMap needle
= {
1074 .translated_addr
= addr
,
1076 const DMAMap
*result
= vhost_iova_tree_find_iova(v
->shared
->iova_tree
,
1081 if (unlikely(!result
)) {
1082 error_report("Unable to find SVQ address to unmap");
1086 size
= ROUND_UP(result
->size
, qemu_real_host_page_size());
1087 r
= vhost_vdpa_dma_unmap(v
->shared
, v
->address_space_id
, result
->iova
,
1089 if (unlikely(r
< 0)) {
1090 error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r
), -r
);
1094 vhost_iova_tree_remove(v
->shared
->iova_tree
, *result
);
1097 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev
*dev
,
1098 const VhostShadowVirtqueue
*svq
)
1100 struct vhost_vdpa
*v
= dev
->opaque
;
1101 struct vhost_vring_addr svq_addr
;
1103 vhost_svq_get_vring_addr(svq
, &svq_addr
);
1105 vhost_vdpa_svq_unmap_ring(v
, svq_addr
.desc_user_addr
);
1107 vhost_vdpa_svq_unmap_ring(v
, svq_addr
.used_user_addr
);
1111 * Map the SVQ area in the device
1113 * @v: Vhost-vdpa device
1114 * @needle: The area to search iova
1115 * @errorp: Error pointer
1117 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa
*v
, DMAMap
*needle
,
1122 r
= vhost_iova_tree_map_alloc(v
->shared
->iova_tree
, needle
);
1123 if (unlikely(r
!= IOVA_OK
)) {
1124 error_setg(errp
, "Cannot allocate iova (%d)", r
);
1128 r
= vhost_vdpa_dma_map(v
->shared
, v
->address_space_id
, needle
->iova
,
1130 (void *)(uintptr_t)needle
->translated_addr
,
1131 needle
->perm
== IOMMU_RO
);
1132 if (unlikely(r
!= 0)) {
1133 error_setg_errno(errp
, -r
, "Cannot map region to device");
1134 vhost_iova_tree_remove(v
->shared
->iova_tree
, *needle
);
1141 * Map the shadow virtqueue rings in the device
1143 * @dev: The vhost device
1144 * @svq: The shadow virtqueue
1145 * @addr: Assigned IOVA addresses
1146 * @errp: Error pointer
1148 static bool vhost_vdpa_svq_map_rings(struct vhost_dev
*dev
,
1149 const VhostShadowVirtqueue
*svq
,
1150 struct vhost_vring_addr
*addr
,
1154 DMAMap device_region
, driver_region
;
1155 struct vhost_vring_addr svq_addr
;
1156 struct vhost_vdpa
*v
= dev
->opaque
;
1157 size_t device_size
= vhost_svq_device_area_size(svq
);
1158 size_t driver_size
= vhost_svq_driver_area_size(svq
);
1159 size_t avail_offset
;
1162 vhost_svq_get_vring_addr(svq
, &svq_addr
);
1164 driver_region
= (DMAMap
) {
1165 .translated_addr
= svq_addr
.desc_user_addr
,
1166 .size
= driver_size
- 1,
1169 ok
= vhost_vdpa_svq_map_ring(v
, &driver_region
, errp
);
1170 if (unlikely(!ok
)) {
1171 error_prepend(errp
, "Cannot create vq driver region: ");
1174 addr
->desc_user_addr
= driver_region
.iova
;
1175 avail_offset
= svq_addr
.avail_user_addr
- svq_addr
.desc_user_addr
;
1176 addr
->avail_user_addr
= driver_region
.iova
+ avail_offset
;
1178 device_region
= (DMAMap
) {
1179 .translated_addr
= svq_addr
.used_user_addr
,
1180 .size
= device_size
- 1,
1183 ok
= vhost_vdpa_svq_map_ring(v
, &device_region
, errp
);
1184 if (unlikely(!ok
)) {
1185 error_prepend(errp
, "Cannot create vq device region: ");
1186 vhost_vdpa_svq_unmap_ring(v
, driver_region
.translated_addr
);
1188 addr
->used_user_addr
= device_region
.iova
;
1193 static bool vhost_vdpa_svq_setup(struct vhost_dev
*dev
,
1194 VhostShadowVirtqueue
*svq
, unsigned idx
,
1197 uint16_t vq_index
= dev
->vq_index
+ idx
;
1198 struct vhost_vring_state s
= {
1203 r
= vhost_vdpa_set_dev_vring_base(dev
, &s
);
1205 error_setg_errno(errp
, -r
, "Cannot set vring base");
1209 r
= vhost_vdpa_svq_set_fds(dev
, svq
, idx
, errp
);
1213 static bool vhost_vdpa_svqs_start(struct vhost_dev
*dev
)
1215 struct vhost_vdpa
*v
= dev
->opaque
;
1219 if (!v
->shadow_vqs_enabled
) {
1223 for (i
= 0; i
< v
->shadow_vqs
->len
; ++i
) {
1224 VirtQueue
*vq
= virtio_get_queue(dev
->vdev
, dev
->vq_index
+ i
);
1225 VhostShadowVirtqueue
*svq
= g_ptr_array_index(v
->shadow_vqs
, i
);
1226 struct vhost_vring_addr addr
= {
1227 .index
= dev
->vq_index
+ i
,
1230 bool ok
= vhost_vdpa_svq_setup(dev
, svq
, i
, &err
);
1231 if (unlikely(!ok
)) {
1235 vhost_svq_start(svq
, dev
->vdev
, vq
, v
->shared
->iova_tree
);
1236 ok
= vhost_vdpa_svq_map_rings(dev
, svq
, &addr
, &err
);
1237 if (unlikely(!ok
)) {
1241 /* Override vring GPA set by vhost subsystem */
1242 r
= vhost_vdpa_set_vring_dev_addr(dev
, &addr
);
1243 if (unlikely(r
!= 0)) {
1244 error_setg_errno(&err
, -r
, "Cannot set device address");
1252 vhost_vdpa_svq_unmap_rings(dev
, g_ptr_array_index(v
->shadow_vqs
, i
));
1255 vhost_svq_stop(g_ptr_array_index(v
->shadow_vqs
, i
));
1258 error_reportf_err(err
, "Cannot setup SVQ %u: ", i
);
1259 for (unsigned j
= 0; j
< i
; ++j
) {
1260 VhostShadowVirtqueue
*svq
= g_ptr_array_index(v
->shadow_vqs
, j
);
1261 vhost_vdpa_svq_unmap_rings(dev
, svq
);
1262 vhost_svq_stop(svq
);
1268 static void vhost_vdpa_svqs_stop(struct vhost_dev
*dev
)
1270 struct vhost_vdpa
*v
= dev
->opaque
;
1272 if (!v
->shadow_vqs_enabled
) {
1276 for (unsigned i
= 0; i
< v
->shadow_vqs
->len
; ++i
) {
1277 VhostShadowVirtqueue
*svq
= g_ptr_array_index(v
->shadow_vqs
, i
);
1279 vhost_svq_stop(svq
);
1280 vhost_vdpa_svq_unmap_rings(dev
, svq
);
1282 event_notifier_cleanup(&svq
->hdev_kick
);
1283 event_notifier_cleanup(&svq
->hdev_call
);
1287 static void vhost_vdpa_suspend(struct vhost_dev
*dev
)
1289 struct vhost_vdpa
*v
= dev
->opaque
;
1292 if (!vhost_vdpa_first_dev(dev
)) {
1296 if (dev
->backend_cap
& BIT_ULL(VHOST_BACKEND_F_SUSPEND
)) {
1297 trace_vhost_vdpa_suspend(dev
);
1298 r
= ioctl(v
->shared
->device_fd
, VHOST_VDPA_SUSPEND
);
1300 error_report("Cannot suspend: %s(%d)", g_strerror(errno
), errno
);
1302 v
->suspended
= true;
1307 vhost_vdpa_reset_device(dev
);
1310 static int vhost_vdpa_dev_start(struct vhost_dev
*dev
, bool started
)
1312 struct vhost_vdpa
*v
= dev
->opaque
;
1314 trace_vhost_vdpa_dev_start(dev
, started
);
1317 vhost_vdpa_host_notifiers_init(dev
);
1318 ok
= vhost_vdpa_svqs_start(dev
);
1319 if (unlikely(!ok
)) {
1323 vhost_vdpa_suspend(dev
);
1324 vhost_vdpa_svqs_stop(dev
);
1325 vhost_vdpa_host_notifiers_uninit(dev
, dev
->nvqs
);
1328 if (!vhost_vdpa_last_dev(dev
)) {
1333 if (vhost_dev_has_iommu(dev
) && (v
->shadow_vqs_enabled
)) {
1334 error_report("SVQ can not work while IOMMU enable, please disable"
1335 "IOMMU and try again");
1338 memory_listener_register(&v
->shared
->listener
, dev
->vdev
->dma_as
);
1340 return vhost_vdpa_add_status(dev
, VIRTIO_CONFIG_S_DRIVER_OK
);
1346 static void vhost_vdpa_reset_status(struct vhost_dev
*dev
)
1348 struct vhost_vdpa
*v
= dev
->opaque
;
1350 if (!vhost_vdpa_last_dev(dev
)) {
1354 vhost_vdpa_reset_device(dev
);
1355 vhost_vdpa_add_status(dev
, VIRTIO_CONFIG_S_ACKNOWLEDGE
|
1356 VIRTIO_CONFIG_S_DRIVER
);
1357 memory_listener_unregister(&v
->shared
->listener
);
1360 static int vhost_vdpa_set_log_base(struct vhost_dev
*dev
, uint64_t base
,
1361 struct vhost_log
*log
)
1363 struct vhost_vdpa
*v
= dev
->opaque
;
1364 if (v
->shadow_vqs_enabled
|| !vhost_vdpa_first_dev(dev
)) {
1368 trace_vhost_vdpa_set_log_base(dev
, base
, log
->size
, log
->refcnt
, log
->fd
,
1370 return vhost_vdpa_call(dev
, VHOST_SET_LOG_BASE
, &base
);
1373 static int vhost_vdpa_set_vring_addr(struct vhost_dev
*dev
,
1374 struct vhost_vring_addr
*addr
)
1376 struct vhost_vdpa
*v
= dev
->opaque
;
1378 if (v
->shadow_vqs_enabled
) {
1380 * Device vring addr was set at device start. SVQ base is handled by
1386 return vhost_vdpa_set_vring_dev_addr(dev
, addr
);
1389 static int vhost_vdpa_set_vring_num(struct vhost_dev
*dev
,
1390 struct vhost_vring_state
*ring
)
1392 trace_vhost_vdpa_set_vring_num(dev
, ring
->index
, ring
->num
);
1393 return vhost_vdpa_call(dev
, VHOST_SET_VRING_NUM
, ring
);
1396 static int vhost_vdpa_set_vring_base(struct vhost_dev
*dev
,
1397 struct vhost_vring_state
*ring
)
1399 struct vhost_vdpa
*v
= dev
->opaque
;
1401 if (v
->shadow_vqs_enabled
) {
1403 * Device vring base was set at device start. SVQ base is handled by
1409 return vhost_vdpa_set_dev_vring_base(dev
, ring
);
1412 static int vhost_vdpa_get_vring_base(struct vhost_dev
*dev
,
1413 struct vhost_vring_state
*ring
)
1415 struct vhost_vdpa
*v
= dev
->opaque
;
1418 if (v
->shadow_vqs_enabled
) {
1419 ring
->num
= virtio_queue_get_last_avail_idx(dev
->vdev
, ring
->index
);
1420 trace_vhost_vdpa_get_vring_base(dev
, ring
->index
, ring
->num
, true);
1424 if (!v
->suspended
) {
1426 * Cannot trust in value returned by device, let vhost recover used
1432 ret
= vhost_vdpa_call(dev
, VHOST_GET_VRING_BASE
, ring
);
1433 trace_vhost_vdpa_get_vring_base(dev
, ring
->index
, ring
->num
, false);
1437 static int vhost_vdpa_set_vring_kick(struct vhost_dev
*dev
,
1438 struct vhost_vring_file
*file
)
1440 struct vhost_vdpa
*v
= dev
->opaque
;
1441 int vdpa_idx
= file
->index
- dev
->vq_index
;
1443 if (v
->shadow_vqs_enabled
) {
1444 VhostShadowVirtqueue
*svq
= g_ptr_array_index(v
->shadow_vqs
, vdpa_idx
);
1445 vhost_svq_set_svq_kick_fd(svq
, file
->fd
);
1448 return vhost_vdpa_set_vring_dev_kick(dev
, file
);
1452 static int vhost_vdpa_set_vring_call(struct vhost_dev
*dev
,
1453 struct vhost_vring_file
*file
)
1455 struct vhost_vdpa
*v
= dev
->opaque
;
1456 int vdpa_idx
= file
->index
- dev
->vq_index
;
1457 VhostShadowVirtqueue
*svq
= g_ptr_array_index(v
->shadow_vqs
, vdpa_idx
);
1459 /* Remember last call fd because we can switch to SVQ anytime. */
1460 vhost_svq_set_svq_call_fd(svq
, file
->fd
);
1462 * When SVQ is transitioning to off, shadow_vqs_enabled has
1463 * not been set back to false yet, but the underlying call fd
1464 * will have to switch back to the guest notifier to signal the
1465 * passthrough virtqueues. In other situations, SVQ's own call
1466 * fd shall be used to signal the device model.
1468 if (v
->shadow_vqs_enabled
&&
1469 v
->shared
->svq_switching
!= SVQ_TSTATE_DISABLING
) {
1473 return vhost_vdpa_set_vring_dev_call(dev
, file
);
1476 static int vhost_vdpa_get_features(struct vhost_dev
*dev
,
1479 int ret
= vhost_vdpa_get_dev_features(dev
, features
);
1482 /* Add SVQ logging capabilities */
1483 *features
|= BIT_ULL(VHOST_F_LOG_ALL
);
1489 static int vhost_vdpa_set_owner(struct vhost_dev
*dev
)
1491 if (!vhost_vdpa_first_dev(dev
)) {
1495 trace_vhost_vdpa_set_owner(dev
);
1496 return vhost_vdpa_call(dev
, VHOST_SET_OWNER
, NULL
);
1499 static int vhost_vdpa_vq_get_addr(struct vhost_dev
*dev
,
1500 struct vhost_vring_addr
*addr
, struct vhost_virtqueue
*vq
)
1502 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_VDPA
);
1503 addr
->desc_user_addr
= (uint64_t)(unsigned long)vq
->desc_phys
;
1504 addr
->avail_user_addr
= (uint64_t)(unsigned long)vq
->avail_phys
;
1505 addr
->used_user_addr
= (uint64_t)(unsigned long)vq
->used_phys
;
1506 trace_vhost_vdpa_vq_get_addr(dev
, vq
, addr
->desc_user_addr
,
1507 addr
->avail_user_addr
, addr
->used_user_addr
);
1511 static bool vhost_vdpa_force_iommu(struct vhost_dev
*dev
)
1516 const VhostOps vdpa_ops
= {
1517 .backend_type
= VHOST_BACKEND_TYPE_VDPA
,
1518 .vhost_backend_init
= vhost_vdpa_init
,
1519 .vhost_backend_cleanup
= vhost_vdpa_cleanup
,
1520 .vhost_set_log_base
= vhost_vdpa_set_log_base
,
1521 .vhost_set_vring_addr
= vhost_vdpa_set_vring_addr
,
1522 .vhost_set_vring_num
= vhost_vdpa_set_vring_num
,
1523 .vhost_set_vring_base
= vhost_vdpa_set_vring_base
,
1524 .vhost_get_vring_base
= vhost_vdpa_get_vring_base
,
1525 .vhost_set_vring_kick
= vhost_vdpa_set_vring_kick
,
1526 .vhost_set_vring_call
= vhost_vdpa_set_vring_call
,
1527 .vhost_get_features
= vhost_vdpa_get_features
,
1528 .vhost_set_backend_cap
= vhost_vdpa_set_backend_cap
,
1529 .vhost_set_owner
= vhost_vdpa_set_owner
,
1530 .vhost_set_vring_endian
= NULL
,
1531 .vhost_backend_memslots_limit
= vhost_vdpa_memslots_limit
,
1532 .vhost_set_mem_table
= vhost_vdpa_set_mem_table
,
1533 .vhost_set_features
= vhost_vdpa_set_features
,
1534 .vhost_reset_device
= vhost_vdpa_reset_device
,
1535 .vhost_get_vq_index
= vhost_vdpa_get_vq_index
,
1536 .vhost_get_config
= vhost_vdpa_get_config
,
1537 .vhost_set_config
= vhost_vdpa_set_config
,
1538 .vhost_requires_shm_log
= NULL
,
1539 .vhost_migration_done
= NULL
,
1540 .vhost_net_set_mtu
= NULL
,
1541 .vhost_set_iotlb_callback
= NULL
,
1542 .vhost_send_device_iotlb_msg
= NULL
,
1543 .vhost_dev_start
= vhost_vdpa_dev_start
,
1544 .vhost_get_device_id
= vhost_vdpa_get_device_id
,
1545 .vhost_vq_get_addr
= vhost_vdpa_vq_get_addr
,
1546 .vhost_force_iommu
= vhost_vdpa_force_iommu
,
1547 .vhost_set_config_call
= vhost_vdpa_set_config_call
,
1548 .vhost_reset_status
= vhost_vdpa_reset_status
,