4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
18 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 #include "sysemu/runstate.h"
30 #include "standard-headers/linux/virtio_ids.h"
33 * The alignment to use between consumer and producer parts of vring.
34 * x86 pagesize again. This is the default, used by transports like PCI
35 * which don't provide a means for the guest to tell the host the alignment.
37 #define VIRTIO_PCI_VRING_ALIGN 4096
39 typedef struct VRingDesc
47 typedef struct VRingPackedDesc
{
54 typedef struct VRingAvail
61 typedef struct VRingUsedElem
67 typedef struct VRingUsed
74 typedef struct VRingMemoryRegionCaches
{
76 MemoryRegionCache desc
;
77 MemoryRegionCache avail
;
78 MemoryRegionCache used
;
79 } VRingMemoryRegionCaches
;
84 unsigned int num_default
;
89 VRingMemoryRegionCaches
*caches
;
92 typedef struct VRingPackedDescEvent
{
95 } VRingPackedDescEvent
;
100 VirtQueueElement
*used_elems
;
102 /* Next head to pop */
103 uint16_t last_avail_idx
;
104 bool last_avail_wrap_counter
;
106 /* Last avail_idx read from VQ. */
107 uint16_t shadow_avail_idx
;
108 bool shadow_avail_wrap_counter
;
111 bool used_wrap_counter
;
113 /* Last used index value we have signalled on */
114 uint16_t signalled_used
;
116 /* Last used index value we have signalled on */
117 bool signalled_used_valid
;
119 /* Notification enabled? */
122 uint16_t queue_index
;
127 VirtIOHandleOutput handle_output
;
129 EventNotifier guest_notifier
;
130 EventNotifier host_notifier
;
131 bool host_notifier_enabled
;
132 QLIST_ENTRY(VirtQueue
) node
;
135 /* Called within call_rcu(). */
136 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
138 assert(caches
!= NULL
);
139 address_space_cache_destroy(&caches
->desc
);
140 address_space_cache_destroy(&caches
->avail
);
141 address_space_cache_destroy(&caches
->used
);
145 static void virtio_virtqueue_reset_region_cache(struct VirtQueue
*vq
)
147 VRingMemoryRegionCaches
*caches
;
149 caches
= qatomic_read(&vq
->vring
.caches
);
150 qatomic_rcu_set(&vq
->vring
.caches
, NULL
);
152 call_rcu(caches
, virtio_free_region_cache
, rcu
);
156 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
158 VirtQueue
*vq
= &vdev
->vq
[n
];
159 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
160 VRingMemoryRegionCaches
*new = NULL
;
166 addr
= vq
->vring
.desc
;
170 new = g_new0(VRingMemoryRegionCaches
, 1);
171 size
= virtio_queue_get_desc_size(vdev
, n
);
172 packed
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
174 len
= address_space_cache_init(&new->desc
, vdev
->dma_as
,
177 virtio_error(vdev
, "Cannot map desc");
181 size
= virtio_queue_get_used_size(vdev
, n
);
182 len
= address_space_cache_init(&new->used
, vdev
->dma_as
,
183 vq
->vring
.used
, size
, true);
185 virtio_error(vdev
, "Cannot map used");
189 size
= virtio_queue_get_avail_size(vdev
, n
);
190 len
= address_space_cache_init(&new->avail
, vdev
->dma_as
,
191 vq
->vring
.avail
, size
, false);
193 virtio_error(vdev
, "Cannot map avail");
197 qatomic_rcu_set(&vq
->vring
.caches
, new);
199 call_rcu(old
, virtio_free_region_cache
, rcu
);
204 address_space_cache_destroy(&new->avail
);
206 address_space_cache_destroy(&new->used
);
208 address_space_cache_destroy(&new->desc
);
211 virtio_virtqueue_reset_region_cache(vq
);
214 /* virt queue functions */
215 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
217 VRing
*vring
= &vdev
->vq
[n
].vring
;
219 if (!vring
->num
|| !vring
->desc
|| !vring
->align
) {
220 /* not yet setup -> nothing to do */
223 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
224 vring
->used
= vring_align(vring
->avail
+
225 offsetof(VRingAvail
, ring
[vring
->num
]),
227 virtio_init_region_cache(vdev
, n
);
230 /* Called within rcu_read_lock(). */
231 static void vring_split_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
232 MemoryRegionCache
*cache
, int i
)
234 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
235 desc
, sizeof(VRingDesc
));
236 virtio_tswap64s(vdev
, &desc
->addr
);
237 virtio_tswap32s(vdev
, &desc
->len
);
238 virtio_tswap16s(vdev
, &desc
->flags
);
239 virtio_tswap16s(vdev
, &desc
->next
);
242 static void vring_packed_event_read(VirtIODevice
*vdev
,
243 MemoryRegionCache
*cache
,
244 VRingPackedDescEvent
*e
)
246 hwaddr off_off
= offsetof(VRingPackedDescEvent
, off_wrap
);
247 hwaddr off_flags
= offsetof(VRingPackedDescEvent
, flags
);
249 e
->flags
= virtio_lduw_phys_cached(vdev
, cache
, off_flags
);
250 /* Make sure flags is seen before off_wrap */
252 e
->off_wrap
= virtio_lduw_phys_cached(vdev
, cache
, off_off
);
253 virtio_tswap16s(vdev
, &e
->flags
);
256 static void vring_packed_off_wrap_write(VirtIODevice
*vdev
,
257 MemoryRegionCache
*cache
,
260 hwaddr off
= offsetof(VRingPackedDescEvent
, off_wrap
);
262 virtio_stw_phys_cached(vdev
, cache
, off
, off_wrap
);
263 address_space_cache_invalidate(cache
, off
, sizeof(off_wrap
));
266 static void vring_packed_flags_write(VirtIODevice
*vdev
,
267 MemoryRegionCache
*cache
, uint16_t flags
)
269 hwaddr off
= offsetof(VRingPackedDescEvent
, flags
);
271 virtio_stw_phys_cached(vdev
, cache
, off
, flags
);
272 address_space_cache_invalidate(cache
, off
, sizeof(flags
));
275 /* Called within rcu_read_lock(). */
276 static VRingMemoryRegionCaches
*vring_get_region_caches(struct VirtQueue
*vq
)
278 return qatomic_rcu_read(&vq
->vring
.caches
);
281 /* Called within rcu_read_lock(). */
282 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
284 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
285 hwaddr pa
= offsetof(VRingAvail
, flags
);
291 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
294 /* Called within rcu_read_lock(). */
295 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
297 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
298 hwaddr pa
= offsetof(VRingAvail
, idx
);
304 vq
->shadow_avail_idx
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
305 return vq
->shadow_avail_idx
;
308 /* Called within rcu_read_lock(). */
309 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
311 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
312 hwaddr pa
= offsetof(VRingAvail
, ring
[i
]);
318 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
321 /* Called within rcu_read_lock(). */
322 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
324 return vring_avail_ring(vq
, vq
->vring
.num
);
327 /* Called within rcu_read_lock(). */
328 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
331 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
332 hwaddr pa
= offsetof(VRingUsed
, ring
[i
]);
338 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
339 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
340 address_space_write_cached(&caches
->used
, pa
, uelem
, sizeof(VRingUsedElem
));
341 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(VRingUsedElem
));
344 /* Called within rcu_read_lock(). */
345 static uint16_t vring_used_idx(VirtQueue
*vq
)
347 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
348 hwaddr pa
= offsetof(VRingUsed
, idx
);
354 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
357 /* Called within rcu_read_lock(). */
358 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
360 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
361 hwaddr pa
= offsetof(VRingUsed
, idx
);
364 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
365 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
371 /* Called within rcu_read_lock(). */
372 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
374 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
375 VirtIODevice
*vdev
= vq
->vdev
;
376 hwaddr pa
= offsetof(VRingUsed
, flags
);
383 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
384 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
| mask
);
385 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
388 /* Called within rcu_read_lock(). */
389 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
391 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
392 VirtIODevice
*vdev
= vq
->vdev
;
393 hwaddr pa
= offsetof(VRingUsed
, flags
);
400 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
401 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
& ~mask
);
402 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
405 /* Called within rcu_read_lock(). */
406 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
408 VRingMemoryRegionCaches
*caches
;
410 if (!vq
->notification
) {
414 caches
= vring_get_region_caches(vq
);
419 pa
= offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
420 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
421 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
424 static void virtio_queue_split_set_notification(VirtQueue
*vq
, int enable
)
426 RCU_READ_LOCK_GUARD();
428 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
429 vring_set_avail_event(vq
, vring_avail_idx(vq
));
431 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
433 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
436 /* Expose avail event/used flags before caller checks the avail idx. */
441 static void virtio_queue_packed_set_notification(VirtQueue
*vq
, int enable
)
444 VRingPackedDescEvent e
;
445 VRingMemoryRegionCaches
*caches
;
447 RCU_READ_LOCK_GUARD();
448 caches
= vring_get_region_caches(vq
);
453 vring_packed_event_read(vq
->vdev
, &caches
->used
, &e
);
456 e
.flags
= VRING_PACKED_EVENT_FLAG_DISABLE
;
457 } else if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
458 off_wrap
= vq
->shadow_avail_idx
| vq
->shadow_avail_wrap_counter
<< 15;
459 vring_packed_off_wrap_write(vq
->vdev
, &caches
->used
, off_wrap
);
460 /* Make sure off_wrap is wrote before flags */
462 e
.flags
= VRING_PACKED_EVENT_FLAG_DESC
;
464 e
.flags
= VRING_PACKED_EVENT_FLAG_ENABLE
;
467 vring_packed_flags_write(vq
->vdev
, &caches
->used
, e
.flags
);
469 /* Expose avail event/used flags before caller checks the avail idx. */
474 bool virtio_queue_get_notification(VirtQueue
*vq
)
476 return vq
->notification
;
479 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
481 vq
->notification
= enable
;
483 if (!vq
->vring
.desc
) {
487 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
488 virtio_queue_packed_set_notification(vq
, enable
);
490 virtio_queue_split_set_notification(vq
, enable
);
494 int virtio_queue_ready(VirtQueue
*vq
)
496 return vq
->vring
.avail
!= 0;
499 static void vring_packed_desc_read_flags(VirtIODevice
*vdev
,
501 MemoryRegionCache
*cache
,
504 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
506 *flags
= virtio_lduw_phys_cached(vdev
, cache
, off
);
509 static void vring_packed_desc_read(VirtIODevice
*vdev
,
510 VRingPackedDesc
*desc
,
511 MemoryRegionCache
*cache
,
512 int i
, bool strict_order
)
514 hwaddr off
= i
* sizeof(VRingPackedDesc
);
516 vring_packed_desc_read_flags(vdev
, &desc
->flags
, cache
, i
);
519 /* Make sure flags is read before the rest fields. */
523 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, addr
),
524 &desc
->addr
, sizeof(desc
->addr
));
525 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, id
),
526 &desc
->id
, sizeof(desc
->id
));
527 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, len
),
528 &desc
->len
, sizeof(desc
->len
));
529 virtio_tswap64s(vdev
, &desc
->addr
);
530 virtio_tswap16s(vdev
, &desc
->id
);
531 virtio_tswap32s(vdev
, &desc
->len
);
534 static void vring_packed_desc_write_data(VirtIODevice
*vdev
,
535 VRingPackedDesc
*desc
,
536 MemoryRegionCache
*cache
,
539 hwaddr off_id
= i
* sizeof(VRingPackedDesc
) +
540 offsetof(VRingPackedDesc
, id
);
541 hwaddr off_len
= i
* sizeof(VRingPackedDesc
) +
542 offsetof(VRingPackedDesc
, len
);
544 virtio_tswap32s(vdev
, &desc
->len
);
545 virtio_tswap16s(vdev
, &desc
->id
);
546 address_space_write_cached(cache
, off_id
, &desc
->id
, sizeof(desc
->id
));
547 address_space_cache_invalidate(cache
, off_id
, sizeof(desc
->id
));
548 address_space_write_cached(cache
, off_len
, &desc
->len
, sizeof(desc
->len
));
549 address_space_cache_invalidate(cache
, off_len
, sizeof(desc
->len
));
552 static void vring_packed_desc_write_flags(VirtIODevice
*vdev
,
553 VRingPackedDesc
*desc
,
554 MemoryRegionCache
*cache
,
557 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
559 virtio_stw_phys_cached(vdev
, cache
, off
, desc
->flags
);
560 address_space_cache_invalidate(cache
, off
, sizeof(desc
->flags
));
563 static void vring_packed_desc_write(VirtIODevice
*vdev
,
564 VRingPackedDesc
*desc
,
565 MemoryRegionCache
*cache
,
566 int i
, bool strict_order
)
568 vring_packed_desc_write_data(vdev
, desc
, cache
, i
);
570 /* Make sure data is wrote before flags. */
573 vring_packed_desc_write_flags(vdev
, desc
, cache
, i
);
576 static inline bool is_desc_avail(uint16_t flags
, bool wrap_counter
)
580 avail
= !!(flags
& (1 << VRING_PACKED_DESC_F_AVAIL
));
581 used
= !!(flags
& (1 << VRING_PACKED_DESC_F_USED
));
582 return (avail
!= used
) && (avail
== wrap_counter
);
585 /* Fetch avail_idx from VQ memory only when we really need to know if
586 * guest has added some buffers.
587 * Called within rcu_read_lock(). */
588 static int virtio_queue_empty_rcu(VirtQueue
*vq
)
590 if (virtio_device_disabled(vq
->vdev
)) {
594 if (unlikely(!vq
->vring
.avail
)) {
598 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
602 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
605 static int virtio_queue_split_empty(VirtQueue
*vq
)
609 if (virtio_device_disabled(vq
->vdev
)) {
613 if (unlikely(!vq
->vring
.avail
)) {
617 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
621 RCU_READ_LOCK_GUARD();
622 empty
= vring_avail_idx(vq
) == vq
->last_avail_idx
;
626 /* Called within rcu_read_lock(). */
627 static int virtio_queue_packed_empty_rcu(VirtQueue
*vq
)
629 struct VRingPackedDesc desc
;
630 VRingMemoryRegionCaches
*cache
;
632 if (unlikely(!vq
->vring
.desc
)) {
636 cache
= vring_get_region_caches(vq
);
641 vring_packed_desc_read_flags(vq
->vdev
, &desc
.flags
, &cache
->desc
,
644 return !is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
);
647 static int virtio_queue_packed_empty(VirtQueue
*vq
)
649 RCU_READ_LOCK_GUARD();
650 return virtio_queue_packed_empty_rcu(vq
);
653 int virtio_queue_empty(VirtQueue
*vq
)
655 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
656 return virtio_queue_packed_empty(vq
);
658 return virtio_queue_split_empty(vq
);
662 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
665 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
670 for (i
= 0; i
< elem
->in_num
; i
++) {
671 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
673 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
674 elem
->in_sg
[i
].iov_len
,
675 DMA_DIRECTION_FROM_DEVICE
, size
);
680 for (i
= 0; i
< elem
->out_num
; i
++)
681 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
682 elem
->out_sg
[i
].iov_len
,
683 DMA_DIRECTION_TO_DEVICE
,
684 elem
->out_sg
[i
].iov_len
);
687 /* virtqueue_detach_element:
688 * @vq: The #VirtQueue
689 * @elem: The #VirtQueueElement
690 * @len: number of bytes written
692 * Detach the element from the virtqueue. This function is suitable for device
693 * reset or other situations where a #VirtQueueElement is simply freed and will
694 * not be pushed or discarded.
696 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
699 vq
->inuse
-= elem
->ndescs
;
700 virtqueue_unmap_sg(vq
, elem
, len
);
703 static void virtqueue_split_rewind(VirtQueue
*vq
, unsigned int num
)
705 vq
->last_avail_idx
-= num
;
708 static void virtqueue_packed_rewind(VirtQueue
*vq
, unsigned int num
)
710 if (vq
->last_avail_idx
< num
) {
711 vq
->last_avail_idx
= vq
->vring
.num
+ vq
->last_avail_idx
- num
;
712 vq
->last_avail_wrap_counter
^= 1;
714 vq
->last_avail_idx
-= num
;
719 * @vq: The #VirtQueue
720 * @elem: The #VirtQueueElement
721 * @len: number of bytes written
723 * Pretend the most recent element wasn't popped from the virtqueue. The next
724 * call to virtqueue_pop() will refetch the element.
726 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
730 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
731 virtqueue_packed_rewind(vq
, 1);
733 virtqueue_split_rewind(vq
, 1);
736 virtqueue_detach_element(vq
, elem
, len
);
740 * @vq: The #VirtQueue
741 * @num: Number of elements to push back
743 * Pretend that elements weren't popped from the virtqueue. The next
744 * virtqueue_pop() will refetch the oldest element.
746 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
748 * Returns: true on success, false if @num is greater than the number of in use
751 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
753 if (num
> vq
->inuse
) {
758 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
759 virtqueue_packed_rewind(vq
, num
);
761 virtqueue_split_rewind(vq
, num
);
766 static void virtqueue_split_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
767 unsigned int len
, unsigned int idx
)
771 if (unlikely(!vq
->vring
.used
)) {
775 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
777 uelem
.id
= elem
->index
;
779 vring_used_write(vq
, &uelem
, idx
);
782 static void virtqueue_packed_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
783 unsigned int len
, unsigned int idx
)
785 vq
->used_elems
[idx
].index
= elem
->index
;
786 vq
->used_elems
[idx
].len
= len
;
787 vq
->used_elems
[idx
].ndescs
= elem
->ndescs
;
790 static void virtqueue_packed_fill_desc(VirtQueue
*vq
,
791 const VirtQueueElement
*elem
,
796 VRingMemoryRegionCaches
*caches
;
797 VRingPackedDesc desc
= {
801 bool wrap_counter
= vq
->used_wrap_counter
;
803 if (unlikely(!vq
->vring
.desc
)) {
807 head
= vq
->used_idx
+ idx
;
808 if (head
>= vq
->vring
.num
) {
809 head
-= vq
->vring
.num
;
813 desc
.flags
|= (1 << VRING_PACKED_DESC_F_AVAIL
);
814 desc
.flags
|= (1 << VRING_PACKED_DESC_F_USED
);
816 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_AVAIL
);
817 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_USED
);
820 caches
= vring_get_region_caches(vq
);
825 vring_packed_desc_write(vq
->vdev
, &desc
, &caches
->desc
, head
, strict_order
);
828 /* Called within rcu_read_lock(). */
829 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
830 unsigned int len
, unsigned int idx
)
832 trace_virtqueue_fill(vq
, elem
, len
, idx
);
834 virtqueue_unmap_sg(vq
, elem
, len
);
836 if (virtio_device_disabled(vq
->vdev
)) {
840 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
841 virtqueue_packed_fill(vq
, elem
, len
, idx
);
843 virtqueue_split_fill(vq
, elem
, len
, idx
);
847 /* Called within rcu_read_lock(). */
848 static void virtqueue_split_flush(VirtQueue
*vq
, unsigned int count
)
852 if (unlikely(!vq
->vring
.used
)) {
856 /* Make sure buffer is written before we update index. */
858 trace_virtqueue_flush(vq
, count
);
861 vring_used_idx_set(vq
, new);
863 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
864 vq
->signalled_used_valid
= false;
867 static void virtqueue_packed_flush(VirtQueue
*vq
, unsigned int count
)
869 unsigned int i
, ndescs
= 0;
871 if (unlikely(!vq
->vring
.desc
)) {
875 for (i
= 1; i
< count
; i
++) {
876 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[i
], i
, false);
877 ndescs
+= vq
->used_elems
[i
].ndescs
;
879 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[0], 0, true);
880 ndescs
+= vq
->used_elems
[0].ndescs
;
883 vq
->used_idx
+= ndescs
;
884 if (vq
->used_idx
>= vq
->vring
.num
) {
885 vq
->used_idx
-= vq
->vring
.num
;
886 vq
->used_wrap_counter
^= 1;
887 vq
->signalled_used_valid
= false;
891 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
893 if (virtio_device_disabled(vq
->vdev
)) {
898 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
899 virtqueue_packed_flush(vq
, count
);
901 virtqueue_split_flush(vq
, count
);
905 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
908 RCU_READ_LOCK_GUARD();
909 virtqueue_fill(vq
, elem
, len
, 0);
910 virtqueue_flush(vq
, 1);
913 /* Called within rcu_read_lock(). */
914 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
916 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
918 /* Check it isn't doing very strange things with descriptor numbers. */
919 if (num_heads
> vq
->vring
.num
) {
920 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
921 idx
, vq
->shadow_avail_idx
);
924 /* On success, callers read a descriptor at vq->last_avail_idx.
925 * Make sure descriptor read does not bypass avail index read. */
933 /* Called within rcu_read_lock(). */
934 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
937 /* Grab the next descriptor number they're advertising, and increment
938 * the index we've seen. */
939 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
941 /* If their number is silly, that's a fatal mistake. */
942 if (*head
>= vq
->vring
.num
) {
943 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
951 VIRTQUEUE_READ_DESC_ERROR
= -1,
952 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
953 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
956 static int virtqueue_split_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
957 MemoryRegionCache
*desc_cache
,
958 unsigned int max
, unsigned int *next
)
960 /* If this descriptor says it doesn't chain, we're done. */
961 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
962 return VIRTQUEUE_READ_DESC_DONE
;
965 /* Check they're not leading us off end of descriptors. */
967 /* Make sure compiler knows to grab that: we don't want it changing! */
971 virtio_error(vdev
, "Desc next is %u", *next
);
972 return VIRTQUEUE_READ_DESC_ERROR
;
975 vring_split_desc_read(vdev
, desc
, desc_cache
, *next
);
976 return VIRTQUEUE_READ_DESC_MORE
;
979 /* Called within rcu_read_lock(). */
980 static void virtqueue_split_get_avail_bytes(VirtQueue
*vq
,
981 unsigned int *in_bytes
, unsigned int *out_bytes
,
982 unsigned max_in_bytes
, unsigned max_out_bytes
,
983 VRingMemoryRegionCaches
*caches
)
985 VirtIODevice
*vdev
= vq
->vdev
;
986 unsigned int max
, idx
;
987 unsigned int total_bufs
, in_total
, out_total
;
988 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
992 idx
= vq
->last_avail_idx
;
993 total_bufs
= in_total
= out_total
= 0;
997 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
998 MemoryRegionCache
*desc_cache
= &caches
->desc
;
999 unsigned int num_bufs
;
1003 num_bufs
= total_bufs
;
1005 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
1009 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1011 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1012 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1013 virtio_error(vdev
, "Invalid size for indirect buffer table");
1017 /* If we've got too many, that implies a descriptor loop. */
1018 if (num_bufs
>= max
) {
1019 virtio_error(vdev
, "Looped descriptor");
1023 /* loop over the indirect descriptor table */
1024 len
= address_space_cache_init(&indirect_desc_cache
,
1026 desc
.addr
, desc
.len
, false);
1027 desc_cache
= &indirect_desc_cache
;
1028 if (len
< desc
.len
) {
1029 virtio_error(vdev
, "Cannot map indirect buffer");
1033 max
= desc
.len
/ sizeof(VRingDesc
);
1035 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1039 /* If we've got too many, that implies a descriptor loop. */
1040 if (++num_bufs
> max
) {
1041 virtio_error(vdev
, "Looped descriptor");
1045 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1046 in_total
+= desc
.len
;
1048 out_total
+= desc
.len
;
1050 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1054 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1055 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1057 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1061 if (desc_cache
== &indirect_desc_cache
) {
1062 address_space_cache_destroy(&indirect_desc_cache
);
1065 total_bufs
= num_bufs
;
1074 address_space_cache_destroy(&indirect_desc_cache
);
1076 *in_bytes
= in_total
;
1079 *out_bytes
= out_total
;
1084 in_total
= out_total
= 0;
1088 static int virtqueue_packed_read_next_desc(VirtQueue
*vq
,
1089 VRingPackedDesc
*desc
,
1096 /* If this descriptor says it doesn't chain, we're done. */
1097 if (!indirect
&& !(desc
->flags
& VRING_DESC_F_NEXT
)) {
1098 return VIRTQUEUE_READ_DESC_DONE
;
1104 return VIRTQUEUE_READ_DESC_DONE
;
1106 (*next
) -= vq
->vring
.num
;
1110 vring_packed_desc_read(vq
->vdev
, desc
, desc_cache
, *next
, false);
1111 return VIRTQUEUE_READ_DESC_MORE
;
1114 /* Called within rcu_read_lock(). */
1115 static void virtqueue_packed_get_avail_bytes(VirtQueue
*vq
,
1116 unsigned int *in_bytes
,
1117 unsigned int *out_bytes
,
1118 unsigned max_in_bytes
,
1119 unsigned max_out_bytes
,
1120 VRingMemoryRegionCaches
*caches
)
1122 VirtIODevice
*vdev
= vq
->vdev
;
1123 unsigned int max
, idx
;
1124 unsigned int total_bufs
, in_total
, out_total
;
1125 MemoryRegionCache
*desc_cache
;
1126 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1128 VRingPackedDesc desc
;
1131 idx
= vq
->last_avail_idx
;
1132 wrap_counter
= vq
->last_avail_wrap_counter
;
1133 total_bufs
= in_total
= out_total
= 0;
1135 max
= vq
->vring
.num
;
1138 unsigned int num_bufs
= total_bufs
;
1139 unsigned int i
= idx
;
1142 desc_cache
= &caches
->desc
;
1143 vring_packed_desc_read(vdev
, &desc
, desc_cache
, idx
, true);
1144 if (!is_desc_avail(desc
.flags
, wrap_counter
)) {
1148 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1149 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1150 virtio_error(vdev
, "Invalid size for indirect buffer table");
1154 /* If we've got too many, that implies a descriptor loop. */
1155 if (num_bufs
>= max
) {
1156 virtio_error(vdev
, "Looped descriptor");
1160 /* loop over the indirect descriptor table */
1161 len
= address_space_cache_init(&indirect_desc_cache
,
1163 desc
.addr
, desc
.len
, false);
1164 desc_cache
= &indirect_desc_cache
;
1165 if (len
< desc
.len
) {
1166 virtio_error(vdev
, "Cannot map indirect buffer");
1170 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1172 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1176 /* If we've got too many, that implies a descriptor loop. */
1177 if (++num_bufs
> max
) {
1178 virtio_error(vdev
, "Looped descriptor");
1182 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1183 in_total
+= desc
.len
;
1185 out_total
+= desc
.len
;
1187 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1191 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
,
1193 &indirect_desc_cache
);
1194 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1196 if (desc_cache
== &indirect_desc_cache
) {
1197 address_space_cache_destroy(&indirect_desc_cache
);
1201 idx
+= num_bufs
- total_bufs
;
1202 total_bufs
= num_bufs
;
1205 if (idx
>= vq
->vring
.num
) {
1206 idx
-= vq
->vring
.num
;
1211 /* Record the index and wrap counter for a kick we want */
1212 vq
->shadow_avail_idx
= idx
;
1213 vq
->shadow_avail_wrap_counter
= wrap_counter
;
1215 address_space_cache_destroy(&indirect_desc_cache
);
1217 *in_bytes
= in_total
;
1220 *out_bytes
= out_total
;
1225 in_total
= out_total
= 0;
1229 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
1230 unsigned int *out_bytes
,
1231 unsigned max_in_bytes
, unsigned max_out_bytes
)
1234 VRingMemoryRegionCaches
*caches
;
1236 RCU_READ_LOCK_GUARD();
1238 if (unlikely(!vq
->vring
.desc
)) {
1242 caches
= vring_get_region_caches(vq
);
1247 desc_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
1248 sizeof(VRingPackedDesc
) : sizeof(VRingDesc
);
1249 if (caches
->desc
.len
< vq
->vring
.num
* desc_size
) {
1250 virtio_error(vq
->vdev
, "Cannot map descriptor ring");
1254 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1255 virtqueue_packed_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1256 max_in_bytes
, max_out_bytes
,
1259 virtqueue_split_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1260 max_in_bytes
, max_out_bytes
,
1274 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
1275 unsigned int out_bytes
)
1277 unsigned int in_total
, out_total
;
1279 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
1280 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
1283 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
1284 hwaddr
*addr
, struct iovec
*iov
,
1285 unsigned int max_num_sg
, bool is_write
,
1286 hwaddr pa
, size_t sz
)
1289 unsigned num_sg
= *p_num_sg
;
1290 assert(num_sg
<= max_num_sg
);
1293 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
1300 if (num_sg
== max_num_sg
) {
1301 virtio_error(vdev
, "virtio: too many write descriptors in "
1306 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
1308 DMA_DIRECTION_FROM_DEVICE
:
1309 DMA_DIRECTION_TO_DEVICE
,
1310 MEMTXATTRS_UNSPECIFIED
);
1311 if (!iov
[num_sg
].iov_base
) {
1312 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
1316 iov
[num_sg
].iov_len
= len
;
1330 /* Only used by error code paths before we have a VirtQueueElement (therefore
1331 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1334 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
1339 for (i
= 0; i
< out_num
+ in_num
; i
++) {
1340 int is_write
= i
>= out_num
;
1342 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
1347 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
1348 hwaddr
*addr
, unsigned int num_sg
,
1354 for (i
= 0; i
< num_sg
; i
++) {
1355 len
= sg
[i
].iov_len
;
1356 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
1357 addr
[i
], &len
, is_write
?
1358 DMA_DIRECTION_FROM_DEVICE
:
1359 DMA_DIRECTION_TO_DEVICE
,
1360 MEMTXATTRS_UNSPECIFIED
);
1361 if (!sg
[i
].iov_base
) {
1362 error_report("virtio: error trying to map MMIO memory");
1365 if (len
!= sg
[i
].iov_len
) {
1366 error_report("virtio: unexpected memory split");
1372 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
1374 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, elem
->in_num
, true);
1375 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, elem
->out_num
,
1379 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
1381 VirtQueueElement
*elem
;
1382 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
1383 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
1384 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
1385 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
1386 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
1387 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
1389 assert(sz
>= sizeof(VirtQueueElement
));
1390 elem
= g_malloc(out_sg_end
);
1391 trace_virtqueue_alloc_element(elem
, sz
, in_num
, out_num
);
1392 elem
->out_num
= out_num
;
1393 elem
->in_num
= in_num
;
1394 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
1395 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
1396 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
1397 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
1401 static void *virtqueue_split_pop(VirtQueue
*vq
, size_t sz
)
1403 unsigned int i
, head
, max
;
1404 VRingMemoryRegionCaches
*caches
;
1405 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1406 MemoryRegionCache
*desc_cache
;
1408 VirtIODevice
*vdev
= vq
->vdev
;
1409 VirtQueueElement
*elem
= NULL
;
1410 unsigned out_num
, in_num
, elem_entries
;
1411 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1412 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1416 RCU_READ_LOCK_GUARD();
1417 if (virtio_queue_empty_rcu(vq
)) {
1420 /* Needed after virtio_queue_empty(), see comment in
1421 * virtqueue_num_heads(). */
1424 /* When we start there are none of either input nor output. */
1425 out_num
= in_num
= elem_entries
= 0;
1427 max
= vq
->vring
.num
;
1429 if (vq
->inuse
>= vq
->vring
.num
) {
1430 virtio_error(vdev
, "Virtqueue size exceeded");
1434 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
1438 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1439 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1444 caches
= vring_get_region_caches(vq
);
1446 virtio_error(vdev
, "Region caches not initialized");
1450 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1451 virtio_error(vdev
, "Cannot map descriptor ring");
1455 desc_cache
= &caches
->desc
;
1456 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1457 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1458 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1459 virtio_error(vdev
, "Invalid size for indirect buffer table");
1463 /* loop over the indirect descriptor table */
1464 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1465 desc
.addr
, desc
.len
, false);
1466 desc_cache
= &indirect_desc_cache
;
1467 if (len
< desc
.len
) {
1468 virtio_error(vdev
, "Cannot map indirect buffer");
1472 max
= desc
.len
/ sizeof(VRingDesc
);
1474 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1477 /* Collect all the descriptors */
1481 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1482 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1484 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1485 desc
.addr
, desc
.len
);
1488 virtio_error(vdev
, "Incorrect order for descriptors");
1491 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1492 VIRTQUEUE_MAX_SIZE
, false,
1493 desc
.addr
, desc
.len
);
1499 /* If we've got too many, that implies a descriptor loop. */
1500 if (++elem_entries
> max
) {
1501 virtio_error(vdev
, "Looped descriptor");
1505 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1506 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1508 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1512 /* Now copy what we have collected and mapped */
1513 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1516 for (i
= 0; i
< out_num
; i
++) {
1517 elem
->out_addr
[i
] = addr
[i
];
1518 elem
->out_sg
[i
] = iov
[i
];
1520 for (i
= 0; i
< in_num
; i
++) {
1521 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1522 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1527 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1529 address_space_cache_destroy(&indirect_desc_cache
);
1534 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1538 static void *virtqueue_packed_pop(VirtQueue
*vq
, size_t sz
)
1540 unsigned int i
, max
;
1541 VRingMemoryRegionCaches
*caches
;
1542 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1543 MemoryRegionCache
*desc_cache
;
1545 VirtIODevice
*vdev
= vq
->vdev
;
1546 VirtQueueElement
*elem
= NULL
;
1547 unsigned out_num
, in_num
, elem_entries
;
1548 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1549 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1550 VRingPackedDesc desc
;
1554 RCU_READ_LOCK_GUARD();
1555 if (virtio_queue_packed_empty_rcu(vq
)) {
1559 /* When we start there are none of either input nor output. */
1560 out_num
= in_num
= elem_entries
= 0;
1562 max
= vq
->vring
.num
;
1564 if (vq
->inuse
>= vq
->vring
.num
) {
1565 virtio_error(vdev
, "Virtqueue size exceeded");
1569 i
= vq
->last_avail_idx
;
1571 caches
= vring_get_region_caches(vq
);
1573 virtio_error(vdev
, "Region caches not initialized");
1577 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1578 virtio_error(vdev
, "Cannot map descriptor ring");
1582 desc_cache
= &caches
->desc
;
1583 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, true);
1585 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1586 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1587 virtio_error(vdev
, "Invalid size for indirect buffer table");
1591 /* loop over the indirect descriptor table */
1592 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1593 desc
.addr
, desc
.len
, false);
1594 desc_cache
= &indirect_desc_cache
;
1595 if (len
< desc
.len
) {
1596 virtio_error(vdev
, "Cannot map indirect buffer");
1600 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1602 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1605 /* Collect all the descriptors */
1609 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1610 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1612 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1613 desc
.addr
, desc
.len
);
1616 virtio_error(vdev
, "Incorrect order for descriptors");
1619 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1620 VIRTQUEUE_MAX_SIZE
, false,
1621 desc
.addr
, desc
.len
);
1627 /* If we've got too many, that implies a descriptor loop. */
1628 if (++elem_entries
> max
) {
1629 virtio_error(vdev
, "Looped descriptor");
1633 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
, &i
,
1635 &indirect_desc_cache
);
1636 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1638 /* Now copy what we have collected and mapped */
1639 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1640 for (i
= 0; i
< out_num
; i
++) {
1641 elem
->out_addr
[i
] = addr
[i
];
1642 elem
->out_sg
[i
] = iov
[i
];
1644 for (i
= 0; i
< in_num
; i
++) {
1645 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1646 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1650 elem
->ndescs
= (desc_cache
== &indirect_desc_cache
) ? 1 : elem_entries
;
1651 vq
->last_avail_idx
+= elem
->ndescs
;
1652 vq
->inuse
+= elem
->ndescs
;
1654 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1655 vq
->last_avail_idx
-= vq
->vring
.num
;
1656 vq
->last_avail_wrap_counter
^= 1;
1659 vq
->shadow_avail_idx
= vq
->last_avail_idx
;
1660 vq
->shadow_avail_wrap_counter
= vq
->last_avail_wrap_counter
;
1662 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1664 address_space_cache_destroy(&indirect_desc_cache
);
1669 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1673 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
1675 if (virtio_device_disabled(vq
->vdev
)) {
1679 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1680 return virtqueue_packed_pop(vq
, sz
);
1682 return virtqueue_split_pop(vq
, sz
);
1686 static unsigned int virtqueue_packed_drop_all(VirtQueue
*vq
)
1688 VRingMemoryRegionCaches
*caches
;
1689 MemoryRegionCache
*desc_cache
;
1690 unsigned int dropped
= 0;
1691 VirtQueueElement elem
= {};
1692 VirtIODevice
*vdev
= vq
->vdev
;
1693 VRingPackedDesc desc
;
1695 RCU_READ_LOCK_GUARD();
1697 caches
= vring_get_region_caches(vq
);
1702 desc_cache
= &caches
->desc
;
1704 virtio_queue_set_notification(vq
, 0);
1706 while (vq
->inuse
< vq
->vring
.num
) {
1707 unsigned int idx
= vq
->last_avail_idx
;
1709 * works similar to virtqueue_pop but does not map buffers
1710 * and does not allocate any memory.
1712 vring_packed_desc_read(vdev
, &desc
, desc_cache
,
1713 vq
->last_avail_idx
, true);
1714 if (!is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
)) {
1717 elem
.index
= desc
.id
;
1719 while (virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
,
1720 vq
->vring
.num
, &idx
, false)) {
1724 * immediately push the element, nothing to unmap
1725 * as both in_num and out_num are set to 0.
1727 virtqueue_push(vq
, &elem
, 0);
1729 vq
->last_avail_idx
+= elem
.ndescs
;
1730 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1731 vq
->last_avail_idx
-= vq
->vring
.num
;
1732 vq
->last_avail_wrap_counter
^= 1;
1739 static unsigned int virtqueue_split_drop_all(VirtQueue
*vq
)
1741 unsigned int dropped
= 0;
1742 VirtQueueElement elem
= {};
1743 VirtIODevice
*vdev
= vq
->vdev
;
1744 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
1746 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
1747 /* works similar to virtqueue_pop but does not map buffers
1748 * and does not allocate any memory */
1750 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
1754 vq
->last_avail_idx
++;
1756 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1758 /* immediately push the element, nothing to unmap
1759 * as both in_num and out_num are set to 0 */
1760 virtqueue_push(vq
, &elem
, 0);
1767 /* virtqueue_drop_all:
1768 * @vq: The #VirtQueue
1769 * Drops all queued buffers and indicates them to the guest
1770 * as if they are done. Useful when buffers can not be
1771 * processed but must be returned to the guest.
1773 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
1775 struct VirtIODevice
*vdev
= vq
->vdev
;
1777 if (virtio_device_disabled(vq
->vdev
)) {
1781 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1782 return virtqueue_packed_drop_all(vq
);
1784 return virtqueue_split_drop_all(vq
);
1788 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1789 * it is what QEMU has always done by mistake. We can change it sooner
1790 * or later by bumping the version number of the affected vm states.
1791 * In the meanwhile, since the in-memory layout of VirtQueueElement
1792 * has changed, we need to marshal to and from the layout that was
1793 * used before the change.
1795 typedef struct VirtQueueElementOld
{
1797 unsigned int out_num
;
1798 unsigned int in_num
;
1799 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
1800 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
1801 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
1802 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
1803 } VirtQueueElementOld
;
1805 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
1807 VirtQueueElement
*elem
;
1808 VirtQueueElementOld data
;
1811 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1813 /* TODO: teach all callers that this can fail, and return failure instead
1814 * of asserting here.
1815 * This is just one thing (there are probably more) that must be
1816 * fixed before we can allow NDEBUG compilation.
1818 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
1819 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
1821 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
1822 elem
->index
= data
.index
;
1824 for (i
= 0; i
< elem
->in_num
; i
++) {
1825 elem
->in_addr
[i
] = data
.in_addr
[i
];
1828 for (i
= 0; i
< elem
->out_num
; i
++) {
1829 elem
->out_addr
[i
] = data
.out_addr
[i
];
1832 for (i
= 0; i
< elem
->in_num
; i
++) {
1833 /* Base is overwritten by virtqueue_map. */
1834 elem
->in_sg
[i
].iov_base
= 0;
1835 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
1838 for (i
= 0; i
< elem
->out_num
; i
++) {
1839 /* Base is overwritten by virtqueue_map. */
1840 elem
->out_sg
[i
].iov_base
= 0;
1841 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
1844 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1845 qemu_get_be32s(f
, &elem
->ndescs
);
1848 virtqueue_map(vdev
, elem
);
1852 void qemu_put_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
,
1853 VirtQueueElement
*elem
)
1855 VirtQueueElementOld data
;
1858 memset(&data
, 0, sizeof(data
));
1859 data
.index
= elem
->index
;
1860 data
.in_num
= elem
->in_num
;
1861 data
.out_num
= elem
->out_num
;
1863 for (i
= 0; i
< elem
->in_num
; i
++) {
1864 data
.in_addr
[i
] = elem
->in_addr
[i
];
1867 for (i
= 0; i
< elem
->out_num
; i
++) {
1868 data
.out_addr
[i
] = elem
->out_addr
[i
];
1871 for (i
= 0; i
< elem
->in_num
; i
++) {
1872 /* Base is overwritten by virtqueue_map when loading. Do not
1873 * save it, as it would leak the QEMU address space layout. */
1874 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
1877 for (i
= 0; i
< elem
->out_num
; i
++) {
1878 /* Do not save iov_base as above. */
1879 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
1882 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1883 qemu_put_be32s(f
, &elem
->ndescs
);
1886 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1890 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
1892 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1893 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1895 if (virtio_device_disabled(vdev
)) {
1900 k
->notify(qbus
->parent
, vector
);
1904 void virtio_update_irq(VirtIODevice
*vdev
)
1906 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1909 static int virtio_validate_features(VirtIODevice
*vdev
)
1911 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1913 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
1914 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
1918 if (k
->validate_features
) {
1919 return k
->validate_features(vdev
);
1925 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
1927 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1928 trace_virtio_set_status(vdev
, val
);
1930 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1931 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
1932 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1933 int ret
= virtio_validate_features(vdev
);
1941 if ((vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
) !=
1942 (val
& VIRTIO_CONFIG_S_DRIVER_OK
)) {
1943 virtio_set_started(vdev
, val
& VIRTIO_CONFIG_S_DRIVER_OK
);
1946 if (k
->set_status
) {
1947 k
->set_status(vdev
, val
);
1954 static enum virtio_device_endian
virtio_default_endian(void)
1956 if (target_words_bigendian()) {
1957 return VIRTIO_DEVICE_ENDIAN_BIG
;
1959 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1963 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1965 if (cpu_virtio_is_big_endian(current_cpu
)) {
1966 return VIRTIO_DEVICE_ENDIAN_BIG
;
1968 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1972 void virtio_reset(void *opaque
)
1974 VirtIODevice
*vdev
= opaque
;
1975 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1978 virtio_set_status(vdev
, 0);
1980 /* Guest initiated reset */
1981 vdev
->device_endian
= virtio_current_cpu_endian();
1984 vdev
->device_endian
= virtio_default_endian();
1991 vdev
->start_on_kick
= false;
1992 vdev
->started
= false;
1993 vdev
->broken
= false;
1994 vdev
->guest_features
= 0;
1995 vdev
->queue_sel
= 0;
1997 vdev
->disabled
= false;
1998 qatomic_set(&vdev
->isr
, 0);
1999 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2000 virtio_notify_vector(vdev
, vdev
->config_vector
);
2002 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2003 vdev
->vq
[i
].vring
.desc
= 0;
2004 vdev
->vq
[i
].vring
.avail
= 0;
2005 vdev
->vq
[i
].vring
.used
= 0;
2006 vdev
->vq
[i
].last_avail_idx
= 0;
2007 vdev
->vq
[i
].shadow_avail_idx
= 0;
2008 vdev
->vq
[i
].used_idx
= 0;
2009 vdev
->vq
[i
].last_avail_wrap_counter
= true;
2010 vdev
->vq
[i
].shadow_avail_wrap_counter
= true;
2011 vdev
->vq
[i
].used_wrap_counter
= true;
2012 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
2013 vdev
->vq
[i
].signalled_used
= 0;
2014 vdev
->vq
[i
].signalled_used_valid
= false;
2015 vdev
->vq
[i
].notification
= true;
2016 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
2017 vdev
->vq
[i
].inuse
= 0;
2018 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
2022 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
2024 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2027 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2028 return (uint32_t)-1;
2031 k
->get_config(vdev
, vdev
->config
);
2033 val
= ldub_p(vdev
->config
+ addr
);
2037 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
2039 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2042 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2043 return (uint32_t)-1;
2046 k
->get_config(vdev
, vdev
->config
);
2048 val
= lduw_p(vdev
->config
+ addr
);
2052 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
2054 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2057 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2058 return (uint32_t)-1;
2061 k
->get_config(vdev
, vdev
->config
);
2063 val
= ldl_p(vdev
->config
+ addr
);
2067 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2069 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2072 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2076 stb_p(vdev
->config
+ addr
, val
);
2078 if (k
->set_config
) {
2079 k
->set_config(vdev
, vdev
->config
);
2083 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2085 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2086 uint16_t val
= data
;
2088 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2092 stw_p(vdev
->config
+ addr
, val
);
2094 if (k
->set_config
) {
2095 k
->set_config(vdev
, vdev
->config
);
2099 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2101 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2102 uint32_t val
= data
;
2104 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2108 stl_p(vdev
->config
+ addr
, val
);
2110 if (k
->set_config
) {
2111 k
->set_config(vdev
, vdev
->config
);
2115 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
2117 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2120 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2121 return (uint32_t)-1;
2124 k
->get_config(vdev
, vdev
->config
);
2126 val
= ldub_p(vdev
->config
+ addr
);
2130 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
2132 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2135 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2136 return (uint32_t)-1;
2139 k
->get_config(vdev
, vdev
->config
);
2141 val
= lduw_le_p(vdev
->config
+ addr
);
2145 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
2147 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2150 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2151 return (uint32_t)-1;
2154 k
->get_config(vdev
, vdev
->config
);
2156 val
= ldl_le_p(vdev
->config
+ addr
);
2160 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
2161 uint32_t addr
, uint32_t data
)
2163 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2166 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2170 stb_p(vdev
->config
+ addr
, val
);
2172 if (k
->set_config
) {
2173 k
->set_config(vdev
, vdev
->config
);
2177 void virtio_config_modern_writew(VirtIODevice
*vdev
,
2178 uint32_t addr
, uint32_t data
)
2180 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2181 uint16_t val
= data
;
2183 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2187 stw_le_p(vdev
->config
+ addr
, val
);
2189 if (k
->set_config
) {
2190 k
->set_config(vdev
, vdev
->config
);
2194 void virtio_config_modern_writel(VirtIODevice
*vdev
,
2195 uint32_t addr
, uint32_t data
)
2197 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2198 uint32_t val
= data
;
2200 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2204 stl_le_p(vdev
->config
+ addr
, val
);
2206 if (k
->set_config
) {
2207 k
->set_config(vdev
, vdev
->config
);
2211 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
2213 if (!vdev
->vq
[n
].vring
.num
) {
2216 vdev
->vq
[n
].vring
.desc
= addr
;
2217 virtio_queue_update_rings(vdev
, n
);
2220 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
2222 return vdev
->vq
[n
].vring
.desc
;
2225 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
2226 hwaddr avail
, hwaddr used
)
2228 if (!vdev
->vq
[n
].vring
.num
) {
2231 vdev
->vq
[n
].vring
.desc
= desc
;
2232 vdev
->vq
[n
].vring
.avail
= avail
;
2233 vdev
->vq
[n
].vring
.used
= used
;
2234 virtio_init_region_cache(vdev
, n
);
2237 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
2239 /* Don't allow guest to flip queue between existent and
2240 * nonexistent states, or to set it to an invalid size.
2242 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
2243 num
> VIRTQUEUE_MAX_SIZE
||
2247 vdev
->vq
[n
].vring
.num
= num
;
2250 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
2252 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
2255 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
2257 return QLIST_NEXT(vq
, node
);
2260 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
2262 return vdev
->vq
[n
].vring
.num
;
2265 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
2267 return vdev
->vq
[n
].vring
.num_default
;
2270 int virtio_get_num_queues(VirtIODevice
*vdev
)
2274 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2275 if (!virtio_queue_get_num(vdev
, i
)) {
2283 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
2285 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2286 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2288 /* virtio-1 compliant devices cannot change the alignment */
2289 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2290 error_report("tried to modify queue alignment for virtio-1 device");
2293 /* Check that the transport told us it was going to do this
2294 * (so a buggy transport will immediately assert rather than
2295 * silently failing to migrate this state)
2297 assert(k
->has_variable_vring_alignment
);
2300 vdev
->vq
[n
].vring
.align
= align
;
2301 virtio_queue_update_rings(vdev
, n
);
2305 static void virtio_queue_notify_vq(VirtQueue
*vq
)
2307 if (vq
->vring
.desc
&& vq
->handle_output
) {
2308 VirtIODevice
*vdev
= vq
->vdev
;
2310 if (unlikely(vdev
->broken
)) {
2314 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2315 vq
->handle_output(vdev
, vq
);
2317 if (unlikely(vdev
->start_on_kick
)) {
2318 virtio_set_started(vdev
, true);
2323 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
2325 VirtQueue
*vq
= &vdev
->vq
[n
];
2327 if (unlikely(!vq
->vring
.desc
|| vdev
->broken
)) {
2331 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2332 if (vq
->host_notifier_enabled
) {
2333 event_notifier_set(&vq
->host_notifier
);
2334 } else if (vq
->handle_output
) {
2335 vq
->handle_output(vdev
, vq
);
2337 if (unlikely(vdev
->start_on_kick
)) {
2338 virtio_set_started(vdev
, true);
2343 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
2345 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
2349 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
2351 VirtQueue
*vq
= &vdev
->vq
[n
];
2353 if (n
< VIRTIO_QUEUE_MAX
) {
2354 if (vdev
->vector_queues
&&
2355 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
2356 QLIST_REMOVE(vq
, node
);
2358 vdev
->vq
[n
].vector
= vector
;
2359 if (vdev
->vector_queues
&&
2360 vector
!= VIRTIO_NO_VECTOR
) {
2361 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
2366 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
2367 VirtIOHandleOutput handle_output
)
2371 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2372 if (vdev
->vq
[i
].vring
.num
== 0)
2376 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
2379 vdev
->vq
[i
].vring
.num
= queue_size
;
2380 vdev
->vq
[i
].vring
.num_default
= queue_size
;
2381 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
2382 vdev
->vq
[i
].handle_output
= handle_output
;
2383 vdev
->vq
[i
].used_elems
= g_malloc0(sizeof(VirtQueueElement
) *
2386 return &vdev
->vq
[i
];
2389 void virtio_delete_queue(VirtQueue
*vq
)
2392 vq
->vring
.num_default
= 0;
2393 vq
->handle_output
= NULL
;
2394 g_free(vq
->used_elems
);
2395 vq
->used_elems
= NULL
;
2396 virtio_virtqueue_reset_region_cache(vq
);
2399 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
2401 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
2405 virtio_delete_queue(&vdev
->vq
[n
]);
2408 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
2410 uint8_t old
= qatomic_read(&vdev
->isr
);
2412 /* Do not write ISR if it does not change, so that its cacheline remains
2413 * shared in the common case where the guest does not read it.
2415 if ((old
& value
) != value
) {
2416 qatomic_or(&vdev
->isr
, value
);
2420 /* Called within rcu_read_lock(). */
2421 static bool virtio_split_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2425 /* We need to expose used array entries before checking used event. */
2427 /* Always notify when queue is empty (when feature acknowledge) */
2428 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
2429 !vq
->inuse
&& virtio_queue_empty(vq
)) {
2433 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2434 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
2437 v
= vq
->signalled_used_valid
;
2438 vq
->signalled_used_valid
= true;
2439 old
= vq
->signalled_used
;
2440 new = vq
->signalled_used
= vq
->used_idx
;
2441 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
2444 static bool vring_packed_need_event(VirtQueue
*vq
, bool wrap
,
2445 uint16_t off_wrap
, uint16_t new,
2448 int off
= off_wrap
& ~(1 << 15);
2450 if (wrap
!= off_wrap
>> 15) {
2451 off
-= vq
->vring
.num
;
2454 return vring_need_event(off
, new, old
);
2457 /* Called within rcu_read_lock(). */
2458 static bool virtio_packed_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2460 VRingPackedDescEvent e
;
2463 VRingMemoryRegionCaches
*caches
;
2465 caches
= vring_get_region_caches(vq
);
2470 vring_packed_event_read(vdev
, &caches
->avail
, &e
);
2472 old
= vq
->signalled_used
;
2473 new = vq
->signalled_used
= vq
->used_idx
;
2474 v
= vq
->signalled_used_valid
;
2475 vq
->signalled_used_valid
= true;
2477 if (e
.flags
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
2479 } else if (e
.flags
== VRING_PACKED_EVENT_FLAG_ENABLE
) {
2483 return !v
|| vring_packed_need_event(vq
, vq
->used_wrap_counter
,
2484 e
.off_wrap
, new, old
);
2487 /* Called within rcu_read_lock(). */
2488 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2490 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
2491 return virtio_packed_should_notify(vdev
, vq
);
2493 return virtio_split_should_notify(vdev
, vq
);
2497 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
2499 WITH_RCU_READ_LOCK_GUARD() {
2500 if (!virtio_should_notify(vdev
, vq
)) {
2505 trace_virtio_notify_irqfd(vdev
, vq
);
2508 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2509 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2510 * incorrectly polling this bit during crashdump and hibernation
2511 * in MSI mode, causing a hang if this bit is never updated.
2512 * Recent releases of Windows do not really shut down, but rather
2513 * log out and hibernate to make the next startup faster. Hence,
2514 * this manifested as a more serious hang during shutdown with
2516 * Next driver release from 2016 fixed this problem, so working around it
2517 * is not a must, but it's easy to do so let's do it here.
2519 * Note: it's safe to update ISR from any thread as it was switched
2520 * to an atomic operation.
2522 virtio_set_isr(vq
->vdev
, 0x1);
2523 event_notifier_set(&vq
->guest_notifier
);
2526 static void virtio_irq(VirtQueue
*vq
)
2528 virtio_set_isr(vq
->vdev
, 0x1);
2529 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2532 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2534 WITH_RCU_READ_LOCK_GUARD() {
2535 if (!virtio_should_notify(vdev
, vq
)) {
2540 trace_virtio_notify(vdev
, vq
);
2544 void virtio_notify_config(VirtIODevice
*vdev
)
2546 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
2549 virtio_set_isr(vdev
, 0x3);
2551 virtio_notify_vector(vdev
, vdev
->config_vector
);
2554 static bool virtio_device_endian_needed(void *opaque
)
2556 VirtIODevice
*vdev
= opaque
;
2558 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
2559 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2560 return vdev
->device_endian
!= virtio_default_endian();
2562 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2563 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
2566 static bool virtio_64bit_features_needed(void *opaque
)
2568 VirtIODevice
*vdev
= opaque
;
2570 return (vdev
->host_features
>> 32) != 0;
2573 static bool virtio_virtqueue_needed(void *opaque
)
2575 VirtIODevice
*vdev
= opaque
;
2577 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
2580 static bool virtio_packed_virtqueue_needed(void *opaque
)
2582 VirtIODevice
*vdev
= opaque
;
2584 return virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
);
2587 static bool virtio_ringsize_needed(void *opaque
)
2589 VirtIODevice
*vdev
= opaque
;
2592 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2593 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
2600 static bool virtio_extra_state_needed(void *opaque
)
2602 VirtIODevice
*vdev
= opaque
;
2603 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2604 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2606 return k
->has_extra_state
&&
2607 k
->has_extra_state(qbus
->parent
);
2610 static bool virtio_broken_needed(void *opaque
)
2612 VirtIODevice
*vdev
= opaque
;
2614 return vdev
->broken
;
2617 static bool virtio_started_needed(void *opaque
)
2619 VirtIODevice
*vdev
= opaque
;
2621 return vdev
->started
;
2624 static bool virtio_disabled_needed(void *opaque
)
2626 VirtIODevice
*vdev
= opaque
;
2628 return vdev
->disabled
;
2631 static const VMStateDescription vmstate_virtqueue
= {
2632 .name
= "virtqueue_state",
2634 .minimum_version_id
= 1,
2635 .fields
= (VMStateField
[]) {
2636 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
2637 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
2638 VMSTATE_END_OF_LIST()
2642 static const VMStateDescription vmstate_packed_virtqueue
= {
2643 .name
= "packed_virtqueue_state",
2645 .minimum_version_id
= 1,
2646 .fields
= (VMStateField
[]) {
2647 VMSTATE_UINT16(last_avail_idx
, struct VirtQueue
),
2648 VMSTATE_BOOL(last_avail_wrap_counter
, struct VirtQueue
),
2649 VMSTATE_UINT16(used_idx
, struct VirtQueue
),
2650 VMSTATE_BOOL(used_wrap_counter
, struct VirtQueue
),
2651 VMSTATE_UINT32(inuse
, struct VirtQueue
),
2652 VMSTATE_END_OF_LIST()
2656 static const VMStateDescription vmstate_virtio_virtqueues
= {
2657 .name
= "virtio/virtqueues",
2659 .minimum_version_id
= 1,
2660 .needed
= &virtio_virtqueue_needed
,
2661 .fields
= (VMStateField
[]) {
2662 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2663 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
2664 VMSTATE_END_OF_LIST()
2668 static const VMStateDescription vmstate_virtio_packed_virtqueues
= {
2669 .name
= "virtio/packed_virtqueues",
2671 .minimum_version_id
= 1,
2672 .needed
= &virtio_packed_virtqueue_needed
,
2673 .fields
= (VMStateField
[]) {
2674 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2675 VIRTIO_QUEUE_MAX
, 0, vmstate_packed_virtqueue
, VirtQueue
),
2676 VMSTATE_END_OF_LIST()
2680 static const VMStateDescription vmstate_ringsize
= {
2681 .name
= "ringsize_state",
2683 .minimum_version_id
= 1,
2684 .fields
= (VMStateField
[]) {
2685 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
2686 VMSTATE_END_OF_LIST()
2690 static const VMStateDescription vmstate_virtio_ringsize
= {
2691 .name
= "virtio/ringsize",
2693 .minimum_version_id
= 1,
2694 .needed
= &virtio_ringsize_needed
,
2695 .fields
= (VMStateField
[]) {
2696 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2697 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
2698 VMSTATE_END_OF_LIST()
2702 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2703 const VMStateField
*field
)
2705 VirtIODevice
*vdev
= pv
;
2706 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2707 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2709 if (!k
->load_extra_state
) {
2712 return k
->load_extra_state(qbus
->parent
, f
);
2716 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2717 const VMStateField
*field
, JSONWriter
*vmdesc
)
2719 VirtIODevice
*vdev
= pv
;
2720 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2721 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2723 k
->save_extra_state(qbus
->parent
, f
);
2727 static const VMStateInfo vmstate_info_extra_state
= {
2728 .name
= "virtqueue_extra_state",
2729 .get
= get_extra_state
,
2730 .put
= put_extra_state
,
2733 static const VMStateDescription vmstate_virtio_extra_state
= {
2734 .name
= "virtio/extra_state",
2736 .minimum_version_id
= 1,
2737 .needed
= &virtio_extra_state_needed
,
2738 .fields
= (VMStateField
[]) {
2740 .name
= "extra_state",
2742 .field_exists
= NULL
,
2744 .info
= &vmstate_info_extra_state
,
2745 .flags
= VMS_SINGLE
,
2748 VMSTATE_END_OF_LIST()
2752 static const VMStateDescription vmstate_virtio_device_endian
= {
2753 .name
= "virtio/device_endian",
2755 .minimum_version_id
= 1,
2756 .needed
= &virtio_device_endian_needed
,
2757 .fields
= (VMStateField
[]) {
2758 VMSTATE_UINT8(device_endian
, VirtIODevice
),
2759 VMSTATE_END_OF_LIST()
2763 static const VMStateDescription vmstate_virtio_64bit_features
= {
2764 .name
= "virtio/64bit_features",
2766 .minimum_version_id
= 1,
2767 .needed
= &virtio_64bit_features_needed
,
2768 .fields
= (VMStateField
[]) {
2769 VMSTATE_UINT64(guest_features
, VirtIODevice
),
2770 VMSTATE_END_OF_LIST()
2774 static const VMStateDescription vmstate_virtio_broken
= {
2775 .name
= "virtio/broken",
2777 .minimum_version_id
= 1,
2778 .needed
= &virtio_broken_needed
,
2779 .fields
= (VMStateField
[]) {
2780 VMSTATE_BOOL(broken
, VirtIODevice
),
2781 VMSTATE_END_OF_LIST()
2785 static const VMStateDescription vmstate_virtio_started
= {
2786 .name
= "virtio/started",
2788 .minimum_version_id
= 1,
2789 .needed
= &virtio_started_needed
,
2790 .fields
= (VMStateField
[]) {
2791 VMSTATE_BOOL(started
, VirtIODevice
),
2792 VMSTATE_END_OF_LIST()
2796 static const VMStateDescription vmstate_virtio_disabled
= {
2797 .name
= "virtio/disabled",
2799 .minimum_version_id
= 1,
2800 .needed
= &virtio_disabled_needed
,
2801 .fields
= (VMStateField
[]) {
2802 VMSTATE_BOOL(disabled
, VirtIODevice
),
2803 VMSTATE_END_OF_LIST()
2807 static const VMStateDescription vmstate_virtio
= {
2810 .minimum_version_id
= 1,
2811 .fields
= (VMStateField
[]) {
2812 VMSTATE_END_OF_LIST()
2814 .subsections
= (const VMStateDescription
*[]) {
2815 &vmstate_virtio_device_endian
,
2816 &vmstate_virtio_64bit_features
,
2817 &vmstate_virtio_virtqueues
,
2818 &vmstate_virtio_ringsize
,
2819 &vmstate_virtio_broken
,
2820 &vmstate_virtio_extra_state
,
2821 &vmstate_virtio_started
,
2822 &vmstate_virtio_packed_virtqueues
,
2823 &vmstate_virtio_disabled
,
2828 int virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
2830 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2831 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2832 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2833 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
2836 if (k
->save_config
) {
2837 k
->save_config(qbus
->parent
, f
);
2840 qemu_put_8s(f
, &vdev
->status
);
2841 qemu_put_8s(f
, &vdev
->isr
);
2842 qemu_put_be16s(f
, &vdev
->queue_sel
);
2843 qemu_put_be32s(f
, &guest_features_lo
);
2844 qemu_put_be32(f
, vdev
->config_len
);
2845 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
2847 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2848 if (vdev
->vq
[i
].vring
.num
== 0)
2852 qemu_put_be32(f
, i
);
2854 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2855 if (vdev
->vq
[i
].vring
.num
== 0)
2858 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
2859 if (k
->has_variable_vring_alignment
) {
2860 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
2863 * Save desc now, the rest of the ring addresses are saved in
2864 * subsections for VIRTIO-1 devices.
2866 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
2867 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2868 if (k
->save_queue
) {
2869 k
->save_queue(qbus
->parent
, i
, f
);
2873 if (vdc
->save
!= NULL
) {
2878 int ret
= vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
2885 return vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
2888 /* A wrapper for use as a VMState .put function */
2889 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
2890 const VMStateField
*field
, JSONWriter
*vmdesc
)
2892 return virtio_save(VIRTIO_DEVICE(opaque
), f
);
2895 /* A wrapper for use as a VMState .get function */
2896 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
2897 const VMStateField
*field
)
2899 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
2900 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
2902 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
2905 const VMStateInfo virtio_vmstate_info
= {
2907 .get
= virtio_device_get
,
2908 .put
= virtio_device_put
,
2911 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
2913 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2914 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
2916 val
&= vdev
->host_features
;
2917 if (k
->set_features
) {
2918 k
->set_features(vdev
, val
);
2920 vdev
->guest_features
= val
;
2921 return bad
? -1 : 0;
2924 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
2928 * The driver must not attempt to set features after feature negotiation
2931 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2934 ret
= virtio_set_features_nocheck(vdev
, val
);
2935 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2936 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2938 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2939 if (vdev
->vq
[i
].vring
.num
!= 0) {
2940 virtio_init_region_cache(vdev
, i
);
2945 if (!virtio_device_started(vdev
, vdev
->status
) &&
2946 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2947 vdev
->start_on_kick
= true;
2953 size_t virtio_feature_get_config_size(const VirtIOFeature
*feature_sizes
,
2954 uint64_t host_features
)
2956 size_t config_size
= 0;
2959 for (i
= 0; feature_sizes
[i
].flags
!= 0; i
++) {
2960 if (host_features
& feature_sizes
[i
].flags
) {
2961 config_size
= MAX(feature_sizes
[i
].end
, config_size
);
2968 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
2974 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2975 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2976 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2979 * We poison the endianness to ensure it does not get used before
2980 * subsections have been loaded.
2982 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
2984 if (k
->load_config
) {
2985 ret
= k
->load_config(qbus
->parent
, f
);
2990 qemu_get_8s(f
, &vdev
->status
);
2991 qemu_get_8s(f
, &vdev
->isr
);
2992 qemu_get_be16s(f
, &vdev
->queue_sel
);
2993 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
2996 qemu_get_be32s(f
, &features
);
2999 * Temporarily set guest_features low bits - needed by
3000 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3001 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3003 * Note: devices should always test host features in future - don't create
3004 * new dependencies like this.
3006 vdev
->guest_features
= features
;
3008 config_len
= qemu_get_be32(f
);
3011 * There are cases where the incoming config can be bigger or smaller
3012 * than what we have; so load what we have space for, and skip
3013 * any excess that's in the stream.
3015 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
3017 while (config_len
> vdev
->config_len
) {
3022 num
= qemu_get_be32(f
);
3024 if (num
> VIRTIO_QUEUE_MAX
) {
3025 error_report("Invalid number of virtqueues: 0x%x", num
);
3029 for (i
= 0; i
< num
; i
++) {
3030 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
3031 if (k
->has_variable_vring_alignment
) {
3032 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
3034 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
3035 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
3036 vdev
->vq
[i
].signalled_used_valid
= false;
3037 vdev
->vq
[i
].notification
= true;
3039 if (!vdev
->vq
[i
].vring
.desc
&& vdev
->vq
[i
].last_avail_idx
) {
3040 error_report("VQ %d address 0x0 "
3041 "inconsistent with Host index 0x%x",
3042 i
, vdev
->vq
[i
].last_avail_idx
);
3045 if (k
->load_queue
) {
3046 ret
= k
->load_queue(qbus
->parent
, i
, f
);
3052 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
3054 if (vdc
->load
!= NULL
) {
3055 ret
= vdc
->load(vdev
, f
, version_id
);
3062 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
3069 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
3074 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
3075 vdev
->device_endian
= virtio_default_endian();
3078 if (virtio_64bit_features_needed(vdev
)) {
3080 * Subsection load filled vdev->guest_features. Run them
3081 * through virtio_set_features to sanity-check them against
3084 uint64_t features64
= vdev
->guest_features
;
3085 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
3086 error_report("Features 0x%" PRIx64
" unsupported. "
3087 "Allowed features: 0x%" PRIx64
,
3088 features64
, vdev
->host_features
);
3092 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
3093 error_report("Features 0x%x unsupported. "
3094 "Allowed features: 0x%" PRIx64
,
3095 features
, vdev
->host_features
);
3100 if (!virtio_device_started(vdev
, vdev
->status
) &&
3101 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3102 vdev
->start_on_kick
= true;
3105 RCU_READ_LOCK_GUARD();
3106 for (i
= 0; i
< num
; i
++) {
3107 if (vdev
->vq
[i
].vring
.desc
) {
3111 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3112 * only the region cache needs to be set up. Legacy devices need
3113 * to calculate used and avail ring addresses based on the desc
3116 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3117 virtio_init_region_cache(vdev
, i
);
3119 virtio_queue_update_rings(vdev
, i
);
3122 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3123 vdev
->vq
[i
].shadow_avail_idx
= vdev
->vq
[i
].last_avail_idx
;
3124 vdev
->vq
[i
].shadow_avail_wrap_counter
=
3125 vdev
->vq
[i
].last_avail_wrap_counter
;
3129 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
3130 /* Check it isn't doing strange things with descriptor numbers. */
3131 if (nheads
> vdev
->vq
[i
].vring
.num
) {
3132 virtio_error(vdev
, "VQ %d size 0x%x Guest index 0x%x "
3133 "inconsistent with Host index 0x%x: delta 0x%x",
3134 i
, vdev
->vq
[i
].vring
.num
,
3135 vring_avail_idx(&vdev
->vq
[i
]),
3136 vdev
->vq
[i
].last_avail_idx
, nheads
);
3137 vdev
->vq
[i
].used_idx
= 0;
3138 vdev
->vq
[i
].shadow_avail_idx
= 0;
3139 vdev
->vq
[i
].inuse
= 0;
3142 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
3143 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
3146 * Some devices migrate VirtQueueElements that have been popped
3147 * from the avail ring but not yet returned to the used ring.
3148 * Since max ring size < UINT16_MAX it's safe to use modulo
3149 * UINT16_MAX + 1 subtraction.
3151 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
3152 vdev
->vq
[i
].used_idx
);
3153 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
3154 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3156 i
, vdev
->vq
[i
].vring
.num
,
3157 vdev
->vq
[i
].last_avail_idx
,
3158 vdev
->vq
[i
].used_idx
);
3164 if (vdc
->post_load
) {
3165 ret
= vdc
->post_load(vdev
);
3174 void virtio_cleanup(VirtIODevice
*vdev
)
3176 qemu_del_vm_change_state_handler(vdev
->vmstate
);
3179 static void virtio_vmstate_change(void *opaque
, bool running
, RunState state
)
3181 VirtIODevice
*vdev
= opaque
;
3182 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3183 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3184 bool backend_run
= running
&& virtio_device_started(vdev
, vdev
->status
);
3185 vdev
->vm_running
= running
;
3188 virtio_set_status(vdev
, vdev
->status
);
3191 if (k
->vmstate_change
) {
3192 k
->vmstate_change(qbus
->parent
, backend_run
);
3196 virtio_set_status(vdev
, vdev
->status
);
3200 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
3201 size_t vdev_size
, const char *vdev_name
)
3203 DeviceState
*vdev
= data
;
3205 object_initialize_child_with_props(proxy_obj
, "virtio-backend", vdev
,
3206 vdev_size
, vdev_name
, &error_abort
,
3208 qdev_alias_all_properties(vdev
, proxy_obj
);
3211 void virtio_init(VirtIODevice
*vdev
, const char *name
,
3212 uint16_t device_id
, size_t config_size
)
3214 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3215 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3217 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
3220 vdev
->vector_queues
=
3221 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
3224 vdev
->start_on_kick
= false;
3225 vdev
->started
= false;
3226 vdev
->device_id
= device_id
;
3228 qatomic_set(&vdev
->isr
, 0);
3229 vdev
->queue_sel
= 0;
3230 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
3231 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
3232 vdev
->vm_running
= runstate_is_running();
3233 vdev
->broken
= false;
3234 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3235 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
3236 vdev
->vq
[i
].vdev
= vdev
;
3237 vdev
->vq
[i
].queue_index
= i
;
3238 vdev
->vq
[i
].host_notifier_enabled
= false;
3242 vdev
->config_len
= config_size
;
3243 if (vdev
->config_len
) {
3244 vdev
->config
= g_malloc0(config_size
);
3246 vdev
->config
= NULL
;
3248 vdev
->vmstate
= qdev_add_vm_change_state_handler(DEVICE(vdev
),
3249 virtio_vmstate_change
, vdev
);
3250 vdev
->device_endian
= virtio_default_endian();
3251 vdev
->use_guest_notifier_mask
= true;
3255 * Only devices that have already been around prior to defining the virtio
3256 * standard support legacy mode; this includes devices not specified in the
3257 * standard. All newer devices conform to the virtio standard only.
3259 bool virtio_legacy_allowed(VirtIODevice
*vdev
)
3261 switch (vdev
->device_id
) {
3263 case VIRTIO_ID_BLOCK
:
3264 case VIRTIO_ID_CONSOLE
:
3266 case VIRTIO_ID_BALLOON
:
3267 case VIRTIO_ID_RPMSG
:
3268 case VIRTIO_ID_SCSI
:
3270 case VIRTIO_ID_RPROC_SERIAL
:
3271 case VIRTIO_ID_CAIF
:
3278 bool virtio_legacy_check_disabled(VirtIODevice
*vdev
)
3280 return vdev
->disable_legacy_check
;
3283 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
3285 return vdev
->vq
[n
].vring
.desc
;
3288 bool virtio_queue_enabled_legacy(VirtIODevice
*vdev
, int n
)
3290 return virtio_queue_get_desc_addr(vdev
, n
) != 0;
3293 bool virtio_queue_enabled(VirtIODevice
*vdev
, int n
)
3295 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3296 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3298 if (k
->queue_enabled
) {
3299 return k
->queue_enabled(qbus
->parent
, n
);
3301 return virtio_queue_enabled_legacy(vdev
, n
);
3304 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
3306 return vdev
->vq
[n
].vring
.avail
;
3309 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
3311 return vdev
->vq
[n
].vring
.used
;
3314 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
3316 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
3319 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
3323 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3324 return sizeof(struct VRingPackedDescEvent
);
3327 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3328 return offsetof(VRingAvail
, ring
) +
3329 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
+ s
;
3332 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
3336 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3337 return sizeof(struct VRingPackedDescEvent
);
3340 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3341 return offsetof(VRingUsed
, ring
) +
3342 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
+ s
;
3345 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice
*vdev
,
3348 unsigned int avail
, used
;
3350 avail
= vdev
->vq
[n
].last_avail_idx
;
3351 avail
|= ((uint16_t)vdev
->vq
[n
].last_avail_wrap_counter
) << 15;
3353 used
= vdev
->vq
[n
].used_idx
;
3354 used
|= ((uint16_t)vdev
->vq
[n
].used_wrap_counter
) << 15;
3356 return avail
| used
<< 16;
3359 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice
*vdev
,
3362 return vdev
->vq
[n
].last_avail_idx
;
3365 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
3367 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3368 return virtio_queue_packed_get_last_avail_idx(vdev
, n
);
3370 return virtio_queue_split_get_last_avail_idx(vdev
, n
);
3374 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice
*vdev
,
3375 int n
, unsigned int idx
)
3377 struct VirtQueue
*vq
= &vdev
->vq
[n
];
3379 vq
->last_avail_idx
= vq
->shadow_avail_idx
= idx
& 0x7fff;
3380 vq
->last_avail_wrap_counter
=
3381 vq
->shadow_avail_wrap_counter
= !!(idx
& 0x8000);
3383 vq
->used_idx
= idx
& 0x7ffff;
3384 vq
->used_wrap_counter
= !!(idx
& 0x8000);
3387 static void virtio_queue_split_set_last_avail_idx(VirtIODevice
*vdev
,
3388 int n
, unsigned int idx
)
3390 vdev
->vq
[n
].last_avail_idx
= idx
;
3391 vdev
->vq
[n
].shadow_avail_idx
= idx
;
3394 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
,
3397 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3398 virtio_queue_packed_set_last_avail_idx(vdev
, n
, idx
);
3400 virtio_queue_split_set_last_avail_idx(vdev
, n
, idx
);
3404 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice
*vdev
,
3407 /* We don't have a reference like avail idx in shared memory */
3411 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice
*vdev
,
3414 RCU_READ_LOCK_GUARD();
3415 if (vdev
->vq
[n
].vring
.desc
) {
3416 vdev
->vq
[n
].last_avail_idx
= vring_used_idx(&vdev
->vq
[n
]);
3417 vdev
->vq
[n
].shadow_avail_idx
= vdev
->vq
[n
].last_avail_idx
;
3421 void virtio_queue_restore_last_avail_idx(VirtIODevice
*vdev
, int n
)
3423 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3424 virtio_queue_packed_restore_last_avail_idx(vdev
, n
);
3426 virtio_queue_split_restore_last_avail_idx(vdev
, n
);
3430 static void virtio_queue_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3432 /* used idx was updated through set_last_avail_idx() */
3436 static void virtio_split_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3438 RCU_READ_LOCK_GUARD();
3439 if (vdev
->vq
[n
].vring
.desc
) {
3440 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
3444 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
3446 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3447 return virtio_queue_packed_update_used_idx(vdev
, n
);
3449 return virtio_split_packed_update_used_idx(vdev
, n
);
3453 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
3455 vdev
->vq
[n
].signalled_used_valid
= false;
3458 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
3460 return vdev
->vq
+ n
;
3463 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
3465 return vq
->queue_index
;
3468 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
3470 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
3471 if (event_notifier_test_and_clear(n
)) {
3476 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
3479 if (assign
&& !with_irqfd
) {
3480 event_notifier_set_handler(&vq
->guest_notifier
,
3481 virtio_queue_guest_notifier_read
);
3483 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
3486 /* Test and clear notifier before closing it,
3487 * in case poll callback didn't have time to run. */
3488 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
3492 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
3494 return &vq
->guest_notifier
;
3497 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
3499 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3501 virtio_queue_set_notification(vq
, 0);
3504 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
3506 EventNotifier
*n
= opaque
;
3507 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3509 return vq
->vring
.desc
&& !virtio_queue_empty(vq
);
3512 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier
*n
)
3514 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3516 virtio_queue_notify_vq(vq
);
3519 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
3521 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3523 /* Caller polls once more after this to catch requests that race with us */
3524 virtio_queue_set_notification(vq
, 1);
3527 void virtio_queue_aio_attach_host_notifier(VirtQueue
*vq
, AioContext
*ctx
)
3529 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
3530 virtio_queue_host_notifier_read
,
3531 virtio_queue_host_notifier_aio_poll
,
3532 virtio_queue_host_notifier_aio_poll_ready
);
3533 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
3534 virtio_queue_host_notifier_aio_poll_begin
,
3535 virtio_queue_host_notifier_aio_poll_end
);
3538 void virtio_queue_aio_detach_host_notifier(VirtQueue
*vq
, AioContext
*ctx
)
3540 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
, NULL
);
3541 /* Test and clear notifier before after disabling event,
3542 * in case poll callback didn't have time to run. */
3543 virtio_queue_host_notifier_read(&vq
->host_notifier
);
3546 void virtio_queue_host_notifier_read(EventNotifier
*n
)
3548 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3549 if (event_notifier_test_and_clear(n
)) {
3550 virtio_queue_notify_vq(vq
);
3554 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
3556 return &vq
->host_notifier
;
3559 void virtio_queue_set_host_notifier_enabled(VirtQueue
*vq
, bool enabled
)
3561 vq
->host_notifier_enabled
= enabled
;
3564 int virtio_queue_set_host_notifier_mr(VirtIODevice
*vdev
, int n
,
3565 MemoryRegion
*mr
, bool assign
)
3567 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3568 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3570 if (k
->set_host_notifier_mr
) {
3571 return k
->set_host_notifier_mr(qbus
->parent
, n
, mr
, assign
);
3577 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
3579 g_free(vdev
->bus_name
);
3580 vdev
->bus_name
= g_strdup(bus_name
);
3583 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
3588 error_vreport(fmt
, ap
);
3591 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3592 vdev
->status
= vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
;
3593 virtio_notify_config(vdev
);
3596 vdev
->broken
= true;
3599 static void virtio_memory_listener_commit(MemoryListener
*listener
)
3601 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
3604 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3605 if (vdev
->vq
[i
].vring
.num
== 0) {
3608 virtio_init_region_cache(vdev
, i
);
3612 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
3614 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3615 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3618 /* Devices should either use vmsd or the load/save methods */
3619 assert(!vdc
->vmsd
|| !vdc
->load
);
3621 if (vdc
->realize
!= NULL
) {
3622 vdc
->realize(dev
, &err
);
3624 error_propagate(errp
, err
);
3629 virtio_bus_device_plugged(vdev
, &err
);
3631 error_propagate(errp
, err
);
3632 vdc
->unrealize(dev
);
3636 vdev
->listener
.commit
= virtio_memory_listener_commit
;
3637 vdev
->listener
.name
= "virtio";
3638 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
3641 static void virtio_device_unrealize(DeviceState
*dev
)
3643 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3644 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3646 memory_listener_unregister(&vdev
->listener
);
3647 virtio_bus_device_unplugged(vdev
);
3649 if (vdc
->unrealize
!= NULL
) {
3650 vdc
->unrealize(dev
);
3653 g_free(vdev
->bus_name
);
3654 vdev
->bus_name
= NULL
;
3657 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
3664 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3665 if (vdev
->vq
[i
].vring
.num
== 0) {
3668 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
3673 static void virtio_device_instance_finalize(Object
*obj
)
3675 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
3677 virtio_device_free_virtqueues(vdev
);
3679 g_free(vdev
->config
);
3680 g_free(vdev
->vector_queues
);
3683 static Property virtio_properties
[] = {
3684 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
3685 DEFINE_PROP_BOOL("use-started", VirtIODevice
, use_started
, true),
3686 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice
, use_disabled_flag
, true),
3687 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice
,
3688 disable_legacy_check
, false),
3689 DEFINE_PROP_END_OF_LIST(),
3692 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
3694 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3698 * Batch all the host notifiers in a single transaction to avoid
3699 * quadratic time complexity in address_space_update_ioeventfds().
3701 memory_region_transaction_begin();
3702 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3703 VirtQueue
*vq
= &vdev
->vq
[n
];
3704 if (!virtio_queue_get_num(vdev
, n
)) {
3707 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
3712 event_notifier_set_handler(&vq
->host_notifier
,
3713 virtio_queue_host_notifier_read
);
3716 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3717 /* Kick right away to begin processing requests already in vring */
3718 VirtQueue
*vq
= &vdev
->vq
[n
];
3719 if (!vq
->vring
.num
) {
3722 event_notifier_set(&vq
->host_notifier
);
3724 memory_region_transaction_commit();
3728 i
= n
; /* save n for a second iteration after transaction is committed. */
3730 VirtQueue
*vq
= &vdev
->vq
[n
];
3731 if (!virtio_queue_get_num(vdev
, n
)) {
3735 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3736 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3740 * The transaction expects the ioeventfds to be open when it
3741 * commits. Do it now, before the cleanup loop.
3743 memory_region_transaction_commit();
3746 if (!virtio_queue_get_num(vdev
, i
)) {
3749 virtio_bus_cleanup_host_notifier(qbus
, i
);
3754 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
3756 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3757 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3759 return virtio_bus_start_ioeventfd(vbus
);
3762 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
3764 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3768 * Batch all the host notifiers in a single transaction to avoid
3769 * quadratic time complexity in address_space_update_ioeventfds().
3771 memory_region_transaction_begin();
3772 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3773 VirtQueue
*vq
= &vdev
->vq
[n
];
3775 if (!virtio_queue_get_num(vdev
, n
)) {
3778 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3779 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3783 * The transaction expects the ioeventfds to be open when it
3784 * commits. Do it now, before the cleanup loop.
3786 memory_region_transaction_commit();
3788 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3789 if (!virtio_queue_get_num(vdev
, n
)) {
3792 virtio_bus_cleanup_host_notifier(qbus
, n
);
3796 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
3798 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3799 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3801 return virtio_bus_grab_ioeventfd(vbus
);
3804 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
3806 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3807 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3809 virtio_bus_release_ioeventfd(vbus
);
3812 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
3814 /* Set the default value here. */
3815 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
3816 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3818 dc
->realize
= virtio_device_realize
;
3819 dc
->unrealize
= virtio_device_unrealize
;
3820 dc
->bus_type
= TYPE_VIRTIO_BUS
;
3821 device_class_set_props(dc
, virtio_properties
);
3822 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
3823 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
3825 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
3828 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
3830 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3831 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3833 return virtio_bus_ioeventfd_enabled(vbus
);
3836 static const TypeInfo virtio_device_info
= {
3837 .name
= TYPE_VIRTIO_DEVICE
,
3838 .parent
= TYPE_DEVICE
,
3839 .instance_size
= sizeof(VirtIODevice
),
3840 .class_init
= virtio_device_class_init
,
3841 .instance_finalize
= virtio_device_instance_finalize
,
3843 .class_size
= sizeof(VirtioDeviceClass
),
3846 static void virtio_register_types(void)
3848 type_register_static(&virtio_device_info
);
3851 type_init(virtio_register_types
)