]> git.proxmox.com Git - mirror_qemu.git/blame - hw/virtio/virtio.c
virtio: Add typedef for handle_output
[mirror_qemu.git] / hw / virtio / virtio.c
CommitLineData
967f97fa
AL
1/*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
9b8bfe21 14#include "qemu/osdep.h"
da34e65c 15#include "qapi/error.h"
4771d756
PB
16#include "qemu-common.h"
17#include "cpu.h"
64979a4d 18#include "trace.h"
fdfba1a2 19#include "exec/address-spaces.h"
1de7afc9 20#include "qemu/error-report.h"
0d09e41a 21#include "hw/virtio/virtio.h"
1de7afc9 22#include "qemu/atomic.h"
0d09e41a 23#include "hw/virtio/virtio-bus.h"
6b321a3d 24#include "migration/migration.h"
cee3ca00 25#include "hw/virtio/virtio-access.h"
967f97fa 26
6ce69d1c
PM
27/*
28 * The alignment to use between consumer and producer parts of vring.
29 * x86 pagesize again. This is the default, used by transports like PCI
30 * which don't provide a means for the guest to tell the host the alignment.
31 */
f46f15bc
AL
32#define VIRTIO_PCI_VRING_ALIGN 4096
33
967f97fa
AL
34typedef struct VRingDesc
35{
36 uint64_t addr;
37 uint32_t len;
38 uint16_t flags;
39 uint16_t next;
40} VRingDesc;
41
42typedef struct VRingAvail
43{
44 uint16_t flags;
45 uint16_t idx;
46 uint16_t ring[0];
47} VRingAvail;
48
49typedef struct VRingUsedElem
50{
51 uint32_t id;
52 uint32_t len;
53} VRingUsedElem;
54
55typedef struct VRingUsed
56{
57 uint16_t flags;
58 uint16_t idx;
59 VRingUsedElem ring[0];
60} VRingUsed;
61
62typedef struct VRing
63{
64 unsigned int num;
46c5d082 65 unsigned int num_default;
6ce69d1c 66 unsigned int align;
a8170e5e
AK
67 hwaddr desc;
68 hwaddr avail;
69 hwaddr used;
967f97fa
AL
70} VRing;
71
72struct VirtQueue
73{
74 VRing vring;
be1fea9b
VM
75
76 /* Next head to pop */
967f97fa 77 uint16_t last_avail_idx;
b796fcd1 78
be1fea9b
VM
79 /* Last avail_idx read from VQ. */
80 uint16_t shadow_avail_idx;
81
b796fcd1
VM
82 uint16_t used_idx;
83
bcbabae8
MT
84 /* Last used index value we have signalled on */
85 uint16_t signalled_used;
86
87 /* Last used index value we have signalled on */
88 bool signalled_used_valid;
89
90 /* Notification enabled? */
91 bool notification;
92
e78a2b42
JW
93 uint16_t queue_index;
94
967f97fa 95 int inuse;
bcbabae8 96
7055e687 97 uint16_t vector;
bf1780b0
FZ
98 VirtIOHandleOutput handle_output;
99 VirtIOHandleOutput handle_aio_output;
1cbdabe2
MT
100 VirtIODevice *vdev;
101 EventNotifier guest_notifier;
102 EventNotifier host_notifier;
e0d686bf 103 QLIST_ENTRY(VirtQueue) node;
967f97fa
AL
104};
105
967f97fa 106/* virt queue functions */
ab223c95 107void virtio_queue_update_rings(VirtIODevice *vdev, int n)
967f97fa 108{
ab223c95 109 VRing *vring = &vdev->vq[n].vring;
53c25cea 110
ab223c95
CH
111 if (!vring->desc) {
112 /* not yet setup -> nothing to do */
113 return;
114 }
115 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
116 vring->used = vring_align(vring->avail +
117 offsetof(VRingAvail, ring[vring->num]),
118 vring->align);
967f97fa
AL
119}
120
aa570d6f
PB
121static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
122 hwaddr desc_pa, int i)
967f97fa 123{
aa570d6f
PB
124 address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
125 MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
126 virtio_tswap64s(vdev, &desc->addr);
127 virtio_tswap32s(vdev, &desc->len);
128 virtio_tswap16s(vdev, &desc->flags);
129 virtio_tswap16s(vdev, &desc->next);
967f97fa
AL
130}
131
132static inline uint16_t vring_avail_flags(VirtQueue *vq)
133{
a8170e5e 134 hwaddr pa;
967f97fa 135 pa = vq->vring.avail + offsetof(VRingAvail, flags);
cee3ca00 136 return virtio_lduw_phys(vq->vdev, pa);
967f97fa
AL
137}
138
139static inline uint16_t vring_avail_idx(VirtQueue *vq)
140{
a8170e5e 141 hwaddr pa;
967f97fa 142 pa = vq->vring.avail + offsetof(VRingAvail, idx);
be1fea9b
VM
143 vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
144 return vq->shadow_avail_idx;
967f97fa
AL
145}
146
147static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
148{
a8170e5e 149 hwaddr pa;
967f97fa 150 pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
cee3ca00 151 return virtio_lduw_phys(vq->vdev, pa);
967f97fa
AL
152}
153
e9600c6c 154static inline uint16_t vring_get_used_event(VirtQueue *vq)
bcbabae8
MT
155{
156 return vring_avail_ring(vq, vq->vring.num);
157}
158
1cdd2ee5
VM
159static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
160 int i)
967f97fa 161{
a8170e5e 162 hwaddr pa;
1cdd2ee5
VM
163 virtio_tswap32s(vq->vdev, &uelem->id);
164 virtio_tswap32s(vq->vdev, &uelem->len);
165 pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
166 address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
167 (void *)uelem, sizeof(VRingUsedElem));
967f97fa
AL
168}
169
170static uint16_t vring_used_idx(VirtQueue *vq)
171{
a8170e5e 172 hwaddr pa;
967f97fa 173 pa = vq->vring.used + offsetof(VRingUsed, idx);
cee3ca00 174 return virtio_lduw_phys(vq->vdev, pa);
967f97fa
AL
175}
176
bcbabae8 177static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
967f97fa 178{
a8170e5e 179 hwaddr pa;
967f97fa 180 pa = vq->vring.used + offsetof(VRingUsed, idx);
cee3ca00 181 virtio_stw_phys(vq->vdev, pa, val);
b796fcd1 182 vq->used_idx = val;
967f97fa
AL
183}
184
185static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
186{
cee3ca00 187 VirtIODevice *vdev = vq->vdev;
a8170e5e 188 hwaddr pa;
967f97fa 189 pa = vq->vring.used + offsetof(VRingUsed, flags);
cee3ca00 190 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
967f97fa
AL
191}
192
193static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
194{
cee3ca00 195 VirtIODevice *vdev = vq->vdev;
a8170e5e 196 hwaddr pa;
967f97fa 197 pa = vq->vring.used + offsetof(VRingUsed, flags);
cee3ca00 198 virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
967f97fa
AL
199}
200
e9600c6c 201static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
bcbabae8 202{
a8170e5e 203 hwaddr pa;
bcbabae8
MT
204 if (!vq->notification) {
205 return;
206 }
207 pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
cee3ca00 208 virtio_stw_phys(vq->vdev, pa, val);
bcbabae8
MT
209}
210
967f97fa
AL
211void virtio_queue_set_notification(VirtQueue *vq, int enable)
212{
bcbabae8 213 vq->notification = enable;
95129d6f 214 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
e9600c6c 215 vring_set_avail_event(vq, vring_avail_idx(vq));
bcbabae8 216 } else if (enable) {
967f97fa 217 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
bcbabae8 218 } else {
967f97fa 219 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
bcbabae8 220 }
92045d80
MT
221 if (enable) {
222 /* Expose avail event/used flags before caller checks the avail idx. */
223 smp_mb();
224 }
967f97fa
AL
225}
226
227int virtio_queue_ready(VirtQueue *vq)
228{
229 return vq->vring.avail != 0;
230}
231
be1fea9b
VM
232/* Fetch avail_idx from VQ memory only when we really need to know if
233 * guest has added some buffers. */
967f97fa
AL
234int virtio_queue_empty(VirtQueue *vq)
235{
be1fea9b
VM
236 if (vq->shadow_avail_idx != vq->last_avail_idx) {
237 return 0;
238 }
239
967f97fa
AL
240 return vring_avail_idx(vq) == vq->last_avail_idx;
241}
242
ce317461
JW
243static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
244 unsigned int len)
967f97fa
AL
245{
246 unsigned int offset;
247 int i;
248
967f97fa
AL
249 offset = 0;
250 for (i = 0; i < elem->in_num; i++) {
251 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
252
26b258e1
AL
253 cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
254 elem->in_sg[i].iov_len,
255 1, size);
967f97fa 256
0cea71a2 257 offset += size;
967f97fa
AL
258 }
259
26b258e1
AL
260 for (i = 0; i < elem->out_num; i++)
261 cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
262 elem->out_sg[i].iov_len,
263 0, elem->out_sg[i].iov_len);
ce317461
JW
264}
265
29b9f5ef
JW
266void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
267 unsigned int len)
268{
269 vq->last_avail_idx--;
270 virtqueue_unmap_sg(vq, elem, len);
271}
272
ce317461
JW
273void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
274 unsigned int len, unsigned int idx)
275{
1cdd2ee5
VM
276 VRingUsedElem uelem;
277
ce317461
JW
278 trace_virtqueue_fill(vq, elem, len, idx);
279
280 virtqueue_unmap_sg(vq, elem, len);
26b258e1 281
b796fcd1 282 idx = (idx + vq->used_idx) % vq->vring.num;
967f97fa 283
1cdd2ee5
VM
284 uelem.id = elem->index;
285 uelem.len = len;
286 vring_used_write(vq, &uelem, idx);
967f97fa
AL
287}
288
289void virtqueue_flush(VirtQueue *vq, unsigned int count)
290{
bcbabae8 291 uint16_t old, new;
967f97fa 292 /* Make sure buffer is written before we update index. */
b90d2f35 293 smp_wmb();
64979a4d 294 trace_virtqueue_flush(vq, count);
b796fcd1 295 old = vq->used_idx;
bcbabae8
MT
296 new = old + count;
297 vring_used_idx_set(vq, new);
967f97fa 298 vq->inuse -= count;
bcbabae8
MT
299 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
300 vq->signalled_used_valid = false;
967f97fa
AL
301}
302
303void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
304 unsigned int len)
305{
306 virtqueue_fill(vq, elem, len, 0);
307 virtqueue_flush(vq, 1);
308}
309
310static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
311{
312 uint16_t num_heads = vring_avail_idx(vq) - idx;
313
314 /* Check it isn't doing very strange things with descriptor numbers. */
bb6834cf 315 if (num_heads > vq->vring.num) {
ce67ed65 316 error_report("Guest moved used index from %u to %u",
be1fea9b 317 idx, vq->shadow_avail_idx);
bb6834cf
AL
318 exit(1);
319 }
a821ce59
MT
320 /* On success, callers read a descriptor at vq->last_avail_idx.
321 * Make sure descriptor read does not bypass avail index read. */
322 if (num_heads) {
323 smp_rmb();
324 }
967f97fa
AL
325
326 return num_heads;
327}
328
329static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
330{
331 unsigned int head;
332
333 /* Grab the next descriptor number they're advertising, and increment
334 * the index we've seen. */
335 head = vring_avail_ring(vq, idx % vq->vring.num);
336
337 /* If their number is silly, that's a fatal mistake. */
bb6834cf 338 if (head >= vq->vring.num) {
ce67ed65 339 error_report("Guest says index %u is available", head);
bb6834cf
AL
340 exit(1);
341 }
967f97fa
AL
342
343 return head;
344}
345
aa570d6f
PB
346static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
347 hwaddr desc_pa, unsigned int max)
967f97fa
AL
348{
349 unsigned int next;
350
351 /* If this descriptor says it doesn't chain, we're done. */
aa570d6f 352 if (!(desc->flags & VRING_DESC_F_NEXT)) {
5774cf98 353 return max;
cee3ca00 354 }
967f97fa
AL
355
356 /* Check they're not leading us off end of descriptors. */
aa570d6f 357 next = desc->next;
967f97fa 358 /* Make sure compiler knows to grab that: we don't want it changing! */
b90d2f35 359 smp_wmb();
967f97fa 360
5774cf98 361 if (next >= max) {
ce67ed65 362 error_report("Desc next is %u", next);
bb6834cf
AL
363 exit(1);
364 }
967f97fa 365
aa570d6f 366 vring_desc_read(vdev, desc, desc_pa, next);
967f97fa
AL
367 return next;
368}
369
0d8d7690 370void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
e1f7b481
MT
371 unsigned int *out_bytes,
372 unsigned max_in_bytes, unsigned max_out_bytes)
967f97fa 373{
efeea6d0 374 unsigned int idx;
385ce95d 375 unsigned int total_bufs, in_total, out_total;
967f97fa
AL
376
377 idx = vq->last_avail_idx;
378
efeea6d0 379 total_bufs = in_total = out_total = 0;
967f97fa 380 while (virtqueue_num_heads(vq, idx)) {
cee3ca00 381 VirtIODevice *vdev = vq->vdev;
efeea6d0 382 unsigned int max, num_bufs, indirect = 0;
aa570d6f 383 VRingDesc desc;
a8170e5e 384 hwaddr desc_pa;
967f97fa
AL
385 int i;
386
efeea6d0
MM
387 max = vq->vring.num;
388 num_bufs = total_bufs;
967f97fa 389 i = virtqueue_get_head(vq, idx++);
efeea6d0 390 desc_pa = vq->vring.desc;
aa570d6f 391 vring_desc_read(vdev, &desc, desc_pa, i);
efeea6d0 392
aa570d6f
PB
393 if (desc.flags & VRING_DESC_F_INDIRECT) {
394 if (desc.len % sizeof(VRingDesc)) {
ce67ed65 395 error_report("Invalid size for indirect buffer table");
efeea6d0
MM
396 exit(1);
397 }
398
399 /* If we've got too many, that implies a descriptor loop. */
400 if (num_bufs >= max) {
ce67ed65 401 error_report("Looped descriptor");
efeea6d0
MM
402 exit(1);
403 }
404
405 /* loop over the indirect descriptor table */
406 indirect = 1;
aa570d6f
PB
407 max = desc.len / sizeof(VRingDesc);
408 desc_pa = desc.addr;
1ae2757c 409 num_bufs = i = 0;
aa570d6f 410 vring_desc_read(vdev, &desc, desc_pa, i);
efeea6d0
MM
411 }
412
967f97fa
AL
413 do {
414 /* If we've got too many, that implies a descriptor loop. */
5774cf98 415 if (++num_bufs > max) {
ce67ed65 416 error_report("Looped descriptor");
bb6834cf
AL
417 exit(1);
418 }
967f97fa 419
aa570d6f
PB
420 if (desc.flags & VRING_DESC_F_WRITE) {
421 in_total += desc.len;
967f97fa 422 } else {
aa570d6f 423 out_total += desc.len;
967f97fa 424 }
e1f7b481
MT
425 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
426 goto done;
427 }
aa570d6f 428 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
efeea6d0
MM
429
430 if (!indirect)
431 total_bufs = num_bufs;
432 else
433 total_bufs++;
967f97fa 434 }
e1f7b481 435done:
0d8d7690
AS
436 if (in_bytes) {
437 *in_bytes = in_total;
438 }
439 if (out_bytes) {
440 *out_bytes = out_total;
441 }
442}
967f97fa 443
0d8d7690
AS
444int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
445 unsigned int out_bytes)
446{
447 unsigned int in_total, out_total;
448
e1f7b481
MT
449 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
450 return in_bytes <= in_total && out_bytes <= out_total;
967f97fa
AL
451}
452
3b3b0628
PB
453static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
454 unsigned int max_num_sg, bool is_write,
455 hwaddr pa, size_t sz)
456{
457 unsigned num_sg = *p_num_sg;
458 assert(num_sg <= max_num_sg);
459
460 while (sz) {
461 hwaddr len = sz;
462
463 if (num_sg == max_num_sg) {
464 error_report("virtio: too many write descriptors in indirect table");
465 exit(1);
466 }
467
468 iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
469 iov[num_sg].iov_len = len;
470 addr[num_sg] = pa;
471
472 sz -= len;
473 pa += len;
474 num_sg++;
475 }
476 *p_num_sg = num_sg;
477}
478
8059feee
MT
479static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
480 unsigned int *num_sg, unsigned int max_size,
481 int is_write)
42fb2e07
KW
482{
483 unsigned int i;
a8170e5e 484 hwaddr len;
42fb2e07 485
8059feee
MT
486 /* Note: this function MUST validate input, some callers
487 * are passing in num_sg values received over the network.
488 */
489 /* TODO: teach all callers that this can fail, and return failure instead
490 * of asserting here.
491 * When we do, we might be able to re-enable NDEBUG below.
492 */
493#ifdef NDEBUG
494#error building with NDEBUG is not supported
495#endif
496 assert(*num_sg <= max_size);
497
498 for (i = 0; i < *num_sg; i++) {
42fb2e07
KW
499 len = sg[i].iov_len;
500 sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
8059feee 501 if (!sg[i].iov_base) {
1a285899 502 error_report("virtio: error trying to map MMIO memory");
42fb2e07
KW
503 exit(1);
504 }
3b3b0628
PB
505 if (len != sg[i].iov_len) {
506 error_report("virtio: unexpected memory split");
8059feee
MT
507 exit(1);
508 }
42fb2e07
KW
509 }
510}
511
8059feee
MT
512void virtqueue_map(VirtQueueElement *elem)
513{
514 virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
3724650d 515 VIRTQUEUE_MAX_SIZE, 1);
8059feee 516 virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
3724650d
PB
517 VIRTQUEUE_MAX_SIZE, 0);
518}
519
520void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
521{
522 VirtQueueElement *elem;
523 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
524 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
525 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
526 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
527 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
528 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
529
530 assert(sz >= sizeof(VirtQueueElement));
531 elem = g_malloc(out_sg_end);
532 elem->out_num = out_num;
533 elem->in_num = in_num;
534 elem->in_addr = (void *)elem + in_addr_ofs;
535 elem->out_addr = (void *)elem + out_addr_ofs;
536 elem->in_sg = (void *)elem + in_sg_ofs;
537 elem->out_sg = (void *)elem + out_sg_ofs;
538 return elem;
8059feee
MT
539}
540
51b19ebe 541void *virtqueue_pop(VirtQueue *vq, size_t sz)
967f97fa 542{
5774cf98 543 unsigned int i, head, max;
a8170e5e 544 hwaddr desc_pa = vq->vring.desc;
cee3ca00 545 VirtIODevice *vdev = vq->vdev;
51b19ebe 546 VirtQueueElement *elem;
3b3b0628
PB
547 unsigned out_num, in_num;
548 hwaddr addr[VIRTQUEUE_MAX_SIZE];
549 struct iovec iov[VIRTQUEUE_MAX_SIZE];
aa570d6f 550 VRingDesc desc;
967f97fa 551
be1fea9b 552 if (virtio_queue_empty(vq)) {
51b19ebe
PB
553 return NULL;
554 }
be1fea9b
VM
555 /* Needed after virtio_queue_empty(), see comment in
556 * virtqueue_num_heads(). */
557 smp_rmb();
967f97fa
AL
558
559 /* When we start there are none of either input nor output. */
3b3b0628 560 out_num = in_num = 0;
967f97fa 561
5774cf98
MM
562 max = vq->vring.num;
563
967f97fa 564 i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
95129d6f 565 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
e9600c6c 566 vring_set_avail_event(vq, vq->last_avail_idx);
bcbabae8 567 }
efeea6d0 568
aa570d6f
PB
569 vring_desc_read(vdev, &desc, desc_pa, i);
570 if (desc.flags & VRING_DESC_F_INDIRECT) {
571 if (desc.len % sizeof(VRingDesc)) {
ce67ed65 572 error_report("Invalid size for indirect buffer table");
efeea6d0
MM
573 exit(1);
574 }
575
576 /* loop over the indirect descriptor table */
aa570d6f
PB
577 max = desc.len / sizeof(VRingDesc);
578 desc_pa = desc.addr;
efeea6d0 579 i = 0;
aa570d6f 580 vring_desc_read(vdev, &desc, desc_pa, i);
efeea6d0
MM
581 }
582
42fb2e07 583 /* Collect all the descriptors */
967f97fa 584 do {
aa570d6f 585 if (desc.flags & VRING_DESC_F_WRITE) {
3b3b0628 586 virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
aa570d6f 587 VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
42fb2e07 588 } else {
3b3b0628
PB
589 if (in_num) {
590 error_report("Incorrect order for descriptors");
c8eac1cf
MT
591 exit(1);
592 }
3b3b0628 593 virtqueue_map_desc(&out_num, addr, iov,
aa570d6f 594 VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
42fb2e07 595 }
967f97fa 596
967f97fa 597 /* If we've got too many, that implies a descriptor loop. */
3b3b0628 598 if ((in_num + out_num) > max) {
ce67ed65 599 error_report("Looped descriptor");
bb6834cf
AL
600 exit(1);
601 }
aa570d6f 602 } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
967f97fa 603
3b3b0628
PB
604 /* Now copy what we have collected and mapped */
605 elem = virtqueue_alloc_element(sz, out_num, in_num);
967f97fa 606 elem->index = head;
3b3b0628
PB
607 for (i = 0; i < out_num; i++) {
608 elem->out_addr[i] = addr[i];
609 elem->out_sg[i] = iov[i];
610 }
611 for (i = 0; i < in_num; i++) {
612 elem->in_addr[i] = addr[out_num + i];
613 elem->in_sg[i] = iov[out_num + i];
614 }
967f97fa
AL
615
616 vq->inuse++;
617
64979a4d 618 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
51b19ebe 619 return elem;
967f97fa
AL
620}
621
3724650d
PB
622/* Reading and writing a structure directly to QEMUFile is *awful*, but
623 * it is what QEMU has always done by mistake. We can change it sooner
624 * or later by bumping the version number of the affected vm states.
625 * In the meanwhile, since the in-memory layout of VirtQueueElement
626 * has changed, we need to marshal to and from the layout that was
627 * used before the change.
628 */
629typedef struct VirtQueueElementOld {
630 unsigned int index;
631 unsigned int out_num;
632 unsigned int in_num;
633 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
634 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
635 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
636 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
637} VirtQueueElementOld;
638
ab281c17
PB
639void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
640{
3724650d
PB
641 VirtQueueElement *elem;
642 VirtQueueElementOld data;
643 int i;
644
645 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
646
647 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
648 elem->index = data.index;
649
650 for (i = 0; i < elem->in_num; i++) {
651 elem->in_addr[i] = data.in_addr[i];
652 }
653
654 for (i = 0; i < elem->out_num; i++) {
655 elem->out_addr[i] = data.out_addr[i];
656 }
657
658 for (i = 0; i < elem->in_num; i++) {
659 /* Base is overwritten by virtqueue_map. */
660 elem->in_sg[i].iov_base = 0;
661 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
662 }
663
664 for (i = 0; i < elem->out_num; i++) {
665 /* Base is overwritten by virtqueue_map. */
666 elem->out_sg[i].iov_base = 0;
667 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
668 }
669
ab281c17
PB
670 virtqueue_map(elem);
671 return elem;
672}
673
674void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
675{
3724650d
PB
676 VirtQueueElementOld data;
677 int i;
678
679 memset(&data, 0, sizeof(data));
680 data.index = elem->index;
681 data.in_num = elem->in_num;
682 data.out_num = elem->out_num;
683
684 for (i = 0; i < elem->in_num; i++) {
685 data.in_addr[i] = elem->in_addr[i];
686 }
687
688 for (i = 0; i < elem->out_num; i++) {
689 data.out_addr[i] = elem->out_addr[i];
690 }
691
692 for (i = 0; i < elem->in_num; i++) {
693 /* Base is overwritten by virtqueue_map when loading. Do not
694 * save it, as it would leak the QEMU address space layout. */
695 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
696 }
697
698 for (i = 0; i < elem->out_num; i++) {
699 /* Do not save iov_base as above. */
700 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
701 }
702 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
ab281c17
PB
703}
704
967f97fa 705/* virtio device */
7055e687
MT
706static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
707{
1c819449
FK
708 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
709 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
710
711 if (k->notify) {
712 k->notify(qbus->parent, vector);
7055e687
MT
713 }
714}
967f97fa 715
53c25cea 716void virtio_update_irq(VirtIODevice *vdev)
967f97fa 717{
7055e687 718 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
967f97fa
AL
719}
720
0b352fd6
CH
721static int virtio_validate_features(VirtIODevice *vdev)
722{
723 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
724
725 if (k->validate_features) {
726 return k->validate_features(vdev);
727 } else {
728 return 0;
729 }
730}
731
732int virtio_set_status(VirtIODevice *vdev, uint8_t val)
4e1837f8 733{
181103cd 734 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
4e1837f8
SH
735 trace_virtio_set_status(vdev, val);
736
95129d6f 737 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
0b352fd6
CH
738 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
739 val & VIRTIO_CONFIG_S_FEATURES_OK) {
740 int ret = virtio_validate_features(vdev);
741
742 if (ret) {
743 return ret;
744 }
745 }
746 }
181103cd
FK
747 if (k->set_status) {
748 k->set_status(vdev, val);
4e1837f8
SH
749 }
750 vdev->status = val;
0b352fd6 751 return 0;
4e1837f8
SH
752}
753
616a6552
GK
754bool target_words_bigendian(void);
755static enum virtio_device_endian virtio_default_endian(void)
756{
757 if (target_words_bigendian()) {
758 return VIRTIO_DEVICE_ENDIAN_BIG;
759 } else {
760 return VIRTIO_DEVICE_ENDIAN_LITTLE;
761 }
762}
763
764static enum virtio_device_endian virtio_current_cpu_endian(void)
765{
766 CPUClass *cc = CPU_GET_CLASS(current_cpu);
767
768 if (cc->virtio_is_big_endian(current_cpu)) {
769 return VIRTIO_DEVICE_ENDIAN_BIG;
770 } else {
771 return VIRTIO_DEVICE_ENDIAN_LITTLE;
772 }
773}
774
53c25cea 775void virtio_reset(void *opaque)
967f97fa
AL
776{
777 VirtIODevice *vdev = opaque;
181103cd 778 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
779 int i;
780
e0c472d8 781 virtio_set_status(vdev, 0);
616a6552
GK
782 if (current_cpu) {
783 /* Guest initiated reset */
784 vdev->device_endian = virtio_current_cpu_endian();
785 } else {
786 /* System reset */
787 vdev->device_endian = virtio_default_endian();
788 }
e0c472d8 789
181103cd
FK
790 if (k->reset) {
791 k->reset(vdev);
792 }
967f97fa 793
704a76fc 794 vdev->guest_features = 0;
967f97fa
AL
795 vdev->queue_sel = 0;
796 vdev->status = 0;
797 vdev->isr = 0;
7055e687
MT
798 vdev->config_vector = VIRTIO_NO_VECTOR;
799 virtio_notify_vector(vdev, vdev->config_vector);
967f97fa 800
87b3bd1c 801 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
967f97fa
AL
802 vdev->vq[i].vring.desc = 0;
803 vdev->vq[i].vring.avail = 0;
804 vdev->vq[i].vring.used = 0;
805 vdev->vq[i].last_avail_idx = 0;
be1fea9b 806 vdev->vq[i].shadow_avail_idx = 0;
b796fcd1 807 vdev->vq[i].used_idx = 0;
e0d686bf 808 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
bcbabae8
MT
809 vdev->vq[i].signalled_used = 0;
810 vdev->vq[i].signalled_used_valid = false;
811 vdev->vq[i].notification = true;
46c5d082 812 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
967f97fa
AL
813 }
814}
815
53c25cea 816uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
967f97fa 817{
181103cd 818 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
819 uint8_t val;
820
5f5a1318 821 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 822 return (uint32_t)-1;
5f5a1318
JW
823 }
824
825 k->get_config(vdev, vdev->config);
967f97fa 826
06dbfc6f 827 val = ldub_p(vdev->config + addr);
967f97fa
AL
828 return val;
829}
830
53c25cea 831uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
967f97fa 832{
181103cd 833 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
834 uint16_t val;
835
5f5a1318 836 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 837 return (uint32_t)-1;
5f5a1318
JW
838 }
839
840 k->get_config(vdev, vdev->config);
967f97fa 841
06dbfc6f 842 val = lduw_p(vdev->config + addr);
967f97fa
AL
843 return val;
844}
845
53c25cea 846uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
967f97fa 847{
181103cd 848 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
849 uint32_t val;
850
5f5a1318 851 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 852 return (uint32_t)-1;
5f5a1318
JW
853 }
854
855 k->get_config(vdev, vdev->config);
967f97fa 856
06dbfc6f 857 val = ldl_p(vdev->config + addr);
967f97fa
AL
858 return val;
859}
860
53c25cea 861void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
967f97fa 862{
181103cd 863 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
864 uint8_t val = data;
865
5f5a1318 866 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 867 return;
5f5a1318 868 }
967f97fa 869
06dbfc6f 870 stb_p(vdev->config + addr, val);
967f97fa 871
181103cd
FK
872 if (k->set_config) {
873 k->set_config(vdev, vdev->config);
874 }
967f97fa
AL
875}
876
53c25cea 877void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
967f97fa 878{
181103cd 879 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
880 uint16_t val = data;
881
5f5a1318 882 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 883 return;
5f5a1318 884 }
967f97fa 885
06dbfc6f 886 stw_p(vdev->config + addr, val);
967f97fa 887
181103cd
FK
888 if (k->set_config) {
889 k->set_config(vdev, vdev->config);
890 }
967f97fa
AL
891}
892
53c25cea 893void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
967f97fa 894{
181103cd 895 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa
AL
896 uint32_t val = data;
897
5f5a1318 898 if (addr + sizeof(val) > vdev->config_len) {
967f97fa 899 return;
5f5a1318 900 }
967f97fa 901
06dbfc6f 902 stl_p(vdev->config + addr, val);
967f97fa 903
181103cd
FK
904 if (k->set_config) {
905 k->set_config(vdev, vdev->config);
906 }
967f97fa
AL
907}
908
adfb743c
MT
909uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
910{
911 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
912 uint8_t val;
913
914 if (addr + sizeof(val) > vdev->config_len) {
915 return (uint32_t)-1;
916 }
917
918 k->get_config(vdev, vdev->config);
919
920 val = ldub_p(vdev->config + addr);
921 return val;
922}
923
924uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
925{
926 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
927 uint16_t val;
928
929 if (addr + sizeof(val) > vdev->config_len) {
930 return (uint32_t)-1;
931 }
932
933 k->get_config(vdev, vdev->config);
934
935 val = lduw_le_p(vdev->config + addr);
936 return val;
937}
938
939uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
940{
941 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
942 uint32_t val;
943
944 if (addr + sizeof(val) > vdev->config_len) {
945 return (uint32_t)-1;
946 }
947
948 k->get_config(vdev, vdev->config);
949
950 val = ldl_le_p(vdev->config + addr);
951 return val;
952}
953
954void virtio_config_modern_writeb(VirtIODevice *vdev,
955 uint32_t addr, uint32_t data)
956{
957 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
958 uint8_t val = data;
959
960 if (addr + sizeof(val) > vdev->config_len) {
961 return;
962 }
963
964 stb_p(vdev->config + addr, val);
965
966 if (k->set_config) {
967 k->set_config(vdev, vdev->config);
968 }
969}
970
971void virtio_config_modern_writew(VirtIODevice *vdev,
972 uint32_t addr, uint32_t data)
973{
974 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
975 uint16_t val = data;
976
977 if (addr + sizeof(val) > vdev->config_len) {
978 return;
979 }
980
981 stw_le_p(vdev->config + addr, val);
982
983 if (k->set_config) {
984 k->set_config(vdev, vdev->config);
985 }
986}
987
988void virtio_config_modern_writel(VirtIODevice *vdev,
989 uint32_t addr, uint32_t data)
990{
991 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
992 uint32_t val = data;
993
994 if (addr + sizeof(val) > vdev->config_len) {
995 return;
996 }
997
998 stl_le_p(vdev->config + addr, val);
999
1000 if (k->set_config) {
1001 k->set_config(vdev, vdev->config);
1002 }
1003}
1004
a8170e5e 1005void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
967f97fa 1006{
ab223c95
CH
1007 vdev->vq[n].vring.desc = addr;
1008 virtio_queue_update_rings(vdev, n);
53c25cea
PB
1009}
1010
a8170e5e 1011hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
53c25cea 1012{
ab223c95
CH
1013 return vdev->vq[n].vring.desc;
1014}
1015
1016void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1017 hwaddr avail, hwaddr used)
1018{
1019 vdev->vq[n].vring.desc = desc;
1020 vdev->vq[n].vring.avail = avail;
1021 vdev->vq[n].vring.used = used;
53c25cea
PB
1022}
1023
e63c0ba1
PM
1024void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1025{
f6049f44
PM
1026 /* Don't allow guest to flip queue between existent and
1027 * nonexistent states, or to set it to an invalid size.
1028 */
1029 if (!!num != !!vdev->vq[n].vring.num ||
1030 num > VIRTQUEUE_MAX_SIZE ||
1031 num < 0) {
1032 return;
e63c0ba1 1033 }
f6049f44 1034 vdev->vq[n].vring.num = num;
e63c0ba1
PM
1035}
1036
e0d686bf
JW
1037VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1038{
1039 return QLIST_FIRST(&vdev->vector_queues[vector]);
1040}
1041
1042VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1043{
1044 return QLIST_NEXT(vq, node);
1045}
1046
53c25cea
PB
1047int virtio_queue_get_num(VirtIODevice *vdev, int n)
1048{
1049 return vdev->vq[n].vring.num;
1050}
967f97fa 1051
8ad176aa
JW
1052int virtio_get_num_queues(VirtIODevice *vdev)
1053{
1054 int i;
1055
87b3bd1c 1056 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
8ad176aa
JW
1057 if (!virtio_queue_get_num(vdev, i)) {
1058 break;
1059 }
1060 }
1061
1062 return i;
1063}
1064
6ce69d1c
PM
1065void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1066{
1067 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1068 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1069
ab223c95 1070 /* virtio-1 compliant devices cannot change the alignment */
95129d6f 1071 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
ab223c95
CH
1072 error_report("tried to modify queue alignment for virtio-1 device");
1073 return;
1074 }
6ce69d1c
PM
1075 /* Check that the transport told us it was going to do this
1076 * (so a buggy transport will immediately assert rather than
1077 * silently failing to migrate this state)
1078 */
1079 assert(k->has_variable_vring_alignment);
1080
1081 vdev->vq[n].vring.align = align;
ab223c95 1082 virtio_queue_update_rings(vdev, n);
6ce69d1c
PM
1083}
1084
344dc16f
MT
1085static void virtio_queue_notify_aio_vq(VirtQueue *vq)
1086{
1087 if (vq->vring.desc && vq->handle_aio_output) {
1088 VirtIODevice *vdev = vq->vdev;
1089
1090 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1091 vq->handle_aio_output(vdev, vq);
1092 }
1093}
1094
2b2cbcad 1095static void virtio_queue_notify_vq(VirtQueue *vq)
25db9ebe 1096{
9e0f5b81 1097 if (vq->vring.desc && vq->handle_output) {
25db9ebe 1098 VirtIODevice *vdev = vq->vdev;
9e0f5b81 1099
25db9ebe
SH
1100 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1101 vq->handle_output(vdev, vq);
1102 }
1103}
1104
53c25cea
PB
1105void virtio_queue_notify(VirtIODevice *vdev, int n)
1106{
7157e2e2 1107 virtio_queue_notify_vq(&vdev->vq[n]);
967f97fa
AL
1108}
1109
7055e687
MT
1110uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1111{
87b3bd1c 1112 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
7055e687
MT
1113 VIRTIO_NO_VECTOR;
1114}
1115
1116void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1117{
e0d686bf
JW
1118 VirtQueue *vq = &vdev->vq[n];
1119
87b3bd1c 1120 if (n < VIRTIO_QUEUE_MAX) {
e0d686bf
JW
1121 if (vdev->vector_queues &&
1122 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1123 QLIST_REMOVE(vq, node);
1124 }
7055e687 1125 vdev->vq[n].vector = vector;
e0d686bf
JW
1126 if (vdev->vector_queues &&
1127 vector != VIRTIO_NO_VECTOR) {
1128 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1129 }
1130 }
7055e687
MT
1131}
1132
967f97fa 1133VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
bf1780b0 1134 VirtIOHandleOutput handle_output)
967f97fa
AL
1135{
1136 int i;
1137
87b3bd1c 1138 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
967f97fa
AL
1139 if (vdev->vq[i].vring.num == 0)
1140 break;
1141 }
1142
87b3bd1c 1143 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
967f97fa
AL
1144 abort();
1145
1146 vdev->vq[i].vring.num = queue_size;
46c5d082 1147 vdev->vq[i].vring.num_default = queue_size;
6ce69d1c 1148 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
967f97fa 1149 vdev->vq[i].handle_output = handle_output;
344dc16f 1150 vdev->vq[i].handle_aio_output = NULL;
967f97fa
AL
1151
1152 return &vdev->vq[i];
1153}
1154
f23fd811
JW
1155void virtio_del_queue(VirtIODevice *vdev, int n)
1156{
87b3bd1c 1157 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
f23fd811
JW
1158 abort();
1159 }
1160
1161 vdev->vq[n].vring.num = 0;
46c5d082 1162 vdev->vq[n].vring.num_default = 0;
f23fd811
JW
1163}
1164
1cbdabe2
MT
1165void virtio_irq(VirtQueue *vq)
1166{
64979a4d 1167 trace_virtio_irq(vq);
1cbdabe2
MT
1168 vq->vdev->isr |= 0x01;
1169 virtio_notify_vector(vq->vdev, vq->vector);
1170}
1171
adb3feda 1172bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
bcbabae8
MT
1173{
1174 uint16_t old, new;
1175 bool v;
a281ebc1
MT
1176 /* We need to expose used array entries before checking used event. */
1177 smp_mb();
97b83deb 1178 /* Always notify when queue is empty (when feature acknowledge) */
95129d6f 1179 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
be1fea9b 1180 !vq->inuse && virtio_queue_empty(vq)) {
bcbabae8
MT
1181 return true;
1182 }
1183
95129d6f 1184 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
bcbabae8
MT
1185 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1186 }
1187
1188 v = vq->signalled_used_valid;
1189 vq->signalled_used_valid = true;
1190 old = vq->signalled_used;
b796fcd1 1191 new = vq->signalled_used = vq->used_idx;
e9600c6c 1192 return !v || vring_need_event(vring_get_used_event(vq), new, old);
bcbabae8
MT
1193}
1194
1195void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1196{
adb3feda 1197 if (!virtio_should_notify(vdev, vq)) {
967f97fa 1198 return;
bcbabae8 1199 }
967f97fa 1200
64979a4d 1201 trace_virtio_notify(vdev, vq);
967f97fa 1202 vdev->isr |= 0x01;
7055e687 1203 virtio_notify_vector(vdev, vq->vector);
967f97fa
AL
1204}
1205
1206void virtio_notify_config(VirtIODevice *vdev)
1207{
7625162c
AL
1208 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1209 return;
1210
967f97fa 1211 vdev->isr |= 0x03;
b8f05908 1212 vdev->generation++;
7055e687 1213 virtio_notify_vector(vdev, vdev->config_vector);
967f97fa
AL
1214}
1215
616a6552
GK
1216static bool virtio_device_endian_needed(void *opaque)
1217{
1218 VirtIODevice *vdev = opaque;
1219
1220 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
95129d6f 1221 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3c185597
CH
1222 return vdev->device_endian != virtio_default_endian();
1223 }
1224 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1225 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
616a6552
GK
1226}
1227
019a3edb
GH
1228static bool virtio_64bit_features_needed(void *opaque)
1229{
1230 VirtIODevice *vdev = opaque;
1231
1232 return (vdev->host_features >> 32) != 0;
1233}
1234
74aae7b2
JW
1235static bool virtio_virtqueue_needed(void *opaque)
1236{
1237 VirtIODevice *vdev = opaque;
1238
1239 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1240}
1241
46c5d082
CH
1242static bool virtio_ringsize_needed(void *opaque)
1243{
1244 VirtIODevice *vdev = opaque;
1245 int i;
1246
1247 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1248 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1249 return true;
1250 }
1251 }
1252 return false;
1253}
1254
a6df8adf
JW
1255static bool virtio_extra_state_needed(void *opaque)
1256{
1257 VirtIODevice *vdev = opaque;
1258 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1259 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1260
1261 return k->has_extra_state &&
1262 k->has_extra_state(qbus->parent);
1263}
1264
50e5ae4d 1265static const VMStateDescription vmstate_virtqueue = {
74aae7b2 1266 .name = "virtqueue_state",
50e5ae4d
DDAG
1267 .version_id = 1,
1268 .minimum_version_id = 1,
1269 .fields = (VMStateField[]) {
1270 VMSTATE_UINT64(vring.avail, struct VirtQueue),
1271 VMSTATE_UINT64(vring.used, struct VirtQueue),
1272 VMSTATE_END_OF_LIST()
1273 }
74aae7b2
JW
1274};
1275
1276static const VMStateDescription vmstate_virtio_virtqueues = {
1277 .name = "virtio/virtqueues",
1278 .version_id = 1,
1279 .minimum_version_id = 1,
1280 .needed = &virtio_virtqueue_needed,
1281 .fields = (VMStateField[]) {
3e996cc5
DDAG
1282 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1283 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
74aae7b2
JW
1284 VMSTATE_END_OF_LIST()
1285 }
1286};
1287
50e5ae4d 1288static const VMStateDescription vmstate_ringsize = {
46c5d082 1289 .name = "ringsize_state",
50e5ae4d
DDAG
1290 .version_id = 1,
1291 .minimum_version_id = 1,
1292 .fields = (VMStateField[]) {
1293 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1294 VMSTATE_END_OF_LIST()
1295 }
46c5d082
CH
1296};
1297
1298static const VMStateDescription vmstate_virtio_ringsize = {
1299 .name = "virtio/ringsize",
1300 .version_id = 1,
1301 .minimum_version_id = 1,
1302 .needed = &virtio_ringsize_needed,
1303 .fields = (VMStateField[]) {
3e996cc5
DDAG
1304 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1305 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
46c5d082
CH
1306 VMSTATE_END_OF_LIST()
1307 }
1308};
1309
a6df8adf
JW
1310static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1311{
1312 VirtIODevice *vdev = pv;
1313 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1314 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1315
1316 if (!k->load_extra_state) {
1317 return -1;
1318 } else {
1319 return k->load_extra_state(qbus->parent, f);
1320 }
1321}
1322
1323static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1324{
1325 VirtIODevice *vdev = pv;
1326 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1327 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1328
1329 k->save_extra_state(qbus->parent, f);
1330}
1331
1332static const VMStateInfo vmstate_info_extra_state = {
1333 .name = "virtqueue_extra_state",
1334 .get = get_extra_state,
1335 .put = put_extra_state,
1336};
1337
1338static const VMStateDescription vmstate_virtio_extra_state = {
1339 .name = "virtio/extra_state",
1340 .version_id = 1,
1341 .minimum_version_id = 1,
1342 .needed = &virtio_extra_state_needed,
1343 .fields = (VMStateField[]) {
1344 {
1345 .name = "extra_state",
1346 .version_id = 0,
1347 .field_exists = NULL,
1348 .size = 0,
1349 .info = &vmstate_info_extra_state,
1350 .flags = VMS_SINGLE,
1351 .offset = 0,
1352 },
1353 VMSTATE_END_OF_LIST()
1354 }
1355};
1356
616a6552
GK
1357static const VMStateDescription vmstate_virtio_device_endian = {
1358 .name = "virtio/device_endian",
1359 .version_id = 1,
1360 .minimum_version_id = 1,
5cd8cada 1361 .needed = &virtio_device_endian_needed,
616a6552
GK
1362 .fields = (VMStateField[]) {
1363 VMSTATE_UINT8(device_endian, VirtIODevice),
1364 VMSTATE_END_OF_LIST()
1365 }
1366};
1367
019a3edb
GH
1368static const VMStateDescription vmstate_virtio_64bit_features = {
1369 .name = "virtio/64bit_features",
1370 .version_id = 1,
1371 .minimum_version_id = 1,
5cd8cada 1372 .needed = &virtio_64bit_features_needed,
019a3edb
GH
1373 .fields = (VMStateField[]) {
1374 VMSTATE_UINT64(guest_features, VirtIODevice),
1375 VMSTATE_END_OF_LIST()
1376 }
1377};
1378
6b321a3d
GK
1379static const VMStateDescription vmstate_virtio = {
1380 .name = "virtio",
1381 .version_id = 1,
1382 .minimum_version_id = 1,
1383 .minimum_version_id_old = 1,
1384 .fields = (VMStateField[]) {
1385 VMSTATE_END_OF_LIST()
616a6552 1386 },
5cd8cada
JQ
1387 .subsections = (const VMStateDescription*[]) {
1388 &vmstate_virtio_device_endian,
1389 &vmstate_virtio_64bit_features,
74aae7b2 1390 &vmstate_virtio_virtqueues,
46c5d082 1391 &vmstate_virtio_ringsize,
a6df8adf 1392 &vmstate_virtio_extra_state,
5cd8cada 1393 NULL
6b321a3d
GK
1394 }
1395};
1396
967f97fa
AL
1397void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1398{
1c819449
FK
1399 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1400 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1b5fc0de 1401 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
019a3edb 1402 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
967f97fa
AL
1403 int i;
1404
1c819449
FK
1405 if (k->save_config) {
1406 k->save_config(qbus->parent, f);
1407 }
967f97fa 1408
967f97fa
AL
1409 qemu_put_8s(f, &vdev->status);
1410 qemu_put_8s(f, &vdev->isr);
1411 qemu_put_be16s(f, &vdev->queue_sel);
019a3edb 1412 qemu_put_be32s(f, &guest_features_lo);
967f97fa
AL
1413 qemu_put_be32(f, vdev->config_len);
1414 qemu_put_buffer(f, vdev->config, vdev->config_len);
1415
87b3bd1c 1416 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
967f97fa
AL
1417 if (vdev->vq[i].vring.num == 0)
1418 break;
1419 }
1420
1421 qemu_put_be32(f, i);
1422
87b3bd1c 1423 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
967f97fa
AL
1424 if (vdev->vq[i].vring.num == 0)
1425 break;
1426
1427 qemu_put_be32(f, vdev->vq[i].vring.num);
6ce69d1c
PM
1428 if (k->has_variable_vring_alignment) {
1429 qemu_put_be32(f, vdev->vq[i].vring.align);
1430 }
ab223c95
CH
1431 /* XXX virtio-1 devices */
1432 qemu_put_be64(f, vdev->vq[i].vring.desc);
967f97fa 1433 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1c819449
FK
1434 if (k->save_queue) {
1435 k->save_queue(qbus->parent, i, f);
1436 }
967f97fa 1437 }
1b5fc0de
GK
1438
1439 if (vdc->save != NULL) {
1440 vdc->save(vdev, f);
1441 }
6b321a3d
GK
1442
1443 /* Subsections */
8118f095 1444 vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
967f97fa
AL
1445}
1446
6c0196d7 1447static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
ad0c9332 1448{
181103cd 1449 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
6b8f1020 1450 bool bad = (val & ~(vdev->host_features)) != 0;
ad0c9332 1451
6b8f1020 1452 val &= vdev->host_features;
181103cd
FK
1453 if (k->set_features) {
1454 k->set_features(vdev, val);
ad0c9332
PB
1455 }
1456 vdev->guest_features = val;
1457 return bad ? -1 : 0;
1458}
1459
6c0196d7
CH
1460int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1461{
1462 /*
1463 * The driver must not attempt to set features after feature negotiation
1464 * has finished.
1465 */
1466 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1467 return -EINVAL;
1468 }
1469 return virtio_set_features_nocheck(vdev, val);
1470}
1471
1b5fc0de 1472int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
967f97fa 1473{
cc459952 1474 int i, ret;
a890a2f9 1475 int32_t config_len;
cc459952 1476 uint32_t num;
6d74ca5a 1477 uint32_t features;
1c819449
FK
1478 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1479 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1b5fc0de 1480 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
967f97fa 1481
616a6552
GK
1482 /*
1483 * We poison the endianness to ensure it does not get used before
1484 * subsections have been loaded.
1485 */
1486 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1487
1c819449
FK
1488 if (k->load_config) {
1489 ret = k->load_config(qbus->parent, f);
ff24bd58
MT
1490 if (ret)
1491 return ret;
1492 }
967f97fa 1493
967f97fa
AL
1494 qemu_get_8s(f, &vdev->status);
1495 qemu_get_8s(f, &vdev->isr);
1496 qemu_get_be16s(f, &vdev->queue_sel);
87b3bd1c 1497 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
4b53c2c7
MR
1498 return -1;
1499 }
6d74ca5a 1500 qemu_get_be32s(f, &features);
ad0c9332 1501
62cee1a2
MT
1502 /*
1503 * Temporarily set guest_features low bits - needed by
1504 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1505 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1506 *
1507 * Note: devices should always test host features in future - don't create
1508 * new dependencies like this.
1509 */
1510 vdev->guest_features = features;
1511
a890a2f9 1512 config_len = qemu_get_be32(f);
2f5732e9
DDAG
1513
1514 /*
1515 * There are cases where the incoming config can be bigger or smaller
1516 * than what we have; so load what we have space for, and skip
1517 * any excess that's in the stream.
1518 */
1519 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1520
1521 while (config_len > vdev->config_len) {
1522 qemu_get_byte(f);
1523 config_len--;
a890a2f9 1524 }
967f97fa
AL
1525
1526 num = qemu_get_be32(f);
1527
87b3bd1c 1528 if (num > VIRTIO_QUEUE_MAX) {
8a1be662 1529 error_report("Invalid number of virtqueues: 0x%x", num);
cc459952
MT
1530 return -1;
1531 }
1532
967f97fa
AL
1533 for (i = 0; i < num; i++) {
1534 vdev->vq[i].vring.num = qemu_get_be32(f);
6ce69d1c
PM
1535 if (k->has_variable_vring_alignment) {
1536 vdev->vq[i].vring.align = qemu_get_be32(f);
1537 }
ab223c95 1538 vdev->vq[i].vring.desc = qemu_get_be64(f);
967f97fa 1539 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
bcbabae8
MT
1540 vdev->vq[i].signalled_used_valid = false;
1541 vdev->vq[i].notification = true;
967f97fa 1542
ab223c95
CH
1543 if (vdev->vq[i].vring.desc) {
1544 /* XXX virtio-1 devices */
1545 virtio_queue_update_rings(vdev, i);
1abeb5a6
MT
1546 } else if (vdev->vq[i].last_avail_idx) {
1547 error_report("VQ %d address 0x0 "
6daf194d 1548 "inconsistent with Host index 0x%x",
1abeb5a6
MT
1549 i, vdev->vq[i].last_avail_idx);
1550 return -1;
258dc7c9 1551 }
1c819449
FK
1552 if (k->load_queue) {
1553 ret = k->load_queue(qbus->parent, i, f);
ff24bd58
MT
1554 if (ret)
1555 return ret;
7055e687 1556 }
967f97fa
AL
1557 }
1558
7055e687 1559 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1b5fc0de
GK
1560
1561 if (vdc->load != NULL) {
6b321a3d
GK
1562 ret = vdc->load(vdev, f, version_id);
1563 if (ret) {
1564 return ret;
1565 }
1b5fc0de
GK
1566 }
1567
616a6552
GK
1568 /* Subsections */
1569 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1570 if (ret) {
1571 return ret;
1572 }
1573
1574 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1575 vdev->device_endian = virtio_default_endian();
1576 }
1577
019a3edb
GH
1578 if (virtio_64bit_features_needed(vdev)) {
1579 /*
1580 * Subsection load filled vdev->guest_features. Run them
1581 * through virtio_set_features to sanity-check them against
1582 * host_features.
1583 */
1584 uint64_t features64 = vdev->guest_features;
6c0196d7 1585 if (virtio_set_features_nocheck(vdev, features64) < 0) {
019a3edb
GH
1586 error_report("Features 0x%" PRIx64 " unsupported. "
1587 "Allowed features: 0x%" PRIx64,
1588 features64, vdev->host_features);
1589 return -1;
1590 }
1591 } else {
6c0196d7 1592 if (virtio_set_features_nocheck(vdev, features) < 0) {
019a3edb
GH
1593 error_report("Features 0x%x unsupported. "
1594 "Allowed features: 0x%" PRIx64,
1595 features, vdev->host_features);
1596 return -1;
1597 }
1598 }
1599
616a6552 1600 for (i = 0; i < num; i++) {
ab223c95 1601 if (vdev->vq[i].vring.desc) {
616a6552
GK
1602 uint16_t nheads;
1603 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1604 /* Check it isn't doing strange things with descriptor numbers. */
1605 if (nheads > vdev->vq[i].vring.num) {
1606 error_report("VQ %d size 0x%x Guest index 0x%x "
1607 "inconsistent with Host index 0x%x: delta 0x%x",
1608 i, vdev->vq[i].vring.num,
1609 vring_avail_idx(&vdev->vq[i]),
1610 vdev->vq[i].last_avail_idx, nheads);
1611 return -1;
1612 }
b796fcd1 1613 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
be1fea9b 1614 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
616a6552
GK
1615 }
1616 }
1617
1618 return 0;
967f97fa
AL
1619}
1620
6a1a8cc7 1621void virtio_cleanup(VirtIODevice *vdev)
b946a153 1622{
85cf2a8d 1623 qemu_del_vm_change_state_handler(vdev->vmstate);
6f79e06b 1624 g_free(vdev->config);
7267c094 1625 g_free(vdev->vq);
e0d686bf 1626 g_free(vdev->vector_queues);
8e05db92
FK
1627}
1628
1dfb4dd9 1629static void virtio_vmstate_change(void *opaque, int running, RunState state)
85cf2a8d
MT
1630{
1631 VirtIODevice *vdev = opaque;
1c819449
FK
1632 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1633 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
85cf2a8d 1634 bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
9e8e8c48 1635 vdev->vm_running = running;
85cf2a8d
MT
1636
1637 if (backend_run) {
1638 virtio_set_status(vdev, vdev->status);
1639 }
1640
1c819449
FK
1641 if (k->vmstate_change) {
1642 k->vmstate_change(qbus->parent, backend_run);
85cf2a8d
MT
1643 }
1644
1645 if (!backend_run) {
1646 virtio_set_status(vdev, vdev->status);
1647 }
1648}
1649
c8075caf
GA
1650void virtio_instance_init_common(Object *proxy_obj, void *data,
1651 size_t vdev_size, const char *vdev_name)
1652{
1653 DeviceState *vdev = data;
1654
1655 object_initialize(vdev, vdev_size, vdev_name);
1656 object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1657 object_unref(OBJECT(vdev));
1658 qdev_alias_all_properties(vdev, proxy_obj);
1659}
1660
8e05db92
FK
1661void virtio_init(VirtIODevice *vdev, const char *name,
1662 uint16_t device_id, size_t config_size)
967f97fa 1663{
e0d686bf
JW
1664 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1665 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
b8193adb 1666 int i;
e0d686bf
JW
1667 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1668
1669 if (nvectors) {
1670 vdev->vector_queues =
1671 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1672 }
1673
53c25cea 1674 vdev->device_id = device_id;
967f97fa
AL
1675 vdev->status = 0;
1676 vdev->isr = 0;
1677 vdev->queue_sel = 0;
7055e687 1678 vdev->config_vector = VIRTIO_NO_VECTOR;
87b3bd1c 1679 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1354869c 1680 vdev->vm_running = runstate_is_running();
87b3bd1c 1681 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
b8193adb 1682 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1cbdabe2 1683 vdev->vq[i].vdev = vdev;
e78a2b42 1684 vdev->vq[i].queue_index = i;
1cbdabe2 1685 }
967f97fa 1686
967f97fa
AL
1687 vdev->name = name;
1688 vdev->config_len = config_size;
8e05db92 1689 if (vdev->config_len) {
7267c094 1690 vdev->config = g_malloc0(config_size);
8e05db92 1691 } else {
967f97fa 1692 vdev->config = NULL;
8e05db92
FK
1693 }
1694 vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1695 vdev);
616a6552 1696 vdev->device_endian = virtio_default_endian();
5669655a 1697 vdev->use_guest_notifier_mask = true;
8e05db92 1698}
967f97fa 1699
a8170e5e 1700hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1cbdabe2
MT
1701{
1702 return vdev->vq[n].vring.desc;
1703}
1704
a8170e5e 1705hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1cbdabe2
MT
1706{
1707 return vdev->vq[n].vring.avail;
1708}
1709
a8170e5e 1710hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1cbdabe2
MT
1711{
1712 return vdev->vq[n].vring.used;
1713}
1714
a8170e5e 1715hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1cbdabe2
MT
1716{
1717 return vdev->vq[n].vring.desc;
1718}
1719
a8170e5e 1720hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1cbdabe2
MT
1721{
1722 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1723}
1724
a8170e5e 1725hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1cbdabe2
MT
1726{
1727 return offsetof(VRingAvail, ring) +
50764fc8 1728 sizeof(uint16_t) * vdev->vq[n].vring.num;
1cbdabe2
MT
1729}
1730
a8170e5e 1731hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1cbdabe2
MT
1732{
1733 return offsetof(VRingUsed, ring) +
1734 sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1735}
1736
a8170e5e 1737hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1cbdabe2
MT
1738{
1739 return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1740 virtio_queue_get_used_size(vdev, n);
1741}
1742
1743uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1744{
1745 return vdev->vq[n].last_avail_idx;
1746}
1747
1748void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1749{
1750 vdev->vq[n].last_avail_idx = idx;
be1fea9b 1751 vdev->vq[n].shadow_avail_idx = idx;
1cbdabe2
MT
1752}
1753
6793dfd1
SH
1754void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1755{
1756 vdev->vq[n].signalled_used_valid = false;
1757}
1758
1cbdabe2
MT
1759VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1760{
1761 return vdev->vq + n;
1762}
1763
e78a2b42
JW
1764uint16_t virtio_get_queue_index(VirtQueue *vq)
1765{
1766 return vq->queue_index;
1767}
1768
15b2bd18
PB
1769static void virtio_queue_guest_notifier_read(EventNotifier *n)
1770{
1771 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1772 if (event_notifier_test_and_clear(n)) {
1773 virtio_irq(vq);
1774 }
1775}
1776
1777void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1778 bool with_irqfd)
1779{
1780 if (assign && !with_irqfd) {
54e18d35 1781 event_notifier_set_handler(&vq->guest_notifier, false,
15b2bd18
PB
1782 virtio_queue_guest_notifier_read);
1783 } else {
54e18d35 1784 event_notifier_set_handler(&vq->guest_notifier, false, NULL);
15b2bd18
PB
1785 }
1786 if (!assign) {
1787 /* Test and clear notifier before closing it,
1788 * in case poll callback didn't have time to run. */
1789 virtio_queue_guest_notifier_read(&vq->guest_notifier);
1790 }
1791}
1792
1cbdabe2
MT
1793EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1794{
1795 return &vq->guest_notifier;
1796}
b1f416aa 1797
344dc16f 1798static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
b1f416aa
PB
1799{
1800 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1801 if (event_notifier_test_and_clear(n)) {
344dc16f 1802 virtio_queue_notify_aio_vq(vq);
b1f416aa
PB
1803 }
1804}
1805
a1afb606 1806void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
bf1780b0 1807 VirtIOHandleOutput handle_output)
a1afb606 1808{
a378b49a
PB
1809 if (handle_output) {
1810 vq->handle_aio_output = handle_output;
a1afb606 1811 aio_set_event_notifier(ctx, &vq->host_notifier, true,
344dc16f 1812 virtio_queue_host_notifier_aio_read);
a1afb606
PB
1813 } else {
1814 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
a1afb606
PB
1815 /* Test and clear notifier before after disabling event,
1816 * in case poll callback didn't have time to run. */
344dc16f 1817 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
a378b49a 1818 vq->handle_aio_output = NULL;
344dc16f
MT
1819 }
1820}
1821
1822static void virtio_queue_host_notifier_read(EventNotifier *n)
1823{
1824 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1825 if (event_notifier_test_and_clear(n)) {
1826 virtio_queue_notify_vq(vq);
a1afb606
PB
1827 }
1828}
1829
26b9b5fe
PB
1830void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1831 bool set_handler)
b1f416aa 1832{
26b9b5fe 1833 if (assign && set_handler) {
14560d69 1834 event_notifier_set_handler(&vq->host_notifier, true,
b1f416aa
PB
1835 virtio_queue_host_notifier_read);
1836 } else {
14560d69 1837 event_notifier_set_handler(&vq->host_notifier, true, NULL);
26b9b5fe
PB
1838 }
1839 if (!assign) {
b1f416aa
PB
1840 /* Test and clear notifier before after disabling event,
1841 * in case poll callback didn't have time to run. */
1842 virtio_queue_host_notifier_read(&vq->host_notifier);
1843 }
1844}
1845
1cbdabe2
MT
1846EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1847{
1848 return &vq->host_notifier;
1849}
8e05db92 1850
1034e9cf
FK
1851void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1852{
9e288406 1853 g_free(vdev->bus_name);
80e0090a 1854 vdev->bus_name = g_strdup(bus_name);
1034e9cf
FK
1855}
1856
1d244b42
AF
1857static void virtio_device_realize(DeviceState *dev, Error **errp)
1858{
1859 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1860 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1861 Error *err = NULL;
1862
1d244b42
AF
1863 if (vdc->realize != NULL) {
1864 vdc->realize(dev, &err);
1865 if (err != NULL) {
1866 error_propagate(errp, err);
1867 return;
1868 }
8e05db92 1869 }
e8398045
JW
1870
1871 virtio_bus_device_plugged(vdev, &err);
1872 if (err != NULL) {
1873 error_propagate(errp, err);
1874 return;
1875 }
8e05db92
FK
1876}
1877
1d244b42 1878static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1034e9cf 1879{
1d244b42 1880 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
306ec6c3
AF
1881 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1882 Error *err = NULL;
1d244b42 1883
83d07047
PB
1884 virtio_bus_device_unplugged(vdev);
1885
306ec6c3
AF
1886 if (vdc->unrealize != NULL) {
1887 vdc->unrealize(dev, &err);
1888 if (err != NULL) {
1889 error_propagate(errp, err);
1890 return;
1891 }
5e96f5d2 1892 }
1d244b42 1893
9e288406
MA
1894 g_free(vdev->bus_name);
1895 vdev->bus_name = NULL;
1034e9cf
FK
1896}
1897
6b8f1020
CH
1898static Property virtio_properties[] = {
1899 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1900 DEFINE_PROP_END_OF_LIST(),
1901};
1902
8e05db92
FK
1903static void virtio_device_class_init(ObjectClass *klass, void *data)
1904{
1905 /* Set the default value here. */
1906 DeviceClass *dc = DEVICE_CLASS(klass);
1d244b42
AF
1907
1908 dc->realize = virtio_device_realize;
1909 dc->unrealize = virtio_device_unrealize;
8e05db92 1910 dc->bus_type = TYPE_VIRTIO_BUS;
6b8f1020 1911 dc->props = virtio_properties;
8e05db92
FK
1912}
1913
1914static const TypeInfo virtio_device_info = {
1915 .name = TYPE_VIRTIO_DEVICE,
1916 .parent = TYPE_DEVICE,
1917 .instance_size = sizeof(VirtIODevice),
1918 .class_init = virtio_device_class_init,
1919 .abstract = true,
1920 .class_size = sizeof(VirtioDeviceClass),
1921};
1922
1923static void virtio_register_types(void)
1924{
1925 type_register_static(&virtio_device_info);
1926}
1927
1928type_init(virtio_register_types)