1 /* Simple test of virtio code, entirely in userpsace. */
5 #include <linux/kernel.h>
7 #include <linux/virtio.h>
8 #include <linux/vringh.h>
9 #include <linux/virtio_ring.h>
10 #include <linux/virtio_config.h>
11 #include <linux/uaccess.h>
12 #include <sys/types.h>
18 #define USER_MEM (1024*1024)
19 void *__user_addr_min
, *__user_addr_max
;
20 void *__kmalloc_fake
, *__kfree_ignore_start
, *__kfree_ignore_end
;
21 static u64 user_addr_offset
;
26 static bool never_notify_host(struct virtqueue
*vq
)
31 static void never_callback_guest(struct virtqueue
*vq
)
36 static bool getrange_iov(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
)
38 if (addr
< (u64
)(unsigned long)__user_addr_min
- user_addr_offset
)
40 if (addr
>= (u64
)(unsigned long)__user_addr_max
- user_addr_offset
)
43 r
->start
= (u64
)(unsigned long)__user_addr_min
- user_addr_offset
;
44 r
->end_incl
= (u64
)(unsigned long)__user_addr_max
- 1 - user_addr_offset
;
45 r
->offset
= user_addr_offset
;
49 /* We return single byte ranges. */
50 static bool getrange_slow(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
)
52 if (addr
< (u64
)(unsigned long)__user_addr_min
- user_addr_offset
)
54 if (addr
>= (u64
)(unsigned long)__user_addr_max
- user_addr_offset
)
58 r
->end_incl
= r
->start
;
59 r
->offset
= user_addr_offset
;
63 struct guest_virtio_device
{
64 struct virtio_device vdev
;
66 unsigned long notifies
;
69 static bool parallel_notify_host(struct virtqueue
*vq
)
72 struct guest_virtio_device
*gvdev
;
74 gvdev
= container_of(vq
->vdev
, struct guest_virtio_device
, vdev
);
75 rc
= write(gvdev
->to_host_fd
, "", 1);
82 static bool no_notify_host(struct virtqueue
*vq
)
87 #define NUM_XFERS (10000000)
89 /* We aim for two "distant" cpus. */
90 static void find_cpus(unsigned int *first
, unsigned int *last
)
96 for (i
= 0; i
< 4096; i
++) {
100 if (sched_setaffinity(getpid(), sizeof(set
), &set
) == 0) {
109 /* Opencoded version for fast mode */
110 static inline int vringh_get_head(struct vringh
*vrh
, u16
*head
)
115 err
= get_user(avail_idx
, &vrh
->vring
.avail
->idx
);
119 if (vrh
->last_avail_idx
== avail_idx
)
122 /* Only get avail ring entries after they have been exposed by guest. */
123 virtio_rmb(vrh
->weak_barriers
);
125 i
= vrh
->last_avail_idx
& (vrh
->vring
.num
- 1);
127 err
= get_user(*head
, &vrh
->vring
.avail
->ring
[i
]);
131 vrh
->last_avail_idx
++;
135 static int parallel_test(u64 features
,
136 bool (*getrange
)(struct vringh
*vrh
,
137 u64 addr
, struct vringh_range
*r
),
140 void *host_map
, *guest_map
;
141 int fd
, mapsize
, to_guest
[2], to_host
[2];
142 unsigned long xfers
= 0, notifies
= 0, receives
= 0;
143 unsigned int first_cpu
, last_cpu
;
147 /* Create real file to mmap. */
148 fd
= open("/tmp/vringh_test-file", O_RDWR
|O_CREAT
|O_TRUNC
, 0600);
150 err(1, "Opening /tmp/vringh_test-file");
152 /* Extra room at the end for some data, and indirects */
153 mapsize
= vring_size(RINGSIZE
, ALIGN
)
154 + RINGSIZE
* 2 * sizeof(int)
155 + RINGSIZE
* 6 * sizeof(struct vring_desc
);
156 mapsize
= (mapsize
+ getpagesize() - 1) & ~(getpagesize() - 1);
157 ftruncate(fd
, mapsize
);
159 /* Parent and child use separate addresses, to check our mapping logic! */
160 host_map
= mmap(NULL
, mapsize
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
161 guest_map
= mmap(NULL
, mapsize
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
167 find_cpus(&first_cpu
, &last_cpu
);
168 printf("Using CPUS %u and %u\n", first_cpu
, last_cpu
);
173 int status
, err
, rlen
= 0;
176 /* We are the host: never access guest addresses! */
177 munmap(guest_map
, mapsize
);
179 __user_addr_min
= host_map
;
180 __user_addr_max
= __user_addr_min
+ mapsize
;
181 user_addr_offset
= host_map
- guest_map
;
182 assert(user_addr_offset
);
187 vring_init(&vrh
.vring
, RINGSIZE
, host_map
, ALIGN
);
188 vringh_init_user(&vrh
, features
, RINGSIZE
, true,
189 vrh
.vring
.desc
, vrh
.vring
.avail
, vrh
.vring
.used
);
190 CPU_SET(first_cpu
, &cpu_set
);
191 if (sched_setaffinity(getpid(), sizeof(cpu_set
), &cpu_set
))
192 errx(1, "Could not set affinity to cpu %u", first_cpu
);
194 while (xfers
< NUM_XFERS
) {
195 struct iovec host_riov
[2], host_wiov
[2];
196 struct vringh_iov riov
, wiov
;
201 err
= vringh_get_head(&vrh
, &head
);
204 err
= vringh_need_notify_user(&vrh
);
206 errx(1, "vringh_need_notify_user: %i",
209 write(to_guest
[1], "", 1);
214 errx(1, "vringh_get_head");
218 vringh_iov_init(&riov
,
220 ARRAY_SIZE(host_riov
));
221 vringh_iov_init(&wiov
,
223 ARRAY_SIZE(host_wiov
));
225 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
,
229 err
= vringh_need_notify_user(&vrh
);
231 errx(1, "vringh_need_notify_user: %i",
234 write(to_guest
[1], "", 1);
238 if (!vringh_notify_enable_user(&vrh
))
241 /* Swallow all notifies at once. */
242 if (read(to_host
[0], buf
, sizeof(buf
)) < 1)
245 vringh_notify_disable_user(&vrh
);
250 errx(1, "vringh_getdesc_user: %i", err
);
252 /* We simply copy bytes. */
254 rlen
= vringh_iov_pull_user(&riov
, rbuf
,
257 errx(1, "vringh_iov_pull_user: %i",
259 assert(riov
.i
== riov
.used
);
262 err
= vringh_iov_push_user(&wiov
, rbuf
, rlen
);
264 errx(1, "vringh_iov_push_user: %i",
266 assert(wiov
.i
== wiov
.used
);
272 err
= vringh_complete_user(&vrh
, head
, written
);
274 errx(1, "vringh_complete_user: %i", err
);
277 err
= vringh_need_notify_user(&vrh
);
279 errx(1, "vringh_need_notify_user: %i", err
);
281 write(to_guest
[1], "", 1);
285 if (!WIFEXITED(status
))
286 errx(1, "Child died with signal %i?", WTERMSIG(status
));
287 if (WEXITSTATUS(status
) != 0)
288 errx(1, "Child exited %i?", WEXITSTATUS(status
));
289 printf("Host: notified %lu, pinged %lu\n", notifies
, receives
);
292 struct guest_virtio_device gvdev
;
293 struct virtqueue
*vq
;
295 struct vring_desc
*indirects
;
296 unsigned int finished
= 0;
298 /* We pass sg[]s pointing into here, but we need RINGSIZE+1 */
299 data
= guest_map
+ vring_size(RINGSIZE
, ALIGN
);
300 indirects
= (void *)data
+ (RINGSIZE
+ 1) * 2 * sizeof(int);
302 /* We are the guest. */
303 munmap(host_map
, mapsize
);
308 gvdev
.vdev
.features
= features
;
309 gvdev
.to_host_fd
= to_host
[1];
312 CPU_SET(first_cpu
, &cpu_set
);
313 if (sched_setaffinity(getpid(), sizeof(cpu_set
), &cpu_set
))
314 err(1, "Could not set affinity to cpu %u", first_cpu
);
316 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &gvdev
.vdev
, true,
318 fast_vringh
? no_notify_host
319 : parallel_notify_host
,
320 never_callback_guest
, "guest vq");
322 /* Don't kfree indirects. */
323 __kfree_ignore_start
= indirects
;
324 __kfree_ignore_end
= indirects
+ RINGSIZE
* 6;
326 while (xfers
< NUM_XFERS
) {
327 struct scatterlist sg
[4];
328 unsigned int num_sg
, len
;
330 bool output
= !(xfers
% 2);
333 while ((dbuf
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
335 assert(*dbuf
== finished
- 1);
336 else if (!fast_vringh
)
337 assert(*dbuf
== finished
);
341 /* Produce a buffer. */
342 dbuf
= data
+ (xfers
% (RINGSIZE
+ 1));
349 switch ((xfers
/ sizeof(*dbuf
)) % 4) {
351 /* Nasty three-element sg list. */
352 sg_init_table(sg
, num_sg
= 3);
353 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
354 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 2);
355 sg_set_buf(&sg
[2], (void *)dbuf
+ 3, 1);
358 sg_init_table(sg
, num_sg
= 2);
359 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
360 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 3);
363 sg_init_table(sg
, num_sg
= 1);
364 sg_set_buf(&sg
[0], (void *)dbuf
, 4);
367 sg_init_table(sg
, num_sg
= 4);
368 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
369 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 1);
370 sg_set_buf(&sg
[2], (void *)dbuf
+ 2, 1);
371 sg_set_buf(&sg
[3], (void *)dbuf
+ 3, 1);
375 /* May allocate an indirect, so force it to allocate
377 __kmalloc_fake
= indirects
+ (xfers
% RINGSIZE
) * 4;
379 err
= virtqueue_add_outbuf(vq
, sg
, num_sg
, dbuf
,
382 err
= virtqueue_add_inbuf(vq
, sg
, num_sg
,
385 if (err
== -ENOSPC
) {
386 if (!virtqueue_enable_cb_delayed(vq
))
388 /* Swallow all notifies at once. */
389 if (read(to_guest
[0], buf
, sizeof(buf
)) < 1)
393 virtqueue_disable_cb(vq
);
398 errx(1, "virtqueue_add_in/outbuf: %i", err
);
405 while (finished
!= xfers
) {
410 dbuf
= virtqueue_get_buf(vq
, &len
);
413 assert(*dbuf
== finished
- 1);
420 if (!virtqueue_enable_cb_delayed(vq
))
422 if (read(to_guest
[0], buf
, sizeof(buf
)) < 1)
426 virtqueue_disable_cb(vq
);
429 printf("Guest: notified %lu, pinged %lu\n",
430 gvdev
.notifies
, receives
);
431 vring_del_virtqueue(vq
);
436 int main(int argc
, char *argv
[])
438 struct virtio_device vdev
;
439 struct virtqueue
*vq
;
441 struct scatterlist guest_sg
[RINGSIZE
], *sgs
[2];
442 struct iovec host_riov
[2], host_wiov
[2];
443 struct vringh_iov riov
, wiov
;
444 struct vring_used_elem used
[RINGSIZE
];
450 bool (*getrange
)(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
);
451 bool fast_vringh
= false, parallel
= false;
453 getrange
= getrange_iov
;
457 if (strcmp(argv
[1], "--indirect") == 0)
458 __virtio_set_bit(&vdev
, VIRTIO_RING_F_INDIRECT_DESC
);
459 else if (strcmp(argv
[1], "--eventidx") == 0)
460 __virtio_set_bit(&vdev
, VIRTIO_RING_F_EVENT_IDX
);
461 else if (strcmp(argv
[1], "--virtio-1") == 0)
462 __virtio_set_bit(&vdev
, VIRTIO_F_VERSION_1
);
463 else if (strcmp(argv
[1], "--slow-range") == 0)
464 getrange
= getrange_slow
;
465 else if (strcmp(argv
[1], "--fast-vringh") == 0)
467 else if (strcmp(argv
[1], "--parallel") == 0)
470 errx(1, "Unknown arg %s", argv
[1]);
475 return parallel_test(vdev
.features
, getrange
, fast_vringh
);
477 if (posix_memalign(&__user_addr_min
, PAGE_SIZE
, USER_MEM
) != 0)
479 __user_addr_max
= __user_addr_min
+ USER_MEM
;
480 memset(__user_addr_min
, 0, vring_size(RINGSIZE
, ALIGN
));
482 /* Set up guest side. */
483 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &vdev
, true, false,
485 never_notify_host
, never_callback_guest
,
488 /* Set up host side. */
489 vring_init(&vrh
.vring
, RINGSIZE
, __user_addr_min
, ALIGN
);
490 vringh_init_user(&vrh
, vdev
.features
, RINGSIZE
, true,
491 vrh
.vring
.desc
, vrh
.vring
.avail
, vrh
.vring
.used
);
493 /* No descriptor to get yet... */
494 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
496 errx(1, "vringh_getdesc_user: %i", err
);
498 /* Guest puts in a descriptor. */
499 memcpy(__user_addr_max
- 1, "a", 1);
500 sg_init_table(guest_sg
, 1);
501 sg_set_buf(&guest_sg
[0], __user_addr_max
- 1, 1);
502 sg_init_table(guest_sg
+1, 1);
503 sg_set_buf(&guest_sg
[1], __user_addr_max
- 3, 2);
504 sgs
[0] = &guest_sg
[0];
505 sgs
[1] = &guest_sg
[1];
507 /* May allocate an indirect, so force it to allocate user addr */
508 __kmalloc_fake
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
509 err
= virtqueue_add_sgs(vq
, sgs
, 1, 1, &err
, GFP_KERNEL
);
511 errx(1, "virtqueue_add_sgs: %i", err
);
512 __kmalloc_fake
= NULL
;
514 /* Host retreives it. */
515 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
516 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
518 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
520 errx(1, "vringh_getdesc_user: %i", err
);
522 assert(riov
.used
== 1);
523 assert(riov
.iov
[0].iov_base
== __user_addr_max
- 1);
524 assert(riov
.iov
[0].iov_len
== 1);
525 if (getrange
!= getrange_slow
) {
526 assert(wiov
.used
== 1);
527 assert(wiov
.iov
[0].iov_base
== __user_addr_max
- 3);
528 assert(wiov
.iov
[0].iov_len
== 2);
530 assert(wiov
.used
== 2);
531 assert(wiov
.iov
[0].iov_base
== __user_addr_max
- 3);
532 assert(wiov
.iov
[0].iov_len
== 1);
533 assert(wiov
.iov
[1].iov_base
== __user_addr_max
- 2);
534 assert(wiov
.iov
[1].iov_len
== 1);
537 err
= vringh_iov_pull_user(&riov
, buf
, 5);
539 errx(1, "vringh_iov_pull_user: %i", err
);
540 assert(buf
[0] == 'a');
542 assert(vringh_iov_pull_user(&riov
, buf
, 5) == 0);
544 memcpy(buf
, "bcdef", 5);
545 err
= vringh_iov_push_user(&wiov
, buf
, 5);
547 errx(1, "vringh_iov_push_user: %i", err
);
548 assert(memcmp(__user_addr_max
- 3, "bc", 2) == 0);
549 assert(wiov
.i
== wiov
.used
);
550 assert(vringh_iov_push_user(&wiov
, buf
, 5) == 0);
553 err
= vringh_complete_user(&vrh
, head
, err
);
555 errx(1, "vringh_complete_user: %i", err
);
557 /* Guest should see used token now. */
558 __kfree_ignore_start
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
559 __kfree_ignore_end
= __kfree_ignore_start
+ 1;
560 ret
= virtqueue_get_buf(vq
, &i
);
562 errx(1, "virtqueue_get_buf: %p", ret
);
565 /* Guest puts in a huge descriptor. */
566 sg_init_table(guest_sg
, RINGSIZE
);
567 for (i
= 0; i
< RINGSIZE
; i
++) {
568 sg_set_buf(&guest_sg
[i
],
569 __user_addr_max
- USER_MEM
/4, USER_MEM
/4);
572 /* Fill contents with recognisable garbage. */
573 for (i
= 0; i
< USER_MEM
/4; i
++)
574 ((char *)__user_addr_max
- USER_MEM
/4)[i
] = i
;
576 /* This will allocate an indirect, so force it to allocate user addr */
577 __kmalloc_fake
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
578 err
= virtqueue_add_outbuf(vq
, guest_sg
, RINGSIZE
, &err
, GFP_KERNEL
);
580 errx(1, "virtqueue_add_outbuf (large): %i", err
);
581 __kmalloc_fake
= NULL
;
583 /* Host picks it up (allocates new iov). */
584 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
585 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
587 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
589 errx(1, "vringh_getdesc_user: %i", err
);
591 assert(riov
.max_num
& VRINGH_IOV_ALLOCATED
);
592 assert(riov
.iov
!= host_riov
);
593 if (getrange
!= getrange_slow
)
594 assert(riov
.used
== RINGSIZE
);
596 assert(riov
.used
== RINGSIZE
* USER_MEM
/4);
598 assert(!(wiov
.max_num
& VRINGH_IOV_ALLOCATED
));
599 assert(wiov
.used
== 0);
601 /* Pull data back out (in odd chunks), should be as expected. */
602 for (i
= 0; i
< RINGSIZE
* USER_MEM
/4; i
+= 3) {
603 err
= vringh_iov_pull_user(&riov
, buf
, 3);
604 if (err
!= 3 && i
+ err
!= RINGSIZE
* USER_MEM
/4)
605 errx(1, "vringh_iov_pull_user large: %i", err
);
606 assert(buf
[0] == (char)i
);
607 assert(err
< 2 || buf
[1] == (char)(i
+ 1));
608 assert(err
< 3 || buf
[2] == (char)(i
+ 2));
610 assert(riov
.i
== riov
.used
);
611 vringh_iov_cleanup(&riov
);
612 vringh_iov_cleanup(&wiov
);
614 /* Complete using multi interface, just because we can. */
617 err
= vringh_complete_multi_user(&vrh
, used
, 1);
619 errx(1, "vringh_complete_multi_user(1): %i", err
);
621 /* Free up those descriptors. */
622 ret
= virtqueue_get_buf(vq
, &i
);
624 errx(1, "virtqueue_get_buf: %p", ret
);
626 /* Add lots of descriptors. */
627 sg_init_table(guest_sg
, 1);
628 sg_set_buf(&guest_sg
[0], __user_addr_max
- 1, 1);
629 for (i
= 0; i
< RINGSIZE
; i
++) {
630 err
= virtqueue_add_outbuf(vq
, guest_sg
, 1, &err
, GFP_KERNEL
);
632 errx(1, "virtqueue_add_outbuf (multiple): %i", err
);
635 /* Now get many, and consume them all at once. */
636 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
637 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
639 for (i
= 0; i
< RINGSIZE
; i
++) {
640 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
642 errx(1, "vringh_getdesc_user: %i", err
);
646 /* Make sure it wraps around ring, to test! */
647 assert(vrh
.vring
.used
->idx
% RINGSIZE
!= 0);
648 err
= vringh_complete_multi_user(&vrh
, used
, RINGSIZE
);
650 errx(1, "vringh_complete_multi_user: %i", err
);
652 /* Free those buffers. */
653 for (i
= 0; i
< RINGSIZE
; i
++) {
655 assert(virtqueue_get_buf(vq
, &len
) != NULL
);
658 /* Test weird (but legal!) indirect. */
659 if (__virtio_test_bit(&vdev
, VIRTIO_RING_F_INDIRECT_DESC
)) {
660 char *data
= __user_addr_max
- USER_MEM
/4;
661 struct vring_desc
*d
= __user_addr_max
- USER_MEM
/2;
664 /* Force creation of direct, which we modify. */
665 __virtio_clear_bit(&vdev
, VIRTIO_RING_F_INDIRECT_DESC
);
666 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &vdev
, true,
667 false, __user_addr_min
,
669 never_callback_guest
,
672 sg_init_table(guest_sg
, 4);
673 sg_set_buf(&guest_sg
[0], d
, sizeof(*d
)*2);
674 sg_set_buf(&guest_sg
[1], d
+ 2, sizeof(*d
)*1);
675 sg_set_buf(&guest_sg
[2], data
+ 6, 4);
676 sg_set_buf(&guest_sg
[3], d
+ 3, sizeof(*d
)*3);
678 err
= virtqueue_add_outbuf(vq
, guest_sg
, 4, &err
, GFP_KERNEL
);
680 errx(1, "virtqueue_add_outbuf (indirect): %i", err
);
682 vring_init(&vring
, RINGSIZE
, __user_addr_min
, ALIGN
);
684 /* They're used in order, but double-check... */
685 assert(vring
.desc
[0].addr
== (unsigned long)d
);
686 assert(vring
.desc
[1].addr
== (unsigned long)(d
+2));
687 assert(vring
.desc
[2].addr
== (unsigned long)data
+ 6);
688 assert(vring
.desc
[3].addr
== (unsigned long)(d
+3));
689 vring
.desc
[0].flags
|= VRING_DESC_F_INDIRECT
;
690 vring
.desc
[1].flags
|= VRING_DESC_F_INDIRECT
;
691 vring
.desc
[3].flags
|= VRING_DESC_F_INDIRECT
;
694 d
[0].addr
= (unsigned long)data
;
696 d
[0].flags
= VRING_DESC_F_NEXT
;
698 d
[1].addr
= (unsigned long)data
+ 1;
702 /* Second indirect */
703 d
[2].addr
= (unsigned long)data
+ 3;
708 d
[3].addr
= (unsigned long)data
+ 10;
710 d
[3].flags
= VRING_DESC_F_NEXT
;
712 d
[4].addr
= (unsigned long)data
+ 15;
714 d
[4].flags
= VRING_DESC_F_NEXT
;
716 d
[5].addr
= (unsigned long)data
+ 21;
720 /* Host picks it up (allocates new iov). */
721 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
722 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
724 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
726 errx(1, "vringh_getdesc_user: %i", err
);
729 errx(1, "vringh_getdesc_user: head %i not 0", head
);
731 assert(riov
.max_num
& VRINGH_IOV_ALLOCATED
);
732 if (getrange
!= getrange_slow
)
733 assert(riov
.used
== 7);
735 assert(riov
.used
== 28);
736 err
= vringh_iov_pull_user(&riov
, buf
, 29);
739 /* Data should be linear. */
740 for (i
= 0; i
< err
; i
++)
742 vringh_iov_cleanup(&riov
);
745 /* Don't leak memory... */
746 vring_del_virtqueue(vq
);
747 free(__user_addr_min
);