]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1515c5ce RR |
2 | /* Simple test of virtio code, entirely in userpsace. */ |
3 | #define _GNU_SOURCE | |
4 | #include <sched.h> | |
5 | #include <err.h> | |
6 | #include <linux/kernel.h> | |
7 | #include <linux/err.h> | |
8 | #include <linux/virtio.h> | |
9 | #include <linux/vringh.h> | |
10 | #include <linux/virtio_ring.h> | |
2eaa614b | 11 | #include <linux/virtio_config.h> |
1515c5ce RR |
12 | #include <linux/uaccess.h> |
13 | #include <sys/types.h> | |
14 | #include <sys/stat.h> | |
15 | #include <sys/mman.h> | |
16 | #include <sys/wait.h> | |
17 | #include <fcntl.h> | |
18 | ||
19 | #define USER_MEM (1024*1024) | |
20 | void *__user_addr_min, *__user_addr_max; | |
21 | void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; | |
22 | static u64 user_addr_offset; | |
23 | ||
24 | #define RINGSIZE 256 | |
25 | #define ALIGN 4096 | |
26 | ||
46f9c2b9 | 27 | static bool never_notify_host(struct virtqueue *vq) |
1515c5ce RR |
28 | { |
29 | abort(); | |
30 | } | |
31 | ||
32 | static void never_callback_guest(struct virtqueue *vq) | |
33 | { | |
34 | abort(); | |
35 | } | |
36 | ||
37 | static bool getrange_iov(struct vringh *vrh, u64 addr, struct vringh_range *r) | |
38 | { | |
39 | if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset) | |
40 | return false; | |
41 | if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset) | |
42 | return false; | |
43 | ||
44 | r->start = (u64)(unsigned long)__user_addr_min - user_addr_offset; | |
45 | r->end_incl = (u64)(unsigned long)__user_addr_max - 1 - user_addr_offset; | |
46 | r->offset = user_addr_offset; | |
47 | return true; | |
48 | } | |
49 | ||
50 | /* We return single byte ranges. */ | |
51 | static bool getrange_slow(struct vringh *vrh, u64 addr, struct vringh_range *r) | |
52 | { | |
53 | if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset) | |
54 | return false; | |
55 | if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset) | |
56 | return false; | |
57 | ||
58 | r->start = addr; | |
59 | r->end_incl = r->start; | |
60 | r->offset = user_addr_offset; | |
61 | return true; | |
62 | } | |
63 | ||
64 | struct guest_virtio_device { | |
65 | struct virtio_device vdev; | |
66 | int to_host_fd; | |
67 | unsigned long notifies; | |
68 | }; | |
69 | ||
46f9c2b9 | 70 | static bool parallel_notify_host(struct virtqueue *vq) |
1515c5ce | 71 | { |
46f9c2b9 | 72 | int rc; |
1515c5ce RR |
73 | struct guest_virtio_device *gvdev; |
74 | ||
75 | gvdev = container_of(vq->vdev, struct guest_virtio_device, vdev); | |
46f9c2b9 HG |
76 | rc = write(gvdev->to_host_fd, "", 1); |
77 | if (rc < 0) | |
78 | return false; | |
1515c5ce | 79 | gvdev->notifies++; |
46f9c2b9 | 80 | return true; |
1515c5ce RR |
81 | } |
82 | ||
46f9c2b9 | 83 | static bool no_notify_host(struct virtqueue *vq) |
1515c5ce | 84 | { |
46f9c2b9 | 85 | return true; |
1515c5ce RR |
86 | } |
87 | ||
88 | #define NUM_XFERS (10000000) | |
89 | ||
90 | /* We aim for two "distant" cpus. */ | |
91 | static void find_cpus(unsigned int *first, unsigned int *last) | |
92 | { | |
93 | unsigned int i; | |
94 | ||
95 | *first = -1U; | |
96 | *last = 0; | |
97 | for (i = 0; i < 4096; i++) { | |
98 | cpu_set_t set; | |
99 | CPU_ZERO(&set); | |
100 | CPU_SET(i, &set); | |
101 | if (sched_setaffinity(getpid(), sizeof(set), &set) == 0) { | |
102 | if (i < *first) | |
103 | *first = i; | |
104 | if (i > *last) | |
105 | *last = i; | |
106 | } | |
107 | } | |
108 | } | |
109 | ||
110 | /* Opencoded version for fast mode */ | |
111 | static inline int vringh_get_head(struct vringh *vrh, u16 *head) | |
112 | { | |
113 | u16 avail_idx, i; | |
114 | int err; | |
115 | ||
116 | err = get_user(avail_idx, &vrh->vring.avail->idx); | |
117 | if (err) | |
118 | return err; | |
119 | ||
120 | if (vrh->last_avail_idx == avail_idx) | |
121 | return 0; | |
122 | ||
123 | /* Only get avail ring entries after they have been exposed by guest. */ | |
124 | virtio_rmb(vrh->weak_barriers); | |
125 | ||
126 | i = vrh->last_avail_idx & (vrh->vring.num - 1); | |
127 | ||
128 | err = get_user(*head, &vrh->vring.avail->ring[i]); | |
129 | if (err) | |
130 | return err; | |
131 | ||
132 | vrh->last_avail_idx++; | |
133 | return 1; | |
134 | } | |
135 | ||
e3edf032 | 136 | static int parallel_test(u64 features, |
1515c5ce RR |
137 | bool (*getrange)(struct vringh *vrh, |
138 | u64 addr, struct vringh_range *r), | |
139 | bool fast_vringh) | |
140 | { | |
141 | void *host_map, *guest_map; | |
142 | int fd, mapsize, to_guest[2], to_host[2]; | |
143 | unsigned long xfers = 0, notifies = 0, receives = 0; | |
144 | unsigned int first_cpu, last_cpu; | |
145 | cpu_set_t cpu_set; | |
146 | char buf[128]; | |
147 | ||
148 | /* Create real file to mmap. */ | |
149 | fd = open("/tmp/vringh_test-file", O_RDWR|O_CREAT|O_TRUNC, 0600); | |
150 | if (fd < 0) | |
151 | err(1, "Opening /tmp/vringh_test-file"); | |
152 | ||
153 | /* Extra room at the end for some data, and indirects */ | |
154 | mapsize = vring_size(RINGSIZE, ALIGN) | |
155 | + RINGSIZE * 2 * sizeof(int) | |
156 | + RINGSIZE * 6 * sizeof(struct vring_desc); | |
157 | mapsize = (mapsize + getpagesize() - 1) & ~(getpagesize() - 1); | |
158 | ftruncate(fd, mapsize); | |
159 | ||
160 | /* Parent and child use separate addresses, to check our mapping logic! */ | |
161 | host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); | |
162 | guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); | |
163 | ||
164 | pipe(to_guest); | |
165 | pipe(to_host); | |
166 | ||
167 | CPU_ZERO(&cpu_set); | |
168 | find_cpus(&first_cpu, &last_cpu); | |
169 | printf("Using CPUS %u and %u\n", first_cpu, last_cpu); | |
170 | fflush(stdout); | |
171 | ||
172 | if (fork() != 0) { | |
173 | struct vringh vrh; | |
174 | int status, err, rlen = 0; | |
175 | char rbuf[5]; | |
176 | ||
177 | /* We are the host: never access guest addresses! */ | |
178 | munmap(guest_map, mapsize); | |
179 | ||
180 | __user_addr_min = host_map; | |
181 | __user_addr_max = __user_addr_min + mapsize; | |
182 | user_addr_offset = host_map - guest_map; | |
183 | assert(user_addr_offset); | |
184 | ||
185 | close(to_guest[0]); | |
186 | close(to_host[1]); | |
187 | ||
188 | vring_init(&vrh.vring, RINGSIZE, host_map, ALIGN); | |
189 | vringh_init_user(&vrh, features, RINGSIZE, true, | |
190 | vrh.vring.desc, vrh.vring.avail, vrh.vring.used); | |
191 | CPU_SET(first_cpu, &cpu_set); | |
192 | if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set)) | |
193 | errx(1, "Could not set affinity to cpu %u", first_cpu); | |
194 | ||
195 | while (xfers < NUM_XFERS) { | |
196 | struct iovec host_riov[2], host_wiov[2]; | |
197 | struct vringh_iov riov, wiov; | |
198 | u16 head, written; | |
199 | ||
200 | if (fast_vringh) { | |
201 | for (;;) { | |
202 | err = vringh_get_head(&vrh, &head); | |
203 | if (err != 0) | |
204 | break; | |
205 | err = vringh_need_notify_user(&vrh); | |
206 | if (err < 0) | |
207 | errx(1, "vringh_need_notify_user: %i", | |
208 | err); | |
209 | if (err) { | |
210 | write(to_guest[1], "", 1); | |
211 | notifies++; | |
212 | } | |
213 | } | |
214 | if (err != 1) | |
215 | errx(1, "vringh_get_head"); | |
216 | written = 0; | |
217 | goto complete; | |
218 | } else { | |
219 | vringh_iov_init(&riov, | |
220 | host_riov, | |
221 | ARRAY_SIZE(host_riov)); | |
222 | vringh_iov_init(&wiov, | |
223 | host_wiov, | |
224 | ARRAY_SIZE(host_wiov)); | |
225 | ||
226 | err = vringh_getdesc_user(&vrh, &riov, &wiov, | |
227 | getrange, &head); | |
228 | } | |
229 | if (err == 0) { | |
230 | err = vringh_need_notify_user(&vrh); | |
231 | if (err < 0) | |
232 | errx(1, "vringh_need_notify_user: %i", | |
233 | err); | |
234 | if (err) { | |
235 | write(to_guest[1], "", 1); | |
236 | notifies++; | |
237 | } | |
238 | ||
239 | if (!vringh_notify_enable_user(&vrh)) | |
240 | continue; | |
241 | ||
242 | /* Swallow all notifies at once. */ | |
243 | if (read(to_host[0], buf, sizeof(buf)) < 1) | |
244 | break; | |
245 | ||
246 | vringh_notify_disable_user(&vrh); | |
247 | receives++; | |
248 | continue; | |
249 | } | |
250 | if (err != 1) | |
251 | errx(1, "vringh_getdesc_user: %i", err); | |
252 | ||
253 | /* We simply copy bytes. */ | |
254 | if (riov.used) { | |
255 | rlen = vringh_iov_pull_user(&riov, rbuf, | |
256 | sizeof(rbuf)); | |
257 | if (rlen != 4) | |
258 | errx(1, "vringh_iov_pull_user: %i", | |
259 | rlen); | |
260 | assert(riov.i == riov.used); | |
261 | written = 0; | |
262 | } else { | |
263 | err = vringh_iov_push_user(&wiov, rbuf, rlen); | |
264 | if (err != rlen) | |
265 | errx(1, "vringh_iov_push_user: %i", | |
266 | err); | |
267 | assert(wiov.i == wiov.used); | |
268 | written = err; | |
269 | } | |
270 | complete: | |
271 | xfers++; | |
272 | ||
273 | err = vringh_complete_user(&vrh, head, written); | |
274 | if (err != 0) | |
275 | errx(1, "vringh_complete_user: %i", err); | |
276 | } | |
277 | ||
278 | err = vringh_need_notify_user(&vrh); | |
279 | if (err < 0) | |
280 | errx(1, "vringh_need_notify_user: %i", err); | |
281 | if (err) { | |
282 | write(to_guest[1], "", 1); | |
283 | notifies++; | |
284 | } | |
285 | wait(&status); | |
286 | if (!WIFEXITED(status)) | |
287 | errx(1, "Child died with signal %i?", WTERMSIG(status)); | |
288 | if (WEXITSTATUS(status) != 0) | |
289 | errx(1, "Child exited %i?", WEXITSTATUS(status)); | |
290 | printf("Host: notified %lu, pinged %lu\n", notifies, receives); | |
291 | return 0; | |
292 | } else { | |
293 | struct guest_virtio_device gvdev; | |
294 | struct virtqueue *vq; | |
295 | unsigned int *data; | |
296 | struct vring_desc *indirects; | |
297 | unsigned int finished = 0; | |
298 | ||
299 | /* We pass sg[]s pointing into here, but we need RINGSIZE+1 */ | |
300 | data = guest_map + vring_size(RINGSIZE, ALIGN); | |
301 | indirects = (void *)data + (RINGSIZE + 1) * 2 * sizeof(int); | |
302 | ||
303 | /* We are the guest. */ | |
304 | munmap(host_map, mapsize); | |
305 | ||
306 | close(to_guest[1]); | |
307 | close(to_host[0]); | |
308 | ||
e16e12be | 309 | gvdev.vdev.features = features; |
1515c5ce RR |
310 | gvdev.to_host_fd = to_host[1]; |
311 | gvdev.notifies = 0; | |
312 | ||
313 | CPU_SET(first_cpu, &cpu_set); | |
314 | if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set)) | |
315 | err(1, "Could not set affinity to cpu %u", first_cpu); | |
316 | ||
317 | vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &gvdev.vdev, true, | |
0a12ae40 SN |
318 | false, guest_map, |
319 | fast_vringh ? no_notify_host | |
1515c5ce RR |
320 | : parallel_notify_host, |
321 | never_callback_guest, "guest vq"); | |
322 | ||
323 | /* Don't kfree indirects. */ | |
324 | __kfree_ignore_start = indirects; | |
325 | __kfree_ignore_end = indirects + RINGSIZE * 6; | |
326 | ||
327 | while (xfers < NUM_XFERS) { | |
328 | struct scatterlist sg[4]; | |
329 | unsigned int num_sg, len; | |
330 | int *dbuf, err; | |
331 | bool output = !(xfers % 2); | |
332 | ||
333 | /* Consume bufs. */ | |
334 | while ((dbuf = virtqueue_get_buf(vq, &len)) != NULL) { | |
335 | if (len == 4) | |
336 | assert(*dbuf == finished - 1); | |
337 | else if (!fast_vringh) | |
338 | assert(*dbuf == finished); | |
339 | finished++; | |
340 | } | |
341 | ||
342 | /* Produce a buffer. */ | |
343 | dbuf = data + (xfers % (RINGSIZE + 1)); | |
344 | ||
345 | if (output) | |
346 | *dbuf = xfers; | |
347 | else | |
348 | *dbuf = -1; | |
349 | ||
350 | switch ((xfers / sizeof(*dbuf)) % 4) { | |
351 | case 0: | |
352 | /* Nasty three-element sg list. */ | |
353 | sg_init_table(sg, num_sg = 3); | |
354 | sg_set_buf(&sg[0], (void *)dbuf, 1); | |
355 | sg_set_buf(&sg[1], (void *)dbuf + 1, 2); | |
356 | sg_set_buf(&sg[2], (void *)dbuf + 3, 1); | |
357 | break; | |
358 | case 1: | |
359 | sg_init_table(sg, num_sg = 2); | |
360 | sg_set_buf(&sg[0], (void *)dbuf, 1); | |
361 | sg_set_buf(&sg[1], (void *)dbuf + 1, 3); | |
362 | break; | |
363 | case 2: | |
364 | sg_init_table(sg, num_sg = 1); | |
365 | sg_set_buf(&sg[0], (void *)dbuf, 4); | |
366 | break; | |
367 | case 3: | |
368 | sg_init_table(sg, num_sg = 4); | |
369 | sg_set_buf(&sg[0], (void *)dbuf, 1); | |
370 | sg_set_buf(&sg[1], (void *)dbuf + 1, 1); | |
371 | sg_set_buf(&sg[2], (void *)dbuf + 2, 1); | |
372 | sg_set_buf(&sg[3], (void *)dbuf + 3, 1); | |
373 | break; | |
374 | } | |
375 | ||
376 | /* May allocate an indirect, so force it to allocate | |
377 | * user addr */ | |
378 | __kmalloc_fake = indirects + (xfers % RINGSIZE) * 4; | |
379 | if (output) | |
e538ebaf RR |
380 | err = virtqueue_add_outbuf(vq, sg, num_sg, dbuf, |
381 | GFP_KERNEL); | |
1515c5ce | 382 | else |
e538ebaf RR |
383 | err = virtqueue_add_inbuf(vq, sg, num_sg, |
384 | dbuf, GFP_KERNEL); | |
1515c5ce RR |
385 | |
386 | if (err == -ENOSPC) { | |
387 | if (!virtqueue_enable_cb_delayed(vq)) | |
388 | continue; | |
389 | /* Swallow all notifies at once. */ | |
390 | if (read(to_guest[0], buf, sizeof(buf)) < 1) | |
391 | break; | |
392 | ||
393 | receives++; | |
394 | virtqueue_disable_cb(vq); | |
395 | continue; | |
396 | } | |
397 | ||
398 | if (err) | |
cf994e0a | 399 | errx(1, "virtqueue_add_in/outbuf: %i", err); |
1515c5ce RR |
400 | |
401 | xfers++; | |
402 | virtqueue_kick(vq); | |
403 | } | |
404 | ||
405 | /* Any extra? */ | |
406 | while (finished != xfers) { | |
407 | int *dbuf; | |
408 | unsigned int len; | |
409 | ||
410 | /* Consume bufs. */ | |
411 | dbuf = virtqueue_get_buf(vq, &len); | |
412 | if (dbuf) { | |
413 | if (len == 4) | |
414 | assert(*dbuf == finished - 1); | |
415 | else | |
416 | assert(len == 0); | |
417 | finished++; | |
418 | continue; | |
419 | } | |
420 | ||
421 | if (!virtqueue_enable_cb_delayed(vq)) | |
422 | continue; | |
423 | if (read(to_guest[0], buf, sizeof(buf)) < 1) | |
424 | break; | |
425 | ||
426 | receives++; | |
427 | virtqueue_disable_cb(vq); | |
428 | } | |
429 | ||
430 | printf("Guest: notified %lu, pinged %lu\n", | |
431 | gvdev.notifies, receives); | |
432 | vring_del_virtqueue(vq); | |
433 | return 0; | |
434 | } | |
435 | } | |
436 | ||
437 | int main(int argc, char *argv[]) | |
438 | { | |
439 | struct virtio_device vdev; | |
440 | struct virtqueue *vq; | |
441 | struct vringh vrh; | |
cf994e0a | 442 | struct scatterlist guest_sg[RINGSIZE], *sgs[2]; |
1515c5ce RR |
443 | struct iovec host_riov[2], host_wiov[2]; |
444 | struct vringh_iov riov, wiov; | |
445 | struct vring_used_elem used[RINGSIZE]; | |
446 | char buf[28]; | |
447 | u16 head; | |
448 | int err; | |
449 | unsigned i; | |
450 | void *ret; | |
451 | bool (*getrange)(struct vringh *vrh, u64 addr, struct vringh_range *r); | |
452 | bool fast_vringh = false, parallel = false; | |
453 | ||
454 | getrange = getrange_iov; | |
e16e12be | 455 | vdev.features = 0; |
1515c5ce RR |
456 | |
457 | while (argv[1]) { | |
458 | if (strcmp(argv[1], "--indirect") == 0) | |
e16e12be | 459 | __virtio_set_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC); |
1515c5ce | 460 | else if (strcmp(argv[1], "--eventidx") == 0) |
e16e12be | 461 | __virtio_set_bit(&vdev, VIRTIO_RING_F_EVENT_IDX); |
8dc32b29 MT |
462 | else if (strcmp(argv[1], "--virtio-1") == 0) |
463 | __virtio_set_bit(&vdev, VIRTIO_F_VERSION_1); | |
1515c5ce RR |
464 | else if (strcmp(argv[1], "--slow-range") == 0) |
465 | getrange = getrange_slow; | |
466 | else if (strcmp(argv[1], "--fast-vringh") == 0) | |
467 | fast_vringh = true; | |
468 | else if (strcmp(argv[1], "--parallel") == 0) | |
469 | parallel = true; | |
470 | else | |
471 | errx(1, "Unknown arg %s", argv[1]); | |
472 | argv++; | |
473 | } | |
474 | ||
475 | if (parallel) | |
e16e12be | 476 | return parallel_test(vdev.features, getrange, fast_vringh); |
1515c5ce RR |
477 | |
478 | if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0) | |
479 | abort(); | |
480 | __user_addr_max = __user_addr_min + USER_MEM; | |
481 | memset(__user_addr_min, 0, vring_size(RINGSIZE, ALIGN)); | |
482 | ||
483 | /* Set up guest side. */ | |
0a12ae40 | 484 | vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true, false, |
1515c5ce RR |
485 | __user_addr_min, |
486 | never_notify_host, never_callback_guest, | |
487 | "guest vq"); | |
488 | ||
489 | /* Set up host side. */ | |
490 | vring_init(&vrh.vring, RINGSIZE, __user_addr_min, ALIGN); | |
e16e12be | 491 | vringh_init_user(&vrh, vdev.features, RINGSIZE, true, |
1515c5ce RR |
492 | vrh.vring.desc, vrh.vring.avail, vrh.vring.used); |
493 | ||
494 | /* No descriptor to get yet... */ | |
495 | err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head); | |
496 | if (err != 0) | |
497 | errx(1, "vringh_getdesc_user: %i", err); | |
498 | ||
499 | /* Guest puts in a descriptor. */ | |
500 | memcpy(__user_addr_max - 1, "a", 1); | |
501 | sg_init_table(guest_sg, 1); | |
502 | sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1); | |
503 | sg_init_table(guest_sg+1, 1); | |
504 | sg_set_buf(&guest_sg[1], __user_addr_max - 3, 2); | |
cf994e0a RR |
505 | sgs[0] = &guest_sg[0]; |
506 | sgs[1] = &guest_sg[1]; | |
1515c5ce RR |
507 | |
508 | /* May allocate an indirect, so force it to allocate user addr */ | |
509 | __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN); | |
cf994e0a | 510 | err = virtqueue_add_sgs(vq, sgs, 1, 1, &err, GFP_KERNEL); |
1515c5ce | 511 | if (err) |
cf994e0a | 512 | errx(1, "virtqueue_add_sgs: %i", err); |
1515c5ce RR |
513 | __kmalloc_fake = NULL; |
514 | ||
515 | /* Host retreives it. */ | |
516 | vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov)); | |
517 | vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov)); | |
518 | ||
519 | err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head); | |
520 | if (err != 1) | |
521 | errx(1, "vringh_getdesc_user: %i", err); | |
522 | ||
523 | assert(riov.used == 1); | |
524 | assert(riov.iov[0].iov_base == __user_addr_max - 1); | |
525 | assert(riov.iov[0].iov_len == 1); | |
526 | if (getrange != getrange_slow) { | |
527 | assert(wiov.used == 1); | |
528 | assert(wiov.iov[0].iov_base == __user_addr_max - 3); | |
529 | assert(wiov.iov[0].iov_len == 2); | |
530 | } else { | |
531 | assert(wiov.used == 2); | |
532 | assert(wiov.iov[0].iov_base == __user_addr_max - 3); | |
533 | assert(wiov.iov[0].iov_len == 1); | |
534 | assert(wiov.iov[1].iov_base == __user_addr_max - 2); | |
535 | assert(wiov.iov[1].iov_len == 1); | |
536 | } | |
537 | ||
538 | err = vringh_iov_pull_user(&riov, buf, 5); | |
539 | if (err != 1) | |
540 | errx(1, "vringh_iov_pull_user: %i", err); | |
541 | assert(buf[0] == 'a'); | |
542 | assert(riov.i == 1); | |
543 | assert(vringh_iov_pull_user(&riov, buf, 5) == 0); | |
544 | ||
545 | memcpy(buf, "bcdef", 5); | |
546 | err = vringh_iov_push_user(&wiov, buf, 5); | |
547 | if (err != 2) | |
548 | errx(1, "vringh_iov_push_user: %i", err); | |
549 | assert(memcmp(__user_addr_max - 3, "bc", 2) == 0); | |
550 | assert(wiov.i == wiov.used); | |
551 | assert(vringh_iov_push_user(&wiov, buf, 5) == 0); | |
552 | ||
553 | /* Host is done. */ | |
554 | err = vringh_complete_user(&vrh, head, err); | |
555 | if (err != 0) | |
556 | errx(1, "vringh_complete_user: %i", err); | |
557 | ||
558 | /* Guest should see used token now. */ | |
559 | __kfree_ignore_start = __user_addr_min + vring_size(RINGSIZE, ALIGN); | |
560 | __kfree_ignore_end = __kfree_ignore_start + 1; | |
561 | ret = virtqueue_get_buf(vq, &i); | |
562 | if (ret != &err) | |
563 | errx(1, "virtqueue_get_buf: %p", ret); | |
564 | assert(i == 2); | |
565 | ||
566 | /* Guest puts in a huge descriptor. */ | |
567 | sg_init_table(guest_sg, RINGSIZE); | |
568 | for (i = 0; i < RINGSIZE; i++) { | |
569 | sg_set_buf(&guest_sg[i], | |
570 | __user_addr_max - USER_MEM/4, USER_MEM/4); | |
571 | } | |
572 | ||
573 | /* Fill contents with recognisable garbage. */ | |
574 | for (i = 0; i < USER_MEM/4; i++) | |
575 | ((char *)__user_addr_max - USER_MEM/4)[i] = i; | |
576 | ||
577 | /* This will allocate an indirect, so force it to allocate user addr */ | |
578 | __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN); | |
cf994e0a | 579 | err = virtqueue_add_outbuf(vq, guest_sg, RINGSIZE, &err, GFP_KERNEL); |
1515c5ce | 580 | if (err) |
cf994e0a | 581 | errx(1, "virtqueue_add_outbuf (large): %i", err); |
1515c5ce RR |
582 | __kmalloc_fake = NULL; |
583 | ||
584 | /* Host picks it up (allocates new iov). */ | |
585 | vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov)); | |
586 | vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov)); | |
587 | ||
588 | err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head); | |
589 | if (err != 1) | |
590 | errx(1, "vringh_getdesc_user: %i", err); | |
591 | ||
592 | assert(riov.max_num & VRINGH_IOV_ALLOCATED); | |
593 | assert(riov.iov != host_riov); | |
594 | if (getrange != getrange_slow) | |
595 | assert(riov.used == RINGSIZE); | |
596 | else | |
597 | assert(riov.used == RINGSIZE * USER_MEM/4); | |
598 | ||
599 | assert(!(wiov.max_num & VRINGH_IOV_ALLOCATED)); | |
600 | assert(wiov.used == 0); | |
601 | ||
602 | /* Pull data back out (in odd chunks), should be as expected. */ | |
603 | for (i = 0; i < RINGSIZE * USER_MEM/4; i += 3) { | |
604 | err = vringh_iov_pull_user(&riov, buf, 3); | |
605 | if (err != 3 && i + err != RINGSIZE * USER_MEM/4) | |
606 | errx(1, "vringh_iov_pull_user large: %i", err); | |
607 | assert(buf[0] == (char)i); | |
608 | assert(err < 2 || buf[1] == (char)(i + 1)); | |
609 | assert(err < 3 || buf[2] == (char)(i + 2)); | |
610 | } | |
611 | assert(riov.i == riov.used); | |
612 | vringh_iov_cleanup(&riov); | |
613 | vringh_iov_cleanup(&wiov); | |
614 | ||
615 | /* Complete using multi interface, just because we can. */ | |
616 | used[0].id = head; | |
617 | used[0].len = 0; | |
618 | err = vringh_complete_multi_user(&vrh, used, 1); | |
619 | if (err) | |
620 | errx(1, "vringh_complete_multi_user(1): %i", err); | |
621 | ||
622 | /* Free up those descriptors. */ | |
623 | ret = virtqueue_get_buf(vq, &i); | |
624 | if (ret != &err) | |
625 | errx(1, "virtqueue_get_buf: %p", ret); | |
626 | ||
627 | /* Add lots of descriptors. */ | |
628 | sg_init_table(guest_sg, 1); | |
629 | sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1); | |
630 | for (i = 0; i < RINGSIZE; i++) { | |
cf994e0a | 631 | err = virtqueue_add_outbuf(vq, guest_sg, 1, &err, GFP_KERNEL); |
1515c5ce | 632 | if (err) |
cf994e0a | 633 | errx(1, "virtqueue_add_outbuf (multiple): %i", err); |
1515c5ce RR |
634 | } |
635 | ||
636 | /* Now get many, and consume them all at once. */ | |
637 | vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov)); | |
638 | vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov)); | |
639 | ||
640 | for (i = 0; i < RINGSIZE; i++) { | |
641 | err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head); | |
642 | if (err != 1) | |
643 | errx(1, "vringh_getdesc_user: %i", err); | |
644 | used[i].id = head; | |
645 | used[i].len = 0; | |
646 | } | |
647 | /* Make sure it wraps around ring, to test! */ | |
648 | assert(vrh.vring.used->idx % RINGSIZE != 0); | |
649 | err = vringh_complete_multi_user(&vrh, used, RINGSIZE); | |
650 | if (err) | |
651 | errx(1, "vringh_complete_multi_user: %i", err); | |
652 | ||
653 | /* Free those buffers. */ | |
654 | for (i = 0; i < RINGSIZE; i++) { | |
655 | unsigned len; | |
656 | assert(virtqueue_get_buf(vq, &len) != NULL); | |
657 | } | |
658 | ||
659 | /* Test weird (but legal!) indirect. */ | |
e16e12be | 660 | if (__virtio_test_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC)) { |
1515c5ce RR |
661 | char *data = __user_addr_max - USER_MEM/4; |
662 | struct vring_desc *d = __user_addr_max - USER_MEM/2; | |
663 | struct vring vring; | |
664 | ||
665 | /* Force creation of direct, which we modify. */ | |
e16e12be | 666 | __virtio_clear_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC); |
1515c5ce | 667 | vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true, |
0a12ae40 | 668 | false, __user_addr_min, |
1515c5ce RR |
669 | never_notify_host, |
670 | never_callback_guest, | |
671 | "guest vq"); | |
672 | ||
673 | sg_init_table(guest_sg, 4); | |
674 | sg_set_buf(&guest_sg[0], d, sizeof(*d)*2); | |
675 | sg_set_buf(&guest_sg[1], d + 2, sizeof(*d)*1); | |
676 | sg_set_buf(&guest_sg[2], data + 6, 4); | |
677 | sg_set_buf(&guest_sg[3], d + 3, sizeof(*d)*3); | |
678 | ||
cf994e0a | 679 | err = virtqueue_add_outbuf(vq, guest_sg, 4, &err, GFP_KERNEL); |
1515c5ce | 680 | if (err) |
cf994e0a | 681 | errx(1, "virtqueue_add_outbuf (indirect): %i", err); |
1515c5ce RR |
682 | |
683 | vring_init(&vring, RINGSIZE, __user_addr_min, ALIGN); | |
684 | ||
685 | /* They're used in order, but double-check... */ | |
686 | assert(vring.desc[0].addr == (unsigned long)d); | |
687 | assert(vring.desc[1].addr == (unsigned long)(d+2)); | |
688 | assert(vring.desc[2].addr == (unsigned long)data + 6); | |
689 | assert(vring.desc[3].addr == (unsigned long)(d+3)); | |
690 | vring.desc[0].flags |= VRING_DESC_F_INDIRECT; | |
691 | vring.desc[1].flags |= VRING_DESC_F_INDIRECT; | |
692 | vring.desc[3].flags |= VRING_DESC_F_INDIRECT; | |
693 | ||
694 | /* First indirect */ | |
695 | d[0].addr = (unsigned long)data; | |
696 | d[0].len = 1; | |
697 | d[0].flags = VRING_DESC_F_NEXT; | |
698 | d[0].next = 1; | |
699 | d[1].addr = (unsigned long)data + 1; | |
700 | d[1].len = 2; | |
701 | d[1].flags = 0; | |
702 | ||
703 | /* Second indirect */ | |
704 | d[2].addr = (unsigned long)data + 3; | |
705 | d[2].len = 3; | |
706 | d[2].flags = 0; | |
707 | ||
708 | /* Third indirect */ | |
709 | d[3].addr = (unsigned long)data + 10; | |
710 | d[3].len = 5; | |
711 | d[3].flags = VRING_DESC_F_NEXT; | |
712 | d[3].next = 1; | |
713 | d[4].addr = (unsigned long)data + 15; | |
714 | d[4].len = 6; | |
715 | d[4].flags = VRING_DESC_F_NEXT; | |
716 | d[4].next = 2; | |
717 | d[5].addr = (unsigned long)data + 21; | |
718 | d[5].len = 7; | |
719 | d[5].flags = 0; | |
720 | ||
721 | /* Host picks it up (allocates new iov). */ | |
722 | vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov)); | |
723 | vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov)); | |
724 | ||
725 | err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head); | |
726 | if (err != 1) | |
727 | errx(1, "vringh_getdesc_user: %i", err); | |
728 | ||
729 | if (head != 0) | |
730 | errx(1, "vringh_getdesc_user: head %i not 0", head); | |
731 | ||
732 | assert(riov.max_num & VRINGH_IOV_ALLOCATED); | |
733 | if (getrange != getrange_slow) | |
734 | assert(riov.used == 7); | |
735 | else | |
736 | assert(riov.used == 28); | |
737 | err = vringh_iov_pull_user(&riov, buf, 29); | |
738 | assert(err == 28); | |
739 | ||
740 | /* Data should be linear. */ | |
741 | for (i = 0; i < err; i++) | |
742 | assert(buf[i] == i); | |
743 | vringh_iov_cleanup(&riov); | |
744 | } | |
745 | ||
746 | /* Don't leak memory... */ | |
747 | vring_del_virtqueue(vq); | |
748 | free(__user_addr_min); | |
749 | ||
750 | return 0; | |
751 | } |