1 /* Simple test of virtio code, entirely in userpsace. */
5 #include <linux/kernel.h>
7 #include <linux/virtio.h>
8 #include <linux/vringh.h>
9 #include <linux/virtio_ring.h>
10 #include <linux/uaccess.h>
11 #include <sys/types.h>
17 #define USER_MEM (1024*1024)
18 void *__user_addr_min
, *__user_addr_max
;
19 void *__kmalloc_fake
, *__kfree_ignore_start
, *__kfree_ignore_end
;
20 static u64 user_addr_offset
;
25 static void never_notify_host(struct virtqueue
*vq
)
30 static void never_callback_guest(struct virtqueue
*vq
)
35 static bool getrange_iov(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
)
37 if (addr
< (u64
)(unsigned long)__user_addr_min
- user_addr_offset
)
39 if (addr
>= (u64
)(unsigned long)__user_addr_max
- user_addr_offset
)
42 r
->start
= (u64
)(unsigned long)__user_addr_min
- user_addr_offset
;
43 r
->end_incl
= (u64
)(unsigned long)__user_addr_max
- 1 - user_addr_offset
;
44 r
->offset
= user_addr_offset
;
48 /* We return single byte ranges. */
49 static bool getrange_slow(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
)
51 if (addr
< (u64
)(unsigned long)__user_addr_min
- user_addr_offset
)
53 if (addr
>= (u64
)(unsigned long)__user_addr_max
- user_addr_offset
)
57 r
->end_incl
= r
->start
;
58 r
->offset
= user_addr_offset
;
62 struct guest_virtio_device
{
63 struct virtio_device vdev
;
65 unsigned long notifies
;
68 static void parallel_notify_host(struct virtqueue
*vq
)
70 struct guest_virtio_device
*gvdev
;
72 gvdev
= container_of(vq
->vdev
, struct guest_virtio_device
, vdev
);
73 write(gvdev
->to_host_fd
, "", 1);
77 static void no_notify_host(struct virtqueue
*vq
)
81 #define NUM_XFERS (10000000)
83 /* We aim for two "distant" cpus. */
84 static void find_cpus(unsigned int *first
, unsigned int *last
)
90 for (i
= 0; i
< 4096; i
++) {
94 if (sched_setaffinity(getpid(), sizeof(set
), &set
) == 0) {
103 /* Opencoded version for fast mode */
104 static inline int vringh_get_head(struct vringh
*vrh
, u16
*head
)
109 err
= get_user(avail_idx
, &vrh
->vring
.avail
->idx
);
113 if (vrh
->last_avail_idx
== avail_idx
)
116 /* Only get avail ring entries after they have been exposed by guest. */
117 virtio_rmb(vrh
->weak_barriers
);
119 i
= vrh
->last_avail_idx
& (vrh
->vring
.num
- 1);
121 err
= get_user(*head
, &vrh
->vring
.avail
->ring
[i
]);
125 vrh
->last_avail_idx
++;
129 static int parallel_test(unsigned long features
,
130 bool (*getrange
)(struct vringh
*vrh
,
131 u64 addr
, struct vringh_range
*r
),
134 void *host_map
, *guest_map
;
135 int fd
, mapsize
, to_guest
[2], to_host
[2];
136 unsigned long xfers
= 0, notifies
= 0, receives
= 0;
137 unsigned int first_cpu
, last_cpu
;
141 /* Create real file to mmap. */
142 fd
= open("/tmp/vringh_test-file", O_RDWR
|O_CREAT
|O_TRUNC
, 0600);
144 err(1, "Opening /tmp/vringh_test-file");
146 /* Extra room at the end for some data, and indirects */
147 mapsize
= vring_size(RINGSIZE
, ALIGN
)
148 + RINGSIZE
* 2 * sizeof(int)
149 + RINGSIZE
* 6 * sizeof(struct vring_desc
);
150 mapsize
= (mapsize
+ getpagesize() - 1) & ~(getpagesize() - 1);
151 ftruncate(fd
, mapsize
);
153 /* Parent and child use separate addresses, to check our mapping logic! */
154 host_map
= mmap(NULL
, mapsize
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
155 guest_map
= mmap(NULL
, mapsize
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
161 find_cpus(&first_cpu
, &last_cpu
);
162 printf("Using CPUS %u and %u\n", first_cpu
, last_cpu
);
167 int status
, err
, rlen
= 0;
170 /* We are the host: never access guest addresses! */
171 munmap(guest_map
, mapsize
);
173 __user_addr_min
= host_map
;
174 __user_addr_max
= __user_addr_min
+ mapsize
;
175 user_addr_offset
= host_map
- guest_map
;
176 assert(user_addr_offset
);
181 vring_init(&vrh
.vring
, RINGSIZE
, host_map
, ALIGN
);
182 vringh_init_user(&vrh
, features
, RINGSIZE
, true,
183 vrh
.vring
.desc
, vrh
.vring
.avail
, vrh
.vring
.used
);
184 CPU_SET(first_cpu
, &cpu_set
);
185 if (sched_setaffinity(getpid(), sizeof(cpu_set
), &cpu_set
))
186 errx(1, "Could not set affinity to cpu %u", first_cpu
);
188 while (xfers
< NUM_XFERS
) {
189 struct iovec host_riov
[2], host_wiov
[2];
190 struct vringh_iov riov
, wiov
;
195 err
= vringh_get_head(&vrh
, &head
);
198 err
= vringh_need_notify_user(&vrh
);
200 errx(1, "vringh_need_notify_user: %i",
203 write(to_guest
[1], "", 1);
208 errx(1, "vringh_get_head");
212 vringh_iov_init(&riov
,
214 ARRAY_SIZE(host_riov
));
215 vringh_iov_init(&wiov
,
217 ARRAY_SIZE(host_wiov
));
219 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
,
223 err
= vringh_need_notify_user(&vrh
);
225 errx(1, "vringh_need_notify_user: %i",
228 write(to_guest
[1], "", 1);
232 if (!vringh_notify_enable_user(&vrh
))
235 /* Swallow all notifies at once. */
236 if (read(to_host
[0], buf
, sizeof(buf
)) < 1)
239 vringh_notify_disable_user(&vrh
);
244 errx(1, "vringh_getdesc_user: %i", err
);
246 /* We simply copy bytes. */
248 rlen
= vringh_iov_pull_user(&riov
, rbuf
,
251 errx(1, "vringh_iov_pull_user: %i",
253 assert(riov
.i
== riov
.used
);
256 err
= vringh_iov_push_user(&wiov
, rbuf
, rlen
);
258 errx(1, "vringh_iov_push_user: %i",
260 assert(wiov
.i
== wiov
.used
);
266 err
= vringh_complete_user(&vrh
, head
, written
);
268 errx(1, "vringh_complete_user: %i", err
);
271 err
= vringh_need_notify_user(&vrh
);
273 errx(1, "vringh_need_notify_user: %i", err
);
275 write(to_guest
[1], "", 1);
279 if (!WIFEXITED(status
))
280 errx(1, "Child died with signal %i?", WTERMSIG(status
));
281 if (WEXITSTATUS(status
) != 0)
282 errx(1, "Child exited %i?", WEXITSTATUS(status
));
283 printf("Host: notified %lu, pinged %lu\n", notifies
, receives
);
286 struct guest_virtio_device gvdev
;
287 struct virtqueue
*vq
;
289 struct vring_desc
*indirects
;
290 unsigned int finished
= 0;
292 /* We pass sg[]s pointing into here, but we need RINGSIZE+1 */
293 data
= guest_map
+ vring_size(RINGSIZE
, ALIGN
);
294 indirects
= (void *)data
+ (RINGSIZE
+ 1) * 2 * sizeof(int);
296 /* We are the guest. */
297 munmap(host_map
, mapsize
);
302 gvdev
.vdev
.features
[0] = features
;
303 gvdev
.to_host_fd
= to_host
[1];
306 CPU_SET(first_cpu
, &cpu_set
);
307 if (sched_setaffinity(getpid(), sizeof(cpu_set
), &cpu_set
))
308 err(1, "Could not set affinity to cpu %u", first_cpu
);
310 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &gvdev
.vdev
, true,
311 guest_map
, fast_vringh
? no_notify_host
312 : parallel_notify_host
,
313 never_callback_guest
, "guest vq");
315 /* Don't kfree indirects. */
316 __kfree_ignore_start
= indirects
;
317 __kfree_ignore_end
= indirects
+ RINGSIZE
* 6;
319 while (xfers
< NUM_XFERS
) {
320 struct scatterlist sg
[4];
321 unsigned int num_sg
, len
;
323 bool output
= !(xfers
% 2);
326 while ((dbuf
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
328 assert(*dbuf
== finished
- 1);
329 else if (!fast_vringh
)
330 assert(*dbuf
== finished
);
334 /* Produce a buffer. */
335 dbuf
= data
+ (xfers
% (RINGSIZE
+ 1));
342 switch ((xfers
/ sizeof(*dbuf
)) % 4) {
344 /* Nasty three-element sg list. */
345 sg_init_table(sg
, num_sg
= 3);
346 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
347 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 2);
348 sg_set_buf(&sg
[2], (void *)dbuf
+ 3, 1);
351 sg_init_table(sg
, num_sg
= 2);
352 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
353 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 3);
356 sg_init_table(sg
, num_sg
= 1);
357 sg_set_buf(&sg
[0], (void *)dbuf
, 4);
360 sg_init_table(sg
, num_sg
= 4);
361 sg_set_buf(&sg
[0], (void *)dbuf
, 1);
362 sg_set_buf(&sg
[1], (void *)dbuf
+ 1, 1);
363 sg_set_buf(&sg
[2], (void *)dbuf
+ 2, 1);
364 sg_set_buf(&sg
[3], (void *)dbuf
+ 3, 1);
368 /* May allocate an indirect, so force it to allocate
370 __kmalloc_fake
= indirects
+ (xfers
% RINGSIZE
) * 4;
372 err
= virtqueue_add_outbuf(vq
, sg
, num_sg
, dbuf
,
375 err
= virtqueue_add_inbuf(vq
, sg
, num_sg
,
378 if (err
== -ENOSPC
) {
379 if (!virtqueue_enable_cb_delayed(vq
))
381 /* Swallow all notifies at once. */
382 if (read(to_guest
[0], buf
, sizeof(buf
)) < 1)
386 virtqueue_disable_cb(vq
);
391 errx(1, "virtqueue_add_in/outbuf: %i", err
);
398 while (finished
!= xfers
) {
403 dbuf
= virtqueue_get_buf(vq
, &len
);
406 assert(*dbuf
== finished
- 1);
413 if (!virtqueue_enable_cb_delayed(vq
))
415 if (read(to_guest
[0], buf
, sizeof(buf
)) < 1)
419 virtqueue_disable_cb(vq
);
422 printf("Guest: notified %lu, pinged %lu\n",
423 gvdev
.notifies
, receives
);
424 vring_del_virtqueue(vq
);
429 int main(int argc
, char *argv
[])
431 struct virtio_device vdev
;
432 struct virtqueue
*vq
;
434 struct scatterlist guest_sg
[RINGSIZE
], *sgs
[2];
435 struct iovec host_riov
[2], host_wiov
[2];
436 struct vringh_iov riov
, wiov
;
437 struct vring_used_elem used
[RINGSIZE
];
443 bool (*getrange
)(struct vringh
*vrh
, u64 addr
, struct vringh_range
*r
);
444 bool fast_vringh
= false, parallel
= false;
446 getrange
= getrange_iov
;
447 vdev
.features
[0] = 0;
450 if (strcmp(argv
[1], "--indirect") == 0)
451 vdev
.features
[0] |= (1 << VIRTIO_RING_F_INDIRECT_DESC
);
452 else if (strcmp(argv
[1], "--eventidx") == 0)
453 vdev
.features
[0] |= (1 << VIRTIO_RING_F_EVENT_IDX
);
454 else if (strcmp(argv
[1], "--slow-range") == 0)
455 getrange
= getrange_slow
;
456 else if (strcmp(argv
[1], "--fast-vringh") == 0)
458 else if (strcmp(argv
[1], "--parallel") == 0)
461 errx(1, "Unknown arg %s", argv
[1]);
466 return parallel_test(vdev
.features
[0], getrange
, fast_vringh
);
468 if (posix_memalign(&__user_addr_min
, PAGE_SIZE
, USER_MEM
) != 0)
470 __user_addr_max
= __user_addr_min
+ USER_MEM
;
471 memset(__user_addr_min
, 0, vring_size(RINGSIZE
, ALIGN
));
473 /* Set up guest side. */
474 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &vdev
, true,
476 never_notify_host
, never_callback_guest
,
479 /* Set up host side. */
480 vring_init(&vrh
.vring
, RINGSIZE
, __user_addr_min
, ALIGN
);
481 vringh_init_user(&vrh
, vdev
.features
[0], RINGSIZE
, true,
482 vrh
.vring
.desc
, vrh
.vring
.avail
, vrh
.vring
.used
);
484 /* No descriptor to get yet... */
485 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
487 errx(1, "vringh_getdesc_user: %i", err
);
489 /* Guest puts in a descriptor. */
490 memcpy(__user_addr_max
- 1, "a", 1);
491 sg_init_table(guest_sg
, 1);
492 sg_set_buf(&guest_sg
[0], __user_addr_max
- 1, 1);
493 sg_init_table(guest_sg
+1, 1);
494 sg_set_buf(&guest_sg
[1], __user_addr_max
- 3, 2);
495 sgs
[0] = &guest_sg
[0];
496 sgs
[1] = &guest_sg
[1];
498 /* May allocate an indirect, so force it to allocate user addr */
499 __kmalloc_fake
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
500 err
= virtqueue_add_sgs(vq
, sgs
, 1, 1, &err
, GFP_KERNEL
);
502 errx(1, "virtqueue_add_sgs: %i", err
);
503 __kmalloc_fake
= NULL
;
505 /* Host retreives it. */
506 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
507 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
509 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
511 errx(1, "vringh_getdesc_user: %i", err
);
513 assert(riov
.used
== 1);
514 assert(riov
.iov
[0].iov_base
== __user_addr_max
- 1);
515 assert(riov
.iov
[0].iov_len
== 1);
516 if (getrange
!= getrange_slow
) {
517 assert(wiov
.used
== 1);
518 assert(wiov
.iov
[0].iov_base
== __user_addr_max
- 3);
519 assert(wiov
.iov
[0].iov_len
== 2);
521 assert(wiov
.used
== 2);
522 assert(wiov
.iov
[0].iov_base
== __user_addr_max
- 3);
523 assert(wiov
.iov
[0].iov_len
== 1);
524 assert(wiov
.iov
[1].iov_base
== __user_addr_max
- 2);
525 assert(wiov
.iov
[1].iov_len
== 1);
528 err
= vringh_iov_pull_user(&riov
, buf
, 5);
530 errx(1, "vringh_iov_pull_user: %i", err
);
531 assert(buf
[0] == 'a');
533 assert(vringh_iov_pull_user(&riov
, buf
, 5) == 0);
535 memcpy(buf
, "bcdef", 5);
536 err
= vringh_iov_push_user(&wiov
, buf
, 5);
538 errx(1, "vringh_iov_push_user: %i", err
);
539 assert(memcmp(__user_addr_max
- 3, "bc", 2) == 0);
540 assert(wiov
.i
== wiov
.used
);
541 assert(vringh_iov_push_user(&wiov
, buf
, 5) == 0);
544 err
= vringh_complete_user(&vrh
, head
, err
);
546 errx(1, "vringh_complete_user: %i", err
);
548 /* Guest should see used token now. */
549 __kfree_ignore_start
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
550 __kfree_ignore_end
= __kfree_ignore_start
+ 1;
551 ret
= virtqueue_get_buf(vq
, &i
);
553 errx(1, "virtqueue_get_buf: %p", ret
);
556 /* Guest puts in a huge descriptor. */
557 sg_init_table(guest_sg
, RINGSIZE
);
558 for (i
= 0; i
< RINGSIZE
; i
++) {
559 sg_set_buf(&guest_sg
[i
],
560 __user_addr_max
- USER_MEM
/4, USER_MEM
/4);
563 /* Fill contents with recognisable garbage. */
564 for (i
= 0; i
< USER_MEM
/4; i
++)
565 ((char *)__user_addr_max
- USER_MEM
/4)[i
] = i
;
567 /* This will allocate an indirect, so force it to allocate user addr */
568 __kmalloc_fake
= __user_addr_min
+ vring_size(RINGSIZE
, ALIGN
);
569 err
= virtqueue_add_outbuf(vq
, guest_sg
, RINGSIZE
, &err
, GFP_KERNEL
);
571 errx(1, "virtqueue_add_outbuf (large): %i", err
);
572 __kmalloc_fake
= NULL
;
574 /* Host picks it up (allocates new iov). */
575 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
576 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
578 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
580 errx(1, "vringh_getdesc_user: %i", err
);
582 assert(riov
.max_num
& VRINGH_IOV_ALLOCATED
);
583 assert(riov
.iov
!= host_riov
);
584 if (getrange
!= getrange_slow
)
585 assert(riov
.used
== RINGSIZE
);
587 assert(riov
.used
== RINGSIZE
* USER_MEM
/4);
589 assert(!(wiov
.max_num
& VRINGH_IOV_ALLOCATED
));
590 assert(wiov
.used
== 0);
592 /* Pull data back out (in odd chunks), should be as expected. */
593 for (i
= 0; i
< RINGSIZE
* USER_MEM
/4; i
+= 3) {
594 err
= vringh_iov_pull_user(&riov
, buf
, 3);
595 if (err
!= 3 && i
+ err
!= RINGSIZE
* USER_MEM
/4)
596 errx(1, "vringh_iov_pull_user large: %i", err
);
597 assert(buf
[0] == (char)i
);
598 assert(err
< 2 || buf
[1] == (char)(i
+ 1));
599 assert(err
< 3 || buf
[2] == (char)(i
+ 2));
601 assert(riov
.i
== riov
.used
);
602 vringh_iov_cleanup(&riov
);
603 vringh_iov_cleanup(&wiov
);
605 /* Complete using multi interface, just because we can. */
608 err
= vringh_complete_multi_user(&vrh
, used
, 1);
610 errx(1, "vringh_complete_multi_user(1): %i", err
);
612 /* Free up those descriptors. */
613 ret
= virtqueue_get_buf(vq
, &i
);
615 errx(1, "virtqueue_get_buf: %p", ret
);
617 /* Add lots of descriptors. */
618 sg_init_table(guest_sg
, 1);
619 sg_set_buf(&guest_sg
[0], __user_addr_max
- 1, 1);
620 for (i
= 0; i
< RINGSIZE
; i
++) {
621 err
= virtqueue_add_outbuf(vq
, guest_sg
, 1, &err
, GFP_KERNEL
);
623 errx(1, "virtqueue_add_outbuf (multiple): %i", err
);
626 /* Now get many, and consume them all at once. */
627 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
628 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
630 for (i
= 0; i
< RINGSIZE
; i
++) {
631 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
633 errx(1, "vringh_getdesc_user: %i", err
);
637 /* Make sure it wraps around ring, to test! */
638 assert(vrh
.vring
.used
->idx
% RINGSIZE
!= 0);
639 err
= vringh_complete_multi_user(&vrh
, used
, RINGSIZE
);
641 errx(1, "vringh_complete_multi_user: %i", err
);
643 /* Free those buffers. */
644 for (i
= 0; i
< RINGSIZE
; i
++) {
646 assert(virtqueue_get_buf(vq
, &len
) != NULL
);
649 /* Test weird (but legal!) indirect. */
650 if (vdev
.features
[0] & (1 << VIRTIO_RING_F_INDIRECT_DESC
)) {
651 char *data
= __user_addr_max
- USER_MEM
/4;
652 struct vring_desc
*d
= __user_addr_max
- USER_MEM
/2;
655 /* Force creation of direct, which we modify. */
656 vdev
.features
[0] &= ~(1 << VIRTIO_RING_F_INDIRECT_DESC
);
657 vq
= vring_new_virtqueue(0, RINGSIZE
, ALIGN
, &vdev
, true,
660 never_callback_guest
,
663 sg_init_table(guest_sg
, 4);
664 sg_set_buf(&guest_sg
[0], d
, sizeof(*d
)*2);
665 sg_set_buf(&guest_sg
[1], d
+ 2, sizeof(*d
)*1);
666 sg_set_buf(&guest_sg
[2], data
+ 6, 4);
667 sg_set_buf(&guest_sg
[3], d
+ 3, sizeof(*d
)*3);
669 err
= virtqueue_add_outbuf(vq
, guest_sg
, 4, &err
, GFP_KERNEL
);
671 errx(1, "virtqueue_add_outbuf (indirect): %i", err
);
673 vring_init(&vring
, RINGSIZE
, __user_addr_min
, ALIGN
);
675 /* They're used in order, but double-check... */
676 assert(vring
.desc
[0].addr
== (unsigned long)d
);
677 assert(vring
.desc
[1].addr
== (unsigned long)(d
+2));
678 assert(vring
.desc
[2].addr
== (unsigned long)data
+ 6);
679 assert(vring
.desc
[3].addr
== (unsigned long)(d
+3));
680 vring
.desc
[0].flags
|= VRING_DESC_F_INDIRECT
;
681 vring
.desc
[1].flags
|= VRING_DESC_F_INDIRECT
;
682 vring
.desc
[3].flags
|= VRING_DESC_F_INDIRECT
;
685 d
[0].addr
= (unsigned long)data
;
687 d
[0].flags
= VRING_DESC_F_NEXT
;
689 d
[1].addr
= (unsigned long)data
+ 1;
693 /* Second indirect */
694 d
[2].addr
= (unsigned long)data
+ 3;
699 d
[3].addr
= (unsigned long)data
+ 10;
701 d
[3].flags
= VRING_DESC_F_NEXT
;
703 d
[4].addr
= (unsigned long)data
+ 15;
705 d
[4].flags
= VRING_DESC_F_NEXT
;
707 d
[5].addr
= (unsigned long)data
+ 21;
711 /* Host picks it up (allocates new iov). */
712 vringh_iov_init(&riov
, host_riov
, ARRAY_SIZE(host_riov
));
713 vringh_iov_init(&wiov
, host_wiov
, ARRAY_SIZE(host_wiov
));
715 err
= vringh_getdesc_user(&vrh
, &riov
, &wiov
, getrange
, &head
);
717 errx(1, "vringh_getdesc_user: %i", err
);
720 errx(1, "vringh_getdesc_user: head %i not 0", head
);
722 assert(riov
.max_num
& VRINGH_IOV_ALLOCATED
);
723 if (getrange
!= getrange_slow
)
724 assert(riov
.used
== 7);
726 assert(riov
.used
== 28);
727 err
= vringh_iov_pull_user(&riov
, buf
, 29);
730 /* Data should be linear. */
731 for (i
= 0; i
< err
; i
++)
733 vringh_iov_cleanup(&riov
);
736 /* Don't leak memory... */
737 vring_del_virtqueue(vq
);
738 free(__user_addr_min
);