2 * vhost transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * This work is licensed under the terms of the GNU GPL, version 2.
10 #include <linux/miscdevice.h>
11 #include <linux/atomic.h>
12 #include <linux/module.h>
13 #include <linux/mutex.h>
14 #include <linux/vmalloc.h>
16 #include <linux/virtio_vsock.h>
17 #include <linux/vhost.h>
19 #include <net/af_vsock.h>
22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2
25 VHOST_VSOCK_FEATURES
= VHOST_FEATURES
,
28 /* Used to track all the vhost_vsock instances on the system. */
29 static DEFINE_SPINLOCK(vhost_vsock_lock
);
30 static LIST_HEAD(vhost_vsock_list
);
34 struct vhost_virtqueue vqs
[2];
36 /* Link to global vhost_vsock_list, protected by vhost_vsock_lock */
37 struct list_head list
;
39 struct vhost_work send_pkt_work
;
40 spinlock_t send_pkt_list_lock
;
41 struct list_head send_pkt_list
; /* host->guest pending packets */
43 atomic_t queued_replies
;
48 static u32
vhost_transport_get_local_cid(void)
50 return VHOST_VSOCK_DEFAULT_HOST_CID
;
53 static struct vhost_vsock
*__vhost_vsock_get(u32 guest_cid
)
55 struct vhost_vsock
*vsock
;
57 list_for_each_entry(vsock
, &vhost_vsock_list
, list
) {
58 u32 other_cid
= vsock
->guest_cid
;
60 /* Skip instances that have no CID yet */
64 if (other_cid
== guest_cid
) {
72 static struct vhost_vsock
*vhost_vsock_get(u32 guest_cid
)
74 struct vhost_vsock
*vsock
;
76 spin_lock_bh(&vhost_vsock_lock
);
77 vsock
= __vhost_vsock_get(guest_cid
);
78 spin_unlock_bh(&vhost_vsock_lock
);
84 vhost_transport_do_send_pkt(struct vhost_vsock
*vsock
,
85 struct vhost_virtqueue
*vq
)
87 struct vhost_virtqueue
*tx_vq
= &vsock
->vqs
[VSOCK_VQ_TX
];
89 bool restart_tx
= false;
91 mutex_lock(&vq
->mutex
);
93 if (!vq
->private_data
)
96 /* Avoid further vmexits, we're already processing the virtqueue */
97 vhost_disable_notify(&vsock
->dev
, vq
);
100 struct virtio_vsock_pkt
*pkt
;
101 struct iov_iter iov_iter
;
107 spin_lock_bh(&vsock
->send_pkt_list_lock
);
108 if (list_empty(&vsock
->send_pkt_list
)) {
109 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
110 vhost_enable_notify(&vsock
->dev
, vq
);
114 pkt
= list_first_entry(&vsock
->send_pkt_list
,
115 struct virtio_vsock_pkt
, list
);
116 list_del_init(&pkt
->list
);
117 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
119 head
= vhost_get_vq_desc(vq
, vq
->iov
, ARRAY_SIZE(vq
->iov
),
120 &out
, &in
, NULL
, NULL
);
122 spin_lock_bh(&vsock
->send_pkt_list_lock
);
123 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
124 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
128 if (head
== vq
->num
) {
129 spin_lock_bh(&vsock
->send_pkt_list_lock
);
130 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
131 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
133 /* We cannot finish yet if more buffers snuck in while
134 * re-enabling notify.
136 if (unlikely(vhost_enable_notify(&vsock
->dev
, vq
))) {
137 vhost_disable_notify(&vsock
->dev
, vq
);
144 virtio_transport_free_pkt(pkt
);
145 vq_err(vq
, "Expected 0 output buffers, got %u\n", out
);
149 len
= iov_length(&vq
->iov
[out
], in
);
150 iov_iter_init(&iov_iter
, READ
, &vq
->iov
[out
], in
, len
);
152 nbytes
= copy_to_iter(&pkt
->hdr
, sizeof(pkt
->hdr
), &iov_iter
);
153 if (nbytes
!= sizeof(pkt
->hdr
)) {
154 virtio_transport_free_pkt(pkt
);
155 vq_err(vq
, "Faulted on copying pkt hdr\n");
159 nbytes
= copy_to_iter(pkt
->buf
, pkt
->len
, &iov_iter
);
160 if (nbytes
!= pkt
->len
) {
161 virtio_transport_free_pkt(pkt
);
162 vq_err(vq
, "Faulted on copying pkt buf\n");
166 vhost_add_used(vq
, head
, sizeof(pkt
->hdr
) + pkt
->len
);
172 val
= atomic_dec_return(&vsock
->queued_replies
);
174 /* Do we have resources to resume tx processing? */
175 if (val
+ 1 == tx_vq
->num
)
179 virtio_transport_free_pkt(pkt
);
182 vhost_signal(&vsock
->dev
, vq
);
185 mutex_unlock(&vq
->mutex
);
188 vhost_poll_queue(&tx_vq
->poll
);
191 static void vhost_transport_send_pkt_work(struct vhost_work
*work
)
193 struct vhost_virtqueue
*vq
;
194 struct vhost_vsock
*vsock
;
196 vsock
= container_of(work
, struct vhost_vsock
, send_pkt_work
);
197 vq
= &vsock
->vqs
[VSOCK_VQ_RX
];
199 vhost_transport_do_send_pkt(vsock
, vq
);
203 vhost_transport_send_pkt(struct virtio_vsock_pkt
*pkt
)
205 struct vhost_vsock
*vsock
;
208 /* Find the vhost_vsock according to guest context id */
209 vsock
= vhost_vsock_get(le64_to_cpu(pkt
->hdr
.dst_cid
));
211 virtio_transport_free_pkt(pkt
);
216 atomic_inc(&vsock
->queued_replies
);
218 spin_lock_bh(&vsock
->send_pkt_list_lock
);
219 list_add_tail(&pkt
->list
, &vsock
->send_pkt_list
);
220 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
222 vhost_work_queue(&vsock
->dev
, &vsock
->send_pkt_work
);
226 static struct virtio_vsock_pkt
*
227 vhost_vsock_alloc_pkt(struct vhost_virtqueue
*vq
,
228 unsigned int out
, unsigned int in
)
230 struct virtio_vsock_pkt
*pkt
;
231 struct iov_iter iov_iter
;
236 vq_err(vq
, "Expected 0 input buffers, got %u\n", in
);
240 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
244 len
= iov_length(vq
->iov
, out
);
245 iov_iter_init(&iov_iter
, WRITE
, vq
->iov
, out
, len
);
247 nbytes
= copy_from_iter(&pkt
->hdr
, sizeof(pkt
->hdr
), &iov_iter
);
248 if (nbytes
!= sizeof(pkt
->hdr
)) {
249 vq_err(vq
, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
250 sizeof(pkt
->hdr
), nbytes
);
255 if (le16_to_cpu(pkt
->hdr
.type
) == VIRTIO_VSOCK_TYPE_STREAM
)
256 pkt
->len
= le32_to_cpu(pkt
->hdr
.len
);
262 /* The pkt is too big */
263 if (pkt
->len
> VIRTIO_VSOCK_MAX_PKT_BUF_SIZE
) {
268 pkt
->buf
= kmalloc(pkt
->len
, GFP_KERNEL
);
274 nbytes
= copy_from_iter(pkt
->buf
, pkt
->len
, &iov_iter
);
275 if (nbytes
!= pkt
->len
) {
276 vq_err(vq
, "Expected %u byte payload, got %zu bytes\n",
278 virtio_transport_free_pkt(pkt
);
285 /* Is there space left for replies to rx packets? */
286 static bool vhost_vsock_more_replies(struct vhost_vsock
*vsock
)
288 struct vhost_virtqueue
*vq
= &vsock
->vqs
[VSOCK_VQ_TX
];
291 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
292 val
= atomic_read(&vsock
->queued_replies
);
294 return val
< vq
->num
;
297 static void vhost_vsock_handle_tx_kick(struct vhost_work
*work
)
299 struct vhost_virtqueue
*vq
= container_of(work
, struct vhost_virtqueue
,
301 struct vhost_vsock
*vsock
= container_of(vq
->dev
, struct vhost_vsock
,
303 struct virtio_vsock_pkt
*pkt
;
305 unsigned int out
, in
;
308 mutex_lock(&vq
->mutex
);
310 if (!vq
->private_data
)
313 vhost_disable_notify(&vsock
->dev
, vq
);
317 if (!vhost_vsock_more_replies(vsock
)) {
318 /* Stop tx until the device processes already
319 * pending replies. Leave tx virtqueue
320 * callbacks disabled.
322 goto no_more_replies
;
325 head
= vhost_get_vq_desc(vq
, vq
->iov
, ARRAY_SIZE(vq
->iov
),
326 &out
, &in
, NULL
, NULL
);
330 if (head
== vq
->num
) {
331 if (unlikely(vhost_enable_notify(&vsock
->dev
, vq
))) {
332 vhost_disable_notify(&vsock
->dev
, vq
);
338 pkt
= vhost_vsock_alloc_pkt(vq
, out
, in
);
340 vq_err(vq
, "Faulted on pkt\n");
346 /* Only accept correctly addressed packets */
347 if (le64_to_cpu(pkt
->hdr
.src_cid
) == vsock
->guest_cid
)
348 virtio_transport_recv_pkt(pkt
);
350 virtio_transport_free_pkt(pkt
);
352 vhost_add_used(vq
, head
, sizeof(pkt
->hdr
) + len
);
358 vhost_signal(&vsock
->dev
, vq
);
361 mutex_unlock(&vq
->mutex
);
364 static void vhost_vsock_handle_rx_kick(struct vhost_work
*work
)
366 struct vhost_virtqueue
*vq
= container_of(work
, struct vhost_virtqueue
,
368 struct vhost_vsock
*vsock
= container_of(vq
->dev
, struct vhost_vsock
,
371 vhost_transport_do_send_pkt(vsock
, vq
);
374 static int vhost_vsock_start(struct vhost_vsock
*vsock
)
376 struct vhost_virtqueue
*vq
;
380 mutex_lock(&vsock
->dev
.mutex
);
382 ret
= vhost_dev_check_owner(&vsock
->dev
);
386 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
389 mutex_lock(&vq
->mutex
);
391 if (!vhost_vq_access_ok(vq
)) {
396 if (!vq
->private_data
) {
397 vq
->private_data
= vsock
;
398 ret
= vhost_vq_init_access(vq
);
403 mutex_unlock(&vq
->mutex
);
406 mutex_unlock(&vsock
->dev
.mutex
);
410 vq
->private_data
= NULL
;
411 mutex_unlock(&vq
->mutex
);
413 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
416 mutex_lock(&vq
->mutex
);
417 vq
->private_data
= NULL
;
418 mutex_unlock(&vq
->mutex
);
421 mutex_unlock(&vsock
->dev
.mutex
);
425 static int vhost_vsock_stop(struct vhost_vsock
*vsock
)
430 mutex_lock(&vsock
->dev
.mutex
);
432 ret
= vhost_dev_check_owner(&vsock
->dev
);
436 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
437 struct vhost_virtqueue
*vq
= &vsock
->vqs
[i
];
439 mutex_lock(&vq
->mutex
);
440 vq
->private_data
= NULL
;
441 mutex_unlock(&vq
->mutex
);
445 mutex_unlock(&vsock
->dev
.mutex
);
449 static void vhost_vsock_free(struct vhost_vsock
*vsock
)
454 static int vhost_vsock_dev_open(struct inode
*inode
, struct file
*file
)
456 struct vhost_virtqueue
**vqs
;
457 struct vhost_vsock
*vsock
;
460 /* This struct is large and allocation could fail, fall back to vmalloc
461 * if there is no other way.
463 vsock
= kzalloc(sizeof(*vsock
), GFP_KERNEL
| __GFP_NOWARN
| __GFP_REPEAT
);
465 vsock
= vmalloc(sizeof(*vsock
));
470 vqs
= kmalloc_array(ARRAY_SIZE(vsock
->vqs
), sizeof(*vqs
), GFP_KERNEL
);
476 atomic_set(&vsock
->queued_replies
, 0);
478 vqs
[VSOCK_VQ_TX
] = &vsock
->vqs
[VSOCK_VQ_TX
];
479 vqs
[VSOCK_VQ_RX
] = &vsock
->vqs
[VSOCK_VQ_RX
];
480 vsock
->vqs
[VSOCK_VQ_TX
].handle_kick
= vhost_vsock_handle_tx_kick
;
481 vsock
->vqs
[VSOCK_VQ_RX
].handle_kick
= vhost_vsock_handle_rx_kick
;
483 vhost_dev_init(&vsock
->dev
, vqs
, ARRAY_SIZE(vsock
->vqs
));
485 file
->private_data
= vsock
;
486 spin_lock_init(&vsock
->send_pkt_list_lock
);
487 INIT_LIST_HEAD(&vsock
->send_pkt_list
);
488 vhost_work_init(&vsock
->send_pkt_work
, vhost_transport_send_pkt_work
);
490 spin_lock_bh(&vhost_vsock_lock
);
491 list_add_tail(&vsock
->list
, &vhost_vsock_list
);
492 spin_unlock_bh(&vhost_vsock_lock
);
496 vhost_vsock_free(vsock
);
500 static void vhost_vsock_flush(struct vhost_vsock
*vsock
)
504 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++)
505 if (vsock
->vqs
[i
].handle_kick
)
506 vhost_poll_flush(&vsock
->vqs
[i
].poll
);
507 vhost_work_flush(&vsock
->dev
, &vsock
->send_pkt_work
);
510 static void vhost_vsock_reset_orphans(struct sock
*sk
)
512 struct vsock_sock
*vsk
= vsock_sk(sk
);
514 /* vmci_transport.c doesn't take sk_lock here either. At least we're
515 * under vsock_table_lock so the sock cannot disappear while we're
519 if (!vhost_vsock_get(vsk
->remote_addr
.svm_cid
)) {
520 sock_set_flag(sk
, SOCK_DONE
);
521 vsk
->peer_shutdown
= SHUTDOWN_MASK
;
522 sk
->sk_state
= SS_UNCONNECTED
;
523 sk
->sk_err
= ECONNRESET
;
524 sk
->sk_error_report(sk
);
528 static int vhost_vsock_dev_release(struct inode
*inode
, struct file
*file
)
530 struct vhost_vsock
*vsock
= file
->private_data
;
532 spin_lock_bh(&vhost_vsock_lock
);
533 list_del(&vsock
->list
);
534 spin_unlock_bh(&vhost_vsock_lock
);
536 /* Iterating over all connections for all CIDs to find orphans is
537 * inefficient. Room for improvement here. */
538 vsock_for_each_connected_socket(vhost_vsock_reset_orphans
);
540 vhost_vsock_stop(vsock
);
541 vhost_vsock_flush(vsock
);
542 vhost_dev_stop(&vsock
->dev
);
544 spin_lock_bh(&vsock
->send_pkt_list_lock
);
545 while (!list_empty(&vsock
->send_pkt_list
)) {
546 struct virtio_vsock_pkt
*pkt
;
548 pkt
= list_first_entry(&vsock
->send_pkt_list
,
549 struct virtio_vsock_pkt
, list
);
550 list_del_init(&pkt
->list
);
551 virtio_transport_free_pkt(pkt
);
553 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
555 vhost_dev_cleanup(&vsock
->dev
, false);
556 kfree(vsock
->dev
.vqs
);
557 vhost_vsock_free(vsock
);
561 static int vhost_vsock_set_cid(struct vhost_vsock
*vsock
, u64 guest_cid
)
563 struct vhost_vsock
*other
;
565 /* Refuse reserved CIDs */
566 if (guest_cid
<= VMADDR_CID_HOST
||
567 guest_cid
== U32_MAX
)
570 /* 64-bit CIDs are not yet supported */
571 if (guest_cid
> U32_MAX
)
574 /* Refuse if CID is already in use */
575 spin_lock_bh(&vhost_vsock_lock
);
576 other
= __vhost_vsock_get(guest_cid
);
577 if (other
&& other
!= vsock
) {
578 spin_unlock_bh(&vhost_vsock_lock
);
581 vsock
->guest_cid
= guest_cid
;
582 spin_unlock_bh(&vhost_vsock_lock
);
587 static int vhost_vsock_set_features(struct vhost_vsock
*vsock
, u64 features
)
589 struct vhost_virtqueue
*vq
;
592 if (features
& ~VHOST_VSOCK_FEATURES
)
595 mutex_lock(&vsock
->dev
.mutex
);
596 if ((features
& (1 << VHOST_F_LOG_ALL
)) &&
597 !vhost_log_access_ok(&vsock
->dev
)) {
598 mutex_unlock(&vsock
->dev
.mutex
);
602 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
604 mutex_lock(&vq
->mutex
);
605 vq
->acked_features
= features
;
606 mutex_unlock(&vq
->mutex
);
608 mutex_unlock(&vsock
->dev
.mutex
);
612 static long vhost_vsock_dev_ioctl(struct file
*f
, unsigned int ioctl
,
615 struct vhost_vsock
*vsock
= f
->private_data
;
616 void __user
*argp
= (void __user
*)arg
;
623 case VHOST_VSOCK_SET_GUEST_CID
:
624 if (copy_from_user(&guest_cid
, argp
, sizeof(guest_cid
)))
626 return vhost_vsock_set_cid(vsock
, guest_cid
);
627 case VHOST_VSOCK_SET_RUNNING
:
628 if (copy_from_user(&start
, argp
, sizeof(start
)))
631 return vhost_vsock_start(vsock
);
633 return vhost_vsock_stop(vsock
);
634 case VHOST_GET_FEATURES
:
635 features
= VHOST_VSOCK_FEATURES
;
636 if (copy_to_user(argp
, &features
, sizeof(features
)))
639 case VHOST_SET_FEATURES
:
640 if (copy_from_user(&features
, argp
, sizeof(features
)))
642 return vhost_vsock_set_features(vsock
, features
);
644 mutex_lock(&vsock
->dev
.mutex
);
645 r
= vhost_dev_ioctl(&vsock
->dev
, ioctl
, argp
);
646 if (r
== -ENOIOCTLCMD
)
647 r
= vhost_vring_ioctl(&vsock
->dev
, ioctl
, argp
);
649 vhost_vsock_flush(vsock
);
650 mutex_unlock(&vsock
->dev
.mutex
);
655 static const struct file_operations vhost_vsock_fops
= {
656 .owner
= THIS_MODULE
,
657 .open
= vhost_vsock_dev_open
,
658 .release
= vhost_vsock_dev_release
,
659 .llseek
= noop_llseek
,
660 .unlocked_ioctl
= vhost_vsock_dev_ioctl
,
663 static struct miscdevice vhost_vsock_misc
= {
664 .minor
= MISC_DYNAMIC_MINOR
,
665 .name
= "vhost-vsock",
666 .fops
= &vhost_vsock_fops
,
669 static struct virtio_transport vhost_transport
= {
671 .get_local_cid
= vhost_transport_get_local_cid
,
673 .init
= virtio_transport_do_socket_init
,
674 .destruct
= virtio_transport_destruct
,
675 .release
= virtio_transport_release
,
676 .connect
= virtio_transport_connect
,
677 .shutdown
= virtio_transport_shutdown
,
679 .dgram_enqueue
= virtio_transport_dgram_enqueue
,
680 .dgram_dequeue
= virtio_transport_dgram_dequeue
,
681 .dgram_bind
= virtio_transport_dgram_bind
,
682 .dgram_allow
= virtio_transport_dgram_allow
,
684 .stream_enqueue
= virtio_transport_stream_enqueue
,
685 .stream_dequeue
= virtio_transport_stream_dequeue
,
686 .stream_has_data
= virtio_transport_stream_has_data
,
687 .stream_has_space
= virtio_transport_stream_has_space
,
688 .stream_rcvhiwat
= virtio_transport_stream_rcvhiwat
,
689 .stream_is_active
= virtio_transport_stream_is_active
,
690 .stream_allow
= virtio_transport_stream_allow
,
692 .notify_poll_in
= virtio_transport_notify_poll_in
,
693 .notify_poll_out
= virtio_transport_notify_poll_out
,
694 .notify_recv_init
= virtio_transport_notify_recv_init
,
695 .notify_recv_pre_block
= virtio_transport_notify_recv_pre_block
,
696 .notify_recv_pre_dequeue
= virtio_transport_notify_recv_pre_dequeue
,
697 .notify_recv_post_dequeue
= virtio_transport_notify_recv_post_dequeue
,
698 .notify_send_init
= virtio_transport_notify_send_init
,
699 .notify_send_pre_block
= virtio_transport_notify_send_pre_block
,
700 .notify_send_pre_enqueue
= virtio_transport_notify_send_pre_enqueue
,
701 .notify_send_post_enqueue
= virtio_transport_notify_send_post_enqueue
,
703 .set_buffer_size
= virtio_transport_set_buffer_size
,
704 .set_min_buffer_size
= virtio_transport_set_min_buffer_size
,
705 .set_max_buffer_size
= virtio_transport_set_max_buffer_size
,
706 .get_buffer_size
= virtio_transport_get_buffer_size
,
707 .get_min_buffer_size
= virtio_transport_get_min_buffer_size
,
708 .get_max_buffer_size
= virtio_transport_get_max_buffer_size
,
711 .send_pkt
= vhost_transport_send_pkt
,
714 static int __init
vhost_vsock_init(void)
718 ret
= vsock_core_init(&vhost_transport
.transport
);
721 return misc_register(&vhost_vsock_misc
);
724 static void __exit
vhost_vsock_exit(void)
726 misc_deregister(&vhost_vsock_misc
);
730 module_init(vhost_vsock_init
);
731 module_exit(vhost_vsock_exit
);
732 MODULE_LICENSE("GPL v2");
733 MODULE_AUTHOR("Asias He");
734 MODULE_DESCRIPTION("vhost transport for vsock ");