2 * vhost transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * This work is licensed under the terms of the GNU GPL, version 2.
10 #include <linux/miscdevice.h>
11 #include <linux/atomic.h>
12 #include <linux/module.h>
13 #include <linux/mutex.h>
14 #include <linux/vmalloc.h>
16 #include <linux/virtio_vsock.h>
17 #include <linux/vhost.h>
19 #include <net/af_vsock.h>
22 #define VHOST_VSOCK_DEFAULT_HOST_CID 2
25 VHOST_VSOCK_FEATURES
= VHOST_FEATURES
,
28 /* Used to track all the vhost_vsock instances on the system. */
29 static DEFINE_SPINLOCK(vhost_vsock_lock
);
30 static LIST_HEAD(vhost_vsock_list
);
34 struct vhost_virtqueue vqs
[2];
36 /* Link to global vhost_vsock_list, protected by vhost_vsock_lock */
37 struct list_head list
;
39 struct vhost_work send_pkt_work
;
40 spinlock_t send_pkt_list_lock
;
41 struct list_head send_pkt_list
; /* host->guest pending packets */
43 atomic_t queued_replies
;
48 static u32
vhost_transport_get_local_cid(void)
50 return VHOST_VSOCK_DEFAULT_HOST_CID
;
53 static struct vhost_vsock
*vhost_vsock_get(u32 guest_cid
)
55 struct vhost_vsock
*vsock
;
57 spin_lock_bh(&vhost_vsock_lock
);
58 list_for_each_entry(vsock
, &vhost_vsock_list
, list
) {
59 u32 other_cid
= vsock
->guest_cid
;
61 /* Skip instances that have no CID yet */
65 if (other_cid
== guest_cid
) {
66 spin_unlock_bh(&vhost_vsock_lock
);
70 spin_unlock_bh(&vhost_vsock_lock
);
76 vhost_transport_do_send_pkt(struct vhost_vsock
*vsock
,
77 struct vhost_virtqueue
*vq
)
79 struct vhost_virtqueue
*tx_vq
= &vsock
->vqs
[VSOCK_VQ_TX
];
81 bool restart_tx
= false;
83 mutex_lock(&vq
->mutex
);
85 if (!vq
->private_data
)
88 /* Avoid further vmexits, we're already processing the virtqueue */
89 vhost_disable_notify(&vsock
->dev
, vq
);
92 struct virtio_vsock_pkt
*pkt
;
93 struct iov_iter iov_iter
;
99 spin_lock_bh(&vsock
->send_pkt_list_lock
);
100 if (list_empty(&vsock
->send_pkt_list
)) {
101 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
102 vhost_enable_notify(&vsock
->dev
, vq
);
106 pkt
= list_first_entry(&vsock
->send_pkt_list
,
107 struct virtio_vsock_pkt
, list
);
108 list_del_init(&pkt
->list
);
109 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
111 head
= vhost_get_vq_desc(vq
, vq
->iov
, ARRAY_SIZE(vq
->iov
),
112 &out
, &in
, NULL
, NULL
);
114 spin_lock_bh(&vsock
->send_pkt_list_lock
);
115 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
116 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
120 if (head
== vq
->num
) {
121 spin_lock_bh(&vsock
->send_pkt_list_lock
);
122 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
123 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
125 /* We cannot finish yet if more buffers snuck in while
126 * re-enabling notify.
128 if (unlikely(vhost_enable_notify(&vsock
->dev
, vq
))) {
129 vhost_disable_notify(&vsock
->dev
, vq
);
136 virtio_transport_free_pkt(pkt
);
137 vq_err(vq
, "Expected 0 output buffers, got %u\n", out
);
141 len
= iov_length(&vq
->iov
[out
], in
);
142 iov_iter_init(&iov_iter
, READ
, &vq
->iov
[out
], in
, len
);
144 nbytes
= copy_to_iter(&pkt
->hdr
, sizeof(pkt
->hdr
), &iov_iter
);
145 if (nbytes
!= sizeof(pkt
->hdr
)) {
146 virtio_transport_free_pkt(pkt
);
147 vq_err(vq
, "Faulted on copying pkt hdr\n");
151 nbytes
= copy_to_iter(pkt
->buf
, pkt
->len
, &iov_iter
);
152 if (nbytes
!= pkt
->len
) {
153 virtio_transport_free_pkt(pkt
);
154 vq_err(vq
, "Faulted on copying pkt buf\n");
158 vhost_add_used(vq
, head
, sizeof(pkt
->hdr
) + pkt
->len
);
164 val
= atomic_dec_return(&vsock
->queued_replies
);
166 /* Do we have resources to resume tx processing? */
167 if (val
+ 1 == tx_vq
->num
)
171 virtio_transport_free_pkt(pkt
);
174 vhost_signal(&vsock
->dev
, vq
);
177 mutex_unlock(&vq
->mutex
);
180 vhost_poll_queue(&tx_vq
->poll
);
183 static void vhost_transport_send_pkt_work(struct vhost_work
*work
)
185 struct vhost_virtqueue
*vq
;
186 struct vhost_vsock
*vsock
;
188 vsock
= container_of(work
, struct vhost_vsock
, send_pkt_work
);
189 vq
= &vsock
->vqs
[VSOCK_VQ_RX
];
191 vhost_transport_do_send_pkt(vsock
, vq
);
195 vhost_transport_send_pkt(struct virtio_vsock_pkt
*pkt
)
197 struct vhost_vsock
*vsock
;
198 struct vhost_virtqueue
*vq
;
201 /* Find the vhost_vsock according to guest context id */
202 vsock
= vhost_vsock_get(le64_to_cpu(pkt
->hdr
.dst_cid
));
204 virtio_transport_free_pkt(pkt
);
208 vq
= &vsock
->vqs
[VSOCK_VQ_RX
];
211 atomic_inc(&vsock
->queued_replies
);
213 spin_lock_bh(&vsock
->send_pkt_list_lock
);
214 list_add_tail(&pkt
->list
, &vsock
->send_pkt_list
);
215 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
217 vhost_work_queue(&vsock
->dev
, &vsock
->send_pkt_work
);
221 static struct virtio_vsock_pkt
*
222 vhost_vsock_alloc_pkt(struct vhost_virtqueue
*vq
,
223 unsigned int out
, unsigned int in
)
225 struct virtio_vsock_pkt
*pkt
;
226 struct iov_iter iov_iter
;
231 vq_err(vq
, "Expected 0 input buffers, got %u\n", in
);
235 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
239 len
= iov_length(vq
->iov
, out
);
240 iov_iter_init(&iov_iter
, WRITE
, vq
->iov
, out
, len
);
242 nbytes
= copy_from_iter(&pkt
->hdr
, sizeof(pkt
->hdr
), &iov_iter
);
243 if (nbytes
!= sizeof(pkt
->hdr
)) {
244 vq_err(vq
, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",
245 sizeof(pkt
->hdr
), nbytes
);
250 if (le16_to_cpu(pkt
->hdr
.type
) == VIRTIO_VSOCK_TYPE_STREAM
)
251 pkt
->len
= le32_to_cpu(pkt
->hdr
.len
);
257 /* The pkt is too big */
258 if (pkt
->len
> VIRTIO_VSOCK_MAX_PKT_BUF_SIZE
) {
263 pkt
->buf
= kmalloc(pkt
->len
, GFP_KERNEL
);
269 nbytes
= copy_from_iter(pkt
->buf
, pkt
->len
, &iov_iter
);
270 if (nbytes
!= pkt
->len
) {
271 vq_err(vq
, "Expected %u byte payload, got %zu bytes\n",
273 virtio_transport_free_pkt(pkt
);
280 /* Is there space left for replies to rx packets? */
281 static bool vhost_vsock_more_replies(struct vhost_vsock
*vsock
)
283 struct vhost_virtqueue
*vq
= &vsock
->vqs
[VSOCK_VQ_TX
];
286 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
287 val
= atomic_read(&vsock
->queued_replies
);
289 return val
< vq
->num
;
292 static void vhost_vsock_handle_tx_kick(struct vhost_work
*work
)
294 struct vhost_virtqueue
*vq
= container_of(work
, struct vhost_virtqueue
,
296 struct vhost_vsock
*vsock
= container_of(vq
->dev
, struct vhost_vsock
,
298 struct virtio_vsock_pkt
*pkt
;
300 unsigned int out
, in
;
303 mutex_lock(&vq
->mutex
);
305 if (!vq
->private_data
)
308 vhost_disable_notify(&vsock
->dev
, vq
);
310 if (!vhost_vsock_more_replies(vsock
)) {
311 /* Stop tx until the device processes already
312 * pending replies. Leave tx virtqueue
313 * callbacks disabled.
315 goto no_more_replies
;
318 head
= vhost_get_vq_desc(vq
, vq
->iov
, ARRAY_SIZE(vq
->iov
),
319 &out
, &in
, NULL
, NULL
);
323 if (head
== vq
->num
) {
324 if (unlikely(vhost_enable_notify(&vsock
->dev
, vq
))) {
325 vhost_disable_notify(&vsock
->dev
, vq
);
331 pkt
= vhost_vsock_alloc_pkt(vq
, out
, in
);
333 vq_err(vq
, "Faulted on pkt\n");
337 /* Only accept correctly addressed packets */
338 if (le64_to_cpu(pkt
->hdr
.src_cid
) == vsock
->guest_cid
)
339 virtio_transport_recv_pkt(pkt
);
341 virtio_transport_free_pkt(pkt
);
343 vhost_add_used(vq
, head
, sizeof(pkt
->hdr
) + pkt
->len
);
349 vhost_signal(&vsock
->dev
, vq
);
352 mutex_unlock(&vq
->mutex
);
355 static void vhost_vsock_handle_rx_kick(struct vhost_work
*work
)
357 struct vhost_virtqueue
*vq
= container_of(work
, struct vhost_virtqueue
,
359 struct vhost_vsock
*vsock
= container_of(vq
->dev
, struct vhost_vsock
,
362 vhost_transport_do_send_pkt(vsock
, vq
);
365 static int vhost_vsock_start(struct vhost_vsock
*vsock
)
370 mutex_lock(&vsock
->dev
.mutex
);
372 ret
= vhost_dev_check_owner(&vsock
->dev
);
376 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
377 struct vhost_virtqueue
*vq
= &vsock
->vqs
[i
];
379 mutex_lock(&vq
->mutex
);
381 if (!vhost_vq_access_ok(vq
)) {
383 mutex_unlock(&vq
->mutex
);
387 if (!vq
->private_data
) {
388 vq
->private_data
= vsock
;
389 vhost_vq_init_access(vq
);
392 mutex_unlock(&vq
->mutex
);
395 mutex_unlock(&vsock
->dev
.mutex
);
399 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
400 struct vhost_virtqueue
*vq
= &vsock
->vqs
[i
];
402 mutex_lock(&vq
->mutex
);
403 vq
->private_data
= NULL
;
404 mutex_unlock(&vq
->mutex
);
407 mutex_unlock(&vsock
->dev
.mutex
);
411 static int vhost_vsock_stop(struct vhost_vsock
*vsock
)
416 mutex_lock(&vsock
->dev
.mutex
);
418 ret
= vhost_dev_check_owner(&vsock
->dev
);
422 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
423 struct vhost_virtqueue
*vq
= &vsock
->vqs
[i
];
425 mutex_lock(&vq
->mutex
);
426 vq
->private_data
= NULL
;
427 mutex_unlock(&vq
->mutex
);
431 mutex_unlock(&vsock
->dev
.mutex
);
435 static void vhost_vsock_free(struct vhost_vsock
*vsock
)
440 static int vhost_vsock_dev_open(struct inode
*inode
, struct file
*file
)
442 struct vhost_virtqueue
**vqs
;
443 struct vhost_vsock
*vsock
;
446 /* This struct is large and allocation could fail, fall back to vmalloc
447 * if there is no other way.
449 vsock
= kzalloc(sizeof(*vsock
), GFP_KERNEL
| __GFP_NOWARN
| __GFP_REPEAT
);
451 vsock
= vmalloc(sizeof(*vsock
));
456 vqs
= kmalloc_array(ARRAY_SIZE(vsock
->vqs
), sizeof(*vqs
), GFP_KERNEL
);
462 atomic_set(&vsock
->queued_replies
, 0);
464 vqs
[VSOCK_VQ_TX
] = &vsock
->vqs
[VSOCK_VQ_TX
];
465 vqs
[VSOCK_VQ_RX
] = &vsock
->vqs
[VSOCK_VQ_RX
];
466 vsock
->vqs
[VSOCK_VQ_TX
].handle_kick
= vhost_vsock_handle_tx_kick
;
467 vsock
->vqs
[VSOCK_VQ_RX
].handle_kick
= vhost_vsock_handle_rx_kick
;
469 vhost_dev_init(&vsock
->dev
, vqs
, ARRAY_SIZE(vsock
->vqs
));
471 file
->private_data
= vsock
;
472 spin_lock_init(&vsock
->send_pkt_list_lock
);
473 INIT_LIST_HEAD(&vsock
->send_pkt_list
);
474 vhost_work_init(&vsock
->send_pkt_work
, vhost_transport_send_pkt_work
);
476 spin_lock_bh(&vhost_vsock_lock
);
477 list_add_tail(&vsock
->list
, &vhost_vsock_list
);
478 spin_unlock_bh(&vhost_vsock_lock
);
482 vhost_vsock_free(vsock
);
486 static void vhost_vsock_flush(struct vhost_vsock
*vsock
)
490 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++)
491 if (vsock
->vqs
[i
].handle_kick
)
492 vhost_poll_flush(&vsock
->vqs
[i
].poll
);
493 vhost_work_flush(&vsock
->dev
, &vsock
->send_pkt_work
);
496 static void vhost_vsock_reset_orphans(struct sock
*sk
)
498 struct vsock_sock
*vsk
= vsock_sk(sk
);
500 /* vmci_transport.c doesn't take sk_lock here either. At least we're
501 * under vsock_table_lock so the sock cannot disappear while we're
505 if (!vhost_vsock_get(vsk
->local_addr
.svm_cid
)) {
506 sock_set_flag(sk
, SOCK_DONE
);
507 vsk
->peer_shutdown
= SHUTDOWN_MASK
;
508 sk
->sk_state
= SS_UNCONNECTED
;
509 sk
->sk_err
= ECONNRESET
;
510 sk
->sk_error_report(sk
);
514 static int vhost_vsock_dev_release(struct inode
*inode
, struct file
*file
)
516 struct vhost_vsock
*vsock
= file
->private_data
;
518 spin_lock_bh(&vhost_vsock_lock
);
519 list_del(&vsock
->list
);
520 spin_unlock_bh(&vhost_vsock_lock
);
522 /* Iterating over all connections for all CIDs to find orphans is
523 * inefficient. Room for improvement here. */
524 vsock_for_each_connected_socket(vhost_vsock_reset_orphans
);
526 vhost_vsock_stop(vsock
);
527 vhost_vsock_flush(vsock
);
528 vhost_dev_stop(&vsock
->dev
);
530 spin_lock_bh(&vsock
->send_pkt_list_lock
);
531 while (!list_empty(&vsock
->send_pkt_list
)) {
532 struct virtio_vsock_pkt
*pkt
;
534 pkt
= list_first_entry(&vsock
->send_pkt_list
,
535 struct virtio_vsock_pkt
, list
);
536 list_del_init(&pkt
->list
);
537 virtio_transport_free_pkt(pkt
);
539 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
541 vhost_dev_cleanup(&vsock
->dev
, false);
542 kfree(vsock
->dev
.vqs
);
543 vhost_vsock_free(vsock
);
547 static int vhost_vsock_set_cid(struct vhost_vsock
*vsock
, u64 guest_cid
)
549 struct vhost_vsock
*other
;
551 /* Refuse reserved CIDs */
552 if (guest_cid
<= VMADDR_CID_HOST
||
553 guest_cid
== U32_MAX
)
556 /* 64-bit CIDs are not yet supported */
557 if (guest_cid
> U32_MAX
)
560 /* Refuse if CID is already in use */
561 other
= vhost_vsock_get(guest_cid
);
562 if (other
&& other
!= vsock
)
565 spin_lock_bh(&vhost_vsock_lock
);
566 vsock
->guest_cid
= guest_cid
;
567 spin_unlock_bh(&vhost_vsock_lock
);
572 static int vhost_vsock_set_features(struct vhost_vsock
*vsock
, u64 features
)
574 struct vhost_virtqueue
*vq
;
577 if (features
& ~VHOST_VSOCK_FEATURES
)
580 mutex_lock(&vsock
->dev
.mutex
);
581 if ((features
& (1 << VHOST_F_LOG_ALL
)) &&
582 !vhost_log_access_ok(&vsock
->dev
)) {
583 mutex_unlock(&vsock
->dev
.mutex
);
587 for (i
= 0; i
< ARRAY_SIZE(vsock
->vqs
); i
++) {
589 mutex_lock(&vq
->mutex
);
590 vq
->acked_features
= features
;
591 mutex_unlock(&vq
->mutex
);
593 mutex_unlock(&vsock
->dev
.mutex
);
597 static long vhost_vsock_dev_ioctl(struct file
*f
, unsigned int ioctl
,
600 struct vhost_vsock
*vsock
= f
->private_data
;
601 void __user
*argp
= (void __user
*)arg
;
608 case VHOST_VSOCK_SET_GUEST_CID
:
609 if (copy_from_user(&guest_cid
, argp
, sizeof(guest_cid
)))
611 return vhost_vsock_set_cid(vsock
, guest_cid
);
612 case VHOST_VSOCK_SET_RUNNING
:
613 if (copy_from_user(&start
, argp
, sizeof(start
)))
616 return vhost_vsock_start(vsock
);
618 return vhost_vsock_stop(vsock
);
619 case VHOST_GET_FEATURES
:
620 features
= VHOST_VSOCK_FEATURES
;
621 if (copy_to_user(argp
, &features
, sizeof(features
)))
624 case VHOST_SET_FEATURES
:
625 if (copy_from_user(&features
, argp
, sizeof(features
)))
627 return vhost_vsock_set_features(vsock
, features
);
629 mutex_lock(&vsock
->dev
.mutex
);
630 r
= vhost_dev_ioctl(&vsock
->dev
, ioctl
, argp
);
631 if (r
== -ENOIOCTLCMD
)
632 r
= vhost_vring_ioctl(&vsock
->dev
, ioctl
, argp
);
634 vhost_vsock_flush(vsock
);
635 mutex_unlock(&vsock
->dev
.mutex
);
640 static const struct file_operations vhost_vsock_fops
= {
641 .owner
= THIS_MODULE
,
642 .open
= vhost_vsock_dev_open
,
643 .release
= vhost_vsock_dev_release
,
644 .llseek
= noop_llseek
,
645 .unlocked_ioctl
= vhost_vsock_dev_ioctl
,
648 static struct miscdevice vhost_vsock_misc
= {
649 .minor
= MISC_DYNAMIC_MINOR
,
650 .name
= "vhost-vsock",
651 .fops
= &vhost_vsock_fops
,
654 static struct virtio_transport vhost_transport
= {
656 .get_local_cid
= vhost_transport_get_local_cid
,
658 .init
= virtio_transport_do_socket_init
,
659 .destruct
= virtio_transport_destruct
,
660 .release
= virtio_transport_release
,
661 .connect
= virtio_transport_connect
,
662 .shutdown
= virtio_transport_shutdown
,
664 .dgram_enqueue
= virtio_transport_dgram_enqueue
,
665 .dgram_dequeue
= virtio_transport_dgram_dequeue
,
666 .dgram_bind
= virtio_transport_dgram_bind
,
667 .dgram_allow
= virtio_transport_dgram_allow
,
669 .stream_enqueue
= virtio_transport_stream_enqueue
,
670 .stream_dequeue
= virtio_transport_stream_dequeue
,
671 .stream_has_data
= virtio_transport_stream_has_data
,
672 .stream_has_space
= virtio_transport_stream_has_space
,
673 .stream_rcvhiwat
= virtio_transport_stream_rcvhiwat
,
674 .stream_is_active
= virtio_transport_stream_is_active
,
675 .stream_allow
= virtio_transport_stream_allow
,
677 .notify_poll_in
= virtio_transport_notify_poll_in
,
678 .notify_poll_out
= virtio_transport_notify_poll_out
,
679 .notify_recv_init
= virtio_transport_notify_recv_init
,
680 .notify_recv_pre_block
= virtio_transport_notify_recv_pre_block
,
681 .notify_recv_pre_dequeue
= virtio_transport_notify_recv_pre_dequeue
,
682 .notify_recv_post_dequeue
= virtio_transport_notify_recv_post_dequeue
,
683 .notify_send_init
= virtio_transport_notify_send_init
,
684 .notify_send_pre_block
= virtio_transport_notify_send_pre_block
,
685 .notify_send_pre_enqueue
= virtio_transport_notify_send_pre_enqueue
,
686 .notify_send_post_enqueue
= virtio_transport_notify_send_post_enqueue
,
688 .set_buffer_size
= virtio_transport_set_buffer_size
,
689 .set_min_buffer_size
= virtio_transport_set_min_buffer_size
,
690 .set_max_buffer_size
= virtio_transport_set_max_buffer_size
,
691 .get_buffer_size
= virtio_transport_get_buffer_size
,
692 .get_min_buffer_size
= virtio_transport_get_min_buffer_size
,
693 .get_max_buffer_size
= virtio_transport_get_max_buffer_size
,
696 .send_pkt
= vhost_transport_send_pkt
,
699 static int __init
vhost_vsock_init(void)
703 ret
= vsock_core_init(&vhost_transport
.transport
);
706 return misc_register(&vhost_vsock_misc
);
709 static void __exit
vhost_vsock_exit(void)
711 misc_deregister(&vhost_vsock_misc
);
715 module_init(vhost_vsock_init
);
716 module_exit(vhost_vsock_exit
);
717 MODULE_LICENSE("GPL v2");
718 MODULE_AUTHOR("Asias He");
719 MODULE_DESCRIPTION("vhost transport for vsock ");