2 * virtio transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s
9 * early virtio-vsock proof-of-concept bits.
11 * This work is licensed under the terms of the GNU GPL, version 2.
13 #include <linux/spinlock.h>
14 #include <linux/module.h>
15 #include <linux/list.h>
16 #include <linux/atomic.h>
17 #include <linux/virtio.h>
18 #include <linux/virtio_ids.h>
19 #include <linux/virtio_config.h>
20 #include <linux/virtio_vsock.h>
22 #include <linux/mutex.h>
23 #include <net/af_vsock.h>
25 static struct workqueue_struct
*virtio_vsock_workqueue
;
26 static struct virtio_vsock
*the_virtio_vsock
;
27 static DEFINE_MUTEX(the_virtio_vsock_mutex
); /* protects the_virtio_vsock */
30 struct virtio_device
*vdev
;
31 struct virtqueue
*vqs
[VSOCK_VQ_MAX
];
33 /* Virtqueue processing is deferred to a workqueue */
34 struct work_struct tx_work
;
35 struct work_struct rx_work
;
36 struct work_struct event_work
;
38 /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX]
39 * must be accessed with tx_lock held.
43 struct work_struct send_pkt_work
;
44 spinlock_t send_pkt_list_lock
;
45 struct list_head send_pkt_list
;
47 struct work_struct loopback_work
;
48 spinlock_t loopback_list_lock
; /* protects loopback_list */
49 struct list_head loopback_list
;
51 atomic_t queued_replies
;
53 /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX]
54 * must be accessed with rx_lock held.
60 /* The following fields are protected by event_lock.
61 * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held.
63 struct mutex event_lock
;
64 struct virtio_vsock_event event_list
[8];
69 static struct virtio_vsock
*virtio_vsock_get(void)
71 return the_virtio_vsock
;
74 static u32
virtio_transport_get_local_cid(void)
76 struct virtio_vsock
*vsock
= virtio_vsock_get();
78 return vsock
->guest_cid
;
81 static void virtio_transport_loopback_work(struct work_struct
*work
)
83 struct virtio_vsock
*vsock
=
84 container_of(work
, struct virtio_vsock
, loopback_work
);
87 spin_lock_bh(&vsock
->loopback_list_lock
);
88 list_splice_init(&vsock
->loopback_list
, &pkts
);
89 spin_unlock_bh(&vsock
->loopback_list_lock
);
91 mutex_lock(&vsock
->rx_lock
);
92 while (!list_empty(&pkts
)) {
93 struct virtio_vsock_pkt
*pkt
;
95 pkt
= list_first_entry(&pkts
, struct virtio_vsock_pkt
, list
);
96 list_del_init(&pkt
->list
);
98 virtio_transport_recv_pkt(pkt
);
100 mutex_unlock(&vsock
->rx_lock
);
103 static int virtio_transport_send_pkt_loopback(struct virtio_vsock
*vsock
,
104 struct virtio_vsock_pkt
*pkt
)
108 spin_lock_bh(&vsock
->loopback_list_lock
);
109 list_add_tail(&pkt
->list
, &vsock
->loopback_list
);
110 spin_unlock_bh(&vsock
->loopback_list_lock
);
112 queue_work(virtio_vsock_workqueue
, &vsock
->loopback_work
);
118 virtio_transport_send_pkt_work(struct work_struct
*work
)
120 struct virtio_vsock
*vsock
=
121 container_of(work
, struct virtio_vsock
, send_pkt_work
);
122 struct virtqueue
*vq
;
124 bool restart_rx
= false;
126 mutex_lock(&vsock
->tx_lock
);
128 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
131 struct virtio_vsock_pkt
*pkt
;
132 struct scatterlist hdr
, buf
, *sgs
[2];
133 int ret
, in_sg
= 0, out_sg
= 0;
136 spin_lock_bh(&vsock
->send_pkt_list_lock
);
137 if (list_empty(&vsock
->send_pkt_list
)) {
138 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
142 pkt
= list_first_entry(&vsock
->send_pkt_list
,
143 struct virtio_vsock_pkt
, list
);
144 list_del_init(&pkt
->list
);
145 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
147 virtio_transport_deliver_tap_pkt(pkt
);
151 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
152 sgs
[out_sg
++] = &hdr
;
154 sg_init_one(&buf
, pkt
->buf
, pkt
->len
);
155 sgs
[out_sg
++] = &buf
;
158 ret
= virtqueue_add_sgs(vq
, sgs
, out_sg
, in_sg
, pkt
, GFP_KERNEL
);
159 /* Usually this means that there is no more space available in
163 spin_lock_bh(&vsock
->send_pkt_list_lock
);
164 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
165 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
170 struct virtqueue
*rx_vq
= vsock
->vqs
[VSOCK_VQ_RX
];
173 val
= atomic_dec_return(&vsock
->queued_replies
);
175 /* Do we now have resources to resume rx processing? */
176 if (val
+ 1 == virtqueue_get_vring_size(rx_vq
))
186 mutex_unlock(&vsock
->tx_lock
);
189 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
193 virtio_transport_send_pkt(struct virtio_vsock_pkt
*pkt
)
195 struct virtio_vsock
*vsock
;
198 vsock
= virtio_vsock_get();
200 virtio_transport_free_pkt(pkt
);
204 if (le64_to_cpu(pkt
->hdr
.dst_cid
) == vsock
->guest_cid
)
205 return virtio_transport_send_pkt_loopback(vsock
, pkt
);
208 atomic_inc(&vsock
->queued_replies
);
210 spin_lock_bh(&vsock
->send_pkt_list_lock
);
211 list_add_tail(&pkt
->list
, &vsock
->send_pkt_list
);
212 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
214 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
219 virtio_transport_cancel_pkt(struct vsock_sock
*vsk
)
221 struct virtio_vsock
*vsock
;
222 struct virtio_vsock_pkt
*pkt
, *n
;
226 vsock
= virtio_vsock_get();
231 spin_lock_bh(&vsock
->send_pkt_list_lock
);
232 list_for_each_entry_safe(pkt
, n
, &vsock
->send_pkt_list
, list
) {
235 list_move(&pkt
->list
, &freeme
);
237 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
239 list_for_each_entry_safe(pkt
, n
, &freeme
, list
) {
242 list_del(&pkt
->list
);
243 virtio_transport_free_pkt(pkt
);
247 struct virtqueue
*rx_vq
= vsock
->vqs
[VSOCK_VQ_RX
];
250 new_cnt
= atomic_sub_return(cnt
, &vsock
->queued_replies
);
251 if (new_cnt
+ cnt
>= virtqueue_get_vring_size(rx_vq
) &&
252 new_cnt
< virtqueue_get_vring_size(rx_vq
))
253 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
259 static void virtio_vsock_rx_fill(struct virtio_vsock
*vsock
)
261 int buf_len
= VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE
;
262 struct virtio_vsock_pkt
*pkt
;
263 struct scatterlist hdr
, buf
, *sgs
[2];
264 struct virtqueue
*vq
;
267 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
270 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
274 pkt
->buf
= kmalloc(buf_len
, GFP_KERNEL
);
276 virtio_transport_free_pkt(pkt
);
282 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
285 sg_init_one(&buf
, pkt
->buf
, buf_len
);
287 ret
= virtqueue_add_sgs(vq
, sgs
, 0, 2, pkt
, GFP_KERNEL
);
289 virtio_transport_free_pkt(pkt
);
293 } while (vq
->num_free
);
294 if (vsock
->rx_buf_nr
> vsock
->rx_buf_max_nr
)
295 vsock
->rx_buf_max_nr
= vsock
->rx_buf_nr
;
299 static void virtio_transport_tx_work(struct work_struct
*work
)
301 struct virtio_vsock
*vsock
=
302 container_of(work
, struct virtio_vsock
, tx_work
);
303 struct virtqueue
*vq
;
306 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
307 mutex_lock(&vsock
->tx_lock
);
309 struct virtio_vsock_pkt
*pkt
;
312 virtqueue_disable_cb(vq
);
313 while ((pkt
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
314 virtio_transport_free_pkt(pkt
);
317 } while (!virtqueue_enable_cb(vq
));
318 mutex_unlock(&vsock
->tx_lock
);
321 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
324 /* Is there space left for replies to rx packets? */
325 static bool virtio_transport_more_replies(struct virtio_vsock
*vsock
)
327 struct virtqueue
*vq
= vsock
->vqs
[VSOCK_VQ_RX
];
330 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
331 val
= atomic_read(&vsock
->queued_replies
);
333 return val
< virtqueue_get_vring_size(vq
);
336 static void virtio_transport_rx_work(struct work_struct
*work
)
338 struct virtio_vsock
*vsock
=
339 container_of(work
, struct virtio_vsock
, rx_work
);
340 struct virtqueue
*vq
;
342 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
344 mutex_lock(&vsock
->rx_lock
);
347 virtqueue_disable_cb(vq
);
349 struct virtio_vsock_pkt
*pkt
;
352 if (!virtio_transport_more_replies(vsock
)) {
353 /* Stop rx until the device processes already
354 * pending replies. Leave rx virtqueue
355 * callbacks disabled.
360 pkt
= virtqueue_get_buf(vq
, &len
);
367 /* Drop short/long packets */
368 if (unlikely(len
< sizeof(pkt
->hdr
) ||
369 len
> sizeof(pkt
->hdr
) + pkt
->len
)) {
370 virtio_transport_free_pkt(pkt
);
374 pkt
->len
= len
- sizeof(pkt
->hdr
);
375 virtio_transport_deliver_tap_pkt(pkt
);
376 virtio_transport_recv_pkt(pkt
);
378 } while (!virtqueue_enable_cb(vq
));
381 if (vsock
->rx_buf_nr
< vsock
->rx_buf_max_nr
/ 2)
382 virtio_vsock_rx_fill(vsock
);
383 mutex_unlock(&vsock
->rx_lock
);
386 /* event_lock must be held */
387 static int virtio_vsock_event_fill_one(struct virtio_vsock
*vsock
,
388 struct virtio_vsock_event
*event
)
390 struct scatterlist sg
;
391 struct virtqueue
*vq
;
393 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
395 sg_init_one(&sg
, event
, sizeof(*event
));
397 return virtqueue_add_inbuf(vq
, &sg
, 1, event
, GFP_KERNEL
);
400 /* event_lock must be held */
401 static void virtio_vsock_event_fill(struct virtio_vsock
*vsock
)
405 for (i
= 0; i
< ARRAY_SIZE(vsock
->event_list
); i
++) {
406 struct virtio_vsock_event
*event
= &vsock
->event_list
[i
];
408 virtio_vsock_event_fill_one(vsock
, event
);
411 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
414 static void virtio_vsock_reset_sock(struct sock
*sk
)
417 sk
->sk_state
= TCP_CLOSE
;
418 sk
->sk_err
= ECONNRESET
;
419 sk
->sk_error_report(sk
);
423 static void virtio_vsock_update_guest_cid(struct virtio_vsock
*vsock
)
425 struct virtio_device
*vdev
= vsock
->vdev
;
428 vdev
->config
->get(vdev
, offsetof(struct virtio_vsock_config
, guest_cid
),
429 &guest_cid
, sizeof(guest_cid
));
430 vsock
->guest_cid
= le64_to_cpu(guest_cid
);
433 /* event_lock must be held */
434 static void virtio_vsock_event_handle(struct virtio_vsock
*vsock
,
435 struct virtio_vsock_event
*event
)
437 switch (le32_to_cpu(event
->id
)) {
438 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET
:
439 virtio_vsock_update_guest_cid(vsock
);
440 vsock_for_each_connected_socket(virtio_vsock_reset_sock
);
445 static void virtio_transport_event_work(struct work_struct
*work
)
447 struct virtio_vsock
*vsock
=
448 container_of(work
, struct virtio_vsock
, event_work
);
449 struct virtqueue
*vq
;
451 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
453 mutex_lock(&vsock
->event_lock
);
456 struct virtio_vsock_event
*event
;
459 virtqueue_disable_cb(vq
);
460 while ((event
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
461 if (len
== sizeof(*event
))
462 virtio_vsock_event_handle(vsock
, event
);
464 virtio_vsock_event_fill_one(vsock
, event
);
466 } while (!virtqueue_enable_cb(vq
));
468 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
470 mutex_unlock(&vsock
->event_lock
);
473 static void virtio_vsock_event_done(struct virtqueue
*vq
)
475 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
479 queue_work(virtio_vsock_workqueue
, &vsock
->event_work
);
482 static void virtio_vsock_tx_done(struct virtqueue
*vq
)
484 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
488 queue_work(virtio_vsock_workqueue
, &vsock
->tx_work
);
491 static void virtio_vsock_rx_done(struct virtqueue
*vq
)
493 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
497 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
500 static struct virtio_transport virtio_transport
= {
502 .get_local_cid
= virtio_transport_get_local_cid
,
504 .init
= virtio_transport_do_socket_init
,
505 .destruct
= virtio_transport_destruct
,
506 .release
= virtio_transport_release
,
507 .connect
= virtio_transport_connect
,
508 .shutdown
= virtio_transport_shutdown
,
509 .cancel_pkt
= virtio_transport_cancel_pkt
,
511 .dgram_bind
= virtio_transport_dgram_bind
,
512 .dgram_dequeue
= virtio_transport_dgram_dequeue
,
513 .dgram_enqueue
= virtio_transport_dgram_enqueue
,
514 .dgram_allow
= virtio_transport_dgram_allow
,
516 .stream_dequeue
= virtio_transport_stream_dequeue
,
517 .stream_enqueue
= virtio_transport_stream_enqueue
,
518 .stream_has_data
= virtio_transport_stream_has_data
,
519 .stream_has_space
= virtio_transport_stream_has_space
,
520 .stream_rcvhiwat
= virtio_transport_stream_rcvhiwat
,
521 .stream_is_active
= virtio_transport_stream_is_active
,
522 .stream_allow
= virtio_transport_stream_allow
,
524 .notify_poll_in
= virtio_transport_notify_poll_in
,
525 .notify_poll_out
= virtio_transport_notify_poll_out
,
526 .notify_recv_init
= virtio_transport_notify_recv_init
,
527 .notify_recv_pre_block
= virtio_transport_notify_recv_pre_block
,
528 .notify_recv_pre_dequeue
= virtio_transport_notify_recv_pre_dequeue
,
529 .notify_recv_post_dequeue
= virtio_transport_notify_recv_post_dequeue
,
530 .notify_send_init
= virtio_transport_notify_send_init
,
531 .notify_send_pre_block
= virtio_transport_notify_send_pre_block
,
532 .notify_send_pre_enqueue
= virtio_transport_notify_send_pre_enqueue
,
533 .notify_send_post_enqueue
= virtio_transport_notify_send_post_enqueue
,
535 .set_buffer_size
= virtio_transport_set_buffer_size
,
536 .set_min_buffer_size
= virtio_transport_set_min_buffer_size
,
537 .set_max_buffer_size
= virtio_transport_set_max_buffer_size
,
538 .get_buffer_size
= virtio_transport_get_buffer_size
,
539 .get_min_buffer_size
= virtio_transport_get_min_buffer_size
,
540 .get_max_buffer_size
= virtio_transport_get_max_buffer_size
,
543 .send_pkt
= virtio_transport_send_pkt
,
546 static int virtio_vsock_probe(struct virtio_device
*vdev
)
548 vq_callback_t
*callbacks
[] = {
549 virtio_vsock_rx_done
,
550 virtio_vsock_tx_done
,
551 virtio_vsock_event_done
,
553 static const char * const names
[] = {
558 struct virtio_vsock
*vsock
= NULL
;
561 ret
= mutex_lock_interruptible(&the_virtio_vsock_mutex
);
565 /* Only one virtio-vsock device per guest is supported */
566 if (the_virtio_vsock
) {
571 vsock
= kzalloc(sizeof(*vsock
), GFP_KERNEL
);
579 ret
= virtio_find_vqs(vsock
->vdev
, VSOCK_VQ_MAX
,
580 vsock
->vqs
, callbacks
, names
,
585 virtio_vsock_update_guest_cid(vsock
);
587 ret
= vsock_core_init(&virtio_transport
.transport
);
591 vsock
->rx_buf_nr
= 0;
592 vsock
->rx_buf_max_nr
= 0;
593 atomic_set(&vsock
->queued_replies
, 0);
596 the_virtio_vsock
= vsock
;
597 mutex_init(&vsock
->tx_lock
);
598 mutex_init(&vsock
->rx_lock
);
599 mutex_init(&vsock
->event_lock
);
600 spin_lock_init(&vsock
->send_pkt_list_lock
);
601 INIT_LIST_HEAD(&vsock
->send_pkt_list
);
602 spin_lock_init(&vsock
->loopback_list_lock
);
603 INIT_LIST_HEAD(&vsock
->loopback_list
);
604 INIT_WORK(&vsock
->rx_work
, virtio_transport_rx_work
);
605 INIT_WORK(&vsock
->tx_work
, virtio_transport_tx_work
);
606 INIT_WORK(&vsock
->event_work
, virtio_transport_event_work
);
607 INIT_WORK(&vsock
->send_pkt_work
, virtio_transport_send_pkt_work
);
608 INIT_WORK(&vsock
->loopback_work
, virtio_transport_loopback_work
);
610 mutex_lock(&vsock
->rx_lock
);
611 virtio_vsock_rx_fill(vsock
);
612 mutex_unlock(&vsock
->rx_lock
);
614 mutex_lock(&vsock
->event_lock
);
615 virtio_vsock_event_fill(vsock
);
616 mutex_unlock(&vsock
->event_lock
);
618 mutex_unlock(&the_virtio_vsock_mutex
);
622 vsock
->vdev
->config
->del_vqs(vsock
->vdev
);
625 mutex_unlock(&the_virtio_vsock_mutex
);
629 static void virtio_vsock_remove(struct virtio_device
*vdev
)
631 struct virtio_vsock
*vsock
= vdev
->priv
;
632 struct virtio_vsock_pkt
*pkt
;
634 flush_work(&vsock
->loopback_work
);
635 flush_work(&vsock
->rx_work
);
636 flush_work(&vsock
->tx_work
);
637 flush_work(&vsock
->event_work
);
638 flush_work(&vsock
->send_pkt_work
);
640 vdev
->config
->reset(vdev
);
642 mutex_lock(&vsock
->rx_lock
);
643 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_RX
])))
644 virtio_transport_free_pkt(pkt
);
645 mutex_unlock(&vsock
->rx_lock
);
647 mutex_lock(&vsock
->tx_lock
);
648 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_TX
])))
649 virtio_transport_free_pkt(pkt
);
650 mutex_unlock(&vsock
->tx_lock
);
652 spin_lock_bh(&vsock
->send_pkt_list_lock
);
653 while (!list_empty(&vsock
->send_pkt_list
)) {
654 pkt
= list_first_entry(&vsock
->send_pkt_list
,
655 struct virtio_vsock_pkt
, list
);
656 list_del(&pkt
->list
);
657 virtio_transport_free_pkt(pkt
);
659 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
661 spin_lock_bh(&vsock
->loopback_list_lock
);
662 while (!list_empty(&vsock
->loopback_list
)) {
663 pkt
= list_first_entry(&vsock
->loopback_list
,
664 struct virtio_vsock_pkt
, list
);
665 list_del(&pkt
->list
);
666 virtio_transport_free_pkt(pkt
);
668 spin_unlock_bh(&vsock
->loopback_list_lock
);
670 mutex_lock(&the_virtio_vsock_mutex
);
671 the_virtio_vsock
= NULL
;
673 mutex_unlock(&the_virtio_vsock_mutex
);
675 vdev
->config
->del_vqs(vdev
);
680 static struct virtio_device_id id_table
[] = {
681 { VIRTIO_ID_VSOCK
, VIRTIO_DEV_ANY_ID
},
685 static unsigned int features
[] = {
688 static struct virtio_driver virtio_vsock_driver
= {
689 .feature_table
= features
,
690 .feature_table_size
= ARRAY_SIZE(features
),
691 .driver
.name
= KBUILD_MODNAME
,
692 .driver
.owner
= THIS_MODULE
,
693 .id_table
= id_table
,
694 .probe
= virtio_vsock_probe
,
695 .remove
= virtio_vsock_remove
,
698 static int __init
virtio_vsock_init(void)
702 virtio_vsock_workqueue
= alloc_workqueue("virtio_vsock", 0, 0);
703 if (!virtio_vsock_workqueue
)
705 ret
= register_virtio_driver(&virtio_vsock_driver
);
707 destroy_workqueue(virtio_vsock_workqueue
);
711 static void __exit
virtio_vsock_exit(void)
713 unregister_virtio_driver(&virtio_vsock_driver
);
714 destroy_workqueue(virtio_vsock_workqueue
);
717 module_init(virtio_vsock_init
);
718 module_exit(virtio_vsock_exit
);
719 MODULE_LICENSE("GPL v2");
720 MODULE_AUTHOR("Asias He");
721 MODULE_DESCRIPTION("virtio transport for vsock");
722 MODULE_DEVICE_TABLE(virtio
, id_table
);