2 * virtio transport for vsock
4 * Copyright (C) 2013-2015 Red Hat, Inc.
5 * Author: Asias He <asias@redhat.com>
6 * Stefan Hajnoczi <stefanha@redhat.com>
8 * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s
9 * early virtio-vsock proof-of-concept bits.
11 * This work is licensed under the terms of the GNU GPL, version 2.
13 #include <linux/spinlock.h>
14 #include <linux/module.h>
15 #include <linux/list.h>
16 #include <linux/atomic.h>
17 #include <linux/virtio.h>
18 #include <linux/virtio_ids.h>
19 #include <linux/virtio_config.h>
20 #include <linux/virtio_vsock.h>
22 #include <linux/mutex.h>
23 #include <net/af_vsock.h>
25 static struct workqueue_struct
*virtio_vsock_workqueue
;
26 static struct virtio_vsock
*the_virtio_vsock
;
27 static DEFINE_MUTEX(the_virtio_vsock_mutex
); /* protects the_virtio_vsock */
30 struct virtio_device
*vdev
;
31 struct virtqueue
*vqs
[VSOCK_VQ_MAX
];
33 /* Virtqueue processing is deferred to a workqueue */
34 struct work_struct tx_work
;
35 struct work_struct rx_work
;
36 struct work_struct event_work
;
38 /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX]
39 * must be accessed with tx_lock held.
43 struct work_struct send_pkt_work
;
44 spinlock_t send_pkt_list_lock
;
45 struct list_head send_pkt_list
;
47 struct work_struct loopback_work
;
48 spinlock_t loopback_list_lock
; /* protects loopback_list */
49 struct list_head loopback_list
;
51 atomic_t queued_replies
;
53 /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX]
54 * must be accessed with rx_lock held.
60 /* The following fields are protected by event_lock.
61 * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held.
63 struct mutex event_lock
;
64 struct virtio_vsock_event event_list
[8];
69 static struct virtio_vsock
*virtio_vsock_get(void)
71 return the_virtio_vsock
;
74 static u32
virtio_transport_get_local_cid(void)
76 struct virtio_vsock
*vsock
= virtio_vsock_get();
78 return vsock
->guest_cid
;
81 static void virtio_transport_loopback_work(struct work_struct
*work
)
83 struct virtio_vsock
*vsock
=
84 container_of(work
, struct virtio_vsock
, loopback_work
);
87 spin_lock_bh(&vsock
->loopback_list_lock
);
88 list_splice_init(&vsock
->loopback_list
, &pkts
);
89 spin_unlock_bh(&vsock
->loopback_list_lock
);
91 mutex_lock(&vsock
->rx_lock
);
92 while (!list_empty(&pkts
)) {
93 struct virtio_vsock_pkt
*pkt
;
95 pkt
= list_first_entry(&pkts
, struct virtio_vsock_pkt
, list
);
96 list_del_init(&pkt
->list
);
98 virtio_transport_recv_pkt(pkt
);
100 mutex_unlock(&vsock
->rx_lock
);
103 static int virtio_transport_send_pkt_loopback(struct virtio_vsock
*vsock
,
104 struct virtio_vsock_pkt
*pkt
)
108 spin_lock_bh(&vsock
->loopback_list_lock
);
109 list_add_tail(&pkt
->list
, &vsock
->loopback_list
);
110 spin_unlock_bh(&vsock
->loopback_list_lock
);
112 queue_work(virtio_vsock_workqueue
, &vsock
->loopback_work
);
118 virtio_transport_send_pkt_work(struct work_struct
*work
)
120 struct virtio_vsock
*vsock
=
121 container_of(work
, struct virtio_vsock
, send_pkt_work
);
122 struct virtqueue
*vq
;
124 bool restart_rx
= false;
126 mutex_lock(&vsock
->tx_lock
);
128 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
131 struct virtio_vsock_pkt
*pkt
;
132 struct scatterlist hdr
, buf
, *sgs
[2];
133 int ret
, in_sg
= 0, out_sg
= 0;
136 spin_lock_bh(&vsock
->send_pkt_list_lock
);
137 if (list_empty(&vsock
->send_pkt_list
)) {
138 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
142 pkt
= list_first_entry(&vsock
->send_pkt_list
,
143 struct virtio_vsock_pkt
, list
);
144 list_del_init(&pkt
->list
);
145 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
149 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
150 sgs
[out_sg
++] = &hdr
;
152 sg_init_one(&buf
, pkt
->buf
, pkt
->len
);
153 sgs
[out_sg
++] = &buf
;
156 ret
= virtqueue_add_sgs(vq
, sgs
, out_sg
, in_sg
, pkt
, GFP_KERNEL
);
157 /* Usually this means that there is no more space available in
161 spin_lock_bh(&vsock
->send_pkt_list_lock
);
162 list_add(&pkt
->list
, &vsock
->send_pkt_list
);
163 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
168 struct virtqueue
*rx_vq
= vsock
->vqs
[VSOCK_VQ_RX
];
171 val
= atomic_dec_return(&vsock
->queued_replies
);
173 /* Do we now have resources to resume rx processing? */
174 if (val
+ 1 == virtqueue_get_vring_size(rx_vq
))
184 mutex_unlock(&vsock
->tx_lock
);
187 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
191 virtio_transport_send_pkt(struct virtio_vsock_pkt
*pkt
)
193 struct virtio_vsock
*vsock
;
196 vsock
= virtio_vsock_get();
198 virtio_transport_free_pkt(pkt
);
202 if (le32_to_cpu(pkt
->hdr
.dst_cid
) == vsock
->guest_cid
)
203 return virtio_transport_send_pkt_loopback(vsock
, pkt
);
206 atomic_inc(&vsock
->queued_replies
);
208 spin_lock_bh(&vsock
->send_pkt_list_lock
);
209 list_add_tail(&pkt
->list
, &vsock
->send_pkt_list
);
210 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
212 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
216 static void virtio_vsock_rx_fill(struct virtio_vsock
*vsock
)
218 int buf_len
= VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE
;
219 struct virtio_vsock_pkt
*pkt
;
220 struct scatterlist hdr
, buf
, *sgs
[2];
221 struct virtqueue
*vq
;
224 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
227 pkt
= kzalloc(sizeof(*pkt
), GFP_KERNEL
);
231 pkt
->buf
= kmalloc(buf_len
, GFP_KERNEL
);
233 virtio_transport_free_pkt(pkt
);
239 sg_init_one(&hdr
, &pkt
->hdr
, sizeof(pkt
->hdr
));
242 sg_init_one(&buf
, pkt
->buf
, buf_len
);
244 ret
= virtqueue_add_sgs(vq
, sgs
, 0, 2, pkt
, GFP_KERNEL
);
246 virtio_transport_free_pkt(pkt
);
250 } while (vq
->num_free
);
251 if (vsock
->rx_buf_nr
> vsock
->rx_buf_max_nr
)
252 vsock
->rx_buf_max_nr
= vsock
->rx_buf_nr
;
256 static void virtio_transport_tx_work(struct work_struct
*work
)
258 struct virtio_vsock
*vsock
=
259 container_of(work
, struct virtio_vsock
, tx_work
);
260 struct virtqueue
*vq
;
263 vq
= vsock
->vqs
[VSOCK_VQ_TX
];
264 mutex_lock(&vsock
->tx_lock
);
266 struct virtio_vsock_pkt
*pkt
;
269 virtqueue_disable_cb(vq
);
270 while ((pkt
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
271 virtio_transport_free_pkt(pkt
);
274 } while (!virtqueue_enable_cb(vq
));
275 mutex_unlock(&vsock
->tx_lock
);
278 queue_work(virtio_vsock_workqueue
, &vsock
->send_pkt_work
);
281 /* Is there space left for replies to rx packets? */
282 static bool virtio_transport_more_replies(struct virtio_vsock
*vsock
)
284 struct virtqueue
*vq
= vsock
->vqs
[VSOCK_VQ_RX
];
287 smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */
288 val
= atomic_read(&vsock
->queued_replies
);
290 return val
< virtqueue_get_vring_size(vq
);
293 static void virtio_transport_rx_work(struct work_struct
*work
)
295 struct virtio_vsock
*vsock
=
296 container_of(work
, struct virtio_vsock
, rx_work
);
297 struct virtqueue
*vq
;
299 vq
= vsock
->vqs
[VSOCK_VQ_RX
];
301 mutex_lock(&vsock
->rx_lock
);
304 virtqueue_disable_cb(vq
);
306 struct virtio_vsock_pkt
*pkt
;
309 if (!virtio_transport_more_replies(vsock
)) {
310 /* Stop rx until the device processes already
311 * pending replies. Leave rx virtqueue
312 * callbacks disabled.
317 pkt
= virtqueue_get_buf(vq
, &len
);
324 /* Drop short/long packets */
325 if (unlikely(len
< sizeof(pkt
->hdr
) ||
326 len
> sizeof(pkt
->hdr
) + pkt
->len
)) {
327 virtio_transport_free_pkt(pkt
);
331 pkt
->len
= len
- sizeof(pkt
->hdr
);
332 virtio_transport_recv_pkt(pkt
);
334 } while (!virtqueue_enable_cb(vq
));
337 if (vsock
->rx_buf_nr
< vsock
->rx_buf_max_nr
/ 2)
338 virtio_vsock_rx_fill(vsock
);
339 mutex_unlock(&vsock
->rx_lock
);
342 /* event_lock must be held */
343 static int virtio_vsock_event_fill_one(struct virtio_vsock
*vsock
,
344 struct virtio_vsock_event
*event
)
346 struct scatterlist sg
;
347 struct virtqueue
*vq
;
349 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
351 sg_init_one(&sg
, event
, sizeof(*event
));
353 return virtqueue_add_inbuf(vq
, &sg
, 1, event
, GFP_KERNEL
);
356 /* event_lock must be held */
357 static void virtio_vsock_event_fill(struct virtio_vsock
*vsock
)
361 for (i
= 0; i
< ARRAY_SIZE(vsock
->event_list
); i
++) {
362 struct virtio_vsock_event
*event
= &vsock
->event_list
[i
];
364 virtio_vsock_event_fill_one(vsock
, event
);
367 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
370 static void virtio_vsock_reset_sock(struct sock
*sk
)
373 sk
->sk_state
= SS_UNCONNECTED
;
374 sk
->sk_err
= ECONNRESET
;
375 sk
->sk_error_report(sk
);
379 static void virtio_vsock_update_guest_cid(struct virtio_vsock
*vsock
)
381 struct virtio_device
*vdev
= vsock
->vdev
;
384 vdev
->config
->get(vdev
, offsetof(struct virtio_vsock_config
, guest_cid
),
385 &guest_cid
, sizeof(guest_cid
));
386 vsock
->guest_cid
= le64_to_cpu(guest_cid
);
389 /* event_lock must be held */
390 static void virtio_vsock_event_handle(struct virtio_vsock
*vsock
,
391 struct virtio_vsock_event
*event
)
393 switch (le32_to_cpu(event
->id
)) {
394 case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET
:
395 virtio_vsock_update_guest_cid(vsock
);
396 vsock_for_each_connected_socket(virtio_vsock_reset_sock
);
401 static void virtio_transport_event_work(struct work_struct
*work
)
403 struct virtio_vsock
*vsock
=
404 container_of(work
, struct virtio_vsock
, event_work
);
405 struct virtqueue
*vq
;
407 vq
= vsock
->vqs
[VSOCK_VQ_EVENT
];
409 mutex_lock(&vsock
->event_lock
);
412 struct virtio_vsock_event
*event
;
415 virtqueue_disable_cb(vq
);
416 while ((event
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
417 if (len
== sizeof(*event
))
418 virtio_vsock_event_handle(vsock
, event
);
420 virtio_vsock_event_fill_one(vsock
, event
);
422 } while (!virtqueue_enable_cb(vq
));
424 virtqueue_kick(vsock
->vqs
[VSOCK_VQ_EVENT
]);
426 mutex_unlock(&vsock
->event_lock
);
429 static void virtio_vsock_event_done(struct virtqueue
*vq
)
431 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
435 queue_work(virtio_vsock_workqueue
, &vsock
->event_work
);
438 static void virtio_vsock_tx_done(struct virtqueue
*vq
)
440 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
444 queue_work(virtio_vsock_workqueue
, &vsock
->tx_work
);
447 static void virtio_vsock_rx_done(struct virtqueue
*vq
)
449 struct virtio_vsock
*vsock
= vq
->vdev
->priv
;
453 queue_work(virtio_vsock_workqueue
, &vsock
->rx_work
);
456 static struct virtio_transport virtio_transport
= {
458 .get_local_cid
= virtio_transport_get_local_cid
,
460 .init
= virtio_transport_do_socket_init
,
461 .destruct
= virtio_transport_destruct
,
462 .release
= virtio_transport_release
,
463 .connect
= virtio_transport_connect
,
464 .shutdown
= virtio_transport_shutdown
,
466 .dgram_bind
= virtio_transport_dgram_bind
,
467 .dgram_dequeue
= virtio_transport_dgram_dequeue
,
468 .dgram_enqueue
= virtio_transport_dgram_enqueue
,
469 .dgram_allow
= virtio_transport_dgram_allow
,
471 .stream_dequeue
= virtio_transport_stream_dequeue
,
472 .stream_enqueue
= virtio_transport_stream_enqueue
,
473 .stream_has_data
= virtio_transport_stream_has_data
,
474 .stream_has_space
= virtio_transport_stream_has_space
,
475 .stream_rcvhiwat
= virtio_transport_stream_rcvhiwat
,
476 .stream_is_active
= virtio_transport_stream_is_active
,
477 .stream_allow
= virtio_transport_stream_allow
,
479 .notify_poll_in
= virtio_transport_notify_poll_in
,
480 .notify_poll_out
= virtio_transport_notify_poll_out
,
481 .notify_recv_init
= virtio_transport_notify_recv_init
,
482 .notify_recv_pre_block
= virtio_transport_notify_recv_pre_block
,
483 .notify_recv_pre_dequeue
= virtio_transport_notify_recv_pre_dequeue
,
484 .notify_recv_post_dequeue
= virtio_transport_notify_recv_post_dequeue
,
485 .notify_send_init
= virtio_transport_notify_send_init
,
486 .notify_send_pre_block
= virtio_transport_notify_send_pre_block
,
487 .notify_send_pre_enqueue
= virtio_transport_notify_send_pre_enqueue
,
488 .notify_send_post_enqueue
= virtio_transport_notify_send_post_enqueue
,
490 .set_buffer_size
= virtio_transport_set_buffer_size
,
491 .set_min_buffer_size
= virtio_transport_set_min_buffer_size
,
492 .set_max_buffer_size
= virtio_transport_set_max_buffer_size
,
493 .get_buffer_size
= virtio_transport_get_buffer_size
,
494 .get_min_buffer_size
= virtio_transport_get_min_buffer_size
,
495 .get_max_buffer_size
= virtio_transport_get_max_buffer_size
,
498 .send_pkt
= virtio_transport_send_pkt
,
501 static int virtio_vsock_probe(struct virtio_device
*vdev
)
503 vq_callback_t
*callbacks
[] = {
504 virtio_vsock_rx_done
,
505 virtio_vsock_tx_done
,
506 virtio_vsock_event_done
,
508 static const char * const names
[] = {
513 struct virtio_vsock
*vsock
= NULL
;
516 ret
= mutex_lock_interruptible(&the_virtio_vsock_mutex
);
520 /* Only one virtio-vsock device per guest is supported */
521 if (the_virtio_vsock
) {
526 vsock
= kzalloc(sizeof(*vsock
), GFP_KERNEL
);
534 ret
= vsock
->vdev
->config
->find_vqs(vsock
->vdev
, VSOCK_VQ_MAX
,
535 vsock
->vqs
, callbacks
, names
);
539 virtio_vsock_update_guest_cid(vsock
);
541 ret
= vsock_core_init(&virtio_transport
.transport
);
545 vsock
->rx_buf_nr
= 0;
546 vsock
->rx_buf_max_nr
= 0;
547 atomic_set(&vsock
->queued_replies
, 0);
550 the_virtio_vsock
= vsock
;
551 mutex_init(&vsock
->tx_lock
);
552 mutex_init(&vsock
->rx_lock
);
553 mutex_init(&vsock
->event_lock
);
554 spin_lock_init(&vsock
->send_pkt_list_lock
);
555 INIT_LIST_HEAD(&vsock
->send_pkt_list
);
556 spin_lock_init(&vsock
->loopback_list_lock
);
557 INIT_LIST_HEAD(&vsock
->loopback_list
);
558 INIT_WORK(&vsock
->rx_work
, virtio_transport_rx_work
);
559 INIT_WORK(&vsock
->tx_work
, virtio_transport_tx_work
);
560 INIT_WORK(&vsock
->event_work
, virtio_transport_event_work
);
561 INIT_WORK(&vsock
->send_pkt_work
, virtio_transport_send_pkt_work
);
562 INIT_WORK(&vsock
->loopback_work
, virtio_transport_loopback_work
);
564 mutex_lock(&vsock
->rx_lock
);
565 virtio_vsock_rx_fill(vsock
);
566 mutex_unlock(&vsock
->rx_lock
);
568 mutex_lock(&vsock
->event_lock
);
569 virtio_vsock_event_fill(vsock
);
570 mutex_unlock(&vsock
->event_lock
);
572 mutex_unlock(&the_virtio_vsock_mutex
);
576 vsock
->vdev
->config
->del_vqs(vsock
->vdev
);
579 mutex_unlock(&the_virtio_vsock_mutex
);
583 static void virtio_vsock_remove(struct virtio_device
*vdev
)
585 struct virtio_vsock
*vsock
= vdev
->priv
;
586 struct virtio_vsock_pkt
*pkt
;
588 flush_work(&vsock
->loopback_work
);
589 flush_work(&vsock
->rx_work
);
590 flush_work(&vsock
->tx_work
);
591 flush_work(&vsock
->event_work
);
592 flush_work(&vsock
->send_pkt_work
);
594 vdev
->config
->reset(vdev
);
596 mutex_lock(&vsock
->rx_lock
);
597 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_RX
])))
598 virtio_transport_free_pkt(pkt
);
599 mutex_unlock(&vsock
->rx_lock
);
601 mutex_lock(&vsock
->tx_lock
);
602 while ((pkt
= virtqueue_detach_unused_buf(vsock
->vqs
[VSOCK_VQ_TX
])))
603 virtio_transport_free_pkt(pkt
);
604 mutex_unlock(&vsock
->tx_lock
);
606 spin_lock_bh(&vsock
->send_pkt_list_lock
);
607 while (!list_empty(&vsock
->send_pkt_list
)) {
608 pkt
= list_first_entry(&vsock
->send_pkt_list
,
609 struct virtio_vsock_pkt
, list
);
610 list_del(&pkt
->list
);
611 virtio_transport_free_pkt(pkt
);
613 spin_unlock_bh(&vsock
->send_pkt_list_lock
);
615 spin_lock_bh(&vsock
->loopback_list_lock
);
616 while (!list_empty(&vsock
->loopback_list
)) {
617 pkt
= list_first_entry(&vsock
->loopback_list
,
618 struct virtio_vsock_pkt
, list
);
619 list_del(&pkt
->list
);
620 virtio_transport_free_pkt(pkt
);
622 spin_unlock_bh(&vsock
->loopback_list_lock
);
624 mutex_lock(&the_virtio_vsock_mutex
);
625 the_virtio_vsock
= NULL
;
627 mutex_unlock(&the_virtio_vsock_mutex
);
629 vdev
->config
->del_vqs(vdev
);
634 static struct virtio_device_id id_table
[] = {
635 { VIRTIO_ID_VSOCK
, VIRTIO_DEV_ANY_ID
},
639 static unsigned int features
[] = {
642 static struct virtio_driver virtio_vsock_driver
= {
643 .feature_table
= features
,
644 .feature_table_size
= ARRAY_SIZE(features
),
645 .driver
.name
= KBUILD_MODNAME
,
646 .driver
.owner
= THIS_MODULE
,
647 .id_table
= id_table
,
648 .probe
= virtio_vsock_probe
,
649 .remove
= virtio_vsock_remove
,
652 static int __init
virtio_vsock_init(void)
656 virtio_vsock_workqueue
= alloc_workqueue("virtio_vsock", 0, 0);
657 if (!virtio_vsock_workqueue
)
659 ret
= register_virtio_driver(&virtio_vsock_driver
);
661 destroy_workqueue(virtio_vsock_workqueue
);
665 static void __exit
virtio_vsock_exit(void)
667 unregister_virtio_driver(&virtio_vsock_driver
);
668 destroy_workqueue(virtio_vsock_workqueue
);
671 module_init(virtio_vsock_init
);
672 module_exit(virtio_vsock_exit
);
673 MODULE_LICENSE("GPL v2");
674 MODULE_AUTHOR("Asias He");
675 MODULE_DESCRIPTION("virtio transport for vsock");
676 MODULE_DEVICE_TABLE(virtio
, id_table
);