1 // SPDX-License-Identifier: GPL-2.0-only
3 * VMware vSockets Driver
5 * Copyright (C) 2007-2013 VMware, Inc. All rights reserved.
8 #include <linux/types.h>
9 #include <linux/bitops.h>
10 #include <linux/cred.h>
11 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/kmod.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/net.h>
19 #include <linux/poll.h>
20 #include <linux/skbuff.h>
21 #include <linux/smp.h>
22 #include <linux/socket.h>
23 #include <linux/stddef.h>
24 #include <linux/unistd.h>
25 #include <linux/wait.h>
26 #include <linux/workqueue.h>
28 #include <net/af_vsock.h>
30 #include "vmci_transport_notify.h"
32 static int vmci_transport_recv_dgram_cb(void *data
, struct vmci_datagram
*dg
);
33 static int vmci_transport_recv_stream_cb(void *data
, struct vmci_datagram
*dg
);
34 static void vmci_transport_peer_detach_cb(u32 sub_id
,
35 const struct vmci_event_data
*ed
,
37 static void vmci_transport_recv_pkt_work(struct work_struct
*work
);
38 static void vmci_transport_cleanup(struct work_struct
*work
);
39 static int vmci_transport_recv_listen(struct sock
*sk
,
40 struct vmci_transport_packet
*pkt
);
41 static int vmci_transport_recv_connecting_server(
44 struct vmci_transport_packet
*pkt
);
45 static int vmci_transport_recv_connecting_client(
47 struct vmci_transport_packet
*pkt
);
48 static int vmci_transport_recv_connecting_client_negotiate(
50 struct vmci_transport_packet
*pkt
);
51 static int vmci_transport_recv_connecting_client_invalid(
53 struct vmci_transport_packet
*pkt
);
54 static int vmci_transport_recv_connected(struct sock
*sk
,
55 struct vmci_transport_packet
*pkt
);
56 static bool vmci_transport_old_proto_override(bool *old_pkt_proto
);
57 static u16
vmci_transport_new_proto_supported_versions(void);
58 static bool vmci_transport_proto_to_notify_struct(struct sock
*sk
, u16
*proto
,
60 static bool vmci_check_transport(struct vsock_sock
*vsk
);
62 struct vmci_transport_recv_pkt_info
{
63 struct work_struct work
;
65 struct vmci_transport_packet pkt
;
68 static LIST_HEAD(vmci_transport_cleanup_list
);
69 static DEFINE_SPINLOCK(vmci_transport_cleanup_lock
);
70 static DECLARE_WORK(vmci_transport_cleanup_work
, vmci_transport_cleanup
);
72 static struct vmci_handle vmci_transport_stream_handle
= { VMCI_INVALID_ID
,
74 static u32 vmci_transport_qp_resumed_sub_id
= VMCI_INVALID_ID
;
76 static int PROTOCOL_OVERRIDE
= -1;
78 /* Helper function to convert from a VMCI error code to a VSock error code. */
80 static s32
vmci_transport_error_to_vsock_error(s32 vmci_error
)
83 case VMCI_ERROR_NO_MEM
:
85 case VMCI_ERROR_DUPLICATE_ENTRY
:
86 case VMCI_ERROR_ALREADY_EXISTS
:
88 case VMCI_ERROR_NO_ACCESS
:
90 case VMCI_ERROR_NO_RESOURCES
:
92 case VMCI_ERROR_INVALID_RESOURCE
:
94 case VMCI_ERROR_INVALID_ARGS
:
101 static u32
vmci_transport_peer_rid(u32 peer_cid
)
103 if (VMADDR_CID_HYPERVISOR
== peer_cid
)
104 return VMCI_TRANSPORT_HYPERVISOR_PACKET_RID
;
106 return VMCI_TRANSPORT_PACKET_RID
;
110 vmci_transport_packet_init(struct vmci_transport_packet
*pkt
,
111 struct sockaddr_vm
*src
,
112 struct sockaddr_vm
*dst
,
116 struct vmci_transport_waiting_info
*wait
,
118 struct vmci_handle handle
)
120 /* We register the stream control handler as an any cid handle so we
121 * must always send from a source address of VMADDR_CID_ANY
123 pkt
->dg
.src
= vmci_make_handle(VMADDR_CID_ANY
,
124 VMCI_TRANSPORT_PACKET_RID
);
125 pkt
->dg
.dst
= vmci_make_handle(dst
->svm_cid
,
126 vmci_transport_peer_rid(dst
->svm_cid
));
127 pkt
->dg
.payload_size
= sizeof(*pkt
) - sizeof(pkt
->dg
);
128 pkt
->version
= VMCI_TRANSPORT_PACKET_VERSION
;
130 pkt
->src_port
= src
->svm_port
;
131 pkt
->dst_port
= dst
->svm_port
;
132 memset(&pkt
->proto
, 0, sizeof(pkt
->proto
));
133 memset(&pkt
->_reserved2
, 0, sizeof(pkt
->_reserved2
));
136 case VMCI_TRANSPORT_PACKET_TYPE_INVALID
:
140 case VMCI_TRANSPORT_PACKET_TYPE_REQUEST
:
141 case VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE
:
145 case VMCI_TRANSPORT_PACKET_TYPE_OFFER
:
146 case VMCI_TRANSPORT_PACKET_TYPE_ATTACH
:
147 pkt
->u
.handle
= handle
;
150 case VMCI_TRANSPORT_PACKET_TYPE_WROTE
:
151 case VMCI_TRANSPORT_PACKET_TYPE_READ
:
152 case VMCI_TRANSPORT_PACKET_TYPE_RST
:
156 case VMCI_TRANSPORT_PACKET_TYPE_SHUTDOWN
:
160 case VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ
:
161 case VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE
:
162 memcpy(&pkt
->u
.wait
, wait
, sizeof(pkt
->u
.wait
));
165 case VMCI_TRANSPORT_PACKET_TYPE_REQUEST2
:
166 case VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE2
:
174 vmci_transport_packet_get_addresses(struct vmci_transport_packet
*pkt
,
175 struct sockaddr_vm
*local
,
176 struct sockaddr_vm
*remote
)
178 vsock_addr_init(local
, pkt
->dg
.dst
.context
, pkt
->dst_port
);
179 vsock_addr_init(remote
, pkt
->dg
.src
.context
, pkt
->src_port
);
183 __vmci_transport_send_control_pkt(struct vmci_transport_packet
*pkt
,
184 struct sockaddr_vm
*src
,
185 struct sockaddr_vm
*dst
,
186 enum vmci_transport_packet_type type
,
189 struct vmci_transport_waiting_info
*wait
,
191 struct vmci_handle handle
,
196 vmci_transport_packet_init(pkt
, src
, dst
, type
, size
, mode
, wait
,
198 err
= vmci_datagram_send(&pkt
->dg
);
199 if (convert_error
&& (err
< 0))
200 return vmci_transport_error_to_vsock_error(err
);
206 vmci_transport_reply_control_pkt_fast(struct vmci_transport_packet
*pkt
,
207 enum vmci_transport_packet_type type
,
210 struct vmci_transport_waiting_info
*wait
,
211 struct vmci_handle handle
)
213 struct vmci_transport_packet reply
;
214 struct sockaddr_vm src
, dst
;
216 if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_RST
) {
219 vmci_transport_packet_get_addresses(pkt
, &src
, &dst
);
220 return __vmci_transport_send_control_pkt(&reply
, &src
, &dst
,
229 vmci_transport_send_control_pkt_bh(struct sockaddr_vm
*src
,
230 struct sockaddr_vm
*dst
,
231 enum vmci_transport_packet_type type
,
234 struct vmci_transport_waiting_info
*wait
,
235 struct vmci_handle handle
)
237 /* Note that it is safe to use a single packet across all CPUs since
238 * two tasklets of the same type are guaranteed to not ever run
239 * simultaneously. If that ever changes, or VMCI stops using tasklets,
240 * we can use per-cpu packets.
242 static struct vmci_transport_packet pkt
;
244 return __vmci_transport_send_control_pkt(&pkt
, src
, dst
, type
,
246 VSOCK_PROTO_INVALID
, handle
,
251 vmci_transport_alloc_send_control_pkt(struct sockaddr_vm
*src
,
252 struct sockaddr_vm
*dst
,
253 enum vmci_transport_packet_type type
,
256 struct vmci_transport_waiting_info
*wait
,
258 struct vmci_handle handle
)
260 struct vmci_transport_packet
*pkt
;
263 pkt
= kmalloc(sizeof(*pkt
), GFP_KERNEL
);
267 err
= __vmci_transport_send_control_pkt(pkt
, src
, dst
, type
, size
,
268 mode
, wait
, proto
, handle
,
276 vmci_transport_send_control_pkt(struct sock
*sk
,
277 enum vmci_transport_packet_type type
,
280 struct vmci_transport_waiting_info
*wait
,
282 struct vmci_handle handle
)
284 struct vsock_sock
*vsk
;
288 if (!vsock_addr_bound(&vsk
->local_addr
))
291 if (!vsock_addr_bound(&vsk
->remote_addr
))
294 return vmci_transport_alloc_send_control_pkt(&vsk
->local_addr
,
297 wait
, proto
, handle
);
300 static int vmci_transport_send_reset_bh(struct sockaddr_vm
*dst
,
301 struct sockaddr_vm
*src
,
302 struct vmci_transport_packet
*pkt
)
304 if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_RST
)
306 return vmci_transport_send_control_pkt_bh(
308 VMCI_TRANSPORT_PACKET_TYPE_RST
, 0,
309 0, NULL
, VMCI_INVALID_HANDLE
);
312 static int vmci_transport_send_reset(struct sock
*sk
,
313 struct vmci_transport_packet
*pkt
)
315 struct sockaddr_vm
*dst_ptr
;
316 struct sockaddr_vm dst
;
317 struct vsock_sock
*vsk
;
319 if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_RST
)
324 if (!vsock_addr_bound(&vsk
->local_addr
))
327 if (vsock_addr_bound(&vsk
->remote_addr
)) {
328 dst_ptr
= &vsk
->remote_addr
;
330 vsock_addr_init(&dst
, pkt
->dg
.src
.context
,
334 return vmci_transport_alloc_send_control_pkt(&vsk
->local_addr
, dst_ptr
,
335 VMCI_TRANSPORT_PACKET_TYPE_RST
,
336 0, 0, NULL
, VSOCK_PROTO_INVALID
,
337 VMCI_INVALID_HANDLE
);
340 static int vmci_transport_send_negotiate(struct sock
*sk
, size_t size
)
342 return vmci_transport_send_control_pkt(
344 VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE
,
347 VMCI_INVALID_HANDLE
);
350 static int vmci_transport_send_negotiate2(struct sock
*sk
, size_t size
,
353 return vmci_transport_send_control_pkt(
355 VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE2
,
356 size
, 0, NULL
, version
,
357 VMCI_INVALID_HANDLE
);
360 static int vmci_transport_send_qp_offer(struct sock
*sk
,
361 struct vmci_handle handle
)
363 return vmci_transport_send_control_pkt(
364 sk
, VMCI_TRANSPORT_PACKET_TYPE_OFFER
, 0,
366 VSOCK_PROTO_INVALID
, handle
);
369 static int vmci_transport_send_attach(struct sock
*sk
,
370 struct vmci_handle handle
)
372 return vmci_transport_send_control_pkt(
373 sk
, VMCI_TRANSPORT_PACKET_TYPE_ATTACH
,
374 0, 0, NULL
, VSOCK_PROTO_INVALID
,
378 static int vmci_transport_reply_reset(struct vmci_transport_packet
*pkt
)
380 return vmci_transport_reply_control_pkt_fast(
382 VMCI_TRANSPORT_PACKET_TYPE_RST
,
384 VMCI_INVALID_HANDLE
);
387 static int vmci_transport_send_invalid_bh(struct sockaddr_vm
*dst
,
388 struct sockaddr_vm
*src
)
390 return vmci_transport_send_control_pkt_bh(
392 VMCI_TRANSPORT_PACKET_TYPE_INVALID
,
393 0, 0, NULL
, VMCI_INVALID_HANDLE
);
396 int vmci_transport_send_wrote_bh(struct sockaddr_vm
*dst
,
397 struct sockaddr_vm
*src
)
399 return vmci_transport_send_control_pkt_bh(
401 VMCI_TRANSPORT_PACKET_TYPE_WROTE
, 0,
402 0, NULL
, VMCI_INVALID_HANDLE
);
405 int vmci_transport_send_read_bh(struct sockaddr_vm
*dst
,
406 struct sockaddr_vm
*src
)
408 return vmci_transport_send_control_pkt_bh(
410 VMCI_TRANSPORT_PACKET_TYPE_READ
, 0,
411 0, NULL
, VMCI_INVALID_HANDLE
);
414 int vmci_transport_send_wrote(struct sock
*sk
)
416 return vmci_transport_send_control_pkt(
417 sk
, VMCI_TRANSPORT_PACKET_TYPE_WROTE
, 0,
418 0, NULL
, VSOCK_PROTO_INVALID
,
419 VMCI_INVALID_HANDLE
);
422 int vmci_transport_send_read(struct sock
*sk
)
424 return vmci_transport_send_control_pkt(
425 sk
, VMCI_TRANSPORT_PACKET_TYPE_READ
, 0,
426 0, NULL
, VSOCK_PROTO_INVALID
,
427 VMCI_INVALID_HANDLE
);
430 int vmci_transport_send_waiting_write(struct sock
*sk
,
431 struct vmci_transport_waiting_info
*wait
)
433 return vmci_transport_send_control_pkt(
434 sk
, VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE
,
435 0, 0, wait
, VSOCK_PROTO_INVALID
,
436 VMCI_INVALID_HANDLE
);
439 int vmci_transport_send_waiting_read(struct sock
*sk
,
440 struct vmci_transport_waiting_info
*wait
)
442 return vmci_transport_send_control_pkt(
443 sk
, VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ
,
444 0, 0, wait
, VSOCK_PROTO_INVALID
,
445 VMCI_INVALID_HANDLE
);
448 static int vmci_transport_shutdown(struct vsock_sock
*vsk
, int mode
)
450 return vmci_transport_send_control_pkt(
452 VMCI_TRANSPORT_PACKET_TYPE_SHUTDOWN
,
455 VMCI_INVALID_HANDLE
);
458 static int vmci_transport_send_conn_request(struct sock
*sk
, size_t size
)
460 return vmci_transport_send_control_pkt(sk
,
461 VMCI_TRANSPORT_PACKET_TYPE_REQUEST
,
464 VMCI_INVALID_HANDLE
);
467 static int vmci_transport_send_conn_request2(struct sock
*sk
, size_t size
,
470 return vmci_transport_send_control_pkt(
471 sk
, VMCI_TRANSPORT_PACKET_TYPE_REQUEST2
,
472 size
, 0, NULL
, version
,
473 VMCI_INVALID_HANDLE
);
476 static struct sock
*vmci_transport_get_pending(
477 struct sock
*listener
,
478 struct vmci_transport_packet
*pkt
)
480 struct vsock_sock
*vlistener
;
481 struct vsock_sock
*vpending
;
482 struct sock
*pending
;
483 struct sockaddr_vm src
;
485 vsock_addr_init(&src
, pkt
->dg
.src
.context
, pkt
->src_port
);
487 vlistener
= vsock_sk(listener
);
489 list_for_each_entry(vpending
, &vlistener
->pending_links
,
491 if (vsock_addr_equals_addr(&src
, &vpending
->remote_addr
) &&
492 pkt
->dst_port
== vpending
->local_addr
.svm_port
) {
493 pending
= sk_vsock(vpending
);
505 static void vmci_transport_release_pending(struct sock
*pending
)
510 /* We allow two kinds of sockets to communicate with a restricted VM: 1)
511 * trusted sockets 2) sockets from applications running as the same user as the
512 * VM (this is only true for the host side and only when using hosted products)
515 static bool vmci_transport_is_trusted(struct vsock_sock
*vsock
, u32 peer_cid
)
517 return vsock
->trusted
||
518 vmci_is_context_owner(peer_cid
, vsock
->owner
->uid
);
521 /* We allow sending datagrams to and receiving datagrams from a restricted VM
522 * only if it is trusted as described in vmci_transport_is_trusted.
525 static bool vmci_transport_allow_dgram(struct vsock_sock
*vsock
, u32 peer_cid
)
527 if (VMADDR_CID_HYPERVISOR
== peer_cid
)
530 if (vsock
->cached_peer
!= peer_cid
) {
531 vsock
->cached_peer
= peer_cid
;
532 if (!vmci_transport_is_trusted(vsock
, peer_cid
) &&
533 (vmci_context_get_priv_flags(peer_cid
) &
534 VMCI_PRIVILEGE_FLAG_RESTRICTED
)) {
535 vsock
->cached_peer_allow_dgram
= false;
537 vsock
->cached_peer_allow_dgram
= true;
541 return vsock
->cached_peer_allow_dgram
;
545 vmci_transport_queue_pair_alloc(struct vmci_qp
**qpair
,
546 struct vmci_handle
*handle
,
549 u32 peer
, u32 flags
, bool trusted
)
554 /* Try to allocate our queue pair as trusted. This will only
555 * work if vsock is running in the host.
558 err
= vmci_qpair_alloc(qpair
, handle
, produce_size
,
561 VMCI_PRIVILEGE_FLAG_TRUSTED
);
562 if (err
!= VMCI_ERROR_NO_ACCESS
)
567 err
= vmci_qpair_alloc(qpair
, handle
, produce_size
, consume_size
,
568 peer
, flags
, VMCI_NO_PRIVILEGE_FLAGS
);
571 pr_err_once("Could not attach to queue pair with %d\n", err
);
572 err
= vmci_transport_error_to_vsock_error(err
);
579 vmci_transport_datagram_create_hnd(u32 resource_id
,
581 vmci_datagram_recv_cb recv_cb
,
583 struct vmci_handle
*out_handle
)
587 /* Try to allocate our datagram handler as trusted. This will only work
588 * if vsock is running in the host.
591 err
= vmci_datagram_create_handle_priv(resource_id
, flags
,
592 VMCI_PRIVILEGE_FLAG_TRUSTED
,
594 client_data
, out_handle
);
596 if (err
== VMCI_ERROR_NO_ACCESS
)
597 err
= vmci_datagram_create_handle(resource_id
, flags
,
598 recv_cb
, client_data
,
604 /* This is invoked as part of a tasklet that's scheduled when the VMCI
605 * interrupt fires. This is run in bottom-half context and if it ever needs to
606 * sleep it should defer that work to a work queue.
609 static int vmci_transport_recv_dgram_cb(void *data
, struct vmci_datagram
*dg
)
614 struct vsock_sock
*vsk
;
616 sk
= (struct sock
*)data
;
618 /* This handler is privileged when this module is running on the host.
619 * We will get datagrams from all endpoints (even VMs that are in a
620 * restricted context). If we get one from a restricted context then
621 * the destination socket must be trusted.
623 * NOTE: We access the socket struct without holding the lock here.
624 * This is ok because the field we are interested is never modified
625 * outside of the create and destruct socket functions.
628 if (!vmci_transport_allow_dgram(vsk
, dg
->src
.context
))
629 return VMCI_ERROR_NO_ACCESS
;
631 size
= VMCI_DG_SIZE(dg
);
633 /* Attach the packet to the socket's receive queue as an sk_buff. */
634 skb
= alloc_skb(size
, GFP_ATOMIC
);
636 return VMCI_ERROR_NO_MEM
;
638 /* sk_receive_skb() will do a sock_put(), so hold here. */
641 memcpy(skb
->data
, dg
, size
);
642 sk_receive_skb(sk
, skb
, 0);
647 static bool vmci_transport_stream_allow(u32 cid
, u32 port
)
649 static const u32 non_socket_contexts
[] = {
654 BUILD_BUG_ON(sizeof(cid
) != sizeof(*non_socket_contexts
));
656 for (i
= 0; i
< ARRAY_SIZE(non_socket_contexts
); i
++) {
657 if (cid
== non_socket_contexts
[i
])
664 /* This is invoked as part of a tasklet that's scheduled when the VMCI
665 * interrupt fires. This is run in bottom-half context but it defers most of
666 * its work to the packet handling work queue.
669 static int vmci_transport_recv_stream_cb(void *data
, struct vmci_datagram
*dg
)
672 struct sockaddr_vm dst
;
673 struct sockaddr_vm src
;
674 struct vmci_transport_packet
*pkt
;
675 struct vsock_sock
*vsk
;
681 bh_process_pkt
= false;
683 /* Ignore incoming packets from contexts without sockets, or resources
684 * that aren't vsock implementations.
687 if (!vmci_transport_stream_allow(dg
->src
.context
, -1)
688 || vmci_transport_peer_rid(dg
->src
.context
) != dg
->src
.resource
)
689 return VMCI_ERROR_NO_ACCESS
;
691 if (VMCI_DG_SIZE(dg
) < sizeof(*pkt
))
692 /* Drop datagrams that do not contain full VSock packets. */
693 return VMCI_ERROR_INVALID_ARGS
;
695 pkt
= (struct vmci_transport_packet
*)dg
;
697 /* Find the socket that should handle this packet. First we look for a
698 * connected socket and if there is none we look for a socket bound to
699 * the destintation address.
701 vsock_addr_init(&src
, pkt
->dg
.src
.context
, pkt
->src_port
);
702 vsock_addr_init(&dst
, pkt
->dg
.dst
.context
, pkt
->dst_port
);
704 sk
= vsock_find_connected_socket(&src
, &dst
);
706 sk
= vsock_find_bound_socket(&dst
);
708 /* We could not find a socket for this specified
709 * address. If this packet is a RST, we just drop it.
710 * If it is another packet, we send a RST. Note that
711 * we do not send a RST reply to RSTs so that we do not
712 * continually send RSTs between two endpoints.
714 * Note that since this is a reply, dst is src and src
717 if (vmci_transport_send_reset_bh(&dst
, &src
, pkt
) < 0)
718 pr_err("unable to send reset\n");
720 err
= VMCI_ERROR_NOT_FOUND
;
725 /* If the received packet type is beyond all types known to this
726 * implementation, reply with an invalid message. Hopefully this will
727 * help when implementing backwards compatibility in the future.
729 if (pkt
->type
>= VMCI_TRANSPORT_PACKET_TYPE_MAX
) {
730 vmci_transport_send_invalid_bh(&dst
, &src
);
731 err
= VMCI_ERROR_INVALID_ARGS
;
735 /* This handler is privileged when this module is running on the host.
736 * We will get datagram connect requests from all endpoints (even VMs
737 * that are in a restricted context). If we get one from a restricted
738 * context then the destination socket must be trusted.
740 * NOTE: We access the socket struct without holding the lock here.
741 * This is ok because the field we are interested is never modified
742 * outside of the create and destruct socket functions.
745 if (!vmci_transport_allow_dgram(vsk
, pkt
->dg
.src
.context
)) {
746 err
= VMCI_ERROR_NO_ACCESS
;
750 /* We do most everything in a work queue, but let's fast path the
751 * notification of reads and writes to help data transfer performance.
752 * We can only do this if there is no process context code executing
753 * for this socket since that may change the state.
757 if (!sock_owned_by_user(sk
)) {
758 /* The local context ID may be out of date, update it. */
759 vsk
->local_addr
.svm_cid
= dst
.svm_cid
;
761 if (sk
->sk_state
== TCP_ESTABLISHED
)
762 vmci_trans(vsk
)->notify_ops
->handle_notify_pkt(
763 sk
, pkt
, true, &dst
, &src
,
769 if (!bh_process_pkt
) {
770 struct vmci_transport_recv_pkt_info
*recv_pkt_info
;
772 recv_pkt_info
= kmalloc(sizeof(*recv_pkt_info
), GFP_ATOMIC
);
773 if (!recv_pkt_info
) {
774 if (vmci_transport_send_reset_bh(&dst
, &src
, pkt
) < 0)
775 pr_err("unable to send reset\n");
777 err
= VMCI_ERROR_NO_MEM
;
781 recv_pkt_info
->sk
= sk
;
782 memcpy(&recv_pkt_info
->pkt
, pkt
, sizeof(recv_pkt_info
->pkt
));
783 INIT_WORK(&recv_pkt_info
->work
, vmci_transport_recv_pkt_work
);
785 schedule_work(&recv_pkt_info
->work
);
786 /* Clear sk so that the reference count incremented by one of
787 * the Find functions above is not decremented below. We need
788 * that reference count for the packet handler we've scheduled
801 static void vmci_transport_handle_detach(struct sock
*sk
)
803 struct vsock_sock
*vsk
;
806 if (!vmci_handle_is_invalid(vmci_trans(vsk
)->qp_handle
)) {
807 sock_set_flag(sk
, SOCK_DONE
);
809 /* On a detach the peer will not be sending or receiving
812 vsk
->peer_shutdown
= SHUTDOWN_MASK
;
814 /* We should not be sending anymore since the peer won't be
815 * there to receive, but we can still receive if there is data
816 * left in our consume queue. If the local endpoint is a host,
817 * we can't call vsock_stream_has_data, since that may block,
818 * but a host endpoint can't read data once the VM has
819 * detached, so there is no available data in that case.
821 if (vsk
->local_addr
.svm_cid
== VMADDR_CID_HOST
||
822 vsock_stream_has_data(vsk
) <= 0) {
823 if (sk
->sk_state
== TCP_SYN_SENT
) {
824 /* The peer may detach from a queue pair while
825 * we are still in the connecting state, i.e.,
826 * if the peer VM is killed after attaching to
827 * a queue pair, but before we complete the
828 * handshake. In that case, we treat the detach
829 * event like a reset.
832 sk
->sk_state
= TCP_CLOSE
;
833 sk
->sk_err
= ECONNRESET
;
834 sk
->sk_error_report(sk
);
837 sk
->sk_state
= TCP_CLOSE
;
839 sk
->sk_state_change(sk
);
843 static void vmci_transport_peer_detach_cb(u32 sub_id
,
844 const struct vmci_event_data
*e_data
,
847 struct vmci_transport
*trans
= client_data
;
848 const struct vmci_event_payload_qp
*e_payload
;
850 e_payload
= vmci_event_data_const_payload(e_data
);
852 /* XXX This is lame, we should provide a way to lookup sockets by
855 if (vmci_handle_is_invalid(e_payload
->handle
) ||
856 !vmci_handle_is_equal(trans
->qp_handle
, e_payload
->handle
))
859 /* We don't ask for delayed CBs when we subscribe to this event (we
860 * pass 0 as flags to vmci_event_subscribe()). VMCI makes no
861 * guarantees in that case about what context we might be running in,
862 * so it could be BH or process, blockable or non-blockable. So we
863 * need to account for all possible contexts here.
865 spin_lock_bh(&trans
->lock
);
869 /* Apart from here, trans->lock is only grabbed as part of sk destruct,
870 * where trans->sk isn't locked.
872 bh_lock_sock(trans
->sk
);
874 vmci_transport_handle_detach(trans
->sk
);
876 bh_unlock_sock(trans
->sk
);
878 spin_unlock_bh(&trans
->lock
);
881 static void vmci_transport_qp_resumed_cb(u32 sub_id
,
882 const struct vmci_event_data
*e_data
,
885 vsock_for_each_connected_socket(vmci_transport_handle_detach
);
888 static void vmci_transport_recv_pkt_work(struct work_struct
*work
)
890 struct vmci_transport_recv_pkt_info
*recv_pkt_info
;
891 struct vmci_transport_packet
*pkt
;
895 container_of(work
, struct vmci_transport_recv_pkt_info
, work
);
896 sk
= recv_pkt_info
->sk
;
897 pkt
= &recv_pkt_info
->pkt
;
901 /* The local context ID may be out of date. */
902 vsock_sk(sk
)->local_addr
.svm_cid
= pkt
->dg
.dst
.context
;
904 switch (sk
->sk_state
) {
906 vmci_transport_recv_listen(sk
, pkt
);
909 /* Processing of pending connections for servers goes through
910 * the listening socket, so see vmci_transport_recv_listen()
913 vmci_transport_recv_connecting_client(sk
, pkt
);
915 case TCP_ESTABLISHED
:
916 vmci_transport_recv_connected(sk
, pkt
);
919 /* Because this function does not run in the same context as
920 * vmci_transport_recv_stream_cb it is possible that the
921 * socket has closed. We need to let the other side know or it
922 * could be sitting in a connect and hang forever. Send a
923 * reset to prevent that.
925 vmci_transport_send_reset(sk
, pkt
);
930 kfree(recv_pkt_info
);
931 /* Release reference obtained in the stream callback when we fetched
932 * this socket out of the bound or connected list.
937 static int vmci_transport_recv_listen(struct sock
*sk
,
938 struct vmci_transport_packet
*pkt
)
940 struct sock
*pending
;
941 struct vsock_sock
*vpending
;
944 bool old_request
= false;
945 bool old_pkt_proto
= false;
949 /* Because we are in the listen state, we could be receiving a packet
950 * for ourself or any previous connection requests that we received.
951 * If it's the latter, we try to find a socket in our list of pending
952 * connections and, if we do, call the appropriate handler for the
953 * state that that socket is in. Otherwise we try to service the
954 * connection request.
956 pending
= vmci_transport_get_pending(sk
, pkt
);
960 /* The local context ID may be out of date. */
961 vsock_sk(pending
)->local_addr
.svm_cid
= pkt
->dg
.dst
.context
;
963 switch (pending
->sk_state
) {
965 err
= vmci_transport_recv_connecting_server(sk
,
970 vmci_transport_send_reset(pending
, pkt
);
975 vsock_remove_pending(sk
, pending
);
977 release_sock(pending
);
978 vmci_transport_release_pending(pending
);
983 /* The listen state only accepts connection requests. Reply with a
984 * reset unless we received a reset.
987 if (!(pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_REQUEST
||
988 pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_REQUEST2
)) {
989 vmci_transport_reply_reset(pkt
);
993 if (pkt
->u
.size
== 0) {
994 vmci_transport_reply_reset(pkt
);
998 /* If this socket can't accommodate this connection request, we send a
999 * reset. Otherwise we create and initialize a child socket and reply
1000 * with a connection negotiation.
1002 if (sk
->sk_ack_backlog
>= sk
->sk_max_ack_backlog
) {
1003 vmci_transport_reply_reset(pkt
);
1004 return -ECONNREFUSED
;
1007 pending
= vsock_create_connected(sk
);
1009 vmci_transport_send_reset(sk
, pkt
);
1013 vpending
= vsock_sk(pending
);
1015 vsock_addr_init(&vpending
->local_addr
, pkt
->dg
.dst
.context
,
1017 vsock_addr_init(&vpending
->remote_addr
, pkt
->dg
.src
.context
,
1020 err
= vsock_assign_transport(vpending
, vsock_sk(sk
));
1021 /* Transport assigned (looking at remote_addr) must be the same
1022 * where we received the request.
1024 if (err
|| !vmci_check_transport(vpending
)) {
1025 vmci_transport_send_reset(sk
, pkt
);
1030 /* If the proposed size fits within our min/max, accept it. Otherwise
1031 * propose our own size.
1033 if (pkt
->u
.size
>= vpending
->buffer_min_size
&&
1034 pkt
->u
.size
<= vpending
->buffer_max_size
) {
1035 qp_size
= pkt
->u
.size
;
1037 qp_size
= vpending
->buffer_size
;
1040 /* Figure out if we are using old or new requests based on the
1041 * overrides pkt types sent by our peer.
1043 if (vmci_transport_old_proto_override(&old_pkt_proto
)) {
1044 old_request
= old_pkt_proto
;
1046 if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_REQUEST
)
1048 else if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_REQUEST2
)
1049 old_request
= false;
1054 /* Handle a REQUEST (or override) */
1055 u16 version
= VSOCK_PROTO_INVALID
;
1056 if (vmci_transport_proto_to_notify_struct(
1057 pending
, &version
, true))
1058 err
= vmci_transport_send_negotiate(pending
, qp_size
);
1063 /* Handle a REQUEST2 (or override) */
1064 int proto_int
= pkt
->proto
;
1066 u16 active_proto_version
= 0;
1068 /* The list of possible protocols is the intersection of all
1069 * protocols the client supports ... plus all the protocols we
1072 proto_int
&= vmci_transport_new_proto_supported_versions();
1074 /* We choose the highest possible protocol version and use that
1077 pos
= fls(proto_int
);
1079 active_proto_version
= (1 << (pos
- 1));
1080 if (vmci_transport_proto_to_notify_struct(
1081 pending
, &active_proto_version
, false))
1082 err
= vmci_transport_send_negotiate2(pending
,
1084 active_proto_version
);
1094 vmci_transport_send_reset(sk
, pkt
);
1096 err
= vmci_transport_error_to_vsock_error(err
);
1100 vsock_add_pending(sk
, pending
);
1101 sk_acceptq_added(sk
);
1103 pending
->sk_state
= TCP_SYN_SENT
;
1104 vmci_trans(vpending
)->produce_size
=
1105 vmci_trans(vpending
)->consume_size
= qp_size
;
1106 vpending
->buffer_size
= qp_size
;
1108 vmci_trans(vpending
)->notify_ops
->process_request(pending
);
1110 /* We might never receive another message for this socket and it's not
1111 * connected to any process, so we have to ensure it gets cleaned up
1112 * ourself. Our delayed work function will take care of that. Note
1113 * that we do not ever cancel this function since we have few
1114 * guarantees about its state when calling cancel_delayed_work().
1115 * Instead we hold a reference on the socket for that function and make
1116 * it capable of handling cases where it needs to do nothing but
1117 * release that reference.
1119 vpending
->listener
= sk
;
1122 schedule_delayed_work(&vpending
->pending_work
, HZ
);
1129 vmci_transport_recv_connecting_server(struct sock
*listener
,
1130 struct sock
*pending
,
1131 struct vmci_transport_packet
*pkt
)
1133 struct vsock_sock
*vpending
;
1134 struct vmci_handle handle
;
1135 struct vmci_qp
*qpair
;
1142 vpending
= vsock_sk(pending
);
1143 detach_sub_id
= VMCI_INVALID_ID
;
1145 switch (pkt
->type
) {
1146 case VMCI_TRANSPORT_PACKET_TYPE_OFFER
:
1147 if (vmci_handle_is_invalid(pkt
->u
.handle
)) {
1148 vmci_transport_send_reset(pending
, pkt
);
1155 /* Close and cleanup the connection. */
1156 vmci_transport_send_reset(pending
, pkt
);
1158 err
= pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_RST
? 0 : -EINVAL
;
1162 /* In order to complete the connection we need to attach to the offered
1163 * queue pair and send an attach notification. We also subscribe to the
1164 * detach event so we know when our peer goes away, and we do that
1165 * before attaching so we don't miss an event. If all this succeeds,
1166 * we update our state and wakeup anything waiting in accept() for a
1170 /* We don't care about attach since we ensure the other side has
1171 * attached by specifying the ATTACH_ONLY flag below.
1173 err
= vmci_event_subscribe(VMCI_EVENT_QP_PEER_DETACH
,
1174 vmci_transport_peer_detach_cb
,
1175 vmci_trans(vpending
), &detach_sub_id
);
1176 if (err
< VMCI_SUCCESS
) {
1177 vmci_transport_send_reset(pending
, pkt
);
1178 err
= vmci_transport_error_to_vsock_error(err
);
1183 vmci_trans(vpending
)->detach_sub_id
= detach_sub_id
;
1185 /* Now attach to the queue pair the client created. */
1186 handle
= pkt
->u
.handle
;
1188 /* vpending->local_addr always has a context id so we do not need to
1189 * worry about VMADDR_CID_ANY in this case.
1192 vpending
->remote_addr
.svm_cid
== vpending
->local_addr
.svm_cid
;
1193 flags
= VMCI_QPFLAG_ATTACH_ONLY
;
1194 flags
|= is_local
? VMCI_QPFLAG_LOCAL
: 0;
1196 err
= vmci_transport_queue_pair_alloc(
1199 vmci_trans(vpending
)->produce_size
,
1200 vmci_trans(vpending
)->consume_size
,
1201 pkt
->dg
.src
.context
,
1203 vmci_transport_is_trusted(
1205 vpending
->remote_addr
.svm_cid
));
1207 vmci_transport_send_reset(pending
, pkt
);
1212 vmci_trans(vpending
)->qp_handle
= handle
;
1213 vmci_trans(vpending
)->qpair
= qpair
;
1215 /* When we send the attach message, we must be ready to handle incoming
1216 * control messages on the newly connected socket. So we move the
1217 * pending socket to the connected state before sending the attach
1218 * message. Otherwise, an incoming packet triggered by the attach being
1219 * received by the peer may be processed concurrently with what happens
1220 * below after sending the attach message, and that incoming packet
1221 * will find the listening socket instead of the (currently) pending
1222 * socket. Note that enqueueing the socket increments the reference
1223 * count, so even if a reset comes before the connection is accepted,
1224 * the socket will be valid until it is removed from the queue.
1226 * If we fail sending the attach below, we remove the socket from the
1227 * connected list and move the socket to TCP_CLOSE before
1228 * releasing the lock, so a pending slow path processing of an incoming
1229 * packet will not see the socket in the connected state in that case.
1231 pending
->sk_state
= TCP_ESTABLISHED
;
1233 vsock_insert_connected(vpending
);
1235 /* Notify our peer of our attach. */
1236 err
= vmci_transport_send_attach(pending
, handle
);
1238 vsock_remove_connected(vpending
);
1239 pr_err("Could not send attach\n");
1240 vmci_transport_send_reset(pending
, pkt
);
1241 err
= vmci_transport_error_to_vsock_error(err
);
1246 /* We have a connection. Move the now connected socket from the
1247 * listener's pending list to the accept queue so callers of accept()
1250 vsock_remove_pending(listener
, pending
);
1251 vsock_enqueue_accept(listener
, pending
);
1253 /* Callers of accept() will be be waiting on the listening socket, not
1254 * the pending socket.
1256 listener
->sk_data_ready(listener
);
1261 pending
->sk_err
= skerr
;
1262 pending
->sk_state
= TCP_CLOSE
;
1263 /* As long as we drop our reference, all necessary cleanup will handle
1264 * when the cleanup function drops its reference and our destruct
1265 * implementation is called. Note that since the listen handler will
1266 * remove pending from the pending list upon our failure, the cleanup
1267 * function won't drop the additional reference, which is why we do it
1276 vmci_transport_recv_connecting_client(struct sock
*sk
,
1277 struct vmci_transport_packet
*pkt
)
1279 struct vsock_sock
*vsk
;
1285 switch (pkt
->type
) {
1286 case VMCI_TRANSPORT_PACKET_TYPE_ATTACH
:
1287 if (vmci_handle_is_invalid(pkt
->u
.handle
) ||
1288 !vmci_handle_is_equal(pkt
->u
.handle
,
1289 vmci_trans(vsk
)->qp_handle
)) {
1295 /* Signify the socket is connected and wakeup the waiter in
1296 * connect(). Also place the socket in the connected table for
1297 * accounting (it can already be found since it's in the bound
1300 sk
->sk_state
= TCP_ESTABLISHED
;
1301 sk
->sk_socket
->state
= SS_CONNECTED
;
1302 vsock_insert_connected(vsk
);
1303 sk
->sk_state_change(sk
);
1306 case VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE
:
1307 case VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE2
:
1308 if (pkt
->u
.size
== 0
1309 || pkt
->dg
.src
.context
!= vsk
->remote_addr
.svm_cid
1310 || pkt
->src_port
!= vsk
->remote_addr
.svm_port
1311 || !vmci_handle_is_invalid(vmci_trans(vsk
)->qp_handle
)
1312 || vmci_trans(vsk
)->qpair
1313 || vmci_trans(vsk
)->produce_size
!= 0
1314 || vmci_trans(vsk
)->consume_size
!= 0
1315 || vmci_trans(vsk
)->detach_sub_id
!= VMCI_INVALID_ID
) {
1322 err
= vmci_transport_recv_connecting_client_negotiate(sk
, pkt
);
1329 case VMCI_TRANSPORT_PACKET_TYPE_INVALID
:
1330 err
= vmci_transport_recv_connecting_client_invalid(sk
, pkt
);
1337 case VMCI_TRANSPORT_PACKET_TYPE_RST
:
1338 /* Older versions of the linux code (WS 6.5 / ESX 4.0) used to
1339 * continue processing here after they sent an INVALID packet.
1340 * This meant that we got a RST after the INVALID. We ignore a
1341 * RST after an INVALID. The common code doesn't send the RST
1342 * ... so we can hang if an old version of the common code
1343 * fails between getting a REQUEST and sending an OFFER back.
1344 * Not much we can do about it... except hope that it doesn't
1347 if (vsk
->ignore_connecting_rst
) {
1348 vsk
->ignore_connecting_rst
= false;
1357 /* Close and cleanup the connection. */
1366 vmci_transport_send_reset(sk
, pkt
);
1368 sk
->sk_state
= TCP_CLOSE
;
1370 sk
->sk_error_report(sk
);
1374 static int vmci_transport_recv_connecting_client_negotiate(
1376 struct vmci_transport_packet
*pkt
)
1379 struct vsock_sock
*vsk
;
1380 struct vmci_handle handle
;
1381 struct vmci_qp
*qpair
;
1385 bool old_proto
= true;
1390 handle
= VMCI_INVALID_HANDLE
;
1391 detach_sub_id
= VMCI_INVALID_ID
;
1393 /* If we have gotten here then we should be past the point where old
1394 * linux vsock could have sent the bogus rst.
1396 vsk
->sent_request
= false;
1397 vsk
->ignore_connecting_rst
= false;
1399 /* Verify that we're OK with the proposed queue pair size */
1400 if (pkt
->u
.size
< vsk
->buffer_min_size
||
1401 pkt
->u
.size
> vsk
->buffer_max_size
) {
1406 /* At this point we know the CID the peer is using to talk to us. */
1408 if (vsk
->local_addr
.svm_cid
== VMADDR_CID_ANY
)
1409 vsk
->local_addr
.svm_cid
= pkt
->dg
.dst
.context
;
1411 /* Setup the notify ops to be the highest supported version that both
1412 * the server and the client support.
1415 if (vmci_transport_old_proto_override(&old_pkt_proto
)) {
1416 old_proto
= old_pkt_proto
;
1418 if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE
)
1420 else if (pkt
->type
== VMCI_TRANSPORT_PACKET_TYPE_NEGOTIATE2
)
1426 version
= VSOCK_PROTO_INVALID
;
1428 version
= pkt
->proto
;
1430 if (!vmci_transport_proto_to_notify_struct(sk
, &version
, old_proto
)) {
1435 /* Subscribe to detach events first.
1437 * XXX We attach once for each queue pair created for now so it is easy
1438 * to find the socket (it's provided), but later we should only
1439 * subscribe once and add a way to lookup sockets by queue pair handle.
1441 err
= vmci_event_subscribe(VMCI_EVENT_QP_PEER_DETACH
,
1442 vmci_transport_peer_detach_cb
,
1443 vmci_trans(vsk
), &detach_sub_id
);
1444 if (err
< VMCI_SUCCESS
) {
1445 err
= vmci_transport_error_to_vsock_error(err
);
1449 /* Make VMCI select the handle for us. */
1450 handle
= VMCI_INVALID_HANDLE
;
1451 is_local
= vsk
->remote_addr
.svm_cid
== vsk
->local_addr
.svm_cid
;
1452 flags
= is_local
? VMCI_QPFLAG_LOCAL
: 0;
1454 err
= vmci_transport_queue_pair_alloc(&qpair
,
1458 vsk
->remote_addr
.svm_cid
,
1460 vmci_transport_is_trusted(
1463 remote_addr
.svm_cid
));
1467 err
= vmci_transport_send_qp_offer(sk
, handle
);
1469 err
= vmci_transport_error_to_vsock_error(err
);
1473 vmci_trans(vsk
)->qp_handle
= handle
;
1474 vmci_trans(vsk
)->qpair
= qpair
;
1476 vmci_trans(vsk
)->produce_size
= vmci_trans(vsk
)->consume_size
=
1479 vmci_trans(vsk
)->detach_sub_id
= detach_sub_id
;
1481 vmci_trans(vsk
)->notify_ops
->process_negotiate(sk
);
1486 if (detach_sub_id
!= VMCI_INVALID_ID
)
1487 vmci_event_unsubscribe(detach_sub_id
);
1489 if (!vmci_handle_is_invalid(handle
))
1490 vmci_qpair_detach(&qpair
);
1496 vmci_transport_recv_connecting_client_invalid(struct sock
*sk
,
1497 struct vmci_transport_packet
*pkt
)
1500 struct vsock_sock
*vsk
= vsock_sk(sk
);
1502 if (vsk
->sent_request
) {
1503 vsk
->sent_request
= false;
1504 vsk
->ignore_connecting_rst
= true;
1506 err
= vmci_transport_send_conn_request(sk
, vsk
->buffer_size
);
1508 err
= vmci_transport_error_to_vsock_error(err
);
1517 static int vmci_transport_recv_connected(struct sock
*sk
,
1518 struct vmci_transport_packet
*pkt
)
1520 struct vsock_sock
*vsk
;
1521 bool pkt_processed
= false;
1523 /* In cases where we are closing the connection, it's sufficient to
1524 * mark the state change (and maybe error) and wake up any waiting
1525 * threads. Since this is a connected socket, it's owned by a user
1526 * process and will be cleaned up when the failure is passed back on
1527 * the current or next system call. Our system call implementations
1528 * must therefore check for error and state changes on entry and when
1531 switch (pkt
->type
) {
1532 case VMCI_TRANSPORT_PACKET_TYPE_SHUTDOWN
:
1536 vsk
->peer_shutdown
|= pkt
->u
.mode
;
1537 sk
->sk_state_change(sk
);
1541 case VMCI_TRANSPORT_PACKET_TYPE_RST
:
1543 /* It is possible that we sent our peer a message (e.g a
1544 * WAITING_READ) right before we got notified that the peer had
1545 * detached. If that happens then we can get a RST pkt back
1546 * from our peer even though there is data available for us to
1547 * read. In that case, don't shutdown the socket completely but
1548 * instead allow the local client to finish reading data off
1549 * the queuepair. Always treat a RST pkt in connected mode like
1552 sock_set_flag(sk
, SOCK_DONE
);
1553 vsk
->peer_shutdown
= SHUTDOWN_MASK
;
1554 if (vsock_stream_has_data(vsk
) <= 0)
1555 sk
->sk_state
= TCP_CLOSING
;
1557 sk
->sk_state_change(sk
);
1562 vmci_trans(vsk
)->notify_ops
->handle_notify_pkt(
1563 sk
, pkt
, false, NULL
, NULL
,
1574 static int vmci_transport_socket_init(struct vsock_sock
*vsk
,
1575 struct vsock_sock
*psk
)
1577 vsk
->trans
= kmalloc(sizeof(struct vmci_transport
), GFP_KERNEL
);
1581 vmci_trans(vsk
)->dg_handle
= VMCI_INVALID_HANDLE
;
1582 vmci_trans(vsk
)->qp_handle
= VMCI_INVALID_HANDLE
;
1583 vmci_trans(vsk
)->qpair
= NULL
;
1584 vmci_trans(vsk
)->produce_size
= vmci_trans(vsk
)->consume_size
= 0;
1585 vmci_trans(vsk
)->detach_sub_id
= VMCI_INVALID_ID
;
1586 vmci_trans(vsk
)->notify_ops
= NULL
;
1587 INIT_LIST_HEAD(&vmci_trans(vsk
)->elem
);
1588 vmci_trans(vsk
)->sk
= &vsk
->sk
;
1589 spin_lock_init(&vmci_trans(vsk
)->lock
);
1594 static void vmci_transport_free_resources(struct list_head
*transport_list
)
1596 while (!list_empty(transport_list
)) {
1597 struct vmci_transport
*transport
=
1598 list_first_entry(transport_list
, struct vmci_transport
,
1600 list_del(&transport
->elem
);
1602 if (transport
->detach_sub_id
!= VMCI_INVALID_ID
) {
1603 vmci_event_unsubscribe(transport
->detach_sub_id
);
1604 transport
->detach_sub_id
= VMCI_INVALID_ID
;
1607 if (!vmci_handle_is_invalid(transport
->qp_handle
)) {
1608 vmci_qpair_detach(&transport
->qpair
);
1609 transport
->qp_handle
= VMCI_INVALID_HANDLE
;
1610 transport
->produce_size
= 0;
1611 transport
->consume_size
= 0;
1618 static void vmci_transport_cleanup(struct work_struct
*work
)
1622 spin_lock_bh(&vmci_transport_cleanup_lock
);
1623 list_replace_init(&vmci_transport_cleanup_list
, &pending
);
1624 spin_unlock_bh(&vmci_transport_cleanup_lock
);
1625 vmci_transport_free_resources(&pending
);
1628 static void vmci_transport_destruct(struct vsock_sock
*vsk
)
1630 /* transport can be NULL if we hit a failure at init() time */
1631 if (!vmci_trans(vsk
))
1634 /* Ensure that the detach callback doesn't use the sk/vsk
1635 * we are about to destruct.
1637 spin_lock_bh(&vmci_trans(vsk
)->lock
);
1638 vmci_trans(vsk
)->sk
= NULL
;
1639 spin_unlock_bh(&vmci_trans(vsk
)->lock
);
1641 if (vmci_trans(vsk
)->notify_ops
)
1642 vmci_trans(vsk
)->notify_ops
->socket_destruct(vsk
);
1644 spin_lock_bh(&vmci_transport_cleanup_lock
);
1645 list_add(&vmci_trans(vsk
)->elem
, &vmci_transport_cleanup_list
);
1646 spin_unlock_bh(&vmci_transport_cleanup_lock
);
1647 schedule_work(&vmci_transport_cleanup_work
);
1652 static void vmci_transport_release(struct vsock_sock
*vsk
)
1654 vsock_remove_sock(vsk
);
1656 if (!vmci_handle_is_invalid(vmci_trans(vsk
)->dg_handle
)) {
1657 vmci_datagram_destroy_handle(vmci_trans(vsk
)->dg_handle
);
1658 vmci_trans(vsk
)->dg_handle
= VMCI_INVALID_HANDLE
;
1662 static int vmci_transport_dgram_bind(struct vsock_sock
*vsk
,
1663 struct sockaddr_vm
*addr
)
1669 /* VMCI will select a resource ID for us if we provide
1672 port
= addr
->svm_port
== VMADDR_PORT_ANY
?
1673 VMCI_INVALID_ID
: addr
->svm_port
;
1675 if (port
<= LAST_RESERVED_PORT
&& !capable(CAP_NET_BIND_SERVICE
))
1678 flags
= addr
->svm_cid
== VMADDR_CID_ANY
?
1679 VMCI_FLAG_ANYCID_DG_HND
: 0;
1681 err
= vmci_transport_datagram_create_hnd(port
, flags
,
1682 vmci_transport_recv_dgram_cb
,
1684 &vmci_trans(vsk
)->dg_handle
);
1685 if (err
< VMCI_SUCCESS
)
1686 return vmci_transport_error_to_vsock_error(err
);
1687 vsock_addr_init(&vsk
->local_addr
, addr
->svm_cid
,
1688 vmci_trans(vsk
)->dg_handle
.resource
);
1693 static int vmci_transport_dgram_enqueue(
1694 struct vsock_sock
*vsk
,
1695 struct sockaddr_vm
*remote_addr
,
1700 struct vmci_datagram
*dg
;
1702 if (len
> VMCI_MAX_DG_PAYLOAD_SIZE
)
1705 if (!vmci_transport_allow_dgram(vsk
, remote_addr
->svm_cid
))
1708 /* Allocate a buffer for the user's message and our packet header. */
1709 dg
= kmalloc(len
+ sizeof(*dg
), GFP_KERNEL
);
1713 memcpy_from_msg(VMCI_DG_PAYLOAD(dg
), msg
, len
);
1715 dg
->dst
= vmci_make_handle(remote_addr
->svm_cid
,
1716 remote_addr
->svm_port
);
1717 dg
->src
= vmci_make_handle(vsk
->local_addr
.svm_cid
,
1718 vsk
->local_addr
.svm_port
);
1719 dg
->payload_size
= len
;
1721 err
= vmci_datagram_send(dg
);
1724 return vmci_transport_error_to_vsock_error(err
);
1726 return err
- sizeof(*dg
);
1729 static int vmci_transport_dgram_dequeue(struct vsock_sock
*vsk
,
1730 struct msghdr
*msg
, size_t len
,
1735 struct vmci_datagram
*dg
;
1737 struct sk_buff
*skb
;
1739 noblock
= flags
& MSG_DONTWAIT
;
1741 if (flags
& MSG_OOB
|| flags
& MSG_ERRQUEUE
)
1744 /* Retrieve the head sk_buff from the socket's receive queue. */
1746 skb
= skb_recv_datagram(&vsk
->sk
, flags
, noblock
, &err
);
1750 dg
= (struct vmci_datagram
*)skb
->data
;
1752 /* err is 0, meaning we read zero bytes. */
1755 payload_len
= dg
->payload_size
;
1756 /* Ensure the sk_buff matches the payload size claimed in the packet. */
1757 if (payload_len
!= skb
->len
- sizeof(*dg
)) {
1762 if (payload_len
> len
) {
1764 msg
->msg_flags
|= MSG_TRUNC
;
1767 /* Place the datagram payload in the user's iovec. */
1768 err
= skb_copy_datagram_msg(skb
, sizeof(*dg
), msg
, payload_len
);
1772 if (msg
->msg_name
) {
1773 /* Provide the address of the sender. */
1774 DECLARE_SOCKADDR(struct sockaddr_vm
*, vm_addr
, msg
->msg_name
);
1775 vsock_addr_init(vm_addr
, dg
->src
.context
, dg
->src
.resource
);
1776 msg
->msg_namelen
= sizeof(*vm_addr
);
1781 skb_free_datagram(&vsk
->sk
, skb
);
1785 static bool vmci_transport_dgram_allow(u32 cid
, u32 port
)
1787 if (cid
== VMADDR_CID_HYPERVISOR
) {
1788 /* Registrations of PBRPC Servers do not modify VMX/Hypervisor
1789 * state and are allowed.
1791 return port
== VMCI_UNITY_PBRPC_REGISTER
;
1797 static int vmci_transport_connect(struct vsock_sock
*vsk
)
1800 bool old_pkt_proto
= false;
1801 struct sock
*sk
= &vsk
->sk
;
1803 if (vmci_transport_old_proto_override(&old_pkt_proto
) &&
1805 err
= vmci_transport_send_conn_request(sk
, vsk
->buffer_size
);
1807 sk
->sk_state
= TCP_CLOSE
;
1811 int supported_proto_versions
=
1812 vmci_transport_new_proto_supported_versions();
1813 err
= vmci_transport_send_conn_request2(sk
, vsk
->buffer_size
,
1814 supported_proto_versions
);
1816 sk
->sk_state
= TCP_CLOSE
;
1820 vsk
->sent_request
= true;
1826 static ssize_t
vmci_transport_stream_dequeue(
1827 struct vsock_sock
*vsk
,
1832 if (flags
& MSG_PEEK
)
1833 return vmci_qpair_peekv(vmci_trans(vsk
)->qpair
, msg
, len
, 0);
1835 return vmci_qpair_dequev(vmci_trans(vsk
)->qpair
, msg
, len
, 0);
1838 static ssize_t
vmci_transport_stream_enqueue(
1839 struct vsock_sock
*vsk
,
1843 return vmci_qpair_enquev(vmci_trans(vsk
)->qpair
, msg
, len
, 0);
1846 static s64
vmci_transport_stream_has_data(struct vsock_sock
*vsk
)
1848 return vmci_qpair_consume_buf_ready(vmci_trans(vsk
)->qpair
);
1851 static s64
vmci_transport_stream_has_space(struct vsock_sock
*vsk
)
1853 return vmci_qpair_produce_free_space(vmci_trans(vsk
)->qpair
);
1856 static u64
vmci_transport_stream_rcvhiwat(struct vsock_sock
*vsk
)
1858 return vmci_trans(vsk
)->consume_size
;
1861 static bool vmci_transport_stream_is_active(struct vsock_sock
*vsk
)
1863 return !vmci_handle_is_invalid(vmci_trans(vsk
)->qp_handle
);
1866 static int vmci_transport_notify_poll_in(
1867 struct vsock_sock
*vsk
,
1869 bool *data_ready_now
)
1871 return vmci_trans(vsk
)->notify_ops
->poll_in(
1872 &vsk
->sk
, target
, data_ready_now
);
1875 static int vmci_transport_notify_poll_out(
1876 struct vsock_sock
*vsk
,
1878 bool *space_available_now
)
1880 return vmci_trans(vsk
)->notify_ops
->poll_out(
1881 &vsk
->sk
, target
, space_available_now
);
1884 static int vmci_transport_notify_recv_init(
1885 struct vsock_sock
*vsk
,
1887 struct vsock_transport_recv_notify_data
*data
)
1889 return vmci_trans(vsk
)->notify_ops
->recv_init(
1891 (struct vmci_transport_recv_notify_data
*)data
);
1894 static int vmci_transport_notify_recv_pre_block(
1895 struct vsock_sock
*vsk
,
1897 struct vsock_transport_recv_notify_data
*data
)
1899 return vmci_trans(vsk
)->notify_ops
->recv_pre_block(
1901 (struct vmci_transport_recv_notify_data
*)data
);
1904 static int vmci_transport_notify_recv_pre_dequeue(
1905 struct vsock_sock
*vsk
,
1907 struct vsock_transport_recv_notify_data
*data
)
1909 return vmci_trans(vsk
)->notify_ops
->recv_pre_dequeue(
1911 (struct vmci_transport_recv_notify_data
*)data
);
1914 static int vmci_transport_notify_recv_post_dequeue(
1915 struct vsock_sock
*vsk
,
1919 struct vsock_transport_recv_notify_data
*data
)
1921 return vmci_trans(vsk
)->notify_ops
->recv_post_dequeue(
1922 &vsk
->sk
, target
, copied
, data_read
,
1923 (struct vmci_transport_recv_notify_data
*)data
);
1926 static int vmci_transport_notify_send_init(
1927 struct vsock_sock
*vsk
,
1928 struct vsock_transport_send_notify_data
*data
)
1930 return vmci_trans(vsk
)->notify_ops
->send_init(
1932 (struct vmci_transport_send_notify_data
*)data
);
1935 static int vmci_transport_notify_send_pre_block(
1936 struct vsock_sock
*vsk
,
1937 struct vsock_transport_send_notify_data
*data
)
1939 return vmci_trans(vsk
)->notify_ops
->send_pre_block(
1941 (struct vmci_transport_send_notify_data
*)data
);
1944 static int vmci_transport_notify_send_pre_enqueue(
1945 struct vsock_sock
*vsk
,
1946 struct vsock_transport_send_notify_data
*data
)
1948 return vmci_trans(vsk
)->notify_ops
->send_pre_enqueue(
1950 (struct vmci_transport_send_notify_data
*)data
);
1953 static int vmci_transport_notify_send_post_enqueue(
1954 struct vsock_sock
*vsk
,
1956 struct vsock_transport_send_notify_data
*data
)
1958 return vmci_trans(vsk
)->notify_ops
->send_post_enqueue(
1960 (struct vmci_transport_send_notify_data
*)data
);
1963 static bool vmci_transport_old_proto_override(bool *old_pkt_proto
)
1965 if (PROTOCOL_OVERRIDE
!= -1) {
1966 if (PROTOCOL_OVERRIDE
== 0)
1967 *old_pkt_proto
= true;
1969 *old_pkt_proto
= false;
1971 pr_info("Proto override in use\n");
1978 static bool vmci_transport_proto_to_notify_struct(struct sock
*sk
,
1982 struct vsock_sock
*vsk
= vsock_sk(sk
);
1984 if (old_pkt_proto
) {
1985 if (*proto
!= VSOCK_PROTO_INVALID
) {
1986 pr_err("Can't set both an old and new protocol\n");
1989 vmci_trans(vsk
)->notify_ops
= &vmci_transport_notify_pkt_ops
;
1994 case VSOCK_PROTO_PKT_ON_NOTIFY
:
1995 vmci_trans(vsk
)->notify_ops
=
1996 &vmci_transport_notify_pkt_q_state_ops
;
1999 pr_err("Unknown notify protocol version\n");
2004 vmci_trans(vsk
)->notify_ops
->socket_init(sk
);
2008 static u16
vmci_transport_new_proto_supported_versions(void)
2010 if (PROTOCOL_OVERRIDE
!= -1)
2011 return PROTOCOL_OVERRIDE
;
2013 return VSOCK_PROTO_ALL_SUPPORTED
;
2016 static u32
vmci_transport_get_local_cid(void)
2018 return vmci_get_context_id();
2021 static struct vsock_transport vmci_transport
= {
2022 .module
= THIS_MODULE
,
2023 .init
= vmci_transport_socket_init
,
2024 .destruct
= vmci_transport_destruct
,
2025 .release
= vmci_transport_release
,
2026 .connect
= vmci_transport_connect
,
2027 .dgram_bind
= vmci_transport_dgram_bind
,
2028 .dgram_dequeue
= vmci_transport_dgram_dequeue
,
2029 .dgram_enqueue
= vmci_transport_dgram_enqueue
,
2030 .dgram_allow
= vmci_transport_dgram_allow
,
2031 .stream_dequeue
= vmci_transport_stream_dequeue
,
2032 .stream_enqueue
= vmci_transport_stream_enqueue
,
2033 .stream_has_data
= vmci_transport_stream_has_data
,
2034 .stream_has_space
= vmci_transport_stream_has_space
,
2035 .stream_rcvhiwat
= vmci_transport_stream_rcvhiwat
,
2036 .stream_is_active
= vmci_transport_stream_is_active
,
2037 .stream_allow
= vmci_transport_stream_allow
,
2038 .notify_poll_in
= vmci_transport_notify_poll_in
,
2039 .notify_poll_out
= vmci_transport_notify_poll_out
,
2040 .notify_recv_init
= vmci_transport_notify_recv_init
,
2041 .notify_recv_pre_block
= vmci_transport_notify_recv_pre_block
,
2042 .notify_recv_pre_dequeue
= vmci_transport_notify_recv_pre_dequeue
,
2043 .notify_recv_post_dequeue
= vmci_transport_notify_recv_post_dequeue
,
2044 .notify_send_init
= vmci_transport_notify_send_init
,
2045 .notify_send_pre_block
= vmci_transport_notify_send_pre_block
,
2046 .notify_send_pre_enqueue
= vmci_transport_notify_send_pre_enqueue
,
2047 .notify_send_post_enqueue
= vmci_transport_notify_send_post_enqueue
,
2048 .shutdown
= vmci_transport_shutdown
,
2049 .get_local_cid
= vmci_transport_get_local_cid
,
2052 static bool vmci_check_transport(struct vsock_sock
*vsk
)
2054 return vsk
->transport
== &vmci_transport
;
2057 static void vmci_vsock_transport_cb(bool is_host
)
2062 features
= VSOCK_TRANSPORT_F_H2G
;
2064 features
= VSOCK_TRANSPORT_F_G2H
;
2066 vsock_core_register(&vmci_transport
, features
);
2069 static int __init
vmci_transport_init(void)
2073 /* Create the datagram handle that we will use to send and receive all
2074 * VSocket control messages for this context.
2076 err
= vmci_transport_datagram_create_hnd(VMCI_TRANSPORT_PACKET_RID
,
2077 VMCI_FLAG_ANYCID_DG_HND
,
2078 vmci_transport_recv_stream_cb
,
2080 &vmci_transport_stream_handle
);
2081 if (err
< VMCI_SUCCESS
) {
2082 pr_err("Unable to create datagram handle. (%d)\n", err
);
2083 return vmci_transport_error_to_vsock_error(err
);
2085 err
= vmci_event_subscribe(VMCI_EVENT_QP_RESUMED
,
2086 vmci_transport_qp_resumed_cb
,
2087 NULL
, &vmci_transport_qp_resumed_sub_id
);
2088 if (err
< VMCI_SUCCESS
) {
2089 pr_err("Unable to subscribe to resumed event. (%d)\n", err
);
2090 err
= vmci_transport_error_to_vsock_error(err
);
2091 vmci_transport_qp_resumed_sub_id
= VMCI_INVALID_ID
;
2092 goto err_destroy_stream_handle
;
2095 /* Register only with dgram feature, other features (H2G, G2H) will be
2096 * registered when the first host or guest becomes active.
2098 err
= vsock_core_register(&vmci_transport
, VSOCK_TRANSPORT_F_DGRAM
);
2100 goto err_unsubscribe
;
2102 err
= vmci_register_vsock_callback(vmci_vsock_transport_cb
);
2104 goto err_unregister
;
2109 vsock_core_unregister(&vmci_transport
);
2111 vmci_event_unsubscribe(vmci_transport_qp_resumed_sub_id
);
2112 err_destroy_stream_handle
:
2113 vmci_datagram_destroy_handle(vmci_transport_stream_handle
);
2116 module_init(vmci_transport_init
);
2118 static void __exit
vmci_transport_exit(void)
2120 cancel_work_sync(&vmci_transport_cleanup_work
);
2121 vmci_transport_free_resources(&vmci_transport_cleanup_list
);
2123 if (!vmci_handle_is_invalid(vmci_transport_stream_handle
)) {
2124 if (vmci_datagram_destroy_handle(
2125 vmci_transport_stream_handle
) != VMCI_SUCCESS
)
2126 pr_err("Couldn't destroy datagram handle\n");
2127 vmci_transport_stream_handle
= VMCI_INVALID_HANDLE
;
2130 if (vmci_transport_qp_resumed_sub_id
!= VMCI_INVALID_ID
) {
2131 vmci_event_unsubscribe(vmci_transport_qp_resumed_sub_id
);
2132 vmci_transport_qp_resumed_sub_id
= VMCI_INVALID_ID
;
2135 vmci_register_vsock_callback(NULL
);
2136 vsock_core_unregister(&vmci_transport
);
2138 module_exit(vmci_transport_exit
);
2140 MODULE_AUTHOR("VMware, Inc.");
2141 MODULE_DESCRIPTION("VMCI transport for Virtual Sockets");
2142 MODULE_VERSION("1.0.5.0-k");
2143 MODULE_LICENSE("GPL v2");
2144 MODULE_ALIAS("vmware_vsock");
2145 MODULE_ALIAS_NETPROTO(PF_VSOCK
);