1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2017 - 2019, Intel Corporation.
7 #define pr_fmt(fmt) "MPTCP: " fmt
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <crypto/algapi.h>
13 #include <crypto/sha2.h>
15 #include <net/inet_common.h>
16 #include <net/inet_hashtables.h>
17 #include <net/protocol.h>
19 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
20 #include <net/ip6_route.h>
22 #include <net/mptcp.h>
23 #include <uapi/linux/mptcp.h>
27 static void SUBFLOW_REQ_INC_STATS(struct request_sock
*req
,
28 enum linux_mptcp_mib_field field
)
30 MPTCP_INC_STATS(sock_net(req_to_sk(req
)), field
);
33 static void subflow_req_destructor(struct request_sock
*req
)
35 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
37 pr_debug("subflow_req=%p", subflow_req
);
40 sock_put((struct sock
*)subflow_req
->msk
);
42 mptcp_token_destroy_request(req
);
43 tcp_request_sock_ops
.destructor(req
);
46 static void subflow_generate_hmac(u64 key1
, u64 key2
, u32 nonce1
, u32 nonce2
,
51 put_unaligned_be32(nonce1
, &msg
[0]);
52 put_unaligned_be32(nonce2
, &msg
[4]);
54 mptcp_crypto_hmac_sha(key1
, key2
, msg
, 8, hmac
);
57 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock
*msk
)
59 return mptcp_is_fully_established((void *)msk
) &&
60 READ_ONCE(msk
->pm
.accept_subflow
);
63 /* validate received token and create truncated hmac and nonce for SYN-ACK */
64 static struct mptcp_sock
*subflow_token_join_request(struct request_sock
*req
,
65 const struct sk_buff
*skb
)
67 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
68 u8 hmac
[SHA256_DIGEST_SIZE
];
69 struct mptcp_sock
*msk
;
72 msk
= mptcp_token_get_sock(subflow_req
->token
);
74 SUBFLOW_REQ_INC_STATS(req
, MPTCP_MIB_JOINNOTOKEN
);
78 local_id
= mptcp_pm_get_local_id(msk
, (struct sock_common
*)req
);
80 sock_put((struct sock
*)msk
);
83 subflow_req
->local_id
= local_id
;
85 get_random_bytes(&subflow_req
->local_nonce
, sizeof(u32
));
87 subflow_generate_hmac(msk
->local_key
, msk
->remote_key
,
88 subflow_req
->local_nonce
,
89 subflow_req
->remote_nonce
, hmac
);
91 subflow_req
->thmac
= get_unaligned_be64(hmac
);
95 static void subflow_init_req(struct request_sock
*req
, const struct sock
*sk_listener
)
97 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
99 subflow_req
->mp_capable
= 0;
100 subflow_req
->mp_join
= 0;
101 subflow_req
->msk
= NULL
;
102 mptcp_token_init_request(req
);
105 /* Init mptcp request socket.
107 * Returns an error code if a JOIN has failed and a TCP reset
110 static int subflow_check_req(struct request_sock
*req
,
111 const struct sock
*sk_listener
,
114 struct mptcp_subflow_context
*listener
= mptcp_subflow_ctx(sk_listener
);
115 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
116 struct mptcp_options_received mp_opt
;
118 pr_debug("subflow_req=%p, listener=%p", subflow_req
, listener
);
120 #ifdef CONFIG_TCP_MD5SIG
121 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
124 if (rcu_access_pointer(tcp_sk(sk_listener
)->md5sig_info
))
128 mptcp_get_options(skb
, &mp_opt
);
130 if (mp_opt
.mp_capable
) {
131 SUBFLOW_REQ_INC_STATS(req
, MPTCP_MIB_MPCAPABLEPASSIVE
);
135 } else if (mp_opt
.mp_join
) {
136 SUBFLOW_REQ_INC_STATS(req
, MPTCP_MIB_JOINSYNRX
);
139 if (mp_opt
.mp_capable
&& listener
->request_mptcp
) {
140 int err
, retries
= 4;
142 subflow_req
->ssn_offset
= TCP_SKB_CB(skb
)->seq
;
145 get_random_bytes(&subflow_req
->local_key
, sizeof(subflow_req
->local_key
));
146 } while (subflow_req
->local_key
== 0);
148 if (unlikely(req
->syncookie
)) {
149 mptcp_crypto_key_sha(subflow_req
->local_key
,
152 if (mptcp_token_exists(subflow_req
->token
)) {
156 subflow_req
->mp_capable
= 1;
161 err
= mptcp_token_new_request(req
);
163 subflow_req
->mp_capable
= 1;
164 else if (retries
-- > 0)
167 } else if (mp_opt
.mp_join
&& listener
->request_mptcp
) {
168 subflow_req
->ssn_offset
= TCP_SKB_CB(skb
)->seq
;
169 subflow_req
->mp_join
= 1;
170 subflow_req
->backup
= mp_opt
.backup
;
171 subflow_req
->remote_id
= mp_opt
.join_id
;
172 subflow_req
->token
= mp_opt
.token
;
173 subflow_req
->remote_nonce
= mp_opt
.nonce
;
174 subflow_req
->msk
= subflow_token_join_request(req
, skb
);
176 /* Can't fall back to TCP in this case. */
177 if (!subflow_req
->msk
)
180 if (unlikely(req
->syncookie
)) {
181 if (mptcp_can_accept_new_subflow(subflow_req
->msk
))
182 subflow_init_req_cookie_join_save(subflow_req
, skb
);
185 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req
->token
,
186 subflow_req
->remote_nonce
, subflow_req
->msk
);
192 int mptcp_subflow_init_cookie_req(struct request_sock
*req
,
193 const struct sock
*sk_listener
,
196 struct mptcp_subflow_context
*listener
= mptcp_subflow_ctx(sk_listener
);
197 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
198 struct mptcp_options_received mp_opt
;
201 subflow_init_req(req
, sk_listener
);
202 mptcp_get_options(skb
, &mp_opt
);
204 if (mp_opt
.mp_capable
&& mp_opt
.mp_join
)
207 if (mp_opt
.mp_capable
&& listener
->request_mptcp
) {
208 if (mp_opt
.sndr_key
== 0)
211 subflow_req
->local_key
= mp_opt
.rcvr_key
;
212 err
= mptcp_token_new_request(req
);
216 subflow_req
->mp_capable
= 1;
217 subflow_req
->ssn_offset
= TCP_SKB_CB(skb
)->seq
- 1;
218 } else if (mp_opt
.mp_join
&& listener
->request_mptcp
) {
219 if (!mptcp_token_join_cookie_init_state(subflow_req
, skb
))
222 if (mptcp_can_accept_new_subflow(subflow_req
->msk
))
223 subflow_req
->mp_join
= 1;
225 subflow_req
->ssn_offset
= TCP_SKB_CB(skb
)->seq
- 1;
230 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req
);
232 static struct dst_entry
*subflow_v4_route_req(const struct sock
*sk
,
235 struct request_sock
*req
)
237 struct dst_entry
*dst
;
240 tcp_rsk(req
)->is_mptcp
= 1;
241 subflow_init_req(req
, sk
);
243 dst
= tcp_request_sock_ipv4_ops
.route_req(sk
, skb
, fl
, req
);
247 err
= subflow_check_req(req
, sk
, skb
);
253 tcp_request_sock_ops
.send_reset(sk
, skb
);
257 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
258 static struct dst_entry
*subflow_v6_route_req(const struct sock
*sk
,
261 struct request_sock
*req
)
263 struct dst_entry
*dst
;
266 tcp_rsk(req
)->is_mptcp
= 1;
267 subflow_init_req(req
, sk
);
269 dst
= tcp_request_sock_ipv6_ops
.route_req(sk
, skb
, fl
, req
);
273 err
= subflow_check_req(req
, sk
, skb
);
279 tcp6_request_sock_ops
.send_reset(sk
, skb
);
284 /* validate received truncated hmac and create hmac for third ACK */
285 static bool subflow_thmac_valid(struct mptcp_subflow_context
*subflow
)
287 u8 hmac
[SHA256_DIGEST_SIZE
];
290 subflow_generate_hmac(subflow
->remote_key
, subflow
->local_key
,
291 subflow
->remote_nonce
, subflow
->local_nonce
,
294 thmac
= get_unaligned_be64(hmac
);
295 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
296 subflow
, subflow
->token
,
297 (unsigned long long)thmac
,
298 (unsigned long long)subflow
->thmac
);
300 return thmac
== subflow
->thmac
;
303 void mptcp_subflow_reset(struct sock
*ssk
)
305 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
306 struct sock
*sk
= subflow
->conn
;
308 /* must hold: tcp_done() could drop last reference on parent */
311 tcp_set_state(ssk
, TCP_CLOSE
);
312 tcp_send_active_reset(ssk
, GFP_ATOMIC
);
314 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW
, &mptcp_sk(sk
)->flags
) &&
315 schedule_work(&mptcp_sk(sk
)->work
))
316 return; /* worker will put sk for us */
321 static void subflow_finish_connect(struct sock
*sk
, const struct sk_buff
*skb
)
323 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
324 struct mptcp_options_received mp_opt
;
325 struct sock
*parent
= subflow
->conn
;
327 subflow
->icsk_af_ops
->sk_rx_dst_set(sk
, skb
);
329 if (inet_sk_state_load(parent
) == TCP_SYN_SENT
) {
330 inet_sk_state_store(parent
, TCP_ESTABLISHED
);
331 parent
->sk_state_change(parent
);
334 /* be sure no special action on any packet other than syn-ack */
335 if (subflow
->conn_finished
)
338 subflow
->rel_write_seq
= 1;
339 subflow
->conn_finished
= 1;
340 subflow
->ssn_offset
= TCP_SKB_CB(skb
)->seq
;
341 pr_debug("subflow=%p synack seq=%x", subflow
, subflow
->ssn_offset
);
343 mptcp_get_options(skb
, &mp_opt
);
344 if (subflow
->request_mptcp
) {
345 if (!mp_opt
.mp_capable
) {
346 MPTCP_INC_STATS(sock_net(sk
),
347 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK
);
348 mptcp_do_fallback(sk
);
349 pr_fallback(mptcp_sk(subflow
->conn
));
353 subflow
->mp_capable
= 1;
354 subflow
->can_ack
= 1;
355 subflow
->remote_key
= mp_opt
.sndr_key
;
356 pr_debug("subflow=%p, remote_key=%llu", subflow
,
357 subflow
->remote_key
);
358 mptcp_finish_connect(sk
);
359 } else if (subflow
->request_join
) {
360 u8 hmac
[SHA256_DIGEST_SIZE
];
365 subflow
->thmac
= mp_opt
.thmac
;
366 subflow
->remote_nonce
= mp_opt
.nonce
;
367 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow
,
368 subflow
->thmac
, subflow
->remote_nonce
);
370 if (!subflow_thmac_valid(subflow
)) {
371 MPTCP_INC_STATS(sock_net(sk
), MPTCP_MIB_JOINACKMAC
);
375 subflow_generate_hmac(subflow
->local_key
, subflow
->remote_key
,
376 subflow
->local_nonce
,
377 subflow
->remote_nonce
,
379 memcpy(subflow
->hmac
, hmac
, MPTCPOPT_HMAC_LEN
);
381 if (!mptcp_finish_join(sk
))
384 subflow
->mp_join
= 1;
385 MPTCP_INC_STATS(sock_net(sk
), MPTCP_MIB_JOINSYNACKRX
);
386 } else if (mptcp_check_fallback(sk
)) {
388 mptcp_rcv_space_init(mptcp_sk(parent
), sk
);
393 mptcp_subflow_reset(sk
);
396 struct request_sock_ops mptcp_subflow_request_sock_ops
;
397 EXPORT_SYMBOL_GPL(mptcp_subflow_request_sock_ops
);
398 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops
;
400 static int subflow_v4_conn_request(struct sock
*sk
, struct sk_buff
*skb
)
402 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
404 pr_debug("subflow=%p", subflow
);
406 /* Never answer to SYNs sent to broadcast or multicast */
407 if (skb_rtable(skb
)->rt_flags
& (RTCF_BROADCAST
| RTCF_MULTICAST
))
410 return tcp_conn_request(&mptcp_subflow_request_sock_ops
,
411 &subflow_request_sock_ipv4_ops
,
418 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
419 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops
;
420 static struct inet_connection_sock_af_ops subflow_v6_specific
;
421 static struct inet_connection_sock_af_ops subflow_v6m_specific
;
423 static int subflow_v6_conn_request(struct sock
*sk
, struct sk_buff
*skb
)
425 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
427 pr_debug("subflow=%p", subflow
);
429 if (skb
->protocol
== htons(ETH_P_IP
))
430 return subflow_v4_conn_request(sk
, skb
);
432 if (!ipv6_unicast_destination(skb
))
435 if (ipv6_addr_v4mapped(&ipv6_hdr(skb
)->saddr
)) {
436 __IP6_INC_STATS(sock_net(sk
), NULL
, IPSTATS_MIB_INHDRERRORS
);
440 return tcp_conn_request(&mptcp_subflow_request_sock_ops
,
441 &subflow_request_sock_ipv6_ops
, sk
, skb
);
445 return 0; /* don't send reset */
449 /* validate hmac received in third ACK */
450 static bool subflow_hmac_valid(const struct request_sock
*req
,
451 const struct mptcp_options_received
*mp_opt
)
453 const struct mptcp_subflow_request_sock
*subflow_req
;
454 u8 hmac
[SHA256_DIGEST_SIZE
];
455 struct mptcp_sock
*msk
;
457 subflow_req
= mptcp_subflow_rsk(req
);
458 msk
= subflow_req
->msk
;
462 subflow_generate_hmac(msk
->remote_key
, msk
->local_key
,
463 subflow_req
->remote_nonce
,
464 subflow_req
->local_nonce
, hmac
);
466 return !crypto_memneq(hmac
, mp_opt
->hmac
, MPTCPOPT_HMAC_LEN
);
469 static void mptcp_sock_destruct(struct sock
*sk
)
471 /* if new mptcp socket isn't accepted, it is free'd
472 * from the tcp listener sockets request queue, linked
473 * from req->sk. The tcp socket is released.
474 * This calls the ULP release function which will
475 * also remove the mptcp socket, via
476 * sock_put(ctx->conn).
478 * Problem is that the mptcp socket will be in
479 * ESTABLISHED state and will not have the SOCK_DEAD flag.
480 * Both result in warnings from inet_sock_destruct.
482 if ((1 << sk
->sk_state
) & (TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
)) {
483 sk
->sk_state
= TCP_CLOSE
;
484 WARN_ON_ONCE(sk
->sk_socket
);
488 mptcp_destroy_common(mptcp_sk(sk
));
489 inet_sock_destruct(sk
);
492 static void mptcp_force_close(struct sock
*sk
)
494 inet_sk_state_store(sk
, TCP_CLOSE
);
495 sk_common_release(sk
);
498 static void subflow_ulp_fallback(struct sock
*sk
,
499 struct mptcp_subflow_context
*old_ctx
)
501 struct inet_connection_sock
*icsk
= inet_csk(sk
);
503 mptcp_subflow_tcp_fallback(sk
, old_ctx
);
504 icsk
->icsk_ulp_ops
= NULL
;
505 rcu_assign_pointer(icsk
->icsk_ulp_data
, NULL
);
506 tcp_sk(sk
)->is_mptcp
= 0;
509 static void subflow_drop_ctx(struct sock
*ssk
)
511 struct mptcp_subflow_context
*ctx
= mptcp_subflow_ctx(ssk
);
516 subflow_ulp_fallback(ssk
, ctx
);
523 void mptcp_subflow_fully_established(struct mptcp_subflow_context
*subflow
,
524 struct mptcp_options_received
*mp_opt
)
526 struct mptcp_sock
*msk
= mptcp_sk(subflow
->conn
);
528 subflow
->remote_key
= mp_opt
->sndr_key
;
529 subflow
->fully_established
= 1;
530 subflow
->can_ack
= 1;
531 WRITE_ONCE(msk
->fully_established
, true);
534 static struct sock
*subflow_syn_recv_sock(const struct sock
*sk
,
536 struct request_sock
*req
,
537 struct dst_entry
*dst
,
538 struct request_sock
*req_unhash
,
541 struct mptcp_subflow_context
*listener
= mptcp_subflow_ctx(sk
);
542 struct mptcp_subflow_request_sock
*subflow_req
;
543 struct mptcp_options_received mp_opt
;
544 bool fallback
, fallback_is_fatal
;
545 struct sock
*new_msk
= NULL
;
548 pr_debug("listener=%p, req=%p, conn=%p", listener
, req
, listener
->conn
);
550 /* After child creation we must look for 'mp_capable' even when options
553 mp_opt
.mp_capable
= 0;
555 /* hopefully temporary handling for MP_JOIN+syncookie */
556 subflow_req
= mptcp_subflow_rsk(req
);
557 fallback_is_fatal
= tcp_rsk(req
)->is_mptcp
&& subflow_req
->mp_join
;
558 fallback
= !tcp_rsk(req
)->is_mptcp
;
562 /* if the sk is MP_CAPABLE, we try to fetch the client key */
563 if (subflow_req
->mp_capable
) {
564 /* we can receive and accept an in-window, out-of-order pkt,
565 * which may not carry the MP_CAPABLE opt even on mptcp enabled
566 * paths: always try to extract the peer key, and fallback
567 * for packets missing it.
568 * Even OoO DSS packets coming legitly after dropped or
569 * reordered MPC will cause fallback, but we don't have other
572 mptcp_get_options(skb
, &mp_opt
);
573 if (!mp_opt
.mp_capable
) {
578 new_msk
= mptcp_sk_clone(listener
->conn
, &mp_opt
, req
);
581 } else if (subflow_req
->mp_join
) {
582 mptcp_get_options(skb
, &mp_opt
);
583 if (!mp_opt
.mp_join
|| !subflow_hmac_valid(req
, &mp_opt
) ||
584 !mptcp_can_accept_new_subflow(subflow_req
->msk
)) {
585 SUBFLOW_REQ_INC_STATS(req
, MPTCP_MIB_JOINACKMAC
);
591 child
= listener
->icsk_af_ops
->syn_recv_sock(sk
, skb
, req
, dst
,
592 req_unhash
, own_req
);
594 if (child
&& *own_req
) {
595 struct mptcp_subflow_context
*ctx
= mptcp_subflow_ctx(child
);
597 tcp_rsk(req
)->drop_req
= false;
599 /* we need to fallback on ctx allocation failure and on pre-reqs
600 * checking above. In the latter scenario we additionally need
601 * to reset the context to non MPTCP status.
603 if (!ctx
|| fallback
) {
604 if (fallback_is_fatal
)
607 subflow_drop_ctx(child
);
611 if (ctx
->mp_capable
) {
612 /* this can't race with mptcp_close(), as the msk is
613 * not yet exposted to user-space
615 inet_sk_state_store((void *)new_msk
, TCP_ESTABLISHED
);
617 /* record the newly created socket as the first msk
618 * subflow, but don't link it yet into conn_list
620 WRITE_ONCE(mptcp_sk(new_msk
)->first
, child
);
622 /* new mpc subflow takes ownership of the newly
623 * created mptcp socket
625 new_msk
->sk_destruct
= mptcp_sock_destruct
;
626 mptcp_pm_new_connection(mptcp_sk(new_msk
), 1);
627 mptcp_token_accept(subflow_req
, mptcp_sk(new_msk
));
631 /* with OoO packets we can reach here without ingress
634 if (mp_opt
.mp_capable
)
635 mptcp_subflow_fully_established(ctx
, &mp_opt
);
636 } else if (ctx
->mp_join
) {
637 struct mptcp_sock
*owner
;
639 owner
= subflow_req
->msk
;
643 /* move the msk reference ownership to the subflow */
644 subflow_req
->msk
= NULL
;
645 ctx
->conn
= (struct sock
*)owner
;
646 if (!mptcp_finish_join(child
))
649 SUBFLOW_REQ_INC_STATS(req
, MPTCP_MIB_JOINACKRX
);
650 tcp_rsk(req
)->drop_req
= true;
655 /* dispose of the left over mptcp master, if any */
656 if (unlikely(new_msk
))
657 mptcp_force_close(new_msk
);
659 /* check for expected invariant - should never trigger, just help
660 * catching eariler subtle bugs
662 WARN_ON_ONCE(child
&& *own_req
&& tcp_sk(child
)->is_mptcp
&&
663 (!mptcp_subflow_ctx(child
) ||
664 !mptcp_subflow_ctx(child
)->conn
));
668 subflow_drop_ctx(child
);
669 tcp_rsk(req
)->drop_req
= true;
670 inet_csk_prepare_for_destroy_sock(child
);
672 req
->rsk_ops
->send_reset(sk
, skb
);
674 /* The last child reference will be released by the caller */
678 static struct inet_connection_sock_af_ops subflow_specific
;
680 enum mapping_status
{
688 static u64
expand_seq(u64 old_seq
, u16 old_data_len
, u64 seq
)
690 if ((u32
)seq
== (u32
)old_seq
)
693 /* Assume map covers data not mapped yet. */
694 return seq
| ((old_seq
+ old_data_len
+ 1) & GENMASK_ULL(63, 32));
697 static void warn_bad_map(struct mptcp_subflow_context
*subflow
, u32 ssn
)
699 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d",
700 ssn
, subflow
->map_subflow_seq
, subflow
->map_data_len
);
703 static bool skb_is_fully_mapped(struct sock
*ssk
, struct sk_buff
*skb
)
705 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
706 unsigned int skb_consumed
;
708 skb_consumed
= tcp_sk(ssk
)->copied_seq
- TCP_SKB_CB(skb
)->seq
;
709 if (WARN_ON_ONCE(skb_consumed
>= skb
->len
))
712 return skb
->len
- skb_consumed
<= subflow
->map_data_len
-
713 mptcp_subflow_get_map_offset(subflow
);
716 static bool validate_mapping(struct sock
*ssk
, struct sk_buff
*skb
)
718 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
719 u32 ssn
= tcp_sk(ssk
)->copied_seq
- subflow
->ssn_offset
;
721 if (unlikely(before(ssn
, subflow
->map_subflow_seq
))) {
722 /* Mapping covers data later in the subflow stream,
723 * currently unsupported.
725 warn_bad_map(subflow
, ssn
);
728 if (unlikely(!before(ssn
, subflow
->map_subflow_seq
+
729 subflow
->map_data_len
))) {
730 /* Mapping does covers past subflow data, invalid */
731 warn_bad_map(subflow
, ssn
+ skb
->len
);
737 static enum mapping_status
get_mapping_status(struct sock
*ssk
,
738 struct mptcp_sock
*msk
)
740 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
741 struct mptcp_ext
*mpext
;
746 skb
= skb_peek(&ssk
->sk_receive_queue
);
748 return MAPPING_EMPTY
;
750 if (mptcp_check_fallback(ssk
))
751 return MAPPING_DUMMY
;
753 mpext
= mptcp_get_ext(skb
);
754 if (!mpext
|| !mpext
->use_map
) {
755 if (!subflow
->map_valid
&& !skb
->len
) {
756 /* the TCP stack deliver 0 len FIN pkt to the receive
757 * queue, that is the only 0len pkts ever expected here,
758 * and we can admit no mapping only for 0 len pkts
760 if (!(TCP_SKB_CB(skb
)->tcp_flags
& TCPHDR_FIN
))
761 WARN_ONCE(1, "0len seq %d:%d flags %x",
762 TCP_SKB_CB(skb
)->seq
,
763 TCP_SKB_CB(skb
)->end_seq
,
764 TCP_SKB_CB(skb
)->tcp_flags
);
765 sk_eat_skb(ssk
, skb
);
766 return MAPPING_EMPTY
;
769 if (!subflow
->map_valid
)
770 return MAPPING_INVALID
;
775 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d",
776 mpext
->data_seq
, mpext
->dsn64
, mpext
->subflow_seq
,
777 mpext
->data_len
, mpext
->data_fin
);
779 data_len
= mpext
->data_len
;
781 MPTCP_INC_STATS(sock_net(ssk
), MPTCP_MIB_INFINITEMAPRX
);
782 return MAPPING_INVALID
;
785 if (mpext
->data_fin
== 1) {
787 bool updated
= mptcp_update_rcv_data_fin(msk
, mpext
->data_seq
,
789 pr_debug("DATA_FIN with no payload seq=%llu", mpext
->data_seq
);
790 if (subflow
->map_valid
) {
791 /* A DATA_FIN might arrive in a DSS
792 * option before the previous mapping
793 * has been fully consumed. Continue
794 * handling the existing mapping.
796 skb_ext_del(skb
, SKB_EXT_MPTCP
);
799 if (updated
&& schedule_work(&msk
->work
))
800 sock_hold((struct sock
*)msk
);
802 return MAPPING_DATA_FIN
;
805 u64 data_fin_seq
= mpext
->data_seq
+ data_len
- 1;
807 /* If mpext->data_seq is a 32-bit value, data_fin_seq
808 * must also be limited to 32 bits.
811 data_fin_seq
&= GENMASK_ULL(31, 0);
813 mptcp_update_rcv_data_fin(msk
, data_fin_seq
, mpext
->dsn64
);
814 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d",
815 data_fin_seq
, mpext
->dsn64
);
818 /* Adjust for DATA_FIN using 1 byte of sequence space */
823 map_seq
= expand_seq(subflow
->map_seq
, subflow
->map_data_len
,
825 pr_debug("expanded seq=%llu", subflow
->map_seq
);
827 map_seq
= mpext
->data_seq
;
829 WRITE_ONCE(mptcp_sk(subflow
->conn
)->use_64bit_ack
, !!mpext
->dsn64
);
831 if (subflow
->map_valid
) {
832 /* Allow replacing only with an identical map */
833 if (subflow
->map_seq
== map_seq
&&
834 subflow
->map_subflow_seq
== mpext
->subflow_seq
&&
835 subflow
->map_data_len
== data_len
) {
836 skb_ext_del(skb
, SKB_EXT_MPTCP
);
840 /* If this skb data are fully covered by the current mapping,
841 * the new map would need caching, which is not supported
843 if (skb_is_fully_mapped(ssk
, skb
)) {
844 MPTCP_INC_STATS(sock_net(ssk
), MPTCP_MIB_DSSNOMATCH
);
845 return MAPPING_INVALID
;
848 /* will validate the next map after consuming the current one */
852 subflow
->map_seq
= map_seq
;
853 subflow
->map_subflow_seq
= mpext
->subflow_seq
;
854 subflow
->map_data_len
= data_len
;
855 subflow
->map_valid
= 1;
856 subflow
->mpc_map
= mpext
->mpc_map
;
857 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u",
858 subflow
->map_seq
, subflow
->map_subflow_seq
,
859 subflow
->map_data_len
);
862 /* we revalidate valid mapping on new skb, because we must ensure
863 * the current skb is completely covered by the available mapping
865 if (!validate_mapping(ssk
, skb
))
866 return MAPPING_INVALID
;
868 skb_ext_del(skb
, SKB_EXT_MPTCP
);
872 static void mptcp_subflow_discard_data(struct sock
*ssk
, struct sk_buff
*skb
,
875 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
876 bool fin
= TCP_SKB_CB(skb
)->tcp_flags
& TCPHDR_FIN
;
879 incr
= limit
>= skb
->len
? skb
->len
+ fin
: limit
;
881 pr_debug("discarding=%d len=%d seq=%d", incr
, skb
->len
,
882 subflow
->map_subflow_seq
);
883 MPTCP_INC_STATS(sock_net(ssk
), MPTCP_MIB_DUPDATA
);
884 tcp_sk(ssk
)->copied_seq
+= incr
;
885 if (!before(tcp_sk(ssk
)->copied_seq
, TCP_SKB_CB(skb
)->end_seq
))
886 sk_eat_skb(ssk
, skb
);
887 if (mptcp_subflow_get_map_offset(subflow
) >= subflow
->map_data_len
)
888 subflow
->map_valid
= 0;
891 static bool subflow_check_data_avail(struct sock
*ssk
)
893 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
894 enum mapping_status status
;
895 struct mptcp_sock
*msk
;
898 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow
->conn
, ssk
,
899 subflow
->data_avail
, skb_peek(&ssk
->sk_receive_queue
));
900 if (!skb_peek(&ssk
->sk_receive_queue
))
901 subflow
->data_avail
= 0;
902 if (subflow
->data_avail
)
905 msk
= mptcp_sk(subflow
->conn
);
910 status
= get_mapping_status(ssk
, msk
);
911 if (unlikely(status
== MAPPING_INVALID
))
914 if (unlikely(status
== MAPPING_DUMMY
))
917 if (status
!= MAPPING_OK
)
920 skb
= skb_peek(&ssk
->sk_receive_queue
);
921 if (WARN_ON_ONCE(!skb
))
924 /* if msk lacks the remote key, this subflow must provide an
925 * MP_CAPABLE-based mapping
927 if (unlikely(!READ_ONCE(msk
->can_ack
))) {
928 if (!subflow
->mpc_map
)
930 WRITE_ONCE(msk
->remote_key
, subflow
->remote_key
);
931 WRITE_ONCE(msk
->ack_seq
, subflow
->map_seq
);
932 WRITE_ONCE(msk
->can_ack
, true);
935 old_ack
= READ_ONCE(msk
->ack_seq
);
936 ack_seq
= mptcp_subflow_get_mapped_dsn(subflow
);
937 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack
,
939 if (ack_seq
== old_ack
) {
940 subflow
->data_avail
= MPTCP_SUBFLOW_DATA_AVAIL
;
942 } else if (after64(ack_seq
, old_ack
)) {
943 subflow
->data_avail
= MPTCP_SUBFLOW_OOO_DATA
;
947 /* only accept in-sequence mapping. Old values are spurious
950 mptcp_subflow_discard_data(ssk
, skb
, old_ack
- ack_seq
);
955 /* RFC 8684 section 3.7. */
956 if (subflow
->mp_join
|| subflow
->fully_established
) {
957 /* fatal protocol error, close the socket.
958 * subflow_error_report() will introduce the appropriate barriers
960 ssk
->sk_err
= EBADMSG
;
961 ssk
->sk_error_report(ssk
);
962 tcp_set_state(ssk
, TCP_CLOSE
);
963 tcp_send_active_reset(ssk
, GFP_ATOMIC
);
964 subflow
->data_avail
= 0;
968 __mptcp_do_fallback(msk
);
969 skb
= skb_peek(&ssk
->sk_receive_queue
);
970 subflow
->map_valid
= 1;
971 subflow
->map_seq
= READ_ONCE(msk
->ack_seq
);
972 subflow
->map_data_len
= skb
->len
;
973 subflow
->map_subflow_seq
= tcp_sk(ssk
)->copied_seq
- subflow
->ssn_offset
;
974 subflow
->data_avail
= MPTCP_SUBFLOW_DATA_AVAIL
;
978 bool mptcp_subflow_data_available(struct sock
*sk
)
980 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
982 /* check if current mapping is still valid */
983 if (subflow
->map_valid
&&
984 mptcp_subflow_get_map_offset(subflow
) >= subflow
->map_data_len
) {
985 subflow
->map_valid
= 0;
986 subflow
->data_avail
= 0;
988 pr_debug("Done with mapping: seq=%u data_len=%u",
989 subflow
->map_subflow_seq
,
990 subflow
->map_data_len
);
993 return subflow_check_data_avail(sk
);
996 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
999 * In mptcp, rwin is about the mptcp-level connection data.
1001 * Data that is still on the ssk rx queue can thus be ignored,
1002 * as far as mptcp peer is concerened that data is still inflight.
1003 * DSS ACK is updated when skb is moved to the mptcp rx queue.
1005 void mptcp_space(const struct sock
*ssk
, int *space
, int *full_space
)
1007 const struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(ssk
);
1008 const struct sock
*sk
= subflow
->conn
;
1010 *space
= __mptcp_space(sk
);
1011 *full_space
= tcp_full_space(sk
);
1014 static void subflow_data_ready(struct sock
*sk
)
1016 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
1017 u16 state
= 1 << inet_sk_state_load(sk
);
1018 struct sock
*parent
= subflow
->conn
;
1019 struct mptcp_sock
*msk
;
1021 msk
= mptcp_sk(parent
);
1022 if (state
& TCPF_LISTEN
) {
1023 /* MPJ subflow are removed from accept queue before reaching here,
1024 * avoid stray wakeups
1026 if (reqsk_queue_empty(&inet_csk(sk
)->icsk_accept_queue
))
1029 set_bit(MPTCP_DATA_READY
, &msk
->flags
);
1030 parent
->sk_data_ready(parent
);
1034 WARN_ON_ONCE(!__mptcp_check_fallback(msk
) && !subflow
->mp_capable
&&
1035 !subflow
->mp_join
&& !(state
& TCPF_CLOSE
));
1037 if (mptcp_subflow_data_available(sk
))
1038 mptcp_data_ready(parent
, sk
);
1041 static void subflow_write_space(struct sock
*ssk
)
1043 /* we take action in __mptcp_clean_una() */
1046 void __mptcp_error_report(struct sock
*sk
)
1048 struct mptcp_subflow_context
*subflow
;
1049 struct mptcp_sock
*msk
= mptcp_sk(sk
);
1051 mptcp_for_each_subflow(msk
, subflow
) {
1052 struct sock
*ssk
= mptcp_subflow_tcp_sock(subflow
);
1053 int err
= sock_error(ssk
);
1058 /* only propagate errors on fallen-back sockets or
1061 if (sk
->sk_state
!= TCP_SYN_SENT
&& !__mptcp_check_fallback(msk
))
1064 inet_sk_state_store(sk
, inet_sk_state_load(ssk
));
1067 /* This barrier is coupled with smp_rmb() in mptcp_poll() */
1069 sk
->sk_error_report(sk
);
1074 static void subflow_error_report(struct sock
*ssk
)
1076 struct sock
*sk
= mptcp_subflow_ctx(ssk
)->conn
;
1078 mptcp_data_lock(sk
);
1079 if (!sock_owned_by_user(sk
))
1080 __mptcp_error_report(sk
);
1082 set_bit(MPTCP_ERROR_REPORT
, &mptcp_sk(sk
)->flags
);
1083 mptcp_data_unlock(sk
);
1086 static struct inet_connection_sock_af_ops
*
1087 subflow_default_af_ops(struct sock
*sk
)
1089 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1090 if (sk
->sk_family
== AF_INET6
)
1091 return &subflow_v6_specific
;
1093 return &subflow_specific
;
1096 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1097 void mptcpv6_handle_mapped(struct sock
*sk
, bool mapped
)
1099 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
1100 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1101 struct inet_connection_sock_af_ops
*target
;
1103 target
= mapped
? &subflow_v6m_specific
: subflow_default_af_ops(sk
);
1105 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d",
1106 subflow
, sk
->sk_family
, icsk
->icsk_af_ops
, target
, mapped
);
1108 if (likely(icsk
->icsk_af_ops
== target
))
1111 subflow
->icsk_af_ops
= icsk
->icsk_af_ops
;
1112 icsk
->icsk_af_ops
= target
;
1116 static void mptcp_info2sockaddr(const struct mptcp_addr_info
*info
,
1117 struct sockaddr_storage
*addr
)
1119 memset(addr
, 0, sizeof(*addr
));
1120 addr
->ss_family
= info
->family
;
1121 if (addr
->ss_family
== AF_INET
) {
1122 struct sockaddr_in
*in_addr
= (struct sockaddr_in
*)addr
;
1124 in_addr
->sin_addr
= info
->addr
;
1125 in_addr
->sin_port
= info
->port
;
1127 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1128 else if (addr
->ss_family
== AF_INET6
) {
1129 struct sockaddr_in6
*in6_addr
= (struct sockaddr_in6
*)addr
;
1131 in6_addr
->sin6_addr
= info
->addr6
;
1132 in6_addr
->sin6_port
= info
->port
;
1137 int __mptcp_subflow_connect(struct sock
*sk
, const struct mptcp_addr_info
*loc
,
1138 const struct mptcp_addr_info
*remote
)
1140 struct mptcp_sock
*msk
= mptcp_sk(sk
);
1141 struct mptcp_subflow_context
*subflow
;
1142 struct sockaddr_storage addr
;
1143 int remote_id
= remote
->id
;
1144 int local_id
= loc
->id
;
1151 if (!mptcp_is_fully_established(sk
))
1154 err
= mptcp_subflow_create_socket(sk
, &sf
);
1159 subflow
= mptcp_subflow_ctx(ssk
);
1161 get_random_bytes(&subflow
->local_nonce
, sizeof(u32
));
1162 } while (!subflow
->local_nonce
);
1165 err
= mptcp_pm_get_local_id(msk
, (struct sock_common
*)ssk
);
1172 subflow
->remote_key
= msk
->remote_key
;
1173 subflow
->local_key
= msk
->local_key
;
1174 subflow
->token
= msk
->token
;
1175 mptcp_info2sockaddr(loc
, &addr
);
1177 addrlen
= sizeof(struct sockaddr_in
);
1178 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1179 if (loc
->family
== AF_INET6
)
1180 addrlen
= sizeof(struct sockaddr_in6
);
1182 ssk
->sk_bound_dev_if
= loc
->ifindex
;
1183 err
= kernel_bind(sf
, (struct sockaddr
*)&addr
, addrlen
);
1187 mptcp_crypto_key_sha(subflow
->remote_key
, &remote_token
, NULL
);
1188 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk
,
1189 remote_token
, local_id
, remote_id
);
1190 subflow
->remote_token
= remote_token
;
1191 subflow
->local_id
= local_id
;
1192 subflow
->remote_id
= remote_id
;
1193 subflow
->request_join
= 1;
1194 subflow
->request_bkup
= !!(loc
->flags
& MPTCP_PM_ADDR_FLAG_BACKUP
);
1195 mptcp_info2sockaddr(remote
, &addr
);
1197 mptcp_add_pending_subflow(msk
, subflow
);
1198 err
= kernel_connect(sf
, (struct sockaddr
*)&addr
, addrlen
, O_NONBLOCK
);
1199 if (err
&& err
!= -EINPROGRESS
)
1202 /* discard the subflow socket */
1203 mptcp_sock_graft(ssk
, sk
->sk_socket
);
1204 iput(SOCK_INODE(sf
));
1208 spin_lock_bh(&msk
->join_list_lock
);
1209 list_del(&subflow
->node
);
1210 spin_unlock_bh(&msk
->join_list_lock
);
1211 sock_put(mptcp_subflow_tcp_sock(subflow
));
1214 subflow
->disposable
= 1;
1219 static void mptcp_attach_cgroup(struct sock
*parent
, struct sock
*child
)
1221 #ifdef CONFIG_SOCK_CGROUP_DATA
1222 struct sock_cgroup_data
*parent_skcd
= &parent
->sk_cgrp_data
,
1223 *child_skcd
= &child
->sk_cgrp_data
;
1225 /* only the additional subflows created by kworkers have to be modified */
1226 if (cgroup_id(sock_cgroup_ptr(parent_skcd
)) !=
1227 cgroup_id(sock_cgroup_ptr(child_skcd
))) {
1229 struct mem_cgroup
*memcg
= parent
->sk_memcg
;
1231 mem_cgroup_sk_free(child
);
1232 if (memcg
&& css_tryget(&memcg
->css
))
1233 child
->sk_memcg
= memcg
;
1234 #endif /* CONFIG_MEMCG */
1236 cgroup_sk_free(child_skcd
);
1237 *child_skcd
= *parent_skcd
;
1238 cgroup_sk_clone(child_skcd
);
1240 #endif /* CONFIG_SOCK_CGROUP_DATA */
1243 int mptcp_subflow_create_socket(struct sock
*sk
, struct socket
**new_sock
)
1245 struct mptcp_subflow_context
*subflow
;
1246 struct net
*net
= sock_net(sk
);
1250 /* un-accepted server sockets can reach here - on bad configuration
1251 * bail early to avoid greater trouble later
1253 if (unlikely(!sk
->sk_socket
))
1256 err
= sock_create_kern(net
, sk
->sk_family
, SOCK_STREAM
, IPPROTO_TCP
,
1263 /* the newly created socket has to be in the same cgroup as its parent */
1264 mptcp_attach_cgroup(sk
, sf
->sk
);
1266 /* kernel sockets do not by default acquire net ref, but TCP timer
1269 sf
->sk
->sk_net_refcnt
= 1;
1271 #ifdef CONFIG_PROC_FS
1272 this_cpu_add(*net
->core
.sock_inuse
, 1);
1274 err
= tcp_set_ulp(sf
->sk
, "mptcp");
1275 release_sock(sf
->sk
);
1282 /* the newly created socket really belongs to the owning MPTCP master
1283 * socket, even if for additional subflows the allocation is performed
1284 * by a kernel workqueue. Adjust inode references, so that the
1285 * procfs/diag interaces really show this one belonging to the correct
1288 SOCK_INODE(sf
)->i_ino
= SOCK_INODE(sk
->sk_socket
)->i_ino
;
1289 SOCK_INODE(sf
)->i_uid
= SOCK_INODE(sk
->sk_socket
)->i_uid
;
1290 SOCK_INODE(sf
)->i_gid
= SOCK_INODE(sk
->sk_socket
)->i_gid
;
1292 subflow
= mptcp_subflow_ctx(sf
->sk
);
1293 pr_debug("subflow=%p", subflow
);
1302 static struct mptcp_subflow_context
*subflow_create_ctx(struct sock
*sk
,
1305 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1306 struct mptcp_subflow_context
*ctx
;
1308 ctx
= kzalloc(sizeof(*ctx
), priority
);
1312 rcu_assign_pointer(icsk
->icsk_ulp_data
, ctx
);
1313 INIT_LIST_HEAD(&ctx
->node
);
1315 pr_debug("subflow=%p", ctx
);
1322 static void __subflow_state_change(struct sock
*sk
)
1324 struct socket_wq
*wq
;
1327 wq
= rcu_dereference(sk
->sk_wq
);
1328 if (skwq_has_sleeper(wq
))
1329 wake_up_interruptible_all(&wq
->wait
);
1333 static bool subflow_is_done(const struct sock
*sk
)
1335 return sk
->sk_shutdown
& RCV_SHUTDOWN
|| sk
->sk_state
== TCP_CLOSE
;
1338 static void subflow_state_change(struct sock
*sk
)
1340 struct mptcp_subflow_context
*subflow
= mptcp_subflow_ctx(sk
);
1341 struct sock
*parent
= subflow
->conn
;
1343 __subflow_state_change(sk
);
1345 if (subflow_simultaneous_connect(sk
)) {
1346 mptcp_do_fallback(sk
);
1347 mptcp_rcv_space_init(mptcp_sk(parent
), sk
);
1348 pr_fallback(mptcp_sk(parent
));
1349 subflow
->conn_finished
= 1;
1350 if (inet_sk_state_load(parent
) == TCP_SYN_SENT
) {
1351 inet_sk_state_store(parent
, TCP_ESTABLISHED
);
1352 parent
->sk_state_change(parent
);
1356 /* as recvmsg() does not acquire the subflow socket for ssk selection
1357 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1358 * the data available machinery here.
1360 if (mptcp_subflow_data_available(sk
))
1361 mptcp_data_ready(parent
, sk
);
1363 if (__mptcp_check_fallback(mptcp_sk(parent
)) &&
1364 !subflow
->rx_eof
&& subflow_is_done(sk
)) {
1365 subflow
->rx_eof
= 1;
1366 mptcp_subflow_eof(parent
);
1370 static int subflow_ulp_init(struct sock
*sk
)
1372 struct inet_connection_sock
*icsk
= inet_csk(sk
);
1373 struct mptcp_subflow_context
*ctx
;
1374 struct tcp_sock
*tp
= tcp_sk(sk
);
1377 /* disallow attaching ULP to a socket unless it has been
1378 * created with sock_create_kern()
1380 if (!sk
->sk_kern_sock
) {
1385 ctx
= subflow_create_ctx(sk
, GFP_KERNEL
);
1391 pr_debug("subflow=%p, family=%d", ctx
, sk
->sk_family
);
1394 ctx
->icsk_af_ops
= icsk
->icsk_af_ops
;
1395 icsk
->icsk_af_ops
= subflow_default_af_ops(sk
);
1396 ctx
->tcp_data_ready
= sk
->sk_data_ready
;
1397 ctx
->tcp_state_change
= sk
->sk_state_change
;
1398 ctx
->tcp_write_space
= sk
->sk_write_space
;
1399 ctx
->tcp_error_report
= sk
->sk_error_report
;
1400 sk
->sk_data_ready
= subflow_data_ready
;
1401 sk
->sk_write_space
= subflow_write_space
;
1402 sk
->sk_state_change
= subflow_state_change
;
1403 sk
->sk_error_report
= subflow_error_report
;
1408 static void subflow_ulp_release(struct sock
*ssk
)
1410 struct mptcp_subflow_context
*ctx
= mptcp_subflow_ctx(ssk
);
1411 bool release
= true;
1419 /* if the msk has been orphaned, keep the ctx
1420 * alive, will be freed by __mptcp_close_ssk(),
1421 * when the subflow is still unaccepted
1423 release
= ctx
->disposable
|| list_empty(&ctx
->node
);
1428 kfree_rcu(ctx
, rcu
);
1431 static void subflow_ulp_clone(const struct request_sock
*req
,
1433 const gfp_t priority
)
1435 struct mptcp_subflow_request_sock
*subflow_req
= mptcp_subflow_rsk(req
);
1436 struct mptcp_subflow_context
*old_ctx
= mptcp_subflow_ctx(newsk
);
1437 struct mptcp_subflow_context
*new_ctx
;
1439 if (!tcp_rsk(req
)->is_mptcp
||
1440 (!subflow_req
->mp_capable
&& !subflow_req
->mp_join
)) {
1441 subflow_ulp_fallback(newsk
, old_ctx
);
1445 new_ctx
= subflow_create_ctx(newsk
, priority
);
1447 subflow_ulp_fallback(newsk
, old_ctx
);
1451 new_ctx
->conn_finished
= 1;
1452 new_ctx
->icsk_af_ops
= old_ctx
->icsk_af_ops
;
1453 new_ctx
->tcp_data_ready
= old_ctx
->tcp_data_ready
;
1454 new_ctx
->tcp_state_change
= old_ctx
->tcp_state_change
;
1455 new_ctx
->tcp_write_space
= old_ctx
->tcp_write_space
;
1456 new_ctx
->tcp_error_report
= old_ctx
->tcp_error_report
;
1457 new_ctx
->rel_write_seq
= 1;
1458 new_ctx
->tcp_sock
= newsk
;
1460 if (subflow_req
->mp_capable
) {
1461 /* see comments in subflow_syn_recv_sock(), MPTCP connection
1462 * is fully established only after we receive the remote key
1464 new_ctx
->mp_capable
= 1;
1465 new_ctx
->local_key
= subflow_req
->local_key
;
1466 new_ctx
->token
= subflow_req
->token
;
1467 new_ctx
->ssn_offset
= subflow_req
->ssn_offset
;
1468 new_ctx
->idsn
= subflow_req
->idsn
;
1469 } else if (subflow_req
->mp_join
) {
1470 new_ctx
->ssn_offset
= subflow_req
->ssn_offset
;
1471 new_ctx
->mp_join
= 1;
1472 new_ctx
->fully_established
= 1;
1473 new_ctx
->backup
= subflow_req
->backup
;
1474 new_ctx
->local_id
= subflow_req
->local_id
;
1475 new_ctx
->remote_id
= subflow_req
->remote_id
;
1476 new_ctx
->token
= subflow_req
->token
;
1477 new_ctx
->thmac
= subflow_req
->thmac
;
1481 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly
= {
1483 .owner
= THIS_MODULE
,
1484 .init
= subflow_ulp_init
,
1485 .release
= subflow_ulp_release
,
1486 .clone
= subflow_ulp_clone
,
1489 static int subflow_ops_init(struct request_sock_ops
*subflow_ops
)
1491 subflow_ops
->obj_size
= sizeof(struct mptcp_subflow_request_sock
);
1492 subflow_ops
->slab_name
= "request_sock_subflow";
1494 subflow_ops
->slab
= kmem_cache_create(subflow_ops
->slab_name
,
1495 subflow_ops
->obj_size
, 0,
1497 SLAB_TYPESAFE_BY_RCU
,
1499 if (!subflow_ops
->slab
)
1502 subflow_ops
->destructor
= subflow_req_destructor
;
1507 void __init
mptcp_subflow_init(void)
1509 mptcp_subflow_request_sock_ops
= tcp_request_sock_ops
;
1510 if (subflow_ops_init(&mptcp_subflow_request_sock_ops
) != 0)
1511 panic("MPTCP: failed to init subflow request sock ops\n");
1513 subflow_request_sock_ipv4_ops
= tcp_request_sock_ipv4_ops
;
1514 subflow_request_sock_ipv4_ops
.route_req
= subflow_v4_route_req
;
1516 subflow_specific
= ipv4_specific
;
1517 subflow_specific
.conn_request
= subflow_v4_conn_request
;
1518 subflow_specific
.syn_recv_sock
= subflow_syn_recv_sock
;
1519 subflow_specific
.sk_rx_dst_set
= subflow_finish_connect
;
1521 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1522 subflow_request_sock_ipv6_ops
= tcp_request_sock_ipv6_ops
;
1523 subflow_request_sock_ipv6_ops
.route_req
= subflow_v6_route_req
;
1525 subflow_v6_specific
= ipv6_specific
;
1526 subflow_v6_specific
.conn_request
= subflow_v6_conn_request
;
1527 subflow_v6_specific
.syn_recv_sock
= subflow_syn_recv_sock
;
1528 subflow_v6_specific
.sk_rx_dst_set
= subflow_finish_connect
;
1530 subflow_v6m_specific
= subflow_v6_specific
;
1531 subflow_v6m_specific
.queue_xmit
= ipv4_specific
.queue_xmit
;
1532 subflow_v6m_specific
.send_check
= ipv4_specific
.send_check
;
1533 subflow_v6m_specific
.net_header_len
= ipv4_specific
.net_header_len
;
1534 subflow_v6m_specific
.mtu_reduced
= ipv4_specific
.mtu_reduced
;
1535 subflow_v6m_specific
.net_frag_header_len
= 0;
1538 mptcp_diag_subflow_init(&subflow_ulp_ops
);
1540 if (tcp_register_ulp(&subflow_ulp_ops
) != 0)
1541 panic("MPTCP: failed to register subflows to ULP\n");