1 // SPDX-License-Identifier: GPL-2.0
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * Implementation of the Transmission Control Protocol(TCP).
10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Mark Evans, <evansmp@uhura.aston.ac.uk>
12 * Corey Minyard <wf-rch!minyard@relay.EU.net>
13 * Florian La Roche, <flla@stud.uni-sb.de>
14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15 * Linus Torvalds, <torvalds@cs.helsinki.fi>
16 * Alan Cox, <gw4pts@gw4pts.ampr.org>
17 * Matthew Dillon, <dillon@apollo.west.oic.com>
18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19 * Jorge Cwik, <jorge@laser.satlink.net>
24 * Pedro Roque : Fast Retransmit/Recovery.
26 * Retransmit queue handled by TCP.
27 * Better retransmit timer handling.
28 * New congestion avoidance.
32 * Eric : Fast Retransmit.
33 * Randy Scott : MSS option defines.
34 * Eric Schenk : Fixes to slow start algorithm.
35 * Eric Schenk : Yet another double ACK bug.
36 * Eric Schenk : Delayed ACK bug fixes.
37 * Eric Schenk : Floyd style fast retrans war avoidance.
38 * David S. Miller : Don't allow zero congestion window.
39 * Eric Schenk : Fix retransmitter so that it sends
40 * next packet on ack of previous packet.
41 * Andi Kleen : Moved open_request checking here
42 * and process RSTs for open_requests.
43 * Andi Kleen : Better prune_queue, and other fixes.
44 * Andrey Savochkin: Fix RTT measurements in the presence of
46 * Andrey Savochkin: Check sequence numbers correctly when
47 * removing SACKs due to in sequence incoming
49 * Andi Kleen: Make sure we never ack data there is not
50 * enough room for. Also make this condition
51 * a fatal error if it might still happen.
52 * Andi Kleen: Add tcp_measure_rcv_mss to make
53 * connections with MSS<min(MTU,ann. MSS)
54 * work without delayed acks.
55 * Andi Kleen: Process packets with PSH set in the
57 * J Hadi Salim: ECN support
60 * Panu Kuhlberg: Experimental audit of TCP (re)transmission
61 * engine. Lots of bugs are found.
62 * Pasi Sarolahti: F-RTO for dealing with spurious RTOs
65 #define pr_fmt(fmt) "TCP: " fmt
68 #include <linux/slab.h>
69 #include <linux/module.h>
70 #include <linux/sysctl.h>
71 #include <linux/kernel.h>
72 #include <linux/prefetch.h>
75 #include <net/inet_common.h>
76 #include <linux/ipsec.h>
77 #include <asm/unaligned.h>
78 #include <linux/errqueue.h>
79 #include <trace/events/tcp.h>
80 #include <linux/jump_label_ratelimit.h>
81 #include <net/busy_poll.h>
82 #include <net/mptcp.h>
84 int sysctl_tcp_max_orphans __read_mostly
= NR_FILE
;
86 #define FLAG_DATA 0x01 /* Incoming frame contained data. */
87 #define FLAG_WIN_UPDATE 0x02 /* Incoming ACK was a window update. */
88 #define FLAG_DATA_ACKED 0x04 /* This ACK acknowledged new data. */
89 #define FLAG_RETRANS_DATA_ACKED 0x08 /* "" "" some of which was retransmitted. */
90 #define FLAG_SYN_ACKED 0x10 /* This ACK acknowledged SYN. */
91 #define FLAG_DATA_SACKED 0x20 /* New SACK. */
92 #define FLAG_ECE 0x40 /* ECE in this ACK */
93 #define FLAG_LOST_RETRANS 0x80 /* This ACK marks some retransmission lost */
94 #define FLAG_SLOWPATH 0x100 /* Do not skip RFC checks for window update.*/
95 #define FLAG_ORIG_SACK_ACKED 0x200 /* Never retransmitted data are (s)acked */
96 #define FLAG_SND_UNA_ADVANCED 0x400 /* Snd_una was changed (!= FLAG_DATA_ACKED) */
97 #define FLAG_DSACKING_ACK 0x800 /* SACK blocks contained D-SACK info */
98 #define FLAG_SET_XMIT_TIMER 0x1000 /* Set TLP or RTO timer */
99 #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */
100 #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */
101 #define FLAG_NO_CHALLENGE_ACK 0x8000 /* do not call tcp_send_challenge_ack() */
102 #define FLAG_ACK_MAYBE_DELAYED 0x10000 /* Likely a delayed ACK */
103 #define FLAG_DSACK_TLP 0x20000 /* DSACK for tail loss probe */
105 #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED)
106 #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED)
107 #define FLAG_CA_ALERT (FLAG_DATA_SACKED|FLAG_ECE|FLAG_DSACKING_ACK)
108 #define FLAG_FORWARD_PROGRESS (FLAG_ACKED|FLAG_DATA_SACKED)
110 #define TCP_REMNANT (TCP_FLAG_FIN|TCP_FLAG_URG|TCP_FLAG_SYN|TCP_FLAG_PSH)
111 #define TCP_HP_BITS (~(TCP_RESERVED_BITS|TCP_FLAG_PSH))
113 #define REXMIT_NONE 0 /* no loss recovery to do */
114 #define REXMIT_LOST 1 /* retransmit packets marked lost */
115 #define REXMIT_NEW 2 /* FRTO-style transmit of unsent/new packets */
117 #if IS_ENABLED(CONFIG_TLS_DEVICE)
118 static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled
, HZ
);
120 void clean_acked_data_enable(struct inet_connection_sock
*icsk
,
121 void (*cad
)(struct sock
*sk
, u32 ack_seq
))
123 icsk
->icsk_clean_acked
= cad
;
124 static_branch_deferred_inc(&clean_acked_data_enabled
);
126 EXPORT_SYMBOL_GPL(clean_acked_data_enable
);
128 void clean_acked_data_disable(struct inet_connection_sock
*icsk
)
130 static_branch_slow_dec_deferred(&clean_acked_data_enabled
);
131 icsk
->icsk_clean_acked
= NULL
;
133 EXPORT_SYMBOL_GPL(clean_acked_data_disable
);
135 void clean_acked_data_flush(void)
137 static_key_deferred_flush(&clean_acked_data_enabled
);
139 EXPORT_SYMBOL_GPL(clean_acked_data_flush
);
142 #ifdef CONFIG_CGROUP_BPF
143 static void bpf_skops_parse_hdr(struct sock
*sk
, struct sk_buff
*skb
)
145 bool unknown_opt
= tcp_sk(sk
)->rx_opt
.saw_unknown
&&
146 BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk
),
147 BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG
);
148 bool parse_all_opt
= BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk
),
149 BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG
);
150 struct bpf_sock_ops_kern sock_ops
;
152 if (likely(!unknown_opt
&& !parse_all_opt
))
155 /* The skb will be handled in the
156 * bpf_skops_established() or
157 * bpf_skops_write_hdr_opt().
159 switch (sk
->sk_state
) {
166 sock_owned_by_me(sk
);
168 memset(&sock_ops
, 0, offsetof(struct bpf_sock_ops_kern
, temp
));
169 sock_ops
.op
= BPF_SOCK_OPS_PARSE_HDR_OPT_CB
;
170 sock_ops
.is_fullsock
= 1;
172 bpf_skops_init_skb(&sock_ops
, skb
, tcp_hdrlen(skb
));
174 BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops
);
177 static void bpf_skops_established(struct sock
*sk
, int bpf_op
,
180 struct bpf_sock_ops_kern sock_ops
;
182 sock_owned_by_me(sk
);
184 memset(&sock_ops
, 0, offsetof(struct bpf_sock_ops_kern
, temp
));
185 sock_ops
.op
= bpf_op
;
186 sock_ops
.is_fullsock
= 1;
188 /* sk with TCP_REPAIR_ON does not have skb in tcp_finish_connect */
190 bpf_skops_init_skb(&sock_ops
, skb
, tcp_hdrlen(skb
));
192 BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops
);
195 static void bpf_skops_parse_hdr(struct sock
*sk
, struct sk_buff
*skb
)
199 static void bpf_skops_established(struct sock
*sk
, int bpf_op
,
205 static void tcp_gro_dev_warn(struct sock
*sk
, const struct sk_buff
*skb
,
208 static bool __once __read_mostly
;
211 struct net_device
*dev
;
216 dev
= dev_get_by_index_rcu(sock_net(sk
), skb
->skb_iif
);
217 if (!dev
|| len
>= dev
->mtu
)
218 pr_warn("%s: Driver has suspect GRO implementation, TCP performance may be compromised.\n",
219 dev
? dev
->name
: "Unknown driver");
224 /* Adapt the MSS value used to make delayed ack decision to the
227 static void tcp_measure_rcv_mss(struct sock
*sk
, const struct sk_buff
*skb
)
229 struct inet_connection_sock
*icsk
= inet_csk(sk
);
230 const unsigned int lss
= icsk
->icsk_ack
.last_seg_size
;
233 icsk
->icsk_ack
.last_seg_size
= 0;
235 /* skb->len may jitter because of SACKs, even if peer
236 * sends good full-sized frames.
238 len
= skb_shinfo(skb
)->gso_size
? : skb
->len
;
239 if (len
>= icsk
->icsk_ack
.rcv_mss
) {
240 icsk
->icsk_ack
.rcv_mss
= min_t(unsigned int, len
,
242 /* Account for possibly-removed options */
243 if (unlikely(len
> icsk
->icsk_ack
.rcv_mss
+
244 MAX_TCP_OPTION_SPACE
))
245 tcp_gro_dev_warn(sk
, skb
, len
);
247 /* Otherwise, we make more careful check taking into account,
248 * that SACKs block is variable.
250 * "len" is invariant segment length, including TCP header.
252 len
+= skb
->data
- skb_transport_header(skb
);
253 if (len
>= TCP_MSS_DEFAULT
+ sizeof(struct tcphdr
) ||
254 /* If PSH is not set, packet should be
255 * full sized, provided peer TCP is not badly broken.
256 * This observation (if it is correct 8)) allows
257 * to handle super-low mtu links fairly.
259 (len
>= TCP_MIN_MSS
+ sizeof(struct tcphdr
) &&
260 !(tcp_flag_word(tcp_hdr(skb
)) & TCP_REMNANT
))) {
261 /* Subtract also invariant (if peer is RFC compliant),
262 * tcp header plus fixed timestamp option length.
263 * Resulting "len" is MSS free of SACK jitter.
265 len
-= tcp_sk(sk
)->tcp_header_len
;
266 icsk
->icsk_ack
.last_seg_size
= len
;
268 icsk
->icsk_ack
.rcv_mss
= len
;
272 if (icsk
->icsk_ack
.pending
& ICSK_ACK_PUSHED
)
273 icsk
->icsk_ack
.pending
|= ICSK_ACK_PUSHED2
;
274 icsk
->icsk_ack
.pending
|= ICSK_ACK_PUSHED
;
278 static void tcp_incr_quickack(struct sock
*sk
, unsigned int max_quickacks
)
280 struct inet_connection_sock
*icsk
= inet_csk(sk
);
281 unsigned int quickacks
= tcp_sk(sk
)->rcv_wnd
/ (2 * icsk
->icsk_ack
.rcv_mss
);
285 quickacks
= min(quickacks
, max_quickacks
);
286 if (quickacks
> icsk
->icsk_ack
.quick
)
287 icsk
->icsk_ack
.quick
= quickacks
;
290 void tcp_enter_quickack_mode(struct sock
*sk
, unsigned int max_quickacks
)
292 struct inet_connection_sock
*icsk
= inet_csk(sk
);
294 tcp_incr_quickack(sk
, max_quickacks
);
295 inet_csk_exit_pingpong_mode(sk
);
296 icsk
->icsk_ack
.ato
= TCP_ATO_MIN
;
298 EXPORT_SYMBOL(tcp_enter_quickack_mode
);
300 /* Send ACKs quickly, if "quick" count is not exhausted
301 * and the session is not interactive.
304 static bool tcp_in_quickack_mode(struct sock
*sk
)
306 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
307 const struct dst_entry
*dst
= __sk_dst_get(sk
);
309 return (dst
&& dst_metric(dst
, RTAX_QUICKACK
)) ||
310 (icsk
->icsk_ack
.quick
&& !inet_csk_in_pingpong_mode(sk
));
313 static void tcp_ecn_queue_cwr(struct tcp_sock
*tp
)
315 if (tp
->ecn_flags
& TCP_ECN_OK
)
316 tp
->ecn_flags
|= TCP_ECN_QUEUE_CWR
;
319 static void tcp_ecn_accept_cwr(struct sock
*sk
, const struct sk_buff
*skb
)
321 if (tcp_hdr(skb
)->cwr
) {
322 tcp_sk(sk
)->ecn_flags
&= ~TCP_ECN_DEMAND_CWR
;
324 /* If the sender is telling us it has entered CWR, then its
325 * cwnd may be very low (even just 1 packet), so we should ACK
328 if (TCP_SKB_CB(skb
)->seq
!= TCP_SKB_CB(skb
)->end_seq
)
329 inet_csk(sk
)->icsk_ack
.pending
|= ICSK_ACK_NOW
;
333 static void tcp_ecn_withdraw_cwr(struct tcp_sock
*tp
)
335 tp
->ecn_flags
&= ~TCP_ECN_QUEUE_CWR
;
338 static void __tcp_ecn_check_ce(struct sock
*sk
, const struct sk_buff
*skb
)
340 struct tcp_sock
*tp
= tcp_sk(sk
);
342 switch (TCP_SKB_CB(skb
)->ip_dsfield
& INET_ECN_MASK
) {
343 case INET_ECN_NOT_ECT
:
344 /* Funny extension: if ECT is not set on a segment,
345 * and we already seen ECT on a previous segment,
346 * it is probably a retransmit.
348 if (tp
->ecn_flags
& TCP_ECN_SEEN
)
349 tcp_enter_quickack_mode(sk
, 2);
352 if (tcp_ca_needs_ecn(sk
))
353 tcp_ca_event(sk
, CA_EVENT_ECN_IS_CE
);
355 if (!(tp
->ecn_flags
& TCP_ECN_DEMAND_CWR
)) {
356 /* Better not delay acks, sender can have a very low cwnd */
357 tcp_enter_quickack_mode(sk
, 2);
358 tp
->ecn_flags
|= TCP_ECN_DEMAND_CWR
;
360 tp
->ecn_flags
|= TCP_ECN_SEEN
;
363 if (tcp_ca_needs_ecn(sk
))
364 tcp_ca_event(sk
, CA_EVENT_ECN_NO_CE
);
365 tp
->ecn_flags
|= TCP_ECN_SEEN
;
370 static void tcp_ecn_check_ce(struct sock
*sk
, const struct sk_buff
*skb
)
372 if (tcp_sk(sk
)->ecn_flags
& TCP_ECN_OK
)
373 __tcp_ecn_check_ce(sk
, skb
);
376 static void tcp_ecn_rcv_synack(struct tcp_sock
*tp
, const struct tcphdr
*th
)
378 if ((tp
->ecn_flags
& TCP_ECN_OK
) && (!th
->ece
|| th
->cwr
))
379 tp
->ecn_flags
&= ~TCP_ECN_OK
;
382 static void tcp_ecn_rcv_syn(struct tcp_sock
*tp
, const struct tcphdr
*th
)
384 if ((tp
->ecn_flags
& TCP_ECN_OK
) && (!th
->ece
|| !th
->cwr
))
385 tp
->ecn_flags
&= ~TCP_ECN_OK
;
388 static bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock
*tp
, const struct tcphdr
*th
)
390 if (th
->ece
&& !th
->syn
&& (tp
->ecn_flags
& TCP_ECN_OK
))
395 /* Buffer size and advertised window tuning.
397 * 1. Tuning sk->sk_sndbuf, when connection enters established state.
400 static void tcp_sndbuf_expand(struct sock
*sk
)
402 const struct tcp_sock
*tp
= tcp_sk(sk
);
403 const struct tcp_congestion_ops
*ca_ops
= inet_csk(sk
)->icsk_ca_ops
;
407 /* Worst case is non GSO/TSO : each frame consumes one skb
408 * and skb->head is kmalloced using power of two area of memory
410 per_mss
= max_t(u32
, tp
->rx_opt
.mss_clamp
, tp
->mss_cache
) +
412 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
414 per_mss
= roundup_pow_of_two(per_mss
) +
415 SKB_DATA_ALIGN(sizeof(struct sk_buff
));
417 nr_segs
= max_t(u32
, TCP_INIT_CWND
, tcp_snd_cwnd(tp
));
418 nr_segs
= max_t(u32
, nr_segs
, tp
->reordering
+ 1);
420 /* Fast Recovery (RFC 5681 3.2) :
421 * Cubic needs 1.7 factor, rounded to 2 to include
422 * extra cushion (application might react slowly to EPOLLOUT)
424 sndmem
= ca_ops
->sndbuf_expand
? ca_ops
->sndbuf_expand(sk
) : 2;
425 sndmem
*= nr_segs
* per_mss
;
427 if (sk
->sk_sndbuf
< sndmem
)
428 WRITE_ONCE(sk
->sk_sndbuf
,
429 min(sndmem
, sock_net(sk
)->ipv4
.sysctl_tcp_wmem
[2]));
432 /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)
434 * All tcp_full_space() is split to two parts: "network" buffer, allocated
435 * forward and advertised in receiver window (tp->rcv_wnd) and
436 * "application buffer", required to isolate scheduling/application
437 * latencies from network.
438 * window_clamp is maximal advertised window. It can be less than
439 * tcp_full_space(), in this case tcp_full_space() - window_clamp
440 * is reserved for "application" buffer. The less window_clamp is
441 * the smoother our behaviour from viewpoint of network, but the lower
442 * throughput and the higher sensitivity of the connection to losses. 8)
444 * rcv_ssthresh is more strict window_clamp used at "slow start"
445 * phase to predict further behaviour of this connection.
446 * It is used for two goals:
447 * - to enforce header prediction at sender, even when application
448 * requires some significant "application buffer". It is check #1.
449 * - to prevent pruning of receive queue because of misprediction
450 * of receiver window. Check #2.
452 * The scheme does not work when sender sends good segments opening
453 * window and then starts to feed us spaghetti. But it should work
454 * in common situations. Otherwise, we have to rely on queue collapsing.
457 /* Slow part of check#2. */
458 static int __tcp_grow_window(const struct sock
*sk
, const struct sk_buff
*skb
,
459 unsigned int skbtruesize
)
461 struct tcp_sock
*tp
= tcp_sk(sk
);
463 int truesize
= tcp_win_from_space(sk
, skbtruesize
) >> 1;
464 int window
= tcp_win_from_space(sk
, sock_net(sk
)->ipv4
.sysctl_tcp_rmem
[2]) >> 1;
466 while (tp
->rcv_ssthresh
<= window
) {
467 if (truesize
<= skb
->len
)
468 return 2 * inet_csk(sk
)->icsk_ack
.rcv_mss
;
476 /* Even if skb appears to have a bad len/truesize ratio, TCP coalescing
477 * can play nice with us, as sk_buff and skb->head might be either
478 * freed or shared with up to MAX_SKB_FRAGS segments.
479 * Only give a boost to drivers using page frag(s) to hold the frame(s),
480 * and if no payload was pulled in skb->head before reaching us.
482 static u32
truesize_adjust(bool adjust
, const struct sk_buff
*skb
)
484 u32 truesize
= skb
->truesize
;
486 if (adjust
&& !skb_headlen(skb
)) {
487 truesize
-= SKB_TRUESIZE(skb_end_offset(skb
));
488 /* paranoid check, some drivers might be buggy */
489 if (unlikely((int)truesize
< (int)skb
->len
))
490 truesize
= skb
->truesize
;
495 static void tcp_grow_window(struct sock
*sk
, const struct sk_buff
*skb
,
498 struct tcp_sock
*tp
= tcp_sk(sk
);
501 room
= min_t(int, tp
->window_clamp
, tcp_space(sk
)) - tp
->rcv_ssthresh
;
507 if (!tcp_under_memory_pressure(sk
)) {
508 unsigned int truesize
= truesize_adjust(adjust
, skb
);
511 /* Check #2. Increase window, if skb with such overhead
512 * will fit to rcvbuf in future.
514 if (tcp_win_from_space(sk
, truesize
) <= skb
->len
)
515 incr
= 2 * tp
->advmss
;
517 incr
= __tcp_grow_window(sk
, skb
, truesize
);
520 incr
= max_t(int, incr
, 2 * skb
->len
);
521 tp
->rcv_ssthresh
+= min(room
, incr
);
522 inet_csk(sk
)->icsk_ack
.quick
|= 1;
526 * Adjust rcv_ssthresh according to reserved mem
528 tcp_adjust_rcv_ssthresh(sk
);
532 /* 3. Try to fixup all. It is made immediately after connection enters
535 static void tcp_init_buffer_space(struct sock
*sk
)
537 int tcp_app_win
= sock_net(sk
)->ipv4
.sysctl_tcp_app_win
;
538 struct tcp_sock
*tp
= tcp_sk(sk
);
541 if (!(sk
->sk_userlocks
& SOCK_SNDBUF_LOCK
))
542 tcp_sndbuf_expand(sk
);
544 tcp_mstamp_refresh(tp
);
545 tp
->rcvq_space
.time
= tp
->tcp_mstamp
;
546 tp
->rcvq_space
.seq
= tp
->copied_seq
;
548 maxwin
= tcp_full_space(sk
);
550 if (tp
->window_clamp
>= maxwin
) {
551 tp
->window_clamp
= maxwin
;
553 if (tcp_app_win
&& maxwin
> 4 * tp
->advmss
)
554 tp
->window_clamp
= max(maxwin
-
555 (maxwin
>> tcp_app_win
),
559 /* Force reservation of one segment. */
561 tp
->window_clamp
> 2 * tp
->advmss
&&
562 tp
->window_clamp
+ tp
->advmss
> maxwin
)
563 tp
->window_clamp
= max(2 * tp
->advmss
, maxwin
- tp
->advmss
);
565 tp
->rcv_ssthresh
= min(tp
->rcv_ssthresh
, tp
->window_clamp
);
566 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
567 tp
->rcvq_space
.space
= min3(tp
->rcv_ssthresh
, tp
->rcv_wnd
,
568 (u32
)TCP_INIT_CWND
* tp
->advmss
);
571 /* 4. Recalculate window clamp after socket hit its memory bounds. */
572 static void tcp_clamp_window(struct sock
*sk
)
574 struct tcp_sock
*tp
= tcp_sk(sk
);
575 struct inet_connection_sock
*icsk
= inet_csk(sk
);
576 struct net
*net
= sock_net(sk
);
578 icsk
->icsk_ack
.quick
= 0;
580 if (sk
->sk_rcvbuf
< net
->ipv4
.sysctl_tcp_rmem
[2] &&
581 !(sk
->sk_userlocks
& SOCK_RCVBUF_LOCK
) &&
582 !tcp_under_memory_pressure(sk
) &&
583 sk_memory_allocated(sk
) < sk_prot_mem_limits(sk
, 0)) {
584 WRITE_ONCE(sk
->sk_rcvbuf
,
585 min(atomic_read(&sk
->sk_rmem_alloc
),
586 net
->ipv4
.sysctl_tcp_rmem
[2]));
588 if (atomic_read(&sk
->sk_rmem_alloc
) > sk
->sk_rcvbuf
)
589 tp
->rcv_ssthresh
= min(tp
->window_clamp
, 2U * tp
->advmss
);
592 /* Initialize RCV_MSS value.
593 * RCV_MSS is an our guess about MSS used by the peer.
594 * We haven't any direct information about the MSS.
595 * It's better to underestimate the RCV_MSS rather than overestimate.
596 * Overestimations make us ACKing less frequently than needed.
597 * Underestimations are more easy to detect and fix by tcp_measure_rcv_mss().
599 void tcp_initialize_rcv_mss(struct sock
*sk
)
601 const struct tcp_sock
*tp
= tcp_sk(sk
);
602 unsigned int hint
= min_t(unsigned int, tp
->advmss
, tp
->mss_cache
);
604 hint
= min(hint
, tp
->rcv_wnd
/ 2);
605 hint
= min(hint
, TCP_MSS_DEFAULT
);
606 hint
= max(hint
, TCP_MIN_MSS
);
608 inet_csk(sk
)->icsk_ack
.rcv_mss
= hint
;
610 EXPORT_SYMBOL(tcp_initialize_rcv_mss
);
612 /* Receiver "autotuning" code.
614 * The algorithm for RTT estimation w/o timestamps is based on
615 * Dynamic Right-Sizing (DRS) by Wu Feng and Mike Fisk of LANL.
616 * <https://public.lanl.gov/radiant/pubs.html#DRS>
618 * More detail on this code can be found at
619 * <http://staff.psc.edu/jheffner/>,
620 * though this reference is out of date. A new paper
623 static void tcp_rcv_rtt_update(struct tcp_sock
*tp
, u32 sample
, int win_dep
)
625 u32 new_sample
= tp
->rcv_rtt_est
.rtt_us
;
628 if (new_sample
!= 0) {
629 /* If we sample in larger samples in the non-timestamp
630 * case, we could grossly overestimate the RTT especially
631 * with chatty applications or bulk transfer apps which
632 * are stalled on filesystem I/O.
634 * Also, since we are only going for a minimum in the
635 * non-timestamp case, we do not smooth things out
636 * else with timestamps disabled convergence takes too
640 m
-= (new_sample
>> 3);
648 /* No previous measure. */
652 tp
->rcv_rtt_est
.rtt_us
= new_sample
;
655 static inline void tcp_rcv_rtt_measure(struct tcp_sock
*tp
)
659 if (tp
->rcv_rtt_est
.time
== 0)
661 if (before(tp
->rcv_nxt
, tp
->rcv_rtt_est
.seq
))
663 delta_us
= tcp_stamp_us_delta(tp
->tcp_mstamp
, tp
->rcv_rtt_est
.time
);
666 tcp_rcv_rtt_update(tp
, delta_us
, 1);
669 tp
->rcv_rtt_est
.seq
= tp
->rcv_nxt
+ tp
->rcv_wnd
;
670 tp
->rcv_rtt_est
.time
= tp
->tcp_mstamp
;
673 static inline void tcp_rcv_rtt_measure_ts(struct sock
*sk
,
674 const struct sk_buff
*skb
)
676 struct tcp_sock
*tp
= tcp_sk(sk
);
678 if (tp
->rx_opt
.rcv_tsecr
== tp
->rcv_rtt_last_tsecr
)
680 tp
->rcv_rtt_last_tsecr
= tp
->rx_opt
.rcv_tsecr
;
682 if (TCP_SKB_CB(skb
)->end_seq
-
683 TCP_SKB_CB(skb
)->seq
>= inet_csk(sk
)->icsk_ack
.rcv_mss
) {
684 u32 delta
= tcp_time_stamp(tp
) - tp
->rx_opt
.rcv_tsecr
;
687 if (likely(delta
< INT_MAX
/ (USEC_PER_SEC
/ TCP_TS_HZ
))) {
690 delta_us
= delta
* (USEC_PER_SEC
/ TCP_TS_HZ
);
691 tcp_rcv_rtt_update(tp
, delta_us
, 0);
697 * This function should be called every time data is copied to user space.
698 * It calculates the appropriate TCP receive buffer space.
700 void tcp_rcv_space_adjust(struct sock
*sk
)
702 struct tcp_sock
*tp
= tcp_sk(sk
);
706 trace_tcp_rcv_space_adjust(sk
);
708 tcp_mstamp_refresh(tp
);
709 time
= tcp_stamp_us_delta(tp
->tcp_mstamp
, tp
->rcvq_space
.time
);
710 if (time
< (tp
->rcv_rtt_est
.rtt_us
>> 3) || tp
->rcv_rtt_est
.rtt_us
== 0)
713 /* Number of bytes copied to user in last RTT */
714 copied
= tp
->copied_seq
- tp
->rcvq_space
.seq
;
715 if (copied
<= tp
->rcvq_space
.space
)
719 * copied = bytes received in previous RTT, our base window
720 * To cope with packet losses, we need a 2x factor
721 * To cope with slow start, and sender growing its cwin by 100 %
722 * every RTT, we need a 4x factor, because the ACK we are sending
723 * now is for the next RTT, not the current one :
724 * <prev RTT . ><current RTT .. ><next RTT .... >
727 if (sock_net(sk
)->ipv4
.sysctl_tcp_moderate_rcvbuf
&&
728 !(sk
->sk_userlocks
& SOCK_RCVBUF_LOCK
)) {
732 /* minimal window to cope with packet losses, assuming
733 * steady state. Add some cushion because of small variations.
735 rcvwin
= ((u64
)copied
<< 1) + 16 * tp
->advmss
;
737 /* Accommodate for sender rate increase (eg. slow start) */
738 grow
= rcvwin
* (copied
- tp
->rcvq_space
.space
);
739 do_div(grow
, tp
->rcvq_space
.space
);
740 rcvwin
+= (grow
<< 1);
742 rcvmem
= SKB_TRUESIZE(tp
->advmss
+ MAX_TCP_HEADER
);
743 while (tcp_win_from_space(sk
, rcvmem
) < tp
->advmss
)
746 do_div(rcvwin
, tp
->advmss
);
747 rcvbuf
= min_t(u64
, rcvwin
* rcvmem
,
748 sock_net(sk
)->ipv4
.sysctl_tcp_rmem
[2]);
749 if (rcvbuf
> sk
->sk_rcvbuf
) {
750 WRITE_ONCE(sk
->sk_rcvbuf
, rcvbuf
);
752 /* Make the window clamp follow along. */
753 tp
->window_clamp
= tcp_win_from_space(sk
, rcvbuf
);
756 tp
->rcvq_space
.space
= copied
;
759 tp
->rcvq_space
.seq
= tp
->copied_seq
;
760 tp
->rcvq_space
.time
= tp
->tcp_mstamp
;
763 /* There is something which you must keep in mind when you analyze the
764 * behavior of the tp->ato delayed ack timeout interval. When a
765 * connection starts up, we want to ack as quickly as possible. The
766 * problem is that "good" TCP's do slow start at the beginning of data
767 * transmission. The means that until we send the first few ACK's the
768 * sender will sit on his end and only queue most of his data, because
769 * he can only send snd_cwnd unacked packets at any given time. For
770 * each ACK we send, he increments snd_cwnd and transmits more of his
773 static void tcp_event_data_recv(struct sock
*sk
, struct sk_buff
*skb
)
775 struct tcp_sock
*tp
= tcp_sk(sk
);
776 struct inet_connection_sock
*icsk
= inet_csk(sk
);
779 inet_csk_schedule_ack(sk
);
781 tcp_measure_rcv_mss(sk
, skb
);
783 tcp_rcv_rtt_measure(tp
);
787 if (!icsk
->icsk_ack
.ato
) {
788 /* The _first_ data packet received, initialize
789 * delayed ACK engine.
791 tcp_incr_quickack(sk
, TCP_MAX_QUICKACKS
);
792 icsk
->icsk_ack
.ato
= TCP_ATO_MIN
;
794 int m
= now
- icsk
->icsk_ack
.lrcvtime
;
796 if (m
<= TCP_ATO_MIN
/ 2) {
797 /* The fastest case is the first. */
798 icsk
->icsk_ack
.ato
= (icsk
->icsk_ack
.ato
>> 1) + TCP_ATO_MIN
/ 2;
799 } else if (m
< icsk
->icsk_ack
.ato
) {
800 icsk
->icsk_ack
.ato
= (icsk
->icsk_ack
.ato
>> 1) + m
;
801 if (icsk
->icsk_ack
.ato
> icsk
->icsk_rto
)
802 icsk
->icsk_ack
.ato
= icsk
->icsk_rto
;
803 } else if (m
> icsk
->icsk_rto
) {
804 /* Too long gap. Apparently sender failed to
805 * restart window, so that we send ACKs quickly.
807 tcp_incr_quickack(sk
, TCP_MAX_QUICKACKS
);
810 icsk
->icsk_ack
.lrcvtime
= now
;
812 tcp_ecn_check_ce(sk
, skb
);
815 tcp_grow_window(sk
, skb
, true);
818 /* Called to compute a smoothed rtt estimate. The data fed to this
819 * routine either comes from timestamps, or from segments that were
820 * known _not_ to have been retransmitted [see Karn/Partridge
821 * Proceedings SIGCOMM 87]. The algorithm is from the SIGCOMM 88
822 * piece by Van Jacobson.
823 * NOTE: the next three routines used to be one big routine.
824 * To save cycles in the RFC 1323 implementation it was better to break
825 * it up into three procedures. -- erics
827 static void tcp_rtt_estimator(struct sock
*sk
, long mrtt_us
)
829 struct tcp_sock
*tp
= tcp_sk(sk
);
830 long m
= mrtt_us
; /* RTT */
831 u32 srtt
= tp
->srtt_us
;
833 /* The following amusing code comes from Jacobson's
834 * article in SIGCOMM '88. Note that rtt and mdev
835 * are scaled versions of rtt and mean deviation.
836 * This is designed to be as fast as possible
837 * m stands for "measurement".
839 * On a 1990 paper the rto value is changed to:
840 * RTO = rtt + 4 * mdev
842 * Funny. This algorithm seems to be very broken.
843 * These formulae increase RTO, when it should be decreased, increase
844 * too slowly, when it should be increased quickly, decrease too quickly
845 * etc. I guess in BSD RTO takes ONE value, so that it is absolutely
846 * does not matter how to _calculate_ it. Seems, it was trap
847 * that VJ failed to avoid. 8)
850 m
-= (srtt
>> 3); /* m is now error in rtt est */
851 srtt
+= m
; /* rtt = 7/8 rtt + 1/8 new */
853 m
= -m
; /* m is now abs(error) */
854 m
-= (tp
->mdev_us
>> 2); /* similar update on mdev */
855 /* This is similar to one of Eifel findings.
856 * Eifel blocks mdev updates when rtt decreases.
857 * This solution is a bit different: we use finer gain
858 * for mdev in this case (alpha*beta).
859 * Like Eifel it also prevents growth of rto,
860 * but also it limits too fast rto decreases,
861 * happening in pure Eifel.
866 m
-= (tp
->mdev_us
>> 2); /* similar update on mdev */
868 tp
->mdev_us
+= m
; /* mdev = 3/4 mdev + 1/4 new */
869 if (tp
->mdev_us
> tp
->mdev_max_us
) {
870 tp
->mdev_max_us
= tp
->mdev_us
;
871 if (tp
->mdev_max_us
> tp
->rttvar_us
)
872 tp
->rttvar_us
= tp
->mdev_max_us
;
874 if (after(tp
->snd_una
, tp
->rtt_seq
)) {
875 if (tp
->mdev_max_us
< tp
->rttvar_us
)
876 tp
->rttvar_us
-= (tp
->rttvar_us
- tp
->mdev_max_us
) >> 2;
877 tp
->rtt_seq
= tp
->snd_nxt
;
878 tp
->mdev_max_us
= tcp_rto_min_us(sk
);
883 /* no previous measure. */
884 srtt
= m
<< 3; /* take the measured time to be rtt */
885 tp
->mdev_us
= m
<< 1; /* make sure rto = 3*rtt */
886 tp
->rttvar_us
= max(tp
->mdev_us
, tcp_rto_min_us(sk
));
887 tp
->mdev_max_us
= tp
->rttvar_us
;
888 tp
->rtt_seq
= tp
->snd_nxt
;
892 tp
->srtt_us
= max(1U, srtt
);
895 static void tcp_update_pacing_rate(struct sock
*sk
)
897 const struct tcp_sock
*tp
= tcp_sk(sk
);
900 /* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
901 rate
= (u64
)tp
->mss_cache
* ((USEC_PER_SEC
/ 100) << 3);
903 /* current rate is (cwnd * mss) / srtt
904 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
905 * In Congestion Avoidance phase, set it to 120 % the current rate.
907 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
908 * If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
909 * end of slow start and should slow down.
911 if (tcp_snd_cwnd(tp
) < tp
->snd_ssthresh
/ 2)
912 rate
*= sock_net(sk
)->ipv4
.sysctl_tcp_pacing_ss_ratio
;
914 rate
*= sock_net(sk
)->ipv4
.sysctl_tcp_pacing_ca_ratio
;
916 rate
*= max(tcp_snd_cwnd(tp
), tp
->packets_out
);
918 if (likely(tp
->srtt_us
))
919 do_div(rate
, tp
->srtt_us
);
921 /* WRITE_ONCE() is needed because sch_fq fetches sk_pacing_rate
922 * without any lock. We want to make sure compiler wont store
923 * intermediate values in this location.
925 WRITE_ONCE(sk
->sk_pacing_rate
, min_t(u64
, rate
,
926 sk
->sk_max_pacing_rate
));
929 /* Calculate rto without backoff. This is the second half of Van Jacobson's
930 * routine referred to above.
932 static void tcp_set_rto(struct sock
*sk
)
934 const struct tcp_sock
*tp
= tcp_sk(sk
);
935 /* Old crap is replaced with new one. 8)
938 * 1. If rtt variance happened to be less 50msec, it is hallucination.
939 * It cannot be less due to utterly erratic ACK generation made
940 * at least by solaris and freebsd. "Erratic ACKs" has _nothing_
941 * to do with delayed acks, because at cwnd>2 true delack timeout
942 * is invisible. Actually, Linux-2.4 also generates erratic
943 * ACKs in some circumstances.
945 inet_csk(sk
)->icsk_rto
= __tcp_set_rto(tp
);
947 /* 2. Fixups made earlier cannot be right.
948 * If we do not estimate RTO correctly without them,
949 * all the algo is pure shit and should be replaced
950 * with correct one. It is exactly, which we pretend to do.
953 /* NOTE: clamping at TCP_RTO_MIN is not required, current algo
954 * guarantees that rto is higher.
959 __u32
tcp_init_cwnd(const struct tcp_sock
*tp
, const struct dst_entry
*dst
)
961 __u32 cwnd
= (dst
? dst_metric(dst
, RTAX_INITCWND
) : 0);
964 cwnd
= TCP_INIT_CWND
;
965 return min_t(__u32
, cwnd
, tp
->snd_cwnd_clamp
);
968 struct tcp_sacktag_state
{
969 /* Timestamps for earliest and latest never-retransmitted segment
970 * that was SACKed. RTO needs the earliest RTT to stay conservative,
971 * but congestion control should still get an accurate delay signal.
978 unsigned int mss_now
;
979 struct rate_sample
*rate
;
982 /* Take a notice that peer is sending D-SACKs. Skip update of data delivery
983 * and spurious retransmission information if this DSACK is unlikely caused by
985 * - DSACKed sequence range is larger than maximum receiver's window.
986 * - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
988 static u32
tcp_dsack_seen(struct tcp_sock
*tp
, u32 start_seq
,
989 u32 end_seq
, struct tcp_sacktag_state
*state
)
991 u32 seq_len
, dup_segs
= 1;
993 if (!before(start_seq
, end_seq
))
996 seq_len
= end_seq
- start_seq
;
997 /* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
998 if (seq_len
> tp
->max_window
)
1000 if (seq_len
> tp
->mss_cache
)
1001 dup_segs
= DIV_ROUND_UP(seq_len
, tp
->mss_cache
);
1002 else if (tp
->tlp_high_seq
&& tp
->tlp_high_seq
== end_seq
)
1003 state
->flag
|= FLAG_DSACK_TLP
;
1005 tp
->dsack_dups
+= dup_segs
;
1006 /* Skip the DSACK if dup segs weren't retransmitted by sender */
1007 if (tp
->dsack_dups
> tp
->total_retrans
)
1010 tp
->rx_opt
.sack_ok
|= TCP_DSACK_SEEN
;
1011 /* We increase the RACK ordering window in rounds where we receive
1012 * DSACKs that may have been due to reordering causing RACK to trigger
1013 * a spurious fast recovery. Thus RACK ignores DSACKs that happen
1014 * without having seen reordering, or that match TLP probes (TLP
1015 * is timer-driven, not triggered by RACK).
1017 if (tp
->reord_seen
&& !(state
->flag
& FLAG_DSACK_TLP
))
1018 tp
->rack
.dsack_seen
= 1;
1020 state
->flag
|= FLAG_DSACKING_ACK
;
1021 /* A spurious retransmission is delivered */
1022 state
->sack_delivered
+= dup_segs
;
1027 /* It's reordering when higher sequence was delivered (i.e. sacked) before
1028 * some lower never-retransmitted sequence ("low_seq"). The maximum reordering
1029 * distance is approximated in full-mss packet distance ("reordering").
1031 static void tcp_check_sack_reordering(struct sock
*sk
, const u32 low_seq
,
1034 struct tcp_sock
*tp
= tcp_sk(sk
);
1035 const u32 mss
= tp
->mss_cache
;
1038 fack
= tcp_highest_sack_seq(tp
);
1039 if (!before(low_seq
, fack
))
1042 metric
= fack
- low_seq
;
1043 if ((metric
> tp
->reordering
* mss
) && mss
) {
1044 #if FASTRETRANS_DEBUG > 1
1045 pr_debug("Disorder%d %d %u f%u s%u rr%d\n",
1046 tp
->rx_opt
.sack_ok
, inet_csk(sk
)->icsk_ca_state
,
1050 tp
->undo_marker
? tp
->undo_retrans
: 0);
1052 tp
->reordering
= min_t(u32
, (metric
+ mss
- 1) / mss
,
1053 sock_net(sk
)->ipv4
.sysctl_tcp_max_reordering
);
1056 /* This exciting event is worth to be remembered. 8) */
1058 NET_INC_STATS(sock_net(sk
),
1059 ts
? LINUX_MIB_TCPTSREORDER
: LINUX_MIB_TCPSACKREORDER
);
1062 /* This must be called before lost_out or retrans_out are updated
1063 * on a new loss, because we want to know if all skbs previously
1064 * known to be lost have already been retransmitted, indicating
1065 * that this newly lost skb is our next skb to retransmit.
1067 static void tcp_verify_retransmit_hint(struct tcp_sock
*tp
, struct sk_buff
*skb
)
1069 if ((!tp
->retransmit_skb_hint
&& tp
->retrans_out
>= tp
->lost_out
) ||
1070 (tp
->retransmit_skb_hint
&&
1071 before(TCP_SKB_CB(skb
)->seq
,
1072 TCP_SKB_CB(tp
->retransmit_skb_hint
)->seq
)))
1073 tp
->retransmit_skb_hint
= skb
;
1076 /* Sum the number of packets on the wire we have marked as lost, and
1077 * notify the congestion control module that the given skb was marked lost.
1079 static void tcp_notify_skb_loss_event(struct tcp_sock
*tp
, const struct sk_buff
*skb
)
1081 tp
->lost
+= tcp_skb_pcount(skb
);
1084 void tcp_mark_skb_lost(struct sock
*sk
, struct sk_buff
*skb
)
1086 __u8 sacked
= TCP_SKB_CB(skb
)->sacked
;
1087 struct tcp_sock
*tp
= tcp_sk(sk
);
1089 if (sacked
& TCPCB_SACKED_ACKED
)
1092 tcp_verify_retransmit_hint(tp
, skb
);
1093 if (sacked
& TCPCB_LOST
) {
1094 if (sacked
& TCPCB_SACKED_RETRANS
) {
1095 /* Account for retransmits that are lost again */
1096 TCP_SKB_CB(skb
)->sacked
&= ~TCPCB_SACKED_RETRANS
;
1097 tp
->retrans_out
-= tcp_skb_pcount(skb
);
1098 NET_ADD_STATS(sock_net(sk
), LINUX_MIB_TCPLOSTRETRANSMIT
,
1099 tcp_skb_pcount(skb
));
1100 tcp_notify_skb_loss_event(tp
, skb
);
1103 tp
->lost_out
+= tcp_skb_pcount(skb
);
1104 TCP_SKB_CB(skb
)->sacked
|= TCPCB_LOST
;
1105 tcp_notify_skb_loss_event(tp
, skb
);
1109 /* Updates the delivered and delivered_ce counts */
1110 static void tcp_count_delivered(struct tcp_sock
*tp
, u32 delivered
,
1113 tp
->delivered
+= delivered
;
1115 tp
->delivered_ce
+= delivered
;
1118 /* This procedure tags the retransmission queue when SACKs arrive.
1120 * We have three tag bits: SACKED(S), RETRANS(R) and LOST(L).
1121 * Packets in queue with these bits set are counted in variables
1122 * sacked_out, retrans_out and lost_out, correspondingly.
1124 * Valid combinations are:
1125 * Tag InFlight Description
1126 * 0 1 - orig segment is in flight.
1127 * S 0 - nothing flies, orig reached receiver.
1128 * L 0 - nothing flies, orig lost by net.
1129 * R 2 - both orig and retransmit are in flight.
1130 * L|R 1 - orig is lost, retransmit is in flight.
1131 * S|R 1 - orig reached receiver, retrans is still in flight.
1132 * (L|S|R is logically valid, it could occur when L|R is sacked,
1133 * but it is equivalent to plain S and code short-curcuits it to S.
1134 * L|S is logically invalid, it would mean -1 packet in flight 8))
1136 * These 6 states form finite state machine, controlled by the following events:
1137 * 1. New ACK (+SACK) arrives. (tcp_sacktag_write_queue())
1138 * 2. Retransmission. (tcp_retransmit_skb(), tcp_xmit_retransmit_queue())
1139 * 3. Loss detection event of two flavors:
1140 * A. Scoreboard estimator decided the packet is lost.
1141 * A'. Reno "three dupacks" marks head of queue lost.
1142 * B. SACK arrives sacking SND.NXT at the moment, when the
1143 * segment was retransmitted.
1144 * 4. D-SACK added new rule: D-SACK changes any tag to S.
1146 * It is pleasant to note, that state diagram turns out to be commutative,
1147 * so that we are allowed not to be bothered by order of our actions,
1148 * when multiple events arrive simultaneously. (see the function below).
1150 * Reordering detection.
1151 * --------------------
1152 * Reordering metric is maximal distance, which a packet can be displaced
1153 * in packet stream. With SACKs we can estimate it:
1155 * 1. SACK fills old hole and the corresponding segment was not
1156 * ever retransmitted -> reordering. Alas, we cannot use it
1157 * when segment was retransmitted.
1158 * 2. The last flaw is solved with D-SACK. D-SACK arrives
1159 * for retransmitted and already SACKed segment -> reordering..
1160 * Both of these heuristics are not used in Loss state, when we cannot
1161 * account for retransmits accurately.
1163 * SACK block validation.
1164 * ----------------------
1166 * SACK block range validation checks that the received SACK block fits to
1167 * the expected sequence limits, i.e., it is between SND.UNA and SND.NXT.
1168 * Note that SND.UNA is not included to the range though being valid because
1169 * it means that the receiver is rather inconsistent with itself reporting
1170 * SACK reneging when it should advance SND.UNA. Such SACK block this is
1171 * perfectly valid, however, in light of RFC2018 which explicitly states
1172 * that "SACK block MUST reflect the newest segment. Even if the newest
1173 * segment is going to be discarded ...", not that it looks very clever
1174 * in case of head skb. Due to potentional receiver driven attacks, we
1175 * choose to avoid immediate execution of a walk in write queue due to
1176 * reneging and defer head skb's loss recovery to standard loss recovery
1177 * procedure that will eventually trigger (nothing forbids us doing this).
1179 * Implements also blockage to start_seq wrap-around. Problem lies in the
1180 * fact that though start_seq (s) is before end_seq (i.e., not reversed),
1181 * there's no guarantee that it will be before snd_nxt (n). The problem
1182 * happens when start_seq resides between end_seq wrap (e_w) and snd_nxt
1185 * <- outs wnd -> <- wrapzone ->
1186 * u e n u_w e_w s n_w
1188 * |<------------+------+----- TCP seqno space --------------+---------->|
1189 * ...-- <2^31 ->| |<--------...
1190 * ...---- >2^31 ------>| |<--------...
1192 * Current code wouldn't be vulnerable but it's better still to discard such
1193 * crazy SACK blocks. Doing this check for start_seq alone closes somewhat
1194 * similar case (end_seq after snd_nxt wrap) as earlier reversed check in
1195 * snd_nxt wrap -> snd_una region will then become "well defined", i.e.,
1196 * equal to the ideal case (infinite seqno space without wrap caused issues).
1198 * With D-SACK the lower bound is extended to cover sequence space below
1199 * SND.UNA down to undo_marker, which is the last point of interest. Yet
1200 * again, D-SACK block must not to go across snd_una (for the same reason as
1201 * for the normal SACK blocks, explained above). But there all simplicity
1202 * ends, TCP might receive valid D-SACKs below that. As long as they reside
1203 * fully below undo_marker they do not affect behavior in anyway and can
1204 * therefore be safely ignored. In rare cases (which are more or less
1205 * theoretical ones), the D-SACK will nicely cross that boundary due to skb
1206 * fragmentation and packet reordering past skb's retransmission. To consider
1207 * them correctly, the acceptable range must be extended even more though
1208 * the exact amount is rather hard to quantify. However, tp->max_window can
1209 * be used as an exaggerated estimate.
1211 static bool tcp_is_sackblock_valid(struct tcp_sock
*tp
, bool is_dsack
,
1212 u32 start_seq
, u32 end_seq
)
1214 /* Too far in future, or reversed (interpretation is ambiguous) */
1215 if (after(end_seq
, tp
->snd_nxt
) || !before(start_seq
, end_seq
))
1218 /* Nasty start_seq wrap-around check (see comments above) */
1219 if (!before(start_seq
, tp
->snd_nxt
))
1222 /* In outstanding window? ...This is valid exit for D-SACKs too.
1223 * start_seq == snd_una is non-sensical (see comments above)
1225 if (after(start_seq
, tp
->snd_una
))
1228 if (!is_dsack
|| !tp
->undo_marker
)
1231 /* ...Then it's D-SACK, and must reside below snd_una completely */
1232 if (after(end_seq
, tp
->snd_una
))
1235 if (!before(start_seq
, tp
->undo_marker
))
1239 if (!after(end_seq
, tp
->undo_marker
))
1242 /* Undo_marker boundary crossing (overestimates a lot). Known already:
1243 * start_seq < undo_marker and end_seq >= undo_marker.
1245 return !before(start_seq
, end_seq
- tp
->max_window
);
1248 static bool tcp_check_dsack(struct sock
*sk
, const struct sk_buff
*ack_skb
,
1249 struct tcp_sack_block_wire
*sp
, int num_sacks
,
1250 u32 prior_snd_una
, struct tcp_sacktag_state
*state
)
1252 struct tcp_sock
*tp
= tcp_sk(sk
);
1253 u32 start_seq_0
= get_unaligned_be32(&sp
[0].start_seq
);
1254 u32 end_seq_0
= get_unaligned_be32(&sp
[0].end_seq
);
1257 if (before(start_seq_0
, TCP_SKB_CB(ack_skb
)->ack_seq
)) {
1258 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPDSACKRECV
);
1259 } else if (num_sacks
> 1) {
1260 u32 end_seq_1
= get_unaligned_be32(&sp
[1].end_seq
);
1261 u32 start_seq_1
= get_unaligned_be32(&sp
[1].start_seq
);
1263 if (after(end_seq_0
, end_seq_1
) || before(start_seq_0
, start_seq_1
))
1265 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPDSACKOFORECV
);
1270 dup_segs
= tcp_dsack_seen(tp
, start_seq_0
, end_seq_0
, state
);
1271 if (!dup_segs
) { /* Skip dubious DSACK */
1272 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS
);
1276 NET_ADD_STATS(sock_net(sk
), LINUX_MIB_TCPDSACKRECVSEGS
, dup_segs
);
1278 /* D-SACK for already forgotten data... Do dumb counting. */
1279 if (tp
->undo_marker
&& tp
->undo_retrans
> 0 &&
1280 !after(end_seq_0
, prior_snd_una
) &&
1281 after(end_seq_0
, tp
->undo_marker
))
1282 tp
->undo_retrans
= max_t(int, 0, tp
->undo_retrans
- dup_segs
);
1287 /* Check if skb is fully within the SACK block. In presence of GSO skbs,
1288 * the incoming SACK may not exactly match but we can find smaller MSS
1289 * aligned portion of it that matches. Therefore we might need to fragment
1290 * which may fail and creates some hassle (caller must handle error case
1293 * FIXME: this could be merged to shift decision code
1295 static int tcp_match_skb_to_sack(struct sock
*sk
, struct sk_buff
*skb
,
1296 u32 start_seq
, u32 end_seq
)
1300 unsigned int pkt_len
;
1303 in_sack
= !after(start_seq
, TCP_SKB_CB(skb
)->seq
) &&
1304 !before(end_seq
, TCP_SKB_CB(skb
)->end_seq
);
1306 if (tcp_skb_pcount(skb
) > 1 && !in_sack
&&
1307 after(TCP_SKB_CB(skb
)->end_seq
, start_seq
)) {
1308 mss
= tcp_skb_mss(skb
);
1309 in_sack
= !after(start_seq
, TCP_SKB_CB(skb
)->seq
);
1312 pkt_len
= start_seq
- TCP_SKB_CB(skb
)->seq
;
1316 pkt_len
= end_seq
- TCP_SKB_CB(skb
)->seq
;
1321 /* Round if necessary so that SACKs cover only full MSSes
1322 * and/or the remaining small portion (if present)
1324 if (pkt_len
> mss
) {
1325 unsigned int new_len
= (pkt_len
/ mss
) * mss
;
1326 if (!in_sack
&& new_len
< pkt_len
)
1331 if (pkt_len
>= skb
->len
&& !in_sack
)
1334 err
= tcp_fragment(sk
, TCP_FRAG_IN_RTX_QUEUE
, skb
,
1335 pkt_len
, mss
, GFP_ATOMIC
);
1343 /* Mark the given newly-SACKed range as such, adjusting counters and hints. */
1344 static u8
tcp_sacktag_one(struct sock
*sk
,
1345 struct tcp_sacktag_state
*state
, u8 sacked
,
1346 u32 start_seq
, u32 end_seq
,
1347 int dup_sack
, int pcount
,
1350 struct tcp_sock
*tp
= tcp_sk(sk
);
1352 /* Account D-SACK for retransmitted packet. */
1353 if (dup_sack
&& (sacked
& TCPCB_RETRANS
)) {
1354 if (tp
->undo_marker
&& tp
->undo_retrans
> 0 &&
1355 after(end_seq
, tp
->undo_marker
))
1356 tp
->undo_retrans
= max_t(int, 0, tp
->undo_retrans
- pcount
);
1357 if ((sacked
& TCPCB_SACKED_ACKED
) &&
1358 before(start_seq
, state
->reord
))
1359 state
->reord
= start_seq
;
1362 /* Nothing to do; acked frame is about to be dropped (was ACKed). */
1363 if (!after(end_seq
, tp
->snd_una
))
1366 if (!(sacked
& TCPCB_SACKED_ACKED
)) {
1367 tcp_rack_advance(tp
, sacked
, end_seq
, xmit_time
);
1369 if (sacked
& TCPCB_SACKED_RETRANS
) {
1370 /* If the segment is not tagged as lost,
1371 * we do not clear RETRANS, believing
1372 * that retransmission is still in flight.
1374 if (sacked
& TCPCB_LOST
) {
1375 sacked
&= ~(TCPCB_LOST
|TCPCB_SACKED_RETRANS
);
1376 tp
->lost_out
-= pcount
;
1377 tp
->retrans_out
-= pcount
;
1380 if (!(sacked
& TCPCB_RETRANS
)) {
1381 /* New sack for not retransmitted frame,
1382 * which was in hole. It is reordering.
1384 if (before(start_seq
,
1385 tcp_highest_sack_seq(tp
)) &&
1386 before(start_seq
, state
->reord
))
1387 state
->reord
= start_seq
;
1389 if (!after(end_seq
, tp
->high_seq
))
1390 state
->flag
|= FLAG_ORIG_SACK_ACKED
;
1391 if (state
->first_sackt
== 0)
1392 state
->first_sackt
= xmit_time
;
1393 state
->last_sackt
= xmit_time
;
1396 if (sacked
& TCPCB_LOST
) {
1397 sacked
&= ~TCPCB_LOST
;
1398 tp
->lost_out
-= pcount
;
1402 sacked
|= TCPCB_SACKED_ACKED
;
1403 state
->flag
|= FLAG_DATA_SACKED
;
1404 tp
->sacked_out
+= pcount
;
1405 /* Out-of-order packets delivered */
1406 state
->sack_delivered
+= pcount
;
1408 /* Lost marker hint past SACKed? Tweak RFC3517 cnt */
1409 if (tp
->lost_skb_hint
&&
1410 before(start_seq
, TCP_SKB_CB(tp
->lost_skb_hint
)->seq
))
1411 tp
->lost_cnt_hint
+= pcount
;
1414 /* D-SACK. We can detect redundant retransmission in S|R and plain R
1415 * frames and clear it. undo_retrans is decreased above, L|R frames
1416 * are accounted above as well.
1418 if (dup_sack
&& (sacked
& TCPCB_SACKED_RETRANS
)) {
1419 sacked
&= ~TCPCB_SACKED_RETRANS
;
1420 tp
->retrans_out
-= pcount
;
1426 /* Shift newly-SACKed bytes from this skb to the immediately previous
1427 * already-SACKed sk_buff. Mark the newly-SACKed bytes as such.
1429 static bool tcp_shifted_skb(struct sock
*sk
, struct sk_buff
*prev
,
1430 struct sk_buff
*skb
,
1431 struct tcp_sacktag_state
*state
,
1432 unsigned int pcount
, int shifted
, int mss
,
1435 struct tcp_sock
*tp
= tcp_sk(sk
);
1436 u32 start_seq
= TCP_SKB_CB(skb
)->seq
; /* start of newly-SACKed */
1437 u32 end_seq
= start_seq
+ shifted
; /* end of newly-SACKed */
1441 /* Adjust counters and hints for the newly sacked sequence
1442 * range but discard the return value since prev is already
1443 * marked. We must tag the range first because the seq
1444 * advancement below implicitly advances
1445 * tcp_highest_sack_seq() when skb is highest_sack.
1447 tcp_sacktag_one(sk
, state
, TCP_SKB_CB(skb
)->sacked
,
1448 start_seq
, end_seq
, dup_sack
, pcount
,
1449 tcp_skb_timestamp_us(skb
));
1450 tcp_rate_skb_delivered(sk
, skb
, state
->rate
);
1452 if (skb
== tp
->lost_skb_hint
)
1453 tp
->lost_cnt_hint
+= pcount
;
1455 TCP_SKB_CB(prev
)->end_seq
+= shifted
;
1456 TCP_SKB_CB(skb
)->seq
+= shifted
;
1458 tcp_skb_pcount_add(prev
, pcount
);
1459 WARN_ON_ONCE(tcp_skb_pcount(skb
) < pcount
);
1460 tcp_skb_pcount_add(skb
, -pcount
);
1462 /* When we're adding to gso_segs == 1, gso_size will be zero,
1463 * in theory this shouldn't be necessary but as long as DSACK
1464 * code can come after this skb later on it's better to keep
1465 * setting gso_size to something.
1467 if (!TCP_SKB_CB(prev
)->tcp_gso_size
)
1468 TCP_SKB_CB(prev
)->tcp_gso_size
= mss
;
1470 /* CHECKME: To clear or not to clear? Mimics normal skb currently */
1471 if (tcp_skb_pcount(skb
) <= 1)
1472 TCP_SKB_CB(skb
)->tcp_gso_size
= 0;
1474 /* Difference in this won't matter, both ACKed by the same cumul. ACK */
1475 TCP_SKB_CB(prev
)->sacked
|= (TCP_SKB_CB(skb
)->sacked
& TCPCB_EVER_RETRANS
);
1478 BUG_ON(!tcp_skb_pcount(skb
));
1479 NET_INC_STATS(sock_net(sk
), LINUX_MIB_SACKSHIFTED
);
1483 /* Whole SKB was eaten :-) */
1485 if (skb
== tp
->retransmit_skb_hint
)
1486 tp
->retransmit_skb_hint
= prev
;
1487 if (skb
== tp
->lost_skb_hint
) {
1488 tp
->lost_skb_hint
= prev
;
1489 tp
->lost_cnt_hint
-= tcp_skb_pcount(prev
);
1492 TCP_SKB_CB(prev
)->tcp_flags
|= TCP_SKB_CB(skb
)->tcp_flags
;
1493 TCP_SKB_CB(prev
)->eor
= TCP_SKB_CB(skb
)->eor
;
1494 if (TCP_SKB_CB(skb
)->tcp_flags
& TCPHDR_FIN
)
1495 TCP_SKB_CB(prev
)->end_seq
++;
1497 if (skb
== tcp_highest_sack(sk
))
1498 tcp_advance_highest_sack(sk
, skb
);
1500 tcp_skb_collapse_tstamp(prev
, skb
);
1501 if (unlikely(TCP_SKB_CB(prev
)->tx
.delivered_mstamp
))
1502 TCP_SKB_CB(prev
)->tx
.delivered_mstamp
= 0;
1504 tcp_rtx_queue_unlink_and_free(skb
, sk
);
1506 NET_INC_STATS(sock_net(sk
), LINUX_MIB_SACKMERGED
);
1511 /* I wish gso_size would have a bit more sane initialization than
1512 * something-or-zero which complicates things
1514 static int tcp_skb_seglen(const struct sk_buff
*skb
)
1516 return tcp_skb_pcount(skb
) == 1 ? skb
->len
: tcp_skb_mss(skb
);
1519 /* Shifting pages past head area doesn't work */
1520 static int skb_can_shift(const struct sk_buff
*skb
)
1522 return !skb_headlen(skb
) && skb_is_nonlinear(skb
);
1525 int tcp_skb_shift(struct sk_buff
*to
, struct sk_buff
*from
,
1526 int pcount
, int shiftlen
)
1528 /* TCP min gso_size is 8 bytes (TCP_MIN_GSO_SIZE)
1529 * Since TCP_SKB_CB(skb)->tcp_gso_segs is 16 bits, we need
1530 * to make sure not storing more than 65535 * 8 bytes per skb,
1531 * even if current MSS is bigger.
1533 if (unlikely(to
->len
+ shiftlen
>= 65535 * TCP_MIN_GSO_SIZE
))
1535 if (unlikely(tcp_skb_pcount(to
) + pcount
> 65535))
1537 return skb_shift(to
, from
, shiftlen
);
1540 /* Try collapsing SACK blocks spanning across multiple skbs to a single
1543 static struct sk_buff
*tcp_shift_skb_data(struct sock
*sk
, struct sk_buff
*skb
,
1544 struct tcp_sacktag_state
*state
,
1545 u32 start_seq
, u32 end_seq
,
1548 struct tcp_sock
*tp
= tcp_sk(sk
);
1549 struct sk_buff
*prev
;
1555 /* Normally R but no L won't result in plain S */
1557 (TCP_SKB_CB(skb
)->sacked
& (TCPCB_LOST
|TCPCB_SACKED_RETRANS
)) == TCPCB_SACKED_RETRANS
)
1559 if (!skb_can_shift(skb
))
1561 /* This frame is about to be dropped (was ACKed). */
1562 if (!after(TCP_SKB_CB(skb
)->end_seq
, tp
->snd_una
))
1565 /* Can only happen with delayed DSACK + discard craziness */
1566 prev
= skb_rb_prev(skb
);
1570 if ((TCP_SKB_CB(prev
)->sacked
& TCPCB_TAGBITS
) != TCPCB_SACKED_ACKED
)
1573 if (!tcp_skb_can_collapse(prev
, skb
))
1576 in_sack
= !after(start_seq
, TCP_SKB_CB(skb
)->seq
) &&
1577 !before(end_seq
, TCP_SKB_CB(skb
)->end_seq
);
1581 pcount
= tcp_skb_pcount(skb
);
1582 mss
= tcp_skb_seglen(skb
);
1584 /* TODO: Fix DSACKs to not fragment already SACKed and we can
1585 * drop this restriction as unnecessary
1587 if (mss
!= tcp_skb_seglen(prev
))
1590 if (!after(TCP_SKB_CB(skb
)->end_seq
, start_seq
))
1592 /* CHECKME: This is non-MSS split case only?, this will
1593 * cause skipped skbs due to advancing loop btw, original
1594 * has that feature too
1596 if (tcp_skb_pcount(skb
) <= 1)
1599 in_sack
= !after(start_seq
, TCP_SKB_CB(skb
)->seq
);
1601 /* TODO: head merge to next could be attempted here
1602 * if (!after(TCP_SKB_CB(skb)->end_seq, end_seq)),
1603 * though it might not be worth of the additional hassle
1605 * ...we can probably just fallback to what was done
1606 * previously. We could try merging non-SACKed ones
1607 * as well but it probably isn't going to buy off
1608 * because later SACKs might again split them, and
1609 * it would make skb timestamp tracking considerably
1615 len
= end_seq
- TCP_SKB_CB(skb
)->seq
;
1617 BUG_ON(len
> skb
->len
);
1619 /* MSS boundaries should be honoured or else pcount will
1620 * severely break even though it makes things bit trickier.
1621 * Optimize common case to avoid most of the divides
1623 mss
= tcp_skb_mss(skb
);
1625 /* TODO: Fix DSACKs to not fragment already SACKed and we can
1626 * drop this restriction as unnecessary
1628 if (mss
!= tcp_skb_seglen(prev
))
1633 } else if (len
< mss
) {
1641 /* tcp_sacktag_one() won't SACK-tag ranges below snd_una */
1642 if (!after(TCP_SKB_CB(skb
)->seq
+ len
, tp
->snd_una
))
1645 if (!tcp_skb_shift(prev
, skb
, pcount
, len
))
1647 if (!tcp_shifted_skb(sk
, prev
, skb
, state
, pcount
, len
, mss
, dup_sack
))
1650 /* Hole filled allows collapsing with the next as well, this is very
1651 * useful when hole on every nth skb pattern happens
1653 skb
= skb_rb_next(prev
);
1657 if (!skb_can_shift(skb
) ||
1658 ((TCP_SKB_CB(skb
)->sacked
& TCPCB_TAGBITS
) != TCPCB_SACKED_ACKED
) ||
1659 (mss
!= tcp_skb_seglen(skb
)))
1662 if (!tcp_skb_can_collapse(prev
, skb
))
1665 pcount
= tcp_skb_pcount(skb
);
1666 if (tcp_skb_shift(prev
, skb
, pcount
, len
))
1667 tcp_shifted_skb(sk
, prev
, skb
, state
, pcount
,
1677 NET_INC_STATS(sock_net(sk
), LINUX_MIB_SACKSHIFTFALLBACK
);
1681 static struct sk_buff
*tcp_sacktag_walk(struct sk_buff
*skb
, struct sock
*sk
,
1682 struct tcp_sack_block
*next_dup
,
1683 struct tcp_sacktag_state
*state
,
1684 u32 start_seq
, u32 end_seq
,
1687 struct tcp_sock
*tp
= tcp_sk(sk
);
1688 struct sk_buff
*tmp
;
1690 skb_rbtree_walk_from(skb
) {
1692 bool dup_sack
= dup_sack_in
;
1694 /* queue is in-order => we can short-circuit the walk early */
1695 if (!before(TCP_SKB_CB(skb
)->seq
, end_seq
))
1699 before(TCP_SKB_CB(skb
)->seq
, next_dup
->end_seq
)) {
1700 in_sack
= tcp_match_skb_to_sack(sk
, skb
,
1701 next_dup
->start_seq
,
1707 /* skb reference here is a bit tricky to get right, since
1708 * shifting can eat and free both this skb and the next,
1709 * so not even _safe variant of the loop is enough.
1712 tmp
= tcp_shift_skb_data(sk
, skb
, state
,
1713 start_seq
, end_seq
, dup_sack
);
1722 in_sack
= tcp_match_skb_to_sack(sk
, skb
,
1728 if (unlikely(in_sack
< 0))
1732 TCP_SKB_CB(skb
)->sacked
=
1735 TCP_SKB_CB(skb
)->sacked
,
1736 TCP_SKB_CB(skb
)->seq
,
1737 TCP_SKB_CB(skb
)->end_seq
,
1739 tcp_skb_pcount(skb
),
1740 tcp_skb_timestamp_us(skb
));
1741 tcp_rate_skb_delivered(sk
, skb
, state
->rate
);
1742 if (TCP_SKB_CB(skb
)->sacked
& TCPCB_SACKED_ACKED
)
1743 list_del_init(&skb
->tcp_tsorted_anchor
);
1745 if (!before(TCP_SKB_CB(skb
)->seq
,
1746 tcp_highest_sack_seq(tp
)))
1747 tcp_advance_highest_sack(sk
, skb
);
1753 static struct sk_buff
*tcp_sacktag_bsearch(struct sock
*sk
, u32 seq
)
1755 struct rb_node
*parent
, **p
= &sk
->tcp_rtx_queue
.rb_node
;
1756 struct sk_buff
*skb
;
1760 skb
= rb_to_skb(parent
);
1761 if (before(seq
, TCP_SKB_CB(skb
)->seq
)) {
1762 p
= &parent
->rb_left
;
1765 if (!before(seq
, TCP_SKB_CB(skb
)->end_seq
)) {
1766 p
= &parent
->rb_right
;
1774 static struct sk_buff
*tcp_sacktag_skip(struct sk_buff
*skb
, struct sock
*sk
,
1777 if (skb
&& after(TCP_SKB_CB(skb
)->seq
, skip_to_seq
))
1780 return tcp_sacktag_bsearch(sk
, skip_to_seq
);
1783 static struct sk_buff
*tcp_maybe_skipping_dsack(struct sk_buff
*skb
,
1785 struct tcp_sack_block
*next_dup
,
1786 struct tcp_sacktag_state
*state
,
1792 if (before(next_dup
->start_seq
, skip_to_seq
)) {
1793 skb
= tcp_sacktag_skip(skb
, sk
, next_dup
->start_seq
);
1794 skb
= tcp_sacktag_walk(skb
, sk
, NULL
, state
,
1795 next_dup
->start_seq
, next_dup
->end_seq
,
1802 static int tcp_sack_cache_ok(const struct tcp_sock
*tp
, const struct tcp_sack_block
*cache
)
1804 return cache
< tp
->recv_sack_cache
+ ARRAY_SIZE(tp
->recv_sack_cache
);
1808 tcp_sacktag_write_queue(struct sock
*sk
, const struct sk_buff
*ack_skb
,
1809 u32 prior_snd_una
, struct tcp_sacktag_state
*state
)
1811 struct tcp_sock
*tp
= tcp_sk(sk
);
1812 const unsigned char *ptr
= (skb_transport_header(ack_skb
) +
1813 TCP_SKB_CB(ack_skb
)->sacked
);
1814 struct tcp_sack_block_wire
*sp_wire
= (struct tcp_sack_block_wire
*)(ptr
+2);
1815 struct tcp_sack_block sp
[TCP_NUM_SACKS
];
1816 struct tcp_sack_block
*cache
;
1817 struct sk_buff
*skb
;
1818 int num_sacks
= min(TCP_NUM_SACKS
, (ptr
[1] - TCPOLEN_SACK_BASE
) >> 3);
1820 bool found_dup_sack
= false;
1822 int first_sack_index
;
1825 state
->reord
= tp
->snd_nxt
;
1827 if (!tp
->sacked_out
)
1828 tcp_highest_sack_reset(sk
);
1830 found_dup_sack
= tcp_check_dsack(sk
, ack_skb
, sp_wire
,
1831 num_sacks
, prior_snd_una
, state
);
1833 /* Eliminate too old ACKs, but take into
1834 * account more or less fresh ones, they can
1835 * contain valid SACK info.
1837 if (before(TCP_SKB_CB(ack_skb
)->ack_seq
, prior_snd_una
- tp
->max_window
))
1840 if (!tp
->packets_out
)
1844 first_sack_index
= 0;
1845 for (i
= 0; i
< num_sacks
; i
++) {
1846 bool dup_sack
= !i
&& found_dup_sack
;
1848 sp
[used_sacks
].start_seq
= get_unaligned_be32(&sp_wire
[i
].start_seq
);
1849 sp
[used_sacks
].end_seq
= get_unaligned_be32(&sp_wire
[i
].end_seq
);
1851 if (!tcp_is_sackblock_valid(tp
, dup_sack
,
1852 sp
[used_sacks
].start_seq
,
1853 sp
[used_sacks
].end_seq
)) {
1857 if (!tp
->undo_marker
)
1858 mib_idx
= LINUX_MIB_TCPDSACKIGNOREDNOUNDO
;
1860 mib_idx
= LINUX_MIB_TCPDSACKIGNOREDOLD
;
1862 /* Don't count olds caused by ACK reordering */
1863 if ((TCP_SKB_CB(ack_skb
)->ack_seq
!= tp
->snd_una
) &&
1864 !after(sp
[used_sacks
].end_seq
, tp
->snd_una
))
1866 mib_idx
= LINUX_MIB_TCPSACKDISCARD
;
1869 NET_INC_STATS(sock_net(sk
), mib_idx
);
1871 first_sack_index
= -1;
1875 /* Ignore very old stuff early */
1876 if (!after(sp
[used_sacks
].end_seq
, prior_snd_una
)) {
1878 first_sack_index
= -1;
1885 /* order SACK blocks to allow in order walk of the retrans queue */
1886 for (i
= used_sacks
- 1; i
> 0; i
--) {
1887 for (j
= 0; j
< i
; j
++) {
1888 if (after(sp
[j
].start_seq
, sp
[j
+ 1].start_seq
)) {
1889 swap(sp
[j
], sp
[j
+ 1]);
1891 /* Track where the first SACK block goes to */
1892 if (j
== first_sack_index
)
1893 first_sack_index
= j
+ 1;
1898 state
->mss_now
= tcp_current_mss(sk
);
1902 if (!tp
->sacked_out
) {
1903 /* It's already past, so skip checking against it */
1904 cache
= tp
->recv_sack_cache
+ ARRAY_SIZE(tp
->recv_sack_cache
);
1906 cache
= tp
->recv_sack_cache
;
1907 /* Skip empty blocks in at head of the cache */
1908 while (tcp_sack_cache_ok(tp
, cache
) && !cache
->start_seq
&&
1913 while (i
< used_sacks
) {
1914 u32 start_seq
= sp
[i
].start_seq
;
1915 u32 end_seq
= sp
[i
].end_seq
;
1916 bool dup_sack
= (found_dup_sack
&& (i
== first_sack_index
));
1917 struct tcp_sack_block
*next_dup
= NULL
;
1919 if (found_dup_sack
&& ((i
+ 1) == first_sack_index
))
1920 next_dup
= &sp
[i
+ 1];
1922 /* Skip too early cached blocks */
1923 while (tcp_sack_cache_ok(tp
, cache
) &&
1924 !before(start_seq
, cache
->end_seq
))
1927 /* Can skip some work by looking recv_sack_cache? */
1928 if (tcp_sack_cache_ok(tp
, cache
) && !dup_sack
&&
1929 after(end_seq
, cache
->start_seq
)) {
1932 if (before(start_seq
, cache
->start_seq
)) {
1933 skb
= tcp_sacktag_skip(skb
, sk
, start_seq
);
1934 skb
= tcp_sacktag_walk(skb
, sk
, next_dup
,
1941 /* Rest of the block already fully processed? */
1942 if (!after(end_seq
, cache
->end_seq
))
1945 skb
= tcp_maybe_skipping_dsack(skb
, sk
, next_dup
,
1949 /* ...tail remains todo... */
1950 if (tcp_highest_sack_seq(tp
) == cache
->end_seq
) {
1951 /* ...but better entrypoint exists! */
1952 skb
= tcp_highest_sack(sk
);
1959 skb
= tcp_sacktag_skip(skb
, sk
, cache
->end_seq
);
1960 /* Check overlap against next cached too (past this one already) */
1965 if (!before(start_seq
, tcp_highest_sack_seq(tp
))) {
1966 skb
= tcp_highest_sack(sk
);
1970 skb
= tcp_sacktag_skip(skb
, sk
, start_seq
);
1973 skb
= tcp_sacktag_walk(skb
, sk
, next_dup
, state
,
1974 start_seq
, end_seq
, dup_sack
);
1980 /* Clear the head of the cache sack blocks so we can skip it next time */
1981 for (i
= 0; i
< ARRAY_SIZE(tp
->recv_sack_cache
) - used_sacks
; i
++) {
1982 tp
->recv_sack_cache
[i
].start_seq
= 0;
1983 tp
->recv_sack_cache
[i
].end_seq
= 0;
1985 for (j
= 0; j
< used_sacks
; j
++)
1986 tp
->recv_sack_cache
[i
++] = sp
[j
];
1988 if (inet_csk(sk
)->icsk_ca_state
!= TCP_CA_Loss
|| tp
->undo_marker
)
1989 tcp_check_sack_reordering(sk
, state
->reord
, 0);
1991 tcp_verify_left_out(tp
);
1994 #if FASTRETRANS_DEBUG > 0
1995 WARN_ON((int)tp
->sacked_out
< 0);
1996 WARN_ON((int)tp
->lost_out
< 0);
1997 WARN_ON((int)tp
->retrans_out
< 0);
1998 WARN_ON((int)tcp_packets_in_flight(tp
) < 0);
2003 /* Limits sacked_out so that sum with lost_out isn't ever larger than
2004 * packets_out. Returns false if sacked_out adjustement wasn't necessary.
2006 static bool tcp_limit_reno_sacked(struct tcp_sock
*tp
)
2010 holes
= max(tp
->lost_out
, 1U);
2011 holes
= min(holes
, tp
->packets_out
);
2013 if ((tp
->sacked_out
+ holes
) > tp
->packets_out
) {
2014 tp
->sacked_out
= tp
->packets_out
- holes
;
2020 /* If we receive more dupacks than we expected counting segments
2021 * in assumption of absent reordering, interpret this as reordering.
2022 * The only another reason could be bug in receiver TCP.
2024 static void tcp_check_reno_reordering(struct sock
*sk
, const int addend
)
2026 struct tcp_sock
*tp
= tcp_sk(sk
);
2028 if (!tcp_limit_reno_sacked(tp
))
2031 tp
->reordering
= min_t(u32
, tp
->packets_out
+ addend
,
2032 sock_net(sk
)->ipv4
.sysctl_tcp_max_reordering
);
2034 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPRENOREORDER
);
2037 /* Emulate SACKs for SACKless connection: account for a new dupack. */
2039 static void tcp_add_reno_sack(struct sock
*sk
, int num_dupack
, bool ece_ack
)
2042 struct tcp_sock
*tp
= tcp_sk(sk
);
2043 u32 prior_sacked
= tp
->sacked_out
;
2046 tp
->sacked_out
+= num_dupack
;
2047 tcp_check_reno_reordering(sk
, 0);
2048 delivered
= tp
->sacked_out
- prior_sacked
;
2050 tcp_count_delivered(tp
, delivered
, ece_ack
);
2051 tcp_verify_left_out(tp
);
2055 /* Account for ACK, ACKing some data in Reno Recovery phase. */
2057 static void tcp_remove_reno_sacks(struct sock
*sk
, int acked
, bool ece_ack
)
2059 struct tcp_sock
*tp
= tcp_sk(sk
);
2062 /* One ACK acked hole. The rest eat duplicate ACKs. */
2063 tcp_count_delivered(tp
, max_t(int, acked
- tp
->sacked_out
, 1),
2065 if (acked
- 1 >= tp
->sacked_out
)
2068 tp
->sacked_out
-= acked
- 1;
2070 tcp_check_reno_reordering(sk
, acked
);
2071 tcp_verify_left_out(tp
);
2074 static inline void tcp_reset_reno_sack(struct tcp_sock
*tp
)
2079 void tcp_clear_retrans(struct tcp_sock
*tp
)
2081 tp
->retrans_out
= 0;
2083 tp
->undo_marker
= 0;
2084 tp
->undo_retrans
= -1;
2088 static inline void tcp_init_undo(struct tcp_sock
*tp
)
2090 tp
->undo_marker
= tp
->snd_una
;
2091 /* Retransmission still in flight may cause DSACKs later. */
2092 tp
->undo_retrans
= tp
->retrans_out
? : -1;
2095 static bool tcp_is_rack(const struct sock
*sk
)
2097 return sock_net(sk
)->ipv4
.sysctl_tcp_recovery
& TCP_RACK_LOSS_DETECTION
;
2100 /* If we detect SACK reneging, forget all SACK information
2101 * and reset tags completely, otherwise preserve SACKs. If receiver
2102 * dropped its ofo queue, we will know this due to reneging detection.
2104 static void tcp_timeout_mark_lost(struct sock
*sk
)
2106 struct tcp_sock
*tp
= tcp_sk(sk
);
2107 struct sk_buff
*skb
, *head
;
2108 bool is_reneg
; /* is receiver reneging on SACKs? */
2110 head
= tcp_rtx_queue_head(sk
);
2111 is_reneg
= head
&& (TCP_SKB_CB(head
)->sacked
& TCPCB_SACKED_ACKED
);
2113 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPSACKRENEGING
);
2115 /* Mark SACK reneging until we recover from this loss event. */
2116 tp
->is_sack_reneg
= 1;
2117 } else if (tcp_is_reno(tp
)) {
2118 tcp_reset_reno_sack(tp
);
2122 skb_rbtree_walk_from(skb
) {
2124 TCP_SKB_CB(skb
)->sacked
&= ~TCPCB_SACKED_ACKED
;
2125 else if (tcp_is_rack(sk
) && skb
!= head
&&
2126 tcp_rack_skb_timeout(tp
, skb
, 0) > 0)
2127 continue; /* Don't mark recently sent ones lost yet */
2128 tcp_mark_skb_lost(sk
, skb
);
2130 tcp_verify_left_out(tp
);
2131 tcp_clear_all_retrans_hints(tp
);
2134 /* Enter Loss state. */
2135 void tcp_enter_loss(struct sock
*sk
)
2137 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
2138 struct tcp_sock
*tp
= tcp_sk(sk
);
2139 struct net
*net
= sock_net(sk
);
2140 bool new_recovery
= icsk
->icsk_ca_state
< TCP_CA_Recovery
;
2142 tcp_timeout_mark_lost(sk
);
2144 /* Reduce ssthresh if it has not yet been made inside this window. */
2145 if (icsk
->icsk_ca_state
<= TCP_CA_Disorder
||
2146 !after(tp
->high_seq
, tp
->snd_una
) ||
2147 (icsk
->icsk_ca_state
== TCP_CA_Loss
&& !icsk
->icsk_retransmits
)) {
2148 tp
->prior_ssthresh
= tcp_current_ssthresh(sk
);
2149 tp
->prior_cwnd
= tcp_snd_cwnd(tp
);
2150 tp
->snd_ssthresh
= icsk
->icsk_ca_ops
->ssthresh(sk
);
2151 tcp_ca_event(sk
, CA_EVENT_LOSS
);
2154 tcp_snd_cwnd_set(tp
, tcp_packets_in_flight(tp
) + 1);
2155 tp
->snd_cwnd_cnt
= 0;
2156 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
2158 /* Timeout in disordered state after receiving substantial DUPACKs
2159 * suggests that the degree of reordering is over-estimated.
2161 if (icsk
->icsk_ca_state
<= TCP_CA_Disorder
&&
2162 tp
->sacked_out
>= net
->ipv4
.sysctl_tcp_reordering
)
2163 tp
->reordering
= min_t(unsigned int, tp
->reordering
,
2164 net
->ipv4
.sysctl_tcp_reordering
);
2165 tcp_set_ca_state(sk
, TCP_CA_Loss
);
2166 tp
->high_seq
= tp
->snd_nxt
;
2167 tcp_ecn_queue_cwr(tp
);
2169 /* F-RTO RFC5682 sec 3.1 step 1: retransmit SND.UNA if no previous
2170 * loss recovery is underway except recurring timeout(s) on
2171 * the same SND.UNA (sec 3.2). Disable F-RTO on path MTU probing
2173 tp
->frto
= net
->ipv4
.sysctl_tcp_frto
&&
2174 (new_recovery
|| icsk
->icsk_retransmits
) &&
2175 !inet_csk(sk
)->icsk_mtup
.probe_size
;
2178 /* If ACK arrived pointing to a remembered SACK, it means that our
2179 * remembered SACKs do not reflect real state of receiver i.e.
2180 * receiver _host_ is heavily congested (or buggy).
2182 * To avoid big spurious retransmission bursts due to transient SACK
2183 * scoreboard oddities that look like reneging, we give the receiver a
2184 * little time (max(RTT/2, 10ms)) to send us some more ACKs that will
2185 * restore sanity to the SACK scoreboard. If the apparent reneging
2186 * persists until this RTO then we'll clear the SACK scoreboard.
2188 static bool tcp_check_sack_reneging(struct sock
*sk
, int flag
)
2190 if (flag
& FLAG_SACK_RENEGING
) {
2191 struct tcp_sock
*tp
= tcp_sk(sk
);
2192 unsigned long delay
= max(usecs_to_jiffies(tp
->srtt_us
>> 4),
2193 msecs_to_jiffies(10));
2195 inet_csk_reset_xmit_timer(sk
, ICSK_TIME_RETRANS
,
2196 delay
, TCP_RTO_MAX
);
2202 /* Heurestics to calculate number of duplicate ACKs. There's no dupACKs
2203 * counter when SACK is enabled (without SACK, sacked_out is used for
2206 * With reordering, holes may still be in flight, so RFC3517 recovery
2207 * uses pure sacked_out (total number of SACKed segments) even though
2208 * it violates the RFC that uses duplicate ACKs, often these are equal
2209 * but when e.g. out-of-window ACKs or packet duplication occurs,
2210 * they differ. Since neither occurs due to loss, TCP should really
2213 static inline int tcp_dupack_heuristics(const struct tcp_sock
*tp
)
2215 return tp
->sacked_out
+ 1;
2218 /* Linux NewReno/SACK/ECN state machine.
2219 * --------------------------------------
2221 * "Open" Normal state, no dubious events, fast path.
2222 * "Disorder" In all the respects it is "Open",
2223 * but requires a bit more attention. It is entered when
2224 * we see some SACKs or dupacks. It is split of "Open"
2225 * mainly to move some processing from fast path to slow one.
2226 * "CWR" CWND was reduced due to some Congestion Notification event.
2227 * It can be ECN, ICMP source quench, local device congestion.
2228 * "Recovery" CWND was reduced, we are fast-retransmitting.
2229 * "Loss" CWND was reduced due to RTO timeout or SACK reneging.
2231 * tcp_fastretrans_alert() is entered:
2232 * - each incoming ACK, if state is not "Open"
2233 * - when arrived ACK is unusual, namely:
2238 * Counting packets in flight is pretty simple.
2240 * in_flight = packets_out - left_out + retrans_out
2242 * packets_out is SND.NXT-SND.UNA counted in packets.
2244 * retrans_out is number of retransmitted segments.
2246 * left_out is number of segments left network, but not ACKed yet.
2248 * left_out = sacked_out + lost_out
2250 * sacked_out: Packets, which arrived to receiver out of order
2251 * and hence not ACKed. With SACKs this number is simply
2252 * amount of SACKed data. Even without SACKs
2253 * it is easy to give pretty reliable estimate of this number,
2254 * counting duplicate ACKs.
2256 * lost_out: Packets lost by network. TCP has no explicit
2257 * "loss notification" feedback from network (for now).
2258 * It means that this number can be only _guessed_.
2259 * Actually, it is the heuristics to predict lossage that
2260 * distinguishes different algorithms.
2262 * F.e. after RTO, when all the queue is considered as lost,
2263 * lost_out = packets_out and in_flight = retrans_out.
2265 * Essentially, we have now a few algorithms detecting
2268 * If the receiver supports SACK:
2270 * RFC6675/3517: It is the conventional algorithm. A packet is
2271 * considered lost if the number of higher sequence packets
2272 * SACKed is greater than or equal the DUPACK thoreshold
2273 * (reordering). This is implemented in tcp_mark_head_lost and
2274 * tcp_update_scoreboard.
2276 * RACK (draft-ietf-tcpm-rack-01): it is a newer algorithm
2277 * (2017-) that checks timing instead of counting DUPACKs.
2278 * Essentially a packet is considered lost if it's not S/ACKed
2279 * after RTT + reordering_window, where both metrics are
2280 * dynamically measured and adjusted. This is implemented in
2281 * tcp_rack_mark_lost.
2283 * If the receiver does not support SACK:
2285 * NewReno (RFC6582): in Recovery we assume that one segment
2286 * is lost (classic Reno). While we are in Recovery and
2287 * a partial ACK arrives, we assume that one more packet
2288 * is lost (NewReno). This heuristics are the same in NewReno
2291 * Really tricky (and requiring careful tuning) part of algorithm
2292 * is hidden in functions tcp_time_to_recover() and tcp_xmit_retransmit_queue().
2293 * The first determines the moment _when_ we should reduce CWND and,
2294 * hence, slow down forward transmission. In fact, it determines the moment
2295 * when we decide that hole is caused by loss, rather than by a reorder.
2297 * tcp_xmit_retransmit_queue() decides, _what_ we should retransmit to fill
2298 * holes, caused by lost packets.
2300 * And the most logically complicated part of algorithm is undo
2301 * heuristics. We detect false retransmits due to both too early
2302 * fast retransmit (reordering) and underestimated RTO, analyzing
2303 * timestamps and D-SACKs. When we detect that some segments were
2304 * retransmitted by mistake and CWND reduction was wrong, we undo
2305 * window reduction and abort recovery phase. This logic is hidden
2306 * inside several functions named tcp_try_undo_<something>.
2309 /* This function decides, when we should leave Disordered state
2310 * and enter Recovery phase, reducing congestion window.
2312 * Main question: may we further continue forward transmission
2313 * with the same cwnd?
2315 static bool tcp_time_to_recover(struct sock
*sk
, int flag
)
2317 struct tcp_sock
*tp
= tcp_sk(sk
);
2319 /* Trick#1: The loss is proven. */
2323 /* Not-A-Trick#2 : Classic rule... */
2324 if (!tcp_is_rack(sk
) && tcp_dupack_heuristics(tp
) > tp
->reordering
)
2330 /* Detect loss in event "A" above by marking head of queue up as lost.
2331 * For RFC3517 SACK, a segment is considered lost if it
2332 * has at least tp->reordering SACKed seqments above it; "packets" refers to
2333 * the maximum SACKed segments to pass before reaching this limit.
2335 static void tcp_mark_head_lost(struct sock
*sk
, int packets
, int mark_head
)
2337 struct tcp_sock
*tp
= tcp_sk(sk
);
2338 struct sk_buff
*skb
;
2340 /* Use SACK to deduce losses of new sequences sent during recovery */
2341 const u32 loss_high
= tp
->snd_nxt
;
2343 WARN_ON(packets
> tp
->packets_out
);
2344 skb
= tp
->lost_skb_hint
;
2346 /* Head already handled? */
2347 if (mark_head
&& after(TCP_SKB_CB(skb
)->seq
, tp
->snd_una
))
2349 cnt
= tp
->lost_cnt_hint
;
2351 skb
= tcp_rtx_queue_head(sk
);
2355 skb_rbtree_walk_from(skb
) {
2356 /* TODO: do this better */
2357 /* this is not the most efficient way to do this... */
2358 tp
->lost_skb_hint
= skb
;
2359 tp
->lost_cnt_hint
= cnt
;
2361 if (after(TCP_SKB_CB(skb
)->end_seq
, loss_high
))
2364 if (TCP_SKB_CB(skb
)->sacked
& TCPCB_SACKED_ACKED
)
2365 cnt
+= tcp_skb_pcount(skb
);
2370 if (!(TCP_SKB_CB(skb
)->sacked
& TCPCB_LOST
))
2371 tcp_mark_skb_lost(sk
, skb
);
2376 tcp_verify_left_out(tp
);
2379 /* Account newly detected lost packet(s) */
2381 static void tcp_update_scoreboard(struct sock
*sk
, int fast_rexmit
)
2383 struct tcp_sock
*tp
= tcp_sk(sk
);
2385 if (tcp_is_sack(tp
)) {
2386 int sacked_upto
= tp
->sacked_out
- tp
->reordering
;
2387 if (sacked_upto
>= 0)
2388 tcp_mark_head_lost(sk
, sacked_upto
, 0);
2389 else if (fast_rexmit
)
2390 tcp_mark_head_lost(sk
, 1, 1);
2394 static bool tcp_tsopt_ecr_before(const struct tcp_sock
*tp
, u32 when
)
2396 return tp
->rx_opt
.saw_tstamp
&& tp
->rx_opt
.rcv_tsecr
&&
2397 before(tp
->rx_opt
.rcv_tsecr
, when
);
2400 /* skb is spurious retransmitted if the returned timestamp echo
2401 * reply is prior to the skb transmission time
2403 static bool tcp_skb_spurious_retrans(const struct tcp_sock
*tp
,
2404 const struct sk_buff
*skb
)
2406 return (TCP_SKB_CB(skb
)->sacked
& TCPCB_RETRANS
) &&
2407 tcp_tsopt_ecr_before(tp
, tcp_skb_timestamp(skb
));
2410 /* Nothing was retransmitted or returned timestamp is less
2411 * than timestamp of the first retransmission.
2413 static inline bool tcp_packet_delayed(const struct tcp_sock
*tp
)
2415 return tp
->retrans_stamp
&&
2416 tcp_tsopt_ecr_before(tp
, tp
->retrans_stamp
);
2419 /* Undo procedures. */
2421 /* We can clear retrans_stamp when there are no retransmissions in the
2422 * window. It would seem that it is trivially available for us in
2423 * tp->retrans_out, however, that kind of assumptions doesn't consider
2424 * what will happen if errors occur when sending retransmission for the
2425 * second time. ...It could the that such segment has only
2426 * TCPCB_EVER_RETRANS set at the present time. It seems that checking
2427 * the head skb is enough except for some reneging corner cases that
2428 * are not worth the effort.
2430 * Main reason for all this complexity is the fact that connection dying
2431 * time now depends on the validity of the retrans_stamp, in particular,
2432 * that successive retransmissions of a segment must not advance
2433 * retrans_stamp under any conditions.
2435 static bool tcp_any_retrans_done(const struct sock
*sk
)
2437 const struct tcp_sock
*tp
= tcp_sk(sk
);
2438 struct sk_buff
*skb
;
2440 if (tp
->retrans_out
)
2443 skb
= tcp_rtx_queue_head(sk
);
2444 if (unlikely(skb
&& TCP_SKB_CB(skb
)->sacked
& TCPCB_EVER_RETRANS
))
2450 static void DBGUNDO(struct sock
*sk
, const char *msg
)
2452 #if FASTRETRANS_DEBUG > 1
2453 struct tcp_sock
*tp
= tcp_sk(sk
);
2454 struct inet_sock
*inet
= inet_sk(sk
);
2456 if (sk
->sk_family
== AF_INET
) {
2457 pr_debug("Undo %s %pI4/%u c%u l%u ss%u/%u p%u\n",
2459 &inet
->inet_daddr
, ntohs(inet
->inet_dport
),
2460 tcp_snd_cwnd(tp
), tcp_left_out(tp
),
2461 tp
->snd_ssthresh
, tp
->prior_ssthresh
,
2464 #if IS_ENABLED(CONFIG_IPV6)
2465 else if (sk
->sk_family
== AF_INET6
) {
2466 pr_debug("Undo %s %pI6/%u c%u l%u ss%u/%u p%u\n",
2468 &sk
->sk_v6_daddr
, ntohs(inet
->inet_dport
),
2469 tcp_snd_cwnd(tp
), tcp_left_out(tp
),
2470 tp
->snd_ssthresh
, tp
->prior_ssthresh
,
2477 static void tcp_undo_cwnd_reduction(struct sock
*sk
, bool unmark_loss
)
2479 struct tcp_sock
*tp
= tcp_sk(sk
);
2482 struct sk_buff
*skb
;
2484 skb_rbtree_walk(skb
, &sk
->tcp_rtx_queue
) {
2485 TCP_SKB_CB(skb
)->sacked
&= ~TCPCB_LOST
;
2488 tcp_clear_all_retrans_hints(tp
);
2491 if (tp
->prior_ssthresh
) {
2492 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
2494 tcp_snd_cwnd_set(tp
, icsk
->icsk_ca_ops
->undo_cwnd(sk
));
2496 if (tp
->prior_ssthresh
> tp
->snd_ssthresh
) {
2497 tp
->snd_ssthresh
= tp
->prior_ssthresh
;
2498 tcp_ecn_withdraw_cwr(tp
);
2501 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
2502 tp
->undo_marker
= 0;
2503 tp
->rack
.advanced
= 1; /* Force RACK to re-exam losses */
2506 static inline bool tcp_may_undo(const struct tcp_sock
*tp
)
2508 return tp
->undo_marker
&& (!tp
->undo_retrans
|| tcp_packet_delayed(tp
));
2511 /* People celebrate: "We love our President!" */
2512 static bool tcp_try_undo_recovery(struct sock
*sk
)
2514 struct tcp_sock
*tp
= tcp_sk(sk
);
2516 if (tcp_may_undo(tp
)) {
2519 /* Happy end! We did not retransmit anything
2520 * or our original transmission succeeded.
2522 DBGUNDO(sk
, inet_csk(sk
)->icsk_ca_state
== TCP_CA_Loss
? "loss" : "retrans");
2523 tcp_undo_cwnd_reduction(sk
, false);
2524 if (inet_csk(sk
)->icsk_ca_state
== TCP_CA_Loss
)
2525 mib_idx
= LINUX_MIB_TCPLOSSUNDO
;
2527 mib_idx
= LINUX_MIB_TCPFULLUNDO
;
2529 NET_INC_STATS(sock_net(sk
), mib_idx
);
2530 } else if (tp
->rack
.reo_wnd_persist
) {
2531 tp
->rack
.reo_wnd_persist
--;
2533 if (tp
->snd_una
== tp
->high_seq
&& tcp_is_reno(tp
)) {
2534 /* Hold old state until something *above* high_seq
2535 * is ACKed. For Reno it is MUST to prevent false
2536 * fast retransmits (RFC2582). SACK TCP is safe. */
2537 if (!tcp_any_retrans_done(sk
))
2538 tp
->retrans_stamp
= 0;
2541 tcp_set_ca_state(sk
, TCP_CA_Open
);
2542 tp
->is_sack_reneg
= 0;
2546 /* Try to undo cwnd reduction, because D-SACKs acked all retransmitted data */
2547 static bool tcp_try_undo_dsack(struct sock
*sk
)
2549 struct tcp_sock
*tp
= tcp_sk(sk
);
2551 if (tp
->undo_marker
&& !tp
->undo_retrans
) {
2552 tp
->rack
.reo_wnd_persist
= min(TCP_RACK_RECOVERY_THRESH
,
2553 tp
->rack
.reo_wnd_persist
+ 1);
2554 DBGUNDO(sk
, "D-SACK");
2555 tcp_undo_cwnd_reduction(sk
, false);
2556 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPDSACKUNDO
);
2562 /* Undo during loss recovery after partial ACK or using F-RTO. */
2563 static bool tcp_try_undo_loss(struct sock
*sk
, bool frto_undo
)
2565 struct tcp_sock
*tp
= tcp_sk(sk
);
2567 if (frto_undo
|| tcp_may_undo(tp
)) {
2568 tcp_undo_cwnd_reduction(sk
, true);
2570 DBGUNDO(sk
, "partial loss");
2571 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPLOSSUNDO
);
2573 NET_INC_STATS(sock_net(sk
),
2574 LINUX_MIB_TCPSPURIOUSRTOS
);
2575 inet_csk(sk
)->icsk_retransmits
= 0;
2576 if (frto_undo
|| tcp_is_sack(tp
)) {
2577 tcp_set_ca_state(sk
, TCP_CA_Open
);
2578 tp
->is_sack_reneg
= 0;
2585 /* The cwnd reduction in CWR and Recovery uses the PRR algorithm in RFC 6937.
2586 * It computes the number of packets to send (sndcnt) based on packets newly
2588 * 1) If the packets in flight is larger than ssthresh, PRR spreads the
2589 * cwnd reductions across a full RTT.
2590 * 2) Otherwise PRR uses packet conservation to send as much as delivered.
2591 * But when SND_UNA is acked without further losses,
2592 * slow starts cwnd up to ssthresh to speed up the recovery.
2594 static void tcp_init_cwnd_reduction(struct sock
*sk
)
2596 struct tcp_sock
*tp
= tcp_sk(sk
);
2598 tp
->high_seq
= tp
->snd_nxt
;
2599 tp
->tlp_high_seq
= 0;
2600 tp
->snd_cwnd_cnt
= 0;
2601 tp
->prior_cwnd
= tcp_snd_cwnd(tp
);
2602 tp
->prr_delivered
= 0;
2604 tp
->snd_ssthresh
= inet_csk(sk
)->icsk_ca_ops
->ssthresh(sk
);
2605 tcp_ecn_queue_cwr(tp
);
2608 void tcp_cwnd_reduction(struct sock
*sk
, int newly_acked_sacked
, int newly_lost
, int flag
)
2610 struct tcp_sock
*tp
= tcp_sk(sk
);
2612 int delta
= tp
->snd_ssthresh
- tcp_packets_in_flight(tp
);
2614 if (newly_acked_sacked
<= 0 || WARN_ON_ONCE(!tp
->prior_cwnd
))
2617 tp
->prr_delivered
+= newly_acked_sacked
;
2619 u64 dividend
= (u64
)tp
->snd_ssthresh
* tp
->prr_delivered
+
2621 sndcnt
= div_u64(dividend
, tp
->prior_cwnd
) - tp
->prr_out
;
2623 sndcnt
= max_t(int, tp
->prr_delivered
- tp
->prr_out
,
2624 newly_acked_sacked
);
2625 if (flag
& FLAG_SND_UNA_ADVANCED
&& !newly_lost
)
2627 sndcnt
= min(delta
, sndcnt
);
2629 /* Force a fast retransmit upon entering fast recovery */
2630 sndcnt
= max(sndcnt
, (tp
->prr_out
? 0 : 1));
2631 tcp_snd_cwnd_set(tp
, tcp_packets_in_flight(tp
) + sndcnt
);
2634 static inline void tcp_end_cwnd_reduction(struct sock
*sk
)
2636 struct tcp_sock
*tp
= tcp_sk(sk
);
2638 if (inet_csk(sk
)->icsk_ca_ops
->cong_control
)
2641 /* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
2642 if (tp
->snd_ssthresh
< TCP_INFINITE_SSTHRESH
&&
2643 (inet_csk(sk
)->icsk_ca_state
== TCP_CA_CWR
|| tp
->undo_marker
)) {
2644 tcp_snd_cwnd_set(tp
, tp
->snd_ssthresh
);
2645 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
2647 tcp_ca_event(sk
, CA_EVENT_COMPLETE_CWR
);
2650 /* Enter CWR state. Disable cwnd undo since congestion is proven with ECN */
2651 void tcp_enter_cwr(struct sock
*sk
)
2653 struct tcp_sock
*tp
= tcp_sk(sk
);
2655 tp
->prior_ssthresh
= 0;
2656 if (inet_csk(sk
)->icsk_ca_state
< TCP_CA_CWR
) {
2657 tp
->undo_marker
= 0;
2658 tcp_init_cwnd_reduction(sk
);
2659 tcp_set_ca_state(sk
, TCP_CA_CWR
);
2662 EXPORT_SYMBOL(tcp_enter_cwr
);
2664 static void tcp_try_keep_open(struct sock
*sk
)
2666 struct tcp_sock
*tp
= tcp_sk(sk
);
2667 int state
= TCP_CA_Open
;
2669 if (tcp_left_out(tp
) || tcp_any_retrans_done(sk
))
2670 state
= TCP_CA_Disorder
;
2672 if (inet_csk(sk
)->icsk_ca_state
!= state
) {
2673 tcp_set_ca_state(sk
, state
);
2674 tp
->high_seq
= tp
->snd_nxt
;
2678 static void tcp_try_to_open(struct sock
*sk
, int flag
)
2680 struct tcp_sock
*tp
= tcp_sk(sk
);
2682 tcp_verify_left_out(tp
);
2684 if (!tcp_any_retrans_done(sk
))
2685 tp
->retrans_stamp
= 0;
2687 if (flag
& FLAG_ECE
)
2690 if (inet_csk(sk
)->icsk_ca_state
!= TCP_CA_CWR
) {
2691 tcp_try_keep_open(sk
);
2695 static void tcp_mtup_probe_failed(struct sock
*sk
)
2697 struct inet_connection_sock
*icsk
= inet_csk(sk
);
2699 icsk
->icsk_mtup
.search_high
= icsk
->icsk_mtup
.probe_size
- 1;
2700 icsk
->icsk_mtup
.probe_size
= 0;
2701 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPMTUPFAIL
);
2704 static void tcp_mtup_probe_success(struct sock
*sk
)
2706 struct tcp_sock
*tp
= tcp_sk(sk
);
2707 struct inet_connection_sock
*icsk
= inet_csk(sk
);
2710 tp
->prior_ssthresh
= tcp_current_ssthresh(sk
);
2712 val
= (u64
)tcp_snd_cwnd(tp
) * tcp_mss_to_mtu(sk
, tp
->mss_cache
);
2713 do_div(val
, icsk
->icsk_mtup
.probe_size
);
2714 DEBUG_NET_WARN_ON_ONCE((u32
)val
!= val
);
2715 tcp_snd_cwnd_set(tp
, max_t(u32
, 1U, val
));
2717 tp
->snd_cwnd_cnt
= 0;
2718 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
2719 tp
->snd_ssthresh
= tcp_current_ssthresh(sk
);
2721 icsk
->icsk_mtup
.search_low
= icsk
->icsk_mtup
.probe_size
;
2722 icsk
->icsk_mtup
.probe_size
= 0;
2723 tcp_sync_mss(sk
, icsk
->icsk_pmtu_cookie
);
2724 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPMTUPSUCCESS
);
2727 /* Do a simple retransmit without using the backoff mechanisms in
2728 * tcp_timer. This is used for path mtu discovery.
2729 * The socket is already locked here.
2731 void tcp_simple_retransmit(struct sock
*sk
)
2733 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
2734 struct tcp_sock
*tp
= tcp_sk(sk
);
2735 struct sk_buff
*skb
;
2738 /* A fastopen SYN request is stored as two separate packets within
2739 * the retransmit queue, this is done by tcp_send_syn_data().
2740 * As a result simply checking the MSS of the frames in the queue
2741 * will not work for the SYN packet.
2743 * Us being here is an indication of a path MTU issue so we can
2744 * assume that the fastopen SYN was lost and just mark all the
2745 * frames in the retransmit queue as lost. We will use an MSS of
2746 * -1 to mark all frames as lost, otherwise compute the current MSS.
2748 if (tp
->syn_data
&& sk
->sk_state
== TCP_SYN_SENT
)
2751 mss
= tcp_current_mss(sk
);
2753 skb_rbtree_walk(skb
, &sk
->tcp_rtx_queue
) {
2754 if (tcp_skb_seglen(skb
) > mss
)
2755 tcp_mark_skb_lost(sk
, skb
);
2758 tcp_clear_retrans_hints_partial(tp
);
2763 if (tcp_is_reno(tp
))
2764 tcp_limit_reno_sacked(tp
);
2766 tcp_verify_left_out(tp
);
2768 /* Don't muck with the congestion window here.
2769 * Reason is that we do not increase amount of _data_
2770 * in network, but units changed and effective
2771 * cwnd/ssthresh really reduced now.
2773 if (icsk
->icsk_ca_state
!= TCP_CA_Loss
) {
2774 tp
->high_seq
= tp
->snd_nxt
;
2775 tp
->snd_ssthresh
= tcp_current_ssthresh(sk
);
2776 tp
->prior_ssthresh
= 0;
2777 tp
->undo_marker
= 0;
2778 tcp_set_ca_state(sk
, TCP_CA_Loss
);
2780 tcp_xmit_retransmit_queue(sk
);
2782 EXPORT_SYMBOL(tcp_simple_retransmit
);
2784 void tcp_enter_recovery(struct sock
*sk
, bool ece_ack
)
2786 struct tcp_sock
*tp
= tcp_sk(sk
);
2789 if (tcp_is_reno(tp
))
2790 mib_idx
= LINUX_MIB_TCPRENORECOVERY
;
2792 mib_idx
= LINUX_MIB_TCPSACKRECOVERY
;
2794 NET_INC_STATS(sock_net(sk
), mib_idx
);
2796 tp
->prior_ssthresh
= 0;
2799 if (!tcp_in_cwnd_reduction(sk
)) {
2801 tp
->prior_ssthresh
= tcp_current_ssthresh(sk
);
2802 tcp_init_cwnd_reduction(sk
);
2804 tcp_set_ca_state(sk
, TCP_CA_Recovery
);
2807 /* Process an ACK in CA_Loss state. Move to CA_Open if lost data are
2808 * recovered or spurious. Otherwise retransmits more on partial ACKs.
2810 static void tcp_process_loss(struct sock
*sk
, int flag
, int num_dupack
,
2813 struct tcp_sock
*tp
= tcp_sk(sk
);
2814 bool recovered
= !before(tp
->snd_una
, tp
->high_seq
);
2816 if ((flag
& FLAG_SND_UNA_ADVANCED
|| rcu_access_pointer(tp
->fastopen_rsk
)) &&
2817 tcp_try_undo_loss(sk
, false))
2820 if (tp
->frto
) { /* F-RTO RFC5682 sec 3.1 (sack enhanced version). */
2821 /* Step 3.b. A timeout is spurious if not all data are
2822 * lost, i.e., never-retransmitted data are (s)acked.
2824 if ((flag
& FLAG_ORIG_SACK_ACKED
) &&
2825 tcp_try_undo_loss(sk
, true))
2828 if (after(tp
->snd_nxt
, tp
->high_seq
)) {
2829 if (flag
& FLAG_DATA_SACKED
|| num_dupack
)
2830 tp
->frto
= 0; /* Step 3.a. loss was real */
2831 } else if (flag
& FLAG_SND_UNA_ADVANCED
&& !recovered
) {
2832 tp
->high_seq
= tp
->snd_nxt
;
2833 /* Step 2.b. Try send new data (but deferred until cwnd
2834 * is updated in tcp_ack()). Otherwise fall back to
2835 * the conventional recovery.
2837 if (!tcp_write_queue_empty(sk
) &&
2838 after(tcp_wnd_end(tp
), tp
->snd_nxt
)) {
2839 *rexmit
= REXMIT_NEW
;
2847 /* F-RTO RFC5682 sec 3.1 step 2.a and 1st part of step 3.a */
2848 tcp_try_undo_recovery(sk
);
2851 if (tcp_is_reno(tp
)) {
2852 /* A Reno DUPACK means new data in F-RTO step 2.b above are
2853 * delivered. Lower inflight to clock out (re)tranmissions.
2855 if (after(tp
->snd_nxt
, tp
->high_seq
) && num_dupack
)
2856 tcp_add_reno_sack(sk
, num_dupack
, flag
& FLAG_ECE
);
2857 else if (flag
& FLAG_SND_UNA_ADVANCED
)
2858 tcp_reset_reno_sack(tp
);
2860 *rexmit
= REXMIT_LOST
;
2863 static bool tcp_force_fast_retransmit(struct sock
*sk
)
2865 struct tcp_sock
*tp
= tcp_sk(sk
);
2867 return after(tcp_highest_sack_seq(tp
),
2868 tp
->snd_una
+ tp
->reordering
* tp
->mss_cache
);
2871 /* Undo during fast recovery after partial ACK. */
2872 static bool tcp_try_undo_partial(struct sock
*sk
, u32 prior_snd_una
,
2875 struct tcp_sock
*tp
= tcp_sk(sk
);
2877 if (tp
->undo_marker
&& tcp_packet_delayed(tp
)) {
2878 /* Plain luck! Hole if filled with delayed
2879 * packet, rather than with a retransmit. Check reordering.
2881 tcp_check_sack_reordering(sk
, prior_snd_una
, 1);
2883 /* We are getting evidence that the reordering degree is higher
2884 * than we realized. If there are no retransmits out then we
2885 * can undo. Otherwise we clock out new packets but do not
2886 * mark more packets lost or retransmit more.
2888 if (tp
->retrans_out
)
2891 if (!tcp_any_retrans_done(sk
))
2892 tp
->retrans_stamp
= 0;
2894 DBGUNDO(sk
, "partial recovery");
2895 tcp_undo_cwnd_reduction(sk
, true);
2896 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPPARTIALUNDO
);
2897 tcp_try_keep_open(sk
);
2899 /* Partial ACK arrived. Force fast retransmit. */
2900 *do_lost
= tcp_force_fast_retransmit(sk
);
2905 static void tcp_identify_packet_loss(struct sock
*sk
, int *ack_flag
)
2907 struct tcp_sock
*tp
= tcp_sk(sk
);
2909 if (tcp_rtx_queue_empty(sk
))
2912 if (unlikely(tcp_is_reno(tp
))) {
2913 tcp_newreno_mark_lost(sk
, *ack_flag
& FLAG_SND_UNA_ADVANCED
);
2914 } else if (tcp_is_rack(sk
)) {
2915 u32 prior_retrans
= tp
->retrans_out
;
2917 if (tcp_rack_mark_lost(sk
))
2918 *ack_flag
&= ~FLAG_SET_XMIT_TIMER
;
2919 if (prior_retrans
> tp
->retrans_out
)
2920 *ack_flag
|= FLAG_LOST_RETRANS
;
2924 /* Process an event, which can update packets-in-flight not trivially.
2925 * Main goal of this function is to calculate new estimate for left_out,
2926 * taking into account both packets sitting in receiver's buffer and
2927 * packets lost by network.
2929 * Besides that it updates the congestion state when packet loss or ECN
2930 * is detected. But it does not reduce the cwnd, it is done by the
2931 * congestion control later.
2933 * It does _not_ decide what to send, it is made in function
2934 * tcp_xmit_retransmit_queue().
2936 static void tcp_fastretrans_alert(struct sock
*sk
, const u32 prior_snd_una
,
2937 int num_dupack
, int *ack_flag
, int *rexmit
)
2939 struct inet_connection_sock
*icsk
= inet_csk(sk
);
2940 struct tcp_sock
*tp
= tcp_sk(sk
);
2941 int fast_rexmit
= 0, flag
= *ack_flag
;
2942 bool ece_ack
= flag
& FLAG_ECE
;
2943 bool do_lost
= num_dupack
|| ((flag
& FLAG_DATA_SACKED
) &&
2944 tcp_force_fast_retransmit(sk
));
2946 if (!tp
->packets_out
&& tp
->sacked_out
)
2949 /* Now state machine starts.
2950 * A. ECE, hence prohibit cwnd undoing, the reduction is required. */
2952 tp
->prior_ssthresh
= 0;
2954 /* B. In all the states check for reneging SACKs. */
2955 if (tcp_check_sack_reneging(sk
, flag
))
2958 /* C. Check consistency of the current state. */
2959 tcp_verify_left_out(tp
);
2961 /* D. Check state exit conditions. State can be terminated
2962 * when high_seq is ACKed. */
2963 if (icsk
->icsk_ca_state
== TCP_CA_Open
) {
2964 WARN_ON(tp
->retrans_out
!= 0 && !tp
->syn_data
);
2965 tp
->retrans_stamp
= 0;
2966 } else if (!before(tp
->snd_una
, tp
->high_seq
)) {
2967 switch (icsk
->icsk_ca_state
) {
2969 /* CWR is to be held something *above* high_seq
2970 * is ACKed for CWR bit to reach receiver. */
2971 if (tp
->snd_una
!= tp
->high_seq
) {
2972 tcp_end_cwnd_reduction(sk
);
2973 tcp_set_ca_state(sk
, TCP_CA_Open
);
2977 case TCP_CA_Recovery
:
2978 if (tcp_is_reno(tp
))
2979 tcp_reset_reno_sack(tp
);
2980 if (tcp_try_undo_recovery(sk
))
2982 tcp_end_cwnd_reduction(sk
);
2987 /* E. Process state. */
2988 switch (icsk
->icsk_ca_state
) {
2989 case TCP_CA_Recovery
:
2990 if (!(flag
& FLAG_SND_UNA_ADVANCED
)) {
2991 if (tcp_is_reno(tp
))
2992 tcp_add_reno_sack(sk
, num_dupack
, ece_ack
);
2993 } else if (tcp_try_undo_partial(sk
, prior_snd_una
, &do_lost
))
2996 if (tcp_try_undo_dsack(sk
))
2997 tcp_try_keep_open(sk
);
2999 tcp_identify_packet_loss(sk
, ack_flag
);
3000 if (icsk
->icsk_ca_state
!= TCP_CA_Recovery
) {
3001 if (!tcp_time_to_recover(sk
, flag
))
3003 /* Undo reverts the recovery state. If loss is evident,
3004 * starts a new recovery (e.g. reordering then loss);
3006 tcp_enter_recovery(sk
, ece_ack
);
3010 tcp_process_loss(sk
, flag
, num_dupack
, rexmit
);
3011 tcp_identify_packet_loss(sk
, ack_flag
);
3012 if (!(icsk
->icsk_ca_state
== TCP_CA_Open
||
3013 (*ack_flag
& FLAG_LOST_RETRANS
)))
3015 /* Change state if cwnd is undone or retransmits are lost */
3018 if (tcp_is_reno(tp
)) {
3019 if (flag
& FLAG_SND_UNA_ADVANCED
)
3020 tcp_reset_reno_sack(tp
);
3021 tcp_add_reno_sack(sk
, num_dupack
, ece_ack
);
3024 if (icsk
->icsk_ca_state
<= TCP_CA_Disorder
)
3025 tcp_try_undo_dsack(sk
);
3027 tcp_identify_packet_loss(sk
, ack_flag
);
3028 if (!tcp_time_to_recover(sk
, flag
)) {
3029 tcp_try_to_open(sk
, flag
);
3033 /* MTU probe failure: don't reduce cwnd */
3034 if (icsk
->icsk_ca_state
< TCP_CA_CWR
&&
3035 icsk
->icsk_mtup
.probe_size
&&
3036 tp
->snd_una
== tp
->mtu_probe
.probe_seq_start
) {
3037 tcp_mtup_probe_failed(sk
);
3038 /* Restores the reduction we did in tcp_mtup_probe() */
3039 tcp_snd_cwnd_set(tp
, tcp_snd_cwnd(tp
) + 1);
3040 tcp_simple_retransmit(sk
);
3044 /* Otherwise enter Recovery state */
3045 tcp_enter_recovery(sk
, ece_ack
);
3049 if (!tcp_is_rack(sk
) && do_lost
)
3050 tcp_update_scoreboard(sk
, fast_rexmit
);
3051 *rexmit
= REXMIT_LOST
;
3054 static void tcp_update_rtt_min(struct sock
*sk
, u32 rtt_us
, const int flag
)
3056 u32 wlen
= sock_net(sk
)->ipv4
.sysctl_tcp_min_rtt_wlen
* HZ
;
3057 struct tcp_sock
*tp
= tcp_sk(sk
);
3059 if ((flag
& FLAG_ACK_MAYBE_DELAYED
) && rtt_us
> tcp_min_rtt(tp
)) {
3060 /* If the remote keeps returning delayed ACKs, eventually
3061 * the min filter would pick it up and overestimate the
3062 * prop. delay when it expires. Skip suspected delayed ACKs.
3066 minmax_running_min(&tp
->rtt_min
, wlen
, tcp_jiffies32
,
3067 rtt_us
? : jiffies_to_usecs(1));
3070 static bool tcp_ack_update_rtt(struct sock
*sk
, const int flag
,
3071 long seq_rtt_us
, long sack_rtt_us
,
3072 long ca_rtt_us
, struct rate_sample
*rs
)
3074 const struct tcp_sock
*tp
= tcp_sk(sk
);
3076 /* Prefer RTT measured from ACK's timing to TS-ECR. This is because
3077 * broken middle-boxes or peers may corrupt TS-ECR fields. But
3078 * Karn's algorithm forbids taking RTT if some retransmitted data
3079 * is acked (RFC6298).
3082 seq_rtt_us
= sack_rtt_us
;
3084 /* RTTM Rule: A TSecr value received in a segment is used to
3085 * update the averaged RTT measurement only if the segment
3086 * acknowledges some new data, i.e., only if it advances the
3087 * left edge of the send window.
3088 * See draft-ietf-tcplw-high-performance-00, section 3.3.
3090 if (seq_rtt_us
< 0 && tp
->rx_opt
.saw_tstamp
&& tp
->rx_opt
.rcv_tsecr
&&
3091 flag
& FLAG_ACKED
) {
3092 u32 delta
= tcp_time_stamp(tp
) - tp
->rx_opt
.rcv_tsecr
;
3094 if (likely(delta
< INT_MAX
/ (USEC_PER_SEC
/ TCP_TS_HZ
))) {
3097 seq_rtt_us
= delta
* (USEC_PER_SEC
/ TCP_TS_HZ
);
3098 ca_rtt_us
= seq_rtt_us
;
3101 rs
->rtt_us
= ca_rtt_us
; /* RTT of last (S)ACKed packet (or -1) */
3105 /* ca_rtt_us >= 0 is counting on the invariant that ca_rtt_us is
3106 * always taken together with ACK, SACK, or TS-opts. Any negative
3107 * values will be skipped with the seq_rtt_us < 0 check above.
3109 tcp_update_rtt_min(sk
, ca_rtt_us
, flag
);
3110 tcp_rtt_estimator(sk
, seq_rtt_us
);
3113 /* RFC6298: only reset backoff on valid RTT measurement. */
3114 inet_csk(sk
)->icsk_backoff
= 0;
3118 /* Compute time elapsed between (last) SYNACK and the ACK completing 3WHS. */
3119 void tcp_synack_rtt_meas(struct sock
*sk
, struct request_sock
*req
)
3121 struct rate_sample rs
;
3124 if (req
&& !req
->num_retrans
&& tcp_rsk(req
)->snt_synack
)
3125 rtt_us
= tcp_stamp_us_delta(tcp_clock_us(), tcp_rsk(req
)->snt_synack
);
3127 tcp_ack_update_rtt(sk
, FLAG_SYN_ACKED
, rtt_us
, -1L, rtt_us
, &rs
);
3131 static void tcp_cong_avoid(struct sock
*sk
, u32 ack
, u32 acked
)
3133 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
3135 icsk
->icsk_ca_ops
->cong_avoid(sk
, ack
, acked
);
3136 tcp_sk(sk
)->snd_cwnd_stamp
= tcp_jiffies32
;
3139 /* Restart timer after forward progress on connection.
3140 * RFC2988 recommends to restart timer to now+rto.
3142 void tcp_rearm_rto(struct sock
*sk
)
3144 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
3145 struct tcp_sock
*tp
= tcp_sk(sk
);
3147 /* If the retrans timer is currently being used by Fast Open
3148 * for SYN-ACK retrans purpose, stay put.
3150 if (rcu_access_pointer(tp
->fastopen_rsk
))
3153 if (!tp
->packets_out
) {
3154 inet_csk_clear_xmit_timer(sk
, ICSK_TIME_RETRANS
);
3156 u32 rto
= inet_csk(sk
)->icsk_rto
;
3157 /* Offset the time elapsed after installing regular RTO */
3158 if (icsk
->icsk_pending
== ICSK_TIME_REO_TIMEOUT
||
3159 icsk
->icsk_pending
== ICSK_TIME_LOSS_PROBE
) {
3160 s64 delta_us
= tcp_rto_delta_us(sk
);
3161 /* delta_us may not be positive if the socket is locked
3162 * when the retrans timer fires and is rescheduled.
3164 rto
= usecs_to_jiffies(max_t(int, delta_us
, 1));
3166 tcp_reset_xmit_timer(sk
, ICSK_TIME_RETRANS
, rto
,
3171 /* Try to schedule a loss probe; if that doesn't work, then schedule an RTO. */
3172 static void tcp_set_xmit_timer(struct sock
*sk
)
3174 if (!tcp_schedule_loss_probe(sk
, true))
3178 /* If we get here, the whole TSO packet has not been acked. */
3179 static u32
tcp_tso_acked(struct sock
*sk
, struct sk_buff
*skb
)
3181 struct tcp_sock
*tp
= tcp_sk(sk
);
3184 BUG_ON(!after(TCP_SKB_CB(skb
)->end_seq
, tp
->snd_una
));
3186 packets_acked
= tcp_skb_pcount(skb
);
3187 if (tcp_trim_head(sk
, skb
, tp
->snd_una
- TCP_SKB_CB(skb
)->seq
))
3189 packets_acked
-= tcp_skb_pcount(skb
);
3191 if (packets_acked
) {
3192 BUG_ON(tcp_skb_pcount(skb
) == 0);
3193 BUG_ON(!before(TCP_SKB_CB(skb
)->seq
, TCP_SKB_CB(skb
)->end_seq
));
3196 return packets_acked
;
3199 static void tcp_ack_tstamp(struct sock
*sk
, struct sk_buff
*skb
,
3200 const struct sk_buff
*ack_skb
, u32 prior_snd_una
)
3202 const struct skb_shared_info
*shinfo
;
3204 /* Avoid cache line misses to get skb_shinfo() and shinfo->tx_flags */
3205 if (likely(!TCP_SKB_CB(skb
)->txstamp_ack
))
3208 shinfo
= skb_shinfo(skb
);
3209 if (!before(shinfo
->tskey
, prior_snd_una
) &&
3210 before(shinfo
->tskey
, tcp_sk(sk
)->snd_una
)) {
3211 tcp_skb_tsorted_save(skb
) {
3212 __skb_tstamp_tx(skb
, ack_skb
, NULL
, sk
, SCM_TSTAMP_ACK
);
3213 } tcp_skb_tsorted_restore(skb
);
3217 /* Remove acknowledged frames from the retransmission queue. If our packet
3218 * is before the ack sequence we can discard it as it's confirmed to have
3219 * arrived at the other end.
3221 static int tcp_clean_rtx_queue(struct sock
*sk
, const struct sk_buff
*ack_skb
,
3222 u32 prior_fack
, u32 prior_snd_una
,
3223 struct tcp_sacktag_state
*sack
, bool ece_ack
)
3225 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
3226 u64 first_ackt
, last_ackt
;
3227 struct tcp_sock
*tp
= tcp_sk(sk
);
3228 u32 prior_sacked
= tp
->sacked_out
;
3229 u32 reord
= tp
->snd_nxt
; /* lowest acked un-retx un-sacked seq */
3230 struct sk_buff
*skb
, *next
;
3231 bool fully_acked
= true;
3232 long sack_rtt_us
= -1L;
3233 long seq_rtt_us
= -1L;
3234 long ca_rtt_us
= -1L;
3241 for (skb
= skb_rb_first(&sk
->tcp_rtx_queue
); skb
; skb
= next
) {
3242 struct tcp_skb_cb
*scb
= TCP_SKB_CB(skb
);
3243 const u32 start_seq
= scb
->seq
;
3244 u8 sacked
= scb
->sacked
;
3247 /* Determine how many packets and what bytes were acked, tso and else */
3248 if (after(scb
->end_seq
, tp
->snd_una
)) {
3249 if (tcp_skb_pcount(skb
) == 1 ||
3250 !after(tp
->snd_una
, scb
->seq
))
3253 acked_pcount
= tcp_tso_acked(sk
, skb
);
3256 fully_acked
= false;
3258 acked_pcount
= tcp_skb_pcount(skb
);
3261 if (unlikely(sacked
& TCPCB_RETRANS
)) {
3262 if (sacked
& TCPCB_SACKED_RETRANS
)
3263 tp
->retrans_out
-= acked_pcount
;
3264 flag
|= FLAG_RETRANS_DATA_ACKED
;
3265 } else if (!(sacked
& TCPCB_SACKED_ACKED
)) {
3266 last_ackt
= tcp_skb_timestamp_us(skb
);
3267 WARN_ON_ONCE(last_ackt
== 0);
3269 first_ackt
= last_ackt
;
3271 if (before(start_seq
, reord
))
3273 if (!after(scb
->end_seq
, tp
->high_seq
))
3274 flag
|= FLAG_ORIG_SACK_ACKED
;
3277 if (sacked
& TCPCB_SACKED_ACKED
) {
3278 tp
->sacked_out
-= acked_pcount
;
3279 } else if (tcp_is_sack(tp
)) {
3280 tcp_count_delivered(tp
, acked_pcount
, ece_ack
);
3281 if (!tcp_skb_spurious_retrans(tp
, skb
))
3282 tcp_rack_advance(tp
, sacked
, scb
->end_seq
,
3283 tcp_skb_timestamp_us(skb
));
3285 if (sacked
& TCPCB_LOST
)
3286 tp
->lost_out
-= acked_pcount
;
3288 tp
->packets_out
-= acked_pcount
;
3289 pkts_acked
+= acked_pcount
;
3290 tcp_rate_skb_delivered(sk
, skb
, sack
->rate
);
3292 /* Initial outgoing SYN's get put onto the write_queue
3293 * just like anything else we transmit. It is not
3294 * true data, and if we misinform our callers that
3295 * this ACK acks real data, we will erroneously exit
3296 * connection startup slow start one packet too
3297 * quickly. This is severely frowned upon behavior.
3299 if (likely(!(scb
->tcp_flags
& TCPHDR_SYN
))) {
3300 flag
|= FLAG_DATA_ACKED
;
3302 flag
|= FLAG_SYN_ACKED
;
3303 tp
->retrans_stamp
= 0;
3309 tcp_ack_tstamp(sk
, skb
, ack_skb
, prior_snd_una
);
3311 next
= skb_rb_next(skb
);
3312 if (unlikely(skb
== tp
->retransmit_skb_hint
))
3313 tp
->retransmit_skb_hint
= NULL
;
3314 if (unlikely(skb
== tp
->lost_skb_hint
))
3315 tp
->lost_skb_hint
= NULL
;
3316 tcp_highest_sack_replace(sk
, skb
, next
);
3317 tcp_rtx_queue_unlink_and_free(skb
, sk
);
3321 tcp_chrono_stop(sk
, TCP_CHRONO_BUSY
);
3323 if (likely(between(tp
->snd_up
, prior_snd_una
, tp
->snd_una
)))
3324 tp
->snd_up
= tp
->snd_una
;
3327 tcp_ack_tstamp(sk
, skb
, ack_skb
, prior_snd_una
);
3328 if (TCP_SKB_CB(skb
)->sacked
& TCPCB_SACKED_ACKED
)
3329 flag
|= FLAG_SACK_RENEGING
;
3332 if (likely(first_ackt
) && !(flag
& FLAG_RETRANS_DATA_ACKED
)) {
3333 seq_rtt_us
= tcp_stamp_us_delta(tp
->tcp_mstamp
, first_ackt
);
3334 ca_rtt_us
= tcp_stamp_us_delta(tp
->tcp_mstamp
, last_ackt
);
3336 if (pkts_acked
== 1 && fully_acked
&& !prior_sacked
&&
3337 (tp
->snd_una
- prior_snd_una
) < tp
->mss_cache
&&
3338 sack
->rate
->prior_delivered
+ 1 == tp
->delivered
&&
3339 !(flag
& (FLAG_CA_ALERT
| FLAG_SYN_ACKED
))) {
3340 /* Conservatively mark a delayed ACK. It's typically
3341 * from a lone runt packet over the round trip to
3342 * a receiver w/o out-of-order or CE events.
3344 flag
|= FLAG_ACK_MAYBE_DELAYED
;
3347 if (sack
->first_sackt
) {
3348 sack_rtt_us
= tcp_stamp_us_delta(tp
->tcp_mstamp
, sack
->first_sackt
);
3349 ca_rtt_us
= tcp_stamp_us_delta(tp
->tcp_mstamp
, sack
->last_sackt
);
3351 rtt_update
= tcp_ack_update_rtt(sk
, flag
, seq_rtt_us
, sack_rtt_us
,
3352 ca_rtt_us
, sack
->rate
);
3354 if (flag
& FLAG_ACKED
) {
3355 flag
|= FLAG_SET_XMIT_TIMER
; /* set TLP or RTO timer */
3356 if (unlikely(icsk
->icsk_mtup
.probe_size
&&
3357 !after(tp
->mtu_probe
.probe_seq_end
, tp
->snd_una
))) {
3358 tcp_mtup_probe_success(sk
);
3361 if (tcp_is_reno(tp
)) {
3362 tcp_remove_reno_sacks(sk
, pkts_acked
, ece_ack
);
3364 /* If any of the cumulatively ACKed segments was
3365 * retransmitted, non-SACK case cannot confirm that
3366 * progress was due to original transmission due to
3367 * lack of TCPCB_SACKED_ACKED bits even if some of
3368 * the packets may have been never retransmitted.
3370 if (flag
& FLAG_RETRANS_DATA_ACKED
)
3371 flag
&= ~FLAG_ORIG_SACK_ACKED
;
3375 /* Non-retransmitted hole got filled? That's reordering */
3376 if (before(reord
, prior_fack
))
3377 tcp_check_sack_reordering(sk
, reord
, 0);
3379 delta
= prior_sacked
- tp
->sacked_out
;
3380 tp
->lost_cnt_hint
-= min(tp
->lost_cnt_hint
, delta
);
3382 } else if (skb
&& rtt_update
&& sack_rtt_us
>= 0 &&
3383 sack_rtt_us
> tcp_stamp_us_delta(tp
->tcp_mstamp
,
3384 tcp_skb_timestamp_us(skb
))) {
3385 /* Do not re-arm RTO if the sack RTT is measured from data sent
3386 * after when the head was last (re)transmitted. Otherwise the
3387 * timeout may continue to extend in loss recovery.
3389 flag
|= FLAG_SET_XMIT_TIMER
; /* set TLP or RTO timer */
3392 if (icsk
->icsk_ca_ops
->pkts_acked
) {
3393 struct ack_sample sample
= { .pkts_acked
= pkts_acked
,
3394 .rtt_us
= sack
->rate
->rtt_us
};
3396 sample
.in_flight
= tp
->mss_cache
*
3397 (tp
->delivered
- sack
->rate
->prior_delivered
);
3398 icsk
->icsk_ca_ops
->pkts_acked(sk
, &sample
);
3401 #if FASTRETRANS_DEBUG > 0
3402 WARN_ON((int)tp
->sacked_out
< 0);
3403 WARN_ON((int)tp
->lost_out
< 0);
3404 WARN_ON((int)tp
->retrans_out
< 0);
3405 if (!tp
->packets_out
&& tcp_is_sack(tp
)) {
3406 icsk
= inet_csk(sk
);
3408 pr_debug("Leak l=%u %d\n",
3409 tp
->lost_out
, icsk
->icsk_ca_state
);
3412 if (tp
->sacked_out
) {
3413 pr_debug("Leak s=%u %d\n",
3414 tp
->sacked_out
, icsk
->icsk_ca_state
);
3417 if (tp
->retrans_out
) {
3418 pr_debug("Leak r=%u %d\n",
3419 tp
->retrans_out
, icsk
->icsk_ca_state
);
3420 tp
->retrans_out
= 0;
3427 static void tcp_ack_probe(struct sock
*sk
)
3429 struct inet_connection_sock
*icsk
= inet_csk(sk
);
3430 struct sk_buff
*head
= tcp_send_head(sk
);
3431 const struct tcp_sock
*tp
= tcp_sk(sk
);
3433 /* Was it a usable window open? */
3436 if (!after(TCP_SKB_CB(head
)->end_seq
, tcp_wnd_end(tp
))) {
3437 icsk
->icsk_backoff
= 0;
3438 icsk
->icsk_probes_tstamp
= 0;
3439 inet_csk_clear_xmit_timer(sk
, ICSK_TIME_PROBE0
);
3440 /* Socket must be waked up by subsequent tcp_data_snd_check().
3441 * This function is not for random using!
3444 unsigned long when
= tcp_probe0_when(sk
, TCP_RTO_MAX
);
3446 when
= tcp_clamp_probe0_to_user_timeout(sk
, when
);
3447 tcp_reset_xmit_timer(sk
, ICSK_TIME_PROBE0
, when
, TCP_RTO_MAX
);
3451 static inline bool tcp_ack_is_dubious(const struct sock
*sk
, const int flag
)
3453 return !(flag
& FLAG_NOT_DUP
) || (flag
& FLAG_CA_ALERT
) ||
3454 inet_csk(sk
)->icsk_ca_state
!= TCP_CA_Open
;
3457 /* Decide wheather to run the increase function of congestion control. */
3458 static inline bool tcp_may_raise_cwnd(const struct sock
*sk
, const int flag
)
3460 /* If reordering is high then always grow cwnd whenever data is
3461 * delivered regardless of its ordering. Otherwise stay conservative
3462 * and only grow cwnd on in-order delivery (RFC5681). A stretched ACK w/
3463 * new SACK or ECE mark may first advance cwnd here and later reduce
3464 * cwnd in tcp_fastretrans_alert() based on more states.
3466 if (tcp_sk(sk
)->reordering
> sock_net(sk
)->ipv4
.sysctl_tcp_reordering
)
3467 return flag
& FLAG_FORWARD_PROGRESS
;
3469 return flag
& FLAG_DATA_ACKED
;
3472 /* The "ultimate" congestion control function that aims to replace the rigid
3473 * cwnd increase and decrease control (tcp_cong_avoid,tcp_*cwnd_reduction).
3474 * It's called toward the end of processing an ACK with precise rate
3475 * information. All transmission or retransmission are delayed afterwards.
3477 static void tcp_cong_control(struct sock
*sk
, u32 ack
, u32 acked_sacked
,
3478 int flag
, const struct rate_sample
*rs
)
3480 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
3482 if (icsk
->icsk_ca_ops
->cong_control
) {
3483 icsk
->icsk_ca_ops
->cong_control(sk
, rs
);
3487 if (tcp_in_cwnd_reduction(sk
)) {
3488 /* Reduce cwnd if state mandates */
3489 tcp_cwnd_reduction(sk
, acked_sacked
, rs
->losses
, flag
);
3490 } else if (tcp_may_raise_cwnd(sk
, flag
)) {
3491 /* Advance cwnd if state allows */
3492 tcp_cong_avoid(sk
, ack
, acked_sacked
);
3494 tcp_update_pacing_rate(sk
);
3497 /* Check that window update is acceptable.
3498 * The function assumes that snd_una<=ack<=snd_next.
3500 static inline bool tcp_may_update_window(const struct tcp_sock
*tp
,
3501 const u32 ack
, const u32 ack_seq
,
3504 return after(ack
, tp
->snd_una
) ||
3505 after(ack_seq
, tp
->snd_wl1
) ||
3506 (ack_seq
== tp
->snd_wl1
&& nwin
> tp
->snd_wnd
);
3509 /* If we update tp->snd_una, also update tp->bytes_acked */
3510 static void tcp_snd_una_update(struct tcp_sock
*tp
, u32 ack
)
3512 u32 delta
= ack
- tp
->snd_una
;
3514 sock_owned_by_me((struct sock
*)tp
);
3515 tp
->bytes_acked
+= delta
;
3519 /* If we update tp->rcv_nxt, also update tp->bytes_received */
3520 static void tcp_rcv_nxt_update(struct tcp_sock
*tp
, u32 seq
)
3522 u32 delta
= seq
- tp
->rcv_nxt
;
3524 sock_owned_by_me((struct sock
*)tp
);
3525 tp
->bytes_received
+= delta
;
3526 WRITE_ONCE(tp
->rcv_nxt
, seq
);
3529 /* Update our send window.
3531 * Window update algorithm, described in RFC793/RFC1122 (used in linux-2.2
3532 * and in FreeBSD. NetBSD's one is even worse.) is wrong.
3534 static int tcp_ack_update_window(struct sock
*sk
, const struct sk_buff
*skb
, u32 ack
,
3537 struct tcp_sock
*tp
= tcp_sk(sk
);
3539 u32 nwin
= ntohs(tcp_hdr(skb
)->window
);
3541 if (likely(!tcp_hdr(skb
)->syn
))
3542 nwin
<<= tp
->rx_opt
.snd_wscale
;
3544 if (tcp_may_update_window(tp
, ack
, ack_seq
, nwin
)) {
3545 flag
|= FLAG_WIN_UPDATE
;
3546 tcp_update_wl(tp
, ack_seq
);
3548 if (tp
->snd_wnd
!= nwin
) {
3551 /* Note, it is the only place, where
3552 * fast path is recovered for sending TCP.
3555 tcp_fast_path_check(sk
);
3557 if (!tcp_write_queue_empty(sk
))
3558 tcp_slow_start_after_idle_check(sk
);
3560 if (nwin
> tp
->max_window
) {
3561 tp
->max_window
= nwin
;
3562 tcp_sync_mss(sk
, inet_csk(sk
)->icsk_pmtu_cookie
);
3567 tcp_snd_una_update(tp
, ack
);
3572 static bool __tcp_oow_rate_limited(struct net
*net
, int mib_idx
,
3573 u32
*last_oow_ack_time
)
3575 if (*last_oow_ack_time
) {
3576 s32 elapsed
= (s32
)(tcp_jiffies32
- *last_oow_ack_time
);
3578 if (0 <= elapsed
&& elapsed
< net
->ipv4
.sysctl_tcp_invalid_ratelimit
) {
3579 NET_INC_STATS(net
, mib_idx
);
3580 return true; /* rate-limited: don't send yet! */
3584 *last_oow_ack_time
= tcp_jiffies32
;
3586 return false; /* not rate-limited: go ahead, send dupack now! */
3589 /* Return true if we're currently rate-limiting out-of-window ACKs and
3590 * thus shouldn't send a dupack right now. We rate-limit dupacks in
3591 * response to out-of-window SYNs or ACKs to mitigate ACK loops or DoS
3592 * attacks that send repeated SYNs or ACKs for the same connection. To
3593 * do this, we do not send a duplicate SYNACK or ACK if the remote
3594 * endpoint is sending out-of-window SYNs or pure ACKs at a high rate.
3596 bool tcp_oow_rate_limited(struct net
*net
, const struct sk_buff
*skb
,
3597 int mib_idx
, u32
*last_oow_ack_time
)
3599 /* Data packets without SYNs are not likely part of an ACK loop. */
3600 if ((TCP_SKB_CB(skb
)->seq
!= TCP_SKB_CB(skb
)->end_seq
) &&
3604 return __tcp_oow_rate_limited(net
, mib_idx
, last_oow_ack_time
);
3607 /* RFC 5961 7 [ACK Throttling] */
3608 static void tcp_send_challenge_ack(struct sock
*sk
)
3610 /* unprotected vars, we dont care of overwrites */
3611 static u32 challenge_timestamp
;
3612 static unsigned int challenge_count
;
3613 struct tcp_sock
*tp
= tcp_sk(sk
);
3614 struct net
*net
= sock_net(sk
);
3617 /* First check our per-socket dupack rate limit. */
3618 if (__tcp_oow_rate_limited(net
,
3619 LINUX_MIB_TCPACKSKIPPEDCHALLENGE
,
3620 &tp
->last_oow_ack_time
))
3623 /* Then check host-wide RFC 5961 rate limit. */
3625 if (now
!= challenge_timestamp
) {
3626 u32 ack_limit
= net
->ipv4
.sysctl_tcp_challenge_ack_limit
;
3627 u32 half
= (ack_limit
+ 1) >> 1;
3629 challenge_timestamp
= now
;
3630 WRITE_ONCE(challenge_count
, half
+ prandom_u32_max(ack_limit
));
3632 count
= READ_ONCE(challenge_count
);
3634 WRITE_ONCE(challenge_count
, count
- 1);
3635 NET_INC_STATS(net
, LINUX_MIB_TCPCHALLENGEACK
);
3640 static void tcp_store_ts_recent(struct tcp_sock
*tp
)
3642 tp
->rx_opt
.ts_recent
= tp
->rx_opt
.rcv_tsval
;
3643 tp
->rx_opt
.ts_recent_stamp
= ktime_get_seconds();
3646 static void tcp_replace_ts_recent(struct tcp_sock
*tp
, u32 seq
)
3648 if (tp
->rx_opt
.saw_tstamp
&& !after(seq
, tp
->rcv_wup
)) {
3649 /* PAWS bug workaround wrt. ACK frames, the PAWS discard
3650 * extra check below makes sure this can only happen
3651 * for pure ACK frames. -DaveM
3653 * Not only, also it occurs for expired timestamps.
3656 if (tcp_paws_check(&tp
->rx_opt
, 0))
3657 tcp_store_ts_recent(tp
);
3661 /* This routine deals with acks during a TLP episode and ends an episode by
3662 * resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack
3664 static void tcp_process_tlp_ack(struct sock
*sk
, u32 ack
, int flag
)
3666 struct tcp_sock
*tp
= tcp_sk(sk
);
3668 if (before(ack
, tp
->tlp_high_seq
))
3671 if (!tp
->tlp_retrans
) {
3672 /* TLP of new data has been acknowledged */
3673 tp
->tlp_high_seq
= 0;
3674 } else if (flag
& FLAG_DSACK_TLP
) {
3675 /* This DSACK means original and TLP probe arrived; no loss */
3676 tp
->tlp_high_seq
= 0;
3677 } else if (after(ack
, tp
->tlp_high_seq
)) {
3678 /* ACK advances: there was a loss, so reduce cwnd. Reset
3679 * tlp_high_seq in tcp_init_cwnd_reduction()
3681 tcp_init_cwnd_reduction(sk
);
3682 tcp_set_ca_state(sk
, TCP_CA_CWR
);
3683 tcp_end_cwnd_reduction(sk
);
3684 tcp_try_keep_open(sk
);
3685 NET_INC_STATS(sock_net(sk
),
3686 LINUX_MIB_TCPLOSSPROBERECOVERY
);
3687 } else if (!(flag
& (FLAG_SND_UNA_ADVANCED
|
3688 FLAG_NOT_DUP
| FLAG_DATA_SACKED
))) {
3689 /* Pure dupack: original and TLP probe arrived; no loss */
3690 tp
->tlp_high_seq
= 0;
3694 static inline void tcp_in_ack_event(struct sock
*sk
, u32 flags
)
3696 const struct inet_connection_sock
*icsk
= inet_csk(sk
);
3698 if (icsk
->icsk_ca_ops
->in_ack_event
)
3699 icsk
->icsk_ca_ops
->in_ack_event(sk
, flags
);
3702 /* Congestion control has updated the cwnd already. So if we're in
3703 * loss recovery then now we do any new sends (for FRTO) or
3704 * retransmits (for CA_Loss or CA_recovery) that make sense.
3706 static void tcp_xmit_recovery(struct sock
*sk
, int rexmit
)
3708 struct tcp_sock
*tp
= tcp_sk(sk
);
3710 if (rexmit
== REXMIT_NONE
|| sk
->sk_state
== TCP_SYN_SENT
)
3713 if (unlikely(rexmit
== REXMIT_NEW
)) {
3714 __tcp_push_pending_frames(sk
, tcp_current_mss(sk
),
3716 if (after(tp
->snd_nxt
, tp
->high_seq
))
3720 tcp_xmit_retransmit_queue(sk
);
3723 /* Returns the number of packets newly acked or sacked by the current ACK */
3724 static u32
tcp_newly_delivered(struct sock
*sk
, u32 prior_delivered
, int flag
)
3726 const struct net
*net
= sock_net(sk
);
3727 struct tcp_sock
*tp
= tcp_sk(sk
);
3730 delivered
= tp
->delivered
- prior_delivered
;
3731 NET_ADD_STATS(net
, LINUX_MIB_TCPDELIVERED
, delivered
);
3732 if (flag
& FLAG_ECE
)
3733 NET_ADD_STATS(net
, LINUX_MIB_TCPDELIVEREDCE
, delivered
);
3738 /* This routine deals with incoming acks, but not outgoing ones. */
3739 static int tcp_ack(struct sock
*sk
, const struct sk_buff
*skb
, int flag
)
3741 struct inet_connection_sock
*icsk
= inet_csk(sk
);
3742 struct tcp_sock
*tp
= tcp_sk(sk
);
3743 struct tcp_sacktag_state sack_state
;
3744 struct rate_sample rs
= { .prior_delivered
= 0 };
3745 u32 prior_snd_una
= tp
->snd_una
;
3746 bool is_sack_reneg
= tp
->is_sack_reneg
;
3747 u32 ack_seq
= TCP_SKB_CB(skb
)->seq
;
3748 u32 ack
= TCP_SKB_CB(skb
)->ack_seq
;
3750 int prior_packets
= tp
->packets_out
;
3751 u32 delivered
= tp
->delivered
;
3752 u32 lost
= tp
->lost
;
3753 int rexmit
= REXMIT_NONE
; /* Flag to (re)transmit to recover losses */
3756 sack_state
.first_sackt
= 0;
3757 sack_state
.rate
= &rs
;
3758 sack_state
.sack_delivered
= 0;
3760 /* We very likely will need to access rtx queue. */
3761 prefetch(sk
->tcp_rtx_queue
.rb_node
);
3763 /* If the ack is older than previous acks
3764 * then we can probably ignore it.
3766 if (before(ack
, prior_snd_una
)) {
3767 /* RFC 5961 5.2 [Blind Data Injection Attack].[Mitigation] */
3768 if (before(ack
, prior_snd_una
- tp
->max_window
)) {
3769 if (!(flag
& FLAG_NO_CHALLENGE_ACK
))
3770 tcp_send_challenge_ack(sk
);
3771 return -SKB_DROP_REASON_TCP_TOO_OLD_ACK
;
3776 /* If the ack includes data we haven't sent yet, discard
3777 * this segment (RFC793 Section 3.9).
3779 if (after(ack
, tp
->snd_nxt
))
3780 return -SKB_DROP_REASON_TCP_ACK_UNSENT_DATA
;
3782 if (after(ack
, prior_snd_una
)) {
3783 flag
|= FLAG_SND_UNA_ADVANCED
;
3784 icsk
->icsk_retransmits
= 0;
3786 #if IS_ENABLED(CONFIG_TLS_DEVICE)
3787 if (static_branch_unlikely(&clean_acked_data_enabled
.key
))
3788 if (icsk
->icsk_clean_acked
)
3789 icsk
->icsk_clean_acked(sk
, ack
);
3793 prior_fack
= tcp_is_sack(tp
) ? tcp_highest_sack_seq(tp
) : tp
->snd_una
;
3794 rs
.prior_in_flight
= tcp_packets_in_flight(tp
);
3796 /* ts_recent update must be made after we are sure that the packet
3799 if (flag
& FLAG_UPDATE_TS_RECENT
)
3800 tcp_replace_ts_recent(tp
, TCP_SKB_CB(skb
)->seq
);
3802 if ((flag
& (FLAG_SLOWPATH
| FLAG_SND_UNA_ADVANCED
)) ==
3803 FLAG_SND_UNA_ADVANCED
) {
3804 /* Window is constant, pure forward advance.
3805 * No more checks are required.
3806 * Note, we use the fact that SND.UNA>=SND.WL2.
3808 tcp_update_wl(tp
, ack_seq
);
3809 tcp_snd_una_update(tp
, ack
);
3810 flag
|= FLAG_WIN_UPDATE
;
3812 tcp_in_ack_event(sk
, CA_ACK_WIN_UPDATE
);
3814 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPHPACKS
);
3816 u32 ack_ev_flags
= CA_ACK_SLOWPATH
;
3818 if (ack_seq
!= TCP_SKB_CB(skb
)->end_seq
)
3821 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPPUREACKS
);
3823 flag
|= tcp_ack_update_window(sk
, skb
, ack
, ack_seq
);
3825 if (TCP_SKB_CB(skb
)->sacked
)
3826 flag
|= tcp_sacktag_write_queue(sk
, skb
, prior_snd_una
,
3829 if (tcp_ecn_rcv_ecn_echo(tp
, tcp_hdr(skb
))) {
3831 ack_ev_flags
|= CA_ACK_ECE
;
3834 if (sack_state
.sack_delivered
)
3835 tcp_count_delivered(tp
, sack_state
.sack_delivered
,
3838 if (flag
& FLAG_WIN_UPDATE
)
3839 ack_ev_flags
|= CA_ACK_WIN_UPDATE
;
3841 tcp_in_ack_event(sk
, ack_ev_flags
);
3844 /* This is a deviation from RFC3168 since it states that:
3845 * "When the TCP data sender is ready to set the CWR bit after reducing
3846 * the congestion window, it SHOULD set the CWR bit only on the first
3847 * new data packet that it transmits."
3848 * We accept CWR on pure ACKs to be more robust
3849 * with widely-deployed TCP implementations that do this.
3851 tcp_ecn_accept_cwr(sk
, skb
);
3853 /* We passed data and got it acked, remove any soft error
3854 * log. Something worked...
3856 sk
->sk_err_soft
= 0;
3857 icsk
->icsk_probes_out
= 0;
3858 tp
->rcv_tstamp
= tcp_jiffies32
;
3862 /* See if we can take anything off of the retransmit queue. */
3863 flag
|= tcp_clean_rtx_queue(sk
, skb
, prior_fack
, prior_snd_una
,
3864 &sack_state
, flag
& FLAG_ECE
);
3866 tcp_rack_update_reo_wnd(sk
, &rs
);
3868 if (tp
->tlp_high_seq
)
3869 tcp_process_tlp_ack(sk
, ack
, flag
);
3871 if (tcp_ack_is_dubious(sk
, flag
)) {
3872 if (!(flag
& (FLAG_SND_UNA_ADVANCED
|
3873 FLAG_NOT_DUP
| FLAG_DSACKING_ACK
))) {
3875 /* Consider if pure acks were aggregated in tcp_add_backlog() */
3876 if (!(flag
& FLAG_DATA
))
3877 num_dupack
= max_t(u16
, 1, skb_shinfo(skb
)->gso_segs
);
3879 tcp_fastretrans_alert(sk
, prior_snd_una
, num_dupack
, &flag
,
3883 /* If needed, reset TLP/RTO timer when RACK doesn't set. */
3884 if (flag
& FLAG_SET_XMIT_TIMER
)
3885 tcp_set_xmit_timer(sk
);
3887 if ((flag
& FLAG_FORWARD_PROGRESS
) || !(flag
& FLAG_NOT_DUP
))
3890 delivered
= tcp_newly_delivered(sk
, delivered
, flag
);
3891 lost
= tp
->lost
- lost
; /* freshly marked lost */
3892 rs
.is_ack_delayed
= !!(flag
& FLAG_ACK_MAYBE_DELAYED
);
3893 tcp_rate_gen(sk
, delivered
, lost
, is_sack_reneg
, sack_state
.rate
);
3894 tcp_cong_control(sk
, ack
, delivered
, flag
, sack_state
.rate
);
3895 tcp_xmit_recovery(sk
, rexmit
);
3899 /* If data was DSACKed, see if we can undo a cwnd reduction. */
3900 if (flag
& FLAG_DSACKING_ACK
) {
3901 tcp_fastretrans_alert(sk
, prior_snd_una
, num_dupack
, &flag
,
3903 tcp_newly_delivered(sk
, delivered
, flag
);
3905 /* If this ack opens up a zero window, clear backoff. It was
3906 * being used to time the probes, and is probably far higher than
3907 * it needs to be for normal retransmission.
3911 if (tp
->tlp_high_seq
)
3912 tcp_process_tlp_ack(sk
, ack
, flag
);
3916 /* If data was SACKed, tag it and see if we should send more data.
3917 * If data was DSACKed, see if we can undo a cwnd reduction.
3919 if (TCP_SKB_CB(skb
)->sacked
) {
3920 flag
|= tcp_sacktag_write_queue(sk
, skb
, prior_snd_una
,
3922 tcp_fastretrans_alert(sk
, prior_snd_una
, num_dupack
, &flag
,
3924 tcp_newly_delivered(sk
, delivered
, flag
);
3925 tcp_xmit_recovery(sk
, rexmit
);
3931 static void tcp_parse_fastopen_option(int len
, const unsigned char *cookie
,
3932 bool syn
, struct tcp_fastopen_cookie
*foc
,
3935 /* Valid only in SYN or SYN-ACK with an even length. */
3936 if (!foc
|| !syn
|| len
< 0 || (len
& 1))
3939 if (len
>= TCP_FASTOPEN_COOKIE_MIN
&&
3940 len
<= TCP_FASTOPEN_COOKIE_MAX
)
3941 memcpy(foc
->val
, cookie
, len
);
3948 static bool smc_parse_options(const struct tcphdr
*th
,
3949 struct tcp_options_received
*opt_rx
,
3950 const unsigned char *ptr
,
3953 #if IS_ENABLED(CONFIG_SMC)
3954 if (static_branch_unlikely(&tcp_have_smc
)) {
3955 if (th
->syn
&& !(opsize
& 1) &&
3956 opsize
>= TCPOLEN_EXP_SMC_BASE
&&
3957 get_unaligned_be32(ptr
) == TCPOPT_SMC_MAGIC
) {
3966 /* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
3969 u16
tcp_parse_mss_option(const struct tcphdr
*th
, u16 user_mss
)
3971 const unsigned char *ptr
= (const unsigned char *)(th
+ 1);
3972 int length
= (th
->doff
* 4) - sizeof(struct tcphdr
);
3975 while (length
> 0) {
3976 int opcode
= *ptr
++;
3982 case TCPOPT_NOP
: /* Ref: RFC 793 section 3.1 */
3989 if (opsize
< 2) /* "silly options" */
3991 if (opsize
> length
)
3992 return mss
; /* fail on partial options */
3993 if (opcode
== TCPOPT_MSS
&& opsize
== TCPOLEN_MSS
) {
3994 u16 in_mss
= get_unaligned_be16(ptr
);
3997 if (user_mss
&& user_mss
< in_mss
)
4008 EXPORT_SYMBOL_GPL(tcp_parse_mss_option
);
4010 /* Look for tcp options. Normally only called on SYN and SYNACK packets.
4011 * But, this can also be called on packets in the established flow when
4012 * the fast version below fails.
4014 void tcp_parse_options(const struct net
*net
,
4015 const struct sk_buff
*skb
,
4016 struct tcp_options_received
*opt_rx
, int estab
,
4017 struct tcp_fastopen_cookie
*foc
)
4019 const unsigned char *ptr
;
4020 const struct tcphdr
*th
= tcp_hdr(skb
);
4021 int length
= (th
->doff
* 4) - sizeof(struct tcphdr
);
4023 ptr
= (const unsigned char *)(th
+ 1);
4024 opt_rx
->saw_tstamp
= 0;
4025 opt_rx
->saw_unknown
= 0;
4027 while (length
> 0) {
4028 int opcode
= *ptr
++;
4034 case TCPOPT_NOP
: /* Ref: RFC 793 section 3.1 */
4041 if (opsize
< 2) /* "silly options" */
4043 if (opsize
> length
)
4044 return; /* don't parse partial options */
4047 if (opsize
== TCPOLEN_MSS
&& th
->syn
&& !estab
) {
4048 u16 in_mss
= get_unaligned_be16(ptr
);
4050 if (opt_rx
->user_mss
&&
4051 opt_rx
->user_mss
< in_mss
)
4052 in_mss
= opt_rx
->user_mss
;
4053 opt_rx
->mss_clamp
= in_mss
;
4058 if (opsize
== TCPOLEN_WINDOW
&& th
->syn
&&
4059 !estab
&& net
->ipv4
.sysctl_tcp_window_scaling
) {
4060 __u8 snd_wscale
= *(__u8
*)ptr
;
4061 opt_rx
->wscale_ok
= 1;
4062 if (snd_wscale
> TCP_MAX_WSCALE
) {
4063 net_info_ratelimited("%s: Illegal window scaling value %d > %u received\n",
4067 snd_wscale
= TCP_MAX_WSCALE
;
4069 opt_rx
->snd_wscale
= snd_wscale
;
4072 case TCPOPT_TIMESTAMP
:
4073 if ((opsize
== TCPOLEN_TIMESTAMP
) &&
4074 ((estab
&& opt_rx
->tstamp_ok
) ||
4075 (!estab
&& net
->ipv4
.sysctl_tcp_timestamps
))) {
4076 opt_rx
->saw_tstamp
= 1;
4077 opt_rx
->rcv_tsval
= get_unaligned_be32(ptr
);
4078 opt_rx
->rcv_tsecr
= get_unaligned_be32(ptr
+ 4);
4081 case TCPOPT_SACK_PERM
:
4082 if (opsize
== TCPOLEN_SACK_PERM
&& th
->syn
&&
4083 !estab
&& net
->ipv4
.sysctl_tcp_sack
) {
4084 opt_rx
->sack_ok
= TCP_SACK_SEEN
;
4085 tcp_sack_reset(opt_rx
);
4090 if ((opsize
>= (TCPOLEN_SACK_BASE
+ TCPOLEN_SACK_PERBLOCK
)) &&
4091 !((opsize
- TCPOLEN_SACK_BASE
) % TCPOLEN_SACK_PERBLOCK
) &&
4093 TCP_SKB_CB(skb
)->sacked
= (ptr
- 2) - (unsigned char *)th
;
4096 #ifdef CONFIG_TCP_MD5SIG
4099 * The MD5 Hash has already been
4100 * checked (see tcp_v{4,6}_do_rcv()).
4104 case TCPOPT_FASTOPEN
:
4105 tcp_parse_fastopen_option(
4106 opsize
- TCPOLEN_FASTOPEN_BASE
,
4107 ptr
, th
->syn
, foc
, false);
4111 /* Fast Open option shares code 254 using a
4112 * 16 bits magic number.
4114 if (opsize
>= TCPOLEN_EXP_FASTOPEN_BASE
&&
4115 get_unaligned_be16(ptr
) ==
4116 TCPOPT_FASTOPEN_MAGIC
) {
4117 tcp_parse_fastopen_option(opsize
-
4118 TCPOLEN_EXP_FASTOPEN_BASE
,
4119 ptr
+ 2, th
->syn
, foc
, true);
4123 if (smc_parse_options(th
, opt_rx
, ptr
, opsize
))
4126 opt_rx
->saw_unknown
= 1;
4130 opt_rx
->saw_unknown
= 1;
4137 EXPORT_SYMBOL(tcp_parse_options
);
4139 static bool tcp_parse_aligned_timestamp(struct tcp_sock
*tp
, const struct tcphdr
*th
)
4141 const __be32
*ptr
= (const __be32
*)(th
+ 1);
4143 if (*ptr
== htonl((TCPOPT_NOP
<< 24) | (TCPOPT_NOP
<< 16)
4144 | (TCPOPT_TIMESTAMP
<< 8) | TCPOLEN_TIMESTAMP
)) {
4145 tp
->rx_opt
.saw_tstamp
= 1;
4147 tp
->rx_opt
.rcv_tsval
= ntohl(*ptr
);
4150 tp
->rx_opt
.rcv_tsecr
= ntohl(*ptr
) - tp
->tsoffset
;
4152 tp
->rx_opt
.rcv_tsecr
= 0;
4158 /* Fast parse options. This hopes to only see timestamps.
4159 * If it is wrong it falls back on tcp_parse_options().
4161 static bool tcp_fast_parse_options(const struct net
*net
,
4162 const struct sk_buff
*skb
,
4163 const struct tcphdr
*th
, struct tcp_sock
*tp
)
4165 /* In the spirit of fast parsing, compare doff directly to constant
4166 * values. Because equality is used, short doff can be ignored here.
4168 if (th
->doff
== (sizeof(*th
) / 4)) {
4169 tp
->rx_opt
.saw_tstamp
= 0;
4171 } else if (tp
->rx_opt
.tstamp_ok
&&
4172 th
->doff
== ((sizeof(*th
) + TCPOLEN_TSTAMP_ALIGNED
) / 4)) {
4173 if (tcp_parse_aligned_timestamp(tp
, th
))
4177 tcp_parse_options(net
, skb
, &tp
->rx_opt
, 1, NULL
);
4178 if (tp
->rx_opt
.saw_tstamp
&& tp
->rx_opt
.rcv_tsecr
)
4179 tp
->rx_opt
.rcv_tsecr
-= tp
->tsoffset
;
4184 #ifdef CONFIG_TCP_MD5SIG
4186 * Parse MD5 Signature option
4188 const u8
*tcp_parse_md5sig_option(const struct tcphdr
*th
)
4190 int length
= (th
->doff
<< 2) - sizeof(*th
);
4191 const u8
*ptr
= (const u8
*)(th
+ 1);
4193 /* If not enough data remaining, we can short cut */
4194 while (length
>= TCPOLEN_MD5SIG
) {
4195 int opcode
= *ptr
++;
4206 if (opsize
< 2 || opsize
> length
)
4208 if (opcode
== TCPOPT_MD5SIG
)
4209 return opsize
== TCPOLEN_MD5SIG
? ptr
: NULL
;
4216 EXPORT_SYMBOL(tcp_parse_md5sig_option
);
4219 /* Sorry, PAWS as specified is broken wrt. pure-ACKs -DaveM
4221 * It is not fatal. If this ACK does _not_ change critical state (seqs, window)
4222 * it can pass through stack. So, the following predicate verifies that
4223 * this segment is not used for anything but congestion avoidance or
4224 * fast retransmit. Moreover, we even are able to eliminate most of such
4225 * second order effects, if we apply some small "replay" window (~RTO)
4226 * to timestamp space.
4228 * All these measures still do not guarantee that we reject wrapped ACKs
4229 * on networks with high bandwidth, when sequence space is recycled fastly,
4230 * but it guarantees that such events will be very rare and do not affect
4231 * connection seriously. This doesn't look nice, but alas, PAWS is really
4234 * [ Later note. Even worse! It is buggy for segments _with_ data. RFC
4235 * states that events when retransmit arrives after original data are rare.
4236 * It is a blatant lie. VJ forgot about fast retransmit! 8)8) It is
4237 * the biggest problem on large power networks even with minor reordering.
4238 * OK, let's give it small replay window. If peer clock is even 1hz, it is safe
4239 * up to bandwidth of 18Gigabit/sec. 8) ]
4242 static int tcp_disordered_ack(const struct sock
*sk
, const struct sk_buff
*skb
)
4244 const struct tcp_sock
*tp
= tcp_sk(sk
);
4245 const struct tcphdr
*th
= tcp_hdr(skb
);
4246 u32 seq
= TCP_SKB_CB(skb
)->seq
;
4247 u32 ack
= TCP_SKB_CB(skb
)->ack_seq
;
4249 return (/* 1. Pure ACK with correct sequence number. */
4250 (th
->ack
&& seq
== TCP_SKB_CB(skb
)->end_seq
&& seq
== tp
->rcv_nxt
) &&
4252 /* 2. ... and duplicate ACK. */
4253 ack
== tp
->snd_una
&&
4255 /* 3. ... and does not update window. */
4256 !tcp_may_update_window(tp
, ack
, seq
, ntohs(th
->window
) << tp
->rx_opt
.snd_wscale
) &&
4258 /* 4. ... and sits in replay window. */
4259 (s32
)(tp
->rx_opt
.ts_recent
- tp
->rx_opt
.rcv_tsval
) <= (inet_csk(sk
)->icsk_rto
* 1024) / HZ
);
4262 static inline bool tcp_paws_discard(const struct sock
*sk
,
4263 const struct sk_buff
*skb
)
4265 const struct tcp_sock
*tp
= tcp_sk(sk
);
4267 return !tcp_paws_check(&tp
->rx_opt
, TCP_PAWS_WINDOW
) &&
4268 !tcp_disordered_ack(sk
, skb
);
4271 /* Check segment sequence number for validity.
4273 * Segment controls are considered valid, if the segment
4274 * fits to the window after truncation to the window. Acceptability
4275 * of data (and SYN, FIN, of course) is checked separately.
4276 * See tcp_data_queue(), for example.
4278 * Also, controls (RST is main one) are accepted using RCV.WUP instead
4279 * of RCV.NXT. Peer still did not advance his SND.UNA when we
4280 * delayed ACK, so that hisSND.UNA<=ourRCV.WUP.
4281 * (borrowed from freebsd)
4284 static inline bool tcp_sequence(const struct tcp_sock
*tp
, u32 seq
, u32 end_seq
)
4286 return !before(end_seq
, tp
->rcv_wup
) &&
4287 !after(seq
, tp
->rcv_nxt
+ tcp_receive_window(tp
));
4290 /* When we get a reset we do this. */
4291 void tcp_reset(struct sock
*sk
, struct sk_buff
*skb
)
4293 trace_tcp_receive_reset(sk
);
4295 /* mptcp can't tell us to ignore reset pkts,
4296 * so just ignore the return value of mptcp_incoming_options().
4298 if (sk_is_mptcp(sk
))
4299 mptcp_incoming_options(sk
, skb
);
4301 /* We want the right error as BSD sees it (and indeed as we do). */
4302 switch (sk
->sk_state
) {
4304 sk
->sk_err
= ECONNREFUSED
;
4306 case TCP_CLOSE_WAIT
:
4312 sk
->sk_err
= ECONNRESET
;
4314 /* This barrier is coupled with smp_rmb() in tcp_poll() */
4317 tcp_write_queue_purge(sk
);
4320 if (!sock_flag(sk
, SOCK_DEAD
))
4321 sk_error_report(sk
);
4325 * Process the FIN bit. This now behaves as it is supposed to work
4326 * and the FIN takes effect when it is validly part of sequence
4327 * space. Not before when we get holes.
4329 * If we are ESTABLISHED, a received fin moves us to CLOSE-WAIT
4330 * (and thence onto LAST-ACK and finally, CLOSE, we never enter
4333 * If we are in FINWAIT-1, a received FIN indicates simultaneous
4334 * close and we go into CLOSING (and later onto TIME-WAIT)
4336 * If we are in FINWAIT-2, a received FIN moves us to TIME-WAIT.
4338 void tcp_fin(struct sock
*sk
)
4340 struct tcp_sock
*tp
= tcp_sk(sk
);
4342 inet_csk_schedule_ack(sk
);
4344 sk
->sk_shutdown
|= RCV_SHUTDOWN
;
4345 sock_set_flag(sk
, SOCK_DONE
);
4347 switch (sk
->sk_state
) {
4349 case TCP_ESTABLISHED
:
4350 /* Move to CLOSE_WAIT */
4351 tcp_set_state(sk
, TCP_CLOSE_WAIT
);
4352 inet_csk_enter_pingpong_mode(sk
);
4355 case TCP_CLOSE_WAIT
:
4357 /* Received a retransmission of the FIN, do
4362 /* RFC793: Remain in the LAST-ACK state. */
4366 /* This case occurs when a simultaneous close
4367 * happens, we must ack the received FIN and
4368 * enter the CLOSING state.
4371 tcp_set_state(sk
, TCP_CLOSING
);
4374 /* Received a FIN -- send ACK and enter TIME_WAIT. */
4376 tcp_time_wait(sk
, TCP_TIME_WAIT
, 0);
4379 /* Only TCP_LISTEN and TCP_CLOSE are left, in these
4380 * cases we should never reach this piece of code.
4382 pr_err("%s: Impossible, sk->sk_state=%d\n",
4383 __func__
, sk
->sk_state
);
4387 /* It _is_ possible, that we have something out-of-order _after_ FIN.
4388 * Probably, we should reset in this case. For now drop them.
4390 skb_rbtree_purge(&tp
->out_of_order_queue
);
4391 if (tcp_is_sack(tp
))
4392 tcp_sack_reset(&tp
->rx_opt
);
4394 if (!sock_flag(sk
, SOCK_DEAD
)) {
4395 sk
->sk_state_change(sk
);
4397 /* Do not send POLL_HUP for half duplex close. */
4398 if (sk
->sk_shutdown
== SHUTDOWN_MASK
||
4399 sk
->sk_state
== TCP_CLOSE
)
4400 sk_wake_async(sk
, SOCK_WAKE_WAITD
, POLL_HUP
);
4402 sk_wake_async(sk
, SOCK_WAKE_WAITD
, POLL_IN
);
4406 static inline bool tcp_sack_extend(struct tcp_sack_block
*sp
, u32 seq
,
4409 if (!after(seq
, sp
->end_seq
) && !after(sp
->start_seq
, end_seq
)) {
4410 if (before(seq
, sp
->start_seq
))
4411 sp
->start_seq
= seq
;
4412 if (after(end_seq
, sp
->end_seq
))
4413 sp
->end_seq
= end_seq
;
4419 static void tcp_dsack_set(struct sock
*sk
, u32 seq
, u32 end_seq
)
4421 struct tcp_sock
*tp
= tcp_sk(sk
);
4423 if (tcp_is_sack(tp
) && sock_net(sk
)->ipv4
.sysctl_tcp_dsack
) {
4426 if (before(seq
, tp
->rcv_nxt
))
4427 mib_idx
= LINUX_MIB_TCPDSACKOLDSENT
;
4429 mib_idx
= LINUX_MIB_TCPDSACKOFOSENT
;
4431 NET_INC_STATS(sock_net(sk
), mib_idx
);
4433 tp
->rx_opt
.dsack
= 1;
4434 tp
->duplicate_sack
[0].start_seq
= seq
;
4435 tp
->duplicate_sack
[0].end_seq
= end_seq
;
4439 static void tcp_dsack_extend(struct sock
*sk
, u32 seq
, u32 end_seq
)
4441 struct tcp_sock
*tp
= tcp_sk(sk
);
4443 if (!tp
->rx_opt
.dsack
)
4444 tcp_dsack_set(sk
, seq
, end_seq
);
4446 tcp_sack_extend(tp
->duplicate_sack
, seq
, end_seq
);
4449 static void tcp_rcv_spurious_retrans(struct sock
*sk
, const struct sk_buff
*skb
)
4451 /* When the ACK path fails or drops most ACKs, the sender would
4452 * timeout and spuriously retransmit the same segment repeatedly.
4453 * The receiver remembers and reflects via DSACKs. Leverage the
4454 * DSACK state and change the txhash to re-route speculatively.
4456 if (TCP_SKB_CB(skb
)->seq
== tcp_sk(sk
)->duplicate_sack
[0].start_seq
&&
4457 sk_rethink_txhash(sk
))
4458 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPDUPLICATEDATAREHASH
);
4461 static void tcp_send_dupack(struct sock
*sk
, const struct sk_buff
*skb
)
4463 struct tcp_sock
*tp
= tcp_sk(sk
);
4465 if (TCP_SKB_CB(skb
)->end_seq
!= TCP_SKB_CB(skb
)->seq
&&
4466 before(TCP_SKB_CB(skb
)->seq
, tp
->rcv_nxt
)) {
4467 NET_INC_STATS(sock_net(sk
), LINUX_MIB_DELAYEDACKLOST
);
4468 tcp_enter_quickack_mode(sk
, TCP_MAX_QUICKACKS
);
4470 if (tcp_is_sack(tp
) && sock_net(sk
)->ipv4
.sysctl_tcp_dsack
) {
4471 u32 end_seq
= TCP_SKB_CB(skb
)->end_seq
;
4473 tcp_rcv_spurious_retrans(sk
, skb
);
4474 if (after(TCP_SKB_CB(skb
)->end_seq
, tp
->rcv_nxt
))
4475 end_seq
= tp
->rcv_nxt
;
4476 tcp_dsack_set(sk
, TCP_SKB_CB(skb
)->seq
, end_seq
);
4483 /* These routines update the SACK block as out-of-order packets arrive or
4484 * in-order packets close up the sequence space.
4486 static void tcp_sack_maybe_coalesce(struct tcp_sock
*tp
)
4489 struct tcp_sack_block
*sp
= &tp
->selective_acks
[0];
4490 struct tcp_sack_block
*swalk
= sp
+ 1;
4492 /* See if the recent change to the first SACK eats into
4493 * or hits the sequence space of other SACK blocks, if so coalesce.
4495 for (this_sack
= 1; this_sack
< tp
->rx_opt
.num_sacks
;) {
4496 if (tcp_sack_extend(sp
, swalk
->start_seq
, swalk
->end_seq
)) {
4499 /* Zap SWALK, by moving every further SACK up by one slot.
4500 * Decrease num_sacks.
4502 tp
->rx_opt
.num_sacks
--;
4503 for (i
= this_sack
; i
< tp
->rx_opt
.num_sacks
; i
++)
4512 static void tcp_sack_compress_send_ack(struct sock
*sk
)
4514 struct tcp_sock
*tp
= tcp_sk(sk
);
4516 if (!tp
->compressed_ack
)
4519 if (hrtimer_try_to_cancel(&tp
->compressed_ack_timer
) == 1)
4522 /* Since we have to send one ack finally,
4523 * substract one from tp->compressed_ack to keep
4524 * LINUX_MIB_TCPACKCOMPRESSED accurate.
4526 NET_ADD_STATS(sock_net(sk
), LINUX_MIB_TCPACKCOMPRESSED
,
4527 tp
->compressed_ack
- 1);
4529 tp
->compressed_ack
= 0;
4533 /* Reasonable amount of sack blocks included in TCP SACK option
4534 * The max is 4, but this becomes 3 if TCP timestamps are there.
4535 * Given that SACK packets might be lost, be conservative and use 2.
4537 #define TCP_SACK_BLOCKS_EXPECTED 2
4539 static void tcp_sack_new_ofo_skb(struct sock
*sk
, u32 seq
, u32 end_seq
)
4541 struct tcp_sock
*tp
= tcp_sk(sk
);
4542 struct tcp_sack_block
*sp
= &tp
->selective_acks
[0];
4543 int cur_sacks
= tp
->rx_opt
.num_sacks
;
4549 for (this_sack
= 0; this_sack
< cur_sacks
; this_sack
++, sp
++) {
4550 if (tcp_sack_extend(sp
, seq
, end_seq
)) {
4551 if (this_sack
>= TCP_SACK_BLOCKS_EXPECTED
)
4552 tcp_sack_compress_send_ack(sk
);
4553 /* Rotate this_sack to the first one. */
4554 for (; this_sack
> 0; this_sack
--, sp
--)
4555 swap(*sp
, *(sp
- 1));
4557 tcp_sack_maybe_coalesce(tp
);
4562 if (this_sack
>= TCP_SACK_BLOCKS_EXPECTED
)
4563 tcp_sack_compress_send_ack(sk
);
4565 /* Could not find an adjacent existing SACK, build a new one,
4566 * put it at the front, and shift everyone else down. We
4567 * always know there is at least one SACK present already here.
4569 * If the sack array is full, forget about the last one.
4571 if (this_sack
>= TCP_NUM_SACKS
) {
4573 tp
->rx_opt
.num_sacks
--;
4576 for (; this_sack
> 0; this_sack
--, sp
--)
4580 /* Build the new head SACK, and we're done. */
4581 sp
->start_seq
= seq
;
4582 sp
->end_seq
= end_seq
;
4583 tp
->rx_opt
.num_sacks
++;
4586 /* RCV.NXT advances, some SACKs should be eaten. */
4588 static void tcp_sack_remove(struct tcp_sock
*tp
)
4590 struct tcp_sack_block
*sp
= &tp
->selective_acks
[0];
4591 int num_sacks
= tp
->rx_opt
.num_sacks
;
4594 /* Empty ofo queue, hence, all the SACKs are eaten. Clear. */
4595 if (RB_EMPTY_ROOT(&tp
->out_of_order_queue
)) {
4596 tp
->rx_opt
.num_sacks
= 0;
4600 for (this_sack
= 0; this_sack
< num_sacks
;) {
4601 /* Check if the start of the sack is covered by RCV.NXT. */
4602 if (!before(tp
->rcv_nxt
, sp
->start_seq
)) {
4605 /* RCV.NXT must cover all the block! */
4606 WARN_ON(before(tp
->rcv_nxt
, sp
->end_seq
));
4608 /* Zap this SACK, by moving forward any other SACKS. */
4609 for (i
= this_sack
+1; i
< num_sacks
; i
++)
4610 tp
->selective_acks
[i
-1] = tp
->selective_acks
[i
];
4617 tp
->rx_opt
.num_sacks
= num_sacks
;
4621 * tcp_try_coalesce - try to merge skb to prior one
4624 * @from: buffer to add in queue
4625 * @fragstolen: pointer to boolean
4627 * Before queueing skb @from after @to, try to merge them
4628 * to reduce overall memory use and queue lengths, if cost is small.
4629 * Packets in ofo or receive queues can stay a long time.
4630 * Better try to coalesce them right now to avoid future collapses.
4631 * Returns true if caller should free @from instead of queueing it
4633 static bool tcp_try_coalesce(struct sock
*sk
,
4635 struct sk_buff
*from
,
4640 *fragstolen
= false;
4642 /* Its possible this segment overlaps with prior segment in queue */
4643 if (TCP_SKB_CB(from
)->seq
!= TCP_SKB_CB(to
)->end_seq
)
4646 if (!mptcp_skb_can_collapse(to
, from
))
4649 #ifdef CONFIG_TLS_DEVICE
4650 if (from
->decrypted
!= to
->decrypted
)
4654 if (!skb_try_coalesce(to
, from
, fragstolen
, &delta
))
4657 atomic_add(delta
, &sk
->sk_rmem_alloc
);
4658 sk_mem_charge(sk
, delta
);
4659 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPRCVCOALESCE
);
4660 TCP_SKB_CB(to
)->end_seq
= TCP_SKB_CB(from
)->end_seq
;
4661 TCP_SKB_CB(to
)->ack_seq
= TCP_SKB_CB(from
)->ack_seq
;
4662 TCP_SKB_CB(to
)->tcp_flags
|= TCP_SKB_CB(from
)->tcp_flags
;
4664 if (TCP_SKB_CB(from
)->has_rxtstamp
) {
4665 TCP_SKB_CB(to
)->has_rxtstamp
= true;
4666 to
->tstamp
= from
->tstamp
;
4667 skb_hwtstamps(to
)->hwtstamp
= skb_hwtstamps(from
)->hwtstamp
;
4673 static bool tcp_ooo_try_coalesce(struct sock
*sk
,
4675 struct sk_buff
*from
,
4678 bool res
= tcp_try_coalesce(sk
, to
, from
, fragstolen
);
4680 /* In case tcp_drop_reason() is called later, update to->gso_segs */
4682 u32 gso_segs
= max_t(u16
, 1, skb_shinfo(to
)->gso_segs
) +
4683 max_t(u16
, 1, skb_shinfo(from
)->gso_segs
);
4685 skb_shinfo(to
)->gso_segs
= min_t(u32
, gso_segs
, 0xFFFF);
4690 static void tcp_drop_reason(struct sock
*sk
, struct sk_buff
*skb
,
4691 enum skb_drop_reason reason
)
4693 sk_drops_add(sk
, skb
);
4694 kfree_skb_reason(skb
, reason
);
4697 /* This one checks to see if we can put data from the
4698 * out_of_order queue into the receive_queue.
4700 static void tcp_ofo_queue(struct sock
*sk
)
4702 struct tcp_sock
*tp
= tcp_sk(sk
);
4703 __u32 dsack_high
= tp
->rcv_nxt
;
4704 bool fin
, fragstolen
, eaten
;
4705 struct sk_buff
*skb
, *tail
;
4708 p
= rb_first(&tp
->out_of_order_queue
);
4711 if (after(TCP_SKB_CB(skb
)->seq
, tp
->rcv_nxt
))
4714 if (before(TCP_SKB_CB(skb
)->seq
, dsack_high
)) {
4715 __u32 dsack
= dsack_high
;
4716 if (before(TCP_SKB_CB(skb
)->end_seq
, dsack_high
))
4717 dsack_high
= TCP_SKB_CB(skb
)->end_seq
;
4718 tcp_dsack_extend(sk
, TCP_SKB_CB(skb
)->seq
, dsack
);
4721 rb_erase(&skb
->rbnode
, &tp
->out_of_order_queue
);
4723 if (unlikely(!after(TCP_SKB_CB(skb
)->end_seq
, tp
->rcv_nxt
))) {
4724 tcp_drop_reason(sk
, skb
, SKB_DROP_REASON_TCP_OFO_DROP
);
4728 tail
= skb_peek_tail(&sk
->sk_receive_queue
);
4729 eaten
= tail
&& tcp_try_coalesce(sk
, tail
, skb
, &fragstolen
);
4730 tcp_rcv_nxt_update(tp
, TCP_SKB_CB(skb
)->end_seq
);
4731 fin
= TCP_SKB_CB(skb
)->tcp_flags
& TCPHDR_FIN
;
4733 __skb_queue_tail(&sk
->sk_receive_queue
, skb
);
4735 kfree_skb_partial(skb
, fragstolen
);
4737 if (unlikely(fin
)) {
4739 /* tcp_fin() purges tp->out_of_order_queue,
4740 * so we must end this loop right now.
4747 static bool tcp_prune_ofo_queue(struct sock
*sk
);
4748 static int tcp_prune_queue(struct sock
*sk
);
4750 static int tcp_try_rmem_schedule(struct sock
*sk
, struct sk_buff
*skb
,
4753 if (atomic_read(&sk
->sk_rmem_alloc
) > sk
->sk_rcvbuf
||
4754 !sk_rmem_schedule(sk
, skb
, size
)) {
4756 if (tcp_prune_queue(sk
) < 0)
4759 while (!sk_rmem_schedule(sk
, skb
, size
)) {
4760 if (!tcp_prune_ofo_queue(sk
))
4767 static void tcp_data_queue_ofo(struct sock
*sk
, struct sk_buff
*skb
)
4769 struct tcp_sock
*tp
= tcp_sk(sk
);
4770 struct rb_node
**p
, *parent
;
4771 struct sk_buff
*skb1
;
4775 tcp_ecn_check_ce(sk
, skb
);
4777 if (unlikely(tcp_try_rmem_schedule(sk
, skb
, skb
->truesize
))) {
4778 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPOFODROP
);
4779 sk
->sk_data_ready(sk
);
4780 tcp_drop_reason(sk
, skb
, SKB_DROP_REASON_PROTO_MEM
);
4784 /* Disable header prediction. */
4786 inet_csk_schedule_ack(sk
);
4788 tp
->rcv_ooopack
+= max_t(u16
, 1, skb_shinfo(skb
)->gso_segs
);
4789 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPOFOQUEUE
);
4790 seq
= TCP_SKB_CB(skb
)->seq
;
4791 end_seq
= TCP_SKB_CB(skb
)->end_seq
;
4793 p
= &tp
->out_of_order_queue
.rb_node
;
4794 if (RB_EMPTY_ROOT(&tp
->out_of_order_queue
)) {
4795 /* Initial out of order segment, build 1 SACK. */
4796 if (tcp_is_sack(tp
)) {
4797 tp
->rx_opt
.num_sacks
= 1;
4798 tp
->selective_acks
[0].start_seq
= seq
;
4799 tp
->selective_acks
[0].end_seq
= end_seq
;
4801 rb_link_node(&skb
->rbnode
, NULL
, p
);
4802 rb_insert_color(&skb
->rbnode
, &tp
->out_of_order_queue
);
4803 tp
->ooo_last_skb
= skb
;
4807 /* In the typical case, we are adding an skb to the end of the list.
4808 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
4810 if (tcp_ooo_try_coalesce(sk
, tp
->ooo_last_skb
,
4811 skb
, &fragstolen
)) {
4813 /* For non sack flows, do not grow window to force DUPACK
4814 * and trigger fast retransmit.
4816 if (tcp_is_sack(tp
))
4817 tcp_grow_window(sk
, skb
, true);
4818 kfree_skb_partial(skb
, fragstolen
);
4822 /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
4823 if (!before(seq
, TCP_SKB_CB(tp
->ooo_last_skb
)->end_seq
)) {
4824 parent
= &tp
->ooo_last_skb
->rbnode
;
4825 p
= &parent
->rb_right
;
4829 /* Find place to insert this segment. Handle overlaps on the way. */
4833 skb1
= rb_to_skb(parent
);
4834 if (before(seq
, TCP_SKB_CB(skb1
)->seq
)) {
4835 p
= &parent
->rb_left
;
4838 if (before(seq
, TCP_SKB_CB(skb1
)->end_seq
)) {
4839 if (!after(end_seq
, TCP_SKB_CB(skb1
)->end_seq
)) {
4840 /* All the bits are present. Drop. */
4841 NET_INC_STATS(sock_net(sk
),
4842 LINUX_MIB_TCPOFOMERGE
);
4843 tcp_drop_reason(sk
, skb
,
4844 SKB_DROP_REASON_TCP_OFOMERGE
);
4846 tcp_dsack_set(sk
, seq
, end_seq
);
4849 if (after(seq
, TCP_SKB_CB(skb1
)->seq
)) {
4850 /* Partial overlap. */
4851 tcp_dsack_set(sk
, seq
, TCP_SKB_CB(skb1
)->end_seq
);
4853 /* skb's seq == skb1's seq and skb covers skb1.
4854 * Replace skb1 with skb.
4856 rb_replace_node(&skb1
->rbnode
, &skb
->rbnode
,
4857 &tp
->out_of_order_queue
);
4858 tcp_dsack_extend(sk
,
4859 TCP_SKB_CB(skb1
)->seq
,
4860 TCP_SKB_CB(skb1
)->end_seq
);
4861 NET_INC_STATS(sock_net(sk
),
4862 LINUX_MIB_TCPOFOMERGE
);
4863 tcp_drop_reason(sk
, skb1
,
4864 SKB_DROP_REASON_TCP_OFOMERGE
);
4867 } else if (tcp_ooo_try_coalesce(sk
, skb1
,
4868 skb
, &fragstolen
)) {
4871 p
= &parent
->rb_right
;
4874 /* Insert segment into RB tree. */
4875 rb_link_node(&skb
->rbnode
, parent
, p
);
4876 rb_insert_color(&skb
->rbnode
, &tp
->out_of_order_queue
);
4879 /* Remove other segments covered by skb. */
4880 while ((skb1
= skb_rb_next(skb
)) != NULL
) {
4881 if (!after(end_seq
, TCP_SKB_CB(skb1
)->seq
))
4883 if (before(end_seq
, TCP_SKB_CB(skb1
)->end_seq
)) {
4884 tcp_dsack_extend(sk
, TCP_SKB_CB(skb1
)->seq
,
4888 rb_erase(&skb1
->rbnode
, &tp
->out_of_order_queue
);
4889 tcp_dsack_extend(sk
, TCP_SKB_CB(skb1
)->seq
,
4890 TCP_SKB_CB(skb1
)->end_seq
);
4891 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPOFOMERGE
);
4892 tcp_drop_reason(sk
, skb1
, SKB_DROP_REASON_TCP_OFOMERGE
);
4894 /* If there is no skb after us, we are the last_skb ! */
4896 tp
->ooo_last_skb
= skb
;
4899 if (tcp_is_sack(tp
))
4900 tcp_sack_new_ofo_skb(sk
, seq
, end_seq
);
4903 /* For non sack flows, do not grow window to force DUPACK
4904 * and trigger fast retransmit.
4906 if (tcp_is_sack(tp
))
4907 tcp_grow_window(sk
, skb
, false);
4909 skb_set_owner_r(skb
, sk
);
4913 static int __must_check
tcp_queue_rcv(struct sock
*sk
, struct sk_buff
*skb
,
4917 struct sk_buff
*tail
= skb_peek_tail(&sk
->sk_receive_queue
);
4920 tcp_try_coalesce(sk
, tail
,
4921 skb
, fragstolen
)) ? 1 : 0;
4922 tcp_rcv_nxt_update(tcp_sk(sk
), TCP_SKB_CB(skb
)->end_seq
);
4924 __skb_queue_tail(&sk
->sk_receive_queue
, skb
);
4925 skb_set_owner_r(skb
, sk
);
4930 int tcp_send_rcvq(struct sock
*sk
, struct msghdr
*msg
, size_t size
)
4932 struct sk_buff
*skb
;
4940 if (size
> PAGE_SIZE
) {
4941 int npages
= min_t(size_t, size
>> PAGE_SHIFT
, MAX_SKB_FRAGS
);
4943 data_len
= npages
<< PAGE_SHIFT
;
4944 size
= data_len
+ (size
& ~PAGE_MASK
);
4946 skb
= alloc_skb_with_frags(size
- data_len
, data_len
,
4947 PAGE_ALLOC_COSTLY_ORDER
,
4948 &err
, sk
->sk_allocation
);
4952 skb_put(skb
, size
- data_len
);
4953 skb
->data_len
= data_len
;
4956 if (tcp_try_rmem_schedule(sk
, skb
, skb
->truesize
)) {
4957 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPRCVQDROP
);
4961 err
= skb_copy_datagram_from_iter(skb
, 0, &msg
->msg_iter
, size
);
4965 TCP_SKB_CB(skb
)->seq
= tcp_sk(sk
)->rcv_nxt
;
4966 TCP_SKB_CB(skb
)->end_seq
= TCP_SKB_CB(skb
)->seq
+ size
;
4967 TCP_SKB_CB(skb
)->ack_seq
= tcp_sk(sk
)->snd_una
- 1;
4969 if (tcp_queue_rcv(sk
, skb
, &fragstolen
)) {
4970 WARN_ON_ONCE(fragstolen
); /* should not happen */
4982 void tcp_data_ready(struct sock
*sk
)
4984 if (tcp_epollin_ready(sk
, sk
->sk_rcvlowat
) || sock_flag(sk
, SOCK_DONE
))
4985 sk
->sk_data_ready(sk
);
4988 static void tcp_data_queue(struct sock
*sk
, struct sk_buff
*skb
)
4990 struct tcp_sock
*tp
= tcp_sk(sk
);
4991 enum skb_drop_reason reason
;
4995 /* If a subflow has been reset, the packet should not continue
4996 * to be processed, drop the packet.
4998 if (sk_is_mptcp(sk
) && !mptcp_incoming_options(sk
, skb
)) {
5003 if (TCP_SKB_CB(skb
)->seq
== TCP_SKB_CB(skb
)->end_seq
) {
5008 __skb_pull(skb
, tcp_hdr(skb
)->doff
* 4);
5010 reason
= SKB_DROP_REASON_NOT_SPECIFIED
;
5011 tp
->rx_opt
.dsack
= 0;
5013 /* Queue data for delivery to the user.
5014 * Packets in sequence go to the receive queue.
5015 * Out of sequence packets to the out_of_order_queue.
5017 if (TCP_SKB_CB(skb
)->seq
== tp
->rcv_nxt
) {
5018 if (tcp_receive_window(tp
) == 0) {
5019 reason
= SKB_DROP_REASON_TCP_ZEROWINDOW
;
5020 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPZEROWINDOWDROP
);
5024 /* Ok. In sequence. In window. */
5026 if (skb_queue_len(&sk
->sk_receive_queue
) == 0)
5027 sk_forced_mem_schedule(sk
, skb
->truesize
);
5028 else if (tcp_try_rmem_schedule(sk
, skb
, skb
->truesize
)) {
5029 reason
= SKB_DROP_REASON_PROTO_MEM
;
5030 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPRCVQDROP
);
5031 sk
->sk_data_ready(sk
);
5035 eaten
= tcp_queue_rcv(sk
, skb
, &fragstolen
);
5037 tcp_event_data_recv(sk
, skb
);
5038 if (TCP_SKB_CB(skb
)->tcp_flags
& TCPHDR_FIN
)
5041 if (!RB_EMPTY_ROOT(&tp
->out_of_order_queue
)) {
5044 /* RFC5681. 4.2. SHOULD send immediate ACK, when
5045 * gap in queue is filled.
5047 if (RB_EMPTY_ROOT(&tp
->out_of_order_queue
))
5048 inet_csk(sk
)->icsk_ack
.pending
|= ICSK_ACK_NOW
;
5051 if (tp
->rx_opt
.num_sacks
)
5052 tcp_sack_remove(tp
);
5054 tcp_fast_path_check(sk
);
5057 kfree_skb_partial(skb
, fragstolen
);
5058 if (!sock_flag(sk
, SOCK_DEAD
))
5063 if (!after(TCP_SKB_CB(skb
)->end_seq
, tp
->rcv_nxt
)) {
5064 tcp_rcv_spurious_retrans(sk
, skb
);
5065 /* A retransmit, 2nd most common case. Force an immediate ack. */
5066 reason
= SKB_DROP_REASON_TCP_OLD_DATA
;
5067 NET_INC_STATS(sock_net(sk
), LINUX_MIB_DELAYEDACKLOST
);
5068 tcp_dsack_set(sk
, TCP_SKB_CB(skb
)->seq
, TCP_SKB_CB(skb
)->end_seq
);
5071 tcp_enter_quickack_mode(sk
, TCP_MAX_QUICKACKS
);
5072 inet_csk_schedule_ack(sk
);
5074 tcp_drop_reason(sk
, skb
, reason
);
5078 /* Out of window. F.e. zero window probe. */
5079 if (!before(TCP_SKB_CB(skb
)->seq
,
5080 tp
->rcv_nxt
+ tcp_receive_window(tp
))) {
5081 reason
= SKB_DROP_REASON_TCP_OVERWINDOW
;
5085 if (before(TCP_SKB_CB(skb
)->seq
, tp
->rcv_nxt
)) {
5086 /* Partial packet, seq < rcv_next < end_seq */
5087 tcp_dsack_set(sk
, TCP_SKB_CB(skb
)->seq
, tp
->rcv_nxt
);
5089 /* If window is closed, drop tail of packet. But after
5090 * remembering D-SACK for its head made in previous line.
5092 if (!tcp_receive_window(tp
)) {
5093 reason
= SKB_DROP_REASON_TCP_ZEROWINDOW
;
5094 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPZEROWINDOWDROP
);
5100 tcp_data_queue_ofo(sk
, skb
);
5103 static struct sk_buff
*tcp_skb_next(struct sk_buff
*skb
, struct sk_buff_head
*list
)
5106 return !skb_queue_is_last(list
, skb
) ? skb
->next
: NULL
;
5108 return skb_rb_next(skb
);
5111 static struct sk_buff
*tcp_collapse_one(struct sock
*sk
, struct sk_buff
*skb
,
5112 struct sk_buff_head
*list
,
5113 struct rb_root
*root
)
5115 struct sk_buff
*next
= tcp_skb_next(skb
, list
);
5118 __skb_unlink(skb
, list
);
5120 rb_erase(&skb
->rbnode
, root
);
5123 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPRCVCOLLAPSED
);
5128 /* Insert skb into rb tree, ordered by TCP_SKB_CB(skb)->seq */
5129 void tcp_rbtree_insert(struct rb_root
*root
, struct sk_buff
*skb
)
5131 struct rb_node
**p
= &root
->rb_node
;
5132 struct rb_node
*parent
= NULL
;
5133 struct sk_buff
*skb1
;
5137 skb1
= rb_to_skb(parent
);
5138 if (before(TCP_SKB_CB(skb
)->seq
, TCP_SKB_CB(skb1
)->seq
))
5139 p
= &parent
->rb_left
;
5141 p
= &parent
->rb_right
;
5143 rb_link_node(&skb
->rbnode
, parent
, p
);
5144 rb_insert_color(&skb
->rbnode
, root
);
5147 /* Collapse contiguous sequence of skbs head..tail with
5148 * sequence numbers start..end.
5150 * If tail is NULL, this means until the end of the queue.
5152 * Segments with FIN/SYN are not collapsed (only because this
5156 tcp_collapse(struct sock
*sk
, struct sk_buff_head
*list
, struct rb_root
*root
,
5157 struct sk_buff
*head
, struct sk_buff
*tail
, u32 start
, u32 end
)
5159 struct sk_buff
*skb
= head
, *n
;
5160 struct sk_buff_head tmp
;
5163 /* First, check that queue is collapsible and find
5164 * the point where collapsing can be useful.
5167 for (end_of_skbs
= true; skb
!= NULL
&& skb
!= tail
; skb
= n
) {
5168 n
= tcp_skb_next(skb
, list
);
5170 /* No new bits? It is possible on ofo queue. */
5171 if (!before(start
, TCP_SKB_CB(skb
)->end_seq
)) {
5172 skb
= tcp_collapse_one(sk
, skb
, list
, root
);
5178 /* The first skb to collapse is:
5180 * - bloated or contains data before "start" or
5181 * overlaps to the next one and mptcp allow collapsing.
5183 if (!(TCP_SKB_CB(skb
)->tcp_flags
& (TCPHDR_SYN
| TCPHDR_FIN
)) &&
5184 (tcp_win_from_space(sk
, skb
->truesize
) > skb
->len
||
5185 before(TCP_SKB_CB(skb
)->seq
, start
))) {
5186 end_of_skbs
= false;
5190 if (n
&& n
!= tail
&& mptcp_skb_can_collapse(skb
, n
) &&
5191 TCP_SKB_CB(skb
)->end_seq
!= TCP_SKB_CB(n
)->seq
) {
5192 end_of_skbs
= false;
5196 /* Decided to skip this, advance start seq. */
5197 start
= TCP_SKB_CB(skb
)->end_seq
;
5200 (TCP_SKB_CB(skb
)->tcp_flags
& (TCPHDR_SYN
| TCPHDR_FIN
)))
5203 __skb_queue_head_init(&tmp
);
5205 while (before(start
, end
)) {
5206 int copy
= min_t(int, SKB_MAX_ORDER(0, 0), end
- start
);
5207 struct sk_buff
*nskb
;
5209 nskb
= alloc_skb(copy
, GFP_ATOMIC
);
5213 memcpy(nskb
->cb
, skb
->cb
, sizeof(skb
->cb
));
5214 #ifdef CONFIG_TLS_DEVICE
5215 nskb
->decrypted
= skb
->decrypted
;
5217 TCP_SKB_CB(nskb
)->seq
= TCP_SKB_CB(nskb
)->end_seq
= start
;
5219 __skb_queue_before(list
, skb
, nskb
);
5221 __skb_queue_tail(&tmp
, nskb
); /* defer rbtree insertion */
5222 skb_set_owner_r(nskb
, sk
);
5223 mptcp_skb_ext_move(nskb
, skb
);
5225 /* Copy data, releasing collapsed skbs. */
5227 int offset
= start
- TCP_SKB_CB(skb
)->seq
;
5228 int size
= TCP_SKB_CB(skb
)->end_seq
- start
;
5232 size
= min(copy
, size
);
5233 if (skb_copy_bits(skb
, offset
, skb_put(nskb
, size
), size
))
5235 TCP_SKB_CB(nskb
)->end_seq
+= size
;
5239 if (!before(start
, TCP_SKB_CB(skb
)->end_seq
)) {
5240 skb
= tcp_collapse_one(sk
, skb
, list
, root
);
5243 !mptcp_skb_can_collapse(nskb
, skb
) ||
5244 (TCP_SKB_CB(skb
)->tcp_flags
& (TCPHDR_SYN
| TCPHDR_FIN
)))
5246 #ifdef CONFIG_TLS_DEVICE
5247 if (skb
->decrypted
!= nskb
->decrypted
)
5254 skb_queue_walk_safe(&tmp
, skb
, n
)
5255 tcp_rbtree_insert(root
, skb
);
5258 /* Collapse ofo queue. Algorithm: select contiguous sequence of skbs
5259 * and tcp_collapse() them until all the queue is collapsed.
5261 static void tcp_collapse_ofo_queue(struct sock
*sk
)
5263 struct tcp_sock
*tp
= tcp_sk(sk
);
5264 u32 range_truesize
, sum_tiny
= 0;
5265 struct sk_buff
*skb
, *head
;
5268 skb
= skb_rb_first(&tp
->out_of_order_queue
);
5271 tp
->ooo_last_skb
= skb_rb_last(&tp
->out_of_order_queue
);
5274 start
= TCP_SKB_CB(skb
)->seq
;
5275 end
= TCP_SKB_CB(skb
)->end_seq
;
5276 range_truesize
= skb
->truesize
;
5278 for (head
= skb
;;) {
5279 skb
= skb_rb_next(skb
);
5281 /* Range is terminated when we see a gap or when
5282 * we are at the queue end.
5285 after(TCP_SKB_CB(skb
)->seq
, end
) ||
5286 before(TCP_SKB_CB(skb
)->end_seq
, start
)) {
5287 /* Do not attempt collapsing tiny skbs */
5288 if (range_truesize
!= head
->truesize
||
5289 end
- start
>= SKB_WITH_OVERHEAD(PAGE_SIZE
)) {
5290 tcp_collapse(sk
, NULL
, &tp
->out_of_order_queue
,
5291 head
, skb
, start
, end
);
5293 sum_tiny
+= range_truesize
;
5294 if (sum_tiny
> sk
->sk_rcvbuf
>> 3)
5300 range_truesize
+= skb
->truesize
;
5301 if (unlikely(before(TCP_SKB_CB(skb
)->seq
, start
)))
5302 start
= TCP_SKB_CB(skb
)->seq
;
5303 if (after(TCP_SKB_CB(skb
)->end_seq
, end
))
5304 end
= TCP_SKB_CB(skb
)->end_seq
;
5309 * Clean the out-of-order queue to make room.
5310 * We drop high sequences packets to :
5311 * 1) Let a chance for holes to be filled.
5312 * 2) not add too big latencies if thousands of packets sit there.
5313 * (But if application shrinks SO_RCVBUF, we could still end up
5314 * freeing whole queue here)
5315 * 3) Drop at least 12.5 % of sk_rcvbuf to avoid malicious attacks.
5317 * Return true if queue has shrunk.
5319 static bool tcp_prune_ofo_queue(struct sock
*sk
)
5321 struct tcp_sock
*tp
= tcp_sk(sk
);
5322 struct rb_node
*node
, *prev
;
5325 if (RB_EMPTY_ROOT(&tp
->out_of_order_queue
))
5328 NET_INC_STATS(sock_net(sk
), LINUX_MIB_OFOPRUNED
);
5329 goal
= sk
->sk_rcvbuf
>> 3;
5330 node
= &tp
->ooo_last_skb
->rbnode
;
5332 prev
= rb_prev(node
);
5333 rb_erase(node
, &tp
->out_of_order_queue
);
5334 goal
-= rb_to_skb(node
)->truesize
;
5335 tcp_drop_reason(sk
, rb_to_skb(node
),
5336 SKB_DROP_REASON_TCP_OFO_QUEUE_PRUNE
);
5337 if (!prev
|| goal
<= 0) {
5338 if (atomic_read(&sk
->sk_rmem_alloc
) <= sk
->sk_rcvbuf
&&
5339 !tcp_under_memory_pressure(sk
))
5341 goal
= sk
->sk_rcvbuf
>> 3;
5345 tp
->ooo_last_skb
= rb_to_skb(prev
);
5347 /* Reset SACK state. A conforming SACK implementation will
5348 * do the same at a timeout based retransmit. When a connection
5349 * is in a sad state like this, we care only about integrity
5350 * of the connection not performance.
5352 if (tp
->rx_opt
.sack_ok
)
5353 tcp_sack_reset(&tp
->rx_opt
);
5357 /* Reduce allocated memory if we can, trying to get
5358 * the socket within its memory limits again.
5360 * Return less than zero if we should start dropping frames
5361 * until the socket owning process reads some of the data
5362 * to stabilize the situation.
5364 static int tcp_prune_queue(struct sock
*sk
)
5366 struct tcp_sock
*tp
= tcp_sk(sk
);
5368 NET_INC_STATS(sock_net(sk
), LINUX_MIB_PRUNECALLED
);
5370 if (atomic_read(&sk
->sk_rmem_alloc
) >= sk
->sk_rcvbuf
)
5371 tcp_clamp_window(sk
);
5372 else if (tcp_under_memory_pressure(sk
))
5373 tcp_adjust_rcv_ssthresh(sk
);
5375 if (atomic_read(&sk
->sk_rmem_alloc
) <= sk
->sk_rcvbuf
)
5378 tcp_collapse_ofo_queue(sk
);
5379 if (!skb_queue_empty(&sk
->sk_receive_queue
))
5380 tcp_collapse(sk
, &sk
->sk_receive_queue
, NULL
,
5381 skb_peek(&sk
->sk_receive_queue
),
5383 tp
->copied_seq
, tp
->rcv_nxt
);
5385 if (atomic_read(&sk
->sk_rmem_alloc
) <= sk
->sk_rcvbuf
)
5388 /* Collapsing did not help, destructive actions follow.
5389 * This must not ever occur. */
5391 tcp_prune_ofo_queue(sk
);
5393 if (atomic_read(&sk
->sk_rmem_alloc
) <= sk
->sk_rcvbuf
)
5396 /* If we are really being abused, tell the caller to silently
5397 * drop receive data on the floor. It will get retransmitted
5398 * and hopefully then we'll have sufficient space.
5400 NET_INC_STATS(sock_net(sk
), LINUX_MIB_RCVPRUNED
);
5402 /* Massive buffer overcommit. */
5407 static bool tcp_should_expand_sndbuf(struct sock
*sk
)
5409 const struct tcp_sock
*tp
= tcp_sk(sk
);
5411 /* If the user specified a specific send buffer setting, do
5414 if (sk
->sk_userlocks
& SOCK_SNDBUF_LOCK
)
5417 /* If we are under global TCP memory pressure, do not expand. */
5418 if (tcp_under_memory_pressure(sk
)) {
5419 int unused_mem
= sk_unused_reserved_mem(sk
);
5421 /* Adjust sndbuf according to reserved mem. But make sure
5422 * it never goes below SOCK_MIN_SNDBUF.
5423 * See sk_stream_moderate_sndbuf() for more details.
5425 if (unused_mem
> SOCK_MIN_SNDBUF
)
5426 WRITE_ONCE(sk
->sk_sndbuf
, unused_mem
);
5431 /* If we are under soft global TCP memory pressure, do not expand. */
5432 if (sk_memory_allocated(sk
) >= sk_prot_mem_limits(sk
, 0))
5435 /* If we filled the congestion window, do not expand. */
5436 if (tcp_packets_in_flight(tp
) >= tcp_snd_cwnd(tp
))
5442 static void tcp_new_space(struct sock
*sk
)
5444 struct tcp_sock
*tp
= tcp_sk(sk
);
5446 if (tcp_should_expand_sndbuf(sk
)) {
5447 tcp_sndbuf_expand(sk
);
5448 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
5451 INDIRECT_CALL_1(sk
->sk_write_space
, sk_stream_write_space
, sk
);
5454 /* Caller made space either from:
5455 * 1) Freeing skbs in rtx queues (after tp->snd_una has advanced)
5456 * 2) Sent skbs from output queue (and thus advancing tp->snd_nxt)
5458 * We might be able to generate EPOLLOUT to the application if:
5459 * 1) Space consumed in output/rtx queues is below sk->sk_sndbuf/2
5460 * 2) notsent amount (tp->write_seq - tp->snd_nxt) became
5461 * small enough that tcp_stream_memory_free() decides it
5462 * is time to generate EPOLLOUT.
5464 void tcp_check_space(struct sock
*sk
)
5466 /* pairs with tcp_poll() */
5468 if (sk
->sk_socket
&&
5469 test_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
)) {
5471 if (!test_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
))
5472 tcp_chrono_stop(sk
, TCP_CHRONO_SNDBUF_LIMITED
);
5476 static inline void tcp_data_snd_check(struct sock
*sk
)
5478 tcp_push_pending_frames(sk
);
5479 tcp_check_space(sk
);
5483 * Check if sending an ack is needed.
5485 static void __tcp_ack_snd_check(struct sock
*sk
, int ofo_possible
)
5487 struct tcp_sock
*tp
= tcp_sk(sk
);
5488 unsigned long rtt
, delay
;
5490 /* More than one full frame received... */
5491 if (((tp
->rcv_nxt
- tp
->rcv_wup
) > inet_csk(sk
)->icsk_ack
.rcv_mss
&&
5492 /* ... and right edge of window advances far enough.
5493 * (tcp_recvmsg() will send ACK otherwise).
5494 * If application uses SO_RCVLOWAT, we want send ack now if
5495 * we have not received enough bytes to satisfy the condition.
5497 (tp
->rcv_nxt
- tp
->copied_seq
< sk
->sk_rcvlowat
||
5498 __tcp_select_window(sk
) >= tp
->rcv_wnd
)) ||
5499 /* We ACK each frame or... */
5500 tcp_in_quickack_mode(sk
) ||
5501 /* Protocol state mandates a one-time immediate ACK */
5502 inet_csk(sk
)->icsk_ack
.pending
& ICSK_ACK_NOW
) {
5508 if (!ofo_possible
|| RB_EMPTY_ROOT(&tp
->out_of_order_queue
)) {
5509 tcp_send_delayed_ack(sk
);
5513 if (!tcp_is_sack(tp
) ||
5514 tp
->compressed_ack
>= sock_net(sk
)->ipv4
.sysctl_tcp_comp_sack_nr
)
5517 if (tp
->compressed_ack_rcv_nxt
!= tp
->rcv_nxt
) {
5518 tp
->compressed_ack_rcv_nxt
= tp
->rcv_nxt
;
5519 tp
->dup_ack_counter
= 0;
5521 if (tp
->dup_ack_counter
< TCP_FASTRETRANS_THRESH
) {
5522 tp
->dup_ack_counter
++;
5525 tp
->compressed_ack
++;
5526 if (hrtimer_is_queued(&tp
->compressed_ack_timer
))
5529 /* compress ack timer : 5 % of rtt, but no more than tcp_comp_sack_delay_ns */
5531 rtt
= tp
->rcv_rtt_est
.rtt_us
;
5532 if (tp
->srtt_us
&& tp
->srtt_us
< rtt
)
5535 delay
= min_t(unsigned long, sock_net(sk
)->ipv4
.sysctl_tcp_comp_sack_delay_ns
,
5536 rtt
* (NSEC_PER_USEC
>> 3)/20);
5538 hrtimer_start_range_ns(&tp
->compressed_ack_timer
, ns_to_ktime(delay
),
5539 sock_net(sk
)->ipv4
.sysctl_tcp_comp_sack_slack_ns
,
5540 HRTIMER_MODE_REL_PINNED_SOFT
);
5543 static inline void tcp_ack_snd_check(struct sock
*sk
)
5545 if (!inet_csk_ack_scheduled(sk
)) {
5546 /* We sent a data segment already. */
5549 __tcp_ack_snd_check(sk
, 1);
5553 * This routine is only called when we have urgent data
5554 * signaled. Its the 'slow' part of tcp_urg. It could be
5555 * moved inline now as tcp_urg is only called from one
5556 * place. We handle URGent data wrong. We have to - as
5557 * BSD still doesn't use the correction from RFC961.
5558 * For 1003.1g we should support a new option TCP_STDURG to permit
5559 * either form (or just set the sysctl tcp_stdurg).
5562 static void tcp_check_urg(struct sock
*sk
, const struct tcphdr
*th
)
5564 struct tcp_sock
*tp
= tcp_sk(sk
);
5565 u32 ptr
= ntohs(th
->urg_ptr
);
5567 if (ptr
&& !sock_net(sk
)->ipv4
.sysctl_tcp_stdurg
)
5569 ptr
+= ntohl(th
->seq
);
5571 /* Ignore urgent data that we've already seen and read. */
5572 if (after(tp
->copied_seq
, ptr
))
5575 /* Do not replay urg ptr.
5577 * NOTE: interesting situation not covered by specs.
5578 * Misbehaving sender may send urg ptr, pointing to segment,
5579 * which we already have in ofo queue. We are not able to fetch
5580 * such data and will stay in TCP_URG_NOTYET until will be eaten
5581 * by recvmsg(). Seems, we are not obliged to handle such wicked
5582 * situations. But it is worth to think about possibility of some
5583 * DoSes using some hypothetical application level deadlock.
5585 if (before(ptr
, tp
->rcv_nxt
))
5588 /* Do we already have a newer (or duplicate) urgent pointer? */
5589 if (tp
->urg_data
&& !after(ptr
, tp
->urg_seq
))
5592 /* Tell the world about our new urgent pointer. */
5595 /* We may be adding urgent data when the last byte read was
5596 * urgent. To do this requires some care. We cannot just ignore
5597 * tp->copied_seq since we would read the last urgent byte again
5598 * as data, nor can we alter copied_seq until this data arrives
5599 * or we break the semantics of SIOCATMARK (and thus sockatmark())
5601 * NOTE. Double Dutch. Rendering to plain English: author of comment
5602 * above did something sort of send("A", MSG_OOB); send("B", MSG_OOB);
5603 * and expect that both A and B disappear from stream. This is _wrong_.
5604 * Though this happens in BSD with high probability, this is occasional.
5605 * Any application relying on this is buggy. Note also, that fix "works"
5606 * only in this artificial test. Insert some normal data between A and B and we will
5607 * decline of BSD again. Verdict: it is better to remove to trap
5610 if (tp
->urg_seq
== tp
->copied_seq
&& tp
->urg_data
&&
5611 !sock_flag(sk
, SOCK_URGINLINE
) && tp
->copied_seq
!= tp
->rcv_nxt
) {
5612 struct sk_buff
*skb
= skb_peek(&sk
->sk_receive_queue
);
5614 if (skb
&& !before(tp
->copied_seq
, TCP_SKB_CB(skb
)->end_seq
)) {
5615 __skb_unlink(skb
, &sk
->sk_receive_queue
);
5620 WRITE_ONCE(tp
->urg_data
, TCP_URG_NOTYET
);
5621 WRITE_ONCE(tp
->urg_seq
, ptr
);
5623 /* Disable header prediction. */
5627 /* This is the 'fast' part of urgent handling. */
5628 static void tcp_urg(struct sock
*sk
, struct sk_buff
*skb
, const struct tcphdr
*th
)
5630 struct tcp_sock
*tp
= tcp_sk(sk
);
5632 /* Check if we get a new urgent pointer - normally not. */
5633 if (unlikely(th
->urg
))
5634 tcp_check_urg(sk
, th
);
5636 /* Do we wait for any urgent data? - normally not... */
5637 if (unlikely(tp
->urg_data
== TCP_URG_NOTYET
)) {
5638 u32 ptr
= tp
->urg_seq
- ntohl(th
->seq
) + (th
->doff
* 4) -
5641 /* Is the urgent pointer pointing into this packet? */
5642 if (ptr
< skb
->len
) {
5644 if (skb_copy_bits(skb
, ptr
, &tmp
, 1))
5646 WRITE_ONCE(tp
->urg_data
, TCP_URG_VALID
| tmp
);
5647 if (!sock_flag(sk
, SOCK_DEAD
))
5648 sk
->sk_data_ready(sk
);
5653 /* Accept RST for rcv_nxt - 1 after a FIN.
5654 * When tcp connections are abruptly terminated from Mac OSX (via ^C), a
5655 * FIN is sent followed by a RST packet. The RST is sent with the same
5656 * sequence number as the FIN, and thus according to RFC 5961 a challenge
5657 * ACK should be sent. However, Mac OSX rate limits replies to challenge
5658 * ACKs on the closed socket. In addition middleboxes can drop either the
5659 * challenge ACK or a subsequent RST.
5661 static bool tcp_reset_check(const struct sock
*sk
, const struct sk_buff
*skb
)
5663 struct tcp_sock
*tp
= tcp_sk(sk
);
5665 return unlikely(TCP_SKB_CB(skb
)->seq
== (tp
->rcv_nxt
- 1) &&
5666 (1 << sk
->sk_state
) & (TCPF_CLOSE_WAIT
| TCPF_LAST_ACK
|
5670 /* Does PAWS and seqno based validation of an incoming segment, flags will
5671 * play significant role here.
5673 static bool tcp_validate_incoming(struct sock
*sk
, struct sk_buff
*skb
,
5674 const struct tcphdr
*th
, int syn_inerr
)
5676 struct tcp_sock
*tp
= tcp_sk(sk
);
5679 /* RFC1323: H1. Apply PAWS check first. */
5680 if (tcp_fast_parse_options(sock_net(sk
), skb
, th
, tp
) &&
5681 tp
->rx_opt
.saw_tstamp
&&
5682 tcp_paws_discard(sk
, skb
)) {
5684 NET_INC_STATS(sock_net(sk
), LINUX_MIB_PAWSESTABREJECTED
);
5685 if (!tcp_oow_rate_limited(sock_net(sk
), skb
,
5686 LINUX_MIB_TCPACKSKIPPEDPAWS
,
5687 &tp
->last_oow_ack_time
))
5688 tcp_send_dupack(sk
, skb
);
5689 SKB_DR_SET(reason
, TCP_RFC7323_PAWS
);
5692 /* Reset is accepted even if it did not pass PAWS. */
5695 /* Step 1: check sequence number */
5696 if (!tcp_sequence(tp
, TCP_SKB_CB(skb
)->seq
, TCP_SKB_CB(skb
)->end_seq
)) {
5697 /* RFC793, page 37: "In all states except SYN-SENT, all reset
5698 * (RST) segments are validated by checking their SEQ-fields."
5699 * And page 69: "If an incoming segment is not acceptable,
5700 * an acknowledgment should be sent in reply (unless the RST
5701 * bit is set, if so drop the segment and return)".
5706 if (!tcp_oow_rate_limited(sock_net(sk
), skb
,
5707 LINUX_MIB_TCPACKSKIPPEDSEQ
,
5708 &tp
->last_oow_ack_time
))
5709 tcp_send_dupack(sk
, skb
);
5710 } else if (tcp_reset_check(sk
, skb
)) {
5713 SKB_DR_SET(reason
, TCP_INVALID_SEQUENCE
);
5717 /* Step 2: check RST bit */
5719 /* RFC 5961 3.2 (extend to match against (RCV.NXT - 1) after a
5720 * FIN and SACK too if available):
5721 * If seq num matches RCV.NXT or (RCV.NXT - 1) after a FIN, or
5722 * the right-most SACK block,
5724 * RESET the connection
5726 * Send a challenge ACK
5728 if (TCP_SKB_CB(skb
)->seq
== tp
->rcv_nxt
||
5729 tcp_reset_check(sk
, skb
))
5732 if (tcp_is_sack(tp
) && tp
->rx_opt
.num_sacks
> 0) {
5733 struct tcp_sack_block
*sp
= &tp
->selective_acks
[0];
5734 int max_sack
= sp
[0].end_seq
;
5737 for (this_sack
= 1; this_sack
< tp
->rx_opt
.num_sacks
;
5739 max_sack
= after(sp
[this_sack
].end_seq
,
5741 sp
[this_sack
].end_seq
: max_sack
;
5744 if (TCP_SKB_CB(skb
)->seq
== max_sack
)
5748 /* Disable TFO if RST is out-of-order
5749 * and no data has been received
5750 * for current active TFO socket
5752 if (tp
->syn_fastopen
&& !tp
->data_segs_in
&&
5753 sk
->sk_state
== TCP_ESTABLISHED
)
5754 tcp_fastopen_active_disable(sk
);
5755 tcp_send_challenge_ack(sk
);
5756 SKB_DR_SET(reason
, TCP_RESET
);
5760 /* step 3: check security and precedence [ignored] */
5762 /* step 4: Check for a SYN
5763 * RFC 5961 4.2 : Send a challenge ack
5768 TCP_INC_STATS(sock_net(sk
), TCP_MIB_INERRS
);
5769 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPSYNCHALLENGE
);
5770 tcp_send_challenge_ack(sk
);
5771 SKB_DR_SET(reason
, TCP_INVALID_SYN
);
5775 bpf_skops_parse_hdr(sk
, skb
);
5780 tcp_drop_reason(sk
, skb
, reason
);
5790 * TCP receive function for the ESTABLISHED state.
5792 * It is split into a fast path and a slow path. The fast path is
5794 * - A zero window was announced from us - zero window probing
5795 * is only handled properly in the slow path.
5796 * - Out of order segments arrived.
5797 * - Urgent data is expected.
5798 * - There is no buffer space left
5799 * - Unexpected TCP flags/window values/header lengths are received
5800 * (detected by checking the TCP header against pred_flags)
5801 * - Data is sent in both directions. Fast path only supports pure senders
5802 * or pure receivers (this means either the sequence number or the ack
5803 * value must stay constant)
5804 * - Unexpected TCP option.
5806 * When these conditions are not satisfied it drops into a standard
5807 * receive procedure patterned after RFC793 to handle all cases.
5808 * The first three cases are guaranteed by proper pred_flags setting,
5809 * the rest is checked inline. Fast processing is turned on in
5810 * tcp_data_queue when everything is OK.
5812 void tcp_rcv_established(struct sock
*sk
, struct sk_buff
*skb
)
5814 enum skb_drop_reason reason
= SKB_DROP_REASON_NOT_SPECIFIED
;
5815 const struct tcphdr
*th
= (const struct tcphdr
*)skb
->data
;
5816 struct tcp_sock
*tp
= tcp_sk(sk
);
5817 unsigned int len
= skb
->len
;
5819 /* TCP congestion window tracking */
5820 trace_tcp_probe(sk
, skb
);
5822 tcp_mstamp_refresh(tp
);
5823 if (unlikely(!rcu_access_pointer(sk
->sk_rx_dst
)))
5824 inet_csk(sk
)->icsk_af_ops
->sk_rx_dst_set(sk
, skb
);
5826 * Header prediction.
5827 * The code loosely follows the one in the famous
5828 * "30 instruction TCP receive" Van Jacobson mail.
5830 * Van's trick is to deposit buffers into socket queue
5831 * on a device interrupt, to call tcp_recv function
5832 * on the receive process context and checksum and copy
5833 * the buffer to user space. smart...
5835 * Our current scheme is not silly either but we take the
5836 * extra cost of the net_bh soft interrupt processing...
5837 * We do checksum and copy also but from device to kernel.
5840 tp
->rx_opt
.saw_tstamp
= 0;
5842 /* pred_flags is 0xS?10 << 16 + snd_wnd
5843 * if header_prediction is to be made
5844 * 'S' will always be tp->tcp_header_len >> 2
5845 * '?' will be 0 for the fast path, otherwise pred_flags is 0 to
5846 * turn it off (when there are holes in the receive
5847 * space for instance)
5848 * PSH flag is ignored.
5851 if ((tcp_flag_word(th
) & TCP_HP_BITS
) == tp
->pred_flags
&&
5852 TCP_SKB_CB(skb
)->seq
== tp
->rcv_nxt
&&
5853 !after(TCP_SKB_CB(skb
)->ack_seq
, tp
->snd_nxt
)) {
5854 int tcp_header_len
= tp
->tcp_header_len
;
5856 /* Timestamp header prediction: tcp_header_len
5857 * is automatically equal to th->doff*4 due to pred_flags
5861 /* Check timestamp */
5862 if (tcp_header_len
== sizeof(struct tcphdr
) + TCPOLEN_TSTAMP_ALIGNED
) {
5863 /* No? Slow path! */
5864 if (!tcp_parse_aligned_timestamp(tp
, th
))
5867 /* If PAWS failed, check it more carefully in slow path */
5868 if ((s32
)(tp
->rx_opt
.rcv_tsval
- tp
->rx_opt
.ts_recent
) < 0)
5871 /* DO NOT update ts_recent here, if checksum fails
5872 * and timestamp was corrupted part, it will result
5873 * in a hung connection since we will drop all
5874 * future packets due to the PAWS test.
5878 if (len
<= tcp_header_len
) {
5879 /* Bulk data transfer: sender */
5880 if (len
== tcp_header_len
) {
5881 /* Predicted packet is in window by definition.
5882 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
5883 * Hence, check seq<=rcv_wup reduces to:
5885 if (tcp_header_len
==
5886 (sizeof(struct tcphdr
) + TCPOLEN_TSTAMP_ALIGNED
) &&
5887 tp
->rcv_nxt
== tp
->rcv_wup
)
5888 tcp_store_ts_recent(tp
);
5890 /* We know that such packets are checksummed
5893 tcp_ack(sk
, skb
, 0);
5895 tcp_data_snd_check(sk
);
5896 /* When receiving pure ack in fast path, update
5897 * last ts ecr directly instead of calling
5898 * tcp_rcv_rtt_measure_ts()
5900 tp
->rcv_rtt_last_tsecr
= tp
->rx_opt
.rcv_tsecr
;
5902 } else { /* Header too small */
5903 reason
= SKB_DROP_REASON_PKT_TOO_SMALL
;
5904 TCP_INC_STATS(sock_net(sk
), TCP_MIB_INERRS
);
5909 bool fragstolen
= false;
5911 if (tcp_checksum_complete(skb
))
5914 if ((int)skb
->truesize
> sk
->sk_forward_alloc
)
5917 /* Predicted packet is in window by definition.
5918 * seq == rcv_nxt and rcv_wup <= rcv_nxt.
5919 * Hence, check seq<=rcv_wup reduces to:
5921 if (tcp_header_len
==
5922 (sizeof(struct tcphdr
) + TCPOLEN_TSTAMP_ALIGNED
) &&
5923 tp
->rcv_nxt
== tp
->rcv_wup
)
5924 tcp_store_ts_recent(tp
);
5926 tcp_rcv_rtt_measure_ts(sk
, skb
);
5928 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPHPHITS
);
5930 /* Bulk data transfer: receiver */
5932 __skb_pull(skb
, tcp_header_len
);
5933 eaten
= tcp_queue_rcv(sk
, skb
, &fragstolen
);
5935 tcp_event_data_recv(sk
, skb
);
5937 if (TCP_SKB_CB(skb
)->ack_seq
!= tp
->snd_una
) {
5938 /* Well, only one small jumplet in fast path... */
5939 tcp_ack(sk
, skb
, FLAG_DATA
);
5940 tcp_data_snd_check(sk
);
5941 if (!inet_csk_ack_scheduled(sk
))
5944 tcp_update_wl(tp
, TCP_SKB_CB(skb
)->seq
);
5947 __tcp_ack_snd_check(sk
, 0);
5950 kfree_skb_partial(skb
, fragstolen
);
5957 if (len
< (th
->doff
<< 2) || tcp_checksum_complete(skb
))
5960 if (!th
->ack
&& !th
->rst
&& !th
->syn
) {
5961 reason
= SKB_DROP_REASON_TCP_FLAGS
;
5966 * Standard slow path.
5969 if (!tcp_validate_incoming(sk
, skb
, th
, 1))
5973 reason
= tcp_ack(sk
, skb
, FLAG_SLOWPATH
| FLAG_UPDATE_TS_RECENT
);
5974 if ((int)reason
< 0) {
5978 tcp_rcv_rtt_measure_ts(sk
, skb
);
5980 /* Process urgent data. */
5981 tcp_urg(sk
, skb
, th
);
5983 /* step 7: process the segment text */
5984 tcp_data_queue(sk
, skb
);
5986 tcp_data_snd_check(sk
);
5987 tcp_ack_snd_check(sk
);
5991 reason
= SKB_DROP_REASON_TCP_CSUM
;
5992 trace_tcp_bad_csum(skb
);
5993 TCP_INC_STATS(sock_net(sk
), TCP_MIB_CSUMERRORS
);
5994 TCP_INC_STATS(sock_net(sk
), TCP_MIB_INERRS
);
5997 tcp_drop_reason(sk
, skb
, reason
);
5999 EXPORT_SYMBOL(tcp_rcv_established
);
6001 void tcp_init_transfer(struct sock
*sk
, int bpf_op
, struct sk_buff
*skb
)
6003 struct inet_connection_sock
*icsk
= inet_csk(sk
);
6004 struct tcp_sock
*tp
= tcp_sk(sk
);
6007 icsk
->icsk_af_ops
->rebuild_header(sk
);
6008 tcp_init_metrics(sk
);
6010 /* Initialize the congestion window to start the transfer.
6011 * Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
6012 * retransmitted. In light of RFC6298 more aggressive 1sec
6013 * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
6014 * retransmission has occurred.
6016 if (tp
->total_retrans
> 1 && tp
->undo_marker
)
6017 tcp_snd_cwnd_set(tp
, 1);
6019 tcp_snd_cwnd_set(tp
, tcp_init_cwnd(tp
, __sk_dst_get(sk
)));
6020 tp
->snd_cwnd_stamp
= tcp_jiffies32
;
6022 bpf_skops_established(sk
, bpf_op
, skb
);
6023 /* Initialize congestion control unless BPF initialized it already: */
6024 if (!icsk
->icsk_ca_initialized
)
6025 tcp_init_congestion_control(sk
);
6026 tcp_init_buffer_space(sk
);
6029 void tcp_finish_connect(struct sock
*sk
, struct sk_buff
*skb
)
6031 struct tcp_sock
*tp
= tcp_sk(sk
);
6032 struct inet_connection_sock
*icsk
= inet_csk(sk
);
6034 tcp_set_state(sk
, TCP_ESTABLISHED
);
6035 icsk
->icsk_ack
.lrcvtime
= tcp_jiffies32
;
6038 icsk
->icsk_af_ops
->sk_rx_dst_set(sk
, skb
);
6039 security_inet_conn_established(sk
, skb
);
6040 sk_mark_napi_id(sk
, skb
);
6043 tcp_init_transfer(sk
, BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB
, skb
);
6045 /* Prevent spurious tcp_cwnd_restart() on first data
6048 tp
->lsndtime
= tcp_jiffies32
;
6050 if (sock_flag(sk
, SOCK_KEEPOPEN
))
6051 inet_csk_reset_keepalive_timer(sk
, keepalive_time_when(tp
));
6053 if (!tp
->rx_opt
.snd_wscale
)
6054 __tcp_fast_path_on(tp
, tp
->snd_wnd
);
6059 static bool tcp_rcv_fastopen_synack(struct sock
*sk
, struct sk_buff
*synack
,
6060 struct tcp_fastopen_cookie
*cookie
)
6062 struct tcp_sock
*tp
= tcp_sk(sk
);
6063 struct sk_buff
*data
= tp
->syn_data
? tcp_rtx_queue_head(sk
) : NULL
;
6064 u16 mss
= tp
->rx_opt
.mss_clamp
, try_exp
= 0;
6065 bool syn_drop
= false;
6067 if (mss
== tp
->rx_opt
.user_mss
) {
6068 struct tcp_options_received opt
;
6070 /* Get original SYNACK MSS value if user MSS sets mss_clamp */
6071 tcp_clear_options(&opt
);
6072 opt
.user_mss
= opt
.mss_clamp
= 0;
6073 tcp_parse_options(sock_net(sk
), synack
, &opt
, 0, NULL
);
6074 mss
= opt
.mss_clamp
;
6077 if (!tp
->syn_fastopen
) {
6078 /* Ignore an unsolicited cookie */
6080 } else if (tp
->total_retrans
) {
6081 /* SYN timed out and the SYN-ACK neither has a cookie nor
6082 * acknowledges data. Presumably the remote received only
6083 * the retransmitted (regular) SYNs: either the original
6084 * SYN-data or the corresponding SYN-ACK was dropped.
6086 syn_drop
= (cookie
->len
< 0 && data
);
6087 } else if (cookie
->len
< 0 && !tp
->syn_data
) {
6088 /* We requested a cookie but didn't get it. If we did not use
6089 * the (old) exp opt format then try so next time (try_exp=1).
6090 * Otherwise we go back to use the RFC7413 opt (try_exp=2).
6092 try_exp
= tp
->syn_fastopen_exp
? 2 : 1;
6095 tcp_fastopen_cache_set(sk
, mss
, cookie
, syn_drop
, try_exp
);
6097 if (data
) { /* Retransmit unacked data in SYN */
6098 if (tp
->total_retrans
)
6099 tp
->fastopen_client_fail
= TFO_SYN_RETRANSMITTED
;
6101 tp
->fastopen_client_fail
= TFO_DATA_NOT_ACKED
;
6102 skb_rbtree_walk_from(data
)
6103 tcp_mark_skb_lost(sk
, data
);
6104 tcp_xmit_retransmit_queue(sk
);
6105 NET_INC_STATS(sock_net(sk
),
6106 LINUX_MIB_TCPFASTOPENACTIVEFAIL
);
6109 tp
->syn_data_acked
= tp
->syn_data
;
6110 if (tp
->syn_data_acked
) {
6111 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPFASTOPENACTIVE
);
6112 /* SYN-data is counted as two separate packets in tcp_ack() */
6113 if (tp
->delivered
> 1)
6117 tcp_fastopen_add_skb(sk
, synack
);
6122 static void smc_check_reset_syn(struct tcp_sock
*tp
)
6124 #if IS_ENABLED(CONFIG_SMC)
6125 if (static_branch_unlikely(&tcp_have_smc
)) {
6126 if (tp
->syn_smc
&& !tp
->rx_opt
.smc_ok
)
6132 static void tcp_try_undo_spurious_syn(struct sock
*sk
)
6134 struct tcp_sock
*tp
= tcp_sk(sk
);
6137 /* undo_marker is set when SYN or SYNACK times out. The timeout is
6138 * spurious if the ACK's timestamp option echo value matches the
6139 * original SYN timestamp.
6141 syn_stamp
= tp
->retrans_stamp
;
6142 if (tp
->undo_marker
&& syn_stamp
&& tp
->rx_opt
.saw_tstamp
&&
6143 syn_stamp
== tp
->rx_opt
.rcv_tsecr
)
6144 tp
->undo_marker
= 0;
6147 static int tcp_rcv_synsent_state_process(struct sock
*sk
, struct sk_buff
*skb
,
6148 const struct tcphdr
*th
)
6150 struct inet_connection_sock
*icsk
= inet_csk(sk
);
6151 struct tcp_sock
*tp
= tcp_sk(sk
);
6152 struct tcp_fastopen_cookie foc
= { .len
= -1 };
6153 int saved_clamp
= tp
->rx_opt
.mss_clamp
;
6157 tcp_parse_options(sock_net(sk
), skb
, &tp
->rx_opt
, 0, &foc
);
6158 if (tp
->rx_opt
.saw_tstamp
&& tp
->rx_opt
.rcv_tsecr
)
6159 tp
->rx_opt
.rcv_tsecr
-= tp
->tsoffset
;
6163 * "If the state is SYN-SENT then
6164 * first check the ACK bit
6165 * If the ACK bit is set
6166 * If SEG.ACK =< ISS, or SEG.ACK > SND.NXT, send
6167 * a reset (unless the RST bit is set, if so drop
6168 * the segment and return)"
6170 if (!after(TCP_SKB_CB(skb
)->ack_seq
, tp
->snd_una
) ||
6171 after(TCP_SKB_CB(skb
)->ack_seq
, tp
->snd_nxt
)) {
6172 /* Previous FIN/ACK or RST/ACK might be ignored. */
6173 if (icsk
->icsk_retransmits
== 0)
6174 inet_csk_reset_xmit_timer(sk
,
6176 TCP_TIMEOUT_MIN
, TCP_RTO_MAX
);
6177 goto reset_and_undo
;
6180 if (tp
->rx_opt
.saw_tstamp
&& tp
->rx_opt
.rcv_tsecr
&&
6181 !between(tp
->rx_opt
.rcv_tsecr
, tp
->retrans_stamp
,
6182 tcp_time_stamp(tp
))) {
6183 NET_INC_STATS(sock_net(sk
),
6184 LINUX_MIB_PAWSACTIVEREJECTED
);
6185 goto reset_and_undo
;
6188 /* Now ACK is acceptable.
6190 * "If the RST bit is set
6191 * If the ACK was acceptable then signal the user "error:
6192 * connection reset", drop the segment, enter CLOSED state,
6193 * delete TCB, and return."
6204 * "fifth, if neither of the SYN or RST bits is set then
6205 * drop the segment and return."
6211 SKB_DR_SET(reason
, TCP_FLAGS
);
6212 goto discard_and_undo
;
6215 * "If the SYN bit is on ...
6216 * are acceptable then ...
6217 * (our SYN has been ACKed), change the connection
6218 * state to ESTABLISHED..."
6221 tcp_ecn_rcv_synack(tp
, th
);
6223 tcp_init_wl(tp
, TCP_SKB_CB(skb
)->seq
);
6224 tcp_try_undo_spurious_syn(sk
);
6225 tcp_ack(sk
, skb
, FLAG_SLOWPATH
);
6227 /* Ok.. it's good. Set up sequence numbers and
6228 * move to established.
6230 WRITE_ONCE(tp
->rcv_nxt
, TCP_SKB_CB(skb
)->seq
+ 1);
6231 tp
->rcv_wup
= TCP_SKB_CB(skb
)->seq
+ 1;
6233 /* RFC1323: The window in SYN & SYN/ACK segments is
6236 tp
->snd_wnd
= ntohs(th
->window
);
6238 if (!tp
->rx_opt
.wscale_ok
) {
6239 tp
->rx_opt
.snd_wscale
= tp
->rx_opt
.rcv_wscale
= 0;
6240 tp
->window_clamp
= min(tp
->window_clamp
, 65535U);
6243 if (tp
->rx_opt
.saw_tstamp
) {
6244 tp
->rx_opt
.tstamp_ok
= 1;
6245 tp
->tcp_header_len
=
6246 sizeof(struct tcphdr
) + TCPOLEN_TSTAMP_ALIGNED
;
6247 tp
->advmss
-= TCPOLEN_TSTAMP_ALIGNED
;
6248 tcp_store_ts_recent(tp
);
6250 tp
->tcp_header_len
= sizeof(struct tcphdr
);
6253 tcp_sync_mss(sk
, icsk
->icsk_pmtu_cookie
);
6254 tcp_initialize_rcv_mss(sk
);
6256 /* Remember, tcp_poll() does not lock socket!
6257 * Change state from SYN-SENT only after copied_seq
6258 * is initialized. */
6259 WRITE_ONCE(tp
->copied_seq
, tp
->rcv_nxt
);
6261 smc_check_reset_syn(tp
);
6265 tcp_finish_connect(sk
, skb
);
6267 fastopen_fail
= (tp
->syn_fastopen
|| tp
->syn_data
) &&
6268 tcp_rcv_fastopen_synack(sk
, skb
, &foc
);
6270 if (!sock_flag(sk
, SOCK_DEAD
)) {
6271 sk
->sk_state_change(sk
);
6272 sk_wake_async(sk
, SOCK_WAKE_IO
, POLL_OUT
);
6276 if (sk
->sk_write_pending
||
6277 icsk
->icsk_accept_queue
.rskq_defer_accept
||
6278 inet_csk_in_pingpong_mode(sk
)) {
6279 /* Save one ACK. Data will be ready after
6280 * several ticks, if write_pending is set.
6282 * It may be deleted, but with this feature tcpdumps
6283 * look so _wonderfully_ clever, that I was not able
6284 * to stand against the temptation 8) --ANK
6286 inet_csk_schedule_ack(sk
);
6287 tcp_enter_quickack_mode(sk
, TCP_MAX_QUICKACKS
);
6288 inet_csk_reset_xmit_timer(sk
, ICSK_TIME_DACK
,
6289 TCP_DELACK_MAX
, TCP_RTO_MAX
);
6296 /* No ACK in the segment */
6300 * "If the RST bit is set
6302 * Otherwise (no ACK) drop the segment and return."
6304 SKB_DR_SET(reason
, TCP_RESET
);
6305 goto discard_and_undo
;
6309 if (tp
->rx_opt
.ts_recent_stamp
&& tp
->rx_opt
.saw_tstamp
&&
6310 tcp_paws_reject(&tp
->rx_opt
, 0)) {
6311 SKB_DR_SET(reason
, TCP_RFC7323_PAWS
);
6312 goto discard_and_undo
;
6315 /* We see SYN without ACK. It is attempt of
6316 * simultaneous connect with crossed SYNs.
6317 * Particularly, it can be connect to self.
6319 tcp_set_state(sk
, TCP_SYN_RECV
);
6321 if (tp
->rx_opt
.saw_tstamp
) {
6322 tp
->rx_opt
.tstamp_ok
= 1;
6323 tcp_store_ts_recent(tp
);
6324 tp
->tcp_header_len
=
6325 sizeof(struct tcphdr
) + TCPOLEN_TSTAMP_ALIGNED
;
6327 tp
->tcp_header_len
= sizeof(struct tcphdr
);
6330 WRITE_ONCE(tp
->rcv_nxt
, TCP_SKB_CB(skb
)->seq
+ 1);
6331 WRITE_ONCE(tp
->copied_seq
, tp
->rcv_nxt
);
6332 tp
->rcv_wup
= TCP_SKB_CB(skb
)->seq
+ 1;
6334 /* RFC1323: The window in SYN & SYN/ACK segments is
6337 tp
->snd_wnd
= ntohs(th
->window
);
6338 tp
->snd_wl1
= TCP_SKB_CB(skb
)->seq
;
6339 tp
->max_window
= tp
->snd_wnd
;
6341 tcp_ecn_rcv_syn(tp
, th
);
6344 tcp_sync_mss(sk
, icsk
->icsk_pmtu_cookie
);
6345 tcp_initialize_rcv_mss(sk
);
6347 tcp_send_synack(sk
);
6349 /* Note, we could accept data and URG from this segment.
6350 * There are no obstacles to make this (except that we must
6351 * either change tcp_recvmsg() to prevent it from returning data
6352 * before 3WHS completes per RFC793, or employ TCP Fast Open).
6354 * However, if we ignore data in ACKless segments sometimes,
6355 * we have no reasons to accept it sometimes.
6356 * Also, seems the code doing it in step6 of tcp_rcv_state_process
6357 * is not flawless. So, discard packet for sanity.
6358 * Uncomment this return to process the data.
6365 /* "fifth, if neither of the SYN or RST bits is set then
6366 * drop the segment and return."
6370 tcp_clear_options(&tp
->rx_opt
);
6371 tp
->rx_opt
.mss_clamp
= saved_clamp
;
6372 tcp_drop_reason(sk
, skb
, reason
);
6376 tcp_clear_options(&tp
->rx_opt
);
6377 tp
->rx_opt
.mss_clamp
= saved_clamp
;
6381 static void tcp_rcv_synrecv_state_fastopen(struct sock
*sk
)
6383 struct request_sock
*req
;
6385 /* If we are still handling the SYNACK RTO, see if timestamp ECR allows
6386 * undo. If peer SACKs triggered fast recovery, we can't undo here.
6388 if (inet_csk(sk
)->icsk_ca_state
== TCP_CA_Loss
)
6389 tcp_try_undo_loss(sk
, false);
6391 /* Reset rtx states to prevent spurious retransmits_timed_out() */
6392 tcp_sk(sk
)->retrans_stamp
= 0;
6393 inet_csk(sk
)->icsk_retransmits
= 0;
6395 /* Once we leave TCP_SYN_RECV or TCP_FIN_WAIT_1,
6396 * we no longer need req so release it.
6398 req
= rcu_dereference_protected(tcp_sk(sk
)->fastopen_rsk
,
6399 lockdep_sock_is_held(sk
));
6400 reqsk_fastopen_remove(sk
, req
, false);
6402 /* Re-arm the timer because data may have been sent out.
6403 * This is similar to the regular data transmission case
6404 * when new data has just been ack'ed.
6406 * (TFO) - we could try to be more aggressive and
6407 * retransmitting any data sooner based on when they
6414 * This function implements the receiving procedure of RFC 793 for
6415 * all states except ESTABLISHED and TIME_WAIT.
6416 * It's called from both tcp_v4_rcv and tcp_v6_rcv and should be
6417 * address independent.
6420 int tcp_rcv_state_process(struct sock
*sk
, struct sk_buff
*skb
)
6422 struct tcp_sock
*tp
= tcp_sk(sk
);
6423 struct inet_connection_sock
*icsk
= inet_csk(sk
);
6424 const struct tcphdr
*th
= tcp_hdr(skb
);
6425 struct request_sock
*req
;
6430 switch (sk
->sk_state
) {
6432 SKB_DR_SET(reason
, TCP_CLOSE
);
6440 SKB_DR_SET(reason
, TCP_RESET
);
6445 SKB_DR_SET(reason
, TCP_FLAGS
);
6448 /* It is possible that we process SYN packets from backlog,
6449 * so we need to make sure to disable BH and RCU right there.
6453 acceptable
= icsk
->icsk_af_ops
->conn_request(sk
, skb
) >= 0;
6462 SKB_DR_SET(reason
, TCP_FLAGS
);
6466 tp
->rx_opt
.saw_tstamp
= 0;
6467 tcp_mstamp_refresh(tp
);
6468 queued
= tcp_rcv_synsent_state_process(sk
, skb
, th
);
6472 /* Do step6 onward by hand. */
6473 tcp_urg(sk
, skb
, th
);
6475 tcp_data_snd_check(sk
);
6479 tcp_mstamp_refresh(tp
);
6480 tp
->rx_opt
.saw_tstamp
= 0;
6481 req
= rcu_dereference_protected(tp
->fastopen_rsk
,
6482 lockdep_sock_is_held(sk
));
6486 WARN_ON_ONCE(sk
->sk_state
!= TCP_SYN_RECV
&&
6487 sk
->sk_state
!= TCP_FIN_WAIT1
);
6489 if (!tcp_check_req(sk
, skb
, req
, true, &req_stolen
)) {
6490 SKB_DR_SET(reason
, TCP_FASTOPEN
);
6495 if (!th
->ack
&& !th
->rst
&& !th
->syn
) {
6496 SKB_DR_SET(reason
, TCP_FLAGS
);
6499 if (!tcp_validate_incoming(sk
, skb
, th
, 0))
6502 /* step 5: check the ACK field */
6503 acceptable
= tcp_ack(sk
, skb
, FLAG_SLOWPATH
|
6504 FLAG_UPDATE_TS_RECENT
|
6505 FLAG_NO_CHALLENGE_ACK
) > 0;
6508 if (sk
->sk_state
== TCP_SYN_RECV
)
6509 return 1; /* send one RST */
6510 tcp_send_challenge_ack(sk
);
6511 SKB_DR_SET(reason
, TCP_OLD_ACK
);
6514 switch (sk
->sk_state
) {
6516 tp
->delivered
++; /* SYN-ACK delivery isn't tracked in tcp_ack */
6518 tcp_synack_rtt_meas(sk
, req
);
6521 tcp_rcv_synrecv_state_fastopen(sk
);
6523 tcp_try_undo_spurious_syn(sk
);
6524 tp
->retrans_stamp
= 0;
6525 tcp_init_transfer(sk
, BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB
,
6527 WRITE_ONCE(tp
->copied_seq
, tp
->rcv_nxt
);
6530 tcp_set_state(sk
, TCP_ESTABLISHED
);
6531 sk
->sk_state_change(sk
);
6533 /* Note, that this wakeup is only for marginal crossed SYN case.
6534 * Passively open sockets are not waked up, because
6535 * sk->sk_sleep == NULL and sk->sk_socket == NULL.
6538 sk_wake_async(sk
, SOCK_WAKE_IO
, POLL_OUT
);
6540 tp
->snd_una
= TCP_SKB_CB(skb
)->ack_seq
;
6541 tp
->snd_wnd
= ntohs(th
->window
) << tp
->rx_opt
.snd_wscale
;
6542 tcp_init_wl(tp
, TCP_SKB_CB(skb
)->seq
);
6544 if (tp
->rx_opt
.tstamp_ok
)
6545 tp
->advmss
-= TCPOLEN_TSTAMP_ALIGNED
;
6547 if (!inet_csk(sk
)->icsk_ca_ops
->cong_control
)
6548 tcp_update_pacing_rate(sk
);
6550 /* Prevent spurious tcp_cwnd_restart() on first data packet */
6551 tp
->lsndtime
= tcp_jiffies32
;
6553 tcp_initialize_rcv_mss(sk
);
6554 tcp_fast_path_on(tp
);
6557 case TCP_FIN_WAIT1
: {
6561 tcp_rcv_synrecv_state_fastopen(sk
);
6563 if (tp
->snd_una
!= tp
->write_seq
)
6566 tcp_set_state(sk
, TCP_FIN_WAIT2
);
6567 sk
->sk_shutdown
|= SEND_SHUTDOWN
;
6571 if (!sock_flag(sk
, SOCK_DEAD
)) {
6572 /* Wake up lingering close() */
6573 sk
->sk_state_change(sk
);
6577 if (tp
->linger2
< 0) {
6579 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPABORTONDATA
);
6582 if (TCP_SKB_CB(skb
)->end_seq
!= TCP_SKB_CB(skb
)->seq
&&
6583 after(TCP_SKB_CB(skb
)->end_seq
- th
->fin
, tp
->rcv_nxt
)) {
6584 /* Receive out of order FIN after close() */
6585 if (tp
->syn_fastopen
&& th
->fin
)
6586 tcp_fastopen_active_disable(sk
);
6588 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPABORTONDATA
);
6592 tmo
= tcp_fin_time(sk
);
6593 if (tmo
> TCP_TIMEWAIT_LEN
) {
6594 inet_csk_reset_keepalive_timer(sk
, tmo
- TCP_TIMEWAIT_LEN
);
6595 } else if (th
->fin
|| sock_owned_by_user(sk
)) {
6596 /* Bad case. We could lose such FIN otherwise.
6597 * It is not a big problem, but it looks confusing
6598 * and not so rare event. We still can lose it now,
6599 * if it spins in bh_lock_sock(), but it is really
6602 inet_csk_reset_keepalive_timer(sk
, tmo
);
6604 tcp_time_wait(sk
, TCP_FIN_WAIT2
, tmo
);
6611 if (tp
->snd_una
== tp
->write_seq
) {
6612 tcp_time_wait(sk
, TCP_TIME_WAIT
, 0);
6618 if (tp
->snd_una
== tp
->write_seq
) {
6619 tcp_update_metrics(sk
);
6626 /* step 6: check the URG bit */
6627 tcp_urg(sk
, skb
, th
);
6629 /* step 7: process the segment text */
6630 switch (sk
->sk_state
) {
6631 case TCP_CLOSE_WAIT
:
6634 if (!before(TCP_SKB_CB(skb
)->seq
, tp
->rcv_nxt
)) {
6635 /* If a subflow has been reset, the packet should not
6636 * continue to be processed, drop the packet.
6638 if (sk_is_mptcp(sk
) && !mptcp_incoming_options(sk
, skb
))
6645 /* RFC 793 says to queue data in these states,
6646 * RFC 1122 says we MUST send a reset.
6647 * BSD 4.4 also does reset.
6649 if (sk
->sk_shutdown
& RCV_SHUTDOWN
) {
6650 if (TCP_SKB_CB(skb
)->end_seq
!= TCP_SKB_CB(skb
)->seq
&&
6651 after(TCP_SKB_CB(skb
)->end_seq
- th
->fin
, tp
->rcv_nxt
)) {
6652 NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPABORTONDATA
);
6658 case TCP_ESTABLISHED
:
6659 tcp_data_queue(sk
, skb
);
6664 /* tcp_data could move socket to TIME-WAIT */
6665 if (sk
->sk_state
!= TCP_CLOSE
) {
6666 tcp_data_snd_check(sk
);
6667 tcp_ack_snd_check(sk
);
6672 tcp_drop_reason(sk
, skb
, reason
);
6680 EXPORT_SYMBOL(tcp_rcv_state_process
);
6682 static inline void pr_drop_req(struct request_sock
*req
, __u16 port
, int family
)
6684 struct inet_request_sock
*ireq
= inet_rsk(req
);
6686 if (family
== AF_INET
)
6687 net_dbg_ratelimited("drop open request from %pI4/%u\n",
6688 &ireq
->ir_rmt_addr
, port
);
6689 #if IS_ENABLED(CONFIG_IPV6)
6690 else if (family
== AF_INET6
)
6691 net_dbg_ratelimited("drop open request from %pI6/%u\n",
6692 &ireq
->ir_v6_rmt_addr
, port
);
6696 /* RFC3168 : 6.1.1 SYN packets must not have ECT/ECN bits set
6698 * If we receive a SYN packet with these bits set, it means a
6699 * network is playing bad games with TOS bits. In order to
6700 * avoid possible false congestion notifications, we disable
6701 * TCP ECN negotiation.
6703 * Exception: tcp_ca wants ECN. This is required for DCTCP
6704 * congestion control: Linux DCTCP asserts ECT on all packets,
6705 * including SYN, which is most optimal solution; however,
6706 * others, such as FreeBSD do not.
6708 * Exception: At least one of the reserved bits of the TCP header (th->res1) is
6709 * set, indicating the use of a future TCP extension (such as AccECN). See
6710 * RFC8311 §4.3 which updates RFC3168 to allow the development of such
6713 static void tcp_ecn_create_request(struct request_sock
*req
,
6714 const struct sk_buff
*skb
,
6715 const struct sock
*listen_sk
,
6716 const struct dst_entry
*dst
)
6718 const struct tcphdr
*th
= tcp_hdr(skb
);
6719 const struct net
*net
= sock_net(listen_sk
);
6720 bool th_ecn
= th
->ece
&& th
->cwr
;
6727 ect
= !INET_ECN_is_not_ect(TCP_SKB_CB(skb
)->ip_dsfield
);
6728 ecn_ok_dst
= dst_feature(dst
, DST_FEATURE_ECN_MASK
);
6729 ecn_ok
= net
->ipv4
.sysctl_tcp_ecn
|| ecn_ok_dst
;
6731 if (((!ect
|| th
->res1
) && ecn_ok
) || tcp_ca_needs_ecn(listen_sk
) ||
6732 (ecn_ok_dst
& DST_FEATURE_ECN_CA
) ||
6733 tcp_bpf_ca_needs_ecn((struct sock
*)req
))
6734 inet_rsk(req
)->ecn_ok
= 1;
6737 static void tcp_openreq_init(struct request_sock
*req
,
6738 const struct tcp_options_received
*rx_opt
,
6739 struct sk_buff
*skb
, const struct sock
*sk
)
6741 struct inet_request_sock
*ireq
= inet_rsk(req
);
6743 req
->rsk_rcv_wnd
= 0; /* So that tcp_send_synack() knows! */
6744 tcp_rsk(req
)->rcv_isn
= TCP_SKB_CB(skb
)->seq
;
6745 tcp_rsk(req
)->rcv_nxt
= TCP_SKB_CB(skb
)->seq
+ 1;
6746 tcp_rsk(req
)->snt_synack
= 0;
6747 tcp_rsk(req
)->last_oow_ack_time
= 0;
6748 req
->mss
= rx_opt
->mss_clamp
;
6749 req
->ts_recent
= rx_opt
->saw_tstamp
? rx_opt
->rcv_tsval
: 0;
6750 ireq
->tstamp_ok
= rx_opt
->tstamp_ok
;
6751 ireq
->sack_ok
= rx_opt
->sack_ok
;
6752 ireq
->snd_wscale
= rx_opt
->snd_wscale
;
6753 ireq
->wscale_ok
= rx_opt
->wscale_ok
;
6756 ireq
->ir_rmt_port
= tcp_hdr(skb
)->source
;
6757 ireq
->ir_num
= ntohs(tcp_hdr(skb
)->dest
);
6758 ireq
->ir_mark
= inet_request_mark(sk
, skb
);
6759 #if IS_ENABLED(CONFIG_SMC)
6760 ireq
->smc_ok
= rx_opt
->smc_ok
&& !(tcp_sk(sk
)->smc_hs_congested
&&
6761 tcp_sk(sk
)->smc_hs_congested(sk
));
6765 struct request_sock
*inet_reqsk_alloc(const struct request_sock_ops
*ops
,
6766 struct sock
*sk_listener
,
6767 bool attach_listener
)
6769 struct request_sock
*req
= reqsk_alloc(ops
, sk_listener
,
6773 struct inet_request_sock
*ireq
= inet_rsk(req
);
6775 ireq
->ireq_opt
= NULL
;
6776 #if IS_ENABLED(CONFIG_IPV6)
6777 ireq
->pktopts
= NULL
;
6779 atomic64_set(&ireq
->ir_cookie
, 0);
6780 ireq
->ireq_state
= TCP_NEW_SYN_RECV
;
6781 write_pnet(&ireq
->ireq_net
, sock_net(sk_listener
));
6782 ireq
->ireq_family
= sk_listener
->sk_family
;
6783 req
->timeout
= TCP_TIMEOUT_INIT
;
6788 EXPORT_SYMBOL(inet_reqsk_alloc
);
6791 * Return true if a syncookie should be sent
6793 static bool tcp_syn_flood_action(const struct sock
*sk
, const char *proto
)
6795 struct request_sock_queue
*queue
= &inet_csk(sk
)->icsk_accept_queue
;
6796 const char *msg
= "Dropping request";
6797 bool want_cookie
= false;
6798 struct net
*net
= sock_net(sk
);
6800 #ifdef CONFIG_SYN_COOKIES
6801 if (net
->ipv4
.sysctl_tcp_syncookies
) {
6802 msg
= "Sending cookies";
6804 __NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPREQQFULLDOCOOKIES
);
6807 __NET_INC_STATS(sock_net(sk
), LINUX_MIB_TCPREQQFULLDROP
);
6809 if (!queue
->synflood_warned
&&
6810 net
->ipv4
.sysctl_tcp_syncookies
!= 2 &&
6811 xchg(&queue
->synflood_warned
, 1) == 0)
6812 net_info_ratelimited("%s: Possible SYN flooding on port %d. %s. Check SNMP counters.\n",
6813 proto
, sk
->sk_num
, msg
);
6818 static void tcp_reqsk_record_syn(const struct sock
*sk
,
6819 struct request_sock
*req
,
6820 const struct sk_buff
*skb
)
6822 if (tcp_sk(sk
)->save_syn
) {
6823 u32 len
= skb_network_header_len(skb
) + tcp_hdrlen(skb
);
6824 struct saved_syn
*saved_syn
;
6828 if (tcp_sk(sk
)->save_syn
== 2) { /* Save full header. */
6829 base
= skb_mac_header(skb
);
6830 mac_hdrlen
= skb_mac_header_len(skb
);
6833 base
= skb_network_header(skb
);
6837 saved_syn
= kmalloc(struct_size(saved_syn
, data
, len
),
6840 saved_syn
->mac_hdrlen
= mac_hdrlen
;
6841 saved_syn
->network_hdrlen
= skb_network_header_len(skb
);
6842 saved_syn
->tcp_hdrlen
= tcp_hdrlen(skb
);
6843 memcpy(saved_syn
->data
, base
, len
);
6844 req
->saved_syn
= saved_syn
;
6849 /* If a SYN cookie is required and supported, returns a clamped MSS value to be
6850 * used for SYN cookie generation.
6852 u16
tcp_get_syncookie_mss(struct request_sock_ops
*rsk_ops
,
6853 const struct tcp_request_sock_ops
*af_ops
,
6854 struct sock
*sk
, struct tcphdr
*th
)
6856 struct tcp_sock
*tp
= tcp_sk(sk
);
6859 if (sock_net(sk
)->ipv4
.sysctl_tcp_syncookies
!= 2 &&
6860 !inet_csk_reqsk_queue_is_full(sk
))
6863 if (!tcp_syn_flood_action(sk
, rsk_ops
->slab_name
))
6866 if (sk_acceptq_is_full(sk
)) {
6867 NET_INC_STATS(sock_net(sk
), LINUX_MIB_LISTENOVERFLOWS
);
6871 mss
= tcp_parse_mss_option(th
, tp
->rx_opt
.user_mss
);
6873 mss
= af_ops
->mss_clamp
;
6877 EXPORT_SYMBOL_GPL(tcp_get_syncookie_mss
);
6879 int tcp_conn_request(struct request_sock_ops
*rsk_ops
,
6880 const struct tcp_request_sock_ops
*af_ops
,
6881 struct sock
*sk
, struct sk_buff
*skb
)
6883 struct tcp_fastopen_cookie foc
= { .len
= -1 };
6884 __u32 isn
= TCP_SKB_CB(skb
)->tcp_tw_isn
;
6885 struct tcp_options_received tmp_opt
;
6886 struct tcp_sock
*tp
= tcp_sk(sk
);
6887 struct net
*net
= sock_net(sk
);
6888 struct sock
*fastopen_sk
= NULL
;
6889 struct request_sock
*req
;
6890 bool want_cookie
= false;
6891 struct dst_entry
*dst
;
6894 /* TW buckets are converted to open requests without
6895 * limitations, they conserve resources and peer is
6896 * evidently real one.
6898 if ((net
->ipv4
.sysctl_tcp_syncookies
== 2 ||
6899 inet_csk_reqsk_queue_is_full(sk
)) && !isn
) {
6900 want_cookie
= tcp_syn_flood_action(sk
, rsk_ops
->slab_name
);
6905 if (sk_acceptq_is_full(sk
)) {
6906 NET_INC_STATS(sock_net(sk
), LINUX_MIB_LISTENOVERFLOWS
);
6910 req
= inet_reqsk_alloc(rsk_ops
, sk
, !want_cookie
);
6914 req
->syncookie
= want_cookie
;
6915 tcp_rsk(req
)->af_specific
= af_ops
;
6916 tcp_rsk(req
)->ts_off
= 0;
6917 #if IS_ENABLED(CONFIG_MPTCP)
6918 tcp_rsk(req
)->is_mptcp
= 0;
6921 tcp_clear_options(&tmp_opt
);
6922 tmp_opt
.mss_clamp
= af_ops
->mss_clamp
;
6923 tmp_opt
.user_mss
= tp
->rx_opt
.user_mss
;
6924 tcp_parse_options(sock_net(sk
), skb
, &tmp_opt
, 0,
6925 want_cookie
? NULL
: &foc
);
6927 if (want_cookie
&& !tmp_opt
.saw_tstamp
)
6928 tcp_clear_options(&tmp_opt
);
6930 if (IS_ENABLED(CONFIG_SMC
) && want_cookie
)
6933 tmp_opt
.tstamp_ok
= tmp_opt
.saw_tstamp
;
6934 tcp_openreq_init(req
, &tmp_opt
, skb
, sk
);
6935 inet_rsk(req
)->no_srccheck
= inet_sk(sk
)->transparent
;
6937 /* Note: tcp_v6_init_req() might override ir_iif for link locals */
6938 inet_rsk(req
)->ir_iif
= inet_request_bound_dev_if(sk
, skb
);
6940 dst
= af_ops
->route_req(sk
, skb
, &fl
, req
);
6944 if (tmp_opt
.tstamp_ok
)
6945 tcp_rsk(req
)->ts_off
= af_ops
->init_ts_off(net
, skb
);
6947 if (!want_cookie
&& !isn
) {
6948 /* Kill the following clause, if you dislike this way. */
6949 if (!net
->ipv4
.sysctl_tcp_syncookies
&&
6950 (net
->ipv4
.sysctl_max_syn_backlog
- inet_csk_reqsk_queue_len(sk
) <
6951 (net
->ipv4
.sysctl_max_syn_backlog
>> 2)) &&
6952 !tcp_peer_is_proven(req
, dst
)) {
6953 /* Without syncookies last quarter of
6954 * backlog is filled with destinations,
6955 * proven to be alive.
6956 * It means that we continue to communicate
6957 * to destinations, already remembered
6958 * to the moment of synflood.
6960 pr_drop_req(req
, ntohs(tcp_hdr(skb
)->source
),
6962 goto drop_and_release
;
6965 isn
= af_ops
->init_seq(skb
);
6968 tcp_ecn_create_request(req
, skb
, sk
, dst
);
6971 isn
= cookie_init_sequence(af_ops
, sk
, skb
, &req
->mss
);
6972 if (!tmp_opt
.tstamp_ok
)
6973 inet_rsk(req
)->ecn_ok
= 0;
6976 tcp_rsk(req
)->snt_isn
= isn
;
6977 tcp_rsk(req
)->txhash
= net_tx_rndhash();
6978 tcp_rsk(req
)->syn_tos
= TCP_SKB_CB(skb
)->ip_dsfield
;
6979 tcp_openreq_init_rwin(req
, sk
, dst
);
6980 sk_rx_queue_set(req_to_sk(req
), skb
);
6982 tcp_reqsk_record_syn(sk
, req
, skb
);
6983 fastopen_sk
= tcp_try_fastopen(sk
, skb
, req
, &foc
, dst
);
6986 af_ops
->send_synack(fastopen_sk
, dst
, &fl
, req
,
6987 &foc
, TCP_SYNACK_FASTOPEN
, skb
);
6988 /* Add the child socket directly into the accept queue */
6989 if (!inet_csk_reqsk_queue_add(sk
, req
, fastopen_sk
)) {
6990 reqsk_fastopen_remove(fastopen_sk
, req
, false);
6991 bh_unlock_sock(fastopen_sk
);
6992 sock_put(fastopen_sk
);
6995 sk
->sk_data_ready(sk
);
6996 bh_unlock_sock(fastopen_sk
);
6997 sock_put(fastopen_sk
);
6999 tcp_rsk(req
)->tfo_listener
= false;
7001 req
->timeout
= tcp_timeout_init((struct sock
*)req
);
7002 inet_csk_reqsk_queue_hash_add(sk
, req
, req
->timeout
);
7004 af_ops
->send_synack(sk
, dst
, &fl
, req
, &foc
,
7005 !want_cookie
? TCP_SYNACK_NORMAL
:
7024 EXPORT_SYMBOL(tcp_conn_request
);