2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
10 * TCPv4 GSO/GRO support
13 #include <linux/skbuff.h>
15 #include <net/protocol.h>
17 struct sk_buff
*tcp_gso_segment(struct sk_buff
*skb
,
18 netdev_features_t features
)
20 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
27 struct sk_buff
*gso_skb
= skb
;
29 bool ooo_okay
, copy_destructor
;
31 if (!pskb_may_pull(skb
, sizeof(*th
)))
36 if (thlen
< sizeof(*th
))
39 if (!pskb_may_pull(skb
, thlen
))
42 oldlen
= (u16
)~skb
->len
;
43 __skb_pull(skb
, thlen
);
45 mss
= tcp_skb_mss(skb
);
46 if (unlikely(skb
->len
<= mss
))
49 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
50 /* Packet is from an untrusted source, reset gso_segs. */
51 int type
= skb_shinfo(skb
)->gso_type
;
64 !(type
& (SKB_GSO_TCPV4
| SKB_GSO_TCPV6
))))
67 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
73 copy_destructor
= gso_skb
->destructor
== tcp_wfree
;
74 ooo_okay
= gso_skb
->ooo_okay
;
75 /* All segments but the first should have ooo_okay cleared */
78 segs
= skb_segment(skb
, features
);
82 /* Only first segment might have ooo_okay set */
83 segs
->ooo_okay
= ooo_okay
;
85 delta
= htonl(oldlen
+ (thlen
+ mss
));
91 newcheck
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
95 th
->fin
= th
->psh
= 0;
98 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
100 csum_fold(csum_partial(skb_transport_header(skb
),
104 if (copy_destructor
) {
105 skb
->destructor
= gso_skb
->destructor
;
106 skb
->sk
= gso_skb
->sk
;
107 /* {tcp|sock}_wfree() use exact truesize accounting :
108 * sum(skb->truesize) MUST be exactly be gso_skb->truesize
109 * So we account mss bytes of 'true size' for each segment.
110 * The last segment will contain the remaining.
113 gso_skb
->truesize
-= mss
;
118 th
->seq
= htonl(seq
);
122 /* Following permits TCP Small Queues to work well with GSO :
123 * The callback to TCP stack will be called at the time last frag
124 * is freed at TX completion, and not right now when gso_skb
125 * is freed by GSO engine
127 if (copy_destructor
) {
128 swap(gso_skb
->sk
, skb
->sk
);
129 swap(gso_skb
->destructor
, skb
->destructor
);
130 swap(gso_skb
->truesize
, skb
->truesize
);
133 delta
= htonl(oldlen
+ (skb_tail_pointer(skb
) -
134 skb_transport_header(skb
)) +
136 th
->check
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
137 (__force u32
)delta
));
138 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
139 th
->check
= csum_fold(csum_partial(skb_transport_header(skb
),
144 EXPORT_SYMBOL(tcp_gso_segment
);
146 struct sk_buff
**tcp_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
148 struct sk_buff
**pp
= NULL
;
155 unsigned int mss
= 1;
161 off
= skb_gro_offset(skb
);
162 hlen
= off
+ sizeof(*th
);
163 th
= skb_gro_header_fast(skb
, off
);
164 if (skb_gro_header_hard(skb
, hlen
)) {
165 th
= skb_gro_header_slow(skb
, hlen
, off
);
170 thlen
= th
->doff
* 4;
171 if (thlen
< sizeof(*th
))
175 if (skb_gro_header_hard(skb
, hlen
)) {
176 th
= skb_gro_header_slow(skb
, hlen
, off
);
181 skb_gro_pull(skb
, thlen
);
183 len
= skb_gro_len(skb
);
184 flags
= tcp_flag_word(th
);
186 for (; (p
= *head
); head
= &p
->next
) {
187 if (!NAPI_GRO_CB(p
)->same_flow
)
192 if (*(u32
*)&th
->source
^ *(u32
*)&th2
->source
) {
193 NAPI_GRO_CB(p
)->same_flow
= 0;
200 goto out_check_final
;
203 flush
= NAPI_GRO_CB(p
)->flush
;
204 flush
|= (__force
int)(flags
& TCP_FLAG_CWR
);
205 flush
|= (__force
int)((flags
^ tcp_flag_word(th2
)) &
206 ~(TCP_FLAG_CWR
| TCP_FLAG_FIN
| TCP_FLAG_PSH
));
207 flush
|= (__force
int)(th
->ack_seq
^ th2
->ack_seq
);
208 for (i
= sizeof(*th
); i
< thlen
; i
+= 4)
209 flush
|= *(u32
*)((u8
*)th
+ i
) ^
210 *(u32
*)((u8
*)th2
+ i
);
212 mss
= tcp_skb_mss(p
);
214 flush
|= (len
- 1) >= mss
;
215 flush
|= (ntohl(th2
->seq
) + skb_gro_len(p
)) ^ ntohl(th
->seq
);
217 if (flush
|| skb_gro_receive(head
, skb
)) {
219 goto out_check_final
;
224 tcp_flag_word(th2
) |= flags
& (TCP_FLAG_FIN
| TCP_FLAG_PSH
);
228 flush
|= (__force
int)(flags
& (TCP_FLAG_URG
| TCP_FLAG_PSH
|
229 TCP_FLAG_RST
| TCP_FLAG_SYN
|
232 if (p
&& (!NAPI_GRO_CB(skb
)->same_flow
|| flush
))
236 NAPI_GRO_CB(skb
)->flush
|= flush
;
240 EXPORT_SYMBOL(tcp_gro_receive
);
242 int tcp_gro_complete(struct sk_buff
*skb
)
244 struct tcphdr
*th
= tcp_hdr(skb
);
246 skb
->csum_start
= skb_transport_header(skb
) - skb
->head
;
247 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
248 skb
->ip_summed
= CHECKSUM_PARTIAL
;
250 skb_shinfo(skb
)->gso_segs
= NAPI_GRO_CB(skb
)->count
;
253 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_ECN
;
257 EXPORT_SYMBOL(tcp_gro_complete
);
259 static int tcp_v4_gso_send_check(struct sk_buff
*skb
)
261 const struct iphdr
*iph
;
264 if (!pskb_may_pull(skb
, sizeof(*th
)))
271 skb
->ip_summed
= CHECKSUM_PARTIAL
;
272 __tcp_v4_send_check(skb
, iph
->saddr
, iph
->daddr
);
276 static struct sk_buff
**tcp4_gro_receive(struct sk_buff
**head
, struct sk_buff
*skb
)
278 const struct iphdr
*iph
= skb_gro_network_header(skb
);
282 switch (skb
->ip_summed
) {
283 case CHECKSUM_COMPLETE
:
284 if (!tcp_v4_check(skb_gro_len(skb
), iph
->saddr
, iph
->daddr
,
286 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
290 NAPI_GRO_CB(skb
)->flush
= 1;
294 wsum
= csum_tcpudp_nofold(iph
->saddr
, iph
->daddr
,
295 skb_gro_len(skb
), IPPROTO_TCP
, 0);
296 sum
= csum_fold(skb_checksum(skb
,
303 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
307 return tcp_gro_receive(head
, skb
);
310 static int tcp4_gro_complete(struct sk_buff
*skb
)
312 const struct iphdr
*iph
= ip_hdr(skb
);
313 struct tcphdr
*th
= tcp_hdr(skb
);
315 th
->check
= ~tcp_v4_check(skb
->len
- skb_transport_offset(skb
),
316 iph
->saddr
, iph
->daddr
, 0);
317 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
319 return tcp_gro_complete(skb
);
322 static const struct net_offload tcpv4_offload
= {
324 .gso_send_check
= tcp_v4_gso_send_check
,
325 .gso_segment
= tcp_gso_segment
,
326 .gro_receive
= tcp4_gro_receive
,
327 .gro_complete
= tcp4_gro_complete
,
331 int __init
tcpv4_offload_init(void)
333 return inet_add_offload(&tcpv4_offload
, IPPROTO_TCP
);