2 * IPV6 GSO/GRO offload support
3 * Linux INET6 implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
11 #include <linux/kernel.h>
12 #include <linux/socket.h>
13 #include <linux/netdevice.h>
14 #include <linux/skbuff.h>
15 #include <linux/printk.h>
17 #include <net/protocol.h>
19 #include <net/inet_common.h>
21 #include "ip6_offload.h"
23 static int ipv6_gso_pull_exthdrs(struct sk_buff
*skb
, int proto
)
25 const struct net_offload
*ops
= NULL
;
28 struct ipv6_opt_hdr
*opth
;
31 if (proto
!= NEXTHDR_HOP
) {
32 ops
= rcu_dereference(inet6_offloads
[proto
]);
37 if (!(ops
->flags
& INET6_PROTO_GSO_EXTHDR
))
41 if (unlikely(!pskb_may_pull(skb
, 8)))
44 opth
= (void *)skb
->data
;
45 len
= ipv6_optlen(opth
);
47 if (unlikely(!pskb_may_pull(skb
, len
)))
50 opth
= (void *)skb
->data
;
51 proto
= opth
->nexthdr
;
58 static struct sk_buff
*ipv6_gso_segment(struct sk_buff
*skb
,
59 netdev_features_t features
)
61 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
62 struct ipv6hdr
*ipv6h
;
63 const struct net_offload
*ops
;
65 struct frag_hdr
*fptr
;
66 unsigned int payload_len
;
73 skb_reset_network_header(skb
);
74 nhoff
= skb_network_header(skb
) - skb_mac_header(skb
);
75 if (unlikely(!pskb_may_pull(skb
, sizeof(*ipv6h
))))
78 encap
= SKB_GSO_CB(skb
)->encap_level
> 0;
80 features
&= skb
->dev
->hw_enc_features
;
81 SKB_GSO_CB(skb
)->encap_level
+= sizeof(*ipv6h
);
83 ipv6h
= ipv6_hdr(skb
);
84 __skb_pull(skb
, sizeof(*ipv6h
));
85 segs
= ERR_PTR(-EPROTONOSUPPORT
);
87 proto
= ipv6_gso_pull_exthdrs(skb
, ipv6h
->nexthdr
);
89 if (skb
->encapsulation
&&
90 skb_shinfo(skb
)->gso_type
& (SKB_GSO_IPXIP4
| SKB_GSO_IPXIP6
))
91 udpfrag
= proto
== IPPROTO_UDP
&& encap
;
93 udpfrag
= proto
== IPPROTO_UDP
&& !skb
->encapsulation
;
95 ops
= rcu_dereference(inet6_offloads
[proto
]);
96 if (likely(ops
&& ops
->callbacks
.gso_segment
)) {
97 skb_reset_transport_header(skb
);
98 segs
= ops
->callbacks
.gso_segment(skb
, features
);
101 if (IS_ERR_OR_NULL(segs
))
104 gso_partial
= !!(skb_shinfo(segs
)->gso_type
& SKB_GSO_PARTIAL
);
106 for (skb
= segs
; skb
; skb
= skb
->next
) {
107 ipv6h
= (struct ipv6hdr
*)(skb_mac_header(skb
) + nhoff
);
109 payload_len
= skb_shinfo(skb
)->gso_size
+
110 SKB_GSO_CB(skb
)->data_offset
+
111 skb
->head
- (unsigned char *)(ipv6h
+ 1);
113 payload_len
= skb
->len
- nhoff
- sizeof(*ipv6h
);
114 ipv6h
->payload_len
= htons(payload_len
);
115 skb
->network_header
= (u8
*)ipv6h
- skb
->head
;
118 int err
= ip6_find_1stfragopt(skb
, &prevhdr
);
121 fptr
= (struct frag_hdr
*)((u8
*)ipv6h
+ err
);
122 fptr
->frag_off
= htons(offset
);
124 fptr
->frag_off
|= htons(IP6_MF
);
125 offset
+= (ntohs(ipv6h
->payload_len
) -
126 sizeof(struct frag_hdr
));
129 skb_reset_inner_headers(skb
);
136 /* Return the total length of all the extension hdrs, following the same
137 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
139 static int ipv6_exthdrs_len(struct ipv6hdr
*iph
,
140 const struct net_offload
**opps
)
142 struct ipv6_opt_hdr
*opth
= (void *)iph
;
143 int len
= 0, proto
, optlen
= sizeof(*iph
);
145 proto
= iph
->nexthdr
;
147 if (proto
!= NEXTHDR_HOP
) {
148 *opps
= rcu_dereference(inet6_offloads
[proto
]);
149 if (unlikely(!(*opps
)))
151 if (!((*opps
)->flags
& INET6_PROTO_GSO_EXTHDR
))
154 opth
= (void *)opth
+ optlen
;
155 optlen
= ipv6_optlen(opth
);
157 proto
= opth
->nexthdr
;
162 static struct sk_buff
**ipv6_gro_receive(struct sk_buff
**head
,
165 const struct net_offload
*ops
;
166 struct sk_buff
**pp
= NULL
;
175 off
= skb_gro_offset(skb
);
176 hlen
= off
+ sizeof(*iph
);
177 iph
= skb_gro_header_fast(skb
, off
);
178 if (skb_gro_header_hard(skb
, hlen
)) {
179 iph
= skb_gro_header_slow(skb
, hlen
, off
);
184 skb_set_network_header(skb
, off
);
185 skb_gro_pull(skb
, sizeof(*iph
));
186 skb_set_transport_header(skb
, skb_gro_offset(skb
));
188 flush
+= ntohs(iph
->payload_len
) != skb_gro_len(skb
);
191 proto
= iph
->nexthdr
;
192 ops
= rcu_dereference(inet6_offloads
[proto
]);
193 if (!ops
|| !ops
->callbacks
.gro_receive
) {
194 __pskb_pull(skb
, skb_gro_offset(skb
));
195 skb_gro_frag0_invalidate(skb
);
196 proto
= ipv6_gso_pull_exthdrs(skb
, proto
);
197 skb_gro_pull(skb
, -skb_transport_offset(skb
));
198 skb_reset_transport_header(skb
);
199 __skb_push(skb
, skb_gro_offset(skb
));
201 ops
= rcu_dereference(inet6_offloads
[proto
]);
202 if (!ops
|| !ops
->callbacks
.gro_receive
)
208 NAPI_GRO_CB(skb
)->proto
= proto
;
211 nlen
= skb_network_header_len(skb
);
213 for (p
= *head
; p
; p
= p
->next
) {
214 const struct ipv6hdr
*iph2
;
215 __be32 first_word
; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
217 if (!NAPI_GRO_CB(p
)->same_flow
)
220 iph2
= (struct ipv6hdr
*)(p
->data
+ off
);
221 first_word
= *(__be32
*)iph
^ *(__be32
*)iph2
;
223 /* All fields must match except length and Traffic Class.
224 * XXX skbs on the gro_list have all been parsed and pulled
225 * already so we don't need to compare nlen
226 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
227 * memcmp() alone below is suffcient, right?
229 if ((first_word
& htonl(0xF00FFFFF)) ||
230 memcmp(&iph
->nexthdr
, &iph2
->nexthdr
,
231 nlen
- offsetof(struct ipv6hdr
, nexthdr
))) {
232 NAPI_GRO_CB(p
)->same_flow
= 0;
235 /* flush if Traffic Class fields are different */
236 NAPI_GRO_CB(p
)->flush
|= !!(first_word
& htonl(0x0FF00000));
237 NAPI_GRO_CB(p
)->flush
|= flush
;
239 /* If the previous IP ID value was based on an atomic
240 * datagram we can overwrite the value and ignore it.
242 if (NAPI_GRO_CB(skb
)->is_atomic
)
243 NAPI_GRO_CB(p
)->flush_id
= 0;
246 NAPI_GRO_CB(skb
)->is_atomic
= true;
247 NAPI_GRO_CB(skb
)->flush
|= flush
;
249 skb_gro_postpull_rcsum(skb
, iph
, nlen
);
251 pp
= call_gro_receive(ops
->callbacks
.gro_receive
, head
, skb
);
257 NAPI_GRO_CB(skb
)->flush
|= flush
;
262 static struct sk_buff
**sit_ip6ip6_gro_receive(struct sk_buff
**head
,
265 /* Common GRO receive for SIT and IP6IP6 */
267 if (NAPI_GRO_CB(skb
)->encap_mark
) {
268 NAPI_GRO_CB(skb
)->flush
= 1;
272 NAPI_GRO_CB(skb
)->encap_mark
= 1;
274 return ipv6_gro_receive(head
, skb
);
277 static struct sk_buff
**ip4ip6_gro_receive(struct sk_buff
**head
,
280 /* Common GRO receive for SIT and IP6IP6 */
282 if (NAPI_GRO_CB(skb
)->encap_mark
) {
283 NAPI_GRO_CB(skb
)->flush
= 1;
287 NAPI_GRO_CB(skb
)->encap_mark
= 1;
289 return inet_gro_receive(head
, skb
);
292 static int ipv6_gro_complete(struct sk_buff
*skb
, int nhoff
)
294 const struct net_offload
*ops
;
295 struct ipv6hdr
*iph
= (struct ipv6hdr
*)(skb
->data
+ nhoff
);
298 if (skb
->encapsulation
) {
299 skb_set_inner_protocol(skb
, cpu_to_be16(ETH_P_IPV6
));
300 skb_set_inner_network_header(skb
, nhoff
);
303 iph
->payload_len
= htons(skb
->len
- nhoff
- sizeof(*iph
));
307 nhoff
+= sizeof(*iph
) + ipv6_exthdrs_len(iph
, &ops
);
308 if (WARN_ON(!ops
|| !ops
->callbacks
.gro_complete
))
311 err
= ops
->callbacks
.gro_complete(skb
, nhoff
);
319 static int sit_gro_complete(struct sk_buff
*skb
, int nhoff
)
321 skb
->encapsulation
= 1;
322 skb_shinfo(skb
)->gso_type
|= SKB_GSO_IPXIP4
;
323 return ipv6_gro_complete(skb
, nhoff
);
326 static int ip6ip6_gro_complete(struct sk_buff
*skb
, int nhoff
)
328 skb
->encapsulation
= 1;
329 skb_shinfo(skb
)->gso_type
|= SKB_GSO_IPXIP6
;
330 return ipv6_gro_complete(skb
, nhoff
);
333 static int ip4ip6_gro_complete(struct sk_buff
*skb
, int nhoff
)
335 skb
->encapsulation
= 1;
336 skb_shinfo(skb
)->gso_type
|= SKB_GSO_IPXIP6
;
337 return inet_gro_complete(skb
, nhoff
);
340 static struct packet_offload ipv6_packet_offload __read_mostly
= {
341 .type
= cpu_to_be16(ETH_P_IPV6
),
343 .gso_segment
= ipv6_gso_segment
,
344 .gro_receive
= ipv6_gro_receive
,
345 .gro_complete
= ipv6_gro_complete
,
349 static const struct net_offload sit_offload
= {
351 .gso_segment
= ipv6_gso_segment
,
352 .gro_receive
= sit_ip6ip6_gro_receive
,
353 .gro_complete
= sit_gro_complete
,
357 static const struct net_offload ip4ip6_offload
= {
359 .gso_segment
= inet_gso_segment
,
360 .gro_receive
= ip4ip6_gro_receive
,
361 .gro_complete
= ip4ip6_gro_complete
,
365 static const struct net_offload ip6ip6_offload
= {
367 .gso_segment
= ipv6_gso_segment
,
368 .gro_receive
= sit_ip6ip6_gro_receive
,
369 .gro_complete
= ip6ip6_gro_complete
,
372 static int __init
ipv6_offload_init(void)
375 if (tcpv6_offload_init() < 0)
376 pr_crit("%s: Cannot add TCP protocol offload\n", __func__
);
377 if (ipv6_exthdrs_offload_init() < 0)
378 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__
);
380 dev_add_offload(&ipv6_packet_offload
);
382 inet_add_offload(&sit_offload
, IPPROTO_IPV6
);
383 inet6_add_offload(&ip6ip6_offload
, IPPROTO_IPV6
);
384 inet6_add_offload(&ip4ip6_offload
, IPPROTO_IPIP
);
389 fs_initcall(ipv6_offload_init
);