1 // SPDX-License-Identifier: GPL-2.0-only
3 * IPV6 GSO/GRO offload support
4 * Linux INET implementation
6 * Copyright (C) 2016 secunet Security Networks AG
7 * Author: Steffen Klassert <steffen.klassert@secunet.com>
12 #include <linux/skbuff.h>
13 #include <linux/init.h>
14 #include <net/protocol.h>
15 #include <crypto/aead.h>
16 #include <crypto/authenc.h>
17 #include <linux/err.h>
18 #include <linux/module.h>
22 #include <linux/scatterlist.h>
23 #include <linux/kernel.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
26 #include <net/ip6_route.h>
28 #include <linux/icmpv6.h>
30 static __u16
esp6_nexthdr_esp_offset(struct ipv6hdr
*ipv6_hdr
, int nhlen
)
32 int off
= sizeof(struct ipv6hdr
);
33 struct ipv6_opt_hdr
*exthdr
;
35 if (likely(ipv6_hdr
->nexthdr
== NEXTHDR_ESP
))
36 return offsetof(struct ipv6hdr
, nexthdr
);
39 exthdr
= (void *)ipv6_hdr
+ off
;
40 if (exthdr
->nexthdr
== NEXTHDR_ESP
)
43 off
+= ipv6_optlen(exthdr
);
49 static struct sk_buff
*esp6_gro_receive(struct list_head
*head
,
52 int offset
= skb_gro_offset(skb
);
53 struct xfrm_offload
*xo
;
60 if (!pskb_pull(skb
, offset
))
63 if ((err
= xfrm_parse_spi(skb
, IPPROTO_ESP
, &spi
, &seq
)) != 0)
66 xo
= xfrm_offload(skb
);
67 if (!xo
|| !(xo
->flags
& CRYPTO_DONE
)) {
68 struct sec_path
*sp
= secpath_set(skb
);
73 if (sp
->len
== XFRM_MAX_DEPTH
)
76 x
= xfrm_state_lookup(dev_net(skb
->dev
), skb
->mark
,
77 (xfrm_address_t
*)&ipv6_hdr(skb
)->daddr
,
78 spi
, IPPROTO_ESP
, AF_INET6
);
82 sp
->xvec
[sp
->len
++] = x
;
85 xo
= xfrm_offload(skb
);
92 xo
->flags
|= XFRM_GRO
;
94 nhoff
= esp6_nexthdr_esp_offset(ipv6_hdr(skb
), offset
);
98 IP6CB(skb
)->nhoff
= nhoff
;
99 XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
= NULL
;
100 XFRM_SPI_SKB_CB(skb
)->family
= AF_INET6
;
101 XFRM_SPI_SKB_CB(skb
)->daddroff
= offsetof(struct ipv6hdr
, daddr
);
102 XFRM_SPI_SKB_CB(skb
)->seq
= seq
;
104 /* We don't need to handle errors from xfrm_input, it does all
105 * the error handling and frees the resources on error. */
106 xfrm_input(skb
, IPPROTO_ESP
, spi
, -2);
108 return ERR_PTR(-EINPROGRESS
);
112 skb_push(skb
, offset
);
113 NAPI_GRO_CB(skb
)->same_flow
= 0;
114 NAPI_GRO_CB(skb
)->flush
= 1;
119 static void esp6_gso_encap(struct xfrm_state
*x
, struct sk_buff
*skb
)
121 struct ip_esp_hdr
*esph
;
122 struct ipv6hdr
*iph
= ipv6_hdr(skb
);
123 struct xfrm_offload
*xo
= xfrm_offload(skb
);
124 int proto
= iph
->nexthdr
;
126 skb_push(skb
, -skb_network_offset(skb
));
127 esph
= ip_esp_hdr(skb
);
128 *skb_mac_header(skb
) = IPPROTO_ESP
;
130 esph
->spi
= x
->id
.spi
;
131 esph
->seq_no
= htonl(XFRM_SKB_CB(skb
)->seq
.output
.low
);
136 static struct sk_buff
*xfrm6_tunnel_gso_segment(struct xfrm_state
*x
,
138 netdev_features_t features
)
140 __skb_push(skb
, skb
->mac_len
);
141 return skb_mac_gso_segment(skb
, features
);
144 static struct sk_buff
*xfrm6_transport_gso_segment(struct xfrm_state
*x
,
146 netdev_features_t features
)
148 const struct net_offload
*ops
;
149 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
150 struct xfrm_offload
*xo
= xfrm_offload(skb
);
152 skb
->transport_header
+= x
->props
.header_len
;
153 ops
= rcu_dereference(inet6_offloads
[xo
->proto
]);
154 if (likely(ops
&& ops
->callbacks
.gso_segment
))
155 segs
= ops
->callbacks
.gso_segment(skb
, features
);
160 static struct sk_buff
*xfrm6_outer_mode_gso_segment(struct xfrm_state
*x
,
162 netdev_features_t features
)
164 switch (x
->outer_mode
.encap
) {
165 case XFRM_MODE_TUNNEL
:
166 return xfrm6_tunnel_gso_segment(x
, skb
, features
);
167 case XFRM_MODE_TRANSPORT
:
168 return xfrm6_transport_gso_segment(x
, skb
, features
);
171 return ERR_PTR(-EOPNOTSUPP
);
174 static struct sk_buff
*esp6_gso_segment(struct sk_buff
*skb
,
175 netdev_features_t features
)
177 struct xfrm_state
*x
;
178 struct ip_esp_hdr
*esph
;
179 struct crypto_aead
*aead
;
180 netdev_features_t esp_features
= features
;
181 struct xfrm_offload
*xo
= xfrm_offload(skb
);
185 return ERR_PTR(-EINVAL
);
187 if (!(skb_shinfo(skb
)->gso_type
& SKB_GSO_ESP
))
188 return ERR_PTR(-EINVAL
);
190 sp
= skb_sec_path(skb
);
191 x
= sp
->xvec
[sp
->len
- 1];
193 esph
= ip_esp_hdr(skb
);
195 if (esph
->spi
!= x
->id
.spi
)
196 return ERR_PTR(-EINVAL
);
198 if (!pskb_may_pull(skb
, sizeof(*esph
) + crypto_aead_ivsize(aead
)))
199 return ERR_PTR(-EINVAL
);
201 __skb_pull(skb
, sizeof(*esph
) + crypto_aead_ivsize(aead
));
203 skb
->encap_hdr_csum
= 1;
205 if (!(features
& NETIF_F_HW_ESP
) || x
->xso
.dev
!= skb
->dev
)
206 esp_features
= features
& ~(NETIF_F_SG
| NETIF_F_CSUM_MASK
);
207 else if (!(features
& NETIF_F_HW_ESP_TX_CSUM
))
208 esp_features
= features
& ~NETIF_F_CSUM_MASK
;
210 xo
->flags
|= XFRM_GSO_SEGMENT
;
212 return xfrm6_outer_mode_gso_segment(x
, skb
, esp_features
);
215 static int esp6_input_tail(struct xfrm_state
*x
, struct sk_buff
*skb
)
217 struct crypto_aead
*aead
= x
->data
;
218 struct xfrm_offload
*xo
= xfrm_offload(skb
);
220 if (!pskb_may_pull(skb
, sizeof(struct ip_esp_hdr
) + crypto_aead_ivsize(aead
)))
223 if (!(xo
->flags
& CRYPTO_DONE
))
224 skb
->ip_summed
= CHECKSUM_NONE
;
226 return esp6_input_done2(skb
, 0);
229 static int esp6_xmit(struct xfrm_state
*x
, struct sk_buff
*skb
, netdev_features_t features
)
235 struct xfrm_offload
*xo
;
236 struct ip_esp_hdr
*esph
;
237 struct crypto_aead
*aead
;
239 bool hw_offload
= true;
244 xo
= xfrm_offload(skb
);
249 if (!(features
& NETIF_F_HW_ESP
) || x
->xso
.dev
!= skb
->dev
) {
250 xo
->flags
|= CRYPTO_FALLBACK
;
254 esp
.proto
= xo
->proto
;
256 /* skb is pure payload to encrypt */
259 alen
= crypto_aead_authsize(aead
);
262 /* XXX: Add support for tfc padding here. */
264 blksize
= ALIGN(crypto_aead_blocksize(aead
), 4);
265 esp
.clen
= ALIGN(skb
->len
+ 2 + esp
.tfclen
, blksize
);
266 esp
.plen
= esp
.clen
- skb
->len
- esp
.tfclen
;
267 esp
.tailen
= esp
.tfclen
+ esp
.plen
+ alen
;
269 if (!hw_offload
|| (hw_offload
&& !skb_is_gso(skb
))) {
270 esp
.nfrags
= esp6_output_head(x
, skb
, &esp
);
277 esph
= ip_esp_hdr(skb
);
278 esph
->spi
= x
->id
.spi
;
280 skb_push(skb
, -skb_network_offset(skb
));
282 if (xo
->flags
& XFRM_GSO_SEGMENT
) {
283 esph
->seq_no
= htonl(seq
);
285 if (!skb_is_gso(skb
))
288 xo
->seq
.low
+= skb_shinfo(skb
)->gso_segs
;
291 esp
.seqno
= cpu_to_be64(xo
->seq
.low
+ ((u64
)xo
->seq
.hi
<< 32));
293 len
= skb
->len
- sizeof(struct ipv6hdr
);
294 if (len
> IPV6_MAXPLEN
)
297 ipv6_hdr(skb
)->payload_len
= htons(len
);
302 err
= esp6_output_tail(x
, skb
, &esp
);
311 static const struct net_offload esp6_offload
= {
313 .gro_receive
= esp6_gro_receive
,
314 .gso_segment
= esp6_gso_segment
,
318 static const struct xfrm_type_offload esp6_type_offload
= {
319 .description
= "ESP6 OFFLOAD",
320 .owner
= THIS_MODULE
,
321 .proto
= IPPROTO_ESP
,
322 .input_tail
= esp6_input_tail
,
324 .encap
= esp6_gso_encap
,
327 static int __init
esp6_offload_init(void)
329 if (xfrm_register_type_offload(&esp6_type_offload
, AF_INET6
) < 0) {
330 pr_info("%s: can't add xfrm type offload\n", __func__
);
334 return inet6_add_offload(&esp6_offload
, IPPROTO_ESP
);
337 static void __exit
esp6_offload_exit(void)
339 xfrm_unregister_type_offload(&esp6_type_offload
, AF_INET6
);
340 inet6_del_offload(&esp6_offload
, IPPROTO_ESP
);
343 module_init(esp6_offload_init
);
344 module_exit(esp6_offload_exit
);
345 MODULE_LICENSE("GPL");
346 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
347 MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6
, XFRM_PROTO_ESP
);