1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C)2002 USAGI/WIDE Project
7 * Mitsuru KANDA @USAGI : IPv6 Support
8 * Kazunori MIYAZAWA @USAGI :
9 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
11 * This file is derived from net/ipv4/esp.c
14 #define pr_fmt(fmt) "IPv6: " fmt
16 #include <crypto/aead.h>
17 #include <crypto/authenc.h>
18 #include <linux/err.h>
19 #include <linux/module.h>
23 #include <linux/scatterlist.h>
24 #include <linux/kernel.h>
25 #include <linux/pfkeyv2.h>
26 #include <linux/random.h>
27 #include <linux/slab.h>
28 #include <linux/spinlock.h>
29 #include <net/ip6_checksum.h>
30 #include <net/ip6_route.h>
33 #include <net/protocol.h>
35 #include <linux/icmpv6.h>
37 #include <net/espintcp.h>
38 #include <net/inet6_hashtables.h>
40 #include <linux/highmem.h>
43 struct xfrm_skb_cb xfrm
;
47 struct esp_output_extra
{
52 #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))
55 * Allocate an AEAD request structure with extra space for SG and IV.
57 * For alignment considerations the upper 32 bits of the sequence number are
58 * placed at the front, if present. Followed by the IV, the request and finally
61 * TODO: Use spare space in skb for this where possible.
63 static void *esp_alloc_tmp(struct crypto_aead
*aead
, int nfrags
, int seqihlen
)
69 len
+= crypto_aead_ivsize(aead
);
72 len
+= crypto_aead_alignmask(aead
) &
73 ~(crypto_tfm_ctx_alignment() - 1);
74 len
= ALIGN(len
, crypto_tfm_ctx_alignment());
77 len
+= sizeof(struct aead_request
) + crypto_aead_reqsize(aead
);
78 len
= ALIGN(len
, __alignof__(struct scatterlist
));
80 len
+= sizeof(struct scatterlist
) * nfrags
;
82 return kmalloc(len
, GFP_ATOMIC
);
85 static inline void *esp_tmp_extra(void *tmp
)
87 return PTR_ALIGN(tmp
, __alignof__(struct esp_output_extra
));
90 static inline u8
*esp_tmp_iv(struct crypto_aead
*aead
, void *tmp
, int seqhilen
)
92 return crypto_aead_ivsize(aead
) ?
93 PTR_ALIGN((u8
*)tmp
+ seqhilen
,
94 crypto_aead_alignmask(aead
) + 1) : tmp
+ seqhilen
;
97 static inline struct aead_request
*esp_tmp_req(struct crypto_aead
*aead
, u8
*iv
)
99 struct aead_request
*req
;
101 req
= (void *)PTR_ALIGN(iv
+ crypto_aead_ivsize(aead
),
102 crypto_tfm_ctx_alignment());
103 aead_request_set_tfm(req
, aead
);
107 static inline struct scatterlist
*esp_req_sg(struct crypto_aead
*aead
,
108 struct aead_request
*req
)
110 return (void *)ALIGN((unsigned long)(req
+ 1) +
111 crypto_aead_reqsize(aead
),
112 __alignof__(struct scatterlist
));
115 static void esp_ssg_unref(struct xfrm_state
*x
, void *tmp
)
117 struct esp_output_extra
*extra
= esp_tmp_extra(tmp
);
118 struct crypto_aead
*aead
= x
->data
;
121 struct aead_request
*req
;
122 struct scatterlist
*sg
;
124 if (x
->props
.flags
& XFRM_STATE_ESN
)
125 extralen
+= sizeof(*extra
);
127 iv
= esp_tmp_iv(aead
, tmp
, extralen
);
128 req
= esp_tmp_req(aead
, iv
);
130 /* Unref skb_frag_pages in the src scatterlist if necessary.
131 * Skip the first sg which comes from skb->data.
133 if (req
->src
!= req
->dst
)
134 for (sg
= sg_next(req
->src
); sg
; sg
= sg_next(sg
))
135 put_page(sg_page(sg
));
138 #ifdef CONFIG_INET6_ESPINTCP
144 static void esp_free_tcp_sk(struct rcu_head
*head
)
146 struct esp_tcp_sk
*esk
= container_of(head
, struct esp_tcp_sk
, rcu
);
152 static struct sock
*esp6_find_tcp_sk(struct xfrm_state
*x
)
154 struct xfrm_encap_tmpl
*encap
= x
->encap
;
155 struct esp_tcp_sk
*esk
;
160 sk
= rcu_dereference(x
->encap_sk
);
161 if (sk
&& sk
->sk_state
== TCP_ESTABLISHED
)
164 spin_lock_bh(&x
->lock
);
165 sport
= encap
->encap_sport
;
166 dport
= encap
->encap_dport
;
167 nsk
= rcu_dereference_protected(x
->encap_sk
,
168 lockdep_is_held(&x
->lock
));
169 if (sk
&& sk
== nsk
) {
170 esk
= kmalloc(sizeof(*esk
), GFP_ATOMIC
);
172 spin_unlock_bh(&x
->lock
);
173 return ERR_PTR(-ENOMEM
);
175 RCU_INIT_POINTER(x
->encap_sk
, NULL
);
177 call_rcu(&esk
->rcu
, esp_free_tcp_sk
);
179 spin_unlock_bh(&x
->lock
);
181 sk
= __inet6_lookup_established(xs_net(x
), &tcp_hashinfo
, &x
->id
.daddr
.in6
,
182 dport
, &x
->props
.saddr
.in6
, ntohs(sport
), 0, 0);
184 return ERR_PTR(-ENOENT
);
186 if (!tcp_is_ulp_esp(sk
)) {
188 return ERR_PTR(-EINVAL
);
191 spin_lock_bh(&x
->lock
);
192 nsk
= rcu_dereference_protected(x
->encap_sk
,
193 lockdep_is_held(&x
->lock
));
194 if (encap
->encap_sport
!= sport
||
195 encap
->encap_dport
!= dport
) {
197 sk
= nsk
?: ERR_PTR(-EREMCHG
);
198 } else if (sk
== nsk
) {
201 rcu_assign_pointer(x
->encap_sk
, sk
);
203 spin_unlock_bh(&x
->lock
);
208 static int esp_output_tcp_finish(struct xfrm_state
*x
, struct sk_buff
*skb
)
215 sk
= esp6_find_tcp_sk(x
);
216 err
= PTR_ERR_OR_ZERO(sk
);
221 if (sock_owned_by_user(sk
))
222 err
= espintcp_queue_out(sk
, skb
);
224 err
= espintcp_push_skb(sk
, skb
);
232 static int esp_output_tcp_encap_cb(struct net
*net
, struct sock
*sk
,
235 struct dst_entry
*dst
= skb_dst(skb
);
236 struct xfrm_state
*x
= dst
->xfrm
;
238 return esp_output_tcp_finish(x
, skb
);
241 static int esp_output_tail_tcp(struct xfrm_state
*x
, struct sk_buff
*skb
)
246 err
= xfrm_trans_queue_net(xs_net(x
), skb
, esp_output_tcp_encap_cb
);
249 /* EINPROGRESS just happens to do the right thing. It
250 * actually means that the skb has been consumed and
253 return err
?: -EINPROGRESS
;
256 static int esp_output_tail_tcp(struct xfrm_state
*x
, struct sk_buff
*skb
)
264 static void esp_output_encap_csum(struct sk_buff
*skb
)
266 /* UDP encap with IPv6 requires a valid checksum */
267 if (*skb_mac_header(skb
) == IPPROTO_UDP
) {
268 struct udphdr
*uh
= udp_hdr(skb
);
269 struct ipv6hdr
*ip6h
= ipv6_hdr(skb
);
270 int len
= ntohs(uh
->len
);
271 unsigned int offset
= skb_transport_offset(skb
);
272 __wsum csum
= skb_checksum(skb
, offset
, skb
->len
- offset
, 0);
274 uh
->check
= csum_ipv6_magic(&ip6h
->saddr
, &ip6h
->daddr
,
275 len
, IPPROTO_UDP
, csum
);
277 uh
->check
= CSUM_MANGLED_0
;
281 static void esp_output_done(struct crypto_async_request
*base
, int err
)
283 struct sk_buff
*skb
= base
->data
;
284 struct xfrm_offload
*xo
= xfrm_offload(skb
);
286 struct xfrm_state
*x
;
288 if (xo
&& (xo
->flags
& XFRM_DEV_RESUME
)) {
289 struct sec_path
*sp
= skb_sec_path(skb
);
291 x
= sp
->xvec
[sp
->len
- 1];
293 x
= skb_dst(skb
)->xfrm
;
296 tmp
= ESP_SKB_CB(skb
)->tmp
;
297 esp_ssg_unref(x
, tmp
);
300 esp_output_encap_csum(skb
);
302 if (xo
&& (xo
->flags
& XFRM_DEV_RESUME
)) {
304 XFRM_INC_STATS(xs_net(x
), LINUX_MIB_XFRMOUTSTATEPROTOERROR
);
309 skb_push(skb
, skb
->data
- skb_mac_header(skb
));
311 xfrm_dev_resume(skb
);
314 x
->encap
&& x
->encap
->encap_type
== TCP_ENCAP_ESPINTCP
)
315 esp_output_tail_tcp(x
, skb
);
317 xfrm_output_resume(skb
->sk
, skb
, err
);
321 /* Move ESP header back into place. */
322 static void esp_restore_header(struct sk_buff
*skb
, unsigned int offset
)
324 struct ip_esp_hdr
*esph
= (void *)(skb
->data
+ offset
);
325 void *tmp
= ESP_SKB_CB(skb
)->tmp
;
326 __be32
*seqhi
= esp_tmp_extra(tmp
);
328 esph
->seq_no
= esph
->spi
;
332 static void esp_output_restore_header(struct sk_buff
*skb
)
334 void *tmp
= ESP_SKB_CB(skb
)->tmp
;
335 struct esp_output_extra
*extra
= esp_tmp_extra(tmp
);
337 esp_restore_header(skb
, skb_transport_offset(skb
) + extra
->esphoff
-
341 static struct ip_esp_hdr
*esp_output_set_esn(struct sk_buff
*skb
,
342 struct xfrm_state
*x
,
343 struct ip_esp_hdr
*esph
,
344 struct esp_output_extra
*extra
)
346 /* For ESN we move the header forward by 4 bytes to
347 * accomodate the high bits. We will move it back after
350 if ((x
->props
.flags
& XFRM_STATE_ESN
)) {
352 struct xfrm_offload
*xo
= xfrm_offload(skb
);
357 seqhi
= XFRM_SKB_CB(skb
)->seq
.output
.hi
;
359 extra
->esphoff
= (unsigned char *)esph
-
360 skb_transport_header(skb
);
361 esph
= (struct ip_esp_hdr
*)((unsigned char *)esph
- 4);
362 extra
->seqhi
= esph
->spi
;
363 esph
->seq_no
= htonl(seqhi
);
366 esph
->spi
= x
->id
.spi
;
371 static void esp_output_done_esn(struct crypto_async_request
*base
, int err
)
373 struct sk_buff
*skb
= base
->data
;
375 esp_output_restore_header(skb
);
376 esp_output_done(base
, err
);
379 static struct ip_esp_hdr
*esp6_output_udp_encap(struct sk_buff
*skb
,
381 struct esp_info
*esp
,
389 len
= skb
->len
+ esp
->tailen
- skb_transport_offset(skb
);
391 return ERR_PTR(-EMSGSIZE
);
393 uh
= (struct udphdr
*)esp
->esph
;
396 uh
->len
= htons(len
);
399 *skb_mac_header(skb
) = IPPROTO_UDP
;
401 if (encap_type
== UDP_ENCAP_ESPINUDP_NON_IKE
) {
402 udpdata32
= (__be32
*)(uh
+ 1);
403 udpdata32
[0] = udpdata32
[1] = 0;
404 return (struct ip_esp_hdr
*)(udpdata32
+ 2);
407 return (struct ip_esp_hdr
*)(uh
+ 1);
410 #ifdef CONFIG_INET6_ESPINTCP
411 static struct ip_esp_hdr
*esp6_output_tcp_encap(struct xfrm_state
*x
,
413 struct esp_info
*esp
)
415 __be16
*lenp
= (void *)esp
->esph
;
416 struct ip_esp_hdr
*esph
;
420 len
= skb
->len
+ esp
->tailen
- skb_transport_offset(skb
);
421 if (len
> IP_MAX_MTU
)
422 return ERR_PTR(-EMSGSIZE
);
425 sk
= esp6_find_tcp_sk(x
);
432 esph
= (struct ip_esp_hdr
*)(lenp
+ 1);
437 static struct ip_esp_hdr
*esp6_output_tcp_encap(struct xfrm_state
*x
,
439 struct esp_info
*esp
)
441 return ERR_PTR(-EOPNOTSUPP
);
445 static int esp6_output_encap(struct xfrm_state
*x
, struct sk_buff
*skb
,
446 struct esp_info
*esp
)
448 struct xfrm_encap_tmpl
*encap
= x
->encap
;
449 struct ip_esp_hdr
*esph
;
453 spin_lock_bh(&x
->lock
);
454 sport
= encap
->encap_sport
;
455 dport
= encap
->encap_dport
;
456 encap_type
= encap
->encap_type
;
457 spin_unlock_bh(&x
->lock
);
459 switch (encap_type
) {
461 case UDP_ENCAP_ESPINUDP
:
462 case UDP_ENCAP_ESPINUDP_NON_IKE
:
463 esph
= esp6_output_udp_encap(skb
, encap_type
, esp
, sport
, dport
);
465 case TCP_ENCAP_ESPINTCP
:
466 esph
= esp6_output_tcp_encap(x
, skb
, esp
);
471 return PTR_ERR(esph
);
478 int esp6_output_head(struct xfrm_state
*x
, struct sk_buff
*skb
, struct esp_info
*esp
)
484 struct sk_buff
*trailer
;
485 int tailen
= esp
->tailen
;
488 int err
= esp6_output_encap(x
, skb
, esp
);
494 if (!skb_cloned(skb
)) {
495 if (tailen
<= skb_tailroom(skb
)) {
498 tail
= skb_tail_pointer(trailer
);
501 } else if ((skb_shinfo(skb
)->nr_frags
< MAX_SKB_FRAGS
)
502 && !skb_has_frag_list(skb
)) {
504 struct sock
*sk
= skb
->sk
;
505 struct page_frag
*pfrag
= &x
->xfrag
;
507 esp
->inplace
= false;
509 allocsize
= ALIGN(tailen
, L1_CACHE_BYTES
);
511 spin_lock_bh(&x
->lock
);
513 if (unlikely(!skb_page_frag_refill(allocsize
, pfrag
, GFP_ATOMIC
))) {
514 spin_unlock_bh(&x
->lock
);
521 tail
= page_address(page
) + pfrag
->offset
;
523 esp_output_fill_trailer(tail
, esp
->tfclen
, esp
->plen
, esp
->proto
);
525 nfrags
= skb_shinfo(skb
)->nr_frags
;
527 __skb_fill_page_desc(skb
, nfrags
, page
, pfrag
->offset
,
529 skb_shinfo(skb
)->nr_frags
= ++nfrags
;
531 pfrag
->offset
= pfrag
->offset
+ allocsize
;
533 spin_unlock_bh(&x
->lock
);
538 skb
->data_len
+= tailen
;
539 skb
->truesize
+= tailen
;
540 if (sk
&& sk_fullsock(sk
))
541 refcount_add(tailen
, &sk
->sk_wmem_alloc
);
548 esph_offset
= (unsigned char *)esp
->esph
- skb_transport_header(skb
);
550 nfrags
= skb_cow_data(skb
, tailen
, &trailer
);
553 tail
= skb_tail_pointer(trailer
);
554 esp
->esph
= (struct ip_esp_hdr
*)(skb_transport_header(skb
) + esph_offset
);
557 esp_output_fill_trailer(tail
, esp
->tfclen
, esp
->plen
, esp
->proto
);
558 pskb_put(skb
, trailer
, tailen
);
563 EXPORT_SYMBOL_GPL(esp6_output_head
);
565 int esp6_output_tail(struct xfrm_state
*x
, struct sk_buff
*skb
, struct esp_info
*esp
)
574 struct ip_esp_hdr
*esph
;
575 struct aead_request
*req
;
576 struct crypto_aead
*aead
;
577 struct scatterlist
*sg
, *dsg
;
578 struct esp_output_extra
*extra
;
581 assoclen
= sizeof(struct ip_esp_hdr
);
584 if (x
->props
.flags
& XFRM_STATE_ESN
) {
585 extralen
+= sizeof(*extra
);
586 assoclen
+= sizeof(__be32
);
590 alen
= crypto_aead_authsize(aead
);
591 ivlen
= crypto_aead_ivsize(aead
);
593 tmp
= esp_alloc_tmp(aead
, esp
->nfrags
+ 2, extralen
);
597 extra
= esp_tmp_extra(tmp
);
598 iv
= esp_tmp_iv(aead
, tmp
, extralen
);
599 req
= esp_tmp_req(aead
, iv
);
600 sg
= esp_req_sg(aead
, req
);
605 dsg
= &sg
[esp
->nfrags
];
607 esph
= esp_output_set_esn(skb
, x
, esp
->esph
, extra
);
610 sg_init_table(sg
, esp
->nfrags
);
611 err
= skb_to_sgvec(skb
, sg
,
612 (unsigned char *)esph
- skb
->data
,
613 assoclen
+ ivlen
+ esp
->clen
+ alen
);
614 if (unlikely(err
< 0))
619 struct page_frag
*pfrag
= &x
->xfrag
;
621 allocsize
= ALIGN(skb
->data_len
, L1_CACHE_BYTES
);
623 spin_lock_bh(&x
->lock
);
624 if (unlikely(!skb_page_frag_refill(allocsize
, pfrag
, GFP_ATOMIC
))) {
625 spin_unlock_bh(&x
->lock
);
629 skb_shinfo(skb
)->nr_frags
= 1;
633 /* replace page frags in skb with new page */
634 __skb_fill_page_desc(skb
, 0, page
, pfrag
->offset
, skb
->data_len
);
635 pfrag
->offset
= pfrag
->offset
+ allocsize
;
636 spin_unlock_bh(&x
->lock
);
638 sg_init_table(dsg
, skb_shinfo(skb
)->nr_frags
+ 1);
639 err
= skb_to_sgvec(skb
, dsg
,
640 (unsigned char *)esph
- skb
->data
,
641 assoclen
+ ivlen
+ esp
->clen
+ alen
);
642 if (unlikely(err
< 0))
646 if ((x
->props
.flags
& XFRM_STATE_ESN
))
647 aead_request_set_callback(req
, 0, esp_output_done_esn
, skb
);
649 aead_request_set_callback(req
, 0, esp_output_done
, skb
);
651 aead_request_set_crypt(req
, sg
, dsg
, ivlen
+ esp
->clen
, iv
);
652 aead_request_set_ad(req
, assoclen
);
654 memset(iv
, 0, ivlen
);
655 memcpy(iv
+ ivlen
- min(ivlen
, 8), (u8
*)&esp
->seqno
+ 8 - min(ivlen
, 8),
658 ESP_SKB_CB(skb
)->tmp
= tmp
;
659 err
= crypto_aead_encrypt(req
);
670 if ((x
->props
.flags
& XFRM_STATE_ESN
))
671 esp_output_restore_header(skb
);
672 esp_output_encap_csum(skb
);
676 esp_ssg_unref(x
, tmp
);
678 if (!err
&& x
->encap
&& x
->encap
->encap_type
== TCP_ENCAP_ESPINTCP
)
679 err
= esp_output_tail_tcp(x
, skb
);
686 EXPORT_SYMBOL_GPL(esp6_output_tail
);
688 static int esp6_output(struct xfrm_state
*x
, struct sk_buff
*skb
)
692 struct ip_esp_hdr
*esph
;
693 struct crypto_aead
*aead
;
698 esp
.proto
= *skb_mac_header(skb
);
699 *skb_mac_header(skb
) = IPPROTO_ESP
;
701 /* skb is pure payload to encrypt */
704 alen
= crypto_aead_authsize(aead
);
708 struct xfrm_dst
*dst
= (struct xfrm_dst
*)skb_dst(skb
);
711 padto
= min(x
->tfcpad
, __xfrm_state_mtu(x
, dst
->child_mtu_cached
));
712 if (skb
->len
< padto
)
713 esp
.tfclen
= padto
- skb
->len
;
715 blksize
= ALIGN(crypto_aead_blocksize(aead
), 4);
716 esp
.clen
= ALIGN(skb
->len
+ 2 + esp
.tfclen
, blksize
);
717 esp
.plen
= esp
.clen
- skb
->len
- esp
.tfclen
;
718 esp
.tailen
= esp
.tfclen
+ esp
.plen
+ alen
;
720 esp
.esph
= ip_esp_hdr(skb
);
722 esp
.nfrags
= esp6_output_head(x
, skb
, &esp
);
727 esph
->spi
= x
->id
.spi
;
729 esph
->seq_no
= htonl(XFRM_SKB_CB(skb
)->seq
.output
.low
);
730 esp
.seqno
= cpu_to_be64(XFRM_SKB_CB(skb
)->seq
.output
.low
+
731 ((u64
)XFRM_SKB_CB(skb
)->seq
.output
.hi
<< 32));
733 skb_push(skb
, -skb_network_offset(skb
));
735 return esp6_output_tail(x
, skb
, &esp
);
738 static inline int esp_remove_trailer(struct sk_buff
*skb
)
740 struct xfrm_state
*x
= xfrm_input_state(skb
);
741 struct xfrm_offload
*xo
= xfrm_offload(skb
);
742 struct crypto_aead
*aead
= x
->data
;
743 int alen
, hlen
, elen
;
749 alen
= crypto_aead_authsize(aead
);
750 hlen
= sizeof(struct ip_esp_hdr
) + crypto_aead_ivsize(aead
);
751 elen
= skb
->len
- hlen
;
753 if (xo
&& (xo
->flags
& XFRM_ESP_NO_TRAILER
)) {
758 ret
= skb_copy_bits(skb
, skb
->len
- alen
- 2, nexthdr
, 2);
763 if (padlen
+ 2 + alen
>= elen
) {
764 net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
765 padlen
+ 2, elen
- alen
);
769 trimlen
= alen
+ padlen
+ 2;
770 if (skb
->ip_summed
== CHECKSUM_COMPLETE
) {
771 csumdiff
= skb_checksum(skb
, skb
->len
- trimlen
, trimlen
, 0);
772 skb
->csum
= csum_block_sub(skb
->csum
, csumdiff
,
775 pskb_trim(skb
, skb
->len
- trimlen
);
783 int esp6_input_done2(struct sk_buff
*skb
, int err
)
785 struct xfrm_state
*x
= xfrm_input_state(skb
);
786 struct xfrm_offload
*xo
= xfrm_offload(skb
);
787 struct crypto_aead
*aead
= x
->data
;
788 int hlen
= sizeof(struct ip_esp_hdr
) + crypto_aead_ivsize(aead
);
789 int hdr_len
= skb_network_header_len(skb
);
791 if (!xo
|| !(xo
->flags
& CRYPTO_DONE
))
792 kfree(ESP_SKB_CB(skb
)->tmp
);
797 err
= esp_remove_trailer(skb
);
798 if (unlikely(err
< 0))
802 const struct ipv6hdr
*ip6h
= ipv6_hdr(skb
);
803 int offset
= skb_network_offset(skb
) + sizeof(*ip6h
);
804 struct xfrm_encap_tmpl
*encap
= x
->encap
;
805 u8 nexthdr
= ip6h
->nexthdr
;
806 __be16 frag_off
, source
;
810 offset
= ipv6_skip_exthdr(skb
, offset
, &nexthdr
, &frag_off
);
811 uh
= (void *)(skb
->data
+ offset
);
812 th
= (void *)(skb
->data
+ offset
);
815 switch (x
->encap
->encap_type
) {
816 case TCP_ENCAP_ESPINTCP
:
819 case UDP_ENCAP_ESPINUDP
:
820 case UDP_ENCAP_ESPINUDP_NON_IKE
:
830 * 1) if the NAT-T peer's IP or port changed then
831 * advertize the change to the keying daemon.
832 * This is an inbound SA, so just compare
835 if (!ipv6_addr_equal(&ip6h
->saddr
, &x
->props
.saddr
.in6
) ||
836 source
!= encap
->encap_sport
) {
837 xfrm_address_t ipaddr
;
839 memcpy(&ipaddr
.a6
, &ip6h
->saddr
.s6_addr
, sizeof(ipaddr
.a6
));
840 km_new_mapping(x
, &ipaddr
, source
);
842 /* XXX: perhaps add an extra
843 * policy check here, to see
844 * if we should allow or
845 * reject a packet from a
852 * 2) ignore UDP/TCP checksums in case
853 * of NAT-T in Transport Mode, or
854 * perform other post-processing fixes
855 * as per draft-ietf-ipsec-udp-encaps-06,
858 if (x
->props
.mode
== XFRM_MODE_TRANSPORT
)
859 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
862 skb_postpull_rcsum(skb
, skb_network_header(skb
),
863 skb_network_header_len(skb
));
864 skb_pull_rcsum(skb
, hlen
);
865 if (x
->props
.mode
== XFRM_MODE_TUNNEL
)
866 skb_reset_transport_header(skb
);
868 skb_set_transport_header(skb
, -hdr_len
);
870 /* RFC4303: Drop dummy packets without any error */
871 if (err
== IPPROTO_NONE
)
877 EXPORT_SYMBOL_GPL(esp6_input_done2
);
879 static void esp_input_done(struct crypto_async_request
*base
, int err
)
881 struct sk_buff
*skb
= base
->data
;
883 xfrm_input_resume(skb
, esp6_input_done2(skb
, err
));
886 static void esp_input_restore_header(struct sk_buff
*skb
)
888 esp_restore_header(skb
, 0);
892 static void esp_input_set_header(struct sk_buff
*skb
, __be32
*seqhi
)
894 struct xfrm_state
*x
= xfrm_input_state(skb
);
896 /* For ESN we move the header forward by 4 bytes to
897 * accomodate the high bits. We will move it back after
900 if ((x
->props
.flags
& XFRM_STATE_ESN
)) {
901 struct ip_esp_hdr
*esph
= skb_push(skb
, 4);
904 esph
->spi
= esph
->seq_no
;
905 esph
->seq_no
= XFRM_SKB_CB(skb
)->seq
.input
.hi
;
909 static void esp_input_done_esn(struct crypto_async_request
*base
, int err
)
911 struct sk_buff
*skb
= base
->data
;
913 esp_input_restore_header(skb
);
914 esp_input_done(base
, err
);
917 static int esp6_input(struct xfrm_state
*x
, struct sk_buff
*skb
)
919 struct crypto_aead
*aead
= x
->data
;
920 struct aead_request
*req
;
921 struct sk_buff
*trailer
;
922 int ivlen
= crypto_aead_ivsize(aead
);
923 int elen
= skb
->len
- sizeof(struct ip_esp_hdr
) - ivlen
;
931 struct scatterlist
*sg
;
933 if (!pskb_may_pull(skb
, sizeof(struct ip_esp_hdr
) + ivlen
)) {
943 assoclen
= sizeof(struct ip_esp_hdr
);
946 if (x
->props
.flags
& XFRM_STATE_ESN
) {
947 seqhilen
+= sizeof(__be32
);
948 assoclen
+= seqhilen
;
951 if (!skb_cloned(skb
)) {
952 if (!skb_is_nonlinear(skb
)) {
956 } else if (!skb_has_frag_list(skb
)) {
957 nfrags
= skb_shinfo(skb
)->nr_frags
;
964 nfrags
= skb_cow_data(skb
, 0, &trailer
);
972 tmp
= esp_alloc_tmp(aead
, nfrags
, seqhilen
);
976 ESP_SKB_CB(skb
)->tmp
= tmp
;
977 seqhi
= esp_tmp_extra(tmp
);
978 iv
= esp_tmp_iv(aead
, tmp
, seqhilen
);
979 req
= esp_tmp_req(aead
, iv
);
980 sg
= esp_req_sg(aead
, req
);
982 esp_input_set_header(skb
, seqhi
);
984 sg_init_table(sg
, nfrags
);
985 ret
= skb_to_sgvec(skb
, sg
, 0, skb
->len
);
986 if (unlikely(ret
< 0)) {
991 skb
->ip_summed
= CHECKSUM_NONE
;
993 if ((x
->props
.flags
& XFRM_STATE_ESN
))
994 aead_request_set_callback(req
, 0, esp_input_done_esn
, skb
);
996 aead_request_set_callback(req
, 0, esp_input_done
, skb
);
998 aead_request_set_crypt(req
, sg
, sg
, elen
+ ivlen
, iv
);
999 aead_request_set_ad(req
, assoclen
);
1001 ret
= crypto_aead_decrypt(req
);
1002 if (ret
== -EINPROGRESS
)
1005 if ((x
->props
.flags
& XFRM_STATE_ESN
))
1006 esp_input_restore_header(skb
);
1008 ret
= esp6_input_done2(skb
, ret
);
1014 static int esp6_err(struct sk_buff
*skb
, struct inet6_skb_parm
*opt
,
1015 u8 type
, u8 code
, int offset
, __be32 info
)
1017 struct net
*net
= dev_net(skb
->dev
);
1018 const struct ipv6hdr
*iph
= (const struct ipv6hdr
*)skb
->data
;
1019 struct ip_esp_hdr
*esph
= (struct ip_esp_hdr
*)(skb
->data
+ offset
);
1020 struct xfrm_state
*x
;
1022 if (type
!= ICMPV6_PKT_TOOBIG
&&
1023 type
!= NDISC_REDIRECT
)
1026 x
= xfrm_state_lookup(net
, skb
->mark
, (const xfrm_address_t
*)&iph
->daddr
,
1027 esph
->spi
, IPPROTO_ESP
, AF_INET6
);
1031 if (type
== NDISC_REDIRECT
)
1032 ip6_redirect(skb
, net
, skb
->dev
->ifindex
, 0,
1033 sock_net_uid(net
, NULL
));
1035 ip6_update_pmtu(skb
, net
, info
, 0, 0, sock_net_uid(net
, NULL
));
1041 static void esp6_destroy(struct xfrm_state
*x
)
1043 struct crypto_aead
*aead
= x
->data
;
1048 crypto_free_aead(aead
);
1051 static int esp_init_aead(struct xfrm_state
*x
)
1053 char aead_name
[CRYPTO_MAX_ALG_NAME
];
1054 struct crypto_aead
*aead
;
1057 err
= -ENAMETOOLONG
;
1058 if (snprintf(aead_name
, CRYPTO_MAX_ALG_NAME
, "%s(%s)",
1059 x
->geniv
, x
->aead
->alg_name
) >= CRYPTO_MAX_ALG_NAME
)
1062 aead
= crypto_alloc_aead(aead_name
, 0, 0);
1063 err
= PTR_ERR(aead
);
1069 err
= crypto_aead_setkey(aead
, x
->aead
->alg_key
,
1070 (x
->aead
->alg_key_len
+ 7) / 8);
1074 err
= crypto_aead_setauthsize(aead
, x
->aead
->alg_icv_len
/ 8);
1082 static int esp_init_authenc(struct xfrm_state
*x
)
1084 struct crypto_aead
*aead
;
1085 struct crypto_authenc_key_param
*param
;
1089 char authenc_name
[CRYPTO_MAX_ALG_NAME
];
1090 unsigned int keylen
;
1097 err
= -ENAMETOOLONG
;
1099 if ((x
->props
.flags
& XFRM_STATE_ESN
)) {
1100 if (snprintf(authenc_name
, CRYPTO_MAX_ALG_NAME
,
1101 "%s%sauthencesn(%s,%s)%s",
1102 x
->geniv
?: "", x
->geniv
? "(" : "",
1103 x
->aalg
? x
->aalg
->alg_name
: "digest_null",
1105 x
->geniv
? ")" : "") >= CRYPTO_MAX_ALG_NAME
)
1108 if (snprintf(authenc_name
, CRYPTO_MAX_ALG_NAME
,
1109 "%s%sauthenc(%s,%s)%s",
1110 x
->geniv
?: "", x
->geniv
? "(" : "",
1111 x
->aalg
? x
->aalg
->alg_name
: "digest_null",
1113 x
->geniv
? ")" : "") >= CRYPTO_MAX_ALG_NAME
)
1117 aead
= crypto_alloc_aead(authenc_name
, 0, 0);
1118 err
= PTR_ERR(aead
);
1124 keylen
= (x
->aalg
? (x
->aalg
->alg_key_len
+ 7) / 8 : 0) +
1125 (x
->ealg
->alg_key_len
+ 7) / 8 + RTA_SPACE(sizeof(*param
));
1127 key
= kmalloc(keylen
, GFP_KERNEL
);
1133 rta
->rta_type
= CRYPTO_AUTHENC_KEYA_PARAM
;
1134 rta
->rta_len
= RTA_LENGTH(sizeof(*param
));
1135 param
= RTA_DATA(rta
);
1136 p
+= RTA_SPACE(sizeof(*param
));
1139 struct xfrm_algo_desc
*aalg_desc
;
1141 memcpy(p
, x
->aalg
->alg_key
, (x
->aalg
->alg_key_len
+ 7) / 8);
1142 p
+= (x
->aalg
->alg_key_len
+ 7) / 8;
1144 aalg_desc
= xfrm_aalg_get_byname(x
->aalg
->alg_name
, 0);
1148 if (aalg_desc
->uinfo
.auth
.icv_fullbits
/ 8 !=
1149 crypto_aead_authsize(aead
)) {
1150 pr_info("ESP: %s digestsize %u != %u\n",
1152 crypto_aead_authsize(aead
),
1153 aalg_desc
->uinfo
.auth
.icv_fullbits
/ 8);
1157 err
= crypto_aead_setauthsize(
1158 aead
, x
->aalg
->alg_trunc_len
/ 8);
1163 param
->enckeylen
= cpu_to_be32((x
->ealg
->alg_key_len
+ 7) / 8);
1164 memcpy(p
, x
->ealg
->alg_key
, (x
->ealg
->alg_key_len
+ 7) / 8);
1166 err
= crypto_aead_setkey(aead
, key
, keylen
);
1175 static int esp6_init_state(struct xfrm_state
*x
)
1177 struct crypto_aead
*aead
;
1184 err
= esp_init_aead(x
);
1186 err
= esp_init_authenc(x
);
1193 x
->props
.header_len
= sizeof(struct ip_esp_hdr
) +
1194 crypto_aead_ivsize(aead
);
1195 switch (x
->props
.mode
) {
1196 case XFRM_MODE_BEET
:
1197 if (x
->sel
.family
!= AF_INET6
)
1198 x
->props
.header_len
+= IPV4_BEET_PHMAXLEN
+
1199 (sizeof(struct ipv6hdr
) - sizeof(struct iphdr
));
1202 case XFRM_MODE_TRANSPORT
:
1204 case XFRM_MODE_TUNNEL
:
1205 x
->props
.header_len
+= sizeof(struct ipv6hdr
);
1210 struct xfrm_encap_tmpl
*encap
= x
->encap
;
1212 switch (encap
->encap_type
) {
1216 case UDP_ENCAP_ESPINUDP
:
1217 x
->props
.header_len
+= sizeof(struct udphdr
);
1219 case UDP_ENCAP_ESPINUDP_NON_IKE
:
1220 x
->props
.header_len
+= sizeof(struct udphdr
) + 2 * sizeof(u32
);
1222 #ifdef CONFIG_INET6_ESPINTCP
1223 case TCP_ENCAP_ESPINTCP
:
1224 /* only the length field, TCP encap is done by
1227 x
->props
.header_len
+= 2;
1233 align
= ALIGN(crypto_aead_blocksize(aead
), 4);
1234 x
->props
.trailer_len
= align
+ 1 + crypto_aead_authsize(aead
);
1240 static int esp6_rcv_cb(struct sk_buff
*skb
, int err
)
1245 static const struct xfrm_type esp6_type
= {
1246 .owner
= THIS_MODULE
,
1247 .proto
= IPPROTO_ESP
,
1248 .flags
= XFRM_TYPE_REPLAY_PROT
,
1249 .init_state
= esp6_init_state
,
1250 .destructor
= esp6_destroy
,
1251 .input
= esp6_input
,
1252 .output
= esp6_output
,
1255 static struct xfrm6_protocol esp6_protocol
= {
1256 .handler
= xfrm6_rcv
,
1257 .input_handler
= xfrm_input
,
1258 .cb_handler
= esp6_rcv_cb
,
1259 .err_handler
= esp6_err
,
1263 static int __init
esp6_init(void)
1265 if (xfrm_register_type(&esp6_type
, AF_INET6
) < 0) {
1266 pr_info("%s: can't add xfrm type\n", __func__
);
1269 if (xfrm6_protocol_register(&esp6_protocol
, IPPROTO_ESP
) < 0) {
1270 pr_info("%s: can't add protocol\n", __func__
);
1271 xfrm_unregister_type(&esp6_type
, AF_INET6
);
1278 static void __exit
esp6_fini(void)
1280 if (xfrm6_protocol_deregister(&esp6_protocol
, IPPROTO_ESP
) < 0)
1281 pr_info("%s: can't remove protocol\n", __func__
);
1282 xfrm_unregister_type(&esp6_type
, AF_INET6
);
1285 module_init(esp6_init
);
1286 module_exit(esp6_fini
);
1288 MODULE_LICENSE("GPL");
1289 MODULE_ALIAS_XFRM_TYPE(AF_INET6
, XFRM_PROTO_ESP
);