1 // SPDX-License-Identifier: GPL-2.0
6 * YOSHIFUJI Hideaki @USAGI
7 * Split up af-specific portion
11 #include <linux/bottom_half.h>
12 #include <linux/interrupt.h>
13 #include <linux/slab.h>
14 #include <linux/module.h>
15 #include <linux/netdevice.h>
16 #include <linux/percpu.h>
20 #include <net/ip_tunnels.h>
21 #include <net/ip6_tunnel.h>
23 struct xfrm_trans_tasklet
{
24 struct tasklet_struct tasklet
;
25 struct sk_buff_head queue
;
28 struct xfrm_trans_cb
{
29 int (*finish
)(struct net
*net
, struct sock
*sk
, struct sk_buff
*skb
);
32 #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
34 static struct kmem_cache
*secpath_cachep __read_mostly
;
36 static DEFINE_SPINLOCK(xfrm_input_afinfo_lock
);
37 static struct xfrm_input_afinfo
const __rcu
*xfrm_input_afinfo
[AF_INET6
+ 1];
39 static struct gro_cells gro_cells
;
40 static struct net_device xfrm_napi_dev
;
42 static DEFINE_PER_CPU(struct xfrm_trans_tasklet
, xfrm_trans_tasklet
);
44 int xfrm_input_register_afinfo(const struct xfrm_input_afinfo
*afinfo
)
48 if (WARN_ON(afinfo
->family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
51 spin_lock_bh(&xfrm_input_afinfo_lock
);
52 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != NULL
))
55 rcu_assign_pointer(xfrm_input_afinfo
[afinfo
->family
], afinfo
);
56 spin_unlock_bh(&xfrm_input_afinfo_lock
);
59 EXPORT_SYMBOL(xfrm_input_register_afinfo
);
61 int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo
*afinfo
)
65 spin_lock_bh(&xfrm_input_afinfo_lock
);
66 if (likely(xfrm_input_afinfo
[afinfo
->family
] != NULL
)) {
67 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != afinfo
))
70 RCU_INIT_POINTER(xfrm_input_afinfo
[afinfo
->family
], NULL
);
72 spin_unlock_bh(&xfrm_input_afinfo_lock
);
76 EXPORT_SYMBOL(xfrm_input_unregister_afinfo
);
78 static const struct xfrm_input_afinfo
*xfrm_input_get_afinfo(unsigned int family
)
80 const struct xfrm_input_afinfo
*afinfo
;
82 if (WARN_ON_ONCE(family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
86 afinfo
= rcu_dereference(xfrm_input_afinfo
[family
]);
87 if (unlikely(!afinfo
))
92 static int xfrm_rcv_cb(struct sk_buff
*skb
, unsigned int family
, u8 protocol
,
96 const struct xfrm_input_afinfo
*afinfo
= xfrm_input_get_afinfo(family
);
101 ret
= afinfo
->callback(skb
, protocol
, err
);
107 void __secpath_destroy(struct sec_path
*sp
)
110 for (i
= 0; i
< sp
->len
; i
++)
111 xfrm_state_put(sp
->xvec
[i
]);
112 kmem_cache_free(secpath_cachep
, sp
);
114 EXPORT_SYMBOL(__secpath_destroy
);
116 struct sec_path
*secpath_dup(struct sec_path
*src
)
120 sp
= kmem_cache_alloc(secpath_cachep
, GFP_ATOMIC
);
127 memset(sp
->ovec
, 0, sizeof(sp
->ovec
[XFRM_MAX_OFFLOAD_DEPTH
]));
132 memcpy(sp
, src
, sizeof(*sp
));
133 for (i
= 0; i
< sp
->len
; i
++)
134 xfrm_state_hold(sp
->xvec
[i
]);
136 refcount_set(&sp
->refcnt
, 1);
139 EXPORT_SYMBOL(secpath_dup
);
141 int secpath_set(struct sk_buff
*skb
)
145 /* Allocate new secpath or COW existing one. */
146 if (!skb
->sp
|| refcount_read(&skb
->sp
->refcnt
) != 1) {
147 sp
= secpath_dup(skb
->sp
);
152 secpath_put(skb
->sp
);
157 EXPORT_SYMBOL(secpath_set
);
159 /* Fetch spi and seq from ipsec header */
161 int xfrm_parse_spi(struct sk_buff
*skb
, u8 nexthdr
, __be32
*spi
, __be32
*seq
)
163 int offset
, offset_seq
;
168 hlen
= sizeof(struct ip_auth_hdr
);
169 offset
= offsetof(struct ip_auth_hdr
, spi
);
170 offset_seq
= offsetof(struct ip_auth_hdr
, seq_no
);
173 hlen
= sizeof(struct ip_esp_hdr
);
174 offset
= offsetof(struct ip_esp_hdr
, spi
);
175 offset_seq
= offsetof(struct ip_esp_hdr
, seq_no
);
178 if (!pskb_may_pull(skb
, sizeof(struct ip_comp_hdr
)))
180 *spi
= htonl(ntohs(*(__be16
*)(skb_transport_header(skb
) + 2)));
187 if (!pskb_may_pull(skb
, hlen
))
190 *spi
= *(__be32
*)(skb_transport_header(skb
) + offset
);
191 *seq
= *(__be32
*)(skb_transport_header(skb
) + offset_seq
);
194 EXPORT_SYMBOL(xfrm_parse_spi
);
196 int xfrm_prepare_input(struct xfrm_state
*x
, struct sk_buff
*skb
)
198 struct xfrm_mode
*inner_mode
= x
->inner_mode
;
201 err
= x
->outer_mode
->afinfo
->extract_input(x
, skb
);
205 if (x
->sel
.family
== AF_UNSPEC
) {
206 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
207 if (inner_mode
== NULL
)
208 return -EAFNOSUPPORT
;
211 skb
->protocol
= inner_mode
->afinfo
->eth_proto
;
212 return inner_mode
->input2(x
, skb
);
214 EXPORT_SYMBOL(xfrm_prepare_input
);
216 int xfrm_input(struct sk_buff
*skb
, int nexthdr
, __be32 spi
, int encap_type
)
218 struct net
*net
= dev_net(skb
->dev
);
222 struct xfrm_state
*x
= NULL
;
223 xfrm_address_t
*daddr
;
224 struct xfrm_mode
*inner_mode
;
225 u32 mark
= skb
->mark
;
226 unsigned int family
= AF_UNSPEC
;
229 bool xfrm_gro
= false;
230 bool crypto_done
= false;
231 struct xfrm_offload
*xo
= xfrm_offload(skb
);
233 if (encap_type
< 0) {
234 x
= xfrm_input_state(skb
);
236 if (unlikely(x
->km
.state
!= XFRM_STATE_VALID
)) {
237 if (x
->km
.state
== XFRM_STATE_ACQ
)
238 XFRM_INC_STATS(net
, LINUX_MIB_XFRMACQUIREERROR
);
241 LINUX_MIB_XFRMINSTATEINVALID
);
245 family
= x
->outer_mode
->afinfo
->family
;
247 /* An encap_type of -1 indicates async resumption. */
248 if (encap_type
== -1) {
250 seq
= XFRM_SKB_CB(skb
)->seq
.input
.low
;
254 /* encap_type < -1 indicates a GRO call. */
256 seq
= XFRM_SPI_SKB_CB(skb
)->seq
;
258 if (xo
&& (xo
->flags
& CRYPTO_DONE
)) {
260 x
= xfrm_input_state(skb
);
261 family
= XFRM_SPI_SKB_CB(skb
)->family
;
263 if (!(xo
->status
& CRYPTO_SUCCESS
)) {
265 (CRYPTO_TRANSPORT_AH_AUTH_FAILED
|
266 CRYPTO_TRANSPORT_ESP_AUTH_FAILED
|
267 CRYPTO_TUNNEL_AH_AUTH_FAILED
|
268 CRYPTO_TUNNEL_ESP_AUTH_FAILED
)) {
270 xfrm_audit_state_icvfail(x
, skb
,
272 x
->stats
.integrity_failed
++;
273 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
277 if (xo
->status
& CRYPTO_INVALID_PROTOCOL
) {
278 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
282 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINBUFFERERROR
);
286 if ((err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
)) != 0) {
287 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
295 family
= XFRM_SPI_SKB_CB(skb
)->family
;
297 /* if tunnel is present override skb->mark value with tunnel i_key */
300 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
)
301 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
->parms
.i_key
);
304 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
)
305 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
->parms
.i_key
);
309 err
= secpath_set(skb
);
311 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINERROR
);
316 if (!spi
&& (err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
)) != 0) {
317 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
321 daddr
= (xfrm_address_t
*)(skb_network_header(skb
) +
322 XFRM_SPI_SKB_CB(skb
)->daddroff
);
324 if (skb
->sp
->len
== XFRM_MAX_DEPTH
) {
325 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINBUFFERERROR
);
329 x
= xfrm_state_lookup(net
, mark
, daddr
, spi
, nexthdr
, family
);
331 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINNOSTATES
);
332 xfrm_audit_state_notfound(skb
, family
, spi
, seq
);
336 skb
->sp
->xvec
[skb
->sp
->len
++] = x
;
341 if (unlikely(x
->km
.state
!= XFRM_STATE_VALID
)) {
342 if (x
->km
.state
== XFRM_STATE_ACQ
)
343 XFRM_INC_STATS(net
, LINUX_MIB_XFRMACQUIREERROR
);
346 LINUX_MIB_XFRMINSTATEINVALID
);
350 if ((x
->encap
? x
->encap
->encap_type
: 0) != encap_type
) {
351 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMISMATCH
);
355 if (x
->repl
->check(x
, skb
, seq
)) {
356 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
360 if (xfrm_state_check_expire(x
)) {
361 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEEXPIRED
);
365 spin_unlock(&x
->lock
);
367 if (xfrm_tunnel_check(skb
, x
, family
)) {
368 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
372 seq_hi
= htonl(xfrm_replay_seqhi(x
, seq
));
374 XFRM_SKB_CB(skb
)->seq
.input
.low
= seq
;
375 XFRM_SKB_CB(skb
)->seq
.input
.hi
= seq_hi
;
381 nexthdr
= x
->type_offload
->input_tail(x
, skb
);
383 nexthdr
= x
->type
->input(x
, skb
);
385 if (nexthdr
== -EINPROGRESS
)
392 if (nexthdr
== -EBADMSG
) {
393 xfrm_audit_state_icvfail(x
, skb
,
395 x
->stats
.integrity_failed
++;
397 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
401 /* only the first xfrm gets the encap type */
404 if (async
&& x
->repl
->recheck(x
, skb
, seq
)) {
405 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
409 x
->repl
->advance(x
, seq
);
411 x
->curlft
.bytes
+= skb
->len
;
414 spin_unlock(&x
->lock
);
416 XFRM_MODE_SKB_CB(skb
)->protocol
= nexthdr
;
418 inner_mode
= x
->inner_mode
;
420 if (x
->sel
.family
== AF_UNSPEC
) {
421 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
422 if (inner_mode
== NULL
) {
423 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
428 if (inner_mode
->input(x
, skb
)) {
429 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
433 if (x
->outer_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
) {
439 * We need the inner address. However, we only get here for
440 * transport mode so the outer address is identical.
442 daddr
= &x
->id
.daddr
;
443 family
= x
->outer_mode
->afinfo
->family
;
445 err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
);
447 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
452 err
= xfrm_rcv_cb(skb
, family
, x
->type
->proto
, 0);
462 gro_cells_receive(&gro_cells
, skb
);
465 xo
= xfrm_offload(skb
);
467 xfrm_gro
= xo
->flags
& XFRM_GRO
;
469 err
= x
->inner_mode
->afinfo
->transport_finish(skb
, xfrm_gro
|| async
);
474 gro_cells_receive(&gro_cells
, skb
);
482 spin_unlock(&x
->lock
);
484 xfrm_rcv_cb(skb
, family
, x
&& x
->type
? x
->type
->proto
: nexthdr
, -1);
488 EXPORT_SYMBOL(xfrm_input
);
490 int xfrm_input_resume(struct sk_buff
*skb
, int nexthdr
)
492 return xfrm_input(skb
, nexthdr
, 0, -1);
494 EXPORT_SYMBOL(xfrm_input_resume
);
496 static void xfrm_trans_reinject(unsigned long data
)
498 struct xfrm_trans_tasklet
*trans
= (void *)data
;
499 struct sk_buff_head queue
;
502 __skb_queue_head_init(&queue
);
503 skb_queue_splice_init(&trans
->queue
, &queue
);
505 while ((skb
= __skb_dequeue(&queue
)))
506 XFRM_TRANS_SKB_CB(skb
)->finish(dev_net(skb
->dev
), NULL
, skb
);
509 int xfrm_trans_queue(struct sk_buff
*skb
,
510 int (*finish
)(struct net
*, struct sock
*,
513 struct xfrm_trans_tasklet
*trans
;
515 trans
= this_cpu_ptr(&xfrm_trans_tasklet
);
517 if (skb_queue_len(&trans
->queue
) >= netdev_max_backlog
)
520 XFRM_TRANS_SKB_CB(skb
)->finish
= finish
;
521 skb_queue_tail(&trans
->queue
, skb
);
522 tasklet_schedule(&trans
->tasklet
);
525 EXPORT_SYMBOL(xfrm_trans_queue
);
527 void __init
xfrm_input_init(void)
532 init_dummy_netdev(&xfrm_napi_dev
);
533 err
= gro_cells_init(&gro_cells
, &xfrm_napi_dev
);
535 gro_cells
.cells
= NULL
;
537 secpath_cachep
= kmem_cache_create("secpath_cache",
538 sizeof(struct sec_path
),
539 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
,
542 for_each_possible_cpu(i
) {
543 struct xfrm_trans_tasklet
*trans
;
545 trans
= &per_cpu(xfrm_trans_tasklet
, i
);
546 __skb_queue_head_init(&trans
->queue
);
547 tasklet_init(&trans
->tasklet
, xfrm_trans_reinject
,
548 (unsigned long)trans
);