1 // SPDX-License-Identifier: GPL-2.0-only
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
10 * YOSHIFUJI Hideaki @USAGI
11 * Split up af-specific functions
12 * Derek Atkins <derek@ihtfp.com>
13 * Add UDP Encapsulation
17 #include <linux/workqueue.h>
19 #include <linux/pfkeyv2.h>
20 #include <linux/ipsec.h>
21 #include <linux/module.h>
22 #include <linux/cache.h>
23 #include <linux/audit.h>
24 #include <linux/uaccess.h>
25 #include <linux/ktime.h>
26 #include <linux/slab.h>
27 #include <linux/interrupt.h>
28 #include <linux/kernel.h>
30 #include <crypto/aead.h>
32 #include "xfrm_hash.h"
34 #define xfrm_state_deref_prot(table, net) \
35 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37 static void xfrm_state_gc_task(struct work_struct
*work
);
39 /* Each xfrm_state may be linked to two tables:
41 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
42 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
43 destination/tunnel endpoint. (output)
46 static unsigned int xfrm_state_hashmax __read_mostly
= 1 * 1024 * 1024;
47 static struct kmem_cache
*xfrm_state_cache __ro_after_init
;
49 static DECLARE_WORK(xfrm_state_gc_work
, xfrm_state_gc_task
);
50 static HLIST_HEAD(xfrm_state_gc_list
);
52 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu
*x
)
54 return refcount_inc_not_zero(&x
->refcnt
);
57 static inline unsigned int xfrm_dst_hash(struct net
*net
,
58 const xfrm_address_t
*daddr
,
59 const xfrm_address_t
*saddr
,
61 unsigned short family
)
63 return __xfrm_dst_hash(daddr
, saddr
, reqid
, family
, net
->xfrm
.state_hmask
);
66 static inline unsigned int xfrm_src_hash(struct net
*net
,
67 const xfrm_address_t
*daddr
,
68 const xfrm_address_t
*saddr
,
69 unsigned short family
)
71 return __xfrm_src_hash(daddr
, saddr
, family
, net
->xfrm
.state_hmask
);
74 static inline unsigned int
75 xfrm_spi_hash(struct net
*net
, const xfrm_address_t
*daddr
,
76 __be32 spi
, u8 proto
, unsigned short family
)
78 return __xfrm_spi_hash(daddr
, spi
, proto
, family
, net
->xfrm
.state_hmask
);
81 static unsigned int xfrm_seq_hash(struct net
*net
, u32 seq
)
83 return __xfrm_seq_hash(seq
, net
->xfrm
.state_hmask
);
86 static void xfrm_hash_transfer(struct hlist_head
*list
,
87 struct hlist_head
*ndsttable
,
88 struct hlist_head
*nsrctable
,
89 struct hlist_head
*nspitable
,
90 struct hlist_head
*nseqtable
,
91 unsigned int nhashmask
)
93 struct hlist_node
*tmp
;
96 hlist_for_each_entry_safe(x
, tmp
, list
, bydst
) {
99 h
= __xfrm_dst_hash(&x
->id
.daddr
, &x
->props
.saddr
,
100 x
->props
.reqid
, x
->props
.family
,
102 hlist_add_head_rcu(&x
->bydst
, ndsttable
+ h
);
104 h
= __xfrm_src_hash(&x
->id
.daddr
, &x
->props
.saddr
,
107 hlist_add_head_rcu(&x
->bysrc
, nsrctable
+ h
);
110 h
= __xfrm_spi_hash(&x
->id
.daddr
, x
->id
.spi
,
111 x
->id
.proto
, x
->props
.family
,
113 hlist_add_head_rcu(&x
->byspi
, nspitable
+ h
);
117 h
= __xfrm_seq_hash(x
->km
.seq
, nhashmask
);
118 hlist_add_head_rcu(&x
->byseq
, nseqtable
+ h
);
123 static unsigned long xfrm_hash_new_size(unsigned int state_hmask
)
125 return ((state_hmask
+ 1) << 1) * sizeof(struct hlist_head
);
128 static void xfrm_hash_resize(struct work_struct
*work
)
130 struct net
*net
= container_of(work
, struct net
, xfrm
.state_hash_work
);
131 struct hlist_head
*ndst
, *nsrc
, *nspi
, *nseq
, *odst
, *osrc
, *ospi
, *oseq
;
132 unsigned long nsize
, osize
;
133 unsigned int nhashmask
, ohashmask
;
136 nsize
= xfrm_hash_new_size(net
->xfrm
.state_hmask
);
137 ndst
= xfrm_hash_alloc(nsize
);
140 nsrc
= xfrm_hash_alloc(nsize
);
142 xfrm_hash_free(ndst
, nsize
);
145 nspi
= xfrm_hash_alloc(nsize
);
147 xfrm_hash_free(ndst
, nsize
);
148 xfrm_hash_free(nsrc
, nsize
);
151 nseq
= xfrm_hash_alloc(nsize
);
153 xfrm_hash_free(ndst
, nsize
);
154 xfrm_hash_free(nsrc
, nsize
);
155 xfrm_hash_free(nspi
, nsize
);
159 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
160 write_seqcount_begin(&net
->xfrm
.xfrm_state_hash_generation
);
162 nhashmask
= (nsize
/ sizeof(struct hlist_head
)) - 1U;
163 odst
= xfrm_state_deref_prot(net
->xfrm
.state_bydst
, net
);
164 for (i
= net
->xfrm
.state_hmask
; i
>= 0; i
--)
165 xfrm_hash_transfer(odst
+ i
, ndst
, nsrc
, nspi
, nseq
, nhashmask
);
167 osrc
= xfrm_state_deref_prot(net
->xfrm
.state_bysrc
, net
);
168 ospi
= xfrm_state_deref_prot(net
->xfrm
.state_byspi
, net
);
169 oseq
= xfrm_state_deref_prot(net
->xfrm
.state_byseq
, net
);
170 ohashmask
= net
->xfrm
.state_hmask
;
172 rcu_assign_pointer(net
->xfrm
.state_bydst
, ndst
);
173 rcu_assign_pointer(net
->xfrm
.state_bysrc
, nsrc
);
174 rcu_assign_pointer(net
->xfrm
.state_byspi
, nspi
);
175 rcu_assign_pointer(net
->xfrm
.state_byseq
, nseq
);
176 net
->xfrm
.state_hmask
= nhashmask
;
178 write_seqcount_end(&net
->xfrm
.xfrm_state_hash_generation
);
179 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
181 osize
= (ohashmask
+ 1) * sizeof(struct hlist_head
);
185 xfrm_hash_free(odst
, osize
);
186 xfrm_hash_free(osrc
, osize
);
187 xfrm_hash_free(ospi
, osize
);
188 xfrm_hash_free(oseq
, osize
);
191 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock
);
192 static struct xfrm_state_afinfo __rcu
*xfrm_state_afinfo
[NPROTO
];
194 static DEFINE_SPINLOCK(xfrm_state_gc_lock
);
196 int __xfrm_state_delete(struct xfrm_state
*x
);
198 int km_query(struct xfrm_state
*x
, struct xfrm_tmpl
*t
, struct xfrm_policy
*pol
);
199 static bool km_is_alive(const struct km_event
*c
);
200 void km_state_expired(struct xfrm_state
*x
, int hard
, u32 portid
);
202 int xfrm_register_type(const struct xfrm_type
*type
, unsigned short family
)
204 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
208 return -EAFNOSUPPORT
;
210 #define X(afi, T, name) do { \
211 WARN_ON((afi)->type_ ## name); \
212 (afi)->type_ ## name = (T); \
215 switch (type
->proto
) {
217 X(afinfo
, type
, comp
);
223 X(afinfo
, type
, esp
);
226 X(afinfo
, type
, ipip
);
228 case IPPROTO_DSTOPTS
:
229 X(afinfo
, type
, dstopts
);
231 case IPPROTO_ROUTING
:
232 X(afinfo
, type
, routing
);
235 X(afinfo
, type
, ipip6
);
239 err
= -EPROTONOSUPPORT
;
246 EXPORT_SYMBOL(xfrm_register_type
);
248 void xfrm_unregister_type(const struct xfrm_type
*type
, unsigned short family
)
250 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
252 if (unlikely(afinfo
== NULL
))
255 #define X(afi, T, name) do { \
256 WARN_ON((afi)->type_ ## name != (T)); \
257 (afi)->type_ ## name = NULL; \
260 switch (type
->proto
) {
262 X(afinfo
, type
, comp
);
268 X(afinfo
, type
, esp
);
271 X(afinfo
, type
, ipip
);
273 case IPPROTO_DSTOPTS
:
274 X(afinfo
, type
, dstopts
);
276 case IPPROTO_ROUTING
:
277 X(afinfo
, type
, routing
);
280 X(afinfo
, type
, ipip6
);
289 EXPORT_SYMBOL(xfrm_unregister_type
);
291 static const struct xfrm_type
*xfrm_get_type(u8 proto
, unsigned short family
)
293 const struct xfrm_type
*type
= NULL
;
294 struct xfrm_state_afinfo
*afinfo
;
295 int modload_attempted
= 0;
298 afinfo
= xfrm_state_get_afinfo(family
);
299 if (unlikely(afinfo
== NULL
))
304 type
= afinfo
->type_comp
;
307 type
= afinfo
->type_ah
;
310 type
= afinfo
->type_esp
;
313 type
= afinfo
->type_ipip
;
315 case IPPROTO_DSTOPTS
:
316 type
= afinfo
->type_dstopts
;
318 case IPPROTO_ROUTING
:
319 type
= afinfo
->type_routing
;
322 type
= afinfo
->type_ipip6
;
328 if (unlikely(type
&& !try_module_get(type
->owner
)))
333 if (!type
&& !modload_attempted
) {
334 request_module("xfrm-type-%d-%d", family
, proto
);
335 modload_attempted
= 1;
342 static void xfrm_put_type(const struct xfrm_type
*type
)
344 module_put(type
->owner
);
347 int xfrm_register_type_offload(const struct xfrm_type_offload
*type
,
348 unsigned short family
)
350 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
353 if (unlikely(afinfo
== NULL
))
354 return -EAFNOSUPPORT
;
356 switch (type
->proto
) {
358 WARN_ON(afinfo
->type_offload_esp
);
359 afinfo
->type_offload_esp
= type
;
363 err
= -EPROTONOSUPPORT
;
370 EXPORT_SYMBOL(xfrm_register_type_offload
);
372 void xfrm_unregister_type_offload(const struct xfrm_type_offload
*type
,
373 unsigned short family
)
375 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
377 if (unlikely(afinfo
== NULL
))
380 switch (type
->proto
) {
382 WARN_ON(afinfo
->type_offload_esp
!= type
);
383 afinfo
->type_offload_esp
= NULL
;
391 EXPORT_SYMBOL(xfrm_unregister_type_offload
);
393 static const struct xfrm_type_offload
*
394 xfrm_get_type_offload(u8 proto
, unsigned short family
, bool try_load
)
396 const struct xfrm_type_offload
*type
= NULL
;
397 struct xfrm_state_afinfo
*afinfo
;
400 afinfo
= xfrm_state_get_afinfo(family
);
401 if (unlikely(afinfo
== NULL
))
406 type
= afinfo
->type_offload_esp
;
412 if ((type
&& !try_module_get(type
->owner
)))
417 if (!type
&& try_load
) {
418 request_module("xfrm-offload-%d-%d", family
, proto
);
426 static void xfrm_put_type_offload(const struct xfrm_type_offload
*type
)
428 module_put(type
->owner
);
431 static const struct xfrm_mode xfrm4_mode_map
[XFRM_MODE_MAX
] = {
433 .encap
= XFRM_MODE_BEET
,
434 .flags
= XFRM_MODE_FLAG_TUNNEL
,
437 [XFRM_MODE_TRANSPORT
] = {
438 .encap
= XFRM_MODE_TRANSPORT
,
441 [XFRM_MODE_TUNNEL
] = {
442 .encap
= XFRM_MODE_TUNNEL
,
443 .flags
= XFRM_MODE_FLAG_TUNNEL
,
448 static const struct xfrm_mode xfrm6_mode_map
[XFRM_MODE_MAX
] = {
450 .encap
= XFRM_MODE_BEET
,
451 .flags
= XFRM_MODE_FLAG_TUNNEL
,
454 [XFRM_MODE_ROUTEOPTIMIZATION
] = {
455 .encap
= XFRM_MODE_ROUTEOPTIMIZATION
,
458 [XFRM_MODE_TRANSPORT
] = {
459 .encap
= XFRM_MODE_TRANSPORT
,
462 [XFRM_MODE_TUNNEL
] = {
463 .encap
= XFRM_MODE_TUNNEL
,
464 .flags
= XFRM_MODE_FLAG_TUNNEL
,
469 static const struct xfrm_mode
*xfrm_get_mode(unsigned int encap
, int family
)
471 const struct xfrm_mode
*mode
;
473 if (unlikely(encap
>= XFRM_MODE_MAX
))
478 mode
= &xfrm4_mode_map
[encap
];
479 if (mode
->family
== family
)
483 mode
= &xfrm6_mode_map
[encap
];
484 if (mode
->family
== family
)
494 void xfrm_state_free(struct xfrm_state
*x
)
496 kmem_cache_free(xfrm_state_cache
, x
);
498 EXPORT_SYMBOL(xfrm_state_free
);
500 static void ___xfrm_state_destroy(struct xfrm_state
*x
)
502 hrtimer_cancel(&x
->mtimer
);
503 del_timer_sync(&x
->rtimer
);
510 kfree(x
->replay_esn
);
511 kfree(x
->preplay_esn
);
513 xfrm_put_type_offload(x
->type_offload
);
515 x
->type
->destructor(x
);
516 xfrm_put_type(x
->type
);
519 put_page(x
->xfrag
.page
);
520 xfrm_dev_state_free(x
);
521 security_xfrm_state_free(x
);
525 static void xfrm_state_gc_task(struct work_struct
*work
)
527 struct xfrm_state
*x
;
528 struct hlist_node
*tmp
;
529 struct hlist_head gc_list
;
531 spin_lock_bh(&xfrm_state_gc_lock
);
532 hlist_move_list(&xfrm_state_gc_list
, &gc_list
);
533 spin_unlock_bh(&xfrm_state_gc_lock
);
537 hlist_for_each_entry_safe(x
, tmp
, &gc_list
, gclist
)
538 ___xfrm_state_destroy(x
);
541 static enum hrtimer_restart
xfrm_timer_handler(struct hrtimer
*me
)
543 struct xfrm_state
*x
= container_of(me
, struct xfrm_state
, mtimer
);
544 enum hrtimer_restart ret
= HRTIMER_NORESTART
;
545 time64_t now
= ktime_get_real_seconds();
546 time64_t next
= TIME64_MAX
;
551 if (x
->km
.state
== XFRM_STATE_DEAD
)
553 if (x
->km
.state
== XFRM_STATE_EXPIRED
)
555 if (x
->lft
.hard_add_expires_seconds
) {
556 long tmo
= x
->lft
.hard_add_expires_seconds
+
557 x
->curlft
.add_time
- now
;
559 if (x
->xflags
& XFRM_SOFT_EXPIRE
) {
560 /* enter hard expire without soft expire first?!
561 * setting a new date could trigger this.
562 * workaround: fix x->curflt.add_time by below:
564 x
->curlft
.add_time
= now
- x
->saved_tmo
- 1;
565 tmo
= x
->lft
.hard_add_expires_seconds
- x
->saved_tmo
;
572 if (x
->lft
.hard_use_expires_seconds
) {
573 long tmo
= x
->lft
.hard_use_expires_seconds
+
574 (x
->curlft
.use_time
? : now
) - now
;
582 if (x
->lft
.soft_add_expires_seconds
) {
583 long tmo
= x
->lft
.soft_add_expires_seconds
+
584 x
->curlft
.add_time
- now
;
587 x
->xflags
&= ~XFRM_SOFT_EXPIRE
;
588 } else if (tmo
< next
) {
590 x
->xflags
|= XFRM_SOFT_EXPIRE
;
594 if (x
->lft
.soft_use_expires_seconds
) {
595 long tmo
= x
->lft
.soft_use_expires_seconds
+
596 (x
->curlft
.use_time
? : now
) - now
;
605 km_state_expired(x
, 0, 0);
607 if (next
!= TIME64_MAX
) {
608 hrtimer_forward_now(&x
->mtimer
, ktime_set(next
, 0));
609 ret
= HRTIMER_RESTART
;
615 if (x
->km
.state
== XFRM_STATE_ACQ
&& x
->id
.spi
== 0)
616 x
->km
.state
= XFRM_STATE_EXPIRED
;
618 err
= __xfrm_state_delete(x
);
620 km_state_expired(x
, 1, 0);
622 xfrm_audit_state_delete(x
, err
? 0 : 1, true);
625 spin_unlock(&x
->lock
);
629 static void xfrm_replay_timer_handler(struct timer_list
*t
);
631 struct xfrm_state
*xfrm_state_alloc(struct net
*net
)
633 struct xfrm_state
*x
;
635 x
= kmem_cache_zalloc(xfrm_state_cache
, GFP_ATOMIC
);
638 write_pnet(&x
->xs_net
, net
);
639 refcount_set(&x
->refcnt
, 1);
640 atomic_set(&x
->tunnel_users
, 0);
641 INIT_LIST_HEAD(&x
->km
.all
);
642 INIT_HLIST_NODE(&x
->bydst
);
643 INIT_HLIST_NODE(&x
->bysrc
);
644 INIT_HLIST_NODE(&x
->byspi
);
645 INIT_HLIST_NODE(&x
->byseq
);
646 hrtimer_init(&x
->mtimer
, CLOCK_BOOTTIME
, HRTIMER_MODE_ABS_SOFT
);
647 x
->mtimer
.function
= xfrm_timer_handler
;
648 timer_setup(&x
->rtimer
, xfrm_replay_timer_handler
, 0);
649 x
->curlft
.add_time
= ktime_get_real_seconds();
650 x
->lft
.soft_byte_limit
= XFRM_INF
;
651 x
->lft
.soft_packet_limit
= XFRM_INF
;
652 x
->lft
.hard_byte_limit
= XFRM_INF
;
653 x
->lft
.hard_packet_limit
= XFRM_INF
;
654 x
->replay_maxage
= 0;
655 x
->replay_maxdiff
= 0;
656 spin_lock_init(&x
->lock
);
660 EXPORT_SYMBOL(xfrm_state_alloc
);
662 void __xfrm_state_destroy(struct xfrm_state
*x
, bool sync
)
664 WARN_ON(x
->km
.state
!= XFRM_STATE_DEAD
);
668 ___xfrm_state_destroy(x
);
670 spin_lock_bh(&xfrm_state_gc_lock
);
671 hlist_add_head(&x
->gclist
, &xfrm_state_gc_list
);
672 spin_unlock_bh(&xfrm_state_gc_lock
);
673 schedule_work(&xfrm_state_gc_work
);
676 EXPORT_SYMBOL(__xfrm_state_destroy
);
678 int __xfrm_state_delete(struct xfrm_state
*x
)
680 struct net
*net
= xs_net(x
);
683 if (x
->km
.state
!= XFRM_STATE_DEAD
) {
684 x
->km
.state
= XFRM_STATE_DEAD
;
685 spin_lock(&net
->xfrm
.xfrm_state_lock
);
686 list_del(&x
->km
.all
);
687 hlist_del_rcu(&x
->bydst
);
688 hlist_del_rcu(&x
->bysrc
);
690 hlist_del_rcu(&x
->byseq
);
692 hlist_del_rcu(&x
->byspi
);
693 net
->xfrm
.state_num
--;
694 spin_unlock(&net
->xfrm
.xfrm_state_lock
);
697 sock_put(rcu_dereference_raw(x
->encap_sk
));
699 xfrm_dev_state_delete(x
);
701 /* All xfrm_state objects are created by xfrm_state_alloc.
702 * The xfrm_state_alloc call gives a reference, and that
703 * is what we are dropping here.
711 EXPORT_SYMBOL(__xfrm_state_delete
);
713 int xfrm_state_delete(struct xfrm_state
*x
)
717 spin_lock_bh(&x
->lock
);
718 err
= __xfrm_state_delete(x
);
719 spin_unlock_bh(&x
->lock
);
723 EXPORT_SYMBOL(xfrm_state_delete
);
725 #ifdef CONFIG_SECURITY_NETWORK_XFRM
727 xfrm_state_flush_secctx_check(struct net
*net
, u8 proto
, bool task_valid
)
731 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
732 struct xfrm_state
*x
;
734 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
735 if (xfrm_id_proto_match(x
->id
.proto
, proto
) &&
736 (err
= security_xfrm_state_delete(x
)) != 0) {
737 xfrm_audit_state_delete(x
, 0, task_valid
);
747 xfrm_dev_state_flush_secctx_check(struct net
*net
, struct net_device
*dev
, bool task_valid
)
751 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
752 struct xfrm_state
*x
;
753 struct xfrm_state_offload
*xso
;
755 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
758 if (xso
->dev
== dev
&&
759 (err
= security_xfrm_state_delete(x
)) != 0) {
760 xfrm_audit_state_delete(x
, 0, task_valid
);
770 xfrm_state_flush_secctx_check(struct net
*net
, u8 proto
, bool task_valid
)
776 xfrm_dev_state_flush_secctx_check(struct net
*net
, struct net_device
*dev
, bool task_valid
)
782 int xfrm_state_flush(struct net
*net
, u8 proto
, bool task_valid
, bool sync
)
784 int i
, err
= 0, cnt
= 0;
786 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
787 err
= xfrm_state_flush_secctx_check(net
, proto
, task_valid
);
792 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
793 struct xfrm_state
*x
;
795 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
796 if (!xfrm_state_kern(x
) &&
797 xfrm_id_proto_match(x
->id
.proto
, proto
)) {
799 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
801 err
= xfrm_state_delete(x
);
802 xfrm_audit_state_delete(x
, err
? 0 : 1,
805 xfrm_state_put_sync(x
);
811 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
817 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
823 EXPORT_SYMBOL(xfrm_state_flush
);
825 int xfrm_dev_state_flush(struct net
*net
, struct net_device
*dev
, bool task_valid
)
827 int i
, err
= 0, cnt
= 0;
829 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
830 err
= xfrm_dev_state_flush_secctx_check(net
, dev
, task_valid
);
835 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
836 struct xfrm_state
*x
;
837 struct xfrm_state_offload
*xso
;
839 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
842 if (!xfrm_state_kern(x
) && xso
->dev
== dev
) {
844 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
846 err
= xfrm_state_delete(x
);
847 xfrm_audit_state_delete(x
, err
? 0 : 1,
853 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
862 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
865 EXPORT_SYMBOL(xfrm_dev_state_flush
);
867 void xfrm_sad_getinfo(struct net
*net
, struct xfrmk_sadinfo
*si
)
869 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
870 si
->sadcnt
= net
->xfrm
.state_num
;
871 si
->sadhcnt
= net
->xfrm
.state_hmask
+ 1;
872 si
->sadhmcnt
= xfrm_state_hashmax
;
873 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
875 EXPORT_SYMBOL(xfrm_sad_getinfo
);
878 __xfrm4_init_tempsel(struct xfrm_selector
*sel
, const struct flowi
*fl
)
880 const struct flowi4
*fl4
= &fl
->u
.ip4
;
882 sel
->daddr
.a4
= fl4
->daddr
;
883 sel
->saddr
.a4
= fl4
->saddr
;
884 sel
->dport
= xfrm_flowi_dport(fl
, &fl4
->uli
);
885 sel
->dport_mask
= htons(0xffff);
886 sel
->sport
= xfrm_flowi_sport(fl
, &fl4
->uli
);
887 sel
->sport_mask
= htons(0xffff);
888 sel
->family
= AF_INET
;
889 sel
->prefixlen_d
= 32;
890 sel
->prefixlen_s
= 32;
891 sel
->proto
= fl4
->flowi4_proto
;
892 sel
->ifindex
= fl4
->flowi4_oif
;
896 __xfrm6_init_tempsel(struct xfrm_selector
*sel
, const struct flowi
*fl
)
898 const struct flowi6
*fl6
= &fl
->u
.ip6
;
900 /* Initialize temporary selector matching only to current session. */
901 *(struct in6_addr
*)&sel
->daddr
= fl6
->daddr
;
902 *(struct in6_addr
*)&sel
->saddr
= fl6
->saddr
;
903 sel
->dport
= xfrm_flowi_dport(fl
, &fl6
->uli
);
904 sel
->dport_mask
= htons(0xffff);
905 sel
->sport
= xfrm_flowi_sport(fl
, &fl6
->uli
);
906 sel
->sport_mask
= htons(0xffff);
907 sel
->family
= AF_INET6
;
908 sel
->prefixlen_d
= 128;
909 sel
->prefixlen_s
= 128;
910 sel
->proto
= fl6
->flowi6_proto
;
911 sel
->ifindex
= fl6
->flowi6_oif
;
915 xfrm_init_tempstate(struct xfrm_state
*x
, const struct flowi
*fl
,
916 const struct xfrm_tmpl
*tmpl
,
917 const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
918 unsigned short family
)
922 __xfrm4_init_tempsel(&x
->sel
, fl
);
925 __xfrm6_init_tempsel(&x
->sel
, fl
);
931 switch (tmpl
->encap_family
) {
933 if (x
->id
.daddr
.a4
== 0)
934 x
->id
.daddr
.a4
= daddr
->a4
;
935 x
->props
.saddr
= tmpl
->saddr
;
936 if (x
->props
.saddr
.a4
== 0)
937 x
->props
.saddr
.a4
= saddr
->a4
;
940 if (ipv6_addr_any((struct in6_addr
*)&x
->id
.daddr
))
941 memcpy(&x
->id
.daddr
, daddr
, sizeof(x
->sel
.daddr
));
942 memcpy(&x
->props
.saddr
, &tmpl
->saddr
, sizeof(x
->props
.saddr
));
943 if (ipv6_addr_any((struct in6_addr
*)&x
->props
.saddr
))
944 memcpy(&x
->props
.saddr
, saddr
, sizeof(x
->props
.saddr
));
948 x
->props
.mode
= tmpl
->mode
;
949 x
->props
.reqid
= tmpl
->reqid
;
950 x
->props
.family
= tmpl
->encap_family
;
953 static struct xfrm_state
*__xfrm_state_lookup(struct net
*net
, u32 mark
,
954 const xfrm_address_t
*daddr
,
955 __be32 spi
, u8 proto
,
956 unsigned short family
)
958 unsigned int h
= xfrm_spi_hash(net
, daddr
, spi
, proto
, family
);
959 struct xfrm_state
*x
;
961 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_byspi
+ h
, byspi
) {
962 if (x
->props
.family
!= family
||
964 x
->id
.proto
!= proto
||
965 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
))
968 if ((mark
& x
->mark
.m
) != x
->mark
.v
)
970 if (!xfrm_state_hold_rcu(x
))
978 static struct xfrm_state
*__xfrm_state_lookup_byaddr(struct net
*net
, u32 mark
,
979 const xfrm_address_t
*daddr
,
980 const xfrm_address_t
*saddr
,
981 u8 proto
, unsigned short family
)
983 unsigned int h
= xfrm_src_hash(net
, daddr
, saddr
, family
);
984 struct xfrm_state
*x
;
986 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bysrc
+ h
, bysrc
) {
987 if (x
->props
.family
!= family
||
988 x
->id
.proto
!= proto
||
989 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
) ||
990 !xfrm_addr_equal(&x
->props
.saddr
, saddr
, family
))
993 if ((mark
& x
->mark
.m
) != x
->mark
.v
)
995 if (!xfrm_state_hold_rcu(x
))
1003 static inline struct xfrm_state
*
1004 __xfrm_state_locate(struct xfrm_state
*x
, int use_spi
, int family
)
1006 struct net
*net
= xs_net(x
);
1007 u32 mark
= x
->mark
.v
& x
->mark
.m
;
1010 return __xfrm_state_lookup(net
, mark
, &x
->id
.daddr
,
1011 x
->id
.spi
, x
->id
.proto
, family
);
1013 return __xfrm_state_lookup_byaddr(net
, mark
,
1016 x
->id
.proto
, family
);
1019 static void xfrm_hash_grow_check(struct net
*net
, int have_hash_collision
)
1021 if (have_hash_collision
&&
1022 (net
->xfrm
.state_hmask
+ 1) < xfrm_state_hashmax
&&
1023 net
->xfrm
.state_num
> net
->xfrm
.state_hmask
)
1024 schedule_work(&net
->xfrm
.state_hash_work
);
1027 static void xfrm_state_look_at(struct xfrm_policy
*pol
, struct xfrm_state
*x
,
1028 const struct flowi
*fl
, unsigned short family
,
1029 struct xfrm_state
**best
, int *acq_in_progress
,
1032 /* Resolution logic:
1033 * 1. There is a valid state with matching selector. Done.
1034 * 2. Valid state with inappropriate selector. Skip.
1036 * Entering area of "sysdeps".
1038 * 3. If state is not valid, selector is temporary, it selects
1039 * only session which triggered previous resolution. Key
1040 * manager will do something to install a state with proper
1043 if (x
->km
.state
== XFRM_STATE_VALID
) {
1044 if ((x
->sel
.family
&&
1045 (x
->sel
.family
!= family
||
1046 !xfrm_selector_match(&x
->sel
, fl
, family
))) ||
1047 !security_xfrm_state_pol_flow_match(x
, pol
,
1048 &fl
->u
.__fl_common
))
1052 (*best
)->km
.dying
> x
->km
.dying
||
1053 ((*best
)->km
.dying
== x
->km
.dying
&&
1054 (*best
)->curlft
.add_time
< x
->curlft
.add_time
))
1056 } else if (x
->km
.state
== XFRM_STATE_ACQ
) {
1057 *acq_in_progress
= 1;
1058 } else if (x
->km
.state
== XFRM_STATE_ERROR
||
1059 x
->km
.state
== XFRM_STATE_EXPIRED
) {
1060 if ((!x
->sel
.family
||
1061 (x
->sel
.family
== family
&&
1062 xfrm_selector_match(&x
->sel
, fl
, family
))) &&
1063 security_xfrm_state_pol_flow_match(x
, pol
,
1064 &fl
->u
.__fl_common
))
1070 xfrm_state_find(const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
1071 const struct flowi
*fl
, struct xfrm_tmpl
*tmpl
,
1072 struct xfrm_policy
*pol
, int *err
,
1073 unsigned short family
, u32 if_id
)
1075 static xfrm_address_t saddr_wildcard
= { };
1076 struct net
*net
= xp_net(pol
);
1077 unsigned int h
, h_wildcard
;
1078 struct xfrm_state
*x
, *x0
, *to_put
;
1079 int acquire_in_progress
= 0;
1081 struct xfrm_state
*best
= NULL
;
1082 u32 mark
= pol
->mark
.v
& pol
->mark
.m
;
1083 unsigned short encap_family
= tmpl
->encap_family
;
1084 unsigned int sequence
;
1089 sequence
= read_seqcount_begin(&net
->xfrm
.xfrm_state_hash_generation
);
1092 h
= xfrm_dst_hash(net
, daddr
, saddr
, tmpl
->reqid
, encap_family
);
1093 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bydst
+ h
, bydst
) {
1094 if (x
->props
.family
== encap_family
&&
1095 x
->props
.reqid
== tmpl
->reqid
&&
1096 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1097 x
->if_id
== if_id
&&
1098 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1099 xfrm_state_addr_check(x
, daddr
, saddr
, encap_family
) &&
1100 tmpl
->mode
== x
->props
.mode
&&
1101 tmpl
->id
.proto
== x
->id
.proto
&&
1102 (tmpl
->id
.spi
== x
->id
.spi
|| !tmpl
->id
.spi
))
1103 xfrm_state_look_at(pol
, x
, fl
, family
,
1104 &best
, &acquire_in_progress
, &error
);
1106 if (best
|| acquire_in_progress
)
1109 h_wildcard
= xfrm_dst_hash(net
, daddr
, &saddr_wildcard
, tmpl
->reqid
, encap_family
);
1110 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bydst
+ h_wildcard
, bydst
) {
1111 if (x
->props
.family
== encap_family
&&
1112 x
->props
.reqid
== tmpl
->reqid
&&
1113 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1114 x
->if_id
== if_id
&&
1115 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1116 xfrm_addr_equal(&x
->id
.daddr
, daddr
, encap_family
) &&
1117 tmpl
->mode
== x
->props
.mode
&&
1118 tmpl
->id
.proto
== x
->id
.proto
&&
1119 (tmpl
->id
.spi
== x
->id
.spi
|| !tmpl
->id
.spi
))
1120 xfrm_state_look_at(pol
, x
, fl
, family
,
1121 &best
, &acquire_in_progress
, &error
);
1126 if (!x
&& !error
&& !acquire_in_progress
) {
1128 (x0
= __xfrm_state_lookup(net
, mark
, daddr
, tmpl
->id
.spi
,
1129 tmpl
->id
.proto
, encap_family
)) != NULL
) {
1136 /* If the KMs have no listeners (yet...), avoid allocating an SA
1137 * for each and every packet - garbage collection might not
1140 if (!km_is_alive(&c
)) {
1145 x
= xfrm_state_alloc(net
);
1150 /* Initialize temporary state matching only
1151 * to current session. */
1152 xfrm_init_tempstate(x
, fl
, tmpl
, daddr
, saddr
, family
);
1153 memcpy(&x
->mark
, &pol
->mark
, sizeof(x
->mark
));
1156 error
= security_xfrm_state_alloc_acquire(x
, pol
->security
, fl
->flowi_secid
);
1158 x
->km
.state
= XFRM_STATE_DEAD
;
1164 if (km_query(x
, tmpl
, pol
) == 0) {
1165 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1166 x
->km
.state
= XFRM_STATE_ACQ
;
1167 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1168 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1169 h
= xfrm_src_hash(net
, daddr
, saddr
, encap_family
);
1170 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1172 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
, encap_family
);
1173 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
1176 h
= xfrm_seq_hash(net
, x
->km
.seq
);
1177 hlist_add_head_rcu(&x
->byseq
, net
->xfrm
.state_byseq
+ h
);
1179 x
->lft
.hard_add_expires_seconds
= net
->xfrm
.sysctl_acq_expires
;
1180 hrtimer_start(&x
->mtimer
,
1181 ktime_set(net
->xfrm
.sysctl_acq_expires
, 0),
1182 HRTIMER_MODE_REL_SOFT
);
1183 net
->xfrm
.state_num
++;
1184 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1185 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1187 x
->km
.state
= XFRM_STATE_DEAD
;
1195 if (!xfrm_state_hold_rcu(x
)) {
1200 *err
= acquire_in_progress
? -EAGAIN
: error
;
1204 xfrm_state_put(to_put
);
1206 if (read_seqcount_retry(&net
->xfrm
.xfrm_state_hash_generation
, sequence
)) {
1218 xfrm_stateonly_find(struct net
*net
, u32 mark
, u32 if_id
,
1219 xfrm_address_t
*daddr
, xfrm_address_t
*saddr
,
1220 unsigned short family
, u8 mode
, u8 proto
, u32 reqid
)
1223 struct xfrm_state
*rx
= NULL
, *x
= NULL
;
1225 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1226 h
= xfrm_dst_hash(net
, daddr
, saddr
, reqid
, family
);
1227 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1228 if (x
->props
.family
== family
&&
1229 x
->props
.reqid
== reqid
&&
1230 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1231 x
->if_id
== if_id
&&
1232 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1233 xfrm_state_addr_check(x
, daddr
, saddr
, family
) &&
1234 mode
== x
->props
.mode
&&
1235 proto
== x
->id
.proto
&&
1236 x
->km
.state
== XFRM_STATE_VALID
) {
1243 xfrm_state_hold(rx
);
1244 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1249 EXPORT_SYMBOL(xfrm_stateonly_find
);
1251 struct xfrm_state
*xfrm_state_lookup_byspi(struct net
*net
, __be32 spi
,
1252 unsigned short family
)
1254 struct xfrm_state
*x
;
1255 struct xfrm_state_walk
*w
;
1257 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1258 list_for_each_entry(w
, &net
->xfrm
.state_all
, all
) {
1259 x
= container_of(w
, struct xfrm_state
, km
);
1260 if (x
->props
.family
!= family
||
1265 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1268 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1271 EXPORT_SYMBOL(xfrm_state_lookup_byspi
);
1273 static void __xfrm_state_insert(struct xfrm_state
*x
)
1275 struct net
*net
= xs_net(x
);
1278 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1280 h
= xfrm_dst_hash(net
, &x
->id
.daddr
, &x
->props
.saddr
,
1281 x
->props
.reqid
, x
->props
.family
);
1282 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1284 h
= xfrm_src_hash(net
, &x
->id
.daddr
, &x
->props
.saddr
, x
->props
.family
);
1285 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1288 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
,
1291 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
1295 h
= xfrm_seq_hash(net
, x
->km
.seq
);
1297 hlist_add_head_rcu(&x
->byseq
, net
->xfrm
.state_byseq
+ h
);
1300 hrtimer_start(&x
->mtimer
, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT
);
1301 if (x
->replay_maxage
)
1302 mod_timer(&x
->rtimer
, jiffies
+ x
->replay_maxage
);
1304 net
->xfrm
.state_num
++;
1306 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1309 /* net->xfrm.xfrm_state_lock is held */
1310 static void __xfrm_state_bump_genids(struct xfrm_state
*xnew
)
1312 struct net
*net
= xs_net(xnew
);
1313 unsigned short family
= xnew
->props
.family
;
1314 u32 reqid
= xnew
->props
.reqid
;
1315 struct xfrm_state
*x
;
1317 u32 mark
= xnew
->mark
.v
& xnew
->mark
.m
;
1318 u32 if_id
= xnew
->if_id
;
1320 h
= xfrm_dst_hash(net
, &xnew
->id
.daddr
, &xnew
->props
.saddr
, reqid
, family
);
1321 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1322 if (x
->props
.family
== family
&&
1323 x
->props
.reqid
== reqid
&&
1324 x
->if_id
== if_id
&&
1325 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1326 xfrm_addr_equal(&x
->id
.daddr
, &xnew
->id
.daddr
, family
) &&
1327 xfrm_addr_equal(&x
->props
.saddr
, &xnew
->props
.saddr
, family
))
1332 void xfrm_state_insert(struct xfrm_state
*x
)
1334 struct net
*net
= xs_net(x
);
1336 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1337 __xfrm_state_bump_genids(x
);
1338 __xfrm_state_insert(x
);
1339 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1341 EXPORT_SYMBOL(xfrm_state_insert
);
1343 /* net->xfrm.xfrm_state_lock is held */
1344 static struct xfrm_state
*__find_acq_core(struct net
*net
,
1345 const struct xfrm_mark
*m
,
1346 unsigned short family
, u8 mode
,
1347 u32 reqid
, u32 if_id
, u8 proto
,
1348 const xfrm_address_t
*daddr
,
1349 const xfrm_address_t
*saddr
,
1352 unsigned int h
= xfrm_dst_hash(net
, daddr
, saddr
, reqid
, family
);
1353 struct xfrm_state
*x
;
1354 u32 mark
= m
->v
& m
->m
;
1356 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1357 if (x
->props
.reqid
!= reqid
||
1358 x
->props
.mode
!= mode
||
1359 x
->props
.family
!= family
||
1360 x
->km
.state
!= XFRM_STATE_ACQ
||
1362 x
->id
.proto
!= proto
||
1363 (mark
& x
->mark
.m
) != x
->mark
.v
||
1364 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
) ||
1365 !xfrm_addr_equal(&x
->props
.saddr
, saddr
, family
))
1375 x
= xfrm_state_alloc(net
);
1379 x
->sel
.daddr
.a4
= daddr
->a4
;
1380 x
->sel
.saddr
.a4
= saddr
->a4
;
1381 x
->sel
.prefixlen_d
= 32;
1382 x
->sel
.prefixlen_s
= 32;
1383 x
->props
.saddr
.a4
= saddr
->a4
;
1384 x
->id
.daddr
.a4
= daddr
->a4
;
1388 x
->sel
.daddr
.in6
= daddr
->in6
;
1389 x
->sel
.saddr
.in6
= saddr
->in6
;
1390 x
->sel
.prefixlen_d
= 128;
1391 x
->sel
.prefixlen_s
= 128;
1392 x
->props
.saddr
.in6
= saddr
->in6
;
1393 x
->id
.daddr
.in6
= daddr
->in6
;
1397 x
->km
.state
= XFRM_STATE_ACQ
;
1398 x
->id
.proto
= proto
;
1399 x
->props
.family
= family
;
1400 x
->props
.mode
= mode
;
1401 x
->props
.reqid
= reqid
;
1405 x
->lft
.hard_add_expires_seconds
= net
->xfrm
.sysctl_acq_expires
;
1407 hrtimer_start(&x
->mtimer
,
1408 ktime_set(net
->xfrm
.sysctl_acq_expires
, 0),
1409 HRTIMER_MODE_REL_SOFT
);
1410 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1411 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1412 h
= xfrm_src_hash(net
, daddr
, saddr
, family
);
1413 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1415 net
->xfrm
.state_num
++;
1417 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1423 static struct xfrm_state
*__xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
);
1425 int xfrm_state_add(struct xfrm_state
*x
)
1427 struct net
*net
= xs_net(x
);
1428 struct xfrm_state
*x1
, *to_put
;
1431 u32 mark
= x
->mark
.v
& x
->mark
.m
;
1432 int use_spi
= xfrm_id_proto_match(x
->id
.proto
, IPSEC_PROTO_ANY
);
1434 family
= x
->props
.family
;
1438 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1440 x1
= __xfrm_state_locate(x
, use_spi
, family
);
1448 if (use_spi
&& x
->km
.seq
) {
1449 x1
= __xfrm_find_acq_byseq(net
, mark
, x
->km
.seq
);
1450 if (x1
&& ((x1
->id
.proto
!= x
->id
.proto
) ||
1451 !xfrm_addr_equal(&x1
->id
.daddr
, &x
->id
.daddr
, family
))) {
1458 x1
= __find_acq_core(net
, &x
->mark
, family
, x
->props
.mode
,
1459 x
->props
.reqid
, x
->if_id
, x
->id
.proto
,
1460 &x
->id
.daddr
, &x
->props
.saddr
, 0);
1462 __xfrm_state_bump_genids(x
);
1463 __xfrm_state_insert(x
);
1467 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1470 xfrm_state_delete(x1
);
1475 xfrm_state_put(to_put
);
1479 EXPORT_SYMBOL(xfrm_state_add
);
1481 #ifdef CONFIG_XFRM_MIGRATE
1482 static inline int clone_security(struct xfrm_state
*x
, struct xfrm_sec_ctx
*security
)
1484 struct xfrm_user_sec_ctx
*uctx
;
1485 int size
= sizeof(*uctx
) + security
->ctx_len
;
1488 uctx
= kmalloc(size
, GFP_KERNEL
);
1492 uctx
->exttype
= XFRMA_SEC_CTX
;
1494 uctx
->ctx_doi
= security
->ctx_doi
;
1495 uctx
->ctx_alg
= security
->ctx_alg
;
1496 uctx
->ctx_len
= security
->ctx_len
;
1497 memcpy(uctx
+ 1, security
->ctx_str
, security
->ctx_len
);
1498 err
= security_xfrm_state_alloc(x
, uctx
);
1506 static struct xfrm_state
*xfrm_state_clone(struct xfrm_state
*orig
,
1507 struct xfrm_encap_tmpl
*encap
)
1509 struct net
*net
= xs_net(orig
);
1510 struct xfrm_state
*x
= xfrm_state_alloc(net
);
1514 memcpy(&x
->id
, &orig
->id
, sizeof(x
->id
));
1515 memcpy(&x
->sel
, &orig
->sel
, sizeof(x
->sel
));
1516 memcpy(&x
->lft
, &orig
->lft
, sizeof(x
->lft
));
1517 x
->props
.mode
= orig
->props
.mode
;
1518 x
->props
.replay_window
= orig
->props
.replay_window
;
1519 x
->props
.reqid
= orig
->props
.reqid
;
1520 x
->props
.family
= orig
->props
.family
;
1521 x
->props
.saddr
= orig
->props
.saddr
;
1524 x
->aalg
= xfrm_algo_auth_clone(orig
->aalg
);
1528 x
->props
.aalgo
= orig
->props
.aalgo
;
1531 x
->aead
= xfrm_algo_aead_clone(orig
->aead
);
1532 x
->geniv
= orig
->geniv
;
1537 x
->ealg
= xfrm_algo_clone(orig
->ealg
);
1541 x
->props
.ealgo
= orig
->props
.ealgo
;
1544 x
->calg
= xfrm_algo_clone(orig
->calg
);
1548 x
->props
.calgo
= orig
->props
.calgo
;
1550 if (encap
|| orig
->encap
) {
1552 x
->encap
= kmemdup(encap
, sizeof(*x
->encap
),
1555 x
->encap
= kmemdup(orig
->encap
, sizeof(*x
->encap
),
1563 if (clone_security(x
, orig
->security
))
1567 x
->coaddr
= kmemdup(orig
->coaddr
, sizeof(*x
->coaddr
),
1573 if (orig
->replay_esn
) {
1574 if (xfrm_replay_clone(x
, orig
))
1578 memcpy(&x
->mark
, &orig
->mark
, sizeof(x
->mark
));
1579 memcpy(&x
->props
.smark
, &orig
->props
.smark
, sizeof(x
->props
.smark
));
1581 if (xfrm_init_state(x
) < 0)
1584 x
->props
.flags
= orig
->props
.flags
;
1585 x
->props
.extra_flags
= orig
->props
.extra_flags
;
1587 x
->if_id
= orig
->if_id
;
1588 x
->tfcpad
= orig
->tfcpad
;
1589 x
->replay_maxdiff
= orig
->replay_maxdiff
;
1590 x
->replay_maxage
= orig
->replay_maxage
;
1591 memcpy(&x
->curlft
, &orig
->curlft
, sizeof(x
->curlft
));
1592 x
->km
.state
= orig
->km
.state
;
1593 x
->km
.seq
= orig
->km
.seq
;
1594 x
->replay
= orig
->replay
;
1595 x
->preplay
= orig
->preplay
;
1605 struct xfrm_state
*xfrm_migrate_state_find(struct xfrm_migrate
*m
, struct net
*net
)
1608 struct xfrm_state
*x
= NULL
;
1610 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1613 h
= xfrm_dst_hash(net
, &m
->old_daddr
, &m
->old_saddr
,
1614 m
->reqid
, m
->old_family
);
1615 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1616 if (x
->props
.mode
!= m
->mode
||
1617 x
->id
.proto
!= m
->proto
)
1619 if (m
->reqid
&& x
->props
.reqid
!= m
->reqid
)
1621 if (!xfrm_addr_equal(&x
->id
.daddr
, &m
->old_daddr
,
1623 !xfrm_addr_equal(&x
->props
.saddr
, &m
->old_saddr
,
1630 h
= xfrm_src_hash(net
, &m
->old_daddr
, &m
->old_saddr
,
1632 hlist_for_each_entry(x
, net
->xfrm
.state_bysrc
+h
, bysrc
) {
1633 if (x
->props
.mode
!= m
->mode
||
1634 x
->id
.proto
!= m
->proto
)
1636 if (!xfrm_addr_equal(&x
->id
.daddr
, &m
->old_daddr
,
1638 !xfrm_addr_equal(&x
->props
.saddr
, &m
->old_saddr
,
1646 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1650 EXPORT_SYMBOL(xfrm_migrate_state_find
);
1652 struct xfrm_state
*xfrm_state_migrate(struct xfrm_state
*x
,
1653 struct xfrm_migrate
*m
,
1654 struct xfrm_encap_tmpl
*encap
)
1656 struct xfrm_state
*xc
;
1658 xc
= xfrm_state_clone(x
, encap
);
1662 memcpy(&xc
->id
.daddr
, &m
->new_daddr
, sizeof(xc
->id
.daddr
));
1663 memcpy(&xc
->props
.saddr
, &m
->new_saddr
, sizeof(xc
->props
.saddr
));
1666 if (xfrm_addr_equal(&x
->id
.daddr
, &m
->new_daddr
, m
->new_family
)) {
1667 /* a care is needed when the destination address of the
1668 state is to be updated as it is a part of triplet */
1669 xfrm_state_insert(xc
);
1671 if (xfrm_state_add(xc
) < 0)
1680 EXPORT_SYMBOL(xfrm_state_migrate
);
1683 int xfrm_state_update(struct xfrm_state
*x
)
1685 struct xfrm_state
*x1
, *to_put
;
1687 int use_spi
= xfrm_id_proto_match(x
->id
.proto
, IPSEC_PROTO_ANY
);
1688 struct net
*net
= xs_net(x
);
1692 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1693 x1
= __xfrm_state_locate(x
, use_spi
, x
->props
.family
);
1699 if (xfrm_state_kern(x1
)) {
1705 if (x1
->km
.state
== XFRM_STATE_ACQ
) {
1706 __xfrm_state_insert(x
);
1712 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1715 xfrm_state_put(to_put
);
1721 xfrm_state_delete(x1
);
1727 spin_lock_bh(&x1
->lock
);
1728 if (likely(x1
->km
.state
== XFRM_STATE_VALID
)) {
1729 if (x
->encap
&& x1
->encap
&&
1730 x
->encap
->encap_type
== x1
->encap
->encap_type
)
1731 memcpy(x1
->encap
, x
->encap
, sizeof(*x1
->encap
));
1732 else if (x
->encap
|| x1
->encap
)
1735 if (x
->coaddr
&& x1
->coaddr
) {
1736 memcpy(x1
->coaddr
, x
->coaddr
, sizeof(*x1
->coaddr
));
1738 if (!use_spi
&& memcmp(&x1
->sel
, &x
->sel
, sizeof(x1
->sel
)))
1739 memcpy(&x1
->sel
, &x
->sel
, sizeof(x1
->sel
));
1740 memcpy(&x1
->lft
, &x
->lft
, sizeof(x1
->lft
));
1743 hrtimer_start(&x1
->mtimer
, ktime_set(1, 0),
1744 HRTIMER_MODE_REL_SOFT
);
1745 if (x1
->curlft
.use_time
)
1746 xfrm_state_check_expire(x1
);
1748 if (x
->props
.smark
.m
|| x
->props
.smark
.v
|| x
->if_id
) {
1749 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1751 if (x
->props
.smark
.m
|| x
->props
.smark
.v
)
1752 x1
->props
.smark
= x
->props
.smark
;
1755 x1
->if_id
= x
->if_id
;
1757 __xfrm_state_bump_genids(x1
);
1758 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1762 x
->km
.state
= XFRM_STATE_DEAD
;
1763 __xfrm_state_put(x
);
1767 spin_unlock_bh(&x1
->lock
);
1773 EXPORT_SYMBOL(xfrm_state_update
);
1775 int xfrm_state_check_expire(struct xfrm_state
*x
)
1777 if (!x
->curlft
.use_time
)
1778 x
->curlft
.use_time
= ktime_get_real_seconds();
1780 if (x
->curlft
.bytes
>= x
->lft
.hard_byte_limit
||
1781 x
->curlft
.packets
>= x
->lft
.hard_packet_limit
) {
1782 x
->km
.state
= XFRM_STATE_EXPIRED
;
1783 hrtimer_start(&x
->mtimer
, 0, HRTIMER_MODE_REL_SOFT
);
1788 (x
->curlft
.bytes
>= x
->lft
.soft_byte_limit
||
1789 x
->curlft
.packets
>= x
->lft
.soft_packet_limit
)) {
1791 km_state_expired(x
, 0, 0);
1795 EXPORT_SYMBOL(xfrm_state_check_expire
);
1798 xfrm_state_lookup(struct net
*net
, u32 mark
, const xfrm_address_t
*daddr
, __be32 spi
,
1799 u8 proto
, unsigned short family
)
1801 struct xfrm_state
*x
;
1804 x
= __xfrm_state_lookup(net
, mark
, daddr
, spi
, proto
, family
);
1808 EXPORT_SYMBOL(xfrm_state_lookup
);
1811 xfrm_state_lookup_byaddr(struct net
*net
, u32 mark
,
1812 const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
1813 u8 proto
, unsigned short family
)
1815 struct xfrm_state
*x
;
1817 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1818 x
= __xfrm_state_lookup_byaddr(net
, mark
, daddr
, saddr
, proto
, family
);
1819 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1822 EXPORT_SYMBOL(xfrm_state_lookup_byaddr
);
1825 xfrm_find_acq(struct net
*net
, const struct xfrm_mark
*mark
, u8 mode
, u32 reqid
,
1826 u32 if_id
, u8 proto
, const xfrm_address_t
*daddr
,
1827 const xfrm_address_t
*saddr
, int create
, unsigned short family
)
1829 struct xfrm_state
*x
;
1831 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1832 x
= __find_acq_core(net
, mark
, family
, mode
, reqid
, if_id
, proto
, daddr
, saddr
, create
);
1833 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1837 EXPORT_SYMBOL(xfrm_find_acq
);
1839 #ifdef CONFIG_XFRM_SUB_POLICY
1840 #if IS_ENABLED(CONFIG_IPV6)
1841 /* distribution counting sort function for xfrm_state and xfrm_tmpl */
1843 __xfrm6_sort(void **dst
, void **src
, int n
,
1844 int (*cmp
)(const void *p
), int maxclass
)
1846 int count
[XFRM_MAX_DEPTH
] = { };
1847 int class[XFRM_MAX_DEPTH
];
1850 for (i
= 0; i
< n
; i
++) {
1851 int c
= cmp(src
[i
]);
1857 for (i
= 2; i
< maxclass
; i
++)
1858 count
[i
] += count
[i
- 1];
1860 for (i
= 0; i
< n
; i
++) {
1861 dst
[count
[class[i
] - 1]++] = src
[i
];
1866 /* Rule for xfrm_state:
1868 * rule 1: select IPsec transport except AH
1869 * rule 2: select MIPv6 RO or inbound trigger
1870 * rule 3: select IPsec transport AH
1871 * rule 4: select IPsec tunnel
1874 static int __xfrm6_state_sort_cmp(const void *p
)
1876 const struct xfrm_state
*v
= p
;
1878 switch (v
->props
.mode
) {
1879 case XFRM_MODE_TRANSPORT
:
1880 if (v
->id
.proto
!= IPPROTO_AH
)
1884 #if IS_ENABLED(CONFIG_IPV6_MIP6)
1885 case XFRM_MODE_ROUTEOPTIMIZATION
:
1886 case XFRM_MODE_IN_TRIGGER
:
1889 case XFRM_MODE_TUNNEL
:
1890 case XFRM_MODE_BEET
:
1896 /* Rule for xfrm_tmpl:
1898 * rule 1: select IPsec transport
1899 * rule 2: select MIPv6 RO or inbound trigger
1900 * rule 3: select IPsec tunnel
1903 static int __xfrm6_tmpl_sort_cmp(const void *p
)
1905 const struct xfrm_tmpl
*v
= p
;
1908 case XFRM_MODE_TRANSPORT
:
1910 #if IS_ENABLED(CONFIG_IPV6_MIP6)
1911 case XFRM_MODE_ROUTEOPTIMIZATION
:
1912 case XFRM_MODE_IN_TRIGGER
:
1915 case XFRM_MODE_TUNNEL
:
1916 case XFRM_MODE_BEET
:
1922 static inline int __xfrm6_state_sort_cmp(const void *p
) { return 5; }
1923 static inline int __xfrm6_tmpl_sort_cmp(const void *p
) { return 4; }
1926 __xfrm6_sort(void **dst
, void **src
, int n
,
1927 int (*cmp
)(const void *p
), int maxclass
)
1931 for (i
= 0; i
< n
; i
++)
1934 #endif /* CONFIG_IPV6 */
1937 xfrm_tmpl_sort(struct xfrm_tmpl
**dst
, struct xfrm_tmpl
**src
, int n
,
1938 unsigned short family
)
1942 if (family
== AF_INET6
)
1943 __xfrm6_sort((void **)dst
, (void **)src
, n
,
1944 __xfrm6_tmpl_sort_cmp
, 5);
1946 for (i
= 0; i
< n
; i
++)
1951 xfrm_state_sort(struct xfrm_state
**dst
, struct xfrm_state
**src
, int n
,
1952 unsigned short family
)
1956 if (family
== AF_INET6
)
1957 __xfrm6_sort((void **)dst
, (void **)src
, n
,
1958 __xfrm6_state_sort_cmp
, 6);
1960 for (i
= 0; i
< n
; i
++)
1965 /* Silly enough, but I'm lazy to build resolution list */
1967 static struct xfrm_state
*__xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
)
1969 unsigned int h
= xfrm_seq_hash(net
, seq
);
1970 struct xfrm_state
*x
;
1972 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_byseq
+ h
, byseq
) {
1973 if (x
->km
.seq
== seq
&&
1974 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1975 x
->km
.state
== XFRM_STATE_ACQ
) {
1984 struct xfrm_state
*xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
)
1986 struct xfrm_state
*x
;
1988 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1989 x
= __xfrm_find_acq_byseq(net
, mark
, seq
);
1990 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1993 EXPORT_SYMBOL(xfrm_find_acq_byseq
);
1995 u32
xfrm_get_acqseq(void)
1998 static atomic_t acqseq
;
2001 res
= atomic_inc_return(&acqseq
);
2006 EXPORT_SYMBOL(xfrm_get_acqseq
);
2008 int verify_spi_info(u8 proto
, u32 min
, u32 max
)
2016 /* IPCOMP spi is 16-bits. */
2030 EXPORT_SYMBOL(verify_spi_info
);
2032 int xfrm_alloc_spi(struct xfrm_state
*x
, u32 low
, u32 high
)
2034 struct net
*net
= xs_net(x
);
2036 struct xfrm_state
*x0
;
2038 __be32 minspi
= htonl(low
);
2039 __be32 maxspi
= htonl(high
);
2041 u32 mark
= x
->mark
.v
& x
->mark
.m
;
2043 spin_lock_bh(&x
->lock
);
2044 if (x
->km
.state
== XFRM_STATE_DEAD
)
2053 if (minspi
== maxspi
) {
2054 x0
= xfrm_state_lookup(net
, mark
, &x
->id
.daddr
, minspi
, x
->id
.proto
, x
->props
.family
);
2062 for (h
= 0; h
< high
-low
+1; h
++) {
2063 spi
= low
+ prandom_u32()%(high
-low
+1);
2064 x0
= xfrm_state_lookup(net
, mark
, &x
->id
.daddr
, htonl(spi
), x
->id
.proto
, x
->props
.family
);
2066 newspi
= htonl(spi
);
2073 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2075 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
, x
->props
.family
);
2076 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
2077 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2083 spin_unlock_bh(&x
->lock
);
2087 EXPORT_SYMBOL(xfrm_alloc_spi
);
2089 static bool __xfrm_state_filter_match(struct xfrm_state
*x
,
2090 struct xfrm_address_filter
*filter
)
2093 if ((filter
->family
== AF_INET
||
2094 filter
->family
== AF_INET6
) &&
2095 x
->props
.family
!= filter
->family
)
2098 return addr_match(&x
->props
.saddr
, &filter
->saddr
,
2100 addr_match(&x
->id
.daddr
, &filter
->daddr
,
2106 int xfrm_state_walk(struct net
*net
, struct xfrm_state_walk
*walk
,
2107 int (*func
)(struct xfrm_state
*, int, void*),
2110 struct xfrm_state
*state
;
2111 struct xfrm_state_walk
*x
;
2114 if (walk
->seq
!= 0 && list_empty(&walk
->all
))
2117 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2118 if (list_empty(&walk
->all
))
2119 x
= list_first_entry(&net
->xfrm
.state_all
, struct xfrm_state_walk
, all
);
2121 x
= list_first_entry(&walk
->all
, struct xfrm_state_walk
, all
);
2122 list_for_each_entry_from(x
, &net
->xfrm
.state_all
, all
) {
2123 if (x
->state
== XFRM_STATE_DEAD
)
2125 state
= container_of(x
, struct xfrm_state
, km
);
2126 if (!xfrm_id_proto_match(state
->id
.proto
, walk
->proto
))
2128 if (!__xfrm_state_filter_match(state
, walk
->filter
))
2130 err
= func(state
, walk
->seq
, data
);
2132 list_move_tail(&walk
->all
, &x
->all
);
2137 if (walk
->seq
== 0) {
2141 list_del_init(&walk
->all
);
2143 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2146 EXPORT_SYMBOL(xfrm_state_walk
);
2148 void xfrm_state_walk_init(struct xfrm_state_walk
*walk
, u8 proto
,
2149 struct xfrm_address_filter
*filter
)
2151 INIT_LIST_HEAD(&walk
->all
);
2152 walk
->proto
= proto
;
2153 walk
->state
= XFRM_STATE_DEAD
;
2155 walk
->filter
= filter
;
2157 EXPORT_SYMBOL(xfrm_state_walk_init
);
2159 void xfrm_state_walk_done(struct xfrm_state_walk
*walk
, struct net
*net
)
2161 kfree(walk
->filter
);
2163 if (list_empty(&walk
->all
))
2166 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2167 list_del(&walk
->all
);
2168 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2170 EXPORT_SYMBOL(xfrm_state_walk_done
);
2172 static void xfrm_replay_timer_handler(struct timer_list
*t
)
2174 struct xfrm_state
*x
= from_timer(x
, t
, rtimer
);
2176 spin_lock(&x
->lock
);
2178 if (x
->km
.state
== XFRM_STATE_VALID
) {
2179 if (xfrm_aevent_is_on(xs_net(x
)))
2180 xfrm_replay_notify(x
, XFRM_REPLAY_TIMEOUT
);
2182 x
->xflags
|= XFRM_TIME_DEFER
;
2185 spin_unlock(&x
->lock
);
2188 static LIST_HEAD(xfrm_km_list
);
2190 void km_policy_notify(struct xfrm_policy
*xp
, int dir
, const struct km_event
*c
)
2192 struct xfrm_mgr
*km
;
2195 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
)
2196 if (km
->notify_policy
)
2197 km
->notify_policy(xp
, dir
, c
);
2201 void km_state_notify(struct xfrm_state
*x
, const struct km_event
*c
)
2203 struct xfrm_mgr
*km
;
2205 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
)
2211 EXPORT_SYMBOL(km_policy_notify
);
2212 EXPORT_SYMBOL(km_state_notify
);
2214 void km_state_expired(struct xfrm_state
*x
, int hard
, u32 portid
)
2220 c
.event
= XFRM_MSG_EXPIRE
;
2221 km_state_notify(x
, &c
);
2224 EXPORT_SYMBOL(km_state_expired
);
2226 * We send to all registered managers regardless of failure
2227 * We are happy with one success
2229 int km_query(struct xfrm_state
*x
, struct xfrm_tmpl
*t
, struct xfrm_policy
*pol
)
2231 int err
= -EINVAL
, acqret
;
2232 struct xfrm_mgr
*km
;
2235 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2236 acqret
= km
->acquire(x
, t
, pol
);
2243 EXPORT_SYMBOL(km_query
);
2245 int km_new_mapping(struct xfrm_state
*x
, xfrm_address_t
*ipaddr
, __be16 sport
)
2248 struct xfrm_mgr
*km
;
2251 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2252 if (km
->new_mapping
)
2253 err
= km
->new_mapping(x
, ipaddr
, sport
);
2260 EXPORT_SYMBOL(km_new_mapping
);
2262 void km_policy_expired(struct xfrm_policy
*pol
, int dir
, int hard
, u32 portid
)
2268 c
.event
= XFRM_MSG_POLEXPIRE
;
2269 km_policy_notify(pol
, dir
, &c
);
2271 EXPORT_SYMBOL(km_policy_expired
);
2273 #ifdef CONFIG_XFRM_MIGRATE
2274 int km_migrate(const struct xfrm_selector
*sel
, u8 dir
, u8 type
,
2275 const struct xfrm_migrate
*m
, int num_migrate
,
2276 const struct xfrm_kmaddress
*k
,
2277 const struct xfrm_encap_tmpl
*encap
)
2281 struct xfrm_mgr
*km
;
2284 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2286 ret
= km
->migrate(sel
, dir
, type
, m
, num_migrate
, k
,
2295 EXPORT_SYMBOL(km_migrate
);
2298 int km_report(struct net
*net
, u8 proto
, struct xfrm_selector
*sel
, xfrm_address_t
*addr
)
2302 struct xfrm_mgr
*km
;
2305 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2307 ret
= km
->report(net
, proto
, sel
, addr
);
2315 EXPORT_SYMBOL(km_report
);
2317 static bool km_is_alive(const struct km_event
*c
)
2319 struct xfrm_mgr
*km
;
2320 bool is_alive
= false;
2323 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2324 if (km
->is_alive
&& km
->is_alive(c
)) {
2334 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2335 static DEFINE_SPINLOCK(xfrm_translator_lock
);
2336 static struct xfrm_translator __rcu
*xfrm_translator
;
2338 struct xfrm_translator
*xfrm_get_translator(void)
2340 struct xfrm_translator
*xtr
;
2343 xtr
= rcu_dereference(xfrm_translator
);
2346 if (!try_module_get(xtr
->owner
))
2352 EXPORT_SYMBOL_GPL(xfrm_get_translator
);
2354 void xfrm_put_translator(struct xfrm_translator
*xtr
)
2356 module_put(xtr
->owner
);
2358 EXPORT_SYMBOL_GPL(xfrm_put_translator
);
2360 int xfrm_register_translator(struct xfrm_translator
*xtr
)
2364 spin_lock_bh(&xfrm_translator_lock
);
2365 if (unlikely(xfrm_translator
!= NULL
))
2368 rcu_assign_pointer(xfrm_translator
, xtr
);
2369 spin_unlock_bh(&xfrm_translator_lock
);
2373 EXPORT_SYMBOL_GPL(xfrm_register_translator
);
2375 int xfrm_unregister_translator(struct xfrm_translator
*xtr
)
2379 spin_lock_bh(&xfrm_translator_lock
);
2380 if (likely(xfrm_translator
!= NULL
)) {
2381 if (rcu_access_pointer(xfrm_translator
) != xtr
)
2384 RCU_INIT_POINTER(xfrm_translator
, NULL
);
2386 spin_unlock_bh(&xfrm_translator_lock
);
2391 EXPORT_SYMBOL_GPL(xfrm_unregister_translator
);
2394 int xfrm_user_policy(struct sock
*sk
, int optname
, sockptr_t optval
, int optlen
)
2398 struct xfrm_mgr
*km
;
2399 struct xfrm_policy
*pol
= NULL
;
2401 if (sockptr_is_null(optval
) && !optlen
) {
2402 xfrm_sk_policy_insert(sk
, XFRM_POLICY_IN
, NULL
);
2403 xfrm_sk_policy_insert(sk
, XFRM_POLICY_OUT
, NULL
);
2408 if (optlen
<= 0 || optlen
> PAGE_SIZE
)
2411 data
= memdup_sockptr(optval
, optlen
);
2413 return PTR_ERR(data
);
2415 if (in_compat_syscall()) {
2416 struct xfrm_translator
*xtr
= xfrm_get_translator();
2423 err
= xtr
->xlate_user_policy_sockptr(&data
, optlen
);
2424 xfrm_put_translator(xtr
);
2433 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2434 pol
= km
->compile_policy(sk
, optname
, data
,
2442 xfrm_sk_policy_insert(sk
, err
, pol
);
2451 EXPORT_SYMBOL(xfrm_user_policy
);
2453 static DEFINE_SPINLOCK(xfrm_km_lock
);
2455 int xfrm_register_km(struct xfrm_mgr
*km
)
2457 spin_lock_bh(&xfrm_km_lock
);
2458 list_add_tail_rcu(&km
->list
, &xfrm_km_list
);
2459 spin_unlock_bh(&xfrm_km_lock
);
2462 EXPORT_SYMBOL(xfrm_register_km
);
2464 int xfrm_unregister_km(struct xfrm_mgr
*km
)
2466 spin_lock_bh(&xfrm_km_lock
);
2467 list_del_rcu(&km
->list
);
2468 spin_unlock_bh(&xfrm_km_lock
);
2472 EXPORT_SYMBOL(xfrm_unregister_km
);
2474 int xfrm_state_register_afinfo(struct xfrm_state_afinfo
*afinfo
)
2478 if (WARN_ON(afinfo
->family
>= NPROTO
))
2479 return -EAFNOSUPPORT
;
2481 spin_lock_bh(&xfrm_state_afinfo_lock
);
2482 if (unlikely(xfrm_state_afinfo
[afinfo
->family
] != NULL
))
2485 rcu_assign_pointer(xfrm_state_afinfo
[afinfo
->family
], afinfo
);
2486 spin_unlock_bh(&xfrm_state_afinfo_lock
);
2489 EXPORT_SYMBOL(xfrm_state_register_afinfo
);
2491 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo
*afinfo
)
2493 int err
= 0, family
= afinfo
->family
;
2495 if (WARN_ON(family
>= NPROTO
))
2496 return -EAFNOSUPPORT
;
2498 spin_lock_bh(&xfrm_state_afinfo_lock
);
2499 if (likely(xfrm_state_afinfo
[afinfo
->family
] != NULL
)) {
2500 if (rcu_access_pointer(xfrm_state_afinfo
[family
]) != afinfo
)
2503 RCU_INIT_POINTER(xfrm_state_afinfo
[afinfo
->family
], NULL
);
2505 spin_unlock_bh(&xfrm_state_afinfo_lock
);
2509 EXPORT_SYMBOL(xfrm_state_unregister_afinfo
);
2511 struct xfrm_state_afinfo
*xfrm_state_afinfo_get_rcu(unsigned int family
)
2513 if (unlikely(family
>= NPROTO
))
2516 return rcu_dereference(xfrm_state_afinfo
[family
]);
2518 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu
);
2520 struct xfrm_state_afinfo
*xfrm_state_get_afinfo(unsigned int family
)
2522 struct xfrm_state_afinfo
*afinfo
;
2523 if (unlikely(family
>= NPROTO
))
2526 afinfo
= rcu_dereference(xfrm_state_afinfo
[family
]);
2527 if (unlikely(!afinfo
))
2532 void xfrm_flush_gc(void)
2534 flush_work(&xfrm_state_gc_work
);
2536 EXPORT_SYMBOL(xfrm_flush_gc
);
2538 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
2539 void xfrm_state_delete_tunnel(struct xfrm_state
*x
)
2542 struct xfrm_state
*t
= x
->tunnel
;
2544 if (atomic_read(&t
->tunnel_users
) == 2)
2545 xfrm_state_delete(t
);
2546 atomic_dec(&t
->tunnel_users
);
2547 xfrm_state_put_sync(t
);
2551 EXPORT_SYMBOL(xfrm_state_delete_tunnel
);
2553 u32
__xfrm_state_mtu(struct xfrm_state
*x
, int mtu
)
2555 const struct xfrm_type
*type
= READ_ONCE(x
->type
);
2556 struct crypto_aead
*aead
;
2557 u32 blksize
, net_adj
= 0;
2559 if (x
->km
.state
!= XFRM_STATE_VALID
||
2560 !type
|| type
->proto
!= IPPROTO_ESP
)
2561 return mtu
- x
->props
.header_len
;
2564 blksize
= ALIGN(crypto_aead_blocksize(aead
), 4);
2566 switch (x
->props
.mode
) {
2567 case XFRM_MODE_TRANSPORT
:
2568 case XFRM_MODE_BEET
:
2569 if (x
->props
.family
== AF_INET
)
2570 net_adj
= sizeof(struct iphdr
);
2571 else if (x
->props
.family
== AF_INET6
)
2572 net_adj
= sizeof(struct ipv6hdr
);
2574 case XFRM_MODE_TUNNEL
:
2581 return ((mtu
- x
->props
.header_len
- crypto_aead_authsize(aead
) -
2582 net_adj
) & ~(blksize
- 1)) + net_adj
- 2;
2584 EXPORT_SYMBOL_GPL(__xfrm_state_mtu
);
2586 u32
xfrm_state_mtu(struct xfrm_state
*x
, int mtu
)
2588 mtu
= __xfrm_state_mtu(x
, mtu
);
2590 if (x
->props
.family
== AF_INET6
&& mtu
< IPV6_MIN_MTU
)
2591 return IPV6_MIN_MTU
;
2596 int __xfrm_init_state(struct xfrm_state
*x
, bool init_replay
, bool offload
)
2598 const struct xfrm_mode
*inner_mode
;
2599 const struct xfrm_mode
*outer_mode
;
2600 int family
= x
->props
.family
;
2603 if (family
== AF_INET
&&
2604 xs_net(x
)->ipv4
.sysctl_ip_no_pmtu_disc
)
2605 x
->props
.flags
|= XFRM_STATE_NOPMTUDISC
;
2607 err
= -EPROTONOSUPPORT
;
2609 if (x
->sel
.family
!= AF_UNSPEC
) {
2610 inner_mode
= xfrm_get_mode(x
->props
.mode
, x
->sel
.family
);
2611 if (inner_mode
== NULL
)
2614 if (!(inner_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
) &&
2615 family
!= x
->sel
.family
)
2618 x
->inner_mode
= *inner_mode
;
2620 const struct xfrm_mode
*inner_mode_iaf
;
2621 int iafamily
= AF_INET
;
2623 inner_mode
= xfrm_get_mode(x
->props
.mode
, x
->props
.family
);
2624 if (inner_mode
== NULL
)
2627 if (!(inner_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
))
2630 x
->inner_mode
= *inner_mode
;
2632 if (x
->props
.family
== AF_INET
)
2633 iafamily
= AF_INET6
;
2635 inner_mode_iaf
= xfrm_get_mode(x
->props
.mode
, iafamily
);
2636 if (inner_mode_iaf
) {
2637 if (inner_mode_iaf
->flags
& XFRM_MODE_FLAG_TUNNEL
)
2638 x
->inner_mode_iaf
= *inner_mode_iaf
;
2642 x
->type
= xfrm_get_type(x
->id
.proto
, family
);
2643 if (x
->type
== NULL
)
2646 x
->type_offload
= xfrm_get_type_offload(x
->id
.proto
, family
, offload
);
2648 err
= x
->type
->init_state(x
);
2652 outer_mode
= xfrm_get_mode(x
->props
.mode
, family
);
2654 err
= -EPROTONOSUPPORT
;
2658 x
->outer_mode
= *outer_mode
;
2660 err
= xfrm_init_replay(x
);
2669 EXPORT_SYMBOL(__xfrm_init_state
);
2671 int xfrm_init_state(struct xfrm_state
*x
)
2675 err
= __xfrm_init_state(x
, true, false);
2677 x
->km
.state
= XFRM_STATE_VALID
;
2682 EXPORT_SYMBOL(xfrm_init_state
);
2684 int __net_init
xfrm_state_init(struct net
*net
)
2688 if (net_eq(net
, &init_net
))
2689 xfrm_state_cache
= KMEM_CACHE(xfrm_state
,
2690 SLAB_HWCACHE_ALIGN
| SLAB_PANIC
);
2692 INIT_LIST_HEAD(&net
->xfrm
.state_all
);
2694 sz
= sizeof(struct hlist_head
) * 8;
2696 net
->xfrm
.state_bydst
= xfrm_hash_alloc(sz
);
2697 if (!net
->xfrm
.state_bydst
)
2699 net
->xfrm
.state_bysrc
= xfrm_hash_alloc(sz
);
2700 if (!net
->xfrm
.state_bysrc
)
2702 net
->xfrm
.state_byspi
= xfrm_hash_alloc(sz
);
2703 if (!net
->xfrm
.state_byspi
)
2705 net
->xfrm
.state_byseq
= xfrm_hash_alloc(sz
);
2706 if (!net
->xfrm
.state_byseq
)
2708 net
->xfrm
.state_hmask
= ((sz
/ sizeof(struct hlist_head
)) - 1);
2710 net
->xfrm
.state_num
= 0;
2711 INIT_WORK(&net
->xfrm
.state_hash_work
, xfrm_hash_resize
);
2712 spin_lock_init(&net
->xfrm
.xfrm_state_lock
);
2713 seqcount_spinlock_init(&net
->xfrm
.xfrm_state_hash_generation
,
2714 &net
->xfrm
.xfrm_state_lock
);
2718 xfrm_hash_free(net
->xfrm
.state_byspi
, sz
);
2720 xfrm_hash_free(net
->xfrm
.state_bysrc
, sz
);
2722 xfrm_hash_free(net
->xfrm
.state_bydst
, sz
);
2727 void xfrm_state_fini(struct net
*net
)
2731 flush_work(&net
->xfrm
.state_hash_work
);
2732 flush_work(&xfrm_state_gc_work
);
2733 xfrm_state_flush(net
, 0, false, true);
2735 WARN_ON(!list_empty(&net
->xfrm
.state_all
));
2737 sz
= (net
->xfrm
.state_hmask
+ 1) * sizeof(struct hlist_head
);
2738 WARN_ON(!hlist_empty(net
->xfrm
.state_byseq
));
2739 xfrm_hash_free(net
->xfrm
.state_byseq
, sz
);
2740 WARN_ON(!hlist_empty(net
->xfrm
.state_byspi
));
2741 xfrm_hash_free(net
->xfrm
.state_byspi
, sz
);
2742 WARN_ON(!hlist_empty(net
->xfrm
.state_bysrc
));
2743 xfrm_hash_free(net
->xfrm
.state_bysrc
, sz
);
2744 WARN_ON(!hlist_empty(net
->xfrm
.state_bydst
));
2745 xfrm_hash_free(net
->xfrm
.state_bydst
, sz
);
2748 #ifdef CONFIG_AUDITSYSCALL
2749 static void xfrm_audit_helper_sainfo(struct xfrm_state
*x
,
2750 struct audit_buffer
*audit_buf
)
2752 struct xfrm_sec_ctx
*ctx
= x
->security
;
2753 u32 spi
= ntohl(x
->id
.spi
);
2756 audit_log_format(audit_buf
, " sec_alg=%u sec_doi=%u sec_obj=%s",
2757 ctx
->ctx_alg
, ctx
->ctx_doi
, ctx
->ctx_str
);
2759 switch (x
->props
.family
) {
2761 audit_log_format(audit_buf
, " src=%pI4 dst=%pI4",
2762 &x
->props
.saddr
.a4
, &x
->id
.daddr
.a4
);
2765 audit_log_format(audit_buf
, " src=%pI6 dst=%pI6",
2766 x
->props
.saddr
.a6
, x
->id
.daddr
.a6
);
2770 audit_log_format(audit_buf
, " spi=%u(0x%x)", spi
, spi
);
2773 static void xfrm_audit_helper_pktinfo(struct sk_buff
*skb
, u16 family
,
2774 struct audit_buffer
*audit_buf
)
2776 const struct iphdr
*iph4
;
2777 const struct ipv6hdr
*iph6
;
2782 audit_log_format(audit_buf
, " src=%pI4 dst=%pI4",
2783 &iph4
->saddr
, &iph4
->daddr
);
2786 iph6
= ipv6_hdr(skb
);
2787 audit_log_format(audit_buf
,
2788 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
2789 &iph6
->saddr
, &iph6
->daddr
,
2790 iph6
->flow_lbl
[0] & 0x0f,
2797 void xfrm_audit_state_add(struct xfrm_state
*x
, int result
, bool task_valid
)
2799 struct audit_buffer
*audit_buf
;
2801 audit_buf
= xfrm_audit_start("SAD-add");
2802 if (audit_buf
== NULL
)
2804 xfrm_audit_helper_usrinfo(task_valid
, audit_buf
);
2805 xfrm_audit_helper_sainfo(x
, audit_buf
);
2806 audit_log_format(audit_buf
, " res=%u", result
);
2807 audit_log_end(audit_buf
);
2809 EXPORT_SYMBOL_GPL(xfrm_audit_state_add
);
2811 void xfrm_audit_state_delete(struct xfrm_state
*x
, int result
, bool task_valid
)
2813 struct audit_buffer
*audit_buf
;
2815 audit_buf
= xfrm_audit_start("SAD-delete");
2816 if (audit_buf
== NULL
)
2818 xfrm_audit_helper_usrinfo(task_valid
, audit_buf
);
2819 xfrm_audit_helper_sainfo(x
, audit_buf
);
2820 audit_log_format(audit_buf
, " res=%u", result
);
2821 audit_log_end(audit_buf
);
2823 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete
);
2825 void xfrm_audit_state_replay_overflow(struct xfrm_state
*x
,
2826 struct sk_buff
*skb
)
2828 struct audit_buffer
*audit_buf
;
2831 audit_buf
= xfrm_audit_start("SA-replay-overflow");
2832 if (audit_buf
== NULL
)
2834 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2835 /* don't record the sequence number because it's inherent in this kind
2836 * of audit message */
2837 spi
= ntohl(x
->id
.spi
);
2838 audit_log_format(audit_buf
, " spi=%u(0x%x)", spi
, spi
);
2839 audit_log_end(audit_buf
);
2841 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow
);
2843 void xfrm_audit_state_replay(struct xfrm_state
*x
,
2844 struct sk_buff
*skb
, __be32 net_seq
)
2846 struct audit_buffer
*audit_buf
;
2849 audit_buf
= xfrm_audit_start("SA-replayed-pkt");
2850 if (audit_buf
== NULL
)
2852 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2853 spi
= ntohl(x
->id
.spi
);
2854 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2855 spi
, spi
, ntohl(net_seq
));
2856 audit_log_end(audit_buf
);
2858 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay
);
2860 void xfrm_audit_state_notfound_simple(struct sk_buff
*skb
, u16 family
)
2862 struct audit_buffer
*audit_buf
;
2864 audit_buf
= xfrm_audit_start("SA-notfound");
2865 if (audit_buf
== NULL
)
2867 xfrm_audit_helper_pktinfo(skb
, family
, audit_buf
);
2868 audit_log_end(audit_buf
);
2870 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple
);
2872 void xfrm_audit_state_notfound(struct sk_buff
*skb
, u16 family
,
2873 __be32 net_spi
, __be32 net_seq
)
2875 struct audit_buffer
*audit_buf
;
2878 audit_buf
= xfrm_audit_start("SA-notfound");
2879 if (audit_buf
== NULL
)
2881 xfrm_audit_helper_pktinfo(skb
, family
, audit_buf
);
2882 spi
= ntohl(net_spi
);
2883 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2884 spi
, spi
, ntohl(net_seq
));
2885 audit_log_end(audit_buf
);
2887 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound
);
2889 void xfrm_audit_state_icvfail(struct xfrm_state
*x
,
2890 struct sk_buff
*skb
, u8 proto
)
2892 struct audit_buffer
*audit_buf
;
2896 audit_buf
= xfrm_audit_start("SA-icv-failure");
2897 if (audit_buf
== NULL
)
2899 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2900 if (xfrm_parse_spi(skb
, proto
, &net_spi
, &net_seq
) == 0) {
2901 u32 spi
= ntohl(net_spi
);
2902 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2903 spi
, spi
, ntohl(net_seq
));
2905 audit_log_end(audit_buf
);
2907 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail
);
2908 #endif /* CONFIG_AUDITSYSCALL */