1 // SPDX-License-Identifier: GPL-2.0-only
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
10 * YOSHIFUJI Hideaki @USAGI
11 * Split up af-specific functions
12 * Derek Atkins <derek@ihtfp.com>
13 * Add UDP Encapsulation
17 #include <linux/workqueue.h>
19 #include <linux/pfkeyv2.h>
20 #include <linux/ipsec.h>
21 #include <linux/module.h>
22 #include <linux/cache.h>
23 #include <linux/audit.h>
24 #include <linux/uaccess.h>
25 #include <linux/ktime.h>
26 #include <linux/slab.h>
27 #include <linux/interrupt.h>
28 #include <linux/kernel.h>
30 #include <crypto/aead.h>
32 #include "xfrm_hash.h"
34 #define xfrm_state_deref_prot(table, net) \
35 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37 static void xfrm_state_gc_task(struct work_struct
*work
);
39 /* Each xfrm_state may be linked to two tables:
41 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
42 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
43 destination/tunnel endpoint. (output)
46 static unsigned int xfrm_state_hashmax __read_mostly
= 1 * 1024 * 1024;
47 static struct kmem_cache
*xfrm_state_cache __ro_after_init
;
49 static DECLARE_WORK(xfrm_state_gc_work
, xfrm_state_gc_task
);
50 static HLIST_HEAD(xfrm_state_gc_list
);
52 static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu
*x
)
54 return refcount_inc_not_zero(&x
->refcnt
);
57 static inline unsigned int xfrm_dst_hash(struct net
*net
,
58 const xfrm_address_t
*daddr
,
59 const xfrm_address_t
*saddr
,
61 unsigned short family
)
63 return __xfrm_dst_hash(daddr
, saddr
, reqid
, family
, net
->xfrm
.state_hmask
);
66 static inline unsigned int xfrm_src_hash(struct net
*net
,
67 const xfrm_address_t
*daddr
,
68 const xfrm_address_t
*saddr
,
69 unsigned short family
)
71 return __xfrm_src_hash(daddr
, saddr
, family
, net
->xfrm
.state_hmask
);
74 static inline unsigned int
75 xfrm_spi_hash(struct net
*net
, const xfrm_address_t
*daddr
,
76 __be32 spi
, u8 proto
, unsigned short family
)
78 return __xfrm_spi_hash(daddr
, spi
, proto
, family
, net
->xfrm
.state_hmask
);
81 static unsigned int xfrm_seq_hash(struct net
*net
, u32 seq
)
83 return __xfrm_seq_hash(seq
, net
->xfrm
.state_hmask
);
86 static void xfrm_hash_transfer(struct hlist_head
*list
,
87 struct hlist_head
*ndsttable
,
88 struct hlist_head
*nsrctable
,
89 struct hlist_head
*nspitable
,
90 struct hlist_head
*nseqtable
,
91 unsigned int nhashmask
)
93 struct hlist_node
*tmp
;
96 hlist_for_each_entry_safe(x
, tmp
, list
, bydst
) {
99 h
= __xfrm_dst_hash(&x
->id
.daddr
, &x
->props
.saddr
,
100 x
->props
.reqid
, x
->props
.family
,
102 hlist_add_head_rcu(&x
->bydst
, ndsttable
+ h
);
104 h
= __xfrm_src_hash(&x
->id
.daddr
, &x
->props
.saddr
,
107 hlist_add_head_rcu(&x
->bysrc
, nsrctable
+ h
);
110 h
= __xfrm_spi_hash(&x
->id
.daddr
, x
->id
.spi
,
111 x
->id
.proto
, x
->props
.family
,
113 hlist_add_head_rcu(&x
->byspi
, nspitable
+ h
);
117 h
= __xfrm_seq_hash(x
->km
.seq
, nhashmask
);
118 hlist_add_head_rcu(&x
->byseq
, nseqtable
+ h
);
123 static unsigned long xfrm_hash_new_size(unsigned int state_hmask
)
125 return ((state_hmask
+ 1) << 1) * sizeof(struct hlist_head
);
128 static void xfrm_hash_resize(struct work_struct
*work
)
130 struct net
*net
= container_of(work
, struct net
, xfrm
.state_hash_work
);
131 struct hlist_head
*ndst
, *nsrc
, *nspi
, *nseq
, *odst
, *osrc
, *ospi
, *oseq
;
132 unsigned long nsize
, osize
;
133 unsigned int nhashmask
, ohashmask
;
136 nsize
= xfrm_hash_new_size(net
->xfrm
.state_hmask
);
137 ndst
= xfrm_hash_alloc(nsize
);
140 nsrc
= xfrm_hash_alloc(nsize
);
142 xfrm_hash_free(ndst
, nsize
);
145 nspi
= xfrm_hash_alloc(nsize
);
147 xfrm_hash_free(ndst
, nsize
);
148 xfrm_hash_free(nsrc
, nsize
);
151 nseq
= xfrm_hash_alloc(nsize
);
153 xfrm_hash_free(ndst
, nsize
);
154 xfrm_hash_free(nsrc
, nsize
);
155 xfrm_hash_free(nspi
, nsize
);
159 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
160 write_seqcount_begin(&net
->xfrm
.xfrm_state_hash_generation
);
162 nhashmask
= (nsize
/ sizeof(struct hlist_head
)) - 1U;
163 odst
= xfrm_state_deref_prot(net
->xfrm
.state_bydst
, net
);
164 for (i
= net
->xfrm
.state_hmask
; i
>= 0; i
--)
165 xfrm_hash_transfer(odst
+ i
, ndst
, nsrc
, nspi
, nseq
, nhashmask
);
167 osrc
= xfrm_state_deref_prot(net
->xfrm
.state_bysrc
, net
);
168 ospi
= xfrm_state_deref_prot(net
->xfrm
.state_byspi
, net
);
169 oseq
= xfrm_state_deref_prot(net
->xfrm
.state_byseq
, net
);
170 ohashmask
= net
->xfrm
.state_hmask
;
172 rcu_assign_pointer(net
->xfrm
.state_bydst
, ndst
);
173 rcu_assign_pointer(net
->xfrm
.state_bysrc
, nsrc
);
174 rcu_assign_pointer(net
->xfrm
.state_byspi
, nspi
);
175 rcu_assign_pointer(net
->xfrm
.state_byseq
, nseq
);
176 net
->xfrm
.state_hmask
= nhashmask
;
178 write_seqcount_end(&net
->xfrm
.xfrm_state_hash_generation
);
179 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
181 osize
= (ohashmask
+ 1) * sizeof(struct hlist_head
);
185 xfrm_hash_free(odst
, osize
);
186 xfrm_hash_free(osrc
, osize
);
187 xfrm_hash_free(ospi
, osize
);
188 xfrm_hash_free(oseq
, osize
);
191 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock
);
192 static struct xfrm_state_afinfo __rcu
*xfrm_state_afinfo
[NPROTO
];
194 static DEFINE_SPINLOCK(xfrm_state_gc_lock
);
196 int __xfrm_state_delete(struct xfrm_state
*x
);
198 int km_query(struct xfrm_state
*x
, struct xfrm_tmpl
*t
, struct xfrm_policy
*pol
);
199 static bool km_is_alive(const struct km_event
*c
);
200 void km_state_expired(struct xfrm_state
*x
, int hard
, u32 portid
);
202 int xfrm_register_type(const struct xfrm_type
*type
, unsigned short family
)
204 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
208 return -EAFNOSUPPORT
;
210 #define X(afi, T, name) do { \
211 WARN_ON((afi)->type_ ## name); \
212 (afi)->type_ ## name = (T); \
215 switch (type
->proto
) {
217 X(afinfo
, type
, comp
);
223 X(afinfo
, type
, esp
);
226 X(afinfo
, type
, ipip
);
228 case IPPROTO_DSTOPTS
:
229 X(afinfo
, type
, dstopts
);
231 case IPPROTO_ROUTING
:
232 X(afinfo
, type
, routing
);
235 X(afinfo
, type
, ipip6
);
239 err
= -EPROTONOSUPPORT
;
246 EXPORT_SYMBOL(xfrm_register_type
);
248 void xfrm_unregister_type(const struct xfrm_type
*type
, unsigned short family
)
250 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
252 if (unlikely(afinfo
== NULL
))
255 #define X(afi, T, name) do { \
256 WARN_ON((afi)->type_ ## name != (T)); \
257 (afi)->type_ ## name = NULL; \
260 switch (type
->proto
) {
262 X(afinfo
, type
, comp
);
268 X(afinfo
, type
, esp
);
271 X(afinfo
, type
, ipip
);
273 case IPPROTO_DSTOPTS
:
274 X(afinfo
, type
, dstopts
);
276 case IPPROTO_ROUTING
:
277 X(afinfo
, type
, routing
);
280 X(afinfo
, type
, ipip6
);
289 EXPORT_SYMBOL(xfrm_unregister_type
);
291 static const struct xfrm_type
*xfrm_get_type(u8 proto
, unsigned short family
)
293 const struct xfrm_type
*type
= NULL
;
294 struct xfrm_state_afinfo
*afinfo
;
295 int modload_attempted
= 0;
298 afinfo
= xfrm_state_get_afinfo(family
);
299 if (unlikely(afinfo
== NULL
))
304 type
= afinfo
->type_comp
;
307 type
= afinfo
->type_ah
;
310 type
= afinfo
->type_esp
;
313 type
= afinfo
->type_ipip
;
315 case IPPROTO_DSTOPTS
:
316 type
= afinfo
->type_dstopts
;
318 case IPPROTO_ROUTING
:
319 type
= afinfo
->type_routing
;
322 type
= afinfo
->type_ipip6
;
328 if (unlikely(type
&& !try_module_get(type
->owner
)))
333 if (!type
&& !modload_attempted
) {
334 request_module("xfrm-type-%d-%d", family
, proto
);
335 modload_attempted
= 1;
342 static void xfrm_put_type(const struct xfrm_type
*type
)
344 module_put(type
->owner
);
347 int xfrm_register_type_offload(const struct xfrm_type_offload
*type
,
348 unsigned short family
)
350 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
353 if (unlikely(afinfo
== NULL
))
354 return -EAFNOSUPPORT
;
356 switch (type
->proto
) {
358 WARN_ON(afinfo
->type_offload_esp
);
359 afinfo
->type_offload_esp
= type
;
363 err
= -EPROTONOSUPPORT
;
370 EXPORT_SYMBOL(xfrm_register_type_offload
);
372 void xfrm_unregister_type_offload(const struct xfrm_type_offload
*type
,
373 unsigned short family
)
375 struct xfrm_state_afinfo
*afinfo
= xfrm_state_get_afinfo(family
);
377 if (unlikely(afinfo
== NULL
))
380 switch (type
->proto
) {
382 WARN_ON(afinfo
->type_offload_esp
!= type
);
383 afinfo
->type_offload_esp
= NULL
;
391 EXPORT_SYMBOL(xfrm_unregister_type_offload
);
393 static const struct xfrm_type_offload
*
394 xfrm_get_type_offload(u8 proto
, unsigned short family
, bool try_load
)
396 const struct xfrm_type_offload
*type
= NULL
;
397 struct xfrm_state_afinfo
*afinfo
;
400 afinfo
= xfrm_state_get_afinfo(family
);
401 if (unlikely(afinfo
== NULL
))
406 type
= afinfo
->type_offload_esp
;
412 if ((type
&& !try_module_get(type
->owner
)))
417 if (!type
&& try_load
) {
418 request_module("xfrm-offload-%d-%d", family
, proto
);
426 static void xfrm_put_type_offload(const struct xfrm_type_offload
*type
)
428 module_put(type
->owner
);
431 static const struct xfrm_mode xfrm4_mode_map
[XFRM_MODE_MAX
] = {
433 .encap
= XFRM_MODE_BEET
,
434 .flags
= XFRM_MODE_FLAG_TUNNEL
,
437 [XFRM_MODE_TRANSPORT
] = {
438 .encap
= XFRM_MODE_TRANSPORT
,
441 [XFRM_MODE_TUNNEL
] = {
442 .encap
= XFRM_MODE_TUNNEL
,
443 .flags
= XFRM_MODE_FLAG_TUNNEL
,
448 static const struct xfrm_mode xfrm6_mode_map
[XFRM_MODE_MAX
] = {
450 .encap
= XFRM_MODE_BEET
,
451 .flags
= XFRM_MODE_FLAG_TUNNEL
,
454 [XFRM_MODE_ROUTEOPTIMIZATION
] = {
455 .encap
= XFRM_MODE_ROUTEOPTIMIZATION
,
458 [XFRM_MODE_TRANSPORT
] = {
459 .encap
= XFRM_MODE_TRANSPORT
,
462 [XFRM_MODE_TUNNEL
] = {
463 .encap
= XFRM_MODE_TUNNEL
,
464 .flags
= XFRM_MODE_FLAG_TUNNEL
,
469 static const struct xfrm_mode
*xfrm_get_mode(unsigned int encap
, int family
)
471 const struct xfrm_mode
*mode
;
473 if (unlikely(encap
>= XFRM_MODE_MAX
))
478 mode
= &xfrm4_mode_map
[encap
];
479 if (mode
->family
== family
)
483 mode
= &xfrm6_mode_map
[encap
];
484 if (mode
->family
== family
)
494 void xfrm_state_free(struct xfrm_state
*x
)
496 kmem_cache_free(xfrm_state_cache
, x
);
498 EXPORT_SYMBOL(xfrm_state_free
);
500 static void ___xfrm_state_destroy(struct xfrm_state
*x
)
502 hrtimer_cancel(&x
->mtimer
);
503 del_timer_sync(&x
->rtimer
);
510 kfree(x
->replay_esn
);
511 kfree(x
->preplay_esn
);
513 xfrm_put_type_offload(x
->type_offload
);
515 x
->type
->destructor(x
);
516 xfrm_put_type(x
->type
);
519 put_page(x
->xfrag
.page
);
520 xfrm_dev_state_free(x
);
521 security_xfrm_state_free(x
);
525 static void xfrm_state_gc_task(struct work_struct
*work
)
527 struct xfrm_state
*x
;
528 struct hlist_node
*tmp
;
529 struct hlist_head gc_list
;
531 spin_lock_bh(&xfrm_state_gc_lock
);
532 hlist_move_list(&xfrm_state_gc_list
, &gc_list
);
533 spin_unlock_bh(&xfrm_state_gc_lock
);
537 hlist_for_each_entry_safe(x
, tmp
, &gc_list
, gclist
)
538 ___xfrm_state_destroy(x
);
541 static enum hrtimer_restart
xfrm_timer_handler(struct hrtimer
*me
)
543 struct xfrm_state
*x
= container_of(me
, struct xfrm_state
, mtimer
);
544 enum hrtimer_restart ret
= HRTIMER_NORESTART
;
545 time64_t now
= ktime_get_real_seconds();
546 time64_t next
= TIME64_MAX
;
551 if (x
->km
.state
== XFRM_STATE_DEAD
)
553 if (x
->km
.state
== XFRM_STATE_EXPIRED
)
555 if (x
->lft
.hard_add_expires_seconds
) {
556 long tmo
= x
->lft
.hard_add_expires_seconds
+
557 x
->curlft
.add_time
- now
;
559 if (x
->xflags
& XFRM_SOFT_EXPIRE
) {
560 /* enter hard expire without soft expire first?!
561 * setting a new date could trigger this.
562 * workaround: fix x->curflt.add_time by below:
564 x
->curlft
.add_time
= now
- x
->saved_tmo
- 1;
565 tmo
= x
->lft
.hard_add_expires_seconds
- x
->saved_tmo
;
572 if (x
->lft
.hard_use_expires_seconds
) {
573 long tmo
= x
->lft
.hard_use_expires_seconds
+
574 (x
->curlft
.use_time
? : now
) - now
;
582 if (x
->lft
.soft_add_expires_seconds
) {
583 long tmo
= x
->lft
.soft_add_expires_seconds
+
584 x
->curlft
.add_time
- now
;
587 x
->xflags
&= ~XFRM_SOFT_EXPIRE
;
588 } else if (tmo
< next
) {
590 x
->xflags
|= XFRM_SOFT_EXPIRE
;
594 if (x
->lft
.soft_use_expires_seconds
) {
595 long tmo
= x
->lft
.soft_use_expires_seconds
+
596 (x
->curlft
.use_time
? : now
) - now
;
605 km_state_expired(x
, 0, 0);
607 if (next
!= TIME64_MAX
) {
608 hrtimer_forward_now(&x
->mtimer
, ktime_set(next
, 0));
609 ret
= HRTIMER_RESTART
;
615 if (x
->km
.state
== XFRM_STATE_ACQ
&& x
->id
.spi
== 0)
616 x
->km
.state
= XFRM_STATE_EXPIRED
;
618 err
= __xfrm_state_delete(x
);
620 km_state_expired(x
, 1, 0);
622 xfrm_audit_state_delete(x
, err
? 0 : 1, true);
625 spin_unlock(&x
->lock
);
629 static void xfrm_replay_timer_handler(struct timer_list
*t
);
631 struct xfrm_state
*xfrm_state_alloc(struct net
*net
)
633 struct xfrm_state
*x
;
635 x
= kmem_cache_zalloc(xfrm_state_cache
, GFP_ATOMIC
);
638 write_pnet(&x
->xs_net
, net
);
639 refcount_set(&x
->refcnt
, 1);
640 atomic_set(&x
->tunnel_users
, 0);
641 INIT_LIST_HEAD(&x
->km
.all
);
642 INIT_HLIST_NODE(&x
->bydst
);
643 INIT_HLIST_NODE(&x
->bysrc
);
644 INIT_HLIST_NODE(&x
->byspi
);
645 INIT_HLIST_NODE(&x
->byseq
);
646 hrtimer_init(&x
->mtimer
, CLOCK_BOOTTIME
, HRTIMER_MODE_ABS_SOFT
);
647 x
->mtimer
.function
= xfrm_timer_handler
;
648 timer_setup(&x
->rtimer
, xfrm_replay_timer_handler
, 0);
649 x
->curlft
.add_time
= ktime_get_real_seconds();
650 x
->lft
.soft_byte_limit
= XFRM_INF
;
651 x
->lft
.soft_packet_limit
= XFRM_INF
;
652 x
->lft
.hard_byte_limit
= XFRM_INF
;
653 x
->lft
.hard_packet_limit
= XFRM_INF
;
654 x
->replay_maxage
= 0;
655 x
->replay_maxdiff
= 0;
656 spin_lock_init(&x
->lock
);
660 EXPORT_SYMBOL(xfrm_state_alloc
);
662 void __xfrm_state_destroy(struct xfrm_state
*x
, bool sync
)
664 WARN_ON(x
->km
.state
!= XFRM_STATE_DEAD
);
668 ___xfrm_state_destroy(x
);
670 spin_lock_bh(&xfrm_state_gc_lock
);
671 hlist_add_head(&x
->gclist
, &xfrm_state_gc_list
);
672 spin_unlock_bh(&xfrm_state_gc_lock
);
673 schedule_work(&xfrm_state_gc_work
);
676 EXPORT_SYMBOL(__xfrm_state_destroy
);
678 int __xfrm_state_delete(struct xfrm_state
*x
)
680 struct net
*net
= xs_net(x
);
683 if (x
->km
.state
!= XFRM_STATE_DEAD
) {
684 x
->km
.state
= XFRM_STATE_DEAD
;
685 spin_lock(&net
->xfrm
.xfrm_state_lock
);
686 list_del(&x
->km
.all
);
687 hlist_del_rcu(&x
->bydst
);
688 hlist_del_rcu(&x
->bysrc
);
690 hlist_del_rcu(&x
->byseq
);
692 hlist_del_rcu(&x
->byspi
);
693 net
->xfrm
.state_num
--;
694 spin_unlock(&net
->xfrm
.xfrm_state_lock
);
697 sock_put(rcu_dereference_raw(x
->encap_sk
));
699 xfrm_dev_state_delete(x
);
701 /* All xfrm_state objects are created by xfrm_state_alloc.
702 * The xfrm_state_alloc call gives a reference, and that
703 * is what we are dropping here.
711 EXPORT_SYMBOL(__xfrm_state_delete
);
713 int xfrm_state_delete(struct xfrm_state
*x
)
717 spin_lock_bh(&x
->lock
);
718 err
= __xfrm_state_delete(x
);
719 spin_unlock_bh(&x
->lock
);
723 EXPORT_SYMBOL(xfrm_state_delete
);
725 #ifdef CONFIG_SECURITY_NETWORK_XFRM
727 xfrm_state_flush_secctx_check(struct net
*net
, u8 proto
, bool task_valid
)
731 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
732 struct xfrm_state
*x
;
734 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
735 if (xfrm_id_proto_match(x
->id
.proto
, proto
) &&
736 (err
= security_xfrm_state_delete(x
)) != 0) {
737 xfrm_audit_state_delete(x
, 0, task_valid
);
747 xfrm_dev_state_flush_secctx_check(struct net
*net
, struct net_device
*dev
, bool task_valid
)
751 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
752 struct xfrm_state
*x
;
753 struct xfrm_state_offload
*xso
;
755 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
758 if (xso
->dev
== dev
&&
759 (err
= security_xfrm_state_delete(x
)) != 0) {
760 xfrm_audit_state_delete(x
, 0, task_valid
);
770 xfrm_state_flush_secctx_check(struct net
*net
, u8 proto
, bool task_valid
)
776 xfrm_dev_state_flush_secctx_check(struct net
*net
, struct net_device
*dev
, bool task_valid
)
782 int xfrm_state_flush(struct net
*net
, u8 proto
, bool task_valid
, bool sync
)
784 int i
, err
= 0, cnt
= 0;
786 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
787 err
= xfrm_state_flush_secctx_check(net
, proto
, task_valid
);
792 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
793 struct xfrm_state
*x
;
795 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
796 if (!xfrm_state_kern(x
) &&
797 xfrm_id_proto_match(x
->id
.proto
, proto
)) {
799 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
801 err
= xfrm_state_delete(x
);
802 xfrm_audit_state_delete(x
, err
? 0 : 1,
805 xfrm_state_put_sync(x
);
811 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
817 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
823 EXPORT_SYMBOL(xfrm_state_flush
);
825 int xfrm_dev_state_flush(struct net
*net
, struct net_device
*dev
, bool task_valid
)
827 int i
, err
= 0, cnt
= 0;
829 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
830 err
= xfrm_dev_state_flush_secctx_check(net
, dev
, task_valid
);
835 for (i
= 0; i
<= net
->xfrm
.state_hmask
; i
++) {
836 struct xfrm_state
*x
;
837 struct xfrm_state_offload
*xso
;
839 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+i
, bydst
) {
842 if (!xfrm_state_kern(x
) && xso
->dev
== dev
) {
844 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
846 err
= xfrm_state_delete(x
);
847 xfrm_audit_state_delete(x
, err
? 0 : 1,
853 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
862 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
865 EXPORT_SYMBOL(xfrm_dev_state_flush
);
867 void xfrm_sad_getinfo(struct net
*net
, struct xfrmk_sadinfo
*si
)
869 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
870 si
->sadcnt
= net
->xfrm
.state_num
;
871 si
->sadhcnt
= net
->xfrm
.state_hmask
+ 1;
872 si
->sadhmcnt
= xfrm_state_hashmax
;
873 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
875 EXPORT_SYMBOL(xfrm_sad_getinfo
);
878 __xfrm4_init_tempsel(struct xfrm_selector
*sel
, const struct flowi
*fl
)
880 const struct flowi4
*fl4
= &fl
->u
.ip4
;
882 sel
->daddr
.a4
= fl4
->daddr
;
883 sel
->saddr
.a4
= fl4
->saddr
;
884 sel
->dport
= xfrm_flowi_dport(fl
, &fl4
->uli
);
885 sel
->dport_mask
= htons(0xffff);
886 sel
->sport
= xfrm_flowi_sport(fl
, &fl4
->uli
);
887 sel
->sport_mask
= htons(0xffff);
888 sel
->family
= AF_INET
;
889 sel
->prefixlen_d
= 32;
890 sel
->prefixlen_s
= 32;
891 sel
->proto
= fl4
->flowi4_proto
;
892 sel
->ifindex
= fl4
->flowi4_oif
;
896 __xfrm6_init_tempsel(struct xfrm_selector
*sel
, const struct flowi
*fl
)
898 const struct flowi6
*fl6
= &fl
->u
.ip6
;
900 /* Initialize temporary selector matching only to current session. */
901 *(struct in6_addr
*)&sel
->daddr
= fl6
->daddr
;
902 *(struct in6_addr
*)&sel
->saddr
= fl6
->saddr
;
903 sel
->dport
= xfrm_flowi_dport(fl
, &fl6
->uli
);
904 sel
->dport_mask
= htons(0xffff);
905 sel
->sport
= xfrm_flowi_sport(fl
, &fl6
->uli
);
906 sel
->sport_mask
= htons(0xffff);
907 sel
->family
= AF_INET6
;
908 sel
->prefixlen_d
= 128;
909 sel
->prefixlen_s
= 128;
910 sel
->proto
= fl6
->flowi6_proto
;
911 sel
->ifindex
= fl6
->flowi6_oif
;
915 xfrm_init_tempstate(struct xfrm_state
*x
, const struct flowi
*fl
,
916 const struct xfrm_tmpl
*tmpl
,
917 const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
918 unsigned short family
)
922 __xfrm4_init_tempsel(&x
->sel
, fl
);
925 __xfrm6_init_tempsel(&x
->sel
, fl
);
931 switch (tmpl
->encap_family
) {
933 if (x
->id
.daddr
.a4
== 0)
934 x
->id
.daddr
.a4
= daddr
->a4
;
935 x
->props
.saddr
= tmpl
->saddr
;
936 if (x
->props
.saddr
.a4
== 0)
937 x
->props
.saddr
.a4
= saddr
->a4
;
940 if (ipv6_addr_any((struct in6_addr
*)&x
->id
.daddr
))
941 memcpy(&x
->id
.daddr
, daddr
, sizeof(x
->sel
.daddr
));
942 memcpy(&x
->props
.saddr
, &tmpl
->saddr
, sizeof(x
->props
.saddr
));
943 if (ipv6_addr_any((struct in6_addr
*)&x
->props
.saddr
))
944 memcpy(&x
->props
.saddr
, saddr
, sizeof(x
->props
.saddr
));
948 x
->props
.mode
= tmpl
->mode
;
949 x
->props
.reqid
= tmpl
->reqid
;
950 x
->props
.family
= tmpl
->encap_family
;
953 static struct xfrm_state
*__xfrm_state_lookup(struct net
*net
, u32 mark
,
954 const xfrm_address_t
*daddr
,
955 __be32 spi
, u8 proto
,
956 unsigned short family
)
958 unsigned int h
= xfrm_spi_hash(net
, daddr
, spi
, proto
, family
);
959 struct xfrm_state
*x
;
961 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_byspi
+ h
, byspi
) {
962 if (x
->props
.family
!= family
||
964 x
->id
.proto
!= proto
||
965 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
))
968 if ((mark
& x
->mark
.m
) != x
->mark
.v
)
970 if (!xfrm_state_hold_rcu(x
))
978 static struct xfrm_state
*__xfrm_state_lookup_byaddr(struct net
*net
, u32 mark
,
979 const xfrm_address_t
*daddr
,
980 const xfrm_address_t
*saddr
,
981 u8 proto
, unsigned short family
)
983 unsigned int h
= xfrm_src_hash(net
, daddr
, saddr
, family
);
984 struct xfrm_state
*x
;
986 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bysrc
+ h
, bysrc
) {
987 if (x
->props
.family
!= family
||
988 x
->id
.proto
!= proto
||
989 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
) ||
990 !xfrm_addr_equal(&x
->props
.saddr
, saddr
, family
))
993 if ((mark
& x
->mark
.m
) != x
->mark
.v
)
995 if (!xfrm_state_hold_rcu(x
))
1003 static inline struct xfrm_state
*
1004 __xfrm_state_locate(struct xfrm_state
*x
, int use_spi
, int family
)
1006 struct net
*net
= xs_net(x
);
1007 u32 mark
= x
->mark
.v
& x
->mark
.m
;
1010 return __xfrm_state_lookup(net
, mark
, &x
->id
.daddr
,
1011 x
->id
.spi
, x
->id
.proto
, family
);
1013 return __xfrm_state_lookup_byaddr(net
, mark
,
1016 x
->id
.proto
, family
);
1019 static void xfrm_hash_grow_check(struct net
*net
, int have_hash_collision
)
1021 if (have_hash_collision
&&
1022 (net
->xfrm
.state_hmask
+ 1) < xfrm_state_hashmax
&&
1023 net
->xfrm
.state_num
> net
->xfrm
.state_hmask
)
1024 schedule_work(&net
->xfrm
.state_hash_work
);
1027 static void xfrm_state_look_at(struct xfrm_policy
*pol
, struct xfrm_state
*x
,
1028 const struct flowi
*fl
, unsigned short family
,
1029 struct xfrm_state
**best
, int *acq_in_progress
,
1032 /* Resolution logic:
1033 * 1. There is a valid state with matching selector. Done.
1034 * 2. Valid state with inappropriate selector. Skip.
1036 * Entering area of "sysdeps".
1038 * 3. If state is not valid, selector is temporary, it selects
1039 * only session which triggered previous resolution. Key
1040 * manager will do something to install a state with proper
1043 if (x
->km
.state
== XFRM_STATE_VALID
) {
1044 if ((x
->sel
.family
&&
1045 (x
->sel
.family
!= family
||
1046 !xfrm_selector_match(&x
->sel
, fl
, family
))) ||
1047 !security_xfrm_state_pol_flow_match(x
, pol
,
1048 &fl
->u
.__fl_common
))
1052 (*best
)->km
.dying
> x
->km
.dying
||
1053 ((*best
)->km
.dying
== x
->km
.dying
&&
1054 (*best
)->curlft
.add_time
< x
->curlft
.add_time
))
1056 } else if (x
->km
.state
== XFRM_STATE_ACQ
) {
1057 *acq_in_progress
= 1;
1058 } else if (x
->km
.state
== XFRM_STATE_ERROR
||
1059 x
->km
.state
== XFRM_STATE_EXPIRED
) {
1060 if ((!x
->sel
.family
||
1061 (x
->sel
.family
== family
&&
1062 xfrm_selector_match(&x
->sel
, fl
, family
))) &&
1063 security_xfrm_state_pol_flow_match(x
, pol
,
1064 &fl
->u
.__fl_common
))
1070 xfrm_state_find(const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
1071 const struct flowi
*fl
, struct xfrm_tmpl
*tmpl
,
1072 struct xfrm_policy
*pol
, int *err
,
1073 unsigned short family
, u32 if_id
)
1075 static xfrm_address_t saddr_wildcard
= { };
1076 struct net
*net
= xp_net(pol
);
1077 unsigned int h
, h_wildcard
;
1078 struct xfrm_state
*x
, *x0
, *to_put
;
1079 int acquire_in_progress
= 0;
1081 struct xfrm_state
*best
= NULL
;
1082 u32 mark
= pol
->mark
.v
& pol
->mark
.m
;
1083 unsigned short encap_family
= tmpl
->encap_family
;
1084 unsigned int sequence
;
1089 sequence
= read_seqcount_begin(&net
->xfrm
.xfrm_state_hash_generation
);
1092 h
= xfrm_dst_hash(net
, daddr
, saddr
, tmpl
->reqid
, encap_family
);
1093 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bydst
+ h
, bydst
) {
1094 if (x
->props
.family
== encap_family
&&
1095 x
->props
.reqid
== tmpl
->reqid
&&
1096 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1097 x
->if_id
== if_id
&&
1098 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1099 xfrm_state_addr_check(x
, daddr
, saddr
, encap_family
) &&
1100 tmpl
->mode
== x
->props
.mode
&&
1101 tmpl
->id
.proto
== x
->id
.proto
&&
1102 (tmpl
->id
.spi
== x
->id
.spi
|| !tmpl
->id
.spi
))
1103 xfrm_state_look_at(pol
, x
, fl
, family
,
1104 &best
, &acquire_in_progress
, &error
);
1106 if (best
|| acquire_in_progress
)
1109 h_wildcard
= xfrm_dst_hash(net
, daddr
, &saddr_wildcard
, tmpl
->reqid
, encap_family
);
1110 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_bydst
+ h_wildcard
, bydst
) {
1111 if (x
->props
.family
== encap_family
&&
1112 x
->props
.reqid
== tmpl
->reqid
&&
1113 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1114 x
->if_id
== if_id
&&
1115 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1116 xfrm_addr_equal(&x
->id
.daddr
, daddr
, encap_family
) &&
1117 tmpl
->mode
== x
->props
.mode
&&
1118 tmpl
->id
.proto
== x
->id
.proto
&&
1119 (tmpl
->id
.spi
== x
->id
.spi
|| !tmpl
->id
.spi
))
1120 xfrm_state_look_at(pol
, x
, fl
, family
,
1121 &best
, &acquire_in_progress
, &error
);
1126 if (!x
&& !error
&& !acquire_in_progress
) {
1128 (x0
= __xfrm_state_lookup(net
, mark
, daddr
, tmpl
->id
.spi
,
1129 tmpl
->id
.proto
, encap_family
)) != NULL
) {
1136 /* If the KMs have no listeners (yet...), avoid allocating an SA
1137 * for each and every packet - garbage collection might not
1140 if (!km_is_alive(&c
)) {
1145 x
= xfrm_state_alloc(net
);
1150 /* Initialize temporary state matching only
1151 * to current session. */
1152 xfrm_init_tempstate(x
, fl
, tmpl
, daddr
, saddr
, family
);
1153 memcpy(&x
->mark
, &pol
->mark
, sizeof(x
->mark
));
1156 error
= security_xfrm_state_alloc_acquire(x
, pol
->security
, fl
->flowi_secid
);
1158 x
->km
.state
= XFRM_STATE_DEAD
;
1164 if (km_query(x
, tmpl
, pol
) == 0) {
1165 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1166 x
->km
.state
= XFRM_STATE_ACQ
;
1167 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1168 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1169 h
= xfrm_src_hash(net
, daddr
, saddr
, encap_family
);
1170 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1172 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
, encap_family
);
1173 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
1176 h
= xfrm_seq_hash(net
, x
->km
.seq
);
1177 hlist_add_head_rcu(&x
->byseq
, net
->xfrm
.state_byseq
+ h
);
1179 x
->lft
.hard_add_expires_seconds
= net
->xfrm
.sysctl_acq_expires
;
1180 hrtimer_start(&x
->mtimer
,
1181 ktime_set(net
->xfrm
.sysctl_acq_expires
, 0),
1182 HRTIMER_MODE_REL_SOFT
);
1183 net
->xfrm
.state_num
++;
1184 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1185 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1187 x
->km
.state
= XFRM_STATE_DEAD
;
1195 if (!xfrm_state_hold_rcu(x
)) {
1200 *err
= acquire_in_progress
? -EAGAIN
: error
;
1204 xfrm_state_put(to_put
);
1206 if (read_seqcount_retry(&net
->xfrm
.xfrm_state_hash_generation
, sequence
)) {
1218 xfrm_stateonly_find(struct net
*net
, u32 mark
, u32 if_id
,
1219 xfrm_address_t
*daddr
, xfrm_address_t
*saddr
,
1220 unsigned short family
, u8 mode
, u8 proto
, u32 reqid
)
1223 struct xfrm_state
*rx
= NULL
, *x
= NULL
;
1225 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1226 h
= xfrm_dst_hash(net
, daddr
, saddr
, reqid
, family
);
1227 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1228 if (x
->props
.family
== family
&&
1229 x
->props
.reqid
== reqid
&&
1230 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1231 x
->if_id
== if_id
&&
1232 !(x
->props
.flags
& XFRM_STATE_WILDRECV
) &&
1233 xfrm_state_addr_check(x
, daddr
, saddr
, family
) &&
1234 mode
== x
->props
.mode
&&
1235 proto
== x
->id
.proto
&&
1236 x
->km
.state
== XFRM_STATE_VALID
) {
1243 xfrm_state_hold(rx
);
1244 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1249 EXPORT_SYMBOL(xfrm_stateonly_find
);
1251 struct xfrm_state
*xfrm_state_lookup_byspi(struct net
*net
, __be32 spi
,
1252 unsigned short family
)
1254 struct xfrm_state
*x
;
1255 struct xfrm_state_walk
*w
;
1257 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1258 list_for_each_entry(w
, &net
->xfrm
.state_all
, all
) {
1259 x
= container_of(w
, struct xfrm_state
, km
);
1260 if (x
->props
.family
!= family
||
1265 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1268 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1271 EXPORT_SYMBOL(xfrm_state_lookup_byspi
);
1273 static void __xfrm_state_insert(struct xfrm_state
*x
)
1275 struct net
*net
= xs_net(x
);
1278 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1280 h
= xfrm_dst_hash(net
, &x
->id
.daddr
, &x
->props
.saddr
,
1281 x
->props
.reqid
, x
->props
.family
);
1282 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1284 h
= xfrm_src_hash(net
, &x
->id
.daddr
, &x
->props
.saddr
, x
->props
.family
);
1285 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1288 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
,
1291 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
1295 h
= xfrm_seq_hash(net
, x
->km
.seq
);
1297 hlist_add_head_rcu(&x
->byseq
, net
->xfrm
.state_byseq
+ h
);
1300 hrtimer_start(&x
->mtimer
, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT
);
1301 if (x
->replay_maxage
)
1302 mod_timer(&x
->rtimer
, jiffies
+ x
->replay_maxage
);
1304 net
->xfrm
.state_num
++;
1306 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1309 /* net->xfrm.xfrm_state_lock is held */
1310 static void __xfrm_state_bump_genids(struct xfrm_state
*xnew
)
1312 struct net
*net
= xs_net(xnew
);
1313 unsigned short family
= xnew
->props
.family
;
1314 u32 reqid
= xnew
->props
.reqid
;
1315 struct xfrm_state
*x
;
1317 u32 mark
= xnew
->mark
.v
& xnew
->mark
.m
;
1318 u32 if_id
= xnew
->if_id
;
1320 h
= xfrm_dst_hash(net
, &xnew
->id
.daddr
, &xnew
->props
.saddr
, reqid
, family
);
1321 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1322 if (x
->props
.family
== family
&&
1323 x
->props
.reqid
== reqid
&&
1324 x
->if_id
== if_id
&&
1325 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1326 xfrm_addr_equal(&x
->id
.daddr
, &xnew
->id
.daddr
, family
) &&
1327 xfrm_addr_equal(&x
->props
.saddr
, &xnew
->props
.saddr
, family
))
1332 void xfrm_state_insert(struct xfrm_state
*x
)
1334 struct net
*net
= xs_net(x
);
1336 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1337 __xfrm_state_bump_genids(x
);
1338 __xfrm_state_insert(x
);
1339 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1341 EXPORT_SYMBOL(xfrm_state_insert
);
1343 /* net->xfrm.xfrm_state_lock is held */
1344 static struct xfrm_state
*__find_acq_core(struct net
*net
,
1345 const struct xfrm_mark
*m
,
1346 unsigned short family
, u8 mode
,
1347 u32 reqid
, u32 if_id
, u8 proto
,
1348 const xfrm_address_t
*daddr
,
1349 const xfrm_address_t
*saddr
,
1352 unsigned int h
= xfrm_dst_hash(net
, daddr
, saddr
, reqid
, family
);
1353 struct xfrm_state
*x
;
1354 u32 mark
= m
->v
& m
->m
;
1356 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1357 if (x
->props
.reqid
!= reqid
||
1358 x
->props
.mode
!= mode
||
1359 x
->props
.family
!= family
||
1360 x
->km
.state
!= XFRM_STATE_ACQ
||
1362 x
->id
.proto
!= proto
||
1363 (mark
& x
->mark
.m
) != x
->mark
.v
||
1364 !xfrm_addr_equal(&x
->id
.daddr
, daddr
, family
) ||
1365 !xfrm_addr_equal(&x
->props
.saddr
, saddr
, family
))
1375 x
= xfrm_state_alloc(net
);
1379 x
->sel
.daddr
.a4
= daddr
->a4
;
1380 x
->sel
.saddr
.a4
= saddr
->a4
;
1381 x
->sel
.prefixlen_d
= 32;
1382 x
->sel
.prefixlen_s
= 32;
1383 x
->props
.saddr
.a4
= saddr
->a4
;
1384 x
->id
.daddr
.a4
= daddr
->a4
;
1388 x
->sel
.daddr
.in6
= daddr
->in6
;
1389 x
->sel
.saddr
.in6
= saddr
->in6
;
1390 x
->sel
.prefixlen_d
= 128;
1391 x
->sel
.prefixlen_s
= 128;
1392 x
->props
.saddr
.in6
= saddr
->in6
;
1393 x
->id
.daddr
.in6
= daddr
->in6
;
1397 x
->km
.state
= XFRM_STATE_ACQ
;
1398 x
->id
.proto
= proto
;
1399 x
->props
.family
= family
;
1400 x
->props
.mode
= mode
;
1401 x
->props
.reqid
= reqid
;
1405 x
->lft
.hard_add_expires_seconds
= net
->xfrm
.sysctl_acq_expires
;
1407 hrtimer_start(&x
->mtimer
,
1408 ktime_set(net
->xfrm
.sysctl_acq_expires
, 0),
1409 HRTIMER_MODE_REL_SOFT
);
1410 list_add(&x
->km
.all
, &net
->xfrm
.state_all
);
1411 hlist_add_head_rcu(&x
->bydst
, net
->xfrm
.state_bydst
+ h
);
1412 h
= xfrm_src_hash(net
, daddr
, saddr
, family
);
1413 hlist_add_head_rcu(&x
->bysrc
, net
->xfrm
.state_bysrc
+ h
);
1415 net
->xfrm
.state_num
++;
1417 xfrm_hash_grow_check(net
, x
->bydst
.next
!= NULL
);
1423 static struct xfrm_state
*__xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
);
1425 int xfrm_state_add(struct xfrm_state
*x
)
1427 struct net
*net
= xs_net(x
);
1428 struct xfrm_state
*x1
, *to_put
;
1431 u32 mark
= x
->mark
.v
& x
->mark
.m
;
1432 int use_spi
= xfrm_id_proto_match(x
->id
.proto
, IPSEC_PROTO_ANY
);
1434 family
= x
->props
.family
;
1438 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1440 x1
= __xfrm_state_locate(x
, use_spi
, family
);
1448 if (use_spi
&& x
->km
.seq
) {
1449 x1
= __xfrm_find_acq_byseq(net
, mark
, x
->km
.seq
);
1450 if (x1
&& ((x1
->id
.proto
!= x
->id
.proto
) ||
1451 !xfrm_addr_equal(&x1
->id
.daddr
, &x
->id
.daddr
, family
))) {
1458 x1
= __find_acq_core(net
, &x
->mark
, family
, x
->props
.mode
,
1459 x
->props
.reqid
, x
->if_id
, x
->id
.proto
,
1460 &x
->id
.daddr
, &x
->props
.saddr
, 0);
1462 __xfrm_state_bump_genids(x
);
1463 __xfrm_state_insert(x
);
1467 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1470 xfrm_state_delete(x1
);
1475 xfrm_state_put(to_put
);
1479 EXPORT_SYMBOL(xfrm_state_add
);
1481 #ifdef CONFIG_XFRM_MIGRATE
1482 static inline int clone_security(struct xfrm_state
*x
, struct xfrm_sec_ctx
*security
)
1484 struct xfrm_user_sec_ctx
*uctx
;
1485 int size
= sizeof(*uctx
) + security
->ctx_len
;
1488 uctx
= kmalloc(size
, GFP_KERNEL
);
1492 uctx
->exttype
= XFRMA_SEC_CTX
;
1494 uctx
->ctx_doi
= security
->ctx_doi
;
1495 uctx
->ctx_alg
= security
->ctx_alg
;
1496 uctx
->ctx_len
= security
->ctx_len
;
1497 memcpy(uctx
+ 1, security
->ctx_str
, security
->ctx_len
);
1498 err
= security_xfrm_state_alloc(x
, uctx
);
1506 static struct xfrm_state
*xfrm_state_clone(struct xfrm_state
*orig
,
1507 struct xfrm_encap_tmpl
*encap
)
1509 struct net
*net
= xs_net(orig
);
1510 struct xfrm_state
*x
= xfrm_state_alloc(net
);
1514 memcpy(&x
->id
, &orig
->id
, sizeof(x
->id
));
1515 memcpy(&x
->sel
, &orig
->sel
, sizeof(x
->sel
));
1516 memcpy(&x
->lft
, &orig
->lft
, sizeof(x
->lft
));
1517 x
->props
.mode
= orig
->props
.mode
;
1518 x
->props
.replay_window
= orig
->props
.replay_window
;
1519 x
->props
.reqid
= orig
->props
.reqid
;
1520 x
->props
.family
= orig
->props
.family
;
1521 x
->props
.saddr
= orig
->props
.saddr
;
1524 x
->aalg
= xfrm_algo_auth_clone(orig
->aalg
);
1528 x
->props
.aalgo
= orig
->props
.aalgo
;
1531 x
->aead
= xfrm_algo_aead_clone(orig
->aead
);
1532 x
->geniv
= orig
->geniv
;
1537 x
->ealg
= xfrm_algo_clone(orig
->ealg
);
1541 x
->props
.ealgo
= orig
->props
.ealgo
;
1544 x
->calg
= xfrm_algo_clone(orig
->calg
);
1548 x
->props
.calgo
= orig
->props
.calgo
;
1550 if (encap
|| orig
->encap
) {
1552 x
->encap
= kmemdup(encap
, sizeof(*x
->encap
),
1555 x
->encap
= kmemdup(orig
->encap
, sizeof(*x
->encap
),
1563 if (clone_security(x
, orig
->security
))
1567 x
->coaddr
= kmemdup(orig
->coaddr
, sizeof(*x
->coaddr
),
1573 if (orig
->replay_esn
) {
1574 if (xfrm_replay_clone(x
, orig
))
1578 memcpy(&x
->mark
, &orig
->mark
, sizeof(x
->mark
));
1579 memcpy(&x
->props
.smark
, &orig
->props
.smark
, sizeof(x
->props
.smark
));
1581 if (xfrm_init_state(x
) < 0)
1584 x
->props
.flags
= orig
->props
.flags
;
1585 x
->props
.extra_flags
= orig
->props
.extra_flags
;
1587 x
->if_id
= orig
->if_id
;
1588 x
->tfcpad
= orig
->tfcpad
;
1589 x
->replay_maxdiff
= orig
->replay_maxdiff
;
1590 x
->replay_maxage
= orig
->replay_maxage
;
1591 memcpy(&x
->curlft
, &orig
->curlft
, sizeof(x
->curlft
));
1592 x
->km
.state
= orig
->km
.state
;
1593 x
->km
.seq
= orig
->km
.seq
;
1594 x
->replay
= orig
->replay
;
1595 x
->preplay
= orig
->preplay
;
1596 x
->mapping_maxage
= orig
->mapping_maxage
;
1598 x
->new_mapping_sport
= 0;
1608 struct xfrm_state
*xfrm_migrate_state_find(struct xfrm_migrate
*m
, struct net
*net
)
1611 struct xfrm_state
*x
= NULL
;
1613 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1616 h
= xfrm_dst_hash(net
, &m
->old_daddr
, &m
->old_saddr
,
1617 m
->reqid
, m
->old_family
);
1618 hlist_for_each_entry(x
, net
->xfrm
.state_bydst
+h
, bydst
) {
1619 if (x
->props
.mode
!= m
->mode
||
1620 x
->id
.proto
!= m
->proto
)
1622 if (m
->reqid
&& x
->props
.reqid
!= m
->reqid
)
1624 if (!xfrm_addr_equal(&x
->id
.daddr
, &m
->old_daddr
,
1626 !xfrm_addr_equal(&x
->props
.saddr
, &m
->old_saddr
,
1633 h
= xfrm_src_hash(net
, &m
->old_daddr
, &m
->old_saddr
,
1635 hlist_for_each_entry(x
, net
->xfrm
.state_bysrc
+h
, bysrc
) {
1636 if (x
->props
.mode
!= m
->mode
||
1637 x
->id
.proto
!= m
->proto
)
1639 if (!xfrm_addr_equal(&x
->id
.daddr
, &m
->old_daddr
,
1641 !xfrm_addr_equal(&x
->props
.saddr
, &m
->old_saddr
,
1649 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1653 EXPORT_SYMBOL(xfrm_migrate_state_find
);
1655 struct xfrm_state
*xfrm_state_migrate(struct xfrm_state
*x
,
1656 struct xfrm_migrate
*m
,
1657 struct xfrm_encap_tmpl
*encap
)
1659 struct xfrm_state
*xc
;
1661 xc
= xfrm_state_clone(x
, encap
);
1665 memcpy(&xc
->id
.daddr
, &m
->new_daddr
, sizeof(xc
->id
.daddr
));
1666 memcpy(&xc
->props
.saddr
, &m
->new_saddr
, sizeof(xc
->props
.saddr
));
1669 if (xfrm_addr_equal(&x
->id
.daddr
, &m
->new_daddr
, m
->new_family
)) {
1670 /* a care is needed when the destination address of the
1671 state is to be updated as it is a part of triplet */
1672 xfrm_state_insert(xc
);
1674 if (xfrm_state_add(xc
) < 0)
1683 EXPORT_SYMBOL(xfrm_state_migrate
);
1686 int xfrm_state_update(struct xfrm_state
*x
)
1688 struct xfrm_state
*x1
, *to_put
;
1690 int use_spi
= xfrm_id_proto_match(x
->id
.proto
, IPSEC_PROTO_ANY
);
1691 struct net
*net
= xs_net(x
);
1695 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1696 x1
= __xfrm_state_locate(x
, use_spi
, x
->props
.family
);
1702 if (xfrm_state_kern(x1
)) {
1708 if (x1
->km
.state
== XFRM_STATE_ACQ
) {
1709 __xfrm_state_insert(x
);
1715 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1718 xfrm_state_put(to_put
);
1724 xfrm_state_delete(x1
);
1730 spin_lock_bh(&x1
->lock
);
1731 if (likely(x1
->km
.state
== XFRM_STATE_VALID
)) {
1732 if (x
->encap
&& x1
->encap
&&
1733 x
->encap
->encap_type
== x1
->encap
->encap_type
)
1734 memcpy(x1
->encap
, x
->encap
, sizeof(*x1
->encap
));
1735 else if (x
->encap
|| x1
->encap
)
1738 if (x
->coaddr
&& x1
->coaddr
) {
1739 memcpy(x1
->coaddr
, x
->coaddr
, sizeof(*x1
->coaddr
));
1741 if (!use_spi
&& memcmp(&x1
->sel
, &x
->sel
, sizeof(x1
->sel
)))
1742 memcpy(&x1
->sel
, &x
->sel
, sizeof(x1
->sel
));
1743 memcpy(&x1
->lft
, &x
->lft
, sizeof(x1
->lft
));
1746 hrtimer_start(&x1
->mtimer
, ktime_set(1, 0),
1747 HRTIMER_MODE_REL_SOFT
);
1748 if (x1
->curlft
.use_time
)
1749 xfrm_state_check_expire(x1
);
1751 if (x
->props
.smark
.m
|| x
->props
.smark
.v
|| x
->if_id
) {
1752 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1754 if (x
->props
.smark
.m
|| x
->props
.smark
.v
)
1755 x1
->props
.smark
= x
->props
.smark
;
1758 x1
->if_id
= x
->if_id
;
1760 __xfrm_state_bump_genids(x1
);
1761 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1765 x
->km
.state
= XFRM_STATE_DEAD
;
1766 __xfrm_state_put(x
);
1770 spin_unlock_bh(&x1
->lock
);
1776 EXPORT_SYMBOL(xfrm_state_update
);
1778 int xfrm_state_check_expire(struct xfrm_state
*x
)
1780 if (!x
->curlft
.use_time
)
1781 x
->curlft
.use_time
= ktime_get_real_seconds();
1783 if (x
->curlft
.bytes
>= x
->lft
.hard_byte_limit
||
1784 x
->curlft
.packets
>= x
->lft
.hard_packet_limit
) {
1785 x
->km
.state
= XFRM_STATE_EXPIRED
;
1786 hrtimer_start(&x
->mtimer
, 0, HRTIMER_MODE_REL_SOFT
);
1791 (x
->curlft
.bytes
>= x
->lft
.soft_byte_limit
||
1792 x
->curlft
.packets
>= x
->lft
.soft_packet_limit
)) {
1794 km_state_expired(x
, 0, 0);
1798 EXPORT_SYMBOL(xfrm_state_check_expire
);
1801 xfrm_state_lookup(struct net
*net
, u32 mark
, const xfrm_address_t
*daddr
, __be32 spi
,
1802 u8 proto
, unsigned short family
)
1804 struct xfrm_state
*x
;
1807 x
= __xfrm_state_lookup(net
, mark
, daddr
, spi
, proto
, family
);
1811 EXPORT_SYMBOL(xfrm_state_lookup
);
1814 xfrm_state_lookup_byaddr(struct net
*net
, u32 mark
,
1815 const xfrm_address_t
*daddr
, const xfrm_address_t
*saddr
,
1816 u8 proto
, unsigned short family
)
1818 struct xfrm_state
*x
;
1820 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1821 x
= __xfrm_state_lookup_byaddr(net
, mark
, daddr
, saddr
, proto
, family
);
1822 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1825 EXPORT_SYMBOL(xfrm_state_lookup_byaddr
);
1828 xfrm_find_acq(struct net
*net
, const struct xfrm_mark
*mark
, u8 mode
, u32 reqid
,
1829 u32 if_id
, u8 proto
, const xfrm_address_t
*daddr
,
1830 const xfrm_address_t
*saddr
, int create
, unsigned short family
)
1832 struct xfrm_state
*x
;
1834 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1835 x
= __find_acq_core(net
, mark
, family
, mode
, reqid
, if_id
, proto
, daddr
, saddr
, create
);
1836 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1840 EXPORT_SYMBOL(xfrm_find_acq
);
1842 #ifdef CONFIG_XFRM_SUB_POLICY
1843 #if IS_ENABLED(CONFIG_IPV6)
1844 /* distribution counting sort function for xfrm_state and xfrm_tmpl */
1846 __xfrm6_sort(void **dst
, void **src
, int n
,
1847 int (*cmp
)(const void *p
), int maxclass
)
1849 int count
[XFRM_MAX_DEPTH
] = { };
1850 int class[XFRM_MAX_DEPTH
];
1853 for (i
= 0; i
< n
; i
++) {
1854 int c
= cmp(src
[i
]);
1860 for (i
= 2; i
< maxclass
; i
++)
1861 count
[i
] += count
[i
- 1];
1863 for (i
= 0; i
< n
; i
++) {
1864 dst
[count
[class[i
] - 1]++] = src
[i
];
1869 /* Rule for xfrm_state:
1871 * rule 1: select IPsec transport except AH
1872 * rule 2: select MIPv6 RO or inbound trigger
1873 * rule 3: select IPsec transport AH
1874 * rule 4: select IPsec tunnel
1877 static int __xfrm6_state_sort_cmp(const void *p
)
1879 const struct xfrm_state
*v
= p
;
1881 switch (v
->props
.mode
) {
1882 case XFRM_MODE_TRANSPORT
:
1883 if (v
->id
.proto
!= IPPROTO_AH
)
1887 #if IS_ENABLED(CONFIG_IPV6_MIP6)
1888 case XFRM_MODE_ROUTEOPTIMIZATION
:
1889 case XFRM_MODE_IN_TRIGGER
:
1892 case XFRM_MODE_TUNNEL
:
1893 case XFRM_MODE_BEET
:
1899 /* Rule for xfrm_tmpl:
1901 * rule 1: select IPsec transport
1902 * rule 2: select MIPv6 RO or inbound trigger
1903 * rule 3: select IPsec tunnel
1906 static int __xfrm6_tmpl_sort_cmp(const void *p
)
1908 const struct xfrm_tmpl
*v
= p
;
1911 case XFRM_MODE_TRANSPORT
:
1913 #if IS_ENABLED(CONFIG_IPV6_MIP6)
1914 case XFRM_MODE_ROUTEOPTIMIZATION
:
1915 case XFRM_MODE_IN_TRIGGER
:
1918 case XFRM_MODE_TUNNEL
:
1919 case XFRM_MODE_BEET
:
1925 static inline int __xfrm6_state_sort_cmp(const void *p
) { return 5; }
1926 static inline int __xfrm6_tmpl_sort_cmp(const void *p
) { return 4; }
1929 __xfrm6_sort(void **dst
, void **src
, int n
,
1930 int (*cmp
)(const void *p
), int maxclass
)
1934 for (i
= 0; i
< n
; i
++)
1937 #endif /* CONFIG_IPV6 */
1940 xfrm_tmpl_sort(struct xfrm_tmpl
**dst
, struct xfrm_tmpl
**src
, int n
,
1941 unsigned short family
)
1945 if (family
== AF_INET6
)
1946 __xfrm6_sort((void **)dst
, (void **)src
, n
,
1947 __xfrm6_tmpl_sort_cmp
, 5);
1949 for (i
= 0; i
< n
; i
++)
1954 xfrm_state_sort(struct xfrm_state
**dst
, struct xfrm_state
**src
, int n
,
1955 unsigned short family
)
1959 if (family
== AF_INET6
)
1960 __xfrm6_sort((void **)dst
, (void **)src
, n
,
1961 __xfrm6_state_sort_cmp
, 6);
1963 for (i
= 0; i
< n
; i
++)
1968 /* Silly enough, but I'm lazy to build resolution list */
1970 static struct xfrm_state
*__xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
)
1972 unsigned int h
= xfrm_seq_hash(net
, seq
);
1973 struct xfrm_state
*x
;
1975 hlist_for_each_entry_rcu(x
, net
->xfrm
.state_byseq
+ h
, byseq
) {
1976 if (x
->km
.seq
== seq
&&
1977 (mark
& x
->mark
.m
) == x
->mark
.v
&&
1978 x
->km
.state
== XFRM_STATE_ACQ
) {
1987 struct xfrm_state
*xfrm_find_acq_byseq(struct net
*net
, u32 mark
, u32 seq
)
1989 struct xfrm_state
*x
;
1991 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
1992 x
= __xfrm_find_acq_byseq(net
, mark
, seq
);
1993 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
1996 EXPORT_SYMBOL(xfrm_find_acq_byseq
);
1998 u32
xfrm_get_acqseq(void)
2001 static atomic_t acqseq
;
2004 res
= atomic_inc_return(&acqseq
);
2009 EXPORT_SYMBOL(xfrm_get_acqseq
);
2011 int verify_spi_info(u8 proto
, u32 min
, u32 max
)
2019 /* IPCOMP spi is 16-bits. */
2033 EXPORT_SYMBOL(verify_spi_info
);
2035 int xfrm_alloc_spi(struct xfrm_state
*x
, u32 low
, u32 high
)
2037 struct net
*net
= xs_net(x
);
2039 struct xfrm_state
*x0
;
2041 __be32 minspi
= htonl(low
);
2042 __be32 maxspi
= htonl(high
);
2044 u32 mark
= x
->mark
.v
& x
->mark
.m
;
2046 spin_lock_bh(&x
->lock
);
2047 if (x
->km
.state
== XFRM_STATE_DEAD
)
2056 if (minspi
== maxspi
) {
2057 x0
= xfrm_state_lookup(net
, mark
, &x
->id
.daddr
, minspi
, x
->id
.proto
, x
->props
.family
);
2065 for (h
= 0; h
< high
-low
+1; h
++) {
2066 spi
= low
+ prandom_u32()%(high
-low
+1);
2067 x0
= xfrm_state_lookup(net
, mark
, &x
->id
.daddr
, htonl(spi
), x
->id
.proto
, x
->props
.family
);
2069 newspi
= htonl(spi
);
2076 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2078 h
= xfrm_spi_hash(net
, &x
->id
.daddr
, x
->id
.spi
, x
->id
.proto
, x
->props
.family
);
2079 hlist_add_head_rcu(&x
->byspi
, net
->xfrm
.state_byspi
+ h
);
2080 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2086 spin_unlock_bh(&x
->lock
);
2090 EXPORT_SYMBOL(xfrm_alloc_spi
);
2092 static bool __xfrm_state_filter_match(struct xfrm_state
*x
,
2093 struct xfrm_address_filter
*filter
)
2096 if ((filter
->family
== AF_INET
||
2097 filter
->family
== AF_INET6
) &&
2098 x
->props
.family
!= filter
->family
)
2101 return addr_match(&x
->props
.saddr
, &filter
->saddr
,
2103 addr_match(&x
->id
.daddr
, &filter
->daddr
,
2109 int xfrm_state_walk(struct net
*net
, struct xfrm_state_walk
*walk
,
2110 int (*func
)(struct xfrm_state
*, int, void*),
2113 struct xfrm_state
*state
;
2114 struct xfrm_state_walk
*x
;
2117 if (walk
->seq
!= 0 && list_empty(&walk
->all
))
2120 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2121 if (list_empty(&walk
->all
))
2122 x
= list_first_entry(&net
->xfrm
.state_all
, struct xfrm_state_walk
, all
);
2124 x
= list_first_entry(&walk
->all
, struct xfrm_state_walk
, all
);
2125 list_for_each_entry_from(x
, &net
->xfrm
.state_all
, all
) {
2126 if (x
->state
== XFRM_STATE_DEAD
)
2128 state
= container_of(x
, struct xfrm_state
, km
);
2129 if (!xfrm_id_proto_match(state
->id
.proto
, walk
->proto
))
2131 if (!__xfrm_state_filter_match(state
, walk
->filter
))
2133 err
= func(state
, walk
->seq
, data
);
2135 list_move_tail(&walk
->all
, &x
->all
);
2140 if (walk
->seq
== 0) {
2144 list_del_init(&walk
->all
);
2146 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2149 EXPORT_SYMBOL(xfrm_state_walk
);
2151 void xfrm_state_walk_init(struct xfrm_state_walk
*walk
, u8 proto
,
2152 struct xfrm_address_filter
*filter
)
2154 INIT_LIST_HEAD(&walk
->all
);
2155 walk
->proto
= proto
;
2156 walk
->state
= XFRM_STATE_DEAD
;
2158 walk
->filter
= filter
;
2160 EXPORT_SYMBOL(xfrm_state_walk_init
);
2162 void xfrm_state_walk_done(struct xfrm_state_walk
*walk
, struct net
*net
)
2164 kfree(walk
->filter
);
2166 if (list_empty(&walk
->all
))
2169 spin_lock_bh(&net
->xfrm
.xfrm_state_lock
);
2170 list_del(&walk
->all
);
2171 spin_unlock_bh(&net
->xfrm
.xfrm_state_lock
);
2173 EXPORT_SYMBOL(xfrm_state_walk_done
);
2175 static void xfrm_replay_timer_handler(struct timer_list
*t
)
2177 struct xfrm_state
*x
= from_timer(x
, t
, rtimer
);
2179 spin_lock(&x
->lock
);
2181 if (x
->km
.state
== XFRM_STATE_VALID
) {
2182 if (xfrm_aevent_is_on(xs_net(x
)))
2183 xfrm_replay_notify(x
, XFRM_REPLAY_TIMEOUT
);
2185 x
->xflags
|= XFRM_TIME_DEFER
;
2188 spin_unlock(&x
->lock
);
2191 static LIST_HEAD(xfrm_km_list
);
2193 void km_policy_notify(struct xfrm_policy
*xp
, int dir
, const struct km_event
*c
)
2195 struct xfrm_mgr
*km
;
2198 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
)
2199 if (km
->notify_policy
)
2200 km
->notify_policy(xp
, dir
, c
);
2204 void km_state_notify(struct xfrm_state
*x
, const struct km_event
*c
)
2206 struct xfrm_mgr
*km
;
2208 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
)
2214 EXPORT_SYMBOL(km_policy_notify
);
2215 EXPORT_SYMBOL(km_state_notify
);
2217 void km_state_expired(struct xfrm_state
*x
, int hard
, u32 portid
)
2223 c
.event
= XFRM_MSG_EXPIRE
;
2224 km_state_notify(x
, &c
);
2227 EXPORT_SYMBOL(km_state_expired
);
2229 * We send to all registered managers regardless of failure
2230 * We are happy with one success
2232 int km_query(struct xfrm_state
*x
, struct xfrm_tmpl
*t
, struct xfrm_policy
*pol
)
2234 int err
= -EINVAL
, acqret
;
2235 struct xfrm_mgr
*km
;
2238 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2239 acqret
= km
->acquire(x
, t
, pol
);
2246 EXPORT_SYMBOL(km_query
);
2248 static int __km_new_mapping(struct xfrm_state
*x
, xfrm_address_t
*ipaddr
, __be16 sport
)
2251 struct xfrm_mgr
*km
;
2254 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2255 if (km
->new_mapping
)
2256 err
= km
->new_mapping(x
, ipaddr
, sport
);
2264 int km_new_mapping(struct xfrm_state
*x
, xfrm_address_t
*ipaddr
, __be16 sport
)
2268 if (x
->mapping_maxage
) {
2269 if ((jiffies
/ HZ
- x
->new_mapping
) > x
->mapping_maxage
||
2270 x
->new_mapping_sport
!= sport
) {
2271 x
->new_mapping_sport
= sport
;
2272 x
->new_mapping
= jiffies
/ HZ
;
2273 ret
= __km_new_mapping(x
, ipaddr
, sport
);
2276 ret
= __km_new_mapping(x
, ipaddr
, sport
);
2281 EXPORT_SYMBOL(km_new_mapping
);
2283 void km_policy_expired(struct xfrm_policy
*pol
, int dir
, int hard
, u32 portid
)
2289 c
.event
= XFRM_MSG_POLEXPIRE
;
2290 km_policy_notify(pol
, dir
, &c
);
2292 EXPORT_SYMBOL(km_policy_expired
);
2294 #ifdef CONFIG_XFRM_MIGRATE
2295 int km_migrate(const struct xfrm_selector
*sel
, u8 dir
, u8 type
,
2296 const struct xfrm_migrate
*m
, int num_migrate
,
2297 const struct xfrm_kmaddress
*k
,
2298 const struct xfrm_encap_tmpl
*encap
)
2302 struct xfrm_mgr
*km
;
2305 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2307 ret
= km
->migrate(sel
, dir
, type
, m
, num_migrate
, k
,
2316 EXPORT_SYMBOL(km_migrate
);
2319 int km_report(struct net
*net
, u8 proto
, struct xfrm_selector
*sel
, xfrm_address_t
*addr
)
2323 struct xfrm_mgr
*km
;
2326 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2328 ret
= km
->report(net
, proto
, sel
, addr
);
2336 EXPORT_SYMBOL(km_report
);
2338 static bool km_is_alive(const struct km_event
*c
)
2340 struct xfrm_mgr
*km
;
2341 bool is_alive
= false;
2344 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2345 if (km
->is_alive
&& km
->is_alive(c
)) {
2355 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2356 static DEFINE_SPINLOCK(xfrm_translator_lock
);
2357 static struct xfrm_translator __rcu
*xfrm_translator
;
2359 struct xfrm_translator
*xfrm_get_translator(void)
2361 struct xfrm_translator
*xtr
;
2364 xtr
= rcu_dereference(xfrm_translator
);
2367 if (!try_module_get(xtr
->owner
))
2373 EXPORT_SYMBOL_GPL(xfrm_get_translator
);
2375 void xfrm_put_translator(struct xfrm_translator
*xtr
)
2377 module_put(xtr
->owner
);
2379 EXPORT_SYMBOL_GPL(xfrm_put_translator
);
2381 int xfrm_register_translator(struct xfrm_translator
*xtr
)
2385 spin_lock_bh(&xfrm_translator_lock
);
2386 if (unlikely(xfrm_translator
!= NULL
))
2389 rcu_assign_pointer(xfrm_translator
, xtr
);
2390 spin_unlock_bh(&xfrm_translator_lock
);
2394 EXPORT_SYMBOL_GPL(xfrm_register_translator
);
2396 int xfrm_unregister_translator(struct xfrm_translator
*xtr
)
2400 spin_lock_bh(&xfrm_translator_lock
);
2401 if (likely(xfrm_translator
!= NULL
)) {
2402 if (rcu_access_pointer(xfrm_translator
) != xtr
)
2405 RCU_INIT_POINTER(xfrm_translator
, NULL
);
2407 spin_unlock_bh(&xfrm_translator_lock
);
2412 EXPORT_SYMBOL_GPL(xfrm_unregister_translator
);
2415 int xfrm_user_policy(struct sock
*sk
, int optname
, sockptr_t optval
, int optlen
)
2419 struct xfrm_mgr
*km
;
2420 struct xfrm_policy
*pol
= NULL
;
2422 if (sockptr_is_null(optval
) && !optlen
) {
2423 xfrm_sk_policy_insert(sk
, XFRM_POLICY_IN
, NULL
);
2424 xfrm_sk_policy_insert(sk
, XFRM_POLICY_OUT
, NULL
);
2429 if (optlen
<= 0 || optlen
> PAGE_SIZE
)
2432 data
= memdup_sockptr(optval
, optlen
);
2434 return PTR_ERR(data
);
2436 if (in_compat_syscall()) {
2437 struct xfrm_translator
*xtr
= xfrm_get_translator();
2444 err
= xtr
->xlate_user_policy_sockptr(&data
, optlen
);
2445 xfrm_put_translator(xtr
);
2454 list_for_each_entry_rcu(km
, &xfrm_km_list
, list
) {
2455 pol
= km
->compile_policy(sk
, optname
, data
,
2463 xfrm_sk_policy_insert(sk
, err
, pol
);
2472 EXPORT_SYMBOL(xfrm_user_policy
);
2474 static DEFINE_SPINLOCK(xfrm_km_lock
);
2476 int xfrm_register_km(struct xfrm_mgr
*km
)
2478 spin_lock_bh(&xfrm_km_lock
);
2479 list_add_tail_rcu(&km
->list
, &xfrm_km_list
);
2480 spin_unlock_bh(&xfrm_km_lock
);
2483 EXPORT_SYMBOL(xfrm_register_km
);
2485 int xfrm_unregister_km(struct xfrm_mgr
*km
)
2487 spin_lock_bh(&xfrm_km_lock
);
2488 list_del_rcu(&km
->list
);
2489 spin_unlock_bh(&xfrm_km_lock
);
2493 EXPORT_SYMBOL(xfrm_unregister_km
);
2495 int xfrm_state_register_afinfo(struct xfrm_state_afinfo
*afinfo
)
2499 if (WARN_ON(afinfo
->family
>= NPROTO
))
2500 return -EAFNOSUPPORT
;
2502 spin_lock_bh(&xfrm_state_afinfo_lock
);
2503 if (unlikely(xfrm_state_afinfo
[afinfo
->family
] != NULL
))
2506 rcu_assign_pointer(xfrm_state_afinfo
[afinfo
->family
], afinfo
);
2507 spin_unlock_bh(&xfrm_state_afinfo_lock
);
2510 EXPORT_SYMBOL(xfrm_state_register_afinfo
);
2512 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo
*afinfo
)
2514 int err
= 0, family
= afinfo
->family
;
2516 if (WARN_ON(family
>= NPROTO
))
2517 return -EAFNOSUPPORT
;
2519 spin_lock_bh(&xfrm_state_afinfo_lock
);
2520 if (likely(xfrm_state_afinfo
[afinfo
->family
] != NULL
)) {
2521 if (rcu_access_pointer(xfrm_state_afinfo
[family
]) != afinfo
)
2524 RCU_INIT_POINTER(xfrm_state_afinfo
[afinfo
->family
], NULL
);
2526 spin_unlock_bh(&xfrm_state_afinfo_lock
);
2530 EXPORT_SYMBOL(xfrm_state_unregister_afinfo
);
2532 struct xfrm_state_afinfo
*xfrm_state_afinfo_get_rcu(unsigned int family
)
2534 if (unlikely(family
>= NPROTO
))
2537 return rcu_dereference(xfrm_state_afinfo
[family
]);
2539 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu
);
2541 struct xfrm_state_afinfo
*xfrm_state_get_afinfo(unsigned int family
)
2543 struct xfrm_state_afinfo
*afinfo
;
2544 if (unlikely(family
>= NPROTO
))
2547 afinfo
= rcu_dereference(xfrm_state_afinfo
[family
]);
2548 if (unlikely(!afinfo
))
2553 void xfrm_flush_gc(void)
2555 flush_work(&xfrm_state_gc_work
);
2557 EXPORT_SYMBOL(xfrm_flush_gc
);
2559 /* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
2560 void xfrm_state_delete_tunnel(struct xfrm_state
*x
)
2563 struct xfrm_state
*t
= x
->tunnel
;
2565 if (atomic_read(&t
->tunnel_users
) == 2)
2566 xfrm_state_delete(t
);
2567 atomic_dec(&t
->tunnel_users
);
2568 xfrm_state_put_sync(t
);
2572 EXPORT_SYMBOL(xfrm_state_delete_tunnel
);
2574 u32
xfrm_state_mtu(struct xfrm_state
*x
, int mtu
)
2576 const struct xfrm_type
*type
= READ_ONCE(x
->type
);
2577 struct crypto_aead
*aead
;
2578 u32 blksize
, net_adj
= 0;
2580 if (x
->km
.state
!= XFRM_STATE_VALID
||
2581 !type
|| type
->proto
!= IPPROTO_ESP
)
2582 return mtu
- x
->props
.header_len
;
2585 blksize
= ALIGN(crypto_aead_blocksize(aead
), 4);
2587 switch (x
->props
.mode
) {
2588 case XFRM_MODE_TRANSPORT
:
2589 case XFRM_MODE_BEET
:
2590 if (x
->props
.family
== AF_INET
)
2591 net_adj
= sizeof(struct iphdr
);
2592 else if (x
->props
.family
== AF_INET6
)
2593 net_adj
= sizeof(struct ipv6hdr
);
2595 case XFRM_MODE_TUNNEL
:
2602 return ((mtu
- x
->props
.header_len
- crypto_aead_authsize(aead
) -
2603 net_adj
) & ~(blksize
- 1)) + net_adj
- 2;
2605 EXPORT_SYMBOL_GPL(xfrm_state_mtu
);
2607 int __xfrm_init_state(struct xfrm_state
*x
, bool init_replay
, bool offload
)
2609 const struct xfrm_mode
*inner_mode
;
2610 const struct xfrm_mode
*outer_mode
;
2611 int family
= x
->props
.family
;
2614 if (family
== AF_INET
&&
2615 xs_net(x
)->ipv4
.sysctl_ip_no_pmtu_disc
)
2616 x
->props
.flags
|= XFRM_STATE_NOPMTUDISC
;
2618 err
= -EPROTONOSUPPORT
;
2620 if (x
->sel
.family
!= AF_UNSPEC
) {
2621 inner_mode
= xfrm_get_mode(x
->props
.mode
, x
->sel
.family
);
2622 if (inner_mode
== NULL
)
2625 if (!(inner_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
) &&
2626 family
!= x
->sel
.family
)
2629 x
->inner_mode
= *inner_mode
;
2631 const struct xfrm_mode
*inner_mode_iaf
;
2632 int iafamily
= AF_INET
;
2634 inner_mode
= xfrm_get_mode(x
->props
.mode
, x
->props
.family
);
2635 if (inner_mode
== NULL
)
2638 if (!(inner_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
))
2641 x
->inner_mode
= *inner_mode
;
2643 if (x
->props
.family
== AF_INET
)
2644 iafamily
= AF_INET6
;
2646 inner_mode_iaf
= xfrm_get_mode(x
->props
.mode
, iafamily
);
2647 if (inner_mode_iaf
) {
2648 if (inner_mode_iaf
->flags
& XFRM_MODE_FLAG_TUNNEL
)
2649 x
->inner_mode_iaf
= *inner_mode_iaf
;
2653 x
->type
= xfrm_get_type(x
->id
.proto
, family
);
2654 if (x
->type
== NULL
)
2657 x
->type_offload
= xfrm_get_type_offload(x
->id
.proto
, family
, offload
);
2659 err
= x
->type
->init_state(x
);
2663 outer_mode
= xfrm_get_mode(x
->props
.mode
, family
);
2665 err
= -EPROTONOSUPPORT
;
2669 x
->outer_mode
= *outer_mode
;
2671 err
= xfrm_init_replay(x
);
2680 EXPORT_SYMBOL(__xfrm_init_state
);
2682 int xfrm_init_state(struct xfrm_state
*x
)
2686 err
= __xfrm_init_state(x
, true, false);
2688 x
->km
.state
= XFRM_STATE_VALID
;
2693 EXPORT_SYMBOL(xfrm_init_state
);
2695 int __net_init
xfrm_state_init(struct net
*net
)
2699 if (net_eq(net
, &init_net
))
2700 xfrm_state_cache
= KMEM_CACHE(xfrm_state
,
2701 SLAB_HWCACHE_ALIGN
| SLAB_PANIC
);
2703 INIT_LIST_HEAD(&net
->xfrm
.state_all
);
2705 sz
= sizeof(struct hlist_head
) * 8;
2707 net
->xfrm
.state_bydst
= xfrm_hash_alloc(sz
);
2708 if (!net
->xfrm
.state_bydst
)
2710 net
->xfrm
.state_bysrc
= xfrm_hash_alloc(sz
);
2711 if (!net
->xfrm
.state_bysrc
)
2713 net
->xfrm
.state_byspi
= xfrm_hash_alloc(sz
);
2714 if (!net
->xfrm
.state_byspi
)
2716 net
->xfrm
.state_byseq
= xfrm_hash_alloc(sz
);
2717 if (!net
->xfrm
.state_byseq
)
2719 net
->xfrm
.state_hmask
= ((sz
/ sizeof(struct hlist_head
)) - 1);
2721 net
->xfrm
.state_num
= 0;
2722 INIT_WORK(&net
->xfrm
.state_hash_work
, xfrm_hash_resize
);
2723 spin_lock_init(&net
->xfrm
.xfrm_state_lock
);
2724 seqcount_spinlock_init(&net
->xfrm
.xfrm_state_hash_generation
,
2725 &net
->xfrm
.xfrm_state_lock
);
2729 xfrm_hash_free(net
->xfrm
.state_byspi
, sz
);
2731 xfrm_hash_free(net
->xfrm
.state_bysrc
, sz
);
2733 xfrm_hash_free(net
->xfrm
.state_bydst
, sz
);
2738 void xfrm_state_fini(struct net
*net
)
2742 flush_work(&net
->xfrm
.state_hash_work
);
2743 flush_work(&xfrm_state_gc_work
);
2744 xfrm_state_flush(net
, 0, false, true);
2746 WARN_ON(!list_empty(&net
->xfrm
.state_all
));
2748 sz
= (net
->xfrm
.state_hmask
+ 1) * sizeof(struct hlist_head
);
2749 WARN_ON(!hlist_empty(net
->xfrm
.state_byseq
));
2750 xfrm_hash_free(net
->xfrm
.state_byseq
, sz
);
2751 WARN_ON(!hlist_empty(net
->xfrm
.state_byspi
));
2752 xfrm_hash_free(net
->xfrm
.state_byspi
, sz
);
2753 WARN_ON(!hlist_empty(net
->xfrm
.state_bysrc
));
2754 xfrm_hash_free(net
->xfrm
.state_bysrc
, sz
);
2755 WARN_ON(!hlist_empty(net
->xfrm
.state_bydst
));
2756 xfrm_hash_free(net
->xfrm
.state_bydst
, sz
);
2759 #ifdef CONFIG_AUDITSYSCALL
2760 static void xfrm_audit_helper_sainfo(struct xfrm_state
*x
,
2761 struct audit_buffer
*audit_buf
)
2763 struct xfrm_sec_ctx
*ctx
= x
->security
;
2764 u32 spi
= ntohl(x
->id
.spi
);
2767 audit_log_format(audit_buf
, " sec_alg=%u sec_doi=%u sec_obj=%s",
2768 ctx
->ctx_alg
, ctx
->ctx_doi
, ctx
->ctx_str
);
2770 switch (x
->props
.family
) {
2772 audit_log_format(audit_buf
, " src=%pI4 dst=%pI4",
2773 &x
->props
.saddr
.a4
, &x
->id
.daddr
.a4
);
2776 audit_log_format(audit_buf
, " src=%pI6 dst=%pI6",
2777 x
->props
.saddr
.a6
, x
->id
.daddr
.a6
);
2781 audit_log_format(audit_buf
, " spi=%u(0x%x)", spi
, spi
);
2784 static void xfrm_audit_helper_pktinfo(struct sk_buff
*skb
, u16 family
,
2785 struct audit_buffer
*audit_buf
)
2787 const struct iphdr
*iph4
;
2788 const struct ipv6hdr
*iph6
;
2793 audit_log_format(audit_buf
, " src=%pI4 dst=%pI4",
2794 &iph4
->saddr
, &iph4
->daddr
);
2797 iph6
= ipv6_hdr(skb
);
2798 audit_log_format(audit_buf
,
2799 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
2800 &iph6
->saddr
, &iph6
->daddr
,
2801 iph6
->flow_lbl
[0] & 0x0f,
2808 void xfrm_audit_state_add(struct xfrm_state
*x
, int result
, bool task_valid
)
2810 struct audit_buffer
*audit_buf
;
2812 audit_buf
= xfrm_audit_start("SAD-add");
2813 if (audit_buf
== NULL
)
2815 xfrm_audit_helper_usrinfo(task_valid
, audit_buf
);
2816 xfrm_audit_helper_sainfo(x
, audit_buf
);
2817 audit_log_format(audit_buf
, " res=%u", result
);
2818 audit_log_end(audit_buf
);
2820 EXPORT_SYMBOL_GPL(xfrm_audit_state_add
);
2822 void xfrm_audit_state_delete(struct xfrm_state
*x
, int result
, bool task_valid
)
2824 struct audit_buffer
*audit_buf
;
2826 audit_buf
= xfrm_audit_start("SAD-delete");
2827 if (audit_buf
== NULL
)
2829 xfrm_audit_helper_usrinfo(task_valid
, audit_buf
);
2830 xfrm_audit_helper_sainfo(x
, audit_buf
);
2831 audit_log_format(audit_buf
, " res=%u", result
);
2832 audit_log_end(audit_buf
);
2834 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete
);
2836 void xfrm_audit_state_replay_overflow(struct xfrm_state
*x
,
2837 struct sk_buff
*skb
)
2839 struct audit_buffer
*audit_buf
;
2842 audit_buf
= xfrm_audit_start("SA-replay-overflow");
2843 if (audit_buf
== NULL
)
2845 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2846 /* don't record the sequence number because it's inherent in this kind
2847 * of audit message */
2848 spi
= ntohl(x
->id
.spi
);
2849 audit_log_format(audit_buf
, " spi=%u(0x%x)", spi
, spi
);
2850 audit_log_end(audit_buf
);
2852 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow
);
2854 void xfrm_audit_state_replay(struct xfrm_state
*x
,
2855 struct sk_buff
*skb
, __be32 net_seq
)
2857 struct audit_buffer
*audit_buf
;
2860 audit_buf
= xfrm_audit_start("SA-replayed-pkt");
2861 if (audit_buf
== NULL
)
2863 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2864 spi
= ntohl(x
->id
.spi
);
2865 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2866 spi
, spi
, ntohl(net_seq
));
2867 audit_log_end(audit_buf
);
2869 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay
);
2871 void xfrm_audit_state_notfound_simple(struct sk_buff
*skb
, u16 family
)
2873 struct audit_buffer
*audit_buf
;
2875 audit_buf
= xfrm_audit_start("SA-notfound");
2876 if (audit_buf
== NULL
)
2878 xfrm_audit_helper_pktinfo(skb
, family
, audit_buf
);
2879 audit_log_end(audit_buf
);
2881 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple
);
2883 void xfrm_audit_state_notfound(struct sk_buff
*skb
, u16 family
,
2884 __be32 net_spi
, __be32 net_seq
)
2886 struct audit_buffer
*audit_buf
;
2889 audit_buf
= xfrm_audit_start("SA-notfound");
2890 if (audit_buf
== NULL
)
2892 xfrm_audit_helper_pktinfo(skb
, family
, audit_buf
);
2893 spi
= ntohl(net_spi
);
2894 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2895 spi
, spi
, ntohl(net_seq
));
2896 audit_log_end(audit_buf
);
2898 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound
);
2900 void xfrm_audit_state_icvfail(struct xfrm_state
*x
,
2901 struct sk_buff
*skb
, u8 proto
)
2903 struct audit_buffer
*audit_buf
;
2907 audit_buf
= xfrm_audit_start("SA-icv-failure");
2908 if (audit_buf
== NULL
)
2910 xfrm_audit_helper_pktinfo(skb
, x
->props
.family
, audit_buf
);
2911 if (xfrm_parse_spi(skb
, proto
, &net_spi
, &net_seq
) == 0) {
2912 u32 spi
= ntohl(net_spi
);
2913 audit_log_format(audit_buf
, " spi=%u(0x%x) seqno=%u",
2914 spi
, spi
, ntohl(net_seq
));
2916 audit_log_end(audit_buf
);
2918 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail
);
2919 #endif /* CONFIG_AUDITSYSCALL */