1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/jhash.h>
3 #include <linux/netfilter.h>
4 #include <linux/rcupdate.h>
5 #include <linux/rhashtable.h>
6 #include <linux/vmalloc.h>
7 #include <net/genetlink.h>
9 #include <net/netns/generic.h>
10 #include <uapi/linux/genetlink.h>
13 struct ila_xlat_params
{
19 struct ila_xlat_params xp
;
20 struct rhash_head node
;
21 struct ila_map __rcu
*next
;
25 static unsigned int ila_net_id
;
28 struct rhashtable rhash_table
;
29 spinlock_t
*locks
; /* Bucket locks for entry manipulation */
30 unsigned int locks_mask
;
31 bool hooks_registered
;
34 #define LOCKS_PER_CPU 10
36 static int alloc_ila_locks(struct ila_net
*ilan
)
39 unsigned int nr_pcpus
= num_possible_cpus();
41 nr_pcpus
= min_t(unsigned int, nr_pcpus
, 32UL);
42 size
= roundup_pow_of_two(nr_pcpus
* LOCKS_PER_CPU
);
44 if (sizeof(spinlock_t
) != 0) {
45 ilan
->locks
= kvmalloc(size
* sizeof(spinlock_t
), GFP_KERNEL
);
48 for (i
= 0; i
< size
; i
++)
49 spin_lock_init(&ilan
->locks
[i
]);
51 ilan
->locks_mask
= size
- 1;
56 static u32 hashrnd __read_mostly
;
57 static __always_inline
void __ila_hash_secret_init(void)
59 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
62 static inline u32
ila_locator_hash(struct ila_locator loc
)
64 u32
*v
= (u32
*)loc
.v32
;
66 __ila_hash_secret_init();
67 return jhash_2words(v
[0], v
[1], hashrnd
);
70 static inline spinlock_t
*ila_get_lock(struct ila_net
*ilan
,
71 struct ila_locator loc
)
73 return &ilan
->locks
[ila_locator_hash(loc
) & ilan
->locks_mask
];
76 static inline int ila_cmp_wildcards(struct ila_map
*ila
,
77 struct ila_addr
*iaddr
, int ifindex
)
79 return (ila
->xp
.ifindex
&& ila
->xp
.ifindex
!= ifindex
);
82 static inline int ila_cmp_params(struct ila_map
*ila
,
83 struct ila_xlat_params
*xp
)
85 return (ila
->xp
.ifindex
!= xp
->ifindex
);
88 static int ila_cmpfn(struct rhashtable_compare_arg
*arg
,
91 const struct ila_map
*ila
= obj
;
93 return (ila
->xp
.ip
.locator_match
.v64
!= *(__be64
*)arg
->key
);
96 static inline int ila_order(struct ila_map
*ila
)
106 static const struct rhashtable_params rht_params
= {
108 .head_offset
= offsetof(struct ila_map
, node
),
109 .key_offset
= offsetof(struct ila_map
, xp
.ip
.locator_match
),
110 .key_len
= sizeof(u64
), /* identifier */
113 .automatic_shrinking
= true,
114 .obj_cmpfn
= ila_cmpfn
,
117 static struct genl_family ila_nl_family
;
119 static const struct nla_policy ila_nl_policy
[ILA_ATTR_MAX
+ 1] = {
120 [ILA_ATTR_LOCATOR
] = { .type
= NLA_U64
, },
121 [ILA_ATTR_LOCATOR_MATCH
] = { .type
= NLA_U64
, },
122 [ILA_ATTR_IFINDEX
] = { .type
= NLA_U32
, },
123 [ILA_ATTR_CSUM_MODE
] = { .type
= NLA_U8
, },
126 static int parse_nl_config(struct genl_info
*info
,
127 struct ila_xlat_params
*xp
)
129 memset(xp
, 0, sizeof(*xp
));
131 if (info
->attrs
[ILA_ATTR_LOCATOR
])
132 xp
->ip
.locator
.v64
= (__force __be64
)nla_get_u64(
133 info
->attrs
[ILA_ATTR_LOCATOR
]);
135 if (info
->attrs
[ILA_ATTR_LOCATOR_MATCH
])
136 xp
->ip
.locator_match
.v64
= (__force __be64
)nla_get_u64(
137 info
->attrs
[ILA_ATTR_LOCATOR_MATCH
]);
139 if (info
->attrs
[ILA_ATTR_CSUM_MODE
])
140 xp
->ip
.csum_mode
= nla_get_u8(info
->attrs
[ILA_ATTR_CSUM_MODE
]);
142 if (info
->attrs
[ILA_ATTR_IFINDEX
])
143 xp
->ifindex
= nla_get_s32(info
->attrs
[ILA_ATTR_IFINDEX
]);
148 /* Must be called with rcu readlock */
149 static inline struct ila_map
*ila_lookup_wildcards(struct ila_addr
*iaddr
,
151 struct ila_net
*ilan
)
155 ila
= rhashtable_lookup_fast(&ilan
->rhash_table
, &iaddr
->loc
,
158 if (!ila_cmp_wildcards(ila
, iaddr
, ifindex
))
160 ila
= rcu_access_pointer(ila
->next
);
166 /* Must be called with rcu readlock */
167 static inline struct ila_map
*ila_lookup_by_params(struct ila_xlat_params
*xp
,
168 struct ila_net
*ilan
)
172 ila
= rhashtable_lookup_fast(&ilan
->rhash_table
,
173 &xp
->ip
.locator_match
,
176 if (!ila_cmp_params(ila
, xp
))
178 ila
= rcu_access_pointer(ila
->next
);
184 static inline void ila_release(struct ila_map
*ila
)
189 static void ila_free_cb(void *ptr
, void *arg
)
191 struct ila_map
*ila
= (struct ila_map
*)ptr
, *next
;
193 /* Assume rcu_readlock held */
195 next
= rcu_access_pointer(ila
->next
);
201 static int ila_xlat_addr(struct sk_buff
*skb
, bool set_csum_neutral
);
204 ila_nf_input(void *priv
,
206 const struct nf_hook_state
*state
)
208 ila_xlat_addr(skb
, false);
212 static const struct nf_hook_ops ila_nf_hook_ops
[] = {
214 .hook
= ila_nf_input
,
216 .hooknum
= NF_INET_PRE_ROUTING
,
221 static int ila_add_mapping(struct net
*net
, struct ila_xlat_params
*xp
)
223 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
224 struct ila_map
*ila
, *head
;
225 spinlock_t
*lock
= ila_get_lock(ilan
, xp
->ip
.locator_match
);
228 if (!ilan
->hooks_registered
) {
229 /* We defer registering net hooks in the namespace until the
230 * first mapping is added.
232 err
= nf_register_net_hooks(net
, ila_nf_hook_ops
,
233 ARRAY_SIZE(ila_nf_hook_ops
));
237 ilan
->hooks_registered
= true;
240 ila
= kzalloc(sizeof(*ila
), GFP_KERNEL
);
244 ila_init_saved_csum(&xp
->ip
);
248 order
= ila_order(ila
);
252 head
= rhashtable_lookup_fast(&ilan
->rhash_table
,
253 &xp
->ip
.locator_match
,
256 /* New entry for the rhash_table */
257 err
= rhashtable_lookup_insert_fast(&ilan
->rhash_table
,
258 &ila
->node
, rht_params
);
260 struct ila_map
*tila
= head
, *prev
= NULL
;
263 if (!ila_cmp_params(tila
, xp
)) {
268 if (order
> ila_order(tila
))
272 tila
= rcu_dereference_protected(tila
->next
,
273 lockdep_is_held(lock
));
277 /* Insert in sub list of head */
278 RCU_INIT_POINTER(ila
->next
, tila
);
279 rcu_assign_pointer(prev
->next
, ila
);
281 /* Make this ila new head */
282 RCU_INIT_POINTER(ila
->next
, head
);
283 err
= rhashtable_replace_fast(&ilan
->rhash_table
,
285 &ila
->node
, rht_params
);
300 static int ila_del_mapping(struct net
*net
, struct ila_xlat_params
*xp
)
302 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
303 struct ila_map
*ila
, *head
, *prev
;
304 spinlock_t
*lock
= ila_get_lock(ilan
, xp
->ip
.locator_match
);
309 head
= rhashtable_lookup_fast(&ilan
->rhash_table
,
310 &xp
->ip
.locator_match
, rht_params
);
316 if (ila_cmp_params(ila
, xp
)) {
318 ila
= rcu_dereference_protected(ila
->next
,
319 lockdep_is_held(lock
));
326 /* Not head, just delete from list */
327 rcu_assign_pointer(prev
->next
, ila
->next
);
329 /* It is the head. If there is something in the
330 * sublist we need to make a new head.
332 head
= rcu_dereference_protected(ila
->next
,
333 lockdep_is_held(lock
));
335 /* Put first entry in the sublist into the
338 err
= rhashtable_replace_fast(
339 &ilan
->rhash_table
, &ila
->node
,
340 &head
->node
, rht_params
);
344 /* Entry no longer used */
345 err
= rhashtable_remove_fast(&ilan
->rhash_table
,
362 static int ila_nl_cmd_add_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
364 struct net
*net
= genl_info_net(info
);
365 struct ila_xlat_params p
;
368 err
= parse_nl_config(info
, &p
);
372 return ila_add_mapping(net
, &p
);
375 static int ila_nl_cmd_del_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
377 struct net
*net
= genl_info_net(info
);
378 struct ila_xlat_params xp
;
381 err
= parse_nl_config(info
, &xp
);
385 ila_del_mapping(net
, &xp
);
390 static int ila_fill_info(struct ila_map
*ila
, struct sk_buff
*msg
)
392 if (nla_put_u64_64bit(msg
, ILA_ATTR_LOCATOR
,
393 (__force u64
)ila
->xp
.ip
.locator
.v64
,
395 nla_put_u64_64bit(msg
, ILA_ATTR_LOCATOR_MATCH
,
396 (__force u64
)ila
->xp
.ip
.locator_match
.v64
,
398 nla_put_s32(msg
, ILA_ATTR_IFINDEX
, ila
->xp
.ifindex
) ||
399 nla_put_u32(msg
, ILA_ATTR_CSUM_MODE
, ila
->xp
.ip
.csum_mode
))
405 static int ila_dump_info(struct ila_map
*ila
,
406 u32 portid
, u32 seq
, u32 flags
,
407 struct sk_buff
*skb
, u8 cmd
)
411 hdr
= genlmsg_put(skb
, portid
, seq
, &ila_nl_family
, flags
, cmd
);
415 if (ila_fill_info(ila
, skb
) < 0)
416 goto nla_put_failure
;
418 genlmsg_end(skb
, hdr
);
422 genlmsg_cancel(skb
, hdr
);
426 static int ila_nl_cmd_get_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
428 struct net
*net
= genl_info_net(info
);
429 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
431 struct ila_xlat_params xp
;
435 ret
= parse_nl_config(info
, &xp
);
439 msg
= nlmsg_new(NLMSG_DEFAULT_SIZE
, GFP_KERNEL
);
445 ila
= ila_lookup_by_params(&xp
, ilan
);
447 ret
= ila_dump_info(ila
,
449 info
->snd_seq
, 0, msg
,
458 return genlmsg_reply(msg
, info
);
465 struct ila_dump_iter
{
466 struct rhashtable_iter rhiter
;
469 static int ila_nl_dump_start(struct netlink_callback
*cb
)
471 struct net
*net
= sock_net(cb
->skb
->sk
);
472 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
473 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
[0];
476 iter
= kmalloc(sizeof(*iter
), GFP_KERNEL
);
480 cb
->args
[0] = (long)iter
;
483 return rhashtable_walk_init(&ilan
->rhash_table
, &iter
->rhiter
,
487 static int ila_nl_dump_done(struct netlink_callback
*cb
)
489 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
[0];
491 rhashtable_walk_exit(&iter
->rhiter
);
498 static int ila_nl_dump(struct sk_buff
*skb
, struct netlink_callback
*cb
)
500 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
[0];
501 struct rhashtable_iter
*rhiter
= &iter
->rhiter
;
505 ret
= rhashtable_walk_start(rhiter
);
506 if (ret
&& ret
!= -EAGAIN
)
510 ila
= rhashtable_walk_next(rhiter
);
513 if (PTR_ERR(ila
) == -EAGAIN
)
522 ret
= ila_dump_info(ila
, NETLINK_CB(cb
->skb
).portid
,
523 cb
->nlh
->nlmsg_seq
, NLM_F_MULTI
,
528 ila
= rcu_access_pointer(ila
->next
);
535 rhashtable_walk_stop(rhiter
);
539 static const struct genl_ops ila_nl_ops
[] = {
542 .doit
= ila_nl_cmd_add_mapping
,
543 .policy
= ila_nl_policy
,
544 .flags
= GENL_ADMIN_PERM
,
548 .doit
= ila_nl_cmd_del_mapping
,
549 .policy
= ila_nl_policy
,
550 .flags
= GENL_ADMIN_PERM
,
554 .doit
= ila_nl_cmd_get_mapping
,
555 .start
= ila_nl_dump_start
,
556 .dumpit
= ila_nl_dump
,
557 .done
= ila_nl_dump_done
,
558 .policy
= ila_nl_policy
,
562 static struct genl_family ila_nl_family __ro_after_init
= {
564 .name
= ILA_GENL_NAME
,
565 .version
= ILA_GENL_VERSION
,
566 .maxattr
= ILA_ATTR_MAX
,
568 .parallel_ops
= true,
569 .module
= THIS_MODULE
,
571 .n_ops
= ARRAY_SIZE(ila_nl_ops
),
574 #define ILA_HASH_TABLE_SIZE 1024
576 static __net_init
int ila_init_net(struct net
*net
)
579 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
581 err
= alloc_ila_locks(ilan
);
585 rhashtable_init(&ilan
->rhash_table
, &rht_params
);
590 static __net_exit
void ila_exit_net(struct net
*net
)
592 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
594 rhashtable_free_and_destroy(&ilan
->rhash_table
, ila_free_cb
, NULL
);
598 if (ilan
->hooks_registered
)
599 nf_unregister_net_hooks(net
, ila_nf_hook_ops
,
600 ARRAY_SIZE(ila_nf_hook_ops
));
603 static struct pernet_operations ila_net_ops
= {
604 .init
= ila_init_net
,
605 .exit
= ila_exit_net
,
607 .size
= sizeof(struct ila_net
),
610 static int ila_xlat_addr(struct sk_buff
*skb
, bool set_csum_neutral
)
613 struct ipv6hdr
*ip6h
= ipv6_hdr(skb
);
614 struct net
*net
= dev_net(skb
->dev
);
615 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
616 struct ila_addr
*iaddr
= ila_a2i(&ip6h
->daddr
);
618 /* Assumes skb contains a valid IPv6 header that is pulled */
620 if (!ila_addr_is_ila(iaddr
)) {
621 /* Type indicates this is not an ILA address */
627 ila
= ila_lookup_wildcards(iaddr
, skb
->dev
->ifindex
, ilan
);
629 ila_update_ipv6_locator(skb
, &ila
->xp
.ip
, set_csum_neutral
);
636 int __init
ila_xlat_init(void)
640 ret
= register_pernet_device(&ila_net_ops
);
644 ret
= genl_register_family(&ila_nl_family
);
651 unregister_pernet_device(&ila_net_ops
);
656 void ila_xlat_fini(void)
658 genl_unregister_family(&ila_nl_family
);
659 unregister_pernet_device(&ila_net_ops
);