[mirror_ubuntu-artful-kernel.git] / net / ipv6 / ila / ila_xlat.c

#include <linux/jhash.h>
#include <linux/netfilter.h>
#include <linux/rcupdate.h>
#include <linux/rhashtable.h>
#include <linux/vmalloc.h>
#include <net/genetlink.h>
#include <net/ila.h>
#include <net/netns/generic.h>
#include <uapi/linux/genetlink.h>
#include "ila.h"

struct ila_xlat_params {
	struct ila_params ip;
	int ifindex;
};

struct ila_map {
	struct ila_xlat_params xp;
	struct rhash_head node;
	struct ila_map __rcu *next;
	struct rcu_head rcu;
};

static unsigned int ila_net_id;

struct ila_net {
	struct rhashtable rhash_table;
	spinlock_t *locks; /* Bucket locks for entry manipulation */
	unsigned int locks_mask;
	bool hooks_registered;
};

#define	LOCKS_PER_CPU 10

static int alloc_ila_locks(struct ila_net *ilan)
{
	unsigned int i, size;
	unsigned int nr_pcpus = num_possible_cpus();

	nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
	size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);

	if (sizeof(spinlock_t) != 0) {
		ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL);
		if (!ilan->locks)
			return -ENOMEM;
		for (i = 0; i < size; i++)
			spin_lock_init(&ilan->locks[i]);
	}
	ilan->locks_mask = size - 1;

	return 0;
}

static u32 hashrnd __read_mostly;
static __always_inline void __ila_hash_secret_init(void)
{
	net_get_random_once(&hashrnd, sizeof(hashrnd));
}

static inline u32 ila_locator_hash(struct ila_locator loc)
{
	u32 *v = (u32 *)loc.v32;

	__ila_hash_secret_init();
	return jhash_2words(v[0], v[1], hashrnd);
}

static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
				       struct ila_locator loc)
{
	return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
}

static inline int ila_cmp_wildcards(struct ila_map *ila,
				    struct ila_addr *iaddr, int ifindex)
{
	return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
}

static inline int ila_cmp_params(struct ila_map *ila,
				 struct ila_xlat_params *xp)
{
	return (ila->xp.ifindex != xp->ifindex);
}

static int ila_cmpfn(struct rhashtable_compare_arg *arg,
		     const void *obj)
{
	const struct ila_map *ila = obj;

	return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
}

static inline int ila_order(struct ila_map *ila)
{
	int score = 0;

	if (ila->xp.ifindex)
		score += 1 << 1;

	return score;
}

static const struct rhashtable_params rht_params = {
	.nelem_hint = 1024,
	.head_offset = offsetof(struct ila_map, node),
	.key_offset = offsetof(struct ila_map, xp.ip.locator_match),
	.key_len = sizeof(u64), /* identifier */
	.max_size = 1048576,
	.min_size = 256,
	.automatic_shrinking = true,
	.obj_cmpfn = ila_cmpfn,
};

static struct genl_family ila_nl_family;

static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
	[ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
	[ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
	[ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
	[ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
};

static int parse_nl_config(struct genl_info *info,
			   struct ila_xlat_params *xp)
{
	memset(xp, 0, sizeof(*xp));

	if (info->attrs[ILA_ATTR_LOCATOR])
		xp->ip.locator.v64 = (__force __be64)nla_get_u64(
			info->attrs[ILA_ATTR_LOCATOR]);

	if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
		xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
			info->attrs[ILA_ATTR_LOCATOR_MATCH]);

	if (info->attrs[ILA_ATTR_CSUM_MODE])
		xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);

	if (info->attrs[ILA_ATTR_IFINDEX])
		xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);

	return 0;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
						   int ifindex,
						   struct ila_net *ilan)
{
	struct ila_map *ila;

	ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
				     rht_params);
	while (ila) {
		if (!ila_cmp_wildcards(ila, iaddr, ifindex))
			return ila;
		ila = rcu_access_pointer(ila->next);
	}

	return NULL;
}

/* Must be called with rcu readlock */
static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
						   struct ila_net *ilan)
{
	struct ila_map *ila;

	ila = rhashtable_lookup_fast(&ilan->rhash_table,
				     &xp->ip.locator_match,
				     rht_params);
	while (ila) {
		if (!ila_cmp_params(ila, xp))
			return ila;
		ila = rcu_access_pointer(ila->next);
	}

	return NULL;
}

static inline void ila_release(struct ila_map *ila)
{
	kfree_rcu(ila, rcu);
}

static void ila_free_cb(void *ptr, void *arg)
{
	struct ila_map *ila = (struct ila_map *)ptr, *next;

	/* Assume rcu_readlock held */
	while (ila) {
		next = rcu_access_pointer(ila->next);
		ila_release(ila);
		ila = next;
	}
}

static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);

static unsigned int
ila_nf_input(void *priv,
	     struct sk_buff *skb,
	     const struct nf_hook_state *state)
{
	ila_xlat_addr(skb, false);
	return NF_ACCEPT;
}

static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
	{
		.hook = ila_nf_input,
		.pf = NFPROTO_IPV6,
		.hooknum = NF_INET_PRE_ROUTING,
		.priority = -1,
	},
};

static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_map *ila, *head;
	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
	int err = 0, order;

	if (!ilan->hooks_registered) {
		/* We defer registering net hooks in the namespace until the
		 * first mapping is added.
		 */
		err = nf_register_net_hooks(net, ila_nf_hook_ops,
					    ARRAY_SIZE(ila_nf_hook_ops));
		if (err)
			return err;

		ilan->hooks_registered = true;
	}

	ila = kzalloc(sizeof(*ila), GFP_KERNEL);
	if (!ila)
		return -ENOMEM;

	ila_init_saved_csum(&xp->ip);

	ila->xp = *xp;

	order = ila_order(ila);

	spin_lock(lock);

	head = rhashtable_lookup_fast(&ilan->rhash_table,
				      &xp->ip.locator_match,
				      rht_params);
	if (!head) {
		/* New entry for the rhash_table */
		err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
						    &ila->node, rht_params);
	} else {
		struct ila_map *tila = head, *prev = NULL;

		do {
			if (!ila_cmp_params(tila, xp)) {
				err = -EEXIST;
				goto out;
			}

			if (order > ila_order(tila))
				break;

			prev = tila;
			tila = rcu_dereference_protected(tila->next,
				lockdep_is_held(lock));
		} while (tila);

		if (prev) {
			/* Insert in sub list of head */
			RCU_INIT_POINTER(ila->next, tila);
			rcu_assign_pointer(prev->next, ila);
		} else {
			/* Make this ila new head */
			RCU_INIT_POINTER(ila->next, head);
			err = rhashtable_replace_fast(&ilan->rhash_table,
						      &head->node,
						      &ila->node, rht_params);
			if (err)
				goto out;
		}
	}

out:
	spin_unlock(lock);

	if (err)
		kfree(ila);

	return err;
}

static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_map *ila, *head, *prev;
	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
	int err = -ENOENT;

	spin_lock(lock);

	head = rhashtable_lookup_fast(&ilan->rhash_table,
				      &xp->ip.locator_match, rht_params);
	ila = head;

	prev = NULL;

	while (ila) {
		if (ila_cmp_params(ila, xp)) {
			prev = ila;
			ila = rcu_dereference_protected(ila->next,
							lockdep_is_held(lock));
			continue;
		}

		err = 0;

		if (prev) {
			/* Not head, just delete from list */
			rcu_assign_pointer(prev->next, ila->next);
		} else {
			/* It is the head. If there is something in the
			 * sublist we need to make a new head.
			 */
			head = rcu_dereference_protected(ila->next,
							 lockdep_is_held(lock));
			if (head) {
				/* Put first entry in the sublist into the
				 * table
				 */
				err = rhashtable_replace_fast(
					&ilan->rhash_table, &ila->node,
					&head->node, rht_params);
				if (err)
					goto out;
			} else {
				/* Entry no longer used */
				err = rhashtable_remove_fast(&ilan->rhash_table,
							     &ila->node,
							     rht_params);
			}
		}

		ila_release(ila);

		break;
	}

out:
	spin_unlock(lock);

	return err;
}

static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_xlat_params p;
	int err;

	err = parse_nl_config(info, &p);
	if (err)
		return err;

	return ila_add_mapping(net, &p);
}

static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_xlat_params xp;
	int err;

	err = parse_nl_config(info, &xp);
	if (err)
		return err;

	ila_del_mapping(net, &xp);

	return 0;
}

static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
{
	if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
			      (__force u64)ila->xp.ip.locator.v64,
			      ILA_ATTR_PAD) ||
	    nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
			      (__force u64)ila->xp.ip.locator_match.v64,
			      ILA_ATTR_PAD) ||
	    nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
	    nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
		return -1;

	return 0;
}

static int ila_dump_info(struct ila_map *ila,
			 u32 portid, u32 seq, u32 flags,
			 struct sk_buff *skb, u8 cmd)
{
	void *hdr;

	hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
	if (!hdr)
		return -ENOMEM;

	if (ila_fill_info(ila, skb) < 0)
		goto nla_put_failure;

	genlmsg_end(skb, hdr);
	return 0;

nla_put_failure:
	genlmsg_cancel(skb, hdr);
	return -EMSGSIZE;
}

static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
{
	struct net *net = genl_info_net(info);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct sk_buff *msg;
	struct ila_xlat_params xp;
	struct ila_map *ila;
	int ret;

	ret = parse_nl_config(info, &xp);
	if (ret)
		return ret;

	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
	if (!msg)
		return -ENOMEM;

	rcu_read_lock();

	ila = ila_lookup_by_params(&xp, ilan);
	if (ila) {
		ret = ila_dump_info(ila,
				    info->snd_portid,
				    info->snd_seq, 0, msg,
				    info->genlhdr->cmd);
	}

	rcu_read_unlock();

	if (ret < 0)
		goto out_free;

	return genlmsg_reply(msg, info);

out_free:
	nlmsg_free(msg);
	return ret;
}

struct ila_dump_iter {
	struct rhashtable_iter rhiter;
};

static int ila_nl_dump_start(struct netlink_callback *cb)
{
	struct net *net = sock_net(cb->skb->sk);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];

	if (!iter) {
		iter = kmalloc(sizeof(*iter), GFP_KERNEL);
		if (!iter)
			return -ENOMEM;

		cb->args[0] = (long)iter;
	}

	return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
				    GFP_KERNEL);
}

static int ila_nl_dump_done(struct netlink_callback *cb)
{
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];

	rhashtable_walk_exit(&iter->rhiter);

	kfree(iter);

	return 0;
}

static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
	struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
	struct rhashtable_iter *rhiter = &iter->rhiter;
	struct ila_map *ila;
	int ret;

	ret = rhashtable_walk_start(rhiter);
	if (ret && ret != -EAGAIN)
		goto done;

	for (;;) {
		ila = rhashtable_walk_next(rhiter);

		if (IS_ERR(ila)) {
			if (PTR_ERR(ila) == -EAGAIN)
				continue;
			ret = PTR_ERR(ila);
			goto done;
		} else if (!ila) {
			break;
		}

		while (ila) {
			ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
					     cb->nlh->nlmsg_seq, NLM_F_MULTI,
					     skb, ILA_CMD_GET);
			if (ret)
				goto done;

			ila = rcu_access_pointer(ila->next);
		}
	}

	ret = skb->len;

done:
	rhashtable_walk_stop(rhiter);
	return ret;
}

static const struct genl_ops ila_nl_ops[] = {
	{
		.cmd = ILA_CMD_ADD,
		.doit = ila_nl_cmd_add_mapping,
		.policy = ila_nl_policy,
		.flags = GENL_ADMIN_PERM,
	},
	{
		.cmd = ILA_CMD_DEL,
		.doit = ila_nl_cmd_del_mapping,
		.policy = ila_nl_policy,
		.flags = GENL_ADMIN_PERM,
	},
	{
		.cmd = ILA_CMD_GET,
		.doit = ila_nl_cmd_get_mapping,
		.start = ila_nl_dump_start,
		.dumpit = ila_nl_dump,
		.done = ila_nl_dump_done,
		.policy = ila_nl_policy,
	},
};

static struct genl_family ila_nl_family __ro_after_init = {
	.hdrsize	= 0,
	.name		= ILA_GENL_NAME,
	.version	= ILA_GENL_VERSION,
	.maxattr	= ILA_ATTR_MAX,
	.netnsok	= true,
	.parallel_ops	= true,
	.module		= THIS_MODULE,
	.ops		= ila_nl_ops,
	.n_ops		= ARRAY_SIZE(ila_nl_ops),
};

#define ILA_HASH_TABLE_SIZE 1024

static __net_init int ila_init_net(struct net *net)
{
	int err;
	struct ila_net *ilan = net_generic(net, ila_net_id);

	err = alloc_ila_locks(ilan);
	if (err)
		return err;

	rhashtable_init(&ilan->rhash_table, &rht_params);

	return 0;
}

static __net_exit void ila_exit_net(struct net *net)
{
	struct ila_net *ilan = net_generic(net, ila_net_id);

	rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);

	kvfree(ilan->locks);

	if (ilan->hooks_registered)
		nf_unregister_net_hooks(net, ila_nf_hook_ops,
					ARRAY_SIZE(ila_nf_hook_ops));
}

static struct pernet_operations ila_net_ops = {
	.init = ila_init_net,
	.exit = ila_exit_net,
	.id   = &ila_net_id,
	.size = sizeof(struct ila_net),
};

static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
{
	struct ila_map *ila;
	struct ipv6hdr *ip6h = ipv6_hdr(skb);
	struct net *net = dev_net(skb->dev);
	struct ila_net *ilan = net_generic(net, ila_net_id);
	struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);

	/* Assumes skb contains a valid IPv6 header that is pulled */

	if (!ila_addr_is_ila(iaddr)) {
		/* Type indicates this is not an ILA address */
		return 0;
	}

	rcu_read_lock();

	ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
	if (ila)
		ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);

	rcu_read_unlock();

	return 0;
}

int __init ila_xlat_init(void)
{
	int ret;

	ret = register_pernet_device(&ila_net_ops);
	if (ret)
		goto exit;

	ret = genl_register_family(&ila_nl_family);
	if (ret < 0)
		goto unregister;

	return 0;

unregister:
	unregister_pernet_device(&ila_net_ops);
exit:
	return ret;
}

void ila_xlat_fini(void)
{
	genl_unregister_family(&ila_nl_family);
	unregister_pernet_device(&ila_net_ops);
}
Commit	Line	Data
7f00feaf TH	1	#include <linux/jhash.h>
	2	#include <linux/netfilter.h>
	3	#include <linux/rcupdate.h>
	4	#include <linux/rhashtable.h>
	5	#include <linux/vmalloc.h>
	6	#include <net/genetlink.h>
	7	#include <net/ila.h>
	8	#include <net/netns/generic.h>
	9	#include <uapi/linux/genetlink.h>
	10	#include "ila.h"
	11
	12	struct ila_xlat_params {
	13	struct ila_params ip;
7f00feaf	14	int ifindex;
7f00feaf TH	15	};
	16
	17	struct ila_map {
351596aa	18	struct ila_xlat_params xp;
7f00feaf TH	19	struct rhash_head node;
	20	struct ila_map __rcu *next;
	21	struct rcu_head rcu;
	22	};
	23
	24	static unsigned int ila_net_id;
	25
	26	struct ila_net {
	27	struct rhashtable rhash_table;
	28	spinlock_t locks; / Bucket locks for entry manipulation */
	29	unsigned int locks_mask;
	30	bool hooks_registered;
	31	};
	32
	33	#define LOCKS_PER_CPU 10
	34
	35	static int alloc_ila_locks(struct ila_net *ilan)
	36	{
	37	unsigned int i, size;
	38	unsigned int nr_pcpus = num_possible_cpus();
	39
	40	nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
	41	size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
	42
	43	if (sizeof(spinlock_t) != 0) {
847f716f	44	ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL);
7f00feaf TH	45	if (!ilan->locks)
	46	return -ENOMEM;
	47	for (i = 0; i < size; i++)
	48	spin_lock_init(&ilan->locks[i]);
	49	}
	50	ilan->locks_mask = size - 1;
	51
	52	return 0;
	53	}
	54
	55	static u32 hashrnd __read_mostly;
	56	static __always_inline void __ila_hash_secret_init(void)
	57	{
	58	net_get_random_once(&hashrnd, sizeof(hashrnd));
	59	}
	60
642c2c95	61	static inline u32 ila_locator_hash(struct ila_locator loc)
7f00feaf	62	{
642c2c95	63	u32 v = (u32 )loc.v32;
7f00feaf	64
0db47e3d	65	__ila_hash_secret_init();
7f00feaf TH	66	return jhash_2words(v[0], v[1], hashrnd);
	67	}
	68
351596aa	69	static inline spinlock_t ila_get_lock(struct ila_net ilan,
642c2c95	70	struct ila_locator loc)
7f00feaf	71	{
642c2c95	72	return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
7f00feaf TH	73	}
7f00feaf TH	74
351596aa	75	static inline int ila_cmp_wildcards(struct ila_map *ila,
642c2c95	76	struct ila_addr *iaddr, int ifindex)
7f00feaf	77	{
642c2c95	78	return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
7f00feaf TH	79	}
7f00feaf TH	80
351596aa TH	81	static inline int ila_cmp_params(struct ila_map *ila,
351596aa TH	82	struct ila_xlat_params *xp)
7f00feaf	83	{
642c2c95	84	return (ila->xp.ifindex != xp->ifindex);
7f00feaf TH	85	}
	86
	87	static int ila_cmpfn(struct rhashtable_compare_arg *arg,
	88	const void *obj)
	89	{
	90	const struct ila_map *ila = obj;
	91
642c2c95	92	return (ila->xp.ip.locator_match.v64 != (__be64 )arg->key);
7f00feaf TH	93	}
	94
	95	static inline int ila_order(struct ila_map *ila)
	96	{
	97	int score = 0;
	98
351596aa	99	if (ila->xp.ifindex)
7f00feaf TH	100	score += 1 << 1;
	101
	102	return score;
	103	}
	104
	105	static const struct rhashtable_params rht_params = {
	106	.nelem_hint = 1024,
	107	.head_offset = offsetof(struct ila_map, node),
642c2c95	108	.key_offset = offsetof(struct ila_map, xp.ip.locator_match),
7f00feaf TH	109	.key_len = sizeof(u64), /* identifier */
	110	.max_size = 1048576,
	111	.min_size = 256,
	112	.automatic_shrinking = true,
	113	.obj_cmpfn = ila_cmpfn,
	114	};
	115
489111e5	116	static struct genl_family ila_nl_family;
7f00feaf	117
6501f34f	118	static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
7f00feaf TH	119	[ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
	120	[ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
	121	[ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
90bfe662	122	[ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
7f00feaf TH	123	};
	124
	125	static int parse_nl_config(struct genl_info *info,
351596aa	126	struct ila_xlat_params *xp)
7f00feaf	127	{
351596aa	128	memset(xp, 0, sizeof(*xp));
7f00feaf	129
7f00feaf	130	if (info->attrs[ILA_ATTR_LOCATOR])
351596aa	131	xp->ip.locator.v64 = (__force __be64)nla_get_u64(
7f00feaf TH	132	info->attrs[ILA_ATTR_LOCATOR]);
	133
	134	if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
351596aa	135	xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
7f00feaf TH	136	info->attrs[ILA_ATTR_LOCATOR_MATCH]);
7f00feaf TH	137
90bfe662 TH	138	if (info->attrs[ILA_ATTR_CSUM_MODE])
	139	xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
	140
7f00feaf	141	if (info->attrs[ILA_ATTR_IFINDEX])
351596aa	142	xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
7f00feaf	143
7f00feaf TH	144	return 0;
	145	}
	146
	147	/* Must be called with rcu readlock */
351596aa	148	static inline struct ila_map ila_lookup_wildcards(struct ila_addr iaddr,
7f00feaf	149	int ifindex,
7f00feaf TH	150	struct ila_net *ilan)
	151	{
	152	struct ila_map *ila;
	153
642c2c95	154	ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
351596aa	155	rht_params);
7f00feaf	156	while (ila) {
642c2c95	157	if (!ila_cmp_wildcards(ila, iaddr, ifindex))
7f00feaf TH	158	return ila;
	159	ila = rcu_access_pointer(ila->next);
	160	}
	161
	162	return NULL;
	163	}
	164
	165	/* Must be called with rcu readlock */
351596aa	166	static inline struct ila_map ila_lookup_by_params(struct ila_xlat_params xp,
7f00feaf TH	167	struct ila_net *ilan)
	168	{
	169	struct ila_map *ila;
	170
642c2c95 TH	171	ila = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95 TH	172	&xp->ip.locator_match,
7f00feaf TH	173	rht_params);
7f00feaf TH	174	while (ila) {
351596aa	175	if (!ila_cmp_params(ila, xp))
7f00feaf TH	176	return ila;
	177	ila = rcu_access_pointer(ila->next);
	178	}
	179
	180	return NULL;
	181	}
	182
	183	static inline void ila_release(struct ila_map *ila)
	184	{
	185	kfree_rcu(ila, rcu);
	186	}
	187
	188	static void ila_free_cb(void ptr, void arg)
	189	{
	190	struct ila_map ila = (struct ila_map )ptr, *next;
	191
	192	/* Assume rcu_readlock held */
	193	while (ila) {
	194	next = rcu_access_pointer(ila->next);
	195	ila_release(ila);
	196	ila = next;
	197	}
	198	}
	199
707a2ca4	200	static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
7f00feaf TH	201
	202	static unsigned int
	203	ila_nf_input(void *priv,
	204	struct sk_buff *skb,
	205	const struct nf_hook_state *state)
	206	{
707a2ca4	207	ila_xlat_addr(skb, false);
7f00feaf TH	208	return NF_ACCEPT;
	209	}
	210
	211	static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
	212	{
	213	.hook = ila_nf_input,
	214	.pf = NFPROTO_IPV6,
	215	.hooknum = NF_INET_PRE_ROUTING,
	216	.priority = -1,
	217	},
	218	};
	219
351596aa	220	static int ila_add_mapping(struct net net, struct ila_xlat_params xp)
7f00feaf TH	221	{
	222	struct ila_net *ilan = net_generic(net, ila_net_id);
	223	struct ila_map ila, head;
642c2c95	224	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
7f00feaf TH	225	int err = 0, order;
	226
	227	if (!ilan->hooks_registered) {
	228	/* We defer registering net hooks in the namespace until the
	229	* first mapping is added.
	230	*/
	231	err = nf_register_net_hooks(net, ila_nf_hook_ops,
	232	ARRAY_SIZE(ila_nf_hook_ops));
	233	if (err)
	234	return err;
	235
	236	ilan->hooks_registered = true;
	237	}
	238
	239	ila = kzalloc(sizeof(*ila), GFP_KERNEL);
	240	if (!ila)
	241	return -ENOMEM;
	242
90bfe662	243	ila_init_saved_csum(&xp->ip);
7f00feaf	244
90bfe662	245	ila->xp = *xp;
7f00feaf TH	246
	247	order = ila_order(ila);
	248
	249	spin_lock(lock);
	250
642c2c95 TH	251	head = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95 TH	252	&xp->ip.locator_match,
7f00feaf TH	253	rht_params);
	254	if (!head) {
	255	/* New entry for the rhash_table */
	256	err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
	257	&ila->node, rht_params);
	258	} else {
	259	struct ila_map tila = head, prev = NULL;
	260
	261	do {
351596aa	262	if (!ila_cmp_params(tila, xp)) {
7f00feaf TH	263	err = -EEXIST;
	264	goto out;
	265	}
	266
	267	if (order > ila_order(tila))
	268	break;
	269
	270	prev = tila;
	271	tila = rcu_dereference_protected(tila->next,
	272	lockdep_is_held(lock));
	273	} while (tila);
	274
	275	if (prev) {
	276	/* Insert in sub list of head */
	277	RCU_INIT_POINTER(ila->next, tila);
	278	rcu_assign_pointer(prev->next, ila);
	279	} else {
	280	/* Make this ila new head */
	281	RCU_INIT_POINTER(ila->next, head);
	282	err = rhashtable_replace_fast(&ilan->rhash_table,
	283	&head->node,
	284	&ila->node, rht_params);
	285	if (err)
	286	goto out;
	287	}
	288	}
	289
	290	out:
	291	spin_unlock(lock);
	292
	293	if (err)
	294	kfree(ila);
	295
	296	return err;
	297	}
	298
351596aa	299	static int ila_del_mapping(struct net net, struct ila_xlat_params xp)
7f00feaf TH	300	{
	301	struct ila_net *ilan = net_generic(net, ila_net_id);
	302	struct ila_map ila, head, *prev;
642c2c95	303	spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
7f00feaf TH	304	int err = -ENOENT;
	305
	306	spin_lock(lock);
	307
	308	head = rhashtable_lookup_fast(&ilan->rhash_table,
642c2c95	309	&xp->ip.locator_match, rht_params);
7f00feaf TH	310	ila = head;
	311
	312	prev = NULL;
	313
	314	while (ila) {
351596aa	315	if (ila_cmp_params(ila, xp)) {
7f00feaf TH	316	prev = ila;
	317	ila = rcu_dereference_protected(ila->next,
	318	lockdep_is_held(lock));
	319	continue;
	320	}
	321
	322	err = 0;
	323
	324	if (prev) {
	325	/* Not head, just delete from list */
	326	rcu_assign_pointer(prev->next, ila->next);
	327	} else {
	328	/* It is the head. If there is something in the
	329	* sublist we need to make a new head.
	330	*/
	331	head = rcu_dereference_protected(ila->next,
	332	lockdep_is_held(lock));
	333	if (head) {
	334	/* Put first entry in the sublist into the
	335	* table
	336	*/
	337	err = rhashtable_replace_fast(
	338	&ilan->rhash_table, &ila->node,
	339	&head->node, rht_params);
	340	if (err)
	341	goto out;
	342	} else {
	343	/* Entry no longer used */
	344	err = rhashtable_remove_fast(&ilan->rhash_table,
	345	&ila->node,
	346	rht_params);
	347	}
	348	}
	349
	350	ila_release(ila);
	351
	352	break;
	353	}
	354
	355	out:
	356	spin_unlock(lock);
	357
	358	return err;
	359	}
	360
	361	static int ila_nl_cmd_add_mapping(struct sk_buff skb, struct genl_info info)
	362	{
	363	struct net *net = genl_info_net(info);
	364	struct ila_xlat_params p;
	365	int err;
	366
	367	err = parse_nl_config(info, &p);
	368	if (err)
	369	return err;
	370
	371	return ila_add_mapping(net, &p);
	372	}
	373
	374	static int ila_nl_cmd_del_mapping(struct sk_buff skb, struct genl_info info)
	375	{
	376	struct net *net = genl_info_net(info);
351596aa	377	struct ila_xlat_params xp;
7f00feaf TH	378	int err;
7f00feaf TH	379
351596aa	380	err = parse_nl_config(info, &xp);
7f00feaf TH	381	if (err)
	382	return err;
	383
351596aa	384	ila_del_mapping(net, &xp);
7f00feaf TH	385
	386	return 0;
	387	}
	388
	389	static int ila_fill_info(struct ila_map ila, struct sk_buff msg)
	390	{
642c2c95	391	if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
351596aa	392	(__force u64)ila->xp.ip.locator.v64,
f13a82d8 ND	393	ILA_ATTR_PAD) \|\|
f13a82d8 ND	394	nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
351596aa	395	(__force u64)ila->xp.ip.locator_match.v64,
f13a82d8	396	ILA_ATTR_PAD) \|\|
90bfe662 TH	397	nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) \|\|
90bfe662 TH	398	nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
7f00feaf TH	399	return -1;
	400
	401	return 0;
	402	}
	403
	404	static int ila_dump_info(struct ila_map *ila,
	405	u32 portid, u32 seq, u32 flags,
	406	struct sk_buff *skb, u8 cmd)
	407	{
	408	void *hdr;
	409
	410	hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
	411	if (!hdr)
	412	return -ENOMEM;
	413
	414	if (ila_fill_info(ila, skb) < 0)
	415	goto nla_put_failure;
	416
	417	genlmsg_end(skb, hdr);
	418	return 0;
	419
	420	nla_put_failure:
	421	genlmsg_cancel(skb, hdr);
	422	return -EMSGSIZE;
	423	}
	424
	425	static int ila_nl_cmd_get_mapping(struct sk_buff skb, struct genl_info info)
	426	{
	427	struct net *net = genl_info_net(info);
	428	struct ila_net *ilan = net_generic(net, ila_net_id);
	429	struct sk_buff *msg;
351596aa	430	struct ila_xlat_params xp;
7f00feaf TH	431	struct ila_map *ila;
	432	int ret;
	433
351596aa	434	ret = parse_nl_config(info, &xp);
7f00feaf TH	435	if (ret)
	436	return ret;
	437
	438	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
	439	if (!msg)
	440	return -ENOMEM;
	441
	442	rcu_read_lock();
	443
351596aa	444	ila = ila_lookup_by_params(&xp, ilan);
7f00feaf TH	445	if (ila) {
	446	ret = ila_dump_info(ila,
	447	info->snd_portid,
	448	info->snd_seq, 0, msg,
	449	info->genlhdr->cmd);
	450	}
	451
	452	rcu_read_unlock();
	453
	454	if (ret < 0)
	455	goto out_free;
	456
	457	return genlmsg_reply(msg, info);
	458
	459	out_free:
	460	nlmsg_free(msg);
	461	return ret;
	462	}
	463
	464	struct ila_dump_iter {
	465	struct rhashtable_iter rhiter;
	466	};
	467
	468	static int ila_nl_dump_start(struct netlink_callback *cb)
	469	{
	470	struct net *net = sock_net(cb->skb->sk);
	471	struct ila_net *ilan = net_generic(net, ila_net_id);
1913540a TH	472	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args[0];
	473
	474	if (!iter) {
	475	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
	476	if (!iter)
	477	return -ENOMEM;
	478
	479	cb->args[0] = (long)iter;
	480	}
7f00feaf	481
8f6fd83c BC	482	return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
8f6fd83c BC	483	GFP_KERNEL);
7f00feaf TH	484	}
	485
	486	static int ila_nl_dump_done(struct netlink_callback *cb)
	487	{
1913540a	488	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args[0];
7f00feaf TH	489
	490	rhashtable_walk_exit(&iter->rhiter);
	491
1913540a TH	492	kfree(iter);
1913540a TH	493
7f00feaf TH	494	return 0;
	495	}
	496
	497	static int ila_nl_dump(struct sk_buff skb, struct netlink_callback cb)
	498	{
1913540a	499	struct ila_dump_iter iter = (struct ila_dump_iter )cb->args[0];
7f00feaf TH	500	struct rhashtable_iter *rhiter = &iter->rhiter;
	501	struct ila_map *ila;
	502	int ret;
	503
	504	ret = rhashtable_walk_start(rhiter);
	505	if (ret && ret != -EAGAIN)
	506	goto done;
	507
	508	for (;;) {
	509	ila = rhashtable_walk_next(rhiter);
	510
	511	if (IS_ERR(ila)) {
	512	if (PTR_ERR(ila) == -EAGAIN)
	513	continue;
	514	ret = PTR_ERR(ila);
	515	goto done;
	516	} else if (!ila) {
	517	break;
	518	}
	519
	520	while (ila) {
	521	ret = ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
	522	cb->nlh->nlmsg_seq, NLM_F_MULTI,
	523	skb, ILA_CMD_GET);
	524	if (ret)
	525	goto done;
	526
	527	ila = rcu_access_pointer(ila->next);
	528	}
	529	}
	530
	531	ret = skb->len;
	532
	533	done:
	534	rhashtable_walk_stop(rhiter);
	535	return ret;
	536	}
	537
	538	static const struct genl_ops ila_nl_ops[] = {
	539	{
	540	.cmd = ILA_CMD_ADD,
	541	.doit = ila_nl_cmd_add_mapping,
	542	.policy = ila_nl_policy,
	543	.flags = GENL_ADMIN_PERM,
	544	},
	545	{
	546	.cmd = ILA_CMD_DEL,
	547	.doit = ila_nl_cmd_del_mapping,
	548	.policy = ila_nl_policy,
	549	.flags = GENL_ADMIN_PERM,
	550	},
	551	{
	552	.cmd = ILA_CMD_GET,
	553	.doit = ila_nl_cmd_get_mapping,
	554	.start = ila_nl_dump_start,
	555	.dumpit = ila_nl_dump,
	556	.done = ila_nl_dump_done,
	557	.policy = ila_nl_policy,
	558	},
	559	};
	560
56989f6d	561	static struct genl_family ila_nl_family __ro_after_init = {
489111e5 JB	562	.hdrsize = 0,
	563	.name = ILA_GENL_NAME,
	564	.version = ILA_GENL_VERSION,
	565	.maxattr = ILA_ATTR_MAX,
	566	.netnsok = true,
	567	.parallel_ops = true,
	568	.module = THIS_MODULE,
	569	.ops = ila_nl_ops,
	570	.n_ops = ARRAY_SIZE(ila_nl_ops),
	571	};
	572
7f00feaf TH	573	#define ILA_HASH_TABLE_SIZE 1024
	574
	575	static __net_init int ila_init_net(struct net *net)
	576	{
	577	int err;
	578	struct ila_net *ilan = net_generic(net, ila_net_id);
	579
	580	err = alloc_ila_locks(ilan);
	581	if (err)
	582	return err;
	583
	584	rhashtable_init(&ilan->rhash_table, &rht_params);
	585
	586	return 0;
	587	}
	588
	589	static __net_exit void ila_exit_net(struct net *net)
	590	{
	591	struct ila_net *ilan = net_generic(net, ila_net_id);
	592
	593	rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
	594
	595	kvfree(ilan->locks);
	596
	597	if (ilan->hooks_registered)
	598	nf_unregister_net_hooks(net, ila_nf_hook_ops,
	599	ARRAY_SIZE(ila_nf_hook_ops));
	600	}
	601
	602	static struct pernet_operations ila_net_ops = {
	603	.init = ila_init_net,
	604	.exit = ila_exit_net,
	605	.id = &ila_net_id,
	606	.size = sizeof(struct ila_net),
	607	};
	608
707a2ca4	609	static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
7f00feaf TH	610	{
	611	struct ila_map *ila;
	612	struct ipv6hdr *ip6h = ipv6_hdr(skb);
	613	struct net *net = dev_net(skb->dev);
	614	struct ila_net *ilan = net_generic(net, ila_net_id);
351596aa	615	struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
7f00feaf TH	616
	617	/* Assumes skb contains a valid IPv6 header that is pulled */
	618
642c2c95 TH	619	if (!ila_addr_is_ila(iaddr)) {
	620	/* Type indicates this is not an ILA address */
	621	return 0;
	622	}
7f00feaf TH	623
	624	rcu_read_lock();
	625
642c2c95	626	ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
7f00feaf	627	if (ila)
707a2ca4	628	ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
7f00feaf TH	629
	630	rcu_read_unlock();
	631
	632	return 0;
	633	}
	634
56989f6d	635	int __init ila_xlat_init(void)
7f00feaf TH	636	{
	637	int ret;
	638
	639	ret = register_pernet_device(&ila_net_ops);
	640	if (ret)
	641	goto exit;
	642
489111e5	643	ret = genl_register_family(&ila_nl_family);
7f00feaf TH	644	if (ret < 0)
	645	goto unregister;
	646
	647	return 0;
	648
	649	unregister:
	650	unregister_pernet_device(&ila_net_ops);
	651	exit:
	652	return ret;
	653	}
	654
	655	void ila_xlat_fini(void)
	656	{
	657	genl_unregister_family(&ila_nl_family);
	658	unregister_pernet_device(&ila_net_ops);
	659	}