[mirror_ubuntu-hirsute-kernel.git] / net / core / flow_offload.c

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <net/flow_offload.h>
#include <linux/rtnetlink.h>
#include <linux/mutex.h>
#include <linux/rhashtable.h>

struct flow_rule *flow_rule_alloc(unsigned int num_actions)
{
	struct flow_rule *rule;
	int i;

	rule = kzalloc(struct_size(rule, action.entries, num_actions),
		       GFP_KERNEL);
	if (!rule)
		return NULL;

	rule->action.num_entries = num_actions;
	/* Pre-fill each action hw_stats with DONT_CARE.
	 * Caller can override this if it wants stats for a given action.
	 */
	for (i = 0; i < num_actions; i++)
		rule->action.entries[i].hw_stats = FLOW_ACTION_HW_STATS_DONT_CARE;

	return rule;
}
EXPORT_SYMBOL(flow_rule_alloc);

#define FLOW_DISSECTOR_MATCH(__rule, __type, __out)				\
	const struct flow_match *__m = &(__rule)->match;			\
	struct flow_dissector *__d = (__m)->dissector;				\
										\
	(__out)->key = skb_flow_dissector_target(__d, __type, (__m)->key);	\
	(__out)->mask = skb_flow_dissector_target(__d, __type, (__m)->mask);	\

void flow_rule_match_meta(const struct flow_rule *rule,
			  struct flow_match_meta *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_META, out);
}
EXPORT_SYMBOL(flow_rule_match_meta);

void flow_rule_match_basic(const struct flow_rule *rule,
			   struct flow_match_basic *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_BASIC, out);
}
EXPORT_SYMBOL(flow_rule_match_basic);

void flow_rule_match_control(const struct flow_rule *rule,
			     struct flow_match_control *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CONTROL, out);
}
EXPORT_SYMBOL(flow_rule_match_control);

void flow_rule_match_eth_addrs(const struct flow_rule *rule,
			       struct flow_match_eth_addrs *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS, out);
}
EXPORT_SYMBOL(flow_rule_match_eth_addrs);

void flow_rule_match_vlan(const struct flow_rule *rule,
			  struct flow_match_vlan *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_VLAN, out);
}
EXPORT_SYMBOL(flow_rule_match_vlan);

void flow_rule_match_cvlan(const struct flow_rule *rule,
			   struct flow_match_vlan *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CVLAN, out);
}
EXPORT_SYMBOL(flow_rule_match_cvlan);

void flow_rule_match_ipv4_addrs(const struct flow_rule *rule,
				struct flow_match_ipv4_addrs *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS, out);
}
EXPORT_SYMBOL(flow_rule_match_ipv4_addrs);

void flow_rule_match_ipv6_addrs(const struct flow_rule *rule,
				struct flow_match_ipv6_addrs *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS, out);
}
EXPORT_SYMBOL(flow_rule_match_ipv6_addrs);

void flow_rule_match_ip(const struct flow_rule *rule,
			struct flow_match_ip *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IP, out);
}
EXPORT_SYMBOL(flow_rule_match_ip);

void flow_rule_match_ports(const struct flow_rule *rule,
			   struct flow_match_ports *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PORTS, out);
}
EXPORT_SYMBOL(flow_rule_match_ports);

void flow_rule_match_tcp(const struct flow_rule *rule,
			 struct flow_match_tcp *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_TCP, out);
}
EXPORT_SYMBOL(flow_rule_match_tcp);

void flow_rule_match_icmp(const struct flow_rule *rule,
			  struct flow_match_icmp *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ICMP, out);
}
EXPORT_SYMBOL(flow_rule_match_icmp);

void flow_rule_match_mpls(const struct flow_rule *rule,
			  struct flow_match_mpls *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_MPLS, out);
}
EXPORT_SYMBOL(flow_rule_match_mpls);

void flow_rule_match_enc_control(const struct flow_rule *rule,
				 struct flow_match_control *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_control);

void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule,
				    struct flow_match_ipv4_addrs *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_ipv4_addrs);

void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule,
				    struct flow_match_ipv6_addrs *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_ipv6_addrs);

void flow_rule_match_enc_ip(const struct flow_rule *rule,
			    struct flow_match_ip *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IP, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_ip);

void flow_rule_match_enc_ports(const struct flow_rule *rule,
			       struct flow_match_ports *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_PORTS, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_ports);

void flow_rule_match_enc_keyid(const struct flow_rule *rule,
			       struct flow_match_enc_keyid *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_KEYID, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_keyid);

void flow_rule_match_enc_opts(const struct flow_rule *rule,
			      struct flow_match_enc_opts *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_OPTS, out);
}
EXPORT_SYMBOL(flow_rule_match_enc_opts);

struct flow_action_cookie *flow_action_cookie_create(void *data,
						     unsigned int len,
						     gfp_t gfp)
{
	struct flow_action_cookie *cookie;

	cookie = kmalloc(sizeof(*cookie) + len, gfp);
	if (!cookie)
		return NULL;
	cookie->cookie_len = len;
	memcpy(cookie->cookie, data, len);
	return cookie;
}
EXPORT_SYMBOL(flow_action_cookie_create);

void flow_action_cookie_destroy(struct flow_action_cookie *cookie)
{
	kfree(cookie);
}
EXPORT_SYMBOL(flow_action_cookie_destroy);

void flow_rule_match_ct(const struct flow_rule *rule,
			struct flow_match_ct *out)
{
	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CT, out);
}
EXPORT_SYMBOL(flow_rule_match_ct);

struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb,
					  void *cb_ident, void *cb_priv,
					  void (*release)(void *cb_priv))
{
	struct flow_block_cb *block_cb;

	block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
	if (!block_cb)
		return ERR_PTR(-ENOMEM);

	block_cb->cb = cb;
	block_cb->cb_ident = cb_ident;
	block_cb->cb_priv = cb_priv;
	block_cb->release = release;

	return block_cb;
}
EXPORT_SYMBOL(flow_block_cb_alloc);

void flow_block_cb_free(struct flow_block_cb *block_cb)
{
	if (block_cb->release)
		block_cb->release(block_cb->cb_priv);

	kfree(block_cb);
}
EXPORT_SYMBOL(flow_block_cb_free);

struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block,
					   flow_setup_cb_t *cb, void *cb_ident)
{
	struct flow_block_cb *block_cb;

	list_for_each_entry(block_cb, &block->cb_list, list) {
		if (block_cb->cb == cb &&
		    block_cb->cb_ident == cb_ident)
			return block_cb;
	}

	return NULL;
}
EXPORT_SYMBOL(flow_block_cb_lookup);

void *flow_block_cb_priv(struct flow_block_cb *block_cb)
{
	return block_cb->cb_priv;
}
EXPORT_SYMBOL(flow_block_cb_priv);

void flow_block_cb_incref(struct flow_block_cb *block_cb)
{
	block_cb->refcnt++;
}
EXPORT_SYMBOL(flow_block_cb_incref);

unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb)
{
	return --block_cb->refcnt;
}
EXPORT_SYMBOL(flow_block_cb_decref);

bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident,
			   struct list_head *driver_block_list)
{
	struct flow_block_cb *block_cb;

	list_for_each_entry(block_cb, driver_block_list, driver_list) {
		if (block_cb->cb == cb &&
		    block_cb->cb_ident == cb_ident)
			return true;
	}

	return false;
}
EXPORT_SYMBOL(flow_block_cb_is_busy);

int flow_block_cb_setup_simple(struct flow_block_offload *f,
			       struct list_head *driver_block_list,
			       flow_setup_cb_t *cb,
			       void *cb_ident, void *cb_priv,
			       bool ingress_only)
{
	struct flow_block_cb *block_cb;

	if (ingress_only &&
	    f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
		return -EOPNOTSUPP;

	f->driver_block_list = driver_block_list;

	switch (f->command) {
	case FLOW_BLOCK_BIND:
		if (flow_block_cb_is_busy(cb, cb_ident, driver_block_list))
			return -EBUSY;

		block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, NULL);
		if (IS_ERR(block_cb))
			return PTR_ERR(block_cb);

		flow_block_cb_add(block_cb, f);
		list_add_tail(&block_cb->driver_list, driver_block_list);
		return 0;
	case FLOW_BLOCK_UNBIND:
		block_cb = flow_block_cb_lookup(f->block, cb, cb_ident);
		if (!block_cb)
			return -ENOENT;

		flow_block_cb_remove(block_cb, f);
		list_del(&block_cb->driver_list);
		return 0;
	default:
		return -EOPNOTSUPP;
	}
}
EXPORT_SYMBOL(flow_block_cb_setup_simple);

static DEFINE_MUTEX(flow_indr_block_lock);
static LIST_HEAD(flow_block_indr_list);
static LIST_HEAD(flow_block_indr_dev_list);

struct flow_indr_dev {
	struct list_head		list;
	flow_indr_block_bind_cb_t	*cb;
	void				*cb_priv;
	refcount_t			refcnt;
	struct rcu_head			rcu;
};

static struct flow_indr_dev *flow_indr_dev_alloc(flow_indr_block_bind_cb_t *cb,
						 void *cb_priv)
{
	struct flow_indr_dev *indr_dev;

	indr_dev = kmalloc(sizeof(*indr_dev), GFP_KERNEL);
	if (!indr_dev)
		return NULL;

	indr_dev->cb		= cb;
	indr_dev->cb_priv	= cb_priv;
	refcount_set(&indr_dev->refcnt, 1);

	return indr_dev;
}

int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv)
{
	struct flow_indr_dev *indr_dev;

	mutex_lock(&flow_indr_block_lock);
	list_for_each_entry(indr_dev, &flow_block_indr_dev_list, list) {
		if (indr_dev->cb == cb &&
		    indr_dev->cb_priv == cb_priv) {
			refcount_inc(&indr_dev->refcnt);
			mutex_unlock(&flow_indr_block_lock);
			return 0;
		}
	}

	indr_dev = flow_indr_dev_alloc(cb, cb_priv);
	if (!indr_dev) {
		mutex_unlock(&flow_indr_block_lock);
		return -ENOMEM;
	}

	list_add(&indr_dev->list, &flow_block_indr_dev_list);
	mutex_unlock(&flow_indr_block_lock);

	return 0;
}
EXPORT_SYMBOL(flow_indr_dev_register);

static void __flow_block_indr_cleanup(void (*release)(void *cb_priv),
				      void *cb_priv,
				      struct list_head *cleanup_list)
{
	struct flow_block_cb *this, *next;

	list_for_each_entry_safe(this, next, &flow_block_indr_list, indr.list) {
		if (this->release == release &&
		    this->indr.cb_priv == cb_priv)
			list_move(&this->indr.list, cleanup_list);
	}
}

static void flow_block_indr_notify(struct list_head *cleanup_list)
{
	struct flow_block_cb *this, *next;

	list_for_each_entry_safe(this, next, cleanup_list, indr.list) {
		list_del(&this->indr.list);
		this->indr.cleanup(this);
	}
}

void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv,
			      void (*release)(void *cb_priv))
{
	struct flow_indr_dev *this, *next, *indr_dev = NULL;
	LIST_HEAD(cleanup_list);

	mutex_lock(&flow_indr_block_lock);
	list_for_each_entry_safe(this, next, &flow_block_indr_dev_list, list) {
		if (this->cb == cb &&
		    this->cb_priv == cb_priv &&
		    refcount_dec_and_test(&this->refcnt)) {
			indr_dev = this;
			list_del(&indr_dev->list);
			break;
		}
	}

	if (!indr_dev) {
		mutex_unlock(&flow_indr_block_lock);
		return;
	}

	__flow_block_indr_cleanup(release, cb_priv, &cleanup_list);
	mutex_unlock(&flow_indr_block_lock);

	flow_block_indr_notify(&cleanup_list);
	kfree(indr_dev);
}
EXPORT_SYMBOL(flow_indr_dev_unregister);

static void flow_block_indr_init(struct flow_block_cb *flow_block,
				 struct flow_block_offload *bo,
				 struct net_device *dev, struct Qdisc *sch, void *data,
				 void *cb_priv,
				 void (*cleanup)(struct flow_block_cb *block_cb))
{
	flow_block->indr.binder_type = bo->binder_type;
	flow_block->indr.data = data;
	flow_block->indr.cb_priv = cb_priv;
	flow_block->indr.dev = dev;
	flow_block->indr.sch = sch;
	flow_block->indr.cleanup = cleanup;
}

struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb,
					       void *cb_ident, void *cb_priv,
					       void (*release)(void *cb_priv),
					       struct flow_block_offload *bo,
					       struct net_device *dev,
					       struct Qdisc *sch, void *data,
					       void *indr_cb_priv,
					       void (*cleanup)(struct flow_block_cb *block_cb))
{
	struct flow_block_cb *block_cb;

	block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, release);
	if (IS_ERR(block_cb))
		goto out;

	flow_block_indr_init(block_cb, bo, dev, sch, data, indr_cb_priv, cleanup);
	list_add(&block_cb->indr.list, &flow_block_indr_list);

out:
	return block_cb;
}
EXPORT_SYMBOL(flow_indr_block_cb_alloc);

int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch,
				enum tc_setup_type type, void *data,
				struct flow_block_offload *bo,
				void (*cleanup)(struct flow_block_cb *block_cb))
{
	struct flow_indr_dev *this;

	mutex_lock(&flow_indr_block_lock);
	list_for_each_entry(this, &flow_block_indr_dev_list, list)
		this->cb(dev, sch, this->cb_priv, type, bo, data, cleanup);

	mutex_unlock(&flow_indr_block_lock);

	return list_empty(&bo->cb_list) ? -EOPNOTSUPP : 0;
}
EXPORT_SYMBOL(flow_indr_dev_setup_offload);
Commit	Line	Data
8f256622 PNA	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#include <linux/kernel.h>
	3	#include <linux/slab.h>
	4	#include <net/flow_offload.h>
4e481908	5	#include <linux/rtnetlink.h>
1150ab0f	6	#include <linux/mutex.h>
c2b69f24	7	#include <linux/rhashtable.h>
8f256622	8
e3ab786b	9	struct flow_rule *flow_rule_alloc(unsigned int num_actions)
8f256622	10	{
e3ab786b	11	struct flow_rule *rule;
060b6381	12	int i;
e3ab786b	13
6dca9360	14	rule = kzalloc(struct_size(rule, action.entries, num_actions),
e3ab786b PNA	15	GFP_KERNEL);
	16	if (!rule)
	17	return NULL;
	18
	19	rule->action.num_entries = num_actions;
060b6381 EC	20	/* Pre-fill each action hw_stats with DONT_CARE.
	21	* Caller can override this if it wants stats for a given action.
	22	*/
	23	for (i = 0; i < num_actions; i++)
	24	rule->action.entries[i].hw_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
e3ab786b PNA	25
e3ab786b PNA	26	return rule;
8f256622 PNA	27	}
	28	EXPORT_SYMBOL(flow_rule_alloc);
	29
	30	#define FLOW_DISSECTOR_MATCH(__rule, __type, __out) \
	31	const struct flow_match *__m = &(__rule)->match; \
	32	struct flow_dissector *__d = (__m)->dissector; \
	33	\
	34	(__out)->key = skb_flow_dissector_target(__d, __type, (__m)->key); \
	35	(__out)->mask = skb_flow_dissector_target(__d, __type, (__m)->mask); \
	36
9558a83a JP	37	void flow_rule_match_meta(const struct flow_rule *rule,
	38	struct flow_match_meta *out)
	39	{
	40	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_META, out);
	41	}
	42	EXPORT_SYMBOL(flow_rule_match_meta);
	43
8f256622 PNA	44	void flow_rule_match_basic(const struct flow_rule *rule,
	45	struct flow_match_basic *out)
	46	{
	47	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_BASIC, out);
	48	}
	49	EXPORT_SYMBOL(flow_rule_match_basic);
	50
	51	void flow_rule_match_control(const struct flow_rule *rule,
	52	struct flow_match_control *out)
	53	{
	54	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CONTROL, out);
	55	}
	56	EXPORT_SYMBOL(flow_rule_match_control);
	57
	58	void flow_rule_match_eth_addrs(const struct flow_rule *rule,
	59	struct flow_match_eth_addrs *out)
	60	{
	61	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS, out);
	62	}
	63	EXPORT_SYMBOL(flow_rule_match_eth_addrs);
	64
	65	void flow_rule_match_vlan(const struct flow_rule *rule,
	66	struct flow_match_vlan *out)
	67	{
	68	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_VLAN, out);
	69	}
	70	EXPORT_SYMBOL(flow_rule_match_vlan);
	71
bae9ed69 EC	72	void flow_rule_match_cvlan(const struct flow_rule *rule,
	73	struct flow_match_vlan *out)
	74	{
	75	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CVLAN, out);
	76	}
	77	EXPORT_SYMBOL(flow_rule_match_cvlan);
	78
8f256622 PNA	79	void flow_rule_match_ipv4_addrs(const struct flow_rule *rule,
	80	struct flow_match_ipv4_addrs *out)
	81	{
	82	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS, out);
	83	}
	84	EXPORT_SYMBOL(flow_rule_match_ipv4_addrs);
	85
	86	void flow_rule_match_ipv6_addrs(const struct flow_rule *rule,
	87	struct flow_match_ipv6_addrs *out)
	88	{
	89	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS, out);
	90	}
	91	EXPORT_SYMBOL(flow_rule_match_ipv6_addrs);
	92
	93	void flow_rule_match_ip(const struct flow_rule *rule,
	94	struct flow_match_ip *out)
	95	{
	96	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_IP, out);
	97	}
	98	EXPORT_SYMBOL(flow_rule_match_ip);
	99
	100	void flow_rule_match_ports(const struct flow_rule *rule,
	101	struct flow_match_ports *out)
	102	{
	103	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PORTS, out);
	104	}
	105	EXPORT_SYMBOL(flow_rule_match_ports);
	106
	107	void flow_rule_match_tcp(const struct flow_rule *rule,
	108	struct flow_match_tcp *out)
	109	{
	110	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_TCP, out);
	111	}
	112	EXPORT_SYMBOL(flow_rule_match_tcp);
	113
	114	void flow_rule_match_icmp(const struct flow_rule *rule,
	115	struct flow_match_icmp *out)
	116	{
	117	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ICMP, out);
	118	}
	119	EXPORT_SYMBOL(flow_rule_match_icmp);
	120
	121	void flow_rule_match_mpls(const struct flow_rule *rule,
	122	struct flow_match_mpls *out)
	123	{
	124	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_MPLS, out);
	125	}
	126	EXPORT_SYMBOL(flow_rule_match_mpls);
	127
	128	void flow_rule_match_enc_control(const struct flow_rule *rule,
	129	struct flow_match_control *out)
	130	{
	131	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL, out);
	132	}
	133	EXPORT_SYMBOL(flow_rule_match_enc_control);
	134
	135	void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule,
	136	struct flow_match_ipv4_addrs *out)
	137	{
	138	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, out);
	139	}
	140	EXPORT_SYMBOL(flow_rule_match_enc_ipv4_addrs);
	141
	142	void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule,
143	struct flow_match_ipv6_addrs *out)
144	{
145	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, out);
146	}
147	EXPORT_SYMBOL(flow_rule_match_enc_ipv6_addrs);
148
149	void flow_rule_match_enc_ip(const struct flow_rule *rule,
150	struct flow_match_ip *out)
151	{
152	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_IP, out);
153	}
154	EXPORT_SYMBOL(flow_rule_match_enc_ip);
155
156	void flow_rule_match_enc_ports(const struct flow_rule *rule,
157	struct flow_match_ports *out)
158	{
159	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_PORTS, out);
160	}
161	EXPORT_SYMBOL(flow_rule_match_enc_ports);
162
163	void flow_rule_match_enc_keyid(const struct flow_rule *rule,
164	struct flow_match_enc_keyid *out)
165	{
166	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_KEYID, out);
167	}
168	EXPORT_SYMBOL(flow_rule_match_enc_keyid);
169
170	void flow_rule_match_enc_opts(const struct flow_rule *rule,
171	struct flow_match_enc_opts *out)
172	{
173	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_ENC_OPTS, out);
174	}
175	EXPORT_SYMBOL(flow_rule_match_enc_opts);
4e95bc26	176
2008495d JP	177	struct flow_action_cookie flow_action_cookie_create(void data,
	178	unsigned int len,
	179	gfp_t gfp)
	180	{
	181	struct flow_action_cookie *cookie;
	182
	183	cookie = kmalloc(sizeof(*cookie) + len, gfp);
	184	if (!cookie)
	185	return NULL;
	186	cookie->cookie_len = len;
	187	memcpy(cookie->cookie, data, len);
	188	return cookie;
	189	}
	190	EXPORT_SYMBOL(flow_action_cookie_create);
	191
	192	void flow_action_cookie_destroy(struct flow_action_cookie *cookie)
	193	{
	194	kfree(cookie);
	195	}
	196	EXPORT_SYMBOL(flow_action_cookie_destroy);
	197
ee1c45e8 PB	198	void flow_rule_match_ct(const struct flow_rule *rule,
	199	struct flow_match_ct *out)
	200	{
	201	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_CT, out);
	202	}
	203	EXPORT_SYMBOL(flow_rule_match_ct);
	204
a7323311	205	struct flow_block_cb flow_block_cb_alloc(flow_setup_cb_t cb,
d63db30c PNA	206	void cb_ident, void cb_priv,
	207	void (release)(void cb_priv))
	208	{
	209	struct flow_block_cb *block_cb;
	210
	211	block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
	212	if (!block_cb)
	213	return ERR_PTR(-ENOMEM);
	214
	215	block_cb->cb = cb;
	216	block_cb->cb_ident = cb_ident;
	217	block_cb->cb_priv = cb_priv;
	218	block_cb->release = release;
	219
	220	return block_cb;
	221	}
	222	EXPORT_SYMBOL(flow_block_cb_alloc);
	223
	224	void flow_block_cb_free(struct flow_block_cb *block_cb)
	225	{
	226	if (block_cb->release)
	227	block_cb->release(block_cb->cb_priv);
	228
	229	kfree(block_cb);
	230	}
	231	EXPORT_SYMBOL(flow_block_cb_free);
	232
14bfb13f	233	struct flow_block_cb flow_block_cb_lookup(struct flow_block block,
a7323311	234	flow_setup_cb_t cb, void cb_ident)
da3eeb90 PNA	235	{
	236	struct flow_block_cb *block_cb;
	237
14bfb13f	238	list_for_each_entry(block_cb, &block->cb_list, list) {
0c7294dd	239	if (block_cb->cb == cb &&
da3eeb90 PNA	240	block_cb->cb_ident == cb_ident)
	241	return block_cb;
	242	}
	243
	244	return NULL;
	245	}
	246	EXPORT_SYMBOL(flow_block_cb_lookup);
	247
67bd0d5e PNA	248	void flow_block_cb_priv(struct flow_block_cb block_cb)
	249	{
	250	return block_cb->cb_priv;
	251	}
	252	EXPORT_SYMBOL(flow_block_cb_priv);
	253
	254	void flow_block_cb_incref(struct flow_block_cb *block_cb)
	255	{
	256	block_cb->refcnt++;
	257	}
	258	EXPORT_SYMBOL(flow_block_cb_incref);
	259
	260	unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb)
	261	{
	262	return --block_cb->refcnt;
	263	}
	264	EXPORT_SYMBOL(flow_block_cb_decref);
	265
a7323311	266	bool flow_block_cb_is_busy(flow_setup_cb_t cb, void cb_ident,
0d4fd02e PNA	267	struct list_head *driver_block_list)
	268	{
	269	struct flow_block_cb *block_cb;
	270
	271	list_for_each_entry(block_cb, driver_block_list, driver_list) {
	272	if (block_cb->cb == cb &&
	273	block_cb->cb_ident == cb_ident)
	274	return true;
	275	}
	276
	277	return false;
	278	}
	279	EXPORT_SYMBOL(flow_block_cb_is_busy);
	280
4e95bc26 PNA	281	int flow_block_cb_setup_simple(struct flow_block_offload *f,
4e95bc26 PNA	282	struct list_head *driver_block_list,
a7323311 PNA	283	flow_setup_cb_t *cb,
a7323311 PNA	284	void cb_ident, void cb_priv,
4e95bc26 PNA	285	bool ingress_only)
4e95bc26 PNA	286	{
955bcb6e PNA	287	struct flow_block_cb *block_cb;
955bcb6e PNA	288
4e95bc26	289	if (ingress_only &&
32f8c409	290	f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
4e95bc26 PNA	291	return -EOPNOTSUPP;
	292
	293	f->driver_block_list = driver_block_list;
	294
	295	switch (f->command) {
9c0e189e	296	case FLOW_BLOCK_BIND:
0d4fd02e PNA	297	if (flow_block_cb_is_busy(cb, cb_ident, driver_block_list))
	298	return -EBUSY;
	299
0c7294dd	300	block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, NULL);
955bcb6e PNA	301	if (IS_ERR(block_cb))
	302	return PTR_ERR(block_cb);
	303
	304	flow_block_cb_add(block_cb, f);
	305	list_add_tail(&block_cb->driver_list, driver_block_list);
	306	return 0;
9c0e189e	307	case FLOW_BLOCK_UNBIND:
14bfb13f	308	block_cb = flow_block_cb_lookup(f->block, cb, cb_ident);
955bcb6e PNA	309	if (!block_cb)
	310	return -ENOENT;
	311
	312	flow_block_cb_remove(block_cb, f);
	313	list_del(&block_cb->driver_list);
4e95bc26 PNA	314	return 0;
	315	default:
	316	return -EOPNOTSUPP;
	317	}
	318	}
	319	EXPORT_SYMBOL(flow_block_cb_setup_simple);
4e481908	320
1fac52da PNA	321	static DEFINE_MUTEX(flow_indr_block_lock);
	322	static LIST_HEAD(flow_block_indr_list);
	323	static LIST_HEAD(flow_block_indr_dev_list);
	324
	325	struct flow_indr_dev {
	326	struct list_head list;
	327	flow_indr_block_bind_cb_t *cb;
	328	void *cb_priv;
	329	refcount_t refcnt;
	330	struct rcu_head rcu;
	331	};
	332
	333	static struct flow_indr_dev flow_indr_dev_alloc(flow_indr_block_bind_cb_t cb,
	334	void *cb_priv)
	335	{
	336	struct flow_indr_dev *indr_dev;
	337
	338	indr_dev = kmalloc(sizeof(*indr_dev), GFP_KERNEL);
	339	if (!indr_dev)
	340	return NULL;
	341
	342	indr_dev->cb = cb;
	343	indr_dev->cb_priv = cb_priv;
	344	refcount_set(&indr_dev->refcnt, 1);
	345
	346	return indr_dev;
	347	}
	348
	349	int flow_indr_dev_register(flow_indr_block_bind_cb_t cb, void cb_priv)
	350	{
	351	struct flow_indr_dev *indr_dev;
	352
	353	mutex_lock(&flow_indr_block_lock);
	354	list_for_each_entry(indr_dev, &flow_block_indr_dev_list, list) {
	355	if (indr_dev->cb == cb &&
	356	indr_dev->cb_priv == cb_priv) {
	357	refcount_inc(&indr_dev->refcnt);
	358	mutex_unlock(&flow_indr_block_lock);
	359	return 0;
	360	}
	361	}
	362
	363	indr_dev = flow_indr_dev_alloc(cb, cb_priv);
	364	if (!indr_dev) {
	365	mutex_unlock(&flow_indr_block_lock);
	366	return -ENOMEM;
	367	}
	368
	369	list_add(&indr_dev->list, &flow_block_indr_dev_list);
	370	mutex_unlock(&flow_indr_block_lock);
	371
	372	return 0;
	373	}
	374	EXPORT_SYMBOL(flow_indr_dev_register);
	375
a1db2178	376	static void __flow_block_indr_cleanup(void (release)(void cb_priv),
a1db2178	377	void *cb_priv,
1fac52da PNA	378	struct list_head *cleanup_list)
	379	{
	380	struct flow_block_cb this, next;
	381
	382	list_for_each_entry_safe(this, next, &flow_block_indr_list, indr.list) {
a1db2178	383	if (this->release == release &&
5137d303	384	this->indr.cb_priv == cb_priv)
1fac52da	385	list_move(&this->indr.list, cleanup_list);
1fac52da PNA	386	}
	387	}
	388
	389	static void flow_block_indr_notify(struct list_head *cleanup_list)
	390	{
	391	struct flow_block_cb this, next;
	392
	393	list_for_each_entry_safe(this, next, cleanup_list, indr.list) {
	394	list_del(&this->indr.list);
	395	this->indr.cleanup(this);
	396	}
	397	}
	398
	399	void flow_indr_dev_unregister(flow_indr_block_bind_cb_t cb, void cb_priv,
a1db2178	400	void (release)(void cb_priv))
1fac52da PNA	401	{
	402	struct flow_indr_dev this, next, *indr_dev = NULL;
	403	LIST_HEAD(cleanup_list);
	404
	405	mutex_lock(&flow_indr_block_lock);
	406	list_for_each_entry_safe(this, next, &flow_block_indr_dev_list, list) {
	407	if (this->cb == cb &&
	408	this->cb_priv == cb_priv &&
	409	refcount_dec_and_test(&this->refcnt)) {
	410	indr_dev = this;
	411	list_del(&indr_dev->list);
	412	break;
	413	}
	414	}
	415
	416	if (!indr_dev) {
	417	mutex_unlock(&flow_indr_block_lock);
	418	return;
	419	}
	420
a1db2178	421	__flow_block_indr_cleanup(release, cb_priv, &cleanup_list);
1fac52da PNA	422	mutex_unlock(&flow_indr_block_lock);
	423
	424	flow_block_indr_notify(&cleanup_list);
	425	kfree(indr_dev);
	426	}
	427	EXPORT_SYMBOL(flow_indr_dev_unregister);
	428
	429	static void flow_block_indr_init(struct flow_block_cb *flow_block,
	430	struct flow_block_offload *bo,
c40f4e50	431	struct net_device dev, struct Qdisc sch, void *data,
a1db2178	432	void *cb_priv,
1fac52da PNA	433	void (cleanup)(struct flow_block_cb block_cb))
	434	{
	435	flow_block->indr.binder_type = bo->binder_type;
	436	flow_block->indr.data = data;
a1db2178	437	flow_block->indr.cb_priv = cb_priv;
1fac52da	438	flow_block->indr.dev = dev;
c40f4e50	439	flow_block->indr.sch = sch;
1fac52da PNA	440	flow_block->indr.cleanup = cleanup;
	441	}
	442
26f2eb27	443	struct flow_block_cb flow_indr_block_cb_alloc(flow_setup_cb_t cb,
	444	void cb_ident, void cb_priv,
	445	void (release)(void cb_priv),
	446	struct flow_block_offload *bo,
c40f4e50 PM	447	struct net_device *dev,
c40f4e50 PM	448	struct Qdisc sch, void data,
a1db2178	449	void *indr_cb_priv,
26f2eb27	450	void (cleanup)(struct flow_block_cb block_cb))
	451	{
	452	struct flow_block_cb *block_cb;
	453
	454	block_cb = flow_block_cb_alloc(cb, cb_ident, cb_priv, release);
	455	if (IS_ERR(block_cb))
	456	goto out;
	457
c40f4e50	458	flow_block_indr_init(block_cb, bo, dev, sch, data, indr_cb_priv, cleanup);
26f2eb27	459	list_add(&block_cb->indr.list, &flow_block_indr_list);
	460
	461	out:
	462	return block_cb;
	463	}
	464	EXPORT_SYMBOL(flow_indr_block_cb_alloc);
	465
c40f4e50	466	int flow_indr_dev_setup_offload(struct net_device dev, struct Qdisc sch,
1fac52da PNA	467	enum tc_setup_type type, void *data,
	468	struct flow_block_offload *bo,
	469	void (cleanup)(struct flow_block_cb block_cb))
	470	{
	471	struct flow_indr_dev *this;
	472
	473	mutex_lock(&flow_indr_block_lock);
	474	list_for_each_entry(this, &flow_block_indr_dev_list, list)
c40f4e50	475	this->cb(dev, sch, this->cb_priv, type, bo, data, cleanup);
1fac52da	476
1fac52da PNA	477	mutex_unlock(&flow_indr_block_lock);
	478
	479	return list_empty(&bo->cb_list) ? -EOPNOTSUPP : 0;
	480	}
	481	EXPORT_SYMBOL(flow_indr_dev_setup_offload);