[mirror_ubuntu-hirsute-kernel.git] / net / netfilter / nf_tables_offload.c

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netfilter.h>
#include <net/flow_offload.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>
#include <net/pkt_cls.h>

static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
{
	struct nft_flow_rule *flow;

	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	if (!flow)
		return NULL;

	flow->rule = flow_rule_alloc(num_actions);
	if (!flow->rule) {
		kfree(flow);
		return NULL;
	}

	flow->rule->match.dissector	= &flow->match.dissector;
	flow->rule->match.mask		= &flow->match.mask;
	flow->rule->match.key		= &flow->match.key;

	return flow;
}

void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
				 enum flow_dissector_key_id addr_type)
{
	struct nft_flow_match *match = &flow->match;
	struct nft_flow_key *mask = &match->mask;
	struct nft_flow_key *key = &match->key;

	if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
		return;

	key->control.addr_type = addr_type;
	mask->control.addr_type = 0xffff;
	match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL);
	match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
		offsetof(struct nft_flow_key, control);
}

struct nft_flow_rule *nft_flow_rule_create(struct net *net,
					   const struct nft_rule *rule)
{
	struct nft_offload_ctx *ctx;
	struct nft_flow_rule *flow;
	int num_actions = 0, err;
	struct nft_expr *expr;

	expr = nft_expr_first(rule);
	while (nft_expr_more(rule, expr)) {
		if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
			num_actions++;

		expr = nft_expr_next(expr);
	}

	if (num_actions == 0)
		return ERR_PTR(-EOPNOTSUPP);

	flow = nft_flow_rule_alloc(num_actions);
	if (!flow)
		return ERR_PTR(-ENOMEM);

	expr = nft_expr_first(rule);

	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	if (!ctx) {
		err = -ENOMEM;
		goto err_out;
	}
	ctx->net = net;
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;

	while (nft_expr_more(rule, expr)) {
		if (!expr->ops->offload) {
			err = -EOPNOTSUPP;
			goto err_out;
		}
		err = expr->ops->offload(ctx, flow, expr);
		if (err < 0)
			goto err_out;

		expr = nft_expr_next(expr);
	}
	flow->proto = ctx->dep.l3num;
	kfree(ctx);

	return flow;
err_out:
	kfree(ctx);
	nft_flow_rule_destroy(flow);

	return ERR_PTR(err);
}

void nft_flow_rule_destroy(struct nft_flow_rule *flow)
{
	struct flow_action_entry *entry;
	int i;

	flow_action_for_each(i, entry, &flow->rule->action) {
		switch (entry->id) {
		case FLOW_ACTION_REDIRECT:
		case FLOW_ACTION_MIRRED:
			dev_put(entry->dev);
			break;
		default:
			break;
		}
	}
	kfree(flow->rule);
	kfree(flow);
}

void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
				enum nft_offload_dep_type type)
{
	ctx->dep.type = type;
}

void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
				   const void *data, u32 len)
{
	switch (ctx->dep.type) {
	case NFT_OFFLOAD_DEP_NETWORK:
		WARN_ON(len != sizeof(__u16));
		memcpy(&ctx->dep.l3num, data, sizeof(__u16));
		break;
	case NFT_OFFLOAD_DEP_TRANSPORT:
		WARN_ON(len != sizeof(__u8));
		memcpy(&ctx->dep.protonum, data, sizeof(__u8));
		break;
	default:
		break;
	}
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
}

static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
					 __be16 proto, int priority,
					 struct netlink_ext_ack *extack)
{
	common->protocol = proto;
	common->prio = priority;
	common->extack = extack;
}

static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
			     struct list_head *cb_list)
{
	struct flow_block_cb *block_cb;
	int err;

	list_for_each_entry(block_cb, cb_list, list) {
		err = block_cb->cb(type, type_data, block_cb->cb_priv);
		if (err < 0)
			return err;
	}
	return 0;
}

int nft_chain_offload_priority(struct nft_base_chain *basechain)
{
	if (basechain->ops.priority <= 0 ||
	    basechain->ops.priority > USHRT_MAX)
		return -1;

	return 0;
}

static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
				       const struct nft_base_chain *basechain,
				       const struct nft_rule *rule,
				       const struct nft_flow_rule *flow,
				       struct netlink_ext_ack *extack,
				       enum flow_cls_command command)
{
	__be16 proto = ETH_P_ALL;

	memset(cls_flow, 0, sizeof(*cls_flow));

	if (flow)
		proto = flow->proto;

	nft_flow_offload_common_init(&cls_flow->common, proto,
				     basechain->ops.priority, extack);
	cls_flow->command = command;
	cls_flow->cookie = (unsigned long) rule;
	if (flow)
		cls_flow->rule = flow->rule;
}

static int nft_flow_offload_rule(struct nft_chain *chain,
				 struct nft_rule *rule,
				 struct nft_flow_rule *flow,
				 enum flow_cls_command command)
{
	struct netlink_ext_ack extack = {};
	struct flow_cls_offload cls_flow;
	struct nft_base_chain *basechain;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
				   command);

	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
				 &basechain->flow_block.cb_list);
}

static int nft_flow_offload_bind(struct flow_block_offload *bo,
				 struct nft_base_chain *basechain)
{
	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
	return 0;
}

static int nft_flow_offload_unbind(struct flow_block_offload *bo,
				   struct nft_base_chain *basechain)
{
	struct flow_block_cb *block_cb, *next;
	struct flow_cls_offload cls_flow;
	struct netlink_ext_ack extack;
	struct nft_chain *chain;
	struct nft_rule *rule;

	chain = &basechain->chain;
	list_for_each_entry(rule, &chain->rules, list) {
		memset(&extack, 0, sizeof(extack));
		nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
					   &extack, FLOW_CLS_DESTROY);
		nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
	}

	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
		list_del(&block_cb->list);
		flow_block_cb_free(block_cb);
	}

	return 0;
}

static int nft_block_setup(struct nft_base_chain *basechain,
			   struct flow_block_offload *bo,
			   enum flow_block_command cmd)
{
	int err;

	switch (cmd) {
	case FLOW_BLOCK_BIND:
		err = nft_flow_offload_bind(bo, basechain);
		break;
	case FLOW_BLOCK_UNBIND:
		err = nft_flow_offload_unbind(bo, basechain);
		break;
	default:
		WARN_ON_ONCE(1);
		err = -EOPNOTSUPP;
	}

	return err;
}

static void nft_flow_block_offload_init(struct flow_block_offload *bo,
					struct net *net,
					enum flow_block_command cmd,
					struct nft_base_chain *basechain,
					struct netlink_ext_ack *extack)
{
	memset(bo, 0, sizeof(*bo));
	bo->net		= net;
	bo->block	= &basechain->flow_block;
	bo->command	= cmd;
	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	bo->extack	= extack;
	INIT_LIST_HEAD(&bo->cb_list);
}

static int nft_block_offload_cmd(struct nft_base_chain *chain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	if (err < 0)
		return err;

	return nft_block_setup(chain, &bo, cmd);
}

static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
{
	struct nft_base_chain *basechain = block_cb->indr.data;
	struct net_device *dev = block_cb->indr.dev;
	struct netlink_ext_ack extack = {};
	struct net *net = dev_net(dev);
	struct flow_block_offload bo;

	nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
				    basechain, &extack);
	mutex_lock(&net->nft.commit_mutex);
	list_del(&block_cb->driver_list);
	list_move(&block_cb->list, &bo.cb_list);
	nft_flow_offload_unbind(&bo, basechain);
	mutex_unlock(&net->nft.commit_mutex);
}

static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
				      struct net_device *dev,
				      enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);

	err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
					  nft_indr_block_cleanup);
	if (err < 0)
		return err;

	if (list_empty(&bo.cb_list))
		return -EOPNOTSUPP;

	return nft_block_setup(basechain, &bo, cmd);
}

static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	int err;

	if (dev->netdev_ops->ndo_setup_tc)
		err = nft_block_offload_cmd(basechain, dev, cmd);
	else
		err = nft_indr_block_offload_cmd(basechain, dev, cmd);

	return err;
}

static int nft_flow_block_chain(struct nft_base_chain *basechain,
				const struct net_device *this_dev,
				enum flow_block_command cmd)
{
	struct net_device *dev;
	struct nft_hook *hook;
	int err, i = 0;

	list_for_each_entry(hook, &basechain->hook_list, list) {
		dev = hook->ops.dev;
		if (this_dev && this_dev != dev)
			continue;

		err = nft_chain_offload_cmd(basechain, dev, cmd);
		if (err < 0 && cmd == FLOW_BLOCK_BIND) {
			if (!this_dev)
				goto err_flow_block;

			return err;
		}
		i++;
	}

	return 0;

err_flow_block:
	list_for_each_entry(hook, &basechain->hook_list, list) {
		if (i-- <= 0)
			break;

		dev = hook->ops.dev;
		nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	}
	return err;
}

static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
				  enum flow_block_command cmd)
{
	struct nft_base_chain *basechain;
	u8 policy;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	policy = ppolicy ? *ppolicy : basechain->policy;

	/* Only default policy to accept is supported for now. */
	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
		return -EOPNOTSUPP;

	return nft_flow_block_chain(basechain, NULL, cmd);
}

static void nft_flow_rule_offload_abort(struct net *net,
					struct nft_trans *trans)
{
	int err = 0;

	list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		}

		if (WARN_ON_ONCE(err))
			break;
	}
}

int nft_flow_rule_offload_commit(struct net *net)
{
	struct nft_trans *trans;
	int err = 0;
	u8 policy;

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			if (trans->ctx.flags & NLM_F_REPLACE ||
			    !(trans->ctx.flags & NLM_F_APPEND)) {
				err = -EOPNOTSUPP;
				break;
			}
			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		}

		if (err) {
			nft_flow_rule_offload_abort(net, trans);
			break;
		}
	}

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWRULE:
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			nft_flow_rule_destroy(nft_trans_flow_rule(trans));
			break;
		default:
			break;
		}
	}

	return err;
}

static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
{
	struct nft_base_chain *basechain;
	struct net *net = dev_net(dev);
	struct nft_hook *hook, *found;
	const struct nft_table *table;
	struct nft_chain *chain;

	list_for_each_entry(table, &net->nft.tables, list) {
		if (table->family != NFPROTO_NETDEV)
			continue;

		list_for_each_entry(chain, &table->chains, list) {
			if (!nft_is_base_chain(chain) ||
			    !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			found = NULL;
			basechain = nft_base_chain(chain);
			list_for_each_entry(hook, &basechain->hook_list, list) {
				if (hook->ops.dev != dev)
					continue;

				found = hook;
				break;
			}
			if (!found)
				continue;

			return chain;
		}
	}

	return NULL;
}

static int nft_offload_netdev_event(struct notifier_block *this,
				    unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	if (event != NETDEV_UNREGISTER)
		return NOTIFY_DONE;

	mutex_lock(&net->nft.commit_mutex);
	chain = __nft_offload_get_chain(dev);
	if (chain)
		nft_flow_block_chain(nft_base_chain(chain), dev,
				     FLOW_BLOCK_UNBIND);

	mutex_unlock(&net->nft.commit_mutex);

	return NOTIFY_DONE;
}

static struct notifier_block nft_offload_netdev_notifier = {
	.notifier_call	= nft_offload_netdev_event,
};

int nft_offload_init(void)
{
	return register_netdevice_notifier(&nft_offload_netdev_notifier);
}

void nft_offload_exit(void)
{
	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
}
Commit	Line	Data
c9626a2c PNA	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#include <linux/init.h>
	3	#include <linux/module.h>
	4	#include <linux/netfilter.h>
	5	#include <net/flow_offload.h>
	6	#include <net/netfilter/nf_tables.h>
	7	#include <net/netfilter/nf_tables_offload.h>
	8	#include <net/pkt_cls.h>
	9
	10	static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
	11	{
	12	struct nft_flow_rule *flow;
	13
	14	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	15	if (!flow)
	16	return NULL;
	17
	18	flow->rule = flow_rule_alloc(num_actions);
	19	if (!flow->rule) {
	20	kfree(flow);
	21	return NULL;
	22	}
	23
	24	flow->rule->match.dissector = &flow->match.dissector;
	25	flow->rule->match.mask = &flow->match.mask;
	26	flow->rule->match.key = &flow->match.key;
	27
	28	return flow;
	29	}
	30
3c78e9e0 PNA	31	void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
	32	enum flow_dissector_key_id addr_type)
	33	{
	34	struct nft_flow_match *match = &flow->match;
	35	struct nft_flow_key *mask = &match->mask;
	36	struct nft_flow_key *key = &match->key;
	37
	38	if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
	39	return;
	40
	41	key->control.addr_type = addr_type;
	42	mask->control.addr_type = 0xffff;
	43	match->dissector.used_keys \|= BIT(FLOW_DISSECTOR_KEY_CONTROL);
	44	match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
	45	offsetof(struct nft_flow_key, control);
	46	}
	47
be2861dc PNA	48	struct nft_flow_rule nft_flow_rule_create(struct net net,
be2861dc PNA	49	const struct nft_rule *rule)
c9626a2c	50	{
b44492af	51	struct nft_offload_ctx *ctx;
c9626a2c PNA	52	struct nft_flow_rule *flow;
	53	int num_actions = 0, err;
	54	struct nft_expr *expr;
	55
	56	expr = nft_expr_first(rule);
31cc578a	57	while (nft_expr_more(rule, expr)) {
c9626a2c PNA	58	if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
	59	num_actions++;
	60
	61	expr = nft_expr_next(expr);
	62	}
	63
81ec6107 PNA	64	if (num_actions == 0)
	65	return ERR_PTR(-EOPNOTSUPP);
	66
c9626a2c PNA	67	flow = nft_flow_rule_alloc(num_actions);
	68	if (!flow)
	69	return ERR_PTR(-ENOMEM);
	70
	71	expr = nft_expr_first(rule);
b44492af AB	72
	73	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	74	if (!ctx) {
	75	err = -ENOMEM;
	76	goto err_out;
	77	}
be2861dc	78	ctx->net = net;
b44492af AB	79	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
b44492af AB	80
31cc578a	81	while (nft_expr_more(rule, expr)) {
c9626a2c PNA	82	if (!expr->ops->offload) {
	83	err = -EOPNOTSUPP;
	84	goto err_out;
	85	}
b44492af	86	err = expr->ops->offload(ctx, flow, expr);
c9626a2c PNA	87	if (err < 0)
	88	goto err_out;
	89
	90	expr = nft_expr_next(expr);
	91	}
b44492af AB	92	flow->proto = ctx->dep.l3num;
b44492af AB	93	kfree(ctx);
c9626a2c PNA	94
	95	return flow;
	96	err_out:
b44492af	97	kfree(ctx);
c9626a2c PNA	98	nft_flow_rule_destroy(flow);
	99
	100	return ERR_PTR(err);
	101	}
	102
	103	void nft_flow_rule_destroy(struct nft_flow_rule *flow)
	104	{
be2861dc PNA	105	struct flow_action_entry *entry;
	106	int i;
	107
	108	flow_action_for_each(i, entry, &flow->rule->action) {
	109	switch (entry->id) {
	110	case FLOW_ACTION_REDIRECT:
	111	case FLOW_ACTION_MIRRED:
	112	dev_put(entry->dev);
	113	break;
	114	default:
	115	break;
	116	}
	117	}
c9626a2c PNA	118	kfree(flow->rule);
	119	kfree(flow);
	120	}
	121
	122	void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
	123	enum nft_offload_dep_type type)
	124	{
	125	ctx->dep.type = type;
	126	}
	127
	128	void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
	129	const void *data, u32 len)
	130	{
	131	switch (ctx->dep.type) {
	132	case NFT_OFFLOAD_DEP_NETWORK:
	133	WARN_ON(len != sizeof(__u16));
	134	memcpy(&ctx->dep.l3num, data, sizeof(__u16));
	135	break;
	136	case NFT_OFFLOAD_DEP_TRANSPORT:
	137	WARN_ON(len != sizeof(__u8));
	138	memcpy(&ctx->dep.protonum, data, sizeof(__u8));
	139	break;
	140	default:
	141	break;
	142	}
	143	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
	144	}
	145
	146	static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
3bc158f8 PNA	147	__be16 proto, int priority,
3bc158f8 PNA	148	struct netlink_ext_ack *extack)
c9626a2c PNA	149	{
c9626a2c PNA	150	common->protocol = proto;
3bc158f8	151	common->prio = priority;
c9626a2c PNA	152	common->extack = extack;
	153	}
	154
b5828880 PNA	155	static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
b5828880 PNA	156	struct list_head *cb_list)
c9626a2c PNA	157	{
	158	struct flow_block_cb *block_cb;
	159	int err;
	160
b5828880	161	list_for_each_entry(block_cb, cb_list, list) {
c9626a2c PNA	162	err = block_cb->cb(type, type_data, block_cb->cb_priv);
	163	if (err < 0)
	164	return err;
	165	}
	166	return 0;
	167	}
	168
3bc158f8 PNA	169	int nft_chain_offload_priority(struct nft_base_chain *basechain)
	170	{
	171	if (basechain->ops.priority <= 0 \|\|
	172	basechain->ops.priority > USHRT_MAX)
	173	return -1;
	174
	175	return 0;
	176	}
	177
c5d27527 PNA	178	static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
	179	const struct nft_base_chain *basechain,
	180	const struct nft_rule *rule,
	181	const struct nft_flow_rule *flow,
be193f5e	182	struct netlink_ext_ack *extack,
c5d27527 PNA	183	enum flow_cls_command command)
c5d27527 PNA	184	{
c5d27527 PNA	185	__be16 proto = ETH_P_ALL;
	186
	187	memset(cls_flow, 0, sizeof(*cls_flow));
	188
	189	if (flow)
	190	proto = flow->proto;
	191
	192	nft_flow_offload_common_init(&cls_flow->common, proto,
be193f5e	193	basechain->ops.priority, extack);
c5d27527 PNA	194	cls_flow->command = command;
	195	cls_flow->cookie = (unsigned long) rule;
	196	if (flow)
	197	cls_flow->rule = flow->rule;
	198	}
	199
e211aab7	200	static int nft_flow_offload_rule(struct nft_chain *chain,
	201	struct nft_rule *rule,
	202	struct nft_flow_rule *flow,
c9626a2c PNA	203	enum flow_cls_command command)
c9626a2c PNA	204	{
be193f5e	205	struct netlink_ext_ack extack = {};
c5d27527	206	struct flow_cls_offload cls_flow;
c9626a2c	207	struct nft_base_chain *basechain;
c9626a2c	208
e211aab7	209	if (!nft_is_base_chain(chain))
c9626a2c PNA	210	return -EOPNOTSUPP;
c9626a2c PNA	211
e211aab7	212	basechain = nft_base_chain(chain);
be193f5e PNA	213	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
be193f5e PNA	214	command);
c9626a2c	215
b5828880 PNA	216	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
b5828880 PNA	217	&basechain->flow_block.cb_list);
c9626a2c PNA	218	}
	219
	220	static int nft_flow_offload_bind(struct flow_block_offload *bo,
	221	struct nft_base_chain *basechain)
	222	{
14bfb13f	223	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
c9626a2c PNA	224	return 0;
	225	}
	226
	227	static int nft_flow_offload_unbind(struct flow_block_offload *bo,
	228	struct nft_base_chain *basechain)
	229	{
	230	struct flow_block_cb block_cb, next;
bbaef955	231	struct flow_cls_offload cls_flow;
be193f5e	232	struct netlink_ext_ack extack;
bbaef955 PNA	233	struct nft_chain *chain;
	234	struct nft_rule *rule;
	235
	236	chain = &basechain->chain;
	237	list_for_each_entry(rule, &chain->rules, list) {
be193f5e	238	memset(&extack, 0, sizeof(extack));
bbaef955	239	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
be193f5e	240	&extack, FLOW_CLS_DESTROY);
bbaef955 PNA	241	nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
bbaef955 PNA	242	}
c9626a2c PNA	243
	244	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
	245	list_del(&block_cb->list);
	246	flow_block_cb_free(block_cb);
	247	}
	248
	249	return 0;
	250	}
	251
9a32669f	252	static int nft_block_setup(struct nft_base_chain *basechain,
	253	struct flow_block_offload *bo,
	254	enum flow_block_command cmd)
	255	{
	256	int err;
	257
	258	switch (cmd) {
	259	case FLOW_BLOCK_BIND:
	260	err = nft_flow_offload_bind(bo, basechain);
	261	break;
	262	case FLOW_BLOCK_UNBIND:
	263	err = nft_flow_offload_unbind(bo, basechain);
	264	break;
	265	default:
	266	WARN_ON_ONCE(1);
	267	err = -EOPNOTSUPP;
	268	}
	269
	270	return err;
	271	}
	272
75ceaf86 PNA	273	static void nft_flow_block_offload_init(struct flow_block_offload *bo,
	274	struct net *net,
	275	enum flow_block_command cmd,
	276	struct nft_base_chain *basechain,
	277	struct netlink_ext_ack *extack)
	278	{
	279	memset(bo, 0, sizeof(*bo));
	280	bo->net = net;
	281	bo->block = &basechain->flow_block;
	282	bo->command = cmd;
	283	bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	284	bo->extack = extack;
	285	INIT_LIST_HEAD(&bo->cb_list);
	286	}
	287
9a32669f	288	static int nft_block_offload_cmd(struct nft_base_chain *chain,
	289	struct net_device *dev,
	290	enum flow_block_command cmd)
	291	{
	292	struct netlink_ext_ack extack = {};
75ceaf86	293	struct flow_block_offload bo;
9a32669f	294	int err;
9a32669f	295
75ceaf86	296	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	297
	298	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	299	if (err < 0)
	300	return err;
	301
	302	return nft_block_setup(chain, &bo, cmd);
	303	}
	304
0fdcf78d PNA	305	static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
	306	{
	307	struct nft_base_chain *basechain = block_cb->indr.data;
	308	struct net_device *dev = block_cb->indr.dev;
	309	struct netlink_ext_ack extack = {};
	310	struct net *net = dev_net(dev);
	311	struct flow_block_offload bo;
	312
	313	nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
	314	basechain, &extack);
	315	mutex_lock(&net->nft.commit_mutex);
a1db2178	316	list_del(&block_cb->driver_list);
0fdcf78d PNA	317	list_move(&block_cb->list, &bo.cb_list);
	318	nft_flow_offload_unbind(&bo, basechain);
	319	mutex_unlock(&net->nft.commit_mutex);
	320	}
	321
	322	static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
9a32669f	323	struct net_device *dev,
	324	enum flow_block_command cmd)
	325	{
9a32669f	326	struct netlink_ext_ack extack = {};
75ceaf86	327	struct flow_block_offload bo;
0fdcf78d	328	int err;
9a32669f	329
0fdcf78d	330	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
9a32669f	331
c40f4e50	332	err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
0fdcf78d PNA	333	nft_indr_block_cleanup);
	334	if (err < 0)
	335	return err;
9a32669f	336
	337	if (list_empty(&bo.cb_list))
	338	return -EOPNOTSUPP;
	339
0fdcf78d	340	return nft_block_setup(basechain, &bo, cmd);
9a32669f	341	}
9a32669f	342
6df5490f PNA	343	static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
	344	struct net_device *dev,
	345	enum flow_block_command cmd)
	346	{
	347	int err;
	348
	349	if (dev->netdev_ops->ndo_setup_tc)
	350	err = nft_block_offload_cmd(basechain, dev, cmd);
	351	else
	352	err = nft_indr_block_offload_cmd(basechain, dev, cmd);
	353
	354	return err;
	355	}
	356
ead3952e	357	static int nft_flow_block_chain(struct nft_base_chain *basechain,
d54725cd	358	const struct net_device *this_dev,
ead3952e PNA	359	enum flow_block_command cmd)
ead3952e PNA	360	{
d54725cd PNA	361	struct net_device *dev;
d54725cd PNA	362	struct nft_hook *hook;
671312e1	363	int err, i = 0;
d54725cd PNA	364
	365	list_for_each_entry(hook, &basechain->hook_list, list) {
	366	dev = hook->ops.dev;
	367	if (this_dev && this_dev != dev)
	368	continue;
ead3952e	369
6df5490f	370	err = nft_chain_offload_cmd(basechain, dev, cmd);
671312e1 PNA	371	if (err < 0 && cmd == FLOW_BLOCK_BIND) {
	372	if (!this_dev)
	373	goto err_flow_block;
	374
d54725cd	375	return err;
671312e1 PNA	376	}
671312e1 PNA	377	i++;
d54725cd PNA	378	}
	379
	380	return 0;
671312e1 PNA	381
	382	err_flow_block:
	383	list_for_each_entry(hook, &basechain->hook_list, list) {
	384	if (i-- <= 0)
	385	break;
	386
	387	dev = hook->ops.dev;
	388	nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	389	}
	390	return err;
ead3952e PNA	391	}
ead3952e PNA	392
d54725cd	393	static int nft_flow_offload_chain(struct nft_chain chain, u8 ppolicy,
c9626a2c PNA	394	enum flow_block_command cmd)
c9626a2c PNA	395	{
c9626a2c	396	struct nft_base_chain *basechain;
8fc618c5	397	u8 policy;
c9626a2c PNA	398
	399	if (!nft_is_base_chain(chain))
	400	return -EOPNOTSUPP;
	401
	402	basechain = nft_base_chain(chain);
8fc618c5	403	policy = ppolicy ? *ppolicy : basechain->policy;
8fc618c5	404
c9626a2c	405	/* Only default policy to accept is supported for now. */
ff175d0b	406	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
c9626a2c PNA	407	return -EOPNOTSUPP;
c9626a2c PNA	408
d54725cd	409	return nft_flow_block_chain(basechain, NULL, cmd);
c9626a2c PNA	410	}
c9626a2c PNA	411
63b48c73 PNA	412	static void nft_flow_rule_offload_abort(struct net *net,
	413	struct nft_trans *trans)
	414	{
	415	int err = 0;
	416
	417	list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
	418	if (trans->ctx.family != NFPROTO_NETDEV)
	419	continue;
	420
	421	switch (trans->msg_type) {
	422	case NFT_MSG_NEWCHAIN:
	423	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) \|\|
	424	nft_trans_chain_update(trans))
	425	continue;
	426
	427	err = nft_flow_offload_chain(trans->ctx.chain, NULL,
	428	FLOW_BLOCK_UNBIND);
	429	break;
	430	case NFT_MSG_DELCHAIN:
	431	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	432	continue;
	433
	434	err = nft_flow_offload_chain(trans->ctx.chain, NULL,
	435	FLOW_BLOCK_BIND);
	436	break;
	437	case NFT_MSG_NEWRULE:
	438	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	439	continue;
	440
	441	err = nft_flow_offload_rule(trans->ctx.chain,
	442	nft_trans_rule(trans),
	443	NULL, FLOW_CLS_DESTROY);
	444	break;
	445	case NFT_MSG_DELRULE:
	446	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	447	continue;
	448
	449	err = nft_flow_offload_rule(trans->ctx.chain,
	450	nft_trans_rule(trans),
	451	nft_trans_flow_rule(trans),
	452	FLOW_CLS_REPLACE);
	453	break;
	454	}
	455
	456	if (WARN_ON_ONCE(err))
	457	break;
	458	}
	459	}
	460
c9626a2c PNA	461	int nft_flow_rule_offload_commit(struct net *net)
	462	{
	463	struct nft_trans *trans;
	464	int err = 0;
8fc618c5	465	u8 policy;
c9626a2c PNA	466
	467	list_for_each_entry(trans, &net->nft.commit_list, list) {
	468	if (trans->ctx.family != NFPROTO_NETDEV)
	469	continue;
	470
	471	switch (trans->msg_type) {
	472	case NFT_MSG_NEWCHAIN:
88c74984 PNA	473	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) \|\|
88c74984 PNA	474	nft_trans_chain_update(trans))
c9626a2c PNA	475	continue;
c9626a2c PNA	476
8fc618c5	477	policy = nft_trans_chain_policy(trans);
	478	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
	479	FLOW_BLOCK_BIND);
c9626a2c PNA	480	break;
	481	case NFT_MSG_DELCHAIN:
	482	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	483	continue;
	484
8fc618c5	485	policy = nft_trans_chain_policy(trans);
8fc618c5	486	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
085461c8	487	FLOW_BLOCK_UNBIND);
c9626a2c PNA	488	break;
	489	case NFT_MSG_NEWRULE:
	490	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	491	continue;
	492
	493	if (trans->ctx.flags & NLM_F_REPLACE \|\|
23403cd8 PNA	494	!(trans->ctx.flags & NLM_F_APPEND)) {
	495	err = -EOPNOTSUPP;
	496	break;
	497	}
e211aab7	498	err = nft_flow_offload_rule(trans->ctx.chain,
	499	nft_trans_rule(trans),
	500	nft_trans_flow_rule(trans),
	501	FLOW_CLS_REPLACE);
c9626a2c PNA	502	break;
	503	case NFT_MSG_DELRULE:
	504	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	505	continue;
	506
e211aab7	507	err = nft_flow_offload_rule(trans->ctx.chain,
e211aab7	508	nft_trans_rule(trans),
6ca61c7a	509	NULL, FLOW_CLS_DESTROY);
c9626a2c PNA	510	break;
	511	}
	512
63b48c73 PNA	513	if (err) {
63b48c73 PNA	514	nft_flow_rule_offload_abort(net, trans);
23403cd8	515	break;
63b48c73	516	}
23403cd8 PNA	517	}
	518
	519	list_for_each_entry(trans, &net->nft.commit_list, list) {
	520	if (trans->ctx.family != NFPROTO_NETDEV)
	521	continue;
	522
	523	switch (trans->msg_type) {
	524	case NFT_MSG_NEWRULE:
63b48c73	525	case NFT_MSG_DELRULE:
23403cd8 PNA	526	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	527	continue;
	528
	529	nft_flow_rule_destroy(nft_trans_flow_rule(trans));
	530	break;
	531	default:
	532	break;
	533	}
c9626a2c PNA	534	}
	535
	536	return err;
	537	}
9a32669f	538
504882db	539	static struct nft_chain __nft_offload_get_chain(struct net_device dev)
9a32669f	540	{
504882db	541	struct nft_base_chain *basechain;
9a32669f	542	struct net *net = dev_net(dev);
d54725cd	543	struct nft_hook hook, found;
9a32669f	544	const struct nft_table *table;
504882db	545	struct nft_chain *chain;
9a32669f	546
504882db	547	list_for_each_entry(table, &net->nft.tables, list) {
9a32669f	548	if (table->family != NFPROTO_NETDEV)
	549	continue;
	550
504882db	551	list_for_each_entry(chain, &table->chains, list) {
	552	if (!nft_is_base_chain(chain) \|\|
	553	!(chain->flags & NFT_CHAIN_HW_OFFLOAD))
	554	continue;
	555
d54725cd	556	found = NULL;
504882db	557	basechain = nft_base_chain(chain);
d54725cd PNA	558	list_for_each_entry(hook, &basechain->hook_list, list) {
	559	if (hook->ops.dev != dev)
	560	continue;
	561
	562	found = hook;
	563	break;
	564	}
	565	if (!found)
504882db	566	continue;
	567
	568	return chain;
9a32669f	569	}
9a32669f	570	}
504882db	571
	572	return NULL;
	573	}
	574
06d392cb	575	static int nft_offload_netdev_event(struct notifier_block *this,
	576	unsigned long event, void *ptr)
	577	{
	578	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	579	struct net *net = dev_net(dev);
	580	struct nft_chain *chain;
	581
d1f4c966	582	if (event != NETDEV_UNREGISTER)
	583	return NOTIFY_DONE;
	584
06d392cb	585	mutex_lock(&net->nft.commit_mutex);
	586	chain = __nft_offload_get_chain(dev);
	587	if (chain)
bbaef955 PNA	588	nft_flow_block_chain(nft_base_chain(chain), dev,
	589	FLOW_BLOCK_UNBIND);
	590
06d392cb	591	mutex_unlock(&net->nft.commit_mutex);
	592
	593	return NOTIFY_DONE;
	594	}
	595
06d392cb	596	static struct notifier_block nft_offload_netdev_notifier = {
	597	.notifier_call = nft_offload_netdev_event,
	598	};
	599
	600	int nft_offload_init(void)
3474a2c6	601	{
709ffbe1	602	return register_netdevice_notifier(&nft_offload_netdev_notifier);
3474a2c6 PNA	603	}
	604
	605	void nft_offload_exit(void)
	606	{
06d392cb	607	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
3474a2c6	608	}