[mirror_ubuntu-jammy-kernel.git] / net / netfilter / nf_tables_offload.c

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netfilter.h>
#include <net/flow_offload.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>
#include <net/pkt_cls.h>

static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
{
	struct nft_flow_rule *flow;

	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	if (!flow)
		return NULL;

	flow->rule = flow_rule_alloc(num_actions);
	if (!flow->rule) {
		kfree(flow);
		return NULL;
	}

	flow->rule->match.dissector	= &flow->match.dissector;
	flow->rule->match.mask		= &flow->match.mask;
	flow->rule->match.key		= &flow->match.key;

	return flow;
}

struct nft_flow_rule *nft_flow_rule_create(struct net *net,
					   const struct nft_rule *rule)
{
	struct nft_offload_ctx *ctx;
	struct nft_flow_rule *flow;
	int num_actions = 0, err;
	struct nft_expr *expr;

	expr = nft_expr_first(rule);
	while (expr->ops && expr != nft_expr_last(rule)) {
		if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
			num_actions++;

		expr = nft_expr_next(expr);
	}

	if (num_actions == 0)
		return ERR_PTR(-EOPNOTSUPP);

	flow = nft_flow_rule_alloc(num_actions);
	if (!flow)
		return ERR_PTR(-ENOMEM);

	expr = nft_expr_first(rule);

	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	if (!ctx) {
		err = -ENOMEM;
		goto err_out;
	}
	ctx->net = net;
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;

	while (expr->ops && expr != nft_expr_last(rule)) {
		if (!expr->ops->offload) {
			err = -EOPNOTSUPP;
			goto err_out;
		}
		err = expr->ops->offload(ctx, flow, expr);
		if (err < 0)
			goto err_out;

		expr = nft_expr_next(expr);
	}
	flow->proto = ctx->dep.l3num;
	kfree(ctx);

	return flow;
err_out:
	kfree(ctx);
	nft_flow_rule_destroy(flow);

	return ERR_PTR(err);
}

void nft_flow_rule_destroy(struct nft_flow_rule *flow)
{
	struct flow_action_entry *entry;
	int i;

	flow_action_for_each(i, entry, &flow->rule->action) {
		switch (entry->id) {
		case FLOW_ACTION_REDIRECT:
		case FLOW_ACTION_MIRRED:
			dev_put(entry->dev);
			break;
		default:
			break;
		}
	}
	kfree(flow->rule);
	kfree(flow);
}

void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
				enum nft_offload_dep_type type)
{
	ctx->dep.type = type;
}

void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
				   const void *data, u32 len)
{
	switch (ctx->dep.type) {
	case NFT_OFFLOAD_DEP_NETWORK:
		WARN_ON(len != sizeof(__u16));
		memcpy(&ctx->dep.l3num, data, sizeof(__u16));
		break;
	case NFT_OFFLOAD_DEP_TRANSPORT:
		WARN_ON(len != sizeof(__u8));
		memcpy(&ctx->dep.protonum, data, sizeof(__u8));
		break;
	default:
		break;
	}
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
}

static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
					 __be16 proto, int priority,
					 struct netlink_ext_ack *extack)
{
	common->protocol = proto;
	common->prio = priority;
	common->extack = extack;
}

static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
			     struct list_head *cb_list)
{
	struct flow_block_cb *block_cb;
	int err;

	list_for_each_entry(block_cb, cb_list, list) {
		err = block_cb->cb(type, type_data, block_cb->cb_priv);
		if (err < 0)
			return err;
	}
	return 0;
}

int nft_chain_offload_priority(struct nft_base_chain *basechain)
{
	if (basechain->ops.priority <= 0 ||
	    basechain->ops.priority > USHRT_MAX)
		return -1;

	return 0;
}

static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
				       const struct nft_base_chain *basechain,
				       const struct nft_rule *rule,
				       const struct nft_flow_rule *flow,
				       struct netlink_ext_ack *extack,
				       enum flow_cls_command command)
{
	__be16 proto = ETH_P_ALL;

	memset(cls_flow, 0, sizeof(*cls_flow));

	if (flow)
		proto = flow->proto;

	nft_flow_offload_common_init(&cls_flow->common, proto,
				     basechain->ops.priority, extack);
	cls_flow->command = command;
	cls_flow->cookie = (unsigned long) rule;
	if (flow)
		cls_flow->rule = flow->rule;
}

static int nft_flow_offload_rule(struct nft_chain *chain,
				 struct nft_rule *rule,
				 struct nft_flow_rule *flow,
				 enum flow_cls_command command)
{
	struct netlink_ext_ack extack = {};
	struct flow_cls_offload cls_flow;
	struct nft_base_chain *basechain;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
				   command);

	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
				 &basechain->flow_block.cb_list);
}

static int nft_flow_offload_bind(struct flow_block_offload *bo,
				 struct nft_base_chain *basechain)
{
	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
	return 0;
}

static int nft_flow_offload_unbind(struct flow_block_offload *bo,
				   struct nft_base_chain *basechain)
{
	struct flow_block_cb *block_cb, *next;
	struct flow_cls_offload cls_flow;
	struct netlink_ext_ack extack;
	struct nft_chain *chain;
	struct nft_rule *rule;

	chain = &basechain->chain;
	list_for_each_entry(rule, &chain->rules, list) {
		memset(&extack, 0, sizeof(extack));
		nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
					   &extack, FLOW_CLS_DESTROY);
		nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
	}

	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
		list_del(&block_cb->list);
		flow_block_cb_free(block_cb);
	}

	return 0;
}

static int nft_block_setup(struct nft_base_chain *basechain,
			   struct flow_block_offload *bo,
			   enum flow_block_command cmd)
{
	int err;

	switch (cmd) {
	case FLOW_BLOCK_BIND:
		err = nft_flow_offload_bind(bo, basechain);
		break;
	case FLOW_BLOCK_UNBIND:
		err = nft_flow_offload_unbind(bo, basechain);
		break;
	default:
		WARN_ON_ONCE(1);
		err = -EOPNOTSUPP;
	}

	return err;
}

static void nft_flow_block_offload_init(struct flow_block_offload *bo,
					struct net *net,
					enum flow_block_command cmd,
					struct nft_base_chain *basechain,
					struct netlink_ext_ack *extack)
{
	memset(bo, 0, sizeof(*bo));
	bo->net		= net;
	bo->block	= &basechain->flow_block;
	bo->command	= cmd;
	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	bo->extack	= extack;
	INIT_LIST_HEAD(&bo->cb_list);
}

static int nft_block_offload_cmd(struct nft_base_chain *chain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	if (err < 0)
		return err;

	return nft_block_setup(chain, &bo, cmd);
}

static void nft_indr_block_ing_cmd(struct net_device *dev,
				   struct nft_base_chain *chain,
				   flow_indr_block_bind_cb_t *cb,
				   void *cb_priv,
				   enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;

	if (!chain)
		return;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);

	nft_block_setup(chain, &bo, cmd);
}

static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
				      struct net_device *dev,
				      enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	flow_indr_block_call(dev, &bo, cmd, TC_SETUP_BLOCK);

	if (list_empty(&bo.cb_list))
		return -EOPNOTSUPP;

	return nft_block_setup(chain, &bo, cmd);
}

#define FLOW_SETUP_BLOCK TC_SETUP_BLOCK

static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	int err;

	if (dev->netdev_ops->ndo_setup_tc)
		err = nft_block_offload_cmd(basechain, dev, cmd);
	else
		err = nft_indr_block_offload_cmd(basechain, dev, cmd);

	return err;
}

static int nft_flow_block_chain(struct nft_base_chain *basechain,
				const struct net_device *this_dev,
				enum flow_block_command cmd)
{
	struct net_device *dev;
	struct nft_hook *hook;
	int err, i = 0;

	list_for_each_entry(hook, &basechain->hook_list, list) {
		dev = hook->ops.dev;
		if (this_dev && this_dev != dev)
			continue;

		err = nft_chain_offload_cmd(basechain, dev, cmd);
		if (err < 0 && cmd == FLOW_BLOCK_BIND) {
			if (!this_dev)
				goto err_flow_block;

			return err;
		}
		i++;
	}

	return 0;

err_flow_block:
	list_for_each_entry(hook, &basechain->hook_list, list) {
		if (i-- <= 0)
			break;

		dev = hook->ops.dev;
		nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	}
	return err;
}

static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
				  enum flow_block_command cmd)
{
	struct nft_base_chain *basechain;
	u8 policy;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	policy = ppolicy ? *ppolicy : basechain->policy;

	/* Only default policy to accept is supported for now. */
	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
		return -EOPNOTSUPP;

	return nft_flow_block_chain(basechain, NULL, cmd);
}

static void nft_flow_rule_offload_abort(struct net *net,
					struct nft_trans *trans)
{
	int err = 0;

	list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		}

		if (WARN_ON_ONCE(err))
			break;
	}
}

int nft_flow_rule_offload_commit(struct net *net)
{
	struct nft_trans *trans;
	int err = 0;
	u8 policy;

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			if (trans->ctx.flags & NLM_F_REPLACE ||
			    !(trans->ctx.flags & NLM_F_APPEND)) {
				err = -EOPNOTSUPP;
				break;
			}
			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		}

		if (err) {
			nft_flow_rule_offload_abort(net, trans);
			break;
		}
	}

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWRULE:
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			nft_flow_rule_destroy(nft_trans_flow_rule(trans));
			break;
		default:
			break;
		}
	}

	return err;
}

static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
{
	struct nft_base_chain *basechain;
	struct net *net = dev_net(dev);
	struct nft_hook *hook, *found;
	const struct nft_table *table;
	struct nft_chain *chain;

	list_for_each_entry(table, &net->nft.tables, list) {
		if (table->family != NFPROTO_NETDEV)
			continue;

		list_for_each_entry(chain, &table->chains, list) {
			if (!nft_is_base_chain(chain) ||
			    !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			found = NULL;
			basechain = nft_base_chain(chain);
			list_for_each_entry(hook, &basechain->hook_list, list) {
				if (hook->ops.dev != dev)
					continue;

				found = hook;
				break;
			}
			if (!found)
				continue;

			return chain;
		}
	}

	return NULL;
}

static void nft_indr_block_cb(struct net_device *dev,
			      flow_indr_block_bind_cb_t *cb, void *cb_priv,
			      enum flow_block_command cmd)
{
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	mutex_lock(&net->nft.commit_mutex);
	chain = __nft_offload_get_chain(dev);
	if (chain && chain->flags & NFT_CHAIN_HW_OFFLOAD) {
		struct nft_base_chain *basechain;

		basechain = nft_base_chain(chain);
		nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
	}
	mutex_unlock(&net->nft.commit_mutex);
}

static int nft_offload_netdev_event(struct notifier_block *this,
				    unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	if (event != NETDEV_UNREGISTER)
		return NOTIFY_DONE;

	mutex_lock(&net->nft.commit_mutex);
	chain = __nft_offload_get_chain(dev);
	if (chain)
		nft_flow_block_chain(nft_base_chain(chain), dev,
				     FLOW_BLOCK_UNBIND);

	mutex_unlock(&net->nft.commit_mutex);

	return NOTIFY_DONE;
}

static struct flow_indr_block_entry block_ing_entry = {
	.cb	= nft_indr_block_cb,
	.list	= LIST_HEAD_INIT(block_ing_entry.list),
};

static struct notifier_block nft_offload_netdev_notifier = {
	.notifier_call	= nft_offload_netdev_event,
};

int nft_offload_init(void)
{
	int err;

	err = register_netdevice_notifier(&nft_offload_netdev_notifier);
	if (err < 0)
		return err;

	flow_indr_add_block_cb(&block_ing_entry);

	return 0;
}

void nft_offload_exit(void)
{
	flow_indr_del_block_cb(&block_ing_entry);
	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
}
Commit	Line	Data
c9626a2c PNA	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#include <linux/init.h>
	3	#include <linux/module.h>
	4	#include <linux/netfilter.h>
	5	#include <net/flow_offload.h>
	6	#include <net/netfilter/nf_tables.h>
	7	#include <net/netfilter/nf_tables_offload.h>
	8	#include <net/pkt_cls.h>
	9
	10	static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
	11	{
	12	struct nft_flow_rule *flow;
	13
	14	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	15	if (!flow)
	16	return NULL;
	17
	18	flow->rule = flow_rule_alloc(num_actions);
	19	if (!flow->rule) {
	20	kfree(flow);
	21	return NULL;
	22	}
	23
	24	flow->rule->match.dissector = &flow->match.dissector;
	25	flow->rule->match.mask = &flow->match.mask;
	26	flow->rule->match.key = &flow->match.key;
	27
	28	return flow;
	29	}
	30
be2861dc PNA	31	struct nft_flow_rule nft_flow_rule_create(struct net net,
be2861dc PNA	32	const struct nft_rule *rule)
c9626a2c	33	{
b44492af	34	struct nft_offload_ctx *ctx;
c9626a2c PNA	35	struct nft_flow_rule *flow;
	36	int num_actions = 0, err;
	37	struct nft_expr *expr;
	38
	39	expr = nft_expr_first(rule);
	40	while (expr->ops && expr != nft_expr_last(rule)) {
	41	if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
	42	num_actions++;
	43
	44	expr = nft_expr_next(expr);
	45	}
	46
81ec6107 PNA	47	if (num_actions == 0)
	48	return ERR_PTR(-EOPNOTSUPP);
	49
c9626a2c PNA	50	flow = nft_flow_rule_alloc(num_actions);
	51	if (!flow)
	52	return ERR_PTR(-ENOMEM);
	53
	54	expr = nft_expr_first(rule);
b44492af AB	55
	56	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	57	if (!ctx) {
	58	err = -ENOMEM;
	59	goto err_out;
	60	}
be2861dc	61	ctx->net = net;
b44492af AB	62	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
b44492af AB	63
c9626a2c PNA	64	while (expr->ops && expr != nft_expr_last(rule)) {
	65	if (!expr->ops->offload) {
	66	err = -EOPNOTSUPP;
	67	goto err_out;
	68	}
b44492af	69	err = expr->ops->offload(ctx, flow, expr);
c9626a2c PNA	70	if (err < 0)
	71	goto err_out;
	72
	73	expr = nft_expr_next(expr);
	74	}
b44492af AB	75	flow->proto = ctx->dep.l3num;
b44492af AB	76	kfree(ctx);
c9626a2c PNA	77
	78	return flow;
	79	err_out:
b44492af	80	kfree(ctx);
c9626a2c PNA	81	nft_flow_rule_destroy(flow);
	82
	83	return ERR_PTR(err);
	84	}
	85
	86	void nft_flow_rule_destroy(struct nft_flow_rule *flow)
	87	{
be2861dc PNA	88	struct flow_action_entry *entry;
	89	int i;
	90
	91	flow_action_for_each(i, entry, &flow->rule->action) {
	92	switch (entry->id) {
	93	case FLOW_ACTION_REDIRECT:
	94	case FLOW_ACTION_MIRRED:
	95	dev_put(entry->dev);
	96	break;
	97	default:
	98	break;
	99	}
	100	}
c9626a2c PNA	101	kfree(flow->rule);
	102	kfree(flow);
	103	}
	104
	105	void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
	106	enum nft_offload_dep_type type)
	107	{
	108	ctx->dep.type = type;
	109	}
	110
	111	void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
	112	const void *data, u32 len)
	113	{
	114	switch (ctx->dep.type) {
	115	case NFT_OFFLOAD_DEP_NETWORK:
	116	WARN_ON(len != sizeof(__u16));
	117	memcpy(&ctx->dep.l3num, data, sizeof(__u16));
	118	break;
	119	case NFT_OFFLOAD_DEP_TRANSPORT:
	120	WARN_ON(len != sizeof(__u8));
	121	memcpy(&ctx->dep.protonum, data, sizeof(__u8));
	122	break;
	123	default:
	124	break;
	125	}
	126	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
	127	}
	128
	129	static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
3bc158f8 PNA	130	__be16 proto, int priority,
3bc158f8 PNA	131	struct netlink_ext_ack *extack)
c9626a2c PNA	132	{
c9626a2c PNA	133	common->protocol = proto;
3bc158f8	134	common->prio = priority;
c9626a2c PNA	135	common->extack = extack;
	136	}
	137
b5828880 PNA	138	static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
b5828880 PNA	139	struct list_head *cb_list)
c9626a2c PNA	140	{
	141	struct flow_block_cb *block_cb;
	142	int err;
	143
b5828880	144	list_for_each_entry(block_cb, cb_list, list) {
c9626a2c PNA	145	err = block_cb->cb(type, type_data, block_cb->cb_priv);
	146	if (err < 0)
	147	return err;
	148	}
	149	return 0;
	150	}
	151
3bc158f8 PNA	152	int nft_chain_offload_priority(struct nft_base_chain *basechain)
	153	{
	154	if (basechain->ops.priority <= 0 \|\|
	155	basechain->ops.priority > USHRT_MAX)
	156	return -1;
	157
	158	return 0;
	159	}
	160
c5d27527 PNA	161	static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
	162	const struct nft_base_chain *basechain,
	163	const struct nft_rule *rule,
	164	const struct nft_flow_rule *flow,
be193f5e	165	struct netlink_ext_ack *extack,
c5d27527 PNA	166	enum flow_cls_command command)
c5d27527 PNA	167	{
c5d27527 PNA	168	__be16 proto = ETH_P_ALL;
	169
	170	memset(cls_flow, 0, sizeof(*cls_flow));
	171
	172	if (flow)
	173	proto = flow->proto;
	174
	175	nft_flow_offload_common_init(&cls_flow->common, proto,
be193f5e	176	basechain->ops.priority, extack);
c5d27527 PNA	177	cls_flow->command = command;
	178	cls_flow->cookie = (unsigned long) rule;
	179	if (flow)
	180	cls_flow->rule = flow->rule;
	181	}
	182
e211aab7	183	static int nft_flow_offload_rule(struct nft_chain *chain,
	184	struct nft_rule *rule,
	185	struct nft_flow_rule *flow,
c9626a2c PNA	186	enum flow_cls_command command)
c9626a2c PNA	187	{
be193f5e	188	struct netlink_ext_ack extack = {};
c5d27527	189	struct flow_cls_offload cls_flow;
c9626a2c	190	struct nft_base_chain *basechain;
c9626a2c	191
e211aab7	192	if (!nft_is_base_chain(chain))
c9626a2c PNA	193	return -EOPNOTSUPP;
c9626a2c PNA	194
e211aab7	195	basechain = nft_base_chain(chain);
be193f5e PNA	196	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
be193f5e PNA	197	command);
c9626a2c	198
b5828880 PNA	199	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
b5828880 PNA	200	&basechain->flow_block.cb_list);
c9626a2c PNA	201	}
	202
	203	static int nft_flow_offload_bind(struct flow_block_offload *bo,
	204	struct nft_base_chain *basechain)
	205	{
14bfb13f	206	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
c9626a2c PNA	207	return 0;
	208	}
	209
	210	static int nft_flow_offload_unbind(struct flow_block_offload *bo,
	211	struct nft_base_chain *basechain)
	212	{
	213	struct flow_block_cb block_cb, next;
bbaef955	214	struct flow_cls_offload cls_flow;
be193f5e	215	struct netlink_ext_ack extack;
bbaef955 PNA	216	struct nft_chain *chain;
	217	struct nft_rule *rule;
	218
	219	chain = &basechain->chain;
	220	list_for_each_entry(rule, &chain->rules, list) {
be193f5e	221	memset(&extack, 0, sizeof(extack));
bbaef955	222	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
be193f5e	223	&extack, FLOW_CLS_DESTROY);
bbaef955 PNA	224	nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
bbaef955 PNA	225	}
c9626a2c PNA	226
	227	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
	228	list_del(&block_cb->list);
	229	flow_block_cb_free(block_cb);
	230	}
	231
	232	return 0;
	233	}
	234
9a32669f	235	static int nft_block_setup(struct nft_base_chain *basechain,
	236	struct flow_block_offload *bo,
	237	enum flow_block_command cmd)
	238	{
	239	int err;
	240
	241	switch (cmd) {
	242	case FLOW_BLOCK_BIND:
	243	err = nft_flow_offload_bind(bo, basechain);
	244	break;
	245	case FLOW_BLOCK_UNBIND:
	246	err = nft_flow_offload_unbind(bo, basechain);
	247	break;
	248	default:
	249	WARN_ON_ONCE(1);
	250	err = -EOPNOTSUPP;
	251	}
	252
	253	return err;
	254	}
	255
75ceaf86 PNA	256	static void nft_flow_block_offload_init(struct flow_block_offload *bo,
	257	struct net *net,
	258	enum flow_block_command cmd,
	259	struct nft_base_chain *basechain,
	260	struct netlink_ext_ack *extack)
	261	{
	262	memset(bo, 0, sizeof(*bo));
	263	bo->net = net;
	264	bo->block = &basechain->flow_block;
	265	bo->command = cmd;
	266	bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	267	bo->extack = extack;
	268	INIT_LIST_HEAD(&bo->cb_list);
	269	}
	270
9a32669f	271	static int nft_block_offload_cmd(struct nft_base_chain *chain,
	272	struct net_device *dev,
	273	enum flow_block_command cmd)
	274	{
	275	struct netlink_ext_ack extack = {};
75ceaf86	276	struct flow_block_offload bo;
9a32669f	277	int err;
9a32669f	278
75ceaf86	279	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	280
	281	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	282	if (err < 0)
	283	return err;
	284
	285	return nft_block_setup(chain, &bo, cmd);
	286	}
	287
	288	static void nft_indr_block_ing_cmd(struct net_device *dev,
	289	struct nft_base_chain *chain,
	290	flow_indr_block_bind_cb_t *cb,
	291	void *cb_priv,
	292	enum flow_block_command cmd)
	293	{
	294	struct netlink_ext_ack extack = {};
75ceaf86	295	struct flow_block_offload bo;
9a32669f	296
	297	if (!chain)
	298	return;
	299
75ceaf86	300	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	301
	302	cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
	303
	304	nft_block_setup(chain, &bo, cmd);
	305	}
	306
	307	static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
	308	struct net_device *dev,
	309	enum flow_block_command cmd)
	310	{
9a32669f	311	struct netlink_ext_ack extack = {};
75ceaf86	312	struct flow_block_offload bo;
9a32669f	313
75ceaf86	314	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	315
133a2fe5	316	flow_indr_block_call(dev, &bo, cmd, TC_SETUP_BLOCK);
9a32669f	317
	318	if (list_empty(&bo.cb_list))
	319	return -EOPNOTSUPP;
	320
	321	return nft_block_setup(chain, &bo, cmd);
	322	}
	323
c9626a2c PNA	324	#define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
c9626a2c PNA	325
6df5490f PNA	326	static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
	327	struct net_device *dev,
	328	enum flow_block_command cmd)
	329	{
	330	int err;
	331
	332	if (dev->netdev_ops->ndo_setup_tc)
	333	err = nft_block_offload_cmd(basechain, dev, cmd);
	334	else
	335	err = nft_indr_block_offload_cmd(basechain, dev, cmd);
	336
	337	return err;
	338	}
	339
ead3952e	340	static int nft_flow_block_chain(struct nft_base_chain *basechain,
d54725cd	341	const struct net_device *this_dev,
ead3952e PNA	342	enum flow_block_command cmd)
ead3952e PNA	343	{
d54725cd PNA	344	struct net_device *dev;
d54725cd PNA	345	struct nft_hook *hook;
671312e1	346	int err, i = 0;
d54725cd PNA	347
	348	list_for_each_entry(hook, &basechain->hook_list, list) {
	349	dev = hook->ops.dev;
	350	if (this_dev && this_dev != dev)
	351	continue;
ead3952e	352
6df5490f	353	err = nft_chain_offload_cmd(basechain, dev, cmd);
671312e1 PNA	354	if (err < 0 && cmd == FLOW_BLOCK_BIND) {
	355	if (!this_dev)
	356	goto err_flow_block;
	357
d54725cd	358	return err;
671312e1 PNA	359	}
671312e1 PNA	360	i++;
d54725cd PNA	361	}
	362
	363	return 0;
671312e1 PNA	364
	365	err_flow_block:
	366	list_for_each_entry(hook, &basechain->hook_list, list) {
	367	if (i-- <= 0)
	368	break;
	369
	370	dev = hook->ops.dev;
	371	nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	372	}
	373	return err;
ead3952e PNA	374	}
ead3952e PNA	375
d54725cd	376	static int nft_flow_offload_chain(struct nft_chain chain, u8 ppolicy,
c9626a2c PNA	377	enum flow_block_command cmd)
c9626a2c PNA	378	{
c9626a2c	379	struct nft_base_chain *basechain;
8fc618c5	380	u8 policy;
c9626a2c PNA	381
	382	if (!nft_is_base_chain(chain))
	383	return -EOPNOTSUPP;
	384
	385	basechain = nft_base_chain(chain);
8fc618c5	386	policy = ppolicy ? *ppolicy : basechain->policy;
8fc618c5	387
c9626a2c	388	/* Only default policy to accept is supported for now. */
ff175d0b	389	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
c9626a2c PNA	390	return -EOPNOTSUPP;
c9626a2c PNA	391
d54725cd	392	return nft_flow_block_chain(basechain, NULL, cmd);
c9626a2c PNA	393	}
c9626a2c PNA	394
63b48c73 PNA	395	static void nft_flow_rule_offload_abort(struct net *net,
	396	struct nft_trans *trans)
	397	{
	398	int err = 0;
	399
	400	list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
	401	if (trans->ctx.family != NFPROTO_NETDEV)
	402	continue;
	403
	404	switch (trans->msg_type) {
	405	case NFT_MSG_NEWCHAIN:
	406	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) \|\|
	407	nft_trans_chain_update(trans))
	408	continue;
	409
	410	err = nft_flow_offload_chain(trans->ctx.chain, NULL,
	411	FLOW_BLOCK_UNBIND);
	412	break;
	413	case NFT_MSG_DELCHAIN:
	414	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	415	continue;
	416
	417	err = nft_flow_offload_chain(trans->ctx.chain, NULL,
	418	FLOW_BLOCK_BIND);
	419	break;
	420	case NFT_MSG_NEWRULE:
	421	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	422	continue;
	423
	424	err = nft_flow_offload_rule(trans->ctx.chain,
	425	nft_trans_rule(trans),
	426	NULL, FLOW_CLS_DESTROY);
	427	break;
	428	case NFT_MSG_DELRULE:
	429	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	430	continue;
	431
	432	err = nft_flow_offload_rule(trans->ctx.chain,
	433	nft_trans_rule(trans),
	434	nft_trans_flow_rule(trans),
	435	FLOW_CLS_REPLACE);
	436	break;
	437	}
	438
	439	if (WARN_ON_ONCE(err))
	440	break;
	441	}
	442	}
	443
c9626a2c PNA	444	int nft_flow_rule_offload_commit(struct net *net)
	445	{
	446	struct nft_trans *trans;
	447	int err = 0;
8fc618c5	448	u8 policy;
c9626a2c PNA	449
	450	list_for_each_entry(trans, &net->nft.commit_list, list) {
	451	if (trans->ctx.family != NFPROTO_NETDEV)
	452	continue;
	453
	454	switch (trans->msg_type) {
	455	case NFT_MSG_NEWCHAIN:
88c74984 PNA	456	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) \|\|
88c74984 PNA	457	nft_trans_chain_update(trans))
c9626a2c PNA	458	continue;
c9626a2c PNA	459
8fc618c5	460	policy = nft_trans_chain_policy(trans);
	461	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
	462	FLOW_BLOCK_BIND);
c9626a2c PNA	463	break;
	464	case NFT_MSG_DELCHAIN:
	465	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	466	continue;
	467
8fc618c5	468	policy = nft_trans_chain_policy(trans);
8fc618c5	469	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
085461c8	470	FLOW_BLOCK_UNBIND);
c9626a2c PNA	471	break;
	472	case NFT_MSG_NEWRULE:
	473	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	474	continue;
	475
	476	if (trans->ctx.flags & NLM_F_REPLACE \|\|
23403cd8 PNA	477	!(trans->ctx.flags & NLM_F_APPEND)) {
	478	err = -EOPNOTSUPP;
	479	break;
	480	}
e211aab7	481	err = nft_flow_offload_rule(trans->ctx.chain,
	482	nft_trans_rule(trans),
	483	nft_trans_flow_rule(trans),
	484	FLOW_CLS_REPLACE);
c9626a2c PNA	485	break;
	486	case NFT_MSG_DELRULE:
	487	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	488	continue;
	489
e211aab7	490	err = nft_flow_offload_rule(trans->ctx.chain,
e211aab7	491	nft_trans_rule(trans),
6ca61c7a	492	NULL, FLOW_CLS_DESTROY);
c9626a2c PNA	493	break;
	494	}
	495
63b48c73 PNA	496	if (err) {
63b48c73 PNA	497	nft_flow_rule_offload_abort(net, trans);
23403cd8	498	break;
63b48c73	499	}
23403cd8 PNA	500	}
	501
	502	list_for_each_entry(trans, &net->nft.commit_list, list) {
	503	if (trans->ctx.family != NFPROTO_NETDEV)
	504	continue;
	505
	506	switch (trans->msg_type) {
	507	case NFT_MSG_NEWRULE:
63b48c73	508	case NFT_MSG_DELRULE:
23403cd8 PNA	509	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	510	continue;
	511
	512	nft_flow_rule_destroy(nft_trans_flow_rule(trans));
	513	break;
	514	default:
	515	break;
	516	}
c9626a2c PNA	517	}
	518
	519	return err;
	520	}
9a32669f	521
504882db	522	static struct nft_chain __nft_offload_get_chain(struct net_device dev)
9a32669f	523	{
504882db	524	struct nft_base_chain *basechain;
9a32669f	525	struct net *net = dev_net(dev);
d54725cd	526	struct nft_hook hook, found;
9a32669f	527	const struct nft_table *table;
504882db	528	struct nft_chain *chain;
9a32669f	529
504882db	530	list_for_each_entry(table, &net->nft.tables, list) {
9a32669f	531	if (table->family != NFPROTO_NETDEV)
	532	continue;
	533
504882db	534	list_for_each_entry(chain, &table->chains, list) {
	535	if (!nft_is_base_chain(chain) \|\|
	536	!(chain->flags & NFT_CHAIN_HW_OFFLOAD))
	537	continue;
	538
d54725cd	539	found = NULL;
504882db	540	basechain = nft_base_chain(chain);
d54725cd PNA	541	list_for_each_entry(hook, &basechain->hook_list, list) {
	542	if (hook->ops.dev != dev)
	543	continue;
	544
	545	found = hook;
	546	break;
	547	}
	548	if (!found)
504882db	549	continue;
	550
	551	return chain;
9a32669f	552	}
9a32669f	553	}
504882db	554
	555	return NULL;
	556	}
	557
	558	static void nft_indr_block_cb(struct net_device *dev,
	559	flow_indr_block_bind_cb_t cb, void cb_priv,
	560	enum flow_block_command cmd)
	561	{
	562	struct net *net = dev_net(dev);
	563	struct nft_chain *chain;
	564
	565	mutex_lock(&net->nft.commit_mutex);
	566	chain = __nft_offload_get_chain(dev);
c83de17d	567	if (chain && chain->flags & NFT_CHAIN_HW_OFFLOAD) {
504882db	568	struct nft_base_chain *basechain;
	569
	570	basechain = nft_base_chain(chain);
	571	nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
	572	}
	573	mutex_unlock(&net->nft.commit_mutex);
9a32669f	574	}
3474a2c6	575
06d392cb	576	static int nft_offload_netdev_event(struct notifier_block *this,
	577	unsigned long event, void *ptr)
	578	{
	579	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	580	struct net *net = dev_net(dev);
	581	struct nft_chain *chain;
	582
d1f4c966	583	if (event != NETDEV_UNREGISTER)
	584	return NOTIFY_DONE;
	585
06d392cb	586	mutex_lock(&net->nft.commit_mutex);
	587	chain = __nft_offload_get_chain(dev);
	588	if (chain)
bbaef955 PNA	589	nft_flow_block_chain(nft_base_chain(chain), dev,
	590	FLOW_BLOCK_UNBIND);
	591
06d392cb	592	mutex_unlock(&net->nft.commit_mutex);
	593
	594	return NOTIFY_DONE;
	595	}
	596
dbad3408	597	static struct flow_indr_block_entry block_ing_entry = {
3474a2c6 PNA	598	.cb = nft_indr_block_cb,
	599	.list = LIST_HEAD_INIT(block_ing_entry.list),
	600	};
	601
06d392cb	602	static struct notifier_block nft_offload_netdev_notifier = {
	603	.notifier_call = nft_offload_netdev_event,
	604	};
	605
	606	int nft_offload_init(void)
3474a2c6	607	{
06d392cb	608	int err;
	609
	610	err = register_netdevice_notifier(&nft_offload_netdev_notifier);
	611	if (err < 0)
	612	return err;
	613
dbad3408	614	flow_indr_add_block_cb(&block_ing_entry);
06d392cb	615
06d392cb	616	return 0;
3474a2c6 PNA	617	}
	618
	619	void nft_offload_exit(void)
	620	{
dbad3408	621	flow_indr_del_block_cb(&block_ing_entry);
06d392cb	622	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
3474a2c6	623	}