[mirror_ubuntu-jammy-kernel.git] / net / netfilter / nf_tables_offload.c

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netfilter.h>
#include <net/flow_offload.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>
#include <net/pkt_cls.h>

static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
{
	struct nft_flow_rule *flow;

	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	if (!flow)
		return NULL;

	flow->rule = flow_rule_alloc(num_actions);
	if (!flow->rule) {
		kfree(flow);
		return NULL;
	}

	flow->rule->match.dissector	= &flow->match.dissector;
	flow->rule->match.mask		= &flow->match.mask;
	flow->rule->match.key		= &flow->match.key;

	return flow;
}

struct nft_flow_rule *nft_flow_rule_create(struct net *net,
					   const struct nft_rule *rule)
{
	struct nft_offload_ctx *ctx;
	struct nft_flow_rule *flow;
	int num_actions = 0, err;
	struct nft_expr *expr;

	expr = nft_expr_first(rule);
	while (expr->ops && expr != nft_expr_last(rule)) {
		if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
			num_actions++;

		expr = nft_expr_next(expr);
	}

	flow = nft_flow_rule_alloc(num_actions);
	if (!flow)
		return ERR_PTR(-ENOMEM);

	expr = nft_expr_first(rule);

	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	if (!ctx) {
		err = -ENOMEM;
		goto err_out;
	}
	ctx->net = net;
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;

	while (expr->ops && expr != nft_expr_last(rule)) {
		if (!expr->ops->offload) {
			err = -EOPNOTSUPP;
			goto err_out;
		}
		err = expr->ops->offload(ctx, flow, expr);
		if (err < 0)
			goto err_out;

		expr = nft_expr_next(expr);
	}
	flow->proto = ctx->dep.l3num;
	kfree(ctx);

	return flow;
err_out:
	kfree(ctx);
	nft_flow_rule_destroy(flow);

	return ERR_PTR(err);
}

void nft_flow_rule_destroy(struct nft_flow_rule *flow)
{
	struct flow_action_entry *entry;
	int i;

	flow_action_for_each(i, entry, &flow->rule->action) {
		switch (entry->id) {
		case FLOW_ACTION_REDIRECT:
		case FLOW_ACTION_MIRRED:
			dev_put(entry->dev);
			break;
		default:
			break;
		}
	}
	kfree(flow->rule);
	kfree(flow);
}

void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
				enum nft_offload_dep_type type)
{
	ctx->dep.type = type;
}

void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
				   const void *data, u32 len)
{
	switch (ctx->dep.type) {
	case NFT_OFFLOAD_DEP_NETWORK:
		WARN_ON(len != sizeof(__u16));
		memcpy(&ctx->dep.l3num, data, sizeof(__u16));
		break;
	case NFT_OFFLOAD_DEP_TRANSPORT:
		WARN_ON(len != sizeof(__u8));
		memcpy(&ctx->dep.protonum, data, sizeof(__u8));
		break;
	default:
		break;
	}
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
}

static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
					 __be16 proto, int priority,
					 struct netlink_ext_ack *extack)
{
	common->protocol = proto;
	common->prio = priority;
	common->extack = extack;
}

static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
			     struct list_head *cb_list)
{
	struct flow_block_cb *block_cb;
	int err;

	list_for_each_entry(block_cb, cb_list, list) {
		err = block_cb->cb(type, type_data, block_cb->cb_priv);
		if (err < 0)
			return err;
	}
	return 0;
}

int nft_chain_offload_priority(struct nft_base_chain *basechain)
{
	if (basechain->ops.priority <= 0 ||
	    basechain->ops.priority > USHRT_MAX)
		return -1;

	return 0;
}

static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
				       const struct nft_base_chain *basechain,
				       const struct nft_rule *rule,
				       const struct nft_flow_rule *flow,
				       struct netlink_ext_ack *extack,
				       enum flow_cls_command command)
{
	__be16 proto = ETH_P_ALL;

	memset(cls_flow, 0, sizeof(*cls_flow));

	if (flow)
		proto = flow->proto;

	nft_flow_offload_common_init(&cls_flow->common, proto,
				     basechain->ops.priority, extack);
	cls_flow->command = command;
	cls_flow->cookie = (unsigned long) rule;
	if (flow)
		cls_flow->rule = flow->rule;
}

static int nft_flow_offload_rule(struct nft_chain *chain,
				 struct nft_rule *rule,
				 struct nft_flow_rule *flow,
				 enum flow_cls_command command)
{
	struct netlink_ext_ack extack = {};
	struct flow_cls_offload cls_flow;
	struct nft_base_chain *basechain;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
				   command);

	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
				 &basechain->flow_block.cb_list);
}

static int nft_flow_offload_bind(struct flow_block_offload *bo,
				 struct nft_base_chain *basechain)
{
	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
	return 0;
}

static int nft_flow_offload_unbind(struct flow_block_offload *bo,
				   struct nft_base_chain *basechain)
{
	struct flow_block_cb *block_cb, *next;
	struct flow_cls_offload cls_flow;
	struct netlink_ext_ack extack;
	struct nft_chain *chain;
	struct nft_rule *rule;

	chain = &basechain->chain;
	list_for_each_entry(rule, &chain->rules, list) {
		memset(&extack, 0, sizeof(extack));
		nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
					   &extack, FLOW_CLS_DESTROY);
		nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
	}

	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
		list_del(&block_cb->list);
		flow_block_cb_free(block_cb);
	}

	return 0;
}

static int nft_block_setup(struct nft_base_chain *basechain,
			   struct flow_block_offload *bo,
			   enum flow_block_command cmd)
{
	int err;

	switch (cmd) {
	case FLOW_BLOCK_BIND:
		err = nft_flow_offload_bind(bo, basechain);
		break;
	case FLOW_BLOCK_UNBIND:
		err = nft_flow_offload_unbind(bo, basechain);
		break;
	default:
		WARN_ON_ONCE(1);
		err = -EOPNOTSUPP;
	}

	return err;
}

static void nft_flow_block_offload_init(struct flow_block_offload *bo,
					struct net *net,
					enum flow_block_command cmd,
					struct nft_base_chain *basechain,
					struct netlink_ext_ack *extack)
{
	memset(bo, 0, sizeof(*bo));
	bo->net		= net;
	bo->block	= &basechain->flow_block;
	bo->command	= cmd;
	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	bo->extack	= extack;
	INIT_LIST_HEAD(&bo->cb_list);
}

static int nft_block_offload_cmd(struct nft_base_chain *chain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	if (err < 0)
		return err;

	return nft_block_setup(chain, &bo, cmd);
}

static void nft_indr_block_ing_cmd(struct net_device *dev,
				   struct nft_base_chain *chain,
				   flow_indr_block_bind_cb_t *cb,
				   void *cb_priv,
				   enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;

	if (!chain)
		return;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);

	nft_block_setup(chain, &bo, cmd);
}

static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
				      struct net_device *dev,
				      enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	flow_indr_block_call(dev, &bo, cmd);

	if (list_empty(&bo.cb_list))
		return -EOPNOTSUPP;

	return nft_block_setup(chain, &bo, cmd);
}

#define FLOW_SETUP_BLOCK TC_SETUP_BLOCK

static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	int err;

	if (dev->netdev_ops->ndo_setup_tc)
		err = nft_block_offload_cmd(basechain, dev, cmd);
	else
		err = nft_indr_block_offload_cmd(basechain, dev, cmd);

	return err;
}

static int nft_flow_block_chain(struct nft_base_chain *basechain,
				const struct net_device *this_dev,
				enum flow_block_command cmd)
{
	struct net_device *dev;
	struct nft_hook *hook;
	int err, i = 0;

	list_for_each_entry(hook, &basechain->hook_list, list) {
		dev = hook->ops.dev;
		if (this_dev && this_dev != dev)
			continue;

		err = nft_chain_offload_cmd(basechain, dev, cmd);
		if (err < 0 && cmd == FLOW_BLOCK_BIND) {
			if (!this_dev)
				goto err_flow_block;

			return err;
		}
		i++;
	}

	return 0;

err_flow_block:
	list_for_each_entry(hook, &basechain->hook_list, list) {
		if (i-- <= 0)
			break;

		dev = hook->ops.dev;
		nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	}
	return err;
}

static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
				  enum flow_block_command cmd)
{
	struct nft_base_chain *basechain;
	u8 policy;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	policy = ppolicy ? *ppolicy : basechain->policy;

	/* Only default policy to accept is supported for now. */
	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
		return -EOPNOTSUPP;

	return nft_flow_block_chain(basechain, NULL, cmd);
}

int nft_flow_rule_offload_commit(struct net *net)
{
	struct nft_trans *trans;
	int err = 0;
	u8 policy;

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			if (trans->ctx.flags & NLM_F_REPLACE ||
			    !(trans->ctx.flags & NLM_F_APPEND)) {
				err = -EOPNOTSUPP;
				break;
			}
			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		}

		if (err)
			break;
	}

	list_for_each_entry(trans, &net->nft.commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			nft_flow_rule_destroy(nft_trans_flow_rule(trans));
			break;
		default:
			break;
		}
	}

	return err;
}

static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
{
	struct nft_base_chain *basechain;
	struct net *net = dev_net(dev);
	struct nft_hook *hook, *found;
	const struct nft_table *table;
	struct nft_chain *chain;

	list_for_each_entry(table, &net->nft.tables, list) {
		if (table->family != NFPROTO_NETDEV)
			continue;

		list_for_each_entry(chain, &table->chains, list) {
			if (!nft_is_base_chain(chain) ||
			    !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			found = NULL;
			basechain = nft_base_chain(chain);
			list_for_each_entry(hook, &basechain->hook_list, list) {
				if (hook->ops.dev != dev)
					continue;

				found = hook;
				break;
			}
			if (!found)
				continue;

			return chain;
		}
	}

	return NULL;
}

static void nft_indr_block_cb(struct net_device *dev,
			      flow_indr_block_bind_cb_t *cb, void *cb_priv,
			      enum flow_block_command cmd)
{
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	mutex_lock(&net->nft.commit_mutex);
	chain = __nft_offload_get_chain(dev);
	if (chain) {
		struct nft_base_chain *basechain;

		basechain = nft_base_chain(chain);
		nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
	}
	mutex_unlock(&net->nft.commit_mutex);
}

static int nft_offload_netdev_event(struct notifier_block *this,
				    unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	mutex_lock(&net->nft.commit_mutex);
	chain = __nft_offload_get_chain(dev);
	if (chain)
		nft_flow_block_chain(nft_base_chain(chain), dev,
				     FLOW_BLOCK_UNBIND);

	mutex_unlock(&net->nft.commit_mutex);

	return NOTIFY_DONE;
}

static struct flow_indr_block_ing_entry block_ing_entry = {
	.cb	= nft_indr_block_cb,
	.list	= LIST_HEAD_INIT(block_ing_entry.list),
};

static struct notifier_block nft_offload_netdev_notifier = {
	.notifier_call	= nft_offload_netdev_event,
};

int nft_offload_init(void)
{
	int err;

	err = register_netdevice_notifier(&nft_offload_netdev_notifier);
	if (err < 0)
		return err;

	flow_indr_add_block_ing_cb(&block_ing_entry);

	return 0;
}

void nft_offload_exit(void)
{
	flow_indr_del_block_ing_cb(&block_ing_entry);
	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
}
Commit	Line	Data
c9626a2c PNA	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#include <linux/init.h>
	3	#include <linux/module.h>
	4	#include <linux/netfilter.h>
	5	#include <net/flow_offload.h>
	6	#include <net/netfilter/nf_tables.h>
	7	#include <net/netfilter/nf_tables_offload.h>
	8	#include <net/pkt_cls.h>
	9
	10	static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
	11	{
	12	struct nft_flow_rule *flow;
	13
	14	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	15	if (!flow)
	16	return NULL;
	17
	18	flow->rule = flow_rule_alloc(num_actions);
	19	if (!flow->rule) {
	20	kfree(flow);
	21	return NULL;
	22	}
	23
	24	flow->rule->match.dissector = &flow->match.dissector;
	25	flow->rule->match.mask = &flow->match.mask;
	26	flow->rule->match.key = &flow->match.key;
	27
	28	return flow;
	29	}
	30
be2861dc PNA	31	struct nft_flow_rule nft_flow_rule_create(struct net net,
be2861dc PNA	32	const struct nft_rule *rule)
c9626a2c	33	{
b44492af	34	struct nft_offload_ctx *ctx;
c9626a2c PNA	35	struct nft_flow_rule *flow;
	36	int num_actions = 0, err;
	37	struct nft_expr *expr;
	38
	39	expr = nft_expr_first(rule);
	40	while (expr->ops && expr != nft_expr_last(rule)) {
	41	if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
	42	num_actions++;
	43
	44	expr = nft_expr_next(expr);
	45	}
	46
	47	flow = nft_flow_rule_alloc(num_actions);
	48	if (!flow)
	49	return ERR_PTR(-ENOMEM);
	50
	51	expr = nft_expr_first(rule);
b44492af AB	52
	53	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	54	if (!ctx) {
	55	err = -ENOMEM;
	56	goto err_out;
	57	}
be2861dc	58	ctx->net = net;
b44492af AB	59	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
b44492af AB	60
c9626a2c PNA	61	while (expr->ops && expr != nft_expr_last(rule)) {
	62	if (!expr->ops->offload) {
	63	err = -EOPNOTSUPP;
	64	goto err_out;
	65	}
b44492af	66	err = expr->ops->offload(ctx, flow, expr);
c9626a2c PNA	67	if (err < 0)
	68	goto err_out;
	69
	70	expr = nft_expr_next(expr);
	71	}
b44492af AB	72	flow->proto = ctx->dep.l3num;
b44492af AB	73	kfree(ctx);
c9626a2c PNA	74
	75	return flow;
	76	err_out:
b44492af	77	kfree(ctx);
c9626a2c PNA	78	nft_flow_rule_destroy(flow);
	79
	80	return ERR_PTR(err);
	81	}
	82
	83	void nft_flow_rule_destroy(struct nft_flow_rule *flow)
	84	{
be2861dc PNA	85	struct flow_action_entry *entry;
	86	int i;
	87
	88	flow_action_for_each(i, entry, &flow->rule->action) {
	89	switch (entry->id) {
	90	case FLOW_ACTION_REDIRECT:
	91	case FLOW_ACTION_MIRRED:
	92	dev_put(entry->dev);
	93	break;
	94	default:
	95	break;
	96	}
	97	}
c9626a2c PNA	98	kfree(flow->rule);
	99	kfree(flow);
	100	}
	101
	102	void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
	103	enum nft_offload_dep_type type)
	104	{
	105	ctx->dep.type = type;
	106	}
	107
	108	void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
	109	const void *data, u32 len)
	110	{
	111	switch (ctx->dep.type) {
	112	case NFT_OFFLOAD_DEP_NETWORK:
	113	WARN_ON(len != sizeof(__u16));
	114	memcpy(&ctx->dep.l3num, data, sizeof(__u16));
	115	break;
	116	case NFT_OFFLOAD_DEP_TRANSPORT:
	117	WARN_ON(len != sizeof(__u8));
	118	memcpy(&ctx->dep.protonum, data, sizeof(__u8));
	119	break;
	120	default:
	121	break;
	122	}
	123	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
	124	}
	125
	126	static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
3bc158f8 PNA	127	__be16 proto, int priority,
3bc158f8 PNA	128	struct netlink_ext_ack *extack)
c9626a2c PNA	129	{
c9626a2c PNA	130	common->protocol = proto;
3bc158f8	131	common->prio = priority;
c9626a2c PNA	132	common->extack = extack;
	133	}
	134
b5828880 PNA	135	static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
b5828880 PNA	136	struct list_head *cb_list)
c9626a2c PNA	137	{
	138	struct flow_block_cb *block_cb;
	139	int err;
	140
b5828880	141	list_for_each_entry(block_cb, cb_list, list) {
c9626a2c PNA	142	err = block_cb->cb(type, type_data, block_cb->cb_priv);
	143	if (err < 0)
	144	return err;
	145	}
	146	return 0;
	147	}
	148
3bc158f8 PNA	149	int nft_chain_offload_priority(struct nft_base_chain *basechain)
	150	{
	151	if (basechain->ops.priority <= 0 \|\|
	152	basechain->ops.priority > USHRT_MAX)
	153	return -1;
	154
	155	return 0;
	156	}
	157
c5d27527 PNA	158	static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
	159	const struct nft_base_chain *basechain,
	160	const struct nft_rule *rule,
	161	const struct nft_flow_rule *flow,
be193f5e	162	struct netlink_ext_ack *extack,
c5d27527 PNA	163	enum flow_cls_command command)
c5d27527 PNA	164	{
c5d27527 PNA	165	__be16 proto = ETH_P_ALL;
	166
	167	memset(cls_flow, 0, sizeof(*cls_flow));
	168
	169	if (flow)
	170	proto = flow->proto;
	171
	172	nft_flow_offload_common_init(&cls_flow->common, proto,
be193f5e	173	basechain->ops.priority, extack);
c5d27527 PNA	174	cls_flow->command = command;
	175	cls_flow->cookie = (unsigned long) rule;
	176	if (flow)
	177	cls_flow->rule = flow->rule;
	178	}
	179
e211aab7	180	static int nft_flow_offload_rule(struct nft_chain *chain,
	181	struct nft_rule *rule,
	182	struct nft_flow_rule *flow,
c9626a2c PNA	183	enum flow_cls_command command)
c9626a2c PNA	184	{
be193f5e	185	struct netlink_ext_ack extack = {};
c5d27527	186	struct flow_cls_offload cls_flow;
c9626a2c	187	struct nft_base_chain *basechain;
c9626a2c	188
e211aab7	189	if (!nft_is_base_chain(chain))
c9626a2c PNA	190	return -EOPNOTSUPP;
c9626a2c PNA	191
e211aab7	192	basechain = nft_base_chain(chain);
be193f5e PNA	193	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
be193f5e PNA	194	command);
c9626a2c	195
b5828880 PNA	196	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
b5828880 PNA	197	&basechain->flow_block.cb_list);
c9626a2c PNA	198	}
	199
	200	static int nft_flow_offload_bind(struct flow_block_offload *bo,
	201	struct nft_base_chain *basechain)
	202	{
14bfb13f	203	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
c9626a2c PNA	204	return 0;
	205	}
	206
	207	static int nft_flow_offload_unbind(struct flow_block_offload *bo,
	208	struct nft_base_chain *basechain)
	209	{
	210	struct flow_block_cb block_cb, next;
bbaef955	211	struct flow_cls_offload cls_flow;
be193f5e	212	struct netlink_ext_ack extack;
bbaef955 PNA	213	struct nft_chain *chain;
	214	struct nft_rule *rule;
	215
	216	chain = &basechain->chain;
	217	list_for_each_entry(rule, &chain->rules, list) {
be193f5e	218	memset(&extack, 0, sizeof(extack));
bbaef955	219	nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
be193f5e	220	&extack, FLOW_CLS_DESTROY);
bbaef955 PNA	221	nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
bbaef955 PNA	222	}
c9626a2c PNA	223
	224	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
	225	list_del(&block_cb->list);
	226	flow_block_cb_free(block_cb);
	227	}
	228
	229	return 0;
	230	}
	231
9a32669f	232	static int nft_block_setup(struct nft_base_chain *basechain,
	233	struct flow_block_offload *bo,
	234	enum flow_block_command cmd)
	235	{
	236	int err;
	237
	238	switch (cmd) {
	239	case FLOW_BLOCK_BIND:
	240	err = nft_flow_offload_bind(bo, basechain);
	241	break;
	242	case FLOW_BLOCK_UNBIND:
	243	err = nft_flow_offload_unbind(bo, basechain);
	244	break;
	245	default:
	246	WARN_ON_ONCE(1);
	247	err = -EOPNOTSUPP;
	248	}
	249
	250	return err;
	251	}
	252
75ceaf86 PNA	253	static void nft_flow_block_offload_init(struct flow_block_offload *bo,
	254	struct net *net,
	255	enum flow_block_command cmd,
	256	struct nft_base_chain *basechain,
	257	struct netlink_ext_ack *extack)
	258	{
	259	memset(bo, 0, sizeof(*bo));
	260	bo->net = net;
	261	bo->block = &basechain->flow_block;
	262	bo->command = cmd;
	263	bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	264	bo->extack = extack;
	265	INIT_LIST_HEAD(&bo->cb_list);
	266	}
	267
9a32669f	268	static int nft_block_offload_cmd(struct nft_base_chain *chain,
	269	struct net_device *dev,
	270	enum flow_block_command cmd)
	271	{
	272	struct netlink_ext_ack extack = {};
75ceaf86	273	struct flow_block_offload bo;
9a32669f	274	int err;
9a32669f	275
75ceaf86	276	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	277
	278	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	279	if (err < 0)
	280	return err;
	281
	282	return nft_block_setup(chain, &bo, cmd);
	283	}
	284
	285	static void nft_indr_block_ing_cmd(struct net_device *dev,
	286	struct nft_base_chain *chain,
	287	flow_indr_block_bind_cb_t *cb,
	288	void *cb_priv,
	289	enum flow_block_command cmd)
	290	{
	291	struct netlink_ext_ack extack = {};
75ceaf86	292	struct flow_block_offload bo;
9a32669f	293
	294	if (!chain)
	295	return;
	296
75ceaf86	297	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	298
	299	cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
	300
	301	nft_block_setup(chain, &bo, cmd);
	302	}
	303
	304	static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
	305	struct net_device *dev,
	306	enum flow_block_command cmd)
	307	{
9a32669f	308	struct netlink_ext_ack extack = {};
75ceaf86	309	struct flow_block_offload bo;
9a32669f	310
75ceaf86	311	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
9a32669f	312
	313	flow_indr_block_call(dev, &bo, cmd);
	314
	315	if (list_empty(&bo.cb_list))
	316	return -EOPNOTSUPP;
	317
	318	return nft_block_setup(chain, &bo, cmd);
	319	}
	320
c9626a2c PNA	321	#define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
c9626a2c PNA	322
6df5490f PNA	323	static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
	324	struct net_device *dev,
	325	enum flow_block_command cmd)
	326	{
	327	int err;
	328
	329	if (dev->netdev_ops->ndo_setup_tc)
	330	err = nft_block_offload_cmd(basechain, dev, cmd);
	331	else
	332	err = nft_indr_block_offload_cmd(basechain, dev, cmd);
	333
	334	return err;
	335	}
	336
ead3952e	337	static int nft_flow_block_chain(struct nft_base_chain *basechain,
d54725cd	338	const struct net_device *this_dev,
ead3952e PNA	339	enum flow_block_command cmd)
ead3952e PNA	340	{
d54725cd PNA	341	struct net_device *dev;
d54725cd PNA	342	struct nft_hook *hook;
671312e1	343	int err, i = 0;
d54725cd PNA	344
	345	list_for_each_entry(hook, &basechain->hook_list, list) {
	346	dev = hook->ops.dev;
	347	if (this_dev && this_dev != dev)
	348	continue;
ead3952e	349
6df5490f	350	err = nft_chain_offload_cmd(basechain, dev, cmd);
671312e1 PNA	351	if (err < 0 && cmd == FLOW_BLOCK_BIND) {
	352	if (!this_dev)
	353	goto err_flow_block;
	354
d54725cd	355	return err;
671312e1 PNA	356	}
671312e1 PNA	357	i++;
d54725cd PNA	358	}
	359
	360	return 0;
671312e1 PNA	361
	362	err_flow_block:
	363	list_for_each_entry(hook, &basechain->hook_list, list) {
	364	if (i-- <= 0)
	365	break;
	366
	367	dev = hook->ops.dev;
	368	nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	369	}
	370	return err;
ead3952e PNA	371	}
ead3952e PNA	372
d54725cd	373	static int nft_flow_offload_chain(struct nft_chain chain, u8 ppolicy,
c9626a2c PNA	374	enum flow_block_command cmd)
c9626a2c PNA	375	{
c9626a2c	376	struct nft_base_chain *basechain;
8fc618c5	377	u8 policy;
c9626a2c PNA	378
	379	if (!nft_is_base_chain(chain))
	380	return -EOPNOTSUPP;
	381
	382	basechain = nft_base_chain(chain);
8fc618c5	383	policy = ppolicy ? *ppolicy : basechain->policy;
8fc618c5	384
c9626a2c	385	/* Only default policy to accept is supported for now. */
ff175d0b	386	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
c9626a2c PNA	387	return -EOPNOTSUPP;
c9626a2c PNA	388
d54725cd	389	return nft_flow_block_chain(basechain, NULL, cmd);
c9626a2c PNA	390	}
	391
	392	int nft_flow_rule_offload_commit(struct net *net)
	393	{
	394	struct nft_trans *trans;
	395	int err = 0;
8fc618c5	396	u8 policy;
c9626a2c PNA	397
	398	list_for_each_entry(trans, &net->nft.commit_list, list) {
	399	if (trans->ctx.family != NFPROTO_NETDEV)
	400	continue;
	401
	402	switch (trans->msg_type) {
	403	case NFT_MSG_NEWCHAIN:
88c74984 PNA	404	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) \|\|
88c74984 PNA	405	nft_trans_chain_update(trans))
c9626a2c PNA	406	continue;
c9626a2c PNA	407
8fc618c5	408	policy = nft_trans_chain_policy(trans);
	409	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
	410	FLOW_BLOCK_BIND);
c9626a2c PNA	411	break;
	412	case NFT_MSG_DELCHAIN:
	413	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	414	continue;
	415
8fc618c5	416	policy = nft_trans_chain_policy(trans);
8fc618c5	417	err = nft_flow_offload_chain(trans->ctx.chain, &policy,
085461c8	418	FLOW_BLOCK_UNBIND);
c9626a2c PNA	419	break;
	420	case NFT_MSG_NEWRULE:
	421	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	422	continue;
	423
	424	if (trans->ctx.flags & NLM_F_REPLACE \|\|
23403cd8 PNA	425	!(trans->ctx.flags & NLM_F_APPEND)) {
	426	err = -EOPNOTSUPP;
	427	break;
	428	}
e211aab7	429	err = nft_flow_offload_rule(trans->ctx.chain,
	430	nft_trans_rule(trans),
	431	nft_trans_flow_rule(trans),
	432	FLOW_CLS_REPLACE);
c9626a2c PNA	433	break;
	434	case NFT_MSG_DELRULE:
	435	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	436	continue;
	437
e211aab7	438	err = nft_flow_offload_rule(trans->ctx.chain,
e211aab7	439	nft_trans_rule(trans),
6ca61c7a	440	NULL, FLOW_CLS_DESTROY);
c9626a2c PNA	441	break;
	442	}
	443
	444	if (err)
23403cd8 PNA	445	break;
	446	}
	447
	448	list_for_each_entry(trans, &net->nft.commit_list, list) {
	449	if (trans->ctx.family != NFPROTO_NETDEV)
	450	continue;
	451
	452	switch (trans->msg_type) {
	453	case NFT_MSG_NEWRULE:
	454	if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
	455	continue;
	456
	457	nft_flow_rule_destroy(nft_trans_flow_rule(trans));
	458	break;
	459	default:
	460	break;
	461	}
c9626a2c PNA	462	}
	463
	464	return err;
	465	}
9a32669f	466
504882db	467	static struct nft_chain __nft_offload_get_chain(struct net_device dev)
9a32669f	468	{
504882db	469	struct nft_base_chain *basechain;
9a32669f	470	struct net *net = dev_net(dev);
d54725cd	471	struct nft_hook hook, found;
9a32669f	472	const struct nft_table *table;
504882db	473	struct nft_chain *chain;
9a32669f	474
504882db	475	list_for_each_entry(table, &net->nft.tables, list) {
9a32669f	476	if (table->family != NFPROTO_NETDEV)
	477	continue;
	478
504882db	479	list_for_each_entry(chain, &table->chains, list) {
	480	if (!nft_is_base_chain(chain) \|\|
	481	!(chain->flags & NFT_CHAIN_HW_OFFLOAD))
	482	continue;
	483
d54725cd	484	found = NULL;
504882db	485	basechain = nft_base_chain(chain);
d54725cd PNA	486	list_for_each_entry(hook, &basechain->hook_list, list) {
	487	if (hook->ops.dev != dev)
	488	continue;
	489
	490	found = hook;
	491	break;
	492	}
	493	if (!found)
504882db	494	continue;
	495
	496	return chain;
9a32669f	497	}
9a32669f	498	}
504882db	499
	500	return NULL;
	501	}
	502
	503	static void nft_indr_block_cb(struct net_device *dev,
	504	flow_indr_block_bind_cb_t cb, void cb_priv,
	505	enum flow_block_command cmd)
	506	{
	507	struct net *net = dev_net(dev);
	508	struct nft_chain *chain;
	509
	510	mutex_lock(&net->nft.commit_mutex);
	511	chain = __nft_offload_get_chain(dev);
	512	if (chain) {
	513	struct nft_base_chain *basechain;
	514
	515	basechain = nft_base_chain(chain);
	516	nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
	517	}
	518	mutex_unlock(&net->nft.commit_mutex);
9a32669f	519	}
3474a2c6	520
06d392cb	521	static int nft_offload_netdev_event(struct notifier_block *this,
	522	unsigned long event, void *ptr)
	523	{
	524	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	525	struct net *net = dev_net(dev);
	526	struct nft_chain *chain;
	527
	528	mutex_lock(&net->nft.commit_mutex);
	529	chain = __nft_offload_get_chain(dev);
	530	if (chain)
bbaef955 PNA	531	nft_flow_block_chain(nft_base_chain(chain), dev,
	532	FLOW_BLOCK_UNBIND);
	533
06d392cb	534	mutex_unlock(&net->nft.commit_mutex);
	535
	536	return NOTIFY_DONE;
	537	}
	538
3474a2c6 PNA	539	static struct flow_indr_block_ing_entry block_ing_entry = {
	540	.cb = nft_indr_block_cb,
	541	.list = LIST_HEAD_INIT(block_ing_entry.list),
	542	};
	543
06d392cb	544	static struct notifier_block nft_offload_netdev_notifier = {
	545	.notifier_call = nft_offload_netdev_event,
	546	};
	547
	548	int nft_offload_init(void)
3474a2c6	549	{
06d392cb	550	int err;
	551
	552	err = register_netdevice_notifier(&nft_offload_netdev_notifier);
	553	if (err < 0)
	554	return err;
	555
3474a2c6	556	flow_indr_add_block_ing_cb(&block_ing_entry);
06d392cb	557
06d392cb	558	return 0;
3474a2c6 PNA	559	}
	560
	561	void nft_offload_exit(void)
	562	{
	563	flow_indr_del_block_ing_cb(&block_ing_entry);
06d392cb	564	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
3474a2c6	565	}