]>
Commit | Line | Data |
---|---|---|
c9626a2c PNA |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #include <linux/init.h> | |
3 | #include <linux/module.h> | |
4 | #include <linux/netfilter.h> | |
5 | #include <net/flow_offload.h> | |
6 | #include <net/netfilter/nf_tables.h> | |
7 | #include <net/netfilter/nf_tables_offload.h> | |
8 | #include <net/pkt_cls.h> | |
9 | ||
10 | static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions) | |
11 | { | |
12 | struct nft_flow_rule *flow; | |
13 | ||
14 | flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL); | |
15 | if (!flow) | |
16 | return NULL; | |
17 | ||
18 | flow->rule = flow_rule_alloc(num_actions); | |
19 | if (!flow->rule) { | |
20 | kfree(flow); | |
21 | return NULL; | |
22 | } | |
23 | ||
24 | flow->rule->match.dissector = &flow->match.dissector; | |
25 | flow->rule->match.mask = &flow->match.mask; | |
26 | flow->rule->match.key = &flow->match.key; | |
27 | ||
28 | return flow; | |
29 | } | |
30 | ||
be2861dc PNA |
31 | struct nft_flow_rule *nft_flow_rule_create(struct net *net, |
32 | const struct nft_rule *rule) | |
c9626a2c | 33 | { |
b44492af | 34 | struct nft_offload_ctx *ctx; |
c9626a2c PNA |
35 | struct nft_flow_rule *flow; |
36 | int num_actions = 0, err; | |
37 | struct nft_expr *expr; | |
38 | ||
39 | expr = nft_expr_first(rule); | |
40 | while (expr->ops && expr != nft_expr_last(rule)) { | |
41 | if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION) | |
42 | num_actions++; | |
43 | ||
44 | expr = nft_expr_next(expr); | |
45 | } | |
46 | ||
47 | flow = nft_flow_rule_alloc(num_actions); | |
48 | if (!flow) | |
49 | return ERR_PTR(-ENOMEM); | |
50 | ||
51 | expr = nft_expr_first(rule); | |
b44492af AB |
52 | |
53 | ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL); | |
54 | if (!ctx) { | |
55 | err = -ENOMEM; | |
56 | goto err_out; | |
57 | } | |
be2861dc | 58 | ctx->net = net; |
b44492af AB |
59 | ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; |
60 | ||
c9626a2c PNA |
61 | while (expr->ops && expr != nft_expr_last(rule)) { |
62 | if (!expr->ops->offload) { | |
63 | err = -EOPNOTSUPP; | |
64 | goto err_out; | |
65 | } | |
b44492af | 66 | err = expr->ops->offload(ctx, flow, expr); |
c9626a2c PNA |
67 | if (err < 0) |
68 | goto err_out; | |
69 | ||
70 | expr = nft_expr_next(expr); | |
71 | } | |
b44492af AB |
72 | flow->proto = ctx->dep.l3num; |
73 | kfree(ctx); | |
c9626a2c PNA |
74 | |
75 | return flow; | |
76 | err_out: | |
b44492af | 77 | kfree(ctx); |
c9626a2c PNA |
78 | nft_flow_rule_destroy(flow); |
79 | ||
80 | return ERR_PTR(err); | |
81 | } | |
82 | ||
83 | void nft_flow_rule_destroy(struct nft_flow_rule *flow) | |
84 | { | |
be2861dc PNA |
85 | struct flow_action_entry *entry; |
86 | int i; | |
87 | ||
88 | flow_action_for_each(i, entry, &flow->rule->action) { | |
89 | switch (entry->id) { | |
90 | case FLOW_ACTION_REDIRECT: | |
91 | case FLOW_ACTION_MIRRED: | |
92 | dev_put(entry->dev); | |
93 | break; | |
94 | default: | |
95 | break; | |
96 | } | |
97 | } | |
c9626a2c PNA |
98 | kfree(flow->rule); |
99 | kfree(flow); | |
100 | } | |
101 | ||
102 | void nft_offload_set_dependency(struct nft_offload_ctx *ctx, | |
103 | enum nft_offload_dep_type type) | |
104 | { | |
105 | ctx->dep.type = type; | |
106 | } | |
107 | ||
108 | void nft_offload_update_dependency(struct nft_offload_ctx *ctx, | |
109 | const void *data, u32 len) | |
110 | { | |
111 | switch (ctx->dep.type) { | |
112 | case NFT_OFFLOAD_DEP_NETWORK: | |
113 | WARN_ON(len != sizeof(__u16)); | |
114 | memcpy(&ctx->dep.l3num, data, sizeof(__u16)); | |
115 | break; | |
116 | case NFT_OFFLOAD_DEP_TRANSPORT: | |
117 | WARN_ON(len != sizeof(__u8)); | |
118 | memcpy(&ctx->dep.protonum, data, sizeof(__u8)); | |
119 | break; | |
120 | default: | |
121 | break; | |
122 | } | |
123 | ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; | |
124 | } | |
125 | ||
126 | static void nft_flow_offload_common_init(struct flow_cls_common_offload *common, | |
3bc158f8 PNA |
127 | __be16 proto, int priority, |
128 | struct netlink_ext_ack *extack) | |
c9626a2c PNA |
129 | { |
130 | common->protocol = proto; | |
3bc158f8 | 131 | common->prio = priority; |
c9626a2c PNA |
132 | common->extack = extack; |
133 | } | |
134 | ||
135 | static int nft_setup_cb_call(struct nft_base_chain *basechain, | |
136 | enum tc_setup_type type, void *type_data) | |
137 | { | |
138 | struct flow_block_cb *block_cb; | |
139 | int err; | |
140 | ||
14bfb13f | 141 | list_for_each_entry(block_cb, &basechain->flow_block.cb_list, list) { |
c9626a2c PNA |
142 | err = block_cb->cb(type, type_data, block_cb->cb_priv); |
143 | if (err < 0) | |
144 | return err; | |
145 | } | |
146 | return 0; | |
147 | } | |
148 | ||
3bc158f8 PNA |
149 | int nft_chain_offload_priority(struct nft_base_chain *basechain) |
150 | { | |
151 | if (basechain->ops.priority <= 0 || | |
152 | basechain->ops.priority > USHRT_MAX) | |
153 | return -1; | |
154 | ||
155 | return 0; | |
156 | } | |
157 | ||
e211aab7 | 158 | static int nft_flow_offload_rule(struct nft_chain *chain, |
159 | struct nft_rule *rule, | |
160 | struct nft_flow_rule *flow, | |
c9626a2c PNA |
161 | enum flow_cls_command command) |
162 | { | |
c9626a2c PNA |
163 | struct flow_cls_offload cls_flow = {}; |
164 | struct nft_base_chain *basechain; | |
165 | struct netlink_ext_ack extack; | |
166 | __be16 proto = ETH_P_ALL; | |
167 | ||
e211aab7 | 168 | if (!nft_is_base_chain(chain)) |
c9626a2c PNA |
169 | return -EOPNOTSUPP; |
170 | ||
e211aab7 | 171 | basechain = nft_base_chain(chain); |
c9626a2c PNA |
172 | |
173 | if (flow) | |
174 | proto = flow->proto; | |
175 | ||
3bc158f8 PNA |
176 | nft_flow_offload_common_init(&cls_flow.common, proto, |
177 | basechain->ops.priority, &extack); | |
c9626a2c PNA |
178 | cls_flow.command = command; |
179 | cls_flow.cookie = (unsigned long) rule; | |
180 | if (flow) | |
181 | cls_flow.rule = flow->rule; | |
182 | ||
183 | return nft_setup_cb_call(basechain, TC_SETUP_CLSFLOWER, &cls_flow); | |
184 | } | |
185 | ||
186 | static int nft_flow_offload_bind(struct flow_block_offload *bo, | |
187 | struct nft_base_chain *basechain) | |
188 | { | |
14bfb13f | 189 | list_splice(&bo->cb_list, &basechain->flow_block.cb_list); |
c9626a2c PNA |
190 | return 0; |
191 | } | |
192 | ||
193 | static int nft_flow_offload_unbind(struct flow_block_offload *bo, | |
194 | struct nft_base_chain *basechain) | |
195 | { | |
196 | struct flow_block_cb *block_cb, *next; | |
197 | ||
198 | list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { | |
199 | list_del(&block_cb->list); | |
200 | flow_block_cb_free(block_cb); | |
201 | } | |
202 | ||
203 | return 0; | |
204 | } | |
205 | ||
9a32669f | 206 | static int nft_block_setup(struct nft_base_chain *basechain, |
207 | struct flow_block_offload *bo, | |
208 | enum flow_block_command cmd) | |
209 | { | |
210 | int err; | |
211 | ||
212 | switch (cmd) { | |
213 | case FLOW_BLOCK_BIND: | |
214 | err = nft_flow_offload_bind(bo, basechain); | |
215 | break; | |
216 | case FLOW_BLOCK_UNBIND: | |
217 | err = nft_flow_offload_unbind(bo, basechain); | |
218 | break; | |
219 | default: | |
220 | WARN_ON_ONCE(1); | |
221 | err = -EOPNOTSUPP; | |
222 | } | |
223 | ||
224 | return err; | |
225 | } | |
226 | ||
227 | static int nft_block_offload_cmd(struct nft_base_chain *chain, | |
228 | struct net_device *dev, | |
229 | enum flow_block_command cmd) | |
230 | { | |
231 | struct netlink_ext_ack extack = {}; | |
232 | struct flow_block_offload bo = {}; | |
233 | int err; | |
234 | ||
235 | bo.net = dev_net(dev); | |
236 | bo.block = &chain->flow_block; | |
237 | bo.command = cmd; | |
238 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
239 | bo.extack = &extack; | |
240 | INIT_LIST_HEAD(&bo.cb_list); | |
241 | ||
242 | err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); | |
243 | if (err < 0) | |
244 | return err; | |
245 | ||
246 | return nft_block_setup(chain, &bo, cmd); | |
247 | } | |
248 | ||
249 | static void nft_indr_block_ing_cmd(struct net_device *dev, | |
250 | struct nft_base_chain *chain, | |
251 | flow_indr_block_bind_cb_t *cb, | |
252 | void *cb_priv, | |
253 | enum flow_block_command cmd) | |
254 | { | |
255 | struct netlink_ext_ack extack = {}; | |
256 | struct flow_block_offload bo = {}; | |
257 | ||
258 | if (!chain) | |
259 | return; | |
260 | ||
261 | bo.net = dev_net(dev); | |
262 | bo.block = &chain->flow_block; | |
263 | bo.command = cmd; | |
264 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
265 | bo.extack = &extack; | |
266 | INIT_LIST_HEAD(&bo.cb_list); | |
267 | ||
268 | cb(dev, cb_priv, TC_SETUP_BLOCK, &bo); | |
269 | ||
270 | nft_block_setup(chain, &bo, cmd); | |
271 | } | |
272 | ||
273 | static int nft_indr_block_offload_cmd(struct nft_base_chain *chain, | |
274 | struct net_device *dev, | |
275 | enum flow_block_command cmd) | |
276 | { | |
277 | struct flow_block_offload bo = {}; | |
278 | struct netlink_ext_ack extack = {}; | |
279 | ||
280 | bo.net = dev_net(dev); | |
281 | bo.block = &chain->flow_block; | |
282 | bo.command = cmd; | |
283 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
284 | bo.extack = &extack; | |
285 | INIT_LIST_HEAD(&bo.cb_list); | |
286 | ||
287 | flow_indr_block_call(dev, &bo, cmd); | |
288 | ||
289 | if (list_empty(&bo.cb_list)) | |
290 | return -EOPNOTSUPP; | |
291 | ||
292 | return nft_block_setup(chain, &bo, cmd); | |
293 | } | |
294 | ||
c9626a2c PNA |
295 | #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK |
296 | ||
8fc618c5 | 297 | static int nft_flow_offload_chain(struct nft_chain *chain, |
298 | u8 *ppolicy, | |
c9626a2c PNA |
299 | enum flow_block_command cmd) |
300 | { | |
c9626a2c PNA |
301 | struct nft_base_chain *basechain; |
302 | struct net_device *dev; | |
8fc618c5 | 303 | u8 policy; |
c9626a2c PNA |
304 | |
305 | if (!nft_is_base_chain(chain)) | |
306 | return -EOPNOTSUPP; | |
307 | ||
308 | basechain = nft_base_chain(chain); | |
309 | dev = basechain->ops.dev; | |
9a32669f | 310 | if (!dev) |
c9626a2c PNA |
311 | return -EOPNOTSUPP; |
312 | ||
8fc618c5 | 313 | policy = ppolicy ? *ppolicy : basechain->policy; |
314 | ||
c9626a2c | 315 | /* Only default policy to accept is supported for now. */ |
8fc618c5 | 316 | if (cmd == FLOW_BLOCK_BIND && policy != -1 && policy != NF_ACCEPT) |
c9626a2c PNA |
317 | return -EOPNOTSUPP; |
318 | ||
9a32669f | 319 | if (dev->netdev_ops->ndo_setup_tc) |
320 | return nft_block_offload_cmd(basechain, dev, cmd); | |
321 | else | |
322 | return nft_indr_block_offload_cmd(basechain, dev, cmd); | |
c9626a2c PNA |
323 | } |
324 | ||
325 | int nft_flow_rule_offload_commit(struct net *net) | |
326 | { | |
327 | struct nft_trans *trans; | |
328 | int err = 0; | |
8fc618c5 | 329 | u8 policy; |
c9626a2c PNA |
330 | |
331 | list_for_each_entry(trans, &net->nft.commit_list, list) { | |
332 | if (trans->ctx.family != NFPROTO_NETDEV) | |
333 | continue; | |
334 | ||
335 | switch (trans->msg_type) { | |
336 | case NFT_MSG_NEWCHAIN: | |
337 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
338 | continue; | |
339 | ||
8fc618c5 | 340 | policy = nft_trans_chain_policy(trans); |
341 | err = nft_flow_offload_chain(trans->ctx.chain, &policy, | |
342 | FLOW_BLOCK_BIND); | |
c9626a2c PNA |
343 | break; |
344 | case NFT_MSG_DELCHAIN: | |
345 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
346 | continue; | |
347 | ||
8fc618c5 | 348 | policy = nft_trans_chain_policy(trans); |
349 | err = nft_flow_offload_chain(trans->ctx.chain, &policy, | |
350 | FLOW_BLOCK_BIND); | |
c9626a2c PNA |
351 | break; |
352 | case NFT_MSG_NEWRULE: | |
353 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
354 | continue; | |
355 | ||
356 | if (trans->ctx.flags & NLM_F_REPLACE || | |
357 | !(trans->ctx.flags & NLM_F_APPEND)) | |
358 | return -EOPNOTSUPP; | |
359 | ||
e211aab7 | 360 | err = nft_flow_offload_rule(trans->ctx.chain, |
361 | nft_trans_rule(trans), | |
362 | nft_trans_flow_rule(trans), | |
363 | FLOW_CLS_REPLACE); | |
c9626a2c PNA |
364 | nft_flow_rule_destroy(nft_trans_flow_rule(trans)); |
365 | break; | |
366 | case NFT_MSG_DELRULE: | |
367 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
368 | continue; | |
369 | ||
e211aab7 | 370 | err = nft_flow_offload_rule(trans->ctx.chain, |
371 | nft_trans_rule(trans), | |
372 | nft_trans_flow_rule(trans), | |
373 | FLOW_CLS_DESTROY); | |
c9626a2c PNA |
374 | break; |
375 | } | |
376 | ||
377 | if (err) | |
378 | return err; | |
379 | } | |
380 | ||
381 | return err; | |
382 | } | |
9a32669f | 383 | |
504882db | 384 | static struct nft_chain *__nft_offload_get_chain(struct net_device *dev) |
9a32669f | 385 | { |
504882db | 386 | struct nft_base_chain *basechain; |
9a32669f | 387 | struct net *net = dev_net(dev); |
388 | const struct nft_table *table; | |
504882db | 389 | struct nft_chain *chain; |
9a32669f | 390 | |
504882db | 391 | list_for_each_entry(table, &net->nft.tables, list) { |
9a32669f | 392 | if (table->family != NFPROTO_NETDEV) |
393 | continue; | |
394 | ||
504882db | 395 | list_for_each_entry(chain, &table->chains, list) { |
396 | if (!nft_is_base_chain(chain) || | |
397 | !(chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
398 | continue; | |
399 | ||
400 | basechain = nft_base_chain(chain); | |
401 | if (strncmp(basechain->dev_name, dev->name, IFNAMSIZ)) | |
402 | continue; | |
403 | ||
404 | return chain; | |
9a32669f | 405 | } |
406 | } | |
504882db | 407 | |
408 | return NULL; | |
409 | } | |
410 | ||
411 | static void nft_indr_block_cb(struct net_device *dev, | |
412 | flow_indr_block_bind_cb_t *cb, void *cb_priv, | |
413 | enum flow_block_command cmd) | |
414 | { | |
415 | struct net *net = dev_net(dev); | |
416 | struct nft_chain *chain; | |
417 | ||
418 | mutex_lock(&net->nft.commit_mutex); | |
419 | chain = __nft_offload_get_chain(dev); | |
420 | if (chain) { | |
421 | struct nft_base_chain *basechain; | |
422 | ||
423 | basechain = nft_base_chain(chain); | |
424 | nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd); | |
425 | } | |
426 | mutex_unlock(&net->nft.commit_mutex); | |
9a32669f | 427 | } |
3474a2c6 | 428 | |
06d392cb | 429 | static void nft_offload_chain_clean(struct nft_chain *chain) |
430 | { | |
431 | struct nft_rule *rule; | |
432 | ||
433 | list_for_each_entry(rule, &chain->rules, list) { | |
434 | nft_flow_offload_rule(chain, rule, | |
435 | NULL, FLOW_CLS_DESTROY); | |
436 | } | |
437 | ||
438 | nft_flow_offload_chain(chain, NULL, FLOW_BLOCK_UNBIND); | |
439 | } | |
440 | ||
441 | static int nft_offload_netdev_event(struct notifier_block *this, | |
442 | unsigned long event, void *ptr) | |
443 | { | |
444 | struct net_device *dev = netdev_notifier_info_to_dev(ptr); | |
445 | struct net *net = dev_net(dev); | |
446 | struct nft_chain *chain; | |
447 | ||
448 | mutex_lock(&net->nft.commit_mutex); | |
449 | chain = __nft_offload_get_chain(dev); | |
450 | if (chain) | |
451 | nft_offload_chain_clean(chain); | |
452 | mutex_unlock(&net->nft.commit_mutex); | |
453 | ||
454 | return NOTIFY_DONE; | |
455 | } | |
456 | ||
3474a2c6 PNA |
457 | static struct flow_indr_block_ing_entry block_ing_entry = { |
458 | .cb = nft_indr_block_cb, | |
459 | .list = LIST_HEAD_INIT(block_ing_entry.list), | |
460 | }; | |
461 | ||
06d392cb | 462 | static struct notifier_block nft_offload_netdev_notifier = { |
463 | .notifier_call = nft_offload_netdev_event, | |
464 | }; | |
465 | ||
466 | int nft_offload_init(void) | |
3474a2c6 | 467 | { |
06d392cb | 468 | int err; |
469 | ||
470 | err = register_netdevice_notifier(&nft_offload_netdev_notifier); | |
471 | if (err < 0) | |
472 | return err; | |
473 | ||
3474a2c6 | 474 | flow_indr_add_block_ing_cb(&block_ing_entry); |
06d392cb | 475 | |
476 | return 0; | |
3474a2c6 PNA |
477 | } |
478 | ||
479 | void nft_offload_exit(void) | |
480 | { | |
481 | flow_indr_del_block_ing_cb(&block_ing_entry); | |
06d392cb | 482 | unregister_netdevice_notifier(&nft_offload_netdev_notifier); |
3474a2c6 | 483 | } |