]>
Commit | Line | Data |
---|---|---|
c9626a2c PNA |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #include <linux/init.h> | |
3 | #include <linux/module.h> | |
4 | #include <linux/netfilter.h> | |
5 | #include <net/flow_offload.h> | |
6 | #include <net/netfilter/nf_tables.h> | |
7 | #include <net/netfilter/nf_tables_offload.h> | |
8 | #include <net/pkt_cls.h> | |
9 | ||
10 | static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions) | |
11 | { | |
12 | struct nft_flow_rule *flow; | |
13 | ||
14 | flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL); | |
15 | if (!flow) | |
16 | return NULL; | |
17 | ||
18 | flow->rule = flow_rule_alloc(num_actions); | |
19 | if (!flow->rule) { | |
20 | kfree(flow); | |
21 | return NULL; | |
22 | } | |
23 | ||
24 | flow->rule->match.dissector = &flow->match.dissector; | |
25 | flow->rule->match.mask = &flow->match.mask; | |
26 | flow->rule->match.key = &flow->match.key; | |
27 | ||
28 | return flow; | |
29 | } | |
30 | ||
be2861dc PNA |
31 | struct nft_flow_rule *nft_flow_rule_create(struct net *net, |
32 | const struct nft_rule *rule) | |
c9626a2c | 33 | { |
b44492af | 34 | struct nft_offload_ctx *ctx; |
c9626a2c PNA |
35 | struct nft_flow_rule *flow; |
36 | int num_actions = 0, err; | |
37 | struct nft_expr *expr; | |
38 | ||
39 | expr = nft_expr_first(rule); | |
40 | while (expr->ops && expr != nft_expr_last(rule)) { | |
41 | if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION) | |
42 | num_actions++; | |
43 | ||
44 | expr = nft_expr_next(expr); | |
45 | } | |
46 | ||
47 | flow = nft_flow_rule_alloc(num_actions); | |
48 | if (!flow) | |
49 | return ERR_PTR(-ENOMEM); | |
50 | ||
51 | expr = nft_expr_first(rule); | |
b44492af AB |
52 | |
53 | ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL); | |
54 | if (!ctx) { | |
55 | err = -ENOMEM; | |
56 | goto err_out; | |
57 | } | |
be2861dc | 58 | ctx->net = net; |
b44492af AB |
59 | ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; |
60 | ||
c9626a2c PNA |
61 | while (expr->ops && expr != nft_expr_last(rule)) { |
62 | if (!expr->ops->offload) { | |
63 | err = -EOPNOTSUPP; | |
64 | goto err_out; | |
65 | } | |
b44492af | 66 | err = expr->ops->offload(ctx, flow, expr); |
c9626a2c PNA |
67 | if (err < 0) |
68 | goto err_out; | |
69 | ||
70 | expr = nft_expr_next(expr); | |
71 | } | |
b44492af AB |
72 | flow->proto = ctx->dep.l3num; |
73 | kfree(ctx); | |
c9626a2c PNA |
74 | |
75 | return flow; | |
76 | err_out: | |
b44492af | 77 | kfree(ctx); |
c9626a2c PNA |
78 | nft_flow_rule_destroy(flow); |
79 | ||
80 | return ERR_PTR(err); | |
81 | } | |
82 | ||
83 | void nft_flow_rule_destroy(struct nft_flow_rule *flow) | |
84 | { | |
be2861dc PNA |
85 | struct flow_action_entry *entry; |
86 | int i; | |
87 | ||
88 | flow_action_for_each(i, entry, &flow->rule->action) { | |
89 | switch (entry->id) { | |
90 | case FLOW_ACTION_REDIRECT: | |
91 | case FLOW_ACTION_MIRRED: | |
92 | dev_put(entry->dev); | |
93 | break; | |
94 | default: | |
95 | break; | |
96 | } | |
97 | } | |
c9626a2c PNA |
98 | kfree(flow->rule); |
99 | kfree(flow); | |
100 | } | |
101 | ||
102 | void nft_offload_set_dependency(struct nft_offload_ctx *ctx, | |
103 | enum nft_offload_dep_type type) | |
104 | { | |
105 | ctx->dep.type = type; | |
106 | } | |
107 | ||
108 | void nft_offload_update_dependency(struct nft_offload_ctx *ctx, | |
109 | const void *data, u32 len) | |
110 | { | |
111 | switch (ctx->dep.type) { | |
112 | case NFT_OFFLOAD_DEP_NETWORK: | |
113 | WARN_ON(len != sizeof(__u16)); | |
114 | memcpy(&ctx->dep.l3num, data, sizeof(__u16)); | |
115 | break; | |
116 | case NFT_OFFLOAD_DEP_TRANSPORT: | |
117 | WARN_ON(len != sizeof(__u8)); | |
118 | memcpy(&ctx->dep.protonum, data, sizeof(__u8)); | |
119 | break; | |
120 | default: | |
121 | break; | |
122 | } | |
123 | ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; | |
124 | } | |
125 | ||
126 | static void nft_flow_offload_common_init(struct flow_cls_common_offload *common, | |
3bc158f8 PNA |
127 | __be16 proto, int priority, |
128 | struct netlink_ext_ack *extack) | |
c9626a2c PNA |
129 | { |
130 | common->protocol = proto; | |
3bc158f8 | 131 | common->prio = priority; |
c9626a2c PNA |
132 | common->extack = extack; |
133 | } | |
134 | ||
135 | static int nft_setup_cb_call(struct nft_base_chain *basechain, | |
136 | enum tc_setup_type type, void *type_data) | |
137 | { | |
138 | struct flow_block_cb *block_cb; | |
139 | int err; | |
140 | ||
14bfb13f | 141 | list_for_each_entry(block_cb, &basechain->flow_block.cb_list, list) { |
c9626a2c PNA |
142 | err = block_cb->cb(type, type_data, block_cb->cb_priv); |
143 | if (err < 0) | |
144 | return err; | |
145 | } | |
146 | return 0; | |
147 | } | |
148 | ||
3bc158f8 PNA |
149 | int nft_chain_offload_priority(struct nft_base_chain *basechain) |
150 | { | |
151 | if (basechain->ops.priority <= 0 || | |
152 | basechain->ops.priority > USHRT_MAX) | |
153 | return -1; | |
154 | ||
155 | return 0; | |
156 | } | |
157 | ||
e211aab7 | 158 | static int nft_flow_offload_rule(struct nft_chain *chain, |
159 | struct nft_rule *rule, | |
160 | struct nft_flow_rule *flow, | |
c9626a2c PNA |
161 | enum flow_cls_command command) |
162 | { | |
c9626a2c PNA |
163 | struct flow_cls_offload cls_flow = {}; |
164 | struct nft_base_chain *basechain; | |
165 | struct netlink_ext_ack extack; | |
166 | __be16 proto = ETH_P_ALL; | |
167 | ||
e211aab7 | 168 | if (!nft_is_base_chain(chain)) |
c9626a2c PNA |
169 | return -EOPNOTSUPP; |
170 | ||
e211aab7 | 171 | basechain = nft_base_chain(chain); |
c9626a2c PNA |
172 | |
173 | if (flow) | |
174 | proto = flow->proto; | |
175 | ||
3bc158f8 PNA |
176 | nft_flow_offload_common_init(&cls_flow.common, proto, |
177 | basechain->ops.priority, &extack); | |
c9626a2c PNA |
178 | cls_flow.command = command; |
179 | cls_flow.cookie = (unsigned long) rule; | |
180 | if (flow) | |
181 | cls_flow.rule = flow->rule; | |
182 | ||
183 | return nft_setup_cb_call(basechain, TC_SETUP_CLSFLOWER, &cls_flow); | |
184 | } | |
185 | ||
186 | static int nft_flow_offload_bind(struct flow_block_offload *bo, | |
187 | struct nft_base_chain *basechain) | |
188 | { | |
14bfb13f | 189 | list_splice(&bo->cb_list, &basechain->flow_block.cb_list); |
c9626a2c PNA |
190 | return 0; |
191 | } | |
192 | ||
193 | static int nft_flow_offload_unbind(struct flow_block_offload *bo, | |
194 | struct nft_base_chain *basechain) | |
195 | { | |
196 | struct flow_block_cb *block_cb, *next; | |
197 | ||
198 | list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { | |
199 | list_del(&block_cb->list); | |
200 | flow_block_cb_free(block_cb); | |
201 | } | |
202 | ||
203 | return 0; | |
204 | } | |
205 | ||
9a32669f | 206 | static int nft_block_setup(struct nft_base_chain *basechain, |
207 | struct flow_block_offload *bo, | |
208 | enum flow_block_command cmd) | |
209 | { | |
210 | int err; | |
211 | ||
212 | switch (cmd) { | |
213 | case FLOW_BLOCK_BIND: | |
214 | err = nft_flow_offload_bind(bo, basechain); | |
215 | break; | |
216 | case FLOW_BLOCK_UNBIND: | |
217 | err = nft_flow_offload_unbind(bo, basechain); | |
218 | break; | |
219 | default: | |
220 | WARN_ON_ONCE(1); | |
221 | err = -EOPNOTSUPP; | |
222 | } | |
223 | ||
224 | return err; | |
225 | } | |
226 | ||
227 | static int nft_block_offload_cmd(struct nft_base_chain *chain, | |
228 | struct net_device *dev, | |
229 | enum flow_block_command cmd) | |
230 | { | |
231 | struct netlink_ext_ack extack = {}; | |
232 | struct flow_block_offload bo = {}; | |
233 | int err; | |
234 | ||
235 | bo.net = dev_net(dev); | |
236 | bo.block = &chain->flow_block; | |
237 | bo.command = cmd; | |
238 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
239 | bo.extack = &extack; | |
240 | INIT_LIST_HEAD(&bo.cb_list); | |
241 | ||
242 | err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); | |
243 | if (err < 0) | |
244 | return err; | |
245 | ||
246 | return nft_block_setup(chain, &bo, cmd); | |
247 | } | |
248 | ||
249 | static void nft_indr_block_ing_cmd(struct net_device *dev, | |
250 | struct nft_base_chain *chain, | |
251 | flow_indr_block_bind_cb_t *cb, | |
252 | void *cb_priv, | |
253 | enum flow_block_command cmd) | |
254 | { | |
255 | struct netlink_ext_ack extack = {}; | |
256 | struct flow_block_offload bo = {}; | |
257 | ||
258 | if (!chain) | |
259 | return; | |
260 | ||
261 | bo.net = dev_net(dev); | |
262 | bo.block = &chain->flow_block; | |
263 | bo.command = cmd; | |
264 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
265 | bo.extack = &extack; | |
266 | INIT_LIST_HEAD(&bo.cb_list); | |
267 | ||
268 | cb(dev, cb_priv, TC_SETUP_BLOCK, &bo); | |
269 | ||
270 | nft_block_setup(chain, &bo, cmd); | |
271 | } | |
272 | ||
273 | static int nft_indr_block_offload_cmd(struct nft_base_chain *chain, | |
274 | struct net_device *dev, | |
275 | enum flow_block_command cmd) | |
276 | { | |
277 | struct flow_block_offload bo = {}; | |
278 | struct netlink_ext_ack extack = {}; | |
279 | ||
280 | bo.net = dev_net(dev); | |
281 | bo.block = &chain->flow_block; | |
282 | bo.command = cmd; | |
283 | bo.binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; | |
284 | bo.extack = &extack; | |
285 | INIT_LIST_HEAD(&bo.cb_list); | |
286 | ||
287 | flow_indr_block_call(dev, &bo, cmd); | |
288 | ||
289 | if (list_empty(&bo.cb_list)) | |
290 | return -EOPNOTSUPP; | |
291 | ||
292 | return nft_block_setup(chain, &bo, cmd); | |
293 | } | |
294 | ||
c9626a2c PNA |
295 | #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK |
296 | ||
ead3952e PNA |
297 | static int nft_flow_block_chain(struct nft_base_chain *basechain, |
298 | struct net_device *dev, | |
299 | enum flow_block_command cmd) | |
300 | { | |
301 | if (dev->netdev_ops->ndo_setup_tc) | |
302 | return nft_block_offload_cmd(basechain, dev, cmd); | |
303 | ||
304 | return nft_indr_block_offload_cmd(basechain, dev, cmd); | |
305 | } | |
306 | ||
8fc618c5 | 307 | static int nft_flow_offload_chain(struct nft_chain *chain, |
308 | u8 *ppolicy, | |
c9626a2c PNA |
309 | enum flow_block_command cmd) |
310 | { | |
c9626a2c PNA |
311 | struct nft_base_chain *basechain; |
312 | struct net_device *dev; | |
8fc618c5 | 313 | u8 policy; |
c9626a2c PNA |
314 | |
315 | if (!nft_is_base_chain(chain)) | |
316 | return -EOPNOTSUPP; | |
317 | ||
318 | basechain = nft_base_chain(chain); | |
319 | dev = basechain->ops.dev; | |
9a32669f | 320 | if (!dev) |
c9626a2c PNA |
321 | return -EOPNOTSUPP; |
322 | ||
8fc618c5 | 323 | policy = ppolicy ? *ppolicy : basechain->policy; |
324 | ||
c9626a2c | 325 | /* Only default policy to accept is supported for now. */ |
ff175d0b | 326 | if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP) |
c9626a2c PNA |
327 | return -EOPNOTSUPP; |
328 | ||
ead3952e | 329 | return nft_flow_block_chain(basechain, dev, cmd); |
c9626a2c PNA |
330 | } |
331 | ||
332 | int nft_flow_rule_offload_commit(struct net *net) | |
333 | { | |
334 | struct nft_trans *trans; | |
335 | int err = 0; | |
8fc618c5 | 336 | u8 policy; |
c9626a2c PNA |
337 | |
338 | list_for_each_entry(trans, &net->nft.commit_list, list) { | |
339 | if (trans->ctx.family != NFPROTO_NETDEV) | |
340 | continue; | |
341 | ||
342 | switch (trans->msg_type) { | |
343 | case NFT_MSG_NEWCHAIN: | |
344 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
345 | continue; | |
346 | ||
8fc618c5 | 347 | policy = nft_trans_chain_policy(trans); |
348 | err = nft_flow_offload_chain(trans->ctx.chain, &policy, | |
349 | FLOW_BLOCK_BIND); | |
c9626a2c PNA |
350 | break; |
351 | case NFT_MSG_DELCHAIN: | |
352 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
353 | continue; | |
354 | ||
8fc618c5 | 355 | policy = nft_trans_chain_policy(trans); |
356 | err = nft_flow_offload_chain(trans->ctx.chain, &policy, | |
357 | FLOW_BLOCK_BIND); | |
c9626a2c PNA |
358 | break; |
359 | case NFT_MSG_NEWRULE: | |
360 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
361 | continue; | |
362 | ||
363 | if (trans->ctx.flags & NLM_F_REPLACE || | |
364 | !(trans->ctx.flags & NLM_F_APPEND)) | |
365 | return -EOPNOTSUPP; | |
366 | ||
e211aab7 | 367 | err = nft_flow_offload_rule(trans->ctx.chain, |
368 | nft_trans_rule(trans), | |
369 | nft_trans_flow_rule(trans), | |
370 | FLOW_CLS_REPLACE); | |
c9626a2c PNA |
371 | nft_flow_rule_destroy(nft_trans_flow_rule(trans)); |
372 | break; | |
373 | case NFT_MSG_DELRULE: | |
374 | if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
375 | continue; | |
376 | ||
e211aab7 | 377 | err = nft_flow_offload_rule(trans->ctx.chain, |
378 | nft_trans_rule(trans), | |
379 | nft_trans_flow_rule(trans), | |
380 | FLOW_CLS_DESTROY); | |
c9626a2c PNA |
381 | break; |
382 | } | |
383 | ||
384 | if (err) | |
385 | return err; | |
386 | } | |
387 | ||
388 | return err; | |
389 | } | |
9a32669f | 390 | |
504882db | 391 | static struct nft_chain *__nft_offload_get_chain(struct net_device *dev) |
9a32669f | 392 | { |
504882db | 393 | struct nft_base_chain *basechain; |
9a32669f | 394 | struct net *net = dev_net(dev); |
395 | const struct nft_table *table; | |
504882db | 396 | struct nft_chain *chain; |
9a32669f | 397 | |
504882db | 398 | list_for_each_entry(table, &net->nft.tables, list) { |
9a32669f | 399 | if (table->family != NFPROTO_NETDEV) |
400 | continue; | |
401 | ||
504882db | 402 | list_for_each_entry(chain, &table->chains, list) { |
403 | if (!nft_is_base_chain(chain) || | |
404 | !(chain->flags & NFT_CHAIN_HW_OFFLOAD)) | |
405 | continue; | |
406 | ||
407 | basechain = nft_base_chain(chain); | |
408 | if (strncmp(basechain->dev_name, dev->name, IFNAMSIZ)) | |
409 | continue; | |
410 | ||
411 | return chain; | |
9a32669f | 412 | } |
413 | } | |
504882db | 414 | |
415 | return NULL; | |
416 | } | |
417 | ||
418 | static void nft_indr_block_cb(struct net_device *dev, | |
419 | flow_indr_block_bind_cb_t *cb, void *cb_priv, | |
420 | enum flow_block_command cmd) | |
421 | { | |
422 | struct net *net = dev_net(dev); | |
423 | struct nft_chain *chain; | |
424 | ||
425 | mutex_lock(&net->nft.commit_mutex); | |
426 | chain = __nft_offload_get_chain(dev); | |
427 | if (chain) { | |
428 | struct nft_base_chain *basechain; | |
429 | ||
430 | basechain = nft_base_chain(chain); | |
431 | nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd); | |
432 | } | |
433 | mutex_unlock(&net->nft.commit_mutex); | |
9a32669f | 434 | } |
3474a2c6 | 435 | |
06d392cb | 436 | static void nft_offload_chain_clean(struct nft_chain *chain) |
437 | { | |
438 | struct nft_rule *rule; | |
439 | ||
440 | list_for_each_entry(rule, &chain->rules, list) { | |
441 | nft_flow_offload_rule(chain, rule, | |
442 | NULL, FLOW_CLS_DESTROY); | |
443 | } | |
444 | ||
445 | nft_flow_offload_chain(chain, NULL, FLOW_BLOCK_UNBIND); | |
446 | } | |
447 | ||
448 | static int nft_offload_netdev_event(struct notifier_block *this, | |
449 | unsigned long event, void *ptr) | |
450 | { | |
451 | struct net_device *dev = netdev_notifier_info_to_dev(ptr); | |
452 | struct net *net = dev_net(dev); | |
453 | struct nft_chain *chain; | |
454 | ||
455 | mutex_lock(&net->nft.commit_mutex); | |
456 | chain = __nft_offload_get_chain(dev); | |
457 | if (chain) | |
458 | nft_offload_chain_clean(chain); | |
459 | mutex_unlock(&net->nft.commit_mutex); | |
460 | ||
461 | return NOTIFY_DONE; | |
462 | } | |
463 | ||
3474a2c6 PNA |
464 | static struct flow_indr_block_ing_entry block_ing_entry = { |
465 | .cb = nft_indr_block_cb, | |
466 | .list = LIST_HEAD_INIT(block_ing_entry.list), | |
467 | }; | |
468 | ||
06d392cb | 469 | static struct notifier_block nft_offload_netdev_notifier = { |
470 | .notifier_call = nft_offload_netdev_event, | |
471 | }; | |
472 | ||
473 | int nft_offload_init(void) | |
3474a2c6 | 474 | { |
06d392cb | 475 | int err; |
476 | ||
477 | err = register_netdevice_notifier(&nft_offload_netdev_notifier); | |
478 | if (err < 0) | |
479 | return err; | |
480 | ||
3474a2c6 | 481 | flow_indr_add_block_ing_cb(&block_ing_entry); |
06d392cb | 482 | |
483 | return 0; | |
3474a2c6 PNA |
484 | } |
485 | ||
486 | void nft_offload_exit(void) | |
487 | { | |
488 | flow_indr_del_block_ing_cb(&block_ing_entry); | |
06d392cb | 489 | unregister_netdevice_notifier(&nft_offload_netdev_notifier); |
3474a2c6 | 490 | } |