]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - net/sched/cls_api.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'net-batched-receive-in-GRO-path'
[mirror_ubuntu-hirsute-kernel.git] / net / sched / cls_api.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * net/sched/cls_api.c Packet classifier API.
4 *
1da177e4
LT		 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 *
7 * Changes:
8 *
9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
1da177e4
LT		 10 */
11
1da177e4
LT		 12#include <linux/module.h>
13#include <linux/types.h>
14#include <linux/kernel.h>
1da177e4 15#include <linux/string.h>
1da177e4 16#include <linux/errno.h>
33a48927 17#include <linux/err.h>
1da177e4 18#include <linux/skbuff.h>
1da177e4
LT		 19#include <linux/init.h>
20#include <linux/kmod.h>
5a0e3ad6 21#include <linux/slab.h>
48617387 22#include <linux/idr.h>
7f76fa36 23#include <linux/rhashtable.h>
b854272b
DL		 24#include <net/net_namespace.h>
25#include <net/sock.h>
dc5fc579 26#include <net/netlink.h>
1da177e4
LT		 27#include <net/pkt_sched.h>
28#include <net/pkt_cls.h>
e3ab786b 29#include <net/tc_act/tc_pedit.h>
3a7b6861
PNA		 30#include <net/tc_act/tc_mirred.h>
31#include <net/tc_act/tc_vlan.h>
32#include <net/tc_act/tc_tunnel_key.h>
33#include <net/tc_act/tc_csum.h>
34#include <net/tc_act/tc_gact.h>
8c8cfc6e 35#include <net/tc_act/tc_police.h>
a7a7be60 36#include <net/tc_act/tc_sample.h>
3a7b6861 37#include <net/tc_act/tc_skbedit.h>
b57dc7c1 38#include <net/tc_act/tc_ct.h>
6749d590 39#include <net/tc_act/tc_mpls.h>
1da177e4 40
e331473f
DC		 41extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
42
1da177e4 43/* The list of all installed classifier types */
36272874 44static LIST_HEAD(tcf_proto_base);
1da177e4
LT		 45
46/* Protects list of registered TC modules. It is pure SMP lock. */
47static DEFINE_RWLOCK(cls_mod_lock);
48
49/* Find classifier type by string name */
50
f34e8bff 51static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
1da177e4 52{
dcd76081 53 const struct tcf_proto_ops *t, *res = NULL;
1da177e4
LT		 54
55 if (kind) {
56 read_lock(&cls_mod_lock);
36272874 57 list_for_each_entry(t, &tcf_proto_base, head) {
33a48927 58 if (strcmp(kind, t->kind) == 0) {
dcd76081
ED		 59 if (try_module_get(t->owner))
60 res = t;
1da177e4
LT		 61 break;
62 }
63 }
64 read_unlock(&cls_mod_lock);
65 }
dcd76081 66 return res;
1da177e4
LT		 67}
68
f34e8bff 69static const struct tcf_proto_ops *
12db03b6
VB		 70tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
71 struct netlink_ext_ack *extack)
f34e8bff
JP		 72{
73 const struct tcf_proto_ops *ops;
74
75 ops = __tcf_proto_lookup_ops(kind);
76 if (ops)
77 return ops;
78#ifdef CONFIG_MODULES
12db03b6
VB		 79 if (rtnl_held)
80 rtnl_unlock();
f34e8bff 81 request_module("cls_%s", kind);
12db03b6
VB		 82 if (rtnl_held)
83 rtnl_lock();
f34e8bff
JP		 84 ops = __tcf_proto_lookup_ops(kind);
85 /* We dropped the RTNL semaphore in order to perform
86 * the module load. So, even if we succeeded in loading
87 * the module we have to replay the request. We indicate
88 * this using -EAGAIN.
89 */
90 if (ops) {
91 module_put(ops->owner);
92 return ERR_PTR(-EAGAIN);
93 }
94#endif
95 NL_SET_ERR_MSG(extack, "TC classifier not found");
96 return ERR_PTR(-ENOENT);
97}
98
1da177e4
LT		 99/* Register(unregister) new classifier type */
100
101int register_tcf_proto_ops(struct tcf_proto_ops *ops)
102{
36272874 103 struct tcf_proto_ops *t;
1da177e4
LT		 104 int rc = -EEXIST;
105
106 write_lock(&cls_mod_lock);
36272874 107 list_for_each_entry(t, &tcf_proto_base, head)
1da177e4
LT		 108 if (!strcmp(ops->kind, t->kind))
109 goto out;
110
36272874 111 list_add_tail(&ops->head, &tcf_proto_base);
1da177e4
LT		 112 rc = 0;
113out:
114 write_unlock(&cls_mod_lock);
115 return rc;
116}
aa767bfe 117EXPORT_SYMBOL(register_tcf_proto_ops);
1da177e4 118
7aa0045d
CW		 119static struct workqueue_struct *tc_filter_wq;
120
1da177e4
LT		 121int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
122{
36272874 123 struct tcf_proto_ops *t;
1da177e4
LT		 124 int rc = -ENOENT;
125
c78e1746
DB		 126 /* Wait for outstanding call_rcu()s, if any, from a
127 * tcf_proto_ops's destroy() handler.
128 */
129 rcu_barrier();
7aa0045d 130 flush_workqueue(tc_filter_wq);
c78e1746 131
1da177e4 132 write_lock(&cls_mod_lock);
dcd76081
ED		 133 list_for_each_entry(t, &tcf_proto_base, head) {
134 if (t == ops) {
135 list_del(&t->head);
136 rc = 0;
1da177e4 137 break;
dcd76081
ED		 138 }
139 }
1da177e4
LT		 140 write_unlock(&cls_mod_lock);
141 return rc;
142}
aa767bfe 143EXPORT_SYMBOL(unregister_tcf_proto_ops);
1da177e4 144
aaa908ff 145bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
7aa0045d 146{
aaa908ff
CW		 147 INIT_RCU_WORK(rwork, func);
148 return queue_rcu_work(tc_filter_wq, rwork);
7aa0045d
CW		 149}
150EXPORT_SYMBOL(tcf_queue_work);
151
1da177e4
LT		 152/* Select new prio value from the range, managed by kernel. */
153
aa767bfe 154static inline u32 tcf_auto_prio(struct tcf_proto *tp)
1da177e4 155{
aa767bfe 156 u32 first = TC_H_MAKE(0xC0000000U, 0U);
1da177e4
LT		 157
158 if (tp)
cc7ec456 159 first = tp->prio - 1;
1da177e4 160
7961973a 161 return TC_H_MAJ(first);
1da177e4
LT		 162}
163
470502de
VB		 164static bool tcf_proto_is_unlocked(const char *kind)
165{
166 const struct tcf_proto_ops *ops;
167 bool ret;
168
169 ops = tcf_proto_lookup_ops(kind, false, NULL);
170 /* On error return false to take rtnl lock. Proto lookup/create
171 * functions will perform lookup again and properly handle errors.
172 */
173 if (IS_ERR(ops))
174 return false;
175
176 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
177 module_put(ops->owner);
178 return ret;
179}
180
33a48927 181static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
c35a4acc 182 u32 prio, struct tcf_chain *chain,
12db03b6 183 bool rtnl_held,
c35a4acc 184 struct netlink_ext_ack *extack)
33a48927
JP		 185{
186 struct tcf_proto *tp;
187 int err;
188
189 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
190 if (!tp)
191 return ERR_PTR(-ENOBUFS);
192
12db03b6 193 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
f34e8bff
JP		 194 if (IS_ERR(tp->ops)) {
195 err = PTR_ERR(tp->ops);
d68d75fd 196 goto errout;
33a48927
JP		 197 }
198 tp->classify = tp->ops->classify;
199 tp->protocol = protocol;
200 tp->prio = prio;
5bc17018 201 tp->chain = chain;
8b64678e 202 spin_lock_init(&tp->lock);
4dbfa766 203 refcount_set(&tp->refcnt, 1);
33a48927
JP		 204
205 err = tp->ops->init(tp);
206 if (err) {
207 module_put(tp->ops->owner);
208 goto errout;
209 }
210 return tp;
211
212errout:
213 kfree(tp);
214 return ERR_PTR(err);
215}
216
4dbfa766
VB		 217static void tcf_proto_get(struct tcf_proto *tp)
218{
219 refcount_inc(&tp->refcnt);
220}
221
222static void tcf_chain_put(struct tcf_chain *chain);
223
12db03b6 224static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
715df5ec 225 struct netlink_ext_ack *extack)
cf1facda 226{
12db03b6 227 tp->ops->destroy(tp, rtnl_held, extack);
4dbfa766 228 tcf_chain_put(tp->chain);
763dbf63
WC		 229 module_put(tp->ops->owner);
230 kfree_rcu(tp, rcu);
cf1facda
JP		 231}
232
12db03b6 233static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
4dbfa766
VB		 234 struct netlink_ext_ack *extack)
235{
236 if (refcount_dec_and_test(&tp->refcnt))
12db03b6 237 tcf_proto_destroy(tp, rtnl_held, extack);
4dbfa766
VB		 238}
239
268a351d 240static int walker_check_empty(struct tcf_proto *tp, void *fh,
6676d5e4 241 struct tcf_walker *arg)
8b64678e 242{
268a351d 243 if (fh) {
6676d5e4
VB		 244 arg->nonempty = true;
245 return -1;
246 }
247 return 0;
8b64678e
VB		 248}
249
12db03b6 250static bool tcf_proto_is_empty(struct tcf_proto *tp, bool rtnl_held)
8b64678e 251{
6676d5e4 252 struct tcf_walker walker = { .fn = walker_check_empty, };
8b64678e
VB		 253
254 if (tp->ops->walk) {
12db03b6 255 tp->ops->walk(tp, &walker, rtnl_held);
6676d5e4 256 return !walker.nonempty;
8b64678e
VB		 257 }
258 return true;
259}
260
12db03b6 261static bool tcf_proto_check_delete(struct tcf_proto *tp, bool rtnl_held)
8b64678e
VB		 262{
263 spin_lock(&tp->lock);
12db03b6 264 if (tcf_proto_is_empty(tp, rtnl_held))
8b64678e
VB		 265 tp->deleting = true;
266 spin_unlock(&tp->lock);
267 return tp->deleting;
268}
269
270static void tcf_proto_mark_delete(struct tcf_proto *tp)
271{
272 spin_lock(&tp->lock);
273 tp->deleting = true;
274 spin_unlock(&tp->lock);
275}
276
277static bool tcf_proto_is_deleting(struct tcf_proto *tp)
278{
279 bool deleting;
280
281 spin_lock(&tp->lock);
282 deleting = tp->deleting;
283 spin_unlock(&tp->lock);
284
285 return deleting;
286}
287
c266f64d
VB		 288#define ASSERT_BLOCK_LOCKED(block) \
289 lockdep_assert_held(&(block)->lock)
290
a9b19443
JP		 291struct tcf_filter_chain_list_item {
292 struct list_head list;
293 tcf_chain_head_change_t *chain_head_change;
294 void *chain_head_change_priv;
295};
296
5bc17018
JP		 297static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
298 u32 chain_index)
2190d1d0 299{
5bc17018
JP		 300 struct tcf_chain *chain;
301
c266f64d
VB		 302 ASSERT_BLOCK_LOCKED(block);
303
5bc17018
JP		 304 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
305 if (!chain)
306 return NULL;
307 list_add_tail(&chain->list, &block->chain_list);
ed76f5ed 308 mutex_init(&chain->filter_chain_lock);
5bc17018
JP		 309 chain->block = block;
310 chain->index = chain_index;
e2ef7544 311 chain->refcnt = 1;
f71e0ca4
JP		 312 if (!chain->index)
313 block->chain0.chain = chain;
5bc17018 314 return chain;
2190d1d0
JP		 315}
316
a9b19443
JP		 317static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
318 struct tcf_proto *tp_head)
319{
320 if (item->chain_head_change)
321 item->chain_head_change(tp_head, item->chain_head_change_priv);
322}
f71e0ca4
JP		 323
324static void tcf_chain0_head_change(struct tcf_chain *chain,
325 struct tcf_proto *tp_head)
c7eb7d72 326{
a9b19443 327 struct tcf_filter_chain_list_item *item;
f71e0ca4 328 struct tcf_block *block = chain->block;
a9b19443 329
f71e0ca4
JP		 330 if (chain->index)
331 return;
165f0135
VB		 332
333 mutex_lock(&block->lock);
f71e0ca4 334 list_for_each_entry(item, &block->chain0.filter_chain_list, list)
a9b19443 335 tcf_chain_head_change_item(item, tp_head);
165f0135 336 mutex_unlock(&block->lock);
c7eb7d72
JP		 337}
338
c266f64d
VB		 339/* Returns true if block can be safely freed. */
340
341static bool tcf_chain_detach(struct tcf_chain *chain)
f93e1cdc 342{
efbf7897
CW		 343 struct tcf_block *block = chain->block;
344
c266f64d
VB		 345 ASSERT_BLOCK_LOCKED(block);
346
e2ef7544 347 list_del(&chain->list);
f71e0ca4
JP		 348 if (!chain->index)
349 block->chain0.chain = NULL;
c266f64d
VB		 350
351 if (list_empty(&block->chain_list) &&
352 refcount_read(&block->refcnt) == 0)
353 return true;
354
355 return false;
356}
357
358static void tcf_block_destroy(struct tcf_block *block)
359{
360 mutex_destroy(&block->lock);
361 kfree_rcu(block, rcu);
362}
363
364static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
365{
366 struct tcf_block *block = chain->block;
367
ed76f5ed 368 mutex_destroy(&chain->filter_chain_lock);
ee3bbfe8 369 kfree_rcu(chain, rcu);
c266f64d
VB		 370 if (free_block)
371 tcf_block_destroy(block);
e2ef7544 372}
744a4cf6 373
e2ef7544
CW		 374static void tcf_chain_hold(struct tcf_chain *chain)
375{
c266f64d
VB		 376 ASSERT_BLOCK_LOCKED(chain->block);
377
e2ef7544 378 ++chain->refcnt;
2190d1d0
JP		 379}
380
3d32f4c5 381static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
1f3ed383 382{
c266f64d
VB		 383 ASSERT_BLOCK_LOCKED(chain->block);
384
1f3ed383 385 /* In case all the references are action references, this
3d32f4c5 386 * chain should not be shown to the user.
1f3ed383
JP		 387 */
388 return chain->refcnt == chain->action_refcnt;
389}
390
32a4f5ec
JP		 391static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
392 u32 chain_index)
5bc17018
JP		 393{
394 struct tcf_chain *chain;
395
c266f64d
VB		 396 ASSERT_BLOCK_LOCKED(block);
397
5bc17018 398 list_for_each_entry(chain, &block->chain_list, list) {
32a4f5ec 399 if (chain->index == chain_index)
e2ef7544 400 return chain;
32a4f5ec
JP		 401 }
402 return NULL;
403}
404
405static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
406 u32 seq, u16 flags, int event, bool unicast);
407
53681407
JP		 408static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
409 u32 chain_index, bool create,
410 bool by_act)
32a4f5ec 411{
c266f64d
VB		 412 struct tcf_chain *chain = NULL;
413 bool is_first_reference;
32a4f5ec 414
c266f64d
VB		 415 mutex_lock(&block->lock);
416 chain = tcf_chain_lookup(block, chain_index);
32a4f5ec
JP		 417 if (chain) {
418 tcf_chain_hold(chain);
53681407
JP		 419 } else {
420 if (!create)
c266f64d 421 goto errout;
53681407
JP		 422 chain = tcf_chain_create(block, chain_index);
423 if (!chain)
c266f64d 424 goto errout;
5bc17018 425 }
80532384 426
53681407
JP		 427 if (by_act)
428 ++chain->action_refcnt;
c266f64d
VB		 429 is_first_reference = chain->refcnt - chain->action_refcnt == 1;
430 mutex_unlock(&block->lock);
53681407
JP		 431
432 /* Send notification only in case we got the first
433 * non-action reference. Until then, the chain acts only as
434 * a placeholder for actions pointing to it and user ought
435 * not know about them.
436 */
c266f64d 437 if (is_first_reference && !by_act)
53681407
JP		 438 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
439 RTM_NEWCHAIN, false);
440
32a4f5ec 441 return chain;
c266f64d
VB		 442
443errout:
444 mutex_unlock(&block->lock);
445 return chain;
5bc17018 446}
53681407 447
290b1c8b
JP		 448static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
449 bool create)
53681407
JP		 450{
451 return __tcf_chain_get(block, chain_index, create, false);
452}
5bc17018 453
1f3ed383
JP		 454struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
455{
53681407 456 return __tcf_chain_get(block, chain_index, true, true);
1f3ed383
JP		 457}
458EXPORT_SYMBOL(tcf_chain_get_by_act);
459
a5654820
VB		 460static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
461 void *tmplt_priv);
462static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
463 void *tmplt_priv, u32 chain_index,
464 struct tcf_block *block, struct sk_buff *oskb,
465 u32 seq, u16 flags, bool unicast);
9f407f17 466
91052fa1
VB		 467static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
468 bool explicitly_created)
5bc17018 469{
c266f64d 470 struct tcf_block *block = chain->block;
a5654820 471 const struct tcf_proto_ops *tmplt_ops;
b62989fc 472 bool free_block = false;
c266f64d 473 unsigned int refcnt;
a5654820 474 void *tmplt_priv;
c266f64d
VB		 475
476 mutex_lock(&block->lock);
91052fa1
VB		 477 if (explicitly_created) {
478 if (!chain->explicitly_created) {
479 mutex_unlock(&block->lock);
480 return;
481 }
482 chain->explicitly_created = false;
483 }
484
53681407
JP		 485 if (by_act)
486 chain->action_refcnt--;
c266f64d
VB		 487
488 /* tc_chain_notify_delete can't be called while holding block lock.
489 * However, when block is unlocked chain can be changed concurrently, so
490 * save these to temporary variables.
491 */
492 refcnt = --chain->refcnt;
a5654820
VB		 493 tmplt_ops = chain->tmplt_ops;
494 tmplt_priv = chain->tmplt_priv;
53681407
JP		 495
496 /* The last dropped non-action reference will trigger notification. */
b62989fc
VB		 497 if (refcnt - chain->action_refcnt == 0 && !by_act) {
498 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index,
a5654820 499 block, NULL, 0, 0, false);
726d0612
VB		 500 /* Last reference to chain, no need to lock. */
501 chain->flushing = false;
502 }
53681407 503
b62989fc
VB		 504 if (refcnt == 0)
505 free_block = tcf_chain_detach(chain);
506 mutex_unlock(&block->lock);
507
c266f64d 508 if (refcnt == 0) {
a5654820 509 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
c266f64d 510 tcf_chain_destroy(chain, free_block);
32a4f5ec 511 }
5bc17018 512}
53681407 513
290b1c8b 514static void tcf_chain_put(struct tcf_chain *chain)
53681407 515{
91052fa1 516 __tcf_chain_put(chain, false, false);
53681407 517}
5bc17018 518
1f3ed383
JP		 519void tcf_chain_put_by_act(struct tcf_chain *chain)
520{
91052fa1 521 __tcf_chain_put(chain, true, false);
1f3ed383
JP		 522}
523EXPORT_SYMBOL(tcf_chain_put_by_act);
524
32a4f5ec
JP		 525static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
526{
91052fa1 527 __tcf_chain_put(chain, false, true);
32a4f5ec
JP		 528}
529
12db03b6 530static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
290b1c8b 531{
4dbfa766 532 struct tcf_proto *tp, *tp_next;
290b1c8b 533
ed76f5ed
VB		 534 mutex_lock(&chain->filter_chain_lock);
535 tp = tcf_chain_dereference(chain->filter_chain, chain);
4dbfa766 536 RCU_INIT_POINTER(chain->filter_chain, NULL);
290b1c8b 537 tcf_chain0_head_change(chain, NULL);
726d0612 538 chain->flushing = true;
ed76f5ed
VB		 539 mutex_unlock(&chain->filter_chain_lock);
540
290b1c8b 541 while (tp) {
4dbfa766 542 tp_next = rcu_dereference_protected(tp->next, 1);
12db03b6 543 tcf_proto_put(tp, rtnl_held, NULL);
4dbfa766 544 tp = tp_next;
290b1c8b
JP		 545 }
546}
547
7f76fa36
JH		 548static struct tcf_block *tc_dev_ingress_block(struct net_device *dev)
549{
550 const struct Qdisc_class_ops *cops;
551 struct Qdisc *qdisc;
552
553 if (!dev_ingress_queue(dev))
554 return NULL;
555
556 qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
557 if (!qdisc)
558 return NULL;
559
560 cops = qdisc->ops->cl_ops;
561 if (!cops)
562 return NULL;
563
564 if (!cops->tcf_block)
565 return NULL;
566
567 return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL);
568}
569
570static struct rhashtable indr_setup_block_ht;
571
572struct tc_indr_block_dev {
573 struct rhash_head ht_node;
574 struct net_device *dev;
575 unsigned int refcnt;
576 struct list_head cb_list;
577 struct tcf_block *block;
578};
579
580struct tc_indr_block_cb {
581 struct list_head list;
582 void *cb_priv;
583 tc_indr_block_bind_cb_t *cb;
584 void *cb_ident;
585};
586
587static const struct rhashtable_params tc_indr_setup_block_ht_params = {
588 .key_offset = offsetof(struct tc_indr_block_dev, dev),
589 .head_offset = offsetof(struct tc_indr_block_dev, ht_node),
590 .key_len = sizeof(struct net_device *),
591};
592
593static struct tc_indr_block_dev *
594tc_indr_block_dev_lookup(struct net_device *dev)
595{
596 return rhashtable_lookup_fast(&indr_setup_block_ht, &dev,
597 tc_indr_setup_block_ht_params);
598}
599
600static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev)
601{
602 struct tc_indr_block_dev *indr_dev;
603
604 indr_dev = tc_indr_block_dev_lookup(dev);
605 if (indr_dev)
606 goto inc_ref;
607
608 indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL);
609 if (!indr_dev)
610 return NULL;
611
612 INIT_LIST_HEAD(&indr_dev->cb_list);
613 indr_dev->dev = dev;
614 indr_dev->block = tc_dev_ingress_block(dev);
615 if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node,
616 tc_indr_setup_block_ht_params)) {
617 kfree(indr_dev);
618 return NULL;
619 }
620
621inc_ref:
622 indr_dev->refcnt++;
623 return indr_dev;
624}
625
626static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev)
627{
628 if (--indr_dev->refcnt)
629 return;
630
631 rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node,
632 tc_indr_setup_block_ht_params);
633 kfree(indr_dev);
634}
635
636static struct tc_indr_block_cb *
637tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev,
638 tc_indr_block_bind_cb_t *cb, void *cb_ident)
639{
640 struct tc_indr_block_cb *indr_block_cb;
641
642 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
643 if (indr_block_cb->cb == cb &&
644 indr_block_cb->cb_ident == cb_ident)
645 return indr_block_cb;
646 return NULL;
647}
648
649static struct tc_indr_block_cb *
650tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv,
651 tc_indr_block_bind_cb_t *cb, void *cb_ident)
652{
653 struct tc_indr_block_cb *indr_block_cb;
654
655 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
656 if (indr_block_cb)
657 return ERR_PTR(-EEXIST);
658
659 indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL);
660 if (!indr_block_cb)
661 return ERR_PTR(-ENOMEM);
662
663 indr_block_cb->cb_priv = cb_priv;
664 indr_block_cb->cb = cb;
665 indr_block_cb->cb_ident = cb_ident;
666 list_add(&indr_block_cb->list, &indr_dev->cb_list);
667
668 return indr_block_cb;
669}
670
671static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb)
672{
673 list_del(&indr_block_cb->list);
674 kfree(indr_block_cb);
675}
676
59094b1e
PNA		 677static int tcf_block_setup(struct tcf_block *block,
678 struct flow_block_offload *bo);
679
7f76fa36
JH		 680static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
681 struct tc_indr_block_cb *indr_block_cb,
9c0e189e 682 enum flow_block_command command)
7f76fa36 683{
955bcb6e 684 struct flow_block_offload bo = {
7f76fa36 685 .command = command,
32f8c409 686 .binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
da3eeb90 687 .net = dev_net(indr_dev->dev),
c1a970d0 688 .block_shared = tcf_block_non_null_shared(indr_dev->block),
7f76fa36 689 };
59094b1e 690 INIT_LIST_HEAD(&bo.cb_list);
7f76fa36
JH		 691
692 if (!indr_dev->block)
693 return;
694
14bfb13f
PNA		 695 bo.block = &indr_dev->block->flow_block;
696
7f76fa36
JH		 697 indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
698 &bo);
59094b1e 699 tcf_block_setup(indr_dev->block, &bo);
7f76fa36
JH		 700}
701
702int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
703 tc_indr_block_bind_cb_t *cb, void *cb_ident)
704{
705 struct tc_indr_block_cb *indr_block_cb;
706 struct tc_indr_block_dev *indr_dev;
707 int err;
708
709 indr_dev = tc_indr_block_dev_get(dev);
710 if (!indr_dev)
711 return -ENOMEM;
712
713 indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident);
714 err = PTR_ERR_OR_ZERO(indr_block_cb);
715 if (err)
716 goto err_dev_put;
717
9c0e189e 718 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, FLOW_BLOCK_BIND);
7f76fa36
JH		 719 return 0;
720
721err_dev_put:
722 tc_indr_block_dev_put(indr_dev);
723 return err;
724}
725EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register);
726
727int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
728 tc_indr_block_bind_cb_t *cb, void *cb_ident)
729{
730 int err;
731
732 rtnl_lock();
733 err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident);
734 rtnl_unlock();
735
736 return err;
737}
738EXPORT_SYMBOL_GPL(tc_indr_block_cb_register);
739
740void __tc_indr_block_cb_unregister(struct net_device *dev,
741 tc_indr_block_bind_cb_t *cb, void *cb_ident)
742{
743 struct tc_indr_block_cb *indr_block_cb;
744 struct tc_indr_block_dev *indr_dev;
745
746 indr_dev = tc_indr_block_dev_lookup(dev);
747 if (!indr_dev)
748 return;
749
750 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
751 if (!indr_block_cb)
752 return;
753
754 /* Send unbind message if required to free any block cbs. */
9c0e189e 755 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, FLOW_BLOCK_UNBIND);
7f76fa36
JH		 756 tc_indr_block_cb_del(indr_block_cb);
757 tc_indr_block_dev_put(indr_dev);
758}
759EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister);
760
761void tc_indr_block_cb_unregister(struct net_device *dev,
762 tc_indr_block_bind_cb_t *cb, void *cb_ident)
763{
764 rtnl_lock();
765 __tc_indr_block_cb_unregister(dev, cb, cb_ident);
766 rtnl_unlock();
767}
768EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister);
769
770static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
771 struct tcf_block_ext_info *ei,
9c0e189e 772 enum flow_block_command command,
7f76fa36
JH		 773 struct netlink_ext_ack *extack)
774{
775 struct tc_indr_block_cb *indr_block_cb;
776 struct tc_indr_block_dev *indr_dev;
955bcb6e 777 struct flow_block_offload bo = {
7f76fa36
JH		 778 .command = command,
779 .binder_type = ei->binder_type,
da3eeb90 780 .net = dev_net(dev),
14bfb13f 781 .block = &block->flow_block,
955bcb6e 782 .block_shared = tcf_block_shared(block),
7f76fa36
JH		 783 .extack = extack,
784 };
59094b1e 785 INIT_LIST_HEAD(&bo.cb_list);
7f76fa36
JH		 786
787 indr_dev = tc_indr_block_dev_lookup(dev);
788 if (!indr_dev)
789 return;
790
9c0e189e 791 indr_dev->block = command == FLOW_BLOCK_BIND ? block : NULL;
7f76fa36
JH		 792
793 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
794 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
795 &bo);
59094b1e
PNA		 796
797 tcf_block_setup(block, &bo);
7f76fa36
JH		 798}
799
caa72601
JP		 800static bool tcf_block_offload_in_use(struct tcf_block *block)
801{
802 return block->offloadcnt;
803}
804
805static int tcf_block_offload_cmd(struct tcf_block *block,
806 struct net_device *dev,
807 struct tcf_block_ext_info *ei,
9c0e189e 808 enum flow_block_command command,
60513bd8 809 struct netlink_ext_ack *extack)
8c4083b3 810{
955bcb6e 811 struct flow_block_offload bo = {};
59094b1e 812 int err;
8c4083b3 813
da3eeb90 814 bo.net = dev_net(dev);
8c4083b3
JP		 815 bo.command = command;
816 bo.binder_type = ei->binder_type;
14bfb13f 817 bo.block = &block->flow_block;
955bcb6e 818 bo.block_shared = tcf_block_shared(block);
60513bd8 819 bo.extack = extack;
59094b1e
PNA		 820 INIT_LIST_HEAD(&bo.cb_list);
821
822 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
823 if (err < 0)
824 return err;
825
826 return tcf_block_setup(block, &bo);
8c4083b3
JP		 827}
828
caa72601 829static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
60513bd8
JH		 830 struct tcf_block_ext_info *ei,
831 struct netlink_ext_ack *extack)
8c4083b3 832{
caa72601
JP		 833 struct net_device *dev = q->dev_queue->dev;
834 int err;
835
836 if (!dev->netdev_ops->ndo_setup_tc)
837 goto no_offload_dev_inc;
838
839 /* If tc offload feature is disabled and the block we try to bind
840 * to already has some offloaded filters, forbid to bind.
841 */
60513bd8
JH		 842 if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
843 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
caa72601 844 return -EOPNOTSUPP;
60513bd8 845 }
caa72601 846
9c0e189e 847 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_BIND, extack);
caa72601
JP		 848 if (err == -EOPNOTSUPP)
849 goto no_offload_dev_inc;
7f76fa36
JH		 850 if (err)
851 return err;
852
9c0e189e 853 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
7f76fa36 854 return 0;
caa72601
JP		 855
856no_offload_dev_inc:
857 if (tcf_block_offload_in_use(block))
858 return -EOPNOTSUPP;
859 block->nooffloaddevcnt++;
9c0e189e 860 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_BIND, extack);
caa72601 861 return 0;
8c4083b3
JP		 862}
863
864static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
865 struct tcf_block_ext_info *ei)
866{
caa72601
JP		 867 struct net_device *dev = q->dev_queue->dev;
868 int err;
869
9c0e189e 870 tc_indr_block_call(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
7f76fa36 871
caa72601
JP		 872 if (!dev->netdev_ops->ndo_setup_tc)
873 goto no_offload_dev_dec;
9c0e189e 874 err = tcf_block_offload_cmd(block, dev, ei, FLOW_BLOCK_UNBIND, NULL);
caa72601
JP		 875 if (err == -EOPNOTSUPP)
876 goto no_offload_dev_dec;
877 return;
878
879no_offload_dev_dec:
880 WARN_ON(block->nooffloaddevcnt-- == 0);
8c4083b3
JP		 881}
882
a9b19443 883static int
f71e0ca4
JP		 884tcf_chain0_head_change_cb_add(struct tcf_block *block,
885 struct tcf_block_ext_info *ei,
886 struct netlink_ext_ack *extack)
a9b19443
JP		 887{
888 struct tcf_filter_chain_list_item *item;
165f0135 889 struct tcf_chain *chain0;
a9b19443
JP		 890
891 item = kmalloc(sizeof(*item), GFP_KERNEL);
892 if (!item) {
893 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
894 return -ENOMEM;
895 }
896 item->chain_head_change = ei->chain_head_change;
897 item->chain_head_change_priv = ei->chain_head_change_priv;
165f0135
VB		 898
899 mutex_lock(&block->lock);
900 chain0 = block->chain0.chain;
ed76f5ed
VB		 901 if (chain0)
902 tcf_chain_hold(chain0);
903 else
904 list_add(&item->list, &block->chain0.filter_chain_list);
165f0135
VB		 905 mutex_unlock(&block->lock);
906
ed76f5ed
VB		 907 if (chain0) {
908 struct tcf_proto *tp_head;
909
910 mutex_lock(&chain0->filter_chain_lock);
911
912 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
913 if (tp_head)
914 tcf_chain_head_change_item(item, tp_head);
915
916 mutex_lock(&block->lock);
917 list_add(&item->list, &block->chain0.filter_chain_list);
918 mutex_unlock(&block->lock);
919
920 mutex_unlock(&chain0->filter_chain_lock);
921 tcf_chain_put(chain0);
922 }
923
a9b19443
JP		 924 return 0;
925}
926
927static void
f71e0ca4
JP		 928tcf_chain0_head_change_cb_del(struct tcf_block *block,
929 struct tcf_block_ext_info *ei)
a9b19443
JP		 930{
931 struct tcf_filter_chain_list_item *item;
932
165f0135 933 mutex_lock(&block->lock);
f71e0ca4 934 list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
a9b19443
JP		 935 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
936 (item->chain_head_change == ei->chain_head_change &&
937 item->chain_head_change_priv == ei->chain_head_change_priv)) {
165f0135 938 if (block->chain0.chain)
f71e0ca4 939 tcf_chain_head_change_item(item, NULL);
a9b19443 940 list_del(&item->list);
165f0135
VB		 941 mutex_unlock(&block->lock);
942
a9b19443
JP		 943 kfree(item);
944 return;
945 }
946 }
165f0135 947 mutex_unlock(&block->lock);
a9b19443
JP		 948 WARN_ON(1);
949}
950
48617387 951struct tcf_net {
ab281629 952 spinlock_t idr_lock; /* Protects idr */
48617387
JP		 953 struct idr idr;
954};
955
956static unsigned int tcf_net_id;
957
958static int tcf_block_insert(struct tcf_block *block, struct net *net,
bb047ddd 959 struct netlink_ext_ack *extack)
a9b19443 960{
48617387 961 struct tcf_net *tn = net_generic(net, tcf_net_id);
ab281629
VB		 962 int err;
963
964 idr_preload(GFP_KERNEL);
965 spin_lock(&tn->idr_lock);
966 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
967 GFP_NOWAIT);
968 spin_unlock(&tn->idr_lock);
969 idr_preload_end();
48617387 970
ab281629 971 return err;
a9b19443
JP		 972}
973
48617387
JP		 974static void tcf_block_remove(struct tcf_block *block, struct net *net)
975{
976 struct tcf_net *tn = net_generic(net, tcf_net_id);
977
ab281629 978 spin_lock(&tn->idr_lock);
9c160941 979 idr_remove(&tn->idr, block->index);
ab281629 980 spin_unlock(&tn->idr_lock);
48617387
JP		 981}
982
983static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
bb047ddd 984 u32 block_index,
48617387 985 struct netlink_ext_ack *extack)
6529eaba 986{
48617387 987 struct tcf_block *block;
6529eaba 988
48617387 989 block = kzalloc(sizeof(*block), GFP_KERNEL);
8d1a77f9
AA		 990 if (!block) {
991 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
48617387 992 return ERR_PTR(-ENOMEM);
8d1a77f9 993 }
c266f64d 994 mutex_init(&block->lock);
14bfb13f 995 flow_block_init(&block->flow_block);
5bc17018 996 INIT_LIST_HEAD(&block->chain_list);
f36fe1c4 997 INIT_LIST_HEAD(&block->owner_list);
f71e0ca4 998 INIT_LIST_HEAD(&block->chain0.filter_chain_list);
acb67442 999
cfebd7e2 1000 refcount_set(&block->refcnt, 1);
48617387 1001 block->net = net;
bb047ddd
JP		 1002 block->index = block_index;
1003
1004 /* Don't store q pointer for blocks which are shared */
1005 if (!tcf_block_shared(block))
1006 block->q = q;
48617387 1007 return block;
48617387
JP		 1008}
1009
1010static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
1011{
1012 struct tcf_net *tn = net_generic(net, tcf_net_id);
1013
322d884b 1014 return idr_find(&tn->idr, block_index);
48617387
JP		 1015}
1016
0607e439
VB		 1017static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
1018{
1019 struct tcf_block *block;
1020
1021 rcu_read_lock();
1022 block = tcf_block_lookup(net, block_index);
1023 if (block && !refcount_inc_not_zero(&block->refcnt))
1024 block = NULL;
1025 rcu_read_unlock();
1026
1027 return block;
1028}
1029
bbf73830
VB		 1030static struct tcf_chain *
1031__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
f0023436 1032{
bbf73830
VB		 1033 mutex_lock(&block->lock);
1034 if (chain)
1035 chain = list_is_last(&chain->list, &block->chain_list) ?
1036 NULL : list_next_entry(chain, list);
1037 else
1038 chain = list_first_entry_or_null(&block->chain_list,
1039 struct tcf_chain, list);
f0023436 1040
bbf73830
VB		 1041 /* skip all action-only chains */
1042 while (chain && tcf_chain_held_by_acts_only(chain))
1043 chain = list_is_last(&chain->list, &block->chain_list) ?
1044 NULL : list_next_entry(chain, list);
1045
1046 if (chain)
f0023436 1047 tcf_chain_hold(chain);
bbf73830 1048 mutex_unlock(&block->lock);
f0023436 1049
bbf73830 1050 return chain;
f0023436
VB		 1051}
1052
bbf73830
VB		 1053/* Function to be used by all clients that want to iterate over all chains on
1054 * block. It properly obtains block->lock and takes reference to chain before
1055 * returning it. Users of this function must be tolerant to concurrent chain
1056 * insertion/deletion or ensure that no concurrent chain modification is
1057 * possible. Note that all netlink dump callbacks cannot guarantee to provide
1058 * consistent dump because rtnl lock is released each time skb is filled with
1059 * data and sent to user-space.
1060 */
1061
1062struct tcf_chain *
1063tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
f0023436 1064{
bbf73830 1065 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
f0023436 1066
bbf73830 1067 if (chain)
f0023436 1068 tcf_chain_put(chain);
bbf73830
VB		 1069
1070 return chain_next;
1071}
1072EXPORT_SYMBOL(tcf_get_next_chain);
1073
fe2923af
VB		 1074static struct tcf_proto *
1075__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1076{
8b64678e
VB		 1077 u32 prio = 0;
1078
fe2923af
VB		 1079 ASSERT_RTNL();
1080 mutex_lock(&chain->filter_chain_lock);
1081
8b64678e 1082 if (!tp) {
fe2923af 1083 tp = tcf_chain_dereference(chain->filter_chain, chain);
8b64678e
VB		 1084 } else if (tcf_proto_is_deleting(tp)) {
1085 /* 'deleting' flag is set and chain->filter_chain_lock was
1086 * unlocked, which means next pointer could be invalid. Restart
1087 * search.
1088 */
1089 prio = tp->prio + 1;
1090 tp = tcf_chain_dereference(chain->filter_chain, chain);
1091
1092 for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1093 if (!tp->deleting && tp->prio >= prio)
1094 break;
1095 } else {
fe2923af 1096 tp = tcf_chain_dereference(tp->next, chain);
8b64678e 1097 }
fe2923af
VB		 1098
1099 if (tp)
1100 tcf_proto_get(tp);
1101
1102 mutex_unlock(&chain->filter_chain_lock);
1103
1104 return tp;
1105}
1106
1107/* Function to be used by all clients that want to iterate over all tp's on
1108 * chain. Users of this function must be tolerant to concurrent tp
1109 * insertion/deletion or ensure that no concurrent chain modification is
1110 * possible. Note that all netlink dump callbacks cannot guarantee to provide
1111 * consistent dump because rtnl lock is released each time skb is filled with
1112 * data and sent to user-space.
1113 */
1114
1115struct tcf_proto *
12db03b6
VB		 1116tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp,
1117 bool rtnl_held)
fe2923af
VB		 1118{
1119 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1120
1121 if (tp)
12db03b6 1122 tcf_proto_put(tp, rtnl_held, NULL);
fe2923af
VB		 1123
1124 return tp_next;
1125}
1126EXPORT_SYMBOL(tcf_get_next_proto);
1127
12db03b6 1128static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
bbf73830
VB		 1129{
1130 struct tcf_chain *chain;
1131
1132 /* Last reference to block. At this point chains cannot be added or
1133 * removed concurrently.
1134 */
1135 for (chain = tcf_get_next_chain(block, NULL);
1136 chain;
1137 chain = tcf_get_next_chain(block, chain)) {
1138 tcf_chain_put_explicitly_created(chain);
12db03b6 1139 tcf_chain_flush(chain, rtnl_held);
f0023436
VB		 1140 }
1141}
1142
18d3eefb
VB		 1143/* Lookup Qdisc and increments its reference counter.
1144 * Set parent, if necessary.
1145 */
1146
1147static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1148 u32 *parent, int ifindex, bool rtnl_held,
1149 struct netlink_ext_ack *extack)
1150{
1151 const struct Qdisc_class_ops *cops;
1152 struct net_device *dev;
1153 int err = 0;
1154
1155 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1156 return 0;
1157
1158 rcu_read_lock();
1159
1160 /* Find link */
1161 dev = dev_get_by_index_rcu(net, ifindex);
1162 if (!dev) {
1163 rcu_read_unlock();
1164 return -ENODEV;
1165 }
1166
1167 /* Find qdisc */
1168 if (!*parent) {
1169 *q = dev->qdisc;
1170 *parent = (*q)->handle;
1171 } else {
1172 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1173 if (!*q) {
1174 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1175 err = -EINVAL;
1176 goto errout_rcu;
1177 }
1178 }
1179
1180 *q = qdisc_refcount_inc_nz(*q);
1181 if (!*q) {
1182 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1183 err = -EINVAL;
1184 goto errout_rcu;
1185 }
1186
1187 /* Is it classful? */
1188 cops = (*q)->ops->cl_ops;
1189 if (!cops) {
1190 NL_SET_ERR_MSG(extack, "Qdisc not classful");
1191 err = -EINVAL;
1192 goto errout_qdisc;
1193 }
1194
1195 if (!cops->tcf_block) {
1196 NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1197 err = -EOPNOTSUPP;
1198 goto errout_qdisc;
1199 }
1200
1201errout_rcu:
1202 /* At this point we know that qdisc is not noop_qdisc,
1203 * which means that qdisc holds a reference to net_device
1204 * and we hold a reference to qdisc, so it is safe to release
1205 * rcu read lock.
1206 */
1207 rcu_read_unlock();
1208 return err;
1209
1210errout_qdisc:
1211 rcu_read_unlock();
1212
1213 if (rtnl_held)
1214 qdisc_put(*q);
1215 else
1216 qdisc_put_unlocked(*q);
1217 *q = NULL;
1218
1219 return err;
1220}
1221
1222static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1223 int ifindex, struct netlink_ext_ack *extack)
1224{
1225 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1226 return 0;
1227
1228 /* Do we search for filter, attached to class? */
1229 if (TC_H_MIN(parent)) {
1230 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1231
1232 *cl = cops->find(q, parent);
1233 if (*cl == 0) {
1234 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1235 return -ENOENT;
1236 }
1237 }
1238
1239 return 0;
1240}
1241
1242static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1243 unsigned long cl, int ifindex,
1244 u32 block_index,
1245 struct netlink_ext_ack *extack)
1246{
1247 struct tcf_block *block;
1248
1249 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1250 block = tcf_block_refcnt_get(net, block_index);
1251 if (!block) {
1252 NL_SET_ERR_MSG(extack, "Block of given index was not found");
1253 return ERR_PTR(-EINVAL);
1254 }
1255 } else {
1256 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1257
1258 block = cops->tcf_block(q, cl, extack);
1259 if (!block)
1260 return ERR_PTR(-EINVAL);
1261
1262 if (tcf_block_shared(block)) {
1263 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1264 return ERR_PTR(-EOPNOTSUPP);
1265 }
1266
1267 /* Always take reference to block in order to support execution
1268 * of rules update path of cls API without rtnl lock. Caller
1269 * must release block when it is finished using it. 'if' block
1270 * of this conditional obtain reference to block by calling
1271 * tcf_block_refcnt_get().
1272 */
1273 refcount_inc(&block->refcnt);
1274 }
1275
1276 return block;
1277}
1278
0607e439 1279static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
12db03b6 1280 struct tcf_block_ext_info *ei, bool rtnl_held)
0607e439 1281{
c266f64d 1282 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
0607e439
VB		 1283 /* Flushing/putting all chains will cause the block to be
1284 * deallocated when last chain is freed. However, if chain_list
1285 * is empty, block has to be manually deallocated. After block
1286 * reference counter reached 0, it is no longer possible to
1287 * increment it or add new chains to block.
1288 */
1289 bool free_block = list_empty(&block->chain_list);
1290
c266f64d 1291 mutex_unlock(&block->lock);
0607e439
VB		 1292 if (tcf_block_shared(block))
1293 tcf_block_remove(block, block->net);
0607e439
VB		 1294
1295 if (q)
1296 tcf_block_offload_unbind(block, q, ei);
1297
1298 if (free_block)
c266f64d 1299 tcf_block_destroy(block);
0607e439 1300 else
12db03b6 1301 tcf_block_flush_all_chains(block, rtnl_held);
0607e439
VB		 1302 } else if (q) {
1303 tcf_block_offload_unbind(block, q, ei);
1304 }
1305}
1306
12db03b6 1307static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
0607e439 1308{
12db03b6 1309 __tcf_block_put(block, NULL, NULL, rtnl_held);
0607e439
VB		 1310}
1311
c431f89b
VB		 1312/* Find tcf block.
1313 * Set q, parent, cl when appropriate.
1314 */
1315
1316static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1317 u32 *parent, unsigned long *cl,
1318 int ifindex, u32 block_index,
1319 struct netlink_ext_ack *extack)
1320{
1321 struct tcf_block *block;
e368fdb6 1322 int err = 0;
c431f89b 1323
18d3eefb 1324 ASSERT_RTNL();
e368fdb6 1325
18d3eefb
VB		 1326 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1327 if (err)
1328 goto errout;
c431f89b 1329
18d3eefb
VB		 1330 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1331 if (err)
1332 goto errout_qdisc;
787ce6d0 1333
18d3eefb 1334 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
af736bf0
DC		 1335 if (IS_ERR(block)) {
1336 err = PTR_ERR(block);
18d3eefb 1337 goto errout_qdisc;
af736bf0 1338 }
c431f89b
VB		 1339
1340 return block;
e368fdb6 1341
e368fdb6 1342errout_qdisc:
18d3eefb 1343 if (*q)
e368fdb6 1344 qdisc_put(*q);
18d3eefb
VB		 1345errout:
1346 *q = NULL;
e368fdb6
VB		 1347 return ERR_PTR(err);
1348}
1349
12db03b6
VB		 1350static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1351 bool rtnl_held)
e368fdb6 1352{
787ce6d0 1353 if (!IS_ERR_OR_NULL(block))
12db03b6 1354 tcf_block_refcnt_put(block, rtnl_held);
787ce6d0 1355
470502de
VB		 1356 if (q) {
1357 if (rtnl_held)
1358 qdisc_put(q);
1359 else
1360 qdisc_put_unlocked(q);
1361 }
c431f89b
VB		 1362}
1363
f36fe1c4
JP		 1364struct tcf_block_owner_item {
1365 struct list_head list;
1366 struct Qdisc *q;
32f8c409 1367 enum flow_block_binder_type binder_type;
f36fe1c4
JP		 1368};
1369
1370static void
1371tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1372 struct Qdisc *q,
32f8c409 1373 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1374{
1375 if (block->keep_dst &&
32f8c409
PNA		 1376 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1377 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
f36fe1c4
JP		 1378 netif_keep_dst(qdisc_dev(q));
1379}
1380
1381void tcf_block_netif_keep_dst(struct tcf_block *block)
1382{
1383 struct tcf_block_owner_item *item;
1384
1385 block->keep_dst = true;
1386 list_for_each_entry(item, &block->owner_list, list)
1387 tcf_block_owner_netif_keep_dst(block, item->q,
1388 item->binder_type);
1389}
1390EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1391
1392static int tcf_block_owner_add(struct tcf_block *block,
1393 struct Qdisc *q,
32f8c409 1394 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1395{
1396 struct tcf_block_owner_item *item;
1397
1398 item = kmalloc(sizeof(*item), GFP_KERNEL);
1399 if (!item)
1400 return -ENOMEM;
1401 item->q = q;
1402 item->binder_type = binder_type;
1403 list_add(&item->list, &block->owner_list);
1404 return 0;
1405}
1406
1407static void tcf_block_owner_del(struct tcf_block *block,
1408 struct Qdisc *q,
32f8c409 1409 enum flow_block_binder_type binder_type)
f36fe1c4
JP		 1410{
1411 struct tcf_block_owner_item *item;
1412
1413 list_for_each_entry(item, &block->owner_list, list) {
1414 if (item->q == q && item->binder_type == binder_type) {
1415 list_del(&item->list);
1416 kfree(item);
1417 return;
1418 }
1419 }
1420 WARN_ON(1);
1421}
1422
48617387
JP		 1423int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1424 struct tcf_block_ext_info *ei,
1425 struct netlink_ext_ack *extack)
1426{
1427 struct net *net = qdisc_net(q);
1428 struct tcf_block *block = NULL;
48617387
JP		 1429 int err;
1430
787ce6d0 1431 if (ei->block_index)
48617387 1432 /* block_index not 0 means the shared block is requested */
787ce6d0 1433 block = tcf_block_refcnt_get(net, ei->block_index);
48617387
JP		 1434
1435 if (!block) {
bb047ddd 1436 block = tcf_block_create(net, q, ei->block_index, extack);
48617387
JP		 1437 if (IS_ERR(block))
1438 return PTR_ERR(block);
bb047ddd
JP		 1439 if (tcf_block_shared(block)) {
1440 err = tcf_block_insert(block, net, extack);
48617387
JP		 1441 if (err)
1442 goto err_block_insert;
1443 }
1444 }
1445
f36fe1c4
JP		 1446 err = tcf_block_owner_add(block, q, ei->binder_type);
1447 if (err)
1448 goto err_block_owner_add;
1449
1450 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1451
f71e0ca4 1452 err = tcf_chain0_head_change_cb_add(block, ei, extack);
a9b19443 1453 if (err)
f71e0ca4 1454 goto err_chain0_head_change_cb_add;
caa72601 1455
60513bd8 1456 err = tcf_block_offload_bind(block, q, ei, extack);
caa72601
JP		 1457 if (err)
1458 goto err_block_offload_bind;
1459
6529eaba
JP		 1460 *p_block = block;
1461 return 0;
2190d1d0 1462
caa72601 1463err_block_offload_bind:
f71e0ca4
JP		 1464 tcf_chain0_head_change_cb_del(block, ei);
1465err_chain0_head_change_cb_add:
f36fe1c4
JP		 1466 tcf_block_owner_del(block, q, ei->binder_type);
1467err_block_owner_add:
48617387 1468err_block_insert:
12db03b6 1469 tcf_block_refcnt_put(block, true);
2190d1d0 1470 return err;
6529eaba 1471}
8c4083b3
JP		 1472EXPORT_SYMBOL(tcf_block_get_ext);
1473
c7eb7d72
JP		 1474static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1475{
1476 struct tcf_proto __rcu **p_filter_chain = priv;
1477
1478 rcu_assign_pointer(*p_filter_chain, tp_head);
1479}
1480
8c4083b3 1481int tcf_block_get(struct tcf_block **p_block,
8d1a77f9
AA		 1482 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1483 struct netlink_ext_ack *extack)
8c4083b3 1484{
c7eb7d72
JP		 1485 struct tcf_block_ext_info ei = {
1486 .chain_head_change = tcf_chain_head_change_dflt,
1487 .chain_head_change_priv = p_filter_chain,
1488 };
8c4083b3 1489
c7eb7d72 1490 WARN_ON(!p_filter_chain);
8d1a77f9 1491 return tcf_block_get_ext(p_block, q, &ei, extack);
8c4083b3 1492}
6529eaba
JP		 1493EXPORT_SYMBOL(tcf_block_get);
1494
7aa0045d 1495/* XXX: Standalone actions are not allowed to jump to any chain, and bound
a60b3f51 1496 * actions should be all removed after flushing.
7aa0045d 1497 */
c7eb7d72 1498void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
e1ea2f98 1499 struct tcf_block_ext_info *ei)
7aa0045d 1500{
c30abd5e
DM		 1501 if (!block)
1502 return;
f71e0ca4 1503 tcf_chain0_head_change_cb_del(block, ei);
f36fe1c4 1504 tcf_block_owner_del(block, q, ei->binder_type);
a60b3f51 1505
12db03b6 1506 __tcf_block_put(block, q, ei, true);
6529eaba 1507}
8c4083b3
JP		 1508EXPORT_SYMBOL(tcf_block_put_ext);
1509
1510void tcf_block_put(struct tcf_block *block)
1511{
1512 struct tcf_block_ext_info ei = {0, };
1513
4853f128
JP		 1514 if (!block)
1515 return;
c7eb7d72 1516 tcf_block_put_ext(block, block->q, &ei);
8c4083b3 1517}
e1ea2f98 1518
6529eaba 1519EXPORT_SYMBOL(tcf_block_put);
cf1facda 1520
32636742 1521static int
a7323311 1522tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
32636742
JH		 1523 void *cb_priv, bool add, bool offload_in_use,
1524 struct netlink_ext_ack *extack)
1525{
bbf73830 1526 struct tcf_chain *chain, *chain_prev;
fe2923af 1527 struct tcf_proto *tp, *tp_prev;
32636742
JH		 1528 int err;
1529
bbf73830
VB		 1530 for (chain = __tcf_get_next_chain(block, NULL);
1531 chain;
1532 chain_prev = chain,
1533 chain = __tcf_get_next_chain(block, chain),
1534 tcf_chain_put(chain_prev)) {
fe2923af
VB		 1535 for (tp = __tcf_get_next_proto(chain, NULL); tp;
1536 tp_prev = tp,
1537 tp = __tcf_get_next_proto(chain, tp),
12db03b6 1538 tcf_proto_put(tp_prev, true, NULL)) {
32636742
JH		 1539 if (tp->ops->reoffload) {
1540 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1541 extack);
1542 if (err && add)
1543 goto err_playback_remove;
1544 } else if (add && offload_in_use) {
1545 err = -EOPNOTSUPP;
1546 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1547 goto err_playback_remove;
1548 }
1549 }
1550 }
1551
1552 return 0;
1553
1554err_playback_remove:
12db03b6 1555 tcf_proto_put(tp, true, NULL);
bbf73830 1556 tcf_chain_put(chain);
32636742
JH		 1557 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1558 extack);
1559 return err;
1560}
1561
59094b1e
PNA		 1562static int tcf_block_bind(struct tcf_block *block,
1563 struct flow_block_offload *bo)
1564{
1565 struct flow_block_cb *block_cb, *next;
1566 int err, i = 0;
1567
1568 list_for_each_entry(block_cb, &bo->cb_list, list) {
1569 err = tcf_block_playback_offloads(block, block_cb->cb,
1570 block_cb->cb_priv, true,
1571 tcf_block_offload_in_use(block),
1572 bo->extack);
1573 if (err)
1574 goto err_unroll;
1575
1576 i++;
1577 }
14bfb13f 1578 list_splice(&bo->cb_list, &block->flow_block.cb_list);
59094b1e
PNA		 1579
1580 return 0;
1581
1582err_unroll:
1583 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1584 if (i-- > 0) {
1585 list_del(&block_cb->list);
1586 tcf_block_playback_offloads(block, block_cb->cb,
1587 block_cb->cb_priv, false,
1588 tcf_block_offload_in_use(block),
1589 NULL);
1590 }
1591 flow_block_cb_free(block_cb);
1592 }
1593
1594 return err;
1595}
1596
1597static void tcf_block_unbind(struct tcf_block *block,
1598 struct flow_block_offload *bo)
1599{
1600 struct flow_block_cb *block_cb, *next;
1601
1602 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1603 tcf_block_playback_offloads(block, block_cb->cb,
1604 block_cb->cb_priv, false,
1605 tcf_block_offload_in_use(block),
1606 NULL);
1607 list_del(&block_cb->list);
1608 flow_block_cb_free(block_cb);
1609 }
1610}
1611
1612static int tcf_block_setup(struct tcf_block *block,
1613 struct flow_block_offload *bo)
1614{
1615 int err;
1616
1617 switch (bo->command) {
1618 case FLOW_BLOCK_BIND:
1619 err = tcf_block_bind(block, bo);
1620 break;
1621 case FLOW_BLOCK_UNBIND:
1622 err = 0;
1623 tcf_block_unbind(block, bo);
1624 break;
1625 default:
1626 WARN_ON_ONCE(1);
1627 err = -EOPNOTSUPP;
1628 }
1629
1630 return err;
1631}
1632
87d83093
JP		 1633/* Main classifier routine: scans classifier chain attached
1634 * to this qdisc, (optionally) tests for protocol and asks
1635 * specific classifiers.
1636 */
1637int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1638 struct tcf_result *res, bool compat_mode)
1639{
87d83093
JP		 1640#ifdef CONFIG_NET_CLS_ACT
1641 const int max_reclassify_loop = 4;
ee538dce
JP		 1642 const struct tcf_proto *orig_tp = tp;
1643 const struct tcf_proto *first_tp;
87d83093
JP		 1644 int limit = 0;
1645
1646reclassify:
1647#endif
1648 for (; tp; tp = rcu_dereference_bh(tp->next)) {
cd0c4e70 1649 __be16 protocol = tc_skb_protocol(skb);
87d83093
JP		 1650 int err;
1651
1652 if (tp->protocol != protocol &&
1653 tp->protocol != htons(ETH_P_ALL))
1654 continue;
1655
1656 err = tp->classify(skb, tp, res);
1657#ifdef CONFIG_NET_CLS_ACT
db50514f 1658 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
ee538dce 1659 first_tp = orig_tp;
87d83093 1660 goto reset;
db50514f 1661 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
ee538dce 1662 first_tp = res->goto_tp;
db50514f
JP		 1663 goto reset;
1664 }
87d83093
JP		 1665#endif
1666 if (err >= 0)
1667 return err;
1668 }
1669
1670 return TC_ACT_UNSPEC; /* signal: continue lookup */
1671#ifdef CONFIG_NET_CLS_ACT
1672reset:
1673 if (unlikely(limit++ >= max_reclassify_loop)) {
9d3aaff3
JP		 1674 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1675 tp->chain->block->index,
1676 tp->prio & 0xffff,
87d83093
JP		 1677 ntohs(tp->protocol));
1678 return TC_ACT_SHOT;
1679 }
1680
ee538dce 1681 tp = first_tp;
87d83093
JP		 1682 goto reclassify;
1683#endif
1684}
1685EXPORT_SYMBOL(tcf_classify);
1686
2190d1d0
JP		 1687struct tcf_chain_info {
1688 struct tcf_proto __rcu **pprev;
1689 struct tcf_proto __rcu *next;
1690};
1691
ed76f5ed
VB		 1692static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1693 struct tcf_chain_info *chain_info)
2190d1d0 1694{
ed76f5ed 1695 return tcf_chain_dereference(*chain_info->pprev, chain);
2190d1d0
JP		 1696}
1697
726d0612
VB		 1698static int tcf_chain_tp_insert(struct tcf_chain *chain,
1699 struct tcf_chain_info *chain_info,
1700 struct tcf_proto *tp)
2190d1d0 1701{
726d0612
VB		 1702 if (chain->flushing)
1703 return -EAGAIN;
1704
c7eb7d72 1705 if (*chain_info->pprev == chain->filter_chain)
f71e0ca4 1706 tcf_chain0_head_change(chain, tp);
4dbfa766 1707 tcf_proto_get(tp);
ed76f5ed 1708 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
2190d1d0 1709 rcu_assign_pointer(*chain_info->pprev, tp);
726d0612
VB		 1710
1711 return 0;
2190d1d0
JP		 1712}
1713
1714static void tcf_chain_tp_remove(struct tcf_chain *chain,
1715 struct tcf_chain_info *chain_info,
1716 struct tcf_proto *tp)
1717{
ed76f5ed 1718 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
2190d1d0 1719
8b64678e 1720 tcf_proto_mark_delete(tp);
c7eb7d72 1721 if (tp == chain->filter_chain)
f71e0ca4 1722 tcf_chain0_head_change(chain, next);
2190d1d0
JP		 1723 RCU_INIT_POINTER(*chain_info->pprev, next);
1724}
1725
8b64678e
VB		 1726static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1727 struct tcf_chain_info *chain_info,
1728 u32 protocol, u32 prio,
1729 bool prio_allocate);
1730
1731/* Try to insert new proto.
1732 * If proto with specified priority already exists, free new proto
1733 * and return existing one.
1734 */
1735
1736static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1737 struct tcf_proto *tp_new,
12db03b6
VB		 1738 u32 protocol, u32 prio,
1739 bool rtnl_held)
8b64678e
VB		 1740{
1741 struct tcf_chain_info chain_info;
1742 struct tcf_proto *tp;
726d0612 1743 int err = 0;
8b64678e
VB		 1744
1745 mutex_lock(&chain->filter_chain_lock);
1746
1747 tp = tcf_chain_tp_find(chain, &chain_info,
1748 protocol, prio, false);
1749 if (!tp)
726d0612 1750 err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
8b64678e
VB		 1751 mutex_unlock(&chain->filter_chain_lock);
1752
1753 if (tp) {
12db03b6 1754 tcf_proto_destroy(tp_new, rtnl_held, NULL);
8b64678e 1755 tp_new = tp;
726d0612 1756 } else if (err) {
12db03b6 1757 tcf_proto_destroy(tp_new, rtnl_held, NULL);
726d0612 1758 tp_new = ERR_PTR(err);
8b64678e
VB		 1759 }
1760
1761 return tp_new;
1762}
1763
1764static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
12db03b6 1765 struct tcf_proto *tp, bool rtnl_held,
8b64678e
VB		 1766 struct netlink_ext_ack *extack)
1767{
1768 struct tcf_chain_info chain_info;
1769 struct tcf_proto *tp_iter;
1770 struct tcf_proto **pprev;
1771 struct tcf_proto *next;
1772
1773 mutex_lock(&chain->filter_chain_lock);
1774
1775 /* Atomically find and remove tp from chain. */
1776 for (pprev = &chain->filter_chain;
1777 (tp_iter = tcf_chain_dereference(*pprev, chain));
1778 pprev = &tp_iter->next) {
1779 if (tp_iter == tp) {
1780 chain_info.pprev = pprev;
1781 chain_info.next = tp_iter->next;
1782 WARN_ON(tp_iter->deleting);
1783 break;
1784 }
1785 }
1786 /* Verify that tp still exists and no new filters were inserted
1787 * concurrently.
1788 * Mark tp for deletion if it is empty.
1789 */
12db03b6 1790 if (!tp_iter || !tcf_proto_check_delete(tp, rtnl_held)) {
8b64678e
VB		 1791 mutex_unlock(&chain->filter_chain_lock);
1792 return;
1793 }
1794
1795 next = tcf_chain_dereference(chain_info.next, chain);
1796 if (tp == chain->filter_chain)
1797 tcf_chain0_head_change(chain, next);
1798 RCU_INIT_POINTER(*chain_info.pprev, next);
1799 mutex_unlock(&chain->filter_chain_lock);
1800
12db03b6 1801 tcf_proto_put(tp, rtnl_held, extack);
8b64678e
VB		 1802}
1803
2190d1d0
JP		 1804static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1805 struct tcf_chain_info *chain_info,
1806 u32 protocol, u32 prio,
1807 bool prio_allocate)
1808{
1809 struct tcf_proto **pprev;
1810 struct tcf_proto *tp;
1811
1812 /* Check the chain for existence of proto-tcf with this priority */
1813 for (pprev = &chain->filter_chain;
ed76f5ed
VB		 1814 (tp = tcf_chain_dereference(*pprev, chain));
1815 pprev = &tp->next) {
2190d1d0
JP		 1816 if (tp->prio >= prio) {
1817 if (tp->prio == prio) {
1818 if (prio_allocate ||
1819 (tp->protocol != protocol && protocol))
1820 return ERR_PTR(-EINVAL);
1821 } else {
1822 tp = NULL;
1823 }
1824 break;
1825 }
1826 }
1827 chain_info->pprev = pprev;
4dbfa766
VB		 1828 if (tp) {
1829 chain_info->next = tp->next;
1830 tcf_proto_get(tp);
1831 } else {
1832 chain_info->next = NULL;
1833 }
2190d1d0
JP		 1834 return tp;
1835}
1836
7120371c 1837static int tcf_fill_node(struct net *net, struct sk_buff *skb,
7960d1da
JP		 1838 struct tcf_proto *tp, struct tcf_block *block,
1839 struct Qdisc *q, u32 parent, void *fh,
12db03b6
VB		 1840 u32 portid, u32 seq, u16 flags, int event,
1841 bool rtnl_held)
7120371c
WC		 1842{
1843 struct tcmsg *tcm;
1844 struct nlmsghdr *nlh;
1845 unsigned char *b = skb_tail_pointer(skb);
1846
1847 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1848 if (!nlh)
1849 goto out_nlmsg_trim;
1850 tcm = nlmsg_data(nlh);
1851 tcm->tcm_family = AF_UNSPEC;
1852 tcm->tcm__pad1 = 0;
1853 tcm->tcm__pad2 = 0;
7960d1da
JP		 1854 if (q) {
1855 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1856 tcm->tcm_parent = parent;
1857 } else {
1858 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1859 tcm->tcm_block_index = block->index;
1860 }
7120371c
WC		 1861 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1862 if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1863 goto nla_put_failure;
1864 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1865 goto nla_put_failure;
1866 if (!fh) {
1867 tcm->tcm_handle = 0;
1868 } else {
12db03b6
VB		 1869 if (tp->ops->dump &&
1870 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
7120371c
WC		 1871 goto nla_put_failure;
1872 }
1873 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1874 return skb->len;
1875
1876out_nlmsg_trim:
1877nla_put_failure:
1878 nlmsg_trim(skb, b);
1879 return -1;
1880}
1881
1882static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1883 struct nlmsghdr *n, struct tcf_proto *tp,
7960d1da 1884 struct tcf_block *block, struct Qdisc *q,
12db03b6
VB		 1885 u32 parent, void *fh, int event, bool unicast,
1886 bool rtnl_held)
7120371c
WC		 1887{
1888 struct sk_buff *skb;
1889 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
5b5f99b1 1890 int err = 0;
7120371c
WC		 1891
1892 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1893 if (!skb)
1894 return -ENOBUFS;
1895
7960d1da 1896 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
12db03b6
VB		 1897 n->nlmsg_seq, n->nlmsg_flags, event,
1898 rtnl_held) <= 0) {
7120371c
WC		 1899 kfree_skb(skb);
1900 return -EINVAL;
1901 }
1902
1903 if (unicast)
5b5f99b1
ZW		 1904 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1905 else
1906 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1907 n->nlmsg_flags & NLM_F_ECHO);
7120371c 1908
5b5f99b1
ZW		 1909 if (err > 0)
1910 err = 0;
1911 return err;
7120371c
WC		 1912}
1913
1914static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1915 struct nlmsghdr *n, struct tcf_proto *tp,
7960d1da 1916 struct tcf_block *block, struct Qdisc *q,
c35a4acc 1917 u32 parent, void *fh, bool unicast, bool *last,
12db03b6 1918 bool rtnl_held, struct netlink_ext_ack *extack)
7120371c
WC		 1919{
1920 struct sk_buff *skb;
1921 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1922 int err;
1923
1924 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1925 if (!skb)
1926 return -ENOBUFS;
1927
7960d1da 1928 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
12db03b6
VB		 1929 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER,
1930 rtnl_held) <= 0) {
c35a4acc 1931 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
7120371c
WC		 1932 kfree_skb(skb);
1933 return -EINVAL;
1934 }
1935
12db03b6 1936 err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
7120371c
WC		 1937 if (err) {
1938 kfree_skb(skb);
1939 return err;
1940 }
1941
1942 if (unicast)
5b5f99b1
ZW		 1943 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1944 else
1945 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1946 n->nlmsg_flags & NLM_F_ECHO);
c35a4acc
AA		 1947 if (err < 0)
1948 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
5b5f99b1
ZW		 1949
1950 if (err > 0)
1951 err = 0;
c35a4acc 1952 return err;
7120371c
WC		 1953}
1954
1955static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
7960d1da
JP		 1956 struct tcf_block *block, struct Qdisc *q,
1957 u32 parent, struct nlmsghdr *n,
12db03b6
VB		 1958 struct tcf_chain *chain, int event,
1959 bool rtnl_held)
7120371c
WC		 1960{
1961 struct tcf_proto *tp;
1962
12db03b6
VB		 1963 for (tp = tcf_get_next_proto(chain, NULL, rtnl_held);
1964 tp; tp = tcf_get_next_proto(chain, tp, rtnl_held))
7960d1da 1965 tfilter_notify(net, oskb, n, tp, block,
12db03b6 1966 q, parent, NULL, event, false, rtnl_held);
7120371c
WC		 1967}
1968
7d5509fa
VB		 1969static void tfilter_put(struct tcf_proto *tp, void *fh)
1970{
1971 if (tp->ops->put && fh)
1972 tp->ops->put(tp, fh);
1973}
1974
c431f89b 1975static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
c21ef3e3 1976 struct netlink_ext_ack *extack)
1da177e4 1977{
3b1e0a65 1978 struct net *net = sock_net(skb->sk);
add93b61 1979 struct nlattr *tca[TCA_MAX + 1];
1da177e4
LT		 1980 struct tcmsg *t;
1981 u32 protocol;
1982 u32 prio;
9d36d9e5 1983 bool prio_allocate;
1da177e4 1984 u32 parent;
5bc17018 1985 u32 chain_index;
7960d1da 1986 struct Qdisc *q = NULL;
2190d1d0 1987 struct tcf_chain_info chain_info;
5bc17018 1988 struct tcf_chain *chain = NULL;
6529eaba 1989 struct tcf_block *block;
1da177e4 1990 struct tcf_proto *tp;
1da177e4 1991 unsigned long cl;
8113c095 1992 void *fh;
1da177e4 1993 int err;
628185cf 1994 int tp_created;
470502de 1995 bool rtnl_held = false;
1da177e4 1996
c431f89b 1997 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
dfc47ef8 1998 return -EPERM;
de179c8c 1999
1da177e4 2000replay:
628185cf
DB		 2001 tp_created = 0;
2002
8cb08174
JB		 2003 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2004 rtm_tca_policy, extack);
de179c8c
H		 2005 if (err < 0)
2006 return err;
2007
942b8165 2008 t = nlmsg_data(n);
1da177e4
LT		 2009 protocol = TC_H_MIN(t->tcm_info);
2010 prio = TC_H_MAJ(t->tcm_info);
9d36d9e5 2011 prio_allocate = false;
1da177e4 2012 parent = t->tcm_parent;
4dbfa766 2013 tp = NULL;
1da177e4 2014 cl = 0;
470502de 2015 block = NULL;
1da177e4
LT		 2016
2017 if (prio == 0) {
c431f89b
VB		 2018 /* If no priority is provided by the user,
2019 * we allocate one.
2020 */
2021 if (n->nlmsg_flags & NLM_F_CREATE) {
2022 prio = TC_H_MAKE(0x80000000U, 0U);
2023 prio_allocate = true;
2024 } else {
c35a4acc 2025 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1da177e4 2026 return -ENOENT;
ea7f8277 2027 }
1da177e4
LT		 2028 }
2029
2030 /* Find head of filter chain. */
2031
470502de
VB		 2032 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2033 if (err)
2034 return err;
2035
2036 /* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2037 * block is shared (no qdisc found), qdisc is not unlocked, classifier
2038 * type is not specified, classifier is not unlocked.
2039 */
2040 if (rtnl_held ||
2041 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2042 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) {
2043 rtnl_held = true;
2044 rtnl_lock();
2045 }
2046
2047 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2048 if (err)
2049 goto errout;
2050
2051 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2052 extack);
c431f89b
VB		 2053 if (IS_ERR(block)) {
2054 err = PTR_ERR(block);
2055 goto errout;
6bb16e7a 2056 }
5bc17018
JP		 2057
2058 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2059 if (chain_index > TC_ACT_EXT_VAL_MASK) {
c35a4acc 2060 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
5bc17018
JP		 2061 err = -EINVAL;
2062 goto errout;
2063 }
c431f89b 2064 chain = tcf_chain_get(block, chain_index, true);
5bc17018 2065 if (!chain) {
d5ed72a5 2066 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
c431f89b 2067 err = -ENOMEM;
ea7f8277
DB		 2068 goto errout;
2069 }
1da177e4 2070
ed76f5ed 2071 mutex_lock(&chain->filter_chain_lock);
2190d1d0
JP		 2072 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2073 prio, prio_allocate);
2074 if (IS_ERR(tp)) {
c35a4acc 2075 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2190d1d0 2076 err = PTR_ERR(tp);
ed76f5ed 2077 goto errout_locked;
1da177e4
LT		 2078 }
2079
2080 if (tp == NULL) {
8b64678e
VB		 2081 struct tcf_proto *tp_new = NULL;
2082
726d0612
VB		 2083 if (chain->flushing) {
2084 err = -EAGAIN;
2085 goto errout_locked;
2086 }
2087
1da177e4
LT		 2088 /* Proto-tcf does not exist, create new one */
2089
6bb16e7a 2090 if (tca[TCA_KIND] == NULL || !protocol) {
c35a4acc 2091 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
6bb16e7a 2092 err = -EINVAL;
ed76f5ed 2093 goto errout_locked;
6bb16e7a 2094 }
1da177e4 2095
c431f89b 2096 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
c35a4acc 2097 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
6bb16e7a 2098 err = -ENOENT;
ed76f5ed 2099 goto errout_locked;
6bb16e7a 2100 }
1da177e4 2101
9d36d9e5 2102 if (prio_allocate)
ed76f5ed
VB		 2103 prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2104 &chain_info));
1da177e4 2105
ed76f5ed 2106 mutex_unlock(&chain->filter_chain_lock);
8b64678e 2107 tp_new = tcf_proto_create(nla_data(tca[TCA_KIND]),
12db03b6
VB		 2108 protocol, prio, chain, rtnl_held,
2109 extack);
8b64678e
VB		 2110 if (IS_ERR(tp_new)) {
2111 err = PTR_ERR(tp_new);
726d0612 2112 goto errout_tp;
1da177e4 2113 }
ed76f5ed 2114
12186be7 2115 tp_created = 1;
12db03b6
VB		 2116 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2117 rtnl_held);
726d0612
VB		 2118 if (IS_ERR(tp)) {
2119 err = PTR_ERR(tp);
2120 goto errout_tp;
2121 }
ed76f5ed
VB		 2122 } else {
2123 mutex_unlock(&chain->filter_chain_lock);
6bb16e7a 2124 }
1da177e4 2125
8b64678e
VB		 2126 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2127 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2128 err = -EINVAL;
2129 goto errout;
2130 }
2131
1da177e4
LT		 2132 fh = tp->ops->get(tp, t->tcm_handle);
2133
8113c095 2134 if (!fh) {
c431f89b 2135 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
c35a4acc 2136 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
6bb16e7a 2137 err = -ENOENT;
1da177e4 2138 goto errout;
6bb16e7a 2139 }
c431f89b 2140 } else if (n->nlmsg_flags & NLM_F_EXCL) {
7d5509fa 2141 tfilter_put(tp, fh);
c431f89b
VB		 2142 NL_SET_ERR_MSG(extack, "Filter already exists");
2143 err = -EEXIST;
2144 goto errout;
1da177e4
LT		 2145 }
2146
9f407f17
JP		 2147 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2148 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2149 err = -EINVAL;
2150 goto errout;
2151 }
2152
2f7ef2f8 2153 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
7306db38 2154 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
12db03b6 2155 rtnl_held, extack);
7d5509fa 2156 if (err == 0) {
7960d1da 2157 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
12db03b6 2158 RTM_NEWTFILTER, false, rtnl_held);
7d5509fa 2159 tfilter_put(tp, fh);
503d81d4
VB		 2160 /* q pointer is NULL for shared blocks */
2161 if (q)
2162 q->flags &= ~TCQ_F_CAN_BYPASS;
7d5509fa 2163 }
1da177e4
LT		 2164
2165errout:
8b64678e 2166 if (err && tp_created)
12db03b6 2167 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
726d0612 2168errout_tp:
4dbfa766
VB		 2169 if (chain) {
2170 if (tp && !IS_ERR(tp))
12db03b6 2171 tcf_proto_put(tp, rtnl_held, NULL);
4dbfa766
VB		 2172 if (!tp_created)
2173 tcf_chain_put(chain);
2174 }
12db03b6 2175 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2176
2177 if (rtnl_held)
2178 rtnl_unlock();
2179
2180 if (err == -EAGAIN) {
2181 /* Take rtnl lock in case EAGAIN is caused by concurrent flush
2182 * of target chain.
2183 */
2184 rtnl_held = true;
1da177e4
LT		 2185 /* Replay the request. */
2186 goto replay;
470502de 2187 }
1da177e4 2188 return err;
ed76f5ed
VB		 2189
2190errout_locked:
2191 mutex_unlock(&chain->filter_chain_lock);
2192 goto errout;
1da177e4
LT		 2193}
2194
c431f89b
VB		 2195static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2196 struct netlink_ext_ack *extack)
2197{
2198 struct net *net = sock_net(skb->sk);
2199 struct nlattr *tca[TCA_MAX + 1];
2200 struct tcmsg *t;
2201 u32 protocol;
2202 u32 prio;
2203 u32 parent;
2204 u32 chain_index;
2205 struct Qdisc *q = NULL;
2206 struct tcf_chain_info chain_info;
2207 struct tcf_chain *chain = NULL;
470502de 2208 struct tcf_block *block = NULL;
c431f89b
VB		 2209 struct tcf_proto *tp = NULL;
2210 unsigned long cl = 0;
2211 void *fh = NULL;
2212 int err;
470502de 2213 bool rtnl_held = false;
c431f89b
VB		 2214
2215 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2216 return -EPERM;
2217
8cb08174
JB		 2218 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2219 rtm_tca_policy, extack);
c431f89b
VB		 2220 if (err < 0)
2221 return err;
2222
2223 t = nlmsg_data(n);
2224 protocol = TC_H_MIN(t->tcm_info);
2225 prio = TC_H_MAJ(t->tcm_info);
2226 parent = t->tcm_parent;
2227
2228 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2229 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2230 return -ENOENT;
2231 }
2232
2233 /* Find head of filter chain. */
2234
470502de
VB		 2235 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2236 if (err)
2237 return err;
2238
2239 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2240 * found), qdisc is not unlocked, classifier type is not specified,
2241 * classifier is not unlocked.
2242 */
2243 if (!prio ||
2244 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2245 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) {
2246 rtnl_held = true;
2247 rtnl_lock();
2248 }
2249
2250 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2251 if (err)
2252 goto errout;
2253
2254 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2255 extack);
c431f89b
VB		 2256 if (IS_ERR(block)) {
2257 err = PTR_ERR(block);
2258 goto errout;
2259 }
2260
2261 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2262 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2263 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2264 err = -EINVAL;
2265 goto errout;
2266 }
2267 chain = tcf_chain_get(block, chain_index, false);
2268 if (!chain) {
5ca8a25c
JP		 2269 /* User requested flush on non-existent chain. Nothing to do,
2270 * so just return success.
2271 */
2272 if (prio == 0) {
2273 err = 0;
2274 goto errout;
2275 }
c431f89b 2276 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
b7b4247d 2277 err = -ENOENT;
c431f89b
VB		 2278 goto errout;
2279 }
2280
2281 if (prio == 0) {
2282 tfilter_notify_chain(net, skb, block, q, parent, n,
12db03b6
VB		 2283 chain, RTM_DELTFILTER, rtnl_held);
2284 tcf_chain_flush(chain, rtnl_held);
c431f89b
VB		 2285 err = 0;
2286 goto errout;
2287 }
2288
ed76f5ed 2289 mutex_lock(&chain->filter_chain_lock);
c431f89b
VB		 2290 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2291 prio, false);
2292 if (!tp || IS_ERR(tp)) {
2293 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
0e399035 2294 err = tp ? PTR_ERR(tp) : -ENOENT;
ed76f5ed 2295 goto errout_locked;
c431f89b
VB		 2296 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2297 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2298 err = -EINVAL;
ed76f5ed
VB		 2299 goto errout_locked;
2300 } else if (t->tcm_handle == 0) {
2301 tcf_chain_tp_remove(chain, &chain_info, tp);
2302 mutex_unlock(&chain->filter_chain_lock);
2303
12db03b6 2304 tcf_proto_put(tp, rtnl_held, NULL);
ed76f5ed 2305 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
12db03b6 2306 RTM_DELTFILTER, false, rtnl_held);
ed76f5ed 2307 err = 0;
c431f89b
VB		 2308 goto errout;
2309 }
ed76f5ed 2310 mutex_unlock(&chain->filter_chain_lock);
c431f89b
VB		 2311
2312 fh = tp->ops->get(tp, t->tcm_handle);
2313
2314 if (!fh) {
ed76f5ed
VB		 2315 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2316 err = -ENOENT;
c431f89b
VB		 2317 } else {
2318 bool last;
2319
2320 err = tfilter_del_notify(net, skb, n, tp, block,
2321 q, parent, fh, false, &last,
12db03b6
VB		 2322 rtnl_held, extack);
2323
c431f89b
VB		 2324 if (err)
2325 goto errout;
8b64678e 2326 if (last)
12db03b6 2327 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
c431f89b
VB		 2328 }
2329
2330errout:
4dbfa766
VB		 2331 if (chain) {
2332 if (tp && !IS_ERR(tp))
12db03b6 2333 tcf_proto_put(tp, rtnl_held, NULL);
c431f89b 2334 tcf_chain_put(chain);
4dbfa766 2335 }
12db03b6 2336 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2337
2338 if (rtnl_held)
2339 rtnl_unlock();
2340
c431f89b 2341 return err;
ed76f5ed
VB		 2342
2343errout_locked:
2344 mutex_unlock(&chain->filter_chain_lock);
2345 goto errout;
c431f89b
VB		 2346}
2347
2348static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2349 struct netlink_ext_ack *extack)
2350{
2351 struct net *net = sock_net(skb->sk);
2352 struct nlattr *tca[TCA_MAX + 1];
2353 struct tcmsg *t;
2354 u32 protocol;
2355 u32 prio;
2356 u32 parent;
2357 u32 chain_index;
2358 struct Qdisc *q = NULL;
2359 struct tcf_chain_info chain_info;
2360 struct tcf_chain *chain = NULL;
470502de 2361 struct tcf_block *block = NULL;
c431f89b
VB		 2362 struct tcf_proto *tp = NULL;
2363 unsigned long cl = 0;
2364 void *fh = NULL;
2365 int err;
470502de 2366 bool rtnl_held = false;
c431f89b 2367
8cb08174
JB		 2368 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2369 rtm_tca_policy, extack);
c431f89b
VB		 2370 if (err < 0)
2371 return err;
2372
2373 t = nlmsg_data(n);
2374 protocol = TC_H_MIN(t->tcm_info);
2375 prio = TC_H_MAJ(t->tcm_info);
2376 parent = t->tcm_parent;
2377
2378 if (prio == 0) {
2379 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2380 return -ENOENT;
2381 }
2382
2383 /* Find head of filter chain. */
2384
470502de
VB		 2385 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2386 if (err)
2387 return err;
2388
2389 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2390 * unlocked, classifier type is not specified, classifier is not
2391 * unlocked.
2392 */
2393 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2394 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) {
2395 rtnl_held = true;
2396 rtnl_lock();
2397 }
2398
2399 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2400 if (err)
2401 goto errout;
2402
2403 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2404 extack);
c431f89b
VB		 2405 if (IS_ERR(block)) {
2406 err = PTR_ERR(block);
2407 goto errout;
2408 }
2409
2410 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2411 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2412 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2413 err = -EINVAL;
2414 goto errout;
2415 }
2416 chain = tcf_chain_get(block, chain_index, false);
2417 if (!chain) {
2418 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2419 err = -EINVAL;
2420 goto errout;
2421 }
2422
ed76f5ed 2423 mutex_lock(&chain->filter_chain_lock);
c431f89b
VB		 2424 tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2425 prio, false);
ed76f5ed 2426 mutex_unlock(&chain->filter_chain_lock);
c431f89b
VB		 2427 if (!tp || IS_ERR(tp)) {
2428 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
0e399035 2429 err = tp ? PTR_ERR(tp) : -ENOENT;
c431f89b
VB		 2430 goto errout;
2431 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2432 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2433 err = -EINVAL;
2434 goto errout;
2435 }
2436
2437 fh = tp->ops->get(tp, t->tcm_handle);
2438
2439 if (!fh) {
2440 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2441 err = -ENOENT;
2442 } else {
2443 err = tfilter_notify(net, skb, n, tp, block, q, parent,
12db03b6 2444 fh, RTM_NEWTFILTER, true, rtnl_held);
c431f89b
VB		 2445 if (err < 0)
2446 NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2447 }
2448
7d5509fa 2449 tfilter_put(tp, fh);
c431f89b 2450errout:
4dbfa766
VB		 2451 if (chain) {
2452 if (tp && !IS_ERR(tp))
12db03b6 2453 tcf_proto_put(tp, rtnl_held, NULL);
c431f89b 2454 tcf_chain_put(chain);
4dbfa766 2455 }
12db03b6 2456 tcf_block_release(q, block, rtnl_held);
470502de
VB		 2457
2458 if (rtnl_held)
2459 rtnl_unlock();
2460
c431f89b
VB		 2461 return err;
2462}
2463
aa767bfe 2464struct tcf_dump_args {
1da177e4
LT		 2465 struct tcf_walker w;
2466 struct sk_buff *skb;
2467 struct netlink_callback *cb;
7960d1da 2468 struct tcf_block *block;
a10fa201
JP		 2469 struct Qdisc *q;
2470 u32 parent;
1da177e4
LT		 2471};
2472
8113c095 2473static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1da177e4 2474{
aa767bfe 2475 struct tcf_dump_args *a = (void *)arg;
832d1d5b 2476 struct net *net = sock_net(a->skb->sk);
1da177e4 2477
7960d1da 2478 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
a10fa201 2479 n, NETLINK_CB(a->cb->skb).portid,
5a7a5555 2480 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
12db03b6 2481 RTM_NEWTFILTER, true);
1da177e4
LT		 2482}
2483
a10fa201
JP		 2484static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2485 struct sk_buff *skb, struct netlink_callback *cb,
acb31fae
JP		 2486 long index_start, long *p_index)
2487{
2488 struct net *net = sock_net(skb->sk);
7960d1da 2489 struct tcf_block *block = chain->block;
acb31fae 2490 struct tcmsg *tcm = nlmsg_data(cb->nlh);
fe2923af 2491 struct tcf_proto *tp, *tp_prev;
acb31fae 2492 struct tcf_dump_args arg;
acb31fae 2493
fe2923af
VB		 2494 for (tp = __tcf_get_next_proto(chain, NULL);
2495 tp;
2496 tp_prev = tp,
2497 tp = __tcf_get_next_proto(chain, tp),
12db03b6 2498 tcf_proto_put(tp_prev, true, NULL),
fe2923af 2499 (*p_index)++) {
acb31fae
JP		 2500 if (*p_index < index_start)
2501 continue;
2502 if (TC_H_MAJ(tcm->tcm_info) &&
2503 TC_H_MAJ(tcm->tcm_info) != tp->prio)
2504 continue;
2505 if (TC_H_MIN(tcm->tcm_info) &&
2506 TC_H_MIN(tcm->tcm_info) != tp->protocol)
2507 continue;
2508 if (*p_index > index_start)
2509 memset(&cb->args[1], 0,
2510 sizeof(cb->args) - sizeof(cb->args[0]));
2511 if (cb->args[1] == 0) {
53189183 2512 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
acb31fae
JP		 2513 NETLINK_CB(cb->skb).portid,
2514 cb->nlh->nlmsg_seq, NLM_F_MULTI,
12db03b6 2515 RTM_NEWTFILTER, true) <= 0)
fe2923af 2516 goto errout;
acb31fae
JP		 2517 cb->args[1] = 1;
2518 }
2519 if (!tp->ops->walk)
2520 continue;
2521 arg.w.fn = tcf_node_dump;
2522 arg.skb = skb;
2523 arg.cb = cb;
7960d1da 2524 arg.block = block;
a10fa201
JP		 2525 arg.q = q;
2526 arg.parent = parent;
acb31fae
JP		 2527 arg.w.stop = 0;
2528 arg.w.skip = cb->args[1] - 1;
2529 arg.w.count = 0;
01683a14 2530 arg.w.cookie = cb->args[2];
12db03b6 2531 tp->ops->walk(tp, &arg.w, true);
01683a14 2532 cb->args[2] = arg.w.cookie;
acb31fae
JP		 2533 cb->args[1] = arg.w.count + 1;
2534 if (arg.w.stop)
fe2923af 2535 goto errout;
acb31fae 2536 }
5bc17018 2537 return true;
fe2923af
VB		 2538
2539errout:
12db03b6 2540 tcf_proto_put(tp, true, NULL);
fe2923af 2541 return false;
acb31fae
JP		 2542}
2543
bd27a875 2544/* called with RTNL */
1da177e4
LT		 2545static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2546{
bbf73830 2547 struct tcf_chain *chain, *chain_prev;
3b1e0a65 2548 struct net *net = sock_net(skb->sk);
5bc17018 2549 struct nlattr *tca[TCA_MAX + 1];
7960d1da 2550 struct Qdisc *q = NULL;
6529eaba 2551 struct tcf_block *block;
942b8165 2552 struct tcmsg *tcm = nlmsg_data(cb->nlh);
acb31fae
JP		 2553 long index_start;
2554 long index;
a10fa201 2555 u32 parent;
5bc17018 2556 int err;
1da177e4 2557
573ce260 2558 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
1da177e4 2559 return skb->len;
5bc17018 2560
8cb08174
JB		 2561 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2562 NULL, cb->extack);
5bc17018
JP		 2563 if (err)
2564 return err;
2565
7960d1da 2566 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
787ce6d0 2567 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
7960d1da
JP		 2568 if (!block)
2569 goto out;
d680b352
JP		 2570 /* If we work with block index, q is NULL and parent value
2571 * will never be used in the following code. The check
2572 * in tcf_fill_node prevents it. However, compiler does not
2573 * see that far, so set parent to zero to silence the warning
2574 * about parent being uninitialized.
2575 */
2576 parent = 0;
a10fa201 2577 } else {
7960d1da
JP		 2578 const struct Qdisc_class_ops *cops;
2579 struct net_device *dev;
2580 unsigned long cl = 0;
2581
2582 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2583 if (!dev)
2584 return skb->len;
2585
2586 parent = tcm->tcm_parent;
2587 if (!parent) {
2588 q = dev->qdisc;
2589 parent = q->handle;
2590 } else {
2591 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2592 }
2593 if (!q)
2594 goto out;
2595 cops = q->ops->cl_ops;
2596 if (!cops)
143976ce 2597 goto out;
7960d1da
JP		 2598 if (!cops->tcf_block)
2599 goto out;
2600 if (TC_H_MIN(tcm->tcm_parent)) {
2601 cl = cops->find(q, tcm->tcm_parent);
2602 if (cl == 0)
2603 goto out;
2604 }
2605 block = cops->tcf_block(q, cl, NULL);
2606 if (!block)
2607 goto out;
2608 if (tcf_block_shared(block))
2609 q = NULL;
1da177e4 2610 }
1da177e4 2611
acb31fae
JP		 2612 index_start = cb->args[0];
2613 index = 0;
5bc17018 2614
bbf73830
VB		 2615 for (chain = __tcf_get_next_chain(block, NULL);
2616 chain;
2617 chain_prev = chain,
2618 chain = __tcf_get_next_chain(block, chain),
2619 tcf_chain_put(chain_prev)) {
5bc17018
JP		 2620 if (tca[TCA_CHAIN] &&
2621 nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2622 continue;
a10fa201 2623 if (!tcf_chain_dump(chain, q, parent, skb, cb,
5ae437ad 2624 index_start, &index)) {
bbf73830 2625 tcf_chain_put(chain);
5ae437ad 2626 err = -EMSGSIZE;
5bc17018 2627 break;
5ae437ad 2628 }
5bc17018
JP		 2629 }
2630
787ce6d0 2631 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
12db03b6 2632 tcf_block_refcnt_put(block, true);
acb31fae 2633 cb->args[0] = index;
1da177e4 2634
1da177e4 2635out:
5ae437ad
RK		 2636 /* If we did no progress, the error (EMSGSIZE) is real */
2637 if (skb->len == 0 && err)
2638 return err;
1da177e4
LT		 2639 return skb->len;
2640}
2641
a5654820
VB		 2642static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2643 void *tmplt_priv, u32 chain_index,
2644 struct net *net, struct sk_buff *skb,
2645 struct tcf_block *block,
32a4f5ec
JP		 2646 u32 portid, u32 seq, u16 flags, int event)
2647{
2648 unsigned char *b = skb_tail_pointer(skb);
9f407f17 2649 const struct tcf_proto_ops *ops;
32a4f5ec
JP		 2650 struct nlmsghdr *nlh;
2651 struct tcmsg *tcm;
9f407f17
JP		 2652 void *priv;
2653
a5654820
VB		 2654 ops = tmplt_ops;
2655 priv = tmplt_priv;
32a4f5ec
JP		 2656
2657 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2658 if (!nlh)
2659 goto out_nlmsg_trim;
2660 tcm = nlmsg_data(nlh);
2661 tcm->tcm_family = AF_UNSPEC;
2662 tcm->tcm__pad1 = 0;
2663 tcm->tcm__pad2 = 0;
2664 tcm->tcm_handle = 0;
2665 if (block->q) {
2666 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2667 tcm->tcm_parent = block->q->handle;
2668 } else {
2669 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2670 tcm->tcm_block_index = block->index;
2671 }
2672
a5654820 2673 if (nla_put_u32(skb, TCA_CHAIN, chain_index))
32a4f5ec
JP		 2674 goto nla_put_failure;
2675
9f407f17
JP		 2676 if (ops) {
2677 if (nla_put_string(skb, TCA_KIND, ops->kind))
2678 goto nla_put_failure;
2679 if (ops->tmplt_dump(skb, net, priv) < 0)
2680 goto nla_put_failure;
2681 }
2682
32a4f5ec
JP		 2683 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2684 return skb->len;
2685
2686out_nlmsg_trim:
2687nla_put_failure:
2688 nlmsg_trim(skb, b);
2689 return -EMSGSIZE;
2690}
2691
2692static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2693 u32 seq, u16 flags, int event, bool unicast)
2694{
2695 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2696 struct tcf_block *block = chain->block;
2697 struct net *net = block->net;
2698 struct sk_buff *skb;
5b5f99b1 2699 int err = 0;
32a4f5ec
JP		 2700
2701 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2702 if (!skb)
2703 return -ENOBUFS;
2704
a5654820
VB		 2705 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2706 chain->index, net, skb, block, portid,
32a4f5ec
JP		 2707 seq, flags, event) <= 0) {
2708 kfree_skb(skb);
2709 return -EINVAL;
2710 }
2711
2712 if (unicast)
5b5f99b1
ZW		 2713 err = netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2714 else
2715 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2716 flags & NLM_F_ECHO);
32a4f5ec 2717
5b5f99b1
ZW		 2718 if (err > 0)
2719 err = 0;
2720 return err;
32a4f5ec
JP		 2721}
2722
a5654820
VB		 2723static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2724 void *tmplt_priv, u32 chain_index,
2725 struct tcf_block *block, struct sk_buff *oskb,
2726 u32 seq, u16 flags, bool unicast)
2727{
2728 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2729 struct net *net = block->net;
2730 struct sk_buff *skb;
2731
2732 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2733 if (!skb)
2734 return -ENOBUFS;
2735
2736 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2737 block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2738 kfree_skb(skb);
2739 return -EINVAL;
2740 }
2741
2742 if (unicast)
2743 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2744
2745 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2746}
2747
9f407f17
JP		 2748static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2749 struct nlattr **tca,
2750 struct netlink_ext_ack *extack)
2751{
2752 const struct tcf_proto_ops *ops;
2753 void *tmplt_priv;
2754
2755 /* If kind is not set, user did not specify template. */
2756 if (!tca[TCA_KIND])
2757 return 0;
2758
12db03b6 2759 ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), true, extack);
9f407f17
JP		 2760 if (IS_ERR(ops))
2761 return PTR_ERR(ops);
2762 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2763 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2764 return -EOPNOTSUPP;
2765 }
2766
2767 tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2768 if (IS_ERR(tmplt_priv)) {
2769 module_put(ops->owner);
2770 return PTR_ERR(tmplt_priv);
2771 }
2772 chain->tmplt_ops = ops;
2773 chain->tmplt_priv = tmplt_priv;
2774 return 0;
2775}
2776
a5654820
VB		 2777static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2778 void *tmplt_priv)
9f407f17 2779{
9f407f17 2780 /* If template ops are set, no work to do for us. */
a5654820 2781 if (!tmplt_ops)
9f407f17
JP		 2782 return;
2783
a5654820
VB		 2784 tmplt_ops->tmplt_destroy(tmplt_priv);
2785 module_put(tmplt_ops->owner);
9f407f17
JP		 2786}
2787
32a4f5ec
JP		 2788/* Add/delete/get a chain */
2789
2790static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2791 struct netlink_ext_ack *extack)
2792{
2793 struct net *net = sock_net(skb->sk);
2794 struct nlattr *tca[TCA_MAX + 1];
2795 struct tcmsg *t;
2796 u32 parent;
2797 u32 chain_index;
2798 struct Qdisc *q = NULL;
2799 struct tcf_chain *chain = NULL;
2800 struct tcf_block *block;
2801 unsigned long cl;
2802 int err;
2803
2804 if (n->nlmsg_type != RTM_GETCHAIN &&
2805 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2806 return -EPERM;
2807
2808replay:
8cb08174
JB		 2809 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2810 rtm_tca_policy, extack);
32a4f5ec
JP		 2811 if (err < 0)
2812 return err;
2813
2814 t = nlmsg_data(n);
2815 parent = t->tcm_parent;
2816 cl = 0;
2817
2818 block = tcf_block_find(net, &q, &parent, &cl,
2819 t->tcm_ifindex, t->tcm_block_index, extack);
2820 if (IS_ERR(block))
2821 return PTR_ERR(block);
2822
2823 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2824 if (chain_index > TC_ACT_EXT_VAL_MASK) {
2825 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
e368fdb6
VB		 2826 err = -EINVAL;
2827 goto errout_block;
32a4f5ec 2828 }
2cbfab07
VB		 2829
2830 mutex_lock(&block->lock);
32a4f5ec
JP		 2831 chain = tcf_chain_lookup(block, chain_index);
2832 if (n->nlmsg_type == RTM_NEWCHAIN) {
2833 if (chain) {
3d32f4c5 2834 if (tcf_chain_held_by_acts_only(chain)) {
1f3ed383 2835 /* The chain exists only because there is
3d32f4c5 2836 * some action referencing it.
1f3ed383
JP		 2837 */
2838 tcf_chain_hold(chain);
2839 } else {
2840 NL_SET_ERR_MSG(extack, "Filter chain already exists");
e368fdb6 2841 err = -EEXIST;
2cbfab07 2842 goto errout_block_locked;
1f3ed383
JP		 2843 }
2844 } else {
2845 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2846 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
e368fdb6 2847 err = -ENOENT;
2cbfab07 2848 goto errout_block_locked;
1f3ed383
JP		 2849 }
2850 chain = tcf_chain_create(block, chain_index);
2851 if (!chain) {
2852 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
e368fdb6 2853 err = -ENOMEM;
2cbfab07 2854 goto errout_block_locked;
1f3ed383 2855 }
32a4f5ec
JP		 2856 }
2857 } else {
3d32f4c5 2858 if (!chain || tcf_chain_held_by_acts_only(chain)) {
32a4f5ec 2859 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
e368fdb6 2860 err = -EINVAL;
2cbfab07 2861 goto errout_block_locked;
32a4f5ec
JP		 2862 }
2863 tcf_chain_hold(chain);
2864 }
2865
2cbfab07
VB		 2866 if (n->nlmsg_type == RTM_NEWCHAIN) {
2867 /* Modifying chain requires holding parent block lock. In case
2868 * the chain was successfully added, take a reference to the
2869 * chain. This ensures that an empty chain does not disappear at
2870 * the end of this function.
2871 */
2872 tcf_chain_hold(chain);
2873 chain->explicitly_created = true;
2874 }
2875 mutex_unlock(&block->lock);
2876
32a4f5ec
JP		 2877 switch (n->nlmsg_type) {
2878 case RTM_NEWCHAIN:
9f407f17 2879 err = tc_chain_tmplt_add(chain, net, tca, extack);
2cbfab07
VB		 2880 if (err) {
2881 tcf_chain_put_explicitly_created(chain);
9f407f17 2882 goto errout;
2cbfab07
VB		 2883 }
2884
32a4f5ec
JP		 2885 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2886 RTM_NEWCHAIN, false);
2887 break;
2888 case RTM_DELCHAIN:
f5b9bac7 2889 tfilter_notify_chain(net, skb, block, q, parent, n,
12db03b6 2890 chain, RTM_DELTFILTER, true);
32a4f5ec 2891 /* Flush the chain first as the user requested chain removal. */
12db03b6 2892 tcf_chain_flush(chain, true);
32a4f5ec
JP		 2893 /* In case the chain was successfully deleted, put a reference
2894 * to the chain previously taken during addition.
2895 */
2896 tcf_chain_put_explicitly_created(chain);
2897 break;
2898 case RTM_GETCHAIN:
32a4f5ec
JP		 2899 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2900 n->nlmsg_seq, n->nlmsg_type, true);
2901 if (err < 0)
2902 NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2903 break;
2904 default:
2905 err = -EOPNOTSUPP;
2906 NL_SET_ERR_MSG(extack, "Unsupported message type");
2907 goto errout;
2908 }
2909
2910errout:
2911 tcf_chain_put(chain);
e368fdb6 2912errout_block:
12db03b6 2913 tcf_block_release(q, block, true);
32a4f5ec
JP		 2914 if (err == -EAGAIN)
2915 /* Replay the request. */
2916 goto replay;
2917 return err;
2cbfab07
VB		 2918
2919errout_block_locked:
2920 mutex_unlock(&block->lock);
2921 goto errout_block;
32a4f5ec
JP		 2922}
2923
2924/* called with RTNL */
2925static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2926{
2927 struct net *net = sock_net(skb->sk);
2928 struct nlattr *tca[TCA_MAX + 1];
2929 struct Qdisc *q = NULL;
2930 struct tcf_block *block;
32a4f5ec 2931 struct tcmsg *tcm = nlmsg_data(cb->nlh);
ace4a267 2932 struct tcf_chain *chain;
32a4f5ec
JP		 2933 long index_start;
2934 long index;
2935 u32 parent;
2936 int err;
2937
2938 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2939 return skb->len;
2940
8cb08174
JB		 2941 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2942 rtm_tca_policy, cb->extack);
32a4f5ec
JP		 2943 if (err)
2944 return err;
2945
2946 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
787ce6d0 2947 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
32a4f5ec
JP		 2948 if (!block)
2949 goto out;
2950 /* If we work with block index, q is NULL and parent value
2951 * will never be used in the following code. The check
2952 * in tcf_fill_node prevents it. However, compiler does not
2953 * see that far, so set parent to zero to silence the warning
2954 * about parent being uninitialized.
2955 */
2956 parent = 0;
2957 } else {
2958 const struct Qdisc_class_ops *cops;
2959 struct net_device *dev;
2960 unsigned long cl = 0;
2961
2962 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2963 if (!dev)
2964 return skb->len;
2965
2966 parent = tcm->tcm_parent;
2967 if (!parent) {
2968 q = dev->qdisc;
2969 parent = q->handle;
2970 } else {
2971 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2972 }
2973 if (!q)
2974 goto out;
2975 cops = q->ops->cl_ops;
2976 if (!cops)
2977 goto out;
2978 if (!cops->tcf_block)
2979 goto out;
2980 if (TC_H_MIN(tcm->tcm_parent)) {
2981 cl = cops->find(q, tcm->tcm_parent);
2982 if (cl == 0)
2983 goto out;
2984 }
2985 block = cops->tcf_block(q, cl, NULL);
2986 if (!block)
2987 goto out;
2988 if (tcf_block_shared(block))
2989 q = NULL;
2990 }
2991
2992 index_start = cb->args[0];
2993 index = 0;
2994
ace4a267
VB		 2995 mutex_lock(&block->lock);
2996 list_for_each_entry(chain, &block->chain_list, list) {
32a4f5ec
JP		 2997 if ((tca[TCA_CHAIN] &&
2998 nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2999 continue;
3000 if (index < index_start) {
3001 index++;
3002 continue;
3003 }
ace4a267
VB		 3004 if (tcf_chain_held_by_acts_only(chain))
3005 continue;
a5654820
VB		 3006 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3007 chain->index, net, skb, block,
32a4f5ec
JP		 3008 NETLINK_CB(cb->skb).portid,
3009 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3010 RTM_NEWCHAIN);
ace4a267 3011 if (err <= 0)
32a4f5ec
JP		 3012 break;
3013 index++;
3014 }
ace4a267 3015 mutex_unlock(&block->lock);
32a4f5ec 3016
787ce6d0 3017 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
12db03b6 3018 tcf_block_refcnt_put(block, true);
32a4f5ec
JP		 3019 cb->args[0] = index;
3020
3021out:
3022 /* If we did no progress, the error (EMSGSIZE) is real */
3023 if (skb->len == 0 && err)
3024 return err;
3025 return skb->len;
3026}
3027
18d0264f 3028void tcf_exts_destroy(struct tcf_exts *exts)
1da177e4
LT		 3029{
3030#ifdef CONFIG_NET_CLS_ACT
90b73b77 3031 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
22dc13c8
WC		 3032 kfree(exts->actions);
3033 exts->nr_actions = 0;
1da177e4
LT		 3034#endif
3035}
aa767bfe 3036EXPORT_SYMBOL(tcf_exts_destroy);
1da177e4 3037
c1b52739 3038int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
50a56190 3039 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
ec6743a1 3040 bool rtnl_held, struct netlink_ext_ack *extack)
1da177e4 3041{
1da177e4
LT		 3042#ifdef CONFIG_NET_CLS_ACT
3043 {
1da177e4 3044 struct tc_action *act;
d04e6990 3045 size_t attr_size = 0;
1da177e4 3046
5da57f42 3047 if (exts->police && tb[exts->police]) {
9fb9f251
JP		 3048 act = tcf_action_init_1(net, tp, tb[exts->police],
3049 rate_tlv, "police", ovr,
ec6743a1
VB		 3050 TCA_ACT_BIND, rtnl_held,
3051 extack);
ab27cfb8
PM		 3052 if (IS_ERR(act))
3053 return PTR_ERR(act);
1da177e4 3054
33be6271 3055 act->type = exts->type = TCA_OLD_COMPAT;
22dc13c8
WC		 3056 exts->actions[0] = act;
3057 exts->nr_actions = 1;
5da57f42 3058 } else if (exts->action && tb[exts->action]) {
90b73b77 3059 int err;
22dc13c8 3060
9fb9f251
JP		 3061 err = tcf_action_init(net, tp, tb[exts->action],
3062 rate_tlv, NULL, ovr, TCA_ACT_BIND,
ec6743a1
VB		 3063 exts->actions, &attr_size,
3064 rtnl_held, extack);
90b73b77 3065 if (err < 0)
33be6271 3066 return err;
90b73b77 3067 exts->nr_actions = err;
1da177e4
LT		 3068 }
3069 }
1da177e4 3070#else
5da57f42 3071 if ((exts->action && tb[exts->action]) ||
50a56190
AA		 3072 (exts->police && tb[exts->police])) {
3073 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
1da177e4 3074 return -EOPNOTSUPP;
50a56190 3075 }
1da177e4
LT		 3076#endif
3077
3078 return 0;
3079}
aa767bfe 3080EXPORT_SYMBOL(tcf_exts_validate);
1da177e4 3081
9b0d4446 3082void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
1da177e4
LT		 3083{
3084#ifdef CONFIG_NET_CLS_ACT
22dc13c8
WC		 3085 struct tcf_exts old = *dst;
3086
9b0d4446 3087 *dst = *src;
22dc13c8 3088 tcf_exts_destroy(&old);
1da177e4
LT		 3089#endif
3090}
aa767bfe 3091EXPORT_SYMBOL(tcf_exts_change);
1da177e4 3092
22dc13c8
WC		 3093#ifdef CONFIG_NET_CLS_ACT
3094static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3095{
3096 if (exts->nr_actions == 0)
3097 return NULL;
3098 else
3099 return exts->actions[0];
3100}
3101#endif
33be6271 3102
5da57f42 3103int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
1da177e4
LT		 3104{
3105#ifdef CONFIG_NET_CLS_ACT
9cc63db5
CW		 3106 struct nlattr *nest;
3107
978dfd8d 3108 if (exts->action && tcf_exts_has_actions(exts)) {
1da177e4
LT		 3109 /*
3110 * again for backward compatible mode - we want
3111 * to work with both old and new modes of entering
3112 * tc data even if iproute2 was newer - jhs
3113 */
33be6271 3114 if (exts->type != TCA_OLD_COMPAT) {
ae0be8de 3115 nest = nla_nest_start_noflag(skb, exts->action);
4b3550ef
PM		 3116 if (nest == NULL)
3117 goto nla_put_failure;
22dc13c8 3118
90b73b77 3119 if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
add93b61 3120 goto nla_put_failure;
4b3550ef 3121 nla_nest_end(skb, nest);
5da57f42 3122 } else if (exts->police) {
33be6271 3123 struct tc_action *act = tcf_exts_first_act(exts);
ae0be8de 3124 nest = nla_nest_start_noflag(skb, exts->police);
63acd680 3125 if (nest == NULL || !act)
4b3550ef 3126 goto nla_put_failure;
33be6271 3127 if (tcf_action_dump_old(skb, act, 0, 0) < 0)
add93b61 3128 goto nla_put_failure;
4b3550ef 3129 nla_nest_end(skb, nest);
1da177e4
LT		 3130 }
3131 }
1da177e4 3132 return 0;
9cc63db5
CW		 3133
3134nla_put_failure:
3135 nla_nest_cancel(skb, nest);
1da177e4 3136 return -1;
9cc63db5
CW		 3137#else
3138 return 0;
3139#endif
1da177e4 3140}
aa767bfe 3141EXPORT_SYMBOL(tcf_exts_dump);
1da177e4 3142
aa767bfe 3143
5da57f42 3144int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
1da177e4
LT		 3145{
3146#ifdef CONFIG_NET_CLS_ACT
33be6271 3147 struct tc_action *a = tcf_exts_first_act(exts);
b057df24 3148 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
33be6271 3149 return -1;
1da177e4
LT		 3150#endif
3151 return 0;
1da177e4 3152}
aa767bfe 3153EXPORT_SYMBOL(tcf_exts_dump_stats);
1da177e4 3154
aeb3fecd
CW		 3155int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3156 void *type_data, bool err_stop)
717503b9 3157{
955bcb6e 3158 struct flow_block_cb *block_cb;
aeb3fecd
CW		 3159 int ok_count = 0;
3160 int err;
3161
3162 /* Make sure all netdevs sharing this block are offload-capable. */
3163 if (block->nooffloaddevcnt && err_stop)
3164 return -EOPNOTSUPP;
3165
14bfb13f 3166 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
aeb3fecd
CW		 3167 err = block_cb->cb(type, type_data, block_cb->cb_priv);
3168 if (err) {
3169 if (err_stop)
3170 return err;
3171 } else {
3172 ok_count++;
3173 }
3174 }
3175 return ok_count;
717503b9
JP		 3176}
3177EXPORT_SYMBOL(tc_setup_cb_call);
b3f55bdd 3178
3a7b6861
PNA		 3179int tc_setup_flow_action(struct flow_action *flow_action,
3180 const struct tcf_exts *exts)
3181{
3182 const struct tc_action *act;
3183 int i, j, k;
3184
3185 if (!exts)
3186 return 0;
3187
3188 j = 0;
3189 tcf_exts_for_each_action(i, act, exts) {
3190 struct flow_action_entry *entry;
3191
3192 entry = &flow_action->entries[j];
3193 if (is_tcf_gact_ok(act)) {
3194 entry->id = FLOW_ACTION_ACCEPT;
3195 } else if (is_tcf_gact_shot(act)) {
3196 entry->id = FLOW_ACTION_DROP;
3197 } else if (is_tcf_gact_trap(act)) {
3198 entry->id = FLOW_ACTION_TRAP;
3199 } else if (is_tcf_gact_goto_chain(act)) {
3200 entry->id = FLOW_ACTION_GOTO;
3201 entry->chain_index = tcf_gact_goto_chain_index(act);
3202 } else if (is_tcf_mirred_egress_redirect(act)) {
3203 entry->id = FLOW_ACTION_REDIRECT;
3204 entry->dev = tcf_mirred_dev(act);
3205 } else if (is_tcf_mirred_egress_mirror(act)) {
3206 entry->id = FLOW_ACTION_MIRRED;
3207 entry->dev = tcf_mirred_dev(act);
48e584ac
JH		 3208 } else if (is_tcf_mirred_ingress_redirect(act)) {
3209 entry->id = FLOW_ACTION_REDIRECT_INGRESS;
3210 entry->dev = tcf_mirred_dev(act);
3211 } else if (is_tcf_mirred_ingress_mirror(act)) {
3212 entry->id = FLOW_ACTION_MIRRED_INGRESS;
3213 entry->dev = tcf_mirred_dev(act);
3a7b6861
PNA		 3214 } else if (is_tcf_vlan(act)) {
3215 switch (tcf_vlan_action(act)) {
3216 case TCA_VLAN_ACT_PUSH:
3217 entry->id = FLOW_ACTION_VLAN_PUSH;
3218 entry->vlan.vid = tcf_vlan_push_vid(act);
3219 entry->vlan.proto = tcf_vlan_push_proto(act);
3220 entry->vlan.prio = tcf_vlan_push_prio(act);
3221 break;
3222 case TCA_VLAN_ACT_POP:
3223 entry->id = FLOW_ACTION_VLAN_POP;
3224 break;
3225 case TCA_VLAN_ACT_MODIFY:
3226 entry->id = FLOW_ACTION_VLAN_MANGLE;
3227 entry->vlan.vid = tcf_vlan_push_vid(act);
3228 entry->vlan.proto = tcf_vlan_push_proto(act);
3229 entry->vlan.prio = tcf_vlan_push_prio(act);
3230 break;
3231 default:
3232 goto err_out;
3233 }
3234 } else if (is_tcf_tunnel_set(act)) {
3235 entry->id = FLOW_ACTION_TUNNEL_ENCAP;
3236 entry->tunnel = tcf_tunnel_info(act);
3237 } else if (is_tcf_tunnel_release(act)) {
3238 entry->id = FLOW_ACTION_TUNNEL_DECAP;
3a7b6861
PNA		 3239 } else if (is_tcf_pedit(act)) {
3240 for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3241 switch (tcf_pedit_cmd(act, k)) {
3242 case TCA_PEDIT_KEY_EX_CMD_SET:
3243 entry->id = FLOW_ACTION_MANGLE;
3244 break;
3245 case TCA_PEDIT_KEY_EX_CMD_ADD:
3246 entry->id = FLOW_ACTION_ADD;
3247 break;
3248 default:
3249 goto err_out;
3250 }
3251 entry->mangle.htype = tcf_pedit_htype(act, k);
3252 entry->mangle.mask = tcf_pedit_mask(act, k);
3253 entry->mangle.val = tcf_pedit_val(act, k);
3254 entry->mangle.offset = tcf_pedit_offset(act, k);
3255 entry = &flow_action->entries[++j];
3256 }
3257 } else if (is_tcf_csum(act)) {
3258 entry->id = FLOW_ACTION_CSUM;
3259 entry->csum_flags = tcf_csum_update_flags(act);
3260 } else if (is_tcf_skbedit_mark(act)) {
3261 entry->id = FLOW_ACTION_MARK;
3262 entry->mark = tcf_skbedit_mark(act);
a7a7be60
PJV		 3263 } else if (is_tcf_sample(act)) {
3264 entry->id = FLOW_ACTION_SAMPLE;
3265 entry->sample.psample_group =
3266 tcf_sample_psample_group(act);
3267 entry->sample.trunc_size = tcf_sample_trunc_size(act);
3268 entry->sample.truncate = tcf_sample_truncate(act);
3269 entry->sample.rate = tcf_sample_rate(act);
8c8cfc6e
PJV		 3270 } else if (is_tcf_police(act)) {
3271 entry->id = FLOW_ACTION_POLICE;
3272 entry->police.burst = tcf_police_tcfp_burst(act);
3273 entry->police.rate_bytes_ps =
3274 tcf_police_rate_bytes_ps(act);
b57dc7c1
PB		 3275 } else if (is_tcf_ct(act)) {
3276 entry->id = FLOW_ACTION_CT;
3277 entry->ct.action = tcf_ct_action(act);
3278 entry->ct.zone = tcf_ct_zone(act);
6749d590
JH		 3279 } else if (is_tcf_mpls(act)) {
3280 switch (tcf_mpls_action(act)) {
3281 case TCA_MPLS_ACT_PUSH:
3282 entry->id = FLOW_ACTION_MPLS_PUSH;
3283 entry->mpls_push.proto = tcf_mpls_proto(act);
3284 entry->mpls_push.label = tcf_mpls_label(act);
3285 entry->mpls_push.tc = tcf_mpls_tc(act);
3286 entry->mpls_push.bos = tcf_mpls_bos(act);
3287 entry->mpls_push.ttl = tcf_mpls_ttl(act);
3288 break;
3289 case TCA_MPLS_ACT_POP:
3290 entry->id = FLOW_ACTION_MPLS_POP;
3291 entry->mpls_pop.proto = tcf_mpls_proto(act);
3292 break;
3293 case TCA_MPLS_ACT_MODIFY:
3294 entry->id = FLOW_ACTION_MPLS_MANGLE;
3295 entry->mpls_mangle.label = tcf_mpls_label(act);
3296 entry->mpls_mangle.tc = tcf_mpls_tc(act);
3297 entry->mpls_mangle.bos = tcf_mpls_bos(act);
3298 entry->mpls_mangle.ttl = tcf_mpls_ttl(act);
3299 break;
3300 default:
3301 goto err_out;
3302 }
fb1b775a
JH		 3303 } else if (is_tcf_skbedit_ptype(act)) {
3304 entry->id = FLOW_ACTION_PTYPE;
3305 entry->ptype = tcf_skbedit_ptype(act);
3a7b6861
PNA		 3306 } else {
3307 goto err_out;
3308 }
3309
3310 if (!is_tcf_pedit(act))
3311 j++;
3312 }
3313 return 0;
3314err_out:
3315 return -EOPNOTSUPP;
3316}
3317EXPORT_SYMBOL(tc_setup_flow_action);
3318
e3ab786b
PNA		 3319unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3320{
3321 unsigned int num_acts = 0;
3322 struct tc_action *act;
3323 int i;
3324
3325 tcf_exts_for_each_action(i, act, exts) {
3326 if (is_tcf_pedit(act))
3327 num_acts += tcf_pedit_nkeys(act);
3328 else
3329 num_acts++;
3330 }
3331 return num_acts;
3332}
3333EXPORT_SYMBOL(tcf_exts_num_actions);
3334
48617387
JP		 3335static __net_init int tcf_net_init(struct net *net)
3336{
3337 struct tcf_net *tn = net_generic(net, tcf_net_id);
3338
ab281629 3339 spin_lock_init(&tn->idr_lock);
48617387
JP		 3340 idr_init(&tn->idr);
3341 return 0;
3342}
3343
3344static void __net_exit tcf_net_exit(struct net *net)
3345{
3346 struct tcf_net *tn = net_generic(net, tcf_net_id);
3347
3348 idr_destroy(&tn->idr);
3349}
3350
3351static struct pernet_operations tcf_net_ops = {
3352 .init = tcf_net_init,
3353 .exit = tcf_net_exit,
3354 .id = &tcf_net_id,
3355 .size = sizeof(struct tcf_net),
3356};
3357
1da177e4
LT		 3358static int __init tc_filter_init(void)
3359{
48617387
JP		 3360 int err;
3361
7aa0045d
CW		 3362 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3363 if (!tc_filter_wq)
3364 return -ENOMEM;
3365
48617387
JP		 3366 err = register_pernet_subsys(&tcf_net_ops);
3367 if (err)
3368 goto err_register_pernet_subsys;
3369
7f76fa36
JH		 3370 err = rhashtable_init(&indr_setup_block_ht,
3371 &tc_indr_setup_block_ht_params);
3372 if (err)
3373 goto err_rhash_setup_block_ht;
3374
470502de
VB		 3375 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL,
3376 RTNL_FLAG_DOIT_UNLOCKED);
3377 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL,
3378 RTNL_FLAG_DOIT_UNLOCKED);
c431f89b 3379 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
470502de 3380 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED);
32a4f5ec
JP		 3381 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3382 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3383 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3384 tc_dump_chain, 0);
1da177e4 3385
1da177e4 3386 return 0;
48617387 3387
7f76fa36
JH		 3388err_rhash_setup_block_ht:
3389 unregister_pernet_subsys(&tcf_net_ops);
48617387
JP		 3390err_register_pernet_subsys:
3391 destroy_workqueue(tc_filter_wq);
3392 return err;
1da177e4
LT		 3393}
3394
3395subsys_initcall(tc_filter_init);
Ubuntu Hirsute Linux kernel mirror