1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/act_api.c Packet action API.
5 * Author: Jamal Hadi Salim
8 #include <linux/types.h>
9 #include <linux/kernel.h>
10 #include <linux/string.h>
11 #include <linux/errno.h>
12 #include <linux/slab.h>
13 #include <linux/skbuff.h>
14 #include <linux/init.h>
15 #include <linux/kmod.h>
16 #include <linux/err.h>
17 #include <linux/module.h>
18 #include <net/net_namespace.h>
20 #include <net/sch_generic.h>
21 #include <net/pkt_cls.h>
22 #include <net/act_api.h>
23 #include <net/netlink.h>
25 static void tcf_action_goto_chain_exec(const struct tc_action
*a
,
26 struct tcf_result
*res
)
28 const struct tcf_chain
*chain
= rcu_dereference_bh(a
->goto_chain
);
30 res
->goto_tp
= rcu_dereference_bh(chain
->filter_chain
);
33 static void tcf_free_cookie_rcu(struct rcu_head
*p
)
35 struct tc_cookie
*cookie
= container_of(p
, struct tc_cookie
, rcu
);
41 static void tcf_set_action_cookie(struct tc_cookie __rcu
**old_cookie
,
42 struct tc_cookie
*new_cookie
)
44 struct tc_cookie
*old
;
46 old
= xchg((__force
struct tc_cookie
**)old_cookie
, new_cookie
);
48 call_rcu(&old
->rcu
, tcf_free_cookie_rcu
);
51 int tcf_action_check_ctrlact(int action
, struct tcf_proto
*tp
,
52 struct tcf_chain
**newchain
,
53 struct netlink_ext_ack
*extack
)
55 int opcode
= TC_ACT_EXT_OPCODE(action
), ret
= -EINVAL
;
59 ret
= action
> TC_ACT_VALUE_MAX
? -EINVAL
: 0;
60 else if (opcode
<= TC_ACT_EXT_OPCODE_MAX
|| action
== TC_ACT_UNSPEC
)
63 NL_SET_ERR_MSG(extack
, "invalid control action");
67 if (TC_ACT_EXT_CMP(action
, TC_ACT_GOTO_CHAIN
)) {
68 chain_index
= action
& TC_ACT_EXT_VAL_MASK
;
69 if (!tp
|| !newchain
) {
71 NL_SET_ERR_MSG(extack
,
72 "can't goto NULL proto/chain");
75 *newchain
= tcf_chain_get_by_act(tp
->chain
->block
, chain_index
);
78 NL_SET_ERR_MSG(extack
,
79 "can't allocate goto_chain");
85 EXPORT_SYMBOL(tcf_action_check_ctrlact
);
87 struct tcf_chain
*tcf_action_set_ctrlact(struct tc_action
*a
, int action
,
88 struct tcf_chain
*goto_chain
)
90 a
->tcfa_action
= action
;
91 rcu_swap_protected(a
->goto_chain
, goto_chain
, 1);
94 EXPORT_SYMBOL(tcf_action_set_ctrlact
);
96 /* XXX: For standalone actions, we don't need a RCU grace period either, because
97 * actions are always connected to filters and filters are already destroyed in
98 * RCU callbacks, so after a RCU grace period actions are already disconnected
99 * from filters. Readers later can not find us.
101 static void free_tcf(struct tc_action
*p
)
103 struct tcf_chain
*chain
= rcu_dereference_protected(p
->goto_chain
, 1);
105 free_percpu(p
->cpu_bstats
);
106 free_percpu(p
->cpu_bstats_hw
);
107 free_percpu(p
->cpu_qstats
);
109 tcf_set_action_cookie(&p
->act_cookie
, NULL
);
111 tcf_chain_put_by_act(chain
);
116 static void tcf_action_cleanup(struct tc_action
*p
)
121 gen_kill_estimator(&p
->tcfa_rate_est
);
125 static int __tcf_action_put(struct tc_action
*p
, bool bind
)
127 struct tcf_idrinfo
*idrinfo
= p
->idrinfo
;
129 if (refcount_dec_and_mutex_lock(&p
->tcfa_refcnt
, &idrinfo
->lock
)) {
131 atomic_dec(&p
->tcfa_bindcnt
);
132 idr_remove(&idrinfo
->action_idr
, p
->tcfa_index
);
133 mutex_unlock(&idrinfo
->lock
);
135 tcf_action_cleanup(p
);
140 atomic_dec(&p
->tcfa_bindcnt
);
145 int __tcf_idr_release(struct tc_action
*p
, bool bind
, bool strict
)
149 /* Release with strict==1 and bind==0 is only called through act API
150 * interface (classifiers always bind). Only case when action with
151 * positive reference count and zero bind count can exist is when it was
152 * also created with act API (unbinding last classifier will destroy the
153 * action if it was created by classifier). So only case when bind count
154 * can be changed after initial check is when unbound action is
155 * destroyed by act API while classifier binds to action with same id
156 * concurrently. This result either creation of new action(same behavior
157 * as before), or reusing existing action if concurrent process
158 * increments reference count before action is deleted. Both scenarios
162 if (!bind
&& strict
&& atomic_read(&p
->tcfa_bindcnt
) > 0)
165 if (__tcf_action_put(p
, bind
))
171 EXPORT_SYMBOL(__tcf_idr_release
);
173 static size_t tcf_action_shared_attrs_size(const struct tc_action
*act
)
175 struct tc_cookie
*act_cookie
;
179 act_cookie
= rcu_dereference(act
->act_cookie
);
182 cookie_len
= nla_total_size(act_cookie
->len
);
185 return nla_total_size(0) /* action number nested */
186 + nla_total_size(IFNAMSIZ
) /* TCA_ACT_KIND */
187 + cookie_len
/* TCA_ACT_COOKIE */
188 + nla_total_size(0) /* TCA_ACT_STATS nested */
189 /* TCA_STATS_BASIC */
190 + nla_total_size_64bit(sizeof(struct gnet_stats_basic
))
191 /* TCA_STATS_PKT64 */
192 + nla_total_size_64bit(sizeof(u64
))
193 /* TCA_STATS_QUEUE */
194 + nla_total_size_64bit(sizeof(struct gnet_stats_queue
))
195 + nla_total_size(0) /* TCA_OPTIONS nested */
196 + nla_total_size(sizeof(struct tcf_t
)); /* TCA_GACT_TM */
199 static size_t tcf_action_full_attrs_size(size_t sz
)
201 return NLMSG_HDRLEN
/* struct nlmsghdr */
202 + sizeof(struct tcamsg
)
203 + nla_total_size(0) /* TCA_ACT_TAB nested */
207 static size_t tcf_action_fill_size(const struct tc_action
*act
)
209 size_t sz
= tcf_action_shared_attrs_size(act
);
211 if (act
->ops
->get_fill_size
)
212 return act
->ops
->get_fill_size(act
) + sz
;
216 static int tcf_dump_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
217 struct netlink_callback
*cb
)
219 int err
= 0, index
= -1, s_i
= 0, n_i
= 0;
220 u32 act_flags
= cb
->args
[2];
221 unsigned long jiffy_since
= cb
->args
[3];
223 struct idr
*idr
= &idrinfo
->action_idr
;
225 unsigned long id
= 1;
228 mutex_lock(&idrinfo
->lock
);
232 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
238 time_after(jiffy_since
,
239 (unsigned long)p
->tcfa_tm
.lastuse
))
242 nest
= nla_nest_start_noflag(skb
, n_i
);
245 goto nla_put_failure
;
247 err
= tcf_action_dump_1(skb
, p
, 0, 0);
250 nlmsg_trim(skb
, nest
);
253 nla_nest_end(skb
, nest
);
255 if (!(act_flags
& TCA_FLAG_LARGE_DUMP_ON
) &&
256 n_i
>= TCA_ACT_MAX_PRIO
)
261 cb
->args
[0] = index
+ 1;
263 mutex_unlock(&idrinfo
->lock
);
265 if (act_flags
& TCA_FLAG_LARGE_DUMP_ON
)
271 nla_nest_cancel(skb
, nest
);
275 static int tcf_idr_release_unsafe(struct tc_action
*p
)
277 if (atomic_read(&p
->tcfa_bindcnt
) > 0)
280 if (refcount_dec_and_test(&p
->tcfa_refcnt
)) {
281 idr_remove(&p
->idrinfo
->action_idr
, p
->tcfa_index
);
282 tcf_action_cleanup(p
);
283 return ACT_P_DELETED
;
289 static int tcf_del_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
290 const struct tc_action_ops
*ops
)
295 struct idr
*idr
= &idrinfo
->action_idr
;
297 unsigned long id
= 1;
300 nest
= nla_nest_start_noflag(skb
, 0);
302 goto nla_put_failure
;
303 if (nla_put_string(skb
, TCA_KIND
, ops
->kind
))
304 goto nla_put_failure
;
306 mutex_lock(&idrinfo
->lock
);
307 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
308 ret
= tcf_idr_release_unsafe(p
);
309 if (ret
== ACT_P_DELETED
) {
310 module_put(ops
->owner
);
312 } else if (ret
< 0) {
313 mutex_unlock(&idrinfo
->lock
);
314 goto nla_put_failure
;
317 mutex_unlock(&idrinfo
->lock
);
319 if (nla_put_u32(skb
, TCA_FCNT
, n_i
))
320 goto nla_put_failure
;
321 nla_nest_end(skb
, nest
);
325 nla_nest_cancel(skb
, nest
);
329 int tcf_generic_walker(struct tc_action_net
*tn
, struct sk_buff
*skb
,
330 struct netlink_callback
*cb
, int type
,
331 const struct tc_action_ops
*ops
,
332 struct netlink_ext_ack
*extack
)
334 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
336 if (type
== RTM_DELACTION
) {
337 return tcf_del_walker(idrinfo
, skb
, ops
);
338 } else if (type
== RTM_GETACTION
) {
339 return tcf_dump_walker(idrinfo
, skb
, cb
);
341 WARN(1, "tcf_generic_walker: unknown command %d\n", type
);
342 NL_SET_ERR_MSG(extack
, "tcf_generic_walker: unknown command");
346 EXPORT_SYMBOL(tcf_generic_walker
);
348 int tcf_idr_search(struct tc_action_net
*tn
, struct tc_action
**a
, u32 index
)
350 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
353 mutex_lock(&idrinfo
->lock
);
354 p
= idr_find(&idrinfo
->action_idr
, index
);
358 refcount_inc(&p
->tcfa_refcnt
);
359 mutex_unlock(&idrinfo
->lock
);
367 EXPORT_SYMBOL(tcf_idr_search
);
369 static int tcf_idr_delete_index(struct tcf_idrinfo
*idrinfo
, u32 index
)
374 mutex_lock(&idrinfo
->lock
);
375 p
= idr_find(&idrinfo
->action_idr
, index
);
377 mutex_unlock(&idrinfo
->lock
);
381 if (!atomic_read(&p
->tcfa_bindcnt
)) {
382 if (refcount_dec_and_test(&p
->tcfa_refcnt
)) {
383 struct module
*owner
= p
->ops
->owner
;
385 WARN_ON(p
!= idr_remove(&idrinfo
->action_idr
,
387 mutex_unlock(&idrinfo
->lock
);
389 tcf_action_cleanup(p
);
398 mutex_unlock(&idrinfo
->lock
);
402 int tcf_idr_create(struct tc_action_net
*tn
, u32 index
, struct nlattr
*est
,
403 struct tc_action
**a
, const struct tc_action_ops
*ops
,
404 int bind
, bool cpustats
, u32 flags
)
406 struct tc_action
*p
= kzalloc(ops
->size
, GFP_KERNEL
);
407 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
412 refcount_set(&p
->tcfa_refcnt
, 1);
414 atomic_set(&p
->tcfa_bindcnt
, 1);
417 p
->cpu_bstats
= netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu
);
420 p
->cpu_bstats_hw
= netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu
);
421 if (!p
->cpu_bstats_hw
)
423 p
->cpu_qstats
= alloc_percpu(struct gnet_stats_queue
);
427 spin_lock_init(&p
->tcfa_lock
);
428 p
->tcfa_index
= index
;
429 p
->tcfa_tm
.install
= jiffies
;
430 p
->tcfa_tm
.lastuse
= jiffies
;
431 p
->tcfa_tm
.firstuse
= 0;
432 p
->tcfa_flags
= flags
;
434 err
= gen_new_estimator(&p
->tcfa_bstats
, p
->cpu_bstats
,
436 &p
->tcfa_lock
, NULL
, est
);
441 p
->idrinfo
= idrinfo
;
446 free_percpu(p
->cpu_qstats
);
448 free_percpu(p
->cpu_bstats_hw
);
450 free_percpu(p
->cpu_bstats
);
455 EXPORT_SYMBOL(tcf_idr_create
);
457 int tcf_idr_create_from_flags(struct tc_action_net
*tn
, u32 index
,
458 struct nlattr
*est
, struct tc_action
**a
,
459 const struct tc_action_ops
*ops
, int bind
,
462 /* Set cpustats according to actions flags. */
463 return tcf_idr_create(tn
, index
, est
, a
, ops
, bind
,
464 !(flags
& TCA_ACT_FLAGS_NO_PERCPU_STATS
), flags
);
466 EXPORT_SYMBOL(tcf_idr_create_from_flags
);
468 void tcf_idr_insert(struct tc_action_net
*tn
, struct tc_action
*a
)
470 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
472 mutex_lock(&idrinfo
->lock
);
473 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
474 WARN_ON(!IS_ERR(idr_replace(&idrinfo
->action_idr
, a
, a
->tcfa_index
)));
475 mutex_unlock(&idrinfo
->lock
);
477 EXPORT_SYMBOL(tcf_idr_insert
);
479 /* Cleanup idr index that was allocated but not initialized. */
481 void tcf_idr_cleanup(struct tc_action_net
*tn
, u32 index
)
483 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
485 mutex_lock(&idrinfo
->lock
);
486 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
487 WARN_ON(!IS_ERR(idr_remove(&idrinfo
->action_idr
, index
)));
488 mutex_unlock(&idrinfo
->lock
);
490 EXPORT_SYMBOL(tcf_idr_cleanup
);
492 /* Check if action with specified index exists. If actions is found, increments
493 * its reference and bind counters, and return 1. Otherwise insert temporary
494 * error pointer (to prevent concurrent users from inserting actions with same
495 * index) and return 0.
498 int tcf_idr_check_alloc(struct tc_action_net
*tn
, u32
*index
,
499 struct tc_action
**a
, int bind
)
501 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
506 mutex_lock(&idrinfo
->lock
);
508 p
= idr_find(&idrinfo
->action_idr
, *index
);
510 /* This means that another process allocated
511 * index but did not assign the pointer yet.
513 mutex_unlock(&idrinfo
->lock
);
518 refcount_inc(&p
->tcfa_refcnt
);
520 atomic_inc(&p
->tcfa_bindcnt
);
525 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
528 idr_replace(&idrinfo
->action_idr
,
529 ERR_PTR(-EBUSY
), *index
);
534 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
535 UINT_MAX
, GFP_KERNEL
);
537 idr_replace(&idrinfo
->action_idr
, ERR_PTR(-EBUSY
),
540 mutex_unlock(&idrinfo
->lock
);
543 EXPORT_SYMBOL(tcf_idr_check_alloc
);
545 void tcf_idrinfo_destroy(const struct tc_action_ops
*ops
,
546 struct tcf_idrinfo
*idrinfo
)
548 struct idr
*idr
= &idrinfo
->action_idr
;
551 unsigned long id
= 1;
554 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
555 ret
= __tcf_idr_release(p
, false, true);
556 if (ret
== ACT_P_DELETED
)
557 module_put(ops
->owner
);
561 idr_destroy(&idrinfo
->action_idr
);
563 EXPORT_SYMBOL(tcf_idrinfo_destroy
);
565 static LIST_HEAD(act_base
);
566 static DEFINE_RWLOCK(act_mod_lock
);
568 int tcf_register_action(struct tc_action_ops
*act
,
569 struct pernet_operations
*ops
)
571 struct tc_action_ops
*a
;
574 if (!act
->act
|| !act
->dump
|| !act
->init
|| !act
->walk
|| !act
->lookup
)
577 /* We have to register pernet ops before making the action ops visible,
578 * otherwise tcf_action_init_1() could get a partially initialized
581 ret
= register_pernet_subsys(ops
);
585 write_lock(&act_mod_lock
);
586 list_for_each_entry(a
, &act_base
, head
) {
587 if (act
->id
== a
->id
|| (strcmp(act
->kind
, a
->kind
) == 0)) {
588 write_unlock(&act_mod_lock
);
589 unregister_pernet_subsys(ops
);
593 list_add_tail(&act
->head
, &act_base
);
594 write_unlock(&act_mod_lock
);
598 EXPORT_SYMBOL(tcf_register_action
);
600 int tcf_unregister_action(struct tc_action_ops
*act
,
601 struct pernet_operations
*ops
)
603 struct tc_action_ops
*a
;
606 write_lock(&act_mod_lock
);
607 list_for_each_entry(a
, &act_base
, head
) {
609 list_del(&act
->head
);
614 write_unlock(&act_mod_lock
);
616 unregister_pernet_subsys(ops
);
619 EXPORT_SYMBOL(tcf_unregister_action
);
622 static struct tc_action_ops
*tc_lookup_action_n(char *kind
)
624 struct tc_action_ops
*a
, *res
= NULL
;
627 read_lock(&act_mod_lock
);
628 list_for_each_entry(a
, &act_base
, head
) {
629 if (strcmp(kind
, a
->kind
) == 0) {
630 if (try_module_get(a
->owner
))
635 read_unlock(&act_mod_lock
);
640 /* lookup by nlattr */
641 static struct tc_action_ops
*tc_lookup_action(struct nlattr
*kind
)
643 struct tc_action_ops
*a
, *res
= NULL
;
646 read_lock(&act_mod_lock
);
647 list_for_each_entry(a
, &act_base
, head
) {
648 if (nla_strcmp(kind
, a
->kind
) == 0) {
649 if (try_module_get(a
->owner
))
654 read_unlock(&act_mod_lock
);
659 /*TCA_ACT_MAX_PRIO is 32, there count upto 32 */
660 #define TCA_ACT_MAX_PRIO_MASK 0x1FF
661 int tcf_action_exec(struct sk_buff
*skb
, struct tc_action
**actions
,
662 int nr_actions
, struct tcf_result
*res
)
665 u32 jmp_ttl
= TCA_ACT_MAX_PRIO
; /*matches actions per filter */
669 if (skb_skip_tc_classify(skb
))
673 for (i
= 0; i
< nr_actions
; i
++) {
674 const struct tc_action
*a
= actions
[i
];
676 if (jmp_prgcnt
> 0) {
681 ret
= a
->ops
->act(skb
, a
, res
);
682 if (ret
== TC_ACT_REPEAT
)
683 goto repeat
; /* we need a ttl - JHS */
685 if (TC_ACT_EXT_CMP(ret
, TC_ACT_JUMP
)) {
686 jmp_prgcnt
= ret
& TCA_ACT_MAX_PRIO_MASK
;
687 if (!jmp_prgcnt
|| (jmp_prgcnt
> nr_actions
)) {
688 /* faulty opcode, stop pipeline */
693 goto restart_act_graph
;
694 else /* faulty graph, stop pipeline */
697 } else if (TC_ACT_EXT_CMP(ret
, TC_ACT_GOTO_CHAIN
)) {
698 if (unlikely(!rcu_access_pointer(a
->goto_chain
))) {
699 net_warn_ratelimited("can't go to NULL chain!\n");
702 tcf_action_goto_chain_exec(a
, res
);
705 if (ret
!= TC_ACT_PIPE
)
711 EXPORT_SYMBOL(tcf_action_exec
);
713 int tcf_action_destroy(struct tc_action
*actions
[], int bind
)
715 const struct tc_action_ops
*ops
;
719 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
723 ret
= __tcf_idr_release(a
, bind
, true);
724 if (ret
== ACT_P_DELETED
)
725 module_put(ops
->owner
);
732 static int tcf_action_destroy_1(struct tc_action
*a
, int bind
)
734 struct tc_action
*actions
[] = { a
, NULL
};
736 return tcf_action_destroy(actions
, bind
);
739 static int tcf_action_put(struct tc_action
*p
)
741 return __tcf_action_put(p
, false);
744 /* Put all actions in this array, skip those NULL's. */
745 static void tcf_action_put_many(struct tc_action
*actions
[])
749 for (i
= 0; i
< TCA_ACT_MAX_PRIO
; i
++) {
750 struct tc_action
*a
= actions
[i
];
751 const struct tc_action_ops
*ops
;
756 if (tcf_action_put(a
))
757 module_put(ops
->owner
);
762 tcf_action_dump_old(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
764 return a
->ops
->dump(skb
, a
, bind
, ref
);
768 tcf_action_dump_1(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
771 unsigned char *b
= skb_tail_pointer(skb
);
773 struct tc_cookie
*cookie
;
775 if (nla_put_string(skb
, TCA_KIND
, a
->ops
->kind
))
776 goto nla_put_failure
;
777 if (tcf_action_copy_stats(skb
, a
, 0))
778 goto nla_put_failure
;
781 cookie
= rcu_dereference(a
->act_cookie
);
783 if (nla_put(skb
, TCA_ACT_COOKIE
, cookie
->len
, cookie
->data
)) {
785 goto nla_put_failure
;
791 struct nla_bitfield32 flags
= { a
->tcfa_flags
,
794 if (nla_put(skb
, TCA_ACT_FLAGS
, sizeof(flags
), &flags
))
795 goto nla_put_failure
;
798 nest
= nla_nest_start_noflag(skb
, TCA_OPTIONS
);
800 goto nla_put_failure
;
801 err
= tcf_action_dump_old(skb
, a
, bind
, ref
);
803 nla_nest_end(skb
, nest
);
811 EXPORT_SYMBOL(tcf_action_dump_1
);
813 int tcf_action_dump(struct sk_buff
*skb
, struct tc_action
*actions
[],
817 int err
= -EINVAL
, i
;
820 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
822 nest
= nla_nest_start_noflag(skb
, i
+ 1);
824 goto nla_put_failure
;
825 err
= tcf_action_dump_1(skb
, a
, bind
, ref
);
828 nla_nest_end(skb
, nest
);
836 nla_nest_cancel(skb
, nest
);
840 static struct tc_cookie
*nla_memdup_cookie(struct nlattr
**tb
)
842 struct tc_cookie
*c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
846 c
->data
= nla_memdup(tb
[TCA_ACT_COOKIE
], GFP_KERNEL
);
851 c
->len
= nla_len(tb
[TCA_ACT_COOKIE
]);
856 static const u32 tca_act_flags_allowed
= TCA_ACT_FLAGS_NO_PERCPU_STATS
;
857 static const struct nla_policy tcf_action_policy
[TCA_ACT_MAX
+ 1] = {
858 [TCA_ACT_KIND
] = { .type
= NLA_STRING
},
859 [TCA_ACT_INDEX
] = { .type
= NLA_U32
},
860 [TCA_ACT_COOKIE
] = { .type
= NLA_BINARY
,
861 .len
= TC_COOKIE_MAX_SIZE
},
862 [TCA_ACT_OPTIONS
] = { .type
= NLA_NESTED
},
863 [TCA_ACT_FLAGS
] = { .type
= NLA_BITFIELD32
,
864 .validation_data
= &tca_act_flags_allowed
},
867 struct tc_action
*tcf_action_init_1(struct net
*net
, struct tcf_proto
*tp
,
868 struct nlattr
*nla
, struct nlattr
*est
,
869 char *name
, int ovr
, int bind
,
871 struct netlink_ext_ack
*extack
)
873 struct nla_bitfield32 flags
= { 0, 0 };
875 struct tc_action_ops
*a_o
;
876 struct tc_cookie
*cookie
= NULL
;
877 char act_name
[IFNAMSIZ
];
878 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
883 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
884 tcf_action_policy
, extack
);
888 kind
= tb
[TCA_ACT_KIND
];
890 NL_SET_ERR_MSG(extack
, "TC action kind must be specified");
893 if (nla_strlcpy(act_name
, kind
, IFNAMSIZ
) >= IFNAMSIZ
) {
894 NL_SET_ERR_MSG(extack
, "TC action name too long");
897 if (tb
[TCA_ACT_COOKIE
]) {
898 cookie
= nla_memdup_cookie(tb
);
900 NL_SET_ERR_MSG(extack
, "No memory to generate TC cookie");
905 if (tb
[TCA_ACT_FLAGS
])
906 flags
= nla_get_bitfield32(tb
[TCA_ACT_FLAGS
]);
908 if (strlcpy(act_name
, name
, IFNAMSIZ
) >= IFNAMSIZ
) {
909 NL_SET_ERR_MSG(extack
, "TC action name too long");
915 a_o
= tc_lookup_action_n(act_name
);
917 #ifdef CONFIG_MODULES
920 request_module("act_%s", act_name
);
924 a_o
= tc_lookup_action_n(act_name
);
926 /* We dropped the RTNL semaphore in order to
927 * perform the module load. So, even if we
928 * succeeded in loading the module we have to
929 * tell the caller to replay the request. We
930 * indicate this using -EAGAIN.
937 NL_SET_ERR_MSG(extack
, "Failed to load TC action module");
942 /* backward compatibility for policer */
944 err
= a_o
->init(net
, tb
[TCA_ACT_OPTIONS
], est
, &a
, ovr
, bind
,
945 rtnl_held
, tp
, flags
.value
, extack
);
947 err
= a_o
->init(net
, nla
, est
, &a
, ovr
, bind
, rtnl_held
,
948 tp
, flags
.value
, extack
);
952 if (!name
&& tb
[TCA_ACT_COOKIE
])
953 tcf_set_action_cookie(&a
->act_cookie
, cookie
);
955 /* module count goes up only when brand new policy is created
956 * if it exists and is only bound to in a_o->init() then
957 * ACT_P_CREATED is not returned (a zero is).
959 if (err
!= ACT_P_CREATED
)
960 module_put(a_o
->owner
);
962 if (TC_ACT_EXT_CMP(a
->tcfa_action
, TC_ACT_GOTO_CHAIN
) &&
963 !rcu_access_pointer(a
->goto_chain
)) {
964 tcf_action_destroy_1(a
, bind
);
965 NL_SET_ERR_MSG(extack
, "can't use goto chain with NULL chain");
966 return ERR_PTR(-EINVAL
);
972 module_put(a_o
->owner
);
981 /* Returns numbers of initialized actions or negative error. */
983 int tcf_action_init(struct net
*net
, struct tcf_proto
*tp
, struct nlattr
*nla
,
984 struct nlattr
*est
, char *name
, int ovr
, int bind
,
985 struct tc_action
*actions
[], size_t *attr_size
,
986 bool rtnl_held
, struct netlink_ext_ack
*extack
)
988 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
989 struct tc_action
*act
;
994 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
,
999 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
1000 act
= tcf_action_init_1(net
, tp
, tb
[i
], est
, name
, ovr
, bind
,
1006 sz
+= tcf_action_fill_size(act
);
1007 /* Start from index 0 */
1008 actions
[i
- 1] = act
;
1011 *attr_size
= tcf_action_full_attrs_size(sz
);
1015 tcf_action_destroy(actions
, bind
);
1019 void tcf_action_update_stats(struct tc_action
*a
, u64 bytes
, u32 packets
,
1022 if (a
->cpu_bstats
) {
1023 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats
), bytes
, packets
);
1026 this_cpu_ptr(a
->cpu_qstats
)->drops
+= packets
;
1029 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats_hw
),
1034 _bstats_update(&a
->tcfa_bstats
, bytes
, packets
);
1036 a
->tcfa_qstats
.drops
+= packets
;
1038 _bstats_update(&a
->tcfa_bstats_hw
, bytes
, packets
);
1040 EXPORT_SYMBOL(tcf_action_update_stats
);
1042 int tcf_action_copy_stats(struct sk_buff
*skb
, struct tc_action
*p
,
1051 /* compat_mode being true specifies a call that is supposed
1052 * to add additional backward compatibility statistic TLVs.
1055 if (p
->type
== TCA_OLD_COMPAT
)
1056 err
= gnet_stats_start_copy_compat(skb
, 0,
1064 err
= gnet_stats_start_copy(skb
, TCA_ACT_STATS
,
1065 &p
->tcfa_lock
, &d
, TCA_ACT_PAD
);
1070 if (gnet_stats_copy_basic(NULL
, &d
, p
->cpu_bstats
, &p
->tcfa_bstats
) < 0 ||
1071 gnet_stats_copy_basic_hw(NULL
, &d
, p
->cpu_bstats_hw
,
1072 &p
->tcfa_bstats_hw
) < 0 ||
1073 gnet_stats_copy_rate_est(&d
, &p
->tcfa_rate_est
) < 0 ||
1074 gnet_stats_copy_queue(&d
, p
->cpu_qstats
,
1076 p
->tcfa_qstats
.qlen
) < 0)
1079 if (gnet_stats_finish_copy(&d
) < 0)
1088 static int tca_get_fill(struct sk_buff
*skb
, struct tc_action
*actions
[],
1089 u32 portid
, u32 seq
, u16 flags
, int event
, int bind
,
1093 struct nlmsghdr
*nlh
;
1094 unsigned char *b
= skb_tail_pointer(skb
);
1095 struct nlattr
*nest
;
1097 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*t
), flags
);
1099 goto out_nlmsg_trim
;
1100 t
= nlmsg_data(nlh
);
1101 t
->tca_family
= AF_UNSPEC
;
1105 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1107 goto out_nlmsg_trim
;
1109 if (tcf_action_dump(skb
, actions
, bind
, ref
) < 0)
1110 goto out_nlmsg_trim
;
1112 nla_nest_end(skb
, nest
);
1114 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1123 tcf_get_notify(struct net
*net
, u32 portid
, struct nlmsghdr
*n
,
1124 struct tc_action
*actions
[], int event
,
1125 struct netlink_ext_ack
*extack
)
1127 struct sk_buff
*skb
;
1129 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1132 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, event
,
1134 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1139 return rtnl_unicast(skb
, net
, portid
);
1142 static struct tc_action
*tcf_action_get_1(struct net
*net
, struct nlattr
*nla
,
1143 struct nlmsghdr
*n
, u32 portid
,
1144 struct netlink_ext_ack
*extack
)
1146 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1147 const struct tc_action_ops
*ops
;
1148 struct tc_action
*a
;
1152 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
1153 tcf_action_policy
, extack
);
1158 if (tb
[TCA_ACT_INDEX
] == NULL
||
1159 nla_len(tb
[TCA_ACT_INDEX
]) < sizeof(index
)) {
1160 NL_SET_ERR_MSG(extack
, "Invalid TC action index value");
1163 index
= nla_get_u32(tb
[TCA_ACT_INDEX
]);
1166 ops
= tc_lookup_action(tb
[TCA_ACT_KIND
]);
1167 if (!ops
) { /* could happen in batch of actions */
1168 NL_SET_ERR_MSG(extack
, "Specified TC action kind not found");
1172 if (ops
->lookup(net
, &a
, index
) == 0) {
1173 NL_SET_ERR_MSG(extack
, "TC action with specified index not found");
1177 module_put(ops
->owner
);
1181 module_put(ops
->owner
);
1183 return ERR_PTR(err
);
1186 static int tca_action_flush(struct net
*net
, struct nlattr
*nla
,
1187 struct nlmsghdr
*n
, u32 portid
,
1188 struct netlink_ext_ack
*extack
)
1190 struct sk_buff
*skb
;
1192 struct nlmsghdr
*nlh
;
1194 struct netlink_callback dcb
;
1195 struct nlattr
*nest
;
1196 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1197 const struct tc_action_ops
*ops
;
1198 struct nlattr
*kind
;
1201 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1205 b
= skb_tail_pointer(skb
);
1207 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
1208 tcf_action_policy
, extack
);
1213 kind
= tb
[TCA_ACT_KIND
];
1214 ops
= tc_lookup_action(kind
);
1215 if (!ops
) { /*some idjot trying to flush unknown action */
1216 NL_SET_ERR_MSG(extack
, "Cannot flush unknown TC action");
1220 nlh
= nlmsg_put(skb
, portid
, n
->nlmsg_seq
, RTM_DELACTION
,
1223 NL_SET_ERR_MSG(extack
, "Failed to create TC action flush notification");
1224 goto out_module_put
;
1226 t
= nlmsg_data(nlh
);
1227 t
->tca_family
= AF_UNSPEC
;
1231 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1233 NL_SET_ERR_MSG(extack
, "Failed to add new netlink message");
1234 goto out_module_put
;
1237 err
= ops
->walk(net
, skb
, &dcb
, RTM_DELACTION
, ops
, extack
);
1239 nla_nest_cancel(skb
, nest
);
1240 goto out_module_put
;
1243 nla_nest_end(skb
, nest
);
1245 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1246 nlh
->nlmsg_flags
|= NLM_F_ROOT
;
1247 module_put(ops
->owner
);
1248 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1249 n
->nlmsg_flags
& NLM_F_ECHO
);
1253 NL_SET_ERR_MSG(extack
, "Failed to send TC action flush notification");
1258 module_put(ops
->owner
);
1264 static int tcf_action_delete(struct net
*net
, struct tc_action
*actions
[])
1268 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
1269 struct tc_action
*a
= actions
[i
];
1270 const struct tc_action_ops
*ops
= a
->ops
;
1271 /* Actions can be deleted concurrently so we must save their
1272 * type and id to search again after reference is released.
1274 struct tcf_idrinfo
*idrinfo
= a
->idrinfo
;
1275 u32 act_index
= a
->tcfa_index
;
1278 if (tcf_action_put(a
)) {
1279 /* last reference, action was deleted concurrently */
1280 module_put(ops
->owner
);
1284 /* now do the delete */
1285 ret
= tcf_idr_delete_index(idrinfo
, act_index
);
1294 tcf_del_notify(struct net
*net
, struct nlmsghdr
*n
, struct tc_action
*actions
[],
1295 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1298 struct sk_buff
*skb
;
1300 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1305 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, RTM_DELACTION
,
1307 NL_SET_ERR_MSG(extack
, "Failed to fill netlink TC action attributes");
1312 /* now do the delete */
1313 ret
= tcf_action_delete(net
, actions
);
1315 NL_SET_ERR_MSG(extack
, "Failed to delete TC action");
1320 ret
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1321 n
->nlmsg_flags
& NLM_F_ECHO
);
1328 tca_action_gd(struct net
*net
, struct nlattr
*nla
, struct nlmsghdr
*n
,
1329 u32 portid
, int event
, struct netlink_ext_ack
*extack
)
1332 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1333 struct tc_action
*act
;
1334 size_t attr_size
= 0;
1335 struct tc_action
*actions
[TCA_ACT_MAX_PRIO
] = {};
1337 ret
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
,
1342 if (event
== RTM_DELACTION
&& n
->nlmsg_flags
& NLM_F_ROOT
) {
1344 return tca_action_flush(net
, tb
[1], n
, portid
, extack
);
1346 NL_SET_ERR_MSG(extack
, "Invalid netlink attributes while flushing TC action");
1350 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
1351 act
= tcf_action_get_1(net
, tb
[i
], n
, portid
, extack
);
1356 attr_size
+= tcf_action_fill_size(act
);
1357 actions
[i
- 1] = act
;
1360 attr_size
= tcf_action_full_attrs_size(attr_size
);
1362 if (event
== RTM_GETACTION
)
1363 ret
= tcf_get_notify(net
, portid
, n
, actions
, event
, extack
);
1365 ret
= tcf_del_notify(net
, n
, actions
, portid
, attr_size
, extack
);
1371 tcf_action_put_many(actions
);
1376 tcf_add_notify(struct net
*net
, struct nlmsghdr
*n
, struct tc_action
*actions
[],
1377 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1379 struct sk_buff
*skb
;
1382 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1387 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, n
->nlmsg_flags
,
1388 RTM_NEWACTION
, 0, 0) <= 0) {
1389 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1394 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1395 n
->nlmsg_flags
& NLM_F_ECHO
);
1401 static int tcf_action_add(struct net
*net
, struct nlattr
*nla
,
1402 struct nlmsghdr
*n
, u32 portid
, int ovr
,
1403 struct netlink_ext_ack
*extack
)
1405 size_t attr_size
= 0;
1407 struct tc_action
*actions
[TCA_ACT_MAX_PRIO
] = {};
1409 for (loop
= 0; loop
< 10; loop
++) {
1410 ret
= tcf_action_init(net
, NULL
, nla
, NULL
, NULL
, ovr
, 0,
1411 actions
, &attr_size
, true, extack
);
1418 ret
= tcf_add_notify(net
, n
, actions
, portid
, attr_size
, extack
);
1420 tcf_action_put_many(actions
);
1425 static u32 tcaa_root_flags_allowed
= TCA_FLAG_LARGE_DUMP_ON
;
1426 static const struct nla_policy tcaa_policy
[TCA_ROOT_MAX
+ 1] = {
1427 [TCA_ROOT_FLAGS
] = { .type
= NLA_BITFIELD32
,
1428 .validation_data
= &tcaa_root_flags_allowed
},
1429 [TCA_ROOT_TIME_DELTA
] = { .type
= NLA_U32
},
1432 static int tc_ctl_action(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1433 struct netlink_ext_ack
*extack
)
1435 struct net
*net
= sock_net(skb
->sk
);
1436 struct nlattr
*tca
[TCA_ROOT_MAX
+ 1];
1437 u32 portid
= skb
? NETLINK_CB(skb
).portid
: 0;
1438 int ret
= 0, ovr
= 0;
1440 if ((n
->nlmsg_type
!= RTM_GETACTION
) &&
1441 !netlink_capable(skb
, CAP_NET_ADMIN
))
1444 ret
= nlmsg_parse_deprecated(n
, sizeof(struct tcamsg
), tca
,
1445 TCA_ROOT_MAX
, NULL
, extack
);
1449 if (tca
[TCA_ACT_TAB
] == NULL
) {
1450 NL_SET_ERR_MSG(extack
, "Netlink action attributes missing");
1454 /* n->nlmsg_flags & NLM_F_CREATE */
1455 switch (n
->nlmsg_type
) {
1457 /* we are going to assume all other flags
1458 * imply create only if it doesn't exist
1459 * Note that CREATE | EXCL implies that
1460 * but since we want avoid ambiguity (eg when flags
1461 * is zero) then just set this
1463 if (n
->nlmsg_flags
& NLM_F_REPLACE
)
1465 ret
= tcf_action_add(net
, tca
[TCA_ACT_TAB
], n
, portid
, ovr
,
1469 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1470 portid
, RTM_DELACTION
, extack
);
1473 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1474 portid
, RTM_GETACTION
, extack
);
1483 static struct nlattr
*find_dump_kind(struct nlattr
**nla
)
1485 struct nlattr
*tb1
, *tb2
[TCA_ACT_MAX
+ 1];
1486 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1487 struct nlattr
*kind
;
1489 tb1
= nla
[TCA_ACT_TAB
];
1493 if (nla_parse_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla_data(tb1
), NLMSG_ALIGN(nla_len(tb1
)), NULL
, NULL
) < 0)
1498 if (nla_parse_nested_deprecated(tb2
, TCA_ACT_MAX
, tb
[1], tcf_action_policy
, NULL
) < 0)
1500 kind
= tb2
[TCA_ACT_KIND
];
1505 static int tc_dump_action(struct sk_buff
*skb
, struct netlink_callback
*cb
)
1507 struct net
*net
= sock_net(skb
->sk
);
1508 struct nlmsghdr
*nlh
;
1509 unsigned char *b
= skb_tail_pointer(skb
);
1510 struct nlattr
*nest
;
1511 struct tc_action_ops
*a_o
;
1513 struct tcamsg
*t
= (struct tcamsg
*) nlmsg_data(cb
->nlh
);
1514 struct nlattr
*tb
[TCA_ROOT_MAX
+ 1];
1515 struct nlattr
*count_attr
= NULL
;
1516 unsigned long jiffy_since
= 0;
1517 struct nlattr
*kind
= NULL
;
1518 struct nla_bitfield32 bf
;
1519 u32 msecs_since
= 0;
1522 ret
= nlmsg_parse_deprecated(cb
->nlh
, sizeof(struct tcamsg
), tb
,
1523 TCA_ROOT_MAX
, tcaa_policy
, cb
->extack
);
1527 kind
= find_dump_kind(tb
);
1529 pr_info("tc_dump_action: action bad kind\n");
1533 a_o
= tc_lookup_action(kind
);
1538 if (tb
[TCA_ROOT_FLAGS
]) {
1539 bf
= nla_get_bitfield32(tb
[TCA_ROOT_FLAGS
]);
1540 cb
->args
[2] = bf
.value
;
1543 if (tb
[TCA_ROOT_TIME_DELTA
]) {
1544 msecs_since
= nla_get_u32(tb
[TCA_ROOT_TIME_DELTA
]);
1547 nlh
= nlmsg_put(skb
, NETLINK_CB(cb
->skb
).portid
, cb
->nlh
->nlmsg_seq
,
1548 cb
->nlh
->nlmsg_type
, sizeof(*t
), 0);
1550 goto out_module_put
;
1553 jiffy_since
= jiffies
- msecs_to_jiffies(msecs_since
);
1555 t
= nlmsg_data(nlh
);
1556 t
->tca_family
= AF_UNSPEC
;
1559 cb
->args
[3] = jiffy_since
;
1560 count_attr
= nla_reserve(skb
, TCA_ROOT_COUNT
, sizeof(u32
));
1562 goto out_module_put
;
1564 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1566 goto out_module_put
;
1568 ret
= a_o
->walk(net
, skb
, cb
, RTM_GETACTION
, a_o
, NULL
);
1570 goto out_module_put
;
1573 nla_nest_end(skb
, nest
);
1575 act_count
= cb
->args
[1];
1576 memcpy(nla_data(count_attr
), &act_count
, sizeof(u32
));
1581 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1582 if (NETLINK_CB(cb
->skb
).portid
&& ret
)
1583 nlh
->nlmsg_flags
|= NLM_F_MULTI
;
1584 module_put(a_o
->owner
);
1588 module_put(a_o
->owner
);
1593 static int __init
tc_action_init(void)
1595 rtnl_register(PF_UNSPEC
, RTM_NEWACTION
, tc_ctl_action
, NULL
, 0);
1596 rtnl_register(PF_UNSPEC
, RTM_DELACTION
, tc_ctl_action
, NULL
, 0);
1597 rtnl_register(PF_UNSPEC
, RTM_GETACTION
, tc_ctl_action
, tc_dump_action
,
1603 subsys_initcall(tc_action_init
);