1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/act_api.c Packet action API.
5 * Author: Jamal Hadi Salim
8 #include <linux/types.h>
9 #include <linux/kernel.h>
10 #include <linux/string.h>
11 #include <linux/errno.h>
12 #include <linux/slab.h>
13 #include <linux/skbuff.h>
14 #include <linux/init.h>
15 #include <linux/kmod.h>
16 #include <linux/err.h>
17 #include <linux/module.h>
18 #include <net/net_namespace.h>
20 #include <net/sch_generic.h>
21 #include <net/pkt_cls.h>
22 #include <net/act_api.h>
23 #include <net/netlink.h>
25 static void tcf_action_goto_chain_exec(const struct tc_action
*a
,
26 struct tcf_result
*res
)
28 const struct tcf_chain
*chain
= rcu_dereference_bh(a
->goto_chain
);
30 res
->goto_tp
= rcu_dereference_bh(chain
->filter_chain
);
33 static void tcf_free_cookie_rcu(struct rcu_head
*p
)
35 struct tc_cookie
*cookie
= container_of(p
, struct tc_cookie
, rcu
);
41 static void tcf_set_action_cookie(struct tc_cookie __rcu
**old_cookie
,
42 struct tc_cookie
*new_cookie
)
44 struct tc_cookie
*old
;
46 old
= xchg((__force
struct tc_cookie
**)old_cookie
, new_cookie
);
48 call_rcu(&old
->rcu
, tcf_free_cookie_rcu
);
51 int tcf_action_check_ctrlact(int action
, struct tcf_proto
*tp
,
52 struct tcf_chain
**newchain
,
53 struct netlink_ext_ack
*extack
)
55 int opcode
= TC_ACT_EXT_OPCODE(action
), ret
= -EINVAL
;
59 ret
= action
> TC_ACT_VALUE_MAX
? -EINVAL
: 0;
60 else if (opcode
<= TC_ACT_EXT_OPCODE_MAX
|| action
== TC_ACT_UNSPEC
)
63 NL_SET_ERR_MSG(extack
, "invalid control action");
67 if (TC_ACT_EXT_CMP(action
, TC_ACT_GOTO_CHAIN
)) {
68 chain_index
= action
& TC_ACT_EXT_VAL_MASK
;
69 if (!tp
|| !newchain
) {
71 NL_SET_ERR_MSG(extack
,
72 "can't goto NULL proto/chain");
75 *newchain
= tcf_chain_get_by_act(tp
->chain
->block
, chain_index
);
78 NL_SET_ERR_MSG(extack
,
79 "can't allocate goto_chain");
85 EXPORT_SYMBOL(tcf_action_check_ctrlact
);
87 struct tcf_chain
*tcf_action_set_ctrlact(struct tc_action
*a
, int action
,
88 struct tcf_chain
*goto_chain
)
90 a
->tcfa_action
= action
;
91 goto_chain
= rcu_replace_pointer(a
->goto_chain
, goto_chain
, 1);
94 EXPORT_SYMBOL(tcf_action_set_ctrlact
);
96 /* XXX: For standalone actions, we don't need a RCU grace period either, because
97 * actions are always connected to filters and filters are already destroyed in
98 * RCU callbacks, so after a RCU grace period actions are already disconnected
99 * from filters. Readers later can not find us.
101 static void free_tcf(struct tc_action
*p
)
103 struct tcf_chain
*chain
= rcu_dereference_protected(p
->goto_chain
, 1);
105 free_percpu(p
->cpu_bstats
);
106 free_percpu(p
->cpu_bstats_hw
);
107 free_percpu(p
->cpu_qstats
);
109 tcf_set_action_cookie(&p
->act_cookie
, NULL
);
111 tcf_chain_put_by_act(chain
);
116 static void tcf_action_cleanup(struct tc_action
*p
)
121 gen_kill_estimator(&p
->tcfa_rate_est
);
125 static int __tcf_action_put(struct tc_action
*p
, bool bind
)
127 struct tcf_idrinfo
*idrinfo
= p
->idrinfo
;
129 if (refcount_dec_and_mutex_lock(&p
->tcfa_refcnt
, &idrinfo
->lock
)) {
131 atomic_dec(&p
->tcfa_bindcnt
);
132 idr_remove(&idrinfo
->action_idr
, p
->tcfa_index
);
133 mutex_unlock(&idrinfo
->lock
);
135 tcf_action_cleanup(p
);
140 atomic_dec(&p
->tcfa_bindcnt
);
145 int __tcf_idr_release(struct tc_action
*p
, bool bind
, bool strict
)
149 /* Release with strict==1 and bind==0 is only called through act API
150 * interface (classifiers always bind). Only case when action with
151 * positive reference count and zero bind count can exist is when it was
152 * also created with act API (unbinding last classifier will destroy the
153 * action if it was created by classifier). So only case when bind count
154 * can be changed after initial check is when unbound action is
155 * destroyed by act API while classifier binds to action with same id
156 * concurrently. This result either creation of new action(same behavior
157 * as before), or reusing existing action if concurrent process
158 * increments reference count before action is deleted. Both scenarios
162 if (!bind
&& strict
&& atomic_read(&p
->tcfa_bindcnt
) > 0)
165 if (__tcf_action_put(p
, bind
))
171 EXPORT_SYMBOL(__tcf_idr_release
);
173 static size_t tcf_action_shared_attrs_size(const struct tc_action
*act
)
175 struct tc_cookie
*act_cookie
;
179 act_cookie
= rcu_dereference(act
->act_cookie
);
182 cookie_len
= nla_total_size(act_cookie
->len
);
185 return nla_total_size(0) /* action number nested */
186 + nla_total_size(IFNAMSIZ
) /* TCA_ACT_KIND */
187 + cookie_len
/* TCA_ACT_COOKIE */
188 + nla_total_size(0) /* TCA_ACT_STATS nested */
189 + nla_total_size(sizeof(struct nla_bitfield32
)) /* TCA_ACT_FLAGS */
190 /* TCA_STATS_BASIC */
191 + nla_total_size_64bit(sizeof(struct gnet_stats_basic
))
192 /* TCA_STATS_PKT64 */
193 + nla_total_size_64bit(sizeof(u64
))
194 /* TCA_STATS_QUEUE */
195 + nla_total_size_64bit(sizeof(struct gnet_stats_queue
))
196 + nla_total_size(0) /* TCA_OPTIONS nested */
197 + nla_total_size(sizeof(struct tcf_t
)); /* TCA_GACT_TM */
200 static size_t tcf_action_full_attrs_size(size_t sz
)
202 return NLMSG_HDRLEN
/* struct nlmsghdr */
203 + sizeof(struct tcamsg
)
204 + nla_total_size(0) /* TCA_ACT_TAB nested */
208 static size_t tcf_action_fill_size(const struct tc_action
*act
)
210 size_t sz
= tcf_action_shared_attrs_size(act
);
212 if (act
->ops
->get_fill_size
)
213 return act
->ops
->get_fill_size(act
) + sz
;
217 static int tcf_dump_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
218 struct netlink_callback
*cb
)
220 int err
= 0, index
= -1, s_i
= 0, n_i
= 0;
221 u32 act_flags
= cb
->args
[2];
222 unsigned long jiffy_since
= cb
->args
[3];
224 struct idr
*idr
= &idrinfo
->action_idr
;
226 unsigned long id
= 1;
229 mutex_lock(&idrinfo
->lock
);
233 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
239 time_after(jiffy_since
,
240 (unsigned long)p
->tcfa_tm
.lastuse
))
243 nest
= nla_nest_start_noflag(skb
, n_i
);
246 goto nla_put_failure
;
248 err
= tcf_action_dump_1(skb
, p
, 0, 0);
251 nlmsg_trim(skb
, nest
);
254 nla_nest_end(skb
, nest
);
256 if (!(act_flags
& TCA_FLAG_LARGE_DUMP_ON
) &&
257 n_i
>= TCA_ACT_MAX_PRIO
)
262 cb
->args
[0] = index
+ 1;
264 mutex_unlock(&idrinfo
->lock
);
266 if (act_flags
& TCA_FLAG_LARGE_DUMP_ON
)
272 nla_nest_cancel(skb
, nest
);
276 static int tcf_idr_release_unsafe(struct tc_action
*p
)
278 if (atomic_read(&p
->tcfa_bindcnt
) > 0)
281 if (refcount_dec_and_test(&p
->tcfa_refcnt
)) {
282 idr_remove(&p
->idrinfo
->action_idr
, p
->tcfa_index
);
283 tcf_action_cleanup(p
);
284 return ACT_P_DELETED
;
290 static int tcf_del_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
291 const struct tc_action_ops
*ops
)
296 struct idr
*idr
= &idrinfo
->action_idr
;
298 unsigned long id
= 1;
301 nest
= nla_nest_start_noflag(skb
, 0);
303 goto nla_put_failure
;
304 if (nla_put_string(skb
, TCA_KIND
, ops
->kind
))
305 goto nla_put_failure
;
307 mutex_lock(&idrinfo
->lock
);
308 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
309 ret
= tcf_idr_release_unsafe(p
);
310 if (ret
== ACT_P_DELETED
) {
311 module_put(ops
->owner
);
313 } else if (ret
< 0) {
314 mutex_unlock(&idrinfo
->lock
);
315 goto nla_put_failure
;
318 mutex_unlock(&idrinfo
->lock
);
320 if (nla_put_u32(skb
, TCA_FCNT
, n_i
))
321 goto nla_put_failure
;
322 nla_nest_end(skb
, nest
);
326 nla_nest_cancel(skb
, nest
);
330 int tcf_generic_walker(struct tc_action_net
*tn
, struct sk_buff
*skb
,
331 struct netlink_callback
*cb
, int type
,
332 const struct tc_action_ops
*ops
,
333 struct netlink_ext_ack
*extack
)
335 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
337 if (type
== RTM_DELACTION
) {
338 return tcf_del_walker(idrinfo
, skb
, ops
);
339 } else if (type
== RTM_GETACTION
) {
340 return tcf_dump_walker(idrinfo
, skb
, cb
);
342 WARN(1, "tcf_generic_walker: unknown command %d\n", type
);
343 NL_SET_ERR_MSG(extack
, "tcf_generic_walker: unknown command");
347 EXPORT_SYMBOL(tcf_generic_walker
);
349 int tcf_idr_search(struct tc_action_net
*tn
, struct tc_action
**a
, u32 index
)
351 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
354 mutex_lock(&idrinfo
->lock
);
355 p
= idr_find(&idrinfo
->action_idr
, index
);
359 refcount_inc(&p
->tcfa_refcnt
);
360 mutex_unlock(&idrinfo
->lock
);
368 EXPORT_SYMBOL(tcf_idr_search
);
370 static int tcf_idr_delete_index(struct tcf_idrinfo
*idrinfo
, u32 index
)
375 mutex_lock(&idrinfo
->lock
);
376 p
= idr_find(&idrinfo
->action_idr
, index
);
378 mutex_unlock(&idrinfo
->lock
);
382 if (!atomic_read(&p
->tcfa_bindcnt
)) {
383 if (refcount_dec_and_test(&p
->tcfa_refcnt
)) {
384 struct module
*owner
= p
->ops
->owner
;
386 WARN_ON(p
!= idr_remove(&idrinfo
->action_idr
,
388 mutex_unlock(&idrinfo
->lock
);
390 tcf_action_cleanup(p
);
399 mutex_unlock(&idrinfo
->lock
);
403 int tcf_idr_create(struct tc_action_net
*tn
, u32 index
, struct nlattr
*est
,
404 struct tc_action
**a
, const struct tc_action_ops
*ops
,
405 int bind
, bool cpustats
, u32 flags
)
407 struct tc_action
*p
= kzalloc(ops
->size
, GFP_KERNEL
);
408 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
413 refcount_set(&p
->tcfa_refcnt
, 1);
415 atomic_set(&p
->tcfa_bindcnt
, 1);
418 p
->cpu_bstats
= netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu
);
421 p
->cpu_bstats_hw
= netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu
);
422 if (!p
->cpu_bstats_hw
)
424 p
->cpu_qstats
= alloc_percpu(struct gnet_stats_queue
);
428 spin_lock_init(&p
->tcfa_lock
);
429 p
->tcfa_index
= index
;
430 p
->tcfa_tm
.install
= jiffies
;
431 p
->tcfa_tm
.lastuse
= jiffies
;
432 p
->tcfa_tm
.firstuse
= 0;
433 p
->tcfa_flags
= flags
;
435 err
= gen_new_estimator(&p
->tcfa_bstats
, p
->cpu_bstats
,
437 &p
->tcfa_lock
, NULL
, est
);
442 p
->idrinfo
= idrinfo
;
447 free_percpu(p
->cpu_qstats
);
449 free_percpu(p
->cpu_bstats_hw
);
451 free_percpu(p
->cpu_bstats
);
456 EXPORT_SYMBOL(tcf_idr_create
);
458 int tcf_idr_create_from_flags(struct tc_action_net
*tn
, u32 index
,
459 struct nlattr
*est
, struct tc_action
**a
,
460 const struct tc_action_ops
*ops
, int bind
,
463 /* Set cpustats according to actions flags. */
464 return tcf_idr_create(tn
, index
, est
, a
, ops
, bind
,
465 !(flags
& TCA_ACT_FLAGS_NO_PERCPU_STATS
), flags
);
467 EXPORT_SYMBOL(tcf_idr_create_from_flags
);
469 void tcf_idr_insert(struct tc_action_net
*tn
, struct tc_action
*a
)
471 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
473 mutex_lock(&idrinfo
->lock
);
474 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
475 WARN_ON(!IS_ERR(idr_replace(&idrinfo
->action_idr
, a
, a
->tcfa_index
)));
476 mutex_unlock(&idrinfo
->lock
);
478 EXPORT_SYMBOL(tcf_idr_insert
);
480 /* Cleanup idr index that was allocated but not initialized. */
482 void tcf_idr_cleanup(struct tc_action_net
*tn
, u32 index
)
484 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
486 mutex_lock(&idrinfo
->lock
);
487 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
488 WARN_ON(!IS_ERR(idr_remove(&idrinfo
->action_idr
, index
)));
489 mutex_unlock(&idrinfo
->lock
);
491 EXPORT_SYMBOL(tcf_idr_cleanup
);
493 /* Check if action with specified index exists. If actions is found, increments
494 * its reference and bind counters, and return 1. Otherwise insert temporary
495 * error pointer (to prevent concurrent users from inserting actions with same
496 * index) and return 0.
499 int tcf_idr_check_alloc(struct tc_action_net
*tn
, u32
*index
,
500 struct tc_action
**a
, int bind
)
502 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
507 mutex_lock(&idrinfo
->lock
);
509 p
= idr_find(&idrinfo
->action_idr
, *index
);
511 /* This means that another process allocated
512 * index but did not assign the pointer yet.
514 mutex_unlock(&idrinfo
->lock
);
519 refcount_inc(&p
->tcfa_refcnt
);
521 atomic_inc(&p
->tcfa_bindcnt
);
526 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
529 idr_replace(&idrinfo
->action_idr
,
530 ERR_PTR(-EBUSY
), *index
);
535 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
536 UINT_MAX
, GFP_KERNEL
);
538 idr_replace(&idrinfo
->action_idr
, ERR_PTR(-EBUSY
),
541 mutex_unlock(&idrinfo
->lock
);
544 EXPORT_SYMBOL(tcf_idr_check_alloc
);
546 void tcf_idrinfo_destroy(const struct tc_action_ops
*ops
,
547 struct tcf_idrinfo
*idrinfo
)
549 struct idr
*idr
= &idrinfo
->action_idr
;
552 unsigned long id
= 1;
555 idr_for_each_entry_ul(idr
, p
, tmp
, id
) {
556 ret
= __tcf_idr_release(p
, false, true);
557 if (ret
== ACT_P_DELETED
)
558 module_put(ops
->owner
);
562 idr_destroy(&idrinfo
->action_idr
);
564 EXPORT_SYMBOL(tcf_idrinfo_destroy
);
566 static LIST_HEAD(act_base
);
567 static DEFINE_RWLOCK(act_mod_lock
);
569 int tcf_register_action(struct tc_action_ops
*act
,
570 struct pernet_operations
*ops
)
572 struct tc_action_ops
*a
;
575 if (!act
->act
|| !act
->dump
|| !act
->init
|| !act
->walk
|| !act
->lookup
)
578 /* We have to register pernet ops before making the action ops visible,
579 * otherwise tcf_action_init_1() could get a partially initialized
582 ret
= register_pernet_subsys(ops
);
586 write_lock(&act_mod_lock
);
587 list_for_each_entry(a
, &act_base
, head
) {
588 if (act
->id
== a
->id
|| (strcmp(act
->kind
, a
->kind
) == 0)) {
589 write_unlock(&act_mod_lock
);
590 unregister_pernet_subsys(ops
);
594 list_add_tail(&act
->head
, &act_base
);
595 write_unlock(&act_mod_lock
);
599 EXPORT_SYMBOL(tcf_register_action
);
601 int tcf_unregister_action(struct tc_action_ops
*act
,
602 struct pernet_operations
*ops
)
604 struct tc_action_ops
*a
;
607 write_lock(&act_mod_lock
);
608 list_for_each_entry(a
, &act_base
, head
) {
610 list_del(&act
->head
);
615 write_unlock(&act_mod_lock
);
617 unregister_pernet_subsys(ops
);
620 EXPORT_SYMBOL(tcf_unregister_action
);
623 static struct tc_action_ops
*tc_lookup_action_n(char *kind
)
625 struct tc_action_ops
*a
, *res
= NULL
;
628 read_lock(&act_mod_lock
);
629 list_for_each_entry(a
, &act_base
, head
) {
630 if (strcmp(kind
, a
->kind
) == 0) {
631 if (try_module_get(a
->owner
))
636 read_unlock(&act_mod_lock
);
641 /* lookup by nlattr */
642 static struct tc_action_ops
*tc_lookup_action(struct nlattr
*kind
)
644 struct tc_action_ops
*a
, *res
= NULL
;
647 read_lock(&act_mod_lock
);
648 list_for_each_entry(a
, &act_base
, head
) {
649 if (nla_strcmp(kind
, a
->kind
) == 0) {
650 if (try_module_get(a
->owner
))
655 read_unlock(&act_mod_lock
);
660 /*TCA_ACT_MAX_PRIO is 32, there count upto 32 */
661 #define TCA_ACT_MAX_PRIO_MASK 0x1FF
662 int tcf_action_exec(struct sk_buff
*skb
, struct tc_action
**actions
,
663 int nr_actions
, struct tcf_result
*res
)
666 u32 jmp_ttl
= TCA_ACT_MAX_PRIO
; /*matches actions per filter */
670 if (skb_skip_tc_classify(skb
))
674 for (i
= 0; i
< nr_actions
; i
++) {
675 const struct tc_action
*a
= actions
[i
];
677 if (jmp_prgcnt
> 0) {
682 ret
= a
->ops
->act(skb
, a
, res
);
683 if (ret
== TC_ACT_REPEAT
)
684 goto repeat
; /* we need a ttl - JHS */
686 if (TC_ACT_EXT_CMP(ret
, TC_ACT_JUMP
)) {
687 jmp_prgcnt
= ret
& TCA_ACT_MAX_PRIO_MASK
;
688 if (!jmp_prgcnt
|| (jmp_prgcnt
> nr_actions
)) {
689 /* faulty opcode, stop pipeline */
694 goto restart_act_graph
;
695 else /* faulty graph, stop pipeline */
698 } else if (TC_ACT_EXT_CMP(ret
, TC_ACT_GOTO_CHAIN
)) {
699 if (unlikely(!rcu_access_pointer(a
->goto_chain
))) {
700 net_warn_ratelimited("can't go to NULL chain!\n");
703 tcf_action_goto_chain_exec(a
, res
);
706 if (ret
!= TC_ACT_PIPE
)
712 EXPORT_SYMBOL(tcf_action_exec
);
714 int tcf_action_destroy(struct tc_action
*actions
[], int bind
)
716 const struct tc_action_ops
*ops
;
720 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
724 ret
= __tcf_idr_release(a
, bind
, true);
725 if (ret
== ACT_P_DELETED
)
726 module_put(ops
->owner
);
733 static int tcf_action_destroy_1(struct tc_action
*a
, int bind
)
735 struct tc_action
*actions
[] = { a
, NULL
};
737 return tcf_action_destroy(actions
, bind
);
740 static int tcf_action_put(struct tc_action
*p
)
742 return __tcf_action_put(p
, false);
745 /* Put all actions in this array, skip those NULL's. */
746 static void tcf_action_put_many(struct tc_action
*actions
[])
750 for (i
= 0; i
< TCA_ACT_MAX_PRIO
; i
++) {
751 struct tc_action
*a
= actions
[i
];
752 const struct tc_action_ops
*ops
;
757 if (tcf_action_put(a
))
758 module_put(ops
->owner
);
763 tcf_action_dump_old(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
765 return a
->ops
->dump(skb
, a
, bind
, ref
);
769 tcf_action_dump_1(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
772 unsigned char *b
= skb_tail_pointer(skb
);
774 struct tc_cookie
*cookie
;
776 if (nla_put_string(skb
, TCA_KIND
, a
->ops
->kind
))
777 goto nla_put_failure
;
778 if (tcf_action_copy_stats(skb
, a
, 0))
779 goto nla_put_failure
;
782 cookie
= rcu_dereference(a
->act_cookie
);
784 if (nla_put(skb
, TCA_ACT_COOKIE
, cookie
->len
, cookie
->data
)) {
786 goto nla_put_failure
;
792 struct nla_bitfield32 flags
= { a
->tcfa_flags
,
795 if (nla_put(skb
, TCA_ACT_FLAGS
, sizeof(flags
), &flags
))
796 goto nla_put_failure
;
799 nest
= nla_nest_start_noflag(skb
, TCA_OPTIONS
);
801 goto nla_put_failure
;
802 err
= tcf_action_dump_old(skb
, a
, bind
, ref
);
804 nla_nest_end(skb
, nest
);
812 EXPORT_SYMBOL(tcf_action_dump_1
);
814 int tcf_action_dump(struct sk_buff
*skb
, struct tc_action
*actions
[],
818 int err
= -EINVAL
, i
;
821 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
823 nest
= nla_nest_start_noflag(skb
, i
+ 1);
825 goto nla_put_failure
;
826 err
= tcf_action_dump_1(skb
, a
, bind
, ref
);
829 nla_nest_end(skb
, nest
);
837 nla_nest_cancel(skb
, nest
);
841 static struct tc_cookie
*nla_memdup_cookie(struct nlattr
**tb
)
843 struct tc_cookie
*c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
847 c
->data
= nla_memdup(tb
[TCA_ACT_COOKIE
], GFP_KERNEL
);
852 c
->len
= nla_len(tb
[TCA_ACT_COOKIE
]);
857 static const u32 tca_act_flags_allowed
= TCA_ACT_FLAGS_NO_PERCPU_STATS
;
858 static const struct nla_policy tcf_action_policy
[TCA_ACT_MAX
+ 1] = {
859 [TCA_ACT_KIND
] = { .type
= NLA_STRING
},
860 [TCA_ACT_INDEX
] = { .type
= NLA_U32
},
861 [TCA_ACT_COOKIE
] = { .type
= NLA_BINARY
,
862 .len
= TC_COOKIE_MAX_SIZE
},
863 [TCA_ACT_OPTIONS
] = { .type
= NLA_NESTED
},
864 [TCA_ACT_FLAGS
] = { .type
= NLA_BITFIELD32
,
865 .validation_data
= &tca_act_flags_allowed
},
868 struct tc_action
*tcf_action_init_1(struct net
*net
, struct tcf_proto
*tp
,
869 struct nlattr
*nla
, struct nlattr
*est
,
870 char *name
, int ovr
, int bind
,
872 struct netlink_ext_ack
*extack
)
874 struct nla_bitfield32 flags
= { 0, 0 };
876 struct tc_action_ops
*a_o
;
877 struct tc_cookie
*cookie
= NULL
;
878 char act_name
[IFNAMSIZ
];
879 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
884 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
885 tcf_action_policy
, extack
);
889 kind
= tb
[TCA_ACT_KIND
];
891 NL_SET_ERR_MSG(extack
, "TC action kind must be specified");
894 if (nla_strlcpy(act_name
, kind
, IFNAMSIZ
) >= IFNAMSIZ
) {
895 NL_SET_ERR_MSG(extack
, "TC action name too long");
898 if (tb
[TCA_ACT_COOKIE
]) {
899 cookie
= nla_memdup_cookie(tb
);
901 NL_SET_ERR_MSG(extack
, "No memory to generate TC cookie");
906 if (tb
[TCA_ACT_FLAGS
])
907 flags
= nla_get_bitfield32(tb
[TCA_ACT_FLAGS
]);
909 if (strlcpy(act_name
, name
, IFNAMSIZ
) >= IFNAMSIZ
) {
910 NL_SET_ERR_MSG(extack
, "TC action name too long");
916 a_o
= tc_lookup_action_n(act_name
);
918 #ifdef CONFIG_MODULES
921 request_module("act_%s", act_name
);
925 a_o
= tc_lookup_action_n(act_name
);
927 /* We dropped the RTNL semaphore in order to
928 * perform the module load. So, even if we
929 * succeeded in loading the module we have to
930 * tell the caller to replay the request. We
931 * indicate this using -EAGAIN.
938 NL_SET_ERR_MSG(extack
, "Failed to load TC action module");
943 /* backward compatibility for policer */
945 err
= a_o
->init(net
, tb
[TCA_ACT_OPTIONS
], est
, &a
, ovr
, bind
,
946 rtnl_held
, tp
, flags
.value
, extack
);
948 err
= a_o
->init(net
, nla
, est
, &a
, ovr
, bind
, rtnl_held
,
949 tp
, flags
.value
, extack
);
953 if (!name
&& tb
[TCA_ACT_COOKIE
])
954 tcf_set_action_cookie(&a
->act_cookie
, cookie
);
956 /* module count goes up only when brand new policy is created
957 * if it exists and is only bound to in a_o->init() then
958 * ACT_P_CREATED is not returned (a zero is).
960 if (err
!= ACT_P_CREATED
)
961 module_put(a_o
->owner
);
963 if (TC_ACT_EXT_CMP(a
->tcfa_action
, TC_ACT_GOTO_CHAIN
) &&
964 !rcu_access_pointer(a
->goto_chain
)) {
965 tcf_action_destroy_1(a
, bind
);
966 NL_SET_ERR_MSG(extack
, "can't use goto chain with NULL chain");
967 return ERR_PTR(-EINVAL
);
973 module_put(a_o
->owner
);
982 /* Returns numbers of initialized actions or negative error. */
984 int tcf_action_init(struct net
*net
, struct tcf_proto
*tp
, struct nlattr
*nla
,
985 struct nlattr
*est
, char *name
, int ovr
, int bind
,
986 struct tc_action
*actions
[], size_t *attr_size
,
987 bool rtnl_held
, struct netlink_ext_ack
*extack
)
989 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
990 struct tc_action
*act
;
995 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
,
1000 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
1001 act
= tcf_action_init_1(net
, tp
, tb
[i
], est
, name
, ovr
, bind
,
1007 sz
+= tcf_action_fill_size(act
);
1008 /* Start from index 0 */
1009 actions
[i
- 1] = act
;
1012 *attr_size
= tcf_action_full_attrs_size(sz
);
1016 tcf_action_destroy(actions
, bind
);
1020 void tcf_action_update_stats(struct tc_action
*a
, u64 bytes
, u32 packets
,
1023 if (a
->cpu_bstats
) {
1024 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats
), bytes
, packets
);
1027 this_cpu_ptr(a
->cpu_qstats
)->drops
+= packets
;
1030 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats_hw
),
1035 _bstats_update(&a
->tcfa_bstats
, bytes
, packets
);
1037 a
->tcfa_qstats
.drops
+= packets
;
1039 _bstats_update(&a
->tcfa_bstats_hw
, bytes
, packets
);
1041 EXPORT_SYMBOL(tcf_action_update_stats
);
1043 int tcf_action_copy_stats(struct sk_buff
*skb
, struct tc_action
*p
,
1052 /* compat_mode being true specifies a call that is supposed
1053 * to add additional backward compatibility statistic TLVs.
1056 if (p
->type
== TCA_OLD_COMPAT
)
1057 err
= gnet_stats_start_copy_compat(skb
, 0,
1065 err
= gnet_stats_start_copy(skb
, TCA_ACT_STATS
,
1066 &p
->tcfa_lock
, &d
, TCA_ACT_PAD
);
1071 if (gnet_stats_copy_basic(NULL
, &d
, p
->cpu_bstats
, &p
->tcfa_bstats
) < 0 ||
1072 gnet_stats_copy_basic_hw(NULL
, &d
, p
->cpu_bstats_hw
,
1073 &p
->tcfa_bstats_hw
) < 0 ||
1074 gnet_stats_copy_rate_est(&d
, &p
->tcfa_rate_est
) < 0 ||
1075 gnet_stats_copy_queue(&d
, p
->cpu_qstats
,
1077 p
->tcfa_qstats
.qlen
) < 0)
1080 if (gnet_stats_finish_copy(&d
) < 0)
1089 static int tca_get_fill(struct sk_buff
*skb
, struct tc_action
*actions
[],
1090 u32 portid
, u32 seq
, u16 flags
, int event
, int bind
,
1094 struct nlmsghdr
*nlh
;
1095 unsigned char *b
= skb_tail_pointer(skb
);
1096 struct nlattr
*nest
;
1098 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*t
), flags
);
1100 goto out_nlmsg_trim
;
1101 t
= nlmsg_data(nlh
);
1102 t
->tca_family
= AF_UNSPEC
;
1106 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1108 goto out_nlmsg_trim
;
1110 if (tcf_action_dump(skb
, actions
, bind
, ref
) < 0)
1111 goto out_nlmsg_trim
;
1113 nla_nest_end(skb
, nest
);
1115 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1124 tcf_get_notify(struct net
*net
, u32 portid
, struct nlmsghdr
*n
,
1125 struct tc_action
*actions
[], int event
,
1126 struct netlink_ext_ack
*extack
)
1128 struct sk_buff
*skb
;
1130 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1133 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, event
,
1135 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1140 return rtnl_unicast(skb
, net
, portid
);
1143 static struct tc_action
*tcf_action_get_1(struct net
*net
, struct nlattr
*nla
,
1144 struct nlmsghdr
*n
, u32 portid
,
1145 struct netlink_ext_ack
*extack
)
1147 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1148 const struct tc_action_ops
*ops
;
1149 struct tc_action
*a
;
1153 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
1154 tcf_action_policy
, extack
);
1159 if (tb
[TCA_ACT_INDEX
] == NULL
||
1160 nla_len(tb
[TCA_ACT_INDEX
]) < sizeof(index
)) {
1161 NL_SET_ERR_MSG(extack
, "Invalid TC action index value");
1164 index
= nla_get_u32(tb
[TCA_ACT_INDEX
]);
1167 ops
= tc_lookup_action(tb
[TCA_ACT_KIND
]);
1168 if (!ops
) { /* could happen in batch of actions */
1169 NL_SET_ERR_MSG(extack
, "Specified TC action kind not found");
1173 if (ops
->lookup(net
, &a
, index
) == 0) {
1174 NL_SET_ERR_MSG(extack
, "TC action with specified index not found");
1178 module_put(ops
->owner
);
1182 module_put(ops
->owner
);
1184 return ERR_PTR(err
);
1187 static int tca_action_flush(struct net
*net
, struct nlattr
*nla
,
1188 struct nlmsghdr
*n
, u32 portid
,
1189 struct netlink_ext_ack
*extack
)
1191 struct sk_buff
*skb
;
1193 struct nlmsghdr
*nlh
;
1195 struct netlink_callback dcb
;
1196 struct nlattr
*nest
;
1197 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1198 const struct tc_action_ops
*ops
;
1199 struct nlattr
*kind
;
1202 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1206 b
= skb_tail_pointer(skb
);
1208 err
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX
, nla
,
1209 tcf_action_policy
, extack
);
1214 kind
= tb
[TCA_ACT_KIND
];
1215 ops
= tc_lookup_action(kind
);
1216 if (!ops
) { /*some idjot trying to flush unknown action */
1217 NL_SET_ERR_MSG(extack
, "Cannot flush unknown TC action");
1221 nlh
= nlmsg_put(skb
, portid
, n
->nlmsg_seq
, RTM_DELACTION
,
1224 NL_SET_ERR_MSG(extack
, "Failed to create TC action flush notification");
1225 goto out_module_put
;
1227 t
= nlmsg_data(nlh
);
1228 t
->tca_family
= AF_UNSPEC
;
1232 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1234 NL_SET_ERR_MSG(extack
, "Failed to add new netlink message");
1235 goto out_module_put
;
1238 err
= ops
->walk(net
, skb
, &dcb
, RTM_DELACTION
, ops
, extack
);
1240 nla_nest_cancel(skb
, nest
);
1241 goto out_module_put
;
1244 nla_nest_end(skb
, nest
);
1246 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1247 nlh
->nlmsg_flags
|= NLM_F_ROOT
;
1248 module_put(ops
->owner
);
1249 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1250 n
->nlmsg_flags
& NLM_F_ECHO
);
1254 NL_SET_ERR_MSG(extack
, "Failed to send TC action flush notification");
1259 module_put(ops
->owner
);
1265 static int tcf_action_delete(struct net
*net
, struct tc_action
*actions
[])
1269 for (i
= 0; i
< TCA_ACT_MAX_PRIO
&& actions
[i
]; i
++) {
1270 struct tc_action
*a
= actions
[i
];
1271 const struct tc_action_ops
*ops
= a
->ops
;
1272 /* Actions can be deleted concurrently so we must save their
1273 * type and id to search again after reference is released.
1275 struct tcf_idrinfo
*idrinfo
= a
->idrinfo
;
1276 u32 act_index
= a
->tcfa_index
;
1279 if (tcf_action_put(a
)) {
1280 /* last reference, action was deleted concurrently */
1281 module_put(ops
->owner
);
1285 /* now do the delete */
1286 ret
= tcf_idr_delete_index(idrinfo
, act_index
);
1295 tcf_del_notify(struct net
*net
, struct nlmsghdr
*n
, struct tc_action
*actions
[],
1296 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1299 struct sk_buff
*skb
;
1301 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1306 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, RTM_DELACTION
,
1308 NL_SET_ERR_MSG(extack
, "Failed to fill netlink TC action attributes");
1313 /* now do the delete */
1314 ret
= tcf_action_delete(net
, actions
);
1316 NL_SET_ERR_MSG(extack
, "Failed to delete TC action");
1321 ret
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1322 n
->nlmsg_flags
& NLM_F_ECHO
);
1329 tca_action_gd(struct net
*net
, struct nlattr
*nla
, struct nlmsghdr
*n
,
1330 u32 portid
, int event
, struct netlink_ext_ack
*extack
)
1333 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1334 struct tc_action
*act
;
1335 size_t attr_size
= 0;
1336 struct tc_action
*actions
[TCA_ACT_MAX_PRIO
] = {};
1338 ret
= nla_parse_nested_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
,
1343 if (event
== RTM_DELACTION
&& n
->nlmsg_flags
& NLM_F_ROOT
) {
1345 return tca_action_flush(net
, tb
[1], n
, portid
, extack
);
1347 NL_SET_ERR_MSG(extack
, "Invalid netlink attributes while flushing TC action");
1351 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
1352 act
= tcf_action_get_1(net
, tb
[i
], n
, portid
, extack
);
1357 attr_size
+= tcf_action_fill_size(act
);
1358 actions
[i
- 1] = act
;
1361 attr_size
= tcf_action_full_attrs_size(attr_size
);
1363 if (event
== RTM_GETACTION
)
1364 ret
= tcf_get_notify(net
, portid
, n
, actions
, event
, extack
);
1366 ret
= tcf_del_notify(net
, n
, actions
, portid
, attr_size
, extack
);
1372 tcf_action_put_many(actions
);
1377 tcf_add_notify(struct net
*net
, struct nlmsghdr
*n
, struct tc_action
*actions
[],
1378 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1380 struct sk_buff
*skb
;
1383 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1388 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, n
->nlmsg_flags
,
1389 RTM_NEWACTION
, 0, 0) <= 0) {
1390 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1395 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1396 n
->nlmsg_flags
& NLM_F_ECHO
);
1402 static int tcf_action_add(struct net
*net
, struct nlattr
*nla
,
1403 struct nlmsghdr
*n
, u32 portid
, int ovr
,
1404 struct netlink_ext_ack
*extack
)
1406 size_t attr_size
= 0;
1408 struct tc_action
*actions
[TCA_ACT_MAX_PRIO
] = {};
1410 for (loop
= 0; loop
< 10; loop
++) {
1411 ret
= tcf_action_init(net
, NULL
, nla
, NULL
, NULL
, ovr
, 0,
1412 actions
, &attr_size
, true, extack
);
1419 ret
= tcf_add_notify(net
, n
, actions
, portid
, attr_size
, extack
);
1421 tcf_action_put_many(actions
);
1426 static u32 tcaa_root_flags_allowed
= TCA_FLAG_LARGE_DUMP_ON
;
1427 static const struct nla_policy tcaa_policy
[TCA_ROOT_MAX
+ 1] = {
1428 [TCA_ROOT_FLAGS
] = { .type
= NLA_BITFIELD32
,
1429 .validation_data
= &tcaa_root_flags_allowed
},
1430 [TCA_ROOT_TIME_DELTA
] = { .type
= NLA_U32
},
1433 static int tc_ctl_action(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1434 struct netlink_ext_ack
*extack
)
1436 struct net
*net
= sock_net(skb
->sk
);
1437 struct nlattr
*tca
[TCA_ROOT_MAX
+ 1];
1438 u32 portid
= skb
? NETLINK_CB(skb
).portid
: 0;
1439 int ret
= 0, ovr
= 0;
1441 if ((n
->nlmsg_type
!= RTM_GETACTION
) &&
1442 !netlink_capable(skb
, CAP_NET_ADMIN
))
1445 ret
= nlmsg_parse_deprecated(n
, sizeof(struct tcamsg
), tca
,
1446 TCA_ROOT_MAX
, NULL
, extack
);
1450 if (tca
[TCA_ACT_TAB
] == NULL
) {
1451 NL_SET_ERR_MSG(extack
, "Netlink action attributes missing");
1455 /* n->nlmsg_flags & NLM_F_CREATE */
1456 switch (n
->nlmsg_type
) {
1458 /* we are going to assume all other flags
1459 * imply create only if it doesn't exist
1460 * Note that CREATE | EXCL implies that
1461 * but since we want avoid ambiguity (eg when flags
1462 * is zero) then just set this
1464 if (n
->nlmsg_flags
& NLM_F_REPLACE
)
1466 ret
= tcf_action_add(net
, tca
[TCA_ACT_TAB
], n
, portid
, ovr
,
1470 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1471 portid
, RTM_DELACTION
, extack
);
1474 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1475 portid
, RTM_GETACTION
, extack
);
1484 static struct nlattr
*find_dump_kind(struct nlattr
**nla
)
1486 struct nlattr
*tb1
, *tb2
[TCA_ACT_MAX
+ 1];
1487 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1488 struct nlattr
*kind
;
1490 tb1
= nla
[TCA_ACT_TAB
];
1494 if (nla_parse_deprecated(tb
, TCA_ACT_MAX_PRIO
, nla_data(tb1
), NLMSG_ALIGN(nla_len(tb1
)), NULL
, NULL
) < 0)
1499 if (nla_parse_nested_deprecated(tb2
, TCA_ACT_MAX
, tb
[1], tcf_action_policy
, NULL
) < 0)
1501 kind
= tb2
[TCA_ACT_KIND
];
1506 static int tc_dump_action(struct sk_buff
*skb
, struct netlink_callback
*cb
)
1508 struct net
*net
= sock_net(skb
->sk
);
1509 struct nlmsghdr
*nlh
;
1510 unsigned char *b
= skb_tail_pointer(skb
);
1511 struct nlattr
*nest
;
1512 struct tc_action_ops
*a_o
;
1514 struct tcamsg
*t
= (struct tcamsg
*) nlmsg_data(cb
->nlh
);
1515 struct nlattr
*tb
[TCA_ROOT_MAX
+ 1];
1516 struct nlattr
*count_attr
= NULL
;
1517 unsigned long jiffy_since
= 0;
1518 struct nlattr
*kind
= NULL
;
1519 struct nla_bitfield32 bf
;
1520 u32 msecs_since
= 0;
1523 ret
= nlmsg_parse_deprecated(cb
->nlh
, sizeof(struct tcamsg
), tb
,
1524 TCA_ROOT_MAX
, tcaa_policy
, cb
->extack
);
1528 kind
= find_dump_kind(tb
);
1530 pr_info("tc_dump_action: action bad kind\n");
1534 a_o
= tc_lookup_action(kind
);
1539 if (tb
[TCA_ROOT_FLAGS
]) {
1540 bf
= nla_get_bitfield32(tb
[TCA_ROOT_FLAGS
]);
1541 cb
->args
[2] = bf
.value
;
1544 if (tb
[TCA_ROOT_TIME_DELTA
]) {
1545 msecs_since
= nla_get_u32(tb
[TCA_ROOT_TIME_DELTA
]);
1548 nlh
= nlmsg_put(skb
, NETLINK_CB(cb
->skb
).portid
, cb
->nlh
->nlmsg_seq
,
1549 cb
->nlh
->nlmsg_type
, sizeof(*t
), 0);
1551 goto out_module_put
;
1554 jiffy_since
= jiffies
- msecs_to_jiffies(msecs_since
);
1556 t
= nlmsg_data(nlh
);
1557 t
->tca_family
= AF_UNSPEC
;
1560 cb
->args
[3] = jiffy_since
;
1561 count_attr
= nla_reserve(skb
, TCA_ROOT_COUNT
, sizeof(u32
));
1563 goto out_module_put
;
1565 nest
= nla_nest_start_noflag(skb
, TCA_ACT_TAB
);
1567 goto out_module_put
;
1569 ret
= a_o
->walk(net
, skb
, cb
, RTM_GETACTION
, a_o
, NULL
);
1571 goto out_module_put
;
1574 nla_nest_end(skb
, nest
);
1576 act_count
= cb
->args
[1];
1577 memcpy(nla_data(count_attr
), &act_count
, sizeof(u32
));
1582 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1583 if (NETLINK_CB(cb
->skb
).portid
&& ret
)
1584 nlh
->nlmsg_flags
|= NLM_F_MULTI
;
1585 module_put(a_o
->owner
);
1589 module_put(a_o
->owner
);
1594 static int __init
tc_action_init(void)
1596 rtnl_register(PF_UNSPEC
, RTM_NEWACTION
, tc_ctl_action
, NULL
, 0);
1597 rtnl_register(PF_UNSPEC
, RTM_DELACTION
, tc_ctl_action
, NULL
, 0);
1598 rtnl_register(PF_UNSPEC
, RTM_GETACTION
, tc_ctl_action
, tc_dump_action
,
1604 subsys_initcall(tc_action_init
);