1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/act_mirred.c packet mirroring and redirect actions
5 * Authors: Jamal Hadi Salim (2002-4)
7 * TODO: Add ingress support (and socket redirect support)
10 #include <linux/types.h>
11 #include <linux/kernel.h>
12 #include <linux/string.h>
13 #include <linux/errno.h>
14 #include <linux/skbuff.h>
15 #include <linux/rtnetlink.h>
16 #include <linux/module.h>
17 #include <linux/init.h>
18 #include <linux/gfp.h>
19 #include <linux/if_arp.h>
20 #include <net/net_namespace.h>
21 #include <net/netlink.h>
22 #include <net/pkt_sched.h>
23 #include <net/pkt_cls.h>
24 #include <linux/tc_act/tc_mirred.h>
25 #include <net/tc_act/tc_mirred.h>
27 static LIST_HEAD(mirred_list
);
28 static DEFINE_SPINLOCK(mirred_list_lock
);
30 #define MIRRED_RECURSION_LIMIT 4
31 static DEFINE_PER_CPU(unsigned int, mirred_rec_level
);
33 static bool tcf_mirred_is_act_redirect(int action
)
35 return action
== TCA_EGRESS_REDIR
|| action
== TCA_INGRESS_REDIR
;
38 static bool tcf_mirred_act_wants_ingress(int action
)
41 case TCA_EGRESS_REDIR
:
42 case TCA_EGRESS_MIRROR
:
44 case TCA_INGRESS_REDIR
:
45 case TCA_INGRESS_MIRROR
:
52 static bool tcf_mirred_can_reinsert(int action
)
64 static struct net_device
*tcf_mirred_dev_dereference(struct tcf_mirred
*m
)
66 return rcu_dereference_protected(m
->tcfm_dev
,
67 lockdep_is_held(&m
->tcf_lock
));
70 static void tcf_mirred_release(struct tc_action
*a
)
72 struct tcf_mirred
*m
= to_mirred(a
);
73 struct net_device
*dev
;
75 spin_lock(&mirred_list_lock
);
76 list_del(&m
->tcfm_list
);
77 spin_unlock(&mirred_list_lock
);
79 /* last reference to action, no need to lock */
80 dev
= rcu_dereference_protected(m
->tcfm_dev
, 1);
85 static const struct nla_policy mirred_policy
[TCA_MIRRED_MAX
+ 1] = {
86 [TCA_MIRRED_PARMS
] = { .len
= sizeof(struct tc_mirred
) },
89 static unsigned int mirred_net_id
;
90 static struct tc_action_ops act_mirred_ops
;
92 static int tcf_mirred_init(struct net
*net
, struct nlattr
*nla
,
93 struct nlattr
*est
, struct tc_action
**a
,
94 int ovr
, int bind
, bool rtnl_held
,
96 struct netlink_ext_ack
*extack
)
98 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
99 struct nlattr
*tb
[TCA_MIRRED_MAX
+ 1];
100 struct tcf_chain
*goto_ch
= NULL
;
101 bool mac_header_xmit
= false;
102 struct tc_mirred
*parm
;
103 struct tcf_mirred
*m
;
104 struct net_device
*dev
;
110 NL_SET_ERR_MSG_MOD(extack
, "Mirred requires attributes to be passed");
113 ret
= nla_parse_nested_deprecated(tb
, TCA_MIRRED_MAX
, nla
,
114 mirred_policy
, extack
);
117 if (!tb
[TCA_MIRRED_PARMS
]) {
118 NL_SET_ERR_MSG_MOD(extack
, "Missing required mirred parameters");
121 parm
= nla_data(tb
[TCA_MIRRED_PARMS
]);
123 err
= tcf_idr_check_alloc(tn
, &index
, a
, bind
);
130 switch (parm
->eaction
) {
131 case TCA_EGRESS_MIRROR
:
132 case TCA_EGRESS_REDIR
:
133 case TCA_INGRESS_REDIR
:
134 case TCA_INGRESS_MIRROR
:
138 tcf_idr_release(*a
, bind
);
140 tcf_idr_cleanup(tn
, index
);
141 NL_SET_ERR_MSG_MOD(extack
, "Unknown mirred option");
146 if (!parm
->ifindex
) {
147 tcf_idr_cleanup(tn
, index
);
148 NL_SET_ERR_MSG_MOD(extack
, "Specified device does not exist");
151 ret
= tcf_idr_create(tn
, index
, est
, a
,
152 &act_mirred_ops
, bind
, true);
154 tcf_idr_cleanup(tn
, index
);
159 tcf_idr_release(*a
, bind
);
164 if (ret
== ACT_P_CREATED
)
165 INIT_LIST_HEAD(&m
->tcfm_list
);
167 err
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
171 spin_lock_bh(&m
->tcf_lock
);
174 dev
= dev_get_by_index(net
, parm
->ifindex
);
176 spin_unlock_bh(&m
->tcf_lock
);
180 mac_header_xmit
= dev_is_mac_header_xmit(dev
);
181 rcu_swap_protected(m
->tcfm_dev
, dev
,
182 lockdep_is_held(&m
->tcf_lock
));
185 m
->tcfm_mac_header_xmit
= mac_header_xmit
;
187 goto_ch
= tcf_action_set_ctrlact(*a
, parm
->action
, goto_ch
);
188 m
->tcfm_eaction
= parm
->eaction
;
189 spin_unlock_bh(&m
->tcf_lock
);
191 tcf_chain_put_by_act(goto_ch
);
193 if (ret
== ACT_P_CREATED
) {
194 spin_lock(&mirred_list_lock
);
195 list_add(&m
->tcfm_list
, &mirred_list
);
196 spin_unlock(&mirred_list_lock
);
198 tcf_idr_insert(tn
, *a
);
204 tcf_chain_put_by_act(goto_ch
);
206 tcf_idr_release(*a
, bind
);
210 static int tcf_mirred_act(struct sk_buff
*skb
, const struct tc_action
*a
,
211 struct tcf_result
*res
)
213 struct tcf_mirred
*m
= to_mirred(a
);
214 struct sk_buff
*skb2
= skb
;
215 bool m_mac_header_xmit
;
216 struct net_device
*dev
;
217 unsigned int rec_level
;
225 rec_level
= __this_cpu_inc_return(mirred_rec_level
);
226 if (unlikely(rec_level
> MIRRED_RECURSION_LIMIT
)) {
227 net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n",
228 netdev_name(skb
->dev
));
229 __this_cpu_dec(mirred_rec_level
);
233 tcf_lastuse_update(&m
->tcf_tm
);
234 bstats_cpu_update(this_cpu_ptr(m
->common
.cpu_bstats
), skb
);
236 m_mac_header_xmit
= READ_ONCE(m
->tcfm_mac_header_xmit
);
237 m_eaction
= READ_ONCE(m
->tcfm_eaction
);
238 retval
= READ_ONCE(m
->tcf_action
);
239 dev
= rcu_dereference_bh(m
->tcfm_dev
);
240 if (unlikely(!dev
)) {
241 pr_notice_once("tc mirred: target device is gone\n");
245 if (unlikely(!(dev
->flags
& IFF_UP
))) {
246 net_notice_ratelimited("tc mirred to Houston: device %s is down\n",
251 /* we could easily avoid the clone only if called by ingress and clsact;
252 * since we can't easily detect the clsact caller, skip clone only for
253 * ingress - that covers the TC S/W datapath.
255 is_redirect
= tcf_mirred_is_act_redirect(m_eaction
);
256 use_reinsert
= skb_at_tc_ingress(skb
) && is_redirect
&&
257 tcf_mirred_can_reinsert(retval
);
259 skb2
= skb_clone(skb
, GFP_ATOMIC
);
264 /* If action's target direction differs than filter's direction,
265 * and devices expect a mac header on xmit, then mac push/pull is
268 want_ingress
= tcf_mirred_act_wants_ingress(m_eaction
);
269 if (skb_at_tc_ingress(skb
) != want_ingress
&& m_mac_header_xmit
) {
270 if (!skb_at_tc_ingress(skb
)) {
271 /* caught at egress, act ingress: pull mac */
272 mac_len
= skb_network_header(skb
) - skb_mac_header(skb
);
273 skb_pull_rcsum(skb2
, mac_len
);
275 /* caught at ingress, act egress: push mac */
276 skb_push_rcsum(skb2
, skb
->mac_len
);
280 skb2
->skb_iif
= skb
->dev
->ifindex
;
283 /* mirror is always swallowed */
285 skb2
->tc_redirected
= 1;
286 skb2
->tc_from_ingress
= skb2
->tc_at_ingress
;
287 if (skb2
->tc_from_ingress
)
289 /* let's the caller reinsert the packet, if possible */
291 res
->ingress
= want_ingress
;
292 res
->qstats
= this_cpu_ptr(m
->common
.cpu_qstats
);
293 skb_tc_reinsert(skb
, res
);
294 __this_cpu_dec(mirred_rec_level
);
295 return TC_ACT_CONSUMED
;
300 err
= dev_queue_xmit(skb2
);
302 err
= netif_receive_skb(skb2
);
306 qstats_overlimit_inc(this_cpu_ptr(m
->common
.cpu_qstats
));
307 if (tcf_mirred_is_act_redirect(m_eaction
))
308 retval
= TC_ACT_SHOT
;
310 __this_cpu_dec(mirred_rec_level
);
315 static void tcf_stats_update(struct tc_action
*a
, u64 bytes
, u32 packets
,
316 u64 lastuse
, bool hw
)
318 struct tcf_mirred
*m
= to_mirred(a
);
319 struct tcf_t
*tm
= &m
->tcf_tm
;
321 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats
), bytes
, packets
);
323 _bstats_cpu_update(this_cpu_ptr(a
->cpu_bstats_hw
),
325 tm
->lastuse
= max_t(u64
, tm
->lastuse
, lastuse
);
328 static int tcf_mirred_dump(struct sk_buff
*skb
, struct tc_action
*a
, int bind
,
331 unsigned char *b
= skb_tail_pointer(skb
);
332 struct tcf_mirred
*m
= to_mirred(a
);
333 struct tc_mirred opt
= {
334 .index
= m
->tcf_index
,
335 .refcnt
= refcount_read(&m
->tcf_refcnt
) - ref
,
336 .bindcnt
= atomic_read(&m
->tcf_bindcnt
) - bind
,
338 struct net_device
*dev
;
341 spin_lock_bh(&m
->tcf_lock
);
342 opt
.action
= m
->tcf_action
;
343 opt
.eaction
= m
->tcfm_eaction
;
344 dev
= tcf_mirred_dev_dereference(m
);
346 opt
.ifindex
= dev
->ifindex
;
348 if (nla_put(skb
, TCA_MIRRED_PARMS
, sizeof(opt
), &opt
))
349 goto nla_put_failure
;
351 tcf_tm_dump(&t
, &m
->tcf_tm
);
352 if (nla_put_64bit(skb
, TCA_MIRRED_TM
, sizeof(t
), &t
, TCA_MIRRED_PAD
))
353 goto nla_put_failure
;
354 spin_unlock_bh(&m
->tcf_lock
);
359 spin_unlock_bh(&m
->tcf_lock
);
364 static int tcf_mirred_walker(struct net
*net
, struct sk_buff
*skb
,
365 struct netlink_callback
*cb
, int type
,
366 const struct tc_action_ops
*ops
,
367 struct netlink_ext_ack
*extack
)
369 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
371 return tcf_generic_walker(tn
, skb
, cb
, type
, ops
, extack
);
374 static int tcf_mirred_search(struct net
*net
, struct tc_action
**a
, u32 index
)
376 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
378 return tcf_idr_search(tn
, a
, index
);
381 static int mirred_device_event(struct notifier_block
*unused
,
382 unsigned long event
, void *ptr
)
384 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
385 struct tcf_mirred
*m
;
388 if (event
== NETDEV_UNREGISTER
) {
389 spin_lock(&mirred_list_lock
);
390 list_for_each_entry(m
, &mirred_list
, tcfm_list
) {
391 spin_lock_bh(&m
->tcf_lock
);
392 if (tcf_mirred_dev_dereference(m
) == dev
) {
394 /* Note : no rcu grace period necessary, as
395 * net_device are already rcu protected.
397 RCU_INIT_POINTER(m
->tcfm_dev
, NULL
);
399 spin_unlock_bh(&m
->tcf_lock
);
401 spin_unlock(&mirred_list_lock
);
407 static struct notifier_block mirred_device_notifier
= {
408 .notifier_call
= mirred_device_event
,
411 static struct net_device
*tcf_mirred_get_dev(const struct tc_action
*a
)
413 struct tcf_mirred
*m
= to_mirred(a
);
414 struct net_device
*dev
;
417 dev
= rcu_dereference(m
->tcfm_dev
);
425 static void tcf_mirred_put_dev(struct net_device
*dev
)
430 static size_t tcf_mirred_get_fill_size(const struct tc_action
*act
)
432 return nla_total_size(sizeof(struct tc_mirred
));
435 static struct tc_action_ops act_mirred_ops
= {
438 .owner
= THIS_MODULE
,
439 .act
= tcf_mirred_act
,
440 .stats_update
= tcf_stats_update
,
441 .dump
= tcf_mirred_dump
,
442 .cleanup
= tcf_mirred_release
,
443 .init
= tcf_mirred_init
,
444 .walk
= tcf_mirred_walker
,
445 .lookup
= tcf_mirred_search
,
446 .get_fill_size
= tcf_mirred_get_fill_size
,
447 .size
= sizeof(struct tcf_mirred
),
448 .get_dev
= tcf_mirred_get_dev
,
449 .put_dev
= tcf_mirred_put_dev
,
452 static __net_init
int mirred_init_net(struct net
*net
)
454 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
456 return tc_action_net_init(tn
, &act_mirred_ops
);
459 static void __net_exit
mirred_exit_net(struct list_head
*net_list
)
461 tc_action_net_exit(net_list
, mirred_net_id
);
464 static struct pernet_operations mirred_net_ops
= {
465 .init
= mirred_init_net
,
466 .exit_batch
= mirred_exit_net
,
467 .id
= &mirred_net_id
,
468 .size
= sizeof(struct tc_action_net
),
471 MODULE_AUTHOR("Jamal Hadi Salim(2002)");
472 MODULE_DESCRIPTION("Device Mirror/redirect actions");
473 MODULE_LICENSE("GPL");
475 static int __init
mirred_init_module(void)
477 int err
= register_netdevice_notifier(&mirred_device_notifier
);
481 pr_info("Mirror/redirect action on\n");
482 return tcf_register_action(&act_mirred_ops
, &mirred_net_ops
);
485 static void __exit
mirred_cleanup_module(void)
487 tcf_unregister_action(&act_mirred_ops
, &mirred_net_ops
);
488 unregister_netdevice_notifier(&mirred_device_notifier
);
491 module_init(mirred_init_module
);
492 module_exit(mirred_cleanup_module
);