1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/act_mirred.c packet mirroring and redirect actions
5 * Authors: Jamal Hadi Salim (2002-4)
7 * TODO: Add ingress support (and socket redirect support)
10 #include <linux/types.h>
11 #include <linux/kernel.h>
12 #include <linux/string.h>
13 #include <linux/errno.h>
14 #include <linux/skbuff.h>
15 #include <linux/rtnetlink.h>
16 #include <linux/module.h>
17 #include <linux/init.h>
18 #include <linux/gfp.h>
19 #include <linux/if_arp.h>
20 #include <net/net_namespace.h>
21 #include <net/netlink.h>
22 #include <net/pkt_sched.h>
23 #include <net/pkt_cls.h>
24 #include <linux/tc_act/tc_mirred.h>
25 #include <net/tc_act/tc_mirred.h>
27 static LIST_HEAD(mirred_list
);
28 static DEFINE_SPINLOCK(mirred_list_lock
);
30 #define MIRRED_RECURSION_LIMIT 4
31 static DEFINE_PER_CPU(unsigned int, mirred_rec_level
);
33 static bool tcf_mirred_is_act_redirect(int action
)
35 return action
== TCA_EGRESS_REDIR
|| action
== TCA_INGRESS_REDIR
;
38 static bool tcf_mirred_act_wants_ingress(int action
)
41 case TCA_EGRESS_REDIR
:
42 case TCA_EGRESS_MIRROR
:
44 case TCA_INGRESS_REDIR
:
45 case TCA_INGRESS_MIRROR
:
52 static bool tcf_mirred_can_reinsert(int action
)
64 static struct net_device
*tcf_mirred_dev_dereference(struct tcf_mirred
*m
)
66 return rcu_dereference_protected(m
->tcfm_dev
,
67 lockdep_is_held(&m
->tcf_lock
));
70 static void tcf_mirred_release(struct tc_action
*a
)
72 struct tcf_mirred
*m
= to_mirred(a
);
73 struct net_device
*dev
;
75 spin_lock(&mirred_list_lock
);
76 list_del(&m
->tcfm_list
);
77 spin_unlock(&mirred_list_lock
);
79 /* last reference to action, no need to lock */
80 dev
= rcu_dereference_protected(m
->tcfm_dev
, 1);
85 static const struct nla_policy mirred_policy
[TCA_MIRRED_MAX
+ 1] = {
86 [TCA_MIRRED_PARMS
] = { .len
= sizeof(struct tc_mirred
) },
89 static unsigned int mirred_net_id
;
90 static struct tc_action_ops act_mirred_ops
;
92 static int tcf_mirred_init(struct net
*net
, struct nlattr
*nla
,
93 struct nlattr
*est
, struct tc_action
**a
,
94 int ovr
, int bind
, bool rtnl_held
,
96 u32 flags
, struct netlink_ext_ack
*extack
)
98 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
99 struct nlattr
*tb
[TCA_MIRRED_MAX
+ 1];
100 struct tcf_chain
*goto_ch
= NULL
;
101 bool mac_header_xmit
= false;
102 struct tc_mirred
*parm
;
103 struct tcf_mirred
*m
;
104 struct net_device
*dev
;
110 NL_SET_ERR_MSG_MOD(extack
, "Mirred requires attributes to be passed");
113 ret
= nla_parse_nested_deprecated(tb
, TCA_MIRRED_MAX
, nla
,
114 mirred_policy
, extack
);
117 if (!tb
[TCA_MIRRED_PARMS
]) {
118 NL_SET_ERR_MSG_MOD(extack
, "Missing required mirred parameters");
121 parm
= nla_data(tb
[TCA_MIRRED_PARMS
]);
123 err
= tcf_idr_check_alloc(tn
, &index
, a
, bind
);
130 switch (parm
->eaction
) {
131 case TCA_EGRESS_MIRROR
:
132 case TCA_EGRESS_REDIR
:
133 case TCA_INGRESS_REDIR
:
134 case TCA_INGRESS_MIRROR
:
138 tcf_idr_release(*a
, bind
);
140 tcf_idr_cleanup(tn
, index
);
141 NL_SET_ERR_MSG_MOD(extack
, "Unknown mirred option");
146 if (!parm
->ifindex
) {
147 tcf_idr_cleanup(tn
, index
);
148 NL_SET_ERR_MSG_MOD(extack
, "Specified device does not exist");
151 ret
= tcf_idr_create_from_flags(tn
, index
, est
, a
,
152 &act_mirred_ops
, bind
, flags
);
154 tcf_idr_cleanup(tn
, index
);
159 tcf_idr_release(*a
, bind
);
164 if (ret
== ACT_P_CREATED
)
165 INIT_LIST_HEAD(&m
->tcfm_list
);
167 err
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
171 spin_lock_bh(&m
->tcf_lock
);
174 dev
= dev_get_by_index(net
, parm
->ifindex
);
176 spin_unlock_bh(&m
->tcf_lock
);
180 mac_header_xmit
= dev_is_mac_header_xmit(dev
);
181 rcu_swap_protected(m
->tcfm_dev
, dev
,
182 lockdep_is_held(&m
->tcf_lock
));
185 m
->tcfm_mac_header_xmit
= mac_header_xmit
;
187 goto_ch
= tcf_action_set_ctrlact(*a
, parm
->action
, goto_ch
);
188 m
->tcfm_eaction
= parm
->eaction
;
189 spin_unlock_bh(&m
->tcf_lock
);
191 tcf_chain_put_by_act(goto_ch
);
193 if (ret
== ACT_P_CREATED
) {
194 spin_lock(&mirred_list_lock
);
195 list_add(&m
->tcfm_list
, &mirred_list
);
196 spin_unlock(&mirred_list_lock
);
198 tcf_idr_insert(tn
, *a
);
204 tcf_chain_put_by_act(goto_ch
);
206 tcf_idr_release(*a
, bind
);
210 static int tcf_mirred_act(struct sk_buff
*skb
, const struct tc_action
*a
,
211 struct tcf_result
*res
)
213 struct tcf_mirred
*m
= to_mirred(a
);
214 struct sk_buff
*skb2
= skb
;
215 bool m_mac_header_xmit
;
216 struct net_device
*dev
;
217 unsigned int rec_level
;
225 rec_level
= __this_cpu_inc_return(mirred_rec_level
);
226 if (unlikely(rec_level
> MIRRED_RECURSION_LIMIT
)) {
227 net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n",
228 netdev_name(skb
->dev
));
229 __this_cpu_dec(mirred_rec_level
);
233 tcf_lastuse_update(&m
->tcf_tm
);
234 tcf_action_update_bstats(&m
->common
, skb
);
236 m_mac_header_xmit
= READ_ONCE(m
->tcfm_mac_header_xmit
);
237 m_eaction
= READ_ONCE(m
->tcfm_eaction
);
238 retval
= READ_ONCE(m
->tcf_action
);
239 dev
= rcu_dereference_bh(m
->tcfm_dev
);
240 if (unlikely(!dev
)) {
241 pr_notice_once("tc mirred: target device is gone\n");
245 if (unlikely(!(dev
->flags
& IFF_UP
))) {
246 net_notice_ratelimited("tc mirred to Houston: device %s is down\n",
251 /* we could easily avoid the clone only if called by ingress and clsact;
252 * since we can't easily detect the clsact caller, skip clone only for
253 * ingress - that covers the TC S/W datapath.
255 is_redirect
= tcf_mirred_is_act_redirect(m_eaction
);
256 use_reinsert
= skb_at_tc_ingress(skb
) && is_redirect
&&
257 tcf_mirred_can_reinsert(retval
);
259 skb2
= skb_clone(skb
, GFP_ATOMIC
);
264 /* If action's target direction differs than filter's direction,
265 * and devices expect a mac header on xmit, then mac push/pull is
268 want_ingress
= tcf_mirred_act_wants_ingress(m_eaction
);
269 if (skb_at_tc_ingress(skb
) != want_ingress
&& m_mac_header_xmit
) {
270 if (!skb_at_tc_ingress(skb
)) {
271 /* caught at egress, act ingress: pull mac */
272 mac_len
= skb_network_header(skb
) - skb_mac_header(skb
);
273 skb_pull_rcsum(skb2
, mac_len
);
275 /* caught at ingress, act egress: push mac */
276 skb_push_rcsum(skb2
, skb
->mac_len
);
280 skb2
->skb_iif
= skb
->dev
->ifindex
;
283 /* mirror is always swallowed */
285 skb2
->tc_redirected
= 1;
286 skb2
->tc_from_ingress
= skb2
->tc_at_ingress
;
287 if (skb2
->tc_from_ingress
)
289 /* let's the caller reinsert the packet, if possible */
291 res
->ingress
= want_ingress
;
292 if (skb_tc_reinsert(skb
, res
))
293 tcf_action_inc_overlimit_qstats(&m
->common
);
294 __this_cpu_dec(mirred_rec_level
);
295 return TC_ACT_CONSUMED
;
300 err
= dev_queue_xmit(skb2
);
302 err
= netif_receive_skb(skb2
);
306 tcf_action_inc_overlimit_qstats(&m
->common
);
307 if (tcf_mirred_is_act_redirect(m_eaction
))
308 retval
= TC_ACT_SHOT
;
310 __this_cpu_dec(mirred_rec_level
);
315 static void tcf_stats_update(struct tc_action
*a
, u64 bytes
, u32 packets
,
316 u64 lastuse
, bool hw
)
318 struct tcf_mirred
*m
= to_mirred(a
);
319 struct tcf_t
*tm
= &m
->tcf_tm
;
321 tcf_action_update_stats(a
, bytes
, packets
, false, hw
);
322 tm
->lastuse
= max_t(u64
, tm
->lastuse
, lastuse
);
325 static int tcf_mirred_dump(struct sk_buff
*skb
, struct tc_action
*a
, int bind
,
328 unsigned char *b
= skb_tail_pointer(skb
);
329 struct tcf_mirred
*m
= to_mirred(a
);
330 struct tc_mirred opt
= {
331 .index
= m
->tcf_index
,
332 .refcnt
= refcount_read(&m
->tcf_refcnt
) - ref
,
333 .bindcnt
= atomic_read(&m
->tcf_bindcnt
) - bind
,
335 struct net_device
*dev
;
338 spin_lock_bh(&m
->tcf_lock
);
339 opt
.action
= m
->tcf_action
;
340 opt
.eaction
= m
->tcfm_eaction
;
341 dev
= tcf_mirred_dev_dereference(m
);
343 opt
.ifindex
= dev
->ifindex
;
345 if (nla_put(skb
, TCA_MIRRED_PARMS
, sizeof(opt
), &opt
))
346 goto nla_put_failure
;
348 tcf_tm_dump(&t
, &m
->tcf_tm
);
349 if (nla_put_64bit(skb
, TCA_MIRRED_TM
, sizeof(t
), &t
, TCA_MIRRED_PAD
))
350 goto nla_put_failure
;
351 spin_unlock_bh(&m
->tcf_lock
);
356 spin_unlock_bh(&m
->tcf_lock
);
361 static int tcf_mirred_walker(struct net
*net
, struct sk_buff
*skb
,
362 struct netlink_callback
*cb
, int type
,
363 const struct tc_action_ops
*ops
,
364 struct netlink_ext_ack
*extack
)
366 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
368 return tcf_generic_walker(tn
, skb
, cb
, type
, ops
, extack
);
371 static int tcf_mirred_search(struct net
*net
, struct tc_action
**a
, u32 index
)
373 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
375 return tcf_idr_search(tn
, a
, index
);
378 static int mirred_device_event(struct notifier_block
*unused
,
379 unsigned long event
, void *ptr
)
381 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
382 struct tcf_mirred
*m
;
385 if (event
== NETDEV_UNREGISTER
) {
386 spin_lock(&mirred_list_lock
);
387 list_for_each_entry(m
, &mirred_list
, tcfm_list
) {
388 spin_lock_bh(&m
->tcf_lock
);
389 if (tcf_mirred_dev_dereference(m
) == dev
) {
391 /* Note : no rcu grace period necessary, as
392 * net_device are already rcu protected.
394 RCU_INIT_POINTER(m
->tcfm_dev
, NULL
);
396 spin_unlock_bh(&m
->tcf_lock
);
398 spin_unlock(&mirred_list_lock
);
404 static struct notifier_block mirred_device_notifier
= {
405 .notifier_call
= mirred_device_event
,
408 static void tcf_mirred_dev_put(void *priv
)
410 struct net_device
*dev
= priv
;
415 static struct net_device
*
416 tcf_mirred_get_dev(const struct tc_action
*a
,
417 tc_action_priv_destructor
*destructor
)
419 struct tcf_mirred
*m
= to_mirred(a
);
420 struct net_device
*dev
;
423 dev
= rcu_dereference(m
->tcfm_dev
);
426 *destructor
= tcf_mirred_dev_put
;
433 static size_t tcf_mirred_get_fill_size(const struct tc_action
*act
)
435 return nla_total_size(sizeof(struct tc_mirred
));
438 static struct tc_action_ops act_mirred_ops
= {
441 .owner
= THIS_MODULE
,
442 .act
= tcf_mirred_act
,
443 .stats_update
= tcf_stats_update
,
444 .dump
= tcf_mirred_dump
,
445 .cleanup
= tcf_mirred_release
,
446 .init
= tcf_mirred_init
,
447 .walk
= tcf_mirred_walker
,
448 .lookup
= tcf_mirred_search
,
449 .get_fill_size
= tcf_mirred_get_fill_size
,
450 .size
= sizeof(struct tcf_mirred
),
451 .get_dev
= tcf_mirred_get_dev
,
454 static __net_init
int mirred_init_net(struct net
*net
)
456 struct tc_action_net
*tn
= net_generic(net
, mirred_net_id
);
458 return tc_action_net_init(net
, tn
, &act_mirred_ops
);
461 static void __net_exit
mirred_exit_net(struct list_head
*net_list
)
463 tc_action_net_exit(net_list
, mirred_net_id
);
466 static struct pernet_operations mirred_net_ops
= {
467 .init
= mirred_init_net
,
468 .exit_batch
= mirred_exit_net
,
469 .id
= &mirred_net_id
,
470 .size
= sizeof(struct tc_action_net
),
473 MODULE_AUTHOR("Jamal Hadi Salim(2002)");
474 MODULE_DESCRIPTION("Device Mirror/redirect actions");
475 MODULE_LICENSE("GPL");
477 static int __init
mirred_init_module(void)
479 int err
= register_netdevice_notifier(&mirred_device_notifier
);
483 pr_info("Mirror/redirect action on\n");
484 err
= tcf_register_action(&act_mirred_ops
, &mirred_net_ops
);
486 unregister_netdevice_notifier(&mirred_device_notifier
);
491 static void __exit
mirred_cleanup_module(void)
493 tcf_unregister_action(&act_mirred_ops
, &mirred_net_ops
);
494 unregister_netdevice_notifier(&mirred_device_notifier
);
497 module_init(mirred_init_module
);
498 module_exit(mirred_cleanup_module
);