1 #include <linux/types.h>
2 #include <linux/skbuff.h>
3 #include <linux/socket.h>
4 #include <linux/sysctl.h>
6 #include <linux/module.h>
7 #include <linux/if_arp.h>
8 #include <linux/ipv6.h>
9 #include <linux/mpls.h>
10 #include <linux/netconf.h>
11 #include <linux/vmalloc.h>
12 #include <linux/percpu.h>
17 #include <net/ip_fib.h>
18 #include <net/netevent.h>
19 #include <net/netns/generic.h>
20 #if IS_ENABLED(CONFIG_IPV6)
23 #include <net/addrconf.h>
24 #include <net/nexthop.h>
27 /* max memory we will use for mpls_route */
28 #define MAX_MPLS_ROUTE_MEM 4096
30 /* Maximum number of labels to look ahead at when selecting a path of
33 #define MAX_MP_SELECT_LABELS 4
35 #define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)
39 static int label_limit
= (1 << 20) - 1;
40 static int ttl_max
= 255;
42 static void rtmsg_lfib(int event
, u32 label
, struct mpls_route
*rt
,
43 struct nlmsghdr
*nlh
, struct net
*net
, u32 portid
,
44 unsigned int nlm_flags
);
46 static struct mpls_route
*mpls_route_input_rcu(struct net
*net
, unsigned index
)
48 struct mpls_route
*rt
= NULL
;
50 if (index
< net
->mpls
.platform_labels
) {
51 struct mpls_route __rcu
**platform_label
=
52 rcu_dereference(net
->mpls
.platform_label
);
53 rt
= rcu_dereference(platform_label
[index
]);
58 bool mpls_output_possible(const struct net_device
*dev
)
60 return dev
&& (dev
->flags
& IFF_UP
) && netif_carrier_ok(dev
);
62 EXPORT_SYMBOL_GPL(mpls_output_possible
);
64 static u8
*__mpls_nh_via(struct mpls_route
*rt
, struct mpls_nh
*nh
)
66 return (u8
*)nh
+ rt
->rt_via_offset
;
69 static const u8
*mpls_nh_via(const struct mpls_route
*rt
,
70 const struct mpls_nh
*nh
)
72 return __mpls_nh_via((struct mpls_route
*)rt
, (struct mpls_nh
*)nh
);
75 static unsigned int mpls_nh_header_size(const struct mpls_nh
*nh
)
77 /* The size of the layer 2.5 labels to be added for this route */
78 return nh
->nh_labels
* sizeof(struct mpls_shim_hdr
);
81 unsigned int mpls_dev_mtu(const struct net_device
*dev
)
83 /* The amount of data the layer 2 frame can hold */
86 EXPORT_SYMBOL_GPL(mpls_dev_mtu
);
88 bool mpls_pkt_too_big(const struct sk_buff
*skb
, unsigned int mtu
)
93 if (skb_is_gso(skb
) && skb_gso_validate_mtu(skb
, mtu
))
98 EXPORT_SYMBOL_GPL(mpls_pkt_too_big
);
100 void mpls_stats_inc_outucastpkts(struct net_device
*dev
,
101 const struct sk_buff
*skb
)
103 struct mpls_dev
*mdev
;
105 if (skb
->protocol
== htons(ETH_P_MPLS_UC
)) {
106 mdev
= mpls_dev_get(dev
);
108 MPLS_INC_STATS_LEN(mdev
, skb
->len
,
111 } else if (skb
->protocol
== htons(ETH_P_IP
)) {
112 IP_UPD_PO_STATS(dev_net(dev
), IPSTATS_MIB_OUT
, skb
->len
);
113 #if IS_ENABLED(CONFIG_IPV6)
114 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
115 struct inet6_dev
*in6dev
= __in6_dev_get(dev
);
118 IP6_UPD_PO_STATS(dev_net(dev
), in6dev
,
119 IPSTATS_MIB_OUT
, skb
->len
);
123 EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts
);
125 static u32
mpls_multipath_hash(struct mpls_route
*rt
, struct sk_buff
*skb
)
127 struct mpls_entry_decoded dec
;
128 unsigned int mpls_hdr_len
= 0;
129 struct mpls_shim_hdr
*hdr
;
130 bool eli_seen
= false;
134 for (label_index
= 0; label_index
< MAX_MP_SELECT_LABELS
;
136 mpls_hdr_len
+= sizeof(*hdr
);
137 if (!pskb_may_pull(skb
, mpls_hdr_len
))
140 /* Read and decode the current label */
141 hdr
= mpls_hdr(skb
) + label_index
;
142 dec
= mpls_entry_decode(hdr
);
144 /* RFC6790 - reserved labels MUST NOT be used as keys
145 * for the load-balancing function
147 if (likely(dec
.label
>= MPLS_LABEL_FIRST_UNRESERVED
)) {
148 hash
= jhash_1word(dec
.label
, hash
);
150 /* The entropy label follows the entropy label
151 * indicator, so this means that the entropy
152 * label was just added to the hash - no need to
153 * go any deeper either in the label stack or in the
158 } else if (dec
.label
== MPLS_LABEL_ENTROPY
) {
165 /* found bottom label; does skb have room for a header? */
166 if (pskb_may_pull(skb
, mpls_hdr_len
+ sizeof(struct iphdr
))) {
167 const struct iphdr
*v4hdr
;
169 v4hdr
= (const struct iphdr
*)(hdr
+ 1);
170 if (v4hdr
->version
== 4) {
171 hash
= jhash_3words(ntohl(v4hdr
->saddr
),
173 v4hdr
->protocol
, hash
);
174 } else if (v4hdr
->version
== 6 &&
175 pskb_may_pull(skb
, mpls_hdr_len
+
176 sizeof(struct ipv6hdr
))) {
177 const struct ipv6hdr
*v6hdr
;
179 v6hdr
= (const struct ipv6hdr
*)(hdr
+ 1);
180 hash
= __ipv6_addr_jhash(&v6hdr
->saddr
, hash
);
181 hash
= __ipv6_addr_jhash(&v6hdr
->daddr
, hash
);
182 hash
= jhash_1word(v6hdr
->nexthdr
, hash
);
192 static struct mpls_nh
*mpls_get_nexthop(struct mpls_route
*rt
, u8 index
)
194 return (struct mpls_nh
*)((u8
*)rt
->rt_nh
+ index
* rt
->rt_nh_size
);
197 /* number of alive nexthops (rt->rt_nhn_alive) and the flags for
198 * a next hop (nh->nh_flags) are modified by netdev event handlers.
199 * Since those fields can change at any moment, use READ_ONCE to
202 static struct mpls_nh
*mpls_select_multipath(struct mpls_route
*rt
,
210 /* No need to look further into packet if there's only
216 alive
= READ_ONCE(rt
->rt_nhn_alive
);
220 hash
= mpls_multipath_hash(rt
, skb
);
221 nh_index
= hash
% alive
;
222 if (alive
== rt
->rt_nhn
)
225 unsigned int nh_flags
= READ_ONCE(nh
->nh_flags
);
227 if (nh_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
))
232 } endfor_nexthops(rt
);
235 return mpls_get_nexthop(rt
, nh_index
);
238 static bool mpls_egress(struct net
*net
, struct mpls_route
*rt
,
239 struct sk_buff
*skb
, struct mpls_entry_decoded dec
)
241 enum mpls_payload_type payload_type
;
242 bool success
= false;
244 /* The IPv4 code below accesses through the IPv4 header
245 * checksum, which is 12 bytes into the packet.
246 * The IPv6 code below accesses through the IPv6 hop limit
247 * which is 8 bytes into the packet.
249 * For all supported cases there should always be at least 12
250 * bytes of packet data present. The IPv4 header is 20 bytes
251 * without options and the IPv6 header is always 40 bytes
254 if (!pskb_may_pull(skb
, 12))
257 payload_type
= rt
->rt_payload_type
;
258 if (payload_type
== MPT_UNSPEC
)
259 payload_type
= ip_hdr(skb
)->version
;
261 switch (payload_type
) {
263 struct iphdr
*hdr4
= ip_hdr(skb
);
265 skb
->protocol
= htons(ETH_P_IP
);
267 /* If propagating TTL, take the decremented TTL from
268 * the incoming MPLS header, otherwise decrement the
269 * TTL, but only if not 0 to avoid underflow.
271 if (rt
->rt_ttl_propagate
== MPLS_TTL_PROP_ENABLED
||
272 (rt
->rt_ttl_propagate
== MPLS_TTL_PROP_DEFAULT
&&
273 net
->mpls
.ip_ttl_propagate
))
276 new_ttl
= hdr4
->ttl
? hdr4
->ttl
- 1 : 0;
278 csum_replace2(&hdr4
->check
,
279 htons(hdr4
->ttl
<< 8),
280 htons(new_ttl
<< 8));
286 struct ipv6hdr
*hdr6
= ipv6_hdr(skb
);
287 skb
->protocol
= htons(ETH_P_IPV6
);
289 /* If propagating TTL, take the decremented TTL from
290 * the incoming MPLS header, otherwise decrement the
291 * hop limit, but only if not 0 to avoid underflow.
293 if (rt
->rt_ttl_propagate
== MPLS_TTL_PROP_ENABLED
||
294 (rt
->rt_ttl_propagate
== MPLS_TTL_PROP_DEFAULT
&&
295 net
->mpls
.ip_ttl_propagate
))
296 hdr6
->hop_limit
= dec
.ttl
;
297 else if (hdr6
->hop_limit
)
298 hdr6
->hop_limit
= hdr6
->hop_limit
- 1;
303 /* Should have decided which protocol it is by now */
310 static int mpls_forward(struct sk_buff
*skb
, struct net_device
*dev
,
311 struct packet_type
*pt
, struct net_device
*orig_dev
)
313 struct net
*net
= dev_net(dev
);
314 struct mpls_shim_hdr
*hdr
;
315 struct mpls_route
*rt
;
317 struct mpls_entry_decoded dec
;
318 struct net_device
*out_dev
;
319 struct mpls_dev
*out_mdev
;
320 struct mpls_dev
*mdev
;
322 unsigned int new_header_size
;
326 /* Careful this entire function runs inside of an rcu critical section */
328 mdev
= mpls_dev_get(dev
);
332 MPLS_INC_STATS_LEN(mdev
, skb
->len
, rx_packets
,
335 if (!mdev
->input_enabled
) {
336 MPLS_INC_STATS(mdev
, rx_dropped
);
340 if (skb
->pkt_type
!= PACKET_HOST
)
343 if ((skb
= skb_share_check(skb
, GFP_ATOMIC
)) == NULL
)
346 if (!pskb_may_pull(skb
, sizeof(*hdr
)))
349 /* Read and decode the label */
351 dec
= mpls_entry_decode(hdr
);
353 rt
= mpls_route_input_rcu(net
, dec
.label
);
355 MPLS_INC_STATS(mdev
, rx_noroute
);
359 nh
= mpls_select_multipath(rt
, skb
);
364 skb_pull(skb
, sizeof(*hdr
));
365 skb_reset_network_header(skb
);
369 if (skb_warn_if_lro(skb
))
372 skb_forward_csum(skb
);
374 /* Verify ttl is valid */
379 /* Find the output device */
380 out_dev
= rcu_dereference(nh
->nh_dev
);
381 if (!mpls_output_possible(out_dev
))
384 /* Verify the destination can hold the packet */
385 new_header_size
= mpls_nh_header_size(nh
);
386 mtu
= mpls_dev_mtu(out_dev
);
387 if (mpls_pkt_too_big(skb
, mtu
- new_header_size
))
390 hh_len
= LL_RESERVED_SPACE(out_dev
);
391 if (!out_dev
->header_ops
)
394 /* Ensure there is enough space for the headers in the skb */
395 if (skb_cow(skb
, hh_len
+ new_header_size
))
399 skb
->protocol
= htons(ETH_P_MPLS_UC
);
401 if (unlikely(!new_header_size
&& dec
.bos
)) {
402 /* Penultimate hop popping */
403 if (!mpls_egress(dev_net(out_dev
), rt
, skb
, dec
))
408 skb_push(skb
, new_header_size
);
409 skb_reset_network_header(skb
);
410 /* Push the new labels */
413 for (i
= nh
->nh_labels
- 1; i
>= 0; i
--) {
414 hdr
[i
] = mpls_entry_encode(nh
->nh_label
[i
],
420 mpls_stats_inc_outucastpkts(out_dev
, skb
);
422 /* If via wasn't specified then send out using device address */
423 if (nh
->nh_via_table
== MPLS_NEIGH_TABLE_UNSPEC
)
424 err
= neigh_xmit(NEIGH_LINK_TABLE
, out_dev
,
425 out_dev
->dev_addr
, skb
);
427 err
= neigh_xmit(nh
->nh_via_table
, out_dev
,
428 mpls_nh_via(rt
, nh
), skb
);
430 net_dbg_ratelimited("%s: packet transmission failed: %d\n",
435 out_mdev
= out_dev
? mpls_dev_get(out_dev
) : NULL
;
437 MPLS_INC_STATS(out_mdev
, tx_errors
);
440 MPLS_INC_STATS(mdev
, rx_errors
);
446 static struct packet_type mpls_packet_type __read_mostly
= {
447 .type
= cpu_to_be16(ETH_P_MPLS_UC
),
448 .func
= mpls_forward
,
451 static const struct nla_policy rtm_mpls_policy
[RTA_MAX
+1] = {
452 [RTA_DST
] = { .type
= NLA_U32
},
453 [RTA_OIF
] = { .type
= NLA_U32
},
454 [RTA_TTL_PROPAGATE
] = { .type
= NLA_U8
},
457 struct mpls_route_config
{
462 u8 rc_via
[MAX_VIA_ALEN
];
466 u32 rc_output_label
[MAX_NEW_LABELS
];
468 enum mpls_payload_type rc_payload_type
;
469 struct nl_info rc_nlinfo
;
470 struct rtnexthop
*rc_mp
;
474 /* all nexthops within a route have the same size based on max
475 * number of labels and max via length for a hop
477 static struct mpls_route
*mpls_rt_alloc(u8 num_nh
, u8 max_alen
, u8 max_labels
)
479 u8 nh_size
= MPLS_NH_SIZE(max_labels
, max_alen
);
480 struct mpls_route
*rt
;
483 size
= sizeof(*rt
) + num_nh
* nh_size
;
484 if (size
> MAX_MPLS_ROUTE_MEM
)
485 return ERR_PTR(-EINVAL
);
487 rt
= kzalloc(size
, GFP_KERNEL
);
489 return ERR_PTR(-ENOMEM
);
492 rt
->rt_nhn_alive
= num_nh
;
493 rt
->rt_nh_size
= nh_size
;
494 rt
->rt_via_offset
= MPLS_NH_VIA_OFF(max_labels
);
499 static void mpls_rt_free(struct mpls_route
*rt
)
502 kfree_rcu(rt
, rt_rcu
);
505 static void mpls_notify_route(struct net
*net
, unsigned index
,
506 struct mpls_route
*old
, struct mpls_route
*new,
507 const struct nl_info
*info
)
509 struct nlmsghdr
*nlh
= info
? info
->nlh
: NULL
;
510 unsigned portid
= info
? info
->portid
: 0;
511 int event
= new ? RTM_NEWROUTE
: RTM_DELROUTE
;
512 struct mpls_route
*rt
= new ? new : old
;
513 unsigned nlm_flags
= (old
&& new) ? NLM_F_REPLACE
: 0;
514 /* Ignore reserved labels for now */
515 if (rt
&& (index
>= MPLS_LABEL_FIRST_UNRESERVED
))
516 rtmsg_lfib(event
, index
, rt
, nlh
, net
, portid
, nlm_flags
);
519 static void mpls_route_update(struct net
*net
, unsigned index
,
520 struct mpls_route
*new,
521 const struct nl_info
*info
)
523 struct mpls_route __rcu
**platform_label
;
524 struct mpls_route
*rt
;
528 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
529 rt
= rtnl_dereference(platform_label
[index
]);
530 rcu_assign_pointer(platform_label
[index
], new);
532 mpls_notify_route(net
, index
, rt
, new, info
);
534 /* If we removed a route free it now */
538 static unsigned find_free_label(struct net
*net
)
540 struct mpls_route __rcu
**platform_label
;
541 size_t platform_labels
;
544 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
545 platform_labels
= net
->mpls
.platform_labels
;
546 for (index
= MPLS_LABEL_FIRST_UNRESERVED
; index
< platform_labels
;
548 if (!rtnl_dereference(platform_label
[index
]))
551 return LABEL_NOT_SPECIFIED
;
554 #if IS_ENABLED(CONFIG_INET)
555 static struct net_device
*inet_fib_lookup_dev(struct net
*net
,
558 struct net_device
*dev
;
560 struct in_addr daddr
;
562 memcpy(&daddr
, addr
, sizeof(struct in_addr
));
563 rt
= ip_route_output(net
, daddr
.s_addr
, 0, 0, 0);
575 static struct net_device
*inet_fib_lookup_dev(struct net
*net
,
578 return ERR_PTR(-EAFNOSUPPORT
);
582 #if IS_ENABLED(CONFIG_IPV6)
583 static struct net_device
*inet6_fib_lookup_dev(struct net
*net
,
586 struct net_device
*dev
;
587 struct dst_entry
*dst
;
592 return ERR_PTR(-EAFNOSUPPORT
);
594 memset(&fl6
, 0, sizeof(fl6
));
595 memcpy(&fl6
.daddr
, addr
, sizeof(struct in6_addr
));
596 err
= ipv6_stub
->ipv6_dst_lookup(net
, NULL
, &dst
, &fl6
);
607 static struct net_device
*inet6_fib_lookup_dev(struct net
*net
,
610 return ERR_PTR(-EAFNOSUPPORT
);
614 static struct net_device
*find_outdev(struct net
*net
,
615 struct mpls_route
*rt
,
616 struct mpls_nh
*nh
, int oif
)
618 struct net_device
*dev
= NULL
;
621 switch (nh
->nh_via_table
) {
622 case NEIGH_ARP_TABLE
:
623 dev
= inet_fib_lookup_dev(net
, mpls_nh_via(rt
, nh
));
626 dev
= inet6_fib_lookup_dev(net
, mpls_nh_via(rt
, nh
));
628 case NEIGH_LINK_TABLE
:
632 dev
= dev_get_by_index(net
, oif
);
636 return ERR_PTR(-ENODEV
);
641 /* The caller is holding rtnl anyways, so release the dev reference */
647 static int mpls_nh_assign_dev(struct net
*net
, struct mpls_route
*rt
,
648 struct mpls_nh
*nh
, int oif
)
650 struct net_device
*dev
= NULL
;
653 dev
= find_outdev(net
, rt
, nh
, oif
);
660 /* Ensure this is a supported device */
662 if (!mpls_dev_get(dev
))
665 if ((nh
->nh_via_table
== NEIGH_LINK_TABLE
) &&
666 (dev
->addr_len
!= nh
->nh_via_alen
))
669 RCU_INIT_POINTER(nh
->nh_dev
, dev
);
671 if (!(dev
->flags
& IFF_UP
)) {
672 nh
->nh_flags
|= RTNH_F_DEAD
;
676 flags
= dev_get_flags(dev
);
677 if (!(flags
& (IFF_RUNNING
| IFF_LOWER_UP
)))
678 nh
->nh_flags
|= RTNH_F_LINKDOWN
;
687 static int mpls_nh_build_from_cfg(struct mpls_route_config
*cfg
,
688 struct mpls_route
*rt
)
690 struct net
*net
= cfg
->rc_nlinfo
.nl_net
;
691 struct mpls_nh
*nh
= rt
->rt_nh
;
700 nh
->nh_labels
= cfg
->rc_output_labels
;
701 for (i
= 0; i
< nh
->nh_labels
; i
++)
702 nh
->nh_label
[i
] = cfg
->rc_output_label
[i
];
704 nh
->nh_via_table
= cfg
->rc_via_table
;
705 memcpy(__mpls_nh_via(rt
, nh
), cfg
->rc_via
, cfg
->rc_via_alen
);
706 nh
->nh_via_alen
= cfg
->rc_via_alen
;
708 err
= mpls_nh_assign_dev(net
, rt
, nh
, cfg
->rc_ifindex
);
712 if (nh
->nh_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
))
721 static int mpls_nh_build(struct net
*net
, struct mpls_route
*rt
,
722 struct mpls_nh
*nh
, int oif
, struct nlattr
*via
,
723 struct nlattr
*newdst
, u8 max_labels
)
731 err
= nla_get_labels(newdst
, max_labels
,
732 &nh
->nh_labels
, nh
->nh_label
);
738 err
= nla_get_via(via
, &nh
->nh_via_alen
, &nh
->nh_via_table
,
739 __mpls_nh_via(rt
, nh
));
743 nh
->nh_via_table
= MPLS_NEIGH_TABLE_UNSPEC
;
746 err
= mpls_nh_assign_dev(net
, rt
, nh
, oif
);
756 static u8
mpls_count_nexthops(struct rtnexthop
*rtnh
, int len
,
757 u8 cfg_via_alen
, u8
*max_via_alen
,
766 while (rtnh_ok(rtnh
, remaining
)) {
767 struct nlattr
*nla
, *attrs
= rtnh_attrs(rtnh
);
771 attrlen
= rtnh_attrlen(rtnh
);
772 nla
= nla_find(attrs
, attrlen
, RTA_VIA
);
773 if (nla
&& nla_len(nla
) >=
774 offsetof(struct rtvia
, rtvia_addr
)) {
775 int via_alen
= nla_len(nla
) -
776 offsetof(struct rtvia
, rtvia_addr
);
778 if (via_alen
<= MAX_VIA_ALEN
)
779 *max_via_alen
= max_t(u16
, *max_via_alen
,
783 nla
= nla_find(attrs
, attrlen
, RTA_NEWDST
);
785 nla_get_labels(nla
, MAX_NEW_LABELS
, &n_labels
, NULL
) != 0)
788 *max_labels
= max_t(u8
, *max_labels
, n_labels
);
790 /* number of nexthops is tracked by a u8.
791 * Check for overflow.
797 rtnh
= rtnh_next(rtnh
, &remaining
);
800 /* leftover implies invalid nexthop configuration, discard it */
801 return remaining
> 0 ? 0 : nhs
;
804 static int mpls_nh_build_multi(struct mpls_route_config
*cfg
,
805 struct mpls_route
*rt
, u8 max_labels
)
807 struct rtnexthop
*rtnh
= cfg
->rc_mp
;
808 struct nlattr
*nla_via
, *nla_newdst
;
809 int remaining
= cfg
->rc_mp_len
;
813 change_nexthops(rt
) {
820 if (!rtnh_ok(rtnh
, remaining
))
823 /* neither weighted multipath nor any flags
826 if (rtnh
->rtnh_hops
|| rtnh
->rtnh_flags
)
829 attrlen
= rtnh_attrlen(rtnh
);
831 struct nlattr
*attrs
= rtnh_attrs(rtnh
);
833 nla_via
= nla_find(attrs
, attrlen
, RTA_VIA
);
834 nla_newdst
= nla_find(attrs
, attrlen
, RTA_NEWDST
);
837 err
= mpls_nh_build(cfg
->rc_nlinfo
.nl_net
, rt
, nh
,
838 rtnh
->rtnh_ifindex
, nla_via
, nla_newdst
,
843 if (nh
->nh_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
))
846 rtnh
= rtnh_next(rtnh
, &remaining
);
848 } endfor_nexthops(rt
);
858 static int mpls_route_add(struct mpls_route_config
*cfg
)
860 struct mpls_route __rcu
**platform_label
;
861 struct net
*net
= cfg
->rc_nlinfo
.nl_net
;
862 struct mpls_route
*rt
, *old
;
869 index
= cfg
->rc_label
;
871 /* If a label was not specified during insert pick one */
872 if ((index
== LABEL_NOT_SPECIFIED
) &&
873 (cfg
->rc_nlflags
& NLM_F_CREATE
)) {
874 index
= find_free_label(net
);
877 /* Reserved labels may not be set */
878 if (index
< MPLS_LABEL_FIRST_UNRESERVED
)
881 /* The full 20 bit range may not be supported. */
882 if (index
>= net
->mpls
.platform_labels
)
885 /* Append makes no sense with mpls */
887 if (cfg
->rc_nlflags
& NLM_F_APPEND
)
891 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
892 old
= rtnl_dereference(platform_label
[index
]);
893 if ((cfg
->rc_nlflags
& NLM_F_EXCL
) && old
)
897 if (!(cfg
->rc_nlflags
& NLM_F_REPLACE
) && old
)
901 if (!(cfg
->rc_nlflags
& NLM_F_CREATE
) && !old
)
906 nhs
= mpls_count_nexthops(cfg
->rc_mp
, cfg
->rc_mp_len
,
907 cfg
->rc_via_alen
, &max_via_alen
,
910 max_via_alen
= cfg
->rc_via_alen
;
911 max_labels
= cfg
->rc_output_labels
;
919 rt
= mpls_rt_alloc(nhs
, max_via_alen
, max_labels
);
925 rt
->rt_protocol
= cfg
->rc_protocol
;
926 rt
->rt_payload_type
= cfg
->rc_payload_type
;
927 rt
->rt_ttl_propagate
= cfg
->rc_ttl_propagate
;
930 err
= mpls_nh_build_multi(cfg
, rt
, max_labels
);
932 err
= mpls_nh_build_from_cfg(cfg
, rt
);
936 mpls_route_update(net
, index
, rt
, &cfg
->rc_nlinfo
);
946 static int mpls_route_del(struct mpls_route_config
*cfg
)
948 struct net
*net
= cfg
->rc_nlinfo
.nl_net
;
952 index
= cfg
->rc_label
;
954 /* Reserved labels may not be removed */
955 if (index
< MPLS_LABEL_FIRST_UNRESERVED
)
958 /* The full 20 bit range may not be supported */
959 if (index
>= net
->mpls
.platform_labels
)
962 mpls_route_update(net
, index
, NULL
, &cfg
->rc_nlinfo
);
969 static void mpls_get_stats(struct mpls_dev
*mdev
,
970 struct mpls_link_stats
*stats
)
972 struct mpls_pcpu_stats
*p
;
975 memset(stats
, 0, sizeof(*stats
));
977 for_each_possible_cpu(i
) {
978 struct mpls_link_stats local
;
981 p
= per_cpu_ptr(mdev
->stats
, i
);
983 start
= u64_stats_fetch_begin(&p
->syncp
);
985 } while (u64_stats_fetch_retry(&p
->syncp
, start
));
987 stats
->rx_packets
+= local
.rx_packets
;
988 stats
->rx_bytes
+= local
.rx_bytes
;
989 stats
->tx_packets
+= local
.tx_packets
;
990 stats
->tx_bytes
+= local
.tx_bytes
;
991 stats
->rx_errors
+= local
.rx_errors
;
992 stats
->tx_errors
+= local
.tx_errors
;
993 stats
->rx_dropped
+= local
.rx_dropped
;
994 stats
->tx_dropped
+= local
.tx_dropped
;
995 stats
->rx_noroute
+= local
.rx_noroute
;
999 static int mpls_fill_stats_af(struct sk_buff
*skb
,
1000 const struct net_device
*dev
)
1002 struct mpls_link_stats
*stats
;
1003 struct mpls_dev
*mdev
;
1006 mdev
= mpls_dev_get(dev
);
1010 nla
= nla_reserve_64bit(skb
, MPLS_STATS_LINK
,
1011 sizeof(struct mpls_link_stats
),
1016 stats
= nla_data(nla
);
1017 mpls_get_stats(mdev
, stats
);
1022 static size_t mpls_get_stats_af_size(const struct net_device
*dev
)
1024 struct mpls_dev
*mdev
;
1026 mdev
= mpls_dev_get(dev
);
1030 return nla_total_size_64bit(sizeof(struct mpls_link_stats
));
1033 static int mpls_netconf_fill_devconf(struct sk_buff
*skb
, struct mpls_dev
*mdev
,
1034 u32 portid
, u32 seq
, int event
,
1035 unsigned int flags
, int type
)
1037 struct nlmsghdr
*nlh
;
1038 struct netconfmsg
*ncm
;
1041 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(struct netconfmsg
),
1046 if (type
== NETCONFA_ALL
)
1049 ncm
= nlmsg_data(nlh
);
1050 ncm
->ncm_family
= AF_MPLS
;
1052 if (nla_put_s32(skb
, NETCONFA_IFINDEX
, mdev
->dev
->ifindex
) < 0)
1053 goto nla_put_failure
;
1055 if ((all
|| type
== NETCONFA_INPUT
) &&
1056 nla_put_s32(skb
, NETCONFA_INPUT
,
1057 mdev
->input_enabled
) < 0)
1058 goto nla_put_failure
;
1060 nlmsg_end(skb
, nlh
);
1064 nlmsg_cancel(skb
, nlh
);
1068 static int mpls_netconf_msgsize_devconf(int type
)
1070 int size
= NLMSG_ALIGN(sizeof(struct netconfmsg
))
1071 + nla_total_size(4); /* NETCONFA_IFINDEX */
1074 if (type
== NETCONFA_ALL
)
1077 if (all
|| type
== NETCONFA_INPUT
)
1078 size
+= nla_total_size(4);
1083 static void mpls_netconf_notify_devconf(struct net
*net
, int event
,
1084 int type
, struct mpls_dev
*mdev
)
1086 struct sk_buff
*skb
;
1089 skb
= nlmsg_new(mpls_netconf_msgsize_devconf(type
), GFP_KERNEL
);
1093 err
= mpls_netconf_fill_devconf(skb
, mdev
, 0, 0, event
, 0, type
);
1095 /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1096 WARN_ON(err
== -EMSGSIZE
);
1101 rtnl_notify(skb
, net
, 0, RTNLGRP_MPLS_NETCONF
, NULL
, GFP_KERNEL
);
1105 rtnl_set_sk_err(net
, RTNLGRP_MPLS_NETCONF
, err
);
1108 static const struct nla_policy devconf_mpls_policy
[NETCONFA_MAX
+ 1] = {
1109 [NETCONFA_IFINDEX
] = { .len
= sizeof(int) },
1112 static int mpls_netconf_get_devconf(struct sk_buff
*in_skb
,
1113 struct nlmsghdr
*nlh
,
1114 struct netlink_ext_ack
*extack
)
1116 struct net
*net
= sock_net(in_skb
->sk
);
1117 struct nlattr
*tb
[NETCONFA_MAX
+ 1];
1118 struct netconfmsg
*ncm
;
1119 struct net_device
*dev
;
1120 struct mpls_dev
*mdev
;
1121 struct sk_buff
*skb
;
1125 err
= nlmsg_parse(nlh
, sizeof(*ncm
), tb
, NETCONFA_MAX
,
1126 devconf_mpls_policy
, NULL
);
1131 if (!tb
[NETCONFA_IFINDEX
])
1134 ifindex
= nla_get_s32(tb
[NETCONFA_IFINDEX
]);
1135 dev
= __dev_get_by_index(net
, ifindex
);
1139 mdev
= mpls_dev_get(dev
);
1144 skb
= nlmsg_new(mpls_netconf_msgsize_devconf(NETCONFA_ALL
), GFP_KERNEL
);
1148 err
= mpls_netconf_fill_devconf(skb
, mdev
,
1149 NETLINK_CB(in_skb
).portid
,
1150 nlh
->nlmsg_seq
, RTM_NEWNETCONF
, 0,
1153 /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
1154 WARN_ON(err
== -EMSGSIZE
);
1158 err
= rtnl_unicast(skb
, net
, NETLINK_CB(in_skb
).portid
);
1163 static int mpls_netconf_dump_devconf(struct sk_buff
*skb
,
1164 struct netlink_callback
*cb
)
1166 struct net
*net
= sock_net(skb
->sk
);
1167 struct hlist_head
*head
;
1168 struct net_device
*dev
;
1169 struct mpls_dev
*mdev
;
1174 s_idx
= idx
= cb
->args
[1];
1176 for (h
= s_h
; h
< NETDEV_HASHENTRIES
; h
++, s_idx
= 0) {
1178 head
= &net
->dev_index_head
[h
];
1180 cb
->seq
= net
->dev_base_seq
;
1181 hlist_for_each_entry_rcu(dev
, head
, index_hlist
) {
1184 mdev
= mpls_dev_get(dev
);
1187 if (mpls_netconf_fill_devconf(skb
, mdev
,
1188 NETLINK_CB(cb
->skb
).portid
,
1192 NETCONFA_ALL
) < 0) {
1196 nl_dump_check_consistent(cb
, nlmsg_hdr(skb
));
1209 #define MPLS_PERDEV_SYSCTL_OFFSET(field) \
1210 (&((struct mpls_dev *)0)->field)
1212 static int mpls_conf_proc(struct ctl_table
*ctl
, int write
,
1213 void __user
*buffer
,
1214 size_t *lenp
, loff_t
*ppos
)
1216 int oval
= *(int *)ctl
->data
;
1217 int ret
= proc_dointvec(ctl
, write
, buffer
, lenp
, ppos
);
1220 struct mpls_dev
*mdev
= ctl
->extra1
;
1221 int i
= (int *)ctl
->data
- (int *)mdev
;
1222 struct net
*net
= ctl
->extra2
;
1223 int val
= *(int *)ctl
->data
;
1225 if (i
== offsetof(struct mpls_dev
, input_enabled
) &&
1227 mpls_netconf_notify_devconf(net
, RTM_NEWNETCONF
,
1228 NETCONFA_INPUT
, mdev
);
1235 static const struct ctl_table mpls_dev_table
[] = {
1237 .procname
= "input",
1238 .maxlen
= sizeof(int),
1240 .proc_handler
= mpls_conf_proc
,
1241 .data
= MPLS_PERDEV_SYSCTL_OFFSET(input_enabled
),
1246 static int mpls_dev_sysctl_register(struct net_device
*dev
,
1247 struct mpls_dev
*mdev
)
1249 char path
[sizeof("net/mpls/conf/") + IFNAMSIZ
];
1250 struct net
*net
= dev_net(dev
);
1251 struct ctl_table
*table
;
1254 table
= kmemdup(&mpls_dev_table
, sizeof(mpls_dev_table
), GFP_KERNEL
);
1258 /* Table data contains only offsets relative to the base of
1259 * the mdev at this point, so make them absolute.
1261 for (i
= 0; i
< ARRAY_SIZE(mpls_dev_table
); i
++) {
1262 table
[i
].data
= (char *)mdev
+ (uintptr_t)table
[i
].data
;
1263 table
[i
].extra1
= mdev
;
1264 table
[i
].extra2
= net
;
1267 snprintf(path
, sizeof(path
), "net/mpls/conf/%s", dev
->name
);
1269 mdev
->sysctl
= register_net_sysctl(net
, path
, table
);
1273 mpls_netconf_notify_devconf(net
, RTM_NEWNETCONF
, NETCONFA_ALL
, mdev
);
1282 static void mpls_dev_sysctl_unregister(struct net_device
*dev
,
1283 struct mpls_dev
*mdev
)
1285 struct net
*net
= dev_net(dev
);
1286 struct ctl_table
*table
;
1288 table
= mdev
->sysctl
->ctl_table_arg
;
1289 unregister_net_sysctl_table(mdev
->sysctl
);
1292 mpls_netconf_notify_devconf(net
, RTM_DELNETCONF
, 0, mdev
);
1295 static struct mpls_dev
*mpls_add_dev(struct net_device
*dev
)
1297 struct mpls_dev
*mdev
;
1303 mdev
= kzalloc(sizeof(*mdev
), GFP_KERNEL
);
1305 return ERR_PTR(err
);
1307 mdev
->stats
= alloc_percpu(struct mpls_pcpu_stats
);
1311 for_each_possible_cpu(i
) {
1312 struct mpls_pcpu_stats
*mpls_stats
;
1314 mpls_stats
= per_cpu_ptr(mdev
->stats
, i
);
1315 u64_stats_init(&mpls_stats
->syncp
);
1320 err
= mpls_dev_sysctl_register(dev
, mdev
);
1324 rcu_assign_pointer(dev
->mpls_ptr
, mdev
);
1329 free_percpu(mdev
->stats
);
1331 return ERR_PTR(err
);
1334 static void mpls_dev_destroy_rcu(struct rcu_head
*head
)
1336 struct mpls_dev
*mdev
= container_of(head
, struct mpls_dev
, rcu
);
1338 free_percpu(mdev
->stats
);
1342 static void mpls_ifdown(struct net_device
*dev
, int event
)
1344 struct mpls_route __rcu
**platform_label
;
1345 struct net
*net
= dev_net(dev
);
1349 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
1350 for (index
= 0; index
< net
->mpls
.platform_labels
; index
++) {
1351 struct mpls_route
*rt
= rtnl_dereference(platform_label
[index
]);
1358 change_nexthops(rt
) {
1359 unsigned int nh_flags
= nh
->nh_flags
;
1361 if (rtnl_dereference(nh
->nh_dev
) != dev
)
1366 case NETDEV_UNREGISTER
:
1367 nh_flags
|= RTNH_F_DEAD
;
1370 nh_flags
|= RTNH_F_LINKDOWN
;
1373 if (event
== NETDEV_UNREGISTER
)
1374 RCU_INIT_POINTER(nh
->nh_dev
, NULL
);
1376 if (nh
->nh_flags
!= nh_flags
)
1377 WRITE_ONCE(nh
->nh_flags
, nh_flags
);
1379 if (!(nh_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
)))
1381 if (!rtnl_dereference(nh
->nh_dev
))
1383 } endfor_nexthops(rt
);
1385 WRITE_ONCE(rt
->rt_nhn_alive
, alive
);
1387 /* if there are no more nexthops, delete the route */
1388 if (event
== NETDEV_UNREGISTER
&& deleted
== rt
->rt_nhn
)
1389 mpls_route_update(net
, index
, NULL
, NULL
);
1393 static void mpls_ifup(struct net_device
*dev
, unsigned int flags
)
1395 struct mpls_route __rcu
**platform_label
;
1396 struct net
*net
= dev_net(dev
);
1400 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
1401 for (index
= 0; index
< net
->mpls
.platform_labels
; index
++) {
1402 struct mpls_route
*rt
= rtnl_dereference(platform_label
[index
]);
1408 change_nexthops(rt
) {
1409 unsigned int nh_flags
= nh
->nh_flags
;
1410 struct net_device
*nh_dev
=
1411 rtnl_dereference(nh
->nh_dev
);
1413 if (!(nh_flags
& flags
)) {
1421 WRITE_ONCE(nh
->nh_flags
, flags
);
1422 } endfor_nexthops(rt
);
1424 WRITE_ONCE(rt
->rt_nhn_alive
, alive
);
1428 static int mpls_dev_notify(struct notifier_block
*this, unsigned long event
,
1431 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
1432 struct mpls_dev
*mdev
;
1435 if (event
== NETDEV_REGISTER
) {
1436 /* For now just support Ethernet, IPGRE, SIT and IPIP devices */
1437 if (dev
->type
== ARPHRD_ETHER
||
1438 dev
->type
== ARPHRD_LOOPBACK
||
1439 dev
->type
== ARPHRD_IPGRE
||
1440 dev
->type
== ARPHRD_SIT
||
1441 dev
->type
== ARPHRD_TUNNEL
) {
1442 mdev
= mpls_add_dev(dev
);
1444 return notifier_from_errno(PTR_ERR(mdev
));
1449 mdev
= mpls_dev_get(dev
);
1455 mpls_ifdown(dev
, event
);
1458 flags
= dev_get_flags(dev
);
1459 if (flags
& (IFF_RUNNING
| IFF_LOWER_UP
))
1460 mpls_ifup(dev
, RTNH_F_DEAD
| RTNH_F_LINKDOWN
);
1462 mpls_ifup(dev
, RTNH_F_DEAD
);
1465 flags
= dev_get_flags(dev
);
1466 if (flags
& (IFF_RUNNING
| IFF_LOWER_UP
))
1467 mpls_ifup(dev
, RTNH_F_DEAD
| RTNH_F_LINKDOWN
);
1469 mpls_ifdown(dev
, event
);
1471 case NETDEV_UNREGISTER
:
1472 mpls_ifdown(dev
, event
);
1473 mdev
= mpls_dev_get(dev
);
1475 mpls_dev_sysctl_unregister(dev
, mdev
);
1476 RCU_INIT_POINTER(dev
->mpls_ptr
, NULL
);
1477 call_rcu(&mdev
->rcu
, mpls_dev_destroy_rcu
);
1480 case NETDEV_CHANGENAME
:
1481 mdev
= mpls_dev_get(dev
);
1485 mpls_dev_sysctl_unregister(dev
, mdev
);
1486 err
= mpls_dev_sysctl_register(dev
, mdev
);
1488 return notifier_from_errno(err
);
1495 static struct notifier_block mpls_dev_notifier
= {
1496 .notifier_call
= mpls_dev_notify
,
1499 static int nla_put_via(struct sk_buff
*skb
,
1500 u8 table
, const void *addr
, int alen
)
1502 static const int table_to_family
[NEIGH_NR_TABLES
+ 1] = {
1503 AF_INET
, AF_INET6
, AF_DECnet
, AF_PACKET
,
1507 int family
= AF_UNSPEC
;
1509 nla
= nla_reserve(skb
, RTA_VIA
, alen
+ 2);
1513 if (table
<= NEIGH_NR_TABLES
)
1514 family
= table_to_family
[table
];
1516 via
= nla_data(nla
);
1517 via
->rtvia_family
= family
;
1518 memcpy(via
->rtvia_addr
, addr
, alen
);
1522 int nla_put_labels(struct sk_buff
*skb
, int attrtype
,
1523 u8 labels
, const u32 label
[])
1526 struct mpls_shim_hdr
*nla_label
;
1529 nla
= nla_reserve(skb
, attrtype
, labels
*4);
1533 nla_label
= nla_data(nla
);
1535 for (i
= labels
- 1; i
>= 0; i
--) {
1536 nla_label
[i
] = mpls_entry_encode(label
[i
], 0, 0, bos
);
1542 EXPORT_SYMBOL_GPL(nla_put_labels
);
1544 int nla_get_labels(const struct nlattr
*nla
,
1545 u8 max_labels
, u8
*labels
, u32 label
[])
1547 unsigned len
= nla_len(nla
);
1548 struct mpls_shim_hdr
*nla_label
;
1553 /* len needs to be an even multiple of 4 (the label size). Number
1554 * of labels is a u8 so check for overflow.
1556 if (len
& 3 || len
/ 4 > 255)
1559 /* Limit the number of new labels allowed */
1561 if (nla_labels
> max_labels
)
1564 /* when label == NULL, caller wants number of labels */
1568 nla_label
= nla_data(nla
);
1570 for (i
= nla_labels
- 1; i
>= 0; i
--, bos
= false) {
1571 struct mpls_entry_decoded dec
;
1572 dec
= mpls_entry_decode(nla_label
+ i
);
1574 /* Ensure the bottom of stack flag is properly set
1575 * and ttl and tc are both clear.
1577 if ((dec
.bos
!= bos
) || dec
.ttl
|| dec
.tc
)
1580 switch (dec
.label
) {
1581 case MPLS_LABEL_IMPLNULL
:
1582 /* RFC3032: This is a label that an LSR may
1583 * assign and distribute, but which never
1584 * actually appears in the encapsulation.
1589 label
[i
] = dec
.label
;
1592 *labels
= nla_labels
;
1595 EXPORT_SYMBOL_GPL(nla_get_labels
);
1597 int nla_get_via(const struct nlattr
*nla
, u8
*via_alen
,
1598 u8
*via_table
, u8 via_addr
[])
1600 struct rtvia
*via
= nla_data(nla
);
1604 if (nla_len(nla
) < offsetof(struct rtvia
, rtvia_addr
))
1606 alen
= nla_len(nla
) -
1607 offsetof(struct rtvia
, rtvia_addr
);
1608 if (alen
> MAX_VIA_ALEN
)
1611 /* Validate the address family */
1612 switch (via
->rtvia_family
) {
1614 *via_table
= NEIGH_LINK_TABLE
;
1617 *via_table
= NEIGH_ARP_TABLE
;
1622 *via_table
= NEIGH_ND_TABLE
;
1627 /* Unsupported address family */
1631 memcpy(via_addr
, via
->rtvia_addr
, alen
);
1639 static int rtm_to_route_config(struct sk_buff
*skb
, struct nlmsghdr
*nlh
,
1640 struct mpls_route_config
*cfg
)
1643 struct nlattr
*tb
[RTA_MAX
+1];
1647 err
= nlmsg_parse(nlh
, sizeof(*rtm
), tb
, RTA_MAX
, rtm_mpls_policy
,
1653 rtm
= nlmsg_data(nlh
);
1655 if (rtm
->rtm_family
!= AF_MPLS
)
1657 if (rtm
->rtm_dst_len
!= 20)
1659 if (rtm
->rtm_src_len
!= 0)
1661 if (rtm
->rtm_tos
!= 0)
1663 if (rtm
->rtm_table
!= RT_TABLE_MAIN
)
1665 /* Any value is acceptable for rtm_protocol */
1667 /* As mpls uses destination specific addresses
1668 * (or source specific address in the case of multicast)
1669 * all addresses have universal scope.
1671 if (rtm
->rtm_scope
!= RT_SCOPE_UNIVERSE
)
1673 if (rtm
->rtm_type
!= RTN_UNICAST
)
1675 if (rtm
->rtm_flags
!= 0)
1678 cfg
->rc_label
= LABEL_NOT_SPECIFIED
;
1679 cfg
->rc_protocol
= rtm
->rtm_protocol
;
1680 cfg
->rc_via_table
= MPLS_NEIGH_TABLE_UNSPEC
;
1681 cfg
->rc_ttl_propagate
= MPLS_TTL_PROP_DEFAULT
;
1682 cfg
->rc_nlflags
= nlh
->nlmsg_flags
;
1683 cfg
->rc_nlinfo
.portid
= NETLINK_CB(skb
).portid
;
1684 cfg
->rc_nlinfo
.nlh
= nlh
;
1685 cfg
->rc_nlinfo
.nl_net
= sock_net(skb
->sk
);
1687 for (index
= 0; index
<= RTA_MAX
; index
++) {
1688 struct nlattr
*nla
= tb
[index
];
1694 cfg
->rc_ifindex
= nla_get_u32(nla
);
1697 if (nla_get_labels(nla
, MAX_NEW_LABELS
,
1698 &cfg
->rc_output_labels
,
1699 cfg
->rc_output_label
))
1705 if (nla_get_labels(nla
, 1, &label_count
,
1709 /* Reserved labels may not be set */
1710 if (cfg
->rc_label
< MPLS_LABEL_FIRST_UNRESERVED
)
1717 if (nla_get_via(nla
, &cfg
->rc_via_alen
,
1718 &cfg
->rc_via_table
, cfg
->rc_via
))
1724 cfg
->rc_mp
= nla_data(nla
);
1725 cfg
->rc_mp_len
= nla_len(nla
);
1728 case RTA_TTL_PROPAGATE
:
1730 u8 ttl_propagate
= nla_get_u8(nla
);
1732 if (ttl_propagate
> 1)
1734 cfg
->rc_ttl_propagate
= ttl_propagate
?
1735 MPLS_TTL_PROP_ENABLED
:
1736 MPLS_TTL_PROP_DISABLED
;
1740 /* Unsupported attribute */
1750 static int mpls_rtm_delroute(struct sk_buff
*skb
, struct nlmsghdr
*nlh
,
1751 struct netlink_ext_ack
*extack
)
1753 struct mpls_route_config
*cfg
;
1756 cfg
= kzalloc(sizeof(*cfg
), GFP_KERNEL
);
1760 err
= rtm_to_route_config(skb
, nlh
, cfg
);
1764 err
= mpls_route_del(cfg
);
1772 static int mpls_rtm_newroute(struct sk_buff
*skb
, struct nlmsghdr
*nlh
,
1773 struct netlink_ext_ack
*extack
)
1775 struct mpls_route_config
*cfg
;
1778 cfg
= kzalloc(sizeof(*cfg
), GFP_KERNEL
);
1782 err
= rtm_to_route_config(skb
, nlh
, cfg
);
1786 err
= mpls_route_add(cfg
);
1793 static int mpls_dump_route(struct sk_buff
*skb
, u32 portid
, u32 seq
, int event
,
1794 u32 label
, struct mpls_route
*rt
, int flags
)
1796 struct net_device
*dev
;
1797 struct nlmsghdr
*nlh
;
1800 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*rtm
), flags
);
1804 rtm
= nlmsg_data(nlh
);
1805 rtm
->rtm_family
= AF_MPLS
;
1806 rtm
->rtm_dst_len
= 20;
1807 rtm
->rtm_src_len
= 0;
1809 rtm
->rtm_table
= RT_TABLE_MAIN
;
1810 rtm
->rtm_protocol
= rt
->rt_protocol
;
1811 rtm
->rtm_scope
= RT_SCOPE_UNIVERSE
;
1812 rtm
->rtm_type
= RTN_UNICAST
;
1815 if (nla_put_labels(skb
, RTA_DST
, 1, &label
))
1816 goto nla_put_failure
;
1818 if (rt
->rt_ttl_propagate
!= MPLS_TTL_PROP_DEFAULT
) {
1819 bool ttl_propagate
=
1820 rt
->rt_ttl_propagate
== MPLS_TTL_PROP_ENABLED
;
1822 if (nla_put_u8(skb
, RTA_TTL_PROPAGATE
,
1824 goto nla_put_failure
;
1826 if (rt
->rt_nhn
== 1) {
1827 const struct mpls_nh
*nh
= rt
->rt_nh
;
1829 if (nh
->nh_labels
&&
1830 nla_put_labels(skb
, RTA_NEWDST
, nh
->nh_labels
,
1832 goto nla_put_failure
;
1833 if (nh
->nh_via_table
!= MPLS_NEIGH_TABLE_UNSPEC
&&
1834 nla_put_via(skb
, nh
->nh_via_table
, mpls_nh_via(rt
, nh
),
1836 goto nla_put_failure
;
1837 dev
= rtnl_dereference(nh
->nh_dev
);
1838 if (dev
&& nla_put_u32(skb
, RTA_OIF
, dev
->ifindex
))
1839 goto nla_put_failure
;
1840 if (nh
->nh_flags
& RTNH_F_LINKDOWN
)
1841 rtm
->rtm_flags
|= RTNH_F_LINKDOWN
;
1842 if (nh
->nh_flags
& RTNH_F_DEAD
)
1843 rtm
->rtm_flags
|= RTNH_F_DEAD
;
1845 struct rtnexthop
*rtnh
;
1850 mp
= nla_nest_start(skb
, RTA_MULTIPATH
);
1852 goto nla_put_failure
;
1855 dev
= rtnl_dereference(nh
->nh_dev
);
1859 rtnh
= nla_reserve_nohdr(skb
, sizeof(*rtnh
));
1861 goto nla_put_failure
;
1863 rtnh
->rtnh_ifindex
= dev
->ifindex
;
1864 if (nh
->nh_flags
& RTNH_F_LINKDOWN
) {
1865 rtnh
->rtnh_flags
|= RTNH_F_LINKDOWN
;
1868 if (nh
->nh_flags
& RTNH_F_DEAD
) {
1869 rtnh
->rtnh_flags
|= RTNH_F_DEAD
;
1873 if (nh
->nh_labels
&& nla_put_labels(skb
, RTA_NEWDST
,
1876 goto nla_put_failure
;
1877 if (nh
->nh_via_table
!= MPLS_NEIGH_TABLE_UNSPEC
&&
1878 nla_put_via(skb
, nh
->nh_via_table
,
1879 mpls_nh_via(rt
, nh
),
1881 goto nla_put_failure
;
1883 /* length of rtnetlink header + attributes */
1884 rtnh
->rtnh_len
= nlmsg_get_pos(skb
) - (void *)rtnh
;
1885 } endfor_nexthops(rt
);
1887 if (linkdown
== rt
->rt_nhn
)
1888 rtm
->rtm_flags
|= RTNH_F_LINKDOWN
;
1889 if (dead
== rt
->rt_nhn
)
1890 rtm
->rtm_flags
|= RTNH_F_DEAD
;
1892 nla_nest_end(skb
, mp
);
1895 nlmsg_end(skb
, nlh
);
1899 nlmsg_cancel(skb
, nlh
);
1903 static int mpls_dump_routes(struct sk_buff
*skb
, struct netlink_callback
*cb
)
1905 struct net
*net
= sock_net(skb
->sk
);
1906 struct mpls_route __rcu
**platform_label
;
1907 size_t platform_labels
;
1912 index
= cb
->args
[0];
1913 if (index
< MPLS_LABEL_FIRST_UNRESERVED
)
1914 index
= MPLS_LABEL_FIRST_UNRESERVED
;
1916 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
1917 platform_labels
= net
->mpls
.platform_labels
;
1918 for (; index
< platform_labels
; index
++) {
1919 struct mpls_route
*rt
;
1920 rt
= rtnl_dereference(platform_label
[index
]);
1924 if (mpls_dump_route(skb
, NETLINK_CB(cb
->skb
).portid
,
1925 cb
->nlh
->nlmsg_seq
, RTM_NEWROUTE
,
1926 index
, rt
, NLM_F_MULTI
) < 0)
1929 cb
->args
[0] = index
;
1934 static inline size_t lfib_nlmsg_size(struct mpls_route
*rt
)
1937 NLMSG_ALIGN(sizeof(struct rtmsg
))
1938 + nla_total_size(4) /* RTA_DST */
1939 + nla_total_size(1); /* RTA_TTL_PROPAGATE */
1941 if (rt
->rt_nhn
== 1) {
1942 struct mpls_nh
*nh
= rt
->rt_nh
;
1945 payload
+= nla_total_size(4); /* RTA_OIF */
1946 if (nh
->nh_via_table
!= MPLS_NEIGH_TABLE_UNSPEC
) /* RTA_VIA */
1947 payload
+= nla_total_size(2 + nh
->nh_via_alen
);
1948 if (nh
->nh_labels
) /* RTA_NEWDST */
1949 payload
+= nla_total_size(nh
->nh_labels
* 4);
1951 /* each nexthop is packed in an attribute */
1955 if (!rtnl_dereference(nh
->nh_dev
))
1957 nhsize
+= nla_total_size(sizeof(struct rtnexthop
));
1959 if (nh
->nh_via_table
!= MPLS_NEIGH_TABLE_UNSPEC
)
1960 nhsize
+= nla_total_size(2 + nh
->nh_via_alen
);
1962 nhsize
+= nla_total_size(nh
->nh_labels
* 4);
1963 } endfor_nexthops(rt
);
1964 /* nested attribute */
1965 payload
+= nla_total_size(nhsize
);
1971 static void rtmsg_lfib(int event
, u32 label
, struct mpls_route
*rt
,
1972 struct nlmsghdr
*nlh
, struct net
*net
, u32 portid
,
1973 unsigned int nlm_flags
)
1975 struct sk_buff
*skb
;
1976 u32 seq
= nlh
? nlh
->nlmsg_seq
: 0;
1979 skb
= nlmsg_new(lfib_nlmsg_size(rt
), GFP_KERNEL
);
1983 err
= mpls_dump_route(skb
, portid
, seq
, event
, label
, rt
, nlm_flags
);
1985 /* -EMSGSIZE implies BUG in lfib_nlmsg_size */
1986 WARN_ON(err
== -EMSGSIZE
);
1990 rtnl_notify(skb
, net
, portid
, RTNLGRP_MPLS_ROUTE
, nlh
, GFP_KERNEL
);
1995 rtnl_set_sk_err(net
, RTNLGRP_MPLS_ROUTE
, err
);
1998 static int resize_platform_label_table(struct net
*net
, size_t limit
)
2000 size_t size
= sizeof(struct mpls_route
*) * limit
;
2003 struct mpls_route __rcu
**labels
= NULL
, **old
;
2004 struct mpls_route
*rt0
= NULL
, *rt2
= NULL
;
2008 labels
= kvzalloc(size
, GFP_KERNEL
);
2013 /* In case the predefined labels need to be populated */
2014 if (limit
> MPLS_LABEL_IPV4NULL
) {
2015 struct net_device
*lo
= net
->loopback_dev
;
2016 rt0
= mpls_rt_alloc(1, lo
->addr_len
, 0);
2019 RCU_INIT_POINTER(rt0
->rt_nh
->nh_dev
, lo
);
2020 rt0
->rt_protocol
= RTPROT_KERNEL
;
2021 rt0
->rt_payload_type
= MPT_IPV4
;
2022 rt0
->rt_ttl_propagate
= MPLS_TTL_PROP_DEFAULT
;
2023 rt0
->rt_nh
->nh_via_table
= NEIGH_LINK_TABLE
;
2024 rt0
->rt_nh
->nh_via_alen
= lo
->addr_len
;
2025 memcpy(__mpls_nh_via(rt0
, rt0
->rt_nh
), lo
->dev_addr
,
2028 if (limit
> MPLS_LABEL_IPV6NULL
) {
2029 struct net_device
*lo
= net
->loopback_dev
;
2030 rt2
= mpls_rt_alloc(1, lo
->addr_len
, 0);
2033 RCU_INIT_POINTER(rt2
->rt_nh
->nh_dev
, lo
);
2034 rt2
->rt_protocol
= RTPROT_KERNEL
;
2035 rt2
->rt_payload_type
= MPT_IPV6
;
2036 rt2
->rt_ttl_propagate
= MPLS_TTL_PROP_DEFAULT
;
2037 rt2
->rt_nh
->nh_via_table
= NEIGH_LINK_TABLE
;
2038 rt2
->rt_nh
->nh_via_alen
= lo
->addr_len
;
2039 memcpy(__mpls_nh_via(rt2
, rt2
->rt_nh
), lo
->dev_addr
,
2044 /* Remember the original table */
2045 old
= rtnl_dereference(net
->mpls
.platform_label
);
2046 old_limit
= net
->mpls
.platform_labels
;
2048 /* Free any labels beyond the new table */
2049 for (index
= limit
; index
< old_limit
; index
++)
2050 mpls_route_update(net
, index
, NULL
, NULL
);
2052 /* Copy over the old labels */
2054 if (old_limit
< limit
)
2055 cp_size
= old_limit
* sizeof(struct mpls_route
*);
2057 memcpy(labels
, old
, cp_size
);
2059 /* If needed set the predefined labels */
2060 if ((old_limit
<= MPLS_LABEL_IPV6NULL
) &&
2061 (limit
> MPLS_LABEL_IPV6NULL
)) {
2062 RCU_INIT_POINTER(labels
[MPLS_LABEL_IPV6NULL
], rt2
);
2066 if ((old_limit
<= MPLS_LABEL_IPV4NULL
) &&
2067 (limit
> MPLS_LABEL_IPV4NULL
)) {
2068 RCU_INIT_POINTER(labels
[MPLS_LABEL_IPV4NULL
], rt0
);
2072 /* Update the global pointers */
2073 net
->mpls
.platform_labels
= limit
;
2074 rcu_assign_pointer(net
->mpls
.platform_label
, labels
);
2095 static int mpls_platform_labels(struct ctl_table
*table
, int write
,
2096 void __user
*buffer
, size_t *lenp
, loff_t
*ppos
)
2098 struct net
*net
= table
->data
;
2099 int platform_labels
= net
->mpls
.platform_labels
;
2101 struct ctl_table tmp
= {
2102 .procname
= table
->procname
,
2103 .data
= &platform_labels
,
2104 .maxlen
= sizeof(int),
2105 .mode
= table
->mode
,
2107 .extra2
= &label_limit
,
2110 ret
= proc_dointvec_minmax(&tmp
, write
, buffer
, lenp
, ppos
);
2112 if (write
&& ret
== 0)
2113 ret
= resize_platform_label_table(net
, platform_labels
);
2118 #define MPLS_NS_SYSCTL_OFFSET(field) \
2119 (&((struct net *)0)->field)
2121 static const struct ctl_table mpls_table
[] = {
2123 .procname
= "platform_labels",
2125 .maxlen
= sizeof(int),
2127 .proc_handler
= mpls_platform_labels
,
2130 .procname
= "ip_ttl_propagate",
2131 .data
= MPLS_NS_SYSCTL_OFFSET(mpls
.ip_ttl_propagate
),
2132 .maxlen
= sizeof(int),
2134 .proc_handler
= proc_dointvec_minmax
,
2139 .procname
= "default_ttl",
2140 .data
= MPLS_NS_SYSCTL_OFFSET(mpls
.default_ttl
),
2141 .maxlen
= sizeof(int),
2143 .proc_handler
= proc_dointvec_minmax
,
2150 static int mpls_net_init(struct net
*net
)
2152 struct ctl_table
*table
;
2155 net
->mpls
.platform_labels
= 0;
2156 net
->mpls
.platform_label
= NULL
;
2157 net
->mpls
.ip_ttl_propagate
= 1;
2158 net
->mpls
.default_ttl
= 255;
2160 table
= kmemdup(mpls_table
, sizeof(mpls_table
), GFP_KERNEL
);
2164 /* Table data contains only offsets relative to the base of
2165 * the mdev at this point, so make them absolute.
2167 for (i
= 0; i
< ARRAY_SIZE(mpls_table
) - 1; i
++)
2168 table
[i
].data
= (char *)net
+ (uintptr_t)table
[i
].data
;
2170 net
->mpls
.ctl
= register_net_sysctl(net
, "net/mpls", table
);
2171 if (net
->mpls
.ctl
== NULL
) {
2179 static void mpls_net_exit(struct net
*net
)
2181 struct mpls_route __rcu
**platform_label
;
2182 size_t platform_labels
;
2183 struct ctl_table
*table
;
2186 table
= net
->mpls
.ctl
->ctl_table_arg
;
2187 unregister_net_sysctl_table(net
->mpls
.ctl
);
2190 /* An rcu grace period has passed since there was a device in
2191 * the network namespace (and thus the last in flight packet)
2192 * left this network namespace. This is because
2193 * unregister_netdevice_many and netdev_run_todo has completed
2194 * for each network device that was in this network namespace.
2196 * As such no additional rcu synchronization is necessary when
2197 * freeing the platform_label table.
2200 platform_label
= rtnl_dereference(net
->mpls
.platform_label
);
2201 platform_labels
= net
->mpls
.platform_labels
;
2202 for (index
= 0; index
< platform_labels
; index
++) {
2203 struct mpls_route
*rt
= rtnl_dereference(platform_label
[index
]);
2204 RCU_INIT_POINTER(platform_label
[index
], NULL
);
2205 mpls_notify_route(net
, index
, rt
, NULL
, NULL
);
2210 kvfree(platform_label
);
2213 static struct pernet_operations mpls_net_ops
= {
2214 .init
= mpls_net_init
,
2215 .exit
= mpls_net_exit
,
2218 static struct rtnl_af_ops mpls_af_ops __read_mostly
= {
2220 .fill_stats_af
= mpls_fill_stats_af
,
2221 .get_stats_af_size
= mpls_get_stats_af_size
,
2224 static int __init
mpls_init(void)
2228 BUILD_BUG_ON(sizeof(struct mpls_shim_hdr
) != 4);
2230 err
= register_pernet_subsys(&mpls_net_ops
);
2234 err
= register_netdevice_notifier(&mpls_dev_notifier
);
2236 goto out_unregister_pernet
;
2238 dev_add_pack(&mpls_packet_type
);
2240 rtnl_af_register(&mpls_af_ops
);
2242 rtnl_register(PF_MPLS
, RTM_NEWROUTE
, mpls_rtm_newroute
, NULL
, NULL
);
2243 rtnl_register(PF_MPLS
, RTM_DELROUTE
, mpls_rtm_delroute
, NULL
, NULL
);
2244 rtnl_register(PF_MPLS
, RTM_GETROUTE
, NULL
, mpls_dump_routes
, NULL
);
2245 rtnl_register(PF_MPLS
, RTM_GETNETCONF
, mpls_netconf_get_devconf
,
2246 mpls_netconf_dump_devconf
, NULL
);
2251 out_unregister_pernet
:
2252 unregister_pernet_subsys(&mpls_net_ops
);
2255 module_init(mpls_init
);
2257 static void __exit
mpls_exit(void)
2259 rtnl_unregister_all(PF_MPLS
);
2260 rtnl_af_unregister(&mpls_af_ops
);
2261 dev_remove_pack(&mpls_packet_type
);
2262 unregister_netdevice_notifier(&mpls_dev_notifier
);
2263 unregister_pernet_subsys(&mpls_net_ops
);
2265 module_exit(mpls_exit
);
2267 MODULE_DESCRIPTION("MultiProtocol Label Switching");
2268 MODULE_LICENSE("GPL v2");
2269 MODULE_ALIAS_NETPROTO(PF_MPLS
);