]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - net/ipv4/route.c
projects / mirror_ubuntu-jammy-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'timers-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[mirror_ubuntu-jammy-kernel.git] / net / ipv4 / route.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * ROUTE - implementation of the IP router.
8 *
9 * Authors: Ross Biro
10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Alan Cox, <gw4pts@gw4pts.ampr.org>
12 * Linus Torvalds, <Linus.Torvalds@helsinki.fi>
13 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
14 *
15 * Fixes:
16 * Alan Cox : Verify area fixes.
17 * Alan Cox : cli() protects routing changes
18 * Rui Oliveira : ICMP routing table updates
19 * (rco@di.uminho.pt) Routing table insertion and update
20 * Linus Torvalds : Rewrote bits to be sensible
21 * Alan Cox : Added BSD route gw semantics
22 * Alan Cox : Super /proc >4K
23 * Alan Cox : MTU in route table
24 * Alan Cox : MSS actually. Also added the window
25 * clamper.
26 * Sam Lantinga : Fixed route matching in rt_del()
27 * Alan Cox : Routing cache support.
28 * Alan Cox : Removed compatibility cruft.
29 * Alan Cox : RTF_REJECT support.
30 * Alan Cox : TCP irtt support.
31 * Jonathan Naylor : Added Metric support.
32 * Miquel van Smoorenburg : BSD API fixes.
33 * Miquel van Smoorenburg : Metrics.
34 * Alan Cox : Use __u32 properly
35 * Alan Cox : Aligned routing errors more closely with BSD
36 * our system is still very different.
37 * Alan Cox : Faster /proc handling
38 * Alexey Kuznetsov : Massive rework to support tree based routing,
39 * routing caches and better behaviour.
40 *
41 * Olaf Erb : irtt wasn't being copied right.
42 * Bjorn Ekwall : Kerneld route support.
43 * Alan Cox : Multicast fixed (I hope)
44 * Pavel Krauz : Limited broadcast fixed
45 * Mike McLagan : Routing by source
46 * Alexey Kuznetsov : End of old history. Split to fib.c and
47 * route.c and rewritten from scratch.
48 * Andi Kleen : Load-limit warning messages.
49 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
50 * Vitaly E. Lavrov : Race condition in ip_route_input_slow.
51 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
52 * Vladimir V. Ivanov : IP rule info (flowid) is really useful.
53 * Marc Boucher : routing by fwmark
54 * Robert Olsson : Added rt_cache statistics
55 * Arnaldo C. Melo : Convert proc stuff to seq_file
56 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
57 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
58 * Ilia Sotnikov : Removed TOS from hash calculations
59 */
60
61 #define pr_fmt(fmt) "IPv4: " fmt
62
63 #include <linux/module.h>
64 #include <linux/uaccess.h>
65 #include <linux/bitops.h>
66 #include <linux/types.h>
67 #include <linux/kernel.h>
68 #include <linux/mm.h>
69 #include <linux/string.h>
70 #include <linux/socket.h>
71 #include <linux/sockios.h>
72 #include <linux/errno.h>
73 #include <linux/in.h>
74 #include <linux/inet.h>
75 #include <linux/netdevice.h>
76 #include <linux/proc_fs.h>
77 #include <linux/init.h>
78 #include <linux/skbuff.h>
79 #include <linux/inetdevice.h>
80 #include <linux/igmp.h>
81 #include <linux/pkt_sched.h>
82 #include <linux/mroute.h>
83 #include <linux/netfilter_ipv4.h>
84 #include <linux/random.h>
85 #include <linux/rcupdate.h>
86 #include <linux/times.h>
87 #include <linux/slab.h>
88 #include <linux/jhash.h>
89 #include <net/dst.h>
90 #include <net/dst_metadata.h>
91 #include <net/net_namespace.h>
92 #include <net/protocol.h>
93 #include <net/ip.h>
94 #include <net/route.h>
95 #include <net/inetpeer.h>
96 #include <net/sock.h>
97 #include <net/ip_fib.h>
98 #include <net/arp.h>
99 #include <net/tcp.h>
100 #include <net/icmp.h>
101 #include <net/xfrm.h>
102 #include <net/lwtunnel.h>
103 #include <net/netevent.h>
104 #include <net/rtnetlink.h>
105 #ifdef CONFIG_SYSCTL
106 #include <linux/sysctl.h>
107 #endif
108 #include <net/secure_seq.h>
109 #include <net/ip_tunnels.h>
110 #include <net/l3mdev.h>
111
112 #include "fib_lookup.h"
113
114 #define RT_FL_TOS(oldflp4) \
115 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
116
117 #define RT_GC_TIMEOUT (300*HZ)
118
119 static int ip_rt_max_size;
120 static int ip_rt_redirect_number __read_mostly = 9;
121 static int ip_rt_redirect_load __read_mostly = HZ / 50;
122 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
123 static int ip_rt_error_cost __read_mostly = HZ;
124 static int ip_rt_error_burst __read_mostly = 5 * HZ;
125 static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
126 static u32 ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
127 static int ip_rt_min_advmss __read_mostly = 256;
128
129 static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
130
131 /*
132 * Interface to generic destination cache.
133 */
134
135 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
136 static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
137 static unsigned int ipv4_mtu(const struct dst_entry *dst);
138 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
139 static void ipv4_link_failure(struct sk_buff *skb);
140 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
141 struct sk_buff *skb, u32 mtu);
142 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk,
143 struct sk_buff *skb);
144 static void ipv4_dst_destroy(struct dst_entry *dst);
145
146 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
147 {
148 WARN_ON(1);
149 return NULL;
150 }
151
152 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
153 struct sk_buff *skb,
154 const void *daddr);
155 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
156
157 static struct dst_ops ipv4_dst_ops = {
158 .family = AF_INET,
159 .check = ipv4_dst_check,
160 .default_advmss = ipv4_default_advmss,
161 .mtu = ipv4_mtu,
162 .cow_metrics = ipv4_cow_metrics,
163 .destroy = ipv4_dst_destroy,
164 .negative_advice = ipv4_negative_advice,
165 .link_failure = ipv4_link_failure,
166 .update_pmtu = ip_rt_update_pmtu,
167 .redirect = ip_do_redirect,
168 .local_out = __ip_local_out,
169 .neigh_lookup = ipv4_neigh_lookup,
170 .confirm_neigh = ipv4_confirm_neigh,
171 };
172
173 #define ECN_OR_COST(class) TC_PRIO_##class
174
175 const __u8 ip_tos2prio[16] = {
176 TC_PRIO_BESTEFFORT,
177 ECN_OR_COST(BESTEFFORT),
178 TC_PRIO_BESTEFFORT,
179 ECN_OR_COST(BESTEFFORT),
180 TC_PRIO_BULK,
181 ECN_OR_COST(BULK),
182 TC_PRIO_BULK,
183 ECN_OR_COST(BULK),
184 TC_PRIO_INTERACTIVE,
185 ECN_OR_COST(INTERACTIVE),
186 TC_PRIO_INTERACTIVE,
187 ECN_OR_COST(INTERACTIVE),
188 TC_PRIO_INTERACTIVE_BULK,
189 ECN_OR_COST(INTERACTIVE_BULK),
190 TC_PRIO_INTERACTIVE_BULK,
191 ECN_OR_COST(INTERACTIVE_BULK)
192 };
193 EXPORT_SYMBOL(ip_tos2prio);
194
195 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
196 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
197
198 #ifdef CONFIG_PROC_FS
199 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
200 {
201 if (*pos)
202 return NULL;
203 return SEQ_START_TOKEN;
204 }
205
206 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
207 {
208 ++*pos;
209 return NULL;
210 }
211
212 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
213 {
214 }
215
216 static int rt_cache_seq_show(struct seq_file *seq, void *v)
217 {
218 if (v == SEQ_START_TOKEN)
219 seq_printf(seq, "%-127s\n",
220 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
221 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
222 "HHUptod\tSpecDst");
223 return 0;
224 }
225
226 static const struct seq_operations rt_cache_seq_ops = {
227 .start = rt_cache_seq_start,
228 .next = rt_cache_seq_next,
229 .stop = rt_cache_seq_stop,
230 .show = rt_cache_seq_show,
231 };
232
233 static int rt_cache_seq_open(struct inode *inode, struct file *file)
234 {
235 return seq_open(file, &rt_cache_seq_ops);
236 }
237
238 static const struct file_operations rt_cache_seq_fops = {
239 .open = rt_cache_seq_open,
240 .read = seq_read,
241 .llseek = seq_lseek,
242 .release = seq_release,
243 };
244
245
246 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
247 {
248 int cpu;
249
250 if (*pos == 0)
251 return SEQ_START_TOKEN;
252
253 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
254 if (!cpu_possible(cpu))
255 continue;
256 *pos = cpu+1;
257 return &per_cpu(rt_cache_stat, cpu);
258 }
259 return NULL;
260 }
261
262 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
263 {
264 int cpu;
265
266 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
267 if (!cpu_possible(cpu))
268 continue;
269 *pos = cpu+1;
270 return &per_cpu(rt_cache_stat, cpu);
271 }
272 return NULL;
273
274 }
275
276 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
277 {
278
279 }
280
281 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
282 {
283 struct rt_cache_stat *st = v;
284
285 if (v == SEQ_START_TOKEN) {
286 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
287 return 0;
288 }
289
290 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x "
291 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
292 dst_entries_get_slow(&ipv4_dst_ops),
293 0, /* st->in_hit */
294 st->in_slow_tot,
295 st->in_slow_mc,
296 st->in_no_route,
297 st->in_brd,
298 st->in_martian_dst,
299 st->in_martian_src,
300
301 0, /* st->out_hit */
302 st->out_slow_tot,
303 st->out_slow_mc,
304
305 0, /* st->gc_total */
306 0, /* st->gc_ignored */
307 0, /* st->gc_goal_miss */
308 0, /* st->gc_dst_overflow */
309 0, /* st->in_hlist_search */
310 0 /* st->out_hlist_search */
311 );
312 return 0;
313 }
314
315 static const struct seq_operations rt_cpu_seq_ops = {
316 .start = rt_cpu_seq_start,
317 .next = rt_cpu_seq_next,
318 .stop = rt_cpu_seq_stop,
319 .show = rt_cpu_seq_show,
320 };
321
322
323 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
324 {
325 return seq_open(file, &rt_cpu_seq_ops);
326 }
327
328 static const struct file_operations rt_cpu_seq_fops = {
329 .open = rt_cpu_seq_open,
330 .read = seq_read,
331 .llseek = seq_lseek,
332 .release = seq_release,
333 };
334
335 #ifdef CONFIG_IP_ROUTE_CLASSID
336 static int rt_acct_proc_show(struct seq_file *m, void *v)
337 {
338 struct ip_rt_acct *dst, *src;
339 unsigned int i, j;
340
341 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
342 if (!dst)
343 return -ENOMEM;
344
345 for_each_possible_cpu(i) {
346 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
347 for (j = 0; j < 256; j++) {
348 dst[j].o_bytes += src[j].o_bytes;
349 dst[j].o_packets += src[j].o_packets;
350 dst[j].i_bytes += src[j].i_bytes;
351 dst[j].i_packets += src[j].i_packets;
352 }
353 }
354
355 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
356 kfree(dst);
357 return 0;
358 }
359 #endif
360
361 static int __net_init ip_rt_do_proc_init(struct net *net)
362 {
363 struct proc_dir_entry *pde;
364
365 pde = proc_create("rt_cache", 0444, net->proc_net,
366 &rt_cache_seq_fops);
367 if (!pde)
368 goto err1;
369
370 pde = proc_create("rt_cache", 0444,
371 net->proc_net_stat, &rt_cpu_seq_fops);
372 if (!pde)
373 goto err2;
374
375 #ifdef CONFIG_IP_ROUTE_CLASSID
376 pde = proc_create_single("rt_acct", 0, net->proc_net,
377 rt_acct_proc_show);
378 if (!pde)
379 goto err3;
380 #endif
381 return 0;
382
383 #ifdef CONFIG_IP_ROUTE_CLASSID
384 err3:
385 remove_proc_entry("rt_cache", net->proc_net_stat);
386 #endif
387 err2:
388 remove_proc_entry("rt_cache", net->proc_net);
389 err1:
390 return -ENOMEM;
391 }
392
393 static void __net_exit ip_rt_do_proc_exit(struct net *net)
394 {
395 remove_proc_entry("rt_cache", net->proc_net_stat);
396 remove_proc_entry("rt_cache", net->proc_net);
397 #ifdef CONFIG_IP_ROUTE_CLASSID
398 remove_proc_entry("rt_acct", net->proc_net);
399 #endif
400 }
401
402 static struct pernet_operations ip_rt_proc_ops __net_initdata = {
403 .init = ip_rt_do_proc_init,
404 .exit = ip_rt_do_proc_exit,
405 };
406
407 static int __init ip_rt_proc_init(void)
408 {
409 return register_pernet_subsys(&ip_rt_proc_ops);
410 }
411
412 #else
413 static inline int ip_rt_proc_init(void)
414 {
415 return 0;
416 }
417 #endif /* CONFIG_PROC_FS */
418
419 static inline bool rt_is_expired(const struct rtable *rth)
420 {
421 return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
422 }
423
424 void rt_cache_flush(struct net *net)
425 {
426 rt_genid_bump_ipv4(net);
427 }
428
429 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
430 struct sk_buff *skb,
431 const void *daddr)
432 {
433 const struct rtable *rt = container_of(dst, struct rtable, dst);
434 struct net_device *dev = dst->dev;
435 struct neighbour *n;
436
437 rcu_read_lock_bh();
438
439 if (likely(rt->rt_gw_family == AF_INET)) {
440 n = ip_neigh_gw4(dev, rt->rt_gw4);
441 } else if (rt->rt_gw_family == AF_INET6) {
442 n = ip_neigh_gw6(dev, &rt->rt_gw6);
443 } else {
444 __be32 pkey;
445
446 pkey = skb ? ip_hdr(skb)->daddr : *((__be32 *) daddr);
447 n = ip_neigh_gw4(dev, pkey);
448 }
449
450 if (n && !refcount_inc_not_zero(&n->refcnt))
451 n = NULL;
452
453 rcu_read_unlock_bh();
454
455 return n;
456 }
457
458 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
459 {
460 const struct rtable *rt = container_of(dst, struct rtable, dst);
461 struct net_device *dev = dst->dev;
462 const __be32 *pkey = daddr;
463
464 if (rt->rt_gw_family == AF_INET) {
465 pkey = (const __be32 *)&rt->rt_gw4;
466 } else if (rt->rt_gw_family == AF_INET6) {
467 return __ipv6_confirm_neigh_stub(dev, &rt->rt_gw6);
468 } else if (!daddr ||
469 (rt->rt_flags &
470 (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) {
471 return;
472 }
473 __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
474 }
475
476 #define IP_IDENTS_SZ 2048u
477
478 static atomic_t *ip_idents __read_mostly;
479 static u32 *ip_tstamps __read_mostly;
480
481 /* In order to protect privacy, we add a perturbation to identifiers
482 * if one generator is seldom used. This makes hard for an attacker
483 * to infer how many packets were sent between two points in time.
484 */
485 u32 ip_idents_reserve(u32 hash, int segs)
486 {
487 u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
488 atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
489 u32 old = READ_ONCE(*p_tstamp);
490 u32 now = (u32)jiffies;
491 u32 new, delta = 0;
492
493 if (old != now && cmpxchg(p_tstamp, old, now) == old)
494 delta = prandom_u32_max(now - old);
495
496 /* Do not use atomic_add_return() as it makes UBSAN unhappy */
497 do {
498 old = (u32)atomic_read(p_id);
499 new = old + delta + segs;
500 } while (atomic_cmpxchg(p_id, old, new) != old);
501
502 return new - segs;
503 }
504 EXPORT_SYMBOL(ip_idents_reserve);
505
506 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
507 {
508 u32 hash, id;
509
510 /* Note the following code is not safe, but this is okay. */
511 if (unlikely(siphash_key_is_zero(&net->ipv4.ip_id_key)))
512 get_random_bytes(&net->ipv4.ip_id_key,
513 sizeof(net->ipv4.ip_id_key));
514
515 hash = siphash_3u32((__force u32)iph->daddr,
516 (__force u32)iph->saddr,
517 iph->protocol,
518 &net->ipv4.ip_id_key);
519 id = ip_idents_reserve(hash, segs);
520 iph->id = htons(id);
521 }
522 EXPORT_SYMBOL(__ip_select_ident);
523
524 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
525 const struct sock *sk,
526 const struct iphdr *iph,
527 int oif, u8 tos,
528 u8 prot, u32 mark, int flow_flags)
529 {
530 if (sk) {
531 const struct inet_sock *inet = inet_sk(sk);
532
533 oif = sk->sk_bound_dev_if;
534 mark = sk->sk_mark;
535 tos = RT_CONN_FLAGS(sk);
536 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
537 }
538 flowi4_init_output(fl4, oif, mark, tos,
539 RT_SCOPE_UNIVERSE, prot,
540 flow_flags,
541 iph->daddr, iph->saddr, 0, 0,
542 sock_net_uid(net, sk));
543 }
544
545 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
546 const struct sock *sk)
547 {
548 const struct net *net = dev_net(skb->dev);
549 const struct iphdr *iph = ip_hdr(skb);
550 int oif = skb->dev->ifindex;
551 u8 tos = RT_TOS(iph->tos);
552 u8 prot = iph->protocol;
553 u32 mark = skb->mark;
554
555 __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
556 }
557
558 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
559 {
560 const struct inet_sock *inet = inet_sk(sk);
561 const struct ip_options_rcu *inet_opt;
562 __be32 daddr = inet->inet_daddr;
563
564 rcu_read_lock();
565 inet_opt = rcu_dereference(inet->inet_opt);
566 if (inet_opt && inet_opt->opt.srr)
567 daddr = inet_opt->opt.faddr;
568 flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
569 RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
570 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
571 inet_sk_flowi_flags(sk),
572 daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
573 rcu_read_unlock();
574 }
575
576 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
577 const struct sk_buff *skb)
578 {
579 if (skb)
580 build_skb_flow_key(fl4, skb, sk);
581 else
582 build_sk_flow_key(fl4, sk);
583 }
584
585 static DEFINE_SPINLOCK(fnhe_lock);
586
587 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
588 {
589 struct rtable *rt;
590
591 rt = rcu_dereference(fnhe->fnhe_rth_input);
592 if (rt) {
593 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
594 dst_dev_put(&rt->dst);
595 dst_release(&rt->dst);
596 }
597 rt = rcu_dereference(fnhe->fnhe_rth_output);
598 if (rt) {
599 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
600 dst_dev_put(&rt->dst);
601 dst_release(&rt->dst);
602 }
603 }
604
605 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
606 {
607 struct fib_nh_exception *fnhe, *oldest;
608
609 oldest = rcu_dereference(hash->chain);
610 for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
611 fnhe = rcu_dereference(fnhe->fnhe_next)) {
612 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
613 oldest = fnhe;
614 }
615 fnhe_flush_routes(oldest);
616 return oldest;
617 }
618
619 static inline u32 fnhe_hashfun(__be32 daddr)
620 {
621 static u32 fnhe_hashrnd __read_mostly;
622 u32 hval;
623
624 net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
625 hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
626 return hash_32(hval, FNHE_HASH_SHIFT);
627 }
628
629 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
630 {
631 rt->rt_pmtu = fnhe->fnhe_pmtu;
632 rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
633 rt->dst.expires = fnhe->fnhe_expires;
634
635 if (fnhe->fnhe_gw) {
636 rt->rt_flags |= RTCF_REDIRECTED;
637 rt->rt_gw_family = AF_INET;
638 rt->rt_gw4 = fnhe->fnhe_gw;
639 }
640 }
641
642 static void update_or_create_fnhe(struct fib_nh_common *nhc, __be32 daddr,
643 __be32 gw, u32 pmtu, bool lock,
644 unsigned long expires)
645 {
646 struct fnhe_hash_bucket *hash;
647 struct fib_nh_exception *fnhe;
648 struct rtable *rt;
649 u32 genid, hval;
650 unsigned int i;
651 int depth;
652
653 genid = fnhe_genid(dev_net(nhc->nhc_dev));
654 hval = fnhe_hashfun(daddr);
655
656 spin_lock_bh(&fnhe_lock);
657
658 hash = rcu_dereference(nhc->nhc_exceptions);
659 if (!hash) {
660 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
661 if (!hash)
662 goto out_unlock;
663 rcu_assign_pointer(nhc->nhc_exceptions, hash);
664 }
665
666 hash += hval;
667
668 depth = 0;
669 for (fnhe = rcu_dereference(hash->chain); fnhe;
670 fnhe = rcu_dereference(fnhe->fnhe_next)) {
671 if (fnhe->fnhe_daddr == daddr)
672 break;
673 depth++;
674 }
675
676 if (fnhe) {
677 if (fnhe->fnhe_genid != genid)
678 fnhe->fnhe_genid = genid;
679 if (gw)
680 fnhe->fnhe_gw = gw;
681 if (pmtu) {
682 fnhe->fnhe_pmtu = pmtu;
683 fnhe->fnhe_mtu_locked = lock;
684 }
685 fnhe->fnhe_expires = max(1UL, expires);
686 /* Update all cached dsts too */
687 rt = rcu_dereference(fnhe->fnhe_rth_input);
688 if (rt)
689 fill_route_from_fnhe(rt, fnhe);
690 rt = rcu_dereference(fnhe->fnhe_rth_output);
691 if (rt)
692 fill_route_from_fnhe(rt, fnhe);
693 } else {
694 if (depth > FNHE_RECLAIM_DEPTH)
695 fnhe = fnhe_oldest(hash);
696 else {
697 fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
698 if (!fnhe)
699 goto out_unlock;
700
701 fnhe->fnhe_next = hash->chain;
702 rcu_assign_pointer(hash->chain, fnhe);
703 }
704 fnhe->fnhe_genid = genid;
705 fnhe->fnhe_daddr = daddr;
706 fnhe->fnhe_gw = gw;
707 fnhe->fnhe_pmtu = pmtu;
708 fnhe->fnhe_mtu_locked = lock;
709 fnhe->fnhe_expires = max(1UL, expires);
710
711 /* Exception created; mark the cached routes for the nexthop
712 * stale, so anyone caching it rechecks if this exception
713 * applies to them.
714 */
715 rt = rcu_dereference(nhc->nhc_rth_input);
716 if (rt)
717 rt->dst.obsolete = DST_OBSOLETE_KILL;
718
719 for_each_possible_cpu(i) {
720 struct rtable __rcu **prt;
721 prt = per_cpu_ptr(nhc->nhc_pcpu_rth_output, i);
722 rt = rcu_dereference(*prt);
723 if (rt)
724 rt->dst.obsolete = DST_OBSOLETE_KILL;
725 }
726 }
727
728 fnhe->fnhe_stamp = jiffies;
729
730 out_unlock:
731 spin_unlock_bh(&fnhe_lock);
732 }
733
734 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
735 bool kill_route)
736 {
737 __be32 new_gw = icmp_hdr(skb)->un.gateway;
738 __be32 old_gw = ip_hdr(skb)->saddr;
739 struct net_device *dev = skb->dev;
740 struct in_device *in_dev;
741 struct fib_result res;
742 struct neighbour *n;
743 struct net *net;
744
745 switch (icmp_hdr(skb)->code & 7) {
746 case ICMP_REDIR_NET:
747 case ICMP_REDIR_NETTOS:
748 case ICMP_REDIR_HOST:
749 case ICMP_REDIR_HOSTTOS:
750 break;
751
752 default:
753 return;
754 }
755
756 if (rt->rt_gw_family != AF_INET || rt->rt_gw4 != old_gw)
757 return;
758
759 in_dev = __in_dev_get_rcu(dev);
760 if (!in_dev)
761 return;
762
763 net = dev_net(dev);
764 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
765 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
766 ipv4_is_zeronet(new_gw))
767 goto reject_redirect;
768
769 if (!IN_DEV_SHARED_MEDIA(in_dev)) {
770 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
771 goto reject_redirect;
772 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
773 goto reject_redirect;
774 } else {
775 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
776 goto reject_redirect;
777 }
778
779 n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
780 if (!n)
781 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
782 if (!IS_ERR(n)) {
783 if (!(n->nud_state & NUD_VALID)) {
784 neigh_event_send(n, NULL);
785 } else {
786 if (fib_lookup(net, fl4, &res, 0) == 0) {
787 struct fib_nh_common *nhc = FIB_RES_NHC(res);
788
789 update_or_create_fnhe(nhc, fl4->daddr, new_gw,
790 0, false,
791 jiffies + ip_rt_gc_timeout);
792 }
793 if (kill_route)
794 rt->dst.obsolete = DST_OBSOLETE_KILL;
795 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
796 }
797 neigh_release(n);
798 }
799 return;
800
801 reject_redirect:
802 #ifdef CONFIG_IP_ROUTE_VERBOSE
803 if (IN_DEV_LOG_MARTIANS(in_dev)) {
804 const struct iphdr *iph = (const struct iphdr *) skb->data;
805 __be32 daddr = iph->daddr;
806 __be32 saddr = iph->saddr;
807
808 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
809 " Advised path = %pI4 -> %pI4\n",
810 &old_gw, dev->name, &new_gw,
811 &saddr, &daddr);
812 }
813 #endif
814 ;
815 }
816
817 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
818 {
819 struct rtable *rt;
820 struct flowi4 fl4;
821 const struct iphdr *iph = (const struct iphdr *) skb->data;
822 struct net *net = dev_net(skb->dev);
823 int oif = skb->dev->ifindex;
824 u8 tos = RT_TOS(iph->tos);
825 u8 prot = iph->protocol;
826 u32 mark = skb->mark;
827
828 rt = (struct rtable *) dst;
829
830 __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
831 __ip_do_redirect(rt, skb, &fl4, true);
832 }
833
834 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
835 {
836 struct rtable *rt = (struct rtable *)dst;
837 struct dst_entry *ret = dst;
838
839 if (rt) {
840 if (dst->obsolete > 0) {
841 ip_rt_put(rt);
842 ret = NULL;
843 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
844 rt->dst.expires) {
845 ip_rt_put(rt);
846 ret = NULL;
847 }
848 }
849 return ret;
850 }
851
852 /*
853 * Algorithm:
854 * 1. The first ip_rt_redirect_number redirects are sent
855 * with exponential backoff, then we stop sending them at all,
856 * assuming that the host ignores our redirects.
857 * 2. If we did not see packets requiring redirects
858 * during ip_rt_redirect_silence, we assume that the host
859 * forgot redirected route and start to send redirects again.
860 *
861 * This algorithm is much cheaper and more intelligent than dumb load limiting
862 * in icmp.c.
863 *
864 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
865 * and "frag. need" (breaks PMTU discovery) in icmp.c.
866 */
867
868 void ip_rt_send_redirect(struct sk_buff *skb)
869 {
870 struct rtable *rt = skb_rtable(skb);
871 struct in_device *in_dev;
872 struct inet_peer *peer;
873 struct net *net;
874 int log_martians;
875 int vif;
876
877 rcu_read_lock();
878 in_dev = __in_dev_get_rcu(rt->dst.dev);
879 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
880 rcu_read_unlock();
881 return;
882 }
883 log_martians = IN_DEV_LOG_MARTIANS(in_dev);
884 vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
885 rcu_read_unlock();
886
887 net = dev_net(rt->dst.dev);
888 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
889 if (!peer) {
890 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
891 rt_nexthop(rt, ip_hdr(skb)->daddr));
892 return;
893 }
894
895 /* No redirected packets during ip_rt_redirect_silence;
896 * reset the algorithm.
897 */
898 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
899 peer->rate_tokens = 0;
900 peer->n_redirects = 0;
901 }
902
903 /* Too many ignored redirects; do not send anything
904 * set dst.rate_last to the last seen redirected packet.
905 */
906 if (peer->n_redirects >= ip_rt_redirect_number) {
907 peer->rate_last = jiffies;
908 goto out_put_peer;
909 }
910
911 /* Check for load limit; set rate_last to the latest sent
912 * redirect.
913 */
914 if (peer->rate_tokens == 0 ||
915 time_after(jiffies,
916 (peer->rate_last +
917 (ip_rt_redirect_load << peer->rate_tokens)))) {
918 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
919
920 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
921 peer->rate_last = jiffies;
922 ++peer->rate_tokens;
923 ++peer->n_redirects;
924 #ifdef CONFIG_IP_ROUTE_VERBOSE
925 if (log_martians &&
926 peer->rate_tokens == ip_rt_redirect_number)
927 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
928 &ip_hdr(skb)->saddr, inet_iif(skb),
929 &ip_hdr(skb)->daddr, &gw);
930 #endif
931 }
932 out_put_peer:
933 inet_putpeer(peer);
934 }
935
936 static int ip_error(struct sk_buff *skb)
937 {
938 struct rtable *rt = skb_rtable(skb);
939 struct net_device *dev = skb->dev;
940 struct in_device *in_dev;
941 struct inet_peer *peer;
942 unsigned long now;
943 struct net *net;
944 bool send;
945 int code;
946
947 if (netif_is_l3_master(skb->dev)) {
948 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
949 if (!dev)
950 goto out;
951 }
952
953 in_dev = __in_dev_get_rcu(dev);
954
955 /* IP on this device is disabled. */
956 if (!in_dev)
957 goto out;
958
959 net = dev_net(rt->dst.dev);
960 if (!IN_DEV_FORWARD(in_dev)) {
961 switch (rt->dst.error) {
962 case EHOSTUNREACH:
963 __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
964 break;
965
966 case ENETUNREACH:
967 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
968 break;
969 }
970 goto out;
971 }
972
973 switch (rt->dst.error) {
974 case EINVAL:
975 default:
976 goto out;
977 case EHOSTUNREACH:
978 code = ICMP_HOST_UNREACH;
979 break;
980 case ENETUNREACH:
981 code = ICMP_NET_UNREACH;
982 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
983 break;
984 case EACCES:
985 code = ICMP_PKT_FILTERED;
986 break;
987 }
988
989 peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
990 l3mdev_master_ifindex(skb->dev), 1);
991
992 send = true;
993 if (peer) {
994 now = jiffies;
995 peer->rate_tokens += now - peer->rate_last;
996 if (peer->rate_tokens > ip_rt_error_burst)
997 peer->rate_tokens = ip_rt_error_burst;
998 peer->rate_last = now;
999 if (peer->rate_tokens >= ip_rt_error_cost)
1000 peer->rate_tokens -= ip_rt_error_cost;
1001 else
1002 send = false;
1003 inet_putpeer(peer);
1004 }
1005 if (send)
1006 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1007
1008 out: kfree_skb(skb);
1009 return 0;
1010 }
1011
1012 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1013 {
1014 struct dst_entry *dst = &rt->dst;
1015 u32 old_mtu = ipv4_mtu(dst);
1016 struct fib_result res;
1017 bool lock = false;
1018
1019 if (ip_mtu_locked(dst))
1020 return;
1021
1022 if (old_mtu < mtu)
1023 return;
1024
1025 if (mtu < ip_rt_min_pmtu) {
1026 lock = true;
1027 mtu = min(old_mtu, ip_rt_min_pmtu);
1028 }
1029
1030 if (rt->rt_pmtu == mtu && !lock &&
1031 time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1032 return;
1033
1034 rcu_read_lock();
1035 if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1036 struct fib_nh_common *nhc = FIB_RES_NHC(res);
1037
1038 update_or_create_fnhe(nhc, fl4->daddr, 0, mtu, lock,
1039 jiffies + ip_rt_mtu_expires);
1040 }
1041 rcu_read_unlock();
1042 }
1043
1044 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1045 struct sk_buff *skb, u32 mtu)
1046 {
1047 struct rtable *rt = (struct rtable *) dst;
1048 struct flowi4 fl4;
1049
1050 ip_rt_build_flow_key(&fl4, sk, skb);
1051 __ip_rt_update_pmtu(rt, &fl4, mtu);
1052 }
1053
1054 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1055 int oif, u8 protocol)
1056 {
1057 const struct iphdr *iph = (const struct iphdr *) skb->data;
1058 struct flowi4 fl4;
1059 struct rtable *rt;
1060 u32 mark = IP4_REPLY_MARK(net, skb->mark);
1061
1062 __build_flow_key(net, &fl4, NULL, iph, oif,
1063 RT_TOS(iph->tos), protocol, mark, 0);
1064 rt = __ip_route_output_key(net, &fl4);
1065 if (!IS_ERR(rt)) {
1066 __ip_rt_update_pmtu(rt, &fl4, mtu);
1067 ip_rt_put(rt);
1068 }
1069 }
1070 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1071
1072 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1073 {
1074 const struct iphdr *iph = (const struct iphdr *) skb->data;
1075 struct flowi4 fl4;
1076 struct rtable *rt;
1077
1078 __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1079
1080 if (!fl4.flowi4_mark)
1081 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1082
1083 rt = __ip_route_output_key(sock_net(sk), &fl4);
1084 if (!IS_ERR(rt)) {
1085 __ip_rt_update_pmtu(rt, &fl4, mtu);
1086 ip_rt_put(rt);
1087 }
1088 }
1089
1090 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1091 {
1092 const struct iphdr *iph = (const struct iphdr *) skb->data;
1093 struct flowi4 fl4;
1094 struct rtable *rt;
1095 struct dst_entry *odst = NULL;
1096 bool new = false;
1097 struct net *net = sock_net(sk);
1098
1099 bh_lock_sock(sk);
1100
1101 if (!ip_sk_accept_pmtu(sk))
1102 goto out;
1103
1104 odst = sk_dst_get(sk);
1105
1106 if (sock_owned_by_user(sk) || !odst) {
1107 __ipv4_sk_update_pmtu(skb, sk, mtu);
1108 goto out;
1109 }
1110
1111 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1112
1113 rt = (struct rtable *)odst;
1114 if (odst->obsolete && !odst->ops->check(odst, 0)) {
1115 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1116 if (IS_ERR(rt))
1117 goto out;
1118
1119 new = true;
1120 }
1121
1122 __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1123
1124 if (!dst_check(&rt->dst, 0)) {
1125 if (new)
1126 dst_release(&rt->dst);
1127
1128 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1129 if (IS_ERR(rt))
1130 goto out;
1131
1132 new = true;
1133 }
1134
1135 if (new)
1136 sk_dst_set(sk, &rt->dst);
1137
1138 out:
1139 bh_unlock_sock(sk);
1140 dst_release(odst);
1141 }
1142 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1143
1144 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1145 int oif, u8 protocol)
1146 {
1147 const struct iphdr *iph = (const struct iphdr *) skb->data;
1148 struct flowi4 fl4;
1149 struct rtable *rt;
1150
1151 __build_flow_key(net, &fl4, NULL, iph, oif,
1152 RT_TOS(iph->tos), protocol, 0, 0);
1153 rt = __ip_route_output_key(net, &fl4);
1154 if (!IS_ERR(rt)) {
1155 __ip_do_redirect(rt, skb, &fl4, false);
1156 ip_rt_put(rt);
1157 }
1158 }
1159 EXPORT_SYMBOL_GPL(ipv4_redirect);
1160
1161 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1162 {
1163 const struct iphdr *iph = (const struct iphdr *) skb->data;
1164 struct flowi4 fl4;
1165 struct rtable *rt;
1166 struct net *net = sock_net(sk);
1167
1168 __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1169 rt = __ip_route_output_key(net, &fl4);
1170 if (!IS_ERR(rt)) {
1171 __ip_do_redirect(rt, skb, &fl4, false);
1172 ip_rt_put(rt);
1173 }
1174 }
1175 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1176
1177 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1178 {
1179 struct rtable *rt = (struct rtable *) dst;
1180
1181 /* All IPV4 dsts are created with ->obsolete set to the value
1182 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1183 * into this function always.
1184 *
1185 * When a PMTU/redirect information update invalidates a route,
1186 * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1187 * DST_OBSOLETE_DEAD.
1188 */
1189 if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1190 return NULL;
1191 return dst;
1192 }
1193
1194 static void ipv4_send_dest_unreach(struct sk_buff *skb)
1195 {
1196 struct ip_options opt;
1197 int res;
1198
1199 /* Recompile ip options since IPCB may not be valid anymore.
1200 * Also check we have a reasonable ipv4 header.
1201 */
1202 if (!pskb_network_may_pull(skb, sizeof(struct iphdr)) ||
1203 ip_hdr(skb)->version != 4 || ip_hdr(skb)->ihl < 5)
1204 return;
1205
1206 memset(&opt, 0, sizeof(opt));
1207 if (ip_hdr(skb)->ihl > 5) {
1208 if (!pskb_network_may_pull(skb, ip_hdr(skb)->ihl * 4))
1209 return;
1210 opt.optlen = ip_hdr(skb)->ihl * 4 - sizeof(struct iphdr);
1211
1212 rcu_read_lock();
1213 res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL);
1214 rcu_read_unlock();
1215
1216 if (res)
1217 return;
1218 }
1219 __icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt);
1220 }
1221
1222 static void ipv4_link_failure(struct sk_buff *skb)
1223 {
1224 struct rtable *rt;
1225
1226 ipv4_send_dest_unreach(skb);
1227
1228 rt = skb_rtable(skb);
1229 if (rt)
1230 dst_set_expires(&rt->dst, 0);
1231 }
1232
1233 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1234 {
1235 pr_debug("%s: %pI4 -> %pI4, %s\n",
1236 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1237 skb->dev ? skb->dev->name : "?");
1238 kfree_skb(skb);
1239 WARN_ON(1);
1240 return 0;
1241 }
1242
1243 /*
1244 We do not cache source address of outgoing interface,
1245 because it is used only by IP RR, TS and SRR options,
1246 so that it out of fast path.
1247
1248 BTW remember: "addr" is allowed to be not aligned
1249 in IP options!
1250 */
1251
1252 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1253 {
1254 __be32 src;
1255
1256 if (rt_is_output_route(rt))
1257 src = ip_hdr(skb)->saddr;
1258 else {
1259 struct fib_result res;
1260 struct iphdr *iph = ip_hdr(skb);
1261 struct flowi4 fl4 = {
1262 .daddr = iph->daddr,
1263 .saddr = iph->saddr,
1264 .flowi4_tos = RT_TOS(iph->tos),
1265 .flowi4_oif = rt->dst.dev->ifindex,
1266 .flowi4_iif = skb->dev->ifindex,
1267 .flowi4_mark = skb->mark,
1268 };
1269
1270 rcu_read_lock();
1271 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1272 src = fib_result_prefsrc(dev_net(rt->dst.dev), &res);
1273 else
1274 src = inet_select_addr(rt->dst.dev,
1275 rt_nexthop(rt, iph->daddr),
1276 RT_SCOPE_UNIVERSE);
1277 rcu_read_unlock();
1278 }
1279 memcpy(addr, &src, 4);
1280 }
1281
1282 #ifdef CONFIG_IP_ROUTE_CLASSID
1283 static void set_class_tag(struct rtable *rt, u32 tag)
1284 {
1285 if (!(rt->dst.tclassid & 0xFFFF))
1286 rt->dst.tclassid |= tag & 0xFFFF;
1287 if (!(rt->dst.tclassid & 0xFFFF0000))
1288 rt->dst.tclassid |= tag & 0xFFFF0000;
1289 }
1290 #endif
1291
1292 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1293 {
1294 unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1295 unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1296 ip_rt_min_advmss);
1297
1298 return min(advmss, IPV4_MAX_PMTU - header_size);
1299 }
1300
1301 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1302 {
1303 const struct rtable *rt = (const struct rtable *) dst;
1304 unsigned int mtu = rt->rt_pmtu;
1305
1306 if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1307 mtu = dst_metric_raw(dst, RTAX_MTU);
1308
1309 if (mtu)
1310 return mtu;
1311
1312 mtu = READ_ONCE(dst->dev->mtu);
1313
1314 if (unlikely(ip_mtu_locked(dst))) {
1315 if (rt->rt_gw_family && mtu > 576)
1316 mtu = 576;
1317 }
1318
1319 mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1320
1321 return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1322 }
1323
1324 static void ip_del_fnhe(struct fib_nh_common *nhc, __be32 daddr)
1325 {
1326 struct fnhe_hash_bucket *hash;
1327 struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1328 u32 hval = fnhe_hashfun(daddr);
1329
1330 spin_lock_bh(&fnhe_lock);
1331
1332 hash = rcu_dereference_protected(nhc->nhc_exceptions,
1333 lockdep_is_held(&fnhe_lock));
1334 hash += hval;
1335
1336 fnhe_p = &hash->chain;
1337 fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1338 while (fnhe) {
1339 if (fnhe->fnhe_daddr == daddr) {
1340 rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1341 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1342 /* set fnhe_daddr to 0 to ensure it won't bind with
1343 * new dsts in rt_bind_exception().
1344 */
1345 fnhe->fnhe_daddr = 0;
1346 fnhe_flush_routes(fnhe);
1347 kfree_rcu(fnhe, rcu);
1348 break;
1349 }
1350 fnhe_p = &fnhe->fnhe_next;
1351 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1352 lockdep_is_held(&fnhe_lock));
1353 }
1354
1355 spin_unlock_bh(&fnhe_lock);
1356 }
1357
1358 static struct fib_nh_exception *find_exception(struct fib_nh_common *nhc,
1359 __be32 daddr)
1360 {
1361 struct fnhe_hash_bucket *hash = rcu_dereference(nhc->nhc_exceptions);
1362 struct fib_nh_exception *fnhe;
1363 u32 hval;
1364
1365 if (!hash)
1366 return NULL;
1367
1368 hval = fnhe_hashfun(daddr);
1369
1370 for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1371 fnhe = rcu_dereference(fnhe->fnhe_next)) {
1372 if (fnhe->fnhe_daddr == daddr) {
1373 if (fnhe->fnhe_expires &&
1374 time_after(jiffies, fnhe->fnhe_expires)) {
1375 ip_del_fnhe(nhc, daddr);
1376 break;
1377 }
1378 return fnhe;
1379 }
1380 }
1381 return NULL;
1382 }
1383
1384 /* MTU selection:
1385 * 1. mtu on route is locked - use it
1386 * 2. mtu from nexthop exception
1387 * 3. mtu from egress device
1388 */
1389
1390 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1391 {
1392 struct fib_nh_common *nhc = res->nhc;
1393 struct net_device *dev = nhc->nhc_dev;
1394 struct fib_info *fi = res->fi;
1395 u32 mtu = 0;
1396
1397 if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1398 fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1399 mtu = fi->fib_mtu;
1400
1401 if (likely(!mtu)) {
1402 struct fib_nh_exception *fnhe;
1403
1404 fnhe = find_exception(nhc, daddr);
1405 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1406 mtu = fnhe->fnhe_pmtu;
1407 }
1408
1409 if (likely(!mtu))
1410 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1411
1412 return mtu - lwtunnel_headroom(nhc->nhc_lwtstate, mtu);
1413 }
1414
1415 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1416 __be32 daddr, const bool do_cache)
1417 {
1418 bool ret = false;
1419
1420 spin_lock_bh(&fnhe_lock);
1421
1422 if (daddr == fnhe->fnhe_daddr) {
1423 struct rtable __rcu **porig;
1424 struct rtable *orig;
1425 int genid = fnhe_genid(dev_net(rt->dst.dev));
1426
1427 if (rt_is_input_route(rt))
1428 porig = &fnhe->fnhe_rth_input;
1429 else
1430 porig = &fnhe->fnhe_rth_output;
1431 orig = rcu_dereference(*porig);
1432
1433 if (fnhe->fnhe_genid != genid) {
1434 fnhe->fnhe_genid = genid;
1435 fnhe->fnhe_gw = 0;
1436 fnhe->fnhe_pmtu = 0;
1437 fnhe->fnhe_expires = 0;
1438 fnhe->fnhe_mtu_locked = false;
1439 fnhe_flush_routes(fnhe);
1440 orig = NULL;
1441 }
1442 fill_route_from_fnhe(rt, fnhe);
1443 if (!rt->rt_gw4) {
1444 rt->rt_gw4 = daddr;
1445 rt->rt_gw_family = AF_INET;
1446 }
1447
1448 if (do_cache) {
1449 dst_hold(&rt->dst);
1450 rcu_assign_pointer(*porig, rt);
1451 if (orig) {
1452 dst_dev_put(&orig->dst);
1453 dst_release(&orig->dst);
1454 }
1455 ret = true;
1456 }
1457
1458 fnhe->fnhe_stamp = jiffies;
1459 }
1460 spin_unlock_bh(&fnhe_lock);
1461
1462 return ret;
1463 }
1464
1465 static bool rt_cache_route(struct fib_nh_common *nhc, struct rtable *rt)
1466 {
1467 struct rtable *orig, *prev, **p;
1468 bool ret = true;
1469
1470 if (rt_is_input_route(rt)) {
1471 p = (struct rtable **)&nhc->nhc_rth_input;
1472 } else {
1473 p = (struct rtable **)raw_cpu_ptr(nhc->nhc_pcpu_rth_output);
1474 }
1475 orig = *p;
1476
1477 /* hold dst before doing cmpxchg() to avoid race condition
1478 * on this dst
1479 */
1480 dst_hold(&rt->dst);
1481 prev = cmpxchg(p, orig, rt);
1482 if (prev == orig) {
1483 if (orig) {
1484 dst_dev_put(&orig->dst);
1485 dst_release(&orig->dst);
1486 }
1487 } else {
1488 dst_release(&rt->dst);
1489 ret = false;
1490 }
1491
1492 return ret;
1493 }
1494
1495 struct uncached_list {
1496 spinlock_t lock;
1497 struct list_head head;
1498 };
1499
1500 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1501
1502 void rt_add_uncached_list(struct rtable *rt)
1503 {
1504 struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1505
1506 rt->rt_uncached_list = ul;
1507
1508 spin_lock_bh(&ul->lock);
1509 list_add_tail(&rt->rt_uncached, &ul->head);
1510 spin_unlock_bh(&ul->lock);
1511 }
1512
1513 void rt_del_uncached_list(struct rtable *rt)
1514 {
1515 if (!list_empty(&rt->rt_uncached)) {
1516 struct uncached_list *ul = rt->rt_uncached_list;
1517
1518 spin_lock_bh(&ul->lock);
1519 list_del(&rt->rt_uncached);
1520 spin_unlock_bh(&ul->lock);
1521 }
1522 }
1523
1524 static void ipv4_dst_destroy(struct dst_entry *dst)
1525 {
1526 struct rtable *rt = (struct rtable *)dst;
1527
1528 ip_dst_metrics_put(dst);
1529 rt_del_uncached_list(rt);
1530 }
1531
1532 void rt_flush_dev(struct net_device *dev)
1533 {
1534 struct net *net = dev_net(dev);
1535 struct rtable *rt;
1536 int cpu;
1537
1538 for_each_possible_cpu(cpu) {
1539 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1540
1541 spin_lock_bh(&ul->lock);
1542 list_for_each_entry(rt, &ul->head, rt_uncached) {
1543 if (rt->dst.dev != dev)
1544 continue;
1545 rt->dst.dev = net->loopback_dev;
1546 dev_hold(rt->dst.dev);
1547 dev_put(dev);
1548 }
1549 spin_unlock_bh(&ul->lock);
1550 }
1551 }
1552
1553 static bool rt_cache_valid(const struct rtable *rt)
1554 {
1555 return rt &&
1556 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1557 !rt_is_expired(rt);
1558 }
1559
1560 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1561 const struct fib_result *res,
1562 struct fib_nh_exception *fnhe,
1563 struct fib_info *fi, u16 type, u32 itag,
1564 const bool do_cache)
1565 {
1566 bool cached = false;
1567
1568 if (fi) {
1569 struct fib_nh_common *nhc = FIB_RES_NHC(*res);
1570
1571 if (nhc->nhc_gw_family && nhc->nhc_scope == RT_SCOPE_LINK) {
1572 rt->rt_gw_family = nhc->nhc_gw_family;
1573 /* only INET and INET6 are supported */
1574 if (likely(nhc->nhc_gw_family == AF_INET))
1575 rt->rt_gw4 = nhc->nhc_gw.ipv4;
1576 else
1577 rt->rt_gw6 = nhc->nhc_gw.ipv6;
1578 }
1579
1580 ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
1581
1582 #ifdef CONFIG_IP_ROUTE_CLASSID
1583 {
1584 struct fib_nh *nh;
1585
1586 nh = container_of(nhc, struct fib_nh, nh_common);
1587 rt->dst.tclassid = nh->nh_tclassid;
1588 }
1589 #endif
1590 rt->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);
1591 if (unlikely(fnhe))
1592 cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1593 else if (do_cache)
1594 cached = rt_cache_route(nhc, rt);
1595 if (unlikely(!cached)) {
1596 /* Routes we intend to cache in nexthop exception or
1597 * FIB nexthop have the DST_NOCACHE bit clear.
1598 * However, if we are unsuccessful at storing this
1599 * route into the cache we really need to set it.
1600 */
1601 if (!rt->rt_gw4) {
1602 rt->rt_gw_family = AF_INET;
1603 rt->rt_gw4 = daddr;
1604 }
1605 rt_add_uncached_list(rt);
1606 }
1607 } else
1608 rt_add_uncached_list(rt);
1609
1610 #ifdef CONFIG_IP_ROUTE_CLASSID
1611 #ifdef CONFIG_IP_MULTIPLE_TABLES
1612 set_class_tag(rt, res->tclassid);
1613 #endif
1614 set_class_tag(rt, itag);
1615 #endif
1616 }
1617
1618 struct rtable *rt_dst_alloc(struct net_device *dev,
1619 unsigned int flags, u16 type,
1620 bool nopolicy, bool noxfrm, bool will_cache)
1621 {
1622 struct rtable *rt;
1623
1624 rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1625 (will_cache ? 0 : DST_HOST) |
1626 (nopolicy ? DST_NOPOLICY : 0) |
1627 (noxfrm ? DST_NOXFRM : 0));
1628
1629 if (rt) {
1630 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1631 rt->rt_flags = flags;
1632 rt->rt_type = type;
1633 rt->rt_is_input = 0;
1634 rt->rt_iif = 0;
1635 rt->rt_pmtu = 0;
1636 rt->rt_mtu_locked = 0;
1637 rt->rt_gw_family = 0;
1638 rt->rt_gw4 = 0;
1639 INIT_LIST_HEAD(&rt->rt_uncached);
1640
1641 rt->dst.output = ip_output;
1642 if (flags & RTCF_LOCAL)
1643 rt->dst.input = ip_local_deliver;
1644 }
1645
1646 return rt;
1647 }
1648 EXPORT_SYMBOL(rt_dst_alloc);
1649
1650 /* called in rcu_read_lock() section */
1651 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1652 u8 tos, struct net_device *dev,
1653 struct in_device *in_dev, u32 *itag)
1654 {
1655 int err;
1656
1657 /* Primary sanity checks. */
1658 if (!in_dev)
1659 return -EINVAL;
1660
1661 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1662 skb->protocol != htons(ETH_P_IP))
1663 return -EINVAL;
1664
1665 if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1666 return -EINVAL;
1667
1668 if (ipv4_is_zeronet(saddr)) {
1669 if (!ipv4_is_local_multicast(daddr) &&
1670 ip_hdr(skb)->protocol != IPPROTO_IGMP)
1671 return -EINVAL;
1672 } else {
1673 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1674 in_dev, itag);
1675 if (err < 0)
1676 return err;
1677 }
1678 return 0;
1679 }
1680
1681 /* called in rcu_read_lock() section */
1682 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1683 u8 tos, struct net_device *dev, int our)
1684 {
1685 struct in_device *in_dev = __in_dev_get_rcu(dev);
1686 unsigned int flags = RTCF_MULTICAST;
1687 struct rtable *rth;
1688 u32 itag = 0;
1689 int err;
1690
1691 err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1692 if (err)
1693 return err;
1694
1695 if (our)
1696 flags |= RTCF_LOCAL;
1697
1698 rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1699 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1700 if (!rth)
1701 return -ENOBUFS;
1702
1703 #ifdef CONFIG_IP_ROUTE_CLASSID
1704 rth->dst.tclassid = itag;
1705 #endif
1706 rth->dst.output = ip_rt_bug;
1707 rth->rt_is_input= 1;
1708
1709 #ifdef CONFIG_IP_MROUTE
1710 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1711 rth->dst.input = ip_mr_input;
1712 #endif
1713 RT_CACHE_STAT_INC(in_slow_mc);
1714
1715 skb_dst_set(skb, &rth->dst);
1716 return 0;
1717 }
1718
1719
1720 static void ip_handle_martian_source(struct net_device *dev,
1721 struct in_device *in_dev,
1722 struct sk_buff *skb,
1723 __be32 daddr,
1724 __be32 saddr)
1725 {
1726 RT_CACHE_STAT_INC(in_martian_src);
1727 #ifdef CONFIG_IP_ROUTE_VERBOSE
1728 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1729 /*
1730 * RFC1812 recommendation, if source is martian,
1731 * the only hint is MAC header.
1732 */
1733 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1734 &daddr, &saddr, dev->name);
1735 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1736 print_hex_dump(KERN_WARNING, "ll header: ",
1737 DUMP_PREFIX_OFFSET, 16, 1,
1738 skb_mac_header(skb),
1739 dev->hard_header_len, false);
1740 }
1741 }
1742 #endif
1743 }
1744
1745 /* called in rcu_read_lock() section */
1746 static int __mkroute_input(struct sk_buff *skb,
1747 const struct fib_result *res,
1748 struct in_device *in_dev,
1749 __be32 daddr, __be32 saddr, u32 tos)
1750 {
1751 struct fib_nh_common *nhc = FIB_RES_NHC(*res);
1752 struct net_device *dev = nhc->nhc_dev;
1753 struct fib_nh_exception *fnhe;
1754 struct rtable *rth;
1755 int err;
1756 struct in_device *out_dev;
1757 bool do_cache;
1758 u32 itag = 0;
1759
1760 /* get a working reference to the output device */
1761 out_dev = __in_dev_get_rcu(dev);
1762 if (!out_dev) {
1763 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1764 return -EINVAL;
1765 }
1766
1767 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1768 in_dev->dev, in_dev, &itag);
1769 if (err < 0) {
1770 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1771 saddr);
1772
1773 goto cleanup;
1774 }
1775
1776 do_cache = res->fi && !itag;
1777 if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1778 skb->protocol == htons(ETH_P_IP)) {
1779 __be32 gw;
1780
1781 gw = nhc->nhc_gw_family == AF_INET ? nhc->nhc_gw.ipv4 : 0;
1782 if (IN_DEV_SHARED_MEDIA(out_dev) ||
1783 inet_addr_onlink(out_dev, saddr, gw))
1784 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1785 }
1786
1787 if (skb->protocol != htons(ETH_P_IP)) {
1788 /* Not IP (i.e. ARP). Do not create route, if it is
1789 * invalid for proxy arp. DNAT routes are always valid.
1790 *
1791 * Proxy arp feature have been extended to allow, ARP
1792 * replies back to the same interface, to support
1793 * Private VLAN switch technologies. See arp.c.
1794 */
1795 if (out_dev == in_dev &&
1796 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1797 err = -EINVAL;
1798 goto cleanup;
1799 }
1800 }
1801
1802 fnhe = find_exception(nhc, daddr);
1803 if (do_cache) {
1804 if (fnhe)
1805 rth = rcu_dereference(fnhe->fnhe_rth_input);
1806 else
1807 rth = rcu_dereference(nhc->nhc_rth_input);
1808 if (rt_cache_valid(rth)) {
1809 skb_dst_set_noref(skb, &rth->dst);
1810 goto out;
1811 }
1812 }
1813
1814 rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1815 IN_DEV_CONF_GET(in_dev, NOPOLICY),
1816 IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1817 if (!rth) {
1818 err = -ENOBUFS;
1819 goto cleanup;
1820 }
1821
1822 rth->rt_is_input = 1;
1823 RT_CACHE_STAT_INC(in_slow_tot);
1824
1825 rth->dst.input = ip_forward;
1826
1827 rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1828 do_cache);
1829 lwtunnel_set_redirect(&rth->dst);
1830 skb_dst_set(skb, &rth->dst);
1831 out:
1832 err = 0;
1833 cleanup:
1834 return err;
1835 }
1836
1837 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1838 /* To make ICMP packets follow the right flow, the multipath hash is
1839 * calculated from the inner IP addresses.
1840 */
1841 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1842 struct flow_keys *hash_keys)
1843 {
1844 const struct iphdr *outer_iph = ip_hdr(skb);
1845 const struct iphdr *key_iph = outer_iph;
1846 const struct iphdr *inner_iph;
1847 const struct icmphdr *icmph;
1848 struct iphdr _inner_iph;
1849 struct icmphdr _icmph;
1850
1851 if (likely(outer_iph->protocol != IPPROTO_ICMP))
1852 goto out;
1853
1854 if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1855 goto out;
1856
1857 icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1858 &_icmph);
1859 if (!icmph)
1860 goto out;
1861
1862 if (icmph->type != ICMP_DEST_UNREACH &&
1863 icmph->type != ICMP_REDIRECT &&
1864 icmph->type != ICMP_TIME_EXCEEDED &&
1865 icmph->type != ICMP_PARAMETERPROB)
1866 goto out;
1867
1868 inner_iph = skb_header_pointer(skb,
1869 outer_iph->ihl * 4 + sizeof(_icmph),
1870 sizeof(_inner_iph), &_inner_iph);
1871 if (!inner_iph)
1872 goto out;
1873
1874 key_iph = inner_iph;
1875 out:
1876 hash_keys->addrs.v4addrs.src = key_iph->saddr;
1877 hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1878 }
1879
1880 /* if skb is set it will be used and fl4 can be NULL */
1881 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1882 const struct sk_buff *skb, struct flow_keys *flkeys)
1883 {
1884 u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0;
1885 struct flow_keys hash_keys;
1886 u32 mhash;
1887
1888 switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1889 case 0:
1890 memset(&hash_keys, 0, sizeof(hash_keys));
1891 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1892 if (skb) {
1893 ip_multipath_l3_keys(skb, &hash_keys);
1894 } else {
1895 hash_keys.addrs.v4addrs.src = fl4->saddr;
1896 hash_keys.addrs.v4addrs.dst = fl4->daddr;
1897 }
1898 break;
1899 case 1:
1900 /* skb is currently provided only when forwarding */
1901 if (skb) {
1902 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1903 struct flow_keys keys;
1904
1905 /* short-circuit if we already have L4 hash present */
1906 if (skb->l4_hash)
1907 return skb_get_hash_raw(skb) >> 1;
1908
1909 memset(&hash_keys, 0, sizeof(hash_keys));
1910
1911 if (!flkeys) {
1912 skb_flow_dissect_flow_keys(skb, &keys, flag);
1913 flkeys = &keys;
1914 }
1915
1916 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1917 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1918 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1919 hash_keys.ports.src = flkeys->ports.src;
1920 hash_keys.ports.dst = flkeys->ports.dst;
1921 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1922 } else {
1923 memset(&hash_keys, 0, sizeof(hash_keys));
1924 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1925 hash_keys.addrs.v4addrs.src = fl4->saddr;
1926 hash_keys.addrs.v4addrs.dst = fl4->daddr;
1927 hash_keys.ports.src = fl4->fl4_sport;
1928 hash_keys.ports.dst = fl4->fl4_dport;
1929 hash_keys.basic.ip_proto = fl4->flowi4_proto;
1930 }
1931 break;
1932 }
1933 mhash = flow_hash_from_keys(&hash_keys);
1934
1935 if (multipath_hash)
1936 mhash = jhash_2words(mhash, multipath_hash, 0);
1937
1938 return mhash >> 1;
1939 }
1940 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1941
1942 static int ip_mkroute_input(struct sk_buff *skb,
1943 struct fib_result *res,
1944 struct in_device *in_dev,
1945 __be32 daddr, __be32 saddr, u32 tos,
1946 struct flow_keys *hkeys)
1947 {
1948 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1949 if (res->fi && res->fi->fib_nhs > 1) {
1950 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1951
1952 fib_select_multipath(res, h);
1953 }
1954 #endif
1955
1956 /* create a routing cache entry */
1957 return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1958 }
1959
1960 /*
1961 * NOTE. We drop all the packets that has local source
1962 * addresses, because every properly looped back packet
1963 * must have correct destination already attached by output routine.
1964 *
1965 * Such approach solves two big problems:
1966 * 1. Not simplex devices are handled properly.
1967 * 2. IP spoofing attempts are filtered with 100% of guarantee.
1968 * called with rcu_read_lock()
1969 */
1970
1971 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1972 u8 tos, struct net_device *dev,
1973 struct fib_result *res)
1974 {
1975 struct in_device *in_dev = __in_dev_get_rcu(dev);
1976 struct flow_keys *flkeys = NULL, _flkeys;
1977 struct net *net = dev_net(dev);
1978 struct ip_tunnel_info *tun_info;
1979 int err = -EINVAL;
1980 unsigned int flags = 0;
1981 u32 itag = 0;
1982 struct rtable *rth;
1983 struct flowi4 fl4;
1984 bool do_cache = true;
1985
1986 /* IP on this device is disabled. */
1987
1988 if (!in_dev)
1989 goto out;
1990
1991 /* Check for the most weird martians, which can be not detected
1992 by fib_lookup.
1993 */
1994
1995 tun_info = skb_tunnel_info(skb);
1996 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1997 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1998 else
1999 fl4.flowi4_tun_key.tun_id = 0;
2000 skb_dst_drop(skb);
2001
2002 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
2003 goto martian_source;
2004
2005 res->fi = NULL;
2006 res->table = NULL;
2007 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2008 goto brd_input;
2009
2010 /* Accept zero addresses only to limited broadcast;
2011 * I even do not know to fix it or not. Waiting for complains :-)
2012 */
2013 if (ipv4_is_zeronet(saddr))
2014 goto martian_source;
2015
2016 if (ipv4_is_zeronet(daddr))
2017 goto martian_destination;
2018
2019 /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
2020 * and call it once if daddr or/and saddr are loopback addresses
2021 */
2022 if (ipv4_is_loopback(daddr)) {
2023 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
2024 goto martian_destination;
2025 } else if (ipv4_is_loopback(saddr)) {
2026 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
2027 goto martian_source;
2028 }
2029
2030 /*
2031 * Now we are ready to route packet.
2032 */
2033 fl4.flowi4_oif = 0;
2034 fl4.flowi4_iif = dev->ifindex;
2035 fl4.flowi4_mark = skb->mark;
2036 fl4.flowi4_tos = tos;
2037 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2038 fl4.flowi4_flags = 0;
2039 fl4.daddr = daddr;
2040 fl4.saddr = saddr;
2041 fl4.flowi4_uid = sock_net_uid(net, NULL);
2042
2043 if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
2044 flkeys = &_flkeys;
2045 } else {
2046 fl4.flowi4_proto = 0;
2047 fl4.fl4_sport = 0;
2048 fl4.fl4_dport = 0;
2049 }
2050
2051 err = fib_lookup(net, &fl4, res, 0);
2052 if (err != 0) {
2053 if (!IN_DEV_FORWARD(in_dev))
2054 err = -EHOSTUNREACH;
2055 goto no_route;
2056 }
2057
2058 if (res->type == RTN_BROADCAST) {
2059 if (IN_DEV_BFORWARD(in_dev))
2060 goto make_route;
2061 /* not do cache if bc_forwarding is enabled */
2062 if (IPV4_DEVCONF_ALL(net, BC_FORWARDING))
2063 do_cache = false;
2064 goto brd_input;
2065 }
2066
2067 if (res->type == RTN_LOCAL) {
2068 err = fib_validate_source(skb, saddr, daddr, tos,
2069 0, dev, in_dev, &itag);
2070 if (err < 0)
2071 goto martian_source;
2072 goto local_input;
2073 }
2074
2075 if (!IN_DEV_FORWARD(in_dev)) {
2076 err = -EHOSTUNREACH;
2077 goto no_route;
2078 }
2079 if (res->type != RTN_UNICAST)
2080 goto martian_destination;
2081
2082 make_route:
2083 err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2084 out: return err;
2085
2086 brd_input:
2087 if (skb->protocol != htons(ETH_P_IP))
2088 goto e_inval;
2089
2090 if (!ipv4_is_zeronet(saddr)) {
2091 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2092 in_dev, &itag);
2093 if (err < 0)
2094 goto martian_source;
2095 }
2096 flags |= RTCF_BROADCAST;
2097 res->type = RTN_BROADCAST;
2098 RT_CACHE_STAT_INC(in_brd);
2099
2100 local_input:
2101 do_cache &= res->fi && !itag;
2102 if (do_cache) {
2103 struct fib_nh_common *nhc = FIB_RES_NHC(*res);
2104
2105 rth = rcu_dereference(nhc->nhc_rth_input);
2106 if (rt_cache_valid(rth)) {
2107 skb_dst_set_noref(skb, &rth->dst);
2108 err = 0;
2109 goto out;
2110 }
2111 }
2112
2113 rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2114 flags | RTCF_LOCAL, res->type,
2115 IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2116 if (!rth)
2117 goto e_nobufs;
2118
2119 rth->dst.output= ip_rt_bug;
2120 #ifdef CONFIG_IP_ROUTE_CLASSID
2121 rth->dst.tclassid = itag;
2122 #endif
2123 rth->rt_is_input = 1;
2124
2125 RT_CACHE_STAT_INC(in_slow_tot);
2126 if (res->type == RTN_UNREACHABLE) {
2127 rth->dst.input= ip_error;
2128 rth->dst.error= -err;
2129 rth->rt_flags &= ~RTCF_LOCAL;
2130 }
2131
2132 if (do_cache) {
2133 struct fib_nh_common *nhc = FIB_RES_NHC(*res);
2134
2135 rth->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);
2136 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2137 WARN_ON(rth->dst.input == lwtunnel_input);
2138 rth->dst.lwtstate->orig_input = rth->dst.input;
2139 rth->dst.input = lwtunnel_input;
2140 }
2141
2142 if (unlikely(!rt_cache_route(nhc, rth)))
2143 rt_add_uncached_list(rth);
2144 }
2145 skb_dst_set(skb, &rth->dst);
2146 err = 0;
2147 goto out;
2148
2149 no_route:
2150 RT_CACHE_STAT_INC(in_no_route);
2151 res->type = RTN_UNREACHABLE;
2152 res->fi = NULL;
2153 res->table = NULL;
2154 goto local_input;
2155
2156 /*
2157 * Do not cache martian addresses: they should be logged (RFC1812)
2158 */
2159 martian_destination:
2160 RT_CACHE_STAT_INC(in_martian_dst);
2161 #ifdef CONFIG_IP_ROUTE_VERBOSE
2162 if (IN_DEV_LOG_MARTIANS(in_dev))
2163 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2164 &daddr, &saddr, dev->name);
2165 #endif
2166
2167 e_inval:
2168 err = -EINVAL;
2169 goto out;
2170
2171 e_nobufs:
2172 err = -ENOBUFS;
2173 goto out;
2174
2175 martian_source:
2176 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2177 goto out;
2178 }
2179
2180 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2181 u8 tos, struct net_device *dev)
2182 {
2183 struct fib_result res;
2184 int err;
2185
2186 tos &= IPTOS_RT_MASK;
2187 rcu_read_lock();
2188 err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2189 rcu_read_unlock();
2190
2191 return err;
2192 }
2193 EXPORT_SYMBOL(ip_route_input_noref);
2194
2195 /* called with rcu_read_lock held */
2196 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2197 u8 tos, struct net_device *dev, struct fib_result *res)
2198 {
2199 /* Multicast recognition logic is moved from route cache to here.
2200 The problem was that too many Ethernet cards have broken/missing
2201 hardware multicast filters :-( As result the host on multicasting
2202 network acquires a lot of useless route cache entries, sort of
2203 SDR messages from all the world. Now we try to get rid of them.
2204 Really, provided software IP multicast filter is organized
2205 reasonably (at least, hashed), it does not result in a slowdown
2206 comparing with route cache reject entries.
2207 Note, that multicast routers are not affected, because
2208 route cache entry is created eventually.
2209 */
2210 if (ipv4_is_multicast(daddr)) {
2211 struct in_device *in_dev = __in_dev_get_rcu(dev);
2212 int our = 0;
2213 int err = -EINVAL;
2214
2215 if (!in_dev)
2216 return err;
2217 our = ip_check_mc_rcu(in_dev, daddr, saddr,
2218 ip_hdr(skb)->protocol);
2219
2220 /* check l3 master if no match yet */
2221 if (!our && netif_is_l3_slave(dev)) {
2222 struct in_device *l3_in_dev;
2223
2224 l3_in_dev = __in_dev_get_rcu(skb->dev);
2225 if (l3_in_dev)
2226 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2227 ip_hdr(skb)->protocol);
2228 }
2229
2230 if (our
2231 #ifdef CONFIG_IP_MROUTE
2232 ||
2233 (!ipv4_is_local_multicast(daddr) &&
2234 IN_DEV_MFORWARD(in_dev))
2235 #endif
2236 ) {
2237 err = ip_route_input_mc(skb, daddr, saddr,
2238 tos, dev, our);
2239 }
2240 return err;
2241 }
2242
2243 return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2244 }
2245
2246 /* called with rcu_read_lock() */
2247 static struct rtable *__mkroute_output(const struct fib_result *res,
2248 const struct flowi4 *fl4, int orig_oif,
2249 struct net_device *dev_out,
2250 unsigned int flags)
2251 {
2252 struct fib_info *fi = res->fi;
2253 struct fib_nh_exception *fnhe;
2254 struct in_device *in_dev;
2255 u16 type = res->type;
2256 struct rtable *rth;
2257 bool do_cache;
2258
2259 in_dev = __in_dev_get_rcu(dev_out);
2260 if (!in_dev)
2261 return ERR_PTR(-EINVAL);
2262
2263 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2264 if (ipv4_is_loopback(fl4->saddr) &&
2265 !(dev_out->flags & IFF_LOOPBACK) &&
2266 !netif_is_l3_master(dev_out))
2267 return ERR_PTR(-EINVAL);
2268
2269 if (ipv4_is_lbcast(fl4->daddr))
2270 type = RTN_BROADCAST;
2271 else if (ipv4_is_multicast(fl4->daddr))
2272 type = RTN_MULTICAST;
2273 else if (ipv4_is_zeronet(fl4->daddr))
2274 return ERR_PTR(-EINVAL);
2275
2276 if (dev_out->flags & IFF_LOOPBACK)
2277 flags |= RTCF_LOCAL;
2278
2279 do_cache = true;
2280 if (type == RTN_BROADCAST) {
2281 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2282 fi = NULL;
2283 } else if (type == RTN_MULTICAST) {
2284 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2285 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2286 fl4->flowi4_proto))
2287 flags &= ~RTCF_LOCAL;
2288 else
2289 do_cache = false;
2290 /* If multicast route do not exist use
2291 * default one, but do not gateway in this case.
2292 * Yes, it is hack.
2293 */
2294 if (fi && res->prefixlen < 4)
2295 fi = NULL;
2296 } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2297 (orig_oif != dev_out->ifindex)) {
2298 /* For local routes that require a particular output interface
2299 * we do not want to cache the result. Caching the result
2300 * causes incorrect behaviour when there are multiple source
2301 * addresses on the interface, the end result being that if the
2302 * intended recipient is waiting on that interface for the
2303 * packet he won't receive it because it will be delivered on
2304 * the loopback interface and the IP_PKTINFO ipi_ifindex will
2305 * be set to the loopback interface as well.
2306 */
2307 do_cache = false;
2308 }
2309
2310 fnhe = NULL;
2311 do_cache &= fi != NULL;
2312 if (fi) {
2313 struct fib_nh_common *nhc = FIB_RES_NHC(*res);
2314 struct rtable __rcu **prth;
2315
2316 fnhe = find_exception(nhc, fl4->daddr);
2317 if (!do_cache)
2318 goto add;
2319 if (fnhe) {
2320 prth = &fnhe->fnhe_rth_output;
2321 } else {
2322 if (unlikely(fl4->flowi4_flags &
2323 FLOWI_FLAG_KNOWN_NH &&
2324 !(nhc->nhc_gw_family &&
2325 nhc->nhc_scope == RT_SCOPE_LINK))) {
2326 do_cache = false;
2327 goto add;
2328 }
2329 prth = raw_cpu_ptr(nhc->nhc_pcpu_rth_output);
2330 }
2331 rth = rcu_dereference(*prth);
2332 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2333 return rth;
2334 }
2335
2336 add:
2337 rth = rt_dst_alloc(dev_out, flags, type,
2338 IN_DEV_CONF_GET(in_dev, NOPOLICY),
2339 IN_DEV_CONF_GET(in_dev, NOXFRM),
2340 do_cache);
2341 if (!rth)
2342 return ERR_PTR(-ENOBUFS);
2343
2344 rth->rt_iif = orig_oif;
2345
2346 RT_CACHE_STAT_INC(out_slow_tot);
2347
2348 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2349 if (flags & RTCF_LOCAL &&
2350 !(dev_out->flags & IFF_LOOPBACK)) {
2351 rth->dst.output = ip_mc_output;
2352 RT_CACHE_STAT_INC(out_slow_mc);
2353 }
2354 #ifdef CONFIG_IP_MROUTE
2355 if (type == RTN_MULTICAST) {
2356 if (IN_DEV_MFORWARD(in_dev) &&
2357 !ipv4_is_local_multicast(fl4->daddr)) {
2358 rth->dst.input = ip_mr_input;
2359 rth->dst.output = ip_mc_output;
2360 }
2361 }
2362 #endif
2363 }
2364
2365 rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2366 lwtunnel_set_redirect(&rth->dst);
2367
2368 return rth;
2369 }
2370
2371 /*
2372 * Major route resolver routine.
2373 */
2374
2375 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2376 const struct sk_buff *skb)
2377 {
2378 __u8 tos = RT_FL_TOS(fl4);
2379 struct fib_result res = {
2380 .type = RTN_UNSPEC,
2381 .fi = NULL,
2382 .table = NULL,
2383 .tclassid = 0,
2384 };
2385 struct rtable *rth;
2386
2387 fl4->flowi4_iif = LOOPBACK_IFINDEX;
2388 fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2389 fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2390 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2391
2392 rcu_read_lock();
2393 rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2394 rcu_read_unlock();
2395
2396 return rth;
2397 }
2398 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2399
2400 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2401 struct fib_result *res,
2402 const struct sk_buff *skb)
2403 {
2404 struct net_device *dev_out = NULL;
2405 int orig_oif = fl4->flowi4_oif;
2406 unsigned int flags = 0;
2407 struct rtable *rth;
2408 int err = -ENETUNREACH;
2409
2410 if (fl4->saddr) {
2411 rth = ERR_PTR(-EINVAL);
2412 if (ipv4_is_multicast(fl4->saddr) ||
2413 ipv4_is_lbcast(fl4->saddr) ||
2414 ipv4_is_zeronet(fl4->saddr))
2415 goto out;
2416
2417 /* I removed check for oif == dev_out->oif here.
2418 It was wrong for two reasons:
2419 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2420 is assigned to multiple interfaces.
2421 2. Moreover, we are allowed to send packets with saddr
2422 of another iface. --ANK
2423 */
2424
2425 if (fl4->flowi4_oif == 0 &&
2426 (ipv4_is_multicast(fl4->daddr) ||
2427 ipv4_is_lbcast(fl4->daddr))) {
2428 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2429 dev_out = __ip_dev_find(net, fl4->saddr, false);
2430 if (!dev_out)
2431 goto out;
2432
2433 /* Special hack: user can direct multicasts
2434 and limited broadcast via necessary interface
2435 without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2436 This hack is not just for fun, it allows
2437 vic,vat and friends to work.
2438 They bind socket to loopback, set ttl to zero
2439 and expect that it will work.
2440 From the viewpoint of routing cache they are broken,
2441 because we are not allowed to build multicast path
2442 with loopback source addr (look, routing cache
2443 cannot know, that ttl is zero, so that packet
2444 will not leave this host and route is valid).
2445 Luckily, this hack is good workaround.
2446 */
2447
2448 fl4->flowi4_oif = dev_out->ifindex;
2449 goto make_route;
2450 }
2451
2452 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2453 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2454 if (!__ip_dev_find(net, fl4->saddr, false))
2455 goto out;
2456 }
2457 }
2458
2459
2460 if (fl4->flowi4_oif) {
2461 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2462 rth = ERR_PTR(-ENODEV);
2463 if (!dev_out)
2464 goto out;
2465
2466 /* RACE: Check return value of inet_select_addr instead. */
2467 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2468 rth = ERR_PTR(-ENETUNREACH);
2469 goto out;
2470 }
2471 if (ipv4_is_local_multicast(fl4->daddr) ||
2472 ipv4_is_lbcast(fl4->daddr) ||
2473 fl4->flowi4_proto == IPPROTO_IGMP) {
2474 if (!fl4->saddr)
2475 fl4->saddr = inet_select_addr(dev_out, 0,
2476 RT_SCOPE_LINK);
2477 goto make_route;
2478 }
2479 if (!fl4->saddr) {
2480 if (ipv4_is_multicast(fl4->daddr))
2481 fl4->saddr = inet_select_addr(dev_out, 0,
2482 fl4->flowi4_scope);
2483 else if (!fl4->daddr)
2484 fl4->saddr = inet_select_addr(dev_out, 0,
2485 RT_SCOPE_HOST);
2486 }
2487 }
2488
2489 if (!fl4->daddr) {
2490 fl4->daddr = fl4->saddr;
2491 if (!fl4->daddr)
2492 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2493 dev_out = net->loopback_dev;
2494 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2495 res->type = RTN_LOCAL;
2496 flags |= RTCF_LOCAL;
2497 goto make_route;
2498 }
2499
2500 err = fib_lookup(net, fl4, res, 0);
2501 if (err) {
2502 res->fi = NULL;
2503 res->table = NULL;
2504 if (fl4->flowi4_oif &&
2505 (ipv4_is_multicast(fl4->daddr) ||
2506 !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2507 /* Apparently, routing tables are wrong. Assume,
2508 that the destination is on link.
2509
2510 WHY? DW.
2511 Because we are allowed to send to iface
2512 even if it has NO routes and NO assigned
2513 addresses. When oif is specified, routing
2514 tables are looked up with only one purpose:
2515 to catch if destination is gatewayed, rather than
2516 direct. Moreover, if MSG_DONTROUTE is set,
2517 we send packet, ignoring both routing tables
2518 and ifaddr state. --ANK
2519
2520
2521 We could make it even if oif is unknown,
2522 likely IPv6, but we do not.
2523 */
2524
2525 if (fl4->saddr == 0)
2526 fl4->saddr = inet_select_addr(dev_out, 0,
2527 RT_SCOPE_LINK);
2528 res->type = RTN_UNICAST;
2529 goto make_route;
2530 }
2531 rth = ERR_PTR(err);
2532 goto out;
2533 }
2534
2535 if (res->type == RTN_LOCAL) {
2536 if (!fl4->saddr) {
2537 if (res->fi->fib_prefsrc)
2538 fl4->saddr = res->fi->fib_prefsrc;
2539 else
2540 fl4->saddr = fl4->daddr;
2541 }
2542
2543 /* L3 master device is the loopback for that domain */
2544 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2545 net->loopback_dev;
2546
2547 /* make sure orig_oif points to fib result device even
2548 * though packet rx/tx happens over loopback or l3mdev
2549 */
2550 orig_oif = FIB_RES_OIF(*res);
2551
2552 fl4->flowi4_oif = dev_out->ifindex;
2553 flags |= RTCF_LOCAL;
2554 goto make_route;
2555 }
2556
2557 fib_select_path(net, res, fl4, skb);
2558
2559 dev_out = FIB_RES_DEV(*res);
2560 fl4->flowi4_oif = dev_out->ifindex;
2561
2562
2563 make_route:
2564 rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2565
2566 out:
2567 return rth;
2568 }
2569
2570 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2571 {
2572 return NULL;
2573 }
2574
2575 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2576 {
2577 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2578
2579 return mtu ? : dst->dev->mtu;
2580 }
2581
2582 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2583 struct sk_buff *skb, u32 mtu)
2584 {
2585 }
2586
2587 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2588 struct sk_buff *skb)
2589 {
2590 }
2591
2592 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2593 unsigned long old)
2594 {
2595 return NULL;
2596 }
2597
2598 static struct dst_ops ipv4_dst_blackhole_ops = {
2599 .family = AF_INET,
2600 .check = ipv4_blackhole_dst_check,
2601 .mtu = ipv4_blackhole_mtu,
2602 .default_advmss = ipv4_default_advmss,
2603 .update_pmtu = ipv4_rt_blackhole_update_pmtu,
2604 .redirect = ipv4_rt_blackhole_redirect,
2605 .cow_metrics = ipv4_rt_blackhole_cow_metrics,
2606 .neigh_lookup = ipv4_neigh_lookup,
2607 };
2608
2609 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2610 {
2611 struct rtable *ort = (struct rtable *) dst_orig;
2612 struct rtable *rt;
2613
2614 rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2615 if (rt) {
2616 struct dst_entry *new = &rt->dst;
2617
2618 new->__use = 1;
2619 new->input = dst_discard;
2620 new->output = dst_discard_out;
2621
2622 new->dev = net->loopback_dev;
2623 if (new->dev)
2624 dev_hold(new->dev);
2625
2626 rt->rt_is_input = ort->rt_is_input;
2627 rt->rt_iif = ort->rt_iif;
2628 rt->rt_pmtu = ort->rt_pmtu;
2629 rt->rt_mtu_locked = ort->rt_mtu_locked;
2630
2631 rt->rt_genid = rt_genid_ipv4(net);
2632 rt->rt_flags = ort->rt_flags;
2633 rt->rt_type = ort->rt_type;
2634 rt->rt_gw_family = ort->rt_gw_family;
2635 if (rt->rt_gw_family == AF_INET)
2636 rt->rt_gw4 = ort->rt_gw4;
2637 else if (rt->rt_gw_family == AF_INET6)
2638 rt->rt_gw6 = ort->rt_gw6;
2639
2640 INIT_LIST_HEAD(&rt->rt_uncached);
2641 }
2642
2643 dst_release(dst_orig);
2644
2645 return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2646 }
2647
2648 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2649 const struct sock *sk)
2650 {
2651 struct rtable *rt = __ip_route_output_key(net, flp4);
2652
2653 if (IS_ERR(rt))
2654 return rt;
2655
2656 if (flp4->flowi4_proto)
2657 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2658 flowi4_to_flowi(flp4),
2659 sk, 0);
2660
2661 return rt;
2662 }
2663 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2664
2665 /* called with rcu_read_lock held */
2666 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2667 struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2668 struct sk_buff *skb, u32 portid, u32 seq)
2669 {
2670 struct rtmsg *r;
2671 struct nlmsghdr *nlh;
2672 unsigned long expires = 0;
2673 u32 error;
2674 u32 metrics[RTAX_MAX];
2675
2676 nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2677 if (!nlh)
2678 return -EMSGSIZE;
2679
2680 r = nlmsg_data(nlh);
2681 r->rtm_family = AF_INET;
2682 r->rtm_dst_len = 32;
2683 r->rtm_src_len = 0;
2684 r->rtm_tos = fl4->flowi4_tos;
2685 r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2686 if (nla_put_u32(skb, RTA_TABLE, table_id))
2687 goto nla_put_failure;
2688 r->rtm_type = rt->rt_type;
2689 r->rtm_scope = RT_SCOPE_UNIVERSE;
2690 r->rtm_protocol = RTPROT_UNSPEC;
2691 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2692 if (rt->rt_flags & RTCF_NOTIFY)
2693 r->rtm_flags |= RTM_F_NOTIFY;
2694 if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2695 r->rtm_flags |= RTCF_DOREDIRECT;
2696
2697 if (nla_put_in_addr(skb, RTA_DST, dst))
2698 goto nla_put_failure;
2699 if (src) {
2700 r->rtm_src_len = 32;
2701 if (nla_put_in_addr(skb, RTA_SRC, src))
2702 goto nla_put_failure;
2703 }
2704 if (rt->dst.dev &&
2705 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2706 goto nla_put_failure;
2707 #ifdef CONFIG_IP_ROUTE_CLASSID
2708 if (rt->dst.tclassid &&
2709 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2710 goto nla_put_failure;
2711 #endif
2712 if (!rt_is_input_route(rt) &&
2713 fl4->saddr != src) {
2714 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2715 goto nla_put_failure;
2716 }
2717 if (rt->rt_gw_family == AF_INET &&
2718 nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gw4)) {
2719 goto nla_put_failure;
2720 } else if (rt->rt_gw_family == AF_INET6) {
2721 int alen = sizeof(struct in6_addr);
2722 struct nlattr *nla;
2723 struct rtvia *via;
2724
2725 nla = nla_reserve(skb, RTA_VIA, alen + 2);
2726 if (!nla)
2727 goto nla_put_failure;
2728
2729 via = nla_data(nla);
2730 via->rtvia_family = AF_INET6;
2731 memcpy(via->rtvia_addr, &rt->rt_gw6, alen);
2732 }
2733
2734 expires = rt->dst.expires;
2735 if (expires) {
2736 unsigned long now = jiffies;
2737
2738 if (time_before(now, expires))
2739 expires -= now;
2740 else
2741 expires = 0;
2742 }
2743
2744 memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2745 if (rt->rt_pmtu && expires)
2746 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2747 if (rt->rt_mtu_locked && expires)
2748 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2749 if (rtnetlink_put_metrics(skb, metrics) < 0)
2750 goto nla_put_failure;
2751
2752 if (fl4->flowi4_mark &&
2753 nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2754 goto nla_put_failure;
2755
2756 if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2757 nla_put_u32(skb, RTA_UID,
2758 from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2759 goto nla_put_failure;
2760
2761 error = rt->dst.error;
2762
2763 if (rt_is_input_route(rt)) {
2764 #ifdef CONFIG_IP_MROUTE
2765 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2766 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2767 int err = ipmr_get_route(net, skb,
2768 fl4->saddr, fl4->daddr,
2769 r, portid);
2770
2771 if (err <= 0) {
2772 if (err == 0)
2773 return 0;
2774 goto nla_put_failure;
2775 }
2776 } else
2777 #endif
2778 if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2779 goto nla_put_failure;
2780 }
2781
2782 if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2783 goto nla_put_failure;
2784
2785 nlmsg_end(skb, nlh);
2786 return 0;
2787
2788 nla_put_failure:
2789 nlmsg_cancel(skb, nlh);
2790 return -EMSGSIZE;
2791 }
2792
2793 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2794 u8 ip_proto, __be16 sport,
2795 __be16 dport)
2796 {
2797 struct sk_buff *skb;
2798 struct iphdr *iph;
2799
2800 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2801 if (!skb)
2802 return NULL;
2803
2804 /* Reserve room for dummy headers, this skb can pass
2805 * through good chunk of routing engine.
2806 */
2807 skb_reset_mac_header(skb);
2808 skb_reset_network_header(skb);
2809 skb->protocol = htons(ETH_P_IP);
2810 iph = skb_put(skb, sizeof(struct iphdr));
2811 iph->protocol = ip_proto;
2812 iph->saddr = src;
2813 iph->daddr = dst;
2814 iph->version = 0x4;
2815 iph->frag_off = 0;
2816 iph->ihl = 0x5;
2817 skb_set_transport_header(skb, skb->len);
2818
2819 switch (iph->protocol) {
2820 case IPPROTO_UDP: {
2821 struct udphdr *udph;
2822
2823 udph = skb_put_zero(skb, sizeof(struct udphdr));
2824 udph->source = sport;
2825 udph->dest = dport;
2826 udph->len = sizeof(struct udphdr);
2827 udph->check = 0;
2828 break;
2829 }
2830 case IPPROTO_TCP: {
2831 struct tcphdr *tcph;
2832
2833 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2834 tcph->source = sport;
2835 tcph->dest = dport;
2836 tcph->doff = sizeof(struct tcphdr) / 4;
2837 tcph->rst = 1;
2838 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2839 src, dst, 0);
2840 break;
2841 }
2842 case IPPROTO_ICMP: {
2843 struct icmphdr *icmph;
2844
2845 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2846 icmph->type = ICMP_ECHO;
2847 icmph->code = 0;
2848 }
2849 }
2850
2851 return skb;
2852 }
2853
2854 static int inet_rtm_valid_getroute_req(struct sk_buff *skb,
2855 const struct nlmsghdr *nlh,
2856 struct nlattr **tb,
2857 struct netlink_ext_ack *extack)
2858 {
2859 struct rtmsg *rtm;
2860 int i, err;
2861
2862 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
2863 NL_SET_ERR_MSG(extack,
2864 "ipv4: Invalid header for route get request");
2865 return -EINVAL;
2866 }
2867
2868 if (!netlink_strict_get_check(skb))
2869 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
2870 rtm_ipv4_policy, extack);
2871
2872 rtm = nlmsg_data(nlh);
2873 if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) ||
2874 (rtm->rtm_dst_len && rtm->rtm_dst_len != 32) ||
2875 rtm->rtm_table || rtm->rtm_protocol ||
2876 rtm->rtm_scope || rtm->rtm_type) {
2877 NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request");
2878 return -EINVAL;
2879 }
2880
2881 if (rtm->rtm_flags & ~(RTM_F_NOTIFY |
2882 RTM_F_LOOKUP_TABLE |
2883 RTM_F_FIB_MATCH)) {
2884 NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request");
2885 return -EINVAL;
2886 }
2887
2888 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
2889 rtm_ipv4_policy, extack);
2890 if (err)
2891 return err;
2892
2893 if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
2894 (tb[RTA_DST] && !rtm->rtm_dst_len)) {
2895 NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4");
2896 return -EINVAL;
2897 }
2898
2899 for (i = 0; i <= RTA_MAX; i++) {
2900 if (!tb[i])
2901 continue;
2902
2903 switch (i) {
2904 case RTA_IIF:
2905 case RTA_OIF:
2906 case RTA_SRC:
2907 case RTA_DST:
2908 case RTA_IP_PROTO:
2909 case RTA_SPORT:
2910 case RTA_DPORT:
2911 case RTA_MARK:
2912 case RTA_UID:
2913 break;
2914 default:
2915 NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request");
2916 return -EINVAL;
2917 }
2918 }
2919
2920 return 0;
2921 }
2922
2923 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2924 struct netlink_ext_ack *extack)
2925 {
2926 struct net *net = sock_net(in_skb->sk);
2927 struct nlattr *tb[RTA_MAX+1];
2928 u32 table_id = RT_TABLE_MAIN;
2929 __be16 sport = 0, dport = 0;
2930 struct fib_result res = {};
2931 u8 ip_proto = IPPROTO_UDP;
2932 struct rtable *rt = NULL;
2933 struct sk_buff *skb;
2934 struct rtmsg *rtm;
2935 struct flowi4 fl4 = {};
2936 __be32 dst = 0;
2937 __be32 src = 0;
2938 kuid_t uid;
2939 u32 iif;
2940 int err;
2941 int mark;
2942
2943 err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
2944 if (err < 0)
2945 return err;
2946
2947 rtm = nlmsg_data(nlh);
2948 src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2949 dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2950 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2951 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2952 if (tb[RTA_UID])
2953 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2954 else
2955 uid = (iif ? INVALID_UID : current_uid());
2956
2957 if (tb[RTA_IP_PROTO]) {
2958 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2959 &ip_proto, AF_INET, extack);
2960 if (err)
2961 return err;
2962 }
2963
2964 if (tb[RTA_SPORT])
2965 sport = nla_get_be16(tb[RTA_SPORT]);
2966
2967 if (tb[RTA_DPORT])
2968 dport = nla_get_be16(tb[RTA_DPORT]);
2969
2970 skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2971 if (!skb)
2972 return -ENOBUFS;
2973
2974 fl4.daddr = dst;
2975 fl4.saddr = src;
2976 fl4.flowi4_tos = rtm->rtm_tos;
2977 fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2978 fl4.flowi4_mark = mark;
2979 fl4.flowi4_uid = uid;
2980 if (sport)
2981 fl4.fl4_sport = sport;
2982 if (dport)
2983 fl4.fl4_dport = dport;
2984 fl4.flowi4_proto = ip_proto;
2985
2986 rcu_read_lock();
2987
2988 if (iif) {
2989 struct net_device *dev;
2990
2991 dev = dev_get_by_index_rcu(net, iif);
2992 if (!dev) {
2993 err = -ENODEV;
2994 goto errout_rcu;
2995 }
2996
2997 fl4.flowi4_iif = iif; /* for rt_fill_info */
2998 skb->dev = dev;
2999 skb->mark = mark;
3000 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
3001 dev, &res);
3002
3003 rt = skb_rtable(skb);
3004 if (err == 0 && rt->dst.error)
3005 err = -rt->dst.error;
3006 } else {
3007 fl4.flowi4_iif = LOOPBACK_IFINDEX;
3008 skb->dev = net->loopback_dev;
3009 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
3010 err = 0;
3011 if (IS_ERR(rt))
3012 err = PTR_ERR(rt);
3013 else
3014 skb_dst_set(skb, &rt->dst);
3015 }
3016
3017 if (err)
3018 goto errout_rcu;
3019
3020 if (rtm->rtm_flags & RTM_F_NOTIFY)
3021 rt->rt_flags |= RTCF_NOTIFY;
3022
3023 if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
3024 table_id = res.table ? res.table->tb_id : 0;
3025
3026 /* reset skb for netlink reply msg */
3027 skb_trim(skb, 0);
3028 skb_reset_network_header(skb);
3029 skb_reset_transport_header(skb);
3030 skb_reset_mac_header(skb);
3031
3032 if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
3033 if (!res.fi) {
3034 err = fib_props[res.type].error;
3035 if (!err)
3036 err = -EHOSTUNREACH;
3037 goto errout_rcu;
3038 }
3039 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
3040 nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
3041 rt->rt_type, res.prefix, res.prefixlen,
3042 fl4.flowi4_tos, res.fi, 0);
3043 } else {
3044 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
3045 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
3046 }
3047 if (err < 0)
3048 goto errout_rcu;
3049
3050 rcu_read_unlock();
3051
3052 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3053
3054 errout_free:
3055 return err;
3056 errout_rcu:
3057 rcu_read_unlock();
3058 kfree_skb(skb);
3059 goto errout_free;
3060 }
3061
3062 void ip_rt_multicast_event(struct in_device *in_dev)
3063 {
3064 rt_cache_flush(dev_net(in_dev->dev));
3065 }
3066
3067 #ifdef CONFIG_SYSCTL
3068 static int ip_rt_gc_interval __read_mostly = 60 * HZ;
3069 static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
3070 static int ip_rt_gc_elasticity __read_mostly = 8;
3071 static int ip_min_valid_pmtu __read_mostly = IPV4_MIN_MTU;
3072
3073 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
3074 void __user *buffer,
3075 size_t *lenp, loff_t *ppos)
3076 {
3077 struct net *net = (struct net *)__ctl->extra1;
3078
3079 if (write) {
3080 rt_cache_flush(net);
3081 fnhe_genid_bump(net);
3082 return 0;
3083 }
3084
3085 return -EINVAL;
3086 }
3087
3088 static struct ctl_table ipv4_route_table[] = {
3089 {
3090 .procname = "gc_thresh",
3091 .data = &ipv4_dst_ops.gc_thresh,
3092 .maxlen = sizeof(int),
3093 .mode = 0644,
3094 .proc_handler = proc_dointvec,
3095 },
3096 {
3097 .procname = "max_size",
3098 .data = &ip_rt_max_size,
3099 .maxlen = sizeof(int),
3100 .mode = 0644,
3101 .proc_handler = proc_dointvec,
3102 },
3103 {
3104 /* Deprecated. Use gc_min_interval_ms */
3105
3106 .procname = "gc_min_interval",
3107 .data = &ip_rt_gc_min_interval,
3108 .maxlen = sizeof(int),
3109 .mode = 0644,
3110 .proc_handler = proc_dointvec_jiffies,
3111 },
3112 {
3113 .procname = "gc_min_interval_ms",
3114 .data = &ip_rt_gc_min_interval,
3115 .maxlen = sizeof(int),
3116 .mode = 0644,
3117 .proc_handler = proc_dointvec_ms_jiffies,
3118 },
3119 {
3120 .procname = "gc_timeout",
3121 .data = &ip_rt_gc_timeout,
3122 .maxlen = sizeof(int),
3123 .mode = 0644,
3124 .proc_handler = proc_dointvec_jiffies,
3125 },
3126 {
3127 .procname = "gc_interval",
3128 .data = &ip_rt_gc_interval,
3129 .maxlen = sizeof(int),
3130 .mode = 0644,
3131 .proc_handler = proc_dointvec_jiffies,
3132 },
3133 {
3134 .procname = "redirect_load",
3135 .data = &ip_rt_redirect_load,
3136 .maxlen = sizeof(int),
3137 .mode = 0644,
3138 .proc_handler = proc_dointvec,
3139 },
3140 {
3141 .procname = "redirect_number",
3142 .data = &ip_rt_redirect_number,
3143 .maxlen = sizeof(int),
3144 .mode = 0644,
3145 .proc_handler = proc_dointvec,
3146 },
3147 {
3148 .procname = "redirect_silence",
3149 .data = &ip_rt_redirect_silence,
3150 .maxlen = sizeof(int),
3151 .mode = 0644,
3152 .proc_handler = proc_dointvec,
3153 },
3154 {
3155 .procname = "error_cost",
3156 .data = &ip_rt_error_cost,
3157 .maxlen = sizeof(int),
3158 .mode = 0644,
3159 .proc_handler = proc_dointvec,
3160 },
3161 {
3162 .procname = "error_burst",
3163 .data = &ip_rt_error_burst,
3164 .maxlen = sizeof(int),
3165 .mode = 0644,
3166 .proc_handler = proc_dointvec,
3167 },
3168 {
3169 .procname = "gc_elasticity",
3170 .data = &ip_rt_gc_elasticity,
3171 .maxlen = sizeof(int),
3172 .mode = 0644,
3173 .proc_handler = proc_dointvec,
3174 },
3175 {
3176 .procname = "mtu_expires",
3177 .data = &ip_rt_mtu_expires,
3178 .maxlen = sizeof(int),
3179 .mode = 0644,
3180 .proc_handler = proc_dointvec_jiffies,
3181 },
3182 {
3183 .procname = "min_pmtu",
3184 .data = &ip_rt_min_pmtu,
3185 .maxlen = sizeof(int),
3186 .mode = 0644,
3187 .proc_handler = proc_dointvec_minmax,
3188 .extra1 = &ip_min_valid_pmtu,
3189 },
3190 {
3191 .procname = "min_adv_mss",
3192 .data = &ip_rt_min_advmss,
3193 .maxlen = sizeof(int),
3194 .mode = 0644,
3195 .proc_handler = proc_dointvec,
3196 },
3197 { }
3198 };
3199
3200 static struct ctl_table ipv4_route_flush_table[] = {
3201 {
3202 .procname = "flush",
3203 .maxlen = sizeof(int),
3204 .mode = 0200,
3205 .proc_handler = ipv4_sysctl_rtcache_flush,
3206 },
3207 { },
3208 };
3209
3210 static __net_init int sysctl_route_net_init(struct net *net)
3211 {
3212 struct ctl_table *tbl;
3213
3214 tbl = ipv4_route_flush_table;
3215 if (!net_eq(net, &init_net)) {
3216 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3217 if (!tbl)
3218 goto err_dup;
3219
3220 /* Don't export sysctls to unprivileged users */
3221 if (net->user_ns != &init_user_ns)
3222 tbl[0].procname = NULL;
3223 }
3224 tbl[0].extra1 = net;
3225
3226 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3227 if (!net->ipv4.route_hdr)
3228 goto err_reg;
3229 return 0;
3230
3231 err_reg:
3232 if (tbl != ipv4_route_flush_table)
3233 kfree(tbl);
3234 err_dup:
3235 return -ENOMEM;
3236 }
3237
3238 static __net_exit void sysctl_route_net_exit(struct net *net)
3239 {
3240 struct ctl_table *tbl;
3241
3242 tbl = net->ipv4.route_hdr->ctl_table_arg;
3243 unregister_net_sysctl_table(net->ipv4.route_hdr);
3244 BUG_ON(tbl == ipv4_route_flush_table);
3245 kfree(tbl);
3246 }
3247
3248 static __net_initdata struct pernet_operations sysctl_route_ops = {
3249 .init = sysctl_route_net_init,
3250 .exit = sysctl_route_net_exit,
3251 };
3252 #endif
3253
3254 static __net_init int rt_genid_init(struct net *net)
3255 {
3256 atomic_set(&net->ipv4.rt_genid, 0);
3257 atomic_set(&net->fnhe_genid, 0);
3258 atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3259 return 0;
3260 }
3261
3262 static __net_initdata struct pernet_operations rt_genid_ops = {
3263 .init = rt_genid_init,
3264 };
3265
3266 static int __net_init ipv4_inetpeer_init(struct net *net)
3267 {
3268 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3269
3270 if (!bp)
3271 return -ENOMEM;
3272 inet_peer_base_init(bp);
3273 net->ipv4.peers = bp;
3274 return 0;
3275 }
3276
3277 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3278 {
3279 struct inet_peer_base *bp = net->ipv4.peers;
3280
3281 net->ipv4.peers = NULL;
3282 inetpeer_invalidate_tree(bp);
3283 kfree(bp);
3284 }
3285
3286 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3287 .init = ipv4_inetpeer_init,
3288 .exit = ipv4_inetpeer_exit,
3289 };
3290
3291 #ifdef CONFIG_IP_ROUTE_CLASSID
3292 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3293 #endif /* CONFIG_IP_ROUTE_CLASSID */
3294
3295 int __init ip_rt_init(void)
3296 {
3297 int cpu;
3298
3299 ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3300 GFP_KERNEL);
3301 if (!ip_idents)
3302 panic("IP: failed to allocate ip_idents\n");
3303
3304 prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3305
3306 ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3307 if (!ip_tstamps)
3308 panic("IP: failed to allocate ip_tstamps\n");
3309
3310 for_each_possible_cpu(cpu) {
3311 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3312
3313 INIT_LIST_HEAD(&ul->head);
3314 spin_lock_init(&ul->lock);
3315 }
3316 #ifdef CONFIG_IP_ROUTE_CLASSID
3317 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3318 if (!ip_rt_acct)
3319 panic("IP: failed to allocate ip_rt_acct\n");
3320 #endif
3321
3322 ipv4_dst_ops.kmem_cachep =
3323 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3324 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3325
3326 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3327
3328 if (dst_entries_init(&ipv4_dst_ops) < 0)
3329 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3330
3331 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3332 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3333
3334 ipv4_dst_ops.gc_thresh = ~0;
3335 ip_rt_max_size = INT_MAX;
3336
3337 devinet_init();
3338 ip_fib_init();
3339
3340 if (ip_rt_proc_init())
3341 pr_err("Unable to create route proc files\n");
3342 #ifdef CONFIG_XFRM
3343 xfrm_init();
3344 xfrm4_init();
3345 #endif
3346 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3347 RTNL_FLAG_DOIT_UNLOCKED);
3348
3349 #ifdef CONFIG_SYSCTL
3350 register_pernet_subsys(&sysctl_route_ops);
3351 #endif
3352 register_pernet_subsys(&rt_genid_ops);
3353 register_pernet_subsys(&ipv4_inetpeer_ops);
3354 return 0;
3355 }
3356
3357 #ifdef CONFIG_SYSCTL
3358 /*
3359 * We really need to sanitize the damn ipv4 init order, then all
3360 * this nonsense will go away.
3361 */
3362 void __init ip_static_sysctl_init(void)
3363 {
3364 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3365 }
3366 #endif
Ubuntu Jammy Linux kernel mirror