net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <net/dst.h>
  93 #include <net/net_namespace.h>
  94 #include <net/protocol.h>
  95 #include <net/ip.h>
  96 #include <net/route.h>
  97 #include <net/inetpeer.h>
  98 #include <net/sock.h>
  99 #include <net/ip_fib.h>
 100 #include <net/arp.h>
 101 #include <net/tcp.h>
 102 #include <net/icmp.h>
 103 #include <net/xfrm.h>
 104 #include <net/netevent.h>
 105 #include <net/rtnetlink.h>
 106 #ifdef CONFIG_SYSCTL
 107 #include <linux/sysctl.h>
 108 #include <linux/kmemleak.h>
 109 #endif
 110 #include <net/secure_seq.h>
 111
 112 #define RT_FL_TOS(oldflp4) \
 113         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 114
 115 #define IP_MAX_MTU      0xFFF0
 116
 117 #define RT_GC_TIMEOUT (300*HZ)
 118
 119 static int ip_rt_max_size;
 120 static int ip_rt_redirect_number __read_mostly  = 9;
 121 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 122 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 123 static int ip_rt_error_cost __read_mostly       = HZ;
 124 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 125 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 126 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 127 static int ip_rt_min_advmss __read_mostly       = 256;
 128
 129 /*
 130  *      Interface to generic destination cache.
 131  */
 132
 133 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 134 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 135 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 136 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 137 static void              ipv4_link_failure(struct sk_buff *skb);
 138 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 139                                            struct sk_buff *skb, u32 mtu);
 140 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 141                                         struct sk_buff *skb);
 142 static void             ipv4_dst_destroy(struct dst_entry *dst);
 143
 144 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 145                             int how)
 146 {
 147 }
 148
 149 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 150 {
 151         WARN_ON(1);
 152         return NULL;
 153 }
 154
 155 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 156                                            struct sk_buff *skb,
 157                                            const void *daddr);
 158
 159 static struct dst_ops ipv4_dst_ops = {
 160         .family =               AF_INET,
 161         .protocol =             cpu_to_be16(ETH_P_IP),
 162         .check =                ipv4_dst_check,
 163         .default_advmss =       ipv4_default_advmss,
 164         .mtu =                  ipv4_mtu,
 165         .cow_metrics =          ipv4_cow_metrics,
 166         .destroy =              ipv4_dst_destroy,
 167         .ifdown =               ipv4_dst_ifdown,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174 };
 175
 176 #define ECN_OR_COST(class)      TC_PRIO_##class
 177
 178 const __u8 ip_tos2prio[16] = {
 179         TC_PRIO_BESTEFFORT,
 180         ECN_OR_COST(BESTEFFORT),
 181         TC_PRIO_BESTEFFORT,
 182         ECN_OR_COST(BESTEFFORT),
 183         TC_PRIO_BULK,
 184         ECN_OR_COST(BULK),
 185         TC_PRIO_BULK,
 186         ECN_OR_COST(BULK),
 187         TC_PRIO_INTERACTIVE,
 188         ECN_OR_COST(INTERACTIVE),
 189         TC_PRIO_INTERACTIVE,
 190         ECN_OR_COST(INTERACTIVE),
 191         TC_PRIO_INTERACTIVE_BULK,
 192         ECN_OR_COST(INTERACTIVE_BULK),
 193         TC_PRIO_INTERACTIVE_BULK,
 194         ECN_OR_COST(INTERACTIVE_BULK)
 195 };
 196 EXPORT_SYMBOL(ip_tos2prio);
 197
 198 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 199 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 200
 201 #ifdef CONFIG_PROC_FS
 202 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 203 {
 204         if (*pos)
 205                 return NULL;
 206         return SEQ_START_TOKEN;
 207 }
 208
 209 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 210 {
 211         ++*pos;
 212         return NULL;
 213 }
 214
 215 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 216 {
 217 }
 218
 219 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 220 {
 221         if (v == SEQ_START_TOKEN)
 222                 seq_printf(seq, "%-127s\n",
 223                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 224                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 225                            "HHUptod\tSpecDst");
 226         return 0;
 227 }
 228
 229 static const struct seq_operations rt_cache_seq_ops = {
 230         .start  = rt_cache_seq_start,
 231         .next   = rt_cache_seq_next,
 232         .stop   = rt_cache_seq_stop,
 233         .show   = rt_cache_seq_show,
 234 };
 235
 236 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 237 {
 238         return seq_open(file, &rt_cache_seq_ops);
 239 }
 240
 241 static const struct file_operations rt_cache_seq_fops = {
 242         .owner   = THIS_MODULE,
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    st->in_hit,
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    st->out_hit,
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    st->gc_total,
 310                    st->gc_ignored,
 311                    st->gc_goal_miss,
 312                    st->gc_dst_overflow,
 313                    st->in_hlist_search,
 314                    st->out_hlist_search
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .owner   = THIS_MODULE,
 334         .open    = rt_cpu_seq_open,
 335         .read    = seq_read,
 336         .llseek  = seq_lseek,
 337         .release = seq_release,
 338 };
 339
 340 #ifdef CONFIG_IP_ROUTE_CLASSID
 341 static int rt_acct_proc_show(struct seq_file *m, void *v)
 342 {
 343         struct ip_rt_acct *dst, *src;
 344         unsigned int i, j;
 345
 346         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 347         if (!dst)
 348                 return -ENOMEM;
 349
 350         for_each_possible_cpu(i) {
 351                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 352                 for (j = 0; j < 256; j++) {
 353                         dst[j].o_bytes   += src[j].o_bytes;
 354                         dst[j].o_packets += src[j].o_packets;
 355                         dst[j].i_bytes   += src[j].i_bytes;
 356                         dst[j].i_packets += src[j].i_packets;
 357                 }
 358         }
 359
 360         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 361         kfree(dst);
 362         return 0;
 363 }
 364
 365 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 366 {
 367         return single_open(file, rt_acct_proc_show, NULL);
 368 }
 369
 370 static const struct file_operations rt_acct_proc_fops = {
 371         .owner          = THIS_MODULE,
 372         .open           = rt_acct_proc_open,
 373         .read           = seq_read,
 374         .llseek         = seq_lseek,
 375         .release        = single_release,
 376 };
 377 #endif
 378
 379 static int __net_init ip_rt_do_proc_init(struct net *net)
 380 {
 381         struct proc_dir_entry *pde;
 382
 383         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 384                           &rt_cache_seq_fops);
 385         if (!pde)
 386                 goto err1;
 387
 388         pde = proc_create("rt_cache", S_IRUGO,
 389                           net->proc_net_stat, &rt_cpu_seq_fops);
 390         if (!pde)
 391                 goto err2;
 392
 393 #ifdef CONFIG_IP_ROUTE_CLASSID
 394         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 395         if (!pde)
 396                 goto err3;
 397 #endif
 398         return 0;
 399
 400 #ifdef CONFIG_IP_ROUTE_CLASSID
 401 err3:
 402         remove_proc_entry("rt_cache", net->proc_net_stat);
 403 #endif
 404 err2:
 405         remove_proc_entry("rt_cache", net->proc_net);
 406 err1:
 407         return -ENOMEM;
 408 }
 409
 410 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 411 {
 412         remove_proc_entry("rt_cache", net->proc_net_stat);
 413         remove_proc_entry("rt_cache", net->proc_net);
 414 #ifdef CONFIG_IP_ROUTE_CLASSID
 415         remove_proc_entry("rt_acct", net->proc_net);
 416 #endif
 417 }
 418
 419 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 420         .init = ip_rt_do_proc_init,
 421         .exit = ip_rt_do_proc_exit,
 422 };
 423
 424 static int __init ip_rt_proc_init(void)
 425 {
 426         return register_pernet_subsys(&ip_rt_proc_ops);
 427 }
 428
 429 #else
 430 static inline int ip_rt_proc_init(void)
 431 {
 432         return 0;
 433 }
 434 #endif /* CONFIG_PROC_FS */
 435
 436 static inline bool rt_is_expired(const struct rtable *rth)
 437 {
 438         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 439 }
 440
 441 void rt_cache_flush(struct net *net)
 442 {
 443         rt_genid_bump(net);
 444 }
 445
 446 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 447                                            struct sk_buff *skb,
 448                                            const void *daddr)
 449 {
 450         struct net_device *dev = dst->dev;
 451         const __be32 *pkey = daddr;
 452         const struct rtable *rt;
 453         struct neighbour *n;
 454
 455         rt = (const struct rtable *) dst;
 456         if (rt->rt_gateway)
 457                 pkey = (const __be32 *) &rt->rt_gateway;
 458         else if (skb)
 459                 pkey = &ip_hdr(skb)->daddr;
 460
 461         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 462         if (n)
 463                 return n;
 464         return neigh_create(&arp_tbl, pkey, dev);
 465 }
 466
 467 /*
 468  * Peer allocation may fail only in serious out-of-memory conditions.  However
 469  * we still can generate some output.
 470  * Random ID selection looks a bit dangerous because we have no chances to
 471  * select ID being unique in a reasonable period of time.
 472  * But broken packet identifier may be better than no packet at all.
 473  */
 474 static void ip_select_fb_ident(struct iphdr *iph)
 475 {
 476         static DEFINE_SPINLOCK(ip_fb_id_lock);
 477         static u32 ip_fallback_id;
 478         u32 salt;
 479
 480         spin_lock_bh(&ip_fb_id_lock);
 481         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
 482         iph->id = htons(salt & 0xFFFF);
 483         ip_fallback_id = salt;
 484         spin_unlock_bh(&ip_fb_id_lock);
 485 }
 486
 487 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
 488 {
 489         struct net *net = dev_net(dst->dev);
 490         struct inet_peer *peer;
 491
 492         peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
 493         if (peer) {
 494                 iph->id = htons(inet_getid(peer, more));
 495                 inet_putpeer(peer);
 496                 return;
 497         }
 498
 499         ip_select_fb_ident(iph);
 500 }
 501 EXPORT_SYMBOL(__ip_select_ident);
 502
 503 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 504                              const struct iphdr *iph,
 505                              int oif, u8 tos,
 506                              u8 prot, u32 mark, int flow_flags)
 507 {
 508         if (sk) {
 509                 const struct inet_sock *inet = inet_sk(sk);
 510
 511                 oif = sk->sk_bound_dev_if;
 512                 mark = sk->sk_mark;
 513                 tos = RT_CONN_FLAGS(sk);
 514                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 515         }
 516         flowi4_init_output(fl4, oif, mark, tos,
 517                            RT_SCOPE_UNIVERSE, prot,
 518                            flow_flags,
 519                            iph->daddr, iph->saddr, 0, 0);
 520 }
 521
 522 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 523                                const struct sock *sk)
 524 {
 525         const struct iphdr *iph = ip_hdr(skb);
 526         int oif = skb->dev->ifindex;
 527         u8 tos = RT_TOS(iph->tos);
 528         u8 prot = iph->protocol;
 529         u32 mark = skb->mark;
 530
 531         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 532 }
 533
 534 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 535 {
 536         const struct inet_sock *inet = inet_sk(sk);
 537         const struct ip_options_rcu *inet_opt;
 538         __be32 daddr = inet->inet_daddr;
 539
 540         rcu_read_lock();
 541         inet_opt = rcu_dereference(inet->inet_opt);
 542         if (inet_opt && inet_opt->opt.srr)
 543                 daddr = inet_opt->opt.faddr;
 544         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 545                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 546                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 547                            inet_sk_flowi_flags(sk),
 548                            daddr, inet->inet_saddr, 0, 0);
 549         rcu_read_unlock();
 550 }
 551
 552 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 553                                  const struct sk_buff *skb)
 554 {
 555         if (skb)
 556                 build_skb_flow_key(fl4, skb, sk);
 557         else
 558                 build_sk_flow_key(fl4, sk);
 559 }
 560
 561 static inline void rt_free(struct rtable *rt)
 562 {
 563         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 564 }
 565
 566 static DEFINE_SPINLOCK(fnhe_lock);
 567
 568 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 569 {
 570         struct fib_nh_exception *fnhe, *oldest;
 571         struct rtable *orig;
 572
 573         oldest = rcu_dereference(hash->chain);
 574         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 575              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 576                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 577                         oldest = fnhe;
 578         }
 579         orig = rcu_dereference(oldest->fnhe_rth);
 580         if (orig) {
 581                 RCU_INIT_POINTER(oldest->fnhe_rth, NULL);
 582                 rt_free(orig);
 583         }
 584         return oldest;
 585 }
 586
 587 static inline u32 fnhe_hashfun(__be32 daddr)
 588 {
 589         u32 hval;
 590
 591         hval = (__force u32) daddr;
 592         hval ^= (hval >> 11) ^ (hval >> 22);
 593
 594         return hval & (FNHE_HASH_SIZE - 1);
 595 }
 596
 597 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 598                                   u32 pmtu, unsigned long expires)
 599 {
 600         struct fnhe_hash_bucket *hash;
 601         struct fib_nh_exception *fnhe;
 602         int depth;
 603         u32 hval = fnhe_hashfun(daddr);
 604
 605         spin_lock_bh(&fnhe_lock);
 606
 607         hash = nh->nh_exceptions;
 608         if (!hash) {
 609                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 610                 if (!hash)
 611                         goto out_unlock;
 612                 nh->nh_exceptions = hash;
 613         }
 614
 615         hash += hval;
 616
 617         depth = 0;
 618         for (fnhe = rcu_dereference(hash->chain); fnhe;
 619              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 620                 if (fnhe->fnhe_daddr == daddr)
 621                         break;
 622                 depth++;
 623         }
 624
 625         if (fnhe) {
 626                 if (gw)
 627                         fnhe->fnhe_gw = gw;
 628                 if (pmtu) {
 629                         fnhe->fnhe_pmtu = pmtu;
 630                         fnhe->fnhe_expires = expires;
 631                 }
 632         } else {
 633                 if (depth > FNHE_RECLAIM_DEPTH)
 634                         fnhe = fnhe_oldest(hash);
 635                 else {
 636                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 637                         if (!fnhe)
 638                                 goto out_unlock;
 639
 640                         fnhe->fnhe_next = hash->chain;
 641                         rcu_assign_pointer(hash->chain, fnhe);
 642                 }
 643                 fnhe->fnhe_daddr = daddr;
 644                 fnhe->fnhe_gw = gw;
 645                 fnhe->fnhe_pmtu = pmtu;
 646                 fnhe->fnhe_expires = expires;
 647         }
 648
 649         fnhe->fnhe_stamp = jiffies;
 650
 651 out_unlock:
 652         spin_unlock_bh(&fnhe_lock);
 653         return;
 654 }
 655
 656 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 657                              bool kill_route)
 658 {
 659         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 660         __be32 old_gw = ip_hdr(skb)->saddr;
 661         struct net_device *dev = skb->dev;
 662         struct in_device *in_dev;
 663         struct fib_result res;
 664         struct neighbour *n;
 665         struct net *net;
 666
 667         switch (icmp_hdr(skb)->code & 7) {
 668         case ICMP_REDIR_NET:
 669         case ICMP_REDIR_NETTOS:
 670         case ICMP_REDIR_HOST:
 671         case ICMP_REDIR_HOSTTOS:
 672                 break;
 673
 674         default:
 675                 return;
 676         }
 677
 678         if (rt->rt_gateway != old_gw)
 679                 return;
 680
 681         in_dev = __in_dev_get_rcu(dev);
 682         if (!in_dev)
 683                 return;
 684
 685         net = dev_net(dev);
 686         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 687             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 688             ipv4_is_zeronet(new_gw))
 689                 goto reject_redirect;
 690
 691         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 692                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 693                         goto reject_redirect;
 694                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 695                         goto reject_redirect;
 696         } else {
 697                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 698                         goto reject_redirect;
 699         }
 700
 701         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 702         if (n) {
 703                 if (!(n->nud_state & NUD_VALID)) {
 704                         neigh_event_send(n, NULL);
 705                 } else {
 706                         if (fib_lookup(net, fl4, &res) == 0) {
 707                                 struct fib_nh *nh = &FIB_RES_NH(res);
 708
 709                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 710                                                       0, 0);
 711                         }
 712                         if (kill_route)
 713                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 714                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 715                 }
 716                 neigh_release(n);
 717         }
 718         return;
 719
 720 reject_redirect:
 721 #ifdef CONFIG_IP_ROUTE_VERBOSE
 722         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 723                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 724                 __be32 daddr = iph->daddr;
 725                 __be32 saddr = iph->saddr;
 726
 727                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 728                                      "  Advised path = %pI4 -> %pI4\n",
 729                                      &old_gw, dev->name, &new_gw,
 730                                      &saddr, &daddr);
 731         }
 732 #endif
 733         ;
 734 }
 735
 736 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 737 {
 738         struct rtable *rt;
 739         struct flowi4 fl4;
 740         const struct iphdr *iph = (const struct iphdr *) skb->data;
 741         int oif = skb->dev->ifindex;
 742         u8 tos = RT_TOS(iph->tos);
 743         u8 prot = iph->protocol;
 744         u32 mark = skb->mark;
 745
 746         rt = (struct rtable *) dst;
 747
 748         __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
 749         __ip_do_redirect(rt, skb, &fl4, true);
 750 }
 751
 752 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 753 {
 754         struct rtable *rt = (struct rtable *)dst;
 755         struct dst_entry *ret = dst;
 756
 757         if (rt) {
 758                 if (dst->obsolete > 0) {
 759                         ip_rt_put(rt);
 760                         ret = NULL;
 761                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 762                            rt->dst.expires) {
 763                         ip_rt_put(rt);
 764                         ret = NULL;
 765                 }
 766         }
 767         return ret;
 768 }
 769
 770 /*
 771  * Algorithm:
 772  *      1. The first ip_rt_redirect_number redirects are sent
 773  *         with exponential backoff, then we stop sending them at all,
 774  *         assuming that the host ignores our redirects.
 775  *      2. If we did not see packets requiring redirects
 776  *         during ip_rt_redirect_silence, we assume that the host
 777  *         forgot redirected route and start to send redirects again.
 778  *
 779  * This algorithm is much cheaper and more intelligent than dumb load limiting
 780  * in icmp.c.
 781  *
 782  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 783  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 784  */
 785
 786 void ip_rt_send_redirect(struct sk_buff *skb)
 787 {
 788         struct rtable *rt = skb_rtable(skb);
 789         struct in_device *in_dev;
 790         struct inet_peer *peer;
 791         struct net *net;
 792         int log_martians;
 793
 794         rcu_read_lock();
 795         in_dev = __in_dev_get_rcu(rt->dst.dev);
 796         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 797                 rcu_read_unlock();
 798                 return;
 799         }
 800         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 801         rcu_read_unlock();
 802
 803         net = dev_net(rt->dst.dev);
 804         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 805         if (!peer) {
 806                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 807                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 808                 return;
 809         }
 810
 811         /* No redirected packets during ip_rt_redirect_silence;
 812          * reset the algorithm.
 813          */
 814         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 815                 peer->rate_tokens = 0;
 816
 817         /* Too many ignored redirects; do not send anything
 818          * set dst.rate_last to the last seen redirected packet.
 819          */
 820         if (peer->rate_tokens >= ip_rt_redirect_number) {
 821                 peer->rate_last = jiffies;
 822                 goto out_put_peer;
 823         }
 824
 825         /* Check for load limit; set rate_last to the latest sent
 826          * redirect.
 827          */
 828         if (peer->rate_tokens == 0 ||
 829             time_after(jiffies,
 830                        (peer->rate_last +
 831                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 832                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 833
 834                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 835                 peer->rate_last = jiffies;
 836                 ++peer->rate_tokens;
 837 #ifdef CONFIG_IP_ROUTE_VERBOSE
 838                 if (log_martians &&
 839                     peer->rate_tokens == ip_rt_redirect_number)
 840                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 841                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 842                                              &ip_hdr(skb)->daddr, &gw);
 843 #endif
 844         }
 845 out_put_peer:
 846         inet_putpeer(peer);
 847 }
 848
 849 static int ip_error(struct sk_buff *skb)
 850 {
 851         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 852         struct rtable *rt = skb_rtable(skb);
 853         struct inet_peer *peer;
 854         unsigned long now;
 855         struct net *net;
 856         bool send;
 857         int code;
 858
 859         net = dev_net(rt->dst.dev);
 860         if (!IN_DEV_FORWARD(in_dev)) {
 861                 switch (rt->dst.error) {
 862                 case EHOSTUNREACH:
 863                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 864                         break;
 865
 866                 case ENETUNREACH:
 867                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 868                         break;
 869                 }
 870                 goto out;
 871         }
 872
 873         switch (rt->dst.error) {
 874         case EINVAL:
 875         default:
 876                 goto out;
 877         case EHOSTUNREACH:
 878                 code = ICMP_HOST_UNREACH;
 879                 break;
 880         case ENETUNREACH:
 881                 code = ICMP_NET_UNREACH;
 882                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 883                 break;
 884         case EACCES:
 885                 code = ICMP_PKT_FILTERED;
 886                 break;
 887         }
 888
 889         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 890
 891         send = true;
 892         if (peer) {
 893                 now = jiffies;
 894                 peer->rate_tokens += now - peer->rate_last;
 895                 if (peer->rate_tokens > ip_rt_error_burst)
 896                         peer->rate_tokens = ip_rt_error_burst;
 897                 peer->rate_last = now;
 898                 if (peer->rate_tokens >= ip_rt_error_cost)
 899                         peer->rate_tokens -= ip_rt_error_cost;
 900                 else
 901                         send = false;
 902                 inet_putpeer(peer);
 903         }
 904         if (send)
 905                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 906
 907 out:    kfree_skb(skb);
 908         return 0;
 909 }
 910
 911 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 912 {
 913         struct dst_entry *dst = &rt->dst;
 914         struct fib_result res;
 915
 916         if (dst_metric_locked(dst, RTAX_MTU))
 917                 return;
 918
 919         if (dst->dev->mtu < mtu)
 920                 return;
 921
 922         if (mtu < ip_rt_min_pmtu)
 923                 mtu = ip_rt_min_pmtu;
 924
 925         if (!rt->rt_pmtu) {
 926                 dst->obsolete = DST_OBSOLETE_KILL;
 927         } else {
 928                 rt->rt_pmtu = mtu;
 929                 dst->expires = max(1UL, jiffies + ip_rt_mtu_expires);
 930         }
 931
 932         rcu_read_lock();
 933         if (fib_lookup(dev_net(dst->dev), fl4, &res) == 0) {
 934                 struct fib_nh *nh = &FIB_RES_NH(res);
 935
 936                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 937                                       jiffies + ip_rt_mtu_expires);
 938         }
 939         rcu_read_unlock();
 940 }
 941
 942 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 943                               struct sk_buff *skb, u32 mtu)
 944 {
 945         struct rtable *rt = (struct rtable *) dst;
 946         struct flowi4 fl4;
 947
 948         ip_rt_build_flow_key(&fl4, sk, skb);
 949         __ip_rt_update_pmtu(rt, &fl4, mtu);
 950 }
 951
 952 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
 953                       int oif, u32 mark, u8 protocol, int flow_flags)
 954 {
 955         const struct iphdr *iph = (const struct iphdr *) skb->data;
 956         struct flowi4 fl4;
 957         struct rtable *rt;
 958
 959         if (!mark)
 960                 mark = IP4_REPLY_MARK(net, skb->mark);
 961
 962         __build_flow_key(&fl4, NULL, iph, oif,
 963                          RT_TOS(iph->tos), protocol, mark, flow_flags);
 964         rt = __ip_route_output_key(net, &fl4);
 965         if (!IS_ERR(rt)) {
 966                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 967                 ip_rt_put(rt);
 968         }
 969 }
 970 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
 971
 972 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 973 {
 974         const struct iphdr *iph = (const struct iphdr *) skb->data;
 975         struct flowi4 fl4;
 976         struct rtable *rt;
 977
 978         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 979
 980         if (!fl4.flowi4_mark)
 981                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
 982
 983         rt = __ip_route_output_key(sock_net(sk), &fl4);
 984         if (!IS_ERR(rt)) {
 985                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 986                 ip_rt_put(rt);
 987         }
 988 }
 989
 990 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 991 {
 992         const struct iphdr *iph = (const struct iphdr *) skb->data;
 993         struct flowi4 fl4;
 994         struct rtable *rt;
 995         struct dst_entry *odst = NULL;
 996         bool new = false;
 997
 998         bh_lock_sock(sk);
 999         odst = sk_dst_get(sk);
1000
1001         if (sock_owned_by_user(sk) || !odst) {
1002                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1003                 goto out;
1004         }
1005
1006         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1007
1008         rt = (struct rtable *)odst;
1009         if (odst->obsolete && odst->ops->check(odst, 0) == NULL) {
1010                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1011                 if (IS_ERR(rt))
1012                         goto out;
1013
1014                 new = true;
1015         }
1016
1017         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1018
1019         if (!dst_check(&rt->dst, 0)) {
1020                 if (new)
1021                         dst_release(&rt->dst);
1022
1023                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1024                 if (IS_ERR(rt))
1025                         goto out;
1026
1027                 new = true;
1028         }
1029
1030         if (new)
1031                 sk_dst_set(sk, &rt->dst);
1032
1033 out:
1034         bh_unlock_sock(sk);
1035         dst_release(odst);
1036 }
1037 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1038
1039 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1040                    int oif, u32 mark, u8 protocol, int flow_flags)
1041 {
1042         const struct iphdr *iph = (const struct iphdr *) skb->data;
1043         struct flowi4 fl4;
1044         struct rtable *rt;
1045
1046         __build_flow_key(&fl4, NULL, iph, oif,
1047                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1048         rt = __ip_route_output_key(net, &fl4);
1049         if (!IS_ERR(rt)) {
1050                 __ip_do_redirect(rt, skb, &fl4, false);
1051                 ip_rt_put(rt);
1052         }
1053 }
1054 EXPORT_SYMBOL_GPL(ipv4_redirect);
1055
1056 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1057 {
1058         const struct iphdr *iph = (const struct iphdr *) skb->data;
1059         struct flowi4 fl4;
1060         struct rtable *rt;
1061
1062         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1063         rt = __ip_route_output_key(sock_net(sk), &fl4);
1064         if (!IS_ERR(rt)) {
1065                 __ip_do_redirect(rt, skb, &fl4, false);
1066                 ip_rt_put(rt);
1067         }
1068 }
1069 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1070
1071 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1072 {
1073         struct rtable *rt = (struct rtable *) dst;
1074
1075         /* All IPV4 dsts are created with ->obsolete set to the value
1076          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1077          * into this function always.
1078          *
1079          * When a PMTU/redirect information update invalidates a
1080          * route, this is indicated by setting obsolete to
1081          * DST_OBSOLETE_KILL.
1082          */
1083         if (dst->obsolete == DST_OBSOLETE_KILL || rt_is_expired(rt))
1084                 return NULL;
1085         return dst;
1086 }
1087
1088 static void ipv4_link_failure(struct sk_buff *skb)
1089 {
1090         struct rtable *rt;
1091
1092         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1093
1094         rt = skb_rtable(skb);
1095         if (rt)
1096                 dst_set_expires(&rt->dst, 0);
1097 }
1098
1099 static int ip_rt_bug(struct sk_buff *skb)
1100 {
1101         pr_debug("%s: %pI4 -> %pI4, %s\n",
1102                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1103                  skb->dev ? skb->dev->name : "?");
1104         kfree_skb(skb);
1105         WARN_ON(1);
1106         return 0;
1107 }
1108
1109 /*
1110    We do not cache source address of outgoing interface,
1111    because it is used only by IP RR, TS and SRR options,
1112    so that it out of fast path.
1113
1114    BTW remember: "addr" is allowed to be not aligned
1115    in IP options!
1116  */
1117
1118 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1119 {
1120         __be32 src;
1121
1122         if (rt_is_output_route(rt))
1123                 src = ip_hdr(skb)->saddr;
1124         else {
1125                 struct fib_result res;
1126                 struct flowi4 fl4;
1127                 struct iphdr *iph;
1128
1129                 iph = ip_hdr(skb);
1130
1131                 memset(&fl4, 0, sizeof(fl4));
1132                 fl4.daddr = iph->daddr;
1133                 fl4.saddr = iph->saddr;
1134                 fl4.flowi4_tos = RT_TOS(iph->tos);
1135                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1136                 fl4.flowi4_iif = skb->dev->ifindex;
1137                 fl4.flowi4_mark = skb->mark;
1138
1139                 rcu_read_lock();
1140                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1141                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1142                 else
1143                         src = inet_select_addr(rt->dst.dev,
1144                                                rt_nexthop(rt, iph->daddr),
1145                                                RT_SCOPE_UNIVERSE);
1146                 rcu_read_unlock();
1147         }
1148         memcpy(addr, &src, 4);
1149 }
1150
1151 #ifdef CONFIG_IP_ROUTE_CLASSID
1152 static void set_class_tag(struct rtable *rt, u32 tag)
1153 {
1154         if (!(rt->dst.tclassid & 0xFFFF))
1155                 rt->dst.tclassid |= tag & 0xFFFF;
1156         if (!(rt->dst.tclassid & 0xFFFF0000))
1157                 rt->dst.tclassid |= tag & 0xFFFF0000;
1158 }
1159 #endif
1160
1161 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1162 {
1163         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1164
1165         if (advmss == 0) {
1166                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1167                                ip_rt_min_advmss);
1168                 if (advmss > 65535 - 40)
1169                         advmss = 65535 - 40;
1170         }
1171         return advmss;
1172 }
1173
1174 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1175 {
1176         const struct rtable *rt = (const struct rtable *) dst;
1177         unsigned int mtu = rt->rt_pmtu;
1178
1179         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1180                 mtu = dst_metric_raw(dst, RTAX_MTU);
1181
1182         if (mtu)
1183                 return mtu;
1184
1185         mtu = dst->dev->mtu;
1186
1187         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1188                 if (rt->rt_uses_gateway && mtu > 576)
1189                         mtu = 576;
1190         }
1191
1192         if (mtu > IP_MAX_MTU)
1193                 mtu = IP_MAX_MTU;
1194
1195         return mtu;
1196 }
1197
1198 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1199 {
1200         struct fnhe_hash_bucket *hash = nh->nh_exceptions;
1201         struct fib_nh_exception *fnhe;
1202         u32 hval;
1203
1204         if (!hash)
1205                 return NULL;
1206
1207         hval = fnhe_hashfun(daddr);
1208
1209         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1210              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1211                 if (fnhe->fnhe_daddr == daddr)
1212                         return fnhe;
1213         }
1214         return NULL;
1215 }
1216
1217 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1218                               __be32 daddr)
1219 {
1220         bool ret = false;
1221
1222         spin_lock_bh(&fnhe_lock);
1223
1224         if (daddr == fnhe->fnhe_daddr) {
1225                 struct rtable *orig = rcu_dereference(fnhe->fnhe_rth);
1226                 if (orig && rt_is_expired(orig)) {
1227                         fnhe->fnhe_gw = 0;
1228                         fnhe->fnhe_pmtu = 0;
1229                         fnhe->fnhe_expires = 0;
1230                 }
1231                 if (fnhe->fnhe_pmtu) {
1232                         unsigned long expires = fnhe->fnhe_expires;
1233                         unsigned long diff = expires - jiffies;
1234
1235                         if (time_before(jiffies, expires)) {
1236                                 rt->rt_pmtu = fnhe->fnhe_pmtu;
1237                                 dst_set_expires(&rt->dst, diff);
1238                         }
1239                 }
1240                 if (fnhe->fnhe_gw) {
1241                         rt->rt_flags |= RTCF_REDIRECTED;
1242                         rt->rt_gateway = fnhe->fnhe_gw;
1243                         rt->rt_uses_gateway = 1;
1244                 } else if (!rt->rt_gateway)
1245                         rt->rt_gateway = daddr;
1246
1247                 rcu_assign_pointer(fnhe->fnhe_rth, rt);
1248                 if (orig)
1249                         rt_free(orig);
1250
1251                 fnhe->fnhe_stamp = jiffies;
1252                 ret = true;
1253         }
1254         spin_unlock_bh(&fnhe_lock);
1255
1256         return ret;
1257 }
1258
1259 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1260 {
1261         struct rtable *orig, *prev, **p;
1262         bool ret = true;
1263
1264         if (rt_is_input_route(rt)) {
1265                 p = (struct rtable **)&nh->nh_rth_input;
1266         } else {
1267                 p = (struct rtable **)__this_cpu_ptr(nh->nh_pcpu_rth_output);
1268         }
1269         orig = *p;
1270
1271         prev = cmpxchg(p, orig, rt);
1272         if (prev == orig) {
1273                 if (orig)
1274                         rt_free(orig);
1275         } else
1276                 ret = false;
1277
1278         return ret;
1279 }
1280
1281 static DEFINE_SPINLOCK(rt_uncached_lock);
1282 static LIST_HEAD(rt_uncached_list);
1283
1284 static void rt_add_uncached_list(struct rtable *rt)
1285 {
1286         spin_lock_bh(&rt_uncached_lock);
1287         list_add_tail(&rt->rt_uncached, &rt_uncached_list);
1288         spin_unlock_bh(&rt_uncached_lock);
1289 }
1290
1291 static void ipv4_dst_destroy(struct dst_entry *dst)
1292 {
1293         struct rtable *rt = (struct rtable *) dst;
1294
1295         if (!list_empty(&rt->rt_uncached)) {
1296                 spin_lock_bh(&rt_uncached_lock);
1297                 list_del(&rt->rt_uncached);
1298                 spin_unlock_bh(&rt_uncached_lock);
1299         }
1300 }
1301
1302 void rt_flush_dev(struct net_device *dev)
1303 {
1304         if (!list_empty(&rt_uncached_list)) {
1305                 struct net *net = dev_net(dev);
1306                 struct rtable *rt;
1307
1308                 spin_lock_bh(&rt_uncached_lock);
1309                 list_for_each_entry(rt, &rt_uncached_list, rt_uncached) {
1310                         if (rt->dst.dev != dev)
1311                                 continue;
1312                         rt->dst.dev = net->loopback_dev;
1313                         dev_hold(rt->dst.dev);
1314                         dev_put(dev);
1315                 }
1316                 spin_unlock_bh(&rt_uncached_lock);
1317         }
1318 }
1319
1320 static bool rt_cache_valid(const struct rtable *rt)
1321 {
1322         return  rt &&
1323                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1324                 !rt_is_expired(rt);
1325 }
1326
1327 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1328                            const struct fib_result *res,
1329                            struct fib_nh_exception *fnhe,
1330                            struct fib_info *fi, u16 type, u32 itag)
1331 {
1332         bool cached = false;
1333
1334         if (fi) {
1335                 struct fib_nh *nh = &FIB_RES_NH(*res);
1336
1337                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1338                         rt->rt_gateway = nh->nh_gw;
1339                         rt->rt_uses_gateway = 1;
1340                 }
1341                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1342 #ifdef CONFIG_IP_ROUTE_CLASSID
1343                 rt->dst.tclassid = nh->nh_tclassid;
1344 #endif
1345                 if (unlikely(fnhe))
1346                         cached = rt_bind_exception(rt, fnhe, daddr);
1347                 else if (!(rt->dst.flags & DST_NOCACHE))
1348                         cached = rt_cache_route(nh, rt);
1349                 if (unlikely(!cached)) {
1350                         /* Routes we intend to cache in nexthop exception or
1351                          * FIB nexthop have the DST_NOCACHE bit clear.
1352                          * However, if we are unsuccessful at storing this
1353                          * route into the cache we really need to set it.
1354                          */
1355                         rt->dst.flags |= DST_NOCACHE;
1356                         if (!rt->rt_gateway)
1357                                 rt->rt_gateway = daddr;
1358                         rt_add_uncached_list(rt);
1359                 }
1360         } else
1361                 rt_add_uncached_list(rt);
1362
1363 #ifdef CONFIG_IP_ROUTE_CLASSID
1364 #ifdef CONFIG_IP_MULTIPLE_TABLES
1365         set_class_tag(rt, res->tclassid);
1366 #endif
1367         set_class_tag(rt, itag);
1368 #endif
1369 }
1370
1371 static struct rtable *rt_dst_alloc(struct net_device *dev,
1372                                    bool nopolicy, bool noxfrm, bool will_cache)
1373 {
1374         return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1375                          (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1376                          (nopolicy ? DST_NOPOLICY : 0) |
1377                          (noxfrm ? DST_NOXFRM : 0));
1378 }
1379
1380 /* called in rcu_read_lock() section */
1381 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1382                                 u8 tos, struct net_device *dev, int our)
1383 {
1384         struct rtable *rth;
1385         struct in_device *in_dev = __in_dev_get_rcu(dev);
1386         u32 itag = 0;
1387         int err;
1388
1389         /* Primary sanity checks. */
1390
1391         if (in_dev == NULL)
1392                 return -EINVAL;
1393
1394         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1395             skb->protocol != htons(ETH_P_IP))
1396                 goto e_inval;
1397
1398         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1399                 if (ipv4_is_loopback(saddr))
1400                         goto e_inval;
1401
1402         if (ipv4_is_zeronet(saddr)) {
1403                 if (!ipv4_is_local_multicast(daddr))
1404                         goto e_inval;
1405         } else {
1406                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1407                                           in_dev, &itag);
1408                 if (err < 0)
1409                         goto e_err;
1410         }
1411         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1412                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1413         if (!rth)
1414                 goto e_nobufs;
1415
1416 #ifdef CONFIG_IP_ROUTE_CLASSID
1417         rth->dst.tclassid = itag;
1418 #endif
1419         rth->dst.output = ip_rt_bug;
1420
1421         rth->rt_genid   = rt_genid(dev_net(dev));
1422         rth->rt_flags   = RTCF_MULTICAST;
1423         rth->rt_type    = RTN_MULTICAST;
1424         rth->rt_is_input= 1;
1425         rth->rt_iif     = 0;
1426         rth->rt_pmtu    = 0;
1427         rth->rt_gateway = 0;
1428         rth->rt_uses_gateway = 0;
1429         INIT_LIST_HEAD(&rth->rt_uncached);
1430         if (our) {
1431                 rth->dst.input= ip_local_deliver;
1432                 rth->rt_flags |= RTCF_LOCAL;
1433         }
1434
1435 #ifdef CONFIG_IP_MROUTE
1436         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1437                 rth->dst.input = ip_mr_input;
1438 #endif
1439         RT_CACHE_STAT_INC(in_slow_mc);
1440
1441         skb_dst_set(skb, &rth->dst);
1442         return 0;
1443
1444 e_nobufs:
1445         return -ENOBUFS;
1446 e_inval:
1447         return -EINVAL;
1448 e_err:
1449         return err;
1450 }
1451
1452
1453 static void ip_handle_martian_source(struct net_device *dev,
1454                                      struct in_device *in_dev,
1455                                      struct sk_buff *skb,
1456                                      __be32 daddr,
1457                                      __be32 saddr)
1458 {
1459         RT_CACHE_STAT_INC(in_martian_src);
1460 #ifdef CONFIG_IP_ROUTE_VERBOSE
1461         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1462                 /*
1463                  *      RFC1812 recommendation, if source is martian,
1464                  *      the only hint is MAC header.
1465                  */
1466                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1467                         &daddr, &saddr, dev->name);
1468                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1469                         print_hex_dump(KERN_WARNING, "ll header: ",
1470                                        DUMP_PREFIX_OFFSET, 16, 1,
1471                                        skb_mac_header(skb),
1472                                        dev->hard_header_len, true);
1473                 }
1474         }
1475 #endif
1476 }
1477
1478 /* called in rcu_read_lock() section */
1479 static int __mkroute_input(struct sk_buff *skb,
1480                            const struct fib_result *res,
1481                            struct in_device *in_dev,
1482                            __be32 daddr, __be32 saddr, u32 tos)
1483 {
1484         struct rtable *rth;
1485         int err;
1486         struct in_device *out_dev;
1487         unsigned int flags = 0;
1488         bool do_cache;
1489         u32 itag = 0;
1490
1491         /* get a working reference to the output device */
1492         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1493         if (out_dev == NULL) {
1494                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1495                 return -EINVAL;
1496         }
1497
1498         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1499                                   in_dev->dev, in_dev, &itag);
1500         if (err < 0) {
1501                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1502                                          saddr);
1503
1504                 goto cleanup;
1505         }
1506
1507         do_cache = res->fi && !itag;
1508         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1509             (IN_DEV_SHARED_MEDIA(out_dev) ||
1510              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res)))) {
1511                 flags |= RTCF_DOREDIRECT;
1512                 do_cache = false;
1513         }
1514
1515         if (skb->protocol != htons(ETH_P_IP)) {
1516                 /* Not IP (i.e. ARP). Do not create route, if it is
1517                  * invalid for proxy arp. DNAT routes are always valid.
1518                  *
1519                  * Proxy arp feature have been extended to allow, ARP
1520                  * replies back to the same interface, to support
1521                  * Private VLAN switch technologies. See arp.c.
1522                  */
1523                 if (out_dev == in_dev &&
1524                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1525                         err = -EINVAL;
1526                         goto cleanup;
1527                 }
1528         }
1529
1530         if (do_cache) {
1531                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1532                 if (rt_cache_valid(rth)) {
1533                         skb_dst_set_noref(skb, &rth->dst);
1534                         goto out;
1535                 }
1536         }
1537
1538         rth = rt_dst_alloc(out_dev->dev,
1539                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1540                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1541         if (!rth) {
1542                 err = -ENOBUFS;
1543                 goto cleanup;
1544         }
1545
1546         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
1547         rth->rt_flags = flags;
1548         rth->rt_type = res->type;
1549         rth->rt_is_input = 1;
1550         rth->rt_iif     = 0;
1551         rth->rt_pmtu    = 0;
1552         rth->rt_gateway = 0;
1553         rth->rt_uses_gateway = 0;
1554         INIT_LIST_HEAD(&rth->rt_uncached);
1555         RT_CACHE_STAT_INC(in_slow_tot);
1556
1557         rth->dst.input = ip_forward;
1558         rth->dst.output = ip_output;
1559
1560         rt_set_nexthop(rth, daddr, res, NULL, res->fi, res->type, itag);
1561         skb_dst_set(skb, &rth->dst);
1562 out:
1563         err = 0;
1564  cleanup:
1565         return err;
1566 }
1567
1568 static int ip_mkroute_input(struct sk_buff *skb,
1569                             struct fib_result *res,
1570                             const struct flowi4 *fl4,
1571                             struct in_device *in_dev,
1572                             __be32 daddr, __be32 saddr, u32 tos)
1573 {
1574 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1575         if (res->fi && res->fi->fib_nhs > 1)
1576                 fib_select_multipath(res);
1577 #endif
1578
1579         /* create a routing cache entry */
1580         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1581 }
1582
1583 /*
1584  *      NOTE. We drop all the packets that has local source
1585  *      addresses, because every properly looped back packet
1586  *      must have correct destination already attached by output routine.
1587  *
1588  *      Such approach solves two big problems:
1589  *      1. Not simplex devices are handled properly.
1590  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1591  *      called with rcu_read_lock()
1592  */
1593
1594 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1595                                u8 tos, struct net_device *dev)
1596 {
1597         struct fib_result res;
1598         struct in_device *in_dev = __in_dev_get_rcu(dev);
1599         struct flowi4   fl4;
1600         unsigned int    flags = 0;
1601         u32             itag = 0;
1602         struct rtable   *rth;
1603         int             err = -EINVAL;
1604         struct net    *net = dev_net(dev);
1605         bool do_cache;
1606
1607         /* IP on this device is disabled. */
1608
1609         if (!in_dev)
1610                 goto out;
1611
1612         /* Check for the most weird martians, which can be not detected
1613            by fib_lookup.
1614          */
1615
1616         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1617                 goto martian_source;
1618
1619         res.fi = NULL;
1620         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1621                 goto brd_input;
1622
1623         /* Accept zero addresses only to limited broadcast;
1624          * I even do not know to fix it or not. Waiting for complains :-)
1625          */
1626         if (ipv4_is_zeronet(saddr))
1627                 goto martian_source;
1628
1629         if (ipv4_is_zeronet(daddr))
1630                 goto martian_destination;
1631
1632         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1633          * and call it once if daddr or/and saddr are loopback addresses
1634          */
1635         if (ipv4_is_loopback(daddr)) {
1636                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1637                         goto martian_destination;
1638         } else if (ipv4_is_loopback(saddr)) {
1639                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1640                         goto martian_source;
1641         }
1642
1643         /*
1644          *      Now we are ready to route packet.
1645          */
1646         fl4.flowi4_oif = 0;
1647         fl4.flowi4_iif = dev->ifindex;
1648         fl4.flowi4_mark = skb->mark;
1649         fl4.flowi4_tos = tos;
1650         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1651         fl4.daddr = daddr;
1652         fl4.saddr = saddr;
1653         err = fib_lookup(net, &fl4, &res);
1654         if (err != 0)
1655                 goto no_route;
1656
1657         if (res.type == RTN_BROADCAST)
1658                 goto brd_input;
1659
1660         if (res.type == RTN_LOCAL) {
1661                 err = fib_validate_source(skb, saddr, daddr, tos,
1662                                           LOOPBACK_IFINDEX,
1663                                           dev, in_dev, &itag);
1664                 if (err < 0)
1665                         goto martian_source_keep_err;
1666                 goto local_input;
1667         }
1668
1669         if (!IN_DEV_FORWARD(in_dev))
1670                 goto no_route;
1671         if (res.type != RTN_UNICAST)
1672                 goto martian_destination;
1673
1674         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1675 out:    return err;
1676
1677 brd_input:
1678         if (skb->protocol != htons(ETH_P_IP))
1679                 goto e_inval;
1680
1681         if (!ipv4_is_zeronet(saddr)) {
1682                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1683                                           in_dev, &itag);
1684                 if (err < 0)
1685                         goto martian_source_keep_err;
1686         }
1687         flags |= RTCF_BROADCAST;
1688         res.type = RTN_BROADCAST;
1689         RT_CACHE_STAT_INC(in_brd);
1690
1691 local_input:
1692         do_cache = false;
1693         if (res.fi) {
1694                 if (!itag) {
1695                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1696                         if (rt_cache_valid(rth)) {
1697                                 skb_dst_set_noref(skb, &rth->dst);
1698                                 err = 0;
1699                                 goto out;
1700                         }
1701                         do_cache = true;
1702                 }
1703         }
1704
1705         rth = rt_dst_alloc(net->loopback_dev,
1706                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1707         if (!rth)
1708                 goto e_nobufs;
1709
1710         rth->dst.input= ip_local_deliver;
1711         rth->dst.output= ip_rt_bug;
1712 #ifdef CONFIG_IP_ROUTE_CLASSID
1713         rth->dst.tclassid = itag;
1714 #endif
1715
1716         rth->rt_genid = rt_genid(net);
1717         rth->rt_flags   = flags|RTCF_LOCAL;
1718         rth->rt_type    = res.type;
1719         rth->rt_is_input = 1;
1720         rth->rt_iif     = 0;
1721         rth->rt_pmtu    = 0;
1722         rth->rt_gateway = 0;
1723         rth->rt_uses_gateway = 0;
1724         INIT_LIST_HEAD(&rth->rt_uncached);
1725         RT_CACHE_STAT_INC(in_slow_tot);
1726         if (res.type == RTN_UNREACHABLE) {
1727                 rth->dst.input= ip_error;
1728                 rth->dst.error= -err;
1729                 rth->rt_flags   &= ~RTCF_LOCAL;
1730         }
1731         if (do_cache) {
1732                 if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
1733                         rth->dst.flags |= DST_NOCACHE;
1734                         rt_add_uncached_list(rth);
1735                 }
1736         }
1737         skb_dst_set(skb, &rth->dst);
1738         err = 0;
1739         goto out;
1740
1741 no_route:
1742         RT_CACHE_STAT_INC(in_no_route);
1743         res.type = RTN_UNREACHABLE;
1744         if (err == -ESRCH)
1745                 err = -ENETUNREACH;
1746         goto local_input;
1747
1748         /*
1749          *      Do not cache martian addresses: they should be logged (RFC1812)
1750          */
1751 martian_destination:
1752         RT_CACHE_STAT_INC(in_martian_dst);
1753 #ifdef CONFIG_IP_ROUTE_VERBOSE
1754         if (IN_DEV_LOG_MARTIANS(in_dev))
1755                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1756                                      &daddr, &saddr, dev->name);
1757 #endif
1758
1759 e_inval:
1760         err = -EINVAL;
1761         goto out;
1762
1763 e_nobufs:
1764         err = -ENOBUFS;
1765         goto out;
1766
1767 martian_source:
1768         err = -EINVAL;
1769 martian_source_keep_err:
1770         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1771         goto out;
1772 }
1773
1774 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1775                          u8 tos, struct net_device *dev)
1776 {
1777         int res;
1778
1779         rcu_read_lock();
1780
1781         /* Multicast recognition logic is moved from route cache to here.
1782            The problem was that too many Ethernet cards have broken/missing
1783            hardware multicast filters :-( As result the host on multicasting
1784            network acquires a lot of useless route cache entries, sort of
1785            SDR messages from all the world. Now we try to get rid of them.
1786            Really, provided software IP multicast filter is organized
1787            reasonably (at least, hashed), it does not result in a slowdown
1788            comparing with route cache reject entries.
1789            Note, that multicast routers are not affected, because
1790            route cache entry is created eventually.
1791          */
1792         if (ipv4_is_multicast(daddr)) {
1793                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1794
1795                 if (in_dev) {
1796                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1797                                                   ip_hdr(skb)->protocol);
1798                         if (our
1799 #ifdef CONFIG_IP_MROUTE
1800                                 ||
1801                             (!ipv4_is_local_multicast(daddr) &&
1802                              IN_DEV_MFORWARD(in_dev))
1803 #endif
1804                            ) {
1805                                 int res = ip_route_input_mc(skb, daddr, saddr,
1806                                                             tos, dev, our);
1807                                 rcu_read_unlock();
1808                                 return res;
1809                         }
1810                 }
1811                 rcu_read_unlock();
1812                 return -EINVAL;
1813         }
1814         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1815         rcu_read_unlock();
1816         return res;
1817 }
1818 EXPORT_SYMBOL(ip_route_input_noref);
1819
1820 /* called with rcu_read_lock() */
1821 static struct rtable *__mkroute_output(const struct fib_result *res,
1822                                        const struct flowi4 *fl4, int orig_oif,
1823                                        struct net_device *dev_out,
1824                                        unsigned int flags)
1825 {
1826         struct fib_info *fi = res->fi;
1827         struct fib_nh_exception *fnhe;
1828         struct in_device *in_dev;
1829         u16 type = res->type;
1830         struct rtable *rth;
1831         bool do_cache;
1832
1833         in_dev = __in_dev_get_rcu(dev_out);
1834         if (!in_dev)
1835                 return ERR_PTR(-EINVAL);
1836
1837         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1838                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
1839                         return ERR_PTR(-EINVAL);
1840
1841         if (ipv4_is_lbcast(fl4->daddr))
1842                 type = RTN_BROADCAST;
1843         else if (ipv4_is_multicast(fl4->daddr))
1844                 type = RTN_MULTICAST;
1845         else if (ipv4_is_zeronet(fl4->daddr))
1846                 return ERR_PTR(-EINVAL);
1847
1848         if (dev_out->flags & IFF_LOOPBACK)
1849                 flags |= RTCF_LOCAL;
1850
1851         do_cache = true;
1852         if (type == RTN_BROADCAST) {
1853                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
1854                 fi = NULL;
1855         } else if (type == RTN_MULTICAST) {
1856                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
1857                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
1858                                      fl4->flowi4_proto))
1859                         flags &= ~RTCF_LOCAL;
1860                 else
1861                         do_cache = false;
1862                 /* If multicast route do not exist use
1863                  * default one, but do not gateway in this case.
1864                  * Yes, it is hack.
1865                  */
1866                 if (fi && res->prefixlen < 4)
1867                         fi = NULL;
1868         }
1869
1870         fnhe = NULL;
1871         do_cache &= fi != NULL;
1872         if (do_cache) {
1873                 struct rtable __rcu **prth;
1874                 struct fib_nh *nh = &FIB_RES_NH(*res);
1875
1876                 fnhe = find_exception(nh, fl4->daddr);
1877                 if (fnhe)
1878                         prth = &fnhe->fnhe_rth;
1879                 else {
1880                         if (unlikely(fl4->flowi4_flags &
1881                                      FLOWI_FLAG_KNOWN_NH &&
1882                                      !(nh->nh_gw &&
1883                                        nh->nh_scope == RT_SCOPE_LINK))) {
1884                                 do_cache = false;
1885                                 goto add;
1886                         }
1887                         prth = __this_cpu_ptr(nh->nh_pcpu_rth_output);
1888                 }
1889                 rth = rcu_dereference(*prth);
1890                 if (rt_cache_valid(rth)) {
1891                         dst_hold(&rth->dst);
1892                         return rth;
1893                 }
1894         }
1895
1896 add:
1897         rth = rt_dst_alloc(dev_out,
1898                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1899                            IN_DEV_CONF_GET(in_dev, NOXFRM),
1900                            do_cache);
1901         if (!rth)
1902                 return ERR_PTR(-ENOBUFS);
1903
1904         rth->dst.output = ip_output;
1905
1906         rth->rt_genid = rt_genid(dev_net(dev_out));
1907         rth->rt_flags   = flags;
1908         rth->rt_type    = type;
1909         rth->rt_is_input = 0;
1910         rth->rt_iif     = orig_oif ? : 0;
1911         rth->rt_pmtu    = 0;
1912         rth->rt_gateway = 0;
1913         rth->rt_uses_gateway = 0;
1914         INIT_LIST_HEAD(&rth->rt_uncached);
1915
1916         RT_CACHE_STAT_INC(out_slow_tot);
1917
1918         if (flags & RTCF_LOCAL)
1919                 rth->dst.input = ip_local_deliver;
1920         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
1921                 if (flags & RTCF_LOCAL &&
1922                     !(dev_out->flags & IFF_LOOPBACK)) {
1923                         rth->dst.output = ip_mc_output;
1924                         RT_CACHE_STAT_INC(out_slow_mc);
1925                 }
1926 #ifdef CONFIG_IP_MROUTE
1927                 if (type == RTN_MULTICAST) {
1928                         if (IN_DEV_MFORWARD(in_dev) &&
1929                             !ipv4_is_local_multicast(fl4->daddr)) {
1930                                 rth->dst.input = ip_mr_input;
1931                                 rth->dst.output = ip_mc_output;
1932                         }
1933                 }
1934 #endif
1935         }
1936
1937         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
1938
1939         return rth;
1940 }
1941
1942 /*
1943  * Major route resolver routine.
1944  */
1945
1946 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4)
1947 {
1948         struct net_device *dev_out = NULL;
1949         __u8 tos = RT_FL_TOS(fl4);
1950         unsigned int flags = 0;
1951         struct fib_result res;
1952         struct rtable *rth;
1953         int orig_oif;
1954
1955         res.tclassid    = 0;
1956         res.fi          = NULL;
1957         res.table       = NULL;
1958
1959         orig_oif = fl4->flowi4_oif;
1960
1961         fl4->flowi4_iif = LOOPBACK_IFINDEX;
1962         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
1963         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
1964                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
1965
1966         rcu_read_lock();
1967         if (fl4->saddr) {
1968                 rth = ERR_PTR(-EINVAL);
1969                 if (ipv4_is_multicast(fl4->saddr) ||
1970                     ipv4_is_lbcast(fl4->saddr) ||
1971                     ipv4_is_zeronet(fl4->saddr))
1972                         goto out;
1973
1974                 /* I removed check for oif == dev_out->oif here.
1975                    It was wrong for two reasons:
1976                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
1977                       is assigned to multiple interfaces.
1978                    2. Moreover, we are allowed to send packets with saddr
1979                       of another iface. --ANK
1980                  */
1981
1982                 if (fl4->flowi4_oif == 0 &&
1983                     (ipv4_is_multicast(fl4->daddr) ||
1984                      ipv4_is_lbcast(fl4->daddr))) {
1985                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1986                         dev_out = __ip_dev_find(net, fl4->saddr, false);
1987                         if (dev_out == NULL)
1988                                 goto out;
1989
1990                         /* Special hack: user can direct multicasts
1991                            and limited broadcast via necessary interface
1992                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
1993                            This hack is not just for fun, it allows
1994                            vic,vat and friends to work.
1995                            They bind socket to loopback, set ttl to zero
1996                            and expect that it will work.
1997                            From the viewpoint of routing cache they are broken,
1998                            because we are not allowed to build multicast path
1999                            with loopback source addr (look, routing cache
2000                            cannot know, that ttl is zero, so that packet
2001                            will not leave this host and route is valid).
2002                            Luckily, this hack is good workaround.
2003                          */
2004
2005                         fl4->flowi4_oif = dev_out->ifindex;
2006                         goto make_route;
2007                 }
2008
2009                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2010                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2011                         if (!__ip_dev_find(net, fl4->saddr, false))
2012                                 goto out;
2013                 }
2014         }
2015
2016
2017         if (fl4->flowi4_oif) {
2018                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2019                 rth = ERR_PTR(-ENODEV);
2020                 if (dev_out == NULL)
2021                         goto out;
2022
2023                 /* RACE: Check return value of inet_select_addr instead. */
2024                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2025                         rth = ERR_PTR(-ENETUNREACH);
2026                         goto out;
2027                 }
2028                 if (ipv4_is_local_multicast(fl4->daddr) ||
2029                     ipv4_is_lbcast(fl4->daddr)) {
2030                         if (!fl4->saddr)
2031                                 fl4->saddr = inet_select_addr(dev_out, 0,
2032                                                               RT_SCOPE_LINK);
2033                         goto make_route;
2034                 }
2035                 if (!fl4->saddr) {
2036                         if (ipv4_is_multicast(fl4->daddr))
2037                                 fl4->saddr = inet_select_addr(dev_out, 0,
2038                                                               fl4->flowi4_scope);
2039                         else if (!fl4->daddr)
2040                                 fl4->saddr = inet_select_addr(dev_out, 0,
2041                                                               RT_SCOPE_HOST);
2042                 }
2043         }
2044
2045         if (!fl4->daddr) {
2046                 fl4->daddr = fl4->saddr;
2047                 if (!fl4->daddr)
2048                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2049                 dev_out = net->loopback_dev;
2050                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2051                 res.type = RTN_LOCAL;
2052                 flags |= RTCF_LOCAL;
2053                 goto make_route;
2054         }
2055
2056         if (fib_lookup(net, fl4, &res)) {
2057                 res.fi = NULL;
2058                 res.table = NULL;
2059                 if (fl4->flowi4_oif) {
2060                         /* Apparently, routing tables are wrong. Assume,
2061                            that the destination is on link.
2062
2063                            WHY? DW.
2064                            Because we are allowed to send to iface
2065                            even if it has NO routes and NO assigned
2066                            addresses. When oif is specified, routing
2067                            tables are looked up with only one purpose:
2068                            to catch if destination is gatewayed, rather than
2069                            direct. Moreover, if MSG_DONTROUTE is set,
2070                            we send packet, ignoring both routing tables
2071                            and ifaddr state. --ANK
2072
2073
2074                            We could make it even if oif is unknown,
2075                            likely IPv6, but we do not.
2076                          */
2077
2078                         if (fl4->saddr == 0)
2079                                 fl4->saddr = inet_select_addr(dev_out, 0,
2080                                                               RT_SCOPE_LINK);
2081                         res.type = RTN_UNICAST;
2082                         goto make_route;
2083                 }
2084                 rth = ERR_PTR(-ENETUNREACH);
2085                 goto out;
2086         }
2087
2088         if (res.type == RTN_LOCAL) {
2089                 if (!fl4->saddr) {
2090                         if (res.fi->fib_prefsrc)
2091                                 fl4->saddr = res.fi->fib_prefsrc;
2092                         else
2093                                 fl4->saddr = fl4->daddr;
2094                 }
2095                 dev_out = net->loopback_dev;
2096                 fl4->flowi4_oif = dev_out->ifindex;
2097                 flags |= RTCF_LOCAL;
2098                 goto make_route;
2099         }
2100
2101 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2102         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2103                 fib_select_multipath(&res);
2104         else
2105 #endif
2106         if (!res.prefixlen &&
2107             res.table->tb_num_default > 1 &&
2108             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2109                 fib_select_default(&res);
2110
2111         if (!fl4->saddr)
2112                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2113
2114         dev_out = FIB_RES_DEV(res);
2115         fl4->flowi4_oif = dev_out->ifindex;
2116
2117
2118 make_route:
2119         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2120
2121 out:
2122         rcu_read_unlock();
2123         return rth;
2124 }
2125 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2126
2127 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2128 {
2129         return NULL;
2130 }
2131
2132 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2133 {
2134         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2135
2136         return mtu ? : dst->dev->mtu;
2137 }
2138
2139 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2140                                           struct sk_buff *skb, u32 mtu)
2141 {
2142 }
2143
2144 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2145                                        struct sk_buff *skb)
2146 {
2147 }
2148
2149 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2150                                           unsigned long old)
2151 {
2152         return NULL;
2153 }
2154
2155 static struct dst_ops ipv4_dst_blackhole_ops = {
2156         .family                 =       AF_INET,
2157         .protocol               =       cpu_to_be16(ETH_P_IP),
2158         .check                  =       ipv4_blackhole_dst_check,
2159         .mtu                    =       ipv4_blackhole_mtu,
2160         .default_advmss         =       ipv4_default_advmss,
2161         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2162         .redirect               =       ipv4_rt_blackhole_redirect,
2163         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2164         .neigh_lookup           =       ipv4_neigh_lookup,
2165 };
2166
2167 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2168 {
2169         struct rtable *ort = (struct rtable *) dst_orig;
2170         struct rtable *rt;
2171
2172         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2173         if (rt) {
2174                 struct dst_entry *new = &rt->dst;
2175
2176                 new->__use = 1;
2177                 new->input = dst_discard;
2178                 new->output = dst_discard;
2179
2180                 new->dev = ort->dst.dev;
2181                 if (new->dev)
2182                         dev_hold(new->dev);
2183
2184                 rt->rt_is_input = ort->rt_is_input;
2185                 rt->rt_iif = ort->rt_iif;
2186                 rt->rt_pmtu = ort->rt_pmtu;
2187
2188                 rt->rt_genid = rt_genid(net);
2189                 rt->rt_flags = ort->rt_flags;
2190                 rt->rt_type = ort->rt_type;
2191                 rt->rt_gateway = ort->rt_gateway;
2192                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2193
2194                 INIT_LIST_HEAD(&rt->rt_uncached);
2195
2196                 dst_free(new);
2197         }
2198
2199         dst_release(dst_orig);
2200
2201         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2202 }
2203
2204 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2205                                     struct sock *sk)
2206 {
2207         struct rtable *rt = __ip_route_output_key(net, flp4);
2208
2209         if (IS_ERR(rt))
2210                 return rt;
2211
2212         if (flp4->flowi4_proto)
2213                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2214                                                    flowi4_to_flowi(flp4),
2215                                                    sk, 0);
2216
2217         return rt;
2218 }
2219 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2220
2221 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src,
2222                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2223                         u32 seq, int event, int nowait, unsigned int flags)
2224 {
2225         struct rtable *rt = skb_rtable(skb);
2226         struct rtmsg *r;
2227         struct nlmsghdr *nlh;
2228         unsigned long expires = 0;
2229         u32 error;
2230         u32 metrics[RTAX_MAX];
2231
2232         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2233         if (nlh == NULL)
2234                 return -EMSGSIZE;
2235
2236         r = nlmsg_data(nlh);
2237         r->rtm_family    = AF_INET;
2238         r->rtm_dst_len  = 32;
2239         r->rtm_src_len  = 0;
2240         r->rtm_tos      = fl4->flowi4_tos;
2241         r->rtm_table    = RT_TABLE_MAIN;
2242         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2243                 goto nla_put_failure;
2244         r->rtm_type     = rt->rt_type;
2245         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2246         r->rtm_protocol = RTPROT_UNSPEC;
2247         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2248         if (rt->rt_flags & RTCF_NOTIFY)
2249                 r->rtm_flags |= RTM_F_NOTIFY;
2250
2251         if (nla_put_be32(skb, RTA_DST, dst))
2252                 goto nla_put_failure;
2253         if (src) {
2254                 r->rtm_src_len = 32;
2255                 if (nla_put_be32(skb, RTA_SRC, src))
2256                         goto nla_put_failure;
2257         }
2258         if (rt->dst.dev &&
2259             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2260                 goto nla_put_failure;
2261 #ifdef CONFIG_IP_ROUTE_CLASSID
2262         if (rt->dst.tclassid &&
2263             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2264                 goto nla_put_failure;
2265 #endif
2266         if (!rt_is_input_route(rt) &&
2267             fl4->saddr != src) {
2268                 if (nla_put_be32(skb, RTA_PREFSRC, fl4->saddr))
2269                         goto nla_put_failure;
2270         }
2271         if (rt->rt_uses_gateway &&
2272             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2273                 goto nla_put_failure;
2274
2275         expires = rt->dst.expires;
2276         if (expires) {
2277                 unsigned long now = jiffies;
2278
2279                 if (time_before(now, expires))
2280                         expires -= now;
2281                 else
2282                         expires = 0;
2283         }
2284
2285         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2286         if (rt->rt_pmtu && expires)
2287                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2288         if (rtnetlink_put_metrics(skb, metrics) < 0)
2289                 goto nla_put_failure;
2290
2291         if (fl4->flowi4_mark &&
2292             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2293                 goto nla_put_failure;
2294
2295         error = rt->dst.error;
2296
2297         if (rt_is_input_route(rt)) {
2298 #ifdef CONFIG_IP_MROUTE
2299                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2300                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2301                         int err = ipmr_get_route(net, skb,
2302                                                  fl4->saddr, fl4->daddr,
2303                                                  r, nowait);
2304                         if (err <= 0) {
2305                                 if (!nowait) {
2306                                         if (err == 0)
2307                                                 return 0;
2308                                         goto nla_put_failure;
2309                                 } else {
2310                                         if (err == -EMSGSIZE)
2311                                                 goto nla_put_failure;
2312                                         error = err;
2313                                 }
2314                         }
2315                 } else
2316 #endif
2317                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2318                                 goto nla_put_failure;
2319         }
2320
2321         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2322                 goto nla_put_failure;
2323
2324         return nlmsg_end(skb, nlh);
2325
2326 nla_put_failure:
2327         nlmsg_cancel(skb, nlh);
2328         return -EMSGSIZE;
2329 }
2330
2331 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2332 {
2333         struct net *net = sock_net(in_skb->sk);
2334         struct rtmsg *rtm;
2335         struct nlattr *tb[RTA_MAX+1];
2336         struct rtable *rt = NULL;
2337         struct flowi4 fl4;
2338         __be32 dst = 0;
2339         __be32 src = 0;
2340         u32 iif;
2341         int err;
2342         int mark;
2343         struct sk_buff *skb;
2344
2345         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2346         if (err < 0)
2347                 goto errout;
2348
2349         rtm = nlmsg_data(nlh);
2350
2351         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2352         if (skb == NULL) {
2353                 err = -ENOBUFS;
2354                 goto errout;
2355         }
2356
2357         /* Reserve room for dummy headers, this skb can pass
2358            through good chunk of routing engine.
2359          */
2360         skb_reset_mac_header(skb);
2361         skb_reset_network_header(skb);
2362
2363         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2364         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2365         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2366
2367         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2368         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2369         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2370         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2371
2372         memset(&fl4, 0, sizeof(fl4));
2373         fl4.daddr = dst;
2374         fl4.saddr = src;
2375         fl4.flowi4_tos = rtm->rtm_tos;
2376         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2377         fl4.flowi4_mark = mark;
2378
2379         if (iif) {
2380                 struct net_device *dev;
2381
2382                 dev = __dev_get_by_index(net, iif);
2383                 if (dev == NULL) {
2384                         err = -ENODEV;
2385                         goto errout_free;
2386                 }
2387
2388                 skb->protocol   = htons(ETH_P_IP);
2389                 skb->dev        = dev;
2390                 skb->mark       = mark;
2391                 local_bh_disable();
2392                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2393                 local_bh_enable();
2394
2395                 rt = skb_rtable(skb);
2396                 if (err == 0 && rt->dst.error)
2397                         err = -rt->dst.error;
2398         } else {
2399                 rt = ip_route_output_key(net, &fl4);
2400
2401                 err = 0;
2402                 if (IS_ERR(rt))
2403                         err = PTR_ERR(rt);
2404         }
2405
2406         if (err)
2407                 goto errout_free;
2408
2409         skb_dst_set(skb, &rt->dst);
2410         if (rtm->rtm_flags & RTM_F_NOTIFY)
2411                 rt->rt_flags |= RTCF_NOTIFY;
2412
2413         err = rt_fill_info(net, dst, src, &fl4, skb,
2414                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2415                            RTM_NEWROUTE, 0, 0);
2416         if (err <= 0)
2417                 goto errout_free;
2418
2419         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2420 errout:
2421         return err;
2422
2423 errout_free:
2424         kfree_skb(skb);
2425         goto errout;
2426 }
2427
2428 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2429 {
2430         return skb->len;
2431 }
2432
2433 void ip_rt_multicast_event(struct in_device *in_dev)
2434 {
2435         rt_cache_flush(dev_net(in_dev->dev));
2436 }
2437
2438 #ifdef CONFIG_SYSCTL
2439 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
2440 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2441 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2442 static int ip_rt_gc_elasticity __read_mostly    = 8;
2443
2444 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
2445                                         void __user *buffer,
2446                                         size_t *lenp, loff_t *ppos)
2447 {
2448         if (write) {
2449                 rt_cache_flush((struct net *)__ctl->extra1);
2450                 return 0;
2451         }
2452
2453         return -EINVAL;
2454 }
2455
2456 static ctl_table ipv4_route_table[] = {
2457         {
2458                 .procname       = "gc_thresh",
2459                 .data           = &ipv4_dst_ops.gc_thresh,
2460                 .maxlen         = sizeof(int),
2461                 .mode           = 0644,
2462                 .proc_handler   = proc_dointvec,
2463         },
2464         {
2465                 .procname       = "max_size",
2466                 .data           = &ip_rt_max_size,
2467                 .maxlen         = sizeof(int),
2468                 .mode           = 0644,
2469                 .proc_handler   = proc_dointvec,
2470         },
2471         {
2472                 /*  Deprecated. Use gc_min_interval_ms */
2473
2474                 .procname       = "gc_min_interval",
2475                 .data           = &ip_rt_gc_min_interval,
2476                 .maxlen         = sizeof(int),
2477                 .mode           = 0644,
2478                 .proc_handler   = proc_dointvec_jiffies,
2479         },
2480         {
2481                 .procname       = "gc_min_interval_ms",
2482                 .data           = &ip_rt_gc_min_interval,
2483                 .maxlen         = sizeof(int),
2484                 .mode           = 0644,
2485                 .proc_handler   = proc_dointvec_ms_jiffies,
2486         },
2487         {
2488                 .procname       = "gc_timeout",
2489                 .data           = &ip_rt_gc_timeout,
2490                 .maxlen         = sizeof(int),
2491                 .mode           = 0644,
2492                 .proc_handler   = proc_dointvec_jiffies,
2493         },
2494         {
2495                 .procname       = "gc_interval",
2496                 .data           = &ip_rt_gc_interval,
2497                 .maxlen         = sizeof(int),
2498                 .mode           = 0644,
2499                 .proc_handler   = proc_dointvec_jiffies,
2500         },
2501         {
2502                 .procname       = "redirect_load",
2503                 .data           = &ip_rt_redirect_load,
2504                 .maxlen         = sizeof(int),
2505                 .mode           = 0644,
2506                 .proc_handler   = proc_dointvec,
2507         },
2508         {
2509                 .procname       = "redirect_number",
2510                 .data           = &ip_rt_redirect_number,
2511                 .maxlen         = sizeof(int),
2512                 .mode           = 0644,
2513                 .proc_handler   = proc_dointvec,
2514         },
2515         {
2516                 .procname       = "redirect_silence",
2517                 .data           = &ip_rt_redirect_silence,
2518                 .maxlen         = sizeof(int),
2519                 .mode           = 0644,
2520                 .proc_handler   = proc_dointvec,
2521         },
2522         {
2523                 .procname       = "error_cost",
2524                 .data           = &ip_rt_error_cost,
2525                 .maxlen         = sizeof(int),
2526                 .mode           = 0644,
2527                 .proc_handler   = proc_dointvec,
2528         },
2529         {
2530                 .procname       = "error_burst",
2531                 .data           = &ip_rt_error_burst,
2532                 .maxlen         = sizeof(int),
2533                 .mode           = 0644,
2534                 .proc_handler   = proc_dointvec,
2535         },
2536         {
2537                 .procname       = "gc_elasticity",
2538                 .data           = &ip_rt_gc_elasticity,
2539                 .maxlen         = sizeof(int),
2540                 .mode           = 0644,
2541                 .proc_handler   = proc_dointvec,
2542         },
2543         {
2544                 .procname       = "mtu_expires",
2545                 .data           = &ip_rt_mtu_expires,
2546                 .maxlen         = sizeof(int),
2547                 .mode           = 0644,
2548                 .proc_handler   = proc_dointvec_jiffies,
2549         },
2550         {
2551                 .procname       = "min_pmtu",
2552                 .data           = &ip_rt_min_pmtu,
2553                 .maxlen         = sizeof(int),
2554                 .mode           = 0644,
2555                 .proc_handler   = proc_dointvec,
2556         },
2557         {
2558                 .procname       = "min_adv_mss",
2559                 .data           = &ip_rt_min_advmss,
2560                 .maxlen         = sizeof(int),
2561                 .mode           = 0644,
2562                 .proc_handler   = proc_dointvec,
2563         },
2564         { }
2565 };
2566
2567 static struct ctl_table ipv4_route_flush_table[] = {
2568         {
2569                 .procname       = "flush",
2570                 .maxlen         = sizeof(int),
2571                 .mode           = 0200,
2572                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2573         },
2574         { },
2575 };
2576
2577 static __net_init int sysctl_route_net_init(struct net *net)
2578 {
2579         struct ctl_table *tbl;
2580
2581         tbl = ipv4_route_flush_table;
2582         if (!net_eq(net, &init_net)) {
2583                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2584                 if (tbl == NULL)
2585                         goto err_dup;
2586
2587                 /* Don't export sysctls to unprivileged users */
2588                 if (net->user_ns != &init_user_ns)
2589                         tbl[0].procname = NULL;
2590         }
2591         tbl[0].extra1 = net;
2592
2593         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2594         if (net->ipv4.route_hdr == NULL)
2595                 goto err_reg;
2596         return 0;
2597
2598 err_reg:
2599         if (tbl != ipv4_route_flush_table)
2600                 kfree(tbl);
2601 err_dup:
2602         return -ENOMEM;
2603 }
2604
2605 static __net_exit void sysctl_route_net_exit(struct net *net)
2606 {
2607         struct ctl_table *tbl;
2608
2609         tbl = net->ipv4.route_hdr->ctl_table_arg;
2610         unregister_net_sysctl_table(net->ipv4.route_hdr);
2611         BUG_ON(tbl == ipv4_route_flush_table);
2612         kfree(tbl);
2613 }
2614
2615 static __net_initdata struct pernet_operations sysctl_route_ops = {
2616         .init = sysctl_route_net_init,
2617         .exit = sysctl_route_net_exit,
2618 };
2619 #endif
2620
2621 static __net_init int rt_genid_init(struct net *net)
2622 {
2623         atomic_set(&net->rt_genid, 0);
2624         get_random_bytes(&net->ipv4.dev_addr_genid,
2625                          sizeof(net->ipv4.dev_addr_genid));
2626         return 0;
2627 }
2628
2629 static __net_initdata struct pernet_operations rt_genid_ops = {
2630         .init = rt_genid_init,
2631 };
2632
2633 static int __net_init ipv4_inetpeer_init(struct net *net)
2634 {
2635         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2636
2637         if (!bp)
2638                 return -ENOMEM;
2639         inet_peer_base_init(bp);
2640         net->ipv4.peers = bp;
2641         return 0;
2642 }
2643
2644 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2645 {
2646         struct inet_peer_base *bp = net->ipv4.peers;
2647
2648         net->ipv4.peers = NULL;
2649         inetpeer_invalidate_tree(bp);
2650         kfree(bp);
2651 }
2652
2653 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2654         .init   =       ipv4_inetpeer_init,
2655         .exit   =       ipv4_inetpeer_exit,
2656 };
2657
2658 #ifdef CONFIG_IP_ROUTE_CLASSID
2659 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2660 #endif /* CONFIG_IP_ROUTE_CLASSID */
2661
2662 int __init ip_rt_init(void)
2663 {
2664         int rc = 0;
2665
2666 #ifdef CONFIG_IP_ROUTE_CLASSID
2667         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2668         if (!ip_rt_acct)
2669                 panic("IP: failed to allocate ip_rt_acct\n");
2670 #endif
2671
2672         ipv4_dst_ops.kmem_cachep =
2673                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2674                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2675
2676         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2677
2678         if (dst_entries_init(&ipv4_dst_ops) < 0)
2679                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2680
2681         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2682                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2683
2684         ipv4_dst_ops.gc_thresh = ~0;
2685         ip_rt_max_size = INT_MAX;
2686
2687         devinet_init();
2688         ip_fib_init();
2689
2690         if (ip_rt_proc_init())
2691                 pr_err("Unable to create route proc files\n");
2692 #ifdef CONFIG_XFRM
2693         xfrm_init();
2694         xfrm4_init();
2695 #endif
2696         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2697
2698 #ifdef CONFIG_SYSCTL
2699         register_pernet_subsys(&sysctl_route_ops);
2700 #endif
2701         register_pernet_subsys(&rt_genid_ops);
2702         register_pernet_subsys(&ipv4_inetpeer_ops);
2703         return rc;
2704 }
2705
2706 #ifdef CONFIG_SYSCTL
2707 /*
2708  * We really need to sanitize the damn ipv4 init order, then all
2709  * this nonsense will go away.
2710  */
2711 void __init ip_static_sysctl_init(void)
2712 {
2713         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2714 }
2715 #endif