core/neighbour.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      Generic address resolution entity
   4  *
   5  *      Authors:
   6  *      Pedro Roque             <roque@di.fc.ul.pt>
   7  *      Alexey Kuznetsov        <kuznet@ms2.inr.ac.ru>
   8  *
   9  *      Fixes:
  10  *      Vitaly E. Lavrov        releasing NULL neighbor in neigh_add.
  11  *      Harald Welte            Add neighbour cache statistics like rtstat
  12  */
  13
  14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15
  16 #include <linux/slab.h>
  17 #include <linux/kmemleak.h>
  18 #include <linux/types.h>
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/socket.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/proc_fs.h>
  24 #ifdef CONFIG_SYSCTL
  25 #include <linux/sysctl.h>
  26 #endif
  27 #include <linux/times.h>
  28 #include <net/net_namespace.h>
  29 #include <net/neighbour.h>
  30 #include <net/arp.h>
  31 #include <net/dst.h>
  32 #include <net/sock.h>
  33 #include <net/netevent.h>
  34 #include <net/netlink.h>
  35 #include <linux/rtnetlink.h>
  36 #include <linux/random.h>
  37 #include <linux/string.h>
  38 #include <linux/log2.h>
  39 #include <linux/inetdevice.h>
  40 #include <net/addrconf.h>
  41
  42 #include <trace/events/neigh.h>
  43
  44 #define NEIGH_DEBUG 1
  45 #define neigh_dbg(level, fmt, ...)              \
  46 do {                                            \
  47         if (level <= NEIGH_DEBUG)               \
  48                 pr_debug(fmt, ##__VA_ARGS__);   \
  49 } while (0)
  50
  51 #define PNEIGH_HASHMASK         0xF
  52
  53 static void neigh_timer_handler(struct timer_list *t);
  54 static void __neigh_notify(struct neighbour *n, int type, int flags,
  55                            u32 pid);
  56 static void neigh_update_notify(struct neighbour *neigh, u32 nlmsg_pid);
  57 static int pneigh_ifdown_and_unlock(struct neigh_table *tbl,
  58                                     struct net_device *dev);
  59
  60 #ifdef CONFIG_PROC_FS
  61 static const struct seq_operations neigh_stat_seq_ops;
  62 #endif
  63
  64 /*
  65    Neighbour hash table buckets are protected with rwlock tbl->lock.
  66
  67    - All the scans/updates to hash buckets MUST be made under this lock.
  68    - NOTHING clever should be made under this lock: no callbacks
  69      to protocol backends, no attempts to send something to network.
  70      It will result in deadlocks, if backend/driver wants to use neighbour
  71      cache.
  72    - If the entry requires some non-trivial actions, increase
  73      its reference count and release table lock.
  74
  75    Neighbour entries are protected:
  76    - with reference count.
  77    - with rwlock neigh->lock
  78
  79    Reference count prevents destruction.
  80
  81    neigh->lock mainly serializes ll address data and its validity state.
  82    However, the same lock is used to protect another entry fields:
  83     - timer
  84     - resolution queue
  85
  86    Again, nothing clever shall be made under neigh->lock,
  87    the most complicated procedure, which we allow is dev->hard_header.
  88    It is supposed, that dev->hard_header is simplistic and does
  89    not make callbacks to neighbour tables.
  90  */
  91
  92 static int neigh_blackhole(struct neighbour *neigh, struct sk_buff *skb)
  93 {
  94         kfree_skb(skb);
  95         return -ENETDOWN;
  96 }
  97
  98 static void neigh_cleanup_and_release(struct neighbour *neigh)
  99 {
 100         trace_neigh_cleanup_and_release(neigh, 0);
 101         __neigh_notify(neigh, RTM_DELNEIGH, 0, 0);
 102         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
 103         neigh_release(neigh);
 104 }
 105
 106 /*
 107  * It is random distribution in the interval (1/2)*base...(3/2)*base.
 108  * It corresponds to default IPv6 settings and is not overridable,
 109  * because it is really reasonable choice.
 110  */
 111
 112 unsigned long neigh_rand_reach_time(unsigned long base)
 113 {
 114         return base ? prandom_u32_max(base) + (base >> 1) : 0;
 115 }
 116 EXPORT_SYMBOL(neigh_rand_reach_time);
 117
 118 static void neigh_mark_dead(struct neighbour *n)
 119 {
 120         n->dead = 1;
 121         if (!list_empty(&n->gc_list)) {
 122                 list_del_init(&n->gc_list);
 123                 atomic_dec(&n->tbl->gc_entries);
 124         }
 125         if (!list_empty(&n->managed_list))
 126                 list_del_init(&n->managed_list);
 127 }
 128
 129 static void neigh_update_gc_list(struct neighbour *n)
 130 {
 131         bool on_gc_list, exempt_from_gc;
 132
 133         write_lock_bh(&n->tbl->lock);
 134         write_lock(&n->lock);
 135         if (n->dead)
 136                 goto out;
 137
 138         /* remove from the gc list if new state is permanent or if neighbor
 139          * is externally learned; otherwise entry should be on the gc list
 140          */
 141         exempt_from_gc = n->nud_state & NUD_PERMANENT ||
 142                          n->flags & NTF_EXT_LEARNED;
 143         on_gc_list = !list_empty(&n->gc_list);
 144
 145         if (exempt_from_gc && on_gc_list) {
 146                 list_del_init(&n->gc_list);
 147                 atomic_dec(&n->tbl->gc_entries);
 148         } else if (!exempt_from_gc && !on_gc_list) {
 149                 /* add entries to the tail; cleaning removes from the front */
 150                 list_add_tail(&n->gc_list, &n->tbl->gc_list);
 151                 atomic_inc(&n->tbl->gc_entries);
 152         }
 153 out:
 154         write_unlock(&n->lock);
 155         write_unlock_bh(&n->tbl->lock);
 156 }
 157
 158 static void neigh_update_managed_list(struct neighbour *n)
 159 {
 160         bool on_managed_list, add_to_managed;
 161
 162         write_lock_bh(&n->tbl->lock);
 163         write_lock(&n->lock);
 164         if (n->dead)
 165                 goto out;
 166
 167         add_to_managed = n->flags & NTF_MANAGED;
 168         on_managed_list = !list_empty(&n->managed_list);
 169
 170         if (!add_to_managed && on_managed_list)
 171                 list_del_init(&n->managed_list);
 172         else if (add_to_managed && !on_managed_list)
 173                 list_add_tail(&n->managed_list, &n->tbl->managed_list);
 174 out:
 175         write_unlock(&n->lock);
 176         write_unlock_bh(&n->tbl->lock);
 177 }
 178
 179 static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify,
 180                                bool *gc_update, bool *managed_update)
 181 {
 182         u32 ndm_flags, old_flags = neigh->flags;
 183
 184         if (!(flags & NEIGH_UPDATE_F_ADMIN))
 185                 return;
 186
 187         ndm_flags  = (flags & NEIGH_UPDATE_F_EXT_LEARNED) ? NTF_EXT_LEARNED : 0;
 188         ndm_flags |= (flags & NEIGH_UPDATE_F_MANAGED) ? NTF_MANAGED : 0;
 189
 190         if ((old_flags ^ ndm_flags) & NTF_EXT_LEARNED) {
 191                 if (ndm_flags & NTF_EXT_LEARNED)
 192                         neigh->flags |= NTF_EXT_LEARNED;
 193                 else
 194                         neigh->flags &= ~NTF_EXT_LEARNED;
 195                 *notify = 1;
 196                 *gc_update = true;
 197         }
 198         if ((old_flags ^ ndm_flags) & NTF_MANAGED) {
 199                 if (ndm_flags & NTF_MANAGED)
 200                         neigh->flags |= NTF_MANAGED;
 201                 else
 202                         neigh->flags &= ~NTF_MANAGED;
 203                 *notify = 1;
 204                 *managed_update = true;
 205         }
 206 }
 207
 208 static bool neigh_del(struct neighbour *n, struct neighbour __rcu **np,
 209                       struct neigh_table *tbl)
 210 {
 211         bool retval = false;
 212
 213         write_lock(&n->lock);
 214         if (refcount_read(&n->refcnt) == 1) {
 215                 struct neighbour *neigh;
 216
 217                 neigh = rcu_dereference_protected(n->next,
 218                                                   lockdep_is_held(&tbl->lock));
 219                 rcu_assign_pointer(*np, neigh);
 220                 neigh_mark_dead(n);
 221                 retval = true;
 222         }
 223         write_unlock(&n->lock);
 224         if (retval)
 225                 neigh_cleanup_and_release(n);
 226         return retval;
 227 }
 228
 229 bool neigh_remove_one(struct neighbour *ndel, struct neigh_table *tbl)
 230 {
 231         struct neigh_hash_table *nht;
 232         void *pkey = ndel->primary_key;
 233         u32 hash_val;
 234         struct neighbour *n;
 235         struct neighbour __rcu **np;
 236
 237         nht = rcu_dereference_protected(tbl->nht,
 238                                         lockdep_is_held(&tbl->lock));
 239         hash_val = tbl->hash(pkey, ndel->dev, nht->hash_rnd);
 240         hash_val = hash_val >> (32 - nht->hash_shift);
 241
 242         np = &nht->hash_buckets[hash_val];
 243         while ((n = rcu_dereference_protected(*np,
 244                                               lockdep_is_held(&tbl->lock)))) {
 245                 if (n == ndel)
 246                         return neigh_del(n, np, tbl);
 247                 np = &n->next;
 248         }
 249         return false;
 250 }
 251
 252 static int neigh_forced_gc(struct neigh_table *tbl)
 253 {
 254         int max_clean = atomic_read(&tbl->gc_entries) -
 255                         READ_ONCE(tbl->gc_thresh2);
 256         u64 tmax = ktime_get_ns() + NSEC_PER_MSEC;
 257         unsigned long tref = jiffies - 5 * HZ;
 258         struct neighbour *n, *tmp;
 259         int shrunk = 0;
 260         int loop = 0;
 261
 262         NEIGH_CACHE_STAT_INC(tbl, forced_gc_runs);
 263
 264         write_lock_bh(&tbl->lock);
 265
 266         list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) {
 267                 if (refcount_read(&n->refcnt) == 1) {
 268                         bool remove = false;
 269
 270                         write_lock(&n->lock);
 271                         if ((n->nud_state == NUD_FAILED) ||
 272                             (n->nud_state == NUD_NOARP) ||
 273                             (tbl->is_multicast &&
 274                              tbl->is_multicast(n->primary_key)) ||
 275                             !time_in_range(n->updated, tref, jiffies))
 276                                 remove = true;
 277                         write_unlock(&n->lock);
 278
 279                         if (remove && neigh_remove_one(n, tbl))
 280                                 shrunk++;
 281                         if (shrunk >= max_clean)
 282                                 break;
 283                         if (++loop == 16) {
 284                                 if (ktime_get_ns() > tmax)
 285                                         goto unlock;
 286                                 loop = 0;
 287                         }
 288                 }
 289         }
 290
 291         WRITE_ONCE(tbl->last_flush, jiffies);
 292 unlock:
 293         write_unlock_bh(&tbl->lock);
 294
 295         return shrunk;
 296 }
 297
 298 static void neigh_add_timer(struct neighbour *n, unsigned long when)
 299 {
 300         /* Use safe distance from the jiffies - LONG_MAX point while timer
 301          * is running in DELAY/PROBE state but still show to user space
 302          * large times in the past.
 303          */
 304         unsigned long mint = jiffies - (LONG_MAX - 86400 * HZ);
 305
 306         neigh_hold(n);
 307         if (!time_in_range(n->confirmed, mint, jiffies))
 308                 n->confirmed = mint;
 309         if (time_before(n->used, n->confirmed))
 310                 n->used = n->confirmed;
 311         if (unlikely(mod_timer(&n->timer, when))) {
 312                 printk("NEIGH: BUG, double timer add, state is %x\n",
 313                        n->nud_state);
 314                 dump_stack();
 315         }
 316 }
 317
 318 static int neigh_del_timer(struct neighbour *n)
 319 {
 320         if ((n->nud_state & NUD_IN_TIMER) &&
 321             del_timer(&n->timer)) {
 322                 neigh_release(n);
 323                 return 1;
 324         }
 325         return 0;
 326 }
 327
 328 static struct neigh_parms *neigh_get_dev_parms_rcu(struct net_device *dev,
 329                                                    int family)
 330 {
 331         switch (family) {
 332         case AF_INET:
 333                 return __in_dev_arp_parms_get_rcu(dev);
 334         case AF_INET6:
 335                 return __in6_dev_nd_parms_get_rcu(dev);
 336         }
 337         return NULL;
 338 }
 339
 340 static void neigh_parms_qlen_dec(struct net_device *dev, int family)
 341 {
 342         struct neigh_parms *p;
 343
 344         rcu_read_lock();
 345         p = neigh_get_dev_parms_rcu(dev, family);
 346         if (p)
 347                 p->qlen--;
 348         rcu_read_unlock();
 349 }
 350
 351 static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,
 352                                int family)
 353 {
 354         struct sk_buff_head tmp;
 355         unsigned long flags;
 356         struct sk_buff *skb;
 357
 358         skb_queue_head_init(&tmp);
 359         spin_lock_irqsave(&list->lock, flags);
 360         skb = skb_peek(list);
 361         while (skb != NULL) {
 362                 struct sk_buff *skb_next = skb_peek_next(skb, list);
 363                 struct net_device *dev = skb->dev;
 364
 365                 if (net == NULL || net_eq(dev_net(dev), net)) {
 366                         neigh_parms_qlen_dec(dev, family);
 367                         __skb_unlink(skb, list);
 368                         __skb_queue_tail(&tmp, skb);
 369                 }
 370                 skb = skb_next;
 371         }
 372         spin_unlock_irqrestore(&list->lock, flags);
 373
 374         while ((skb = __skb_dequeue(&tmp))) {
 375                 dev_put(skb->dev);
 376                 kfree_skb(skb);
 377         }
 378 }
 379
 380 static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,
 381                             bool skip_perm)
 382 {
 383         int i;
 384         struct neigh_hash_table *nht;
 385
 386         nht = rcu_dereference_protected(tbl->nht,
 387                                         lockdep_is_held(&tbl->lock));
 388
 389         for (i = 0; i < (1 << nht->hash_shift); i++) {
 390                 struct neighbour *n;
 391                 struct neighbour __rcu **np = &nht->hash_buckets[i];
 392
 393                 while ((n = rcu_dereference_protected(*np,
 394                                         lockdep_is_held(&tbl->lock))) != NULL) {
 395                         if (dev && n->dev != dev) {
 396                                 np = &n->next;
 397                                 continue;
 398                         }
 399                         if (skip_perm && n->nud_state & NUD_PERMANENT) {
 400                                 np = &n->next;
 401                                 continue;
 402                         }
 403                         rcu_assign_pointer(*np,
 404                                    rcu_dereference_protected(n->next,
 405                                                 lockdep_is_held(&tbl->lock)));
 406                         write_lock(&n->lock);
 407                         neigh_del_timer(n);
 408                         neigh_mark_dead(n);
 409                         if (refcount_read(&n->refcnt) != 1) {
 410                                 /* The most unpleasant situation.
 411                                    We must destroy neighbour entry,
 412                                    but someone still uses it.
 413
 414                                    The destroy will be delayed until
 415                                    the last user releases us, but
 416                                    we must kill timers etc. and move
 417                                    it to safe state.
 418                                  */
 419                                 __skb_queue_purge(&n->arp_queue);
 420                                 n->arp_queue_len_bytes = 0;
 421                                 WRITE_ONCE(n->output, neigh_blackhole);
 422                                 if (n->nud_state & NUD_VALID)
 423                                         n->nud_state = NUD_NOARP;
 424                                 else
 425                                         n->nud_state = NUD_NONE;
 426                                 neigh_dbg(2, "neigh %p is stray\n", n);
 427                         }
 428                         write_unlock(&n->lock);
 429                         neigh_cleanup_and_release(n);
 430                 }
 431         }
 432 }
 433
 434 void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev)
 435 {
 436         write_lock_bh(&tbl->lock);
 437         neigh_flush_dev(tbl, dev, false);
 438         write_unlock_bh(&tbl->lock);
 439 }
 440 EXPORT_SYMBOL(neigh_changeaddr);
 441
 442 static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
 443                           bool skip_perm)
 444 {
 445         write_lock_bh(&tbl->lock);
 446         neigh_flush_dev(tbl, dev, skip_perm);
 447         pneigh_ifdown_and_unlock(tbl, dev);
 448         pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL,
 449                            tbl->family);
 450         if (skb_queue_empty_lockless(&tbl->proxy_queue))
 451                 del_timer_sync(&tbl->proxy_timer);
 452         return 0;
 453 }
 454
 455 int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev)
 456 {
 457         __neigh_ifdown(tbl, dev, true);
 458         return 0;
 459 }
 460 EXPORT_SYMBOL(neigh_carrier_down);
 461
 462 int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev)
 463 {
 464         __neigh_ifdown(tbl, dev, false);
 465         return 0;
 466 }
 467 EXPORT_SYMBOL(neigh_ifdown);
 468
 469 static struct neighbour *neigh_alloc(struct neigh_table *tbl,
 470                                      struct net_device *dev,
 471                                      u32 flags, bool exempt_from_gc)
 472 {
 473         struct neighbour *n = NULL;
 474         unsigned long now = jiffies;
 475         int entries, gc_thresh3;
 476
 477         if (exempt_from_gc)
 478                 goto do_alloc;
 479
 480         entries = atomic_inc_return(&tbl->gc_entries) - 1;
 481         gc_thresh3 = READ_ONCE(tbl->gc_thresh3);
 482         if (entries >= gc_thresh3 ||
 483             (entries >= READ_ONCE(tbl->gc_thresh2) &&
 484              time_after(now, READ_ONCE(tbl->last_flush) + 5 * HZ))) {
 485                 if (!neigh_forced_gc(tbl) && entries >= gc_thresh3) {
 486                         net_info_ratelimited("%s: neighbor table overflow!\n",
 487                                              tbl->id);
 488                         NEIGH_CACHE_STAT_INC(tbl, table_fulls);
 489                         goto out_entries;
 490                 }
 491         }
 492
 493 do_alloc:
 494         n = kzalloc(tbl->entry_size + dev->neigh_priv_len, GFP_ATOMIC);
 495         if (!n)
 496                 goto out_entries;
 497
 498         __skb_queue_head_init(&n->arp_queue);
 499         rwlock_init(&n->lock);
 500         seqlock_init(&n->ha_lock);
 501         n->updated        = n->used = now;
 502         n->nud_state      = NUD_NONE;
 503         n->output         = neigh_blackhole;
 504         n->flags          = flags;
 505         seqlock_init(&n->hh.hh_lock);
 506         n->parms          = neigh_parms_clone(&tbl->parms);
 507         timer_setup(&n->timer, neigh_timer_handler, 0);
 508
 509         NEIGH_CACHE_STAT_INC(tbl, allocs);
 510         n->tbl            = tbl;
 511         refcount_set(&n->refcnt, 1);
 512         n->dead           = 1;
 513         INIT_LIST_HEAD(&n->gc_list);
 514         INIT_LIST_HEAD(&n->managed_list);
 515
 516         atomic_inc(&tbl->entries);
 517 out:
 518         return n;
 519
 520 out_entries:
 521         if (!exempt_from_gc)
 522                 atomic_dec(&tbl->gc_entries);
 523         goto out;
 524 }
 525
 526 static void neigh_get_hash_rnd(u32 *x)
 527 {
 528         *x = get_random_u32() | 1;
 529 }
 530
 531 static struct neigh_hash_table *neigh_hash_alloc(unsigned int shift)
 532 {
 533         size_t size = (1 << shift) * sizeof(struct neighbour *);
 534         struct neigh_hash_table *ret;
 535         struct neighbour __rcu **buckets;
 536         int i;
 537
 538         ret = kmalloc(sizeof(*ret), GFP_ATOMIC);
 539         if (!ret)
 540                 return NULL;
 541         if (size <= PAGE_SIZE) {
 542                 buckets = kzalloc(size, GFP_ATOMIC);
 543         } else {
 544                 buckets = (struct neighbour __rcu **)
 545                           __get_free_pages(GFP_ATOMIC | __GFP_ZERO,
 546                                            get_order(size));
 547                 kmemleak_alloc(buckets, size, 1, GFP_ATOMIC);
 548         }
 549         if (!buckets) {
 550                 kfree(ret);
 551                 return NULL;
 552         }
 553         ret->hash_buckets = buckets;
 554         ret->hash_shift = shift;
 555         for (i = 0; i < NEIGH_NUM_HASH_RND; i++)
 556                 neigh_get_hash_rnd(&ret->hash_rnd[i]);
 557         return ret;
 558 }
 559
 560 static void neigh_hash_free_rcu(struct rcu_head *head)
 561 {
 562         struct neigh_hash_table *nht = container_of(head,
 563                                                     struct neigh_hash_table,
 564                                                     rcu);
 565         size_t size = (1 << nht->hash_shift) * sizeof(struct neighbour *);
 566         struct neighbour __rcu **buckets = nht->hash_buckets;
 567
 568         if (size <= PAGE_SIZE) {
 569                 kfree(buckets);
 570         } else {
 571                 kmemleak_free(buckets);
 572                 free_pages((unsigned long)buckets, get_order(size));
 573         }
 574         kfree(nht);
 575 }
 576
 577 static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,
 578                                                 unsigned long new_shift)
 579 {
 580         unsigned int i, hash;
 581         struct neigh_hash_table *new_nht, *old_nht;
 582
 583         NEIGH_CACHE_STAT_INC(tbl, hash_grows);
 584
 585         old_nht = rcu_dereference_protected(tbl->nht,
 586                                             lockdep_is_held(&tbl->lock));
 587         new_nht = neigh_hash_alloc(new_shift);
 588         if (!new_nht)
 589                 return old_nht;
 590
 591         for (i = 0; i < (1 << old_nht->hash_shift); i++) {
 592                 struct neighbour *n, *next;
 593
 594                 for (n = rcu_dereference_protected(old_nht->hash_buckets[i],
 595                                                    lockdep_is_held(&tbl->lock));
 596                      n != NULL;
 597                      n = next) {
 598                         hash = tbl->hash(n->primary_key, n->dev,
 599                                          new_nht->hash_rnd);
 600
 601                         hash >>= (32 - new_nht->hash_shift);
 602                         next = rcu_dereference_protected(n->next,
 603                                                 lockdep_is_held(&tbl->lock));
 604
 605                         rcu_assign_pointer(n->next,
 606                                            rcu_dereference_protected(
 607                                                 new_nht->hash_buckets[hash],
 608                                                 lockdep_is_held(&tbl->lock)));
 609                         rcu_assign_pointer(new_nht->hash_buckets[hash], n);
 610                 }
 611         }
 612
 613         rcu_assign_pointer(tbl->nht, new_nht);
 614         call_rcu(&old_nht->rcu, neigh_hash_free_rcu);
 615         return new_nht;
 616 }
 617
 618 struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
 619                                struct net_device *dev)
 620 {
 621         struct neighbour *n;
 622
 623         NEIGH_CACHE_STAT_INC(tbl, lookups);
 624
 625         rcu_read_lock();
 626         n = __neigh_lookup_noref(tbl, pkey, dev);
 627         if (n) {
 628                 if (!refcount_inc_not_zero(&n->refcnt))
 629                         n = NULL;
 630                 NEIGH_CACHE_STAT_INC(tbl, hits);
 631         }
 632
 633         rcu_read_unlock();
 634         return n;
 635 }
 636 EXPORT_SYMBOL(neigh_lookup);
 637
 638 static struct neighbour *
 639 ___neigh_create(struct neigh_table *tbl, const void *pkey,
 640                 struct net_device *dev, u32 flags,
 641                 bool exempt_from_gc, bool want_ref)
 642 {
 643         u32 hash_val, key_len = tbl->key_len;
 644         struct neighbour *n1, *rc, *n;
 645         struct neigh_hash_table *nht;
 646         int error;
 647
 648         n = neigh_alloc(tbl, dev, flags, exempt_from_gc);
 649         trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);
 650         if (!n) {
 651                 rc = ERR_PTR(-ENOBUFS);
 652                 goto out;
 653         }
 654
 655         memcpy(n->primary_key, pkey, key_len);
 656         n->dev = dev;
 657         netdev_hold(dev, &n->dev_tracker, GFP_ATOMIC);
 658
 659         /* Protocol specific setup. */
 660         if (tbl->constructor && (error = tbl->constructor(n)) < 0) {
 661                 rc = ERR_PTR(error);
 662                 goto out_neigh_release;
 663         }
 664
 665         if (dev->netdev_ops->ndo_neigh_construct) {
 666                 error = dev->netdev_ops->ndo_neigh_construct(dev, n);
 667                 if (error < 0) {
 668                         rc = ERR_PTR(error);
 669                         goto out_neigh_release;
 670                 }
 671         }
 672
 673         /* Device specific setup. */
 674         if (n->parms->neigh_setup &&
 675             (error = n->parms->neigh_setup(n)) < 0) {
 676                 rc = ERR_PTR(error);
 677                 goto out_neigh_release;
 678         }
 679
 680         n->confirmed = jiffies - (NEIGH_VAR(n->parms, BASE_REACHABLE_TIME) << 1);
 681
 682         write_lock_bh(&tbl->lock);
 683         nht = rcu_dereference_protected(tbl->nht,
 684                                         lockdep_is_held(&tbl->lock));
 685
 686         if (atomic_read(&tbl->entries) > (1 << nht->hash_shift))
 687                 nht = neigh_hash_grow(tbl, nht->hash_shift + 1);
 688
 689         hash_val = tbl->hash(n->primary_key, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
 690
 691         if (n->parms->dead) {
 692                 rc = ERR_PTR(-EINVAL);
 693                 goto out_tbl_unlock;
 694         }
 695
 696         for (n1 = rcu_dereference_protected(nht->hash_buckets[hash_val],
 697                                             lockdep_is_held(&tbl->lock));
 698              n1 != NULL;
 699              n1 = rcu_dereference_protected(n1->next,
 700                         lockdep_is_held(&tbl->lock))) {
 701                 if (dev == n1->dev && !memcmp(n1->primary_key, n->primary_key, key_len)) {
 702                         if (want_ref)
 703                                 neigh_hold(n1);
 704                         rc = n1;
 705                         goto out_tbl_unlock;
 706                 }
 707         }
 708
 709         n->dead = 0;
 710         if (!exempt_from_gc)
 711                 list_add_tail(&n->gc_list, &n->tbl->gc_list);
 712         if (n->flags & NTF_MANAGED)
 713                 list_add_tail(&n->managed_list, &n->tbl->managed_list);
 714         if (want_ref)
 715                 neigh_hold(n);
 716         rcu_assign_pointer(n->next,
 717                            rcu_dereference_protected(nht->hash_buckets[hash_val],
 718                                                      lockdep_is_held(&tbl->lock)));
 719         rcu_assign_pointer(nht->hash_buckets[hash_val], n);
 720         write_unlock_bh(&tbl->lock);
 721         neigh_dbg(2, "neigh %p is created\n", n);
 722         rc = n;
 723 out:
 724         return rc;
 725 out_tbl_unlock:
 726         write_unlock_bh(&tbl->lock);
 727 out_neigh_release:
 728         if (!exempt_from_gc)
 729                 atomic_dec(&tbl->gc_entries);
 730         neigh_release(n);
 731         goto out;
 732 }
 733
 734 struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
 735                                  struct net_device *dev, bool want_ref)
 736 {
 737         return ___neigh_create(tbl, pkey, dev, 0, false, want_ref);
 738 }
 739 EXPORT_SYMBOL(__neigh_create);
 740
 741 static u32 pneigh_hash(const void *pkey, unsigned int key_len)
 742 {
 743         u32 hash_val = *(u32 *)(pkey + key_len - 4);
 744         hash_val ^= (hash_val >> 16);
 745         hash_val ^= hash_val >> 8;
 746         hash_val ^= hash_val >> 4;
 747         hash_val &= PNEIGH_HASHMASK;
 748         return hash_val;
 749 }
 750
 751 static struct pneigh_entry *__pneigh_lookup_1(struct pneigh_entry *n,
 752                                               struct net *net,
 753                                               const void *pkey,
 754                                               unsigned int key_len,
 755                                               struct net_device *dev)
 756 {
 757         while (n) {
 758                 if (!memcmp(n->key, pkey, key_len) &&
 759                     net_eq(pneigh_net(n), net) &&
 760                     (n->dev == dev || !n->dev))
 761                         return n;
 762                 n = n->next;
 763         }
 764         return NULL;
 765 }
 766
 767 struct pneigh_entry *__pneigh_lookup(struct neigh_table *tbl,
 768                 struct net *net, const void *pkey, struct net_device *dev)
 769 {
 770         unsigned int key_len = tbl->key_len;
 771         u32 hash_val = pneigh_hash(pkey, key_len);
 772
 773         return __pneigh_lookup_1(tbl->phash_buckets[hash_val],
 774                                  net, pkey, key_len, dev);
 775 }
 776 EXPORT_SYMBOL_GPL(__pneigh_lookup);
 777
 778 struct pneigh_entry * pneigh_lookup(struct neigh_table *tbl,
 779                                     struct net *net, const void *pkey,
 780                                     struct net_device *dev, int creat)
 781 {
 782         struct pneigh_entry *n;
 783         unsigned int key_len = tbl->key_len;
 784         u32 hash_val = pneigh_hash(pkey, key_len);
 785
 786         read_lock_bh(&tbl->lock);
 787         n = __pneigh_lookup_1(tbl->phash_buckets[hash_val],
 788                               net, pkey, key_len, dev);
 789         read_unlock_bh(&tbl->lock);
 790
 791         if (n || !creat)
 792                 goto out;
 793
 794         ASSERT_RTNL();
 795
 796         n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL);
 797         if (!n)
 798                 goto out;
 799
 800         write_pnet(&n->net, net);
 801         memcpy(n->key, pkey, key_len);
 802         n->dev = dev;
 803         netdev_hold(dev, &n->dev_tracker, GFP_KERNEL);
 804
 805         if (tbl->pconstructor && tbl->pconstructor(n)) {
 806                 netdev_put(dev, &n->dev_tracker);
 807                 kfree(n);
 808                 n = NULL;
 809                 goto out;
 810         }
 811
 812         write_lock_bh(&tbl->lock);
 813         n->next = tbl->phash_buckets[hash_val];
 814         tbl->phash_buckets[hash_val] = n;
 815         write_unlock_bh(&tbl->lock);
 816 out:
 817         return n;
 818 }
 819 EXPORT_SYMBOL(pneigh_lookup);
 820
 821
 822 int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,
 823                   struct net_device *dev)
 824 {
 825         struct pneigh_entry *n, **np;
 826         unsigned int key_len = tbl->key_len;
 827         u32 hash_val = pneigh_hash(pkey, key_len);
 828
 829         write_lock_bh(&tbl->lock);
 830         for (np = &tbl->phash_buckets[hash_val]; (n = *np) != NULL;
 831              np = &n->next) {
 832                 if (!memcmp(n->key, pkey, key_len) && n->dev == dev &&
 833                     net_eq(pneigh_net(n), net)) {
 834                         *np = n->next;
 835                         write_unlock_bh(&tbl->lock);
 836                         if (tbl->pdestructor)
 837                                 tbl->pdestructor(n);
 838                         netdev_put(n->dev, &n->dev_tracker);
 839                         kfree(n);
 840                         return 0;
 841                 }
 842         }
 843         write_unlock_bh(&tbl->lock);
 844         return -ENOENT;
 845 }
 846
 847 static int pneigh_ifdown_and_unlock(struct neigh_table *tbl,
 848                                     struct net_device *dev)
 849 {
 850         struct pneigh_entry *n, **np, *freelist = NULL;
 851         u32 h;
 852
 853         for (h = 0; h <= PNEIGH_HASHMASK; h++) {
 854                 np = &tbl->phash_buckets[h];
 855                 while ((n = *np) != NULL) {
 856                         if (!dev || n->dev == dev) {
 857                                 *np = n->next;
 858                                 n->next = freelist;
 859                                 freelist = n;
 860                                 continue;
 861                         }
 862                         np = &n->next;
 863                 }
 864         }
 865         write_unlock_bh(&tbl->lock);
 866         while ((n = freelist)) {
 867                 freelist = n->next;
 868                 n->next = NULL;
 869                 if (tbl->pdestructor)
 870                         tbl->pdestructor(n);
 871                 netdev_put(n->dev, &n->dev_tracker);
 872                 kfree(n);
 873         }
 874         return -ENOENT;
 875 }
 876
 877 static void neigh_parms_destroy(struct neigh_parms *parms);
 878
 879 static inline void neigh_parms_put(struct neigh_parms *parms)
 880 {
 881         if (refcount_dec_and_test(&parms->refcnt))
 882                 neigh_parms_destroy(parms);
 883 }
 884
 885 /*
 886  *      neighbour must already be out of the table;
 887  *
 888  */
 889 void neigh_destroy(struct neighbour *neigh)
 890 {
 891         struct net_device *dev = neigh->dev;
 892
 893         NEIGH_CACHE_STAT_INC(neigh->tbl, destroys);
 894
 895         if (!neigh->dead) {
 896                 pr_warn("Destroying alive neighbour %p\n", neigh);
 897                 dump_stack();
 898                 return;
 899         }
 900
 901         if (neigh_del_timer(neigh))
 902                 pr_warn("Impossible event\n");
 903
 904         write_lock_bh(&neigh->lock);
 905         __skb_queue_purge(&neigh->arp_queue);
 906         write_unlock_bh(&neigh->lock);
 907         neigh->arp_queue_len_bytes = 0;
 908
 909         if (dev->netdev_ops->ndo_neigh_destroy)
 910                 dev->netdev_ops->ndo_neigh_destroy(dev, neigh);
 911
 912         netdev_put(dev, &neigh->dev_tracker);
 913         neigh_parms_put(neigh->parms);
 914
 915         neigh_dbg(2, "neigh %p is destroyed\n", neigh);
 916
 917         atomic_dec(&neigh->tbl->entries);
 918         kfree_rcu(neigh, rcu);
 919 }
 920 EXPORT_SYMBOL(neigh_destroy);
 921
 922 /* Neighbour state is suspicious;
 923    disable fast path.
 924
 925    Called with write_locked neigh.
 926  */
 927 static void neigh_suspect(struct neighbour *neigh)
 928 {
 929         neigh_dbg(2, "neigh %p is suspected\n", neigh);
 930
 931         WRITE_ONCE(neigh->output, neigh->ops->output);
 932 }
 933
 934 /* Neighbour state is OK;
 935    enable fast path.
 936
 937    Called with write_locked neigh.
 938  */
 939 static void neigh_connect(struct neighbour *neigh)
 940 {
 941         neigh_dbg(2, "neigh %p is connected\n", neigh);
 942
 943         WRITE_ONCE(neigh->output, neigh->ops->connected_output);
 944 }
 945
 946 static void neigh_periodic_work(struct work_struct *work)
 947 {
 948         struct neigh_table *tbl = container_of(work, struct neigh_table, gc_work.work);
 949         struct neighbour *n;
 950         struct neighbour __rcu **np;
 951         unsigned int i;
 952         struct neigh_hash_table *nht;
 953
 954         NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs);
 955
 956         write_lock_bh(&tbl->lock);
 957         nht = rcu_dereference_protected(tbl->nht,
 958                                         lockdep_is_held(&tbl->lock));
 959
 960         /*
 961          *      periodically recompute ReachableTime from random function
 962          */
 963
 964         if (time_after(jiffies, tbl->last_rand + 300 * HZ)) {
 965                 struct neigh_parms *p;
 966
 967                 WRITE_ONCE(tbl->last_rand, jiffies);
 968                 list_for_each_entry(p, &tbl->parms_list, list)
 969                         p->reachable_time =
 970                                 neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
 971         }
 972
 973         if (atomic_read(&tbl->entries) < READ_ONCE(tbl->gc_thresh1))
 974                 goto out;
 975
 976         for (i = 0 ; i < (1 << nht->hash_shift); i++) {
 977                 np = &nht->hash_buckets[i];
 978
 979                 while ((n = rcu_dereference_protected(*np,
 980                                 lockdep_is_held(&tbl->lock))) != NULL) {
 981                         unsigned int state;
 982
 983                         write_lock(&n->lock);
 984
 985                         state = n->nud_state;
 986                         if ((state & (NUD_PERMANENT | NUD_IN_TIMER)) ||
 987                             (n->flags & NTF_EXT_LEARNED)) {
 988                                 write_unlock(&n->lock);
 989                                 goto next_elt;
 990                         }
 991
 992                         if (time_before(n->used, n->confirmed) &&
 993                             time_is_before_eq_jiffies(n->confirmed))
 994                                 n->used = n->confirmed;
 995
 996                         if (refcount_read(&n->refcnt) == 1 &&
 997                             (state == NUD_FAILED ||
 998                              !time_in_range_open(jiffies, n->used,
 999                                                  n->used + NEIGH_VAR(n->parms, GC_STALETIME)))) {
1000                                 rcu_assign_pointer(*np,
1001                                         rcu_dereference_protected(n->next,
1002                                                 lockdep_is_held(&tbl->lock)));
1003                                 neigh_mark_dead(n);
1004                                 write_unlock(&n->lock);
1005                                 neigh_cleanup_and_release(n);
1006                                 continue;
1007                         }
1008                         write_unlock(&n->lock);
1009
1010 next_elt:
1011                         np = &n->next;
1012                 }
1013                 /*
1014                  * It's fine to release lock here, even if hash table
1015                  * grows while we are preempted.
1016                  */
1017                 write_unlock_bh(&tbl->lock);
1018                 cond_resched();
1019                 write_lock_bh(&tbl->lock);
1020                 nht = rcu_dereference_protected(tbl->nht,
1021                                                 lockdep_is_held(&tbl->lock));
1022         }
1023 out:
1024         /* Cycle through all hash buckets every BASE_REACHABLE_TIME/2 ticks.
1025          * ARP entry timeouts range from 1/2 BASE_REACHABLE_TIME to 3/2
1026          * BASE_REACHABLE_TIME.
1027          */
1028         queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
1029                               NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME) >> 1);
1030         write_unlock_bh(&tbl->lock);
1031 }
1032
1033 static __inline__ int neigh_max_probes(struct neighbour *n)
1034 {
1035         struct neigh_parms *p = n->parms;
1036         return NEIGH_VAR(p, UCAST_PROBES) + NEIGH_VAR(p, APP_PROBES) +
1037                (n->nud_state & NUD_PROBE ? NEIGH_VAR(p, MCAST_REPROBES) :
1038                 NEIGH_VAR(p, MCAST_PROBES));
1039 }
1040
1041 static void neigh_invalidate(struct neighbour *neigh)
1042         __releases(neigh->lock)
1043         __acquires(neigh->lock)
1044 {
1045         struct sk_buff *skb;
1046
1047         NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed);
1048         neigh_dbg(2, "neigh %p is failed\n", neigh);
1049         neigh->updated = jiffies;
1050
1051         /* It is very thin place. report_unreachable is very complicated
1052            routine. Particularly, it can hit the same neighbour entry!
1053
1054            So that, we try to be accurate and avoid dead loop. --ANK
1055          */
1056         while (neigh->nud_state == NUD_FAILED &&
1057                (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
1058                 write_unlock(&neigh->lock);
1059                 neigh->ops->error_report(neigh, skb);
1060                 write_lock(&neigh->lock);
1061         }
1062         __skb_queue_purge(&neigh->arp_queue);
1063         neigh->arp_queue_len_bytes = 0;
1064 }
1065
1066 static void neigh_probe(struct neighbour *neigh)
1067         __releases(neigh->lock)
1068 {
1069         struct sk_buff *skb = skb_peek_tail(&neigh->arp_queue);
1070         /* keep skb alive even if arp_queue overflows */
1071         if (skb)
1072                 skb = skb_clone(skb, GFP_ATOMIC);
1073         write_unlock(&neigh->lock);
1074         if (neigh->ops->solicit)
1075                 neigh->ops->solicit(neigh, skb);
1076         atomic_inc(&neigh->probes);
1077         consume_skb(skb);
1078 }
1079
1080 /* Called when a timer expires for a neighbour entry. */
1081
1082 static void neigh_timer_handler(struct timer_list *t)
1083 {
1084         unsigned long now, next;
1085         struct neighbour *neigh = from_timer(neigh, t, timer);
1086         unsigned int state;
1087         int notify = 0;
1088
1089         write_lock(&neigh->lock);
1090
1091         state = neigh->nud_state;
1092         now = jiffies;
1093         next = now + HZ;
1094
1095         if (!(state & NUD_IN_TIMER))
1096                 goto out;
1097
1098         if (state & NUD_REACHABLE) {
1099                 if (time_before_eq(now,
1100                                    neigh->confirmed + neigh->parms->reachable_time)) {
1101                         neigh_dbg(2, "neigh %p is still alive\n", neigh);
1102                         next = neigh->confirmed + neigh->parms->reachable_time;
1103                 } else if (time_before_eq(now,
1104                                           neigh->used +
1105                                           NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
1106                         neigh_dbg(2, "neigh %p is delayed\n", neigh);
1107                         WRITE_ONCE(neigh->nud_state, NUD_DELAY);
1108                         neigh->updated = jiffies;
1109                         neigh_suspect(neigh);
1110                         next = now + NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME);
1111                 } else {
1112                         neigh_dbg(2, "neigh %p is suspected\n", neigh);
1113                         WRITE_ONCE(neigh->nud_state, NUD_STALE);
1114                         neigh->updated = jiffies;
1115                         neigh_suspect(neigh);
1116                         notify = 1;
1117                 }
1118         } else if (state & NUD_DELAY) {
1119                 if (time_before_eq(now,
1120                                    neigh->confirmed +
1121                                    NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
1122                         neigh_dbg(2, "neigh %p is now reachable\n", neigh);
1123                         WRITE_ONCE(neigh->nud_state, NUD_REACHABLE);
1124                         neigh->updated = jiffies;
1125                         neigh_connect(neigh);
1126                         notify = 1;
1127                         next = neigh->confirmed + neigh->parms->reachable_time;
1128                 } else {
1129                         neigh_dbg(2, "neigh %p is probed\n", neigh);
1130                         WRITE_ONCE(neigh->nud_state, NUD_PROBE);
1131                         neigh->updated = jiffies;
1132                         atomic_set(&neigh->probes, 0);
1133                         notify = 1;
1134                         next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
1135                                          HZ/100);
1136                 }
1137         } else {
1138                 /* NUD_PROBE|NUD_INCOMPLETE */
1139                 next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME), HZ/100);
1140         }
1141
1142         if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
1143             atomic_read(&neigh->probes) >= neigh_max_probes(neigh)) {
1144                 WRITE_ONCE(neigh->nud_state, NUD_FAILED);
1145                 notify = 1;
1146                 neigh_invalidate(neigh);
1147                 goto out;
1148         }
1149
1150         if (neigh->nud_state & NUD_IN_TIMER) {
1151                 if (time_before(next, jiffies + HZ/100))
1152                         next = jiffies + HZ/100;
1153                 if (!mod_timer(&neigh->timer, next))
1154                         neigh_hold(neigh);
1155         }
1156         if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) {
1157                 neigh_probe(neigh);
1158         } else {
1159 out:
1160                 write_unlock(&neigh->lock);
1161         }
1162
1163         if (notify)
1164                 neigh_update_notify(neigh, 0);
1165
1166         trace_neigh_timer_handler(neigh, 0);
1167
1168         neigh_release(neigh);
1169 }
1170
1171 int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
1172                        const bool immediate_ok)
1173 {
1174         int rc;
1175         bool immediate_probe = false;
1176
1177         write_lock_bh(&neigh->lock);
1178
1179         rc = 0;
1180         if (neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE))
1181                 goto out_unlock_bh;
1182         if (neigh->dead)
1183                 goto out_dead;
1184
1185         if (!(neigh->nud_state & (NUD_STALE | NUD_INCOMPLETE))) {
1186                 if (NEIGH_VAR(neigh->parms, MCAST_PROBES) +
1187                     NEIGH_VAR(neigh->parms, APP_PROBES)) {
1188                         unsigned long next, now = jiffies;
1189
1190                         atomic_set(&neigh->probes,
1191                                    NEIGH_VAR(neigh->parms, UCAST_PROBES));
1192                         neigh_del_timer(neigh);
1193                         WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
1194                         neigh->updated = now;
1195                         if (!immediate_ok) {
1196                                 next = now + 1;
1197                         } else {
1198                                 immediate_probe = true;
1199                                 next = now + max(NEIGH_VAR(neigh->parms,
1200                                                            RETRANS_TIME),
1201                                                  HZ / 100);
1202                         }
1203                         neigh_add_timer(neigh, next);
1204                 } else {
1205                         WRITE_ONCE(neigh->nud_state, NUD_FAILED);
1206                         neigh->updated = jiffies;
1207                         write_unlock_bh(&neigh->lock);
1208
1209                         kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_FAILED);
1210                         return 1;
1211                 }
1212         } else if (neigh->nud_state & NUD_STALE) {
1213                 neigh_dbg(2, "neigh %p is delayed\n", neigh);
1214                 neigh_del_timer(neigh);
1215                 WRITE_ONCE(neigh->nud_state, NUD_DELAY);
1216                 neigh->updated = jiffies;
1217                 neigh_add_timer(neigh, jiffies +
1218                                 NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME));
1219         }
1220
1221         if (neigh->nud_state == NUD_INCOMPLETE) {
1222                 if (skb) {
1223                         while (neigh->arp_queue_len_bytes + skb->truesize >
1224                                NEIGH_VAR(neigh->parms, QUEUE_LEN_BYTES)) {
1225                                 struct sk_buff *buff;
1226
1227                                 buff = __skb_dequeue(&neigh->arp_queue);
1228                                 if (!buff)
1229                                         break;
1230                                 neigh->arp_queue_len_bytes -= buff->truesize;
1231                                 kfree_skb_reason(buff, SKB_DROP_REASON_NEIGH_QUEUEFULL);
1232                                 NEIGH_CACHE_STAT_INC(neigh->tbl, unres_discards);
1233                         }
1234                         skb_dst_force(skb);
1235                         __skb_queue_tail(&neigh->arp_queue, skb);
1236                         neigh->arp_queue_len_bytes += skb->truesize;
1237                 }
1238                 rc = 1;
1239         }
1240 out_unlock_bh:
1241         if (immediate_probe)
1242                 neigh_probe(neigh);
1243         else
1244                 write_unlock(&neigh->lock);
1245         local_bh_enable();
1246         trace_neigh_event_send_done(neigh, rc);
1247         return rc;
1248
1249 out_dead:
1250         if (neigh->nud_state & NUD_STALE)
1251                 goto out_unlock_bh;
1252         write_unlock_bh(&neigh->lock);
1253         kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_DEAD);
1254         trace_neigh_event_send_dead(neigh, 1);
1255         return 1;
1256 }
1257 EXPORT_SYMBOL(__neigh_event_send);
1258
1259 static void neigh_update_hhs(struct neighbour *neigh)
1260 {
1261         struct hh_cache *hh;
1262         void (*update)(struct hh_cache*, const struct net_device*, const unsigned char *)
1263                 = NULL;
1264
1265         if (neigh->dev->header_ops)
1266                 update = neigh->dev->header_ops->cache_update;
1267
1268         if (update) {
1269                 hh = &neigh->hh;
1270                 if (READ_ONCE(hh->hh_len)) {
1271                         write_seqlock_bh(&hh->hh_lock);
1272                         update(hh, neigh->dev, neigh->ha);
1273                         write_sequnlock_bh(&hh->hh_lock);
1274                 }
1275         }
1276 }
1277
1278 /* Generic update routine.
1279    -- lladdr is new lladdr or NULL, if it is not supplied.
1280    -- new    is new state.
1281    -- flags
1282         NEIGH_UPDATE_F_OVERRIDE allows to override existing lladdr,
1283                                 if it is different.
1284         NEIGH_UPDATE_F_WEAK_OVERRIDE will suspect existing "connected"
1285                                 lladdr instead of overriding it
1286                                 if it is different.
1287         NEIGH_UPDATE_F_ADMIN    means that the change is administrative.
1288         NEIGH_UPDATE_F_USE      means that the entry is user triggered.
1289         NEIGH_UPDATE_F_MANAGED  means that the entry will be auto-refreshed.
1290         NEIGH_UPDATE_F_OVERRIDE_ISROUTER allows to override existing
1291                                 NTF_ROUTER flag.
1292         NEIGH_UPDATE_F_ISROUTER indicates if the neighbour is known as
1293                                 a router.
1294
1295    Caller MUST hold reference count on the entry.
1296  */
1297 static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
1298                           u8 new, u32 flags, u32 nlmsg_pid,
1299                           struct netlink_ext_ack *extack)
1300 {
1301         bool gc_update = false, managed_update = false;
1302         int update_isrouter = 0;
1303         struct net_device *dev;
1304         int err, notify = 0;
1305         u8 old;
1306
1307         trace_neigh_update(neigh, lladdr, new, flags, nlmsg_pid);
1308
1309         write_lock_bh(&neigh->lock);
1310
1311         dev    = neigh->dev;
1312         old    = neigh->nud_state;
1313         err    = -EPERM;
1314
1315         if (neigh->dead) {
1316                 NL_SET_ERR_MSG(extack, "Neighbor entry is now dead");
1317                 new = old;
1318                 goto out;
1319         }
1320         if (!(flags & NEIGH_UPDATE_F_ADMIN) &&
1321             (old & (NUD_NOARP | NUD_PERMANENT)))
1322                 goto out;
1323
1324         neigh_update_flags(neigh, flags, &notify, &gc_update, &managed_update);
1325         if (flags & (NEIGH_UPDATE_F_USE | NEIGH_UPDATE_F_MANAGED)) {
1326                 new = old & ~NUD_PERMANENT;
1327                 WRITE_ONCE(neigh->nud_state, new);
1328                 err = 0;
1329                 goto out;
1330         }
1331
1332         if (!(new & NUD_VALID)) {
1333                 neigh_del_timer(neigh);
1334                 if (old & NUD_CONNECTED)
1335                         neigh_suspect(neigh);
1336                 WRITE_ONCE(neigh->nud_state, new);
1337                 err = 0;
1338                 notify = old & NUD_VALID;
1339                 if ((old & (NUD_INCOMPLETE | NUD_PROBE)) &&
1340                     (new & NUD_FAILED)) {
1341                         neigh_invalidate(neigh);
1342                         notify = 1;
1343                 }
1344                 goto out;
1345         }
1346
1347         /* Compare new lladdr with cached one */
1348         if (!dev->addr_len) {
1349                 /* First case: device needs no address. */
1350                 lladdr = neigh->ha;
1351         } else if (lladdr) {
1352                 /* The second case: if something is already cached
1353                    and a new address is proposed:
1354                    - compare new & old
1355                    - if they are different, check override flag
1356                  */
1357                 if ((old & NUD_VALID) &&
1358                     !memcmp(lladdr, neigh->ha, dev->addr_len))
1359                         lladdr = neigh->ha;
1360         } else {
1361                 /* No address is supplied; if we know something,
1362                    use it, otherwise discard the request.
1363                  */
1364                 err = -EINVAL;
1365                 if (!(old & NUD_VALID)) {
1366                         NL_SET_ERR_MSG(extack, "No link layer address given");
1367                         goto out;
1368                 }
1369                 lladdr = neigh->ha;
1370         }
1371
1372         /* Update confirmed timestamp for neighbour entry after we
1373          * received ARP packet even if it doesn't change IP to MAC binding.
1374          */
1375         if (new & NUD_CONNECTED)
1376                 neigh->confirmed = jiffies;
1377
1378         /* If entry was valid and address is not changed,
1379            do not change entry state, if new one is STALE.
1380          */
1381         err = 0;
1382         update_isrouter = flags & NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
1383         if (old & NUD_VALID) {
1384                 if (lladdr != neigh->ha && !(flags & NEIGH_UPDATE_F_OVERRIDE)) {
1385                         update_isrouter = 0;
1386                         if ((flags & NEIGH_UPDATE_F_WEAK_OVERRIDE) &&
1387                             (old & NUD_CONNECTED)) {
1388                                 lladdr = neigh->ha;
1389                                 new = NUD_STALE;
1390                         } else
1391                                 goto out;
1392                 } else {
1393                         if (lladdr == neigh->ha && new == NUD_STALE &&
1394                             !(flags & NEIGH_UPDATE_F_ADMIN))
1395                                 new = old;
1396                 }
1397         }
1398
1399         /* Update timestamp only once we know we will make a change to the
1400          * neighbour entry. Otherwise we risk to move the locktime window with
1401          * noop updates and ignore relevant ARP updates.
1402          */
1403         if (new != old || lladdr != neigh->ha)
1404                 neigh->updated = jiffies;
1405
1406         if (new != old) {
1407                 neigh_del_timer(neigh);
1408                 if (new & NUD_PROBE)
1409                         atomic_set(&neigh->probes, 0);
1410                 if (new & NUD_IN_TIMER)
1411                         neigh_add_timer(neigh, (jiffies +
1412                                                 ((new & NUD_REACHABLE) ?
1413                                                  neigh->parms->reachable_time :
1414                                                  0)));
1415                 WRITE_ONCE(neigh->nud_state, new);
1416                 notify = 1;
1417         }
1418
1419         if (lladdr != neigh->ha) {
1420                 write_seqlock(&neigh->ha_lock);
1421                 memcpy(&neigh->ha, lladdr, dev->addr_len);
1422                 write_sequnlock(&neigh->ha_lock);
1423                 neigh_update_hhs(neigh);
1424                 if (!(new & NUD_CONNECTED))
1425                         neigh->confirmed = jiffies -
1426                                       (NEIGH_VAR(neigh->parms, BASE_REACHABLE_TIME) << 1);
1427                 notify = 1;
1428         }
1429         if (new == old)
1430                 goto out;
1431         if (new & NUD_CONNECTED)
1432                 neigh_connect(neigh);
1433         else
1434                 neigh_suspect(neigh);
1435         if (!(old & NUD_VALID)) {
1436                 struct sk_buff *skb;
1437
1438                 /* Again: avoid dead loop if something went wrong */
1439
1440                 while (neigh->nud_state & NUD_VALID &&
1441                        (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
1442                         struct dst_entry *dst = skb_dst(skb);
1443                         struct neighbour *n2, *n1 = neigh;
1444                         write_unlock_bh(&neigh->lock);
1445
1446                         rcu_read_lock();
1447
1448                         /* Why not just use 'neigh' as-is?  The problem is that
1449                          * things such as shaper, eql, and sch_teql can end up
1450                          * using alternative, different, neigh objects to output
1451                          * the packet in the output path.  So what we need to do
1452                          * here is re-lookup the top-level neigh in the path so
1453                          * we can reinject the packet there.
1454                          */
1455                         n2 = NULL;
1456                         if (dst && dst->obsolete != DST_OBSOLETE_DEAD) {
1457                                 n2 = dst_neigh_lookup_skb(dst, skb);
1458                                 if (n2)
1459                                         n1 = n2;
1460                         }
1461                         READ_ONCE(n1->output)(n1, skb);
1462                         if (n2)
1463                                 neigh_release(n2);
1464                         rcu_read_unlock();
1465
1466                         write_lock_bh(&neigh->lock);
1467                 }
1468                 __skb_queue_purge(&neigh->arp_queue);
1469                 neigh->arp_queue_len_bytes = 0;
1470         }
1471 out:
1472         if (update_isrouter)
1473                 neigh_update_is_router(neigh, flags, &notify);
1474         write_unlock_bh(&neigh->lock);
1475         if (((new ^ old) & NUD_PERMANENT) || gc_update)
1476                 neigh_update_gc_list(neigh);
1477         if (managed_update)
1478                 neigh_update_managed_list(neigh);
1479         if (notify)
1480                 neigh_update_notify(neigh, nlmsg_pid);
1481         trace_neigh_update_done(neigh, err);
1482         return err;
1483 }
1484
1485 int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,
1486                  u32 flags, u32 nlmsg_pid)
1487 {
1488         return __neigh_update(neigh, lladdr, new, flags, nlmsg_pid, NULL);
1489 }
1490 EXPORT_SYMBOL(neigh_update);
1491
1492 /* Update the neigh to listen temporarily for probe responses, even if it is
1493  * in a NUD_FAILED state. The caller has to hold neigh->lock for writing.
1494  */
1495 void __neigh_set_probe_once(struct neighbour *neigh)
1496 {
1497         if (neigh->dead)
1498                 return;
1499         neigh->updated = jiffies;
1500         if (!(neigh->nud_state & NUD_FAILED))
1501                 return;
1502         WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
1503         atomic_set(&neigh->probes, neigh_max_probes(neigh));
1504         neigh_add_timer(neigh,
1505                         jiffies + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
1506                                       HZ/100));
1507 }
1508 EXPORT_SYMBOL(__neigh_set_probe_once);
1509
1510 struct neighbour *neigh_event_ns(struct neigh_table *tbl,
1511                                  u8 *lladdr, void *saddr,
1512                                  struct net_device *dev)
1513 {
1514         struct neighbour *neigh = __neigh_lookup(tbl, saddr, dev,
1515                                                  lladdr || !dev->addr_len);
1516         if (neigh)
1517                 neigh_update(neigh, lladdr, NUD_STALE,
1518                              NEIGH_UPDATE_F_OVERRIDE, 0);
1519         return neigh;
1520 }
1521 EXPORT_SYMBOL(neigh_event_ns);
1522
1523 /* called with read_lock_bh(&n->lock); */
1524 static void neigh_hh_init(struct neighbour *n)
1525 {
1526         struct net_device *dev = n->dev;
1527         __be16 prot = n->tbl->protocol;
1528         struct hh_cache *hh = &n->hh;
1529
1530         write_lock_bh(&n->lock);
1531
1532         /* Only one thread can come in here and initialize the
1533          * hh_cache entry.
1534          */
1535         if (!hh->hh_len)
1536                 dev->header_ops->cache(n, hh, prot);
1537
1538         write_unlock_bh(&n->lock);
1539 }
1540
1541 /* Slow and careful. */
1542
1543 int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb)
1544 {
1545         int rc = 0;
1546
1547         if (!neigh_event_send(neigh, skb)) {
1548                 int err;
1549                 struct net_device *dev = neigh->dev;
1550                 unsigned int seq;
1551
1552                 if (dev->header_ops->cache && !READ_ONCE(neigh->hh.hh_len))
1553                         neigh_hh_init(neigh);
1554
1555                 do {
1556                         __skb_pull(skb, skb_network_offset(skb));
1557                         seq = read_seqbegin(&neigh->ha_lock);
1558                         err = dev_hard_header(skb, dev, ntohs(skb->protocol),
1559                                               neigh->ha, NULL, skb->len);
1560                 } while (read_seqretry(&neigh->ha_lock, seq));
1561
1562                 if (err >= 0)
1563                         rc = dev_queue_xmit(skb);
1564                 else
1565                         goto out_kfree_skb;
1566         }
1567 out:
1568         return rc;
1569 out_kfree_skb:
1570         rc = -EINVAL;
1571         kfree_skb(skb);
1572         goto out;
1573 }
1574 EXPORT_SYMBOL(neigh_resolve_output);
1575
1576 /* As fast as possible without hh cache */
1577
1578 int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb)
1579 {
1580         struct net_device *dev = neigh->dev;
1581         unsigned int seq;
1582         int err;
1583
1584         do {
1585                 __skb_pull(skb, skb_network_offset(skb));
1586                 seq = read_seqbegin(&neigh->ha_lock);
1587                 err = dev_hard_header(skb, dev, ntohs(skb->protocol),
1588                                       neigh->ha, NULL, skb->len);
1589         } while (read_seqretry(&neigh->ha_lock, seq));
1590
1591         if (err >= 0)
1592                 err = dev_queue_xmit(skb);
1593         else {
1594                 err = -EINVAL;
1595                 kfree_skb(skb);
1596         }
1597         return err;
1598 }
1599 EXPORT_SYMBOL(neigh_connected_output);
1600
1601 int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb)
1602 {
1603         return dev_queue_xmit(skb);
1604 }
1605 EXPORT_SYMBOL(neigh_direct_output);
1606
1607 static void neigh_managed_work(struct work_struct *work)
1608 {
1609         struct neigh_table *tbl = container_of(work, struct neigh_table,
1610                                                managed_work.work);
1611         struct neighbour *neigh;
1612
1613         write_lock_bh(&tbl->lock);
1614         list_for_each_entry(neigh, &tbl->managed_list, managed_list)
1615                 neigh_event_send_probe(neigh, NULL, false);
1616         queue_delayed_work(system_power_efficient_wq, &tbl->managed_work,
1617                            NEIGH_VAR(&tbl->parms, INTERVAL_PROBE_TIME_MS));
1618         write_unlock_bh(&tbl->lock);
1619 }
1620
1621 static void neigh_proxy_process(struct timer_list *t)
1622 {
1623         struct neigh_table *tbl = from_timer(tbl, t, proxy_timer);
1624         long sched_next = 0;
1625         unsigned long now = jiffies;
1626         struct sk_buff *skb, *n;
1627
1628         spin_lock(&tbl->proxy_queue.lock);
1629
1630         skb_queue_walk_safe(&tbl->proxy_queue, skb, n) {
1631                 long tdif = NEIGH_CB(skb)->sched_next - now;
1632
1633                 if (tdif <= 0) {
1634                         struct net_device *dev = skb->dev;
1635
1636                         neigh_parms_qlen_dec(dev, tbl->family);
1637                         __skb_unlink(skb, &tbl->proxy_queue);
1638
1639                         if (tbl->proxy_redo && netif_running(dev)) {
1640                                 rcu_read_lock();
1641                                 tbl->proxy_redo(skb);
1642                                 rcu_read_unlock();
1643                         } else {
1644                                 kfree_skb(skb);
1645                         }
1646
1647                         dev_put(dev);
1648                 } else if (!sched_next || tdif < sched_next)
1649                         sched_next = tdif;
1650         }
1651         del_timer(&tbl->proxy_timer);
1652         if (sched_next)
1653                 mod_timer(&tbl->proxy_timer, jiffies + sched_next);
1654         spin_unlock(&tbl->proxy_queue.lock);
1655 }
1656
1657 void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
1658                     struct sk_buff *skb)
1659 {
1660         unsigned long sched_next = jiffies +
1661                         prandom_u32_max(NEIGH_VAR(p, PROXY_DELAY));
1662
1663         if (p->qlen > NEIGH_VAR(p, PROXY_QLEN)) {
1664                 kfree_skb(skb);
1665                 return;
1666         }
1667
1668         NEIGH_CB(skb)->sched_next = sched_next;
1669         NEIGH_CB(skb)->flags |= LOCALLY_ENQUEUED;
1670
1671         spin_lock(&tbl->proxy_queue.lock);
1672         if (del_timer(&tbl->proxy_timer)) {
1673                 if (time_before(tbl->proxy_timer.expires, sched_next))
1674                         sched_next = tbl->proxy_timer.expires;
1675         }
1676         skb_dst_drop(skb);
1677         dev_hold(skb->dev);
1678         __skb_queue_tail(&tbl->proxy_queue, skb);
1679         p->qlen++;
1680         mod_timer(&tbl->proxy_timer, sched_next);
1681         spin_unlock(&tbl->proxy_queue.lock);
1682 }
1683 EXPORT_SYMBOL(pneigh_enqueue);
1684
1685 static inline struct neigh_parms *lookup_neigh_parms(struct neigh_table *tbl,
1686                                                       struct net *net, int ifindex)
1687 {
1688         struct neigh_parms *p;
1689
1690         list_for_each_entry(p, &tbl->parms_list, list) {
1691                 if ((p->dev && p->dev->ifindex == ifindex && net_eq(neigh_parms_net(p), net)) ||
1692                     (!p->dev && !ifindex && net_eq(net, &init_net)))
1693                         return p;
1694         }
1695
1696         return NULL;
1697 }
1698
1699 struct neigh_parms *neigh_parms_alloc(struct net_device *dev,
1700                                       struct neigh_table *tbl)
1701 {
1702         struct neigh_parms *p;
1703         struct net *net = dev_net(dev);
1704         const struct net_device_ops *ops = dev->netdev_ops;
1705
1706         p = kmemdup(&tbl->parms, sizeof(*p), GFP_KERNEL);
1707         if (p) {
1708                 p->tbl            = tbl;
1709                 refcount_set(&p->refcnt, 1);
1710                 p->reachable_time =
1711                                 neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
1712                 p->qlen = 0;
1713                 netdev_hold(dev, &p->dev_tracker, GFP_KERNEL);
1714                 p->dev = dev;
1715                 write_pnet(&p->net, net);
1716                 p->sysctl_table = NULL;
1717
1718                 if (ops->ndo_neigh_setup && ops->ndo_neigh_setup(dev, p)) {
1719                         netdev_put(dev, &p->dev_tracker);
1720                         kfree(p);
1721                         return NULL;
1722                 }
1723
1724                 write_lock_bh(&tbl->lock);
1725                 list_add(&p->list, &tbl->parms.list);
1726                 write_unlock_bh(&tbl->lock);
1727
1728                 neigh_parms_data_state_cleanall(p);
1729         }
1730         return p;
1731 }
1732 EXPORT_SYMBOL(neigh_parms_alloc);
1733
1734 static void neigh_rcu_free_parms(struct rcu_head *head)
1735 {
1736         struct neigh_parms *parms =
1737                 container_of(head, struct neigh_parms, rcu_head);
1738
1739         neigh_parms_put(parms);
1740 }
1741
1742 void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms)
1743 {
1744         if (!parms || parms == &tbl->parms)
1745                 return;
1746         write_lock_bh(&tbl->lock);
1747         list_del(&parms->list);
1748         parms->dead = 1;
1749         write_unlock_bh(&tbl->lock);
1750         netdev_put(parms->dev, &parms->dev_tracker);
1751         call_rcu(&parms->rcu_head, neigh_rcu_free_parms);
1752 }
1753 EXPORT_SYMBOL(neigh_parms_release);
1754
1755 static void neigh_parms_destroy(struct neigh_parms *parms)
1756 {
1757         kfree(parms);
1758 }
1759
1760 static struct lock_class_key neigh_table_proxy_queue_class;
1761
1762 static struct neigh_table *neigh_tables[NEIGH_NR_TABLES] __read_mostly;
1763
1764 void neigh_table_init(int index, struct neigh_table *tbl)
1765 {
1766         unsigned long now = jiffies;
1767         unsigned long phsize;
1768
1769         INIT_LIST_HEAD(&tbl->parms_list);
1770         INIT_LIST_HEAD(&tbl->gc_list);
1771         INIT_LIST_HEAD(&tbl->managed_list);
1772
1773         list_add(&tbl->parms.list, &tbl->parms_list);
1774         write_pnet(&tbl->parms.net, &init_net);
1775         refcount_set(&tbl->parms.refcnt, 1);
1776         tbl->parms.reachable_time =
1777                           neigh_rand_reach_time(NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME));
1778         tbl->parms.qlen = 0;
1779
1780         tbl->stats = alloc_percpu(struct neigh_statistics);
1781         if (!tbl->stats)
1782                 panic("cannot create neighbour cache statistics");
1783
1784 #ifdef CONFIG_PROC_FS
1785         if (!proc_create_seq_data(tbl->id, 0, init_net.proc_net_stat,
1786                               &neigh_stat_seq_ops, tbl))
1787                 panic("cannot create neighbour proc dir entry");
1788 #endif
1789
1790         RCU_INIT_POINTER(tbl->nht, neigh_hash_alloc(3));
1791
1792         phsize = (PNEIGH_HASHMASK + 1) * sizeof(struct pneigh_entry *);
1793         tbl->phash_buckets = kzalloc(phsize, GFP_KERNEL);
1794
1795         if (!tbl->nht || !tbl->phash_buckets)
1796                 panic("cannot allocate neighbour cache hashes");
1797
1798         if (!tbl->entry_size)
1799                 tbl->entry_size = ALIGN(offsetof(struct neighbour, primary_key) +
1800                                         tbl->key_len, NEIGH_PRIV_ALIGN);
1801         else
1802                 WARN_ON(tbl->entry_size % NEIGH_PRIV_ALIGN);
1803
1804         rwlock_init(&tbl->lock);
1805
1806         INIT_DEFERRABLE_WORK(&tbl->gc_work, neigh_periodic_work);
1807         queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
1808                         tbl->parms.reachable_time);
1809         INIT_DEFERRABLE_WORK(&tbl->managed_work, neigh_managed_work);
1810         queue_delayed_work(system_power_efficient_wq, &tbl->managed_work, 0);
1811
1812         timer_setup(&tbl->proxy_timer, neigh_proxy_process, 0);
1813         skb_queue_head_init_class(&tbl->proxy_queue,
1814                         &neigh_table_proxy_queue_class);
1815
1816         tbl->last_flush = now;
1817         tbl->last_rand  = now + tbl->parms.reachable_time * 20;
1818
1819         neigh_tables[index] = tbl;
1820 }
1821 EXPORT_SYMBOL(neigh_table_init);
1822
1823 int neigh_table_clear(int index, struct neigh_table *tbl)
1824 {
1825         neigh_tables[index] = NULL;
1826         /* It is not clean... Fix it to unload IPv6 module safely */
1827         cancel_delayed_work_sync(&tbl->managed_work);
1828         cancel_delayed_work_sync(&tbl->gc_work);
1829         del_timer_sync(&tbl->proxy_timer);
1830         pneigh_queue_purge(&tbl->proxy_queue, NULL, tbl->family);
1831         neigh_ifdown(tbl, NULL);
1832         if (atomic_read(&tbl->entries))
1833                 pr_crit("neighbour leakage\n");
1834
1835         call_rcu(&rcu_dereference_protected(tbl->nht, 1)->rcu,
1836                  neigh_hash_free_rcu);
1837         tbl->nht = NULL;
1838
1839         kfree(tbl->phash_buckets);
1840         tbl->phash_buckets = NULL;
1841
1842         remove_proc_entry(tbl->id, init_net.proc_net_stat);
1843
1844         free_percpu(tbl->stats);
1845         tbl->stats = NULL;
1846
1847         return 0;
1848 }
1849 EXPORT_SYMBOL(neigh_table_clear);
1850
1851 static struct neigh_table *neigh_find_table(int family)
1852 {
1853         struct neigh_table *tbl = NULL;
1854
1855         switch (family) {
1856         case AF_INET:
1857                 tbl = neigh_tables[NEIGH_ARP_TABLE];
1858                 break;
1859         case AF_INET6:
1860                 tbl = neigh_tables[NEIGH_ND_TABLE];
1861                 break;
1862         }
1863
1864         return tbl;
1865 }
1866
1867 const struct nla_policy nda_policy[NDA_MAX+1] = {
1868         [NDA_UNSPEC]            = { .strict_start_type = NDA_NH_ID },
1869         [NDA_DST]               = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
1870         [NDA_LLADDR]            = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
1871         [NDA_CACHEINFO]         = { .len = sizeof(struct nda_cacheinfo) },
1872         [NDA_PROBES]            = { .type = NLA_U32 },
1873         [NDA_VLAN]              = { .type = NLA_U16 },
1874         [NDA_PORT]              = { .type = NLA_U16 },
1875         [NDA_VNI]               = { .type = NLA_U32 },
1876         [NDA_IFINDEX]           = { .type = NLA_U32 },
1877         [NDA_MASTER]            = { .type = NLA_U32 },
1878         [NDA_PROTOCOL]          = { .type = NLA_U8 },
1879         [NDA_NH_ID]             = { .type = NLA_U32 },
1880         [NDA_FLAGS_EXT]         = NLA_POLICY_MASK(NLA_U32, NTF_EXT_MASK),
1881         [NDA_FDB_EXT_ATTRS]     = { .type = NLA_NESTED },
1882 };
1883
1884 static int neigh_delete(struct sk_buff *skb, struct nlmsghdr *nlh,
1885                         struct netlink_ext_ack *extack)
1886 {
1887         struct net *net = sock_net(skb->sk);
1888         struct ndmsg *ndm;
1889         struct nlattr *dst_attr;
1890         struct neigh_table *tbl;
1891         struct neighbour *neigh;
1892         struct net_device *dev = NULL;
1893         int err = -EINVAL;
1894
1895         ASSERT_RTNL();
1896         if (nlmsg_len(nlh) < sizeof(*ndm))
1897                 goto out;
1898
1899         dst_attr = nlmsg_find_attr(nlh, sizeof(*ndm), NDA_DST);
1900         if (!dst_attr) {
1901                 NL_SET_ERR_MSG(extack, "Network address not specified");
1902                 goto out;
1903         }
1904
1905         ndm = nlmsg_data(nlh);
1906         if (ndm->ndm_ifindex) {
1907                 dev = __dev_get_by_index(net, ndm->ndm_ifindex);
1908                 if (dev == NULL) {
1909                         err = -ENODEV;
1910                         goto out;
1911                 }
1912         }
1913
1914         tbl = neigh_find_table(ndm->ndm_family);
1915         if (tbl == NULL)
1916                 return -EAFNOSUPPORT;
1917
1918         if (nla_len(dst_attr) < (int)tbl->key_len) {
1919                 NL_SET_ERR_MSG(extack, "Invalid network address");
1920                 goto out;
1921         }
1922
1923         if (ndm->ndm_flags & NTF_PROXY) {
1924                 err = pneigh_delete(tbl, net, nla_data(dst_attr), dev);
1925                 goto out;
1926         }
1927
1928         if (dev == NULL)
1929                 goto out;
1930
1931         neigh = neigh_lookup(tbl, nla_data(dst_attr), dev);
1932         if (neigh == NULL) {
1933                 err = -ENOENT;
1934                 goto out;
1935         }
1936
1937         err = __neigh_update(neigh, NULL, NUD_FAILED,
1938                              NEIGH_UPDATE_F_OVERRIDE | NEIGH_UPDATE_F_ADMIN,
1939                              NETLINK_CB(skb).portid, extack);
1940         write_lock_bh(&tbl->lock);
1941         neigh_release(neigh);
1942         neigh_remove_one(neigh, tbl);
1943         write_unlock_bh(&tbl->lock);
1944
1945 out:
1946         return err;
1947 }
1948
1949 static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
1950                      struct netlink_ext_ack *extack)
1951 {
1952         int flags = NEIGH_UPDATE_F_ADMIN | NEIGH_UPDATE_F_OVERRIDE |
1953                     NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
1954         struct net *net = sock_net(skb->sk);
1955         struct ndmsg *ndm;
1956         struct nlattr *tb[NDA_MAX+1];
1957         struct neigh_table *tbl;
1958         struct net_device *dev = NULL;
1959         struct neighbour *neigh;
1960         void *dst, *lladdr;
1961         u8 protocol = 0;
1962         u32 ndm_flags;
1963         int err;
1964
1965         ASSERT_RTNL();
1966         err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX,
1967                                      nda_policy, extack);
1968         if (err < 0)
1969                 goto out;
1970
1971         err = -EINVAL;
1972         if (!tb[NDA_DST]) {
1973                 NL_SET_ERR_MSG(extack, "Network address not specified");
1974                 goto out;
1975         }
1976
1977         ndm = nlmsg_data(nlh);
1978         ndm_flags = ndm->ndm_flags;
1979         if (tb[NDA_FLAGS_EXT]) {
1980                 u32 ext = nla_get_u32(tb[NDA_FLAGS_EXT]);
1981
1982                 BUILD_BUG_ON(sizeof(neigh->flags) * BITS_PER_BYTE <
1983                              (sizeof(ndm->ndm_flags) * BITS_PER_BYTE +
1984                               hweight32(NTF_EXT_MASK)));
1985                 ndm_flags |= (ext << NTF_EXT_SHIFT);
1986         }
1987         if (ndm->ndm_ifindex) {
1988                 dev = __dev_get_by_index(net, ndm->ndm_ifindex);
1989                 if (dev == NULL) {
1990                         err = -ENODEV;
1991                         goto out;
1992                 }
1993
1994                 if (tb[NDA_LLADDR] && nla_len(tb[NDA_LLADDR]) < dev->addr_len) {
1995                         NL_SET_ERR_MSG(extack, "Invalid link address");
1996                         goto out;
1997                 }
1998         }
1999
2000         tbl = neigh_find_table(ndm->ndm_family);
2001         if (tbl == NULL)
2002                 return -EAFNOSUPPORT;
2003
2004         if (nla_len(tb[NDA_DST]) < (int)tbl->key_len) {
2005                 NL_SET_ERR_MSG(extack, "Invalid network address");
2006                 goto out;
2007         }
2008
2009         dst = nla_data(tb[NDA_DST]);
2010         lladdr = tb[NDA_LLADDR] ? nla_data(tb[NDA_LLADDR]) : NULL;
2011
2012         if (tb[NDA_PROTOCOL])
2013                 protocol = nla_get_u8(tb[NDA_PROTOCOL]);
2014         if (ndm_flags & NTF_PROXY) {
2015                 struct pneigh_entry *pn;
2016
2017                 if (ndm_flags & NTF_MANAGED) {
2018                         NL_SET_ERR_MSG(extack, "Invalid NTF_* flag combination");
2019                         goto out;
2020                 }
2021
2022                 err = -ENOBUFS;
2023                 pn = pneigh_lookup(tbl, net, dst, dev, 1);
2024                 if (pn) {
2025                         pn->flags = ndm_flags;
2026                         if (protocol)
2027                                 pn->protocol = protocol;
2028                         err = 0;
2029                 }
2030                 goto out;
2031         }
2032
2033         if (!dev) {
2034                 NL_SET_ERR_MSG(extack, "Device not specified");
2035                 goto out;
2036         }
2037
2038         if (tbl->allow_add && !tbl->allow_add(dev, extack)) {
2039                 err = -EINVAL;
2040                 goto out;
2041         }
2042
2043         neigh = neigh_lookup(tbl, dst, dev);
2044         if (neigh == NULL) {
2045                 bool ndm_permanent  = ndm->ndm_state & NUD_PERMANENT;
2046                 bool exempt_from_gc = ndm_permanent ||
2047                                       ndm_flags & NTF_EXT_LEARNED;
2048
2049                 if (!(nlh->nlmsg_flags & NLM_F_CREATE)) {
2050                         err = -ENOENT;
2051                         goto out;
2052                 }
2053                 if (ndm_permanent && (ndm_flags & NTF_MANAGED)) {
2054                         NL_SET_ERR_MSG(extack, "Invalid NTF_* flag for permanent entry");
2055                         err = -EINVAL;
2056                         goto out;
2057                 }
2058
2059                 neigh = ___neigh_create(tbl, dst, dev,
2060                                         ndm_flags &
2061                                         (NTF_EXT_LEARNED | NTF_MANAGED),
2062                                         exempt_from_gc, true);
2063                 if (IS_ERR(neigh)) {
2064                         err = PTR_ERR(neigh);
2065                         goto out;
2066                 }
2067         } else {
2068                 if (nlh->nlmsg_flags & NLM_F_EXCL) {
2069                         err = -EEXIST;
2070                         neigh_release(neigh);
2071                         goto out;
2072                 }
2073
2074                 if (!(nlh->nlmsg_flags & NLM_F_REPLACE))
2075                         flags &= ~(NEIGH_UPDATE_F_OVERRIDE |
2076                                    NEIGH_UPDATE_F_OVERRIDE_ISROUTER);
2077         }
2078
2079         if (protocol)
2080                 neigh->protocol = protocol;
2081         if (ndm_flags & NTF_EXT_LEARNED)
2082                 flags |= NEIGH_UPDATE_F_EXT_LEARNED;
2083         if (ndm_flags & NTF_ROUTER)
2084                 flags |= NEIGH_UPDATE_F_ISROUTER;
2085         if (ndm_flags & NTF_MANAGED)
2086                 flags |= NEIGH_UPDATE_F_MANAGED;
2087         if (ndm_flags & NTF_USE)
2088                 flags |= NEIGH_UPDATE_F_USE;
2089
2090         err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,
2091                              NETLINK_CB(skb).portid, extack);
2092         if (!err && ndm_flags & (NTF_USE | NTF_MANAGED)) {
2093                 neigh_event_send(neigh, NULL);
2094                 err = 0;
2095         }
2096         neigh_release(neigh);
2097 out:
2098         return err;
2099 }
2100
2101 static int neightbl_fill_parms(struct sk_buff *skb, struct neigh_parms *parms)
2102 {
2103         struct nlattr *nest;
2104
2105         nest = nla_nest_start_noflag(skb, NDTA_PARMS);
2106         if (nest == NULL)
2107                 return -ENOBUFS;
2108
2109         if ((parms->dev &&
2110              nla_put_u32(skb, NDTPA_IFINDEX, parms->dev->ifindex)) ||
2111             nla_put_u32(skb, NDTPA_REFCNT, refcount_read(&parms->refcnt)) ||
2112             nla_put_u32(skb, NDTPA_QUEUE_LENBYTES,
2113                         NEIGH_VAR(parms, QUEUE_LEN_BYTES)) ||
2114             /* approximative value for deprecated QUEUE_LEN (in packets) */
2115             nla_put_u32(skb, NDTPA_QUEUE_LEN,
2116                         NEIGH_VAR(parms, QUEUE_LEN_BYTES) / SKB_TRUESIZE(ETH_FRAME_LEN)) ||
2117             nla_put_u32(skb, NDTPA_PROXY_QLEN, NEIGH_VAR(parms, PROXY_QLEN)) ||
2118             nla_put_u32(skb, NDTPA_APP_PROBES, NEIGH_VAR(parms, APP_PROBES)) ||
2119             nla_put_u32(skb, NDTPA_UCAST_PROBES,
2120                         NEIGH_VAR(parms, UCAST_PROBES)) ||
2121             nla_put_u32(skb, NDTPA_MCAST_PROBES,
2122                         NEIGH_VAR(parms, MCAST_PROBES)) ||
2123             nla_put_u32(skb, NDTPA_MCAST_REPROBES,
2124                         NEIGH_VAR(parms, MCAST_REPROBES)) ||
2125             nla_put_msecs(skb, NDTPA_REACHABLE_TIME, parms->reachable_time,
2126                           NDTPA_PAD) ||
2127             nla_put_msecs(skb, NDTPA_BASE_REACHABLE_TIME,
2128                           NEIGH_VAR(parms, BASE_REACHABLE_TIME), NDTPA_PAD) ||
2129             nla_put_msecs(skb, NDTPA_GC_STALETIME,
2130                           NEIGH_VAR(parms, GC_STALETIME), NDTPA_PAD) ||
2131             nla_put_msecs(skb, NDTPA_DELAY_PROBE_TIME,
2132                           NEIGH_VAR(parms, DELAY_PROBE_TIME), NDTPA_PAD) ||
2133             nla_put_msecs(skb, NDTPA_RETRANS_TIME,
2134                           NEIGH_VAR(parms, RETRANS_TIME), NDTPA_PAD) ||
2135             nla_put_msecs(skb, NDTPA_ANYCAST_DELAY,
2136                           NEIGH_VAR(parms, ANYCAST_DELAY), NDTPA_PAD) ||
2137             nla_put_msecs(skb, NDTPA_PROXY_DELAY,
2138                           NEIGH_VAR(parms, PROXY_DELAY), NDTPA_PAD) ||
2139             nla_put_msecs(skb, NDTPA_LOCKTIME,
2140                           NEIGH_VAR(parms, LOCKTIME), NDTPA_PAD) ||
2141             nla_put_msecs(skb, NDTPA_INTERVAL_PROBE_TIME_MS,
2142                           NEIGH_VAR(parms, INTERVAL_PROBE_TIME_MS), NDTPA_PAD))
2143                 goto nla_put_failure;
2144         return nla_nest_end(skb, nest);
2145
2146 nla_put_failure:
2147         nla_nest_cancel(skb, nest);
2148         return -EMSGSIZE;
2149 }
2150
2151 static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
2152                               u32 pid, u32 seq, int type, int flags)
2153 {
2154         struct nlmsghdr *nlh;
2155         struct ndtmsg *ndtmsg;
2156
2157         nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
2158         if (nlh == NULL)
2159                 return -EMSGSIZE;
2160
2161         ndtmsg = nlmsg_data(nlh);
2162
2163         read_lock_bh(&tbl->lock);
2164         ndtmsg->ndtm_family = tbl->family;
2165         ndtmsg->ndtm_pad1   = 0;
2166         ndtmsg->ndtm_pad2   = 0;
2167
2168         if (nla_put_string(skb, NDTA_NAME, tbl->id) ||
2169             nla_put_msecs(skb, NDTA_GC_INTERVAL, READ_ONCE(tbl->gc_interval),
2170                           NDTA_PAD) ||
2171             nla_put_u32(skb, NDTA_THRESH1, READ_ONCE(tbl->gc_thresh1)) ||
2172             nla_put_u32(skb, NDTA_THRESH2, READ_ONCE(tbl->gc_thresh2)) ||
2173             nla_put_u32(skb, NDTA_THRESH3, READ_ONCE(tbl->gc_thresh3)))
2174                 goto nla_put_failure;
2175         {
2176                 unsigned long now = jiffies;
2177                 long flush_delta = now - READ_ONCE(tbl->last_flush);
2178                 long rand_delta = now - READ_ONCE(tbl->last_rand);
2179                 struct neigh_hash_table *nht;
2180                 struct ndt_config ndc = {
2181                         .ndtc_key_len           = tbl->key_len,
2182                         .ndtc_entry_size        = tbl->entry_size,
2183                         .ndtc_entries           = atomic_read(&tbl->entries),
2184                         .ndtc_last_flush        = jiffies_to_msecs(flush_delta),
2185                         .ndtc_last_rand         = jiffies_to_msecs(rand_delta),
2186                         .ndtc_proxy_qlen        = READ_ONCE(tbl->proxy_queue.qlen),
2187                 };
2188
2189                 rcu_read_lock();
2190                 nht = rcu_dereference(tbl->nht);
2191                 ndc.ndtc_hash_rnd = nht->hash_rnd[0];
2192                 ndc.ndtc_hash_mask = ((1 << nht->hash_shift) - 1);
2193                 rcu_read_unlock();
2194
2195                 if (nla_put(skb, NDTA_CONFIG, sizeof(ndc), &ndc))
2196                         goto nla_put_failure;
2197         }
2198
2199         {
2200                 int cpu;
2201                 struct ndt_stats ndst;
2202
2203                 memset(&ndst, 0, sizeof(ndst));
2204
2205                 for_each_possible_cpu(cpu) {
2206                         struct neigh_statistics *st;
2207
2208                         st = per_cpu_ptr(tbl->stats, cpu);
2209                         ndst.ndts_allocs                += READ_ONCE(st->allocs);
2210                         ndst.ndts_destroys              += READ_ONCE(st->destroys);
2211                         ndst.ndts_hash_grows            += READ_ONCE(st->hash_grows);
2212                         ndst.ndts_res_failed            += READ_ONCE(st->res_failed);
2213                         ndst.ndts_lookups               += READ_ONCE(st->lookups);
2214                         ndst.ndts_hits                  += READ_ONCE(st->hits);
2215                         ndst.ndts_rcv_probes_mcast      += READ_ONCE(st->rcv_probes_mcast);
2216                         ndst.ndts_rcv_probes_ucast      += READ_ONCE(st->rcv_probes_ucast);
2217                         ndst.ndts_periodic_gc_runs      += READ_ONCE(st->periodic_gc_runs);
2218                         ndst.ndts_forced_gc_runs        += READ_ONCE(st->forced_gc_runs);
2219                         ndst.ndts_table_fulls           += READ_ONCE(st->table_fulls);
2220                 }
2221
2222                 if (nla_put_64bit(skb, NDTA_STATS, sizeof(ndst), &ndst,
2223                                   NDTA_PAD))
2224                         goto nla_put_failure;
2225         }
2226
2227         BUG_ON(tbl->parms.dev);
2228         if (neightbl_fill_parms(skb, &tbl->parms) < 0)
2229                 goto nla_put_failure;
2230
2231         read_unlock_bh(&tbl->lock);
2232         nlmsg_end(skb, nlh);
2233         return 0;
2234
2235 nla_put_failure:
2236         read_unlock_bh(&tbl->lock);
2237         nlmsg_cancel(skb, nlh);
2238         return -EMSGSIZE;
2239 }
2240
2241 static int neightbl_fill_param_info(struct sk_buff *skb,
2242                                     struct neigh_table *tbl,
2243                                     struct neigh_parms *parms,
2244                                     u32 pid, u32 seq, int type,
2245                                     unsigned int flags)
2246 {
2247         struct ndtmsg *ndtmsg;
2248         struct nlmsghdr *nlh;
2249
2250         nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
2251         if (nlh == NULL)
2252                 return -EMSGSIZE;
2253
2254         ndtmsg = nlmsg_data(nlh);
2255
2256         read_lock_bh(&tbl->lock);
2257         ndtmsg->ndtm_family = tbl->family;
2258         ndtmsg->ndtm_pad1   = 0;
2259         ndtmsg->ndtm_pad2   = 0;
2260
2261         if (nla_put_string(skb, NDTA_NAME, tbl->id) < 0 ||
2262             neightbl_fill_parms(skb, parms) < 0)
2263                 goto errout;
2264
2265         read_unlock_bh(&tbl->lock);
2266         nlmsg_end(skb, nlh);
2267         return 0;
2268 errout:
2269         read_unlock_bh(&tbl->lock);
2270         nlmsg_cancel(skb, nlh);
2271         return -EMSGSIZE;
2272 }
2273
2274 static const struct nla_policy nl_neightbl_policy[NDTA_MAX+1] = {
2275         [NDTA_NAME]             = { .type = NLA_STRING },
2276         [NDTA_THRESH1]          = { .type = NLA_U32 },
2277         [NDTA_THRESH2]          = { .type = NLA_U32 },
2278         [NDTA_THRESH3]          = { .type = NLA_U32 },
2279         [NDTA_GC_INTERVAL]      = { .type = NLA_U64 },
2280         [NDTA_PARMS]            = { .type = NLA_NESTED },
2281 };
2282
2283 static const struct nla_policy nl_ntbl_parm_policy[NDTPA_MAX+1] = {
2284         [NDTPA_IFINDEX]                 = { .type = NLA_U32 },
2285         [NDTPA_QUEUE_LEN]               = { .type = NLA_U32 },
2286         [NDTPA_PROXY_QLEN]              = { .type = NLA_U32 },
2287         [NDTPA_APP_PROBES]              = { .type = NLA_U32 },
2288         [NDTPA_UCAST_PROBES]            = { .type = NLA_U32 },
2289         [NDTPA_MCAST_PROBES]            = { .type = NLA_U32 },
2290         [NDTPA_MCAST_REPROBES]          = { .type = NLA_U32 },
2291         [NDTPA_BASE_REACHABLE_TIME]     = { .type = NLA_U64 },
2292         [NDTPA_GC_STALETIME]            = { .type = NLA_U64 },
2293         [NDTPA_DELAY_PROBE_TIME]        = { .type = NLA_U64 },
2294         [NDTPA_RETRANS_TIME]            = { .type = NLA_U64 },
2295         [NDTPA_ANYCAST_DELAY]           = { .type = NLA_U64 },
2296         [NDTPA_PROXY_DELAY]             = { .type = NLA_U64 },
2297         [NDTPA_LOCKTIME]                = { .type = NLA_U64 },
2298         [NDTPA_INTERVAL_PROBE_TIME_MS]  = { .type = NLA_U64, .min = 1 },
2299 };
2300
2301 static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
2302                         struct netlink_ext_ack *extack)
2303 {
2304         struct net *net = sock_net(skb->sk);
2305         struct neigh_table *tbl;
2306         struct ndtmsg *ndtmsg;
2307         struct nlattr *tb[NDTA_MAX+1];
2308         bool found = false;
2309         int err, tidx;
2310
2311         err = nlmsg_parse_deprecated(nlh, sizeof(*ndtmsg), tb, NDTA_MAX,
2312                                      nl_neightbl_policy, extack);
2313         if (err < 0)
2314                 goto errout;
2315
2316         if (tb[NDTA_NAME] == NULL) {
2317                 err = -EINVAL;
2318                 goto errout;
2319         }
2320
2321         ndtmsg = nlmsg_data(nlh);
2322
2323         for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
2324                 tbl = neigh_tables[tidx];
2325                 if (!tbl)
2326                         continue;
2327                 if (ndtmsg->ndtm_family && tbl->family != ndtmsg->ndtm_family)
2328                         continue;
2329                 if (nla_strcmp(tb[NDTA_NAME], tbl->id) == 0) {
2330                         found = true;
2331                         break;
2332                 }
2333         }
2334
2335         if (!found)
2336                 return -ENOENT;
2337
2338         /*
2339          * We acquire tbl->lock to be nice to the periodic timers and
2340          * make sure they always see a consistent set of values.
2341          */
2342         write_lock_bh(&tbl->lock);
2343
2344         if (tb[NDTA_PARMS]) {
2345                 struct nlattr *tbp[NDTPA_MAX+1];
2346                 struct neigh_parms *p;
2347                 int i, ifindex = 0;
2348
2349                 err = nla_parse_nested_deprecated(tbp, NDTPA_MAX,
2350                                                   tb[NDTA_PARMS],
2351                                                   nl_ntbl_parm_policy, extack);
2352                 if (err < 0)
2353                         goto errout_tbl_lock;
2354
2355                 if (tbp[NDTPA_IFINDEX])
2356                         ifindex = nla_get_u32(tbp[NDTPA_IFINDEX]);
2357
2358                 p = lookup_neigh_parms(tbl, net, ifindex);
2359                 if (p == NULL) {
2360                         err = -ENOENT;
2361                         goto errout_tbl_lock;
2362                 }
2363
2364                 for (i = 1; i <= NDTPA_MAX; i++) {
2365                         if (tbp[i] == NULL)
2366                                 continue;
2367
2368                         switch (i) {
2369                         case NDTPA_QUEUE_LEN:
2370                                 NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
2371                                               nla_get_u32(tbp[i]) *
2372                                               SKB_TRUESIZE(ETH_FRAME_LEN));
2373                                 break;
2374                         case NDTPA_QUEUE_LENBYTES:
2375                                 NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
2376                                               nla_get_u32(tbp[i]));
2377                                 break;
2378                         case NDTPA_PROXY_QLEN:
2379                                 NEIGH_VAR_SET(p, PROXY_QLEN,
2380                                               nla_get_u32(tbp[i]));
2381                                 break;
2382                         case NDTPA_APP_PROBES:
2383                                 NEIGH_VAR_SET(p, APP_PROBES,
2384                                               nla_get_u32(tbp[i]));
2385                                 break;
2386                         case NDTPA_UCAST_PROBES:
2387                                 NEIGH_VAR_SET(p, UCAST_PROBES,
2388                                               nla_get_u32(tbp[i]));
2389                                 break;
2390                         case NDTPA_MCAST_PROBES:
2391                                 NEIGH_VAR_SET(p, MCAST_PROBES,
2392                                               nla_get_u32(tbp[i]));
2393                                 break;
2394                         case NDTPA_MCAST_REPROBES:
2395                                 NEIGH_VAR_SET(p, MCAST_REPROBES,
2396                                               nla_get_u32(tbp[i]));
2397                                 break;
2398                         case NDTPA_BASE_REACHABLE_TIME:
2399                                 NEIGH_VAR_SET(p, BASE_REACHABLE_TIME,
2400                                               nla_get_msecs(tbp[i]));
2401                                 /* update reachable_time as well, otherwise, the change will
2402                                  * only be effective after the next time neigh_periodic_work
2403                                  * decides to recompute it (can be multiple minutes)
2404                                  */
2405                                 p->reachable_time =
2406                                         neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
2407                                 break;
2408                         case NDTPA_GC_STALETIME:
2409                                 NEIGH_VAR_SET(p, GC_STALETIME,
2410                                               nla_get_msecs(tbp[i]));
2411                                 break;
2412                         case NDTPA_DELAY_PROBE_TIME:
2413                                 NEIGH_VAR_SET(p, DELAY_PROBE_TIME,
2414                                               nla_get_msecs(tbp[i]));
2415                                 call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
2416                                 break;
2417                         case NDTPA_INTERVAL_PROBE_TIME_MS:
2418                                 NEIGH_VAR_SET(p, INTERVAL_PROBE_TIME_MS,
2419                                               nla_get_msecs(tbp[i]));
2420                                 break;
2421                         case NDTPA_RETRANS_TIME:
2422                                 NEIGH_VAR_SET(p, RETRANS_TIME,
2423                                               nla_get_msecs(tbp[i]));
2424                                 break;
2425                         case NDTPA_ANYCAST_DELAY:
2426                                 NEIGH_VAR_SET(p, ANYCAST_DELAY,
2427                                               nla_get_msecs(tbp[i]));
2428                                 break;
2429                         case NDTPA_PROXY_DELAY:
2430                                 NEIGH_VAR_SET(p, PROXY_DELAY,
2431                                               nla_get_msecs(tbp[i]));
2432                                 break;
2433                         case NDTPA_LOCKTIME:
2434                                 NEIGH_VAR_SET(p, LOCKTIME,
2435                                               nla_get_msecs(tbp[i]));
2436                                 break;
2437                         }
2438                 }
2439         }
2440
2441         err = -ENOENT;
2442         if ((tb[NDTA_THRESH1] || tb[NDTA_THRESH2] ||
2443              tb[NDTA_THRESH3] || tb[NDTA_GC_INTERVAL]) &&
2444             !net_eq(net, &init_net))
2445                 goto errout_tbl_lock;
2446
2447         if (tb[NDTA_THRESH1])
2448                 WRITE_ONCE(tbl->gc_thresh1, nla_get_u32(tb[NDTA_THRESH1]));
2449
2450         if (tb[NDTA_THRESH2])
2451                 WRITE_ONCE(tbl->gc_thresh2, nla_get_u32(tb[NDTA_THRESH2]));
2452
2453         if (tb[NDTA_THRESH3])
2454                 WRITE_ONCE(tbl->gc_thresh3, nla_get_u32(tb[NDTA_THRESH3]));
2455
2456         if (tb[NDTA_GC_INTERVAL])
2457                 WRITE_ONCE(tbl->gc_interval, nla_get_msecs(tb[NDTA_GC_INTERVAL]));
2458
2459         err = 0;
2460
2461 errout_tbl_lock:
2462         write_unlock_bh(&tbl->lock);
2463 errout:
2464         return err;
2465 }
2466
2467 static int neightbl_valid_dump_info(const struct nlmsghdr *nlh,
2468                                     struct netlink_ext_ack *extack)
2469 {
2470         struct ndtmsg *ndtm;
2471
2472         if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ndtm))) {
2473                 NL_SET_ERR_MSG(extack, "Invalid header for neighbor table dump request");
2474                 return -EINVAL;
2475         }
2476
2477         ndtm = nlmsg_data(nlh);
2478         if (ndtm->ndtm_pad1  || ndtm->ndtm_pad2) {
2479                 NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor table dump request");
2480                 return -EINVAL;
2481         }
2482
2483         if (nlmsg_attrlen(nlh, sizeof(*ndtm))) {
2484                 NL_SET_ERR_MSG(extack, "Invalid data after header in neighbor table dump request");
2485                 return -EINVAL;
2486         }
2487
2488         return 0;
2489 }
2490
2491 static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
2492 {
2493         const struct nlmsghdr *nlh = cb->nlh;
2494         struct net *net = sock_net(skb->sk);
2495         int family, tidx, nidx = 0;
2496         int tbl_skip = cb->args[0];
2497         int neigh_skip = cb->args[1];
2498         struct neigh_table *tbl;
2499
2500         if (cb->strict_check) {
2501                 int err = neightbl_valid_dump_info(nlh, cb->extack);
2502
2503                 if (err < 0)
2504                         return err;
2505         }
2506
2507         family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;
2508
2509         for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
2510                 struct neigh_parms *p;
2511
2512                 tbl = neigh_tables[tidx];
2513                 if (!tbl)
2514                         continue;
2515
2516                 if (tidx < tbl_skip || (family && tbl->family != family))
2517                         continue;
2518
2519                 if (neightbl_fill_info(skb, tbl, NETLINK_CB(cb->skb).portid,
2520                                        nlh->nlmsg_seq, RTM_NEWNEIGHTBL,
2521                                        NLM_F_MULTI) < 0)
2522                         break;
2523
2524                 nidx = 0;
2525                 p = list_next_entry(&tbl->parms, list);
2526                 list_for_each_entry_from(p, &tbl->parms_list, list) {
2527                         if (!net_eq(neigh_parms_net(p), net))
2528                                 continue;
2529
2530                         if (nidx < neigh_skip)
2531                                 goto next;
2532
2533                         if (neightbl_fill_param_info(skb, tbl, p,
2534                                                      NETLINK_CB(cb->skb).portid,
2535                                                      nlh->nlmsg_seq,
2536                                                      RTM_NEWNEIGHTBL,
2537                                                      NLM_F_MULTI) < 0)
2538                                 goto out;
2539                 next:
2540                         nidx++;
2541                 }
2542
2543                 neigh_skip = 0;
2544         }
2545 out:
2546         cb->args[0] = tidx;
2547         cb->args[1] = nidx;
2548
2549         return skb->len;
2550 }
2551
2552 static int neigh_fill_info(struct sk_buff *skb, struct neighbour *neigh,
2553                            u32 pid, u32 seq, int type, unsigned int flags)
2554 {
2555         u32 neigh_flags, neigh_flags_ext;
2556         unsigned long now = jiffies;
2557         struct nda_cacheinfo ci;
2558         struct nlmsghdr *nlh;
2559         struct ndmsg *ndm;
2560
2561         nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
2562         if (nlh == NULL)
2563                 return -EMSGSIZE;
2564
2565         neigh_flags_ext = neigh->flags >> NTF_EXT_SHIFT;
2566         neigh_flags     = neigh->flags & NTF_OLD_MASK;
2567
2568         ndm = nlmsg_data(nlh);
2569         ndm->ndm_family  = neigh->ops->family;
2570         ndm->ndm_pad1    = 0;
2571         ndm->ndm_pad2    = 0;
2572         ndm->ndm_flags   = neigh_flags;
2573         ndm->ndm_type    = neigh->type;
2574         ndm->ndm_ifindex = neigh->dev->ifindex;
2575
2576         if (nla_put(skb, NDA_DST, neigh->tbl->key_len, neigh->primary_key))
2577                 goto nla_put_failure;
2578
2579         read_lock_bh(&neigh->lock);
2580         ndm->ndm_state   = neigh->nud_state;
2581         if (neigh->nud_state & NUD_VALID) {
2582                 char haddr[MAX_ADDR_LEN];
2583
2584                 neigh_ha_snapshot(haddr, neigh, neigh->dev);
2585                 if (nla_put(skb, NDA_LLADDR, neigh->dev->addr_len, haddr) < 0) {
2586                         read_unlock_bh(&neigh->lock);
2587                         goto nla_put_failure;
2588                 }
2589         }
2590
2591         ci.ndm_used      = jiffies_to_clock_t(now - neigh->used);
2592         ci.ndm_confirmed = jiffies_to_clock_t(now - neigh->confirmed);
2593         ci.ndm_updated   = jiffies_to_clock_t(now - neigh->updated);
2594         ci.ndm_refcnt    = refcount_read(&neigh->refcnt) - 1;
2595         read_unlock_bh(&neigh->lock);
2596
2597         if (nla_put_u32(skb, NDA_PROBES, atomic_read(&neigh->probes)) ||
2598             nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci))
2599                 goto nla_put_failure;
2600
2601         if (neigh->protocol && nla_put_u8(skb, NDA_PROTOCOL, neigh->protocol))
2602                 goto nla_put_failure;
2603         if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
2604                 goto nla_put_failure;
2605
2606         nlmsg_end(skb, nlh);
2607         return 0;
2608
2609 nla_put_failure:
2610         nlmsg_cancel(skb, nlh);
2611         return -EMSGSIZE;
2612 }
2613
2614 static int pneigh_fill_info(struct sk_buff *skb, struct pneigh_entry *pn,
2615                             u32 pid, u32 seq, int type, unsigned int flags,
2616                             struct neigh_table *tbl)
2617 {
2618         u32 neigh_flags, neigh_flags_ext;
2619         struct nlmsghdr *nlh;
2620         struct ndmsg *ndm;
2621
2622         nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
2623         if (nlh == NULL)
2624                 return -EMSGSIZE;
2625
2626         neigh_flags_ext = pn->flags >> NTF_EXT_SHIFT;
2627         neigh_flags     = pn->flags & NTF_OLD_MASK;
2628
2629         ndm = nlmsg_data(nlh);
2630         ndm->ndm_family  = tbl->family;
2631         ndm->ndm_pad1    = 0;
2632         ndm->ndm_pad2    = 0;
2633         ndm->ndm_flags   = neigh_flags | NTF_PROXY;
2634         ndm->ndm_type    = RTN_UNICAST;
2635         ndm->ndm_ifindex = pn->dev ? pn->dev->ifindex : 0;
2636         ndm->ndm_state   = NUD_NONE;
2637
2638         if (nla_put(skb, NDA_DST, tbl->key_len, pn->key))
2639                 goto nla_put_failure;
2640
2641         if (pn->protocol && nla_put_u8(skb, NDA_PROTOCOL, pn->protocol))
2642                 goto nla_put_failure;
2643         if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
2644                 goto nla_put_failure;
2645
2646         nlmsg_end(skb, nlh);
2647         return 0;
2648
2649 nla_put_failure:
2650         nlmsg_cancel(skb, nlh);
2651         return -EMSGSIZE;
2652 }
2653
2654 static void neigh_update_notify(struct neighbour *neigh, u32 nlmsg_pid)
2655 {
2656         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
2657         __neigh_notify(neigh, RTM_NEWNEIGH, 0, nlmsg_pid);
2658 }
2659
2660 static bool neigh_master_filtered(struct net_device *dev, int master_idx)
2661 {
2662         struct net_device *master;
2663
2664         if (!master_idx)
2665                 return false;
2666
2667         master = dev ? netdev_master_upper_dev_get(dev) : NULL;
2668
2669         /* 0 is already used to denote NDA_MASTER wasn't passed, therefore need another
2670          * invalid value for ifindex to denote "no master".
2671          */
2672         if (master_idx == -1)
2673                 return !!master;
2674
2675         if (!master || master->ifindex != master_idx)
2676                 return true;
2677
2678         return false;
2679 }
2680
2681 static bool neigh_ifindex_filtered(struct net_device *dev, int filter_idx)
2682 {
2683         if (filter_idx && (!dev || dev->ifindex != filter_idx))
2684                 return true;
2685
2686         return false;
2687 }
2688
2689 struct neigh_dump_filter {
2690         int master_idx;
2691         int dev_idx;
2692 };
2693
2694 static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
2695                             struct netlink_callback *cb,
2696                             struct neigh_dump_filter *filter)
2697 {
2698         struct net *net = sock_net(skb->sk);
2699         struct neighbour *n;
2700         int rc, h, s_h = cb->args[1];
2701         int idx, s_idx = idx = cb->args[2];
2702         struct neigh_hash_table *nht;
2703         unsigned int flags = NLM_F_MULTI;
2704
2705         if (filter->dev_idx || filter->master_idx)
2706                 flags |= NLM_F_DUMP_FILTERED;
2707
2708         rcu_read_lock();
2709         nht = rcu_dereference(tbl->nht);
2710
2711         for (h = s_h; h < (1 << nht->hash_shift); h++) {
2712                 if (h > s_h)
2713                         s_idx = 0;
2714                 for (n = rcu_dereference(nht->hash_buckets[h]), idx = 0;
2715                      n != NULL;
2716                      n = rcu_dereference(n->next)) {
2717                         if (idx < s_idx || !net_eq(dev_net(n->dev), net))
2718                                 goto next;
2719                         if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
2720                             neigh_master_filtered(n->dev, filter->master_idx))
2721                                 goto next;
2722                         if (neigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
2723                                             cb->nlh->nlmsg_seq,
2724                                             RTM_NEWNEIGH,
2725                                             flags) < 0) {
2726                                 rc = -1;
2727                                 goto out;
2728                         }
2729 next:
2730                         idx++;
2731                 }
2732         }
2733         rc = skb->len;
2734 out:
2735         rcu_read_unlock();
2736         cb->args[1] = h;
2737         cb->args[2] = idx;
2738         return rc;
2739 }
2740
2741 static int pneigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
2742                              struct netlink_callback *cb,
2743                              struct neigh_dump_filter *filter)
2744 {
2745         struct pneigh_entry *n;
2746         struct net *net = sock_net(skb->sk);
2747         int rc, h, s_h = cb->args[3];
2748         int idx, s_idx = idx = cb->args[4];
2749         unsigned int flags = NLM_F_MULTI;
2750
2751         if (filter->dev_idx || filter->master_idx)
2752                 flags |= NLM_F_DUMP_FILTERED;
2753
2754         read_lock_bh(&tbl->lock);
2755
2756         for (h = s_h; h <= PNEIGH_HASHMASK; h++) {
2757                 if (h > s_h)
2758                         s_idx = 0;
2759                 for (n = tbl->phash_buckets[h], idx = 0; n; n = n->next) {
2760                         if (idx < s_idx || pneigh_net(n) != net)
2761                                 goto next;
2762                         if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
2763                             neigh_master_filtered(n->dev, filter->master_idx))
2764                                 goto next;
2765                         if (pneigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
2766                                             cb->nlh->nlmsg_seq,
2767                                             RTM_NEWNEIGH, flags, tbl) < 0) {
2768                                 read_unlock_bh(&tbl->lock);
2769                                 rc = -1;
2770                                 goto out;
2771                         }
2772                 next:
2773                         idx++;
2774                 }
2775         }
2776
2777         read_unlock_bh(&tbl->lock);
2778         rc = skb->len;
2779 out:
2780         cb->args[3] = h;
2781         cb->args[4] = idx;
2782         return rc;
2783
2784 }
2785
2786 static int neigh_valid_dump_req(const struct nlmsghdr *nlh,
2787                                 bool strict_check,
2788                                 struct neigh_dump_filter *filter,
2789                                 struct netlink_ext_ack *extack)
2790 {
2791         struct nlattr *tb[NDA_MAX + 1];
2792         int err, i;
2793
2794         if (strict_check) {
2795                 struct ndmsg *ndm;
2796
2797                 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ndm))) {
2798                         NL_SET_ERR_MSG(extack, "Invalid header for neighbor dump request");
2799                         return -EINVAL;
2800                 }
2801
2802                 ndm = nlmsg_data(nlh);
2803                 if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_ifindex ||
2804                     ndm->ndm_state || ndm->ndm_type) {
2805                         NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor dump request");
2806                         return -EINVAL;
2807                 }
2808
2809                 if (ndm->ndm_flags & ~NTF_PROXY) {
2810                         NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor dump request");
2811                         return -EINVAL;
2812                 }
2813
2814                 err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg),
2815                                                     tb, NDA_MAX, nda_policy,
2816                                                     extack);
2817         } else {
2818                 err = nlmsg_parse_deprecated(nlh, sizeof(struct ndmsg), tb,
2819                                              NDA_MAX, nda_policy, extack);
2820         }
2821         if (err < 0)
2822                 return err;
2823
2824         for (i = 0; i <= NDA_MAX; ++i) {
2825                 if (!tb[i])
2826                         continue;
2827
2828                 /* all new attributes should require strict_check */
2829                 switch (i) {
2830                 case NDA_IFINDEX:
2831                         filter->dev_idx = nla_get_u32(tb[i]);
2832                         break;
2833                 case NDA_MASTER:
2834                         filter->master_idx = nla_get_u32(tb[i]);
2835                         break;
2836                 default:
2837                         if (strict_check) {
2838                                 NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor dump request");
2839                                 return -EINVAL;
2840                         }
2841                 }
2842         }
2843
2844         return 0;
2845 }
2846
2847 static int neigh_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
2848 {
2849         const struct nlmsghdr *nlh = cb->nlh;
2850         struct neigh_dump_filter filter = {};
2851         struct neigh_table *tbl;
2852         int t, family, s_t;
2853         int proxy = 0;
2854         int err;
2855
2856         family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;
2857
2858         /* check for full ndmsg structure presence, family member is
2859          * the same for both structures
2860          */
2861         if (nlmsg_len(nlh) >= sizeof(struct ndmsg) &&
2862             ((struct ndmsg *)nlmsg_data(nlh))->ndm_flags == NTF_PROXY)
2863                 proxy = 1;
2864
2865         err = neigh_valid_dump_req(nlh, cb->strict_check, &filter, cb->extack);
2866         if (err < 0 && cb->strict_check)
2867                 return err;
2868
2869         s_t = cb->args[0];
2870
2871         for (t = 0; t < NEIGH_NR_TABLES; t++) {
2872                 tbl = neigh_tables[t];
2873
2874                 if (!tbl)
2875                         continue;
2876                 if (t < s_t || (family && tbl->family != family))
2877                         continue;
2878                 if (t > s_t)
2879                         memset(&cb->args[1], 0, sizeof(cb->args) -
2880                                                 sizeof(cb->args[0]));
2881                 if (proxy)
2882                         err = pneigh_dump_table(tbl, skb, cb, &filter);
2883                 else
2884                         err = neigh_dump_table(tbl, skb, cb, &filter);
2885                 if (err < 0)
2886                         break;
2887         }
2888
2889         cb->args[0] = t;
2890         return skb->len;
2891 }
2892
2893 static int neigh_valid_get_req(const struct nlmsghdr *nlh,
2894                                struct neigh_table **tbl,
2895                                void **dst, int *dev_idx, u8 *ndm_flags,
2896                                struct netlink_ext_ack *extack)
2897 {
2898         struct nlattr *tb[NDA_MAX + 1];
2899         struct ndmsg *ndm;
2900         int err, i;
2901
2902         if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ndm))) {
2903                 NL_SET_ERR_MSG(extack, "Invalid header for neighbor get request");
2904                 return -EINVAL;
2905         }
2906
2907         ndm = nlmsg_data(nlh);
2908         if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_state ||
2909             ndm->ndm_type) {
2910                 NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor get request");
2911                 return -EINVAL;
2912         }
2913
2914         if (ndm->ndm_flags & ~NTF_PROXY) {
2915                 NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor get request");
2916                 return -EINVAL;
2917         }
2918
2919         err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg), tb,
2920                                             NDA_MAX, nda_policy, extack);
2921         if (err < 0)
2922                 return err;
2923
2924         *ndm_flags = ndm->ndm_flags;
2925         *dev_idx = ndm->ndm_ifindex;
2926         *tbl = neigh_find_table(ndm->ndm_family);
2927         if (*tbl == NULL) {
2928                 NL_SET_ERR_MSG(extack, "Unsupported family in header for neighbor get request");
2929                 return -EAFNOSUPPORT;
2930         }
2931
2932         for (i = 0; i <= NDA_MAX; ++i) {
2933                 if (!tb[i])
2934                         continue;
2935
2936                 switch (i) {
2937                 case NDA_DST:
2938                         if (nla_len(tb[i]) != (int)(*tbl)->key_len) {
2939                                 NL_SET_ERR_MSG(extack, "Invalid network address in neighbor get request");
2940                                 return -EINVAL;
2941                         }
2942                         *dst = nla_data(tb[i]);
2943                         break;
2944                 default:
2945                         NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor get request");
2946                         return -EINVAL;
2947                 }
2948         }
2949
2950         return 0;
2951 }
2952
2953 static inline size_t neigh_nlmsg_size(void)
2954 {
2955         return NLMSG_ALIGN(sizeof(struct ndmsg))
2956                + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
2957                + nla_total_size(MAX_ADDR_LEN) /* NDA_LLADDR */
2958                + nla_total_size(sizeof(struct nda_cacheinfo))
2959                + nla_total_size(4)  /* NDA_PROBES */
2960                + nla_total_size(4)  /* NDA_FLAGS_EXT */
2961                + nla_total_size(1); /* NDA_PROTOCOL */
2962 }
2963
2964 static int neigh_get_reply(struct net *net, struct neighbour *neigh,
2965                            u32 pid, u32 seq)
2966 {
2967         struct sk_buff *skb;
2968         int err = 0;
2969
2970         skb = nlmsg_new(neigh_nlmsg_size(), GFP_KERNEL);
2971         if (!skb)
2972                 return -ENOBUFS;
2973
2974         err = neigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0);
2975         if (err) {
2976                 kfree_skb(skb);
2977                 goto errout;
2978         }
2979
2980         err = rtnl_unicast(skb, net, pid);
2981 errout:
2982         return err;
2983 }
2984
2985 static inline size_t pneigh_nlmsg_size(void)
2986 {
2987         return NLMSG_ALIGN(sizeof(struct ndmsg))
2988                + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
2989                + nla_total_size(4)  /* NDA_FLAGS_EXT */
2990                + nla_total_size(1); /* NDA_PROTOCOL */
2991 }
2992
2993 static int pneigh_get_reply(struct net *net, struct pneigh_entry *neigh,
2994                             u32 pid, u32 seq, struct neigh_table *tbl)
2995 {
2996         struct sk_buff *skb;
2997         int err = 0;
2998
2999         skb = nlmsg_new(pneigh_nlmsg_size(), GFP_KERNEL);
3000         if (!skb)
3001                 return -ENOBUFS;
3002
3003         err = pneigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0, tbl);
3004         if (err) {
3005                 kfree_skb(skb);
3006                 goto errout;
3007         }
3008
3009         err = rtnl_unicast(skb, net, pid);
3010 errout:
3011         return err;
3012 }
3013
3014 static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
3015                      struct netlink_ext_ack *extack)
3016 {
3017         struct net *net = sock_net(in_skb->sk);
3018         struct net_device *dev = NULL;
3019         struct neigh_table *tbl = NULL;
3020         struct neighbour *neigh;
3021         void *dst = NULL;
3022         u8 ndm_flags = 0;
3023         int dev_idx = 0;
3024         int err;
3025
3026         err = neigh_valid_get_req(nlh, &tbl, &dst, &dev_idx, &ndm_flags,
3027                                   extack);
3028         if (err < 0)
3029                 return err;
3030
3031         if (dev_idx) {
3032                 dev = __dev_get_by_index(net, dev_idx);
3033                 if (!dev) {
3034                         NL_SET_ERR_MSG(extack, "Unknown device ifindex");
3035                         return -ENODEV;
3036                 }
3037         }
3038
3039         if (!dst) {
3040                 NL_SET_ERR_MSG(extack, "Network address not specified");
3041                 return -EINVAL;
3042         }
3043
3044         if (ndm_flags & NTF_PROXY) {
3045                 struct pneigh_entry *pn;
3046
3047                 pn = pneigh_lookup(tbl, net, dst, dev, 0);
3048                 if (!pn) {
3049                         NL_SET_ERR_MSG(extack, "Proxy neighbour entry not found");
3050                         return -ENOENT;
3051                 }
3052                 return pneigh_get_reply(net, pn, NETLINK_CB(in_skb).portid,
3053                                         nlh->nlmsg_seq, tbl);
3054         }
3055
3056         if (!dev) {
3057                 NL_SET_ERR_MSG(extack, "No device specified");
3058                 return -EINVAL;
3059         }
3060
3061         neigh = neigh_lookup(tbl, dst, dev);
3062         if (!neigh) {
3063                 NL_SET_ERR_MSG(extack, "Neighbour entry not found");
3064                 return -ENOENT;
3065         }
3066
3067         err = neigh_get_reply(net, neigh, NETLINK_CB(in_skb).portid,
3068                               nlh->nlmsg_seq);
3069
3070         neigh_release(neigh);
3071
3072         return err;
3073 }
3074
3075 void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void *), void *cookie)
3076 {
3077         int chain;
3078         struct neigh_hash_table *nht;
3079
3080         rcu_read_lock();
3081         nht = rcu_dereference(tbl->nht);
3082
3083         read_lock_bh(&tbl->lock); /* avoid resizes */
3084         for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
3085                 struct neighbour *n;
3086
3087                 for (n = rcu_dereference(nht->hash_buckets[chain]);
3088                      n != NULL;
3089                      n = rcu_dereference(n->next))
3090                         cb(n, cookie);
3091         }
3092         read_unlock_bh(&tbl->lock);
3093         rcu_read_unlock();
3094 }
3095 EXPORT_SYMBOL(neigh_for_each);
3096
3097 /* The tbl->lock must be held as a writer and BH disabled. */
3098 void __neigh_for_each_release(struct neigh_table *tbl,
3099                               int (*cb)(struct neighbour *))
3100 {
3101         int chain;
3102         struct neigh_hash_table *nht;
3103
3104         nht = rcu_dereference_protected(tbl->nht,
3105                                         lockdep_is_held(&tbl->lock));
3106         for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
3107                 struct neighbour *n;
3108                 struct neighbour __rcu **np;
3109
3110                 np = &nht->hash_buckets[chain];
3111                 while ((n = rcu_dereference_protected(*np,
3112                                         lockdep_is_held(&tbl->lock))) != NULL) {
3113                         int release;
3114
3115                         write_lock(&n->lock);
3116                         release = cb(n);
3117                         if (release) {
3118                                 rcu_assign_pointer(*np,
3119                                         rcu_dereference_protected(n->next,
3120                                                 lockdep_is_held(&tbl->lock)));
3121                                 neigh_mark_dead(n);
3122                         } else
3123                                 np = &n->next;
3124                         write_unlock(&n->lock);
3125                         if (release)
3126                                 neigh_cleanup_and_release(n);
3127                 }
3128         }
3129 }
3130 EXPORT_SYMBOL(__neigh_for_each_release);
3131
3132 int neigh_xmit(int index, struct net_device *dev,
3133                const void *addr, struct sk_buff *skb)
3134 {
3135         int err = -EAFNOSUPPORT;
3136         if (likely(index < NEIGH_NR_TABLES)) {
3137                 struct neigh_table *tbl;
3138                 struct neighbour *neigh;
3139
3140                 tbl = neigh_tables[index];
3141                 if (!tbl)
3142                         goto out;
3143                 rcu_read_lock();
3144                 if (index == NEIGH_ARP_TABLE) {
3145                         u32 key = *((u32 *)addr);
3146
3147                         neigh = __ipv4_neigh_lookup_noref(dev, key);
3148                 } else {
3149                         neigh = __neigh_lookup_noref(tbl, addr, dev);
3150                 }
3151                 if (!neigh)
3152                         neigh = __neigh_create(tbl, addr, dev, false);
3153                 err = PTR_ERR(neigh);
3154                 if (IS_ERR(neigh)) {
3155                         rcu_read_unlock();
3156                         goto out_kfree_skb;
3157                 }
3158                 err = READ_ONCE(neigh->output)(neigh, skb);
3159                 rcu_read_unlock();
3160         }
3161         else if (index == NEIGH_LINK_TABLE) {
3162                 err = dev_hard_header(skb, dev, ntohs(skb->protocol),
3163                                       addr, NULL, skb->len);
3164                 if (err < 0)
3165                         goto out_kfree_skb;
3166                 err = dev_queue_xmit(skb);
3167         }
3168 out:
3169         return err;
3170 out_kfree_skb:
3171         kfree_skb(skb);
3172         goto out;
3173 }
3174 EXPORT_SYMBOL(neigh_xmit);
3175
3176 #ifdef CONFIG_PROC_FS
3177
3178 static struct neighbour *neigh_get_first(struct seq_file *seq)
3179 {
3180         struct neigh_seq_state *state = seq->private;
3181         struct net *net = seq_file_net(seq);
3182         struct neigh_hash_table *nht = state->nht;
3183         struct neighbour *n = NULL;
3184         int bucket;
3185
3186         state->flags &= ~NEIGH_SEQ_IS_PNEIGH;
3187         for (bucket = 0; bucket < (1 << nht->hash_shift); bucket++) {
3188                 n = rcu_dereference(nht->hash_buckets[bucket]);
3189
3190                 while (n) {
3191                         if (!net_eq(dev_net(n->dev), net))
3192                                 goto next;
3193                         if (state->neigh_sub_iter) {
3194                                 loff_t fakep = 0;
3195                                 void *v;
3196
3197                                 v = state->neigh_sub_iter(state, n, &fakep);
3198                                 if (!v)
3199                                         goto next;
3200                         }
3201                         if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
3202                                 break;
3203                         if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
3204                                 break;
3205 next:
3206                         n = rcu_dereference(n->next);
3207                 }
3208
3209                 if (n)
3210                         break;
3211         }
3212         state->bucket = bucket;
3213
3214         return n;
3215 }
3216
3217 static struct neighbour *neigh_get_next(struct seq_file *seq,
3218                                         struct neighbour *n,
3219                                         loff_t *pos)
3220 {
3221         struct neigh_seq_state *state = seq->private;
3222         struct net *net = seq_file_net(seq);
3223         struct neigh_hash_table *nht = state->nht;
3224
3225         if (state->neigh_sub_iter) {
3226                 void *v = state->neigh_sub_iter(state, n, pos);
3227                 if (v)
3228                         return n;
3229         }
3230         n = rcu_dereference(n->next);
3231
3232         while (1) {
3233                 while (n) {
3234                         if (!net_eq(dev_net(n->dev), net))
3235                                 goto next;
3236                         if (state->neigh_sub_iter) {
3237                                 void *v = state->neigh_sub_iter(state, n, pos);
3238                                 if (v)
3239                                         return n;
3240                                 goto next;
3241                         }
3242                         if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
3243                                 break;
3244
3245                         if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
3246                                 break;
3247 next:
3248                         n = rcu_dereference(n->next);
3249                 }
3250
3251                 if (n)
3252                         break;
3253
3254                 if (++state->bucket >= (1 << nht->hash_shift))
3255                         break;
3256
3257                 n = rcu_dereference(nht->hash_buckets[state->bucket]);
3258         }
3259
3260         if (n && pos)
3261                 --(*pos);
3262         return n;
3263 }
3264
3265 static struct neighbour *neigh_get_idx(struct seq_file *seq, loff_t *pos)
3266 {
3267         struct neighbour *n = neigh_get_first(seq);
3268
3269         if (n) {
3270                 --(*pos);
3271                 while (*pos) {
3272                         n = neigh_get_next(seq, n, pos);
3273                         if (!n)
3274                                 break;
3275                 }
3276         }
3277         return *pos ? NULL : n;
3278 }
3279
3280 static struct pneigh_entry *pneigh_get_first(struct seq_file *seq)
3281 {
3282         struct neigh_seq_state *state = seq->private;
3283         struct net *net = seq_file_net(seq);
3284         struct neigh_table *tbl = state->tbl;
3285         struct pneigh_entry *pn = NULL;
3286         int bucket;
3287
3288         state->flags |= NEIGH_SEQ_IS_PNEIGH;
3289         for (bucket = 0; bucket <= PNEIGH_HASHMASK; bucket++) {
3290                 pn = tbl->phash_buckets[bucket];
3291                 while (pn && !net_eq(pneigh_net(pn), net))
3292                         pn = pn->next;
3293                 if (pn)
3294                         break;
3295         }
3296         state->bucket = bucket;
3297
3298         return pn;
3299 }
3300
3301 static struct pneigh_entry *pneigh_get_next(struct seq_file *seq,
3302                                             struct pneigh_entry *pn,
3303                                             loff_t *pos)
3304 {
3305         struct neigh_seq_state *state = seq->private;
3306         struct net *net = seq_file_net(seq);
3307         struct neigh_table *tbl = state->tbl;
3308
3309         do {
3310                 pn = pn->next;
3311         } while (pn && !net_eq(pneigh_net(pn), net));
3312
3313         while (!pn) {
3314                 if (++state->bucket > PNEIGH_HASHMASK)
3315                         break;
3316                 pn = tbl->phash_buckets[state->bucket];
3317                 while (pn && !net_eq(pneigh_net(pn), net))
3318                         pn = pn->next;
3319                 if (pn)
3320                         break;
3321         }
3322
3323         if (pn && pos)
3324                 --(*pos);
3325
3326         return pn;
3327 }
3328
3329 static struct pneigh_entry *pneigh_get_idx(struct seq_file *seq, loff_t *pos)
3330 {
3331         struct pneigh_entry *pn = pneigh_get_first(seq);
3332
3333         if (pn) {
3334                 --(*pos);
3335                 while (*pos) {
3336                         pn = pneigh_get_next(seq, pn, pos);
3337                         if (!pn)
3338                                 break;
3339                 }
3340         }
3341         return *pos ? NULL : pn;
3342 }
3343
3344 static void *neigh_get_idx_any(struct seq_file *seq, loff_t *pos)
3345 {
3346         struct neigh_seq_state *state = seq->private;
3347         void *rc;
3348         loff_t idxpos = *pos;
3349
3350         rc = neigh_get_idx(seq, &idxpos);
3351         if (!rc && !(state->flags & NEIGH_SEQ_NEIGH_ONLY))
3352                 rc = pneigh_get_idx(seq, &idxpos);
3353
3354         return rc;
3355 }
3356
3357 void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl, unsigned int neigh_seq_flags)
3358         __acquires(tbl->lock)
3359         __acquires(rcu)
3360 {
3361         struct neigh_seq_state *state = seq->private;
3362
3363         state->tbl = tbl;
3364         state->bucket = 0;
3365         state->flags = (neigh_seq_flags & ~NEIGH_SEQ_IS_PNEIGH);
3366
3367         rcu_read_lock();
3368         state->nht = rcu_dereference(tbl->nht);
3369         read_lock_bh(&tbl->lock);
3370
3371         return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
3372 }
3373 EXPORT_SYMBOL(neigh_seq_start);
3374
3375 void *neigh_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3376 {
3377         struct neigh_seq_state *state;
3378         void *rc;
3379
3380         if (v == SEQ_START_TOKEN) {
3381                 rc = neigh_get_first(seq);
3382                 goto out;
3383         }
3384
3385         state = seq->private;
3386         if (!(state->flags & NEIGH_SEQ_IS_PNEIGH)) {
3387                 rc = neigh_get_next(seq, v, NULL);
3388                 if (rc)
3389                         goto out;
3390                 if (!(state->flags & NEIGH_SEQ_NEIGH_ONLY))
3391                         rc = pneigh_get_first(seq);
3392         } else {
3393                 BUG_ON(state->flags & NEIGH_SEQ_NEIGH_ONLY);
3394                 rc = pneigh_get_next(seq, v, NULL);
3395         }
3396 out:
3397         ++(*pos);
3398         return rc;
3399 }
3400 EXPORT_SYMBOL(neigh_seq_next);
3401
3402 void neigh_seq_stop(struct seq_file *seq, void *v)
3403         __releases(tbl->lock)
3404         __releases(rcu)
3405 {
3406         struct neigh_seq_state *state = seq->private;
3407         struct neigh_table *tbl = state->tbl;
3408
3409         read_unlock_bh(&tbl->lock);
3410         rcu_read_unlock();
3411 }
3412 EXPORT_SYMBOL(neigh_seq_stop);
3413
3414 /* statistics via seq_file */
3415
3416 static void *neigh_stat_seq_start(struct seq_file *seq, loff_t *pos)
3417 {
3418         struct neigh_table *tbl = pde_data(file_inode(seq->file));
3419         int cpu;
3420
3421         if (*pos == 0)
3422                 return SEQ_START_TOKEN;
3423
3424         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
3425                 if (!cpu_possible(cpu))
3426                         continue;
3427                 *pos = cpu+1;
3428                 return per_cpu_ptr(tbl->stats, cpu);
3429         }
3430         return NULL;
3431 }
3432
3433 static void *neigh_stat_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3434 {
3435         struct neigh_table *tbl = pde_data(file_inode(seq->file));
3436         int cpu;
3437
3438         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
3439                 if (!cpu_possible(cpu))
3440                         continue;
3441                 *pos = cpu+1;
3442                 return per_cpu_ptr(tbl->stats, cpu);
3443         }
3444         (*pos)++;
3445         return NULL;
3446 }
3447
3448 static void neigh_stat_seq_stop(struct seq_file *seq, void *v)
3449 {
3450
3451 }
3452
3453 static int neigh_stat_seq_show(struct seq_file *seq, void *v)
3454 {
3455         struct neigh_table *tbl = pde_data(file_inode(seq->file));
3456         struct neigh_statistics *st = v;
3457
3458         if (v == SEQ_START_TOKEN) {
3459                 seq_puts(seq, "entries  allocs   destroys hash_grows lookups  hits     res_failed rcv_probes_mcast rcv_probes_ucast periodic_gc_runs forced_gc_runs unresolved_discards table_fulls\n");
3460                 return 0;
3461         }
3462
3463         seq_printf(seq, "%08x %08lx %08lx %08lx   %08lx %08lx %08lx   "
3464                         "%08lx         %08lx         %08lx         "
3465                         "%08lx       %08lx            %08lx\n",
3466                    atomic_read(&tbl->entries),
3467
3468                    st->allocs,
3469                    st->destroys,
3470                    st->hash_grows,
3471
3472                    st->lookups,
3473                    st->hits,
3474
3475                    st->res_failed,
3476
3477                    st->rcv_probes_mcast,
3478                    st->rcv_probes_ucast,
3479
3480                    st->periodic_gc_runs,
3481                    st->forced_gc_runs,
3482                    st->unres_discards,
3483                    st->table_fulls
3484                    );
3485
3486         return 0;
3487 }
3488
3489 static const struct seq_operations neigh_stat_seq_ops = {
3490         .start  = neigh_stat_seq_start,
3491         .next   = neigh_stat_seq_next,
3492         .stop   = neigh_stat_seq_stop,
3493         .show   = neigh_stat_seq_show,
3494 };
3495 #endif /* CONFIG_PROC_FS */
3496
3497 static void __neigh_notify(struct neighbour *n, int type, int flags,
3498                            u32 pid)
3499 {
3500         struct net *net = dev_net(n->dev);
3501         struct sk_buff *skb;
3502         int err = -ENOBUFS;
3503
3504         skb = nlmsg_new(neigh_nlmsg_size(), GFP_ATOMIC);
3505         if (skb == NULL)
3506                 goto errout;
3507
3508         err = neigh_fill_info(skb, n, pid, 0, type, flags);
3509         if (err < 0) {
3510                 /* -EMSGSIZE implies BUG in neigh_nlmsg_size() */
3511                 WARN_ON(err == -EMSGSIZE);
3512                 kfree_skb(skb);
3513                 goto errout;
3514         }
3515         rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
3516         return;
3517 errout:
3518         if (err < 0)
3519                 rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
3520 }
3521
3522 void neigh_app_ns(struct neighbour *n)
3523 {
3524         __neigh_notify(n, RTM_GETNEIGH, NLM_F_REQUEST, 0);
3525 }
3526 EXPORT_SYMBOL(neigh_app_ns);
3527
3528 #ifdef CONFIG_SYSCTL
3529 static int unres_qlen_max = INT_MAX / SKB_TRUESIZE(ETH_FRAME_LEN);
3530
3531 static int proc_unres_qlen(struct ctl_table *ctl, int write,
3532                            void *buffer, size_t *lenp, loff_t *ppos)
3533 {
3534         int size, ret;
3535         struct ctl_table tmp = *ctl;
3536
3537         tmp.extra1 = SYSCTL_ZERO;
3538         tmp.extra2 = &unres_qlen_max;
3539         tmp.data = &size;
3540
3541         size = *(int *)ctl->data / SKB_TRUESIZE(ETH_FRAME_LEN);
3542         ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
3543
3544         if (write && !ret)
3545                 *(int *)ctl->data = size * SKB_TRUESIZE(ETH_FRAME_LEN);
3546         return ret;
3547 }
3548
3549 static void neigh_copy_dflt_parms(struct net *net, struct neigh_parms *p,
3550                                   int index)
3551 {
3552         struct net_device *dev;
3553         int family = neigh_parms_family(p);
3554
3555         rcu_read_lock();
3556         for_each_netdev_rcu(net, dev) {
3557                 struct neigh_parms *dst_p =
3558                                 neigh_get_dev_parms_rcu(dev, family);
3559
3560                 if (dst_p && !test_bit(index, dst_p->data_state))
3561                         dst_p->data[index] = p->data[index];
3562         }
3563         rcu_read_unlock();
3564 }
3565
3566 static void neigh_proc_update(struct ctl_table *ctl, int write)
3567 {
3568         struct net_device *dev = ctl->extra1;
3569         struct neigh_parms *p = ctl->extra2;
3570         struct net *net = neigh_parms_net(p);
3571         int index = (int *) ctl->data - p->data;
3572
3573         if (!write)
3574                 return;
3575
3576         set_bit(index, p->data_state);
3577         if (index == NEIGH_VAR_DELAY_PROBE_TIME)
3578                 call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
3579         if (!dev) /* NULL dev means this is default value */
3580                 neigh_copy_dflt_parms(net, p, index);
3581 }
3582
3583 static int neigh_proc_dointvec_zero_intmax(struct ctl_table *ctl, int write,
3584                                            void *buffer, size_t *lenp,
3585                                            loff_t *ppos)
3586 {
3587         struct ctl_table tmp = *ctl;
3588         int ret;
3589
3590         tmp.extra1 = SYSCTL_ZERO;
3591         tmp.extra2 = SYSCTL_INT_MAX;
3592
3593         ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
3594         neigh_proc_update(ctl, write);
3595         return ret;
3596 }
3597
3598 static int neigh_proc_dointvec_ms_jiffies_positive(struct ctl_table *ctl, int write,
3599                                                    void *buffer, size_t *lenp, loff_t *ppos)
3600 {
3601         struct ctl_table tmp = *ctl;
3602         int ret;
3603
3604         int min = msecs_to_jiffies(1);
3605
3606         tmp.extra1 = &min;
3607         tmp.extra2 = NULL;
3608
3609         ret = proc_dointvec_ms_jiffies_minmax(&tmp, write, buffer, lenp, ppos);
3610         neigh_proc_update(ctl, write);
3611         return ret;
3612 }
3613
3614 int neigh_proc_dointvec(struct ctl_table *ctl, int write, void *buffer,
3615                         size_t *lenp, loff_t *ppos)
3616 {
3617         int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
3618
3619         neigh_proc_update(ctl, write);
3620         return ret;
3621 }
3622 EXPORT_SYMBOL(neigh_proc_dointvec);
3623
3624 int neigh_proc_dointvec_jiffies(struct ctl_table *ctl, int write, void *buffer,
3625                                 size_t *lenp, loff_t *ppos)
3626 {
3627         int ret = proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);
3628
3629         neigh_proc_update(ctl, write);
3630         return ret;
3631 }
3632 EXPORT_SYMBOL(neigh_proc_dointvec_jiffies);
3633
3634 static int neigh_proc_dointvec_userhz_jiffies(struct ctl_table *ctl, int write,
3635                                               void *buffer, size_t *lenp,
3636                                               loff_t *ppos)
3637 {
3638         int ret = proc_dointvec_userhz_jiffies(ctl, write, buffer, lenp, ppos);
3639
3640         neigh_proc_update(ctl, write);
3641         return ret;
3642 }
3643
3644 int neigh_proc_dointvec_ms_jiffies(struct ctl_table *ctl, int write,
3645                                    void *buffer, size_t *lenp, loff_t *ppos)
3646 {
3647         int ret = proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);
3648
3649         neigh_proc_update(ctl, write);
3650         return ret;
3651 }
3652 EXPORT_SYMBOL(neigh_proc_dointvec_ms_jiffies);
3653
3654 static int neigh_proc_dointvec_unres_qlen(struct ctl_table *ctl, int write,
3655                                           void *buffer, size_t *lenp,
3656                                           loff_t *ppos)
3657 {
3658         int ret = proc_unres_qlen(ctl, write, buffer, lenp, ppos);
3659
3660         neigh_proc_update(ctl, write);
3661         return ret;
3662 }
3663
3664 static int neigh_proc_base_reachable_time(struct ctl_table *ctl, int write,
3665                                           void *buffer, size_t *lenp,
3666                                           loff_t *ppos)
3667 {
3668         struct neigh_parms *p = ctl->extra2;
3669         int ret;
3670
3671         if (strcmp(ctl->procname, "base_reachable_time") == 0)
3672                 ret = neigh_proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);
3673         else if (strcmp(ctl->procname, "base_reachable_time_ms") == 0)
3674                 ret = neigh_proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);
3675         else
3676                 ret = -1;
3677
3678         if (write && ret == 0) {
3679                 /* update reachable_time as well, otherwise, the change will
3680                  * only be effective after the next time neigh_periodic_work
3681                  * decides to recompute it
3682                  */
3683                 p->reachable_time =
3684                         neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
3685         }
3686         return ret;
3687 }
3688
3689 #define NEIGH_PARMS_DATA_OFFSET(index)  \
3690         (&((struct neigh_parms *) 0)->data[index])
3691
3692 #define NEIGH_SYSCTL_ENTRY(attr, data_attr, name, mval, proc) \
3693         [NEIGH_VAR_ ## attr] = { \
3694                 .procname       = name, \
3695                 .data           = NEIGH_PARMS_DATA_OFFSET(NEIGH_VAR_ ## data_attr), \
3696                 .maxlen         = sizeof(int), \
3697                 .mode           = mval, \
3698                 .proc_handler   = proc, \
3699         }
3700
3701 #define NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(attr, name) \
3702         NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_zero_intmax)
3703
3704 #define NEIGH_SYSCTL_JIFFIES_ENTRY(attr, name) \
3705         NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_jiffies)
3706
3707 #define NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(attr, name) \
3708         NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_userhz_jiffies)
3709
3710 #define NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(attr, name) \
3711         NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_ms_jiffies_positive)
3712
3713 #define NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(attr, data_attr, name) \
3714         NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_ms_jiffies)
3715
3716 #define NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(attr, data_attr, name) \
3717         NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_unres_qlen)
3718
3719 static struct neigh_sysctl_table {
3720         struct ctl_table_header *sysctl_header;
3721         struct ctl_table neigh_vars[NEIGH_VAR_MAX + 1];
3722 } neigh_sysctl_template __read_mostly = {
3723         .neigh_vars = {
3724                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_PROBES, "mcast_solicit"),
3725                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(UCAST_PROBES, "ucast_solicit"),
3726                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(APP_PROBES, "app_solicit"),
3727                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_REPROBES, "mcast_resolicit"),
3728                 NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(RETRANS_TIME, "retrans_time"),
3729                 NEIGH_SYSCTL_JIFFIES_ENTRY(BASE_REACHABLE_TIME, "base_reachable_time"),
3730                 NEIGH_SYSCTL_JIFFIES_ENTRY(DELAY_PROBE_TIME, "delay_first_probe_time"),
3731                 NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(INTERVAL_PROBE_TIME_MS,
3732                                                        "interval_probe_time_ms"),
3733                 NEIGH_SYSCTL_JIFFIES_ENTRY(GC_STALETIME, "gc_stale_time"),
3734                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(QUEUE_LEN_BYTES, "unres_qlen_bytes"),
3735                 NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(PROXY_QLEN, "proxy_qlen"),
3736                 NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(ANYCAST_DELAY, "anycast_delay"),
3737                 NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(PROXY_DELAY, "proxy_delay"),
3738                 NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(LOCKTIME, "locktime"),
3739                 NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(QUEUE_LEN, QUEUE_LEN_BYTES, "unres_qlen"),
3740                 NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(RETRANS_TIME_MS, RETRANS_TIME, "retrans_time_ms"),
3741                 NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(BASE_REACHABLE_TIME_MS, BASE_REACHABLE_TIME, "base_reachable_time_ms"),
3742                 [NEIGH_VAR_GC_INTERVAL] = {
3743                         .procname       = "gc_interval",
3744                         .maxlen         = sizeof(int),
3745                         .mode           = 0644,
3746                         .proc_handler   = proc_dointvec_jiffies,
3747                 },
3748                 [NEIGH_VAR_GC_THRESH1] = {
3749                         .procname       = "gc_thresh1",
3750                         .maxlen         = sizeof(int),
3751                         .mode           = 0644,
3752                         .extra1         = SYSCTL_ZERO,
3753                         .extra2         = SYSCTL_INT_MAX,
3754                         .proc_handler   = proc_dointvec_minmax,
3755                 },
3756                 [NEIGH_VAR_GC_THRESH2] = {
3757                         .procname       = "gc_thresh2",
3758                         .maxlen         = sizeof(int),
3759                         .mode           = 0644,
3760                         .extra1         = SYSCTL_ZERO,
3761                         .extra2         = SYSCTL_INT_MAX,
3762                         .proc_handler   = proc_dointvec_minmax,
3763                 },
3764                 [NEIGH_VAR_GC_THRESH3] = {
3765                         .procname       = "gc_thresh3",
3766                         .maxlen         = sizeof(int),
3767                         .mode           = 0644,
3768                         .extra1         = SYSCTL_ZERO,
3769                         .extra2         = SYSCTL_INT_MAX,
3770                         .proc_handler   = proc_dointvec_minmax,
3771                 },
3772                 {},
3773         },
3774 };
3775
3776 int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
3777                           proc_handler *handler)
3778 {
3779         int i;
3780         struct neigh_sysctl_table *t;
3781         const char *dev_name_source;
3782         char neigh_path[ sizeof("net//neigh/") + IFNAMSIZ + IFNAMSIZ ];
3783         char *p_name;
3784
3785         t = kmemdup(&neigh_sysctl_template, sizeof(*t), GFP_KERNEL_ACCOUNT);
3786         if (!t)
3787                 goto err;
3788
3789         for (i = 0; i < NEIGH_VAR_GC_INTERVAL; i++) {
3790                 t->neigh_vars[i].data += (long) p;
3791                 t->neigh_vars[i].extra1 = dev;
3792                 t->neigh_vars[i].extra2 = p;
3793         }
3794
3795         if (dev) {
3796                 dev_name_source = dev->name;
3797                 /* Terminate the table early */
3798                 memset(&t->neigh_vars[NEIGH_VAR_GC_INTERVAL], 0,
3799                        sizeof(t->neigh_vars[NEIGH_VAR_GC_INTERVAL]));
3800         } else {
3801                 struct neigh_table *tbl = p->tbl;
3802                 dev_name_source = "default";
3803                 t->neigh_vars[NEIGH_VAR_GC_INTERVAL].data = &tbl->gc_interval;
3804                 t->neigh_vars[NEIGH_VAR_GC_THRESH1].data = &tbl->gc_thresh1;
3805                 t->neigh_vars[NEIGH_VAR_GC_THRESH2].data = &tbl->gc_thresh2;
3806                 t->neigh_vars[NEIGH_VAR_GC_THRESH3].data = &tbl->gc_thresh3;
3807         }
3808
3809         if (handler) {
3810                 /* RetransTime */
3811                 t->neigh_vars[NEIGH_VAR_RETRANS_TIME].proc_handler = handler;
3812                 /* ReachableTime */
3813                 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler = handler;
3814                 /* RetransTime (in milliseconds)*/
3815                 t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].proc_handler = handler;
3816                 /* ReachableTime (in milliseconds) */
3817                 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler = handler;
3818         } else {
3819                 /* Those handlers will update p->reachable_time after
3820                  * base_reachable_time(_ms) is set to ensure the new timer starts being
3821                  * applied after the next neighbour update instead of waiting for
3822                  * neigh_periodic_work to update its value (can be multiple minutes)
3823                  * So any handler that replaces them should do this as well
3824                  */
3825                 /* ReachableTime */
3826                 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler =
3827                         neigh_proc_base_reachable_time;
3828                 /* ReachableTime (in milliseconds) */
3829                 t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler =
3830                         neigh_proc_base_reachable_time;
3831         }
3832
3833         switch (neigh_parms_family(p)) {
3834         case AF_INET:
3835               p_name = "ipv4";
3836               break;
3837         case AF_INET6:
3838               p_name = "ipv6";
3839               break;
3840         default:
3841               BUG();
3842         }
3843
3844         snprintf(neigh_path, sizeof(neigh_path), "net/%s/neigh/%s",
3845                 p_name, dev_name_source);
3846         t->sysctl_header =
3847                 register_net_sysctl(neigh_parms_net(p), neigh_path, t->neigh_vars);
3848         if (!t->sysctl_header)
3849                 goto free;
3850
3851         p->sysctl_table = t;
3852         return 0;
3853
3854 free:
3855         kfree(t);
3856 err:
3857         return -ENOBUFS;
3858 }
3859 EXPORT_SYMBOL(neigh_sysctl_register);
3860
3861 void neigh_sysctl_unregister(struct neigh_parms *p)
3862 {
3863         if (p->sysctl_table) {
3864                 struct neigh_sysctl_table *t = p->sysctl_table;
3865                 p->sysctl_table = NULL;
3866                 unregister_net_sysctl_table(t->sysctl_header);
3867                 kfree(t);
3868         }
3869 }
3870 EXPORT_SYMBOL(neigh_sysctl_unregister);
3871
3872 #endif  /* CONFIG_SYSCTL */
3873
3874 static int __init neigh_init(void)
3875 {
3876         rtnl_register(PF_UNSPEC, RTM_NEWNEIGH, neigh_add, NULL, 0);
3877         rtnl_register(PF_UNSPEC, RTM_DELNEIGH, neigh_delete, NULL, 0);
3878         rtnl_register(PF_UNSPEC, RTM_GETNEIGH, neigh_get, neigh_dump_info, 0);
3879
3880         rtnl_register(PF_UNSPEC, RTM_GETNEIGHTBL, NULL, neightbl_dump_info,
3881                       0);
3882         rtnl_register(PF_UNSPEC, RTM_SETNEIGHTBL, neightbl_set, NULL, 0);
3883
3884         return 0;
3885 }
3886
3887 subsys_initcall(neigh_init);