core/dev.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *      NET3    Protocol independent device support routines.
   4  *
   5  *      Derived from the non IP parts of dev.c 1.0.19
   6  *              Authors:        Ross Biro
   7  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
   8  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
   9  *
  10  *      Additional Authors:
  11  *              Florian la Roche <rzsfl@rz.uni-sb.de>
  12  *              Alan Cox <gw4pts@gw4pts.ampr.org>
  13  *              David Hinds <dahinds@users.sourceforge.net>
  14  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  15  *              Adam Sulmicki <adam@cfar.umd.edu>
  16  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  17  *
  18  *      Changes:
  19  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  20  *                                      to 2 if register_netdev gets called
  21  *                                      before net_dev_init & also removed a
  22  *                                      few lines of code in the process.
  23  *              Alan Cox        :       device private ioctl copies fields back.
  24  *              Alan Cox        :       Transmit queue code does relevant
  25  *                                      stunts to keep the queue safe.
  26  *              Alan Cox        :       Fixed double lock.
  27  *              Alan Cox        :       Fixed promisc NULL pointer trap
  28  *              ????????        :       Support the full private ioctl range
  29  *              Alan Cox        :       Moved ioctl permission check into
  30  *                                      drivers
  31  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
  32  *              Alan Cox        :       100 backlog just doesn't cut it when
  33  *                                      you start doing multicast video 8)
  34  *              Alan Cox        :       Rewrote net_bh and list manager.
  35  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
  36  *              Alan Cox        :       Took out transmit every packet pass
  37  *                                      Saved a few bytes in the ioctl handler
  38  *              Alan Cox        :       Network driver sets packet type before
  39  *                                      calling netif_rx. Saves a function
  40  *                                      call a packet.
  41  *              Alan Cox        :       Hashed net_bh()
  42  *              Richard Kooijman:       Timestamp fixes.
  43  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
  44  *              Alan Cox        :       Device lock protection.
  45  *              Alan Cox        :       Fixed nasty side effect of device close
  46  *                                      changes.
  47  *              Rudi Cilibrasi  :       Pass the right thing to
  48  *                                      set_mac_address()
  49  *              Dave Miller     :       32bit quantity for the device lock to
  50  *                                      make it work out on a Sparc.
  51  *              Bjorn Ekwall    :       Added KERNELD hack.
  52  *              Alan Cox        :       Cleaned up the backlog initialise.
  53  *              Craig Metz      :       SIOCGIFCONF fix if space for under
  54  *                                      1 device.
  55  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
  56  *                                      is no device open function.
  57  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
  58  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
  59  *              Cyrus Durgin    :       Cleaned for KMOD
  60  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
  61  *                                      A network device unload needs to purge
  62  *                                      the backlog queue.
  63  *      Paul Rusty Russell      :       SIOCSIFNAME
  64  *              Pekka Riikonen  :       Netdev boot-time settings code
  65  *              Andrew Morton   :       Make unregister_netdevice wait
  66  *                                      indefinitely on dev->refcnt
  67  *              J Hadi Salim    :       - Backlog queue sampling
  68  *                                      - netif_rx() feedback
  69  */
  70
  71 #include <linux/uaccess.h>
  72 #include <linux/bitops.h>
  73 #include <linux/capability.h>
  74 #include <linux/cpu.h>
  75 #include <linux/types.h>
  76 #include <linux/kernel.h>
  77 #include <linux/hash.h>
  78 #include <linux/slab.h>
  79 #include <linux/sched.h>
  80 #include <linux/sched/mm.h>
  81 #include <linux/mutex.h>
  82 #include <linux/rwsem.h>
  83 #include <linux/string.h>
  84 #include <linux/mm.h>
  85 #include <linux/socket.h>
  86 #include <linux/sockios.h>
  87 #include <linux/errno.h>
  88 #include <linux/interrupt.h>
  89 #include <linux/if_ether.h>
  90 #include <linux/netdevice.h>
  91 #include <linux/etherdevice.h>
  92 #include <linux/ethtool.h>
  93 #include <linux/skbuff.h>
  94 #include <linux/kthread.h>
  95 #include <linux/bpf.h>
  96 #include <linux/bpf_trace.h>
  97 #include <net/net_namespace.h>
  98 #include <net/sock.h>
  99 #include <net/busy_poll.h>
 100 #include <linux/rtnetlink.h>
 101 #include <linux/stat.h>
 102 #include <net/dsa.h>
 103 #include <net/dst.h>
 104 #include <net/dst_metadata.h>
 105 #include <net/gro.h>
 106 #include <net/pkt_sched.h>
 107 #include <net/pkt_cls.h>
 108 #include <net/checksum.h>
 109 #include <net/xfrm.h>
 110 #include <linux/highmem.h>
 111 #include <linux/init.h>
 112 #include <linux/module.h>
 113 #include <linux/netpoll.h>
 114 #include <linux/rcupdate.h>
 115 #include <linux/delay.h>
 116 #include <net/iw_handler.h>
 117 #include <asm/current.h>
 118 #include <linux/audit.h>
 119 #include <linux/dmaengine.h>
 120 #include <linux/err.h>
 121 #include <linux/ctype.h>
 122 #include <linux/if_arp.h>
 123 #include <linux/if_vlan.h>
 124 #include <linux/ip.h>
 125 #include <net/ip.h>
 126 #include <net/mpls.h>
 127 #include <linux/ipv6.h>
 128 #include <linux/in.h>
 129 #include <linux/jhash.h>
 130 #include <linux/random.h>
 131 #include <trace/events/napi.h>
 132 #include <trace/events/net.h>
 133 #include <trace/events/skb.h>
 134 #include <trace/events/qdisc.h>
 135 #include <linux/inetdevice.h>
 136 #include <linux/cpu_rmap.h>
 137 #include <linux/static_key.h>
 138 #include <linux/hashtable.h>
 139 #include <linux/vmalloc.h>
 140 #include <linux/if_macvlan.h>
 141 #include <linux/errqueue.h>
 142 #include <linux/hrtimer.h>
 143 #include <linux/netfilter_netdev.h>
 144 #include <linux/crash_dump.h>
 145 #include <linux/sctp.h>
 146 #include <net/udp_tunnel.h>
 147 #include <linux/net_namespace.h>
 148 #include <linux/indirect_call_wrapper.h>
 149 #include <net/devlink.h>
 150 #include <linux/pm_runtime.h>
 151 #include <linux/prandom.h>
 152 #include <linux/once_lite.h>
 153
 154 #include "dev.h"
 155 #include "net-sysfs.h"
 156
 157
 158 static DEFINE_SPINLOCK(ptype_lock);
 159 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 160 struct list_head ptype_all __read_mostly;       /* Taps */
 161
 162 static int netif_rx_internal(struct sk_buff *skb);
 163 static int call_netdevice_notifiers_info(unsigned long val,
 164                                          struct netdev_notifier_info *info);
 165 static int call_netdevice_notifiers_extack(unsigned long val,
 166                                            struct net_device *dev,
 167                                            struct netlink_ext_ack *extack);
 168 static struct napi_struct *napi_by_id(unsigned int napi_id);
 169
 170 /*
 171  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
 172  * semaphore.
 173  *
 174  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
 175  *
 176  * Writers must hold the rtnl semaphore while they loop through the
 177  * dev_base_head list, and hold dev_base_lock for writing when they do the
 178  * actual updates.  This allows pure readers to access the list even
 179  * while a writer is preparing to update it.
 180  *
 181  * To put it another way, dev_base_lock is held for writing only to
 182  * protect against pure readers; the rtnl semaphore provides the
 183  * protection against other writers.
 184  *
 185  * See, for example usages, register_netdevice() and
 186  * unregister_netdevice(), which must be called with the rtnl
 187  * semaphore held.
 188  */
 189 DEFINE_RWLOCK(dev_base_lock);
 190 EXPORT_SYMBOL(dev_base_lock);
 191
 192 static DEFINE_MUTEX(ifalias_mutex);
 193
 194 /* protects napi_hash addition/deletion and napi_gen_id */
 195 static DEFINE_SPINLOCK(napi_hash_lock);
 196
 197 static unsigned int napi_gen_id = NR_CPUS;
 198 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
 199
 200 static DECLARE_RWSEM(devnet_rename_sem);
 201
 202 static inline void dev_base_seq_inc(struct net *net)
 203 {
 204         while (++net->dev_base_seq == 0)
 205                 ;
 206 }
 207
 208 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 209 {
 210         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
 211
 212         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 213 }
 214
 215 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 216 {
 217         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 218 }
 219
 220 static inline void rps_lock_irqsave(struct softnet_data *sd,
 221                                     unsigned long *flags)
 222 {
 223         if (IS_ENABLED(CONFIG_RPS))
 224                 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
 225         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 226                 local_irq_save(*flags);
 227 }
 228
 229 static inline void rps_lock_irq_disable(struct softnet_data *sd)
 230 {
 231         if (IS_ENABLED(CONFIG_RPS))
 232                 spin_lock_irq(&sd->input_pkt_queue.lock);
 233         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 234                 local_irq_disable();
 235 }
 236
 237 static inline void rps_unlock_irq_restore(struct softnet_data *sd,
 238                                           unsigned long *flags)
 239 {
 240         if (IS_ENABLED(CONFIG_RPS))
 241                 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags);
 242         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 243                 local_irq_restore(*flags);
 244 }
 245
 246 static inline void rps_unlock_irq_enable(struct softnet_data *sd)
 247 {
 248         if (IS_ENABLED(CONFIG_RPS))
 249                 spin_unlock_irq(&sd->input_pkt_queue.lock);
 250         else if (!IS_ENABLED(CONFIG_PREEMPT_RT))
 251                 local_irq_enable();
 252 }
 253
 254 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
 255                                                        const char *name)
 256 {
 257         struct netdev_name_node *name_node;
 258
 259         name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
 260         if (!name_node)
 261                 return NULL;
 262         INIT_HLIST_NODE(&name_node->hlist);
 263         name_node->dev = dev;
 264         name_node->name = name;
 265         return name_node;
 266 }
 267
 268 static struct netdev_name_node *
 269 netdev_name_node_head_alloc(struct net_device *dev)
 270 {
 271         struct netdev_name_node *name_node;
 272
 273         name_node = netdev_name_node_alloc(dev, dev->name);
 274         if (!name_node)
 275                 return NULL;
 276         INIT_LIST_HEAD(&name_node->list);
 277         return name_node;
 278 }
 279
 280 static void netdev_name_node_free(struct netdev_name_node *name_node)
 281 {
 282         kfree(name_node);
 283 }
 284
 285 static void netdev_name_node_add(struct net *net,
 286                                  struct netdev_name_node *name_node)
 287 {
 288         hlist_add_head_rcu(&name_node->hlist,
 289                            dev_name_hash(net, name_node->name));
 290 }
 291
 292 static void netdev_name_node_del(struct netdev_name_node *name_node)
 293 {
 294         hlist_del_rcu(&name_node->hlist);
 295 }
 296
 297 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
 298                                                         const char *name)
 299 {
 300         struct hlist_head *head = dev_name_hash(net, name);
 301         struct netdev_name_node *name_node;
 302
 303         hlist_for_each_entry(name_node, head, hlist)
 304                 if (!strcmp(name_node->name, name))
 305                         return name_node;
 306         return NULL;
 307 }
 308
 309 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
 310                                                             const char *name)
 311 {
 312         struct hlist_head *head = dev_name_hash(net, name);
 313         struct netdev_name_node *name_node;
 314
 315         hlist_for_each_entry_rcu(name_node, head, hlist)
 316                 if (!strcmp(name_node->name, name))
 317                         return name_node;
 318         return NULL;
 319 }
 320
 321 bool netdev_name_in_use(struct net *net, const char *name)
 322 {
 323         return netdev_name_node_lookup(net, name);
 324 }
 325 EXPORT_SYMBOL(netdev_name_in_use);
 326
 327 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
 328 {
 329         struct netdev_name_node *name_node;
 330         struct net *net = dev_net(dev);
 331
 332         name_node = netdev_name_node_lookup(net, name);
 333         if (name_node)
 334                 return -EEXIST;
 335         name_node = netdev_name_node_alloc(dev, name);
 336         if (!name_node)
 337                 return -ENOMEM;
 338         netdev_name_node_add(net, name_node);
 339         /* The node that holds dev->name acts as a head of per-device list. */
 340         list_add_tail(&name_node->list, &dev->name_node->list);
 341
 342         return 0;
 343 }
 344
 345 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
 346 {
 347         list_del(&name_node->list);
 348         kfree(name_node->name);
 349         netdev_name_node_free(name_node);
 350 }
 351
 352 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
 353 {
 354         struct netdev_name_node *name_node;
 355         struct net *net = dev_net(dev);
 356
 357         name_node = netdev_name_node_lookup(net, name);
 358         if (!name_node)
 359                 return -ENOENT;
 360         /* lookup might have found our primary name or a name belonging
 361          * to another device.
 362          */
 363         if (name_node == dev->name_node || name_node->dev != dev)
 364                 return -EINVAL;
 365
 366         netdev_name_node_del(name_node);
 367         synchronize_rcu();
 368         __netdev_name_node_alt_destroy(name_node);
 369
 370         return 0;
 371 }
 372
 373 static void netdev_name_node_alt_flush(struct net_device *dev)
 374 {
 375         struct netdev_name_node *name_node, *tmp;
 376
 377         list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
 378                 __netdev_name_node_alt_destroy(name_node);
 379 }
 380
 381 /* Device list insertion */
 382 static void list_netdevice(struct net_device *dev)
 383 {
 384         struct netdev_name_node *name_node;
 385         struct net *net = dev_net(dev);
 386
 387         ASSERT_RTNL();
 388
 389         write_lock(&dev_base_lock);
 390         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 391         netdev_name_node_add(net, dev->name_node);
 392         hlist_add_head_rcu(&dev->index_hlist,
 393                            dev_index_hash(net, dev->ifindex));
 394         write_unlock(&dev_base_lock);
 395
 396         netdev_for_each_altname(dev, name_node)
 397                 netdev_name_node_add(net, name_node);
 398
 399         dev_base_seq_inc(net);
 400 }
 401
 402 /* Device list removal
 403  * caller must respect a RCU grace period before freeing/reusing dev
 404  */
 405 static void unlist_netdevice(struct net_device *dev, bool lock)
 406 {
 407         struct netdev_name_node *name_node;
 408
 409         ASSERT_RTNL();
 410
 411         netdev_for_each_altname(dev, name_node)
 412                 netdev_name_node_del(name_node);
 413
 414         /* Unlink dev from the device chain */
 415         if (lock)
 416                 write_lock(&dev_base_lock);
 417         list_del_rcu(&dev->dev_list);
 418         netdev_name_node_del(dev->name_node);
 419         hlist_del_rcu(&dev->index_hlist);
 420         if (lock)
 421                 write_unlock(&dev_base_lock);
 422
 423         dev_base_seq_inc(dev_net(dev));
 424 }
 425
 426 /*
 427  *      Our notifier list
 428  */
 429
 430 static RAW_NOTIFIER_HEAD(netdev_chain);
 431
 432 /*
 433  *      Device drivers call our routines to queue packets here. We empty the
 434  *      queue in the local softnet handler.
 435  */
 436
 437 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 438 EXPORT_PER_CPU_SYMBOL(softnet_data);
 439
 440 #ifdef CONFIG_LOCKDEP
 441 /*
 442  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
 443  * according to dev->type
 444  */
 445 static const unsigned short netdev_lock_type[] = {
 446          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 447          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 448          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 449          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 450          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 451          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 452          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 453          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 454          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 455          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 456          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 457          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 458          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
 459          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
 460          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
 461
 462 static const char *const netdev_lock_name[] = {
 463         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 464         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 465         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 466         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 467         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 468         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 469         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 470         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 471         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 472         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 473         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 474         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 475         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
 476         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
 477         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
 478
 479 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 480 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 481
 482 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 483 {
 484         int i;
 485
 486         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 487                 if (netdev_lock_type[i] == dev_type)
 488                         return i;
 489         /* the last key is used by default */
 490         return ARRAY_SIZE(netdev_lock_type) - 1;
 491 }
 492
 493 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 494                                                  unsigned short dev_type)
 495 {
 496         int i;
 497
 498         i = netdev_lock_pos(dev_type);
 499         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 500                                    netdev_lock_name[i]);
 501 }
 502
 503 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 504 {
 505         int i;
 506
 507         i = netdev_lock_pos(dev->type);
 508         lockdep_set_class_and_name(&dev->addr_list_lock,
 509                                    &netdev_addr_lock_key[i],
 510                                    netdev_lock_name[i]);
 511 }
 512 #else
 513 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 514                                                  unsigned short dev_type)
 515 {
 516 }
 517
 518 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 519 {
 520 }
 521 #endif
 522
 523 /*******************************************************************************
 524  *
 525  *              Protocol management and registration routines
 526  *
 527  *******************************************************************************/
 528
 529
 530 /*
 531  *      Add a protocol ID to the list. Now that the input handler is
 532  *      smarter we can dispense with all the messy stuff that used to be
 533  *      here.
 534  *
 535  *      BEWARE!!! Protocol handlers, mangling input packets,
 536  *      MUST BE last in hash buckets and checking protocol handlers
 537  *      MUST start from promiscuous ptype_all chain in net_bh.
 538  *      It is true now, do not change it.
 539  *      Explanation follows: if protocol handler, mangling packet, will
 540  *      be the first on list, it is not able to sense, that packet
 541  *      is cloned and should be copied-on-write, so that it will
 542  *      change it and subsequent readers will get broken packet.
 543  *                                                      --ANK (980803)
 544  */
 545
 546 static inline struct list_head *ptype_head(const struct packet_type *pt)
 547 {
 548         if (pt->type == htons(ETH_P_ALL))
 549                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
 550         else
 551                 return pt->dev ? &pt->dev->ptype_specific :
 552                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 553 }
 554
 555 /**
 556  *      dev_add_pack - add packet handler
 557  *      @pt: packet type declaration
 558  *
 559  *      Add a protocol handler to the networking stack. The passed &packet_type
 560  *      is linked into kernel lists and may not be freed until it has been
 561  *      removed from the kernel lists.
 562  *
 563  *      This call does not sleep therefore it can not
 564  *      guarantee all CPU's that are in middle of receiving packets
 565  *      will see the new packet type (until the next received packet).
 566  */
 567
 568 void dev_add_pack(struct packet_type *pt)
 569 {
 570         struct list_head *head = ptype_head(pt);
 571
 572         spin_lock(&ptype_lock);
 573         list_add_rcu(&pt->list, head);
 574         spin_unlock(&ptype_lock);
 575 }
 576 EXPORT_SYMBOL(dev_add_pack);
 577
 578 /**
 579  *      __dev_remove_pack        - remove packet handler
 580  *      @pt: packet type declaration
 581  *
 582  *      Remove a protocol handler that was previously added to the kernel
 583  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 584  *      from the kernel lists and can be freed or reused once this function
 585  *      returns.
 586  *
 587  *      The packet type might still be in use by receivers
 588  *      and must not be freed until after all the CPU's have gone
 589  *      through a quiescent state.
 590  */
 591 void __dev_remove_pack(struct packet_type *pt)
 592 {
 593         struct list_head *head = ptype_head(pt);
 594         struct packet_type *pt1;
 595
 596         spin_lock(&ptype_lock);
 597
 598         list_for_each_entry(pt1, head, list) {
 599                 if (pt == pt1) {
 600                         list_del_rcu(&pt->list);
 601                         goto out;
 602                 }
 603         }
 604
 605         pr_warn("dev_remove_pack: %p not found\n", pt);
 606 out:
 607         spin_unlock(&ptype_lock);
 608 }
 609 EXPORT_SYMBOL(__dev_remove_pack);
 610
 611 /**
 612  *      dev_remove_pack  - remove packet handler
 613  *      @pt: packet type declaration
 614  *
 615  *      Remove a protocol handler that was previously added to the kernel
 616  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
 617  *      from the kernel lists and can be freed or reused once this function
 618  *      returns.
 619  *
 620  *      This call sleeps to guarantee that no CPU is looking at the packet
 621  *      type after return.
 622  */
 623 void dev_remove_pack(struct packet_type *pt)
 624 {
 625         __dev_remove_pack(pt);
 626
 627         synchronize_net();
 628 }
 629 EXPORT_SYMBOL(dev_remove_pack);
 630
 631
 632 /*******************************************************************************
 633  *
 634  *                          Device Interface Subroutines
 635  *
 636  *******************************************************************************/
 637
 638 /**
 639  *      dev_get_iflink  - get 'iflink' value of a interface
 640  *      @dev: targeted interface
 641  *
 642  *      Indicates the ifindex the interface is linked to.
 643  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
 644  */
 645
 646 int dev_get_iflink(const struct net_device *dev)
 647 {
 648         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
 649                 return dev->netdev_ops->ndo_get_iflink(dev);
 650
 651         return dev->ifindex;
 652 }
 653 EXPORT_SYMBOL(dev_get_iflink);
 654
 655 /**
 656  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
 657  *      @dev: targeted interface
 658  *      @skb: The packet.
 659  *
 660  *      For better visibility of tunnel traffic OVS needs to retrieve
 661  *      egress tunnel information for a packet. Following API allows
 662  *      user to get this info.
 663  */
 664 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 665 {
 666         struct ip_tunnel_info *info;
 667
 668         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
 669                 return -EINVAL;
 670
 671         info = skb_tunnel_info_unclone(skb);
 672         if (!info)
 673                 return -ENOMEM;
 674         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
 675                 return -EINVAL;
 676
 677         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
 678 }
 679 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
 680
 681 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
 682 {
 683         int k = stack->num_paths++;
 684
 685         if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
 686                 return NULL;
 687
 688         return &stack->path[k];
 689 }
 690
 691 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
 692                           struct net_device_path_stack *stack)
 693 {
 694         const struct net_device *last_dev;
 695         struct net_device_path_ctx ctx = {
 696                 .dev    = dev,
 697         };
 698         struct net_device_path *path;
 699         int ret = 0;
 700
 701         memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
 702         stack->num_paths = 0;
 703         while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
 704                 last_dev = ctx.dev;
 705                 path = dev_fwd_path(stack);
 706                 if (!path)
 707                         return -1;
 708
 709                 memset(path, 0, sizeof(struct net_device_path));
 710                 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
 711                 if (ret < 0)
 712                         return -1;
 713
 714                 if (WARN_ON_ONCE(last_dev == ctx.dev))
 715                         return -1;
 716         }
 717
 718         if (!ctx.dev)
 719                 return ret;
 720
 721         path = dev_fwd_path(stack);
 722         if (!path)
 723                 return -1;
 724         path->type = DEV_PATH_ETHERNET;
 725         path->dev = ctx.dev;
 726
 727         return ret;
 728 }
 729 EXPORT_SYMBOL_GPL(dev_fill_forward_path);
 730
 731 /**
 732  *      __dev_get_by_name       - find a device by its name
 733  *      @net: the applicable net namespace
 734  *      @name: name to find
 735  *
 736  *      Find an interface by name. Must be called under RTNL semaphore
 737  *      or @dev_base_lock. If the name is found a pointer to the device
 738  *      is returned. If the name is not found then %NULL is returned. The
 739  *      reference counters are not incremented so the caller must be
 740  *      careful with locks.
 741  */
 742
 743 struct net_device *__dev_get_by_name(struct net *net, const char *name)
 744 {
 745         struct netdev_name_node *node_name;
 746
 747         node_name = netdev_name_node_lookup(net, name);
 748         return node_name ? node_name->dev : NULL;
 749 }
 750 EXPORT_SYMBOL(__dev_get_by_name);
 751
 752 /**
 753  * dev_get_by_name_rcu  - find a device by its name
 754  * @net: the applicable net namespace
 755  * @name: name to find
 756  *
 757  * Find an interface by name.
 758  * If the name is found a pointer to the device is returned.
 759  * If the name is not found then %NULL is returned.
 760  * The reference counters are not incremented so the caller must be
 761  * careful with locks. The caller must hold RCU lock.
 762  */
 763
 764 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 765 {
 766         struct netdev_name_node *node_name;
 767
 768         node_name = netdev_name_node_lookup_rcu(net, name);
 769         return node_name ? node_name->dev : NULL;
 770 }
 771 EXPORT_SYMBOL(dev_get_by_name_rcu);
 772
 773 /**
 774  *      dev_get_by_name         - find a device by its name
 775  *      @net: the applicable net namespace
 776  *      @name: name to find
 777  *
 778  *      Find an interface by name. This can be called from any
 779  *      context and does its own locking. The returned handle has
 780  *      the usage count incremented and the caller must use dev_put() to
 781  *      release it when it is no longer needed. %NULL is returned if no
 782  *      matching device is found.
 783  */
 784
 785 struct net_device *dev_get_by_name(struct net *net, const char *name)
 786 {
 787         struct net_device *dev;
 788
 789         rcu_read_lock();
 790         dev = dev_get_by_name_rcu(net, name);
 791         dev_hold(dev);
 792         rcu_read_unlock();
 793         return dev;
 794 }
 795 EXPORT_SYMBOL(dev_get_by_name);
 796
 797 /**
 798  *      __dev_get_by_index - find a device by its ifindex
 799  *      @net: the applicable net namespace
 800  *      @ifindex: index of device
 801  *
 802  *      Search for an interface by index. Returns %NULL if the device
 803  *      is not found or a pointer to the device. The device has not
 804  *      had its reference counter increased so the caller must be careful
 805  *      about locking. The caller must hold either the RTNL semaphore
 806  *      or @dev_base_lock.
 807  */
 808
 809 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 810 {
 811         struct net_device *dev;
 812         struct hlist_head *head = dev_index_hash(net, ifindex);
 813
 814         hlist_for_each_entry(dev, head, index_hlist)
 815                 if (dev->ifindex == ifindex)
 816                         return dev;
 817
 818         return NULL;
 819 }
 820 EXPORT_SYMBOL(__dev_get_by_index);
 821
 822 /**
 823  *      dev_get_by_index_rcu - find a device by its ifindex
 824  *      @net: the applicable net namespace
 825  *      @ifindex: index of device
 826  *
 827  *      Search for an interface by index. Returns %NULL if the device
 828  *      is not found or a pointer to the device. The device has not
 829  *      had its reference counter increased so the caller must be careful
 830  *      about locking. The caller must hold RCU lock.
 831  */
 832
 833 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 834 {
 835         struct net_device *dev;
 836         struct hlist_head *head = dev_index_hash(net, ifindex);
 837
 838         hlist_for_each_entry_rcu(dev, head, index_hlist)
 839                 if (dev->ifindex == ifindex)
 840                         return dev;
 841
 842         return NULL;
 843 }
 844 EXPORT_SYMBOL(dev_get_by_index_rcu);
 845
 846
 847 /**
 848  *      dev_get_by_index - find a device by its ifindex
 849  *      @net: the applicable net namespace
 850  *      @ifindex: index of device
 851  *
 852  *      Search for an interface by index. Returns NULL if the device
 853  *      is not found or a pointer to the device. The device returned has
 854  *      had a reference added and the pointer is safe until the user calls
 855  *      dev_put to indicate they have finished with it.
 856  */
 857
 858 struct net_device *dev_get_by_index(struct net *net, int ifindex)
 859 {
 860         struct net_device *dev;
 861
 862         rcu_read_lock();
 863         dev = dev_get_by_index_rcu(net, ifindex);
 864         dev_hold(dev);
 865         rcu_read_unlock();
 866         return dev;
 867 }
 868 EXPORT_SYMBOL(dev_get_by_index);
 869
 870 /**
 871  *      dev_get_by_napi_id - find a device by napi_id
 872  *      @napi_id: ID of the NAPI struct
 873  *
 874  *      Search for an interface by NAPI ID. Returns %NULL if the device
 875  *      is not found or a pointer to the device. The device has not had
 876  *      its reference counter increased so the caller must be careful
 877  *      about locking. The caller must hold RCU lock.
 878  */
 879
 880 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
 881 {
 882         struct napi_struct *napi;
 883
 884         WARN_ON_ONCE(!rcu_read_lock_held());
 885
 886         if (napi_id < MIN_NAPI_ID)
 887                 return NULL;
 888
 889         napi = napi_by_id(napi_id);
 890
 891         return napi ? napi->dev : NULL;
 892 }
 893 EXPORT_SYMBOL(dev_get_by_napi_id);
 894
 895 /**
 896  *      netdev_get_name - get a netdevice name, knowing its ifindex.
 897  *      @net: network namespace
 898  *      @name: a pointer to the buffer where the name will be stored.
 899  *      @ifindex: the ifindex of the interface to get the name from.
 900  */
 901 int netdev_get_name(struct net *net, char *name, int ifindex)
 902 {
 903         struct net_device *dev;
 904         int ret;
 905
 906         down_read(&devnet_rename_sem);
 907         rcu_read_lock();
 908
 909         dev = dev_get_by_index_rcu(net, ifindex);
 910         if (!dev) {
 911                 ret = -ENODEV;
 912                 goto out;
 913         }
 914
 915         strcpy(name, dev->name);
 916
 917         ret = 0;
 918 out:
 919         rcu_read_unlock();
 920         up_read(&devnet_rename_sem);
 921         return ret;
 922 }
 923
 924 /**
 925  *      dev_getbyhwaddr_rcu - find a device by its hardware address
 926  *      @net: the applicable net namespace
 927  *      @type: media type of device
 928  *      @ha: hardware address
 929  *
 930  *      Search for an interface by MAC address. Returns NULL if the device
 931  *      is not found or a pointer to the device.
 932  *      The caller must hold RCU or RTNL.
 933  *      The returned device has not had its ref count increased
 934  *      and the caller must therefore be careful about locking
 935  *
 936  */
 937
 938 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
 939                                        const char *ha)
 940 {
 941         struct net_device *dev;
 942
 943         for_each_netdev_rcu(net, dev)
 944                 if (dev->type == type &&
 945                     !memcmp(dev->dev_addr, ha, dev->addr_len))
 946                         return dev;
 947
 948         return NULL;
 949 }
 950 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
 951
 952 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 953 {
 954         struct net_device *dev, *ret = NULL;
 955
 956         rcu_read_lock();
 957         for_each_netdev_rcu(net, dev)
 958                 if (dev->type == type) {
 959                         dev_hold(dev);
 960                         ret = dev;
 961                         break;
 962                 }
 963         rcu_read_unlock();
 964         return ret;
 965 }
 966 EXPORT_SYMBOL(dev_getfirstbyhwtype);
 967
 968 /**
 969  *      __dev_get_by_flags - find any device with given flags
 970  *      @net: the applicable net namespace
 971  *      @if_flags: IFF_* values
 972  *      @mask: bitmask of bits in if_flags to check
 973  *
 974  *      Search for any interface with the given flags. Returns NULL if a device
 975  *      is not found or a pointer to the device. Must be called inside
 976  *      rtnl_lock(), and result refcount is unchanged.
 977  */
 978
 979 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
 980                                       unsigned short mask)
 981 {
 982         struct net_device *dev, *ret;
 983
 984         ASSERT_RTNL();
 985
 986         ret = NULL;
 987         for_each_netdev(net, dev) {
 988                 if (((dev->flags ^ if_flags) & mask) == 0) {
 989                         ret = dev;
 990                         break;
 991                 }
 992         }
 993         return ret;
 994 }
 995 EXPORT_SYMBOL(__dev_get_by_flags);
 996
 997 /**
 998  *      dev_valid_name - check if name is okay for network device
 999  *      @name: name string
1000  *
1001  *      Network device names need to be valid file names to
1002  *      allow sysfs to work.  We also disallow any kind of
1003  *      whitespace.
1004  */
1005 bool dev_valid_name(const char *name)
1006 {
1007         if (*name == '\0')
1008                 return false;
1009         if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1010                 return false;
1011         if (!strcmp(name, ".") || !strcmp(name, ".."))
1012                 return false;
1013
1014         while (*name) {
1015                 if (*name == '/' || *name == ':' || isspace(*name))
1016                         return false;
1017                 name++;
1018         }
1019         return true;
1020 }
1021 EXPORT_SYMBOL(dev_valid_name);
1022
1023 /**
1024  *      __dev_alloc_name - allocate a name for a device
1025  *      @net: network namespace to allocate the device name in
1026  *      @name: name format string
1027  *      @buf:  scratch buffer and result name string
1028  *
1029  *      Passed a format string - eg "lt%d" it will try and find a suitable
1030  *      id. It scans list of devices to build up a free map, then chooses
1031  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1032  *      while allocating the name and adding the device in order to avoid
1033  *      duplicates.
1034  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1035  *      Returns the number of the unit assigned or a negative errno code.
1036  */
1037
1038 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1039 {
1040         int i = 0;
1041         const char *p;
1042         const int max_netdevices = 8*PAGE_SIZE;
1043         unsigned long *inuse;
1044         struct net_device *d;
1045
1046         if (!dev_valid_name(name))
1047                 return -EINVAL;
1048
1049         p = strchr(name, '%');
1050         if (p) {
1051                 /*
1052                  * Verify the string as this thing may have come from
1053                  * the user.  There must be either one "%d" and no other "%"
1054                  * characters.
1055                  */
1056                 if (p[1] != 'd' || strchr(p + 2, '%'))
1057                         return -EINVAL;
1058
1059                 /* Use one page as a bit array of possible slots */
1060                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1061                 if (!inuse)
1062                         return -ENOMEM;
1063
1064                 for_each_netdev(net, d) {
1065                         struct netdev_name_node *name_node;
1066
1067                         netdev_for_each_altname(d, name_node) {
1068                                 if (!sscanf(name_node->name, name, &i))
1069                                         continue;
1070                                 if (i < 0 || i >= max_netdevices)
1071                                         continue;
1072
1073                                 /*  avoid cases where sscanf is not exact inverse of printf */
1074                                 snprintf(buf, IFNAMSIZ, name, i);
1075                                 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1076                                         __set_bit(i, inuse);
1077                         }
1078                         if (!sscanf(d->name, name, &i))
1079                                 continue;
1080                         if (i < 0 || i >= max_netdevices)
1081                                 continue;
1082
1083                         /*  avoid cases where sscanf is not exact inverse of printf */
1084                         snprintf(buf, IFNAMSIZ, name, i);
1085                         if (!strncmp(buf, d->name, IFNAMSIZ))
1086                                 __set_bit(i, inuse);
1087                 }
1088
1089                 i = find_first_zero_bit(inuse, max_netdevices);
1090                 free_page((unsigned long) inuse);
1091         }
1092
1093         snprintf(buf, IFNAMSIZ, name, i);
1094         if (!netdev_name_in_use(net, buf))
1095                 return i;
1096
1097         /* It is possible to run out of possible slots
1098          * when the name is long and there isn't enough space left
1099          * for the digits, or if all bits are used.
1100          */
1101         return -ENFILE;
1102 }
1103
1104 static int dev_prep_valid_name(struct net *net, struct net_device *dev,
1105                                const char *want_name, char *out_name)
1106 {
1107         int ret;
1108
1109         if (!dev_valid_name(want_name))
1110                 return -EINVAL;
1111
1112         if (strchr(want_name, '%')) {
1113                 ret = __dev_alloc_name(net, want_name, out_name);
1114                 return ret < 0 ? ret : 0;
1115         } else if (netdev_name_in_use(net, want_name)) {
1116                 return -EEXIST;
1117         } else if (out_name != want_name) {
1118                 strscpy(out_name, want_name, IFNAMSIZ);
1119         }
1120
1121         return 0;
1122 }
1123
1124 static int dev_alloc_name_ns(struct net *net,
1125                              struct net_device *dev,
1126                              const char *name)
1127 {
1128         char buf[IFNAMSIZ];
1129         int ret;
1130
1131         BUG_ON(!net);
1132         ret = __dev_alloc_name(net, name, buf);
1133         if (ret >= 0)
1134                 strscpy(dev->name, buf, IFNAMSIZ);
1135         return ret;
1136 }
1137
1138 /**
1139  *      dev_alloc_name - allocate a name for a device
1140  *      @dev: device
1141  *      @name: name format string
1142  *
1143  *      Passed a format string - eg "lt%d" it will try and find a suitable
1144  *      id. It scans list of devices to build up a free map, then chooses
1145  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1146  *      while allocating the name and adding the device in order to avoid
1147  *      duplicates.
1148  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1149  *      Returns the number of the unit assigned or a negative errno code.
1150  */
1151
1152 int dev_alloc_name(struct net_device *dev, const char *name)
1153 {
1154         return dev_alloc_name_ns(dev_net(dev), dev, name);
1155 }
1156 EXPORT_SYMBOL(dev_alloc_name);
1157
1158 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1159                               const char *name)
1160 {
1161         char buf[IFNAMSIZ];
1162         int ret;
1163
1164         ret = dev_prep_valid_name(net, dev, name, buf);
1165         if (ret >= 0)
1166                 strscpy(dev->name, buf, IFNAMSIZ);
1167         return ret;
1168 }
1169
1170 /**
1171  *      dev_change_name - change name of a device
1172  *      @dev: device
1173  *      @newname: name (or format string) must be at least IFNAMSIZ
1174  *
1175  *      Change name of a device, can pass format strings "eth%d".
1176  *      for wildcarding.
1177  */
1178 int dev_change_name(struct net_device *dev, const char *newname)
1179 {
1180         unsigned char old_assign_type;
1181         char oldname[IFNAMSIZ];
1182         int err = 0;
1183         int ret;
1184         struct net *net;
1185
1186         ASSERT_RTNL();
1187         BUG_ON(!dev_net(dev));
1188
1189         net = dev_net(dev);
1190
1191         /* Some auto-enslaved devices e.g. failover slaves are
1192          * special, as userspace might rename the device after
1193          * the interface had been brought up and running since
1194          * the point kernel initiated auto-enslavement. Allow
1195          * live name change even when these slave devices are
1196          * up and running.
1197          *
1198          * Typically, users of these auto-enslaving devices
1199          * don't actually care about slave name change, as
1200          * they are supposed to operate on master interface
1201          * directly.
1202          */
1203         if (dev->flags & IFF_UP &&
1204             likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1205                 return -EBUSY;
1206
1207         down_write(&devnet_rename_sem);
1208
1209         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1210                 up_write(&devnet_rename_sem);
1211                 return 0;
1212         }
1213
1214         memcpy(oldname, dev->name, IFNAMSIZ);
1215
1216         err = dev_get_valid_name(net, dev, newname);
1217         if (err < 0) {
1218                 up_write(&devnet_rename_sem);
1219                 return err;
1220         }
1221
1222         if (oldname[0] && !strchr(oldname, '%'))
1223                 netdev_info(dev, "renamed from %s\n", oldname);
1224
1225         old_assign_type = dev->name_assign_type;
1226         dev->name_assign_type = NET_NAME_RENAMED;
1227
1228 rollback:
1229         ret = device_rename(&dev->dev, dev->name);
1230         if (ret) {
1231                 memcpy(dev->name, oldname, IFNAMSIZ);
1232                 dev->name_assign_type = old_assign_type;
1233                 up_write(&devnet_rename_sem);
1234                 return ret;
1235         }
1236
1237         up_write(&devnet_rename_sem);
1238
1239         netdev_adjacent_rename_links(dev, oldname);
1240
1241         write_lock(&dev_base_lock);
1242         netdev_name_node_del(dev->name_node);
1243         write_unlock(&dev_base_lock);
1244
1245         synchronize_rcu();
1246
1247         write_lock(&dev_base_lock);
1248         netdev_name_node_add(net, dev->name_node);
1249         write_unlock(&dev_base_lock);
1250
1251         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1252         ret = notifier_to_errno(ret);
1253
1254         if (ret) {
1255                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1256                 if (err >= 0) {
1257                         err = ret;
1258                         down_write(&devnet_rename_sem);
1259                         memcpy(dev->name, oldname, IFNAMSIZ);
1260                         memcpy(oldname, newname, IFNAMSIZ);
1261                         dev->name_assign_type = old_assign_type;
1262                         old_assign_type = NET_NAME_RENAMED;
1263                         goto rollback;
1264                 } else {
1265                         netdev_err(dev, "name change rollback failed: %d\n",
1266                                    ret);
1267                 }
1268         }
1269
1270         return err;
1271 }
1272
1273 /**
1274  *      dev_set_alias - change ifalias of a device
1275  *      @dev: device
1276  *      @alias: name up to IFALIASZ
1277  *      @len: limit of bytes to copy from info
1278  *
1279  *      Set ifalias for a device,
1280  */
1281 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1282 {
1283         struct dev_ifalias *new_alias = NULL;
1284
1285         if (len >= IFALIASZ)
1286                 return -EINVAL;
1287
1288         if (len) {
1289                 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1290                 if (!new_alias)
1291                         return -ENOMEM;
1292
1293                 memcpy(new_alias->ifalias, alias, len);
1294                 new_alias->ifalias[len] = 0;
1295         }
1296
1297         mutex_lock(&ifalias_mutex);
1298         new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1299                                         mutex_is_locked(&ifalias_mutex));
1300         mutex_unlock(&ifalias_mutex);
1301
1302         if (new_alias)
1303                 kfree_rcu(new_alias, rcuhead);
1304
1305         return len;
1306 }
1307 EXPORT_SYMBOL(dev_set_alias);
1308
1309 /**
1310  *      dev_get_alias - get ifalias of a device
1311  *      @dev: device
1312  *      @name: buffer to store name of ifalias
1313  *      @len: size of buffer
1314  *
1315  *      get ifalias for a device.  Caller must make sure dev cannot go
1316  *      away,  e.g. rcu read lock or own a reference count to device.
1317  */
1318 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1319 {
1320         const struct dev_ifalias *alias;
1321         int ret = 0;
1322
1323         rcu_read_lock();
1324         alias = rcu_dereference(dev->ifalias);
1325         if (alias)
1326                 ret = snprintf(name, len, "%s", alias->ifalias);
1327         rcu_read_unlock();
1328
1329         return ret;
1330 }
1331
1332 /**
1333  *      netdev_features_change - device changes features
1334  *      @dev: device to cause notification
1335  *
1336  *      Called to indicate a device has changed features.
1337  */
1338 void netdev_features_change(struct net_device *dev)
1339 {
1340         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1341 }
1342 EXPORT_SYMBOL(netdev_features_change);
1343
1344 /**
1345  *      netdev_state_change - device changes state
1346  *      @dev: device to cause notification
1347  *
1348  *      Called to indicate a device has changed state. This function calls
1349  *      the notifier chains for netdev_chain and sends a NEWLINK message
1350  *      to the routing socket.
1351  */
1352 void netdev_state_change(struct net_device *dev)
1353 {
1354         if (dev->flags & IFF_UP) {
1355                 struct netdev_notifier_change_info change_info = {
1356                         .info.dev = dev,
1357                 };
1358
1359                 call_netdevice_notifiers_info(NETDEV_CHANGE,
1360                                               &change_info.info);
1361                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1362         }
1363 }
1364 EXPORT_SYMBOL(netdev_state_change);
1365
1366 /**
1367  * __netdev_notify_peers - notify network peers about existence of @dev,
1368  * to be called when rtnl lock is already held.
1369  * @dev: network device
1370  *
1371  * Generate traffic such that interested network peers are aware of
1372  * @dev, such as by generating a gratuitous ARP. This may be used when
1373  * a device wants to inform the rest of the network about some sort of
1374  * reconfiguration such as a failover event or virtual machine
1375  * migration.
1376  */
1377 void __netdev_notify_peers(struct net_device *dev)
1378 {
1379         ASSERT_RTNL();
1380         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1381         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1382 }
1383 EXPORT_SYMBOL(__netdev_notify_peers);
1384
1385 /**
1386  * netdev_notify_peers - notify network peers about existence of @dev
1387  * @dev: network device
1388  *
1389  * Generate traffic such that interested network peers are aware of
1390  * @dev, such as by generating a gratuitous ARP. This may be used when
1391  * a device wants to inform the rest of the network about some sort of
1392  * reconfiguration such as a failover event or virtual machine
1393  * migration.
1394  */
1395 void netdev_notify_peers(struct net_device *dev)
1396 {
1397         rtnl_lock();
1398         __netdev_notify_peers(dev);
1399         rtnl_unlock();
1400 }
1401 EXPORT_SYMBOL(netdev_notify_peers);
1402
1403 static int napi_threaded_poll(void *data);
1404
1405 static int napi_kthread_create(struct napi_struct *n)
1406 {
1407         int err = 0;
1408
1409         /* Create and wake up the kthread once to put it in
1410          * TASK_INTERRUPTIBLE mode to avoid the blocked task
1411          * warning and work with loadavg.
1412          */
1413         n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1414                                 n->dev->name, n->napi_id);
1415         if (IS_ERR(n->thread)) {
1416                 err = PTR_ERR(n->thread);
1417                 pr_err("kthread_run failed with err %d\n", err);
1418                 n->thread = NULL;
1419         }
1420
1421         return err;
1422 }
1423
1424 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1425 {
1426         const struct net_device_ops *ops = dev->netdev_ops;
1427         int ret;
1428
1429         ASSERT_RTNL();
1430         dev_addr_check(dev);
1431
1432         if (!netif_device_present(dev)) {
1433                 /* may be detached because parent is runtime-suspended */
1434                 if (dev->dev.parent)
1435                         pm_runtime_resume(dev->dev.parent);
1436                 if (!netif_device_present(dev))
1437                         return -ENODEV;
1438         }
1439
1440         /* Block netpoll from trying to do any rx path servicing.
1441          * If we don't do this there is a chance ndo_poll_controller
1442          * or ndo_poll may be running while we open the device
1443          */
1444         netpoll_poll_disable(dev);
1445
1446         ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1447         ret = notifier_to_errno(ret);
1448         if (ret)
1449                 return ret;
1450
1451         set_bit(__LINK_STATE_START, &dev->state);
1452
1453         if (ops->ndo_validate_addr)
1454                 ret = ops->ndo_validate_addr(dev);
1455
1456         if (!ret && ops->ndo_open)
1457                 ret = ops->ndo_open(dev);
1458
1459         netpoll_poll_enable(dev);
1460
1461         if (ret)
1462                 clear_bit(__LINK_STATE_START, &dev->state);
1463         else {
1464                 dev->flags |= IFF_UP;
1465                 dev_set_rx_mode(dev);
1466                 dev_activate(dev);
1467                 add_device_randomness(dev->dev_addr, dev->addr_len);
1468         }
1469
1470         return ret;
1471 }
1472
1473 /**
1474  *      dev_open        - prepare an interface for use.
1475  *      @dev: device to open
1476  *      @extack: netlink extended ack
1477  *
1478  *      Takes a device from down to up state. The device's private open
1479  *      function is invoked and then the multicast lists are loaded. Finally
1480  *      the device is moved into the up state and a %NETDEV_UP message is
1481  *      sent to the netdev notifier chain.
1482  *
1483  *      Calling this function on an active interface is a nop. On a failure
1484  *      a negative errno code is returned.
1485  */
1486 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1487 {
1488         int ret;
1489
1490         if (dev->flags & IFF_UP)
1491                 return 0;
1492
1493         ret = __dev_open(dev, extack);
1494         if (ret < 0)
1495                 return ret;
1496
1497         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1498         call_netdevice_notifiers(NETDEV_UP, dev);
1499
1500         return ret;
1501 }
1502 EXPORT_SYMBOL(dev_open);
1503
1504 static void __dev_close_many(struct list_head *head)
1505 {
1506         struct net_device *dev;
1507
1508         ASSERT_RTNL();
1509         might_sleep();
1510
1511         list_for_each_entry(dev, head, close_list) {
1512                 /* Temporarily disable netpoll until the interface is down */
1513                 netpoll_poll_disable(dev);
1514
1515                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1516
1517                 clear_bit(__LINK_STATE_START, &dev->state);
1518
1519                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1520                  * can be even on different cpu. So just clear netif_running().
1521                  *
1522                  * dev->stop() will invoke napi_disable() on all of it's
1523                  * napi_struct instances on this device.
1524                  */
1525                 smp_mb__after_atomic(); /* Commit netif_running(). */
1526         }
1527
1528         dev_deactivate_many(head);
1529
1530         list_for_each_entry(dev, head, close_list) {
1531                 const struct net_device_ops *ops = dev->netdev_ops;
1532
1533                 /*
1534                  *      Call the device specific close. This cannot fail.
1535                  *      Only if device is UP
1536                  *
1537                  *      We allow it to be called even after a DETACH hot-plug
1538                  *      event.
1539                  */
1540                 if (ops->ndo_stop)
1541                         ops->ndo_stop(dev);
1542
1543                 dev->flags &= ~IFF_UP;
1544                 netpoll_poll_enable(dev);
1545         }
1546 }
1547
1548 static void __dev_close(struct net_device *dev)
1549 {
1550         LIST_HEAD(single);
1551
1552         list_add(&dev->close_list, &single);
1553         __dev_close_many(&single);
1554         list_del(&single);
1555 }
1556
1557 void dev_close_many(struct list_head *head, bool unlink)
1558 {
1559         struct net_device *dev, *tmp;
1560
1561         /* Remove the devices that don't need to be closed */
1562         list_for_each_entry_safe(dev, tmp, head, close_list)
1563                 if (!(dev->flags & IFF_UP))
1564                         list_del_init(&dev->close_list);
1565
1566         __dev_close_many(head);
1567
1568         list_for_each_entry_safe(dev, tmp, head, close_list) {
1569                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1570                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1571                 if (unlink)
1572                         list_del_init(&dev->close_list);
1573         }
1574 }
1575 EXPORT_SYMBOL(dev_close_many);
1576
1577 /**
1578  *      dev_close - shutdown an interface.
1579  *      @dev: device to shutdown
1580  *
1581  *      This function moves an active device into down state. A
1582  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1583  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1584  *      chain.
1585  */
1586 void dev_close(struct net_device *dev)
1587 {
1588         if (dev->flags & IFF_UP) {
1589                 LIST_HEAD(single);
1590
1591                 list_add(&dev->close_list, &single);
1592                 dev_close_many(&single, true);
1593                 list_del(&single);
1594         }
1595 }
1596 EXPORT_SYMBOL(dev_close);
1597
1598
1599 /**
1600  *      dev_disable_lro - disable Large Receive Offload on a device
1601  *      @dev: device
1602  *
1603  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1604  *      called under RTNL.  This is needed if received packets may be
1605  *      forwarded to another interface.
1606  */
1607 void dev_disable_lro(struct net_device *dev)
1608 {
1609         struct net_device *lower_dev;
1610         struct list_head *iter;
1611
1612         dev->wanted_features &= ~NETIF_F_LRO;
1613         netdev_update_features(dev);
1614
1615         if (unlikely(dev->features & NETIF_F_LRO))
1616                 netdev_WARN(dev, "failed to disable LRO!\n");
1617
1618         netdev_for_each_lower_dev(dev, lower_dev, iter)
1619                 dev_disable_lro(lower_dev);
1620 }
1621 EXPORT_SYMBOL(dev_disable_lro);
1622
1623 /**
1624  *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1625  *      @dev: device
1626  *
1627  *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1628  *      called under RTNL.  This is needed if Generic XDP is installed on
1629  *      the device.
1630  */
1631 static void dev_disable_gro_hw(struct net_device *dev)
1632 {
1633         dev->wanted_features &= ~NETIF_F_GRO_HW;
1634         netdev_update_features(dev);
1635
1636         if (unlikely(dev->features & NETIF_F_GRO_HW))
1637                 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1638 }
1639
1640 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1641 {
1642 #define N(val)                                          \
1643         case NETDEV_##val:                              \
1644                 return "NETDEV_" __stringify(val);
1645         switch (cmd) {
1646         N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1647         N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1648         N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1649         N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1650         N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1651         N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1652         N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1653         N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1654         N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1655         N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1656         N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1657         }
1658 #undef N
1659         return "UNKNOWN_NETDEV_EVENT";
1660 }
1661 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1662
1663 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1664                                    struct net_device *dev)
1665 {
1666         struct netdev_notifier_info info = {
1667                 .dev = dev,
1668         };
1669
1670         return nb->notifier_call(nb, val, &info);
1671 }
1672
1673 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1674                                              struct net_device *dev)
1675 {
1676         int err;
1677
1678         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1679         err = notifier_to_errno(err);
1680         if (err)
1681                 return err;
1682
1683         if (!(dev->flags & IFF_UP))
1684                 return 0;
1685
1686         call_netdevice_notifier(nb, NETDEV_UP, dev);
1687         return 0;
1688 }
1689
1690 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1691                                                 struct net_device *dev)
1692 {
1693         if (dev->flags & IFF_UP) {
1694                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1695                                         dev);
1696                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1697         }
1698         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1699 }
1700
1701 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1702                                                  struct net *net)
1703 {
1704         struct net_device *dev;
1705         int err;
1706
1707         for_each_netdev(net, dev) {
1708                 err = call_netdevice_register_notifiers(nb, dev);
1709                 if (err)
1710                         goto rollback;
1711         }
1712         return 0;
1713
1714 rollback:
1715         for_each_netdev_continue_reverse(net, dev)
1716                 call_netdevice_unregister_notifiers(nb, dev);
1717         return err;
1718 }
1719
1720 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1721                                                     struct net *net)
1722 {
1723         struct net_device *dev;
1724
1725         for_each_netdev(net, dev)
1726                 call_netdevice_unregister_notifiers(nb, dev);
1727 }
1728
1729 static int dev_boot_phase = 1;
1730
1731 /**
1732  * register_netdevice_notifier - register a network notifier block
1733  * @nb: notifier
1734  *
1735  * Register a notifier to be called when network device events occur.
1736  * The notifier passed is linked into the kernel structures and must
1737  * not be reused until it has been unregistered. A negative errno code
1738  * is returned on a failure.
1739  *
1740  * When registered all registration and up events are replayed
1741  * to the new notifier to allow device to have a race free
1742  * view of the network device list.
1743  */
1744
1745 int register_netdevice_notifier(struct notifier_block *nb)
1746 {
1747         struct net *net;
1748         int err;
1749
1750         /* Close race with setup_net() and cleanup_net() */
1751         down_write(&pernet_ops_rwsem);
1752         rtnl_lock();
1753         err = raw_notifier_chain_register(&netdev_chain, nb);
1754         if (err)
1755                 goto unlock;
1756         if (dev_boot_phase)
1757                 goto unlock;
1758         for_each_net(net) {
1759                 err = call_netdevice_register_net_notifiers(nb, net);
1760                 if (err)
1761                         goto rollback;
1762         }
1763
1764 unlock:
1765         rtnl_unlock();
1766         up_write(&pernet_ops_rwsem);
1767         return err;
1768
1769 rollback:
1770         for_each_net_continue_reverse(net)
1771                 call_netdevice_unregister_net_notifiers(nb, net);
1772
1773         raw_notifier_chain_unregister(&netdev_chain, nb);
1774         goto unlock;
1775 }
1776 EXPORT_SYMBOL(register_netdevice_notifier);
1777
1778 /**
1779  * unregister_netdevice_notifier - unregister a network notifier block
1780  * @nb: notifier
1781  *
1782  * Unregister a notifier previously registered by
1783  * register_netdevice_notifier(). The notifier is unlinked into the
1784  * kernel structures and may then be reused. A negative errno code
1785  * is returned on a failure.
1786  *
1787  * After unregistering unregister and down device events are synthesized
1788  * for all devices on the device list to the removed notifier to remove
1789  * the need for special case cleanup code.
1790  */
1791
1792 int unregister_netdevice_notifier(struct notifier_block *nb)
1793 {
1794         struct net *net;
1795         int err;
1796
1797         /* Close race with setup_net() and cleanup_net() */
1798         down_write(&pernet_ops_rwsem);
1799         rtnl_lock();
1800         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1801         if (err)
1802                 goto unlock;
1803
1804         for_each_net(net)
1805                 call_netdevice_unregister_net_notifiers(nb, net);
1806
1807 unlock:
1808         rtnl_unlock();
1809         up_write(&pernet_ops_rwsem);
1810         return err;
1811 }
1812 EXPORT_SYMBOL(unregister_netdevice_notifier);
1813
1814 static int __register_netdevice_notifier_net(struct net *net,
1815                                              struct notifier_block *nb,
1816                                              bool ignore_call_fail)
1817 {
1818         int err;
1819
1820         err = raw_notifier_chain_register(&net->netdev_chain, nb);
1821         if (err)
1822                 return err;
1823         if (dev_boot_phase)
1824                 return 0;
1825
1826         err = call_netdevice_register_net_notifiers(nb, net);
1827         if (err && !ignore_call_fail)
1828                 goto chain_unregister;
1829
1830         return 0;
1831
1832 chain_unregister:
1833         raw_notifier_chain_unregister(&net->netdev_chain, nb);
1834         return err;
1835 }
1836
1837 static int __unregister_netdevice_notifier_net(struct net *net,
1838                                                struct notifier_block *nb)
1839 {
1840         int err;
1841
1842         err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1843         if (err)
1844                 return err;
1845
1846         call_netdevice_unregister_net_notifiers(nb, net);
1847         return 0;
1848 }
1849
1850 /**
1851  * register_netdevice_notifier_net - register a per-netns network notifier block
1852  * @net: network namespace
1853  * @nb: notifier
1854  *
1855  * Register a notifier to be called when network device events occur.
1856  * The notifier passed is linked into the kernel structures and must
1857  * not be reused until it has been unregistered. A negative errno code
1858  * is returned on a failure.
1859  *
1860  * When registered all registration and up events are replayed
1861  * to the new notifier to allow device to have a race free
1862  * view of the network device list.
1863  */
1864
1865 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
1866 {
1867         int err;
1868
1869         rtnl_lock();
1870         err = __register_netdevice_notifier_net(net, nb, false);
1871         rtnl_unlock();
1872         return err;
1873 }
1874 EXPORT_SYMBOL(register_netdevice_notifier_net);
1875
1876 /**
1877  * unregister_netdevice_notifier_net - unregister a per-netns
1878  *                                     network notifier block
1879  * @net: network namespace
1880  * @nb: notifier
1881  *
1882  * Unregister a notifier previously registered by
1883  * register_netdevice_notifier(). The notifier is unlinked into the
1884  * kernel structures and may then be reused. A negative errno code
1885  * is returned on a failure.
1886  *
1887  * After unregistering unregister and down device events are synthesized
1888  * for all devices on the device list to the removed notifier to remove
1889  * the need for special case cleanup code.
1890  */
1891
1892 int unregister_netdevice_notifier_net(struct net *net,
1893                                       struct notifier_block *nb)
1894 {
1895         int err;
1896
1897         rtnl_lock();
1898         err = __unregister_netdevice_notifier_net(net, nb);
1899         rtnl_unlock();
1900         return err;
1901 }
1902 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
1903
1904 int register_netdevice_notifier_dev_net(struct net_device *dev,
1905                                         struct notifier_block *nb,
1906                                         struct netdev_net_notifier *nn)
1907 {
1908         int err;
1909
1910         rtnl_lock();
1911         err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
1912         if (!err) {
1913                 nn->nb = nb;
1914                 list_add(&nn->list, &dev->net_notifier_list);
1915         }
1916         rtnl_unlock();
1917         return err;
1918 }
1919 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
1920
1921 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
1922                                           struct notifier_block *nb,
1923                                           struct netdev_net_notifier *nn)
1924 {
1925         int err;
1926
1927         rtnl_lock();
1928         list_del(&nn->list);
1929         err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
1930         rtnl_unlock();
1931         return err;
1932 }
1933 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
1934
1935 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
1936                                              struct net *net)
1937 {
1938         struct netdev_net_notifier *nn;
1939
1940         list_for_each_entry(nn, &dev->net_notifier_list, list) {
1941                 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
1942                 __register_netdevice_notifier_net(net, nn->nb, true);
1943         }
1944 }
1945
1946 /**
1947  *      call_netdevice_notifiers_info - call all network notifier blocks
1948  *      @val: value passed unmodified to notifier function
1949  *      @info: notifier information data
1950  *
1951  *      Call all network notifier blocks.  Parameters and return value
1952  *      are as for raw_notifier_call_chain().
1953  */
1954
1955 static int call_netdevice_notifiers_info(unsigned long val,
1956                                          struct netdev_notifier_info *info)
1957 {
1958         struct net *net = dev_net(info->dev);
1959         int ret;
1960
1961         ASSERT_RTNL();
1962
1963         /* Run per-netns notifier block chain first, then run the global one.
1964          * Hopefully, one day, the global one is going to be removed after
1965          * all notifier block registrators get converted to be per-netns.
1966          */
1967         ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
1968         if (ret & NOTIFY_STOP_MASK)
1969                 return ret;
1970         return raw_notifier_call_chain(&netdev_chain, val, info);
1971 }
1972
1973 /**
1974  *      call_netdevice_notifiers_info_robust - call per-netns notifier blocks
1975  *                                             for and rollback on error
1976  *      @val_up: value passed unmodified to notifier function
1977  *      @val_down: value passed unmodified to the notifier function when
1978  *                 recovering from an error on @val_up
1979  *      @info: notifier information data
1980  *
1981  *      Call all per-netns network notifier blocks, but not notifier blocks on
1982  *      the global notifier chain. Parameters and return value are as for
1983  *      raw_notifier_call_chain_robust().
1984  */
1985
1986 static int
1987 call_netdevice_notifiers_info_robust(unsigned long val_up,
1988                                      unsigned long val_down,
1989                                      struct netdev_notifier_info *info)
1990 {
1991         struct net *net = dev_net(info->dev);
1992
1993         ASSERT_RTNL();
1994
1995         return raw_notifier_call_chain_robust(&net->netdev_chain,
1996                                               val_up, val_down, info);
1997 }
1998
1999 static int call_netdevice_notifiers_extack(unsigned long val,
2000                                            struct net_device *dev,
2001                                            struct netlink_ext_ack *extack)
2002 {
2003         struct netdev_notifier_info info = {
2004                 .dev = dev,
2005                 .extack = extack,
2006         };
2007
2008         return call_netdevice_notifiers_info(val, &info);
2009 }
2010
2011 /**
2012  *      call_netdevice_notifiers - call all network notifier blocks
2013  *      @val: value passed unmodified to notifier function
2014  *      @dev: net_device pointer passed unmodified to notifier function
2015  *
2016  *      Call all network notifier blocks.  Parameters and return value
2017  *      are as for raw_notifier_call_chain().
2018  */
2019
2020 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2021 {
2022         return call_netdevice_notifiers_extack(val, dev, NULL);
2023 }
2024 EXPORT_SYMBOL(call_netdevice_notifiers);
2025
2026 /**
2027  *      call_netdevice_notifiers_mtu - call all network notifier blocks
2028  *      @val: value passed unmodified to notifier function
2029  *      @dev: net_device pointer passed unmodified to notifier function
2030  *      @arg: additional u32 argument passed to the notifier function
2031  *
2032  *      Call all network notifier blocks.  Parameters and return value
2033  *      are as for raw_notifier_call_chain().
2034  */
2035 static int call_netdevice_notifiers_mtu(unsigned long val,
2036                                         struct net_device *dev, u32 arg)
2037 {
2038         struct netdev_notifier_info_ext info = {
2039                 .info.dev = dev,
2040                 .ext.mtu = arg,
2041         };
2042
2043         BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2044
2045         return call_netdevice_notifiers_info(val, &info.info);
2046 }
2047
2048 #ifdef CONFIG_NET_INGRESS
2049 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2050
2051 void net_inc_ingress_queue(void)
2052 {
2053         static_branch_inc(&ingress_needed_key);
2054 }
2055 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2056
2057 void net_dec_ingress_queue(void)
2058 {
2059         static_branch_dec(&ingress_needed_key);
2060 }
2061 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2062 #endif
2063
2064 #ifdef CONFIG_NET_EGRESS
2065 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2066
2067 void net_inc_egress_queue(void)
2068 {
2069         static_branch_inc(&egress_needed_key);
2070 }
2071 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2072
2073 void net_dec_egress_queue(void)
2074 {
2075         static_branch_dec(&egress_needed_key);
2076 }
2077 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2078 #endif
2079
2080 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2081 EXPORT_SYMBOL(netstamp_needed_key);
2082 #ifdef CONFIG_JUMP_LABEL
2083 static atomic_t netstamp_needed_deferred;
2084 static atomic_t netstamp_wanted;
2085 static void netstamp_clear(struct work_struct *work)
2086 {
2087         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2088         int wanted;
2089
2090         wanted = atomic_add_return(deferred, &netstamp_wanted);
2091         if (wanted > 0)
2092                 static_branch_enable(&netstamp_needed_key);
2093         else
2094                 static_branch_disable(&netstamp_needed_key);
2095 }
2096 static DECLARE_WORK(netstamp_work, netstamp_clear);
2097 #endif
2098
2099 void net_enable_timestamp(void)
2100 {
2101 #ifdef CONFIG_JUMP_LABEL
2102         int wanted;
2103
2104         while (1) {
2105                 wanted = atomic_read(&netstamp_wanted);
2106                 if (wanted <= 0)
2107                         break;
2108                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2109                         return;
2110         }
2111         atomic_inc(&netstamp_needed_deferred);
2112         schedule_work(&netstamp_work);
2113 #else
2114         static_branch_inc(&netstamp_needed_key);
2115 #endif
2116 }
2117 EXPORT_SYMBOL(net_enable_timestamp);
2118
2119 void net_disable_timestamp(void)
2120 {
2121 #ifdef CONFIG_JUMP_LABEL
2122         int wanted;
2123
2124         while (1) {
2125                 wanted = atomic_read(&netstamp_wanted);
2126                 if (wanted <= 1)
2127                         break;
2128                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2129                         return;
2130         }
2131         atomic_dec(&netstamp_needed_deferred);
2132         schedule_work(&netstamp_work);
2133 #else
2134         static_branch_dec(&netstamp_needed_key);
2135 #endif
2136 }
2137 EXPORT_SYMBOL(net_disable_timestamp);
2138
2139 static inline void net_timestamp_set(struct sk_buff *skb)
2140 {
2141         skb->tstamp = 0;
2142         skb->mono_delivery_time = 0;
2143         if (static_branch_unlikely(&netstamp_needed_key))
2144                 skb->tstamp = ktime_get_real();
2145 }
2146
2147 #define net_timestamp_check(COND, SKB)                          \
2148         if (static_branch_unlikely(&netstamp_needed_key)) {     \
2149                 if ((COND) && !(SKB)->tstamp)                   \
2150                         (SKB)->tstamp = ktime_get_real();       \
2151         }                                                       \
2152
2153 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2154 {
2155         return __is_skb_forwardable(dev, skb, true);
2156 }
2157 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2158
2159 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2160                               bool check_mtu)
2161 {
2162         int ret = ____dev_forward_skb(dev, skb, check_mtu);
2163
2164         if (likely(!ret)) {
2165                 skb->protocol = eth_type_trans(skb, dev);
2166                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2167         }
2168
2169         return ret;
2170 }
2171
2172 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2173 {
2174         return __dev_forward_skb2(dev, skb, true);
2175 }
2176 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2177
2178 /**
2179  * dev_forward_skb - loopback an skb to another netif
2180  *
2181  * @dev: destination network device
2182  * @skb: buffer to forward
2183  *
2184  * return values:
2185  *      NET_RX_SUCCESS  (no congestion)
2186  *      NET_RX_DROP     (packet was dropped, but freed)
2187  *
2188  * dev_forward_skb can be used for injecting an skb from the
2189  * start_xmit function of one device into the receive queue
2190  * of another device.
2191  *
2192  * The receiving device may be in another namespace, so
2193  * we have to clear all information in the skb that could
2194  * impact namespace isolation.
2195  */
2196 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2197 {
2198         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2199 }
2200 EXPORT_SYMBOL_GPL(dev_forward_skb);
2201
2202 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2203 {
2204         return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2205 }
2206
2207 static inline int deliver_skb(struct sk_buff *skb,
2208                               struct packet_type *pt_prev,
2209                               struct net_device *orig_dev)
2210 {
2211         if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2212                 return -ENOMEM;
2213         refcount_inc(&skb->users);
2214         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2215 }
2216
2217 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2218                                           struct packet_type **pt,
2219                                           struct net_device *orig_dev,
2220                                           __be16 type,
2221                                           struct list_head *ptype_list)
2222 {
2223         struct packet_type *ptype, *pt_prev = *pt;
2224
2225         list_for_each_entry_rcu(ptype, ptype_list, list) {
2226                 if (ptype->type != type)
2227                         continue;
2228                 if (pt_prev)
2229                         deliver_skb(skb, pt_prev, orig_dev);
2230                 pt_prev = ptype;
2231         }
2232         *pt = pt_prev;
2233 }
2234
2235 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2236 {
2237         if (!ptype->af_packet_priv || !skb->sk)
2238                 return false;
2239
2240         if (ptype->id_match)
2241                 return ptype->id_match(ptype, skb->sk);
2242         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2243                 return true;
2244
2245         return false;
2246 }
2247
2248 /**
2249  * dev_nit_active - return true if any network interface taps are in use
2250  *
2251  * @dev: network device to check for the presence of taps
2252  */
2253 bool dev_nit_active(struct net_device *dev)
2254 {
2255         return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2256 }
2257 EXPORT_SYMBOL_GPL(dev_nit_active);
2258
2259 /*
2260  *      Support routine. Sends outgoing frames to any network
2261  *      taps currently in use.
2262  */
2263
2264 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2265 {
2266         struct packet_type *ptype;
2267         struct sk_buff *skb2 = NULL;
2268         struct packet_type *pt_prev = NULL;
2269         struct list_head *ptype_list = &ptype_all;
2270
2271         rcu_read_lock();
2272 again:
2273         list_for_each_entry_rcu(ptype, ptype_list, list) {
2274                 if (READ_ONCE(ptype->ignore_outgoing))
2275                         continue;
2276
2277                 /* Never send packets back to the socket
2278                  * they originated from - MvS (miquels@drinkel.ow.org)
2279                  */
2280                 if (skb_loop_sk(ptype, skb))
2281                         continue;
2282
2283                 if (pt_prev) {
2284                         deliver_skb(skb2, pt_prev, skb->dev);
2285                         pt_prev = ptype;
2286                         continue;
2287                 }
2288
2289                 /* need to clone skb, done only once */
2290                 skb2 = skb_clone(skb, GFP_ATOMIC);
2291                 if (!skb2)
2292                         goto out_unlock;
2293
2294                 net_timestamp_set(skb2);
2295
2296                 /* skb->nh should be correctly
2297                  * set by sender, so that the second statement is
2298                  * just protection against buggy protocols.
2299                  */
2300                 skb_reset_mac_header(skb2);
2301
2302                 if (skb_network_header(skb2) < skb2->data ||
2303                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2304                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2305                                              ntohs(skb2->protocol),
2306                                              dev->name);
2307                         skb_reset_network_header(skb2);
2308                 }
2309
2310                 skb2->transport_header = skb2->network_header;
2311                 skb2->pkt_type = PACKET_OUTGOING;
2312                 pt_prev = ptype;
2313         }
2314
2315         if (ptype_list == &ptype_all) {
2316                 ptype_list = &dev->ptype_all;
2317                 goto again;
2318         }
2319 out_unlock:
2320         if (pt_prev) {
2321                 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2322                         pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2323                 else
2324                         kfree_skb(skb2);
2325         }
2326         rcu_read_unlock();
2327 }
2328 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2329
2330 /**
2331  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2332  * @dev: Network device
2333  * @txq: number of queues available
2334  *
2335  * If real_num_tx_queues is changed the tc mappings may no longer be
2336  * valid. To resolve this verify the tc mapping remains valid and if
2337  * not NULL the mapping. With no priorities mapping to this
2338  * offset/count pair it will no longer be used. In the worst case TC0
2339  * is invalid nothing can be done so disable priority mappings. If is
2340  * expected that drivers will fix this mapping if they can before
2341  * calling netif_set_real_num_tx_queues.
2342  */
2343 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2344 {
2345         int i;
2346         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2347
2348         /* If TC0 is invalidated disable TC mapping */
2349         if (tc->offset + tc->count > txq) {
2350                 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2351                 dev->num_tc = 0;
2352                 return;
2353         }
2354
2355         /* Invalidated prio to tc mappings set to TC0 */
2356         for (i = 1; i < TC_BITMASK + 1; i++) {
2357                 int q = netdev_get_prio_tc_map(dev, i);
2358
2359                 tc = &dev->tc_to_txq[q];
2360                 if (tc->offset + tc->count > txq) {
2361                         netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2362                                     i, q);
2363                         netdev_set_prio_tc_map(dev, i, 0);
2364                 }
2365         }
2366 }
2367
2368 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2369 {
2370         if (dev->num_tc) {
2371                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2372                 int i;
2373
2374                 /* walk through the TCs and see if it falls into any of them */
2375                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2376                         if ((txq - tc->offset) < tc->count)
2377                                 return i;
2378                 }
2379
2380                 /* didn't find it, just return -1 to indicate no match */
2381                 return -1;
2382         }
2383
2384         return 0;
2385 }
2386 EXPORT_SYMBOL(netdev_txq_to_tc);
2387
2388 #ifdef CONFIG_XPS
2389 static struct static_key xps_needed __read_mostly;
2390 static struct static_key xps_rxqs_needed __read_mostly;
2391 static DEFINE_MUTEX(xps_map_mutex);
2392 #define xmap_dereference(P)             \
2393         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2394
2395 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2396                              struct xps_dev_maps *old_maps, int tci, u16 index)
2397 {
2398         struct xps_map *map = NULL;
2399         int pos;
2400
2401         if (dev_maps)
2402                 map = xmap_dereference(dev_maps->attr_map[tci]);
2403         if (!map)
2404                 return false;
2405
2406         for (pos = map->len; pos--;) {
2407                 if (map->queues[pos] != index)
2408                         continue;
2409
2410                 if (map->len > 1) {
2411                         map->queues[pos] = map->queues[--map->len];
2412                         break;
2413                 }
2414
2415                 if (old_maps)
2416                         RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2417                 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2418                 kfree_rcu(map, rcu);
2419                 return false;
2420         }
2421
2422         return true;
2423 }
2424
2425 static bool remove_xps_queue_cpu(struct net_device *dev,
2426                                  struct xps_dev_maps *dev_maps,
2427                                  int cpu, u16 offset, u16 count)
2428 {
2429         int num_tc = dev_maps->num_tc;
2430         bool active = false;
2431         int tci;
2432
2433         for (tci = cpu * num_tc; num_tc--; tci++) {
2434                 int i, j;
2435
2436                 for (i = count, j = offset; i--; j++) {
2437                         if (!remove_xps_queue(dev_maps, NULL, tci, j))
2438                                 break;
2439                 }
2440
2441                 active |= i < 0;
2442         }
2443
2444         return active;
2445 }
2446
2447 static void reset_xps_maps(struct net_device *dev,
2448                            struct xps_dev_maps *dev_maps,
2449                            enum xps_map_type type)
2450 {
2451         static_key_slow_dec_cpuslocked(&xps_needed);
2452         if (type == XPS_RXQS)
2453                 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2454
2455         RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2456
2457         kfree_rcu(dev_maps, rcu);
2458 }
2459
2460 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2461                            u16 offset, u16 count)
2462 {
2463         struct xps_dev_maps *dev_maps;
2464         bool active = false;
2465         int i, j;
2466
2467         dev_maps = xmap_dereference(dev->xps_maps[type]);
2468         if (!dev_maps)
2469                 return;
2470
2471         for (j = 0; j < dev_maps->nr_ids; j++)
2472                 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2473         if (!active)
2474                 reset_xps_maps(dev, dev_maps, type);
2475
2476         if (type == XPS_CPUS) {
2477                 for (i = offset + (count - 1); count--; i--)
2478                         netdev_queue_numa_node_write(
2479                                 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2480         }
2481 }
2482
2483 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2484                                    u16 count)
2485 {
2486         if (!static_key_false(&xps_needed))
2487                 return;
2488
2489         cpus_read_lock();
2490         mutex_lock(&xps_map_mutex);
2491
2492         if (static_key_false(&xps_rxqs_needed))
2493                 clean_xps_maps(dev, XPS_RXQS, offset, count);
2494
2495         clean_xps_maps(dev, XPS_CPUS, offset, count);
2496
2497         mutex_unlock(&xps_map_mutex);
2498         cpus_read_unlock();
2499 }
2500
2501 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2502 {
2503         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2504 }
2505
2506 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2507                                       u16 index, bool is_rxqs_map)
2508 {
2509         struct xps_map *new_map;
2510         int alloc_len = XPS_MIN_MAP_ALLOC;
2511         int i, pos;
2512
2513         for (pos = 0; map && pos < map->len; pos++) {
2514                 if (map->queues[pos] != index)
2515                         continue;
2516                 return map;
2517         }
2518
2519         /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2520         if (map) {
2521                 if (pos < map->alloc_len)
2522                         return map;
2523
2524                 alloc_len = map->alloc_len * 2;
2525         }
2526
2527         /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2528          *  map
2529          */
2530         if (is_rxqs_map)
2531                 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2532         else
2533                 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2534                                        cpu_to_node(attr_index));
2535         if (!new_map)
2536                 return NULL;
2537
2538         for (i = 0; i < pos; i++)
2539                 new_map->queues[i] = map->queues[i];
2540         new_map->alloc_len = alloc_len;
2541         new_map->len = pos;
2542
2543         return new_map;
2544 }
2545
2546 /* Copy xps maps at a given index */
2547 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2548                               struct xps_dev_maps *new_dev_maps, int index,
2549                               int tc, bool skip_tc)
2550 {
2551         int i, tci = index * dev_maps->num_tc;
2552         struct xps_map *map;
2553
2554         /* copy maps belonging to foreign traffic classes */
2555         for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2556                 if (i == tc && skip_tc)
2557                         continue;
2558
2559                 /* fill in the new device map from the old device map */
2560                 map = xmap_dereference(dev_maps->attr_map[tci]);
2561                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2562         }
2563 }
2564
2565 /* Must be called under cpus_read_lock */
2566 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2567                           u16 index, enum xps_map_type type)
2568 {
2569         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2570         const unsigned long *online_mask = NULL;
2571         bool active = false, copy = false;
2572         int i, j, tci, numa_node_id = -2;
2573         int maps_sz, num_tc = 1, tc = 0;
2574         struct xps_map *map, *new_map;
2575         unsigned int nr_ids;
2576
2577         WARN_ON_ONCE(index >= dev->num_tx_queues);
2578
2579         if (dev->num_tc) {
2580                 /* Do not allow XPS on subordinate device directly */
2581                 num_tc = dev->num_tc;
2582                 if (num_tc < 0)
2583                         return -EINVAL;
2584
2585                 /* If queue belongs to subordinate dev use its map */
2586                 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2587
2588                 tc = netdev_txq_to_tc(dev, index);
2589                 if (tc < 0)
2590                         return -EINVAL;
2591         }
2592
2593         mutex_lock(&xps_map_mutex);
2594
2595         dev_maps = xmap_dereference(dev->xps_maps[type]);
2596         if (type == XPS_RXQS) {
2597                 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2598                 nr_ids = dev->num_rx_queues;
2599         } else {
2600                 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2601                 if (num_possible_cpus() > 1)
2602                         online_mask = cpumask_bits(cpu_online_mask);
2603                 nr_ids = nr_cpu_ids;
2604         }
2605
2606         if (maps_sz < L1_CACHE_BYTES)
2607                 maps_sz = L1_CACHE_BYTES;
2608
2609         /* The old dev_maps could be larger or smaller than the one we're
2610          * setting up now, as dev->num_tc or nr_ids could have been updated in
2611          * between. We could try to be smart, but let's be safe instead and only
2612          * copy foreign traffic classes if the two map sizes match.
2613          */
2614         if (dev_maps &&
2615             dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2616                 copy = true;
2617
2618         /* allocate memory for queue storage */
2619         for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2620              j < nr_ids;) {
2621                 if (!new_dev_maps) {
2622                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2623                         if (!new_dev_maps) {
2624                                 mutex_unlock(&xps_map_mutex);
2625                                 return -ENOMEM;
2626                         }
2627
2628                         new_dev_maps->nr_ids = nr_ids;
2629                         new_dev_maps->num_tc = num_tc;
2630                 }
2631
2632                 tci = j * num_tc + tc;
2633                 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2634
2635                 map = expand_xps_map(map, j, index, type == XPS_RXQS);
2636                 if (!map)
2637                         goto error;
2638
2639                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2640         }
2641
2642         if (!new_dev_maps)
2643                 goto out_no_new_maps;
2644
2645         if (!dev_maps) {
2646                 /* Increment static keys at most once per type */
2647                 static_key_slow_inc_cpuslocked(&xps_needed);
2648                 if (type == XPS_RXQS)
2649                         static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2650         }
2651
2652         for (j = 0; j < nr_ids; j++) {
2653                 bool skip_tc = false;
2654
2655                 tci = j * num_tc + tc;
2656                 if (netif_attr_test_mask(j, mask, nr_ids) &&
2657                     netif_attr_test_online(j, online_mask, nr_ids)) {
2658                         /* add tx-queue to CPU/rx-queue maps */
2659                         int pos = 0;
2660
2661                         skip_tc = true;
2662
2663                         map = xmap_dereference(new_dev_maps->attr_map[tci]);
2664                         while ((pos < map->len) && (map->queues[pos] != index))
2665                                 pos++;
2666
2667                         if (pos == map->len)
2668                                 map->queues[map->len++] = index;
2669 #ifdef CONFIG_NUMA
2670                         if (type == XPS_CPUS) {
2671                                 if (numa_node_id == -2)
2672                                         numa_node_id = cpu_to_node(j);
2673                                 else if (numa_node_id != cpu_to_node(j))
2674                                         numa_node_id = -1;
2675                         }
2676 #endif
2677                 }
2678
2679                 if (copy)
2680                         xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2681                                           skip_tc);
2682         }
2683
2684         rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
2685
2686         /* Cleanup old maps */
2687         if (!dev_maps)
2688                 goto out_no_old_maps;
2689
2690         for (j = 0; j < dev_maps->nr_ids; j++) {
2691                 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
2692                         map = xmap_dereference(dev_maps->attr_map[tci]);
2693                         if (!map)
2694                                 continue;
2695
2696                         if (copy) {
2697                                 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2698                                 if (map == new_map)
2699                                         continue;
2700                         }
2701
2702                         RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2703                         kfree_rcu(map, rcu);
2704                 }
2705         }
2706
2707         old_dev_maps = dev_maps;
2708
2709 out_no_old_maps:
2710         dev_maps = new_dev_maps;
2711         active = true;
2712
2713 out_no_new_maps:
2714         if (type == XPS_CPUS)
2715                 /* update Tx queue numa node */
2716                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2717                                              (numa_node_id >= 0) ?
2718                                              numa_node_id : NUMA_NO_NODE);
2719
2720         if (!dev_maps)
2721                 goto out_no_maps;
2722
2723         /* removes tx-queue from unused CPUs/rx-queues */
2724         for (j = 0; j < dev_maps->nr_ids; j++) {
2725                 tci = j * dev_maps->num_tc;
2726
2727                 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2728                         if (i == tc &&
2729                             netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2730                             netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2731                                 continue;
2732
2733                         active |= remove_xps_queue(dev_maps,
2734                                                    copy ? old_dev_maps : NULL,
2735                                                    tci, index);
2736                 }
2737         }
2738
2739         if (old_dev_maps)
2740                 kfree_rcu(old_dev_maps, rcu);
2741
2742         /* free map if not active */
2743         if (!active)
2744                 reset_xps_maps(dev, dev_maps, type);
2745
2746 out_no_maps:
2747         mutex_unlock(&xps_map_mutex);
2748
2749         return 0;
2750 error:
2751         /* remove any maps that we added */
2752         for (j = 0; j < nr_ids; j++) {
2753                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2754                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2755                         map = copy ?
2756                               xmap_dereference(dev_maps->attr_map[tci]) :
2757                               NULL;
2758                         if (new_map && new_map != map)
2759                                 kfree(new_map);
2760                 }
2761         }
2762
2763         mutex_unlock(&xps_map_mutex);
2764
2765         kfree(new_dev_maps);
2766         return -ENOMEM;
2767 }
2768 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2769
2770 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2771                         u16 index)
2772 {
2773         int ret;
2774
2775         cpus_read_lock();
2776         ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
2777         cpus_read_unlock();
2778
2779         return ret;
2780 }
2781 EXPORT_SYMBOL(netif_set_xps_queue);
2782
2783 #endif
2784 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2785 {
2786         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2787
2788         /* Unbind any subordinate channels */
2789         while (txq-- != &dev->_tx[0]) {
2790                 if (txq->sb_dev)
2791                         netdev_unbind_sb_channel(dev, txq->sb_dev);
2792         }
2793 }
2794
2795 void netdev_reset_tc(struct net_device *dev)
2796 {
2797 #ifdef CONFIG_XPS
2798         netif_reset_xps_queues_gt(dev, 0);
2799 #endif
2800         netdev_unbind_all_sb_channels(dev);
2801
2802         /* Reset TC configuration of device */
2803         dev->num_tc = 0;
2804         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2805         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2806 }
2807 EXPORT_SYMBOL(netdev_reset_tc);
2808
2809 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2810 {
2811         if (tc >= dev->num_tc)
2812                 return -EINVAL;
2813
2814 #ifdef CONFIG_XPS
2815         netif_reset_xps_queues(dev, offset, count);
2816 #endif
2817         dev->tc_to_txq[tc].count = count;
2818         dev->tc_to_txq[tc].offset = offset;
2819         return 0;
2820 }
2821 EXPORT_SYMBOL(netdev_set_tc_queue);
2822
2823 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2824 {
2825         if (num_tc > TC_MAX_QUEUE)
2826                 return -EINVAL;
2827
2828 #ifdef CONFIG_XPS
2829         netif_reset_xps_queues_gt(dev, 0);
2830 #endif
2831         netdev_unbind_all_sb_channels(dev);
2832
2833         dev->num_tc = num_tc;
2834         return 0;
2835 }
2836 EXPORT_SYMBOL(netdev_set_num_tc);
2837
2838 void netdev_unbind_sb_channel(struct net_device *dev,
2839                               struct net_device *sb_dev)
2840 {
2841         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2842
2843 #ifdef CONFIG_XPS
2844         netif_reset_xps_queues_gt(sb_dev, 0);
2845 #endif
2846         memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2847         memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2848
2849         while (txq-- != &dev->_tx[0]) {
2850                 if (txq->sb_dev == sb_dev)
2851                         txq->sb_dev = NULL;
2852         }
2853 }
2854 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2855
2856 int netdev_bind_sb_channel_queue(struct net_device *dev,
2857                                  struct net_device *sb_dev,
2858                                  u8 tc, u16 count, u16 offset)
2859 {
2860         /* Make certain the sb_dev and dev are already configured */
2861         if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2862                 return -EINVAL;
2863
2864         /* We cannot hand out queues we don't have */
2865         if ((offset + count) > dev->real_num_tx_queues)
2866                 return -EINVAL;
2867
2868         /* Record the mapping */
2869         sb_dev->tc_to_txq[tc].count = count;
2870         sb_dev->tc_to_txq[tc].offset = offset;
2871
2872         /* Provide a way for Tx queue to find the tc_to_txq map or
2873          * XPS map for itself.
2874          */
2875         while (count--)
2876                 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2877
2878         return 0;
2879 }
2880 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2881
2882 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2883 {
2884         /* Do not use a multiqueue device to represent a subordinate channel */
2885         if (netif_is_multiqueue(dev))
2886                 return -ENODEV;
2887
2888         /* We allow channels 1 - 32767 to be used for subordinate channels.
2889          * Channel 0 is meant to be "native" mode and used only to represent
2890          * the main root device. We allow writing 0 to reset the device back
2891          * to normal mode after being used as a subordinate channel.
2892          */
2893         if (channel > S16_MAX)
2894                 return -EINVAL;
2895
2896         dev->num_tc = -channel;
2897
2898         return 0;
2899 }
2900 EXPORT_SYMBOL(netdev_set_sb_channel);
2901
2902 /*
2903  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2904  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2905  */
2906 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2907 {
2908         bool disabling;
2909         int rc;
2910
2911         disabling = txq < dev->real_num_tx_queues;
2912
2913         if (txq < 1 || txq > dev->num_tx_queues)
2914                 return -EINVAL;
2915
2916         if (dev->reg_state == NETREG_REGISTERED ||
2917             dev->reg_state == NETREG_UNREGISTERING) {
2918                 ASSERT_RTNL();
2919
2920                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2921                                                   txq);
2922                 if (rc)
2923                         return rc;
2924
2925                 if (dev->num_tc)
2926                         netif_setup_tc(dev, txq);
2927
2928                 dev_qdisc_change_real_num_tx(dev, txq);
2929
2930                 dev->real_num_tx_queues = txq;
2931
2932                 if (disabling) {
2933                         synchronize_net();
2934                         qdisc_reset_all_tx_gt(dev, txq);
2935 #ifdef CONFIG_XPS
2936                         netif_reset_xps_queues_gt(dev, txq);
2937 #endif
2938                 }
2939         } else {
2940                 dev->real_num_tx_queues = txq;
2941         }
2942
2943         return 0;
2944 }
2945 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2946
2947 #ifdef CONFIG_SYSFS
2948 /**
2949  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2950  *      @dev: Network device
2951  *      @rxq: Actual number of RX queues
2952  *
2953  *      This must be called either with the rtnl_lock held or before
2954  *      registration of the net device.  Returns 0 on success, or a
2955  *      negative error code.  If called before registration, it always
2956  *      succeeds.
2957  */
2958 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2959 {
2960         int rc;
2961
2962         if (rxq < 1 || rxq > dev->num_rx_queues)
2963                 return -EINVAL;
2964
2965         if (dev->reg_state == NETREG_REGISTERED) {
2966                 ASSERT_RTNL();
2967
2968                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2969                                                   rxq);
2970                 if (rc)
2971                         return rc;
2972         }
2973
2974         dev->real_num_rx_queues = rxq;
2975         return 0;
2976 }
2977 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2978 #endif
2979
2980 /**
2981  *      netif_set_real_num_queues - set actual number of RX and TX queues used
2982  *      @dev: Network device
2983  *      @txq: Actual number of TX queues
2984  *      @rxq: Actual number of RX queues
2985  *
2986  *      Set the real number of both TX and RX queues.
2987  *      Does nothing if the number of queues is already correct.
2988  */
2989 int netif_set_real_num_queues(struct net_device *dev,
2990                               unsigned int txq, unsigned int rxq)
2991 {
2992         unsigned int old_rxq = dev->real_num_rx_queues;
2993         int err;
2994
2995         if (txq < 1 || txq > dev->num_tx_queues ||
2996             rxq < 1 || rxq > dev->num_rx_queues)
2997                 return -EINVAL;
2998
2999         /* Start from increases, so the error path only does decreases -
3000          * decreases can't fail.
3001          */
3002         if (rxq > dev->real_num_rx_queues) {
3003                 err = netif_set_real_num_rx_queues(dev, rxq);
3004                 if (err)
3005                         return err;
3006         }
3007         if (txq > dev->real_num_tx_queues) {
3008                 err = netif_set_real_num_tx_queues(dev, txq);
3009                 if (err)
3010                         goto undo_rx;
3011         }
3012         if (rxq < dev->real_num_rx_queues)
3013                 WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3014         if (txq < dev->real_num_tx_queues)
3015                 WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3016
3017         return 0;
3018 undo_rx:
3019         WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3020         return err;
3021 }
3022 EXPORT_SYMBOL(netif_set_real_num_queues);
3023
3024 /**
3025  * netif_set_tso_max_size() - set the max size of TSO frames supported
3026  * @dev:        netdev to update
3027  * @size:       max skb->len of a TSO frame
3028  *
3029  * Set the limit on the size of TSO super-frames the device can handle.
3030  * Unless explicitly set the stack will assume the value of
3031  * %GSO_LEGACY_MAX_SIZE.
3032  */
3033 void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3034 {
3035         dev->tso_max_size = min(GSO_MAX_SIZE, size);
3036         if (size < READ_ONCE(dev->gso_max_size))
3037                 netif_set_gso_max_size(dev, size);
3038 }
3039 EXPORT_SYMBOL(netif_set_tso_max_size);
3040
3041 /**
3042  * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3043  * @dev:        netdev to update
3044  * @segs:       max number of TCP segments
3045  *
3046  * Set the limit on the number of TCP segments the device can generate from
3047  * a single TSO super-frame.
3048  * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3049  */
3050 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3051 {
3052         dev->tso_max_segs = segs;
3053         if (segs < READ_ONCE(dev->gso_max_segs))
3054                 netif_set_gso_max_segs(dev, segs);
3055 }
3056 EXPORT_SYMBOL(netif_set_tso_max_segs);
3057
3058 /**
3059  * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3060  * @to:         netdev to update
3061  * @from:       netdev from which to copy the limits
3062  */
3063 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3064 {
3065         netif_set_tso_max_size(to, from->tso_max_size);
3066         netif_set_tso_max_segs(to, from->tso_max_segs);
3067 }
3068 EXPORT_SYMBOL(netif_inherit_tso_max);
3069
3070 /**
3071  * netif_get_num_default_rss_queues - default number of RSS queues
3072  *
3073  * Default value is the number of physical cores if there are only 1 or 2, or
3074  * divided by 2 if there are more.
3075  */
3076 int netif_get_num_default_rss_queues(void)
3077 {
3078         cpumask_var_t cpus;
3079         int cpu, count = 0;
3080
3081         if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3082                 return 1;
3083
3084         cpumask_copy(cpus, cpu_online_mask);
3085         for_each_cpu(cpu, cpus) {
3086                 ++count;
3087                 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3088         }
3089         free_cpumask_var(cpus);
3090
3091         return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3092 }
3093 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3094
3095 static void __netif_reschedule(struct Qdisc *q)
3096 {
3097         struct softnet_data *sd;
3098         unsigned long flags;
3099
3100         local_irq_save(flags);
3101         sd = this_cpu_ptr(&softnet_data);
3102         q->next_sched = NULL;
3103         *sd->output_queue_tailp = q;
3104         sd->output_queue_tailp = &q->next_sched;
3105         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3106         local_irq_restore(flags);
3107 }
3108
3109 void __netif_schedule(struct Qdisc *q)
3110 {
3111         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3112                 __netif_reschedule(q);
3113 }
3114 EXPORT_SYMBOL(__netif_schedule);
3115
3116 struct dev_kfree_skb_cb {
3117         enum skb_free_reason reason;
3118 };
3119
3120 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3121 {
3122         return (struct dev_kfree_skb_cb *)skb->cb;
3123 }
3124
3125 void netif_schedule_queue(struct netdev_queue *txq)
3126 {
3127         rcu_read_lock();
3128         if (!netif_xmit_stopped(txq)) {
3129                 struct Qdisc *q = rcu_dereference(txq->qdisc);
3130
3131                 __netif_schedule(q);
3132         }
3133         rcu_read_unlock();
3134 }
3135 EXPORT_SYMBOL(netif_schedule_queue);
3136
3137 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3138 {
3139         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3140                 struct Qdisc *q;
3141
3142                 rcu_read_lock();
3143                 q = rcu_dereference(dev_queue->qdisc);
3144                 __netif_schedule(q);
3145                 rcu_read_unlock();
3146         }
3147 }
3148 EXPORT_SYMBOL(netif_tx_wake_queue);
3149
3150 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
3151 {
3152         unsigned long flags;
3153
3154         if (unlikely(!skb))
3155                 return;
3156
3157         if (likely(refcount_read(&skb->users) == 1)) {
3158                 smp_rmb();
3159                 refcount_set(&skb->users, 0);
3160         } else if (likely(!refcount_dec_and_test(&skb->users))) {
3161                 return;
3162         }
3163         get_kfree_skb_cb(skb)->reason = reason;
3164         local_irq_save(flags);
3165         skb->next = __this_cpu_read(softnet_data.completion_queue);
3166         __this_cpu_write(softnet_data.completion_queue, skb);
3167         raise_softirq_irqoff(NET_TX_SOFTIRQ);
3168         local_irq_restore(flags);
3169 }
3170 EXPORT_SYMBOL(__dev_kfree_skb_irq);
3171
3172 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
3173 {
3174         if (in_hardirq() || irqs_disabled())
3175                 __dev_kfree_skb_irq(skb, reason);
3176         else if (unlikely(reason == SKB_REASON_DROPPED))
3177                 kfree_skb(skb);
3178         else
3179                 consume_skb(skb);
3180 }
3181 EXPORT_SYMBOL(__dev_kfree_skb_any);
3182
3183
3184 /**
3185  * netif_device_detach - mark device as removed
3186  * @dev: network device
3187  *
3188  * Mark device as removed from system and therefore no longer available.
3189  */
3190 void netif_device_detach(struct net_device *dev)
3191 {
3192         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3193             netif_running(dev)) {
3194                 netif_tx_stop_all_queues(dev);
3195         }
3196 }
3197 EXPORT_SYMBOL(netif_device_detach);
3198
3199 /**
3200  * netif_device_attach - mark device as attached
3201  * @dev: network device
3202  *
3203  * Mark device as attached from system and restart if needed.
3204  */
3205 void netif_device_attach(struct net_device *dev)
3206 {
3207         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3208             netif_running(dev)) {
3209                 netif_tx_wake_all_queues(dev);
3210                 __netdev_watchdog_up(dev);
3211         }
3212 }
3213 EXPORT_SYMBOL(netif_device_attach);
3214
3215 /*
3216  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3217  * to be used as a distribution range.
3218  */
3219 static u16 skb_tx_hash(const struct net_device *dev,
3220                        const struct net_device *sb_dev,
3221                        struct sk_buff *skb)
3222 {
3223         u32 hash;
3224         u16 qoffset = 0;
3225         u16 qcount = dev->real_num_tx_queues;
3226
3227         if (dev->num_tc) {
3228                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3229
3230                 qoffset = sb_dev->tc_to_txq[tc].offset;
3231                 qcount = sb_dev->tc_to_txq[tc].count;
3232                 if (unlikely(!qcount)) {
3233                         net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3234                                              sb_dev->name, qoffset, tc);
3235                         qoffset = 0;
3236                         qcount = dev->real_num_tx_queues;
3237                 }
3238         }
3239
3240         if (skb_rx_queue_recorded(skb)) {
3241                 DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3242                 hash = skb_get_rx_queue(skb);
3243                 if (hash >= qoffset)
3244                         hash -= qoffset;
3245                 while (unlikely(hash >= qcount))
3246                         hash -= qcount;
3247                 return hash + qoffset;
3248         }
3249
3250         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3251 }
3252
3253 static void skb_warn_bad_offload(const struct sk_buff *skb)
3254 {
3255         static const netdev_features_t null_features;
3256         struct net_device *dev = skb->dev;
3257         const char *name = "";
3258
3259         if (!net_ratelimit())
3260                 return;
3261
3262         if (dev) {
3263                 if (dev->dev.parent)
3264                         name = dev_driver_string(dev->dev.parent);
3265                 else
3266                         name = netdev_name(dev);
3267         }
3268         skb_dump(KERN_WARNING, skb, false);
3269         WARN(1, "%s: caps=(%pNF, %pNF)\n",
3270              name, dev ? &dev->features : &null_features,
3271              skb->sk ? &skb->sk->sk_route_caps : &null_features);
3272 }
3273
3274 /*
3275  * Invalidate hardware checksum when packet is to be mangled, and
3276  * complete checksum manually on outgoing path.
3277  */
3278 int skb_checksum_help(struct sk_buff *skb)
3279 {
3280         __wsum csum;
3281         int ret = 0, offset;
3282
3283         if (skb->ip_summed == CHECKSUM_COMPLETE)
3284                 goto out_set_summed;
3285
3286         if (unlikely(skb_is_gso(skb))) {
3287                 skb_warn_bad_offload(skb);
3288                 return -EINVAL;
3289         }
3290
3291         /* Before computing a checksum, we should make sure no frag could
3292          * be modified by an external entity : checksum could be wrong.
3293          */
3294         if (skb_has_shared_frag(skb)) {
3295                 ret = __skb_linearize(skb);
3296                 if (ret)
3297                         goto out;
3298         }
3299
3300         offset = skb_checksum_start_offset(skb);
3301         ret = -EINVAL;
3302         if (unlikely(offset >= skb_headlen(skb))) {
3303                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3304                 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3305                           offset, skb_headlen(skb));
3306                 goto out;
3307         }
3308         csum = skb_checksum(skb, offset, skb->len - offset, 0);
3309
3310         offset += skb->csum_offset;
3311         if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3312                 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3313                 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3314                           offset + sizeof(__sum16), skb_headlen(skb));
3315                 goto out;
3316         }
3317         ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3318         if (ret)
3319                 goto out;
3320
3321         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3322 out_set_summed:
3323         skb->ip_summed = CHECKSUM_NONE;
3324 out:
3325         return ret;
3326 }
3327 EXPORT_SYMBOL(skb_checksum_help);
3328
3329 int skb_crc32c_csum_help(struct sk_buff *skb)
3330 {
3331         __le32 crc32c_csum;
3332         int ret = 0, offset, start;
3333
3334         if (skb->ip_summed != CHECKSUM_PARTIAL)
3335                 goto out;
3336
3337         if (unlikely(skb_is_gso(skb)))
3338                 goto out;
3339
3340         /* Before computing a checksum, we should make sure no frag could
3341          * be modified by an external entity : checksum could be wrong.
3342          */
3343         if (unlikely(skb_has_shared_frag(skb))) {
3344                 ret = __skb_linearize(skb);
3345                 if (ret)
3346                         goto out;
3347         }
3348         start = skb_checksum_start_offset(skb);
3349         offset = start + offsetof(struct sctphdr, checksum);
3350         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3351                 ret = -EINVAL;
3352                 goto out;
3353         }
3354
3355         ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3356         if (ret)
3357                 goto out;
3358
3359         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3360                                                   skb->len - start, ~(__u32)0,
3361                                                   crc32c_csum_stub));
3362         *(__le32 *)(skb->data + offset) = crc32c_csum;
3363         skb->ip_summed = CHECKSUM_NONE;
3364         skb->csum_not_inet = 0;
3365 out:
3366         return ret;
3367 }
3368
3369 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3370 {
3371         __be16 type = skb->protocol;
3372
3373         /* Tunnel gso handlers can set protocol to ethernet. */
3374         if (type == htons(ETH_P_TEB)) {
3375                 struct ethhdr *eth;
3376
3377                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3378                         return 0;
3379
3380                 eth = (struct ethhdr *)skb->data;
3381                 type = eth->h_proto;
3382         }
3383
3384         return vlan_get_protocol_and_depth(skb, type, depth);
3385 }
3386
3387 /* openvswitch calls this on rx path, so we need a different check.
3388  */
3389 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3390 {
3391         if (tx_path)
3392                 return skb->ip_summed != CHECKSUM_PARTIAL &&
3393                        skb->ip_summed != CHECKSUM_UNNECESSARY;
3394
3395         return skb->ip_summed == CHECKSUM_NONE;
3396 }
3397
3398 /**
3399  *      __skb_gso_segment - Perform segmentation on skb.
3400  *      @skb: buffer to segment
3401  *      @features: features for the output path (see dev->features)
3402  *      @tx_path: whether it is called in TX path
3403  *
3404  *      This function segments the given skb and returns a list of segments.
3405  *
3406  *      It may return NULL if the skb requires no segmentation.  This is
3407  *      only possible when GSO is used for verifying header integrity.
3408  *
3409  *      Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
3410  */
3411 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3412                                   netdev_features_t features, bool tx_path)
3413 {
3414         struct sk_buff *segs;
3415
3416         if (unlikely(skb_needs_check(skb, tx_path))) {
3417                 int err;
3418
3419                 /* We're going to init ->check field in TCP or UDP header */
3420                 err = skb_cow_head(skb, 0);
3421                 if (err < 0)
3422                         return ERR_PTR(err);
3423         }
3424
3425         /* Only report GSO partial support if it will enable us to
3426          * support segmentation on this frame without needing additional
3427          * work.
3428          */
3429         if (features & NETIF_F_GSO_PARTIAL) {
3430                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3431                 struct net_device *dev = skb->dev;
3432
3433                 partial_features |= dev->features & dev->gso_partial_features;
3434                 if (!skb_gso_ok(skb, features | partial_features))
3435                         features &= ~NETIF_F_GSO_PARTIAL;
3436         }
3437
3438         BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
3439                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3440
3441         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3442         SKB_GSO_CB(skb)->encap_level = 0;
3443
3444         skb_reset_mac_header(skb);
3445         skb_reset_mac_len(skb);
3446
3447         segs = skb_mac_gso_segment(skb, features);
3448
3449         if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3450                 skb_warn_bad_offload(skb);
3451
3452         return segs;
3453 }
3454 EXPORT_SYMBOL(__skb_gso_segment);
3455
3456 /* Take action when hardware reception checksum errors are detected. */
3457 #ifdef CONFIG_BUG
3458 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3459 {
3460         netdev_err(dev, "hw csum failure\n");
3461         skb_dump(KERN_ERR, skb, true);
3462         dump_stack();
3463 }
3464
3465 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3466 {
3467         DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3468 }
3469 EXPORT_SYMBOL(netdev_rx_csum_fault);
3470 #endif
3471
3472 /* XXX: check that highmem exists at all on the given machine. */
3473 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3474 {
3475 #ifdef CONFIG_HIGHMEM
3476         int i;
3477
3478         if (!(dev->features & NETIF_F_HIGHDMA)) {
3479                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3480                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3481
3482                         if (PageHighMem(skb_frag_page(frag)))
3483                                 return 1;
3484                 }
3485         }
3486 #endif
3487         return 0;
3488 }
3489
3490 /* If MPLS offload request, verify we are testing hardware MPLS features
3491  * instead of standard features for the netdev.
3492  */
3493 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3494 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3495                                            netdev_features_t features,
3496                                            __be16 type)
3497 {
3498         if (eth_p_mpls(type))
3499                 features &= skb->dev->mpls_features;
3500
3501         return features;
3502 }
3503 #else
3504 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3505                                            netdev_features_t features,
3506                                            __be16 type)
3507 {
3508         return features;
3509 }
3510 #endif
3511
3512 static netdev_features_t harmonize_features(struct sk_buff *skb,
3513         netdev_features_t features)
3514 {
3515         __be16 type;
3516
3517         type = skb_network_protocol(skb, NULL);
3518         features = net_mpls_features(skb, features, type);
3519
3520         if (skb->ip_summed != CHECKSUM_NONE &&
3521             !can_checksum_protocol(features, type)) {
3522                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3523         }
3524         if (illegal_highdma(skb->dev, skb))
3525                 features &= ~NETIF_F_SG;
3526
3527         return features;
3528 }
3529
3530 netdev_features_t passthru_features_check(struct sk_buff *skb,
3531                                           struct net_device *dev,
3532                                           netdev_features_t features)
3533 {
3534         return features;
3535 }
3536 EXPORT_SYMBOL(passthru_features_check);
3537
3538 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3539                                              struct net_device *dev,
3540                                              netdev_features_t features)
3541 {
3542         return vlan_features_check(skb, features);
3543 }
3544
3545 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3546                                             struct net_device *dev,
3547                                             netdev_features_t features)
3548 {
3549         u16 gso_segs = skb_shinfo(skb)->gso_segs;
3550
3551         if (gso_segs > READ_ONCE(dev->gso_max_segs))
3552                 return features & ~NETIF_F_GSO_MASK;
3553
3554         if (unlikely(skb->len >= READ_ONCE(dev->gso_max_size)))
3555                 return features & ~NETIF_F_GSO_MASK;
3556
3557         if (!skb_shinfo(skb)->gso_type) {
3558                 skb_warn_bad_offload(skb);
3559                 return features & ~NETIF_F_GSO_MASK;
3560         }
3561
3562         /* Support for GSO partial features requires software
3563          * intervention before we can actually process the packets
3564          * so we need to strip support for any partial features now
3565          * and we can pull them back in after we have partially
3566          * segmented the frame.
3567          */
3568         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3569                 features &= ~dev->gso_partial_features;
3570
3571         /* Make sure to clear the IPv4 ID mangling feature if the
3572          * IPv4 header has the potential to be fragmented.
3573          */
3574         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3575                 struct iphdr *iph = skb->encapsulation ?
3576                                     inner_ip_hdr(skb) : ip_hdr(skb);
3577
3578                 if (!(iph->frag_off & htons(IP_DF)))
3579                         features &= ~NETIF_F_TSO_MANGLEID;
3580         }
3581
3582         return features;
3583 }
3584
3585 netdev_features_t netif_skb_features(struct sk_buff *skb)
3586 {
3587         struct net_device *dev = skb->dev;
3588         netdev_features_t features = dev->features;
3589
3590         if (skb_is_gso(skb))
3591                 features = gso_features_check(skb, dev, features);
3592
3593         /* If encapsulation offload request, verify we are testing
3594          * hardware encapsulation features instead of standard
3595          * features for the netdev
3596          */
3597         if (skb->encapsulation)
3598                 features &= dev->hw_enc_features;
3599
3600         if (skb_vlan_tagged(skb))
3601                 features = netdev_intersect_features(features,
3602                                                      dev->vlan_features |
3603                                                      NETIF_F_HW_VLAN_CTAG_TX |
3604                                                      NETIF_F_HW_VLAN_STAG_TX);
3605
3606         if (dev->netdev_ops->ndo_features_check)
3607                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3608                                                                 features);
3609         else
3610                 features &= dflt_features_check(skb, dev, features);
3611
3612         return harmonize_features(skb, features);
3613 }
3614 EXPORT_SYMBOL(netif_skb_features);
3615
3616 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3617                     struct netdev_queue *txq, bool more)
3618 {
3619         unsigned int len;
3620         int rc;
3621
3622         if (dev_nit_active(dev))
3623                 dev_queue_xmit_nit(skb, dev);
3624
3625         len = skb->len;
3626         trace_net_dev_start_xmit(skb, dev);
3627         rc = netdev_start_xmit(skb, dev, txq, more);
3628         trace_net_dev_xmit(skb, rc, dev, len);
3629
3630         return rc;
3631 }
3632
3633 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3634                                     struct netdev_queue *txq, int *ret)
3635 {
3636         struct sk_buff *skb = first;
3637         int rc = NETDEV_TX_OK;
3638
3639         while (skb) {
3640                 struct sk_buff *next = skb->next;
3641
3642                 skb_mark_not_on_list(skb);
3643                 rc = xmit_one(skb, dev, txq, next != NULL);
3644                 if (unlikely(!dev_xmit_complete(rc))) {
3645                         skb->next = next;
3646                         goto out;
3647                 }
3648
3649                 skb = next;
3650                 if (netif_tx_queue_stopped(txq) && skb) {
3651                         rc = NETDEV_TX_BUSY;
3652                         break;
3653                 }
3654         }
3655
3656 out:
3657         *ret = rc;
3658         return skb;
3659 }
3660
3661 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3662                                           netdev_features_t features)
3663 {
3664         if (skb_vlan_tag_present(skb) &&
3665             !vlan_hw_offload_capable(features, skb->vlan_proto))
3666                 skb = __vlan_hwaccel_push_inside(skb);
3667         return skb;
3668 }
3669
3670 int skb_csum_hwoffload_help(struct sk_buff *skb,
3671                             const netdev_features_t features)
3672 {
3673         if (unlikely(skb_csum_is_sctp(skb)))
3674                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3675                         skb_crc32c_csum_help(skb);
3676
3677         if (features & NETIF_F_HW_CSUM)
3678                 return 0;
3679
3680         if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3681                 switch (skb->csum_offset) {
3682                 case offsetof(struct tcphdr, check):
3683                 case offsetof(struct udphdr, check):
3684                         return 0;
3685                 }
3686         }
3687
3688         return skb_checksum_help(skb);
3689 }
3690 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3691
3692 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3693 {
3694         netdev_features_t features;
3695
3696         features = netif_skb_features(skb);
3697         skb = validate_xmit_vlan(skb, features);
3698         if (unlikely(!skb))
3699                 goto out_null;
3700
3701         skb = sk_validate_xmit_skb(skb, dev);
3702         if (unlikely(!skb))
3703                 goto out_null;
3704
3705         if (netif_needs_gso(skb, features)) {
3706                 struct sk_buff *segs;
3707
3708                 segs = skb_gso_segment(skb, features);
3709                 if (IS_ERR(segs)) {
3710                         goto out_kfree_skb;
3711                 } else if (segs) {
3712                         consume_skb(skb);
3713                         skb = segs;
3714                 }
3715         } else {
3716                 if (skb_needs_linearize(skb, features) &&
3717                     __skb_linearize(skb))
3718                         goto out_kfree_skb;
3719
3720                 /* If packet is not checksummed and device does not
3721                  * support checksumming for this protocol, complete
3722                  * checksumming here.
3723                  */
3724                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3725                         if (skb->encapsulation)
3726                                 skb_set_inner_transport_header(skb,
3727                                                                skb_checksum_start_offset(skb));
3728                         else
3729                                 skb_set_transport_header(skb,
3730                                                          skb_checksum_start_offset(skb));
3731                         if (skb_csum_hwoffload_help(skb, features))
3732                                 goto out_kfree_skb;
3733                 }
3734         }
3735
3736         skb = validate_xmit_xfrm(skb, features, again);
3737
3738         return skb;
3739
3740 out_kfree_skb:
3741         kfree_skb(skb);
3742 out_null:
3743         dev_core_stats_tx_dropped_inc(dev);
3744         return NULL;
3745 }
3746
3747 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3748 {
3749         struct sk_buff *next, *head = NULL, *tail;
3750
3751         for (; skb != NULL; skb = next) {
3752                 next = skb->next;
3753                 skb_mark_not_on_list(skb);
3754
3755                 /* in case skb wont be segmented, point to itself */
3756                 skb->prev = skb;
3757
3758                 skb = validate_xmit_skb(skb, dev, again);
3759                 if (!skb)
3760                         continue;
3761
3762                 if (!head)
3763                         head = skb;
3764                 else
3765                         tail->next = skb;
3766                 /* If skb was segmented, skb->prev points to
3767                  * the last segment. If not, it still contains skb.
3768                  */
3769                 tail = skb->prev;
3770         }
3771         return head;
3772 }
3773 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3774
3775 static void qdisc_pkt_len_init(struct sk_buff *skb)
3776 {
3777         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3778
3779         qdisc_skb_cb(skb)->pkt_len = skb->len;
3780
3781         /* To get more precise estimation of bytes sent on wire,
3782          * we add to pkt_len the headers size of all segments
3783          */
3784         if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
3785                 unsigned int hdr_len;
3786                 u16 gso_segs = shinfo->gso_segs;
3787
3788                 /* mac layer + network layer */
3789                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3790
3791                 /* + transport layer */
3792                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3793                         const struct tcphdr *th;
3794                         struct tcphdr _tcphdr;
3795
3796                         th = skb_header_pointer(skb, skb_transport_offset(skb),
3797                                                 sizeof(_tcphdr), &_tcphdr);
3798                         if (likely(th))
3799                                 hdr_len += __tcp_hdrlen(th);
3800                 } else {
3801                         struct udphdr _udphdr;
3802
3803                         if (skb_header_pointer(skb, skb_transport_offset(skb),
3804                                                sizeof(_udphdr), &_udphdr))
3805                                 hdr_len += sizeof(struct udphdr);
3806                 }
3807
3808                 if (shinfo->gso_type & SKB_GSO_DODGY)
3809                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3810                                                 shinfo->gso_size);
3811
3812                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3813         }
3814 }
3815
3816 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3817                              struct sk_buff **to_free,
3818                              struct netdev_queue *txq)
3819 {
3820         int rc;
3821
3822         rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3823         if (rc == NET_XMIT_SUCCESS)
3824                 trace_qdisc_enqueue(q, txq, skb);
3825         return rc;
3826 }
3827
3828 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3829                                  struct net_device *dev,
3830                                  struct netdev_queue *txq)
3831 {
3832         spinlock_t *root_lock = qdisc_lock(q);
3833         struct sk_buff *to_free = NULL;
3834         bool contended;
3835         int rc;
3836
3837         qdisc_calculate_pkt_len(skb, q);
3838
3839         if (q->flags & TCQ_F_NOLOCK) {
3840                 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3841                     qdisc_run_begin(q)) {
3842                         /* Retest nolock_qdisc_is_empty() within the protection
3843                          * of q->seqlock to protect from racing with requeuing.
3844                          */
3845                         if (unlikely(!nolock_qdisc_is_empty(q))) {
3846                                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3847                                 __qdisc_run(q);
3848                                 qdisc_run_end(q);
3849
3850                                 goto no_lock_out;
3851                         }
3852
3853                         qdisc_bstats_cpu_update(q, skb);
3854                         if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3855                             !nolock_qdisc_is_empty(q))
3856                                 __qdisc_run(q);
3857
3858                         qdisc_run_end(q);
3859                         return NET_XMIT_SUCCESS;
3860                 }
3861
3862                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3863                 qdisc_run(q);
3864
3865 no_lock_out:
3866                 if (unlikely(to_free))
3867                         kfree_skb_list_reason(to_free,
3868                                               SKB_DROP_REASON_QDISC_DROP);
3869                 return rc;
3870         }
3871
3872         /*
3873          * Heuristic to force contended enqueues to serialize on a
3874          * separate lock before trying to get qdisc main lock.
3875          * This permits qdisc->running owner to get the lock more
3876          * often and dequeue packets faster.
3877          * On PREEMPT_RT it is possible to preempt the qdisc owner during xmit
3878          * and then other tasks will only enqueue packets. The packets will be
3879          * sent after the qdisc owner is scheduled again. To prevent this
3880          * scenario the task always serialize on the lock.
3881          */
3882         contended = qdisc_is_running(q) || IS_ENABLED(CONFIG_PREEMPT_RT);
3883         if (unlikely(contended))
3884                 spin_lock(&q->busylock);
3885
3886         spin_lock(root_lock);
3887         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3888                 __qdisc_drop(skb, &to_free);
3889                 rc = NET_XMIT_DROP;
3890         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3891                    qdisc_run_begin(q)) {
3892                 /*
3893                  * This is a work-conserving queue; there are no old skbs
3894                  * waiting to be sent out; and the qdisc is not running -
3895                  * xmit the skb directly.
3896                  */
3897
3898                 qdisc_bstats_update(q, skb);
3899
3900                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3901                         if (unlikely(contended)) {
3902                                 spin_unlock(&q->busylock);
3903                                 contended = false;
3904                         }
3905                         __qdisc_run(q);
3906                 }
3907
3908                 qdisc_run_end(q);
3909                 rc = NET_XMIT_SUCCESS;
3910         } else {
3911                 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
3912                 if (qdisc_run_begin(q)) {
3913                         if (unlikely(contended)) {
3914                                 spin_unlock(&q->busylock);
3915                                 contended = false;
3916                         }
3917                         __qdisc_run(q);
3918                         qdisc_run_end(q);
3919                 }
3920         }
3921         spin_unlock(root_lock);
3922         if (unlikely(to_free))
3923                 kfree_skb_list_reason(to_free, SKB_DROP_REASON_QDISC_DROP);
3924         if (unlikely(contended))
3925                 spin_unlock(&q->busylock);
3926         return rc;
3927 }
3928
3929 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3930 static void skb_update_prio(struct sk_buff *skb)
3931 {
3932         const struct netprio_map *map;
3933         const struct sock *sk;
3934         unsigned int prioidx;
3935
3936         if (skb->priority)
3937                 return;
3938         map = rcu_dereference_bh(skb->dev->priomap);
3939         if (!map)
3940                 return;
3941         sk = skb_to_full_sk(skb);
3942         if (!sk)
3943                 return;
3944
3945         prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3946
3947         if (prioidx < map->priomap_len)
3948                 skb->priority = map->priomap[prioidx];
3949 }
3950 #else
3951 #define skb_update_prio(skb)
3952 #endif
3953
3954 /**
3955  *      dev_loopback_xmit - loop back @skb
3956  *      @net: network namespace this loopback is happening in
3957  *      @sk:  sk needed to be a netfilter okfn
3958  *      @skb: buffer to transmit
3959  */
3960 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3961 {
3962         skb_reset_mac_header(skb);
3963         __skb_pull(skb, skb_network_offset(skb));
3964         skb->pkt_type = PACKET_LOOPBACK;
3965         if (skb->ip_summed == CHECKSUM_NONE)
3966                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3967         DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
3968         skb_dst_force(skb);
3969         netif_rx(skb);
3970         return 0;
3971 }
3972 EXPORT_SYMBOL(dev_loopback_xmit);
3973
3974 #ifdef CONFIG_NET_EGRESS
3975 static struct sk_buff *
3976 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3977 {
3978 #ifdef CONFIG_NET_CLS_ACT
3979         struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3980         struct tcf_result cl_res;
3981
3982         if (!miniq)
3983                 return skb;
3984
3985         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3986         tc_skb_cb(skb)->mru = 0;
3987         tc_skb_cb(skb)->post_ct = false;
3988         mini_qdisc_bstats_cpu_update(miniq, skb);
3989
3990         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
3991         case TC_ACT_OK:
3992         case TC_ACT_RECLASSIFY:
3993                 skb->tc_index = TC_H_MIN(cl_res.classid);
3994                 break;
3995         case TC_ACT_SHOT:
3996                 mini_qdisc_qstats_cpu_drop(miniq);
3997                 *ret = NET_XMIT_DROP;
3998                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
3999                 return NULL;
4000         case TC_ACT_STOLEN:
4001         case TC_ACT_QUEUED:
4002         case TC_ACT_TRAP:
4003                 *ret = NET_XMIT_SUCCESS;
4004                 consume_skb(skb);
4005                 return NULL;
4006         case TC_ACT_REDIRECT:
4007                 /* No need to push/pop skb's mac_header here on egress! */
4008                 skb_do_redirect(skb);
4009                 *ret = NET_XMIT_SUCCESS;
4010                 return NULL;
4011         default:
4012                 break;
4013         }
4014 #endif /* CONFIG_NET_CLS_ACT */
4015
4016         return skb;
4017 }
4018
4019 static struct netdev_queue *
4020 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
4021 {
4022         int qm = skb_get_queue_mapping(skb);
4023
4024         return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
4025 }
4026
4027 static bool netdev_xmit_txqueue_skipped(void)
4028 {
4029         return __this_cpu_read(softnet_data.xmit.skip_txqueue);
4030 }
4031
4032 void netdev_xmit_skip_txqueue(bool skip)
4033 {
4034         __this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
4035 }
4036 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4037 #endif /* CONFIG_NET_EGRESS */
4038
4039 #ifdef CONFIG_XPS
4040 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4041                                struct xps_dev_maps *dev_maps, unsigned int tci)
4042 {
4043         int tc = netdev_get_prio_tc_map(dev, skb->priority);
4044         struct xps_map *map;
4045         int queue_index = -1;
4046
4047         if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4048                 return queue_index;
4049
4050         tci *= dev_maps->num_tc;
4051         tci += tc;
4052
4053         map = rcu_dereference(dev_maps->attr_map[tci]);
4054         if (map) {
4055                 if (map->len == 1)
4056                         queue_index = map->queues[0];
4057                 else
4058                         queue_index = map->queues[reciprocal_scale(
4059                                                 skb_get_hash(skb), map->len)];
4060                 if (unlikely(queue_index >= dev->real_num_tx_queues))
4061                         queue_index = -1;
4062         }
4063         return queue_index;
4064 }
4065 #endif
4066
4067 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4068                          struct sk_buff *skb)
4069 {
4070 #ifdef CONFIG_XPS
4071         struct xps_dev_maps *dev_maps;
4072         struct sock *sk = skb->sk;
4073         int queue_index = -1;
4074
4075         if (!static_key_false(&xps_needed))
4076                 return -1;
4077
4078         rcu_read_lock();
4079         if (!static_key_false(&xps_rxqs_needed))
4080                 goto get_cpus_map;
4081
4082         dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4083         if (dev_maps) {
4084                 int tci = sk_rx_queue_get(sk);
4085
4086                 if (tci >= 0)
4087                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4088                                                           tci);
4089         }
4090
4091 get_cpus_map:
4092         if (queue_index < 0) {
4093                 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4094                 if (dev_maps) {
4095                         unsigned int tci = skb->sender_cpu - 1;
4096
4097                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4098                                                           tci);
4099                 }
4100         }
4101         rcu_read_unlock();
4102
4103         return queue_index;
4104 #else
4105         return -1;
4106 #endif
4107 }
4108
4109 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4110                      struct net_device *sb_dev)
4111 {
4112         return 0;
4113 }
4114 EXPORT_SYMBOL(dev_pick_tx_zero);
4115
4116 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
4117                        struct net_device *sb_dev)
4118 {
4119         return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4120 }
4121 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4122
4123 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4124                      struct net_device *sb_dev)
4125 {
4126         struct sock *sk = skb->sk;
4127         int queue_index = sk_tx_queue_get(sk);
4128
4129         sb_dev = sb_dev ? : dev;
4130
4131         if (queue_index < 0 || skb->ooo_okay ||
4132             queue_index >= dev->real_num_tx_queues) {
4133                 int new_index = get_xps_queue(dev, sb_dev, skb);
4134
4135                 if (new_index < 0)
4136                         new_index = skb_tx_hash(dev, sb_dev, skb);
4137
4138                 if (queue_index != new_index && sk &&
4139                     sk_fullsock(sk) &&
4140                     rcu_access_pointer(sk->sk_dst_cache))
4141                         sk_tx_queue_set(sk, new_index);
4142
4143                 queue_index = new_index;
4144         }
4145
4146         return queue_index;
4147 }
4148 EXPORT_SYMBOL(netdev_pick_tx);
4149
4150 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4151                                          struct sk_buff *skb,
4152                                          struct net_device *sb_dev)
4153 {
4154         int queue_index = 0;
4155
4156 #ifdef CONFIG_XPS
4157         u32 sender_cpu = skb->sender_cpu - 1;
4158
4159         if (sender_cpu >= (u32)NR_CPUS)
4160                 skb->sender_cpu = raw_smp_processor_id() + 1;
4161 #endif
4162
4163         if (dev->real_num_tx_queues != 1) {
4164                 const struct net_device_ops *ops = dev->netdev_ops;
4165
4166                 if (ops->ndo_select_queue)
4167                         queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4168                 else
4169                         queue_index = netdev_pick_tx(dev, skb, sb_dev);
4170
4171                 queue_index = netdev_cap_txqueue(dev, queue_index);
4172         }
4173
4174         skb_set_queue_mapping(skb, queue_index);
4175         return netdev_get_tx_queue(dev, queue_index);
4176 }
4177
4178 /**
4179  * __dev_queue_xmit() - transmit a buffer
4180  * @skb:        buffer to transmit
4181  * @sb_dev:     suboordinate device used for L2 forwarding offload
4182  *
4183  * Queue a buffer for transmission to a network device. The caller must
4184  * have set the device and priority and built the buffer before calling
4185  * this function. The function can be called from an interrupt.
4186  *
4187  * When calling this method, interrupts MUST be enabled. This is because
4188  * the BH enable code must have IRQs enabled so that it will not deadlock.
4189  *
4190  * Regardless of the return value, the skb is consumed, so it is currently
4191  * difficult to retry a send to this method. (You can bump the ref count
4192  * before sending to hold a reference for retry if you are careful.)
4193  *
4194  * Return:
4195  * * 0                          - buffer successfully transmitted
4196  * * positive qdisc return code - NET_XMIT_DROP etc.
4197  * * negative errno             - other errors
4198  */
4199 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4200 {
4201         struct net_device *dev = skb->dev;
4202         struct netdev_queue *txq = NULL;
4203         struct Qdisc *q;
4204         int rc = -ENOMEM;
4205         bool again = false;
4206
4207         skb_reset_mac_header(skb);
4208         skb_assert_len(skb);
4209
4210         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
4211                 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
4212
4213         /* Disable soft irqs for various locks below. Also
4214          * stops preemption for RCU.
4215          */
4216         rcu_read_lock_bh();
4217
4218         skb_update_prio(skb);
4219
4220         qdisc_pkt_len_init(skb);
4221 #ifdef CONFIG_NET_CLS_ACT
4222         skb->tc_at_ingress = 0;
4223 #endif
4224 #ifdef CONFIG_NET_EGRESS
4225         if (static_branch_unlikely(&egress_needed_key)) {
4226                 if (nf_hook_egress_active()) {
4227                         skb = nf_hook_egress(skb, &rc, dev);
4228                         if (!skb)
4229                                 goto out;
4230                 }
4231
4232                 netdev_xmit_skip_txqueue(false);
4233
4234                 nf_skip_egress(skb, true);
4235                 skb = sch_handle_egress(skb, &rc, dev);
4236                 if (!skb)
4237                         goto out;
4238                 nf_skip_egress(skb, false);
4239
4240                 if (netdev_xmit_txqueue_skipped())
4241                         txq = netdev_tx_queue_mapping(dev, skb);
4242         }
4243 #endif
4244         /* If device/qdisc don't need skb->dst, release it right now while
4245          * its hot in this cpu cache.
4246          */
4247         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4248                 skb_dst_drop(skb);
4249         else
4250                 skb_dst_force(skb);
4251
4252         if (!txq)
4253                 txq = netdev_core_pick_tx(dev, skb, sb_dev);
4254
4255         q = rcu_dereference_bh(txq->qdisc);
4256
4257         trace_net_dev_queue(skb);
4258         if (q->enqueue) {
4259                 rc = __dev_xmit_skb(skb, q, dev, txq);
4260                 goto out;
4261         }
4262
4263         /* The device has no queue. Common case for software devices:
4264          * loopback, all the sorts of tunnels...
4265
4266          * Really, it is unlikely that netif_tx_lock protection is necessary
4267          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
4268          * counters.)
4269          * However, it is possible, that they rely on protection
4270          * made by us here.
4271
4272          * Check this and shot the lock. It is not prone from deadlocks.
4273          *Either shot noqueue qdisc, it is even simpler 8)
4274          */
4275         if (dev->flags & IFF_UP) {
4276                 int cpu = smp_processor_id(); /* ok because BHs are off */
4277
4278                 /* Other cpus might concurrently change txq->xmit_lock_owner
4279                  * to -1 or to their cpu id, but not to our id.
4280                  */
4281                 if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
4282                         if (dev_xmit_recursion())
4283                                 goto recursion_alert;
4284
4285                         skb = validate_xmit_skb(skb, dev, &again);
4286                         if (!skb)
4287                                 goto out;
4288
4289                         HARD_TX_LOCK(dev, txq, cpu);
4290
4291                         if (!netif_xmit_stopped(txq)) {
4292                                 dev_xmit_recursion_inc();
4293                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
4294                                 dev_xmit_recursion_dec();
4295                                 if (dev_xmit_complete(rc)) {
4296                                         HARD_TX_UNLOCK(dev, txq);
4297                                         goto out;
4298                                 }
4299                         }
4300                         HARD_TX_UNLOCK(dev, txq);
4301                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4302                                              dev->name);
4303                 } else {
4304                         /* Recursion is detected! It is possible,
4305                          * unfortunately
4306                          */
4307 recursion_alert:
4308                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4309                                              dev->name);
4310                 }
4311         }
4312
4313         rc = -ENETDOWN;
4314         rcu_read_unlock_bh();
4315
4316         dev_core_stats_tx_dropped_inc(dev);
4317         kfree_skb_list(skb);
4318         return rc;
4319 out:
4320         rcu_read_unlock_bh();
4321         return rc;
4322 }
4323 EXPORT_SYMBOL(__dev_queue_xmit);
4324
4325 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
4326 {
4327         struct net_device *dev = skb->dev;
4328         struct sk_buff *orig_skb = skb;
4329         struct netdev_queue *txq;
4330         int ret = NETDEV_TX_BUSY;
4331         bool again = false;
4332
4333         if (unlikely(!netif_running(dev) ||
4334                      !netif_carrier_ok(dev)))
4335                 goto drop;
4336
4337         skb = validate_xmit_skb_list(skb, dev, &again);
4338         if (skb != orig_skb)
4339                 goto drop;
4340
4341         skb_set_queue_mapping(skb, queue_id);
4342         txq = skb_get_tx_queue(dev, skb);
4343
4344         local_bh_disable();
4345
4346         dev_xmit_recursion_inc();
4347         HARD_TX_LOCK(dev, txq, smp_processor_id());
4348         if (!netif_xmit_frozen_or_drv_stopped(txq))
4349                 ret = netdev_start_xmit(skb, dev, txq, false);
4350         HARD_TX_UNLOCK(dev, txq);
4351         dev_xmit_recursion_dec();
4352
4353         local_bh_enable();
4354         return ret;
4355 drop:
4356         dev_core_stats_tx_dropped_inc(dev);
4357         kfree_skb_list(skb);
4358         return NET_XMIT_DROP;
4359 }
4360 EXPORT_SYMBOL(__dev_direct_xmit);
4361
4362 /*************************************************************************
4363  *                      Receiver routines
4364  *************************************************************************/
4365
4366 int netdev_max_backlog __read_mostly = 1000;
4367 EXPORT_SYMBOL(netdev_max_backlog);
4368
4369 int netdev_tstamp_prequeue __read_mostly = 1;
4370 unsigned int sysctl_skb_defer_max __read_mostly = 64;
4371 int netdev_budget __read_mostly = 300;
4372 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4373 unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4374 int weight_p __read_mostly = 64;           /* old backlog weight */
4375 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
4376 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
4377 int dev_rx_weight __read_mostly = 64;
4378 int dev_tx_weight __read_mostly = 64;
4379
4380 /* Called with irq disabled */
4381 static inline void ____napi_schedule(struct softnet_data *sd,
4382                                      struct napi_struct *napi)
4383 {
4384         struct task_struct *thread;
4385
4386         lockdep_assert_irqs_disabled();
4387
4388         if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4389                 /* Paired with smp_mb__before_atomic() in
4390                  * napi_enable()/dev_set_threaded().
4391                  * Use READ_ONCE() to guarantee a complete
4392                  * read on napi->thread. Only call
4393                  * wake_up_process() when it's not NULL.
4394                  */
4395                 thread = READ_ONCE(napi->thread);
4396                 if (thread) {
4397                         /* Avoid doing set_bit() if the thread is in
4398                          * INTERRUPTIBLE state, cause napi_thread_wait()
4399                          * makes sure to proceed with napi polling
4400                          * if the thread is explicitly woken from here.
4401                          */
4402                         if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
4403                                 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4404                         wake_up_process(thread);
4405                         return;
4406                 }
4407         }
4408
4409         list_add_tail(&napi->poll_list, &sd->poll_list);
4410         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4411 }
4412
4413 #ifdef CONFIG_RPS
4414
4415 /* One global table that all flow-based protocols share. */
4416 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4417 EXPORT_SYMBOL(rps_sock_flow_table);
4418 u32 rps_cpu_mask __read_mostly;
4419 EXPORT_SYMBOL(rps_cpu_mask);
4420
4421 struct static_key_false rps_needed __read_mostly;
4422 EXPORT_SYMBOL(rps_needed);
4423 struct static_key_false rfs_needed __read_mostly;
4424 EXPORT_SYMBOL(rfs_needed);
4425
4426 static struct rps_dev_flow *
4427 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4428             struct rps_dev_flow *rflow, u16 next_cpu)
4429 {
4430         if (next_cpu < nr_cpu_ids) {
4431 #ifdef CONFIG_RFS_ACCEL
4432                 struct netdev_rx_queue *rxqueue;
4433                 struct rps_dev_flow_table *flow_table;
4434                 struct rps_dev_flow *old_rflow;
4435                 u32 flow_id;
4436                 u16 rxq_index;
4437                 int rc;
4438
4439                 /* Should we steer this flow to a different hardware queue? */
4440                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4441                     !(dev->features & NETIF_F_NTUPLE))
4442                         goto out;
4443                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4444                 if (rxq_index == skb_get_rx_queue(skb))
4445                         goto out;
4446
4447                 rxqueue = dev->_rx + rxq_index;
4448                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4449                 if (!flow_table)
4450                         goto out;
4451                 flow_id = skb_get_hash(skb) & flow_table->mask;
4452                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4453                                                         rxq_index, flow_id);
4454                 if (rc < 0)
4455                         goto out;
4456                 old_rflow = rflow;
4457                 rflow = &flow_table->flows[flow_id];
4458                 rflow->filter = rc;
4459                 if (old_rflow->filter == rflow->filter)
4460                         old_rflow->filter = RPS_NO_FILTER;
4461         out:
4462 #endif
4463                 rflow->last_qtail =
4464                         per_cpu(softnet_data, next_cpu).input_queue_head;
4465         }
4466
4467         rflow->cpu = next_cpu;
4468         return rflow;
4469 }
4470
4471 /*
4472  * get_rps_cpu is called from netif_receive_skb and returns the target
4473  * CPU from the RPS map of the receiving queue for a given skb.
4474  * rcu_read_lock must be held on entry.
4475  */
4476 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4477                        struct rps_dev_flow **rflowp)
4478 {
4479         const struct rps_sock_flow_table *sock_flow_table;
4480         struct netdev_rx_queue *rxqueue = dev->_rx;
4481         struct rps_dev_flow_table *flow_table;
4482         struct rps_map *map;
4483         int cpu = -1;
4484         u32 tcpu;
4485         u32 hash;
4486
4487         if (skb_rx_queue_recorded(skb)) {
4488                 u16 index = skb_get_rx_queue(skb);
4489
4490                 if (unlikely(index >= dev->real_num_rx_queues)) {
4491                         WARN_ONCE(dev->real_num_rx_queues > 1,
4492                                   "%s received packet on queue %u, but number "
4493                                   "of RX queues is %u\n",
4494                                   dev->name, index, dev->real_num_rx_queues);
4495                         goto done;
4496                 }
4497                 rxqueue += index;
4498         }
4499
4500         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4501
4502         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4503         map = rcu_dereference(rxqueue->rps_map);
4504         if (!flow_table && !map)
4505                 goto done;
4506
4507         skb_reset_network_header(skb);
4508         hash = skb_get_hash(skb);
4509         if (!hash)
4510                 goto done;
4511
4512         sock_flow_table = rcu_dereference(rps_sock_flow_table);
4513         if (flow_table && sock_flow_table) {
4514                 struct rps_dev_flow *rflow;
4515                 u32 next_cpu;
4516                 u32 ident;
4517
4518                 /* First check into global flow table if there is a match.
4519                  * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4520                  */
4521                 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4522                 if ((ident ^ hash) & ~rps_cpu_mask)
4523                         goto try_rps;
4524
4525                 next_cpu = ident & rps_cpu_mask;
4526
4527                 /* OK, now we know there is a match,
4528                  * we can look at the local (per receive queue) flow table
4529                  */
4530                 rflow = &flow_table->flows[hash & flow_table->mask];
4531                 tcpu = rflow->cpu;
4532
4533                 /*
4534                  * If the desired CPU (where last recvmsg was done) is
4535                  * different from current CPU (one in the rx-queue flow
4536                  * table entry), switch if one of the following holds:
4537                  *   - Current CPU is unset (>= nr_cpu_ids).
4538                  *   - Current CPU is offline.
4539                  *   - The current CPU's queue tail has advanced beyond the
4540                  *     last packet that was enqueued using this table entry.
4541                  *     This guarantees that all previous packets for the flow
4542                  *     have been dequeued, thus preserving in order delivery.
4543                  */
4544                 if (unlikely(tcpu != next_cpu) &&
4545                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4546                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4547                       rflow->last_qtail)) >= 0)) {
4548                         tcpu = next_cpu;
4549                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4550                 }
4551
4552                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4553                         *rflowp = rflow;
4554                         cpu = tcpu;
4555                         goto done;
4556                 }
4557         }
4558
4559 try_rps:
4560
4561         if (map) {
4562                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4563                 if (cpu_online(tcpu)) {
4564                         cpu = tcpu;
4565                         goto done;
4566                 }
4567         }
4568
4569 done:
4570         return cpu;
4571 }
4572
4573 #ifdef CONFIG_RFS_ACCEL
4574
4575 /**
4576  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4577  * @dev: Device on which the filter was set
4578  * @rxq_index: RX queue index
4579  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4580  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4581  *
4582  * Drivers that implement ndo_rx_flow_steer() should periodically call
4583  * this function for each installed filter and remove the filters for
4584  * which it returns %true.
4585  */
4586 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4587                          u32 flow_id, u16 filter_id)
4588 {
4589         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4590         struct rps_dev_flow_table *flow_table;
4591         struct rps_dev_flow *rflow;
4592         bool expire = true;
4593         unsigned int cpu;
4594
4595         rcu_read_lock();
4596         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4597         if (flow_table && flow_id <= flow_table->mask) {
4598                 rflow = &flow_table->flows[flow_id];
4599                 cpu = READ_ONCE(rflow->cpu);
4600                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4601                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4602                            rflow->last_qtail) <
4603                      (int)(10 * flow_table->mask)))
4604                         expire = false;
4605         }
4606         rcu_read_unlock();
4607         return expire;
4608 }
4609 EXPORT_SYMBOL(rps_may_expire_flow);
4610
4611 #endif /* CONFIG_RFS_ACCEL */
4612
4613 /* Called from hardirq (IPI) context */
4614 static void rps_trigger_softirq(void *data)
4615 {
4616         struct softnet_data *sd = data;
4617
4618         ____napi_schedule(sd, &sd->backlog);
4619         sd->received_rps++;
4620 }
4621
4622 #endif /* CONFIG_RPS */
4623
4624 /* Called from hardirq (IPI) context */
4625 static void trigger_rx_softirq(void *data)
4626 {
4627         struct softnet_data *sd = data;
4628
4629         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4630         smp_store_release(&sd->defer_ipi_scheduled, 0);
4631 }
4632
4633 /*
4634  * Check if this softnet_data structure is another cpu one
4635  * If yes, queue it to our IPI list and return 1
4636  * If no, return 0
4637  */
4638 static int napi_schedule_rps(struct softnet_data *sd)
4639 {
4640         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4641
4642 #ifdef CONFIG_RPS
4643         if (sd != mysd) {
4644                 sd->rps_ipi_next = mysd->rps_ipi_list;
4645                 mysd->rps_ipi_list = sd;
4646
4647                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4648                 return 1;
4649         }
4650 #endif /* CONFIG_RPS */
4651         __napi_schedule_irqoff(&mysd->backlog);
4652         return 0;
4653 }
4654
4655 #ifdef CONFIG_NET_FLOW_LIMIT
4656 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4657 #endif
4658
4659 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4660 {
4661 #ifdef CONFIG_NET_FLOW_LIMIT
4662         struct sd_flow_limit *fl;
4663         struct softnet_data *sd;
4664         unsigned int old_flow, new_flow;
4665
4666         if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4667                 return false;
4668
4669         sd = this_cpu_ptr(&softnet_data);
4670
4671         rcu_read_lock();
4672         fl = rcu_dereference(sd->flow_limit);
4673         if (fl) {
4674                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4675                 old_flow = fl->history[fl->history_head];
4676                 fl->history[fl->history_head] = new_flow;
4677
4678                 fl->history_head++;
4679                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4680
4681                 if (likely(fl->buckets[old_flow]))
4682                         fl->buckets[old_flow]--;
4683
4684                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4685                         fl->count++;
4686                         rcu_read_unlock();
4687                         return true;
4688                 }
4689         }
4690         rcu_read_unlock();
4691 #endif
4692         return false;
4693 }
4694
4695 /*
4696  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4697  * queue (may be a remote CPU queue).
4698  */
4699 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4700                               unsigned int *qtail)
4701 {
4702         enum skb_drop_reason reason;
4703         struct softnet_data *sd;
4704         unsigned long flags;
4705         unsigned int qlen;
4706
4707         reason = SKB_DROP_REASON_NOT_SPECIFIED;
4708         sd = &per_cpu(softnet_data, cpu);
4709
4710         rps_lock_irqsave(sd, &flags);
4711         if (!netif_running(skb->dev))
4712                 goto drop;
4713         qlen = skb_queue_len(&sd->input_pkt_queue);
4714         if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4715                 if (qlen) {
4716 enqueue:
4717                         __skb_queue_tail(&sd->input_pkt_queue, skb);
4718                         input_queue_tail_incr_save(sd, qtail);
4719                         rps_unlock_irq_restore(sd, &flags);
4720                         return NET_RX_SUCCESS;
4721                 }
4722
4723                 /* Schedule NAPI for backlog device
4724                  * We can use non atomic operation since we own the queue lock
4725                  */
4726                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
4727                         napi_schedule_rps(sd);
4728                 goto enqueue;
4729         }
4730         reason = SKB_DROP_REASON_CPU_BACKLOG;
4731
4732 drop:
4733         sd->dropped++;
4734         rps_unlock_irq_restore(sd, &flags);
4735
4736         dev_core_stats_rx_dropped_inc(skb->dev);
4737         kfree_skb_reason(skb, reason);
4738         return NET_RX_DROP;
4739 }
4740
4741 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4742 {
4743         struct net_device *dev = skb->dev;
4744         struct netdev_rx_queue *rxqueue;
4745
4746         rxqueue = dev->_rx;
4747
4748         if (skb_rx_queue_recorded(skb)) {
4749                 u16 index = skb_get_rx_queue(skb);
4750
4751                 if (unlikely(index >= dev->real_num_rx_queues)) {
4752                         WARN_ONCE(dev->real_num_rx_queues > 1,
4753                                   "%s received packet on queue %u, but number "
4754                                   "of RX queues is %u\n",
4755                                   dev->name, index, dev->real_num_rx_queues);
4756
4757                         return rxqueue; /* Return first rxqueue */
4758                 }
4759                 rxqueue += index;
4760         }
4761         return rxqueue;
4762 }
4763
4764 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4765                              struct bpf_prog *xdp_prog)
4766 {
4767         void *orig_data, *orig_data_end, *hard_start;
4768         struct netdev_rx_queue *rxqueue;
4769         bool orig_bcast, orig_host;
4770         u32 mac_len, frame_sz;
4771         __be16 orig_eth_type;
4772         struct ethhdr *eth;
4773         u32 metalen, act;
4774         int off;
4775
4776         /* The XDP program wants to see the packet starting at the MAC
4777          * header.
4778          */
4779         mac_len = skb->data - skb_mac_header(skb);
4780         hard_start = skb->data - skb_headroom(skb);
4781
4782         /* SKB "head" area always have tailroom for skb_shared_info */
4783         frame_sz = (void *)skb_end_pointer(skb) - hard_start;
4784         frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4785
4786         rxqueue = netif_get_rxqueue(skb);
4787         xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4788         xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4789                          skb_headlen(skb) + mac_len, true);
4790
4791         orig_data_end = xdp->data_end;
4792         orig_data = xdp->data;
4793         eth = (struct ethhdr *)xdp->data;
4794         orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
4795         orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4796         orig_eth_type = eth->h_proto;
4797
4798         act = bpf_prog_run_xdp(xdp_prog, xdp);
4799
4800         /* check if bpf_xdp_adjust_head was used */
4801         off = xdp->data - orig_data;
4802         if (off) {
4803                 if (off > 0)
4804                         __skb_pull(skb, off);
4805                 else if (off < 0)
4806                         __skb_push(skb, -off);
4807
4808                 skb->mac_header += off;
4809                 skb_reset_network_header(skb);
4810         }
4811
4812         /* check if bpf_xdp_adjust_tail was used */
4813         off = xdp->data_end - orig_data_end;
4814         if (off != 0) {
4815                 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4816                 skb->len += off; /* positive on grow, negative on shrink */
4817         }
4818
4819         /* check if XDP changed eth hdr such SKB needs update */
4820         eth = (struct ethhdr *)xdp->data;
4821         if ((orig_eth_type != eth->h_proto) ||
4822             (orig_host != ether_addr_equal_64bits(eth->h_dest,
4823                                                   skb->dev->dev_addr)) ||
4824             (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4825                 __skb_push(skb, ETH_HLEN);
4826                 skb->pkt_type = PACKET_HOST;
4827                 skb->protocol = eth_type_trans(skb, skb->dev);
4828         }
4829
4830         /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4831          * before calling us again on redirect path. We do not call do_redirect
4832          * as we leave that up to the caller.
4833          *
4834          * Caller is responsible for managing lifetime of skb (i.e. calling
4835          * kfree_skb in response to actions it cannot handle/XDP_DROP).
4836          */
4837         switch (act) {
4838         case XDP_REDIRECT:
4839         case XDP_TX:
4840                 __skb_push(skb, mac_len);
4841                 break;
4842         case XDP_PASS:
4843                 metalen = xdp->data - xdp->data_meta;
4844                 if (metalen)
4845                         skb_metadata_set(skb, metalen);
4846                 break;
4847         }
4848
4849         return act;
4850 }
4851
4852 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4853                                      struct xdp_buff *xdp,
4854                                      struct bpf_prog *xdp_prog)
4855 {
4856         u32 act = XDP_DROP;
4857
4858         /* Reinjected packets coming from act_mirred or similar should
4859          * not get XDP generic processing.
4860          */
4861         if (skb_is_redirected(skb))
4862                 return XDP_PASS;
4863
4864         /* XDP packets must be linear and must have sufficient headroom
4865          * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4866          * native XDP provides, thus we need to do it here as well.
4867          */
4868         if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4869             skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4870                 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4871                 int troom = skb->tail + skb->data_len - skb->end;
4872
4873                 /* In case we have to go down the path and also linearize,
4874                  * then lets do the pskb_expand_head() work just once here.
4875                  */
4876                 if (pskb_expand_head(skb,
4877                                      hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4878                                      troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4879                         goto do_drop;
4880                 if (skb_linearize(skb))
4881                         goto do_drop;
4882         }
4883
4884         act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4885         switch (act) {
4886         case XDP_REDIRECT:
4887         case XDP_TX:
4888         case XDP_PASS:
4889                 break;
4890         default:
4891                 bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
4892                 fallthrough;
4893         case XDP_ABORTED:
4894                 trace_xdp_exception(skb->dev, xdp_prog, act);
4895                 fallthrough;
4896         case XDP_DROP:
4897         do_drop:
4898                 kfree_skb(skb);
4899                 break;
4900         }
4901
4902         return act;
4903 }
4904
4905 /* When doing generic XDP we have to bypass the qdisc layer and the
4906  * network taps in order to match in-driver-XDP behavior. This also means
4907  * that XDP packets are able to starve other packets going through a qdisc,
4908  * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
4909  * queues, so they do not have this starvation issue.
4910  */
4911 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4912 {
4913         struct net_device *dev = skb->dev;
4914         struct netdev_queue *txq;
4915         bool free_skb = true;
4916         int cpu, rc;
4917
4918         txq = netdev_core_pick_tx(dev, skb, NULL);
4919         cpu = smp_processor_id();
4920         HARD_TX_LOCK(dev, txq, cpu);
4921         if (!netif_xmit_frozen_or_drv_stopped(txq)) {
4922                 rc = netdev_start_xmit(skb, dev, txq, 0);
4923                 if (dev_xmit_complete(rc))
4924                         free_skb = false;
4925         }
4926         HARD_TX_UNLOCK(dev, txq);
4927         if (free_skb) {
4928                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4929                 dev_core_stats_tx_dropped_inc(dev);
4930                 kfree_skb(skb);
4931         }
4932 }
4933
4934 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4935
4936 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4937 {
4938         if (xdp_prog) {
4939                 struct xdp_buff xdp;
4940                 u32 act;
4941                 int err;
4942
4943                 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4944                 if (act != XDP_PASS) {
4945                         switch (act) {
4946                         case XDP_REDIRECT:
4947                                 err = xdp_do_generic_redirect(skb->dev, skb,
4948                                                               &xdp, xdp_prog);
4949                                 if (err)
4950                                         goto out_redir;
4951                                 break;
4952                         case XDP_TX:
4953                                 generic_xdp_tx(skb, xdp_prog);
4954                                 break;
4955                         }
4956                         return XDP_DROP;
4957                 }
4958         }
4959         return XDP_PASS;
4960 out_redir:
4961         kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
4962         return XDP_DROP;
4963 }
4964 EXPORT_SYMBOL_GPL(do_xdp_generic);
4965
4966 static int netif_rx_internal(struct sk_buff *skb)
4967 {
4968         int ret;
4969
4970         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
4971
4972         trace_netif_rx(skb);
4973
4974 #ifdef CONFIG_RPS
4975         if (static_branch_unlikely(&rps_needed)) {
4976                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4977                 int cpu;
4978
4979                 rcu_read_lock();
4980
4981                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4982                 if (cpu < 0)
4983                         cpu = smp_processor_id();
4984
4985                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4986
4987                 rcu_read_unlock();
4988         } else
4989 #endif
4990         {
4991                 unsigned int qtail;
4992
4993                 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
4994         }
4995         return ret;
4996 }
4997
4998 /**
4999  *      __netif_rx      -       Slightly optimized version of netif_rx
5000  *      @skb: buffer to post
5001  *
5002  *      This behaves as netif_rx except that it does not disable bottom halves.
5003  *      As a result this function may only be invoked from the interrupt context
5004  *      (either hard or soft interrupt).
5005  */
5006 int __netif_rx(struct sk_buff *skb)
5007 {
5008         int ret;
5009
5010         lockdep_assert_once(hardirq_count() | softirq_count());
5011
5012         trace_netif_rx_entry(skb);
5013         ret = netif_rx_internal(skb);
5014         trace_netif_rx_exit(ret);
5015         return ret;
5016 }
5017 EXPORT_SYMBOL(__netif_rx);
5018
5019 /**
5020  *      netif_rx        -       post buffer to the network code
5021  *      @skb: buffer to post
5022  *
5023  *      This function receives a packet from a device driver and queues it for
5024  *      the upper (protocol) levels to process via the backlog NAPI device. It
5025  *      always succeeds. The buffer may be dropped during processing for
5026  *      congestion control or by the protocol layers.
5027  *      The network buffer is passed via the backlog NAPI device. Modern NIC
5028  *      driver should use NAPI and GRO.
5029  *      This function can used from interrupt and from process context. The
5030  *      caller from process context must not disable interrupts before invoking
5031  *      this function.
5032  *
5033  *      return values:
5034  *      NET_RX_SUCCESS  (no congestion)
5035  *      NET_RX_DROP     (packet was dropped)
5036  *
5037  */
5038 int netif_rx(struct sk_buff *skb)
5039 {
5040         bool need_bh_off = !(hardirq_count() | softirq_count());
5041         int ret;
5042
5043         if (need_bh_off)
5044                 local_bh_disable();
5045         trace_netif_rx_entry(skb);
5046         ret = netif_rx_internal(skb);
5047         trace_netif_rx_exit(ret);
5048         if (need_bh_off)
5049                 local_bh_enable();
5050         return ret;
5051 }
5052 EXPORT_SYMBOL(netif_rx);
5053
5054 static __latent_entropy void net_tx_action(struct softirq_action *h)
5055 {
5056         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5057
5058         if (sd->completion_queue) {
5059                 struct sk_buff *clist;
5060
5061                 local_irq_disable();
5062                 clist = sd->completion_queue;
5063                 sd->completion_queue = NULL;
5064                 local_irq_enable();
5065
5066                 while (clist) {
5067                         struct sk_buff *skb = clist;
5068
5069                         clist = clist->next;
5070
5071                         WARN_ON(refcount_read(&skb->users));
5072                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5073                                 trace_consume_skb(skb);
5074                         else
5075                                 trace_kfree_skb(skb, net_tx_action,
5076                                                 SKB_DROP_REASON_NOT_SPECIFIED);
5077
5078                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5079                                 __kfree_skb(skb);
5080                         else
5081                                 __kfree_skb_defer(skb);
5082                 }
5083         }
5084
5085         if (sd->output_queue) {
5086                 struct Qdisc *head;
5087
5088                 local_irq_disable();
5089                 head = sd->output_queue;
5090                 sd->output_queue = NULL;
5091                 sd->output_queue_tailp = &sd->output_queue;
5092                 local_irq_enable();
5093
5094                 rcu_read_lock();
5095
5096                 while (head) {
5097                         struct Qdisc *q = head;
5098                         spinlock_t *root_lock = NULL;
5099
5100                         head = head->next_sched;
5101
5102                         /* We need to make sure head->next_sched is read
5103                          * before clearing __QDISC_STATE_SCHED
5104                          */
5105                         smp_mb__before_atomic();
5106
5107                         if (!(q->flags & TCQ_F_NOLOCK)) {
5108                                 root_lock = qdisc_lock(q);
5109                                 spin_lock(root_lock);
5110                         } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5111                                                      &q->state))) {
5112                                 /* There is a synchronize_net() between
5113                                  * STATE_DEACTIVATED flag being set and
5114                                  * qdisc_reset()/some_qdisc_is_busy() in
5115                                  * dev_deactivate(), so we can safely bail out
5116                                  * early here to avoid data race between
5117                                  * qdisc_deactivate() and some_qdisc_is_busy()
5118                                  * for lockless qdisc.
5119                                  */
5120                                 clear_bit(__QDISC_STATE_SCHED, &q->state);
5121                                 continue;
5122                         }
5123
5124                         clear_bit(__QDISC_STATE_SCHED, &q->state);
5125                         qdisc_run(q);
5126                         if (root_lock)
5127                                 spin_unlock(root_lock);
5128                 }
5129
5130                 rcu_read_unlock();
5131         }
5132
5133         xfrm_dev_backlog(sd);
5134 }
5135
5136 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
5137 /* This hook is defined here for ATM LANE */
5138 int (*br_fdb_test_addr_hook)(struct net_device *dev,
5139                              unsigned char *addr) __read_mostly;
5140 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
5141 #endif
5142
5143 static inline struct sk_buff *
5144 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
5145                    struct net_device *orig_dev, bool *another)
5146 {
5147 #ifdef CONFIG_NET_CLS_ACT
5148         struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
5149         struct tcf_result cl_res;
5150
5151         /* If there's at least one ingress present somewhere (so
5152          * we get here via enabled static key), remaining devices
5153          * that are not configured with an ingress qdisc will bail
5154          * out here.
5155          */
5156         if (!miniq)
5157                 return skb;
5158
5159         if (*pt_prev) {
5160                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5161                 *pt_prev = NULL;
5162         }
5163
5164         qdisc_skb_cb(skb)->pkt_len = skb->len;
5165         tc_skb_cb(skb)->mru = 0;
5166         tc_skb_cb(skb)->post_ct = false;
5167         skb->tc_at_ingress = 1;
5168         mini_qdisc_bstats_cpu_update(miniq, skb);
5169
5170         switch (tcf_classify(skb, miniq->block, miniq->filter_list, &cl_res, false)) {
5171         case TC_ACT_OK:
5172         case TC_ACT_RECLASSIFY:
5173                 skb->tc_index = TC_H_MIN(cl_res.classid);
5174                 break;
5175         case TC_ACT_SHOT:
5176                 mini_qdisc_qstats_cpu_drop(miniq);
5177                 kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS);
5178                 *ret = NET_RX_DROP;
5179                 return NULL;
5180         case TC_ACT_STOLEN:
5181         case TC_ACT_QUEUED:
5182         case TC_ACT_TRAP:
5183                 consume_skb(skb);
5184                 *ret = NET_RX_SUCCESS;
5185                 return NULL;
5186         case TC_ACT_REDIRECT:
5187                 /* skb_mac_header check was done by cls/act_bpf, so
5188                  * we can safely push the L2 header back before
5189                  * redirecting to another netdev
5190                  */
5191                 __skb_push(skb, skb->mac_len);
5192                 if (skb_do_redirect(skb) == -EAGAIN) {
5193                         __skb_pull(skb, skb->mac_len);
5194                         *another = true;
5195                         break;
5196                 }
5197                 *ret = NET_RX_SUCCESS;
5198                 return NULL;
5199         case TC_ACT_CONSUMED:
5200                 *ret = NET_RX_SUCCESS;
5201                 return NULL;
5202         default:
5203                 break;
5204         }
5205 #endif /* CONFIG_NET_CLS_ACT */
5206         return skb;
5207 }
5208
5209 /**
5210  *      netdev_is_rx_handler_busy - check if receive handler is registered
5211  *      @dev: device to check
5212  *
5213  *      Check if a receive handler is already registered for a given device.
5214  *      Return true if there one.
5215  *
5216  *      The caller must hold the rtnl_mutex.
5217  */
5218 bool netdev_is_rx_handler_busy(struct net_device *dev)
5219 {
5220         ASSERT_RTNL();
5221         return dev && rtnl_dereference(dev->rx_handler);
5222 }
5223 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5224
5225 /**
5226  *      netdev_rx_handler_register - register receive handler
5227  *      @dev: device to register a handler for
5228  *      @rx_handler: receive handler to register
5229  *      @rx_handler_data: data pointer that is used by rx handler
5230  *
5231  *      Register a receive handler for a device. This handler will then be
5232  *      called from __netif_receive_skb. A negative errno code is returned
5233  *      on a failure.
5234  *
5235  *      The caller must hold the rtnl_mutex.
5236  *
5237  *      For a general description of rx_handler, see enum rx_handler_result.
5238  */
5239 int netdev_rx_handler_register(struct net_device *dev,
5240                                rx_handler_func_t *rx_handler,
5241                                void *rx_handler_data)
5242 {
5243         if (netdev_is_rx_handler_busy(dev))
5244                 return -EBUSY;
5245
5246         if (dev->priv_flags & IFF_NO_RX_HANDLER)
5247                 return -EINVAL;
5248
5249         /* Note: rx_handler_data must be set before rx_handler */
5250         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
5251         rcu_assign_pointer(dev->rx_handler, rx_handler);
5252
5253         return 0;
5254 }
5255 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5256
5257 /**
5258  *      netdev_rx_handler_unregister - unregister receive handler
5259  *      @dev: device to unregister a handler from
5260  *
5261  *      Unregister a receive handler from a device.
5262  *
5263  *      The caller must hold the rtnl_mutex.
5264  */
5265 void netdev_rx_handler_unregister(struct net_device *dev)
5266 {
5267
5268         ASSERT_RTNL();
5269         RCU_INIT_POINTER(dev->rx_handler, NULL);
5270         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5271          * section has a guarantee to see a non NULL rx_handler_data
5272          * as well.
5273          */
5274         synchronize_net();
5275         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
5276 }
5277 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5278
5279 /*
5280  * Limit the use of PFMEMALLOC reserves to those protocols that implement
5281  * the special handling of PFMEMALLOC skbs.
5282  */
5283 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5284 {
5285         switch (skb->protocol) {
5286         case htons(ETH_P_ARP):
5287         case htons(ETH_P_IP):
5288         case htons(ETH_P_IPV6):
5289         case htons(ETH_P_8021Q):
5290         case htons(ETH_P_8021AD):
5291                 return true;
5292         default:
5293                 return false;
5294         }
5295 }
5296
5297 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5298                              int *ret, struct net_device *orig_dev)
5299 {
5300         if (nf_hook_ingress_active(skb)) {
5301                 int ingress_retval;
5302
5303                 if (*pt_prev) {
5304                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
5305                         *pt_prev = NULL;
5306                 }
5307
5308                 rcu_read_lock();
5309                 ingress_retval = nf_hook_ingress(skb);
5310                 rcu_read_unlock();
5311                 return ingress_retval;
5312         }
5313         return 0;
5314 }
5315
5316 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
5317                                     struct packet_type **ppt_prev)
5318 {
5319         struct packet_type *ptype, *pt_prev;
5320         rx_handler_func_t *rx_handler;
5321         struct sk_buff *skb = *pskb;
5322         struct net_device *orig_dev;
5323         bool deliver_exact = false;
5324         int ret = NET_RX_DROP;
5325         __be16 type;
5326
5327         net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5328
5329         trace_netif_receive_skb(skb);
5330
5331         orig_dev = skb->dev;
5332
5333         skb_reset_network_header(skb);
5334         if (!skb_transport_header_was_set(skb))
5335                 skb_reset_transport_header(skb);
5336         skb_reset_mac_len(skb);
5337
5338         pt_prev = NULL;
5339
5340 another_round:
5341         skb->skb_iif = skb->dev->ifindex;
5342
5343         __this_cpu_inc(softnet_data.processed);
5344
5345         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5346                 int ret2;
5347
5348                 migrate_disable();
5349                 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5350                 migrate_enable();
5351
5352                 if (ret2 != XDP_PASS) {
5353                         ret = NET_RX_DROP;
5354                         goto out;
5355                 }
5356         }
5357
5358         if (eth_type_vlan(skb->protocol)) {
5359                 skb = skb_vlan_untag(skb);
5360                 if (unlikely(!skb))
5361                         goto out;
5362         }
5363
5364         if (skb_skip_tc_classify(skb))
5365                 goto skip_classify;
5366
5367         if (pfmemalloc)
5368                 goto skip_taps;
5369
5370         list_for_each_entry_rcu(ptype, &ptype_all, list) {
5371                 if (pt_prev)
5372                         ret = deliver_skb(skb, pt_prev, orig_dev);
5373                 pt_prev = ptype;
5374         }
5375
5376         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5377                 if (pt_prev)
5378                         ret = deliver_skb(skb, pt_prev, orig_dev);
5379                 pt_prev = ptype;
5380         }
5381
5382 skip_taps:
5383 #ifdef CONFIG_NET_INGRESS
5384         if (static_branch_unlikely(&ingress_needed_key)) {
5385                 bool another = false;
5386
5387                 nf_skip_egress(skb, true);
5388                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5389                                          &another);
5390                 if (another)
5391                         goto another_round;
5392                 if (!skb)
5393                         goto out;
5394
5395                 nf_skip_egress(skb, false);
5396                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
5397                         goto out;
5398         }
5399 #endif
5400         skb_reset_redirect(skb);
5401 skip_classify:
5402         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
5403                 goto drop;
5404
5405         if (skb_vlan_tag_present(skb)) {
5406                 if (pt_prev) {
5407                         ret = deliver_skb(skb, pt_prev, orig_dev);
5408                         pt_prev = NULL;
5409                 }
5410                 if (vlan_do_receive(&skb))
5411                         goto another_round;
5412                 else if (unlikely(!skb))
5413                         goto out;
5414         }
5415
5416         rx_handler = rcu_dereference(skb->dev->rx_handler);
5417         if (rx_handler) {
5418                 if (pt_prev) {
5419                         ret = deliver_skb(skb, pt_prev, orig_dev);
5420                         pt_prev = NULL;
5421                 }
5422                 switch (rx_handler(&skb)) {
5423                 case RX_HANDLER_CONSUMED:
5424                         ret = NET_RX_SUCCESS;
5425                         goto out;
5426                 case RX_HANDLER_ANOTHER:
5427                         goto another_round;
5428                 case RX_HANDLER_EXACT:
5429                         deliver_exact = true;
5430                         break;
5431                 case RX_HANDLER_PASS:
5432                         break;
5433                 default:
5434                         BUG();
5435                 }
5436         }
5437
5438         if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
5439 check_vlan_id:
5440                 if (skb_vlan_tag_get_id(skb)) {
5441                         /* Vlan id is non 0 and vlan_do_receive() above couldn't
5442                          * find vlan device.
5443                          */
5444                         skb->pkt_type = PACKET_OTHERHOST;
5445                 } else if (eth_type_vlan(skb->protocol)) {
5446                         /* Outer header is 802.1P with vlan 0, inner header is
5447                          * 802.1Q or 802.1AD and vlan_do_receive() above could
5448                          * not find vlan dev for vlan id 0.
5449                          */
5450                         __vlan_hwaccel_clear_tag(skb);
5451                         skb = skb_vlan_untag(skb);
5452                         if (unlikely(!skb))
5453                                 goto out;
5454                         if (vlan_do_receive(&skb))
5455                                 /* After stripping off 802.1P header with vlan 0
5456                                  * vlan dev is found for inner header.
5457                                  */
5458                                 goto another_round;
5459                         else if (unlikely(!skb))
5460                                 goto out;
5461                         else
5462                                 /* We have stripped outer 802.1P vlan 0 header.
5463                                  * But could not find vlan dev.
5464                                  * check again for vlan id to set OTHERHOST.
5465                                  */
5466                                 goto check_vlan_id;
5467                 }
5468                 /* Note: we might in the future use prio bits
5469                  * and set skb->priority like in vlan_do_receive()
5470                  * For the time being, just ignore Priority Code Point
5471                  */
5472                 __vlan_hwaccel_clear_tag(skb);
5473         }
5474
5475         type = skb->protocol;
5476
5477         /* deliver only exact match when indicated */
5478         if (likely(!deliver_exact)) {
5479                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5480                                        &ptype_base[ntohs(type) &
5481                                                    PTYPE_HASH_MASK]);
5482         }
5483
5484         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5485                                &orig_dev->ptype_specific);
5486
5487         if (unlikely(skb->dev != orig_dev)) {
5488                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5489                                        &skb->dev->ptype_specific);
5490         }
5491
5492         if (pt_prev) {
5493                 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
5494                         goto drop;
5495                 *ppt_prev = pt_prev;
5496         } else {
5497 drop:
5498                 if (!deliver_exact)
5499                         dev_core_stats_rx_dropped_inc(skb->dev);
5500                 else
5501                         dev_core_stats_rx_nohandler_inc(skb->dev);
5502                 kfree_skb_reason(skb, SKB_DROP_REASON_UNHANDLED_PROTO);
5503                 /* Jamal, now you will not able to escape explaining
5504                  * me how you were going to use this. :-)
5505                  */
5506                 ret = NET_RX_DROP;
5507         }
5508
5509 out:
5510         /* The invariant here is that if *ppt_prev is not NULL
5511          * then skb should also be non-NULL.
5512          *
5513          * Apparently *ppt_prev assignment above holds this invariant due to
5514          * skb dereferencing near it.
5515          */
5516         *pskb = skb;
5517         return ret;
5518 }
5519
5520 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5521 {
5522         struct net_device *orig_dev = skb->dev;
5523         struct packet_type *pt_prev = NULL;
5524         int ret;
5525
5526         ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5527         if (pt_prev)
5528                 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5529                                          skb->dev, pt_prev, orig_dev);
5530         return ret;
5531 }
5532
5533 /**
5534  *      netif_receive_skb_core - special purpose version of netif_receive_skb
5535  *      @skb: buffer to process
5536  *
5537  *      More direct receive version of netif_receive_skb().  It should
5538  *      only be used by callers that have a need to skip RPS and Generic XDP.
5539  *      Caller must also take care of handling if ``(page_is_)pfmemalloc``.
5540  *
5541  *      This function may only be called from softirq context and interrupts
5542  *      should be enabled.
5543  *
5544  *      Return values (usually ignored):
5545  *      NET_RX_SUCCESS: no congestion
5546  *      NET_RX_DROP: packet was dropped
5547  */
5548 int netif_receive_skb_core(struct sk_buff *skb)
5549 {
5550         int ret;
5551
5552         rcu_read_lock();
5553         ret = __netif_receive_skb_one_core(skb, false);
5554         rcu_read_unlock();
5555
5556         return ret;
5557 }
5558 EXPORT_SYMBOL(netif_receive_skb_core);
5559
5560 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5561                                                   struct packet_type *pt_prev,
5562                                                   struct net_device *orig_dev)
5563 {
5564         struct sk_buff *skb, *next;
5565
5566         if (!pt_prev)
5567                 return;
5568         if (list_empty(head))
5569                 return;
5570         if (pt_prev->list_func != NULL)
5571                 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5572                                    ip_list_rcv, head, pt_prev, orig_dev);
5573         else
5574                 list_for_each_entry_safe(skb, next, head, list) {
5575                         skb_list_del_init(skb);
5576                         pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
5577                 }
5578 }
5579
5580 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5581 {
5582         /* Fast-path assumptions:
5583          * - There is no RX handler.
5584          * - Only one packet_type matches.
5585          * If either of these fails, we will end up doing some per-packet
5586          * processing in-line, then handling the 'last ptype' for the whole
5587          * sublist.  This can't cause out-of-order delivery to any single ptype,
5588          * because the 'last ptype' must be constant across the sublist, and all
5589          * other ptypes are handled per-packet.
5590          */
5591         /* Current (common) ptype of sublist */
5592         struct packet_type *pt_curr = NULL;
5593         /* Current (common) orig_dev of sublist */
5594         struct net_device *od_curr = NULL;
5595         struct list_head sublist;
5596         struct sk_buff *skb, *next;
5597
5598         INIT_LIST_HEAD(&sublist);
5599         list_for_each_entry_safe(skb, next, head, list) {
5600                 struct net_device *orig_dev = skb->dev;
5601                 struct packet_type *pt_prev = NULL;
5602
5603                 skb_list_del_init(skb);
5604                 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
5605                 if (!pt_prev)
5606                         continue;
5607                 if (pt_curr != pt_prev || od_curr != orig_dev) {
5608                         /* dispatch old sublist */
5609                         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5610                         /* start new sublist */
5611                         INIT_LIST_HEAD(&sublist);
5612                         pt_curr = pt_prev;
5613                         od_curr = orig_dev;
5614                 }
5615                 list_add_tail(&skb->list, &sublist);
5616         }
5617
5618         /* dispatch final sublist */
5619         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5620 }
5621
5622 static int __netif_receive_skb(struct sk_buff *skb)
5623 {
5624         int ret;
5625
5626         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5627                 unsigned int noreclaim_flag;
5628
5629                 /*
5630                  * PFMEMALLOC skbs are special, they should
5631                  * - be delivered to SOCK_MEMALLOC sockets only
5632                  * - stay away from userspace
5633                  * - have bounded memory usage
5634                  *
5635                  * Use PF_MEMALLOC as this saves us from propagating the allocation
5636                  * context down to all allocation sites.
5637                  */
5638                 noreclaim_flag = memalloc_noreclaim_save();
5639                 ret = __netif_receive_skb_one_core(skb, true);
5640                 memalloc_noreclaim_restore(noreclaim_flag);
5641         } else
5642                 ret = __netif_receive_skb_one_core(skb, false);
5643
5644         return ret;
5645 }
5646
5647 static void __netif_receive_skb_list(struct list_head *head)
5648 {
5649         unsigned long noreclaim_flag = 0;
5650         struct sk_buff *skb, *next;
5651         bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5652
5653         list_for_each_entry_safe(skb, next, head, list) {
5654                 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5655                         struct list_head sublist;
5656
5657                         /* Handle the previous sublist */
5658                         list_cut_before(&sublist, head, &skb->list);
5659                         if (!list_empty(&sublist))
5660                                 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5661                         pfmemalloc = !pfmemalloc;
5662                         /* See comments in __netif_receive_skb */
5663                         if (pfmemalloc)
5664                                 noreclaim_flag = memalloc_noreclaim_save();
5665                         else
5666                                 memalloc_noreclaim_restore(noreclaim_flag);
5667                 }
5668         }
5669         /* Handle the remaining sublist */
5670         if (!list_empty(head))
5671                 __netif_receive_skb_list_core(head, pfmemalloc);
5672         /* Restore pflags */
5673         if (pfmemalloc)
5674                 memalloc_noreclaim_restore(noreclaim_flag);
5675 }
5676
5677 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5678 {
5679         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5680         struct bpf_prog *new = xdp->prog;
5681         int ret = 0;
5682
5683         switch (xdp->command) {
5684         case XDP_SETUP_PROG:
5685                 rcu_assign_pointer(dev->xdp_prog, new);
5686                 if (old)
5687                         bpf_prog_put(old);
5688
5689                 if (old && !new) {
5690                         static_branch_dec(&generic_xdp_needed_key);
5691                 } else if (new && !old) {
5692                         static_branch_inc(&generic_xdp_needed_key);
5693                         dev_disable_lro(dev);
5694                         dev_disable_gro_hw(dev);
5695                 }
5696                 break;
5697
5698         default:
5699                 ret = -EINVAL;
5700                 break;
5701         }
5702
5703         return ret;
5704 }
5705
5706 static int netif_receive_skb_internal(struct sk_buff *skb)
5707 {
5708         int ret;
5709
5710         net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5711
5712         if (skb_defer_rx_timestamp(skb))
5713                 return NET_RX_SUCCESS;
5714
5715         rcu_read_lock();
5716 #ifdef CONFIG_RPS
5717         if (static_branch_unlikely(&rps_needed)) {
5718                 struct rps_dev_flow voidflow, *rflow = &voidflow;
5719                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5720
5721                 if (cpu >= 0) {
5722                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5723                         rcu_read_unlock();
5724                         return ret;
5725                 }
5726         }
5727 #endif
5728         ret = __netif_receive_skb(skb);
5729         rcu_read_unlock();
5730         return ret;
5731 }
5732
5733 void netif_receive_skb_list_internal(struct list_head *head)
5734 {
5735         struct sk_buff *skb, *next;
5736         struct list_head sublist;
5737
5738         INIT_LIST_HEAD(&sublist);
5739         list_for_each_entry_safe(skb, next, head, list) {
5740                 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5741                 skb_list_del_init(skb);
5742                 if (!skb_defer_rx_timestamp(skb))
5743                         list_add_tail(&skb->list, &sublist);
5744         }
5745         list_splice_init(&sublist, head);
5746
5747         rcu_read_lock();
5748 #ifdef CONFIG_RPS
5749         if (static_branch_unlikely(&rps_needed)) {
5750                 list_for_each_entry_safe(skb, next, head, list) {
5751                         struct rps_dev_flow voidflow, *rflow = &voidflow;
5752                         int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5753
5754                         if (cpu >= 0) {
5755                                 /* Will be handled, remove from list */
5756                                 skb_list_del_init(skb);
5757                                 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5758                         }
5759                 }
5760         }
5761 #endif
5762         __netif_receive_skb_list(head);
5763         rcu_read_unlock();
5764 }
5765
5766 /**
5767  *      netif_receive_skb - process receive buffer from network
5768  *      @skb: buffer to process
5769  *
5770  *      netif_receive_skb() is the main receive data processing function.
5771  *      It always succeeds. The buffer may be dropped during processing
5772  *      for congestion control or by the protocol layers.
5773  *
5774  *      This function may only be called from softirq context and interrupts
5775  *      should be enabled.
5776  *
5777  *      Return values (usually ignored):
5778  *      NET_RX_SUCCESS: no congestion
5779  *      NET_RX_DROP: packet was dropped
5780  */
5781 int netif_receive_skb(struct sk_buff *skb)
5782 {
5783         int ret;
5784
5785         trace_netif_receive_skb_entry(skb);
5786
5787         ret = netif_receive_skb_internal(skb);
5788         trace_netif_receive_skb_exit(ret);
5789
5790         return ret;
5791 }
5792 EXPORT_SYMBOL(netif_receive_skb);
5793
5794 /**
5795  *      netif_receive_skb_list - process many receive buffers from network
5796  *      @head: list of skbs to process.
5797  *
5798  *      Since return value of netif_receive_skb() is normally ignored, and
5799  *      wouldn't be meaningful for a list, this function returns void.
5800  *
5801  *      This function may only be called from softirq context and interrupts
5802  *      should be enabled.
5803  */
5804 void netif_receive_skb_list(struct list_head *head)
5805 {
5806         struct sk_buff *skb;
5807
5808         if (list_empty(head))
5809                 return;
5810         if (trace_netif_receive_skb_list_entry_enabled()) {
5811                 list_for_each_entry(skb, head, list)
5812                         trace_netif_receive_skb_list_entry(skb);
5813         }
5814         netif_receive_skb_list_internal(head);
5815         trace_netif_receive_skb_list_exit(0);
5816 }
5817 EXPORT_SYMBOL(netif_receive_skb_list);
5818
5819 static DEFINE_PER_CPU(struct work_struct, flush_works);
5820
5821 /* Network device is going away, flush any packets still pending */
5822 static void flush_backlog(struct work_struct *work)
5823 {
5824         struct sk_buff *skb, *tmp;
5825         struct softnet_data *sd;
5826
5827         local_bh_disable();
5828         sd = this_cpu_ptr(&softnet_data);
5829
5830         rps_lock_irq_disable(sd);
5831         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5832                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5833                         __skb_unlink(skb, &sd->input_pkt_queue);
5834                         dev_kfree_skb_irq(skb);
5835                         input_queue_head_incr(sd);
5836                 }
5837         }
5838         rps_unlock_irq_enable(sd);
5839
5840         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5841                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5842                         __skb_unlink(skb, &sd->process_queue);
5843                         kfree_skb(skb);
5844                         input_queue_head_incr(sd);
5845                 }
5846         }
5847         local_bh_enable();
5848 }
5849
5850 static bool flush_required(int cpu)
5851 {
5852 #if IS_ENABLED(CONFIG_RPS)
5853         struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5854         bool do_flush;
5855
5856         rps_lock_irq_disable(sd);
5857
5858         /* as insertion into process_queue happens with the rps lock held,
5859          * process_queue access may race only with dequeue
5860          */
5861         do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5862                    !skb_queue_empty_lockless(&sd->process_queue);
5863         rps_unlock_irq_enable(sd);
5864
5865         return do_flush;
5866 #endif
5867         /* without RPS we can't safely check input_pkt_queue: during a
5868          * concurrent remote skb_queue_splice() we can detect as empty both
5869          * input_pkt_queue and process_queue even if the latter could end-up
5870          * containing a lot of packets.
5871          */
5872         return true;
5873 }
5874
5875 static void flush_all_backlogs(void)
5876 {
5877         static cpumask_t flush_cpus;
5878         unsigned int cpu;
5879
5880         /* since we are under rtnl lock protection we can use static data
5881          * for the cpumask and avoid allocating on stack the possibly
5882          * large mask
5883          */
5884         ASSERT_RTNL();
5885
5886         cpus_read_lock();
5887
5888         cpumask_clear(&flush_cpus);
5889         for_each_online_cpu(cpu) {
5890                 if (flush_required(cpu)) {
5891                         queue_work_on(cpu, system_highpri_wq,
5892                                       per_cpu_ptr(&flush_works, cpu));
5893                         cpumask_set_cpu(cpu, &flush_cpus);
5894                 }
5895         }
5896
5897         /* we can have in flight packet[s] on the cpus we are not flushing,
5898          * synchronize_net() in unregister_netdevice_many() will take care of
5899          * them
5900          */
5901         for_each_cpu(cpu, &flush_cpus)
5902                 flush_work(per_cpu_ptr(&flush_works, cpu));
5903
5904         cpus_read_unlock();
5905 }
5906
5907 static void net_rps_send_ipi(struct softnet_data *remsd)
5908 {
5909 #ifdef CONFIG_RPS
5910         while (remsd) {
5911                 struct softnet_data *next = remsd->rps_ipi_next;
5912
5913                 if (cpu_online(remsd->cpu))
5914                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5915                 remsd = next;
5916         }
5917 #endif
5918 }
5919
5920 /*
5921  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5922  * Note: called with local irq disabled, but exits with local irq enabled.
5923  */
5924 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5925 {
5926 #ifdef CONFIG_RPS
5927         struct softnet_data *remsd = sd->rps_ipi_list;
5928
5929         if (remsd) {
5930                 sd->rps_ipi_list = NULL;
5931
5932                 local_irq_enable();
5933
5934                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5935                 net_rps_send_ipi(remsd);
5936         } else
5937 #endif
5938                 local_irq_enable();
5939 }
5940
5941 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5942 {
5943 #ifdef CONFIG_RPS
5944         return sd->rps_ipi_list != NULL;
5945 #else
5946         return false;
5947 #endif
5948 }
5949
5950 static int process_backlog(struct napi_struct *napi, int quota)
5951 {
5952         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5953         bool again = true;
5954         int work = 0;
5955
5956         /* Check if we have pending ipi, its better to send them now,
5957          * not waiting net_rx_action() end.
5958          */
5959         if (sd_has_rps_ipi_waiting(sd)) {
5960                 local_irq_disable();
5961                 net_rps_action_and_irq_enable(sd);
5962         }
5963
5964         napi->weight = READ_ONCE(dev_rx_weight);
5965         while (again) {
5966                 struct sk_buff *skb;
5967
5968                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5969                         rcu_read_lock();
5970                         __netif_receive_skb(skb);
5971                         rcu_read_unlock();
5972                         input_queue_head_incr(sd);
5973                         if (++work >= quota)
5974                                 return work;
5975
5976                 }
5977
5978                 rps_lock_irq_disable(sd);
5979                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5980                         /*
5981                          * Inline a custom version of __napi_complete().
5982                          * only current cpu owns and manipulates this napi,
5983                          * and NAPI_STATE_SCHED is the only possible flag set
5984                          * on backlog.
5985                          * We can use a plain write instead of clear_bit(),
5986                          * and we dont need an smp_mb() memory barrier.
5987                          */
5988                         napi->state = 0;
5989                         again = false;
5990                 } else {
5991                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5992                                                    &sd->process_queue);
5993                 }
5994                 rps_unlock_irq_enable(sd);
5995         }
5996
5997         return work;
5998 }
5999
6000 /**
6001  * __napi_schedule - schedule for receive
6002  * @n: entry to schedule
6003  *
6004  * The entry's receive function will be scheduled to run.
6005  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6006  */
6007 void __napi_schedule(struct napi_struct *n)
6008 {
6009         unsigned long flags;
6010
6011         local_irq_save(flags);
6012         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6013         local_irq_restore(flags);
6014 }
6015 EXPORT_SYMBOL(__napi_schedule);
6016
6017 /**
6018  *      napi_schedule_prep - check if napi can be scheduled
6019  *      @n: napi context
6020  *
6021  * Test if NAPI routine is already running, and if not mark
6022  * it as running.  This is used as a condition variable to
6023  * insure only one NAPI poll instance runs.  We also make
6024  * sure there is no pending NAPI disable.
6025  */
6026 bool napi_schedule_prep(struct napi_struct *n)
6027 {
6028         unsigned long val, new;
6029
6030         do {
6031                 val = READ_ONCE(n->state);
6032                 if (unlikely(val & NAPIF_STATE_DISABLE))
6033                         return false;
6034                 new = val | NAPIF_STATE_SCHED;
6035
6036                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6037                  * This was suggested by Alexander Duyck, as compiler
6038                  * emits better code than :
6039                  * if (val & NAPIF_STATE_SCHED)
6040                  *     new |= NAPIF_STATE_MISSED;
6041                  */
6042                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6043                                                    NAPIF_STATE_MISSED;
6044         } while (cmpxchg(&n->state, val, new) != val);
6045
6046         return !(val & NAPIF_STATE_SCHED);
6047 }
6048 EXPORT_SYMBOL(napi_schedule_prep);
6049
6050 /**
6051  * __napi_schedule_irqoff - schedule for receive
6052  * @n: entry to schedule
6053  *
6054  * Variant of __napi_schedule() assuming hard irqs are masked.
6055  *
6056  * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6057  * because the interrupt disabled assumption might not be true
6058  * due to force-threaded interrupts and spinlock substitution.
6059  */
6060 void __napi_schedule_irqoff(struct napi_struct *n)
6061 {
6062         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6063                 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6064         else
6065                 __napi_schedule(n);
6066 }
6067 EXPORT_SYMBOL(__napi_schedule_irqoff);
6068
6069 bool napi_complete_done(struct napi_struct *n, int work_done)
6070 {
6071         unsigned long flags, val, new, timeout = 0;
6072         bool ret = true;
6073
6074         /*
6075          * 1) Don't let napi dequeue from the cpu poll list
6076          *    just in case its running on a different cpu.
6077          * 2) If we are busy polling, do nothing here, we have
6078          *    the guarantee we will be called later.
6079          */
6080         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6081                                  NAPIF_STATE_IN_BUSY_POLL)))
6082                 return false;
6083
6084         if (work_done) {
6085                 if (n->gro_bitmask)
6086                         timeout = READ_ONCE(n->dev->gro_flush_timeout);
6087                 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6088         }
6089         if (n->defer_hard_irqs_count > 0) {
6090                 n->defer_hard_irqs_count--;
6091                 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6092                 if (timeout)
6093                         ret = false;
6094         }
6095         if (n->gro_bitmask) {
6096                 /* When the NAPI instance uses a timeout and keeps postponing
6097                  * it, we need to bound somehow the time packets are kept in
6098                  * the GRO layer
6099                  */
6100                 napi_gro_flush(n, !!timeout);
6101         }
6102
6103         gro_normal_list(n);
6104
6105         if (unlikely(!list_empty(&n->poll_list))) {
6106                 /* If n->poll_list is not empty, we need to mask irqs */
6107                 local_irq_save(flags);
6108                 list_del_init(&n->poll_list);
6109                 local_irq_restore(flags);
6110         }
6111
6112         do {
6113                 val = READ_ONCE(n->state);
6114
6115                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6116
6117                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
6118                               NAPIF_STATE_SCHED_THREADED |
6119                               NAPIF_STATE_PREFER_BUSY_POLL);
6120
6121                 /* If STATE_MISSED was set, leave STATE_SCHED set,
6122                  * because we will call napi->poll() one more time.
6123                  * This C code was suggested by Alexander Duyck to help gcc.
6124                  */
6125                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6126                                                     NAPIF_STATE_SCHED;
6127         } while (cmpxchg(&n->state, val, new) != val);
6128
6129         if (unlikely(val & NAPIF_STATE_MISSED)) {
6130                 __napi_schedule(n);
6131                 return false;
6132         }
6133
6134         if (timeout)
6135                 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6136                               HRTIMER_MODE_REL_PINNED);
6137         return ret;
6138 }
6139 EXPORT_SYMBOL(napi_complete_done);
6140
6141 /* must be called under rcu_read_lock(), as we dont take a reference */
6142 static struct napi_struct *napi_by_id(unsigned int napi_id)
6143 {
6144         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6145         struct napi_struct *napi;
6146
6147         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6148                 if (napi->napi_id == napi_id)
6149                         return napi;
6150
6151         return NULL;
6152 }
6153
6154 #if defined(CONFIG_NET_RX_BUSY_POLL)
6155
6156 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6157 {
6158         if (!skip_schedule) {
6159                 gro_normal_list(napi);
6160                 __napi_schedule(napi);
6161                 return;
6162         }
6163
6164         if (napi->gro_bitmask) {
6165                 /* flush too old packets
6166                  * If HZ < 1000, flush all packets.
6167                  */
6168                 napi_gro_flush(napi, HZ >= 1000);
6169         }
6170
6171         gro_normal_list(napi);
6172         clear_bit(NAPI_STATE_SCHED, &napi->state);
6173 }
6174
6175 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6176                            u16 budget)
6177 {
6178         bool skip_schedule = false;
6179         unsigned long timeout;
6180         int rc;
6181
6182         /* Busy polling means there is a high chance device driver hard irq
6183          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6184          * set in napi_schedule_prep().
6185          * Since we are about to call napi->poll() once more, we can safely
6186          * clear NAPI_STATE_MISSED.
6187          *
6188          * Note: x86 could use a single "lock and ..." instruction
6189          * to perform these two clear_bit()
6190          */
6191         clear_bit(NAPI_STATE_MISSED, &napi->state);
6192         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6193
6194         local_bh_disable();
6195
6196         if (prefer_busy_poll) {
6197                 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6198                 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6199                 if (napi->defer_hard_irqs_count && timeout) {
6200                         hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6201                         skip_schedule = true;
6202                 }
6203         }
6204
6205         /* All we really want here is to re-enable device interrupts.
6206          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6207          */
6208         rc = napi->poll(napi, budget);
6209         /* We can't gro_normal_list() here, because napi->poll() might have
6210          * rearmed the napi (napi_complete_done()) in which case it could
6211          * already be running on another CPU.
6212          */
6213         trace_napi_poll(napi, rc, budget);
6214         netpoll_poll_unlock(have_poll_lock);
6215         if (rc == budget)
6216                 __busy_poll_stop(napi, skip_schedule);
6217         local_bh_enable();
6218 }
6219
6220 void napi_busy_loop(unsigned int napi_id,
6221                     bool (*loop_end)(void *, unsigned long),
6222                     void *loop_end_arg, bool prefer_busy_poll, u16 budget)
6223 {
6224         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6225         int (*napi_poll)(struct napi_struct *napi, int budget);
6226         void *have_poll_lock = NULL;
6227         struct napi_struct *napi;
6228
6229 restart:
6230         napi_poll = NULL;
6231
6232         rcu_read_lock();
6233
6234         napi = napi_by_id(napi_id);
6235         if (!napi)
6236                 goto out;
6237
6238         preempt_disable();
6239         for (;;) {
6240                 int work = 0;
6241
6242                 local_bh_disable();
6243                 if (!napi_poll) {
6244                         unsigned long val = READ_ONCE(napi->state);
6245
6246                         /* If multiple threads are competing for this napi,
6247                          * we avoid dirtying napi->state as much as we can.
6248                          */
6249                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6250                                    NAPIF_STATE_IN_BUSY_POLL)) {
6251                                 if (prefer_busy_poll)
6252                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6253                                 goto count;
6254                         }
6255                         if (cmpxchg(&napi->state, val,
6256                                     val | NAPIF_STATE_IN_BUSY_POLL |
6257                                           NAPIF_STATE_SCHED) != val) {
6258                                 if (prefer_busy_poll)
6259                                         set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6260                                 goto count;
6261                         }
6262                         have_poll_lock = netpoll_poll_lock(napi);
6263                         napi_poll = napi->poll;
6264                 }
6265                 work = napi_poll(napi, budget);
6266                 trace_napi_poll(napi, work, budget);
6267                 gro_normal_list(napi);
6268 count:
6269                 if (work > 0)
6270                         __NET_ADD_STATS(dev_net(napi->dev),
6271                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
6272                 local_bh_enable();
6273
6274                 if (!loop_end || loop_end(loop_end_arg, start_time))
6275                         break;
6276
6277                 if (unlikely(need_resched())) {
6278                         if (napi_poll)
6279                                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6280                         preempt_enable();
6281                         rcu_read_unlock();
6282                         cond_resched();
6283                         if (loop_end(loop_end_arg, start_time))
6284                                 return;
6285                         goto restart;
6286                 }
6287                 cpu_relax();
6288         }
6289         if (napi_poll)
6290                 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6291         preempt_enable();
6292 out:
6293         rcu_read_unlock();
6294 }
6295 EXPORT_SYMBOL(napi_busy_loop);
6296
6297 #endif /* CONFIG_NET_RX_BUSY_POLL */
6298
6299 static void napi_hash_add(struct napi_struct *napi)
6300 {
6301         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6302                 return;
6303
6304         spin_lock(&napi_hash_lock);
6305
6306         /* 0..NR_CPUS range is reserved for sender_cpu use */
6307         do {
6308                 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6309                         napi_gen_id = MIN_NAPI_ID;
6310         } while (napi_by_id(napi_gen_id));
6311         napi->napi_id = napi_gen_id;
6312
6313         hlist_add_head_rcu(&napi->napi_hash_node,
6314                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6315
6316         spin_unlock(&napi_hash_lock);
6317 }
6318
6319 /* Warning : caller is responsible to make sure rcu grace period
6320  * is respected before freeing memory containing @napi
6321  */
6322 static void napi_hash_del(struct napi_struct *napi)
6323 {
6324         spin_lock(&napi_hash_lock);
6325
6326         hlist_del_init_rcu(&napi->napi_hash_node);
6327
6328         spin_unlock(&napi_hash_lock);
6329 }
6330
6331 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6332 {
6333         struct napi_struct *napi;
6334
6335         napi = container_of(timer, struct napi_struct, timer);
6336
6337         /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6338          * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6339          */
6340         if (!napi_disable_pending(napi) &&
6341             !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6342                 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6343                 __napi_schedule_irqoff(napi);
6344         }
6345
6346         return HRTIMER_NORESTART;
6347 }
6348
6349 static void init_gro_hash(struct napi_struct *napi)
6350 {
6351         int i;
6352
6353         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6354                 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6355                 napi->gro_hash[i].count = 0;
6356         }
6357         napi->gro_bitmask = 0;
6358 }
6359
6360 int dev_set_threaded(struct net_device *dev, bool threaded)
6361 {
6362         struct napi_struct *napi;
6363         int err = 0;
6364
6365         if (dev->threaded == threaded)
6366                 return 0;
6367
6368         if (threaded) {
6369                 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6370                         if (!napi->thread) {
6371                                 err = napi_kthread_create(napi);
6372                                 if (err) {
6373                                         threaded = false;
6374                                         break;
6375                                 }
6376                         }
6377                 }
6378         }
6379
6380         dev->threaded = threaded;
6381
6382         /* Make sure kthread is created before THREADED bit
6383          * is set.
6384          */
6385         smp_mb__before_atomic();
6386
6387         /* Setting/unsetting threaded mode on a napi might not immediately
6388          * take effect, if the current napi instance is actively being
6389          * polled. In this case, the switch between threaded mode and
6390          * softirq mode will happen in the next round of napi_schedule().
6391          * This should not cause hiccups/stalls to the live traffic.
6392          */
6393         list_for_each_entry(napi, &dev->napi_list, dev_list) {
6394                 if (threaded)
6395                         set_bit(NAPI_STATE_THREADED, &napi->state);
6396                 else
6397                         clear_bit(NAPI_STATE_THREADED, &napi->state);
6398         }
6399
6400         return err;
6401 }
6402 EXPORT_SYMBOL(dev_set_threaded);
6403
6404 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6405                            int (*poll)(struct napi_struct *, int), int weight)
6406 {
6407         if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6408                 return;
6409
6410         INIT_LIST_HEAD(&napi->poll_list);
6411         INIT_HLIST_NODE(&napi->napi_hash_node);
6412         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6413         napi->timer.function = napi_watchdog;
6414         init_gro_hash(napi);
6415         napi->skb = NULL;
6416         INIT_LIST_HEAD(&napi->rx_list);
6417         napi->rx_count = 0;
6418         napi->poll = poll;
6419         if (weight > NAPI_POLL_WEIGHT)
6420                 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6421                                 weight);
6422         napi->weight = weight;
6423         napi->dev = dev;
6424 #ifdef CONFIG_NETPOLL
6425         napi->poll_owner = -1;
6426 #endif
6427         set_bit(NAPI_STATE_SCHED, &napi->state);
6428         set_bit(NAPI_STATE_NPSVC, &napi->state);
6429         list_add_rcu(&napi->dev_list, &dev->napi_list);
6430         napi_hash_add(napi);
6431         napi_get_frags_check(napi);
6432         /* Create kthread for this napi if dev->threaded is set.
6433          * Clear dev->threaded if kthread creation failed so that
6434          * threaded mode will not be enabled in napi_enable().
6435          */
6436         if (dev->threaded && napi_kthread_create(napi))
6437                 dev->threaded = 0;
6438 }
6439 EXPORT_SYMBOL(netif_napi_add_weight);
6440
6441 void napi_disable(struct napi_struct *n)
6442 {
6443         unsigned long val, new;
6444
6445         might_sleep();
6446         set_bit(NAPI_STATE_DISABLE, &n->state);
6447
6448         for ( ; ; ) {
6449                 val = READ_ONCE(n->state);
6450                 if (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
6451                         usleep_range(20, 200);
6452                         continue;
6453                 }
6454
6455                 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC;
6456                 new &= ~(NAPIF_STATE_THREADED | NAPIF_STATE_PREFER_BUSY_POLL);
6457
6458                 if (cmpxchg(&n->state, val, new) == val)
6459                         break;
6460         }
6461
6462         hrtimer_cancel(&n->timer);
6463
6464         clear_bit(NAPI_STATE_DISABLE, &n->state);
6465 }
6466 EXPORT_SYMBOL(napi_disable);
6467
6468 /**
6469  *      napi_enable - enable NAPI scheduling
6470  *      @n: NAPI context
6471  *
6472  * Resume NAPI from being scheduled on this context.
6473  * Must be paired with napi_disable.
6474  */
6475 void napi_enable(struct napi_struct *n)
6476 {
6477         unsigned long val, new;
6478
6479         do {
6480                 val = READ_ONCE(n->state);
6481                 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
6482
6483                 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC);
6484                 if (n->dev->threaded && n->thread)
6485                         new |= NAPIF_STATE_THREADED;
6486         } while (cmpxchg(&n->state, val, new) != val);
6487 }
6488 EXPORT_SYMBOL(napi_enable);
6489
6490 static void flush_gro_hash(struct napi_struct *napi)
6491 {
6492         int i;
6493
6494         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6495                 struct sk_buff *skb, *n;
6496
6497                 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6498                         kfree_skb(skb);
6499                 napi->gro_hash[i].count = 0;
6500         }
6501 }
6502
6503 /* Must be called in process context */
6504 void __netif_napi_del(struct napi_struct *napi)
6505 {
6506         if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6507                 return;
6508
6509         napi_hash_del(napi);
6510         list_del_rcu(&napi->dev_list);
6511         napi_free_frags(napi);
6512
6513         flush_gro_hash(napi);
6514         napi->gro_bitmask = 0;
6515
6516         if (napi->thread) {
6517                 kthread_stop(napi->thread);
6518                 napi->thread = NULL;
6519         }
6520 }
6521 EXPORT_SYMBOL(__netif_napi_del);
6522
6523 static int __napi_poll(struct napi_struct *n, bool *repoll)
6524 {
6525         int work, weight;
6526
6527         weight = n->weight;
6528
6529         /* This NAPI_STATE_SCHED test is for avoiding a race
6530          * with netpoll's poll_napi().  Only the entity which
6531          * obtains the lock and sees NAPI_STATE_SCHED set will
6532          * actually make the ->poll() call.  Therefore we avoid
6533          * accidentally calling ->poll() when NAPI is not scheduled.
6534          */
6535         work = 0;
6536         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6537                 work = n->poll(n, weight);
6538                 trace_napi_poll(n, work, weight);
6539         }
6540
6541         if (unlikely(work > weight))
6542                 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
6543                                 n->poll, work, weight);
6544
6545         if (likely(work < weight))
6546                 return work;
6547
6548         /* Drivers must not modify the NAPI state if they
6549          * consume the entire weight.  In such cases this code
6550          * still "owns" the NAPI instance and therefore can
6551          * move the instance around on the list at-will.
6552          */
6553         if (unlikely(napi_disable_pending(n))) {
6554                 napi_complete(n);
6555                 return work;
6556         }
6557
6558         /* The NAPI context has more processing work, but busy-polling
6559          * is preferred. Exit early.
6560          */
6561         if (napi_prefer_busy_poll(n)) {
6562                 if (napi_complete_done(n, work)) {
6563                         /* If timeout is not set, we need to make sure
6564                          * that the NAPI is re-scheduled.
6565                          */
6566                         napi_schedule(n);
6567                 }
6568                 return work;
6569         }
6570
6571         if (n->gro_bitmask) {
6572                 /* flush too old packets
6573                  * If HZ < 1000, flush all packets.
6574                  */
6575                 napi_gro_flush(n, HZ >= 1000);
6576         }
6577
6578         gro_normal_list(n);
6579
6580         /* Some drivers may have called napi_schedule
6581          * prior to exhausting their budget.
6582          */
6583         if (unlikely(!list_empty(&n->poll_list))) {
6584                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6585                              n->dev ? n->dev->name : "backlog");
6586                 return work;
6587         }
6588
6589         *repoll = true;
6590
6591         return work;
6592 }
6593
6594 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6595 {
6596         bool do_repoll = false;
6597         void *have;
6598         int work;
6599
6600         list_del_init(&n->poll_list);
6601
6602         have = netpoll_poll_lock(n);
6603
6604         work = __napi_poll(n, &do_repoll);
6605
6606         if (do_repoll)
6607                 list_add_tail(&n->poll_list, repoll);
6608
6609         netpoll_poll_unlock(have);
6610
6611         return work;
6612 }
6613
6614 static int napi_thread_wait(struct napi_struct *napi)
6615 {
6616         bool woken = false;
6617
6618         set_current_state(TASK_INTERRUPTIBLE);
6619
6620         while (!kthread_should_stop()) {
6621                 /* Testing SCHED_THREADED bit here to make sure the current
6622                  * kthread owns this napi and could poll on this napi.
6623                  * Testing SCHED bit is not enough because SCHED bit might be
6624                  * set by some other busy poll thread or by napi_disable().
6625                  */
6626                 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
6627                         WARN_ON(!list_empty(&napi->poll_list));
6628                         __set_current_state(TASK_RUNNING);
6629                         return 0;
6630                 }
6631
6632                 schedule();
6633                 /* woken being true indicates this thread owns this napi. */
6634                 woken = true;
6635                 set_current_state(TASK_INTERRUPTIBLE);
6636         }
6637         __set_current_state(TASK_RUNNING);
6638
6639         return -1;
6640 }
6641
6642 static int napi_threaded_poll(void *data)
6643 {
6644         struct napi_struct *napi = data;
6645         void *have;
6646
6647         while (!napi_thread_wait(napi)) {
6648                 unsigned long last_qs = jiffies;
6649
6650                 for (;;) {
6651                         bool repoll = false;
6652
6653                         local_bh_disable();
6654
6655                         have = netpoll_poll_lock(napi);
6656                         __napi_poll(napi, &repoll);
6657                         netpoll_poll_unlock(have);
6658
6659                         local_bh_enable();
6660
6661                         if (!repoll)
6662                                 break;
6663
6664                         rcu_softirq_qs_periodic(last_qs);
6665                         cond_resched();
6666                 }
6667         }
6668         return 0;
6669 }
6670
6671 static void skb_defer_free_flush(struct softnet_data *sd)
6672 {
6673         struct sk_buff *skb, *next;
6674         unsigned long flags;
6675
6676         /* Paired with WRITE_ONCE() in skb_attempt_defer_free() */
6677         if (!READ_ONCE(sd->defer_list))
6678                 return;
6679
6680         spin_lock_irqsave(&sd->defer_lock, flags);
6681         skb = sd->defer_list;
6682         sd->defer_list = NULL;
6683         sd->defer_count = 0;
6684         spin_unlock_irqrestore(&sd->defer_lock, flags);
6685
6686         while (skb != NULL) {
6687                 next = skb->next;
6688                 napi_consume_skb(skb, 1);
6689                 skb = next;
6690         }
6691 }
6692
6693 static __latent_entropy void net_rx_action(struct softirq_action *h)
6694 {
6695         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6696         unsigned long time_limit = jiffies +
6697                 usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
6698         int budget = READ_ONCE(netdev_budget);
6699         LIST_HEAD(list);
6700         LIST_HEAD(repoll);
6701
6702         local_irq_disable();
6703         list_splice_init(&sd->poll_list, &list);
6704         local_irq_enable();
6705
6706         for (;;) {
6707                 struct napi_struct *n;
6708
6709                 skb_defer_free_flush(sd);
6710
6711                 if (list_empty(&list)) {
6712                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
6713                                 goto end;
6714                         break;
6715                 }
6716
6717                 n = list_first_entry(&list, struct napi_struct, poll_list);
6718                 budget -= napi_poll(n, &repoll);
6719
6720                 /* If softirq window is exhausted then punt.
6721                  * Allow this to run for 2 jiffies since which will allow
6722                  * an average latency of 1.5/HZ.
6723                  */
6724                 if (unlikely(budget <= 0 ||
6725                              time_after_eq(jiffies, time_limit))) {
6726                         sd->time_squeeze++;
6727                         break;
6728                 }
6729         }
6730
6731         local_irq_disable();
6732
6733         list_splice_tail_init(&sd->poll_list, &list);
6734         list_splice_tail(&repoll, &list);
6735         list_splice(&list, &sd->poll_list);
6736         if (!list_empty(&sd->poll_list))
6737                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6738
6739         net_rps_action_and_irq_enable(sd);
6740 end:;
6741 }
6742
6743 struct netdev_adjacent {
6744         struct net_device *dev;
6745         netdevice_tracker dev_tracker;
6746
6747         /* upper master flag, there can only be one master device per list */
6748         bool master;
6749
6750         /* lookup ignore flag */
6751         bool ignore;
6752
6753         /* counter for the number of times this device was added to us */
6754         u16 ref_nr;
6755
6756         /* private field for the users */
6757         void *private;
6758
6759         struct list_head list;
6760         struct rcu_head rcu;
6761 };
6762
6763 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
6764                                                  struct list_head *adj_list)
6765 {
6766         struct netdev_adjacent *adj;
6767
6768         list_for_each_entry(adj, adj_list, list) {
6769                 if (adj->dev == adj_dev)
6770                         return adj;
6771         }
6772         return NULL;
6773 }
6774
6775 static int ____netdev_has_upper_dev(struct net_device *upper_dev,
6776                                     struct netdev_nested_priv *priv)
6777 {
6778         struct net_device *dev = (struct net_device *)priv->data;
6779
6780         return upper_dev == dev;
6781 }
6782
6783 /**
6784  * netdev_has_upper_dev - Check if device is linked to an upper device
6785  * @dev: device
6786  * @upper_dev: upper device to check
6787  *
6788  * Find out if a device is linked to specified upper device and return true
6789  * in case it is. Note that this checks only immediate upper device,
6790  * not through a complete stack of devices. The caller must hold the RTNL lock.
6791  */
6792 bool netdev_has_upper_dev(struct net_device *dev,
6793                           struct net_device *upper_dev)
6794 {
6795         struct netdev_nested_priv priv = {
6796                 .data = (void *)upper_dev,
6797         };
6798
6799         ASSERT_RTNL();
6800
6801         return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6802                                              &priv);
6803 }
6804 EXPORT_SYMBOL(netdev_has_upper_dev);
6805
6806 /**
6807  * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
6808  * @dev: device
6809  * @upper_dev: upper device to check
6810  *
6811  * Find out if a device is linked to specified upper device and return true
6812  * in case it is. Note that this checks the entire upper device chain.
6813  * The caller must hold rcu lock.
6814  */
6815
6816 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6817                                   struct net_device *upper_dev)
6818 {
6819         struct netdev_nested_priv priv = {
6820                 .data = (void *)upper_dev,
6821         };
6822
6823         return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
6824                                                &priv);
6825 }
6826 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6827
6828 /**
6829  * netdev_has_any_upper_dev - Check if device is linked to some device
6830  * @dev: device
6831  *
6832  * Find out if a device is linked to an upper device and return true in case
6833  * it is. The caller must hold the RTNL lock.
6834  */
6835 bool netdev_has_any_upper_dev(struct net_device *dev)
6836 {
6837         ASSERT_RTNL();
6838
6839         return !list_empty(&dev->adj_list.upper);
6840 }
6841 EXPORT_SYMBOL(netdev_has_any_upper_dev);
6842
6843 /**
6844  * netdev_master_upper_dev_get - Get master upper device
6845  * @dev: device
6846  *
6847  * Find a master upper device and return pointer to it or NULL in case
6848  * it's not there. The caller must hold the RTNL lock.
6849  */
6850 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6851 {
6852         struct netdev_adjacent *upper;
6853
6854         ASSERT_RTNL();
6855
6856         if (list_empty(&dev->adj_list.upper))
6857                 return NULL;
6858
6859         upper = list_first_entry(&dev->adj_list.upper,
6860                                  struct netdev_adjacent, list);
6861         if (likely(upper->master))
6862                 return upper->dev;
6863         return NULL;
6864 }
6865 EXPORT_SYMBOL(netdev_master_upper_dev_get);
6866
6867 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
6868 {
6869         struct netdev_adjacent *upper;
6870
6871         ASSERT_RTNL();
6872
6873         if (list_empty(&dev->adj_list.upper))
6874                 return NULL;
6875
6876         upper = list_first_entry(&dev->adj_list.upper,
6877                                  struct netdev_adjacent, list);
6878         if (likely(upper->master) && !upper->ignore)
6879                 return upper->dev;
6880         return NULL;
6881 }
6882
6883 /**
6884  * netdev_has_any_lower_dev - Check if device is linked to some device
6885  * @dev: device
6886  *
6887  * Find out if a device is linked to a lower device and return true in case
6888  * it is. The caller must hold the RTNL lock.
6889  */
6890 static bool netdev_has_any_lower_dev(struct net_device *dev)
6891 {
6892         ASSERT_RTNL();
6893
6894         return !list_empty(&dev->adj_list.lower);
6895 }
6896
6897 void *netdev_adjacent_get_private(struct list_head *adj_list)
6898 {
6899         struct netdev_adjacent *adj;
6900
6901         adj = list_entry(adj_list, struct netdev_adjacent, list);
6902
6903         return adj->private;
6904 }
6905 EXPORT_SYMBOL(netdev_adjacent_get_private);
6906
6907 /**
6908  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6909  * @dev: device
6910  * @iter: list_head ** of the current position
6911  *
6912  * Gets the next device from the dev's upper list, starting from iter
6913  * position. The caller must hold RCU read lock.
6914  */
6915 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6916                                                  struct list_head **iter)
6917 {
6918         struct netdev_adjacent *upper;
6919
6920         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6921
6922         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6923
6924         if (&upper->list == &dev->adj_list.upper)
6925                 return NULL;
6926
6927         *iter = &upper->list;
6928
6929         return upper->dev;
6930 }
6931 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6932
6933 static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
6934                                                   struct list_head **iter,
6935                                                   bool *ignore)
6936 {
6937         struct netdev_adjacent *upper;
6938
6939         upper = list_entry((*iter)->next, struct netdev_adjacent, list);
6940
6941         if (&upper->list == &dev->adj_list.upper)
6942                 return NULL;
6943
6944         *iter = &upper->list;
6945         *ignore = upper->ignore;
6946
6947         return upper->dev;
6948 }
6949
6950 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
6951                                                     struct list_head **iter)
6952 {
6953         struct netdev_adjacent *upper;
6954
6955         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6956
6957         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6958
6959         if (&upper->list == &dev->adj_list.upper)
6960                 return NULL;
6961
6962         *iter = &upper->list;
6963
6964         return upper->dev;
6965 }
6966
6967 static int __netdev_walk_all_upper_dev(struct net_device *dev,
6968                                        int (*fn)(struct net_device *dev,
6969                                          struct netdev_nested_priv *priv),
6970                                        struct netdev_nested_priv *priv)
6971 {
6972         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
6973         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
6974         int ret, cur = 0;
6975         bool ignore;
6976
6977         now = dev;
6978         iter = &dev->adj_list.upper;
6979
6980         while (1) {
6981                 if (now != dev) {
6982                         ret = fn(now, priv);
6983                         if (ret)
6984                                 return ret;
6985                 }
6986
6987                 next = NULL;
6988                 while (1) {
6989                         udev = __netdev_next_upper_dev(now, &iter, &ignore);
6990                         if (!udev)
6991                                 break;
6992                         if (ignore)
6993                                 continue;
6994
6995                         next = udev;
6996                         niter = &udev->adj_list.upper;
6997                         dev_stack[cur] = now;
6998                         iter_stack[cur++] = iter;
6999                         break;
7000                 }
7001
7002                 if (!next) {
7003                         if (!cur)
7004                                 return 0;
7005                         next = dev_stack[--cur];
7006                         niter = iter_stack[cur];
7007                 }
7008
7009                 now = next;
7010                 iter = niter;
7011         }
7012
7013         return 0;
7014 }
7015
7016 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7017                                   int (*fn)(struct net_device *dev,
7018                                             struct netdev_nested_priv *priv),
7019                                   struct netdev_nested_priv *priv)
7020 {
7021         struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7022         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7023         int ret, cur = 0;
7024
7025         now = dev;
7026         iter = &dev->adj_list.upper;
7027
7028         while (1) {
7029                 if (now != dev) {
7030                         ret = fn(now, priv);
7031                         if (ret)
7032                                 return ret;
7033                 }
7034
7035                 next = NULL;
7036                 while (1) {
7037                         udev = netdev_next_upper_dev_rcu(now, &iter);
7038                         if (!udev)
7039                                 break;
7040
7041                         next = udev;
7042                         niter = &udev->adj_list.upper;
7043                         dev_stack[cur] = now;
7044                         iter_stack[cur++] = iter;
7045                         break;
7046                 }
7047
7048                 if (!next) {
7049                         if (!cur)
7050                                 return 0;
7051                         next = dev_stack[--cur];
7052                         niter = iter_stack[cur];
7053                 }
7054
7055                 now = next;
7056                 iter = niter;
7057         }
7058
7059         return 0;
7060 }
7061 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7062
7063 static bool __netdev_has_upper_dev(struct net_device *dev,
7064                                    struct net_device *upper_dev)
7065 {
7066         struct netdev_nested_priv priv = {
7067                 .flags = 0,
7068                 .data = (void *)upper_dev,
7069         };
7070
7071         ASSERT_RTNL();
7072
7073         return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
7074                                            &priv);
7075 }
7076
7077 /**
7078  * netdev_lower_get_next_private - Get the next ->private from the
7079  *                                 lower neighbour list
7080  * @dev: device
7081  * @iter: list_head ** of the current position
7082  *
7083  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7084  * list, starting from iter position. The caller must hold either hold the
7085  * RTNL lock or its own locking that guarantees that the neighbour lower
7086  * list will remain unchanged.
7087  */
7088 void *netdev_lower_get_next_private(struct net_device *dev,
7089                                     struct list_head **iter)
7090 {
7091         struct netdev_adjacent *lower;
7092
7093         lower = list_entry(*iter, struct netdev_adjacent, list);
7094
7095         if (&lower->list == &dev->adj_list.lower)
7096                 return NULL;
7097
7098         *iter = lower->list.next;
7099
7100         return lower->private;
7101 }
7102 EXPORT_SYMBOL(netdev_lower_get_next_private);
7103
7104 /**
7105  * netdev_lower_get_next_private_rcu - Get the next ->private from the
7106  *                                     lower neighbour list, RCU
7107  *                                     variant
7108  * @dev: device
7109  * @iter: list_head ** of the current position
7110  *
7111  * Gets the next netdev_adjacent->private from the dev's lower neighbour
7112  * list, starting from iter position. The caller must hold RCU read lock.
7113  */
7114 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7115                                         struct list_head **iter)
7116 {
7117         struct netdev_adjacent *lower;
7118
7119         WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
7120
7121         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7122
7123         if (&lower->list == &dev->adj_list.lower)
7124                 return NULL;
7125
7126         *iter = &lower->list;
7127
7128         return lower->private;
7129 }
7130 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7131
7132 /**
7133  * netdev_lower_get_next - Get the next device from the lower neighbour
7134  *                         list
7135  * @dev: device
7136  * @iter: list_head ** of the current position
7137  *
7138  * Gets the next netdev_adjacent from the dev's lower neighbour
7139  * list, starting from iter position. The caller must hold RTNL lock or
7140  * its own locking that guarantees that the neighbour lower
7141  * list will remain unchanged.
7142  */
7143 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7144 {
7145         struct netdev_adjacent *lower;
7146
7147         lower = list_entry(*iter, struct netdev_adjacent, list);
7148
7149         if (&lower->list == &dev->adj_list.lower)
7150                 return NULL;
7151
7152         *iter = lower->list.next;
7153
7154         return lower->dev;
7155 }
7156 EXPORT_SYMBOL(netdev_lower_get_next);
7157
7158 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7159                                                 struct list_head **iter)
7160 {
7161         struct netdev_adjacent *lower;
7162
7163         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7164
7165         if (&lower->list == &dev->adj_list.lower)
7166                 return NULL;
7167
7168         *iter = &lower->list;
7169
7170         return lower->dev;
7171 }
7172
7173 static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7174                                                   struct list_head **iter,
7175                                                   bool *ignore)
7176 {
7177         struct netdev_adjacent *lower;
7178
7179         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7180
7181         if (&lower->list == &dev->adj_list.lower)
7182                 return NULL;
7183
7184         *iter = &lower->list;
7185         *ignore = lower->ignore;
7186
7187         return lower->dev;
7188 }
7189
7190 int netdev_walk_all_lower_dev(struct net_device *dev,
7191                               int (*fn)(struct net_device *dev,
7192                                         struct netdev_nested_priv *priv),
7193                               struct netdev_nested_priv *priv)
7194 {
7195         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7196         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7197         int ret, cur = 0;
7198
7199         now = dev;
7200         iter = &dev->adj_list.lower;
7201
7202         while (1) {
7203                 if (now != dev) {
7204                         ret = fn(now, priv);
7205                         if (ret)
7206                                 return ret;
7207                 }
7208
7209                 next = NULL;
7210                 while (1) {
7211                         ldev = netdev_next_lower_dev(now, &iter);
7212                         if (!ldev)
7213                                 break;
7214
7215                         next = ldev;
7216                         niter = &ldev->adj_list.lower;
7217                         dev_stack[cur] = now;
7218                         iter_stack[cur++] = iter;
7219                         break;
7220                 }
7221
7222                 if (!next) {
7223                         if (!cur)
7224                                 return 0;
7225                         next = dev_stack[--cur];
7226                         niter = iter_stack[cur];
7227                 }
7228
7229                 now = next;
7230                 iter = niter;
7231         }
7232
7233         return 0;
7234 }
7235 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7236
7237 static int __netdev_walk_all_lower_dev(struct net_device *dev,
7238                                        int (*fn)(struct net_device *dev,
7239                                          struct netdev_nested_priv *priv),
7240                                        struct netdev_nested_priv *priv)
7241 {
7242         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7243         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7244         int ret, cur = 0;
7245         bool ignore;
7246
7247         now = dev;
7248         iter = &dev->adj_list.lower;
7249
7250         while (1) {
7251                 if (now != dev) {
7252                         ret = fn(now, priv);
7253                         if (ret)
7254                                 return ret;
7255                 }
7256
7257                 next = NULL;
7258                 while (1) {
7259                         ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7260                         if (!ldev)
7261                                 break;
7262                         if (ignore)
7263                                 continue;
7264
7265                         next = ldev;
7266                         niter = &ldev->adj_list.lower;
7267                         dev_stack[cur] = now;
7268                         iter_stack[cur++] = iter;
7269                         break;
7270                 }
7271
7272                 if (!next) {
7273                         if (!cur)
7274                                 return 0;
7275                         next = dev_stack[--cur];
7276                         niter = iter_stack[cur];
7277                 }
7278
7279                 now = next;
7280                 iter = niter;
7281         }
7282
7283         return 0;
7284 }
7285
7286 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7287                                              struct list_head **iter)
7288 {
7289         struct netdev_adjacent *lower;
7290
7291         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7292         if (&lower->list == &dev->adj_list.lower)
7293                 return NULL;
7294
7295         *iter = &lower->list;
7296
7297         return lower->dev;
7298 }
7299 EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
7300
7301 static u8 __netdev_upper_depth(struct net_device *dev)
7302 {
7303         struct net_device *udev;
7304         struct list_head *iter;
7305         u8 max_depth = 0;
7306         bool ignore;
7307
7308         for (iter = &dev->adj_list.upper,
7309              udev = __netdev_next_upper_dev(dev, &iter, &ignore);
7310              udev;
7311              udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7312                 if (ignore)
7313                         continue;
7314                 if (max_depth < udev->upper_level)
7315                         max_depth = udev->upper_level;
7316         }
7317
7318         return max_depth;
7319 }
7320
7321 static u8 __netdev_lower_depth(struct net_device *dev)
7322 {
7323         struct net_device *ldev;
7324         struct list_head *iter;
7325         u8 max_depth = 0;
7326         bool ignore;
7327
7328         for (iter = &dev->adj_list.lower,
7329              ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
7330              ldev;
7331              ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7332                 if (ignore)
7333                         continue;
7334                 if (max_depth < ldev->lower_level)
7335                         max_depth = ldev->lower_level;
7336         }
7337
7338         return max_depth;
7339 }
7340
7341 static int __netdev_update_upper_level(struct net_device *dev,
7342                                        struct netdev_nested_priv *__unused)
7343 {
7344         dev->upper_level = __netdev_upper_depth(dev) + 1;
7345         return 0;
7346 }
7347
7348 #ifdef CONFIG_LOCKDEP
7349 static LIST_HEAD(net_unlink_list);
7350
7351 static void net_unlink_todo(struct net_device *dev)
7352 {
7353         if (list_empty(&dev->unlink_list))
7354                 list_add_tail(&dev->unlink_list, &net_unlink_list);
7355 }
7356 #endif
7357
7358 static int __netdev_update_lower_level(struct net_device *dev,
7359                                        struct netdev_nested_priv *priv)
7360 {
7361         dev->lower_level = __netdev_lower_depth(dev) + 1;
7362
7363 #ifdef CONFIG_LOCKDEP
7364         if (!priv)
7365                 return 0;
7366
7367         if (priv->flags & NESTED_SYNC_IMM)
7368                 dev->nested_level = dev->lower_level - 1;
7369         if (priv->flags & NESTED_SYNC_TODO)
7370                 net_unlink_todo(dev);
7371 #endif
7372         return 0;
7373 }
7374
7375 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7376                                   int (*fn)(struct net_device *dev,
7377                                             struct netdev_nested_priv *priv),
7378                                   struct netdev_nested_priv *priv)
7379 {
7380         struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7381         struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7382         int ret, cur = 0;
7383
7384         now = dev;
7385         iter = &dev->adj_list.lower;
7386
7387         while (1) {
7388                 if (now != dev) {
7389                         ret = fn(now, priv);
7390                         if (ret)
7391                                 return ret;
7392                 }
7393
7394                 next = NULL;
7395                 while (1) {
7396                         ldev = netdev_next_lower_dev_rcu(now, &iter);
7397                         if (!ldev)
7398                                 break;
7399
7400                         next = ldev;
7401                         niter = &ldev->adj_list.lower;
7402                         dev_stack[cur] = now;
7403                         iter_stack[cur++] = iter;
7404                         break;
7405                 }
7406
7407                 if (!next) {
7408                         if (!cur)
7409                                 return 0;
7410                         next = dev_stack[--cur];
7411                         niter = iter_stack[cur];
7412                 }
7413
7414                 now = next;
7415                 iter = niter;
7416         }
7417
7418         return 0;
7419 }
7420 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7421
7422 /**
7423  * netdev_lower_get_first_private_rcu - Get the first ->private from the
7424  *                                     lower neighbour list, RCU
7425  *                                     variant
7426  * @dev: device
7427  *
7428  * Gets the first netdev_adjacent->private from the dev's lower neighbour
7429  * list. The caller must hold RCU read lock.
7430  */
7431 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7432 {
7433         struct netdev_adjacent *lower;
7434
7435         lower = list_first_or_null_rcu(&dev->adj_list.lower,
7436                         struct netdev_adjacent, list);
7437         if (lower)
7438                 return lower->private;
7439         return NULL;
7440 }
7441 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7442
7443 /**
7444  * netdev_master_upper_dev_get_rcu - Get master upper device
7445  * @dev: device
7446  *
7447  * Find a master upper device and return pointer to it or NULL in case
7448  * it's not there. The caller must hold the RCU read lock.
7449  */
7450 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7451 {
7452         struct netdev_adjacent *upper;
7453
7454         upper = list_first_or_null_rcu(&dev->adj_list.upper,
7455                                        struct netdev_adjacent, list);
7456         if (upper && likely(upper->master))
7457                 return upper->dev;
7458         return NULL;
7459 }
7460 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7461
7462 static int netdev_adjacent_sysfs_add(struct net_device *dev,
7463                               struct net_device *adj_dev,
7464                               struct list_head *dev_list)
7465 {
7466         char linkname[IFNAMSIZ+7];
7467
7468         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7469                 "upper_%s" : "lower_%s", adj_dev->name);
7470         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7471                                  linkname);
7472 }
7473 static void netdev_adjacent_sysfs_del(struct net_device *dev,
7474                                char *name,
7475                                struct list_head *dev_list)
7476 {
7477         char linkname[IFNAMSIZ+7];
7478
7479         sprintf(linkname, dev_list == &dev->adj_list.upper ?
7480                 "upper_%s" : "lower_%s", name);
7481         sysfs_remove_link(&(dev->dev.kobj), linkname);
7482 }
7483
7484 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7485                                                  struct net_device *adj_dev,
7486                                                  struct list_head *dev_list)
7487 {
7488         return (dev_list == &dev->adj_list.upper ||
7489                 dev_list == &dev->adj_list.lower) &&
7490                 net_eq(dev_net(dev), dev_net(adj_dev));
7491 }
7492
7493 static int __netdev_adjacent_dev_insert(struct net_device *dev,
7494                                         struct net_device *adj_dev,
7495                                         struct list_head *dev_list,
7496                                         void *private, bool master)
7497 {
7498         struct netdev_adjacent *adj;
7499         int ret;
7500
7501         adj = __netdev_find_adj(adj_dev, dev_list);
7502
7503         if (adj) {
7504                 adj->ref_nr += 1;
7505                 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7506                          dev->name, adj_dev->name, adj->ref_nr);
7507
7508                 return 0;
7509         }
7510
7511         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
7512         if (!adj)
7513                 return -ENOMEM;
7514
7515         adj->dev = adj_dev;
7516         adj->master = master;
7517         adj->ref_nr = 1;
7518         adj->private = private;
7519         adj->ignore = false;
7520         netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
7521
7522         pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
7523                  dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
7524
7525         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
7526                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
7527                 if (ret)
7528                         goto free_adj;
7529         }
7530
7531         /* Ensure that master link is always the first item in list. */
7532         if (master) {
7533                 ret = sysfs_create_link(&(dev->dev.kobj),
7534                                         &(adj_dev->dev.kobj), "master");
7535                 if (ret)
7536                         goto remove_symlinks;
7537
7538                 list_add_rcu(&adj->list, dev_list);
7539         } else {
7540                 list_add_tail_rcu(&adj->list, dev_list);
7541         }
7542
7543         return 0;
7544
7545 remove_symlinks:
7546         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7547                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7548 free_adj:
7549         netdev_put(adj_dev, &adj->dev_tracker);
7550         kfree(adj);
7551
7552         return ret;
7553 }
7554
7555 static void __netdev_adjacent_dev_remove(struct net_device *dev,
7556                                          struct net_device *adj_dev,
7557                                          u16 ref_nr,
7558                                          struct list_head *dev_list)
7559 {
7560         struct netdev_adjacent *adj;
7561
7562         pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
7563                  dev->name, adj_dev->name, ref_nr);
7564
7565         adj = __netdev_find_adj(adj_dev, dev_list);
7566
7567         if (!adj) {
7568                 pr_err("Adjacency does not exist for device %s from %s\n",
7569                        dev->name, adj_dev->name);
7570                 WARN_ON(1);
7571                 return;
7572         }
7573
7574         if (adj->ref_nr > ref_nr) {
7575                 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
7576                          dev->name, adj_dev->name, ref_nr,
7577                          adj->ref_nr - ref_nr);
7578                 adj->ref_nr -= ref_nr;
7579                 return;
7580         }
7581
7582         if (adj->master)
7583                 sysfs_remove_link(&(dev->dev.kobj), "master");
7584
7585         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
7586                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
7587
7588         list_del_rcu(&adj->list);
7589         pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
7590                  adj_dev->name, dev->name, adj_dev->name);
7591         netdev_put(adj_dev, &adj->dev_tracker);
7592         kfree_rcu(adj, rcu);
7593 }
7594
7595 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
7596                                             struct net_device *upper_dev,
7597                                             struct list_head *up_list,
7598                                             struct list_head *down_list,
7599                                             void *private, bool master)
7600 {
7601         int ret;
7602
7603         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
7604                                            private, master);
7605         if (ret)
7606                 return ret;
7607
7608         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
7609                                            private, false);
7610         if (ret) {
7611                 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
7612                 return ret;
7613         }
7614
7615         return 0;
7616 }
7617
7618 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
7619                                                struct net_device *upper_dev,
7620                                                u16 ref_nr,
7621                                                struct list_head *up_list,
7622                                                struct list_head *down_list)
7623 {
7624         __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
7625         __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
7626 }
7627
7628 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
7629                                                 struct net_device *upper_dev,
7630                                                 void *private, bool master)
7631 {
7632         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
7633                                                 &dev->adj_list.upper,
7634                                                 &upper_dev->adj_list.lower,
7635                                                 private, master);
7636 }
7637
7638 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
7639                                                    struct net_device *upper_dev)
7640 {
7641         __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
7642                                            &dev->adj_list.upper,
7643                                            &upper_dev->adj_list.lower);
7644 }
7645
7646 static int __netdev_upper_dev_link(struct net_device *dev,
7647                                    struct net_device *upper_dev, bool master,
7648                                    void *upper_priv, void *upper_info,
7649                                    struct netdev_nested_priv *priv,
7650                                    struct netlink_ext_ack *extack)
7651 {
7652         struct netdev_notifier_changeupper_info changeupper_info = {
7653                 .info = {
7654                         .dev = dev,
7655                         .extack = extack,
7656                 },
7657                 .upper_dev = upper_dev,
7658                 .master = master,
7659                 .linking = true,
7660                 .upper_info = upper_info,
7661         };
7662         struct net_device *master_dev;
7663         int ret = 0;
7664
7665         ASSERT_RTNL();
7666
7667         if (dev == upper_dev)
7668                 return -EBUSY;
7669
7670         /* To prevent loops, check if dev is not upper device to upper_dev. */
7671         if (__netdev_has_upper_dev(upper_dev, dev))
7672                 return -EBUSY;
7673
7674         if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
7675                 return -EMLINK;
7676
7677         if (!master) {
7678                 if (__netdev_has_upper_dev(dev, upper_dev))
7679                         return -EEXIST;
7680         } else {
7681                 master_dev = __netdev_master_upper_dev_get(dev);
7682                 if (master_dev)
7683                         return master_dev == upper_dev ? -EEXIST : -EBUSY;
7684         }
7685
7686         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7687                                             &changeupper_info.info);
7688         ret = notifier_to_errno(ret);
7689         if (ret)
7690                 return ret;
7691
7692         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
7693                                                    master);
7694         if (ret)
7695                 return ret;
7696
7697         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7698                                             &changeupper_info.info);
7699         ret = notifier_to_errno(ret);
7700         if (ret)
7701                 goto rollback;
7702
7703         __netdev_update_upper_level(dev, NULL);
7704         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7705
7706         __netdev_update_lower_level(upper_dev, priv);
7707         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7708                                     priv);
7709
7710         return 0;
7711
7712 rollback:
7713         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7714
7715         return ret;
7716 }
7717
7718 /**
7719  * netdev_upper_dev_link - Add a link to the upper device
7720  * @dev: device
7721  * @upper_dev: new upper device
7722  * @extack: netlink extended ack
7723  *
7724  * Adds a link to device which is upper to this one. The caller must hold
7725  * the RTNL lock. On a failure a negative errno code is returned.
7726  * On success the reference counts are adjusted and the function
7727  * returns zero.
7728  */
7729 int netdev_upper_dev_link(struct net_device *dev,
7730                           struct net_device *upper_dev,
7731                           struct netlink_ext_ack *extack)
7732 {
7733         struct netdev_nested_priv priv = {
7734                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7735                 .data = NULL,
7736         };
7737
7738         return __netdev_upper_dev_link(dev, upper_dev, false,
7739                                        NULL, NULL, &priv, extack);
7740 }
7741 EXPORT_SYMBOL(netdev_upper_dev_link);
7742
7743 /**
7744  * netdev_master_upper_dev_link - Add a master link to the upper device
7745  * @dev: device
7746  * @upper_dev: new upper device
7747  * @upper_priv: upper device private
7748  * @upper_info: upper info to be passed down via notifier
7749  * @extack: netlink extended ack
7750  *
7751  * Adds a link to device which is upper to this one. In this case, only
7752  * one master upper device can be linked, although other non-master devices
7753  * might be linked as well. The caller must hold the RTNL lock.
7754  * On a failure a negative errno code is returned. On success the reference
7755  * counts are adjusted and the function returns zero.
7756  */
7757 int netdev_master_upper_dev_link(struct net_device *dev,
7758                                  struct net_device *upper_dev,
7759                                  void *upper_priv, void *upper_info,
7760                                  struct netlink_ext_ack *extack)
7761 {
7762         struct netdev_nested_priv priv = {
7763                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7764                 .data = NULL,
7765         };
7766
7767         return __netdev_upper_dev_link(dev, upper_dev, true,
7768                                        upper_priv, upper_info, &priv, extack);
7769 }
7770 EXPORT_SYMBOL(netdev_master_upper_dev_link);
7771
7772 static void __netdev_upper_dev_unlink(struct net_device *dev,
7773                                       struct net_device *upper_dev,
7774                                       struct netdev_nested_priv *priv)
7775 {
7776         struct netdev_notifier_changeupper_info changeupper_info = {
7777                 .info = {
7778                         .dev = dev,
7779                 },
7780                 .upper_dev = upper_dev,
7781                 .linking = false,
7782         };
7783
7784         ASSERT_RTNL();
7785
7786         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
7787
7788         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7789                                       &changeupper_info.info);
7790
7791         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7792
7793         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7794                                       &changeupper_info.info);
7795
7796         __netdev_update_upper_level(dev, NULL);
7797         __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
7798
7799         __netdev_update_lower_level(upper_dev, priv);
7800         __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
7801                                     priv);
7802 }
7803
7804 /**
7805  * netdev_upper_dev_unlink - Removes a link to upper device
7806  * @dev: device
7807  * @upper_dev: new upper device
7808  *
7809  * Removes a link to device which is upper to this one. The caller must hold
7810  * the RTNL lock.
7811  */
7812 void netdev_upper_dev_unlink(struct net_device *dev,
7813                              struct net_device *upper_dev)
7814 {
7815         struct netdev_nested_priv priv = {
7816                 .flags = NESTED_SYNC_TODO,
7817                 .data = NULL,
7818         };
7819
7820         __netdev_upper_dev_unlink(dev, upper_dev, &priv);
7821 }
7822 EXPORT_SYMBOL(netdev_upper_dev_unlink);
7823
7824 static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
7825                                       struct net_device *lower_dev,
7826                                       bool val)
7827 {
7828         struct netdev_adjacent *adj;
7829
7830         adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
7831         if (adj)
7832                 adj->ignore = val;
7833
7834         adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
7835         if (adj)
7836                 adj->ignore = val;
7837 }
7838
7839 static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
7840                                         struct net_device *lower_dev)
7841 {
7842         __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
7843 }
7844
7845 static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
7846                                        struct net_device *lower_dev)
7847 {
7848         __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
7849 }
7850
7851 int netdev_adjacent_change_prepare(struct net_device *old_dev,
7852                                    struct net_device *new_dev,
7853                                    struct net_device *dev,
7854                                    struct netlink_ext_ack *extack)
7855 {
7856         struct netdev_nested_priv priv = {
7857                 .flags = 0,
7858                 .data = NULL,
7859         };
7860         int err;
7861
7862         if (!new_dev)
7863                 return 0;
7864
7865         if (old_dev && new_dev != old_dev)
7866                 netdev_adjacent_dev_disable(dev, old_dev);
7867         err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
7868                                       extack);
7869         if (err) {
7870                 if (old_dev && new_dev != old_dev)
7871                         netdev_adjacent_dev_enable(dev, old_dev);
7872                 return err;
7873         }
7874
7875         return 0;
7876 }
7877 EXPORT_SYMBOL(netdev_adjacent_change_prepare);
7878
7879 void netdev_adjacent_change_commit(struct net_device *old_dev,
7880                                    struct net_device *new_dev,
7881                                    struct net_device *dev)
7882 {
7883         struct netdev_nested_priv priv = {
7884                 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
7885                 .data = NULL,
7886         };
7887
7888         if (!new_dev || !old_dev)
7889                 return;
7890
7891         if (new_dev == old_dev)
7892                 return;
7893
7894         netdev_adjacent_dev_enable(dev, old_dev);
7895         __netdev_upper_dev_unlink(old_dev, dev, &priv);
7896 }
7897 EXPORT_SYMBOL(netdev_adjacent_change_commit);
7898
7899 void netdev_adjacent_change_abort(struct net_device *old_dev,
7900                                   struct net_device *new_dev,
7901                                   struct net_device *dev)
7902 {
7903         struct netdev_nested_priv priv = {
7904                 .flags = 0,
7905                 .data = NULL,
7906         };
7907
7908         if (!new_dev)
7909                 return;
7910
7911         if (old_dev && new_dev != old_dev)
7912                 netdev_adjacent_dev_enable(dev, old_dev);
7913
7914         __netdev_upper_dev_unlink(new_dev, dev, &priv);
7915 }
7916 EXPORT_SYMBOL(netdev_adjacent_change_abort);
7917
7918 /**
7919  * netdev_bonding_info_change - Dispatch event about slave change
7920  * @dev: device
7921  * @bonding_info: info to dispatch
7922  *
7923  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7924  * The caller must hold the RTNL lock.
7925  */
7926 void netdev_bonding_info_change(struct net_device *dev,
7927                                 struct netdev_bonding_info *bonding_info)
7928 {
7929         struct netdev_notifier_bonding_info info = {
7930                 .info.dev = dev,
7931         };
7932
7933         memcpy(&info.bonding_info, bonding_info,
7934                sizeof(struct netdev_bonding_info));
7935         call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
7936                                       &info.info);
7937 }
7938 EXPORT_SYMBOL(netdev_bonding_info_change);
7939
7940 static int netdev_offload_xstats_enable_l3(struct net_device *dev,
7941                                            struct netlink_ext_ack *extack)
7942 {
7943         struct netdev_notifier_offload_xstats_info info = {
7944                 .info.dev = dev,
7945                 .info.extack = extack,
7946                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7947         };
7948         int err;
7949         int rc;
7950
7951         dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3),
7952                                          GFP_KERNEL);
7953         if (!dev->offload_xstats_l3)
7954                 return -ENOMEM;
7955
7956         rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
7957                                                   NETDEV_OFFLOAD_XSTATS_DISABLE,
7958                                                   &info.info);
7959         err = notifier_to_errno(rc);
7960         if (err)
7961                 goto free_stats;
7962
7963         return 0;
7964
7965 free_stats:
7966         kfree(dev->offload_xstats_l3);
7967         dev->offload_xstats_l3 = NULL;
7968         return err;
7969 }
7970
7971 int netdev_offload_xstats_enable(struct net_device *dev,
7972                                  enum netdev_offload_xstats_type type,
7973                                  struct netlink_ext_ack *extack)
7974 {
7975         ASSERT_RTNL();
7976
7977         if (netdev_offload_xstats_enabled(dev, type))
7978                 return -EALREADY;
7979
7980         switch (type) {
7981         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
7982                 return netdev_offload_xstats_enable_l3(dev, extack);
7983         }
7984
7985         WARN_ON(1);
7986         return -EINVAL;
7987 }
7988 EXPORT_SYMBOL(netdev_offload_xstats_enable);
7989
7990 static void netdev_offload_xstats_disable_l3(struct net_device *dev)
7991 {
7992         struct netdev_notifier_offload_xstats_info info = {
7993                 .info.dev = dev,
7994                 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
7995         };
7996
7997         call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
7998                                       &info.info);
7999         kfree(dev->offload_xstats_l3);
8000         dev->offload_xstats_l3 = NULL;
8001 }
8002
8003 int netdev_offload_xstats_disable(struct net_device *dev,
8004                                   enum netdev_offload_xstats_type type)
8005 {
8006         ASSERT_RTNL();
8007
8008         if (!netdev_offload_xstats_enabled(dev, type))
8009                 return -EALREADY;
8010
8011         switch (type) {
8012         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8013                 netdev_offload_xstats_disable_l3(dev);
8014                 return 0;
8015         }
8016
8017         WARN_ON(1);
8018         return -EINVAL;
8019 }
8020 EXPORT_SYMBOL(netdev_offload_xstats_disable);
8021
8022 static void netdev_offload_xstats_disable_all(struct net_device *dev)
8023 {
8024         netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
8025 }
8026
8027 static struct rtnl_hw_stats64 *
8028 netdev_offload_xstats_get_ptr(const struct net_device *dev,
8029                               enum netdev_offload_xstats_type type)
8030 {
8031         switch (type) {
8032         case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
8033                 return dev->offload_xstats_l3;
8034         }
8035
8036         WARN_ON(1);
8037         return NULL;
8038 }
8039
8040 bool netdev_offload_xstats_enabled(const struct net_device *dev,
8041                                    enum netdev_offload_xstats_type type)
8042 {
8043         ASSERT_RTNL();
8044
8045         return netdev_offload_xstats_get_ptr(dev, type);
8046 }
8047 EXPORT_SYMBOL(netdev_offload_xstats_enabled);
8048
8049 struct netdev_notifier_offload_xstats_ru {
8050         bool used;
8051 };
8052
8053 struct netdev_notifier_offload_xstats_rd {
8054         struct rtnl_hw_stats64 stats;
8055         bool used;
8056 };
8057
8058 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
8059                                   const struct rtnl_hw_stats64 *src)
8060 {
8061         dest->rx_packets          += src->rx_packets;
8062         dest->tx_packets          += src->tx_packets;
8063         dest->rx_bytes            += src->rx_bytes;
8064         dest->tx_bytes            += src->tx_bytes;
8065         dest->rx_errors           += src->rx_errors;
8066         dest->tx_errors           += src->tx_errors;
8067         dest->rx_dropped          += src->rx_dropped;
8068         dest->tx_dropped          += src->tx_dropped;
8069         dest->multicast           += src->multicast;
8070 }
8071
8072 static int netdev_offload_xstats_get_used(struct net_device *dev,
8073                                           enum netdev_offload_xstats_type type,
8074                                           bool *p_used,
8075                                           struct netlink_ext_ack *extack)
8076 {
8077         struct netdev_notifier_offload_xstats_ru report_used = {};
8078         struct netdev_notifier_offload_xstats_info info = {
8079                 .info.dev = dev,
8080                 .info.extack = extack,
8081                 .type = type,
8082                 .report_used = &report_used,
8083         };
8084         int rc;
8085
8086         WARN_ON(!netdev_offload_xstats_enabled(dev, type));
8087         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
8088                                            &info.info);
8089         *p_used = report_used.used;
8090         return notifier_to_errno(rc);
8091 }
8092
8093 static int netdev_offload_xstats_get_stats(struct net_device *dev,
8094                                            enum netdev_offload_xstats_type type,
8095                                            struct rtnl_hw_stats64 *p_stats,
8096                                            bool *p_used,
8097                                            struct netlink_ext_ack *extack)
8098 {
8099         struct netdev_notifier_offload_xstats_rd report_delta = {};
8100         struct netdev_notifier_offload_xstats_info info = {
8101                 .info.dev = dev,
8102                 .info.extack = extack,
8103                 .type = type,
8104                 .report_delta = &report_delta,
8105         };
8106         struct rtnl_hw_stats64 *stats;
8107         int rc;
8108
8109         stats = netdev_offload_xstats_get_ptr(dev, type);
8110         if (WARN_ON(!stats))
8111                 return -EINVAL;
8112
8113         rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
8114                                            &info.info);
8115
8116         /* Cache whatever we got, even if there was an error, otherwise the
8117          * successful stats retrievals would get lost.
8118          */
8119         netdev_hw_stats64_add(stats, &report_delta.stats);
8120
8121         if (p_stats)
8122                 *p_stats = *stats;
8123         *p_used = report_delta.used;
8124
8125         return notifier_to_errno(rc);
8126 }
8127
8128 int netdev_offload_xstats_get(struct net_device *dev,
8129                               enum netdev_offload_xstats_type type,
8130                               struct rtnl_hw_stats64 *p_stats, bool *p_used,
8131                               struct netlink_ext_ack *extack)
8132 {
8133         ASSERT_RTNL();
8134
8135         if (p_stats)
8136                 return netdev_offload_xstats_get_stats(dev, type, p_stats,
8137                                                        p_used, extack);
8138         else
8139                 return netdev_offload_xstats_get_used(dev, type, p_used,
8140                                                       extack);
8141 }
8142 EXPORT_SYMBOL(netdev_offload_xstats_get);
8143
8144 void
8145 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
8146                                    const struct rtnl_hw_stats64 *stats)
8147 {
8148         report_delta->used = true;
8149         netdev_hw_stats64_add(&report_delta->stats, stats);
8150 }
8151 EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
8152
8153 void
8154 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
8155 {
8156         report_used->used = true;
8157 }
8158 EXPORT_SYMBOL(netdev_offload_xstats_report_used);
8159
8160 void netdev_offload_xstats_push_delta(struct net_device *dev,
8161                                       enum netdev_offload_xstats_type type,
8162                                       const struct rtnl_hw_stats64 *p_stats)
8163 {
8164         struct rtnl_hw_stats64 *stats;
8165
8166         ASSERT_RTNL();
8167
8168         stats = netdev_offload_xstats_get_ptr(dev, type);
8169         if (WARN_ON(!stats))
8170                 return;
8171
8172         netdev_hw_stats64_add(stats, p_stats);
8173 }
8174 EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
8175
8176 /**
8177  * netdev_get_xmit_slave - Get the xmit slave of master device
8178  * @dev: device
8179  * @skb: The packet
8180  * @all_slaves: assume all the slaves are active
8181  *
8182  * The reference counters are not incremented so the caller must be
8183  * careful with locks. The caller must hold RCU lock.
8184  * %NULL is returned if no slave is found.
8185  */
8186
8187 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8188                                          struct sk_buff *skb,
8189                                          bool all_slaves)
8190 {
8191         const struct net_device_ops *ops = dev->netdev_ops;
8192
8193         if (!ops->ndo_get_xmit_slave)
8194                 return NULL;
8195         return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8196 }
8197 EXPORT_SYMBOL(netdev_get_xmit_slave);
8198
8199 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8200                                                   struct sock *sk)
8201 {
8202         const struct net_device_ops *ops = dev->netdev_ops;
8203
8204         if (!ops->ndo_sk_get_lower_dev)
8205                 return NULL;
8206         return ops->ndo_sk_get_lower_dev(dev, sk);
8207 }
8208
8209 /**
8210  * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8211  * @dev: device
8212  * @sk: the socket
8213  *
8214  * %NULL is returned if no lower device is found.
8215  */
8216
8217 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8218                                             struct sock *sk)
8219 {
8220         struct net_device *lower;
8221
8222         lower = netdev_sk_get_lower_dev(dev, sk);
8223         while (lower) {
8224                 dev = lower;
8225                 lower = netdev_sk_get_lower_dev(dev, sk);
8226         }
8227
8228         return dev;
8229 }
8230 EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8231
8232 static void netdev_adjacent_add_links(struct net_device *dev)
8233 {
8234         struct netdev_adjacent *iter;
8235
8236         struct net *net = dev_net(dev);
8237
8238         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8239                 if (!net_eq(net, dev_net(iter->dev)))
8240                         continue;
8241                 netdev_adjacent_sysfs_add(iter->dev, dev,
8242                                           &iter->dev->adj_list.lower);
8243                 netdev_adjacent_sysfs_add(dev, iter->dev,
8244                                           &dev->adj_list.upper);
8245         }
8246
8247         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8248                 if (!net_eq(net, dev_net(iter->dev)))
8249                         continue;
8250                 netdev_adjacent_sysfs_add(iter->dev, dev,
8251                                           &iter->dev->adj_list.upper);
8252                 netdev_adjacent_sysfs_add(dev, iter->dev,
8253                                           &dev->adj_list.lower);
8254         }
8255 }
8256
8257 static void netdev_adjacent_del_links(struct net_device *dev)
8258 {
8259         struct netdev_adjacent *iter;
8260
8261         struct net *net = dev_net(dev);
8262
8263         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8264                 if (!net_eq(net, dev_net(iter->dev)))
8265                         continue;
8266                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8267                                           &iter->dev->adj_list.lower);
8268                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8269                                           &dev->adj_list.upper);
8270         }
8271
8272         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8273                 if (!net_eq(net, dev_net(iter->dev)))
8274                         continue;
8275                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8276                                           &iter->dev->adj_list.upper);
8277                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8278                                           &dev->adj_list.lower);
8279         }
8280 }
8281
8282 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
8283 {
8284         struct netdev_adjacent *iter;
8285
8286         struct net *net = dev_net(dev);
8287
8288         list_for_each_entry(iter, &dev->adj_list.upper, list) {
8289                 if (!net_eq(net, dev_net(iter->dev)))
8290                         continue;
8291                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8292                                           &iter->dev->adj_list.lower);
8293                 netdev_adjacent_sysfs_add(iter->dev, dev,
8294                                           &iter->dev->adj_list.lower);
8295         }
8296
8297         list_for_each_entry(iter, &dev->adj_list.lower, list) {
8298                 if (!net_eq(net, dev_net(iter->dev)))
8299                         continue;
8300                 netdev_adjacent_sysfs_del(iter->dev, oldname,
8301                                           &iter->dev->adj_list.upper);
8302                 netdev_adjacent_sysfs_add(iter->dev, dev,
8303                                           &iter->dev->adj_list.upper);
8304         }
8305 }
8306
8307 void *netdev_lower_dev_get_private(struct net_device *dev,
8308                                    struct net_device *lower_dev)
8309 {
8310         struct netdev_adjacent *lower;
8311
8312         if (!lower_dev)
8313                 return NULL;
8314         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
8315         if (!lower)
8316                 return NULL;
8317
8318         return lower->private;
8319 }
8320 EXPORT_SYMBOL(netdev_lower_dev_get_private);
8321
8322
8323 /**
8324  * netdev_lower_state_changed - Dispatch event about lower device state change
8325  * @lower_dev: device
8326  * @lower_state_info: state to dispatch
8327  *
8328  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8329  * The caller must hold the RTNL lock.
8330  */
8331 void netdev_lower_state_changed(struct net_device *lower_dev,
8332                                 void *lower_state_info)
8333 {
8334         struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8335                 .info.dev = lower_dev,
8336         };
8337
8338         ASSERT_RTNL();
8339         changelowerstate_info.lower_state_info = lower_state_info;
8340         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
8341                                       &changelowerstate_info.info);
8342 }
8343 EXPORT_SYMBOL(netdev_lower_state_changed);
8344
8345 static void dev_change_rx_flags(struct net_device *dev, int flags)
8346 {
8347         const struct net_device_ops *ops = dev->netdev_ops;
8348
8349         if (ops->ndo_change_rx_flags)
8350                 ops->ndo_change_rx_flags(dev, flags);
8351 }
8352
8353 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
8354 {
8355         unsigned int old_flags = dev->flags;
8356         kuid_t uid;
8357         kgid_t gid;
8358
8359         ASSERT_RTNL();
8360
8361         dev->flags |= IFF_PROMISC;
8362         dev->promiscuity += inc;
8363         if (dev->promiscuity == 0) {
8364                 /*
8365                  * Avoid overflow.
8366                  * If inc causes overflow, untouch promisc and return error.
8367                  */
8368                 if (inc < 0)
8369                         dev->flags &= ~IFF_PROMISC;
8370                 else {
8371                         dev->promiscuity -= inc;
8372                         netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
8373                         return -EOVERFLOW;
8374                 }
8375         }
8376         if (dev->flags != old_flags) {
8377                 pr_info("device %s %s promiscuous mode\n",
8378                         dev->name,
8379                         dev->flags & IFF_PROMISC ? "entered" : "left");
8380                 if (audit_enabled) {
8381                         current_uid_gid(&uid, &gid);
8382                         audit_log(audit_context(), GFP_ATOMIC,
8383                                   AUDIT_ANOM_PROMISCUOUS,
8384                                   "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8385                                   dev->name, (dev->flags & IFF_PROMISC),
8386                                   (old_flags & IFF_PROMISC),
8387                                   from_kuid(&init_user_ns, audit_get_loginuid(current)),
8388                                   from_kuid(&init_user_ns, uid),
8389                                   from_kgid(&init_user_ns, gid),
8390                                   audit_get_sessionid(current));
8391                 }
8392
8393                 dev_change_rx_flags(dev, IFF_PROMISC);
8394         }
8395         if (notify)
8396                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
8397         return 0;
8398 }
8399
8400 /**
8401  *      dev_set_promiscuity     - update promiscuity count on a device
8402  *      @dev: device
8403  *      @inc: modifier
8404  *
8405  *      Add or remove promiscuity from a device. While the count in the device
8406  *      remains above zero the interface remains promiscuous. Once it hits zero
8407  *      the device reverts back to normal filtering operation. A negative inc
8408  *      value is used to drop promiscuity on the device.
8409  *      Return 0 if successful or a negative errno code on error.
8410  */
8411 int dev_set_promiscuity(struct net_device *dev, int inc)
8412 {
8413         unsigned int old_flags = dev->flags;
8414         int err;
8415
8416         err = __dev_set_promiscuity(dev, inc, true);
8417         if (err < 0)
8418                 return err;
8419         if (dev->flags != old_flags)
8420                 dev_set_rx_mode(dev);
8421         return err;
8422 }
8423 EXPORT_SYMBOL(dev_set_promiscuity);
8424
8425 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
8426 {
8427         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
8428
8429         ASSERT_RTNL();
8430
8431         dev->flags |= IFF_ALLMULTI;
8432         dev->allmulti += inc;
8433         if (dev->allmulti == 0) {
8434                 /*
8435                  * Avoid overflow.
8436                  * If inc causes overflow, untouch allmulti and return error.
8437                  */
8438                 if (inc < 0)
8439                         dev->flags &= ~IFF_ALLMULTI;
8440                 else {
8441                         dev->allmulti -= inc;
8442                         netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
8443                         return -EOVERFLOW;
8444                 }
8445         }
8446         if (dev->flags ^ old_flags) {
8447                 dev_change_rx_flags(dev, IFF_ALLMULTI);
8448                 dev_set_rx_mode(dev);
8449                 if (notify)
8450                         __dev_notify_flags(dev, old_flags,
8451                                            dev->gflags ^ old_gflags);
8452         }
8453         return 0;
8454 }
8455
8456 /**
8457  *      dev_set_allmulti        - update allmulti count on a device
8458  *      @dev: device
8459  *      @inc: modifier
8460  *
8461  *      Add or remove reception of all multicast frames to a device. While the
8462  *      count in the device remains above zero the interface remains listening
8463  *      to all interfaces. Once it hits zero the device reverts back to normal
8464  *      filtering operation. A negative @inc value is used to drop the counter
8465  *      when releasing a resource needing all multicasts.
8466  *      Return 0 if successful or a negative errno code on error.
8467  */
8468
8469 int dev_set_allmulti(struct net_device *dev, int inc)
8470 {
8471         return __dev_set_allmulti(dev, inc, true);
8472 }
8473 EXPORT_SYMBOL(dev_set_allmulti);
8474
8475 /*
8476  *      Upload unicast and multicast address lists to device and
8477  *      configure RX filtering. When the device doesn't support unicast
8478  *      filtering it is put in promiscuous mode while unicast addresses
8479  *      are present.
8480  */
8481 void __dev_set_rx_mode(struct net_device *dev)
8482 {
8483         const struct net_device_ops *ops = dev->netdev_ops;
8484
8485         /* dev_open will call this function so the list will stay sane. */
8486         if (!(dev->flags&IFF_UP))
8487                 return;
8488
8489         if (!netif_device_present(dev))
8490                 return;
8491
8492         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
8493                 /* Unicast addresses changes may only happen under the rtnl,
8494                  * therefore calling __dev_set_promiscuity here is safe.
8495                  */
8496                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
8497                         __dev_set_promiscuity(dev, 1, false);
8498                         dev->uc_promisc = true;
8499                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
8500                         __dev_set_promiscuity(dev, -1, false);
8501                         dev->uc_promisc = false;
8502                 }
8503         }
8504
8505         if (ops->ndo_set_rx_mode)
8506                 ops->ndo_set_rx_mode(dev);
8507 }
8508
8509 void dev_set_rx_mode(struct net_device *dev)
8510 {
8511         netif_addr_lock_bh(dev);
8512         __dev_set_rx_mode(dev);
8513         netif_addr_unlock_bh(dev);
8514 }
8515
8516 /**
8517  *      dev_get_flags - get flags reported to userspace
8518  *      @dev: device
8519  *
8520  *      Get the combination of flag bits exported through APIs to userspace.
8521  */
8522 unsigned int dev_get_flags(const struct net_device *dev)
8523 {
8524         unsigned int flags;
8525
8526         flags = (dev->flags & ~(IFF_PROMISC |
8527                                 IFF_ALLMULTI |
8528                                 IFF_RUNNING |
8529                                 IFF_LOWER_UP |
8530                                 IFF_DORMANT)) |
8531                 (dev->gflags & (IFF_PROMISC |
8532                                 IFF_ALLMULTI));
8533
8534         if (netif_running(dev)) {
8535                 if (netif_oper_up(dev))
8536                         flags |= IFF_RUNNING;
8537                 if (netif_carrier_ok(dev))
8538                         flags |= IFF_LOWER_UP;
8539                 if (netif_dormant(dev))
8540                         flags |= IFF_DORMANT;
8541         }
8542
8543         return flags;
8544 }
8545 EXPORT_SYMBOL(dev_get_flags);
8546
8547 int __dev_change_flags(struct net_device *dev, unsigned int flags,
8548                        struct netlink_ext_ack *extack)
8549 {
8550         unsigned int old_flags = dev->flags;
8551         int ret;
8552
8553         ASSERT_RTNL();
8554
8555         /*
8556          *      Set the flags on our device.
8557          */
8558
8559         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8560                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8561                                IFF_AUTOMEDIA)) |
8562                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8563                                     IFF_ALLMULTI));
8564
8565         /*
8566          *      Load in the correct multicast list now the flags have changed.
8567          */
8568
8569         if ((old_flags ^ flags) & IFF_MULTICAST)
8570                 dev_change_rx_flags(dev, IFF_MULTICAST);
8571
8572         dev_set_rx_mode(dev);
8573
8574         /*
8575          *      Have we downed the interface. We handle IFF_UP ourselves
8576          *      according to user attempts to set it, rather than blindly
8577          *      setting it.
8578          */
8579
8580         ret = 0;
8581         if ((old_flags ^ flags) & IFF_UP) {
8582                 if (old_flags & IFF_UP)
8583                         __dev_close(dev);
8584                 else
8585                         ret = __dev_open(dev, extack);
8586         }
8587
8588         if ((flags ^ dev->gflags) & IFF_PROMISC) {
8589                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
8590                 unsigned int old_flags = dev->flags;
8591
8592                 dev->gflags ^= IFF_PROMISC;
8593
8594                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8595                         if (dev->flags != old_flags)
8596                                 dev_set_rx_mode(dev);
8597         }
8598
8599         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
8600          * is important. Some (broken) drivers set IFF_PROMISC, when
8601          * IFF_ALLMULTI is requested not asking us and not reporting.
8602          */
8603         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
8604                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8605
8606                 dev->gflags ^= IFF_ALLMULTI;
8607                 __dev_set_allmulti(dev, inc, false);
8608         }
8609
8610         return ret;
8611 }
8612
8613 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8614                         unsigned int gchanges)
8615 {
8616         unsigned int changes = dev->flags ^ old_flags;
8617
8618         if (gchanges)
8619                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
8620
8621         if (changes & IFF_UP) {
8622                 if (dev->flags & IFF_UP)
8623                         call_netdevice_notifiers(NETDEV_UP, dev);
8624                 else
8625                         call_netdevice_notifiers(NETDEV_DOWN, dev);
8626         }
8627
8628         if (dev->flags & IFF_UP &&
8629             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
8630                 struct netdev_notifier_change_info change_info = {
8631                         .info = {
8632                                 .dev = dev,
8633                         },
8634                         .flags_changed = changes,
8635                 };
8636
8637                 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
8638         }
8639 }
8640
8641 /**
8642  *      dev_change_flags - change device settings
8643  *      @dev: device
8644  *      @flags: device state flags
8645  *      @extack: netlink extended ack
8646  *
8647  *      Change settings on device based state flags. The flags are
8648  *      in the userspace exported format.
8649  */
8650 int dev_change_flags(struct net_device *dev, unsigned int flags,
8651                      struct netlink_ext_ack *extack)
8652 {
8653         int ret;
8654         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
8655
8656         ret = __dev_change_flags(dev, flags, extack);
8657         if (ret < 0)
8658                 return ret;
8659
8660         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
8661         __dev_notify_flags(dev, old_flags, changes);
8662         return ret;
8663 }
8664 EXPORT_SYMBOL(dev_change_flags);
8665
8666 int __dev_set_mtu(struct net_device *dev, int new_mtu)
8667 {
8668         const struct net_device_ops *ops = dev->netdev_ops;
8669
8670         if (ops->ndo_change_mtu)
8671                 return ops->ndo_change_mtu(dev, new_mtu);
8672
8673         /* Pairs with all the lockless reads of dev->mtu in the stack */
8674         WRITE_ONCE(dev->mtu, new_mtu);
8675         return 0;
8676 }
8677 EXPORT_SYMBOL(__dev_set_mtu);
8678
8679 int dev_validate_mtu(struct net_device *dev, int new_mtu,
8680                      struct netlink_ext_ack *extack)
8681 {
8682         /* MTU must be positive, and in range */
8683         if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8684                 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8685                 return -EINVAL;
8686         }
8687
8688         if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8689                 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8690                 return -EINVAL;
8691         }
8692         return 0;
8693 }
8694
8695 /**
8696  *      dev_set_mtu_ext - Change maximum transfer unit
8697  *      @dev: device
8698  *      @new_mtu: new transfer unit
8699  *      @extack: netlink extended ack
8700  *
8701  *      Change the maximum transfer size of the network device.
8702  */
8703 int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8704                     struct netlink_ext_ack *extack)
8705 {
8706         int err, orig_mtu;
8707
8708         if (new_mtu == dev->mtu)
8709                 return 0;
8710
8711         err = dev_validate_mtu(dev, new_mtu, extack);
8712         if (err)
8713                 return err;
8714
8715         if (!netif_device_present(dev))
8716                 return -ENODEV;
8717
8718         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8719         err = notifier_to_errno(err);
8720         if (err)
8721                 return err;
8722
8723         orig_mtu = dev->mtu;
8724         err = __dev_set_mtu(dev, new_mtu);
8725
8726         if (!err) {
8727                 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8728                                                    orig_mtu);
8729                 err = notifier_to_errno(err);
8730                 if (err) {
8731                         /* setting mtu back and notifying everyone again,
8732                          * so that they have a chance to revert changes.
8733                          */
8734                         __dev_set_mtu(dev, orig_mtu);
8735                         call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8736                                                      new_mtu);
8737                 }
8738         }
8739         return err;
8740 }
8741
8742 int dev_set_mtu(struct net_device *dev, int new_mtu)
8743 {
8744         struct netlink_ext_ack extack;
8745         int err;
8746
8747         memset(&extack, 0, sizeof(extack));
8748         err = dev_set_mtu_ext(dev, new_mtu, &extack);
8749         if (err && extack._msg)
8750                 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
8751         return err;
8752 }
8753 EXPORT_SYMBOL(dev_set_mtu);
8754
8755 /**
8756  *      dev_change_tx_queue_len - Change TX queue length of a netdevice
8757  *      @dev: device
8758  *      @new_len: new tx queue length
8759  */
8760 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
8761 {
8762         unsigned int orig_len = dev->tx_queue_len;
8763         int res;
8764
8765         if (new_len != (unsigned int)new_len)
8766                 return -ERANGE;
8767
8768         if (new_len != orig_len) {
8769                 dev->tx_queue_len = new_len;
8770                 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
8771                 res = notifier_to_errno(res);
8772                 if (res)
8773                         goto err_rollback;
8774                 res = dev_qdisc_change_tx_queue_len(dev);
8775                 if (res)
8776                         goto err_rollback;
8777         }
8778
8779         return 0;
8780
8781 err_rollback:
8782         netdev_err(dev, "refused to change device tx_queue_len\n");
8783         dev->tx_queue_len = orig_len;
8784         return res;
8785 }
8786
8787 /**
8788  *      dev_set_group - Change group this device belongs to
8789  *      @dev: device
8790  *      @new_group: group this device should belong to
8791  */
8792 void dev_set_group(struct net_device *dev, int new_group)
8793 {
8794         dev->group = new_group;
8795 }
8796
8797 /**
8798  *      dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
8799  *      @dev: device
8800  *      @addr: new address
8801  *      @extack: netlink extended ack
8802  */
8803 int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
8804                               struct netlink_ext_ack *extack)
8805 {
8806         struct netdev_notifier_pre_changeaddr_info info = {
8807                 .info.dev = dev,
8808                 .info.extack = extack,
8809                 .dev_addr = addr,
8810         };
8811         int rc;
8812
8813         rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
8814         return notifier_to_errno(rc);
8815 }
8816 EXPORT_SYMBOL(dev_pre_changeaddr_notify);
8817
8818 /**
8819  *      dev_set_mac_address - Change Media Access Control Address
8820  *      @dev: device
8821  *      @sa: new address
8822  *      @extack: netlink extended ack
8823  *
8824  *      Change the hardware (MAC) address of the device
8825  */
8826 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
8827                         struct netlink_ext_ack *extack)
8828 {
8829         const struct net_device_ops *ops = dev->netdev_ops;
8830         int err;
8831
8832         if (!ops->ndo_set_mac_address)
8833                 return -EOPNOTSUPP;
8834         if (sa->sa_family != dev->type)
8835                 return -EINVAL;
8836         if (!netif_device_present(dev))
8837                 return -ENODEV;
8838         err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
8839         if (err)
8840                 return err;
8841         err = ops->ndo_set_mac_address(dev, sa);
8842         if (err)
8843                 return err;
8844         dev->addr_assign_type = NET_ADDR_SET;
8845         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
8846         add_device_randomness(dev->dev_addr, dev->addr_len);
8847         return 0;
8848 }
8849 EXPORT_SYMBOL(dev_set_mac_address);
8850
8851 static DECLARE_RWSEM(dev_addr_sem);
8852
8853 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
8854                              struct netlink_ext_ack *extack)
8855 {
8856         int ret;
8857
8858         down_write(&dev_addr_sem);
8859         ret = dev_set_mac_address(dev, sa, extack);
8860         up_write(&dev_addr_sem);
8861         return ret;
8862 }
8863 EXPORT_SYMBOL(dev_set_mac_address_user);
8864
8865 int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
8866 {
8867         size_t size = sizeof(sa->sa_data_min);
8868         struct net_device *dev;
8869         int ret = 0;
8870
8871         down_read(&dev_addr_sem);
8872         rcu_read_lock();
8873
8874         dev = dev_get_by_name_rcu(net, dev_name);
8875         if (!dev) {
8876                 ret = -ENODEV;
8877                 goto unlock;
8878         }
8879         if (!dev->addr_len)
8880                 memset(sa->sa_data, 0, size);
8881         else
8882                 memcpy(sa->sa_data, dev->dev_addr,
8883                        min_t(size_t, size, dev->addr_len));
8884         sa->sa_family = dev->type;
8885
8886 unlock:
8887         rcu_read_unlock();
8888         up_read(&dev_addr_sem);
8889         return ret;
8890 }
8891 EXPORT_SYMBOL(dev_get_mac_address);
8892
8893 /**
8894  *      dev_change_carrier - Change device carrier
8895  *      @dev: device
8896  *      @new_carrier: new value
8897  *
8898  *      Change device carrier
8899  */
8900 int dev_change_carrier(struct net_device *dev, bool new_carrier)
8901 {
8902         const struct net_device_ops *ops = dev->netdev_ops;
8903
8904         if (!ops->ndo_change_carrier)
8905                 return -EOPNOTSUPP;
8906         if (!netif_device_present(dev))
8907                 return -ENODEV;
8908         return ops->ndo_change_carrier(dev, new_carrier);
8909 }
8910
8911 /**
8912  *      dev_get_phys_port_id - Get device physical port ID
8913  *      @dev: device
8914  *      @ppid: port ID
8915  *
8916  *      Get device physical port ID
8917  */
8918 int dev_get_phys_port_id(struct net_device *dev,
8919                          struct netdev_phys_item_id *ppid)
8920 {
8921         const struct net_device_ops *ops = dev->netdev_ops;
8922
8923         if (!ops->ndo_get_phys_port_id)
8924                 return -EOPNOTSUPP;
8925         return ops->ndo_get_phys_port_id(dev, ppid);
8926 }
8927
8928 /**
8929  *      dev_get_phys_port_name - Get device physical port name
8930  *      @dev: device
8931  *      @name: port name
8932  *      @len: limit of bytes to copy to name
8933  *
8934  *      Get device physical port name
8935  */
8936 int dev_get_phys_port_name(struct net_device *dev,
8937                            char *name, size_t len)
8938 {
8939         const struct net_device_ops *ops = dev->netdev_ops;
8940         int err;
8941
8942         if (ops->ndo_get_phys_port_name) {
8943                 err = ops->ndo_get_phys_port_name(dev, name, len);
8944                 if (err != -EOPNOTSUPP)
8945                         return err;
8946         }
8947         return devlink_compat_phys_port_name_get(dev, name, len);
8948 }
8949
8950 /**
8951  *      dev_get_port_parent_id - Get the device's port parent identifier
8952  *      @dev: network device
8953  *      @ppid: pointer to a storage for the port's parent identifier
8954  *      @recurse: allow/disallow recursion to lower devices
8955  *
8956  *      Get the devices's port parent identifier
8957  */
8958 int dev_get_port_parent_id(struct net_device *dev,
8959                            struct netdev_phys_item_id *ppid,
8960                            bool recurse)
8961 {
8962         const struct net_device_ops *ops = dev->netdev_ops;
8963         struct netdev_phys_item_id first = { };
8964         struct net_device *lower_dev;
8965         struct list_head *iter;
8966         int err;
8967
8968         if (ops->ndo_get_port_parent_id) {
8969                 err = ops->ndo_get_port_parent_id(dev, ppid);
8970                 if (err != -EOPNOTSUPP)
8971                         return err;
8972         }
8973
8974         err = devlink_compat_switch_id_get(dev, ppid);
8975         if (!recurse || err != -EOPNOTSUPP)
8976                 return err;
8977
8978         netdev_for_each_lower_dev(dev, lower_dev, iter) {
8979                 err = dev_get_port_parent_id(lower_dev, ppid, true);
8980                 if (err)
8981                         break;
8982                 if (!first.id_len)
8983                         first = *ppid;
8984                 else if (memcmp(&first, ppid, sizeof(*ppid)))
8985                         return -EOPNOTSUPP;
8986         }
8987
8988         return err;
8989 }
8990 EXPORT_SYMBOL(dev_get_port_parent_id);
8991
8992 /**
8993  *      netdev_port_same_parent_id - Indicate if two network devices have
8994  *      the same port parent identifier
8995  *      @a: first network device
8996  *      @b: second network device
8997  */
8998 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
8999 {
9000         struct netdev_phys_item_id a_id = { };
9001         struct netdev_phys_item_id b_id = { };
9002
9003         if (dev_get_port_parent_id(a, &a_id, true) ||
9004             dev_get_port_parent_id(b, &b_id, true))
9005                 return false;
9006
9007         return netdev_phys_item_id_same(&a_id, &b_id);
9008 }
9009 EXPORT_SYMBOL(netdev_port_same_parent_id);
9010
9011 /**
9012  *      dev_change_proto_down - set carrier according to proto_down.
9013  *
9014  *      @dev: device
9015  *      @proto_down: new value
9016  */
9017 int dev_change_proto_down(struct net_device *dev, bool proto_down)
9018 {
9019         if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN))
9020                 return -EOPNOTSUPP;
9021         if (!netif_device_present(dev))
9022                 return -ENODEV;
9023         if (proto_down)
9024                 netif_carrier_off(dev);
9025         else
9026                 netif_carrier_on(dev);
9027         dev->proto_down = proto_down;
9028         return 0;
9029 }
9030
9031 /**
9032  *      dev_change_proto_down_reason - proto down reason
9033  *
9034  *      @dev: device
9035  *      @mask: proto down mask
9036  *      @value: proto down value
9037  */
9038 void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9039                                   u32 value)
9040 {
9041         int b;
9042
9043         if (!mask) {
9044                 dev->proto_down_reason = value;
9045         } else {
9046                 for_each_set_bit(b, &mask, 32) {
9047                         if (value & (1 << b))
9048                                 dev->proto_down_reason |= BIT(b);
9049                         else
9050                                 dev->proto_down_reason &= ~BIT(b);
9051                 }
9052         }
9053 }
9054
9055 struct bpf_xdp_link {
9056         struct bpf_link link;
9057         struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9058         int flags;
9059 };
9060
9061 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
9062 {
9063         if (flags & XDP_FLAGS_HW_MODE)
9064                 return XDP_MODE_HW;
9065         if (flags & XDP_FLAGS_DRV_MODE)
9066                 return XDP_MODE_DRV;
9067         if (flags & XDP_FLAGS_SKB_MODE)
9068                 return XDP_MODE_SKB;
9069         return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
9070 }
9071
9072 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9073 {
9074         switch (mode) {
9075         case XDP_MODE_SKB:
9076                 return generic_xdp_install;
9077         case XDP_MODE_DRV:
9078         case XDP_MODE_HW:
9079                 return dev->netdev_ops->ndo_bpf;
9080         default:
9081                 return NULL;
9082         }
9083 }
9084
9085 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9086                                          enum bpf_xdp_mode mode)
9087 {
9088         return dev->xdp_state[mode].link;
9089 }
9090
9091 static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9092                                      enum bpf_xdp_mode mode)
9093 {
9094         struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9095
9096         if (link)
9097                 return link->link.prog;
9098         return dev->xdp_state[mode].prog;
9099 }
9100
9101 u8 dev_xdp_prog_count(struct net_device *dev)
9102 {
9103         u8 count = 0;
9104         int i;
9105
9106         for (i = 0; i < __MAX_XDP_MODE; i++)
9107                 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9108                         count++;
9109         return count;
9110 }
9111 EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
9112
9113 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9114 {
9115         struct bpf_prog *prog = dev_xdp_prog(dev, mode);
9116
9117         return prog ? prog->aux->id : 0;
9118 }
9119
9120 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9121                              struct bpf_xdp_link *link)
9122 {
9123         dev->xdp_state[mode].link = link;
9124         dev->xdp_state[mode].prog = NULL;
9125 }
9126
9127 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9128                              struct bpf_prog *prog)
9129 {
9130         dev->xdp_state[mode].link = NULL;
9131         dev->xdp_state[mode].prog = prog;
9132 }
9133
9134 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9135                            bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9136                            u32 flags, struct bpf_prog *prog)
9137 {
9138         struct netdev_bpf xdp;
9139         int err;
9140
9141         memset(&xdp, 0, sizeof(xdp));
9142         xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
9143         xdp.extack = extack;
9144         xdp.flags = flags;
9145         xdp.prog = prog;
9146
9147         /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9148          * "moved" into driver), so they don't increment it on their own, but
9149          * they do decrement refcnt when program is detached or replaced.
9150          * Given net_device also owns link/prog, we need to bump refcnt here
9151          * to prevent drivers from underflowing it.
9152          */
9153         if (prog)
9154                 bpf_prog_inc(prog);
9155         err = bpf_op(dev, &xdp);
9156         if (err) {
9157                 if (prog)
9158                         bpf_prog_put(prog);
9159                 return err;
9160         }
9161
9162         if (mode != XDP_MODE_HW)
9163                 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
9164
9165         return 0;
9166 }
9167
9168 static void dev_xdp_uninstall(struct net_device *dev)
9169 {
9170         struct bpf_xdp_link *link;
9171         struct bpf_prog *prog;
9172         enum bpf_xdp_mode mode;
9173         bpf_op_t bpf_op;
9174
9175         ASSERT_RTNL();
9176
9177         for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9178                 prog = dev_xdp_prog(dev, mode);
9179                 if (!prog)
9180                         continue;
9181
9182                 bpf_op = dev_xdp_bpf_op(dev, mode);
9183                 if (!bpf_op)
9184                         continue;
9185
9186                 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9187
9188                 /* auto-detach link from net device */
9189                 link = dev_xdp_link(dev, mode);
9190                 if (link)
9191                         link->dev = NULL;
9192                 else
9193                         bpf_prog_put(prog);
9194
9195                 dev_xdp_set_link(dev, mode, NULL);
9196         }
9197 }
9198
9199 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
9200                           struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9201                           struct bpf_prog *old_prog, u32 flags)
9202 {
9203         unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
9204         struct bpf_prog *cur_prog;
9205         struct net_device *upper;
9206         struct list_head *iter;
9207         enum bpf_xdp_mode mode;
9208         bpf_op_t bpf_op;
9209         int err;
9210
9211         ASSERT_RTNL();
9212
9213         /* either link or prog attachment, never both */
9214         if (link && (new_prog || old_prog))
9215                 return -EINVAL;
9216         /* link supports only XDP mode flags */
9217         if (link && (flags & ~XDP_FLAGS_MODES)) {
9218                 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9219                 return -EINVAL;
9220         }
9221         /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9222         if (num_modes > 1) {
9223                 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9224                 return -EINVAL;
9225         }
9226         /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9227         if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9228                 NL_SET_ERR_MSG(extack,
9229                                "More than one program loaded, unset mode is ambiguous");
9230                 return -EINVAL;
9231         }
9232         /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9233         if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9234                 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9235                 return -EINVAL;
9236         }
9237
9238         mode = dev_xdp_mode(dev, flags);
9239         /* can't replace attached link */
9240         if (dev_xdp_link(dev, mode)) {
9241                 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9242                 return -EBUSY;
9243         }
9244
9245         /* don't allow if an upper device already has a program */
9246         netdev_for_each_upper_dev_rcu(dev, upper, iter) {
9247                 if (dev_xdp_prog_count(upper) > 0) {
9248                         NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
9249                         return -EEXIST;
9250                 }
9251         }
9252
9253         cur_prog = dev_xdp_prog(dev, mode);
9254         /* can't replace attached prog with link */
9255         if (link && cur_prog) {
9256                 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9257                 return -EBUSY;
9258         }
9259         if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9260                 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9261                 return -EEXIST;
9262         }
9263
9264         /* put effective new program into new_prog */
9265         if (link)
9266                 new_prog = link->link.prog;
9267
9268         if (new_prog) {
9269                 bool offload = mode == XDP_MODE_HW;
9270                 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9271                                                ? XDP_MODE_DRV : XDP_MODE_SKB;
9272
9273                 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9274                         NL_SET_ERR_MSG(extack, "XDP program already attached");
9275                         return -EBUSY;
9276                 }
9277                 if (!offload && dev_xdp_prog(dev, other_mode)) {
9278                         NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
9279                         return -EEXIST;
9280                 }
9281                 if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
9282                         NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
9283                         return -EINVAL;
9284                 }
9285                 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
9286                         NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
9287                         return -EINVAL;
9288                 }
9289                 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9290                         NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
9291                         return -EINVAL;
9292                 }
9293         }
9294
9295         /* don't call drivers if the effective program didn't change */
9296         if (new_prog != cur_prog) {
9297                 bpf_op = dev_xdp_bpf_op(dev, mode);
9298                 if (!bpf_op) {
9299                         NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9300                         return -EOPNOTSUPP;
9301                 }
9302
9303                 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9304                 if (err)
9305                         return err;
9306         }
9307
9308         if (link)
9309                 dev_xdp_set_link(dev, mode, link);
9310         else
9311                 dev_xdp_set_prog(dev, mode, new_prog);
9312         if (cur_prog)
9313                 bpf_prog_put(cur_prog);
9314
9315         return 0;
9316 }
9317
9318 static int dev_xdp_attach_link(struct net_device *dev,
9319                                struct netlink_ext_ack *extack,
9320                                struct bpf_xdp_link *link)
9321 {
9322         return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9323 }
9324
9325 static int dev_xdp_detach_link(struct net_device *dev,
9326                                struct netlink_ext_ack *extack,
9327                                struct bpf_xdp_link *link)
9328 {
9329         enum bpf_xdp_mode mode;
9330         bpf_op_t bpf_op;
9331
9332         ASSERT_RTNL();
9333
9334         mode = dev_xdp_mode(dev, link->flags);
9335         if (dev_xdp_link(dev, mode) != link)
9336                 return -EINVAL;
9337
9338         bpf_op = dev_xdp_bpf_op(dev, mode);
9339         WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9340         dev_xdp_set_link(dev, mode, NULL);
9341         return 0;
9342 }
9343
9344 static void bpf_xdp_link_release(struct bpf_link *link)
9345 {
9346         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9347
9348         rtnl_lock();
9349
9350         /* if racing with net_device's tear down, xdp_link->dev might be
9351          * already NULL, in which case link was already auto-detached
9352          */
9353         if (xdp_link->dev) {
9354                 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
9355                 xdp_link->dev = NULL;
9356         }
9357
9358         rtnl_unlock();
9359 }
9360
9361 static int bpf_xdp_link_detach(struct bpf_link *link)
9362 {
9363         bpf_xdp_link_release(link);
9364         return 0;
9365 }
9366
9367 static void bpf_xdp_link_dealloc(struct bpf_link *link)
9368 {
9369         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9370
9371         kfree(xdp_link);
9372 }
9373
9374 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9375                                      struct seq_file *seq)
9376 {
9377         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9378         u32 ifindex = 0;
9379
9380         rtnl_lock();
9381         if (xdp_link->dev)
9382                 ifindex = xdp_link->dev->ifindex;
9383         rtnl_unlock();
9384
9385         seq_printf(seq, "ifindex:\t%u\n", ifindex);
9386 }
9387
9388 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9389                                        struct bpf_link_info *info)
9390 {
9391         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9392         u32 ifindex = 0;
9393
9394         rtnl_lock();
9395         if (xdp_link->dev)
9396                 ifindex = xdp_link->dev->ifindex;
9397         rtnl_unlock();
9398
9399         info->xdp.ifindex = ifindex;
9400         return 0;
9401 }
9402
9403 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9404                                struct bpf_prog *old_prog)
9405 {
9406         struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9407         enum bpf_xdp_mode mode;
9408         bpf_op_t bpf_op;
9409         int err = 0;
9410
9411         rtnl_lock();
9412
9413         /* link might have been auto-released already, so fail */
9414         if (!xdp_link->dev) {
9415                 err = -ENOLINK;
9416                 goto out_unlock;
9417         }
9418
9419         if (old_prog && link->prog != old_prog) {
9420                 err = -EPERM;
9421                 goto out_unlock;
9422         }
9423         old_prog = link->prog;
9424         if (old_prog->type != new_prog->type ||
9425             old_prog->expected_attach_type != new_prog->expected_attach_type) {
9426                 err = -EINVAL;
9427                 goto out_unlock;
9428         }
9429
9430         if (old_prog == new_prog) {
9431                 /* no-op, don't disturb drivers */
9432                 bpf_prog_put(new_prog);
9433                 goto out_unlock;
9434         }
9435
9436         mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
9437         bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9438         err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9439                               xdp_link->flags, new_prog);
9440         if (err)
9441                 goto out_unlock;
9442
9443         old_prog = xchg(&link->prog, new_prog);
9444         bpf_prog_put(old_prog);
9445
9446 out_unlock:
9447         rtnl_unlock();
9448         return err;
9449 }
9450
9451 static const struct bpf_link_ops bpf_xdp_link_lops = {
9452         .release = bpf_xdp_link_release,
9453         .dealloc = bpf_xdp_link_dealloc,
9454         .detach = bpf_xdp_link_detach,
9455         .show_fdinfo = bpf_xdp_link_show_fdinfo,
9456         .fill_link_info = bpf_xdp_link_fill_link_info,
9457         .update_prog = bpf_xdp_link_update,
9458 };
9459
9460 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9461 {
9462         struct net *net = current->nsproxy->net_ns;
9463         struct bpf_link_primer link_primer;
9464         struct bpf_xdp_link *link;
9465         struct net_device *dev;
9466         int err, fd;
9467
9468         rtnl_lock();
9469         dev = dev_get_by_index(net, attr->link_create.target_ifindex);
9470         if (!dev) {
9471                 rtnl_unlock();
9472                 return -EINVAL;
9473         }
9474
9475         link = kzalloc(sizeof(*link), GFP_USER);
9476         if (!link) {
9477                 err = -ENOMEM;
9478                 goto unlock;
9479         }
9480
9481         bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9482         link->dev = dev;
9483         link->flags = attr->link_create.flags;
9484
9485         err = bpf_link_prime(&link->link, &link_primer);
9486         if (err) {
9487                 kfree(link);
9488                 goto unlock;
9489         }
9490
9491         err = dev_xdp_attach_link(dev, NULL, link);
9492         rtnl_unlock();
9493
9494         if (err) {
9495                 link->dev = NULL;
9496                 bpf_link_cleanup(&link_primer);
9497                 goto out_put_dev;
9498         }
9499
9500         fd = bpf_link_settle(&link_primer);
9501         /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9502         dev_put(dev);
9503         return fd;
9504
9505 unlock:
9506         rtnl_unlock();
9507
9508 out_put_dev:
9509         dev_put(dev);
9510         return err;
9511 }
9512
9513 /**
9514  *      dev_change_xdp_fd - set or clear a bpf program for a device rx path
9515  *      @dev: device
9516  *      @extack: netlink extended ack
9517  *      @fd: new program fd or negative value to clear
9518  *      @expected_fd: old program fd that userspace expects to replace or clear
9519  *      @flags: xdp-related flags
9520  *
9521  *      Set or clear a bpf program for a device
9522  */
9523 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9524                       int fd, int expected_fd, u32 flags)
9525 {
9526         enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
9527         struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9528         int err;
9529
9530         ASSERT_RTNL();
9531
9532         if (fd >= 0) {
9533                 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9534                                                  mode != XDP_MODE_SKB);
9535                 if (IS_ERR(new_prog))
9536                         return PTR_ERR(new_prog);
9537         }
9538
9539         if (expected_fd >= 0) {
9540                 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9541                                                  mode != XDP_MODE_SKB);
9542                 if (IS_ERR(old_prog)) {
9543                         err = PTR_ERR(old_prog);
9544                         old_prog = NULL;
9545                         goto err_out;
9546                 }
9547         }
9548
9549         err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
9550
9551 err_out:
9552         if (err && new_prog)
9553                 bpf_prog_put(new_prog);
9554         if (old_prog)
9555                 bpf_prog_put(old_prog);
9556         return err;
9557 }
9558
9559 /**
9560  *      dev_new_index   -       allocate an ifindex
9561  *      @net: the applicable net namespace
9562  *
9563  *      Returns a suitable unique value for a new device interface
9564  *      number.  The caller must hold the rtnl semaphore or the
9565  *      dev_base_lock to be sure it remains unique.
9566  */
9567 static int dev_new_index(struct net *net)
9568 {
9569         int ifindex = net->ifindex;
9570
9571         for (;;) {
9572                 if (++ifindex <= 0)
9573                         ifindex = 1;
9574                 if (!__dev_get_by_index(net, ifindex))
9575                         return net->ifindex = ifindex;
9576         }
9577 }
9578
9579 /* Delayed registration/unregisteration */
9580 LIST_HEAD(net_todo_list);
9581 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
9582
9583 static void net_set_todo(struct net_device *dev)
9584 {
9585         list_add_tail(&dev->todo_list, &net_todo_list);
9586         atomic_inc(&dev_net(dev)->dev_unreg_count);
9587 }
9588
9589 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9590         struct net_device *upper, netdev_features_t features)
9591 {
9592         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9593         netdev_features_t feature;
9594         int feature_bit;
9595
9596         for_each_netdev_feature(upper_disables, feature_bit) {
9597                 feature = __NETIF_F_BIT(feature_bit);
9598                 if (!(upper->wanted_features & feature)
9599                     && (features & feature)) {
9600                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9601                                    &feature, upper->name);
9602                         features &= ~feature;
9603                 }
9604         }
9605
9606         return features;
9607 }
9608
9609 static void netdev_sync_lower_features(struct net_device *upper,
9610         struct net_device *lower, netdev_features_t features)
9611 {
9612         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9613         netdev_features_t feature;
9614         int feature_bit;
9615
9616         for_each_netdev_feature(upper_disables, feature_bit) {
9617                 feature = __NETIF_F_BIT(feature_bit);
9618                 if (!(features & feature) && (lower->features & feature)) {
9619                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9620                                    &feature, lower->name);
9621                         lower->wanted_features &= ~feature;
9622                         __netdev_update_features(lower);
9623
9624                         if (unlikely(lower->features & feature))
9625                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9626                                             &feature, lower->name);
9627                         else
9628                                 netdev_features_change(lower);
9629                 }
9630         }
9631 }
9632
9633 static netdev_features_t netdev_fix_features(struct net_device *dev,
9634         netdev_features_t features)
9635 {
9636         /* Fix illegal checksum combinations */
9637         if ((features & NETIF_F_HW_CSUM) &&
9638             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
9639                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
9640                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9641         }
9642
9643         /* TSO requires that SG is present as well. */
9644         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
9645                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
9646                 features &= ~NETIF_F_ALL_TSO;
9647         }
9648
9649         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9650                                         !(features & NETIF_F_IP_CSUM)) {
9651                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9652                 features &= ~NETIF_F_TSO;
9653                 features &= ~NETIF_F_TSO_ECN;
9654         }
9655
9656         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9657                                          !(features & NETIF_F_IPV6_CSUM)) {
9658                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9659                 features &= ~NETIF_F_TSO6;
9660         }
9661
9662         /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9663         if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9664                 features &= ~NETIF_F_TSO_MANGLEID;
9665
9666         /* TSO ECN requires that TSO is present as well. */
9667         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9668                 features &= ~NETIF_F_TSO_ECN;
9669
9670         /* Software GSO depends on SG. */
9671         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
9672                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
9673                 features &= ~NETIF_F_GSO;
9674         }
9675
9676         /* GSO partial features require GSO partial be set */
9677         if ((features & dev->gso_partial_features) &&
9678             !(features & NETIF_F_GSO_PARTIAL)) {
9679                 netdev_dbg(dev,
9680                            "Dropping partially supported GSO features since no GSO partial.\n");
9681                 features &= ~dev->gso_partial_features;
9682         }
9683
9684         if (!(features & NETIF_F_RXCSUM)) {
9685                 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9686                  * successfully merged by hardware must also have the
9687                  * checksum verified by hardware.  If the user does not
9688                  * want to enable RXCSUM, logically, we should disable GRO_HW.
9689                  */
9690                 if (features & NETIF_F_GRO_HW) {
9691                         netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9692                         features &= ~NETIF_F_GRO_HW;
9693                 }
9694         }
9695
9696         /* LRO/HW-GRO features cannot be combined with RX-FCS */
9697         if (features & NETIF_F_RXFCS) {
9698                 if (features & NETIF_F_LRO) {
9699                         netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9700                         features &= ~NETIF_F_LRO;
9701                 }
9702
9703                 if (features & NETIF_F_GRO_HW) {
9704                         netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9705                         features &= ~NETIF_F_GRO_HW;
9706                 }
9707         }
9708
9709         if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
9710                 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
9711                 features &= ~NETIF_F_LRO;
9712         }
9713
9714         if (features & NETIF_F_HW_TLS_TX) {
9715                 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9716                         (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9717                 bool hw_csum = features & NETIF_F_HW_CSUM;
9718
9719                 if (!ip_csum && !hw_csum) {
9720                         netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9721                         features &= ~NETIF_F_HW_TLS_TX;
9722                 }
9723         }
9724
9725         if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9726                 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9727                 features &= ~NETIF_F_HW_TLS_RX;
9728         }
9729
9730         return features;
9731 }
9732
9733 int __netdev_update_features(struct net_device *dev)
9734 {
9735         struct net_device *upper, *lower;
9736         netdev_features_t features;
9737         struct list_head *iter;
9738         int err = -1;
9739
9740         ASSERT_RTNL();
9741
9742         features = netdev_get_wanted_features(dev);
9743
9744         if (dev->netdev_ops->ndo_fix_features)
9745                 features = dev->netdev_ops->ndo_fix_features(dev, features);
9746
9747         /* driver might be less strict about feature dependencies */
9748         features = netdev_fix_features(dev, features);
9749
9750         /* some features can't be enabled if they're off on an upper device */
9751         netdev_for_each_upper_dev_rcu(dev, upper, iter)
9752                 features = netdev_sync_upper_features(dev, upper, features);
9753
9754         if (dev->features == features)
9755                 goto sync_lower;
9756
9757         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
9758                 &dev->features, &features);
9759
9760         if (dev->netdev_ops->ndo_set_features)
9761                 err = dev->netdev_ops->ndo_set_features(dev, features);
9762         else
9763                 err = 0;
9764
9765         if (unlikely(err < 0)) {
9766                 netdev_err(dev,
9767                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
9768                         err, &features, &dev->features);
9769                 /* return non-0 since some features might have changed and
9770                  * it's better to fire a spurious notification than miss it
9771                  */
9772                 return -1;
9773         }
9774
9775 sync_lower:
9776         /* some features must be disabled on lower devices when disabled
9777          * on an upper device (think: bonding master or bridge)
9778          */
9779         netdev_for_each_lower_dev(dev, lower, iter)
9780                 netdev_sync_lower_features(dev, lower, features);
9781
9782         if (!err) {
9783                 netdev_features_t diff = features ^ dev->features;
9784
9785                 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
9786                         /* udp_tunnel_{get,drop}_rx_info both need
9787                          * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
9788                          * device, or they won't do anything.
9789                          * Thus we need to update dev->features
9790                          * *before* calling udp_tunnel_get_rx_info,
9791                          * but *after* calling udp_tunnel_drop_rx_info.
9792                          */
9793                         if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
9794                                 dev->features = features;
9795                                 udp_tunnel_get_rx_info(dev);
9796                         } else {
9797                                 udp_tunnel_drop_rx_info(dev);
9798                         }
9799                 }
9800
9801                 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
9802                         if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
9803                                 dev->features = features;
9804                                 err |= vlan_get_rx_ctag_filter_info(dev);
9805                         } else {
9806                                 vlan_drop_rx_ctag_filter_info(dev);
9807                         }
9808                 }
9809
9810                 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
9811                         if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
9812                                 dev->features = features;
9813                                 err |= vlan_get_rx_stag_filter_info(dev);
9814                         } else {
9815                                 vlan_drop_rx_stag_filter_info(dev);
9816                         }
9817                 }
9818
9819                 dev->features = features;
9820         }
9821
9822         return err < 0 ? 0 : 1;
9823 }
9824
9825 /**
9826  *      netdev_update_features - recalculate device features
9827  *      @dev: the device to check
9828  *
9829  *      Recalculate dev->features set and send notifications if it
9830  *      has changed. Should be called after driver or hardware dependent
9831  *      conditions might have changed that influence the features.
9832  */
9833 void netdev_update_features(struct net_device *dev)
9834 {
9835         if (__netdev_update_features(dev))
9836                 netdev_features_change(dev);
9837 }
9838 EXPORT_SYMBOL(netdev_update_features);
9839
9840 /**
9841  *      netdev_change_features - recalculate device features
9842  *      @dev: the device to check
9843  *
9844  *      Recalculate dev->features set and send notifications even
9845  *      if they have not changed. Should be called instead of
9846  *      netdev_update_features() if also dev->vlan_features might
9847  *      have changed to allow the changes to be propagated to stacked
9848  *      VLAN devices.
9849  */
9850 void netdev_change_features(struct net_device *dev)
9851 {
9852         __netdev_update_features(dev);
9853         netdev_features_change(dev);
9854 }
9855 EXPORT_SYMBOL(netdev_change_features);
9856
9857 /**
9858  *      netif_stacked_transfer_operstate -      transfer operstate
9859  *      @rootdev: the root or lower level device to transfer state from
9860  *      @dev: the device to transfer operstate to
9861  *
9862  *      Transfer operational state from root to device. This is normally
9863  *      called when a stacking relationship exists between the root
9864  *      device and the device(a leaf device).
9865  */
9866 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
9867                                         struct net_device *dev)
9868 {
9869         if (rootdev->operstate == IF_OPER_DORMANT)
9870                 netif_dormant_on(dev);
9871         else
9872                 netif_dormant_off(dev);
9873
9874         if (rootdev->operstate == IF_OPER_TESTING)
9875                 netif_testing_on(dev);
9876         else
9877                 netif_testing_off(dev);
9878
9879         if (netif_carrier_ok(rootdev))
9880                 netif_carrier_on(dev);
9881         else
9882                 netif_carrier_off(dev);
9883 }
9884 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
9885
9886 static int netif_alloc_rx_queues(struct net_device *dev)
9887 {
9888         unsigned int i, count = dev->num_rx_queues;
9889         struct netdev_rx_queue *rx;
9890         size_t sz = count * sizeof(*rx);
9891         int err = 0;
9892
9893         BUG_ON(count < 1);
9894
9895         rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9896         if (!rx)
9897                 return -ENOMEM;
9898
9899         dev->_rx = rx;
9900
9901         for (i = 0; i < count; i++) {
9902                 rx[i].dev = dev;
9903
9904                 /* XDP RX-queue setup */
9905                 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
9906                 if (err < 0)
9907                         goto err_rxq_info;
9908         }
9909         return 0;
9910
9911 err_rxq_info:
9912         /* Rollback successful reg's and free other resources */
9913         while (i--)
9914                 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
9915         kvfree(dev->_rx);
9916         dev->_rx = NULL;
9917         return err;
9918 }
9919
9920 static void netif_free_rx_queues(struct net_device *dev)
9921 {
9922         unsigned int i, count = dev->num_rx_queues;
9923
9924         /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
9925         if (!dev->_rx)
9926                 return;
9927
9928         for (i = 0; i < count; i++)
9929                 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
9930
9931         kvfree(dev->_rx);
9932 }
9933
9934 static void netdev_init_one_queue(struct net_device *dev,
9935                                   struct netdev_queue *queue, void *_unused)
9936 {
9937         /* Initialize queue lock */
9938         spin_lock_init(&queue->_xmit_lock);
9939         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
9940         queue->xmit_lock_owner = -1;
9941         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
9942         queue->dev = dev;
9943 #ifdef CONFIG_BQL
9944         dql_init(&queue->dql, HZ);
9945 #endif
9946 }
9947
9948 static void netif_free_tx_queues(struct net_device *dev)
9949 {
9950         kvfree(dev->_tx);
9951 }
9952
9953 static int netif_alloc_netdev_queues(struct net_device *dev)
9954 {
9955         unsigned int count = dev->num_tx_queues;
9956         struct netdev_queue *tx;
9957         size_t sz = count * sizeof(*tx);
9958
9959         if (count < 1 || count > 0xffff)
9960                 return -EINVAL;
9961
9962         tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
9963         if (!tx)
9964                 return -ENOMEM;
9965
9966         dev->_tx = tx;
9967
9968         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
9969         spin_lock_init(&dev->tx_global_lock);
9970
9971         return 0;
9972 }
9973
9974 void netif_tx_stop_all_queues(struct net_device *dev)
9975 {
9976         unsigned int i;
9977
9978         for (i = 0; i < dev->num_tx_queues; i++) {
9979                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
9980
9981                 netif_tx_stop_queue(txq);
9982         }
9983 }
9984 EXPORT_SYMBOL(netif_tx_stop_all_queues);
9985
9986 /**
9987  * register_netdevice() - register a network device
9988  * @dev: device to register
9989  *
9990  * Take a prepared network device structure and make it externally accessible.
9991  * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
9992  * Callers must hold the rtnl lock - you may want register_netdev()
9993  * instead of this.
9994  */
9995 int register_netdevice(struct net_device *dev)
9996 {
9997         int ret;
9998         struct net *net = dev_net(dev);
9999
10000         BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10001                      NETDEV_FEATURE_COUNT);
10002         BUG_ON(dev_boot_phase);
10003         ASSERT_RTNL();
10004
10005         might_sleep();
10006
10007         /* When net_device's are persistent, this will be fatal. */
10008         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
10009         BUG_ON(!net);
10010
10011         ret = ethtool_check_ops(dev->ethtool_ops);
10012         if (ret)
10013                 return ret;
10014
10015         spin_lock_init(&dev->addr_list_lock);
10016         netdev_set_addr_lockdep_class(dev);
10017
10018         ret = dev_get_valid_name(net, dev, dev->name);
10019         if (ret < 0)
10020                 goto out;
10021
10022         ret = -ENOMEM;
10023         dev->name_node = netdev_name_node_head_alloc(dev);
10024         if (!dev->name_node)
10025                 goto out;
10026
10027         /* Init, if this function is available */
10028         if (dev->netdev_ops->ndo_init) {
10029                 ret = dev->netdev_ops->ndo_init(dev);
10030                 if (ret) {
10031                         if (ret > 0)
10032                                 ret = -EIO;
10033                         goto err_free_name;
10034                 }
10035         }
10036
10037         if (((dev->hw_features | dev->features) &
10038              NETIF_F_HW_VLAN_CTAG_FILTER) &&
10039             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10040              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10041                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10042                 ret = -EINVAL;
10043                 goto err_uninit;
10044         }
10045
10046         ret = -EBUSY;
10047         if (!dev->ifindex)
10048                 dev->ifindex = dev_new_index(net);
10049         else if (__dev_get_by_index(net, dev->ifindex))
10050                 goto err_uninit;
10051
10052         /* Transfer changeable features to wanted_features and enable
10053          * software offloads (GSO and GRO).
10054          */
10055         dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
10056         dev->features |= NETIF_F_SOFT_FEATURES;
10057
10058         if (dev->udp_tunnel_nic_info) {
10059                 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10060                 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10061         }
10062
10063         dev->wanted_features = dev->features & dev->hw_features;
10064
10065         if (!(dev->flags & IFF_LOOPBACK))
10066                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
10067
10068         /* If IPv4 TCP segmentation offload is supported we should also
10069          * allow the device to enable segmenting the frame with the option
10070          * of ignoring a static IP ID value.  This doesn't enable the
10071          * feature itself but allows the user to enable it later.
10072          */
10073         if (dev->hw_features & NETIF_F_TSO)
10074                 dev->hw_features |= NETIF_F_TSO_MANGLEID;
10075         if (dev->vlan_features & NETIF_F_TSO)
10076                 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10077         if (dev->mpls_features & NETIF_F_TSO)
10078                 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10079         if (dev->hw_enc_features & NETIF_F_TSO)
10080                 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
10081
10082         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
10083          */
10084         dev->vlan_features |= NETIF_F_HIGHDMA;
10085
10086         /* Make NETIF_F_SG inheritable to tunnel devices.
10087          */
10088         dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
10089
10090         /* Make NETIF_F_SG inheritable to MPLS.
10091          */
10092         dev->mpls_features |= NETIF_F_SG;
10093
10094         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10095         ret = notifier_to_errno(ret);
10096         if (ret)
10097                 goto err_uninit;
10098
10099         ret = netdev_register_kobject(dev);
10100         write_lock(&dev_base_lock);
10101         dev->reg_state = ret ? NETREG_UNREGISTERED : NETREG_REGISTERED;
10102         write_unlock(&dev_base_lock);
10103         if (ret)
10104                 goto err_uninit;
10105
10106         __netdev_update_features(dev);
10107
10108         /*
10109          *      Default initial state at registry is that the
10110          *      device is present.
10111          */
10112
10113         set_bit(__LINK_STATE_PRESENT, &dev->state);
10114
10115         linkwatch_init_dev(dev);
10116
10117         dev_init_scheduler(dev);
10118
10119         netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
10120         list_netdevice(dev);
10121
10122         add_device_randomness(dev->dev_addr, dev->addr_len);
10123
10124         /* If the device has permanent device address, driver should
10125          * set dev_addr and also addr_assign_type should be set to
10126          * NET_ADDR_PERM (default value).
10127          */
10128         if (dev->addr_assign_type == NET_ADDR_PERM)
10129                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10130
10131         /* Notify protocols, that a new device appeared. */
10132         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
10133         ret = notifier_to_errno(ret);
10134         if (ret) {
10135                 /* Expect explicit free_netdev() on failure */
10136                 dev->needs_free_netdev = false;
10137                 unregister_netdevice_queue(dev, NULL);
10138                 goto out;
10139         }
10140         /*
10141          *      Prevent userspace races by waiting until the network
10142          *      device is fully setup before sending notifications.
10143          */
10144         if (!dev->rtnl_link_ops ||
10145             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10146                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
10147
10148 out:
10149         return ret;
10150
10151 err_uninit:
10152         if (dev->netdev_ops->ndo_uninit)
10153                 dev->netdev_ops->ndo_uninit(dev);
10154         if (dev->priv_destructor)
10155                 dev->priv_destructor(dev);
10156 err_free_name:
10157         netdev_name_node_free(dev->name_node);
10158         goto out;
10159 }
10160 EXPORT_SYMBOL(register_netdevice);
10161
10162 /**
10163  *      init_dummy_netdev       - init a dummy network device for NAPI
10164  *      @dev: device to init
10165  *
10166  *      This takes a network device structure and initialize the minimum
10167  *      amount of fields so it can be used to schedule NAPI polls without
10168  *      registering a full blown interface. This is to be used by drivers
10169  *      that need to tie several hardware interfaces to a single NAPI
10170  *      poll scheduler due to HW limitations.
10171  */
10172 int init_dummy_netdev(struct net_device *dev)
10173 {
10174         /* Clear everything. Note we don't initialize spinlocks
10175          * are they aren't supposed to be taken by any of the
10176          * NAPI code and this dummy netdev is supposed to be
10177          * only ever used for NAPI polls
10178          */
10179         memset(dev, 0, sizeof(struct net_device));
10180
10181         /* make sure we BUG if trying to hit standard
10182          * register/unregister code path
10183          */
10184         dev->reg_state = NETREG_DUMMY;
10185
10186         /* NAPI wants this */
10187         INIT_LIST_HEAD(&dev->napi_list);
10188
10189         /* a dummy interface is started by default */
10190         set_bit(__LINK_STATE_PRESENT, &dev->state);
10191         set_bit(__LINK_STATE_START, &dev->state);
10192
10193         /* napi_busy_loop stats accounting wants this */
10194         dev_net_set(dev, &init_net);
10195
10196         /* Note : We dont allocate pcpu_refcnt for dummy devices,
10197          * because users of this 'device' dont need to change
10198          * its refcount.
10199          */
10200
10201         return 0;
10202 }
10203 EXPORT_SYMBOL_GPL(init_dummy_netdev);
10204
10205
10206 /**
10207  *      register_netdev - register a network device
10208  *      @dev: device to register
10209  *
10210  *      Take a completed network device structure and add it to the kernel
10211  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10212  *      chain. 0 is returned on success. A negative errno code is returned
10213  *      on a failure to set up the device, or if the name is a duplicate.
10214  *
10215  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
10216  *      and expands the device name if you passed a format string to
10217  *      alloc_netdev.
10218  */
10219 int register_netdev(struct net_device *dev)
10220 {
10221         int err;
10222
10223         if (rtnl_lock_killable())
10224                 return -EINTR;
10225         err = register_netdevice(dev);
10226         rtnl_unlock();
10227         return err;
10228 }
10229 EXPORT_SYMBOL(register_netdev);
10230
10231 int netdev_refcnt_read(const struct net_device *dev)
10232 {
10233 #ifdef CONFIG_PCPU_DEV_REFCNT
10234         int i, refcnt = 0;
10235
10236         for_each_possible_cpu(i)
10237                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10238         return refcnt;
10239 #else
10240         return refcount_read(&dev->dev_refcnt);
10241 #endif
10242 }
10243 EXPORT_SYMBOL(netdev_refcnt_read);
10244
10245 int netdev_unregister_timeout_secs __read_mostly = 10;
10246
10247 #define WAIT_REFS_MIN_MSECS 1
10248 #define WAIT_REFS_MAX_MSECS 250
10249 /**
10250  * netdev_wait_allrefs_any - wait until all references are gone.
10251  * @list: list of net_devices to wait on
10252  *
10253  * This is called when unregistering network devices.
10254  *
10255  * Any protocol or device that holds a reference should register
10256  * for netdevice notification, and cleanup and put back the
10257  * reference if they receive an UNREGISTER event.
10258  * We can get stuck here if buggy protocols don't correctly
10259  * call dev_put.
10260  */
10261 static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
10262 {
10263         unsigned long rebroadcast_time, warning_time;
10264         struct net_device *dev;
10265         int wait = 0;
10266
10267         rebroadcast_time = warning_time = jiffies;
10268
10269         list_for_each_entry(dev, list, todo_list)
10270                 if (netdev_refcnt_read(dev) == 1)
10271                         return dev;
10272
10273         while (true) {
10274                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
10275                         rtnl_lock();
10276
10277                         /* Rebroadcast unregister notification */
10278                         list_for_each_entry(dev, list, todo_list)
10279                                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10280
10281                         __rtnl_unlock();
10282                         rcu_barrier();
10283                         rtnl_lock();
10284
10285                         list_for_each_entry(dev, list, todo_list)
10286                                 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10287                                              &dev->state)) {
10288                                         /* We must not have linkwatch events
10289                                          * pending on unregister. If this
10290                                          * happens, we simply run the queue
10291                                          * unscheduled, resulting in a noop
10292                                          * for this device.
10293                                          */
10294                                         linkwatch_run_queue();
10295                                         break;
10296                                 }
10297
10298                         __rtnl_unlock();
10299
10300                         rebroadcast_time = jiffies;
10301                 }
10302
10303                 if (!wait) {
10304                         rcu_barrier();
10305                         wait = WAIT_REFS_MIN_MSECS;
10306                 } else {
10307                         msleep(wait);
10308                         wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10309                 }
10310
10311                 list_for_each_entry(dev, list, todo_list)
10312                         if (netdev_refcnt_read(dev) == 1)
10313                                 return dev;
10314
10315                 if (time_after(jiffies, warning_time +
10316                                READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
10317                         list_for_each_entry(dev, list, todo_list) {
10318                                 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10319                                          dev->name, netdev_refcnt_read(dev));
10320                                 ref_tracker_dir_print(&dev->refcnt_tracker, 10);
10321                         }
10322
10323                         warning_time = jiffies;
10324                 }
10325         }
10326 }
10327
10328 /* The sequence is:
10329  *
10330  *      rtnl_lock();
10331  *      ...
10332  *      register_netdevice(x1);
10333  *      register_netdevice(x2);
10334  *      ...
10335  *      unregister_netdevice(y1);
10336  *      unregister_netdevice(y2);
10337  *      ...
10338  *      rtnl_unlock();
10339  *      free_netdev(y1);
10340  *      free_netdev(y2);
10341  *
10342  * We are invoked by rtnl_unlock().
10343  * This allows us to deal with problems:
10344  * 1) We can delete sysfs objects which invoke hotplug
10345  *    without deadlocking with linkwatch via keventd.
10346  * 2) Since we run with the RTNL semaphore not held, we can sleep
10347  *    safely in order to wait for the netdev refcnt to drop to zero.
10348  *
10349  * We must not return until all unregister events added during
10350  * the interval the lock was held have been completed.
10351  */
10352 void netdev_run_todo(void)
10353 {
10354         struct net_device *dev, *tmp;
10355         struct list_head list;
10356 #ifdef CONFIG_LOCKDEP
10357         struct list_head unlink_list;
10358
10359         list_replace_init(&net_unlink_list, &unlink_list);
10360
10361         while (!list_empty(&unlink_list)) {
10362                 struct net_device *dev = list_first_entry(&unlink_list,
10363                                                           struct net_device,
10364                                                           unlink_list);
10365                 list_del_init(&dev->unlink_list);
10366                 dev->nested_level = dev->lower_level - 1;
10367         }
10368 #endif
10369
10370         /* Snapshot list, allow later requests */
10371         list_replace_init(&net_todo_list, &list);
10372
10373         __rtnl_unlock();
10374
10375         /* Wait for rcu callbacks to finish before next phase */
10376         if (!list_empty(&list))
10377                 rcu_barrier();
10378
10379         list_for_each_entry_safe(dev, tmp, &list, todo_list) {
10380                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
10381                         netdev_WARN(dev, "run_todo but not unregistering\n");
10382                         list_del(&dev->todo_list);
10383                         continue;
10384                 }
10385
10386                 write_lock(&dev_base_lock);
10387                 dev->reg_state = NETREG_UNREGISTERED;
10388                 write_unlock(&dev_base_lock);
10389                 linkwatch_forget_dev(dev);
10390         }
10391
10392         while (!list_empty(&list)) {
10393                 dev = netdev_wait_allrefs_any(&list);
10394                 list_del(&dev->todo_list);
10395
10396                 /* paranoia */
10397                 BUG_ON(netdev_refcnt_read(dev) != 1);
10398                 BUG_ON(!list_empty(&dev->ptype_all));
10399                 BUG_ON(!list_empty(&dev->ptype_specific));
10400                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10401                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
10402
10403                 if (dev->priv_destructor)
10404                         dev->priv_destructor(dev);
10405                 if (dev->needs_free_netdev)
10406                         free_netdev(dev);
10407
10408                 if (atomic_dec_and_test(&dev_net(dev)->dev_unreg_count))
10409                         wake_up(&netdev_unregistering_wq);
10410
10411                 /* Free network device */
10412                 kobject_put(&dev->dev.kobj);
10413         }
10414 }
10415
10416 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10417  * all the same fields in the same order as net_device_stats, with only
10418  * the type differing, but rtnl_link_stats64 may have additional fields
10419  * at the end for newer counters.
10420  */
10421 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10422                              const struct net_device_stats *netdev_stats)
10423 {
10424         size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
10425         const atomic_long_t *src = (atomic_long_t *)netdev_stats;
10426         u64 *dst = (u64 *)stats64;
10427
10428         BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
10429         for (i = 0; i < n; i++)
10430                 dst[i] = (unsigned long)atomic_long_read(&src[i]);
10431         /* zero out counters that only exist in rtnl_link_stats64 */
10432         memset((char *)stats64 + n * sizeof(u64), 0,
10433                sizeof(*stats64) - n * sizeof(u64));
10434 }
10435 EXPORT_SYMBOL(netdev_stats_to_stats64);
10436
10437 struct net_device_core_stats __percpu *netdev_core_stats_alloc(struct net_device *dev)
10438 {
10439         struct net_device_core_stats __percpu *p;
10440
10441         p = alloc_percpu_gfp(struct net_device_core_stats,
10442                              GFP_ATOMIC | __GFP_NOWARN);
10443
10444         if (p && cmpxchg(&dev->core_stats, NULL, p))
10445                 free_percpu(p);
10446
10447         /* This READ_ONCE() pairs with the cmpxchg() above */
10448         return READ_ONCE(dev->core_stats);
10449 }
10450 EXPORT_SYMBOL(netdev_core_stats_alloc);
10451
10452 /**
10453  *      dev_get_stats   - get network device statistics
10454  *      @dev: device to get statistics from
10455  *      @storage: place to store stats
10456  *
10457  *      Get network statistics from device. Return @storage.
10458  *      The device driver may provide its own method by setting
10459  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10460  *      otherwise the internal statistics structure is used.
10461  */
10462 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10463                                         struct rtnl_link_stats64 *storage)
10464 {
10465         const struct net_device_ops *ops = dev->netdev_ops;
10466         const struct net_device_core_stats __percpu *p;
10467
10468         if (ops->ndo_get_stats64) {
10469                 memset(storage, 0, sizeof(*storage));
10470                 ops->ndo_get_stats64(dev, storage);
10471         } else if (ops->ndo_get_stats) {
10472                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
10473         } else {
10474                 netdev_stats_to_stats64(storage, &dev->stats);
10475         }
10476
10477         /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
10478         p = READ_ONCE(dev->core_stats);
10479         if (p) {
10480                 const struct net_device_core_stats *core_stats;
10481                 int i;
10482
10483                 for_each_possible_cpu(i) {
10484                         core_stats = per_cpu_ptr(p, i);
10485                         storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
10486                         storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
10487                         storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
10488                         storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
10489                 }
10490         }
10491         return storage;
10492 }
10493 EXPORT_SYMBOL(dev_get_stats);
10494
10495 /**
10496  *      dev_fetch_sw_netstats - get per-cpu network device statistics
10497  *      @s: place to store stats
10498  *      @netstats: per-cpu network stats to read from
10499  *
10500  *      Read per-cpu network statistics and populate the related fields in @s.
10501  */
10502 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10503                            const struct pcpu_sw_netstats __percpu *netstats)
10504 {
10505         int cpu;
10506
10507         for_each_possible_cpu(cpu) {
10508                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
10509                 const struct pcpu_sw_netstats *stats;
10510                 unsigned int start;
10511
10512                 stats = per_cpu_ptr(netstats, cpu);
10513                 do {
10514                         start = u64_stats_fetch_begin_irq(&stats->syncp);
10515                         rx_packets = u64_stats_read(&stats->rx_packets);
10516                         rx_bytes   = u64_stats_read(&stats->rx_bytes);
10517                         tx_packets = u64_stats_read(&stats->tx_packets);
10518                         tx_bytes   = u64_stats_read(&stats->tx_bytes);
10519                 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
10520
10521                 s->rx_packets += rx_packets;
10522                 s->rx_bytes   += rx_bytes;
10523                 s->tx_packets += tx_packets;
10524                 s->tx_bytes   += tx_bytes;
10525         }
10526 }
10527 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10528
10529 /**
10530  *      dev_get_tstats64 - ndo_get_stats64 implementation
10531  *      @dev: device to get statistics from
10532  *      @s: place to store stats
10533  *
10534  *      Populate @s from dev->stats and dev->tstats. Can be used as
10535  *      ndo_get_stats64() callback.
10536  */
10537 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10538 {
10539         netdev_stats_to_stats64(s, &dev->stats);
10540         dev_fetch_sw_netstats(s, dev->tstats);
10541 }
10542 EXPORT_SYMBOL_GPL(dev_get_tstats64);
10543
10544 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
10545 {
10546         struct netdev_queue *queue = dev_ingress_queue(dev);
10547
10548 #ifdef CONFIG_NET_CLS_ACT
10549         if (queue)
10550                 return queue;
10551         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10552         if (!queue)
10553                 return NULL;
10554         netdev_init_one_queue(dev, queue, NULL);
10555         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
10556         RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
10557         rcu_assign_pointer(dev->ingress_queue, queue);
10558 #endif
10559         return queue;
10560 }
10561
10562 static const struct ethtool_ops default_ethtool_ops;
10563
10564 void netdev_set_default_ethtool_ops(struct net_device *dev,
10565                                     const struct ethtool_ops *ops)
10566 {
10567         if (dev->ethtool_ops == &default_ethtool_ops)
10568                 dev->ethtool_ops = ops;
10569 }
10570 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10571
10572 void netdev_freemem(struct net_device *dev)
10573 {
10574         char *addr = (char *)dev - dev->padded;
10575
10576         kvfree(addr);
10577 }
10578
10579 /**
10580  * alloc_netdev_mqs - allocate network device
10581  * @sizeof_priv: size of private data to allocate space for
10582  * @name: device name format string
10583  * @name_assign_type: origin of device name
10584  * @setup: callback to initialize device
10585  * @txqs: the number of TX subqueues to allocate
10586  * @rxqs: the number of RX subqueues to allocate
10587  *
10588  * Allocates a struct net_device with private data area for driver use
10589  * and performs basic initialization.  Also allocates subqueue structs
10590  * for each queue on the device.
10591  */
10592 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
10593                 unsigned char name_assign_type,
10594                 void (*setup)(struct net_device *),
10595                 unsigned int txqs, unsigned int rxqs)
10596 {
10597         struct net_device *dev;
10598         unsigned int alloc_size;
10599         struct net_device *p;
10600
10601         BUG_ON(strlen(name) >= sizeof(dev->name));
10602
10603         if (txqs < 1) {
10604                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
10605                 return NULL;
10606         }
10607
10608         if (rxqs < 1) {
10609                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
10610                 return NULL;
10611         }
10612
10613         alloc_size = sizeof(struct net_device);
10614         if (sizeof_priv) {
10615                 /* ensure 32-byte alignment of private area */
10616                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
10617                 alloc_size += sizeof_priv;
10618         }
10619         /* ensure 32-byte alignment of whole construct */
10620         alloc_size += NETDEV_ALIGN - 1;
10621
10622         p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
10623         if (!p)
10624                 return NULL;
10625
10626         dev = PTR_ALIGN(p, NETDEV_ALIGN);
10627         dev->padded = (char *)dev - (char *)p;
10628
10629         ref_tracker_dir_init(&dev->refcnt_tracker, 128);
10630 #ifdef CONFIG_PCPU_DEV_REFCNT
10631         dev->pcpu_refcnt = alloc_percpu(int);
10632         if (!dev->pcpu_refcnt)
10633                 goto free_dev;
10634         __dev_hold(dev);
10635 #else
10636         refcount_set(&dev->dev_refcnt, 1);
10637 #endif
10638
10639         if (dev_addr_init(dev))
10640                 goto free_pcpu;
10641
10642         dev_mc_init(dev);
10643         dev_uc_init(dev);
10644
10645         dev_net_set(dev, &init_net);
10646
10647         dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
10648         dev->gso_max_segs = GSO_MAX_SEGS;
10649         dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
10650         dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
10651         dev->tso_max_segs = TSO_MAX_SEGS;
10652         dev->upper_level = 1;
10653         dev->lower_level = 1;
10654 #ifdef CONFIG_LOCKDEP
10655         dev->nested_level = 0;
10656         INIT_LIST_HEAD(&dev->unlink_list);
10657 #endif
10658
10659         INIT_LIST_HEAD(&dev->napi_list);
10660         INIT_LIST_HEAD(&dev->unreg_list);
10661         INIT_LIST_HEAD(&dev->close_list);
10662         INIT_LIST_HEAD(&dev->link_watch_list);
10663         INIT_LIST_HEAD(&dev->adj_list.upper);
10664         INIT_LIST_HEAD(&dev->adj_list.lower);
10665         INIT_LIST_HEAD(&dev->ptype_all);
10666         INIT_LIST_HEAD(&dev->ptype_specific);
10667         INIT_LIST_HEAD(&dev->net_notifier_list);
10668 #ifdef CONFIG_NET_SCHED
10669         hash_init(dev->qdisc_hash);
10670 #endif
10671         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
10672         setup(dev);
10673
10674         if (!dev->tx_queue_len) {
10675                 dev->priv_flags |= IFF_NO_QUEUE;
10676                 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
10677         }
10678
10679         dev->num_tx_queues = txqs;
10680         dev->real_num_tx_queues = txqs;
10681         if (netif_alloc_netdev_queues(dev))
10682                 goto free_all;
10683
10684         dev->num_rx_queues = rxqs;
10685         dev->real_num_rx_queues = rxqs;
10686         if (netif_alloc_rx_queues(dev))
10687                 goto free_all;
10688
10689         strcpy(dev->name, name);
10690         dev->name_assign_type = name_assign_type;
10691         dev->group = INIT_NETDEV_GROUP;
10692         if (!dev->ethtool_ops)
10693                 dev->ethtool_ops = &default_ethtool_ops;
10694
10695         nf_hook_netdev_init(dev);
10696
10697         return dev;
10698
10699 free_all:
10700         free_netdev(dev);
10701         return NULL;
10702
10703 free_pcpu:
10704 #ifdef CONFIG_PCPU_DEV_REFCNT
10705         free_percpu(dev->pcpu_refcnt);
10706 free_dev:
10707 #endif
10708         netdev_freemem(dev);
10709         return NULL;
10710 }
10711 EXPORT_SYMBOL(alloc_netdev_mqs);
10712
10713 /**
10714  * free_netdev - free network device
10715  * @dev: device
10716  *
10717  * This function does the last stage of destroying an allocated device
10718  * interface. The reference to the device object is released. If this
10719  * is the last reference then it will be freed.Must be called in process
10720  * context.
10721  */
10722 void free_netdev(struct net_device *dev)
10723 {
10724         struct napi_struct *p, *n;
10725
10726         might_sleep();
10727
10728         /* When called immediately after register_netdevice() failed the unwind
10729          * handling may still be dismantling the device. Handle that case by
10730          * deferring the free.
10731          */
10732         if (dev->reg_state == NETREG_UNREGISTERING) {
10733                 ASSERT_RTNL();
10734                 dev->needs_free_netdev = true;
10735                 return;
10736         }
10737
10738         netif_free_tx_queues(dev);
10739         netif_free_rx_queues(dev);
10740
10741         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
10742
10743         /* Flush device addresses */
10744         dev_addr_flush(dev);
10745
10746         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10747                 netif_napi_del(p);
10748
10749         ref_tracker_dir_exit(&dev->refcnt_tracker);
10750 #ifdef CONFIG_PCPU_DEV_REFCNT
10751         free_percpu(dev->pcpu_refcnt);
10752         dev->pcpu_refcnt = NULL;
10753 #endif
10754         free_percpu(dev->core_stats);
10755         dev->core_stats = NULL;
10756         free_percpu(dev->xdp_bulkq);
10757         dev->xdp_bulkq = NULL;
10758
10759         /*  Compatibility with error handling in drivers */
10760         if (dev->reg_state == NETREG_UNINITIALIZED) {
10761                 netdev_freemem(dev);
10762                 return;
10763         }
10764
10765         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10766         dev->reg_state = NETREG_RELEASED;
10767
10768         /* will free via device release */
10769         put_device(&dev->dev);
10770 }
10771 EXPORT_SYMBOL(free_netdev);
10772
10773 /**
10774  *      synchronize_net -  Synchronize with packet receive processing
10775  *
10776  *      Wait for packets currently being received to be done.
10777  *      Does not block later packets from starting.
10778  */
10779 void synchronize_net(void)
10780 {
10781         might_sleep();
10782         if (rtnl_is_locked())
10783                 synchronize_rcu_expedited();
10784         else
10785                 synchronize_rcu();
10786 }
10787 EXPORT_SYMBOL(synchronize_net);
10788
10789 /**
10790  *      unregister_netdevice_queue - remove device from the kernel
10791  *      @dev: device
10792  *      @head: list
10793  *
10794  *      This function shuts down a device interface and removes it
10795  *      from the kernel tables.
10796  *      If head not NULL, device is queued to be unregistered later.
10797  *
10798  *      Callers must hold the rtnl semaphore.  You may want
10799  *      unregister_netdev() instead of this.
10800  */
10801
10802 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
10803 {
10804         ASSERT_RTNL();
10805
10806         if (head) {
10807                 list_move_tail(&dev->unreg_list, head);
10808         } else {
10809                 LIST_HEAD(single);
10810
10811                 list_add(&dev->unreg_list, &single);
10812                 unregister_netdevice_many(&single);
10813         }
10814 }
10815 EXPORT_SYMBOL(unregister_netdevice_queue);
10816
10817 /**
10818  *      unregister_netdevice_many - unregister many devices
10819  *      @head: list of devices
10820  *
10821  *  Note: As most callers use a stack allocated list_head,
10822  *  we force a list_del() to make sure stack wont be corrupted later.
10823  */
10824 void unregister_netdevice_many(struct list_head *head)
10825 {
10826         struct net_device *dev, *tmp;
10827         LIST_HEAD(close_head);
10828
10829         BUG_ON(dev_boot_phase);
10830         ASSERT_RTNL();
10831
10832         if (list_empty(head))
10833                 return;
10834
10835         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
10836                 /* Some devices call without registering
10837                  * for initialization unwind. Remove those
10838                  * devices and proceed with the remaining.
10839                  */
10840                 if (dev->reg_state == NETREG_UNINITIALIZED) {
10841                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
10842                                  dev->name, dev);
10843
10844                         WARN_ON(1);
10845                         list_del(&dev->unreg_list);
10846                         continue;
10847                 }
10848                 dev->dismantle = true;
10849                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
10850         }
10851
10852         /* If device is running, close it first. */
10853         list_for_each_entry(dev, head, unreg_list)
10854                 list_add_tail(&dev->close_list, &close_head);
10855         dev_close_many(&close_head, true);
10856
10857         list_for_each_entry(dev, head, unreg_list) {
10858                 /* And unlink it from device chain. */
10859                 write_lock(&dev_base_lock);
10860                 unlist_netdevice(dev, false);
10861                 dev->reg_state = NETREG_UNREGISTERING;
10862                 write_unlock(&dev_base_lock);
10863         }
10864         flush_all_backlogs();
10865
10866         synchronize_net();
10867
10868         list_for_each_entry(dev, head, unreg_list) {
10869                 struct sk_buff *skb = NULL;
10870
10871                 /* Shutdown queueing discipline. */
10872                 dev_shutdown(dev);
10873
10874                 dev_xdp_uninstall(dev);
10875
10876                 netdev_offload_xstats_disable_all(dev);
10877
10878                 /* Notify protocols, that we are about to destroy
10879                  * this device. They should clean all the things.
10880                  */
10881                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
10882
10883                 if (!dev->rtnl_link_ops ||
10884                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
10885                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
10886                                                      GFP_KERNEL, NULL, 0);
10887
10888                 /*
10889                  *      Flush the unicast and multicast chains
10890                  */
10891                 dev_uc_flush(dev);
10892                 dev_mc_flush(dev);
10893
10894                 netdev_name_node_alt_flush(dev);
10895                 netdev_name_node_free(dev->name_node);
10896
10897                 if (dev->netdev_ops->ndo_uninit)
10898                         dev->netdev_ops->ndo_uninit(dev);
10899
10900                 if (skb)
10901                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
10902
10903                 /* Notifier chain MUST detach us all upper devices. */
10904                 WARN_ON(netdev_has_any_upper_dev(dev));
10905                 WARN_ON(netdev_has_any_lower_dev(dev));
10906
10907                 /* Remove entries from kobject tree */
10908                 netdev_unregister_kobject(dev);
10909 #ifdef CONFIG_XPS
10910                 /* Remove XPS queueing entries */
10911                 netif_reset_xps_queues_gt(dev, 0);
10912 #endif
10913         }
10914
10915         synchronize_net();
10916
10917         list_for_each_entry(dev, head, unreg_list) {
10918                 netdev_put(dev, &dev->dev_registered_tracker);
10919                 net_set_todo(dev);
10920         }
10921
10922         list_del(head);
10923 }
10924 EXPORT_SYMBOL(unregister_netdevice_many);
10925
10926 /**
10927  *      unregister_netdev - remove device from the kernel
10928  *      @dev: device
10929  *
10930  *      This function shuts down a device interface and removes it
10931  *      from the kernel tables.
10932  *
10933  *      This is just a wrapper for unregister_netdevice that takes
10934  *      the rtnl semaphore.  In general you want to use this and not
10935  *      unregister_netdevice.
10936  */
10937 void unregister_netdev(struct net_device *dev)
10938 {
10939         rtnl_lock();
10940         unregister_netdevice(dev);
10941         rtnl_unlock();
10942 }
10943 EXPORT_SYMBOL(unregister_netdev);
10944
10945 /**
10946  *      __dev_change_net_namespace - move device to different nethost namespace
10947  *      @dev: device
10948  *      @net: network namespace
10949  *      @pat: If not NULL name pattern to try if the current device name
10950  *            is already taken in the destination network namespace.
10951  *      @new_ifindex: If not zero, specifies device index in the target
10952  *                    namespace.
10953  *
10954  *      This function shuts down a device interface and moves it
10955  *      to a new network namespace. On success 0 is returned, on
10956  *      a failure a netagive errno code is returned.
10957  *
10958  *      Callers must hold the rtnl semaphore.
10959  */
10960
10961 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
10962                                const char *pat, int new_ifindex)
10963 {
10964         struct netdev_name_node *name_node;
10965         struct net *net_old = dev_net(dev);
10966         char new_name[IFNAMSIZ] = {};
10967         int err, new_nsid;
10968
10969         ASSERT_RTNL();
10970
10971         /* Don't allow namespace local devices to be moved. */
10972         err = -EINVAL;
10973         if (dev->features & NETIF_F_NETNS_LOCAL)
10974                 goto out;
10975
10976         /* Ensure the device has been registrered */
10977         if (dev->reg_state != NETREG_REGISTERED)
10978                 goto out;
10979
10980         /* Get out if there is nothing todo */
10981         err = 0;
10982         if (net_eq(net_old, net))
10983                 goto out;
10984
10985         /* Pick the destination device name, and ensure
10986          * we can use it in the destination network namespace.
10987          */
10988         err = -EEXIST;
10989         if (netdev_name_in_use(net, dev->name)) {
10990                 /* We get here if we can't use the current device name */
10991                 if (!pat)
10992                         goto out;
10993                 err = dev_prep_valid_name(net, dev, pat, new_name);
10994                 if (err < 0)
10995                         goto out;
10996         }
10997         /* Check that none of the altnames conflicts. */
10998         err = -EEXIST;
10999         netdev_for_each_altname(dev, name_node)
11000                 if (netdev_name_in_use(net, name_node->name))
11001                         goto out;
11002
11003         /* Check that new_ifindex isn't used yet. */
11004         err = -EBUSY;
11005         if (new_ifindex && __dev_get_by_index(net, new_ifindex))
11006                 goto out;
11007
11008         /*
11009          * And now a mini version of register_netdevice unregister_netdevice.
11010          */
11011
11012         /* If device is running close it first. */
11013         dev_close(dev);
11014
11015         /* And unlink it from device chain */
11016         unlist_netdevice(dev, true);
11017
11018         synchronize_net();
11019
11020         /* Shutdown queueing discipline. */
11021         dev_shutdown(dev);
11022
11023         /* Notify protocols, that we are about to destroy
11024          * this device. They should clean all the things.
11025          *
11026          * Note that dev->reg_state stays at NETREG_REGISTERED.
11027          * This is wanted because this way 8021q and macvlan know
11028          * the device is just moving and can keep their slaves up.
11029          */
11030         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11031         rcu_barrier();
11032
11033         new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
11034         /* If there is an ifindex conflict assign a new one */
11035         if (!new_ifindex) {
11036                 if (__dev_get_by_index(net, dev->ifindex))
11037                         new_ifindex = dev_new_index(net);
11038                 else
11039                         new_ifindex = dev->ifindex;
11040         }
11041
11042         rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11043                             new_ifindex);
11044
11045         /*
11046          *      Flush the unicast and multicast chains
11047          */
11048         dev_uc_flush(dev);
11049         dev_mc_flush(dev);
11050
11051         /* Send a netdev-removed uevent to the old namespace */
11052         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
11053         netdev_adjacent_del_links(dev);
11054
11055         /* Move per-net netdevice notifiers that are following the netdevice */
11056         move_netdevice_notifiers_dev_net(dev, net);
11057
11058         /* Actually switch the network namespace */
11059         dev_net_set(dev, net);
11060         dev->ifindex = new_ifindex;
11061
11062         /* Send a netdev-add uevent to the new namespace */
11063         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
11064         netdev_adjacent_add_links(dev);
11065
11066         if (new_name[0]) /* Rename the netdev to prepared name */
11067                 strscpy(dev->name, new_name, IFNAMSIZ);
11068
11069         /* Fixup kobjects */
11070         err = device_rename(&dev->dev, dev->name);
11071         WARN_ON(err);
11072
11073         /* Adapt owner in case owning user namespace of target network
11074          * namespace is different from the original one.
11075          */
11076         err = netdev_change_owner(dev, net_old, net);
11077         WARN_ON(err);
11078
11079         /* Add the device back in the hashes */
11080         list_netdevice(dev);
11081
11082         /* Notify protocols, that a new device appeared. */
11083         call_netdevice_notifiers(NETDEV_REGISTER, dev);
11084
11085         /*
11086          *      Prevent userspace races by waiting until the network
11087          *      device is fully setup before sending notifications.
11088          */
11089         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
11090
11091         synchronize_net();
11092         err = 0;
11093 out:
11094         return err;
11095 }
11096 EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
11097
11098 static int dev_cpu_dead(unsigned int oldcpu)
11099 {
11100         struct sk_buff **list_skb;
11101         struct sk_buff *skb;
11102         unsigned int cpu;
11103         struct softnet_data *sd, *oldsd, *remsd = NULL;
11104
11105         local_irq_disable();
11106         cpu = smp_processor_id();
11107         sd = &per_cpu(softnet_data, cpu);
11108         oldsd = &per_cpu(softnet_data, oldcpu);
11109
11110         /* Find end of our completion_queue. */
11111         list_skb = &sd->completion_queue;
11112         while (*list_skb)
11113                 list_skb = &(*list_skb)->next;
11114         /* Append completion queue from offline CPU. */
11115         *list_skb = oldsd->completion_queue;
11116         oldsd->completion_queue = NULL;
11117
11118         /* Append output queue from offline CPU. */
11119         if (oldsd->output_queue) {
11120                 *sd->output_queue_tailp = oldsd->output_queue;
11121                 sd->output_queue_tailp = oldsd->output_queue_tailp;
11122                 oldsd->output_queue = NULL;
11123                 oldsd->output_queue_tailp = &oldsd->output_queue;
11124         }
11125         /* Append NAPI poll list from offline CPU, with one exception :
11126          * process_backlog() must be called by cpu owning percpu backlog.
11127          * We properly handle process_queue & input_pkt_queue later.
11128          */
11129         while (!list_empty(&oldsd->poll_list)) {
11130                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11131                                                             struct napi_struct,
11132                                                             poll_list);
11133
11134                 list_del_init(&napi->poll_list);
11135                 if (napi->poll == process_backlog)
11136                         napi->state = 0;
11137                 else
11138                         ____napi_schedule(sd, napi);
11139         }
11140
11141         raise_softirq_irqoff(NET_TX_SOFTIRQ);
11142         local_irq_enable();
11143
11144 #ifdef CONFIG_RPS
11145         remsd = oldsd->rps_ipi_list;
11146         oldsd->rps_ipi_list = NULL;
11147 #endif
11148         /* send out pending IPI's on offline CPU */
11149         net_rps_send_ipi(remsd);
11150
11151         /* Process offline CPU's input_pkt_queue */
11152         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
11153                 netif_rx(skb);
11154                 input_queue_head_incr(oldsd);
11155         }
11156         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
11157                 netif_rx(skb);
11158                 input_queue_head_incr(oldsd);
11159         }
11160
11161         return 0;
11162 }
11163
11164 /**
11165  *      netdev_increment_features - increment feature set by one
11166  *      @all: current feature set
11167  *      @one: new feature set
11168  *      @mask: mask feature set
11169  *
11170  *      Computes a new feature set after adding a device with feature set
11171  *      @one to the master device with current feature set @all.  Will not
11172  *      enable anything that is off in @mask. Returns the new feature set.
11173  */
11174 netdev_features_t netdev_increment_features(netdev_features_t all,
11175         netdev_features_t one, netdev_features_t mask)
11176 {
11177         if (mask & NETIF_F_HW_CSUM)
11178                 mask |= NETIF_F_CSUM_MASK;
11179         mask |= NETIF_F_VLAN_CHALLENGED;
11180
11181         all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
11182         all &= one | ~NETIF_F_ALL_FOR_ALL;
11183
11184         /* If one device supports hw checksumming, set for all. */
11185         if (all & NETIF_F_HW_CSUM)
11186                 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
11187
11188         return all;
11189 }
11190 EXPORT_SYMBOL(netdev_increment_features);
11191
11192 static struct hlist_head * __net_init netdev_create_hash(void)
11193 {
11194         int i;
11195         struct hlist_head *hash;
11196
11197         hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
11198         if (hash != NULL)
11199                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11200                         INIT_HLIST_HEAD(&hash[i]);
11201
11202         return hash;
11203 }
11204
11205 /* Initialize per network namespace state */
11206 static int __net_init netdev_init(struct net *net)
11207 {
11208         BUILD_BUG_ON(GRO_HASH_BUCKETS >
11209                      8 * sizeof_field(struct napi_struct, gro_bitmask));
11210
11211         INIT_LIST_HEAD(&net->dev_base_head);
11212
11213         net->dev_name_head = netdev_create_hash();
11214         if (net->dev_name_head == NULL)
11215                 goto err_name;
11216
11217         net->dev_index_head = netdev_create_hash();
11218         if (net->dev_index_head == NULL)
11219                 goto err_idx;
11220
11221         RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11222
11223         return 0;
11224
11225 err_idx:
11226         kfree(net->dev_name_head);
11227 err_name:
11228         return -ENOMEM;
11229 }
11230
11231 /**
11232  *      netdev_drivername - network driver for the device
11233  *      @dev: network device
11234  *
11235  *      Determine network driver for device.
11236  */
11237 const char *netdev_drivername(const struct net_device *dev)
11238 {
11239         const struct device_driver *driver;
11240         const struct device *parent;
11241         const char *empty = "";
11242
11243         parent = dev->dev.parent;
11244         if (!parent)
11245                 return empty;
11246
11247         driver = parent->driver;
11248         if (driver && driver->name)
11249                 return driver->name;
11250         return empty;
11251 }
11252
11253 static void __netdev_printk(const char *level, const struct net_device *dev,
11254                             struct va_format *vaf)
11255 {
11256         if (dev && dev->dev.parent) {
11257                 dev_printk_emit(level[1] - '0',
11258                                 dev->dev.parent,
11259                                 "%s %s %s%s: %pV",
11260                                 dev_driver_string(dev->dev.parent),
11261                                 dev_name(dev->dev.parent),
11262                                 netdev_name(dev), netdev_reg_state(dev),
11263                                 vaf);
11264         } else if (dev) {
11265                 printk("%s%s%s: %pV",
11266                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
11267         } else {
11268                 printk("%s(NULL net_device): %pV", level, vaf);
11269         }
11270 }
11271
11272 void netdev_printk(const char *level, const struct net_device *dev,
11273                    const char *format, ...)
11274 {
11275         struct va_format vaf;
11276         va_list args;
11277
11278         va_start(args, format);
11279
11280         vaf.fmt = format;
11281         vaf.va = &args;
11282
11283         __netdev_printk(level, dev, &vaf);
11284
11285         va_end(args);
11286 }
11287 EXPORT_SYMBOL(netdev_printk);
11288
11289 #define define_netdev_printk_level(func, level)                 \
11290 void func(const struct net_device *dev, const char *fmt, ...)   \
11291 {                                                               \
11292         struct va_format vaf;                                   \
11293         va_list args;                                           \
11294                                                                 \
11295         va_start(args, fmt);                                    \
11296                                                                 \
11297         vaf.fmt = fmt;                                          \
11298         vaf.va = &args;                                         \
11299                                                                 \
11300         __netdev_printk(level, dev, &vaf);                      \
11301                                                                 \
11302         va_end(args);                                           \
11303 }                                                               \
11304 EXPORT_SYMBOL(func);
11305
11306 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11307 define_netdev_printk_level(netdev_alert, KERN_ALERT);
11308 define_netdev_printk_level(netdev_crit, KERN_CRIT);
11309 define_netdev_printk_level(netdev_err, KERN_ERR);
11310 define_netdev_printk_level(netdev_warn, KERN_WARNING);
11311 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11312 define_netdev_printk_level(netdev_info, KERN_INFO);
11313
11314 static void __net_exit netdev_exit(struct net *net)
11315 {
11316         kfree(net->dev_name_head);
11317         kfree(net->dev_index_head);
11318         if (net != &init_net)
11319                 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
11320 }
11321
11322 static struct pernet_operations __net_initdata netdev_net_ops = {
11323         .init = netdev_init,
11324         .exit = netdev_exit,
11325 };
11326
11327 static void __net_exit default_device_exit_net(struct net *net)
11328 {
11329         struct netdev_name_node *name_node, *tmp;
11330         struct net_device *dev, *aux;
11331         /*
11332          * Push all migratable network devices back to the
11333          * initial network namespace
11334          */
11335         ASSERT_RTNL();
11336         for_each_netdev_safe(net, dev, aux) {
11337                 int err;
11338                 char fb_name[IFNAMSIZ];
11339
11340                 /* Ignore unmoveable devices (i.e. loopback) */
11341                 if (dev->features & NETIF_F_NETNS_LOCAL)
11342                         continue;
11343
11344                 /* Leave virtual devices for the generic cleanup */
11345                 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
11346                         continue;
11347
11348                 /* Push remaining network devices to init_net */
11349                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
11350                 if (netdev_name_in_use(&init_net, fb_name))
11351                         snprintf(fb_name, IFNAMSIZ, "dev%%d");
11352
11353                 netdev_for_each_altname_safe(dev, name_node, tmp)
11354                         if (netdev_name_in_use(&init_net, name_node->name)) {
11355                                 netdev_name_node_del(name_node);
11356                                 synchronize_rcu();
11357                                 __netdev_name_node_alt_destroy(name_node);
11358                         }
11359
11360                 err = dev_change_net_namespace(dev, &init_net, fb_name);
11361                 if (err) {
11362                         pr_emerg("%s: failed to move %s to init_net: %d\n",
11363                                  __func__, dev->name, err);
11364                         BUG();
11365                 }
11366         }
11367 }
11368
11369 static void __net_exit default_device_exit_batch(struct list_head *net_list)
11370 {
11371         /* At exit all network devices most be removed from a network
11372          * namespace.  Do this in the reverse order of registration.
11373          * Do this across as many network namespaces as possible to
11374          * improve batching efficiency.
11375          */
11376         struct net_device *dev;
11377         struct net *net;
11378         LIST_HEAD(dev_kill_list);
11379
11380         rtnl_lock();
11381         list_for_each_entry(net, net_list, exit_list) {
11382                 default_device_exit_net(net);
11383                 cond_resched();
11384         }
11385
11386         list_for_each_entry(net, net_list, exit_list) {
11387                 for_each_netdev_reverse(net, dev) {
11388                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
11389                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11390                         else
11391                                 unregister_netdevice_queue(dev, &dev_kill_list);
11392                 }
11393         }
11394         unregister_netdevice_many(&dev_kill_list);
11395         rtnl_unlock();
11396 }
11397
11398 static struct pernet_operations __net_initdata default_device_ops = {
11399         .exit_batch = default_device_exit_batch,
11400 };
11401
11402 /*
11403  *      Initialize the DEV module. At boot time this walks the device list and
11404  *      unhooks any devices that fail to initialise (normally hardware not
11405  *      present) and leaves us with a valid list of present and active devices.
11406  *
11407  */
11408
11409 /*
11410  *       This is called single threaded during boot, so no need
11411  *       to take the rtnl semaphore.
11412  */
11413 static int __init net_dev_init(void)
11414 {
11415         int i, rc = -ENOMEM;
11416
11417         BUG_ON(!dev_boot_phase);
11418
11419         if (dev_proc_init())
11420                 goto out;
11421
11422         if (netdev_kobject_init())
11423                 goto out;
11424
11425         INIT_LIST_HEAD(&ptype_all);
11426         for (i = 0; i < PTYPE_HASH_SIZE; i++)
11427                 INIT_LIST_HEAD(&ptype_base[i]);
11428
11429         if (register_pernet_subsys(&netdev_net_ops))
11430                 goto out;
11431
11432         /*
11433          *      Initialise the packet receive queues.
11434          */
11435
11436         for_each_possible_cpu(i) {
11437                 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11438                 struct softnet_data *sd = &per_cpu(softnet_data, i);
11439
11440                 INIT_WORK(flush, flush_backlog);
11441
11442                 skb_queue_head_init(&sd->input_pkt_queue);
11443                 skb_queue_head_init(&sd->process_queue);
11444 #ifdef CONFIG_XFRM_OFFLOAD
11445                 skb_queue_head_init(&sd->xfrm_backlog);
11446 #endif
11447                 INIT_LIST_HEAD(&sd->poll_list);
11448                 sd->output_queue_tailp = &sd->output_queue;
11449 #ifdef CONFIG_RPS
11450                 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11451                 sd->cpu = i;
11452 #endif
11453                 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
11454                 spin_lock_init(&sd->defer_lock);
11455
11456                 init_gro_hash(&sd->backlog);
11457                 sd->backlog.poll = process_backlog;
11458                 sd->backlog.weight = weight_p;
11459         }
11460
11461         dev_boot_phase = 0;
11462
11463         /* The loopback device is special if any other network devices
11464          * is present in a network namespace the loopback device must
11465          * be present. Since we now dynamically allocate and free the
11466          * loopback device ensure this invariant is maintained by
11467          * keeping the loopback device as the first device on the
11468          * list of network devices.  Ensuring the loopback devices
11469          * is the first device that appears and the last network device
11470          * that disappears.
11471          */
11472         if (register_pernet_device(&loopback_net_ops))
11473                 goto out;
11474
11475         if (register_pernet_device(&default_device_ops))
11476                 goto out;
11477
11478         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11479         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11480
11481         rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11482                                        NULL, dev_cpu_dead);
11483         WARN_ON(rc < 0);
11484         rc = 0;
11485 out:
11486         return rc;
11487 }
11488
11489 subsys_initcall(net_dev_init);