GNU Linux-libre 4.4-gnu1
[releases.git] / net / core / dev.c
1 /*
2  *      NET3    Protocol independent device support routines.
3  *
4  *              This program is free software; you can redistribute it and/or
5  *              modify it under the terms of the GNU General Public License
6  *              as published by the Free Software Foundation; either version
7  *              2 of the License, or (at your option) any later version.
8  *
9  *      Derived from the non IP parts of dev.c 1.0.19
10  *              Authors:        Ross Biro
11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *      Additional Authors:
15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
17  *              David Hinds <dahinds@users.sourceforge.net>
18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *              Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *      Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *                                      to 2 if register_netdev gets called
25  *                                      before net_dev_init & also removed a
26  *                                      few lines of code in the process.
27  *              Alan Cox        :       device private ioctl copies fields back.
28  *              Alan Cox        :       Transmit queue code does relevant
29  *                                      stunts to keep the queue safe.
30  *              Alan Cox        :       Fixed double lock.
31  *              Alan Cox        :       Fixed promisc NULL pointer trap
32  *              ????????        :       Support the full private ioctl range
33  *              Alan Cox        :       Moved ioctl permission check into
34  *                                      drivers
35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
36  *              Alan Cox        :       100 backlog just doesn't cut it when
37  *                                      you start doing multicast video 8)
38  *              Alan Cox        :       Rewrote net_bh and list manager.
39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
40  *              Alan Cox        :       Took out transmit every packet pass
41  *                                      Saved a few bytes in the ioctl handler
42  *              Alan Cox        :       Network driver sets packet type before
43  *                                      calling netif_rx. Saves a function
44  *                                      call a packet.
45  *              Alan Cox        :       Hashed net_bh()
46  *              Richard Kooijman:       Timestamp fixes.
47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
48  *              Alan Cox        :       Device lock protection.
49  *              Alan Cox        :       Fixed nasty side effect of device close
50  *                                      changes.
51  *              Rudi Cilibrasi  :       Pass the right thing to
52  *                                      set_mac_address()
53  *              Dave Miller     :       32bit quantity for the device lock to
54  *                                      make it work out on a Sparc.
55  *              Bjorn Ekwall    :       Added KERNELD hack.
56  *              Alan Cox        :       Cleaned up the backlog initialise.
57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
58  *                                      1 device.
59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
60  *                                      is no device open function.
61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
63  *              Cyrus Durgin    :       Cleaned for KMOD
64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
65  *                                      A network device unload needs to purge
66  *                                      the backlog queue.
67  *      Paul Rusty Russell      :       SIOCSIFNAME
68  *              Pekka Riikonen  :       Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *                                      indefinitely on dev->refcnt
71  *              J Hadi Salim    :       - Backlog queue sampling
72  *                                      - netif_rx() feedback
73  */
74
75 #include <asm/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <linux/rtnetlink.h>
100 #include <linux/stat.h>
101 #include <net/dst.h>
102 #include <net/dst_metadata.h>
103 #include <net/pkt_sched.h>
104 #include <net/checksum.h>
105 #include <net/xfrm.h>
106 #include <linux/highmem.h>
107 #include <linux/init.h>
108 #include <linux/module.h>
109 #include <linux/netpoll.h>
110 #include <linux/rcupdate.h>
111 #include <linux/delay.h>
112 #include <net/iw_handler.h>
113 #include <asm/current.h>
114 #include <linux/audit.h>
115 #include <linux/dmaengine.h>
116 #include <linux/err.h>
117 #include <linux/ctype.h>
118 #include <linux/if_arp.h>
119 #include <linux/if_vlan.h>
120 #include <linux/ip.h>
121 #include <net/ip.h>
122 #include <net/mpls.h>
123 #include <linux/ipv6.h>
124 #include <linux/in.h>
125 #include <linux/jhash.h>
126 #include <linux/random.h>
127 #include <trace/events/napi.h>
128 #include <trace/events/net.h>
129 #include <trace/events/skb.h>
130 #include <linux/pci.h>
131 #include <linux/inetdevice.h>
132 #include <linux/cpu_rmap.h>
133 #include <linux/static_key.h>
134 #include <linux/hashtable.h>
135 #include <linux/vmalloc.h>
136 #include <linux/if_macvlan.h>
137 #include <linux/errqueue.h>
138 #include <linux/hrtimer.h>
139 #include <linux/netfilter_ingress.h>
140
141 #include "net-sysfs.h"
142
143 /* Instead of increasing this, you should create a hash table. */
144 #define MAX_GRO_SKBS 8
145
146 /* This should be increased if a protocol with a bigger head is added. */
147 #define GRO_MAX_HEAD (MAX_HEADER + 128)
148
149 static DEFINE_SPINLOCK(ptype_lock);
150 static DEFINE_SPINLOCK(offload_lock);
151 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
152 struct list_head ptype_all __read_mostly;       /* Taps */
153 static struct list_head offload_base __read_mostly;
154
155 static int netif_rx_internal(struct sk_buff *skb);
156 static int call_netdevice_notifiers_info(unsigned long val,
157                                          struct net_device *dev,
158                                          struct netdev_notifier_info *info);
159
160 /*
161  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
162  * semaphore.
163  *
164  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
165  *
166  * Writers must hold the rtnl semaphore while they loop through the
167  * dev_base_head list, and hold dev_base_lock for writing when they do the
168  * actual updates.  This allows pure readers to access the list even
169  * while a writer is preparing to update it.
170  *
171  * To put it another way, dev_base_lock is held for writing only to
172  * protect against pure readers; the rtnl semaphore provides the
173  * protection against other writers.
174  *
175  * See, for example usages, register_netdevice() and
176  * unregister_netdevice(), which must be called with the rtnl
177  * semaphore held.
178  */
179 DEFINE_RWLOCK(dev_base_lock);
180 EXPORT_SYMBOL(dev_base_lock);
181
182 /* protects napi_hash addition/deletion and napi_gen_id */
183 static DEFINE_SPINLOCK(napi_hash_lock);
184
185 static unsigned int napi_gen_id;
186 static DEFINE_HASHTABLE(napi_hash, 8);
187
188 static seqcount_t devnet_rename_seq;
189
190 static inline void dev_base_seq_inc(struct net *net)
191 {
192         while (++net->dev_base_seq == 0);
193 }
194
195 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
196 {
197         unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
198
199         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
200 }
201
202 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
203 {
204         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
205 }
206
207 static inline void rps_lock(struct softnet_data *sd)
208 {
209 #ifdef CONFIG_RPS
210         spin_lock(&sd->input_pkt_queue.lock);
211 #endif
212 }
213
214 static inline void rps_unlock(struct softnet_data *sd)
215 {
216 #ifdef CONFIG_RPS
217         spin_unlock(&sd->input_pkt_queue.lock);
218 #endif
219 }
220
221 /* Device list insertion */
222 static void list_netdevice(struct net_device *dev)
223 {
224         struct net *net = dev_net(dev);
225
226         ASSERT_RTNL();
227
228         write_lock_bh(&dev_base_lock);
229         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
230         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
231         hlist_add_head_rcu(&dev->index_hlist,
232                            dev_index_hash(net, dev->ifindex));
233         write_unlock_bh(&dev_base_lock);
234
235         dev_base_seq_inc(net);
236 }
237
238 /* Device list removal
239  * caller must respect a RCU grace period before freeing/reusing dev
240  */
241 static void unlist_netdevice(struct net_device *dev)
242 {
243         ASSERT_RTNL();
244
245         /* Unlink dev from the device chain */
246         write_lock_bh(&dev_base_lock);
247         list_del_rcu(&dev->dev_list);
248         hlist_del_rcu(&dev->name_hlist);
249         hlist_del_rcu(&dev->index_hlist);
250         write_unlock_bh(&dev_base_lock);
251
252         dev_base_seq_inc(dev_net(dev));
253 }
254
255 /*
256  *      Our notifier list
257  */
258
259 static RAW_NOTIFIER_HEAD(netdev_chain);
260
261 /*
262  *      Device drivers call our routines to queue packets here. We empty the
263  *      queue in the local softnet handler.
264  */
265
266 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
267 EXPORT_PER_CPU_SYMBOL(softnet_data);
268
269 #ifdef CONFIG_LOCKDEP
270 /*
271  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
272  * according to dev->type
273  */
274 static const unsigned short netdev_lock_type[] =
275         {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
276          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
277          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
278          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
279          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
280          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
281          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
282          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
283          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
284          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
285          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
286          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
287          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
288          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
289          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
290
291 static const char *const netdev_lock_name[] =
292         {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
293          "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
294          "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
295          "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
296          "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
297          "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
298          "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
299          "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
300          "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
301          "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
302          "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
303          "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
304          "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
305          "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
306          "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
307
308 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
309 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
310
311 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
312 {
313         int i;
314
315         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
316                 if (netdev_lock_type[i] == dev_type)
317                         return i;
318         /* the last key is used by default */
319         return ARRAY_SIZE(netdev_lock_type) - 1;
320 }
321
322 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
323                                                  unsigned short dev_type)
324 {
325         int i;
326
327         i = netdev_lock_pos(dev_type);
328         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
329                                    netdev_lock_name[i]);
330 }
331
332 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
333 {
334         int i;
335
336         i = netdev_lock_pos(dev->type);
337         lockdep_set_class_and_name(&dev->addr_list_lock,
338                                    &netdev_addr_lock_key[i],
339                                    netdev_lock_name[i]);
340 }
341 #else
342 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
343                                                  unsigned short dev_type)
344 {
345 }
346 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
347 {
348 }
349 #endif
350
351 /*******************************************************************************
352
353                 Protocol management and registration routines
354
355 *******************************************************************************/
356
357 /*
358  *      Add a protocol ID to the list. Now that the input handler is
359  *      smarter we can dispense with all the messy stuff that used to be
360  *      here.
361  *
362  *      BEWARE!!! Protocol handlers, mangling input packets,
363  *      MUST BE last in hash buckets and checking protocol handlers
364  *      MUST start from promiscuous ptype_all chain in net_bh.
365  *      It is true now, do not change it.
366  *      Explanation follows: if protocol handler, mangling packet, will
367  *      be the first on list, it is not able to sense, that packet
368  *      is cloned and should be copied-on-write, so that it will
369  *      change it and subsequent readers will get broken packet.
370  *                                                      --ANK (980803)
371  */
372
373 static inline struct list_head *ptype_head(const struct packet_type *pt)
374 {
375         if (pt->type == htons(ETH_P_ALL))
376                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
377         else
378                 return pt->dev ? &pt->dev->ptype_specific :
379                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
380 }
381
382 /**
383  *      dev_add_pack - add packet handler
384  *      @pt: packet type declaration
385  *
386  *      Add a protocol handler to the networking stack. The passed &packet_type
387  *      is linked into kernel lists and may not be freed until it has been
388  *      removed from the kernel lists.
389  *
390  *      This call does not sleep therefore it can not
391  *      guarantee all CPU's that are in middle of receiving packets
392  *      will see the new packet type (until the next received packet).
393  */
394
395 void dev_add_pack(struct packet_type *pt)
396 {
397         struct list_head *head = ptype_head(pt);
398
399         spin_lock(&ptype_lock);
400         list_add_rcu(&pt->list, head);
401         spin_unlock(&ptype_lock);
402 }
403 EXPORT_SYMBOL(dev_add_pack);
404
405 /**
406  *      __dev_remove_pack        - remove packet handler
407  *      @pt: packet type declaration
408  *
409  *      Remove a protocol handler that was previously added to the kernel
410  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
411  *      from the kernel lists and can be freed or reused once this function
412  *      returns.
413  *
414  *      The packet type might still be in use by receivers
415  *      and must not be freed until after all the CPU's have gone
416  *      through a quiescent state.
417  */
418 void __dev_remove_pack(struct packet_type *pt)
419 {
420         struct list_head *head = ptype_head(pt);
421         struct packet_type *pt1;
422
423         spin_lock(&ptype_lock);
424
425         list_for_each_entry(pt1, head, list) {
426                 if (pt == pt1) {
427                         list_del_rcu(&pt->list);
428                         goto out;
429                 }
430         }
431
432         pr_warn("dev_remove_pack: %p not found\n", pt);
433 out:
434         spin_unlock(&ptype_lock);
435 }
436 EXPORT_SYMBOL(__dev_remove_pack);
437
438 /**
439  *      dev_remove_pack  - remove packet handler
440  *      @pt: packet type declaration
441  *
442  *      Remove a protocol handler that was previously added to the kernel
443  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
444  *      from the kernel lists and can be freed or reused once this function
445  *      returns.
446  *
447  *      This call sleeps to guarantee that no CPU is looking at the packet
448  *      type after return.
449  */
450 void dev_remove_pack(struct packet_type *pt)
451 {
452         __dev_remove_pack(pt);
453
454         synchronize_net();
455 }
456 EXPORT_SYMBOL(dev_remove_pack);
457
458
459 /**
460  *      dev_add_offload - register offload handlers
461  *      @po: protocol offload declaration
462  *
463  *      Add protocol offload handlers to the networking stack. The passed
464  *      &proto_offload is linked into kernel lists and may not be freed until
465  *      it has been removed from the kernel lists.
466  *
467  *      This call does not sleep therefore it can not
468  *      guarantee all CPU's that are in middle of receiving packets
469  *      will see the new offload handlers (until the next received packet).
470  */
471 void dev_add_offload(struct packet_offload *po)
472 {
473         struct packet_offload *elem;
474
475         spin_lock(&offload_lock);
476         list_for_each_entry(elem, &offload_base, list) {
477                 if (po->priority < elem->priority)
478                         break;
479         }
480         list_add_rcu(&po->list, elem->list.prev);
481         spin_unlock(&offload_lock);
482 }
483 EXPORT_SYMBOL(dev_add_offload);
484
485 /**
486  *      __dev_remove_offload     - remove offload handler
487  *      @po: packet offload declaration
488  *
489  *      Remove a protocol offload handler that was previously added to the
490  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
491  *      is removed from the kernel lists and can be freed or reused once this
492  *      function returns.
493  *
494  *      The packet type might still be in use by receivers
495  *      and must not be freed until after all the CPU's have gone
496  *      through a quiescent state.
497  */
498 static void __dev_remove_offload(struct packet_offload *po)
499 {
500         struct list_head *head = &offload_base;
501         struct packet_offload *po1;
502
503         spin_lock(&offload_lock);
504
505         list_for_each_entry(po1, head, list) {
506                 if (po == po1) {
507                         list_del_rcu(&po->list);
508                         goto out;
509                 }
510         }
511
512         pr_warn("dev_remove_offload: %p not found\n", po);
513 out:
514         spin_unlock(&offload_lock);
515 }
516
517 /**
518  *      dev_remove_offload       - remove packet offload handler
519  *      @po: packet offload declaration
520  *
521  *      Remove a packet offload handler that was previously added to the kernel
522  *      offload handlers by dev_add_offload(). The passed &offload_type is
523  *      removed from the kernel lists and can be freed or reused once this
524  *      function returns.
525  *
526  *      This call sleeps to guarantee that no CPU is looking at the packet
527  *      type after return.
528  */
529 void dev_remove_offload(struct packet_offload *po)
530 {
531         __dev_remove_offload(po);
532
533         synchronize_net();
534 }
535 EXPORT_SYMBOL(dev_remove_offload);
536
537 /******************************************************************************
538
539                       Device Boot-time Settings Routines
540
541 *******************************************************************************/
542
543 /* Boot time configuration table */
544 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
545
546 /**
547  *      netdev_boot_setup_add   - add new setup entry
548  *      @name: name of the device
549  *      @map: configured settings for the device
550  *
551  *      Adds new setup entry to the dev_boot_setup list.  The function
552  *      returns 0 on error and 1 on success.  This is a generic routine to
553  *      all netdevices.
554  */
555 static int netdev_boot_setup_add(char *name, struct ifmap *map)
556 {
557         struct netdev_boot_setup *s;
558         int i;
559
560         s = dev_boot_setup;
561         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
562                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
563                         memset(s[i].name, 0, sizeof(s[i].name));
564                         strlcpy(s[i].name, name, IFNAMSIZ);
565                         memcpy(&s[i].map, map, sizeof(s[i].map));
566                         break;
567                 }
568         }
569
570         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
571 }
572
573 /**
574  *      netdev_boot_setup_check - check boot time settings
575  *      @dev: the netdevice
576  *
577  *      Check boot time settings for the device.
578  *      The found settings are set for the device to be used
579  *      later in the device probing.
580  *      Returns 0 if no settings found, 1 if they are.
581  */
582 int netdev_boot_setup_check(struct net_device *dev)
583 {
584         struct netdev_boot_setup *s = dev_boot_setup;
585         int i;
586
587         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
588                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
589                     !strcmp(dev->name, s[i].name)) {
590                         dev->irq        = s[i].map.irq;
591                         dev->base_addr  = s[i].map.base_addr;
592                         dev->mem_start  = s[i].map.mem_start;
593                         dev->mem_end    = s[i].map.mem_end;
594                         return 1;
595                 }
596         }
597         return 0;
598 }
599 EXPORT_SYMBOL(netdev_boot_setup_check);
600
601
602 /**
603  *      netdev_boot_base        - get address from boot time settings
604  *      @prefix: prefix for network device
605  *      @unit: id for network device
606  *
607  *      Check boot time settings for the base address of device.
608  *      The found settings are set for the device to be used
609  *      later in the device probing.
610  *      Returns 0 if no settings found.
611  */
612 unsigned long netdev_boot_base(const char *prefix, int unit)
613 {
614         const struct netdev_boot_setup *s = dev_boot_setup;
615         char name[IFNAMSIZ];
616         int i;
617
618         sprintf(name, "%s%d", prefix, unit);
619
620         /*
621          * If device already registered then return base of 1
622          * to indicate not to probe for this interface
623          */
624         if (__dev_get_by_name(&init_net, name))
625                 return 1;
626
627         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
628                 if (!strcmp(name, s[i].name))
629                         return s[i].map.base_addr;
630         return 0;
631 }
632
633 /*
634  * Saves at boot time configured settings for any netdevice.
635  */
636 int __init netdev_boot_setup(char *str)
637 {
638         int ints[5];
639         struct ifmap map;
640
641         str = get_options(str, ARRAY_SIZE(ints), ints);
642         if (!str || !*str)
643                 return 0;
644
645         /* Save settings */
646         memset(&map, 0, sizeof(map));
647         if (ints[0] > 0)
648                 map.irq = ints[1];
649         if (ints[0] > 1)
650                 map.base_addr = ints[2];
651         if (ints[0] > 2)
652                 map.mem_start = ints[3];
653         if (ints[0] > 3)
654                 map.mem_end = ints[4];
655
656         /* Add new entry to the list */
657         return netdev_boot_setup_add(str, &map);
658 }
659
660 __setup("netdev=", netdev_boot_setup);
661
662 /*******************************************************************************
663
664                             Device Interface Subroutines
665
666 *******************************************************************************/
667
668 /**
669  *      dev_get_iflink  - get 'iflink' value of a interface
670  *      @dev: targeted interface
671  *
672  *      Indicates the ifindex the interface is linked to.
673  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
674  */
675
676 int dev_get_iflink(const struct net_device *dev)
677 {
678         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
679                 return dev->netdev_ops->ndo_get_iflink(dev);
680
681         return dev->ifindex;
682 }
683 EXPORT_SYMBOL(dev_get_iflink);
684
685 /**
686  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
687  *      @dev: targeted interface
688  *      @skb: The packet.
689  *
690  *      For better visibility of tunnel traffic OVS needs to retrieve
691  *      egress tunnel information for a packet. Following API allows
692  *      user to get this info.
693  */
694 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
695 {
696         struct ip_tunnel_info *info;
697
698         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
699                 return -EINVAL;
700
701         info = skb_tunnel_info_unclone(skb);
702         if (!info)
703                 return -ENOMEM;
704         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
705                 return -EINVAL;
706
707         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
708 }
709 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
710
711 /**
712  *      __dev_get_by_name       - find a device by its name
713  *      @net: the applicable net namespace
714  *      @name: name to find
715  *
716  *      Find an interface by name. Must be called under RTNL semaphore
717  *      or @dev_base_lock. If the name is found a pointer to the device
718  *      is returned. If the name is not found then %NULL is returned. The
719  *      reference counters are not incremented so the caller must be
720  *      careful with locks.
721  */
722
723 struct net_device *__dev_get_by_name(struct net *net, const char *name)
724 {
725         struct net_device *dev;
726         struct hlist_head *head = dev_name_hash(net, name);
727
728         hlist_for_each_entry(dev, head, name_hlist)
729                 if (!strncmp(dev->name, name, IFNAMSIZ))
730                         return dev;
731
732         return NULL;
733 }
734 EXPORT_SYMBOL(__dev_get_by_name);
735
736 /**
737  *      dev_get_by_name_rcu     - find a device by its name
738  *      @net: the applicable net namespace
739  *      @name: name to find
740  *
741  *      Find an interface by name.
742  *      If the name is found a pointer to the device is returned.
743  *      If the name is not found then %NULL is returned.
744  *      The reference counters are not incremented so the caller must be
745  *      careful with locks. The caller must hold RCU lock.
746  */
747
748 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
749 {
750         struct net_device *dev;
751         struct hlist_head *head = dev_name_hash(net, name);
752
753         hlist_for_each_entry_rcu(dev, head, name_hlist)
754                 if (!strncmp(dev->name, name, IFNAMSIZ))
755                         return dev;
756
757         return NULL;
758 }
759 EXPORT_SYMBOL(dev_get_by_name_rcu);
760
761 /**
762  *      dev_get_by_name         - find a device by its name
763  *      @net: the applicable net namespace
764  *      @name: name to find
765  *
766  *      Find an interface by name. This can be called from any
767  *      context and does its own locking. The returned handle has
768  *      the usage count incremented and the caller must use dev_put() to
769  *      release it when it is no longer needed. %NULL is returned if no
770  *      matching device is found.
771  */
772
773 struct net_device *dev_get_by_name(struct net *net, const char *name)
774 {
775         struct net_device *dev;
776
777         rcu_read_lock();
778         dev = dev_get_by_name_rcu(net, name);
779         if (dev)
780                 dev_hold(dev);
781         rcu_read_unlock();
782         return dev;
783 }
784 EXPORT_SYMBOL(dev_get_by_name);
785
786 /**
787  *      __dev_get_by_index - find a device by its ifindex
788  *      @net: the applicable net namespace
789  *      @ifindex: index of device
790  *
791  *      Search for an interface by index. Returns %NULL if the device
792  *      is not found or a pointer to the device. The device has not
793  *      had its reference counter increased so the caller must be careful
794  *      about locking. The caller must hold either the RTNL semaphore
795  *      or @dev_base_lock.
796  */
797
798 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
799 {
800         struct net_device *dev;
801         struct hlist_head *head = dev_index_hash(net, ifindex);
802
803         hlist_for_each_entry(dev, head, index_hlist)
804                 if (dev->ifindex == ifindex)
805                         return dev;
806
807         return NULL;
808 }
809 EXPORT_SYMBOL(__dev_get_by_index);
810
811 /**
812  *      dev_get_by_index_rcu - find a device by its ifindex
813  *      @net: the applicable net namespace
814  *      @ifindex: index of device
815  *
816  *      Search for an interface by index. Returns %NULL if the device
817  *      is not found or a pointer to the device. The device has not
818  *      had its reference counter increased so the caller must be careful
819  *      about locking. The caller must hold RCU lock.
820  */
821
822 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
823 {
824         struct net_device *dev;
825         struct hlist_head *head = dev_index_hash(net, ifindex);
826
827         hlist_for_each_entry_rcu(dev, head, index_hlist)
828                 if (dev->ifindex == ifindex)
829                         return dev;
830
831         return NULL;
832 }
833 EXPORT_SYMBOL(dev_get_by_index_rcu);
834
835
836 /**
837  *      dev_get_by_index - find a device by its ifindex
838  *      @net: the applicable net namespace
839  *      @ifindex: index of device
840  *
841  *      Search for an interface by index. Returns NULL if the device
842  *      is not found or a pointer to the device. The device returned has
843  *      had a reference added and the pointer is safe until the user calls
844  *      dev_put to indicate they have finished with it.
845  */
846
847 struct net_device *dev_get_by_index(struct net *net, int ifindex)
848 {
849         struct net_device *dev;
850
851         rcu_read_lock();
852         dev = dev_get_by_index_rcu(net, ifindex);
853         if (dev)
854                 dev_hold(dev);
855         rcu_read_unlock();
856         return dev;
857 }
858 EXPORT_SYMBOL(dev_get_by_index);
859
860 /**
861  *      netdev_get_name - get a netdevice name, knowing its ifindex.
862  *      @net: network namespace
863  *      @name: a pointer to the buffer where the name will be stored.
864  *      @ifindex: the ifindex of the interface to get the name from.
865  *
866  *      The use of raw_seqcount_begin() and cond_resched() before
867  *      retrying is required as we want to give the writers a chance
868  *      to complete when CONFIG_PREEMPT is not set.
869  */
870 int netdev_get_name(struct net *net, char *name, int ifindex)
871 {
872         struct net_device *dev;
873         unsigned int seq;
874
875 retry:
876         seq = raw_seqcount_begin(&devnet_rename_seq);
877         rcu_read_lock();
878         dev = dev_get_by_index_rcu(net, ifindex);
879         if (!dev) {
880                 rcu_read_unlock();
881                 return -ENODEV;
882         }
883
884         strcpy(name, dev->name);
885         rcu_read_unlock();
886         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
887                 cond_resched();
888                 goto retry;
889         }
890
891         return 0;
892 }
893
894 /**
895  *      dev_getbyhwaddr_rcu - find a device by its hardware address
896  *      @net: the applicable net namespace
897  *      @type: media type of device
898  *      @ha: hardware address
899  *
900  *      Search for an interface by MAC address. Returns NULL if the device
901  *      is not found or a pointer to the device.
902  *      The caller must hold RCU or RTNL.
903  *      The returned device has not had its ref count increased
904  *      and the caller must therefore be careful about locking
905  *
906  */
907
908 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
909                                        const char *ha)
910 {
911         struct net_device *dev;
912
913         for_each_netdev_rcu(net, dev)
914                 if (dev->type == type &&
915                     !memcmp(dev->dev_addr, ha, dev->addr_len))
916                         return dev;
917
918         return NULL;
919 }
920 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
921
922 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
923 {
924         struct net_device *dev;
925
926         ASSERT_RTNL();
927         for_each_netdev(net, dev)
928                 if (dev->type == type)
929                         return dev;
930
931         return NULL;
932 }
933 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
934
935 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
936 {
937         struct net_device *dev, *ret = NULL;
938
939         rcu_read_lock();
940         for_each_netdev_rcu(net, dev)
941                 if (dev->type == type) {
942                         dev_hold(dev);
943                         ret = dev;
944                         break;
945                 }
946         rcu_read_unlock();
947         return ret;
948 }
949 EXPORT_SYMBOL(dev_getfirstbyhwtype);
950
951 /**
952  *      __dev_get_by_flags - find any device with given flags
953  *      @net: the applicable net namespace
954  *      @if_flags: IFF_* values
955  *      @mask: bitmask of bits in if_flags to check
956  *
957  *      Search for any interface with the given flags. Returns NULL if a device
958  *      is not found or a pointer to the device. Must be called inside
959  *      rtnl_lock(), and result refcount is unchanged.
960  */
961
962 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
963                                       unsigned short mask)
964 {
965         struct net_device *dev, *ret;
966
967         ASSERT_RTNL();
968
969         ret = NULL;
970         for_each_netdev(net, dev) {
971                 if (((dev->flags ^ if_flags) & mask) == 0) {
972                         ret = dev;
973                         break;
974                 }
975         }
976         return ret;
977 }
978 EXPORT_SYMBOL(__dev_get_by_flags);
979
980 /**
981  *      dev_valid_name - check if name is okay for network device
982  *      @name: name string
983  *
984  *      Network device names need to be valid file names to
985  *      to allow sysfs to work.  We also disallow any kind of
986  *      whitespace.
987  */
988 bool dev_valid_name(const char *name)
989 {
990         if (*name == '\0')
991                 return false;
992         if (strlen(name) >= IFNAMSIZ)
993                 return false;
994         if (!strcmp(name, ".") || !strcmp(name, ".."))
995                 return false;
996
997         while (*name) {
998                 if (*name == '/' || *name == ':' || isspace(*name))
999                         return false;
1000                 name++;
1001         }
1002         return true;
1003 }
1004 EXPORT_SYMBOL(dev_valid_name);
1005
1006 /**
1007  *      __dev_alloc_name - allocate a name for a device
1008  *      @net: network namespace to allocate the device name in
1009  *      @name: name format string
1010  *      @buf:  scratch buffer and result name string
1011  *
1012  *      Passed a format string - eg "lt%d" it will try and find a suitable
1013  *      id. It scans list of devices to build up a free map, then chooses
1014  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1015  *      while allocating the name and adding the device in order to avoid
1016  *      duplicates.
1017  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1018  *      Returns the number of the unit assigned or a negative errno code.
1019  */
1020
1021 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1022 {
1023         int i = 0;
1024         const char *p;
1025         const int max_netdevices = 8*PAGE_SIZE;
1026         unsigned long *inuse;
1027         struct net_device *d;
1028
1029         p = strnchr(name, IFNAMSIZ-1, '%');
1030         if (p) {
1031                 /*
1032                  * Verify the string as this thing may have come from
1033                  * the user.  There must be either one "%d" and no other "%"
1034                  * characters.
1035                  */
1036                 if (p[1] != 'd' || strchr(p + 2, '%'))
1037                         return -EINVAL;
1038
1039                 /* Use one page as a bit array of possible slots */
1040                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1041                 if (!inuse)
1042                         return -ENOMEM;
1043
1044                 for_each_netdev(net, d) {
1045                         if (!sscanf(d->name, name, &i))
1046                                 continue;
1047                         if (i < 0 || i >= max_netdevices)
1048                                 continue;
1049
1050                         /*  avoid cases where sscanf is not exact inverse of printf */
1051                         snprintf(buf, IFNAMSIZ, name, i);
1052                         if (!strncmp(buf, d->name, IFNAMSIZ))
1053                                 set_bit(i, inuse);
1054                 }
1055
1056                 i = find_first_zero_bit(inuse, max_netdevices);
1057                 free_page((unsigned long) inuse);
1058         }
1059
1060         if (buf != name)
1061                 snprintf(buf, IFNAMSIZ, name, i);
1062         if (!__dev_get_by_name(net, buf))
1063                 return i;
1064
1065         /* It is possible to run out of possible slots
1066          * when the name is long and there isn't enough space left
1067          * for the digits, or if all bits are used.
1068          */
1069         return -ENFILE;
1070 }
1071
1072 /**
1073  *      dev_alloc_name - allocate a name for a device
1074  *      @dev: device
1075  *      @name: name format string
1076  *
1077  *      Passed a format string - eg "lt%d" it will try and find a suitable
1078  *      id. It scans list of devices to build up a free map, then chooses
1079  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1080  *      while allocating the name and adding the device in order to avoid
1081  *      duplicates.
1082  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1083  *      Returns the number of the unit assigned or a negative errno code.
1084  */
1085
1086 int dev_alloc_name(struct net_device *dev, const char *name)
1087 {
1088         char buf[IFNAMSIZ];
1089         struct net *net;
1090         int ret;
1091
1092         BUG_ON(!dev_net(dev));
1093         net = dev_net(dev);
1094         ret = __dev_alloc_name(net, name, buf);
1095         if (ret >= 0)
1096                 strlcpy(dev->name, buf, IFNAMSIZ);
1097         return ret;
1098 }
1099 EXPORT_SYMBOL(dev_alloc_name);
1100
1101 static int dev_alloc_name_ns(struct net *net,
1102                              struct net_device *dev,
1103                              const char *name)
1104 {
1105         char buf[IFNAMSIZ];
1106         int ret;
1107
1108         ret = __dev_alloc_name(net, name, buf);
1109         if (ret >= 0)
1110                 strlcpy(dev->name, buf, IFNAMSIZ);
1111         return ret;
1112 }
1113
1114 static int dev_get_valid_name(struct net *net,
1115                               struct net_device *dev,
1116                               const char *name)
1117 {
1118         BUG_ON(!net);
1119
1120         if (!dev_valid_name(name))
1121                 return -EINVAL;
1122
1123         if (strchr(name, '%'))
1124                 return dev_alloc_name_ns(net, dev, name);
1125         else if (__dev_get_by_name(net, name))
1126                 return -EEXIST;
1127         else if (dev->name != name)
1128                 strlcpy(dev->name, name, IFNAMSIZ);
1129
1130         return 0;
1131 }
1132
1133 /**
1134  *      dev_change_name - change name of a device
1135  *      @dev: device
1136  *      @newname: name (or format string) must be at least IFNAMSIZ
1137  *
1138  *      Change name of a device, can pass format strings "eth%d".
1139  *      for wildcarding.
1140  */
1141 int dev_change_name(struct net_device *dev, const char *newname)
1142 {
1143         unsigned char old_assign_type;
1144         char oldname[IFNAMSIZ];
1145         int err = 0;
1146         int ret;
1147         struct net *net;
1148
1149         ASSERT_RTNL();
1150         BUG_ON(!dev_net(dev));
1151
1152         net = dev_net(dev);
1153         if (dev->flags & IFF_UP)
1154                 return -EBUSY;
1155
1156         write_seqcount_begin(&devnet_rename_seq);
1157
1158         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1159                 write_seqcount_end(&devnet_rename_seq);
1160                 return 0;
1161         }
1162
1163         memcpy(oldname, dev->name, IFNAMSIZ);
1164
1165         err = dev_get_valid_name(net, dev, newname);
1166         if (err < 0) {
1167                 write_seqcount_end(&devnet_rename_seq);
1168                 return err;
1169         }
1170
1171         if (oldname[0] && !strchr(oldname, '%'))
1172                 netdev_info(dev, "renamed from %s\n", oldname);
1173
1174         old_assign_type = dev->name_assign_type;
1175         dev->name_assign_type = NET_NAME_RENAMED;
1176
1177 rollback:
1178         ret = device_rename(&dev->dev, dev->name);
1179         if (ret) {
1180                 memcpy(dev->name, oldname, IFNAMSIZ);
1181                 dev->name_assign_type = old_assign_type;
1182                 write_seqcount_end(&devnet_rename_seq);
1183                 return ret;
1184         }
1185
1186         write_seqcount_end(&devnet_rename_seq);
1187
1188         netdev_adjacent_rename_links(dev, oldname);
1189
1190         write_lock_bh(&dev_base_lock);
1191         hlist_del_rcu(&dev->name_hlist);
1192         write_unlock_bh(&dev_base_lock);
1193
1194         synchronize_rcu();
1195
1196         write_lock_bh(&dev_base_lock);
1197         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1198         write_unlock_bh(&dev_base_lock);
1199
1200         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1201         ret = notifier_to_errno(ret);
1202
1203         if (ret) {
1204                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1205                 if (err >= 0) {
1206                         err = ret;
1207                         write_seqcount_begin(&devnet_rename_seq);
1208                         memcpy(dev->name, oldname, IFNAMSIZ);
1209                         memcpy(oldname, newname, IFNAMSIZ);
1210                         dev->name_assign_type = old_assign_type;
1211                         old_assign_type = NET_NAME_RENAMED;
1212                         goto rollback;
1213                 } else {
1214                         pr_err("%s: name change rollback failed: %d\n",
1215                                dev->name, ret);
1216                 }
1217         }
1218
1219         return err;
1220 }
1221
1222 /**
1223  *      dev_set_alias - change ifalias of a device
1224  *      @dev: device
1225  *      @alias: name up to IFALIASZ
1226  *      @len: limit of bytes to copy from info
1227  *
1228  *      Set ifalias for a device,
1229  */
1230 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1231 {
1232         char *new_ifalias;
1233
1234         ASSERT_RTNL();
1235
1236         if (len >= IFALIASZ)
1237                 return -EINVAL;
1238
1239         if (!len) {
1240                 kfree(dev->ifalias);
1241                 dev->ifalias = NULL;
1242                 return 0;
1243         }
1244
1245         new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1246         if (!new_ifalias)
1247                 return -ENOMEM;
1248         dev->ifalias = new_ifalias;
1249
1250         strlcpy(dev->ifalias, alias, len+1);
1251         return len;
1252 }
1253
1254
1255 /**
1256  *      netdev_features_change - device changes features
1257  *      @dev: device to cause notification
1258  *
1259  *      Called to indicate a device has changed features.
1260  */
1261 void netdev_features_change(struct net_device *dev)
1262 {
1263         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1264 }
1265 EXPORT_SYMBOL(netdev_features_change);
1266
1267 /**
1268  *      netdev_state_change - device changes state
1269  *      @dev: device to cause notification
1270  *
1271  *      Called to indicate a device has changed state. This function calls
1272  *      the notifier chains for netdev_chain and sends a NEWLINK message
1273  *      to the routing socket.
1274  */
1275 void netdev_state_change(struct net_device *dev)
1276 {
1277         if (dev->flags & IFF_UP) {
1278                 struct netdev_notifier_change_info change_info;
1279
1280                 change_info.flags_changed = 0;
1281                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1282                                               &change_info.info);
1283                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1284         }
1285 }
1286 EXPORT_SYMBOL(netdev_state_change);
1287
1288 /**
1289  *      netdev_notify_peers - notify network peers about existence of @dev
1290  *      @dev: network device
1291  *
1292  * Generate traffic such that interested network peers are aware of
1293  * @dev, such as by generating a gratuitous ARP. This may be used when
1294  * a device wants to inform the rest of the network about some sort of
1295  * reconfiguration such as a failover event or virtual machine
1296  * migration.
1297  */
1298 void netdev_notify_peers(struct net_device *dev)
1299 {
1300         rtnl_lock();
1301         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1302         rtnl_unlock();
1303 }
1304 EXPORT_SYMBOL(netdev_notify_peers);
1305
1306 static int __dev_open(struct net_device *dev)
1307 {
1308         const struct net_device_ops *ops = dev->netdev_ops;
1309         int ret;
1310
1311         ASSERT_RTNL();
1312
1313         if (!netif_device_present(dev))
1314                 return -ENODEV;
1315
1316         /* Block netpoll from trying to do any rx path servicing.
1317          * If we don't do this there is a chance ndo_poll_controller
1318          * or ndo_poll may be running while we open the device
1319          */
1320         netpoll_poll_disable(dev);
1321
1322         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1323         ret = notifier_to_errno(ret);
1324         if (ret)
1325                 return ret;
1326
1327         set_bit(__LINK_STATE_START, &dev->state);
1328
1329         if (ops->ndo_validate_addr)
1330                 ret = ops->ndo_validate_addr(dev);
1331
1332         if (!ret && ops->ndo_open)
1333                 ret = ops->ndo_open(dev);
1334
1335         netpoll_poll_enable(dev);
1336
1337         if (ret)
1338                 clear_bit(__LINK_STATE_START, &dev->state);
1339         else {
1340                 dev->flags |= IFF_UP;
1341                 dev_set_rx_mode(dev);
1342                 dev_activate(dev);
1343                 add_device_randomness(dev->dev_addr, dev->addr_len);
1344         }
1345
1346         return ret;
1347 }
1348
1349 /**
1350  *      dev_open        - prepare an interface for use.
1351  *      @dev:   device to open
1352  *
1353  *      Takes a device from down to up state. The device's private open
1354  *      function is invoked and then the multicast lists are loaded. Finally
1355  *      the device is moved into the up state and a %NETDEV_UP message is
1356  *      sent to the netdev notifier chain.
1357  *
1358  *      Calling this function on an active interface is a nop. On a failure
1359  *      a negative errno code is returned.
1360  */
1361 int dev_open(struct net_device *dev)
1362 {
1363         int ret;
1364
1365         if (dev->flags & IFF_UP)
1366                 return 0;
1367
1368         ret = __dev_open(dev);
1369         if (ret < 0)
1370                 return ret;
1371
1372         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1373         call_netdevice_notifiers(NETDEV_UP, dev);
1374
1375         return ret;
1376 }
1377 EXPORT_SYMBOL(dev_open);
1378
1379 static int __dev_close_many(struct list_head *head)
1380 {
1381         struct net_device *dev;
1382
1383         ASSERT_RTNL();
1384         might_sleep();
1385
1386         list_for_each_entry(dev, head, close_list) {
1387                 /* Temporarily disable netpoll until the interface is down */
1388                 netpoll_poll_disable(dev);
1389
1390                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1391
1392                 clear_bit(__LINK_STATE_START, &dev->state);
1393
1394                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1395                  * can be even on different cpu. So just clear netif_running().
1396                  *
1397                  * dev->stop() will invoke napi_disable() on all of it's
1398                  * napi_struct instances on this device.
1399                  */
1400                 smp_mb__after_atomic(); /* Commit netif_running(). */
1401         }
1402
1403         dev_deactivate_many(head);
1404
1405         list_for_each_entry(dev, head, close_list) {
1406                 const struct net_device_ops *ops = dev->netdev_ops;
1407
1408                 /*
1409                  *      Call the device specific close. This cannot fail.
1410                  *      Only if device is UP
1411                  *
1412                  *      We allow it to be called even after a DETACH hot-plug
1413                  *      event.
1414                  */
1415                 if (ops->ndo_stop)
1416                         ops->ndo_stop(dev);
1417
1418                 dev->flags &= ~IFF_UP;
1419                 netpoll_poll_enable(dev);
1420         }
1421
1422         return 0;
1423 }
1424
1425 static int __dev_close(struct net_device *dev)
1426 {
1427         int retval;
1428         LIST_HEAD(single);
1429
1430         list_add(&dev->close_list, &single);
1431         retval = __dev_close_many(&single);
1432         list_del(&single);
1433
1434         return retval;
1435 }
1436
1437 int dev_close_many(struct list_head *head, bool unlink)
1438 {
1439         struct net_device *dev, *tmp;
1440
1441         /* Remove the devices that don't need to be closed */
1442         list_for_each_entry_safe(dev, tmp, head, close_list)
1443                 if (!(dev->flags & IFF_UP))
1444                         list_del_init(&dev->close_list);
1445
1446         __dev_close_many(head);
1447
1448         list_for_each_entry_safe(dev, tmp, head, close_list) {
1449                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1450                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1451                 if (unlink)
1452                         list_del_init(&dev->close_list);
1453         }
1454
1455         return 0;
1456 }
1457 EXPORT_SYMBOL(dev_close_many);
1458
1459 /**
1460  *      dev_close - shutdown an interface.
1461  *      @dev: device to shutdown
1462  *
1463  *      This function moves an active device into down state. A
1464  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1465  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1466  *      chain.
1467  */
1468 int dev_close(struct net_device *dev)
1469 {
1470         if (dev->flags & IFF_UP) {
1471                 LIST_HEAD(single);
1472
1473                 list_add(&dev->close_list, &single);
1474                 dev_close_many(&single, true);
1475                 list_del(&single);
1476         }
1477         return 0;
1478 }
1479 EXPORT_SYMBOL(dev_close);
1480
1481
1482 /**
1483  *      dev_disable_lro - disable Large Receive Offload on a device
1484  *      @dev: device
1485  *
1486  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1487  *      called under RTNL.  This is needed if received packets may be
1488  *      forwarded to another interface.
1489  */
1490 void dev_disable_lro(struct net_device *dev)
1491 {
1492         struct net_device *lower_dev;
1493         struct list_head *iter;
1494
1495         dev->wanted_features &= ~NETIF_F_LRO;
1496         netdev_update_features(dev);
1497
1498         if (unlikely(dev->features & NETIF_F_LRO))
1499                 netdev_WARN(dev, "failed to disable LRO!\n");
1500
1501         netdev_for_each_lower_dev(dev, lower_dev, iter)
1502                 dev_disable_lro(lower_dev);
1503 }
1504 EXPORT_SYMBOL(dev_disable_lro);
1505
1506 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1507                                    struct net_device *dev)
1508 {
1509         struct netdev_notifier_info info;
1510
1511         netdev_notifier_info_init(&info, dev);
1512         return nb->notifier_call(nb, val, &info);
1513 }
1514
1515 static int dev_boot_phase = 1;
1516
1517 /**
1518  *      register_netdevice_notifier - register a network notifier block
1519  *      @nb: notifier
1520  *
1521  *      Register a notifier to be called when network device events occur.
1522  *      The notifier passed is linked into the kernel structures and must
1523  *      not be reused until it has been unregistered. A negative errno code
1524  *      is returned on a failure.
1525  *
1526  *      When registered all registration and up events are replayed
1527  *      to the new notifier to allow device to have a race free
1528  *      view of the network device list.
1529  */
1530
1531 int register_netdevice_notifier(struct notifier_block *nb)
1532 {
1533         struct net_device *dev;
1534         struct net_device *last;
1535         struct net *net;
1536         int err;
1537
1538         rtnl_lock();
1539         err = raw_notifier_chain_register(&netdev_chain, nb);
1540         if (err)
1541                 goto unlock;
1542         if (dev_boot_phase)
1543                 goto unlock;
1544         for_each_net(net) {
1545                 for_each_netdev(net, dev) {
1546                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1547                         err = notifier_to_errno(err);
1548                         if (err)
1549                                 goto rollback;
1550
1551                         if (!(dev->flags & IFF_UP))
1552                                 continue;
1553
1554                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1555                 }
1556         }
1557
1558 unlock:
1559         rtnl_unlock();
1560         return err;
1561
1562 rollback:
1563         last = dev;
1564         for_each_net(net) {
1565                 for_each_netdev(net, dev) {
1566                         if (dev == last)
1567                                 goto outroll;
1568
1569                         if (dev->flags & IFF_UP) {
1570                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1571                                                         dev);
1572                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1573                         }
1574                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1575                 }
1576         }
1577
1578 outroll:
1579         raw_notifier_chain_unregister(&netdev_chain, nb);
1580         goto unlock;
1581 }
1582 EXPORT_SYMBOL(register_netdevice_notifier);
1583
1584 /**
1585  *      unregister_netdevice_notifier - unregister a network notifier block
1586  *      @nb: notifier
1587  *
1588  *      Unregister a notifier previously registered by
1589  *      register_netdevice_notifier(). The notifier is unlinked into the
1590  *      kernel structures and may then be reused. A negative errno code
1591  *      is returned on a failure.
1592  *
1593  *      After unregistering unregister and down device events are synthesized
1594  *      for all devices on the device list to the removed notifier to remove
1595  *      the need for special case cleanup code.
1596  */
1597
1598 int unregister_netdevice_notifier(struct notifier_block *nb)
1599 {
1600         struct net_device *dev;
1601         struct net *net;
1602         int err;
1603
1604         rtnl_lock();
1605         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1606         if (err)
1607                 goto unlock;
1608
1609         for_each_net(net) {
1610                 for_each_netdev(net, dev) {
1611                         if (dev->flags & IFF_UP) {
1612                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1613                                                         dev);
1614                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1615                         }
1616                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1617                 }
1618         }
1619 unlock:
1620         rtnl_unlock();
1621         return err;
1622 }
1623 EXPORT_SYMBOL(unregister_netdevice_notifier);
1624
1625 /**
1626  *      call_netdevice_notifiers_info - call all network notifier blocks
1627  *      @val: value passed unmodified to notifier function
1628  *      @dev: net_device pointer passed unmodified to notifier function
1629  *      @info: notifier information data
1630  *
1631  *      Call all network notifier blocks.  Parameters and return value
1632  *      are as for raw_notifier_call_chain().
1633  */
1634
1635 static int call_netdevice_notifiers_info(unsigned long val,
1636                                          struct net_device *dev,
1637                                          struct netdev_notifier_info *info)
1638 {
1639         ASSERT_RTNL();
1640         netdev_notifier_info_init(info, dev);
1641         return raw_notifier_call_chain(&netdev_chain, val, info);
1642 }
1643
1644 /**
1645  *      call_netdevice_notifiers - call all network notifier blocks
1646  *      @val: value passed unmodified to notifier function
1647  *      @dev: net_device pointer passed unmodified to notifier function
1648  *
1649  *      Call all network notifier blocks.  Parameters and return value
1650  *      are as for raw_notifier_call_chain().
1651  */
1652
1653 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1654 {
1655         struct netdev_notifier_info info;
1656
1657         return call_netdevice_notifiers_info(val, dev, &info);
1658 }
1659 EXPORT_SYMBOL(call_netdevice_notifiers);
1660
1661 #ifdef CONFIG_NET_INGRESS
1662 static struct static_key ingress_needed __read_mostly;
1663
1664 void net_inc_ingress_queue(void)
1665 {
1666         static_key_slow_inc(&ingress_needed);
1667 }
1668 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1669
1670 void net_dec_ingress_queue(void)
1671 {
1672         static_key_slow_dec(&ingress_needed);
1673 }
1674 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1675 #endif
1676
1677 static struct static_key netstamp_needed __read_mostly;
1678 #ifdef HAVE_JUMP_LABEL
1679 /* We are not allowed to call static_key_slow_dec() from irq context
1680  * If net_disable_timestamp() is called from irq context, defer the
1681  * static_key_slow_dec() calls.
1682  */
1683 static atomic_t netstamp_needed_deferred;
1684 #endif
1685
1686 void net_enable_timestamp(void)
1687 {
1688 #ifdef HAVE_JUMP_LABEL
1689         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1690
1691         if (deferred) {
1692                 while (--deferred)
1693                         static_key_slow_dec(&netstamp_needed);
1694                 return;
1695         }
1696 #endif
1697         static_key_slow_inc(&netstamp_needed);
1698 }
1699 EXPORT_SYMBOL(net_enable_timestamp);
1700
1701 void net_disable_timestamp(void)
1702 {
1703 #ifdef HAVE_JUMP_LABEL
1704         if (in_interrupt()) {
1705                 atomic_inc(&netstamp_needed_deferred);
1706                 return;
1707         }
1708 #endif
1709         static_key_slow_dec(&netstamp_needed);
1710 }
1711 EXPORT_SYMBOL(net_disable_timestamp);
1712
1713 static inline void net_timestamp_set(struct sk_buff *skb)
1714 {
1715         skb->tstamp.tv64 = 0;
1716         if (static_key_false(&netstamp_needed))
1717                 __net_timestamp(skb);
1718 }
1719
1720 #define net_timestamp_check(COND, SKB)                  \
1721         if (static_key_false(&netstamp_needed)) {               \
1722                 if ((COND) && !(SKB)->tstamp.tv64)      \
1723                         __net_timestamp(SKB);           \
1724         }                                               \
1725
1726 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1727 {
1728         unsigned int len;
1729
1730         if (!(dev->flags & IFF_UP))
1731                 return false;
1732
1733         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1734         if (skb->len <= len)
1735                 return true;
1736
1737         /* if TSO is enabled, we don't care about the length as the packet
1738          * could be forwarded without being segmented before
1739          */
1740         if (skb_is_gso(skb))
1741                 return true;
1742
1743         return false;
1744 }
1745 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1746
1747 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1748 {
1749         if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1750             unlikely(!is_skb_forwardable(dev, skb))) {
1751                 atomic_long_inc(&dev->rx_dropped);
1752                 kfree_skb(skb);
1753                 return NET_RX_DROP;
1754         }
1755
1756         skb_scrub_packet(skb, true);
1757         skb->priority = 0;
1758         skb->protocol = eth_type_trans(skb, dev);
1759         skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1760
1761         return 0;
1762 }
1763 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1764
1765 /**
1766  * dev_forward_skb - loopback an skb to another netif
1767  *
1768  * @dev: destination network device
1769  * @skb: buffer to forward
1770  *
1771  * return values:
1772  *      NET_RX_SUCCESS  (no congestion)
1773  *      NET_RX_DROP     (packet was dropped, but freed)
1774  *
1775  * dev_forward_skb can be used for injecting an skb from the
1776  * start_xmit function of one device into the receive queue
1777  * of another device.
1778  *
1779  * The receiving device may be in another namespace, so
1780  * we have to clear all information in the skb that could
1781  * impact namespace isolation.
1782  */
1783 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1784 {
1785         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1786 }
1787 EXPORT_SYMBOL_GPL(dev_forward_skb);
1788
1789 static inline int deliver_skb(struct sk_buff *skb,
1790                               struct packet_type *pt_prev,
1791                               struct net_device *orig_dev)
1792 {
1793         if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1794                 return -ENOMEM;
1795         atomic_inc(&skb->users);
1796         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1797 }
1798
1799 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1800                                           struct packet_type **pt,
1801                                           struct net_device *orig_dev,
1802                                           __be16 type,
1803                                           struct list_head *ptype_list)
1804 {
1805         struct packet_type *ptype, *pt_prev = *pt;
1806
1807         list_for_each_entry_rcu(ptype, ptype_list, list) {
1808                 if (ptype->type != type)
1809                         continue;
1810                 if (pt_prev)
1811                         deliver_skb(skb, pt_prev, orig_dev);
1812                 pt_prev = ptype;
1813         }
1814         *pt = pt_prev;
1815 }
1816
1817 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1818 {
1819         if (!ptype->af_packet_priv || !skb->sk)
1820                 return false;
1821
1822         if (ptype->id_match)
1823                 return ptype->id_match(ptype, skb->sk);
1824         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1825                 return true;
1826
1827         return false;
1828 }
1829
1830 /*
1831  *      Support routine. Sends outgoing frames to any network
1832  *      taps currently in use.
1833  */
1834
1835 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1836 {
1837         struct packet_type *ptype;
1838         struct sk_buff *skb2 = NULL;
1839         struct packet_type *pt_prev = NULL;
1840         struct list_head *ptype_list = &ptype_all;
1841
1842         rcu_read_lock();
1843 again:
1844         list_for_each_entry_rcu(ptype, ptype_list, list) {
1845                 /* Never send packets back to the socket
1846                  * they originated from - MvS (miquels@drinkel.ow.org)
1847                  */
1848                 if (skb_loop_sk(ptype, skb))
1849                         continue;
1850
1851                 if (pt_prev) {
1852                         deliver_skb(skb2, pt_prev, skb->dev);
1853                         pt_prev = ptype;
1854                         continue;
1855                 }
1856
1857                 /* need to clone skb, done only once */
1858                 skb2 = skb_clone(skb, GFP_ATOMIC);
1859                 if (!skb2)
1860                         goto out_unlock;
1861
1862                 net_timestamp_set(skb2);
1863
1864                 /* skb->nh should be correctly
1865                  * set by sender, so that the second statement is
1866                  * just protection against buggy protocols.
1867                  */
1868                 skb_reset_mac_header(skb2);
1869
1870                 if (skb_network_header(skb2) < skb2->data ||
1871                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1872                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1873                                              ntohs(skb2->protocol),
1874                                              dev->name);
1875                         skb_reset_network_header(skb2);
1876                 }
1877
1878                 skb2->transport_header = skb2->network_header;
1879                 skb2->pkt_type = PACKET_OUTGOING;
1880                 pt_prev = ptype;
1881         }
1882
1883         if (ptype_list == &ptype_all) {
1884                 ptype_list = &dev->ptype_all;
1885                 goto again;
1886         }
1887 out_unlock:
1888         if (pt_prev)
1889                 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1890         rcu_read_unlock();
1891 }
1892
1893 /**
1894  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1895  * @dev: Network device
1896  * @txq: number of queues available
1897  *
1898  * If real_num_tx_queues is changed the tc mappings may no longer be
1899  * valid. To resolve this verify the tc mapping remains valid and if
1900  * not NULL the mapping. With no priorities mapping to this
1901  * offset/count pair it will no longer be used. In the worst case TC0
1902  * is invalid nothing can be done so disable priority mappings. If is
1903  * expected that drivers will fix this mapping if they can before
1904  * calling netif_set_real_num_tx_queues.
1905  */
1906 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1907 {
1908         int i;
1909         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1910
1911         /* If TC0 is invalidated disable TC mapping */
1912         if (tc->offset + tc->count > txq) {
1913                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1914                 dev->num_tc = 0;
1915                 return;
1916         }
1917
1918         /* Invalidated prio to tc mappings set to TC0 */
1919         for (i = 1; i < TC_BITMASK + 1; i++) {
1920                 int q = netdev_get_prio_tc_map(dev, i);
1921
1922                 tc = &dev->tc_to_txq[q];
1923                 if (tc->offset + tc->count > txq) {
1924                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1925                                 i, q);
1926                         netdev_set_prio_tc_map(dev, i, 0);
1927                 }
1928         }
1929 }
1930
1931 #ifdef CONFIG_XPS
1932 static DEFINE_MUTEX(xps_map_mutex);
1933 #define xmap_dereference(P)             \
1934         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1935
1936 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1937                                         int cpu, u16 index)
1938 {
1939         struct xps_map *map = NULL;
1940         int pos;
1941
1942         if (dev_maps)
1943                 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1944
1945         for (pos = 0; map && pos < map->len; pos++) {
1946                 if (map->queues[pos] == index) {
1947                         if (map->len > 1) {
1948                                 map->queues[pos] = map->queues[--map->len];
1949                         } else {
1950                                 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1951                                 kfree_rcu(map, rcu);
1952                                 map = NULL;
1953                         }
1954                         break;
1955                 }
1956         }
1957
1958         return map;
1959 }
1960
1961 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1962 {
1963         struct xps_dev_maps *dev_maps;
1964         int cpu, i;
1965         bool active = false;
1966
1967         mutex_lock(&xps_map_mutex);
1968         dev_maps = xmap_dereference(dev->xps_maps);
1969
1970         if (!dev_maps)
1971                 goto out_no_maps;
1972
1973         for_each_possible_cpu(cpu) {
1974                 for (i = index; i < dev->num_tx_queues; i++) {
1975                         if (!remove_xps_queue(dev_maps, cpu, i))
1976                                 break;
1977                 }
1978                 if (i == dev->num_tx_queues)
1979                         active = true;
1980         }
1981
1982         if (!active) {
1983                 RCU_INIT_POINTER(dev->xps_maps, NULL);
1984                 kfree_rcu(dev_maps, rcu);
1985         }
1986
1987         for (i = index; i < dev->num_tx_queues; i++)
1988                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
1989                                              NUMA_NO_NODE);
1990
1991 out_no_maps:
1992         mutex_unlock(&xps_map_mutex);
1993 }
1994
1995 static struct xps_map *expand_xps_map(struct xps_map *map,
1996                                       int cpu, u16 index)
1997 {
1998         struct xps_map *new_map;
1999         int alloc_len = XPS_MIN_MAP_ALLOC;
2000         int i, pos;
2001
2002         for (pos = 0; map && pos < map->len; pos++) {
2003                 if (map->queues[pos] != index)
2004                         continue;
2005                 return map;
2006         }
2007
2008         /* Need to add queue to this CPU's existing map */
2009         if (map) {
2010                 if (pos < map->alloc_len)
2011                         return map;
2012
2013                 alloc_len = map->alloc_len * 2;
2014         }
2015
2016         /* Need to allocate new map to store queue on this CPU's map */
2017         new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2018                                cpu_to_node(cpu));
2019         if (!new_map)
2020                 return NULL;
2021
2022         for (i = 0; i < pos; i++)
2023                 new_map->queues[i] = map->queues[i];
2024         new_map->alloc_len = alloc_len;
2025         new_map->len = pos;
2026
2027         return new_map;
2028 }
2029
2030 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2031                         u16 index)
2032 {
2033         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2034         struct xps_map *map, *new_map;
2035         int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2036         int cpu, numa_node_id = -2;
2037         bool active = false;
2038
2039         mutex_lock(&xps_map_mutex);
2040
2041         dev_maps = xmap_dereference(dev->xps_maps);
2042
2043         /* allocate memory for queue storage */
2044         for_each_online_cpu(cpu) {
2045                 if (!cpumask_test_cpu(cpu, mask))
2046                         continue;
2047
2048                 if (!new_dev_maps)
2049                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2050                 if (!new_dev_maps) {
2051                         mutex_unlock(&xps_map_mutex);
2052                         return -ENOMEM;
2053                 }
2054
2055                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2056                                  NULL;
2057
2058                 map = expand_xps_map(map, cpu, index);
2059                 if (!map)
2060                         goto error;
2061
2062                 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2063         }
2064
2065         if (!new_dev_maps)
2066                 goto out_no_new_maps;
2067
2068         for_each_possible_cpu(cpu) {
2069                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2070                         /* add queue to CPU maps */
2071                         int pos = 0;
2072
2073                         map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2074                         while ((pos < map->len) && (map->queues[pos] != index))
2075                                 pos++;
2076
2077                         if (pos == map->len)
2078                                 map->queues[map->len++] = index;
2079 #ifdef CONFIG_NUMA
2080                         if (numa_node_id == -2)
2081                                 numa_node_id = cpu_to_node(cpu);
2082                         else if (numa_node_id != cpu_to_node(cpu))
2083                                 numa_node_id = -1;
2084 #endif
2085                 } else if (dev_maps) {
2086                         /* fill in the new device map from the old device map */
2087                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2088                         RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2089                 }
2090
2091         }
2092
2093         rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2094
2095         /* Cleanup old maps */
2096         if (dev_maps) {
2097                 for_each_possible_cpu(cpu) {
2098                         new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2099                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2100                         if (map && map != new_map)
2101                                 kfree_rcu(map, rcu);
2102                 }
2103
2104                 kfree_rcu(dev_maps, rcu);
2105         }
2106
2107         dev_maps = new_dev_maps;
2108         active = true;
2109
2110 out_no_new_maps:
2111         /* update Tx queue numa node */
2112         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2113                                      (numa_node_id >= 0) ? numa_node_id :
2114                                      NUMA_NO_NODE);
2115
2116         if (!dev_maps)
2117                 goto out_no_maps;
2118
2119         /* removes queue from unused CPUs */
2120         for_each_possible_cpu(cpu) {
2121                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2122                         continue;
2123
2124                 if (remove_xps_queue(dev_maps, cpu, index))
2125                         active = true;
2126         }
2127
2128         /* free map if not active */
2129         if (!active) {
2130                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2131                 kfree_rcu(dev_maps, rcu);
2132         }
2133
2134 out_no_maps:
2135         mutex_unlock(&xps_map_mutex);
2136
2137         return 0;
2138 error:
2139         /* remove any maps that we added */
2140         for_each_possible_cpu(cpu) {
2141                 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2142                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2143                                  NULL;
2144                 if (new_map && new_map != map)
2145                         kfree(new_map);
2146         }
2147
2148         mutex_unlock(&xps_map_mutex);
2149
2150         kfree(new_dev_maps);
2151         return -ENOMEM;
2152 }
2153 EXPORT_SYMBOL(netif_set_xps_queue);
2154
2155 #endif
2156 /*
2157  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2158  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2159  */
2160 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2161 {
2162         int rc;
2163
2164         if (txq < 1 || txq > dev->num_tx_queues)
2165                 return -EINVAL;
2166
2167         if (dev->reg_state == NETREG_REGISTERED ||
2168             dev->reg_state == NETREG_UNREGISTERING) {
2169                 ASSERT_RTNL();
2170
2171                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2172                                                   txq);
2173                 if (rc)
2174                         return rc;
2175
2176                 if (dev->num_tc)
2177                         netif_setup_tc(dev, txq);
2178
2179                 if (txq < dev->real_num_tx_queues) {
2180                         qdisc_reset_all_tx_gt(dev, txq);
2181 #ifdef CONFIG_XPS
2182                         netif_reset_xps_queues_gt(dev, txq);
2183 #endif
2184                 }
2185         }
2186
2187         dev->real_num_tx_queues = txq;
2188         return 0;
2189 }
2190 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2191
2192 #ifdef CONFIG_SYSFS
2193 /**
2194  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2195  *      @dev: Network device
2196  *      @rxq: Actual number of RX queues
2197  *
2198  *      This must be called either with the rtnl_lock held or before
2199  *      registration of the net device.  Returns 0 on success, or a
2200  *      negative error code.  If called before registration, it always
2201  *      succeeds.
2202  */
2203 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2204 {
2205         int rc;
2206
2207         if (rxq < 1 || rxq > dev->num_rx_queues)
2208                 return -EINVAL;
2209
2210         if (dev->reg_state == NETREG_REGISTERED) {
2211                 ASSERT_RTNL();
2212
2213                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2214                                                   rxq);
2215                 if (rc)
2216                         return rc;
2217         }
2218
2219         dev->real_num_rx_queues = rxq;
2220         return 0;
2221 }
2222 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2223 #endif
2224
2225 /**
2226  * netif_get_num_default_rss_queues - default number of RSS queues
2227  *
2228  * This routine should set an upper limit on the number of RSS queues
2229  * used by default by multiqueue devices.
2230  */
2231 int netif_get_num_default_rss_queues(void)
2232 {
2233         return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2234 }
2235 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2236
2237 static inline void __netif_reschedule(struct Qdisc *q)
2238 {
2239         struct softnet_data *sd;
2240         unsigned long flags;
2241
2242         local_irq_save(flags);
2243         sd = this_cpu_ptr(&softnet_data);
2244         q->next_sched = NULL;
2245         *sd->output_queue_tailp = q;
2246         sd->output_queue_tailp = &q->next_sched;
2247         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2248         local_irq_restore(flags);
2249 }
2250
2251 void __netif_schedule(struct Qdisc *q)
2252 {
2253         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2254                 __netif_reschedule(q);
2255 }
2256 EXPORT_SYMBOL(__netif_schedule);
2257
2258 struct dev_kfree_skb_cb {
2259         enum skb_free_reason reason;
2260 };
2261
2262 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2263 {
2264         return (struct dev_kfree_skb_cb *)skb->cb;
2265 }
2266
2267 void netif_schedule_queue(struct netdev_queue *txq)
2268 {
2269         rcu_read_lock();
2270         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2271                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2272
2273                 __netif_schedule(q);
2274         }
2275         rcu_read_unlock();
2276 }
2277 EXPORT_SYMBOL(netif_schedule_queue);
2278
2279 /**
2280  *      netif_wake_subqueue - allow sending packets on subqueue
2281  *      @dev: network device
2282  *      @queue_index: sub queue index
2283  *
2284  * Resume individual transmit queue of a device with multiple transmit queues.
2285  */
2286 void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2287 {
2288         struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2289
2290         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2291                 struct Qdisc *q;
2292
2293                 rcu_read_lock();
2294                 q = rcu_dereference(txq->qdisc);
2295                 __netif_schedule(q);
2296                 rcu_read_unlock();
2297         }
2298 }
2299 EXPORT_SYMBOL(netif_wake_subqueue);
2300
2301 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2302 {
2303         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2304                 struct Qdisc *q;
2305
2306                 rcu_read_lock();
2307                 q = rcu_dereference(dev_queue->qdisc);
2308                 __netif_schedule(q);
2309                 rcu_read_unlock();
2310         }
2311 }
2312 EXPORT_SYMBOL(netif_tx_wake_queue);
2313
2314 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2315 {
2316         unsigned long flags;
2317
2318         if (likely(atomic_read(&skb->users) == 1)) {
2319                 smp_rmb();
2320                 atomic_set(&skb->users, 0);
2321         } else if (likely(!atomic_dec_and_test(&skb->users))) {
2322                 return;
2323         }
2324         get_kfree_skb_cb(skb)->reason = reason;
2325         local_irq_save(flags);
2326         skb->next = __this_cpu_read(softnet_data.completion_queue);
2327         __this_cpu_write(softnet_data.completion_queue, skb);
2328         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2329         local_irq_restore(flags);
2330 }
2331 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2332
2333 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2334 {
2335         if (in_irq() || irqs_disabled())
2336                 __dev_kfree_skb_irq(skb, reason);
2337         else
2338                 dev_kfree_skb(skb);
2339 }
2340 EXPORT_SYMBOL(__dev_kfree_skb_any);
2341
2342
2343 /**
2344  * netif_device_detach - mark device as removed
2345  * @dev: network device
2346  *
2347  * Mark device as removed from system and therefore no longer available.
2348  */
2349 void netif_device_detach(struct net_device *dev)
2350 {
2351         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2352             netif_running(dev)) {
2353                 netif_tx_stop_all_queues(dev);
2354         }
2355 }
2356 EXPORT_SYMBOL(netif_device_detach);
2357
2358 /**
2359  * netif_device_attach - mark device as attached
2360  * @dev: network device
2361  *
2362  * Mark device as attached from system and restart if needed.
2363  */
2364 void netif_device_attach(struct net_device *dev)
2365 {
2366         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2367             netif_running(dev)) {
2368                 netif_tx_wake_all_queues(dev);
2369                 __netdev_watchdog_up(dev);
2370         }
2371 }
2372 EXPORT_SYMBOL(netif_device_attach);
2373
2374 /*
2375  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2376  * to be used as a distribution range.
2377  */
2378 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2379                   unsigned int num_tx_queues)
2380 {
2381         u32 hash;
2382         u16 qoffset = 0;
2383         u16 qcount = num_tx_queues;
2384
2385         if (skb_rx_queue_recorded(skb)) {
2386                 hash = skb_get_rx_queue(skb);
2387                 while (unlikely(hash >= num_tx_queues))
2388                         hash -= num_tx_queues;
2389                 return hash;
2390         }
2391
2392         if (dev->num_tc) {
2393                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2394                 qoffset = dev->tc_to_txq[tc].offset;
2395                 qcount = dev->tc_to_txq[tc].count;
2396         }
2397
2398         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2399 }
2400 EXPORT_SYMBOL(__skb_tx_hash);
2401
2402 static void skb_warn_bad_offload(const struct sk_buff *skb)
2403 {
2404         static const netdev_features_t null_features = 0;
2405         struct net_device *dev = skb->dev;
2406         const char *name = "";
2407
2408         if (!net_ratelimit())
2409                 return;
2410
2411         if (dev) {
2412                 if (dev->dev.parent)
2413                         name = dev_driver_string(dev->dev.parent);
2414                 else
2415                         name = netdev_name(dev);
2416         }
2417         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2418              "gso_type=%d ip_summed=%d\n",
2419              name, dev ? &dev->features : &null_features,
2420              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2421              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2422              skb_shinfo(skb)->gso_type, skb->ip_summed);
2423 }
2424
2425 /*
2426  * Invalidate hardware checksum when packet is to be mangled, and
2427  * complete checksum manually on outgoing path.
2428  */
2429 int skb_checksum_help(struct sk_buff *skb)
2430 {
2431         __wsum csum;
2432         int ret = 0, offset;
2433
2434         if (skb->ip_summed == CHECKSUM_COMPLETE)
2435                 goto out_set_summed;
2436
2437         if (unlikely(skb_shinfo(skb)->gso_size)) {
2438                 skb_warn_bad_offload(skb);
2439                 return -EINVAL;
2440         }
2441
2442         /* Before computing a checksum, we should make sure no frag could
2443          * be modified by an external entity : checksum could be wrong.
2444          */
2445         if (skb_has_shared_frag(skb)) {
2446                 ret = __skb_linearize(skb);
2447                 if (ret)
2448                         goto out;
2449         }
2450
2451         offset = skb_checksum_start_offset(skb);
2452         BUG_ON(offset >= skb_headlen(skb));
2453         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2454
2455         offset += skb->csum_offset;
2456         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2457
2458         if (skb_cloned(skb) &&
2459             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2460                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2461                 if (ret)
2462                         goto out;
2463         }
2464
2465         *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2466 out_set_summed:
2467         skb->ip_summed = CHECKSUM_NONE;
2468 out:
2469         return ret;
2470 }
2471 EXPORT_SYMBOL(skb_checksum_help);
2472
2473 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2474 {
2475         __be16 type = skb->protocol;
2476
2477         /* Tunnel gso handlers can set protocol to ethernet. */
2478         if (type == htons(ETH_P_TEB)) {
2479                 struct ethhdr *eth;
2480
2481                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2482                         return 0;
2483
2484                 eth = (struct ethhdr *)skb_mac_header(skb);
2485                 type = eth->h_proto;
2486         }
2487
2488         return __vlan_get_protocol(skb, type, depth);
2489 }
2490
2491 /**
2492  *      skb_mac_gso_segment - mac layer segmentation handler.
2493  *      @skb: buffer to segment
2494  *      @features: features for the output path (see dev->features)
2495  */
2496 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2497                                     netdev_features_t features)
2498 {
2499         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2500         struct packet_offload *ptype;
2501         int vlan_depth = skb->mac_len;
2502         __be16 type = skb_network_protocol(skb, &vlan_depth);
2503
2504         if (unlikely(!type))
2505                 return ERR_PTR(-EINVAL);
2506
2507         __skb_pull(skb, vlan_depth);
2508
2509         rcu_read_lock();
2510         list_for_each_entry_rcu(ptype, &offload_base, list) {
2511                 if (ptype->type == type && ptype->callbacks.gso_segment) {
2512                         segs = ptype->callbacks.gso_segment(skb, features);
2513                         break;
2514                 }
2515         }
2516         rcu_read_unlock();
2517
2518         __skb_push(skb, skb->data - skb_mac_header(skb));
2519
2520         return segs;
2521 }
2522 EXPORT_SYMBOL(skb_mac_gso_segment);
2523
2524
2525 /* openvswitch calls this on rx path, so we need a different check.
2526  */
2527 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2528 {
2529         if (tx_path)
2530                 return skb->ip_summed != CHECKSUM_PARTIAL;
2531         else
2532                 return skb->ip_summed == CHECKSUM_NONE;
2533 }
2534
2535 /**
2536  *      __skb_gso_segment - Perform segmentation on skb.
2537  *      @skb: buffer to segment
2538  *      @features: features for the output path (see dev->features)
2539  *      @tx_path: whether it is called in TX path
2540  *
2541  *      This function segments the given skb and returns a list of segments.
2542  *
2543  *      It may return NULL if the skb requires no segmentation.  This is
2544  *      only possible when GSO is used for verifying header integrity.
2545  */
2546 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2547                                   netdev_features_t features, bool tx_path)
2548 {
2549         if (unlikely(skb_needs_check(skb, tx_path))) {
2550                 int err;
2551
2552                 skb_warn_bad_offload(skb);
2553
2554                 err = skb_cow_head(skb, 0);
2555                 if (err < 0)
2556                         return ERR_PTR(err);
2557         }
2558
2559         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2560         SKB_GSO_CB(skb)->encap_level = 0;
2561
2562         skb_reset_mac_header(skb);
2563         skb_reset_mac_len(skb);
2564
2565         return skb_mac_gso_segment(skb, features);
2566 }
2567 EXPORT_SYMBOL(__skb_gso_segment);
2568
2569 /* Take action when hardware reception checksum errors are detected. */
2570 #ifdef CONFIG_BUG
2571 void netdev_rx_csum_fault(struct net_device *dev)
2572 {
2573         if (net_ratelimit()) {
2574                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2575                 dump_stack();
2576         }
2577 }
2578 EXPORT_SYMBOL(netdev_rx_csum_fault);
2579 #endif
2580
2581 /* Actually, we should eliminate this check as soon as we know, that:
2582  * 1. IOMMU is present and allows to map all the memory.
2583  * 2. No high memory really exists on this machine.
2584  */
2585
2586 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2587 {
2588 #ifdef CONFIG_HIGHMEM
2589         int i;
2590         if (!(dev->features & NETIF_F_HIGHDMA)) {
2591                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2592                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2593                         if (PageHighMem(skb_frag_page(frag)))
2594                                 return 1;
2595                 }
2596         }
2597
2598         if (PCI_DMA_BUS_IS_PHYS) {
2599                 struct device *pdev = dev->dev.parent;
2600
2601                 if (!pdev)
2602                         return 0;
2603                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2604                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2605                         dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2606                         if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2607                                 return 1;
2608                 }
2609         }
2610 #endif
2611         return 0;
2612 }
2613
2614 /* If MPLS offload request, verify we are testing hardware MPLS features
2615  * instead of standard features for the netdev.
2616  */
2617 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2618 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2619                                            netdev_features_t features,
2620                                            __be16 type)
2621 {
2622         if (eth_p_mpls(type))
2623                 features &= skb->dev->mpls_features;
2624
2625         return features;
2626 }
2627 #else
2628 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2629                                            netdev_features_t features,
2630                                            __be16 type)
2631 {
2632         return features;
2633 }
2634 #endif
2635
2636 static netdev_features_t harmonize_features(struct sk_buff *skb,
2637         netdev_features_t features)
2638 {
2639         int tmp;
2640         __be16 type;
2641
2642         type = skb_network_protocol(skb, &tmp);
2643         features = net_mpls_features(skb, features, type);
2644
2645         if (skb->ip_summed != CHECKSUM_NONE &&
2646             !can_checksum_protocol(features, type)) {
2647                 features &= ~NETIF_F_ALL_CSUM;
2648         } else if (illegal_highdma(skb->dev, skb)) {
2649                 features &= ~NETIF_F_SG;
2650         }
2651
2652         return features;
2653 }
2654
2655 netdev_features_t passthru_features_check(struct sk_buff *skb,
2656                                           struct net_device *dev,
2657                                           netdev_features_t features)
2658 {
2659         return features;
2660 }
2661 EXPORT_SYMBOL(passthru_features_check);
2662
2663 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2664                                              struct net_device *dev,
2665                                              netdev_features_t features)
2666 {
2667         return vlan_features_check(skb, features);
2668 }
2669
2670 netdev_features_t netif_skb_features(struct sk_buff *skb)
2671 {
2672         struct net_device *dev = skb->dev;
2673         netdev_features_t features = dev->features;
2674         u16 gso_segs = skb_shinfo(skb)->gso_segs;
2675
2676         if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
2677                 features &= ~NETIF_F_GSO_MASK;
2678
2679         /* If encapsulation offload request, verify we are testing
2680          * hardware encapsulation features instead of standard
2681          * features for the netdev
2682          */
2683         if (skb->encapsulation)
2684                 features &= dev->hw_enc_features;
2685
2686         if (skb_vlan_tagged(skb))
2687                 features = netdev_intersect_features(features,
2688                                                      dev->vlan_features |
2689                                                      NETIF_F_HW_VLAN_CTAG_TX |
2690                                                      NETIF_F_HW_VLAN_STAG_TX);
2691
2692         if (dev->netdev_ops->ndo_features_check)
2693                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2694                                                                 features);
2695         else
2696                 features &= dflt_features_check(skb, dev, features);
2697
2698         return harmonize_features(skb, features);
2699 }
2700 EXPORT_SYMBOL(netif_skb_features);
2701
2702 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2703                     struct netdev_queue *txq, bool more)
2704 {
2705         unsigned int len;
2706         int rc;
2707
2708         if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2709                 dev_queue_xmit_nit(skb, dev);
2710
2711         len = skb->len;
2712         trace_net_dev_start_xmit(skb, dev);
2713         rc = netdev_start_xmit(skb, dev, txq, more);
2714         trace_net_dev_xmit(skb, rc, dev, len);
2715
2716         return rc;
2717 }
2718
2719 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2720                                     struct netdev_queue *txq, int *ret)
2721 {
2722         struct sk_buff *skb = first;
2723         int rc = NETDEV_TX_OK;
2724
2725         while (skb) {
2726                 struct sk_buff *next = skb->next;
2727
2728                 skb->next = NULL;
2729                 rc = xmit_one(skb, dev, txq, next != NULL);
2730                 if (unlikely(!dev_xmit_complete(rc))) {
2731                         skb->next = next;
2732                         goto out;
2733                 }
2734
2735                 skb = next;
2736                 if (netif_xmit_stopped(txq) && skb) {
2737                         rc = NETDEV_TX_BUSY;
2738                         break;
2739                 }
2740         }
2741
2742 out:
2743         *ret = rc;
2744         return skb;
2745 }
2746
2747 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2748                                           netdev_features_t features)
2749 {
2750         if (skb_vlan_tag_present(skb) &&
2751             !vlan_hw_offload_capable(features, skb->vlan_proto))
2752                 skb = __vlan_hwaccel_push_inside(skb);
2753         return skb;
2754 }
2755
2756 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2757 {
2758         netdev_features_t features;
2759
2760         if (skb->next)
2761                 return skb;
2762
2763         features = netif_skb_features(skb);
2764         skb = validate_xmit_vlan(skb, features);
2765         if (unlikely(!skb))
2766                 goto out_null;
2767
2768         if (netif_needs_gso(skb, features)) {
2769                 struct sk_buff *segs;
2770
2771                 segs = skb_gso_segment(skb, features);
2772                 if (IS_ERR(segs)) {
2773                         goto out_kfree_skb;
2774                 } else if (segs) {
2775                         consume_skb(skb);
2776                         skb = segs;
2777                 }
2778         } else {
2779                 if (skb_needs_linearize(skb, features) &&
2780                     __skb_linearize(skb))
2781                         goto out_kfree_skb;
2782
2783                 /* If packet is not checksummed and device does not
2784                  * support checksumming for this protocol, complete
2785                  * checksumming here.
2786                  */
2787                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2788                         if (skb->encapsulation)
2789                                 skb_set_inner_transport_header(skb,
2790                                                                skb_checksum_start_offset(skb));
2791                         else
2792                                 skb_set_transport_header(skb,
2793                                                          skb_checksum_start_offset(skb));
2794                         if (!(features & NETIF_F_ALL_CSUM) &&
2795                             skb_checksum_help(skb))
2796                                 goto out_kfree_skb;
2797                 }
2798         }
2799
2800         return skb;
2801
2802 out_kfree_skb:
2803         kfree_skb(skb);
2804 out_null:
2805         return NULL;
2806 }
2807
2808 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2809 {
2810         struct sk_buff *next, *head = NULL, *tail;
2811
2812         for (; skb != NULL; skb = next) {
2813                 next = skb->next;
2814                 skb->next = NULL;
2815
2816                 /* in case skb wont be segmented, point to itself */
2817                 skb->prev = skb;
2818
2819                 skb = validate_xmit_skb(skb, dev);
2820                 if (!skb)
2821                         continue;
2822
2823                 if (!head)
2824                         head = skb;
2825                 else
2826                         tail->next = skb;
2827                 /* If skb was segmented, skb->prev points to
2828                  * the last segment. If not, it still contains skb.
2829                  */
2830                 tail = skb->prev;
2831         }
2832         return head;
2833 }
2834
2835 static void qdisc_pkt_len_init(struct sk_buff *skb)
2836 {
2837         const struct skb_shared_info *shinfo = skb_shinfo(skb);
2838
2839         qdisc_skb_cb(skb)->pkt_len = skb->len;
2840
2841         /* To get more precise estimation of bytes sent on wire,
2842          * we add to pkt_len the headers size of all segments
2843          */
2844         if (shinfo->gso_size)  {
2845                 unsigned int hdr_len;
2846                 u16 gso_segs = shinfo->gso_segs;
2847
2848                 /* mac layer + network layer */
2849                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
2850
2851                 /* + transport layer */
2852                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
2853                         hdr_len += tcp_hdrlen(skb);
2854                 else
2855                         hdr_len += sizeof(struct udphdr);
2856
2857                 if (shinfo->gso_type & SKB_GSO_DODGY)
2858                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
2859                                                 shinfo->gso_size);
2860
2861                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
2862         }
2863 }
2864
2865 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2866                                  struct net_device *dev,
2867                                  struct netdev_queue *txq)
2868 {
2869         spinlock_t *root_lock = qdisc_lock(q);
2870         bool contended;
2871         int rc;
2872
2873         qdisc_pkt_len_init(skb);
2874         qdisc_calculate_pkt_len(skb, q);
2875         /*
2876          * Heuristic to force contended enqueues to serialize on a
2877          * separate lock before trying to get qdisc main lock.
2878          * This permits __QDISC___STATE_RUNNING owner to get the lock more
2879          * often and dequeue packets faster.
2880          */
2881         contended = qdisc_is_running(q);
2882         if (unlikely(contended))
2883                 spin_lock(&q->busylock);
2884
2885         spin_lock(root_lock);
2886         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2887                 kfree_skb(skb);
2888                 rc = NET_XMIT_DROP;
2889         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2890                    qdisc_run_begin(q)) {
2891                 /*
2892                  * This is a work-conserving queue; there are no old skbs
2893                  * waiting to be sent out; and the qdisc is not running -
2894                  * xmit the skb directly.
2895                  */
2896
2897                 qdisc_bstats_update(q, skb);
2898
2899                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
2900                         if (unlikely(contended)) {
2901                                 spin_unlock(&q->busylock);
2902                                 contended = false;
2903                         }
2904                         __qdisc_run(q);
2905                 } else
2906                         qdisc_run_end(q);
2907
2908                 rc = NET_XMIT_SUCCESS;
2909         } else {
2910                 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2911                 if (qdisc_run_begin(q)) {
2912                         if (unlikely(contended)) {
2913                                 spin_unlock(&q->busylock);
2914                                 contended = false;
2915                         }
2916                         __qdisc_run(q);
2917                 }
2918         }
2919         spin_unlock(root_lock);
2920         if (unlikely(contended))
2921                 spin_unlock(&q->busylock);
2922         return rc;
2923 }
2924
2925 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2926 static void skb_update_prio(struct sk_buff *skb)
2927 {
2928         struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
2929
2930         if (!skb->priority && skb->sk && map) {
2931                 unsigned int prioidx = skb->sk->sk_cgrp_prioidx;
2932
2933                 if (prioidx < map->priomap_len)
2934                         skb->priority = map->priomap[prioidx];
2935         }
2936 }
2937 #else
2938 #define skb_update_prio(skb)
2939 #endif
2940
2941 DEFINE_PER_CPU(int, xmit_recursion);
2942 EXPORT_SYMBOL(xmit_recursion);
2943
2944 #define RECURSION_LIMIT 10
2945
2946 /**
2947  *      dev_loopback_xmit - loop back @skb
2948  *      @net: network namespace this loopback is happening in
2949  *      @sk:  sk needed to be a netfilter okfn
2950  *      @skb: buffer to transmit
2951  */
2952 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
2953 {
2954         skb_reset_mac_header(skb);
2955         __skb_pull(skb, skb_network_offset(skb));
2956         skb->pkt_type = PACKET_LOOPBACK;
2957         skb->ip_summed = CHECKSUM_UNNECESSARY;
2958         WARN_ON(!skb_dst(skb));
2959         skb_dst_force(skb);
2960         netif_rx_ni(skb);
2961         return 0;
2962 }
2963 EXPORT_SYMBOL(dev_loopback_xmit);
2964
2965 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2966 {
2967 #ifdef CONFIG_XPS
2968         struct xps_dev_maps *dev_maps;
2969         struct xps_map *map;
2970         int queue_index = -1;
2971
2972         rcu_read_lock();
2973         dev_maps = rcu_dereference(dev->xps_maps);
2974         if (dev_maps) {
2975                 map = rcu_dereference(
2976                     dev_maps->cpu_map[skb->sender_cpu - 1]);
2977                 if (map) {
2978                         if (map->len == 1)
2979                                 queue_index = map->queues[0];
2980                         else
2981                                 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
2982                                                                            map->len)];
2983                         if (unlikely(queue_index >= dev->real_num_tx_queues))
2984                                 queue_index = -1;
2985                 }
2986         }
2987         rcu_read_unlock();
2988
2989         return queue_index;
2990 #else
2991         return -1;
2992 #endif
2993 }
2994
2995 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
2996 {
2997         struct sock *sk = skb->sk;
2998         int queue_index = sk_tx_queue_get(sk);
2999
3000         if (queue_index < 0 || skb->ooo_okay ||
3001             queue_index >= dev->real_num_tx_queues) {
3002                 int new_index = get_xps_queue(dev, skb);
3003                 if (new_index < 0)
3004                         new_index = skb_tx_hash(dev, skb);
3005
3006                 if (queue_index != new_index && sk &&
3007                     sk_fullsock(sk) &&
3008                     rcu_access_pointer(sk->sk_dst_cache))
3009                         sk_tx_queue_set(sk, new_index);
3010
3011                 queue_index = new_index;
3012         }
3013
3014         return queue_index;
3015 }
3016
3017 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3018                                     struct sk_buff *skb,
3019                                     void *accel_priv)
3020 {
3021         int queue_index = 0;
3022
3023 #ifdef CONFIG_XPS
3024         if (skb->sender_cpu == 0)
3025                 skb->sender_cpu = raw_smp_processor_id() + 1;
3026 #endif
3027
3028         if (dev->real_num_tx_queues != 1) {
3029                 const struct net_device_ops *ops = dev->netdev_ops;
3030                 if (ops->ndo_select_queue)
3031                         queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3032                                                             __netdev_pick_tx);
3033                 else
3034                         queue_index = __netdev_pick_tx(dev, skb);
3035
3036                 if (!accel_priv)
3037                         queue_index = netdev_cap_txqueue(dev, queue_index);
3038         }
3039
3040         skb_set_queue_mapping(skb, queue_index);
3041         return netdev_get_tx_queue(dev, queue_index);
3042 }
3043
3044 /**
3045  *      __dev_queue_xmit - transmit a buffer
3046  *      @skb: buffer to transmit
3047  *      @accel_priv: private data used for L2 forwarding offload
3048  *
3049  *      Queue a buffer for transmission to a network device. The caller must
3050  *      have set the device and priority and built the buffer before calling
3051  *      this function. The function can be called from an interrupt.
3052  *
3053  *      A negative errno code is returned on a failure. A success does not
3054  *      guarantee the frame will be transmitted as it may be dropped due
3055  *      to congestion or traffic shaping.
3056  *
3057  * -----------------------------------------------------------------------------------
3058  *      I notice this method can also return errors from the queue disciplines,
3059  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3060  *      be positive.
3061  *
3062  *      Regardless of the return value, the skb is consumed, so it is currently
3063  *      difficult to retry a send to this method.  (You can bump the ref count
3064  *      before sending to hold a reference for retry if you are careful.)
3065  *
3066  *      When calling this method, interrupts MUST be enabled.  This is because
3067  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3068  *          --BLG
3069  */
3070 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3071 {
3072         struct net_device *dev = skb->dev;
3073         struct netdev_queue *txq;
3074         struct Qdisc *q;
3075         int rc = -ENOMEM;
3076
3077         skb_reset_mac_header(skb);
3078
3079         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3080                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3081
3082         /* Disable soft irqs for various locks below. Also
3083          * stops preemption for RCU.
3084          */
3085         rcu_read_lock_bh();
3086
3087         skb_update_prio(skb);
3088
3089         /* If device/qdisc don't need skb->dst, release it right now while
3090          * its hot in this cpu cache.
3091          */
3092         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3093                 skb_dst_drop(skb);
3094         else
3095                 skb_dst_force(skb);
3096
3097 #ifdef CONFIG_NET_SWITCHDEV
3098         /* Don't forward if offload device already forwarded */
3099         if (skb->offload_fwd_mark &&
3100             skb->offload_fwd_mark == dev->offload_fwd_mark) {
3101                 consume_skb(skb);
3102                 rc = NET_XMIT_SUCCESS;
3103                 goto out;
3104         }
3105 #endif
3106
3107         txq = netdev_pick_tx(dev, skb, accel_priv);
3108         q = rcu_dereference_bh(txq->qdisc);
3109
3110 #ifdef CONFIG_NET_CLS_ACT
3111         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3112 #endif
3113         trace_net_dev_queue(skb);
3114         if (q->enqueue) {
3115                 rc = __dev_xmit_skb(skb, q, dev, txq);
3116                 goto out;
3117         }
3118
3119         /* The device has no queue. Common case for software devices:
3120            loopback, all the sorts of tunnels...
3121
3122            Really, it is unlikely that netif_tx_lock protection is necessary
3123            here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3124            counters.)
3125            However, it is possible, that they rely on protection
3126            made by us here.
3127
3128            Check this and shot the lock. It is not prone from deadlocks.
3129            Either shot noqueue qdisc, it is even simpler 8)
3130          */
3131         if (dev->flags & IFF_UP) {
3132                 int cpu = smp_processor_id(); /* ok because BHs are off */
3133
3134                 if (txq->xmit_lock_owner != cpu) {
3135
3136                         if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3137                                 goto recursion_alert;
3138
3139                         skb = validate_xmit_skb(skb, dev);
3140                         if (!skb)
3141                                 goto drop;
3142
3143                         HARD_TX_LOCK(dev, txq, cpu);
3144
3145                         if (!netif_xmit_stopped(txq)) {
3146                                 __this_cpu_inc(xmit_recursion);
3147                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3148                                 __this_cpu_dec(xmit_recursion);
3149                                 if (dev_xmit_complete(rc)) {
3150                                         HARD_TX_UNLOCK(dev, txq);
3151                                         goto out;
3152                                 }
3153                         }
3154                         HARD_TX_UNLOCK(dev, txq);
3155                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3156                                              dev->name);
3157                 } else {
3158                         /* Recursion is detected! It is possible,
3159                          * unfortunately
3160                          */
3161 recursion_alert:
3162                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3163                                              dev->name);
3164                 }
3165         }
3166
3167         rc = -ENETDOWN;
3168 drop:
3169         rcu_read_unlock_bh();
3170
3171         atomic_long_inc(&dev->tx_dropped);
3172         kfree_skb_list(skb);
3173         return rc;
3174 out:
3175         rcu_read_unlock_bh();
3176         return rc;
3177 }
3178
3179 int dev_queue_xmit(struct sk_buff *skb)
3180 {
3181         return __dev_queue_xmit(skb, NULL);
3182 }
3183 EXPORT_SYMBOL(dev_queue_xmit);
3184
3185 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3186 {
3187         return __dev_queue_xmit(skb, accel_priv);
3188 }
3189 EXPORT_SYMBOL(dev_queue_xmit_accel);
3190
3191
3192 /*=======================================================================
3193                         Receiver routines
3194   =======================================================================*/
3195
3196 int netdev_max_backlog __read_mostly = 1000;
3197 EXPORT_SYMBOL(netdev_max_backlog);
3198
3199 int netdev_tstamp_prequeue __read_mostly = 1;
3200 int netdev_budget __read_mostly = 300;
3201 int weight_p __read_mostly = 64;            /* old backlog weight */
3202
3203 /* Called with irq disabled */
3204 static inline void ____napi_schedule(struct softnet_data *sd,
3205                                      struct napi_struct *napi)
3206 {
3207         list_add_tail(&napi->poll_list, &sd->poll_list);
3208         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3209 }
3210
3211 #ifdef CONFIG_RPS
3212
3213 /* One global table that all flow-based protocols share. */
3214 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3215 EXPORT_SYMBOL(rps_sock_flow_table);
3216 u32 rps_cpu_mask __read_mostly;
3217 EXPORT_SYMBOL(rps_cpu_mask);
3218
3219 struct static_key rps_needed __read_mostly;
3220
3221 static struct rps_dev_flow *
3222 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3223             struct rps_dev_flow *rflow, u16 next_cpu)
3224 {
3225         if (next_cpu < nr_cpu_ids) {
3226 #ifdef CONFIG_RFS_ACCEL
3227                 struct netdev_rx_queue *rxqueue;
3228                 struct rps_dev_flow_table *flow_table;
3229                 struct rps_dev_flow *old_rflow;
3230                 u32 flow_id;
3231                 u16 rxq_index;
3232                 int rc;
3233
3234                 /* Should we steer this flow to a different hardware queue? */
3235                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3236                     !(dev->features & NETIF_F_NTUPLE))
3237                         goto out;
3238                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3239                 if (rxq_index == skb_get_rx_queue(skb))
3240                         goto out;
3241
3242                 rxqueue = dev->_rx + rxq_index;
3243                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3244                 if (!flow_table)
3245                         goto out;
3246                 flow_id = skb_get_hash(skb) & flow_table->mask;
3247                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3248                                                         rxq_index, flow_id);
3249                 if (rc < 0)
3250                         goto out;
3251                 old_rflow = rflow;
3252                 rflow = &flow_table->flows[flow_id];
3253                 rflow->filter = rc;
3254                 if (old_rflow->filter == rflow->filter)
3255                         old_rflow->filter = RPS_NO_FILTER;
3256         out:
3257 #endif
3258                 rflow->last_qtail =
3259                         per_cpu(softnet_data, next_cpu).input_queue_head;
3260         }
3261
3262         rflow->cpu = next_cpu;
3263         return rflow;
3264 }
3265
3266 /*
3267  * get_rps_cpu is called from netif_receive_skb and returns the target
3268  * CPU from the RPS map of the receiving queue for a given skb.
3269  * rcu_read_lock must be held on entry.
3270  */
3271 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3272                        struct rps_dev_flow **rflowp)
3273 {
3274         const struct rps_sock_flow_table *sock_flow_table;
3275         struct netdev_rx_queue *rxqueue = dev->_rx;
3276         struct rps_dev_flow_table *flow_table;
3277         struct rps_map *map;
3278         int cpu = -1;
3279         u32 tcpu;
3280         u32 hash;
3281
3282         if (skb_rx_queue_recorded(skb)) {
3283                 u16 index = skb_get_rx_queue(skb);
3284
3285                 if (unlikely(index >= dev->real_num_rx_queues)) {
3286                         WARN_ONCE(dev->real_num_rx_queues > 1,
3287                                   "%s received packet on queue %u, but number "
3288                                   "of RX queues is %u\n",
3289                                   dev->name, index, dev->real_num_rx_queues);
3290                         goto done;
3291                 }
3292                 rxqueue += index;
3293         }
3294
3295         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3296
3297         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3298         map = rcu_dereference(rxqueue->rps_map);
3299         if (!flow_table && !map)
3300                 goto done;
3301
3302         skb_reset_network_header(skb);
3303         hash = skb_get_hash(skb);
3304         if (!hash)
3305                 goto done;
3306
3307         sock_flow_table = rcu_dereference(rps_sock_flow_table);
3308         if (flow_table && sock_flow_table) {
3309                 struct rps_dev_flow *rflow;
3310                 u32 next_cpu;
3311                 u32 ident;
3312
3313                 /* First check into global flow table if there is a match */
3314                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3315                 if ((ident ^ hash) & ~rps_cpu_mask)
3316                         goto try_rps;
3317
3318                 next_cpu = ident & rps_cpu_mask;
3319
3320                 /* OK, now we know there is a match,
3321                  * we can look at the local (per receive queue) flow table
3322                  */
3323                 rflow = &flow_table->flows[hash & flow_table->mask];
3324                 tcpu = rflow->cpu;
3325
3326                 /*
3327                  * If the desired CPU (where last recvmsg was done) is
3328                  * different from current CPU (one in the rx-queue flow
3329                  * table entry), switch if one of the following holds:
3330                  *   - Current CPU is unset (>= nr_cpu_ids).
3331                  *   - Current CPU is offline.
3332                  *   - The current CPU's queue tail has advanced beyond the
3333                  *     last packet that was enqueued using this table entry.
3334                  *     This guarantees that all previous packets for the flow
3335                  *     have been dequeued, thus preserving in order delivery.
3336                  */
3337                 if (unlikely(tcpu != next_cpu) &&
3338                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3339                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3340                       rflow->last_qtail)) >= 0)) {
3341                         tcpu = next_cpu;
3342                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3343                 }
3344
3345                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3346                         *rflowp = rflow;
3347                         cpu = tcpu;
3348                         goto done;
3349                 }
3350         }
3351
3352 try_rps:
3353
3354         if (map) {
3355                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3356                 if (cpu_online(tcpu)) {
3357                         cpu = tcpu;
3358                         goto done;
3359                 }
3360         }
3361
3362 done:
3363         return cpu;
3364 }
3365
3366 #ifdef CONFIG_RFS_ACCEL
3367
3368 /**
3369  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3370  * @dev: Device on which the filter was set
3371  * @rxq_index: RX queue index
3372  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3373  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3374  *
3375  * Drivers that implement ndo_rx_flow_steer() should periodically call
3376  * this function for each installed filter and remove the filters for
3377  * which it returns %true.
3378  */
3379 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3380                          u32 flow_id, u16 filter_id)
3381 {
3382         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3383         struct rps_dev_flow_table *flow_table;
3384         struct rps_dev_flow *rflow;
3385         bool expire = true;
3386         unsigned int cpu;
3387
3388         rcu_read_lock();
3389         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3390         if (flow_table && flow_id <= flow_table->mask) {
3391                 rflow = &flow_table->flows[flow_id];
3392                 cpu = ACCESS_ONCE(rflow->cpu);
3393                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3394                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3395                            rflow->last_qtail) <
3396                      (int)(10 * flow_table->mask)))
3397                         expire = false;
3398         }
3399         rcu_read_unlock();
3400         return expire;
3401 }
3402 EXPORT_SYMBOL(rps_may_expire_flow);
3403
3404 #endif /* CONFIG_RFS_ACCEL */
3405
3406 /* Called from hardirq (IPI) context */
3407 static void rps_trigger_softirq(void *data)
3408 {
3409         struct softnet_data *sd = data;
3410
3411         ____napi_schedule(sd, &sd->backlog);
3412         sd->received_rps++;
3413 }
3414
3415 #endif /* CONFIG_RPS */
3416
3417 /*
3418  * Check if this softnet_data structure is another cpu one
3419  * If yes, queue it to our IPI list and return 1
3420  * If no, return 0
3421  */
3422 static int rps_ipi_queued(struct softnet_data *sd)
3423 {
3424 #ifdef CONFIG_RPS
3425         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3426
3427         if (sd != mysd) {
3428                 sd->rps_ipi_next = mysd->rps_ipi_list;
3429                 mysd->rps_ipi_list = sd;
3430
3431                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3432                 return 1;
3433         }
3434 #endif /* CONFIG_RPS */
3435         return 0;
3436 }
3437
3438 #ifdef CONFIG_NET_FLOW_LIMIT
3439 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3440 #endif
3441
3442 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3443 {
3444 #ifdef CONFIG_NET_FLOW_LIMIT
3445         struct sd_flow_limit *fl;
3446         struct softnet_data *sd;
3447         unsigned int old_flow, new_flow;
3448
3449         if (qlen < (netdev_max_backlog >> 1))
3450                 return false;
3451
3452         sd = this_cpu_ptr(&softnet_data);
3453
3454         rcu_read_lock();
3455         fl = rcu_dereference(sd->flow_limit);
3456         if (fl) {
3457                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3458                 old_flow = fl->history[fl->history_head];
3459                 fl->history[fl->history_head] = new_flow;
3460
3461                 fl->history_head++;
3462                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3463
3464                 if (likely(fl->buckets[old_flow]))
3465                         fl->buckets[old_flow]--;
3466
3467                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3468                         fl->count++;
3469                         rcu_read_unlock();
3470                         return true;
3471                 }
3472         }
3473         rcu_read_unlock();
3474 #endif
3475         return false;
3476 }
3477
3478 /*
3479  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3480  * queue (may be a remote CPU queue).
3481  */
3482 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3483                               unsigned int *qtail)
3484 {
3485         struct softnet_data *sd;
3486         unsigned long flags;
3487         unsigned int qlen;
3488
3489         sd = &per_cpu(softnet_data, cpu);
3490
3491         local_irq_save(flags);
3492
3493         rps_lock(sd);
3494         if (!netif_running(skb->dev))
3495                 goto drop;
3496         qlen = skb_queue_len(&sd->input_pkt_queue);
3497         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3498                 if (qlen) {
3499 enqueue:
3500                         __skb_queue_tail(&sd->input_pkt_queue, skb);
3501                         input_queue_tail_incr_save(sd, qtail);
3502                         rps_unlock(sd);
3503                         local_irq_restore(flags);
3504                         return NET_RX_SUCCESS;
3505                 }
3506
3507                 /* Schedule NAPI for backlog device
3508                  * We can use non atomic operation since we own the queue lock
3509                  */
3510                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3511                         if (!rps_ipi_queued(sd))
3512                                 ____napi_schedule(sd, &sd->backlog);
3513                 }
3514                 goto enqueue;
3515         }
3516
3517 drop:
3518         sd->dropped++;
3519         rps_unlock(sd);
3520
3521         local_irq_restore(flags);
3522
3523         atomic_long_inc(&skb->dev->rx_dropped);
3524         kfree_skb(skb);
3525         return NET_RX_DROP;
3526 }
3527
3528 static int netif_rx_internal(struct sk_buff *skb)
3529 {
3530         int ret;
3531
3532         net_timestamp_check(netdev_tstamp_prequeue, skb);
3533
3534         trace_netif_rx(skb);
3535 #ifdef CONFIG_RPS
3536         if (static_key_false(&rps_needed)) {
3537                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3538                 int cpu;
3539
3540                 preempt_disable();
3541                 rcu_read_lock();
3542
3543                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3544                 if (cpu < 0)
3545                         cpu = smp_processor_id();
3546
3547                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3548
3549                 rcu_read_unlock();
3550                 preempt_enable();
3551         } else
3552 #endif
3553         {
3554                 unsigned int qtail;
3555                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3556                 put_cpu();
3557         }
3558         return ret;
3559 }
3560
3561 /**
3562  *      netif_rx        -       post buffer to the network code
3563  *      @skb: buffer to post
3564  *
3565  *      This function receives a packet from a device driver and queues it for
3566  *      the upper (protocol) levels to process.  It always succeeds. The buffer
3567  *      may be dropped during processing for congestion control or by the
3568  *      protocol layers.
3569  *
3570  *      return values:
3571  *      NET_RX_SUCCESS  (no congestion)
3572  *      NET_RX_DROP     (packet was dropped)
3573  *
3574  */
3575
3576 int netif_rx(struct sk_buff *skb)
3577 {
3578         trace_netif_rx_entry(skb);
3579
3580         return netif_rx_internal(skb);
3581 }
3582 EXPORT_SYMBOL(netif_rx);
3583
3584 int netif_rx_ni(struct sk_buff *skb)
3585 {
3586         int err;
3587
3588         trace_netif_rx_ni_entry(skb);
3589
3590         preempt_disable();
3591         err = netif_rx_internal(skb);
3592         if (local_softirq_pending())
3593                 do_softirq();
3594         preempt_enable();
3595
3596         return err;
3597 }
3598 EXPORT_SYMBOL(netif_rx_ni);
3599
3600 static void net_tx_action(struct softirq_action *h)
3601 {
3602         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3603
3604         if (sd->completion_queue) {
3605                 struct sk_buff *clist;
3606
3607                 local_irq_disable();
3608                 clist = sd->completion_queue;
3609                 sd->completion_queue = NULL;
3610                 local_irq_enable();
3611
3612                 while (clist) {
3613                         struct sk_buff *skb = clist;
3614                         clist = clist->next;
3615
3616                         WARN_ON(atomic_read(&skb->users));
3617                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3618                                 trace_consume_skb(skb);
3619                         else
3620                                 trace_kfree_skb(skb, net_tx_action);
3621                         __kfree_skb(skb);
3622                 }
3623         }
3624
3625         if (sd->output_queue) {
3626                 struct Qdisc *head;
3627
3628                 local_irq_disable();
3629                 head = sd->output_queue;
3630                 sd->output_queue = NULL;
3631                 sd->output_queue_tailp = &sd->output_queue;
3632                 local_irq_enable();
3633
3634                 while (head) {
3635                         struct Qdisc *q = head;
3636                         spinlock_t *root_lock;
3637
3638                         head = head->next_sched;
3639
3640                         root_lock = qdisc_lock(q);
3641                         if (spin_trylock(root_lock)) {
3642                                 smp_mb__before_atomic();
3643                                 clear_bit(__QDISC_STATE_SCHED,
3644                                           &q->state);
3645                                 qdisc_run(q);
3646                                 spin_unlock(root_lock);
3647                         } else {
3648                                 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3649                                               &q->state)) {
3650                                         __netif_reschedule(q);
3651                                 } else {
3652                                         smp_mb__before_atomic();
3653                                         clear_bit(__QDISC_STATE_SCHED,
3654                                                   &q->state);
3655                                 }
3656                         }
3657                 }
3658         }
3659 }
3660
3661 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3662     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3663 /* This hook is defined here for ATM LANE */
3664 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3665                              unsigned char *addr) __read_mostly;
3666 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3667 #endif
3668
3669 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
3670                                          struct packet_type **pt_prev,
3671                                          int *ret, struct net_device *orig_dev)
3672 {
3673 #ifdef CONFIG_NET_CLS_ACT
3674         struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3675         struct tcf_result cl_res;
3676
3677         /* If there's at least one ingress present somewhere (so
3678          * we get here via enabled static key), remaining devices
3679          * that are not configured with an ingress qdisc will bail
3680          * out here.
3681          */
3682         if (!cl)
3683                 return skb;
3684         if (*pt_prev) {
3685                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3686                 *pt_prev = NULL;
3687         }
3688
3689         qdisc_skb_cb(skb)->pkt_len = skb->len;
3690         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3691         qdisc_bstats_cpu_update(cl->q, skb);
3692
3693         switch (tc_classify(skb, cl, &cl_res, false)) {
3694         case TC_ACT_OK:
3695         case TC_ACT_RECLASSIFY:
3696                 skb->tc_index = TC_H_MIN(cl_res.classid);
3697                 break;
3698         case TC_ACT_SHOT:
3699                 qdisc_qstats_cpu_drop(cl->q);
3700         case TC_ACT_STOLEN:
3701         case TC_ACT_QUEUED:
3702                 kfree_skb(skb);
3703                 return NULL;
3704         case TC_ACT_REDIRECT:
3705                 /* skb_mac_header check was done by cls/act_bpf, so
3706                  * we can safely push the L2 header back before
3707                  * redirecting to another netdev
3708                  */
3709                 __skb_push(skb, skb->mac_len);
3710                 skb_do_redirect(skb);
3711                 return NULL;
3712         default:
3713                 break;
3714         }
3715 #endif /* CONFIG_NET_CLS_ACT */
3716         return skb;
3717 }
3718
3719 /**
3720  *      netdev_rx_handler_register - register receive handler
3721  *      @dev: device to register a handler for
3722  *      @rx_handler: receive handler to register
3723  *      @rx_handler_data: data pointer that is used by rx handler
3724  *
3725  *      Register a receive handler for a device. This handler will then be
3726  *      called from __netif_receive_skb. A negative errno code is returned
3727  *      on a failure.
3728  *
3729  *      The caller must hold the rtnl_mutex.
3730  *
3731  *      For a general description of rx_handler, see enum rx_handler_result.
3732  */
3733 int netdev_rx_handler_register(struct net_device *dev,
3734                                rx_handler_func_t *rx_handler,
3735                                void *rx_handler_data)
3736 {
3737         ASSERT_RTNL();
3738
3739         if (dev->rx_handler)
3740                 return -EBUSY;
3741
3742         /* Note: rx_handler_data must be set before rx_handler */
3743         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3744         rcu_assign_pointer(dev->rx_handler, rx_handler);
3745
3746         return 0;
3747 }
3748 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3749
3750 /**
3751  *      netdev_rx_handler_unregister - unregister receive handler
3752  *      @dev: device to unregister a handler from
3753  *
3754  *      Unregister a receive handler from a device.
3755  *
3756  *      The caller must hold the rtnl_mutex.
3757  */
3758 void netdev_rx_handler_unregister(struct net_device *dev)
3759 {
3760
3761         ASSERT_RTNL();
3762         RCU_INIT_POINTER(dev->rx_handler, NULL);
3763         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3764          * section has a guarantee to see a non NULL rx_handler_data
3765          * as well.
3766          */
3767         synchronize_net();
3768         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3769 }
3770 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3771
3772 /*
3773  * Limit the use of PFMEMALLOC reserves to those protocols that implement
3774  * the special handling of PFMEMALLOC skbs.
3775  */
3776 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3777 {
3778         switch (skb->protocol) {
3779         case htons(ETH_P_ARP):
3780         case htons(ETH_P_IP):
3781         case htons(ETH_P_IPV6):
3782         case htons(ETH_P_8021Q):
3783         case htons(ETH_P_8021AD):
3784                 return true;
3785         default:
3786                 return false;
3787         }
3788 }
3789
3790 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
3791                              int *ret, struct net_device *orig_dev)
3792 {
3793 #ifdef CONFIG_NETFILTER_INGRESS
3794         if (nf_hook_ingress_active(skb)) {
3795                 if (*pt_prev) {
3796                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
3797                         *pt_prev = NULL;
3798                 }
3799
3800                 return nf_hook_ingress(skb);
3801         }
3802 #endif /* CONFIG_NETFILTER_INGRESS */
3803         return 0;
3804 }
3805
3806 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
3807 {
3808         struct packet_type *ptype, *pt_prev;
3809         rx_handler_func_t *rx_handler;
3810         struct net_device *orig_dev;
3811         bool deliver_exact = false;
3812         int ret = NET_RX_DROP;
3813         __be16 type;
3814
3815         net_timestamp_check(!netdev_tstamp_prequeue, skb);
3816
3817         trace_netif_receive_skb(skb);
3818
3819         orig_dev = skb->dev;
3820
3821         skb_reset_network_header(skb);
3822         if (!skb_transport_header_was_set(skb))
3823                 skb_reset_transport_header(skb);
3824         skb_reset_mac_len(skb);
3825
3826         pt_prev = NULL;
3827
3828 another_round:
3829         skb->skb_iif = skb->dev->ifindex;
3830
3831         __this_cpu_inc(softnet_data.processed);
3832
3833         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
3834             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
3835                 skb = skb_vlan_untag(skb);
3836                 if (unlikely(!skb))
3837                         goto out;
3838         }
3839
3840 #ifdef CONFIG_NET_CLS_ACT
3841         if (skb->tc_verd & TC_NCLS) {
3842                 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3843                 goto ncls;
3844         }
3845 #endif
3846
3847         if (pfmemalloc)
3848                 goto skip_taps;
3849
3850         list_for_each_entry_rcu(ptype, &ptype_all, list) {
3851                 if (pt_prev)
3852                         ret = deliver_skb(skb, pt_prev, orig_dev);
3853                 pt_prev = ptype;
3854         }
3855
3856         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
3857                 if (pt_prev)
3858                         ret = deliver_skb(skb, pt_prev, orig_dev);
3859                 pt_prev = ptype;
3860         }
3861
3862 skip_taps:
3863 #ifdef CONFIG_NET_INGRESS
3864         if (static_key_false(&ingress_needed)) {
3865                 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3866                 if (!skb)
3867                         goto out;
3868
3869                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
3870                         goto out;
3871         }
3872 #endif
3873 #ifdef CONFIG_NET_CLS_ACT
3874         skb->tc_verd = 0;
3875 ncls:
3876 #endif
3877         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
3878                 goto drop;
3879
3880         if (skb_vlan_tag_present(skb)) {
3881                 if (pt_prev) {
3882                         ret = deliver_skb(skb, pt_prev, orig_dev);
3883                         pt_prev = NULL;
3884                 }
3885                 if (vlan_do_receive(&skb))
3886                         goto another_round;
3887                 else if (unlikely(!skb))
3888                         goto out;
3889         }
3890
3891         rx_handler = rcu_dereference(skb->dev->rx_handler);
3892         if (rx_handler) {
3893                 if (pt_prev) {
3894                         ret = deliver_skb(skb, pt_prev, orig_dev);
3895                         pt_prev = NULL;
3896                 }
3897                 switch (rx_handler(&skb)) {
3898                 case RX_HANDLER_CONSUMED:
3899                         ret = NET_RX_SUCCESS;
3900                         goto out;
3901                 case RX_HANDLER_ANOTHER:
3902                         goto another_round;
3903                 case RX_HANDLER_EXACT:
3904                         deliver_exact = true;
3905                 case RX_HANDLER_PASS:
3906                         break;
3907                 default:
3908                         BUG();
3909                 }
3910         }
3911
3912         if (unlikely(skb_vlan_tag_present(skb))) {
3913                 if (skb_vlan_tag_get_id(skb))
3914                         skb->pkt_type = PACKET_OTHERHOST;
3915                 /* Note: we might in the future use prio bits
3916                  * and set skb->priority like in vlan_do_receive()
3917                  * For the time being, just ignore Priority Code Point
3918                  */
3919                 skb->vlan_tci = 0;
3920         }
3921
3922         type = skb->protocol;
3923
3924         /* deliver only exact match when indicated */
3925         if (likely(!deliver_exact)) {
3926                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3927                                        &ptype_base[ntohs(type) &
3928                                                    PTYPE_HASH_MASK]);
3929         }
3930
3931         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3932                                &orig_dev->ptype_specific);
3933
3934         if (unlikely(skb->dev != orig_dev)) {
3935                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
3936                                        &skb->dev->ptype_specific);
3937         }
3938
3939         if (pt_prev) {
3940                 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
3941                         goto drop;
3942                 else
3943                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3944         } else {
3945 drop:
3946                 atomic_long_inc(&skb->dev->rx_dropped);
3947                 kfree_skb(skb);
3948                 /* Jamal, now you will not able to escape explaining
3949                  * me how you were going to use this. :-)
3950                  */
3951                 ret = NET_RX_DROP;
3952         }
3953
3954 out:
3955         return ret;
3956 }
3957
3958 static int __netif_receive_skb(struct sk_buff *skb)
3959 {
3960         int ret;
3961
3962         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
3963                 unsigned long pflags = current->flags;
3964
3965                 /*
3966                  * PFMEMALLOC skbs are special, they should
3967                  * - be delivered to SOCK_MEMALLOC sockets only
3968                  * - stay away from userspace
3969                  * - have bounded memory usage
3970                  *
3971                  * Use PF_MEMALLOC as this saves us from propagating the allocation
3972                  * context down to all allocation sites.
3973                  */
3974                 current->flags |= PF_MEMALLOC;
3975                 ret = __netif_receive_skb_core(skb, true);
3976                 tsk_restore_flags(current, pflags, PF_MEMALLOC);
3977         } else
3978                 ret = __netif_receive_skb_core(skb, false);
3979
3980         return ret;
3981 }
3982
3983 static int netif_receive_skb_internal(struct sk_buff *skb)
3984 {
3985         int ret;
3986
3987         net_timestamp_check(netdev_tstamp_prequeue, skb);
3988
3989         if (skb_defer_rx_timestamp(skb))
3990                 return NET_RX_SUCCESS;
3991
3992         rcu_read_lock();
3993
3994 #ifdef CONFIG_RPS
3995         if (static_key_false(&rps_needed)) {
3996                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3997                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
3998
3999                 if (cpu >= 0) {
4000                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4001                         rcu_read_unlock();
4002                         return ret;
4003                 }
4004         }
4005 #endif
4006         ret = __netif_receive_skb(skb);
4007         rcu_read_unlock();
4008         return ret;
4009 }
4010
4011 /**
4012  *      netif_receive_skb - process receive buffer from network
4013  *      @skb: buffer to process
4014  *
4015  *      netif_receive_skb() is the main receive data processing function.
4016  *      It always succeeds. The buffer may be dropped during processing
4017  *      for congestion control or by the protocol layers.
4018  *
4019  *      This function may only be called from softirq context and interrupts
4020  *      should be enabled.
4021  *
4022  *      Return values (usually ignored):
4023  *      NET_RX_SUCCESS: no congestion
4024  *      NET_RX_DROP: packet was dropped
4025  */
4026 int netif_receive_skb(struct sk_buff *skb)
4027 {
4028         trace_netif_receive_skb_entry(skb);
4029
4030         return netif_receive_skb_internal(skb);
4031 }
4032 EXPORT_SYMBOL(netif_receive_skb);
4033
4034 /* Network device is going away, flush any packets still pending
4035  * Called with irqs disabled.
4036  */
4037 static void flush_backlog(void *arg)
4038 {
4039         struct net_device *dev = arg;
4040         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4041         struct sk_buff *skb, *tmp;
4042
4043         rps_lock(sd);
4044         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4045                 if (skb->dev == dev) {
4046                         __skb_unlink(skb, &sd->input_pkt_queue);
4047                         kfree_skb(skb);
4048                         input_queue_head_incr(sd);
4049                 }
4050         }
4051         rps_unlock(sd);
4052
4053         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4054                 if (skb->dev == dev) {
4055                         __skb_unlink(skb, &sd->process_queue);
4056                         kfree_skb(skb);
4057                         input_queue_head_incr(sd);
4058                 }
4059         }
4060 }
4061
4062 static int napi_gro_complete(struct sk_buff *skb)
4063 {
4064         struct packet_offload *ptype;
4065         __be16 type = skb->protocol;
4066         struct list_head *head = &offload_base;
4067         int err = -ENOENT;
4068
4069         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4070
4071         if (NAPI_GRO_CB(skb)->count == 1) {
4072                 skb_shinfo(skb)->gso_size = 0;
4073                 goto out;
4074         }
4075
4076         rcu_read_lock();
4077         list_for_each_entry_rcu(ptype, head, list) {
4078                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4079                         continue;
4080
4081                 err = ptype->callbacks.gro_complete(skb, 0);
4082                 break;
4083         }
4084         rcu_read_unlock();
4085
4086         if (err) {
4087                 WARN_ON(&ptype->list == head);
4088                 kfree_skb(skb);
4089                 return NET_RX_SUCCESS;
4090         }
4091
4092 out:
4093         return netif_receive_skb_internal(skb);
4094 }
4095
4096 /* napi->gro_list contains packets ordered by age.
4097  * youngest packets at the head of it.
4098  * Complete skbs in reverse order to reduce latencies.
4099  */
4100 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4101 {
4102         struct sk_buff *skb, *prev = NULL;
4103
4104         /* scan list and build reverse chain */
4105         for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4106                 skb->prev = prev;
4107                 prev = skb;
4108         }
4109
4110         for (skb = prev; skb; skb = prev) {
4111                 skb->next = NULL;
4112
4113                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4114                         return;
4115
4116                 prev = skb->prev;
4117                 napi_gro_complete(skb);
4118                 napi->gro_count--;
4119         }
4120
4121         napi->gro_list = NULL;
4122 }
4123 EXPORT_SYMBOL(napi_gro_flush);
4124
4125 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4126 {
4127         struct sk_buff *p;
4128         unsigned int maclen = skb->dev->hard_header_len;
4129         u32 hash = skb_get_hash_raw(skb);
4130
4131         for (p = napi->gro_list; p; p = p->next) {
4132                 unsigned long diffs;
4133
4134                 NAPI_GRO_CB(p)->flush = 0;
4135
4136                 if (hash != skb_get_hash_raw(p)) {
4137                         NAPI_GRO_CB(p)->same_flow = 0;
4138                         continue;
4139                 }
4140
4141                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4142                 diffs |= p->vlan_tci ^ skb->vlan_tci;
4143                 if (maclen == ETH_HLEN)
4144                         diffs |= compare_ether_header(skb_mac_header(p),
4145                                                       skb_mac_header(skb));
4146                 else if (!diffs)
4147                         diffs = memcmp(skb_mac_header(p),
4148                                        skb_mac_header(skb),
4149                                        maclen);
4150                 NAPI_GRO_CB(p)->same_flow = !diffs;
4151         }
4152 }
4153
4154 static void skb_gro_reset_offset(struct sk_buff *skb)
4155 {
4156         const struct skb_shared_info *pinfo = skb_shinfo(skb);
4157         const skb_frag_t *frag0 = &pinfo->frags[0];
4158
4159         NAPI_GRO_CB(skb)->data_offset = 0;
4160         NAPI_GRO_CB(skb)->frag0 = NULL;
4161         NAPI_GRO_CB(skb)->frag0_len = 0;
4162
4163         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4164             pinfo->nr_frags &&
4165             !PageHighMem(skb_frag_page(frag0))) {
4166                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4167                 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
4168         }
4169 }
4170
4171 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4172 {
4173         struct skb_shared_info *pinfo = skb_shinfo(skb);
4174
4175         BUG_ON(skb->end - skb->tail < grow);
4176
4177         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4178
4179         skb->data_len -= grow;
4180         skb->tail += grow;
4181
4182         pinfo->frags[0].page_offset += grow;
4183         skb_frag_size_sub(&pinfo->frags[0], grow);
4184
4185         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4186                 skb_frag_unref(skb, 0);
4187                 memmove(pinfo->frags, pinfo->frags + 1,
4188                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4189         }
4190 }
4191
4192 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4193 {
4194         struct sk_buff **pp = NULL;
4195         struct packet_offload *ptype;
4196         __be16 type = skb->protocol;
4197         struct list_head *head = &offload_base;
4198         int same_flow;
4199         enum gro_result ret;
4200         int grow;
4201
4202         if (!(skb->dev->features & NETIF_F_GRO))
4203                 goto normal;
4204
4205         if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4206                 goto normal;
4207
4208         gro_list_prepare(napi, skb);
4209
4210         rcu_read_lock();
4211         list_for_each_entry_rcu(ptype, head, list) {
4212                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4213                         continue;
4214
4215                 skb_set_network_header(skb, skb_gro_offset(skb));
4216                 skb_reset_mac_len(skb);
4217                 NAPI_GRO_CB(skb)->same_flow = 0;
4218                 NAPI_GRO_CB(skb)->flush = 0;
4219                 NAPI_GRO_CB(skb)->free = 0;
4220                 NAPI_GRO_CB(skb)->udp_mark = 0;
4221                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4222
4223                 /* Setup for GRO checksum validation */
4224                 switch (skb->ip_summed) {
4225                 case CHECKSUM_COMPLETE:
4226                         NAPI_GRO_CB(skb)->csum = skb->csum;
4227                         NAPI_GRO_CB(skb)->csum_valid = 1;
4228                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4229                         break;
4230                 case CHECKSUM_UNNECESSARY:
4231                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4232                         NAPI_GRO_CB(skb)->csum_valid = 0;
4233                         break;
4234                 default:
4235                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4236                         NAPI_GRO_CB(skb)->csum_valid = 0;
4237                 }
4238
4239                 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4240                 break;
4241         }
4242         rcu_read_unlock();
4243
4244         if (&ptype->list == head)
4245                 goto normal;
4246
4247         same_flow = NAPI_GRO_CB(skb)->same_flow;
4248         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4249
4250         if (pp) {
4251                 struct sk_buff *nskb = *pp;
4252
4253                 *pp = nskb->next;
4254                 nskb->next = NULL;
4255                 napi_gro_complete(nskb);
4256                 napi->gro_count--;
4257         }
4258
4259         if (same_flow)
4260                 goto ok;
4261
4262         if (NAPI_GRO_CB(skb)->flush)
4263                 goto normal;
4264
4265         if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4266                 struct sk_buff *nskb = napi->gro_list;
4267
4268                 /* locate the end of the list to select the 'oldest' flow */
4269                 while (nskb->next) {
4270                         pp = &nskb->next;
4271                         nskb = *pp;
4272                 }
4273                 *pp = NULL;
4274                 nskb->next = NULL;
4275                 napi_gro_complete(nskb);
4276         } else {
4277                 napi->gro_count++;
4278         }
4279         NAPI_GRO_CB(skb)->count = 1;
4280         NAPI_GRO_CB(skb)->age = jiffies;
4281         NAPI_GRO_CB(skb)->last = skb;
4282         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4283         skb->next = napi->gro_list;
4284         napi->gro_list = skb;
4285         ret = GRO_HELD;
4286
4287 pull:
4288         grow = skb_gro_offset(skb) - skb_headlen(skb);
4289         if (grow > 0)
4290                 gro_pull_from_frag0(skb, grow);
4291 ok:
4292         return ret;
4293
4294 normal:
4295         ret = GRO_NORMAL;
4296         goto pull;
4297 }
4298
4299 struct packet_offload *gro_find_receive_by_type(__be16 type)
4300 {
4301         struct list_head *offload_head = &offload_base;
4302         struct packet_offload *ptype;
4303
4304         list_for_each_entry_rcu(ptype, offload_head, list) {
4305                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4306                         continue;
4307                 return ptype;
4308         }
4309         return NULL;
4310 }
4311 EXPORT_SYMBOL(gro_find_receive_by_type);
4312
4313 struct packet_offload *gro_find_complete_by_type(__be16 type)
4314 {
4315         struct list_head *offload_head = &offload_base;
4316         struct packet_offload *ptype;
4317
4318         list_for_each_entry_rcu(ptype, offload_head, list) {
4319                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4320                         continue;
4321                 return ptype;
4322         }
4323         return NULL;
4324 }
4325 EXPORT_SYMBOL(gro_find_complete_by_type);
4326
4327 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4328 {
4329         switch (ret) {
4330         case GRO_NORMAL:
4331                 if (netif_receive_skb_internal(skb))
4332                         ret = GRO_DROP;
4333                 break;
4334
4335         case GRO_DROP:
4336                 kfree_skb(skb);
4337                 break;
4338
4339         case GRO_MERGED_FREE:
4340                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
4341                         kmem_cache_free(skbuff_head_cache, skb);
4342                 else
4343                         __kfree_skb(skb);
4344                 break;
4345
4346         case GRO_HELD:
4347         case GRO_MERGED:
4348                 break;
4349         }
4350
4351         return ret;
4352 }
4353
4354 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4355 {
4356         trace_napi_gro_receive_entry(skb);
4357
4358         skb_gro_reset_offset(skb);
4359
4360         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4361 }
4362 EXPORT_SYMBOL(napi_gro_receive);
4363
4364 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4365 {
4366         if (unlikely(skb->pfmemalloc)) {
4367                 consume_skb(skb);
4368                 return;
4369         }
4370         __skb_pull(skb, skb_headlen(skb));
4371         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4372         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4373         skb->vlan_tci = 0;
4374         skb->dev = napi->dev;
4375         skb->skb_iif = 0;
4376         skb->encapsulation = 0;
4377         skb_shinfo(skb)->gso_type = 0;
4378         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4379
4380         napi->skb = skb;
4381 }
4382
4383 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4384 {
4385         struct sk_buff *skb = napi->skb;
4386
4387         if (!skb) {
4388                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4389                 napi->skb = skb;
4390         }
4391         return skb;
4392 }
4393 EXPORT_SYMBOL(napi_get_frags);
4394
4395 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4396                                       struct sk_buff *skb,
4397                                       gro_result_t ret)
4398 {
4399         switch (ret) {
4400         case GRO_NORMAL:
4401         case GRO_HELD:
4402                 __skb_push(skb, ETH_HLEN);
4403                 skb->protocol = eth_type_trans(skb, skb->dev);
4404                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4405                         ret = GRO_DROP;
4406                 break;
4407
4408         case GRO_DROP:
4409         case GRO_MERGED_FREE:
4410                 napi_reuse_skb(napi, skb);
4411                 break;
4412
4413         case GRO_MERGED:
4414                 break;
4415         }
4416
4417         return ret;
4418 }
4419
4420 /* Upper GRO stack assumes network header starts at gro_offset=0
4421  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4422  * We copy ethernet header into skb->data to have a common layout.
4423  */
4424 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4425 {
4426         struct sk_buff *skb = napi->skb;
4427         const struct ethhdr *eth;
4428         unsigned int hlen = sizeof(*eth);
4429
4430         napi->skb = NULL;
4431
4432         skb_reset_mac_header(skb);
4433         skb_gro_reset_offset(skb);
4434
4435         eth = skb_gro_header_fast(skb, 0);
4436         if (unlikely(skb_gro_header_hard(skb, hlen))) {
4437                 eth = skb_gro_header_slow(skb, hlen, 0);
4438                 if (unlikely(!eth)) {
4439                         napi_reuse_skb(napi, skb);
4440                         return NULL;
4441                 }
4442         } else {
4443                 gro_pull_from_frag0(skb, hlen);
4444                 NAPI_GRO_CB(skb)->frag0 += hlen;
4445                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4446         }
4447         __skb_pull(skb, hlen);
4448
4449         /*
4450          * This works because the only protocols we care about don't require
4451          * special handling.
4452          * We'll fix it up properly in napi_frags_finish()
4453          */
4454         skb->protocol = eth->h_proto;
4455
4456         return skb;
4457 }
4458
4459 gro_result_t napi_gro_frags(struct napi_struct *napi)
4460 {
4461         struct sk_buff *skb = napi_frags_skb(napi);
4462
4463         if (!skb)
4464                 return GRO_DROP;
4465
4466         trace_napi_gro_frags_entry(skb);
4467
4468         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4469 }
4470 EXPORT_SYMBOL(napi_gro_frags);
4471
4472 /* Compute the checksum from gro_offset and return the folded value
4473  * after adding in any pseudo checksum.
4474  */
4475 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4476 {
4477         __wsum wsum;
4478         __sum16 sum;
4479
4480         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4481
4482         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4483         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4484         if (likely(!sum)) {
4485                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4486                     !skb->csum_complete_sw)
4487                         netdev_rx_csum_fault(skb->dev);
4488         }
4489
4490         NAPI_GRO_CB(skb)->csum = wsum;
4491         NAPI_GRO_CB(skb)->csum_valid = 1;
4492
4493         return sum;
4494 }
4495 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4496
4497 /*
4498  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4499  * Note: called with local irq disabled, but exits with local irq enabled.
4500  */
4501 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4502 {
4503 #ifdef CONFIG_RPS
4504         struct softnet_data *remsd = sd->rps_ipi_list;
4505
4506         if (remsd) {
4507                 sd->rps_ipi_list = NULL;
4508
4509                 local_irq_enable();
4510
4511                 /* Send pending IPI's to kick RPS processing on remote cpus. */
4512                 while (remsd) {
4513                         struct softnet_data *next = remsd->rps_ipi_next;
4514
4515                         if (cpu_online(remsd->cpu))
4516                                 smp_call_function_single_async(remsd->cpu,
4517                                                            &remsd->csd);
4518                         remsd = next;
4519                 }
4520         } else
4521 #endif
4522                 local_irq_enable();
4523 }
4524
4525 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4526 {
4527 #ifdef CONFIG_RPS
4528         return sd->rps_ipi_list != NULL;
4529 #else
4530         return false;
4531 #endif
4532 }
4533
4534 static int process_backlog(struct napi_struct *napi, int quota)
4535 {
4536         int work = 0;
4537         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4538
4539         /* Check if we have pending ipi, its better to send them now,
4540          * not waiting net_rx_action() end.
4541          */
4542         if (sd_has_rps_ipi_waiting(sd)) {
4543                 local_irq_disable();
4544                 net_rps_action_and_irq_enable(sd);
4545         }
4546
4547         napi->weight = weight_p;
4548         local_irq_disable();
4549         while (1) {
4550                 struct sk_buff *skb;
4551
4552                 while ((skb = __skb_dequeue(&sd->process_queue))) {
4553                         rcu_read_lock();
4554                         local_irq_enable();
4555                         __netif_receive_skb(skb);
4556                         rcu_read_unlock();
4557                         local_irq_disable();
4558                         input_queue_head_incr(sd);
4559                         if (++work >= quota) {
4560                                 local_irq_enable();
4561                                 return work;
4562                         }
4563                 }
4564
4565                 rps_lock(sd);
4566                 if (skb_queue_empty(&sd->input_pkt_queue)) {
4567                         /*
4568                          * Inline a custom version of __napi_complete().
4569                          * only current cpu owns and manipulates this napi,
4570                          * and NAPI_STATE_SCHED is the only possible flag set
4571                          * on backlog.
4572                          * We can use a plain write instead of clear_bit(),
4573                          * and we dont need an smp_mb() memory barrier.
4574                          */
4575                         napi->state = 0;
4576                         rps_unlock(sd);
4577
4578                         break;
4579                 }
4580
4581                 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4582                                            &sd->process_queue);
4583                 rps_unlock(sd);
4584         }
4585         local_irq_enable();
4586
4587         return work;
4588 }
4589
4590 /**
4591  * __napi_schedule - schedule for receive
4592  * @n: entry to schedule
4593  *
4594  * The entry's receive function will be scheduled to run.
4595  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4596  */
4597 void __napi_schedule(struct napi_struct *n)
4598 {
4599         unsigned long flags;
4600
4601         local_irq_save(flags);
4602         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4603         local_irq_restore(flags);
4604 }
4605 EXPORT_SYMBOL(__napi_schedule);
4606
4607 /**
4608  * __napi_schedule_irqoff - schedule for receive
4609  * @n: entry to schedule
4610  *
4611  * Variant of __napi_schedule() assuming hard irqs are masked
4612  */
4613 void __napi_schedule_irqoff(struct napi_struct *n)
4614 {
4615         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4616 }
4617 EXPORT_SYMBOL(__napi_schedule_irqoff);
4618
4619 void __napi_complete(struct napi_struct *n)
4620 {
4621         BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4622
4623         list_del_init(&n->poll_list);
4624         smp_mb__before_atomic();
4625         clear_bit(NAPI_STATE_SCHED, &n->state);
4626 }
4627 EXPORT_SYMBOL(__napi_complete);
4628
4629 void napi_complete_done(struct napi_struct *n, int work_done)
4630 {
4631         unsigned long flags;
4632
4633         /*
4634          * don't let napi dequeue from the cpu poll list
4635          * just in case its running on a different cpu
4636          */
4637         if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4638                 return;
4639
4640         if (n->gro_list) {
4641                 unsigned long timeout = 0;
4642
4643                 if (work_done)
4644                         timeout = n->dev->gro_flush_timeout;
4645
4646                 if (timeout)
4647                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
4648                                       HRTIMER_MODE_REL_PINNED);
4649                 else
4650                         napi_gro_flush(n, false);
4651         }
4652         if (likely(list_empty(&n->poll_list))) {
4653                 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4654         } else {
4655                 /* If n->poll_list is not empty, we need to mask irqs */
4656                 local_irq_save(flags);
4657                 __napi_complete(n);
4658                 local_irq_restore(flags);
4659         }
4660 }
4661 EXPORT_SYMBOL(napi_complete_done);
4662
4663 /* must be called under rcu_read_lock(), as we dont take a reference */
4664 struct napi_struct *napi_by_id(unsigned int napi_id)
4665 {
4666         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4667         struct napi_struct *napi;
4668
4669         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4670                 if (napi->napi_id == napi_id)
4671                         return napi;
4672
4673         return NULL;
4674 }
4675 EXPORT_SYMBOL_GPL(napi_by_id);
4676
4677 void napi_hash_add(struct napi_struct *napi)
4678 {
4679         if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
4680
4681                 spin_lock(&napi_hash_lock);
4682
4683                 /* 0 is not a valid id, we also skip an id that is taken
4684                  * we expect both events to be extremely rare
4685                  */
4686                 napi->napi_id = 0;
4687                 while (!napi->napi_id) {
4688                         napi->napi_id = ++napi_gen_id;
4689                         if (napi_by_id(napi->napi_id))
4690                                 napi->napi_id = 0;
4691                 }
4692
4693                 hlist_add_head_rcu(&napi->napi_hash_node,
4694                         &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4695
4696                 spin_unlock(&napi_hash_lock);
4697         }
4698 }
4699 EXPORT_SYMBOL_GPL(napi_hash_add);
4700
4701 /* Warning : caller is responsible to make sure rcu grace period
4702  * is respected before freeing memory containing @napi
4703  */
4704 void napi_hash_del(struct napi_struct *napi)
4705 {
4706         spin_lock(&napi_hash_lock);
4707
4708         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
4709                 hlist_del_rcu(&napi->napi_hash_node);
4710
4711         spin_unlock(&napi_hash_lock);
4712 }
4713 EXPORT_SYMBOL_GPL(napi_hash_del);
4714
4715 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
4716 {
4717         struct napi_struct *napi;
4718
4719         napi = container_of(timer, struct napi_struct, timer);
4720         if (napi->gro_list)
4721                 napi_schedule(napi);
4722
4723         return HRTIMER_NORESTART;
4724 }
4725
4726 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
4727                     int (*poll)(struct napi_struct *, int), int weight)
4728 {
4729         INIT_LIST_HEAD(&napi->poll_list);
4730         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
4731         napi->timer.function = napi_watchdog;
4732         napi->gro_count = 0;
4733         napi->gro_list = NULL;
4734         napi->skb = NULL;
4735         napi->poll = poll;
4736         if (weight > NAPI_POLL_WEIGHT)
4737                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
4738                             weight, dev->name);
4739         napi->weight = weight;
4740         list_add(&napi->dev_list, &dev->napi_list);
4741         napi->dev = dev;
4742 #ifdef CONFIG_NETPOLL
4743         spin_lock_init(&napi->poll_lock);
4744         napi->poll_owner = -1;
4745 #endif
4746         set_bit(NAPI_STATE_SCHED, &napi->state);
4747 }
4748 EXPORT_SYMBOL(netif_napi_add);
4749
4750 void napi_disable(struct napi_struct *n)
4751 {
4752         might_sleep();
4753         set_bit(NAPI_STATE_DISABLE, &n->state);
4754
4755         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
4756                 msleep(1);
4757         while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
4758                 msleep(1);
4759
4760         hrtimer_cancel(&n->timer);
4761
4762         clear_bit(NAPI_STATE_DISABLE, &n->state);
4763 }
4764 EXPORT_SYMBOL(napi_disable);
4765
4766 void netif_napi_del(struct napi_struct *napi)
4767 {
4768         list_del_init(&napi->dev_list);
4769         napi_free_frags(napi);
4770
4771         kfree_skb_list(napi->gro_list);
4772         napi->gro_list = NULL;
4773         napi->gro_count = 0;
4774 }
4775 EXPORT_SYMBOL(netif_napi_del);
4776
4777 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
4778 {
4779         void *have;
4780         int work, weight;
4781
4782         list_del_init(&n->poll_list);
4783
4784         have = netpoll_poll_lock(n);
4785
4786         weight = n->weight;
4787
4788         /* This NAPI_STATE_SCHED test is for avoiding a race
4789          * with netpoll's poll_napi().  Only the entity which
4790          * obtains the lock and sees NAPI_STATE_SCHED set will
4791          * actually make the ->poll() call.  Therefore we avoid
4792          * accidentally calling ->poll() when NAPI is not scheduled.
4793          */
4794         work = 0;
4795         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4796                 work = n->poll(n, weight);
4797                 trace_napi_poll(n);
4798         }
4799
4800         WARN_ON_ONCE(work > weight);
4801
4802         if (likely(work < weight))
4803                 goto out_unlock;
4804
4805         /* Drivers must not modify the NAPI state if they
4806          * consume the entire weight.  In such cases this code
4807          * still "owns" the NAPI instance and therefore can
4808          * move the instance around on the list at-will.
4809          */
4810         if (unlikely(napi_disable_pending(n))) {
4811                 napi_complete(n);
4812                 goto out_unlock;
4813         }
4814
4815         if (n->gro_list) {
4816                 /* flush too old packets
4817                  * If HZ < 1000, flush all packets.
4818                  */
4819                 napi_gro_flush(n, HZ >= 1000);
4820         }
4821
4822         /* Some drivers may have called napi_schedule
4823          * prior to exhausting their budget.
4824          */
4825         if (unlikely(!list_empty(&n->poll_list))) {
4826                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
4827                              n->dev ? n->dev->name : "backlog");
4828                 goto out_unlock;
4829         }
4830
4831         list_add_tail(&n->poll_list, repoll);
4832
4833 out_unlock:
4834         netpoll_poll_unlock(have);
4835
4836         return work;
4837 }
4838
4839 static void net_rx_action(struct softirq_action *h)
4840 {
4841         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4842         unsigned long time_limit = jiffies + 2;
4843         int budget = netdev_budget;
4844         LIST_HEAD(list);
4845         LIST_HEAD(repoll);
4846
4847         local_irq_disable();
4848         list_splice_init(&sd->poll_list, &list);
4849         local_irq_enable();
4850
4851         for (;;) {
4852                 struct napi_struct *n;
4853
4854                 if (list_empty(&list)) {
4855                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
4856                                 return;
4857                         break;
4858                 }
4859
4860                 n = list_first_entry(&list, struct napi_struct, poll_list);
4861                 budget -= napi_poll(n, &repoll);
4862
4863                 /* If softirq window is exhausted then punt.
4864                  * Allow this to run for 2 jiffies since which will allow
4865                  * an average latency of 1.5/HZ.
4866                  */
4867                 if (unlikely(budget <= 0 ||
4868                              time_after_eq(jiffies, time_limit))) {
4869                         sd->time_squeeze++;
4870                         break;
4871                 }
4872         }
4873
4874         local_irq_disable();
4875
4876         list_splice_tail_init(&sd->poll_list, &list);
4877         list_splice_tail(&repoll, &list);
4878         list_splice(&list, &sd->poll_list);
4879         if (!list_empty(&sd->poll_list))
4880                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4881
4882         net_rps_action_and_irq_enable(sd);
4883 }
4884
4885 struct netdev_adjacent {
4886         struct net_device *dev;
4887
4888         /* upper master flag, there can only be one master device per list */
4889         bool master;
4890
4891         /* counter for the number of times this device was added to us */
4892         u16 ref_nr;
4893
4894         /* private field for the users */
4895         void *private;
4896
4897         struct list_head list;
4898         struct rcu_head rcu;
4899 };
4900
4901 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
4902                                                  struct list_head *adj_list)
4903 {
4904         struct netdev_adjacent *adj;
4905
4906         list_for_each_entry(adj, adj_list, list) {
4907                 if (adj->dev == adj_dev)
4908                         return adj;
4909         }
4910         return NULL;
4911 }
4912
4913 /**
4914  * netdev_has_upper_dev - Check if device is linked to an upper device
4915  * @dev: device
4916  * @upper_dev: upper device to check
4917  *
4918  * Find out if a device is linked to specified upper device and return true
4919  * in case it is. Note that this checks only immediate upper device,
4920  * not through a complete stack of devices. The caller must hold the RTNL lock.
4921  */
4922 bool netdev_has_upper_dev(struct net_device *dev,
4923                           struct net_device *upper_dev)
4924 {
4925         ASSERT_RTNL();
4926
4927         return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
4928 }
4929 EXPORT_SYMBOL(netdev_has_upper_dev);
4930
4931 /**
4932  * netdev_has_any_upper_dev - Check if device is linked to some device
4933  * @dev: device
4934  *
4935  * Find out if a device is linked to an upper device and return true in case
4936  * it is. The caller must hold the RTNL lock.
4937  */
4938 static bool netdev_has_any_upper_dev(struct net_device *dev)
4939 {
4940         ASSERT_RTNL();
4941
4942         return !list_empty(&dev->all_adj_list.upper);
4943 }
4944
4945 /**
4946  * netdev_master_upper_dev_get - Get master upper device
4947  * @dev: device
4948  *
4949  * Find a master upper device and return pointer to it or NULL in case
4950  * it's not there. The caller must hold the RTNL lock.
4951  */
4952 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
4953 {
4954         struct netdev_adjacent *upper;
4955
4956         ASSERT_RTNL();
4957
4958         if (list_empty(&dev->adj_list.upper))
4959                 return NULL;
4960
4961         upper = list_first_entry(&dev->adj_list.upper,
4962                                  struct netdev_adjacent, list);
4963         if (likely(upper->master))
4964                 return upper->dev;
4965         return NULL;
4966 }
4967 EXPORT_SYMBOL(netdev_master_upper_dev_get);
4968
4969 void *netdev_adjacent_get_private(struct list_head *adj_list)
4970 {
4971         struct netdev_adjacent *adj;
4972
4973         adj = list_entry(adj_list, struct netdev_adjacent, list);
4974
4975         return adj->private;
4976 }
4977 EXPORT_SYMBOL(netdev_adjacent_get_private);
4978
4979 /**
4980  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
4981  * @dev: device
4982  * @iter: list_head ** of the current position
4983  *
4984  * Gets the next device from the dev's upper list, starting from iter
4985  * position. The caller must hold RCU read lock.
4986  */
4987 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
4988                                                  struct list_head **iter)
4989 {
4990         struct netdev_adjacent *upper;
4991
4992         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
4993
4994         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
4995
4996         if (&upper->list == &dev->adj_list.upper)
4997                 return NULL;
4998
4999         *iter = &upper->list;
5000
5001         return upper->dev;
5002 }
5003 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5004
5005 /**
5006  * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
5007  * @dev: device
5008  * @iter: list_head ** of the current position
5009  *
5010  * Gets the next device from the dev's upper list, starting from iter
5011  * position. The caller must hold RCU read lock.
5012  */
5013 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5014                                                      struct list_head **iter)
5015 {
5016         struct netdev_adjacent *upper;
5017
5018         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5019
5020         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5021
5022         if (&upper->list == &dev->all_adj_list.upper)
5023                 return NULL;
5024
5025         *iter = &upper->list;
5026
5027         return upper->dev;
5028 }
5029 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
5030
5031 /**
5032  * netdev_lower_get_next_private - Get the next ->private from the
5033  *                                 lower neighbour list
5034  * @dev: device
5035  * @iter: list_head ** of the current position
5036  *
5037  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5038  * list, starting from iter position. The caller must hold either hold the
5039  * RTNL lock or its own locking that guarantees that the neighbour lower
5040  * list will remain unchanged.
5041  */
5042 void *netdev_lower_get_next_private(struct net_device *dev,
5043                                     struct list_head **iter)
5044 {
5045         struct netdev_adjacent *lower;
5046
5047         lower = list_entry(*iter, struct netdev_adjacent, list);
5048
5049         if (&lower->list == &dev->adj_list.lower)
5050                 return NULL;
5051
5052         *iter = lower->list.next;
5053
5054         return lower->private;
5055 }
5056 EXPORT_SYMBOL(netdev_lower_get_next_private);
5057
5058 /**
5059  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5060  *                                     lower neighbour list, RCU
5061  *                                     variant
5062  * @dev: device
5063  * @iter: list_head ** of the current position
5064  *
5065  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5066  * list, starting from iter position. The caller must hold RCU read lock.
5067  */
5068 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5069                                         struct list_head **iter)
5070 {
5071         struct netdev_adjacent *lower;
5072
5073         WARN_ON_ONCE(!rcu_read_lock_held());
5074
5075         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5076
5077         if (&lower->list == &dev->adj_list.lower)
5078                 return NULL;
5079
5080         *iter = &lower->list;
5081
5082         return lower->private;
5083 }
5084 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5085
5086 /**
5087  * netdev_lower_get_next - Get the next device from the lower neighbour
5088  *                         list
5089  * @dev: device
5090  * @iter: list_head ** of the current position
5091  *
5092  * Gets the next netdev_adjacent from the dev's lower neighbour
5093  * list, starting from iter position. The caller must hold RTNL lock or
5094  * its own locking that guarantees that the neighbour lower
5095  * list will remain unchanged.
5096  */
5097 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5098 {
5099         struct netdev_adjacent *lower;
5100
5101         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
5102
5103         if (&lower->list == &dev->adj_list.lower)
5104                 return NULL;
5105
5106         *iter = &lower->list;
5107
5108         return lower->dev;
5109 }
5110 EXPORT_SYMBOL(netdev_lower_get_next);
5111
5112 /**
5113  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5114  *                                     lower neighbour list, RCU
5115  *                                     variant
5116  * @dev: device
5117  *
5118  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5119  * list. The caller must hold RCU read lock.
5120  */
5121 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5122 {
5123         struct netdev_adjacent *lower;
5124
5125         lower = list_first_or_null_rcu(&dev->adj_list.lower,
5126                         struct netdev_adjacent, list);
5127         if (lower)
5128                 return lower->private;
5129         return NULL;
5130 }
5131 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5132
5133 /**
5134  * netdev_master_upper_dev_get_rcu - Get master upper device
5135  * @dev: device
5136  *
5137  * Find a master upper device and return pointer to it or NULL in case
5138  * it's not there. The caller must hold the RCU read lock.
5139  */
5140 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5141 {
5142         struct netdev_adjacent *upper;
5143
5144         upper = list_first_or_null_rcu(&dev->adj_list.upper,
5145                                        struct netdev_adjacent, list);
5146         if (upper && likely(upper->master))
5147                 return upper->dev;
5148         return NULL;
5149 }
5150 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5151
5152 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5153                               struct net_device *adj_dev,
5154                               struct list_head *dev_list)
5155 {
5156         char linkname[IFNAMSIZ+7];
5157         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5158                 "upper_%s" : "lower_%s", adj_dev->name);
5159         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5160                                  linkname);
5161 }
5162 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5163                                char *name,
5164                                struct list_head *dev_list)
5165 {
5166         char linkname[IFNAMSIZ+7];
5167         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5168                 "upper_%s" : "lower_%s", name);
5169         sysfs_remove_link(&(dev->dev.kobj), linkname);
5170 }
5171
5172 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5173                                                  struct net_device *adj_dev,
5174                                                  struct list_head *dev_list)
5175 {
5176         return (dev_list == &dev->adj_list.upper ||
5177                 dev_list == &dev->adj_list.lower) &&
5178                 net_eq(dev_net(dev), dev_net(adj_dev));
5179 }
5180
5181 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5182                                         struct net_device *adj_dev,
5183                                         struct list_head *dev_list,
5184                                         void *private, bool master)
5185 {
5186         struct netdev_adjacent *adj;
5187         int ret;
5188
5189         adj = __netdev_find_adj(adj_dev, dev_list);
5190
5191         if (adj) {
5192                 adj->ref_nr++;
5193                 return 0;
5194         }
5195
5196         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5197         if (!adj)
5198                 return -ENOMEM;
5199
5200         adj->dev = adj_dev;
5201         adj->master = master;
5202         adj->ref_nr = 1;
5203         adj->private = private;
5204         dev_hold(adj_dev);
5205
5206         pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5207                  adj_dev->name, dev->name, adj_dev->name);
5208
5209         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5210                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5211                 if (ret)
5212                         goto free_adj;
5213         }
5214
5215         /* Ensure that master link is always the first item in list. */
5216         if (master) {
5217                 ret = sysfs_create_link(&(dev->dev.kobj),
5218                                         &(adj_dev->dev.kobj), "master");
5219                 if (ret)
5220                         goto remove_symlinks;
5221
5222                 list_add_rcu(&adj->list, dev_list);
5223         } else {
5224                 list_add_tail_rcu(&adj->list, dev_list);
5225         }
5226
5227         return 0;
5228
5229 remove_symlinks:
5230         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5231                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5232 free_adj:
5233         kfree(adj);
5234         dev_put(adj_dev);
5235
5236         return ret;
5237 }
5238
5239 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5240                                          struct net_device *adj_dev,
5241                                          struct list_head *dev_list)
5242 {
5243         struct netdev_adjacent *adj;
5244
5245         adj = __netdev_find_adj(adj_dev, dev_list);
5246
5247         if (!adj) {
5248                 pr_err("tried to remove device %s from %s\n",
5249                        dev->name, adj_dev->name);
5250                 BUG();
5251         }
5252
5253         if (adj->ref_nr > 1) {
5254                 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5255                          adj->ref_nr-1);
5256                 adj->ref_nr--;
5257                 return;
5258         }
5259
5260         if (adj->master)
5261                 sysfs_remove_link(&(dev->dev.kobj), "master");
5262
5263         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5264                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5265
5266         list_del_rcu(&adj->list);
5267         pr_debug("dev_put for %s, because link removed from %s to %s\n",
5268                  adj_dev->name, dev->name, adj_dev->name);
5269         dev_put(adj_dev);
5270         kfree_rcu(adj, rcu);
5271 }
5272
5273 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5274                                             struct net_device *upper_dev,
5275                                             struct list_head *up_list,
5276                                             struct list_head *down_list,
5277                                             void *private, bool master)
5278 {
5279         int ret;
5280
5281         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5282                                            master);
5283         if (ret)
5284                 return ret;
5285
5286         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5287                                            false);
5288         if (ret) {
5289                 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5290                 return ret;
5291         }
5292
5293         return 0;
5294 }
5295
5296 static int __netdev_adjacent_dev_link(struct net_device *dev,
5297                                       struct net_device *upper_dev)
5298 {
5299         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5300                                                 &dev->all_adj_list.upper,
5301                                                 &upper_dev->all_adj_list.lower,
5302                                                 NULL, false);
5303 }
5304
5305 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5306                                                struct net_device *upper_dev,
5307                                                struct list_head *up_list,
5308                                                struct list_head *down_list)
5309 {
5310         __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5311         __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
5312 }
5313
5314 static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5315                                          struct net_device *upper_dev)
5316 {
5317         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5318                                            &dev->all_adj_list.upper,
5319                                            &upper_dev->all_adj_list.lower);
5320 }
5321
5322 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5323                                                 struct net_device *upper_dev,
5324                                                 void *private, bool master)
5325 {
5326         int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5327
5328         if (ret)
5329                 return ret;
5330
5331         ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5332                                                &dev->adj_list.upper,
5333                                                &upper_dev->adj_list.lower,
5334                                                private, master);
5335         if (ret) {
5336                 __netdev_adjacent_dev_unlink(dev, upper_dev);
5337                 return ret;
5338         }
5339
5340         return 0;
5341 }
5342
5343 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5344                                                    struct net_device *upper_dev)
5345 {
5346         __netdev_adjacent_dev_unlink(dev, upper_dev);
5347         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5348                                            &dev->adj_list.upper,
5349                                            &upper_dev->adj_list.lower);
5350 }
5351
5352 static int __netdev_upper_dev_link(struct net_device *dev,
5353                                    struct net_device *upper_dev, bool master,
5354                                    void *private)
5355 {
5356         struct netdev_notifier_changeupper_info changeupper_info;
5357         struct netdev_adjacent *i, *j, *to_i, *to_j;
5358         int ret = 0;
5359
5360         ASSERT_RTNL();
5361
5362         if (dev == upper_dev)
5363                 return -EBUSY;
5364
5365         /* To prevent loops, check if dev is not upper device to upper_dev. */
5366         if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
5367                 return -EBUSY;
5368
5369         if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
5370                 return -EEXIST;
5371
5372         if (master && netdev_master_upper_dev_get(dev))
5373                 return -EBUSY;
5374
5375         changeupper_info.upper_dev = upper_dev;
5376         changeupper_info.master = master;
5377         changeupper_info.linking = true;
5378
5379         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5380                                             &changeupper_info.info);
5381         ret = notifier_to_errno(ret);
5382         if (ret)
5383                 return ret;
5384
5385         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private,
5386                                                    master);
5387         if (ret)
5388                 return ret;
5389
5390         /* Now that we linked these devs, make all the upper_dev's
5391          * all_adj_list.upper visible to every dev's all_adj_list.lower an
5392          * versa, and don't forget the devices itself. All of these
5393          * links are non-neighbours.
5394          */
5395         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5396                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5397                         pr_debug("Interlinking %s with %s, non-neighbour\n",
5398                                  i->dev->name, j->dev->name);
5399                         ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5400                         if (ret)
5401                                 goto rollback_mesh;
5402                 }
5403         }
5404
5405         /* add dev to every upper_dev's upper device */
5406         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5407                 pr_debug("linking %s's upper device %s with %s\n",
5408                          upper_dev->name, i->dev->name, dev->name);
5409                 ret = __netdev_adjacent_dev_link(dev, i->dev);
5410                 if (ret)
5411                         goto rollback_upper_mesh;
5412         }
5413
5414         /* add upper_dev to every dev's lower device */
5415         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5416                 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5417                          i->dev->name, upper_dev->name);
5418                 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5419                 if (ret)
5420                         goto rollback_lower_mesh;
5421         }
5422
5423         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5424                                       &changeupper_info.info);
5425         return 0;
5426
5427 rollback_lower_mesh:
5428         to_i = i;
5429         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5430                 if (i == to_i)
5431                         break;
5432                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5433         }
5434
5435         i = NULL;
5436
5437 rollback_upper_mesh:
5438         to_i = i;
5439         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5440                 if (i == to_i)
5441                         break;
5442                 __netdev_adjacent_dev_unlink(dev, i->dev);
5443         }
5444
5445         i = j = NULL;
5446
5447 rollback_mesh:
5448         to_i = i;
5449         to_j = j;
5450         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5451                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5452                         if (i == to_i && j == to_j)
5453                                 break;
5454                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5455                 }
5456                 if (i == to_i)
5457                         break;
5458         }
5459
5460         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5461
5462         return ret;
5463 }
5464
5465 /**
5466  * netdev_upper_dev_link - Add a link to the upper device
5467  * @dev: device
5468  * @upper_dev: new upper device
5469  *
5470  * Adds a link to device which is upper to this one. The caller must hold
5471  * the RTNL lock. On a failure a negative errno code is returned.
5472  * On success the reference counts are adjusted and the function
5473  * returns zero.
5474  */
5475 int netdev_upper_dev_link(struct net_device *dev,
5476                           struct net_device *upper_dev)
5477 {
5478         return __netdev_upper_dev_link(dev, upper_dev, false, NULL);
5479 }
5480 EXPORT_SYMBOL(netdev_upper_dev_link);
5481
5482 /**
5483  * netdev_master_upper_dev_link - Add a master link to the upper device
5484  * @dev: device
5485  * @upper_dev: new upper device
5486  *
5487  * Adds a link to device which is upper to this one. In this case, only
5488  * one master upper device can be linked, although other non-master devices
5489  * might be linked as well. The caller must hold the RTNL lock.
5490  * On a failure a negative errno code is returned. On success the reference
5491  * counts are adjusted and the function returns zero.
5492  */
5493 int netdev_master_upper_dev_link(struct net_device *dev,
5494                                  struct net_device *upper_dev)
5495 {
5496         return __netdev_upper_dev_link(dev, upper_dev, true, NULL);
5497 }
5498 EXPORT_SYMBOL(netdev_master_upper_dev_link);
5499
5500 int netdev_master_upper_dev_link_private(struct net_device *dev,
5501                                          struct net_device *upper_dev,
5502                                          void *private)
5503 {
5504         return __netdev_upper_dev_link(dev, upper_dev, true, private);
5505 }
5506 EXPORT_SYMBOL(netdev_master_upper_dev_link_private);
5507
5508 /**
5509  * netdev_upper_dev_unlink - Removes a link to upper device
5510  * @dev: device
5511  * @upper_dev: new upper device
5512  *
5513  * Removes a link to device which is upper to this one. The caller must hold
5514  * the RTNL lock.
5515  */
5516 void netdev_upper_dev_unlink(struct net_device *dev,
5517                              struct net_device *upper_dev)
5518 {
5519         struct netdev_notifier_changeupper_info changeupper_info;
5520         struct netdev_adjacent *i, *j;
5521         ASSERT_RTNL();
5522
5523         changeupper_info.upper_dev = upper_dev;
5524         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5525         changeupper_info.linking = false;
5526
5527         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5528                                       &changeupper_info.info);
5529
5530         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5531
5532         /* Here is the tricky part. We must remove all dev's lower
5533          * devices from all upper_dev's upper devices and vice
5534          * versa, to maintain the graph relationship.
5535          */
5536         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5537                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5538                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5539
5540         /* remove also the devices itself from lower/upper device
5541          * list
5542          */
5543         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5544                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5545
5546         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5547                 __netdev_adjacent_dev_unlink(dev, i->dev);
5548
5549         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5550                                       &changeupper_info.info);
5551 }
5552 EXPORT_SYMBOL(netdev_upper_dev_unlink);
5553
5554 /**
5555  * netdev_bonding_info_change - Dispatch event about slave change
5556  * @dev: device
5557  * @bonding_info: info to dispatch
5558  *
5559  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5560  * The caller must hold the RTNL lock.
5561  */
5562 void netdev_bonding_info_change(struct net_device *dev,
5563                                 struct netdev_bonding_info *bonding_info)
5564 {
5565         struct netdev_notifier_bonding_info     info;
5566
5567         memcpy(&info.bonding_info, bonding_info,
5568                sizeof(struct netdev_bonding_info));
5569         call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5570                                       &info.info);
5571 }
5572 EXPORT_SYMBOL(netdev_bonding_info_change);
5573
5574 static void netdev_adjacent_add_links(struct net_device *dev)
5575 {
5576         struct netdev_adjacent *iter;
5577
5578         struct net *net = dev_net(dev);
5579
5580         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5581                 if (!net_eq(net,dev_net(iter->dev)))
5582                         continue;
5583                 netdev_adjacent_sysfs_add(iter->dev, dev,
5584                                           &iter->dev->adj_list.lower);
5585                 netdev_adjacent_sysfs_add(dev, iter->dev,
5586                                           &dev->adj_list.upper);
5587         }
5588
5589         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5590                 if (!net_eq(net,dev_net(iter->dev)))
5591                         continue;
5592                 netdev_adjacent_sysfs_add(iter->dev, dev,
5593                                           &iter->dev->adj_list.upper);
5594                 netdev_adjacent_sysfs_add(dev, iter->dev,
5595                                           &dev->adj_list.lower);
5596         }
5597 }
5598
5599 static void netdev_adjacent_del_links(struct net_device *dev)
5600 {
5601         struct netdev_adjacent *iter;
5602
5603         struct net *net = dev_net(dev);
5604
5605         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5606                 if (!net_eq(net,dev_net(iter->dev)))
5607                         continue;
5608                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5609                                           &iter->dev->adj_list.lower);
5610                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5611                                           &dev->adj_list.upper);
5612         }
5613
5614         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5615                 if (!net_eq(net,dev_net(iter->dev)))
5616                         continue;
5617                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5618                                           &iter->dev->adj_list.upper);
5619                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5620                                           &dev->adj_list.lower);
5621         }
5622 }
5623
5624 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5625 {
5626         struct netdev_adjacent *iter;
5627
5628         struct net *net = dev_net(dev);
5629
5630         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5631                 if (!net_eq(net,dev_net(iter->dev)))
5632                         continue;
5633                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5634                                           &iter->dev->adj_list.lower);
5635                 netdev_adjacent_sysfs_add(iter->dev, dev,
5636                                           &iter->dev->adj_list.lower);
5637         }
5638
5639         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5640                 if (!net_eq(net,dev_net(iter->dev)))
5641                         continue;
5642                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5643                                           &iter->dev->adj_list.upper);
5644                 netdev_adjacent_sysfs_add(iter->dev, dev,
5645                                           &iter->dev->adj_list.upper);
5646         }
5647 }
5648
5649 void *netdev_lower_dev_get_private(struct net_device *dev,
5650                                    struct net_device *lower_dev)
5651 {
5652         struct netdev_adjacent *lower;
5653
5654         if (!lower_dev)
5655                 return NULL;
5656         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
5657         if (!lower)
5658                 return NULL;
5659
5660         return lower->private;
5661 }
5662 EXPORT_SYMBOL(netdev_lower_dev_get_private);
5663
5664
5665 int dev_get_nest_level(struct net_device *dev,
5666                        bool (*type_check)(struct net_device *dev))
5667 {
5668         struct net_device *lower = NULL;
5669         struct list_head *iter;
5670         int max_nest = -1;
5671         int nest;
5672
5673         ASSERT_RTNL();
5674
5675         netdev_for_each_lower_dev(dev, lower, iter) {
5676                 nest = dev_get_nest_level(lower, type_check);
5677                 if (max_nest < nest)
5678                         max_nest = nest;
5679         }
5680
5681         if (type_check(dev))
5682                 max_nest++;
5683
5684         return max_nest;
5685 }
5686 EXPORT_SYMBOL(dev_get_nest_level);
5687
5688 static void dev_change_rx_flags(struct net_device *dev, int flags)
5689 {
5690         const struct net_device_ops *ops = dev->netdev_ops;
5691
5692         if (ops->ndo_change_rx_flags)
5693                 ops->ndo_change_rx_flags(dev, flags);
5694 }
5695
5696 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
5697 {
5698         unsigned int old_flags = dev->flags;
5699         kuid_t uid;
5700         kgid_t gid;
5701
5702         ASSERT_RTNL();
5703
5704         dev->flags |= IFF_PROMISC;
5705         dev->promiscuity += inc;
5706         if (dev->promiscuity == 0) {
5707                 /*
5708                  * Avoid overflow.
5709                  * If inc causes overflow, untouch promisc and return error.
5710                  */
5711                 if (inc < 0)
5712                         dev->flags &= ~IFF_PROMISC;
5713                 else {
5714                         dev->promiscuity -= inc;
5715                         pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
5716                                 dev->name);
5717                         return -EOVERFLOW;
5718                 }
5719         }
5720         if (dev->flags != old_flags) {
5721                 pr_info("device %s %s promiscuous mode\n",
5722                         dev->name,
5723                         dev->flags & IFF_PROMISC ? "entered" : "left");
5724                 if (audit_enabled) {
5725                         current_uid_gid(&uid, &gid);
5726                         audit_log(current->audit_context, GFP_ATOMIC,
5727                                 AUDIT_ANOM_PROMISCUOUS,
5728                                 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
5729                                 dev->name, (dev->flags & IFF_PROMISC),
5730                                 (old_flags & IFF_PROMISC),
5731                                 from_kuid(&init_user_ns, audit_get_loginuid(current)),
5732                                 from_kuid(&init_user_ns, uid),
5733                                 from_kgid(&init_user_ns, gid),
5734                                 audit_get_sessionid(current));
5735                 }
5736
5737                 dev_change_rx_flags(dev, IFF_PROMISC);
5738         }
5739         if (notify)
5740                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
5741         return 0;
5742 }
5743
5744 /**
5745  *      dev_set_promiscuity     - update promiscuity count on a device
5746  *      @dev: device
5747  *      @inc: modifier
5748  *
5749  *      Add or remove promiscuity from a device. While the count in the device
5750  *      remains above zero the interface remains promiscuous. Once it hits zero
5751  *      the device reverts back to normal filtering operation. A negative inc
5752  *      value is used to drop promiscuity on the device.
5753  *      Return 0 if successful or a negative errno code on error.
5754  */
5755 int dev_set_promiscuity(struct net_device *dev, int inc)
5756 {
5757         unsigned int old_flags = dev->flags;
5758         int err;
5759
5760         err = __dev_set_promiscuity(dev, inc, true);
5761         if (err < 0)
5762                 return err;
5763         if (dev->flags != old_flags)
5764                 dev_set_rx_mode(dev);
5765         return err;
5766 }
5767 EXPORT_SYMBOL(dev_set_promiscuity);
5768
5769 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
5770 {
5771         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
5772
5773         ASSERT_RTNL();
5774
5775         dev->flags |= IFF_ALLMULTI;
5776         dev->allmulti += inc;
5777         if (dev->allmulti == 0) {
5778                 /*
5779                  * Avoid overflow.
5780                  * If inc causes overflow, untouch allmulti and return error.
5781                  */
5782                 if (inc < 0)
5783                         dev->flags &= ~IFF_ALLMULTI;
5784                 else {
5785                         dev->allmulti -= inc;
5786                         pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
5787                                 dev->name);
5788                         return -EOVERFLOW;
5789                 }
5790         }
5791         if (dev->flags ^ old_flags) {
5792                 dev_change_rx_flags(dev, IFF_ALLMULTI);
5793                 dev_set_rx_mode(dev);
5794                 if (notify)
5795                         __dev_notify_flags(dev, old_flags,
5796                                            dev->gflags ^ old_gflags);
5797         }
5798         return 0;
5799 }
5800
5801 /**
5802  *      dev_set_allmulti        - update allmulti count on a device
5803  *      @dev: device
5804  *      @inc: modifier
5805  *
5806  *      Add or remove reception of all multicast frames to a device. While the
5807  *      count in the device remains above zero the interface remains listening
5808  *      to all interfaces. Once it hits zero the device reverts back to normal
5809  *      filtering operation. A negative @inc value is used to drop the counter
5810  *      when releasing a resource needing all multicasts.
5811  *      Return 0 if successful or a negative errno code on error.
5812  */
5813
5814 int dev_set_allmulti(struct net_device *dev, int inc)
5815 {
5816         return __dev_set_allmulti(dev, inc, true);
5817 }
5818 EXPORT_SYMBOL(dev_set_allmulti);
5819
5820 /*
5821  *      Upload unicast and multicast address lists to device and
5822  *      configure RX filtering. When the device doesn't support unicast
5823  *      filtering it is put in promiscuous mode while unicast addresses
5824  *      are present.
5825  */
5826 void __dev_set_rx_mode(struct net_device *dev)
5827 {
5828         const struct net_device_ops *ops = dev->netdev_ops;
5829
5830         /* dev_open will call this function so the list will stay sane. */
5831         if (!(dev->flags&IFF_UP))
5832                 return;
5833
5834         if (!netif_device_present(dev))
5835                 return;
5836
5837         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
5838                 /* Unicast addresses changes may only happen under the rtnl,
5839                  * therefore calling __dev_set_promiscuity here is safe.
5840                  */
5841                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
5842                         __dev_set_promiscuity(dev, 1, false);
5843                         dev->uc_promisc = true;
5844                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
5845                         __dev_set_promiscuity(dev, -1, false);
5846                         dev->uc_promisc = false;
5847                 }
5848         }
5849
5850         if (ops->ndo_set_rx_mode)
5851                 ops->ndo_set_rx_mode(dev);
5852 }
5853
5854 void dev_set_rx_mode(struct net_device *dev)
5855 {
5856         netif_addr_lock_bh(dev);
5857         __dev_set_rx_mode(dev);
5858         netif_addr_unlock_bh(dev);
5859 }
5860
5861 /**
5862  *      dev_get_flags - get flags reported to userspace
5863  *      @dev: device
5864  *
5865  *      Get the combination of flag bits exported through APIs to userspace.
5866  */
5867 unsigned int dev_get_flags(const struct net_device *dev)
5868 {
5869         unsigned int flags;
5870
5871         flags = (dev->flags & ~(IFF_PROMISC |
5872                                 IFF_ALLMULTI |
5873                                 IFF_RUNNING |
5874                                 IFF_LOWER_UP |
5875                                 IFF_DORMANT)) |
5876                 (dev->gflags & (IFF_PROMISC |
5877                                 IFF_ALLMULTI));
5878
5879         if (netif_running(dev)) {
5880                 if (netif_oper_up(dev))
5881                         flags |= IFF_RUNNING;
5882                 if (netif_carrier_ok(dev))
5883                         flags |= IFF_LOWER_UP;
5884                 if (netif_dormant(dev))
5885                         flags |= IFF_DORMANT;
5886         }
5887
5888         return flags;
5889 }
5890 EXPORT_SYMBOL(dev_get_flags);
5891
5892 int __dev_change_flags(struct net_device *dev, unsigned int flags)
5893 {
5894         unsigned int old_flags = dev->flags;
5895         int ret;
5896
5897         ASSERT_RTNL();
5898
5899         /*
5900          *      Set the flags on our device.
5901          */
5902
5903         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
5904                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
5905                                IFF_AUTOMEDIA)) |
5906                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
5907                                     IFF_ALLMULTI));
5908
5909         /*
5910          *      Load in the correct multicast list now the flags have changed.
5911          */
5912
5913         if ((old_flags ^ flags) & IFF_MULTICAST)
5914                 dev_change_rx_flags(dev, IFF_MULTICAST);
5915
5916         dev_set_rx_mode(dev);
5917
5918         /*
5919          *      Have we downed the interface. We handle IFF_UP ourselves
5920          *      according to user attempts to set it, rather than blindly
5921          *      setting it.
5922          */
5923
5924         ret = 0;
5925         if ((old_flags ^ flags) & IFF_UP)
5926                 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
5927
5928         if ((flags ^ dev->gflags) & IFF_PROMISC) {
5929                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
5930                 unsigned int old_flags = dev->flags;
5931
5932                 dev->gflags ^= IFF_PROMISC;
5933
5934                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
5935                         if (dev->flags != old_flags)
5936                                 dev_set_rx_mode(dev);
5937         }
5938
5939         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
5940            is important. Some (broken) drivers set IFF_PROMISC, when
5941            IFF_ALLMULTI is requested not asking us and not reporting.
5942          */
5943         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
5944                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
5945
5946                 dev->gflags ^= IFF_ALLMULTI;
5947                 __dev_set_allmulti(dev, inc, false);
5948         }
5949
5950         return ret;
5951 }
5952
5953 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
5954                         unsigned int gchanges)
5955 {
5956         unsigned int changes = dev->flags ^ old_flags;
5957
5958         if (gchanges)
5959                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
5960
5961         if (changes & IFF_UP) {
5962                 if (dev->flags & IFF_UP)
5963                         call_netdevice_notifiers(NETDEV_UP, dev);
5964                 else
5965                         call_netdevice_notifiers(NETDEV_DOWN, dev);
5966         }
5967
5968         if (dev->flags & IFF_UP &&
5969             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
5970                 struct netdev_notifier_change_info change_info;
5971
5972                 change_info.flags_changed = changes;
5973                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
5974                                               &change_info.info);
5975         }
5976 }
5977
5978 /**
5979  *      dev_change_flags - change device settings
5980  *      @dev: device
5981  *      @flags: device state flags
5982  *
5983  *      Change settings on device based state flags. The flags are
5984  *      in the userspace exported format.
5985  */
5986 int dev_change_flags(struct net_device *dev, unsigned int flags)
5987 {
5988         int ret;
5989         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
5990
5991         ret = __dev_change_flags(dev, flags);
5992         if (ret < 0)
5993                 return ret;
5994
5995         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
5996         __dev_notify_flags(dev, old_flags, changes);
5997         return ret;
5998 }
5999 EXPORT_SYMBOL(dev_change_flags);
6000
6001 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6002 {
6003         const struct net_device_ops *ops = dev->netdev_ops;
6004
6005         if (ops->ndo_change_mtu)
6006                 return ops->ndo_change_mtu(dev, new_mtu);
6007
6008         dev->mtu = new_mtu;
6009         return 0;
6010 }
6011
6012 /**
6013  *      dev_set_mtu - Change maximum transfer unit
6014  *      @dev: device
6015  *      @new_mtu: new transfer unit
6016  *
6017  *      Change the maximum transfer size of the network device.
6018  */
6019 int dev_set_mtu(struct net_device *dev, int new_mtu)
6020 {
6021         int err, orig_mtu;
6022
6023         if (new_mtu == dev->mtu)
6024                 return 0;
6025
6026         /*      MTU must be positive.    */
6027         if (new_mtu < 0)
6028                 return -EINVAL;
6029
6030         if (!netif_device_present(dev))
6031                 return -ENODEV;
6032
6033         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6034         err = notifier_to_errno(err);
6035         if (err)
6036                 return err;
6037
6038         orig_mtu = dev->mtu;
6039         err = __dev_set_mtu(dev, new_mtu);
6040
6041         if (!err) {
6042                 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6043                 err = notifier_to_errno(err);
6044                 if (err) {
6045                         /* setting mtu back and notifying everyone again,
6046                          * so that they have a chance to revert changes.
6047                          */
6048                         __dev_set_mtu(dev, orig_mtu);
6049                         call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6050                 }
6051         }
6052         return err;
6053 }
6054 EXPORT_SYMBOL(dev_set_mtu);
6055
6056 /**
6057  *      dev_set_group - Change group this device belongs to
6058  *      @dev: device
6059  *      @new_group: group this device should belong to
6060  */
6061 void dev_set_group(struct net_device *dev, int new_group)
6062 {
6063         dev->group = new_group;
6064 }
6065 EXPORT_SYMBOL(dev_set_group);
6066
6067 /**
6068  *      dev_set_mac_address - Change Media Access Control Address
6069  *      @dev: device
6070  *      @sa: new address
6071  *
6072  *      Change the hardware (MAC) address of the device
6073  */
6074 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6075 {
6076         const struct net_device_ops *ops = dev->netdev_ops;
6077         int err;
6078
6079         if (!ops->ndo_set_mac_address)
6080                 return -EOPNOTSUPP;
6081         if (sa->sa_family != dev->type)
6082                 return -EINVAL;
6083         if (!netif_device_present(dev))
6084                 return -ENODEV;
6085         err = ops->ndo_set_mac_address(dev, sa);
6086         if (err)
6087                 return err;
6088         dev->addr_assign_type = NET_ADDR_SET;
6089         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6090         add_device_randomness(dev->dev_addr, dev->addr_len);
6091         return 0;
6092 }
6093 EXPORT_SYMBOL(dev_set_mac_address);
6094
6095 /**
6096  *      dev_change_carrier - Change device carrier
6097  *      @dev: device
6098  *      @new_carrier: new value
6099  *
6100  *      Change device carrier
6101  */
6102 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6103 {
6104         const struct net_device_ops *ops = dev->netdev_ops;
6105
6106         if (!ops->ndo_change_carrier)
6107                 return -EOPNOTSUPP;
6108         if (!netif_device_present(dev))
6109                 return -ENODEV;
6110         return ops->ndo_change_carrier(dev, new_carrier);
6111 }
6112 EXPORT_SYMBOL(dev_change_carrier);
6113
6114 /**
6115  *      dev_get_phys_port_id - Get device physical port ID
6116  *      @dev: device
6117  *      @ppid: port ID
6118  *
6119  *      Get device physical port ID
6120  */
6121 int dev_get_phys_port_id(struct net_device *dev,
6122                          struct netdev_phys_item_id *ppid)
6123 {
6124         const struct net_device_ops *ops = dev->netdev_ops;
6125
6126         if (!ops->ndo_get_phys_port_id)
6127                 return -EOPNOTSUPP;
6128         return ops->ndo_get_phys_port_id(dev, ppid);
6129 }
6130 EXPORT_SYMBOL(dev_get_phys_port_id);
6131
6132 /**
6133  *      dev_get_phys_port_name - Get device physical port name
6134  *      @dev: device
6135  *      @name: port name
6136  *
6137  *      Get device physical port name
6138  */
6139 int dev_get_phys_port_name(struct net_device *dev,
6140                            char *name, size_t len)
6141 {
6142         const struct net_device_ops *ops = dev->netdev_ops;
6143
6144         if (!ops->ndo_get_phys_port_name)
6145                 return -EOPNOTSUPP;
6146         return ops->ndo_get_phys_port_name(dev, name, len);
6147 }
6148 EXPORT_SYMBOL(dev_get_phys_port_name);
6149
6150 /**
6151  *      dev_change_proto_down - update protocol port state information
6152  *      @dev: device
6153  *      @proto_down: new value
6154  *
6155  *      This info can be used by switch drivers to set the phys state of the
6156  *      port.
6157  */
6158 int dev_change_proto_down(struct net_device *dev, bool proto_down)
6159 {
6160         const struct net_device_ops *ops = dev->netdev_ops;
6161
6162         if (!ops->ndo_change_proto_down)
6163                 return -EOPNOTSUPP;
6164         if (!netif_device_present(dev))
6165                 return -ENODEV;
6166         return ops->ndo_change_proto_down(dev, proto_down);
6167 }
6168 EXPORT_SYMBOL(dev_change_proto_down);
6169
6170 /**
6171  *      dev_new_index   -       allocate an ifindex
6172  *      @net: the applicable net namespace
6173  *
6174  *      Returns a suitable unique value for a new device interface
6175  *      number.  The caller must hold the rtnl semaphore or the
6176  *      dev_base_lock to be sure it remains unique.
6177  */
6178 static int dev_new_index(struct net *net)
6179 {
6180         int ifindex = net->ifindex;
6181         for (;;) {
6182                 if (++ifindex <= 0)
6183                         ifindex = 1;
6184                 if (!__dev_get_by_index(net, ifindex))
6185                         return net->ifindex = ifindex;
6186         }
6187 }
6188
6189 /* Delayed registration/unregisteration */
6190 static LIST_HEAD(net_todo_list);
6191 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6192
6193 static void net_set_todo(struct net_device *dev)
6194 {
6195         list_add_tail(&dev->todo_list, &net_todo_list);
6196         dev_net(dev)->dev_unreg_count++;
6197 }
6198
6199 static void rollback_registered_many(struct list_head *head)
6200 {
6201         struct net_device *dev, *tmp;
6202         LIST_HEAD(close_head);
6203
6204         BUG_ON(dev_boot_phase);
6205         ASSERT_RTNL();
6206
6207         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6208                 /* Some devices call without registering
6209                  * for initialization unwind. Remove those
6210                  * devices and proceed with the remaining.
6211                  */
6212                 if (dev->reg_state == NETREG_UNINITIALIZED) {
6213                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6214                                  dev->name, dev);
6215
6216                         WARN_ON(1);
6217                         list_del(&dev->unreg_list);
6218                         continue;
6219                 }
6220                 dev->dismantle = true;
6221                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
6222         }
6223
6224         /* If device is running, close it first. */
6225         list_for_each_entry(dev, head, unreg_list)
6226                 list_add_tail(&dev->close_list, &close_head);
6227         dev_close_many(&close_head, true);
6228
6229         list_for_each_entry(dev, head, unreg_list) {
6230                 /* And unlink it from device chain. */
6231                 unlist_netdevice(dev);
6232
6233                 dev->reg_state = NETREG_UNREGISTERING;
6234                 on_each_cpu(flush_backlog, dev, 1);
6235         }
6236
6237         synchronize_net();
6238
6239         list_for_each_entry(dev, head, unreg_list) {
6240                 struct sk_buff *skb = NULL;
6241
6242                 /* Shutdown queueing discipline. */
6243                 dev_shutdown(dev);
6244
6245
6246                 /* Notify protocols, that we are about to destroy
6247                    this device. They should clean all the things.
6248                 */
6249                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6250
6251                 if (!dev->rtnl_link_ops ||
6252                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6253                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6254                                                      GFP_KERNEL);
6255
6256                 /*
6257                  *      Flush the unicast and multicast chains
6258                  */
6259                 dev_uc_flush(dev);
6260                 dev_mc_flush(dev);
6261
6262                 if (dev->netdev_ops->ndo_uninit)
6263                         dev->netdev_ops->ndo_uninit(dev);
6264
6265                 if (skb)
6266                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6267
6268                 /* Notifier chain MUST detach us all upper devices. */
6269                 WARN_ON(netdev_has_any_upper_dev(dev));
6270
6271                 /* Remove entries from kobject tree */
6272                 netdev_unregister_kobject(dev);
6273 #ifdef CONFIG_XPS
6274                 /* Remove XPS queueing entries */
6275                 netif_reset_xps_queues_gt(dev, 0);
6276 #endif
6277         }
6278
6279         synchronize_net();
6280
6281         list_for_each_entry(dev, head, unreg_list)
6282                 dev_put(dev);
6283 }
6284
6285 static void rollback_registered(struct net_device *dev)
6286 {
6287         LIST_HEAD(single);
6288
6289         list_add(&dev->unreg_list, &single);
6290         rollback_registered_many(&single);
6291         list_del(&single);
6292 }
6293
6294 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6295         struct net_device *upper, netdev_features_t features)
6296 {
6297         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6298         netdev_features_t feature;
6299         int feature_bit;
6300
6301         for_each_netdev_feature(&upper_disables, feature_bit) {
6302                 feature = __NETIF_F_BIT(feature_bit);
6303                 if (!(upper->wanted_features & feature)
6304                     && (features & feature)) {
6305                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6306                                    &feature, upper->name);
6307                         features &= ~feature;
6308                 }
6309         }
6310
6311         return features;
6312 }
6313
6314 static void netdev_sync_lower_features(struct net_device *upper,
6315         struct net_device *lower, netdev_features_t features)
6316 {
6317         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6318         netdev_features_t feature;
6319         int feature_bit;
6320
6321         for_each_netdev_feature(&upper_disables, feature_bit) {
6322                 feature = __NETIF_F_BIT(feature_bit);
6323                 if (!(features & feature) && (lower->features & feature)) {
6324                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6325                                    &feature, lower->name);
6326                         lower->wanted_features &= ~feature;
6327                         netdev_update_features(lower);
6328
6329                         if (unlikely(lower->features & feature))
6330                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6331                                             &feature, lower->name);
6332                 }
6333         }
6334 }
6335
6336 static netdev_features_t netdev_fix_features(struct net_device *dev,
6337         netdev_features_t features)
6338 {
6339         /* Fix illegal checksum combinations */
6340         if ((features & NETIF_F_HW_CSUM) &&
6341             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6342                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6343                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6344         }
6345
6346         /* TSO requires that SG is present as well. */
6347         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6348                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6349                 features &= ~NETIF_F_ALL_TSO;
6350         }
6351
6352         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6353                                         !(features & NETIF_F_IP_CSUM)) {
6354                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6355                 features &= ~NETIF_F_TSO;
6356                 features &= ~NETIF_F_TSO_ECN;
6357         }
6358
6359         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6360                                          !(features & NETIF_F_IPV6_CSUM)) {
6361                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6362                 features &= ~NETIF_F_TSO6;
6363         }
6364
6365         /* TSO ECN requires that TSO is present as well. */
6366         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6367                 features &= ~NETIF_F_TSO_ECN;
6368
6369         /* Software GSO depends on SG. */
6370         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6371                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6372                 features &= ~NETIF_F_GSO;
6373         }
6374
6375         /* UFO needs SG and checksumming */
6376         if (features & NETIF_F_UFO) {
6377                 /* maybe split UFO into V4 and V6? */
6378                 if (!((features & NETIF_F_GEN_CSUM) ||
6379                     (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
6380                             == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6381                         netdev_dbg(dev,
6382                                 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6383                         features &= ~NETIF_F_UFO;
6384                 }
6385
6386                 if (!(features & NETIF_F_SG)) {
6387                         netdev_dbg(dev,
6388                                 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6389                         features &= ~NETIF_F_UFO;
6390                 }
6391         }
6392
6393 #ifdef CONFIG_NET_RX_BUSY_POLL
6394         if (dev->netdev_ops->ndo_busy_poll)
6395                 features |= NETIF_F_BUSY_POLL;
6396         else
6397 #endif
6398                 features &= ~NETIF_F_BUSY_POLL;
6399
6400         return features;
6401 }
6402
6403 int __netdev_update_features(struct net_device *dev)
6404 {
6405         struct net_device *upper, *lower;
6406         netdev_features_t features;
6407         struct list_head *iter;
6408         int err = -1;
6409
6410         ASSERT_RTNL();
6411
6412         features = netdev_get_wanted_features(dev);
6413
6414         if (dev->netdev_ops->ndo_fix_features)
6415                 features = dev->netdev_ops->ndo_fix_features(dev, features);
6416
6417         /* driver might be less strict about feature dependencies */
6418         features = netdev_fix_features(dev, features);
6419
6420         /* some features can't be enabled if they're off an an upper device */
6421         netdev_for_each_upper_dev_rcu(dev, upper, iter)
6422                 features = netdev_sync_upper_features(dev, upper, features);
6423
6424         if (dev->features == features)
6425                 goto sync_lower;
6426
6427         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6428                 &dev->features, &features);
6429
6430         if (dev->netdev_ops->ndo_set_features)
6431                 err = dev->netdev_ops->ndo_set_features(dev, features);
6432         else
6433                 err = 0;
6434
6435         if (unlikely(err < 0)) {
6436                 netdev_err(dev,
6437                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
6438                         err, &features, &dev->features);
6439                 /* return non-0 since some features might have changed and
6440                  * it's better to fire a spurious notification than miss it
6441                  */
6442                 return -1;
6443         }
6444
6445 sync_lower:
6446         /* some features must be disabled on lower devices when disabled
6447          * on an upper device (think: bonding master or bridge)
6448          */
6449         netdev_for_each_lower_dev(dev, lower, iter)
6450                 netdev_sync_lower_features(dev, lower, features);
6451
6452         if (!err)
6453                 dev->features = features;
6454
6455         return err < 0 ? 0 : 1;
6456 }
6457
6458 /**
6459  *      netdev_update_features - recalculate device features
6460  *      @dev: the device to check
6461  *
6462  *      Recalculate dev->features set and send notifications if it
6463  *      has changed. Should be called after driver or hardware dependent
6464  *      conditions might have changed that influence the features.
6465  */
6466 void netdev_update_features(struct net_device *dev)
6467 {
6468         if (__netdev_update_features(dev))
6469                 netdev_features_change(dev);
6470 }
6471 EXPORT_SYMBOL(netdev_update_features);
6472
6473 /**
6474  *      netdev_change_features - recalculate device features
6475  *      @dev: the device to check
6476  *
6477  *      Recalculate dev->features set and send notifications even
6478  *      if they have not changed. Should be called instead of
6479  *      netdev_update_features() if also dev->vlan_features might
6480  *      have changed to allow the changes to be propagated to stacked
6481  *      VLAN devices.
6482  */
6483 void netdev_change_features(struct net_device *dev)
6484 {
6485         __netdev_update_features(dev);
6486         netdev_features_change(dev);
6487 }
6488 EXPORT_SYMBOL(netdev_change_features);
6489
6490 /**
6491  *      netif_stacked_transfer_operstate -      transfer operstate
6492  *      @rootdev: the root or lower level device to transfer state from
6493  *      @dev: the device to transfer operstate to
6494  *
6495  *      Transfer operational state from root to device. This is normally
6496  *      called when a stacking relationship exists between the root
6497  *      device and the device(a leaf device).
6498  */
6499 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6500                                         struct net_device *dev)
6501 {
6502         if (rootdev->operstate == IF_OPER_DORMANT)
6503                 netif_dormant_on(dev);
6504         else
6505                 netif_dormant_off(dev);
6506
6507         if (netif_carrier_ok(rootdev)) {
6508                 if (!netif_carrier_ok(dev))
6509                         netif_carrier_on(dev);
6510         } else {
6511                 if (netif_carrier_ok(dev))
6512                         netif_carrier_off(dev);
6513         }
6514 }
6515 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6516
6517 #ifdef CONFIG_SYSFS
6518 static int netif_alloc_rx_queues(struct net_device *dev)
6519 {
6520         unsigned int i, count = dev->num_rx_queues;
6521         struct netdev_rx_queue *rx;
6522         size_t sz = count * sizeof(*rx);
6523
6524         BUG_ON(count < 1);
6525
6526         rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6527         if (!rx) {
6528                 rx = vzalloc(sz);
6529                 if (!rx)
6530                         return -ENOMEM;
6531         }
6532         dev->_rx = rx;
6533
6534         for (i = 0; i < count; i++)
6535                 rx[i].dev = dev;
6536         return 0;
6537 }
6538 #endif
6539
6540 static void netdev_init_one_queue(struct net_device *dev,
6541                                   struct netdev_queue *queue, void *_unused)
6542 {
6543         /* Initialize queue lock */
6544         spin_lock_init(&queue->_xmit_lock);
6545         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6546         queue->xmit_lock_owner = -1;
6547         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
6548         queue->dev = dev;
6549 #ifdef CONFIG_BQL
6550         dql_init(&queue->dql, HZ);
6551 #endif
6552 }
6553
6554 static void netif_free_tx_queues(struct net_device *dev)
6555 {
6556         kvfree(dev->_tx);
6557 }
6558
6559 static int netif_alloc_netdev_queues(struct net_device *dev)
6560 {
6561         unsigned int count = dev->num_tx_queues;
6562         struct netdev_queue *tx;
6563         size_t sz = count * sizeof(*tx);
6564
6565         if (count < 1 || count > 0xffff)
6566                 return -EINVAL;
6567
6568         tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6569         if (!tx) {
6570                 tx = vzalloc(sz);
6571                 if (!tx)
6572                         return -ENOMEM;
6573         }
6574         dev->_tx = tx;
6575
6576         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6577         spin_lock_init(&dev->tx_global_lock);
6578
6579         return 0;
6580 }
6581
6582 void netif_tx_stop_all_queues(struct net_device *dev)
6583 {
6584         unsigned int i;
6585
6586         for (i = 0; i < dev->num_tx_queues; i++) {
6587                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6588                 netif_tx_stop_queue(txq);
6589         }
6590 }
6591 EXPORT_SYMBOL(netif_tx_stop_all_queues);
6592
6593 /**
6594  *      register_netdevice      - register a network device
6595  *      @dev: device to register
6596  *
6597  *      Take a completed network device structure and add it to the kernel
6598  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6599  *      chain. 0 is returned on success. A negative errno code is returned
6600  *      on a failure to set up the device, or if the name is a duplicate.
6601  *
6602  *      Callers must hold the rtnl semaphore. You may want
6603  *      register_netdev() instead of this.
6604  *
6605  *      BUGS:
6606  *      The locking appears insufficient to guarantee two parallel registers
6607  *      will not get the same name.
6608  */
6609
6610 int register_netdevice(struct net_device *dev)
6611 {
6612         int ret;
6613         struct net *net = dev_net(dev);
6614
6615         BUG_ON(dev_boot_phase);
6616         ASSERT_RTNL();
6617
6618         might_sleep();
6619
6620         /* When net_device's are persistent, this will be fatal. */
6621         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
6622         BUG_ON(!net);
6623
6624         spin_lock_init(&dev->addr_list_lock);
6625         netdev_set_addr_lockdep_class(dev);
6626
6627         ret = dev_get_valid_name(net, dev, dev->name);
6628         if (ret < 0)
6629                 goto out;
6630
6631         /* Init, if this function is available */
6632         if (dev->netdev_ops->ndo_init) {
6633                 ret = dev->netdev_ops->ndo_init(dev);
6634                 if (ret) {
6635                         if (ret > 0)
6636                                 ret = -EIO;
6637                         goto out;
6638                 }
6639         }
6640
6641         if (((dev->hw_features | dev->features) &
6642              NETIF_F_HW_VLAN_CTAG_FILTER) &&
6643             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6644              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6645                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6646                 ret = -EINVAL;
6647                 goto err_uninit;
6648         }
6649
6650         ret = -EBUSY;
6651         if (!dev->ifindex)
6652                 dev->ifindex = dev_new_index(net);
6653         else if (__dev_get_by_index(net, dev->ifindex))
6654                 goto err_uninit;
6655
6656         /* Transfer changeable features to wanted_features and enable
6657          * software offloads (GSO and GRO).
6658          */
6659         dev->hw_features |= NETIF_F_SOFT_FEATURES;
6660         dev->features |= NETIF_F_SOFT_FEATURES;
6661         dev->wanted_features = dev->features & dev->hw_features;
6662
6663         if (!(dev->flags & IFF_LOOPBACK)) {
6664                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6665         }
6666
6667         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6668          */
6669         dev->vlan_features |= NETIF_F_HIGHDMA;
6670
6671         /* Make NETIF_F_SG inheritable to tunnel devices.
6672          */
6673         dev->hw_enc_features |= NETIF_F_SG;
6674
6675         /* Make NETIF_F_SG inheritable to MPLS.
6676          */
6677         dev->mpls_features |= NETIF_F_SG;
6678
6679         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6680         ret = notifier_to_errno(ret);
6681         if (ret)
6682                 goto err_uninit;
6683
6684         ret = netdev_register_kobject(dev);
6685         if (ret)
6686                 goto err_uninit;
6687         dev->reg_state = NETREG_REGISTERED;
6688
6689         __netdev_update_features(dev);
6690
6691         /*
6692          *      Default initial state at registry is that the
6693          *      device is present.
6694          */
6695
6696         set_bit(__LINK_STATE_PRESENT, &dev->state);
6697
6698         linkwatch_init_dev(dev);
6699
6700         dev_init_scheduler(dev);
6701         dev_hold(dev);
6702         list_netdevice(dev);
6703         add_device_randomness(dev->dev_addr, dev->addr_len);
6704
6705         /* If the device has permanent device address, driver should
6706          * set dev_addr and also addr_assign_type should be set to
6707          * NET_ADDR_PERM (default value).
6708          */
6709         if (dev->addr_assign_type == NET_ADDR_PERM)
6710                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
6711
6712         /* Notify protocols, that a new device appeared. */
6713         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
6714         ret = notifier_to_errno(ret);
6715         if (ret) {
6716                 rollback_registered(dev);
6717                 dev->reg_state = NETREG_UNREGISTERED;
6718         }
6719         /*
6720          *      Prevent userspace races by waiting until the network
6721          *      device is fully setup before sending notifications.
6722          */
6723         if (!dev->rtnl_link_ops ||
6724             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6725                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
6726
6727 out:
6728         return ret;
6729
6730 err_uninit:
6731         if (dev->netdev_ops->ndo_uninit)
6732                 dev->netdev_ops->ndo_uninit(dev);
6733         goto out;
6734 }
6735 EXPORT_SYMBOL(register_netdevice);
6736
6737 /**
6738  *      init_dummy_netdev       - init a dummy network device for NAPI
6739  *      @dev: device to init
6740  *
6741  *      This takes a network device structure and initialize the minimum
6742  *      amount of fields so it can be used to schedule NAPI polls without
6743  *      registering a full blown interface. This is to be used by drivers
6744  *      that need to tie several hardware interfaces to a single NAPI
6745  *      poll scheduler due to HW limitations.
6746  */
6747 int init_dummy_netdev(struct net_device *dev)
6748 {
6749         /* Clear everything. Note we don't initialize spinlocks
6750          * are they aren't supposed to be taken by any of the
6751          * NAPI code and this dummy netdev is supposed to be
6752          * only ever used for NAPI polls
6753          */
6754         memset(dev, 0, sizeof(struct net_device));
6755
6756         /* make sure we BUG if trying to hit standard
6757          * register/unregister code path
6758          */
6759         dev->reg_state = NETREG_DUMMY;
6760
6761         /* NAPI wants this */
6762         INIT_LIST_HEAD(&dev->napi_list);
6763
6764         /* a dummy interface is started by default */
6765         set_bit(__LINK_STATE_PRESENT, &dev->state);
6766         set_bit(__LINK_STATE_START, &dev->state);
6767
6768         /* Note : We dont allocate pcpu_refcnt for dummy devices,
6769          * because users of this 'device' dont need to change
6770          * its refcount.
6771          */
6772
6773         return 0;
6774 }
6775 EXPORT_SYMBOL_GPL(init_dummy_netdev);
6776
6777
6778 /**
6779  *      register_netdev - register a network device
6780  *      @dev: device to register
6781  *
6782  *      Take a completed network device structure and add it to the kernel
6783  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6784  *      chain. 0 is returned on success. A negative errno code is returned
6785  *      on a failure to set up the device, or if the name is a duplicate.
6786  *
6787  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
6788  *      and expands the device name if you passed a format string to
6789  *      alloc_netdev.
6790  */
6791 int register_netdev(struct net_device *dev)
6792 {
6793         int err;
6794
6795         rtnl_lock();
6796         err = register_netdevice(dev);
6797         rtnl_unlock();
6798         return err;
6799 }
6800 EXPORT_SYMBOL(register_netdev);
6801
6802 int netdev_refcnt_read(const struct net_device *dev)
6803 {
6804         int i, refcnt = 0;
6805
6806         for_each_possible_cpu(i)
6807                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
6808         return refcnt;
6809 }
6810 EXPORT_SYMBOL(netdev_refcnt_read);
6811
6812 /**
6813  * netdev_wait_allrefs - wait until all references are gone.
6814  * @dev: target net_device
6815  *
6816  * This is called when unregistering network devices.
6817  *
6818  * Any protocol or device that holds a reference should register
6819  * for netdevice notification, and cleanup and put back the
6820  * reference if they receive an UNREGISTER event.
6821  * We can get stuck here if buggy protocols don't correctly
6822  * call dev_put.
6823  */
6824 static void netdev_wait_allrefs(struct net_device *dev)
6825 {
6826         unsigned long rebroadcast_time, warning_time;
6827         int refcnt;
6828
6829         linkwatch_forget_dev(dev);
6830
6831         rebroadcast_time = warning_time = jiffies;
6832         refcnt = netdev_refcnt_read(dev);
6833
6834         while (refcnt != 0) {
6835                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
6836                         rtnl_lock();
6837
6838                         /* Rebroadcast unregister notification */
6839                         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6840
6841                         __rtnl_unlock();
6842                         rcu_barrier();
6843                         rtnl_lock();
6844
6845                         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6846                         if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
6847                                      &dev->state)) {
6848                                 /* We must not have linkwatch events
6849                                  * pending on unregister. If this
6850                                  * happens, we simply run the queue
6851                                  * unscheduled, resulting in a noop
6852                                  * for this device.
6853                                  */
6854                                 linkwatch_run_queue();
6855                         }
6856
6857                         __rtnl_unlock();
6858
6859                         rebroadcast_time = jiffies;
6860                 }
6861
6862                 msleep(250);
6863
6864                 refcnt = netdev_refcnt_read(dev);
6865
6866                 if (time_after(jiffies, warning_time + 10 * HZ)) {
6867                         pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
6868                                  dev->name, refcnt);
6869                         warning_time = jiffies;
6870                 }
6871         }
6872 }
6873
6874 /* The sequence is:
6875  *
6876  *      rtnl_lock();
6877  *      ...
6878  *      register_netdevice(x1);
6879  *      register_netdevice(x2);
6880  *      ...
6881  *      unregister_netdevice(y1);
6882  *      unregister_netdevice(y2);
6883  *      ...
6884  *      rtnl_unlock();
6885  *      free_netdev(y1);
6886  *      free_netdev(y2);
6887  *
6888  * We are invoked by rtnl_unlock().
6889  * This allows us to deal with problems:
6890  * 1) We can delete sysfs objects which invoke hotplug
6891  *    without deadlocking with linkwatch via keventd.
6892  * 2) Since we run with the RTNL semaphore not held, we can sleep
6893  *    safely in order to wait for the netdev refcnt to drop to zero.
6894  *
6895  * We must not return until all unregister events added during
6896  * the interval the lock was held have been completed.
6897  */
6898 void netdev_run_todo(void)
6899 {
6900         struct list_head list;
6901
6902         /* Snapshot list, allow later requests */
6903         list_replace_init(&net_todo_list, &list);
6904
6905         __rtnl_unlock();
6906
6907
6908         /* Wait for rcu callbacks to finish before next phase */
6909         if (!list_empty(&list))
6910                 rcu_barrier();
6911
6912         while (!list_empty(&list)) {
6913                 struct net_device *dev
6914                         = list_first_entry(&list, struct net_device, todo_list);
6915                 list_del(&dev->todo_list);
6916
6917                 rtnl_lock();
6918                 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
6919                 __rtnl_unlock();
6920
6921                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
6922                         pr_err("network todo '%s' but state %d\n",
6923                                dev->name, dev->reg_state);
6924                         dump_stack();
6925                         continue;
6926                 }
6927
6928                 dev->reg_state = NETREG_UNREGISTERED;
6929
6930                 netdev_wait_allrefs(dev);
6931
6932                 /* paranoia */
6933                 BUG_ON(netdev_refcnt_read(dev));
6934                 BUG_ON(!list_empty(&dev->ptype_all));
6935                 BUG_ON(!list_empty(&dev->ptype_specific));
6936                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
6937                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
6938                 WARN_ON(dev->dn_ptr);
6939
6940                 if (dev->destructor)
6941                         dev->destructor(dev);
6942
6943                 /* Report a network device has been unregistered */
6944                 rtnl_lock();
6945                 dev_net(dev)->dev_unreg_count--;
6946                 __rtnl_unlock();
6947                 wake_up(&netdev_unregistering_wq);
6948
6949                 /* Free network device */
6950                 kobject_put(&dev->dev.kobj);
6951         }
6952 }
6953
6954 /* Convert net_device_stats to rtnl_link_stats64.  They have the same
6955  * fields in the same order, with only the type differing.
6956  */
6957 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
6958                              const struct net_device_stats *netdev_stats)
6959 {
6960 #if BITS_PER_LONG == 64
6961         BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
6962         memcpy(stats64, netdev_stats, sizeof(*stats64));
6963 #else
6964         size_t i, n = sizeof(*stats64) / sizeof(u64);
6965         const unsigned long *src = (const unsigned long *)netdev_stats;
6966         u64 *dst = (u64 *)stats64;
6967
6968         BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
6969                      sizeof(*stats64) / sizeof(u64));
6970         for (i = 0; i < n; i++)
6971                 dst[i] = src[i];
6972 #endif
6973 }
6974 EXPORT_SYMBOL(netdev_stats_to_stats64);
6975
6976 /**
6977  *      dev_get_stats   - get network device statistics
6978  *      @dev: device to get statistics from
6979  *      @storage: place to store stats
6980  *
6981  *      Get network statistics from device. Return @storage.
6982  *      The device driver may provide its own method by setting
6983  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
6984  *      otherwise the internal statistics structure is used.
6985  */
6986 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
6987                                         struct rtnl_link_stats64 *storage)
6988 {
6989         const struct net_device_ops *ops = dev->netdev_ops;
6990
6991         if (ops->ndo_get_stats64) {
6992                 memset(storage, 0, sizeof(*storage));
6993                 ops->ndo_get_stats64(dev, storage);
6994         } else if (ops->ndo_get_stats) {
6995                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
6996         } else {
6997                 netdev_stats_to_stats64(storage, &dev->stats);
6998         }
6999         storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7000         storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7001         return storage;
7002 }
7003 EXPORT_SYMBOL(dev_get_stats);
7004
7005 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7006 {
7007         struct netdev_queue *queue = dev_ingress_queue(dev);
7008
7009 #ifdef CONFIG_NET_CLS_ACT
7010         if (queue)
7011                 return queue;
7012         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7013         if (!queue)
7014                 return NULL;
7015         netdev_init_one_queue(dev, queue, NULL);
7016         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7017         queue->qdisc_sleeping = &noop_qdisc;
7018         rcu_assign_pointer(dev->ingress_queue, queue);
7019 #endif
7020         return queue;
7021 }
7022
7023 static const struct ethtool_ops default_ethtool_ops;
7024
7025 void netdev_set_default_ethtool_ops(struct net_device *dev,
7026                                     const struct ethtool_ops *ops)
7027 {
7028         if (dev->ethtool_ops == &default_ethtool_ops)
7029                 dev->ethtool_ops = ops;
7030 }
7031 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7032
7033 void netdev_freemem(struct net_device *dev)
7034 {
7035         char *addr = (char *)dev - dev->padded;
7036
7037         kvfree(addr);
7038 }
7039
7040 /**
7041  *      alloc_netdev_mqs - allocate network device
7042  *      @sizeof_priv:           size of private data to allocate space for
7043  *      @name:                  device name format string
7044  *      @name_assign_type:      origin of device name
7045  *      @setup:                 callback to initialize device
7046  *      @txqs:                  the number of TX subqueues to allocate
7047  *      @rxqs:                  the number of RX subqueues to allocate
7048  *
7049  *      Allocates a struct net_device with private data area for driver use
7050  *      and performs basic initialization.  Also allocates subqueue structs
7051  *      for each queue on the device.
7052  */
7053 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7054                 unsigned char name_assign_type,
7055                 void (*setup)(struct net_device *),
7056                 unsigned int txqs, unsigned int rxqs)
7057 {
7058         struct net_device *dev;
7059         size_t alloc_size;
7060         struct net_device *p;
7061
7062         BUG_ON(strlen(name) >= sizeof(dev->name));
7063
7064         if (txqs < 1) {
7065                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7066                 return NULL;
7067         }
7068
7069 #ifdef CONFIG_SYSFS
7070         if (rxqs < 1) {
7071                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7072                 return NULL;
7073         }
7074 #endif
7075
7076         alloc_size = sizeof(struct net_device);
7077         if (sizeof_priv) {
7078                 /* ensure 32-byte alignment of private area */
7079                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7080                 alloc_size += sizeof_priv;
7081         }
7082         /* ensure 32-byte alignment of whole construct */
7083         alloc_size += NETDEV_ALIGN - 1;
7084
7085         p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7086         if (!p)
7087                 p = vzalloc(alloc_size);
7088         if (!p)
7089                 return NULL;
7090
7091         dev = PTR_ALIGN(p, NETDEV_ALIGN);
7092         dev->padded = (char *)dev - (char *)p;
7093
7094         dev->pcpu_refcnt = alloc_percpu(int);
7095         if (!dev->pcpu_refcnt)
7096                 goto free_dev;
7097
7098         if (dev_addr_init(dev))
7099                 goto free_pcpu;
7100
7101         dev_mc_init(dev);
7102         dev_uc_init(dev);
7103
7104         dev_net_set(dev, &init_net);
7105
7106         dev->gso_max_size = GSO_MAX_SIZE;
7107         dev->gso_max_segs = GSO_MAX_SEGS;
7108         dev->gso_min_segs = 0;
7109
7110         INIT_LIST_HEAD(&dev->napi_list);
7111         INIT_LIST_HEAD(&dev->unreg_list);
7112         INIT_LIST_HEAD(&dev->close_list);
7113         INIT_LIST_HEAD(&dev->link_watch_list);
7114         INIT_LIST_HEAD(&dev->adj_list.upper);
7115         INIT_LIST_HEAD(&dev->adj_list.lower);
7116         INIT_LIST_HEAD(&dev->all_adj_list.upper);
7117         INIT_LIST_HEAD(&dev->all_adj_list.lower);
7118         INIT_LIST_HEAD(&dev->ptype_all);
7119         INIT_LIST_HEAD(&dev->ptype_specific);
7120         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7121         setup(dev);
7122
7123         if (!dev->tx_queue_len)
7124                 dev->priv_flags |= IFF_NO_QUEUE;
7125
7126         dev->num_tx_queues = txqs;
7127         dev->real_num_tx_queues = txqs;
7128         if (netif_alloc_netdev_queues(dev))
7129                 goto free_all;
7130
7131 #ifdef CONFIG_SYSFS
7132         dev->num_rx_queues = rxqs;
7133         dev->real_num_rx_queues = rxqs;
7134         if (netif_alloc_rx_queues(dev))
7135                 goto free_all;
7136 #endif
7137
7138         strcpy(dev->name, name);
7139         dev->name_assign_type = name_assign_type;
7140         dev->group = INIT_NETDEV_GROUP;
7141         if (!dev->ethtool_ops)
7142                 dev->ethtool_ops = &default_ethtool_ops;
7143
7144         nf_hook_ingress_init(dev);
7145
7146         return dev;
7147
7148 free_all:
7149         free_netdev(dev);
7150         return NULL;
7151
7152 free_pcpu:
7153         free_percpu(dev->pcpu_refcnt);
7154 free_dev:
7155         netdev_freemem(dev);
7156         return NULL;
7157 }
7158 EXPORT_SYMBOL(alloc_netdev_mqs);
7159
7160 /**
7161  *      free_netdev - free network device
7162  *      @dev: device
7163  *
7164  *      This function does the last stage of destroying an allocated device
7165  *      interface. The reference to the device object is released.
7166  *      If this is the last reference then it will be freed.
7167  */
7168 void free_netdev(struct net_device *dev)
7169 {
7170         struct napi_struct *p, *n;
7171
7172         netif_free_tx_queues(dev);
7173 #ifdef CONFIG_SYSFS
7174         kvfree(dev->_rx);
7175 #endif
7176
7177         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7178
7179         /* Flush device addresses */
7180         dev_addr_flush(dev);
7181
7182         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7183                 netif_napi_del(p);
7184
7185         free_percpu(dev->pcpu_refcnt);
7186         dev->pcpu_refcnt = NULL;
7187
7188         /*  Compatibility with error handling in drivers */
7189         if (dev->reg_state == NETREG_UNINITIALIZED) {
7190                 netdev_freemem(dev);
7191                 return;
7192         }
7193
7194         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7195         dev->reg_state = NETREG_RELEASED;
7196
7197         /* will free via device release */
7198         put_device(&dev->dev);
7199 }
7200 EXPORT_SYMBOL(free_netdev);
7201
7202 /**
7203  *      synchronize_net -  Synchronize with packet receive processing
7204  *
7205  *      Wait for packets currently being received to be done.
7206  *      Does not block later packets from starting.
7207  */
7208 void synchronize_net(void)
7209 {
7210         might_sleep();
7211         if (rtnl_is_locked())
7212                 synchronize_rcu_expedited();
7213         else
7214                 synchronize_rcu();
7215 }
7216 EXPORT_SYMBOL(synchronize_net);
7217
7218 /**
7219  *      unregister_netdevice_queue - remove device from the kernel
7220  *      @dev: device
7221  *      @head: list
7222  *
7223  *      This function shuts down a device interface and removes it
7224  *      from the kernel tables.
7225  *      If head not NULL, device is queued to be unregistered later.
7226  *
7227  *      Callers must hold the rtnl semaphore.  You may want
7228  *      unregister_netdev() instead of this.
7229  */
7230
7231 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7232 {
7233         ASSERT_RTNL();
7234
7235         if (head) {
7236                 list_move_tail(&dev->unreg_list, head);
7237         } else {
7238                 rollback_registered(dev);
7239                 /* Finish processing unregister after unlock */
7240                 net_set_todo(dev);
7241         }
7242 }
7243 EXPORT_SYMBOL(unregister_netdevice_queue);
7244
7245 /**
7246  *      unregister_netdevice_many - unregister many devices
7247  *      @head: list of devices
7248  *
7249  *  Note: As most callers use a stack allocated list_head,
7250  *  we force a list_del() to make sure stack wont be corrupted later.
7251  */
7252 void unregister_netdevice_many(struct list_head *head)
7253 {
7254         struct net_device *dev;
7255
7256         if (!list_empty(head)) {
7257                 rollback_registered_many(head);
7258                 list_for_each_entry(dev, head, unreg_list)
7259                         net_set_todo(dev);
7260                 list_del(head);
7261         }
7262 }
7263 EXPORT_SYMBOL(unregister_netdevice_many);
7264
7265 /**
7266  *      unregister_netdev - remove device from the kernel
7267  *      @dev: device
7268  *
7269  *      This function shuts down a device interface and removes it
7270  *      from the kernel tables.
7271  *
7272  *      This is just a wrapper for unregister_netdevice that takes
7273  *      the rtnl semaphore.  In general you want to use this and not
7274  *      unregister_netdevice.
7275  */
7276 void unregister_netdev(struct net_device *dev)
7277 {
7278         rtnl_lock();
7279         unregister_netdevice(dev);
7280         rtnl_unlock();
7281 }
7282 EXPORT_SYMBOL(unregister_netdev);
7283
7284 /**
7285  *      dev_change_net_namespace - move device to different nethost namespace
7286  *      @dev: device
7287  *      @net: network namespace
7288  *      @pat: If not NULL name pattern to try if the current device name
7289  *            is already taken in the destination network namespace.
7290  *
7291  *      This function shuts down a device interface and moves it
7292  *      to a new network namespace. On success 0 is returned, on
7293  *      a failure a netagive errno code is returned.
7294  *
7295  *      Callers must hold the rtnl semaphore.
7296  */
7297
7298 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7299 {
7300         int err;
7301
7302         ASSERT_RTNL();
7303
7304         /* Don't allow namespace local devices to be moved. */
7305         err = -EINVAL;
7306         if (dev->features & NETIF_F_NETNS_LOCAL)
7307                 goto out;
7308
7309         /* Ensure the device has been registrered */
7310         if (dev->reg_state != NETREG_REGISTERED)
7311                 goto out;
7312
7313         /* Get out if there is nothing todo */
7314         err = 0;
7315         if (net_eq(dev_net(dev), net))
7316                 goto out;
7317
7318         /* Pick the destination device name, and ensure
7319          * we can use it in the destination network namespace.
7320          */
7321         err = -EEXIST;
7322         if (__dev_get_by_name(net, dev->name)) {
7323                 /* We get here if we can't use the current device name */
7324                 if (!pat)
7325                         goto out;
7326                 if (dev_get_valid_name(net, dev, pat) < 0)
7327                         goto out;
7328         }
7329
7330         /*
7331          * And now a mini version of register_netdevice unregister_netdevice.
7332          */
7333
7334         /* If device is running close it first. */
7335         dev_close(dev);
7336
7337         /* And unlink it from device chain */
7338         err = -ENODEV;
7339         unlist_netdevice(dev);
7340
7341         synchronize_net();
7342
7343         /* Shutdown queueing discipline. */
7344         dev_shutdown(dev);
7345
7346         /* Notify protocols, that we are about to destroy
7347            this device. They should clean all the things.
7348
7349            Note that dev->reg_state stays at NETREG_REGISTERED.
7350            This is wanted because this way 8021q and macvlan know
7351            the device is just moving and can keep their slaves up.
7352         */
7353         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7354         rcu_barrier();
7355         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7356         rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7357
7358         /*
7359          *      Flush the unicast and multicast chains
7360          */
7361         dev_uc_flush(dev);
7362         dev_mc_flush(dev);
7363
7364         /* Send a netdev-removed uevent to the old namespace */
7365         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7366         netdev_adjacent_del_links(dev);
7367
7368         /* Actually switch the network namespace */
7369         dev_net_set(dev, net);
7370
7371         /* If there is an ifindex conflict assign a new one */
7372         if (__dev_get_by_index(net, dev->ifindex))
7373                 dev->ifindex = dev_new_index(net);
7374
7375         /* Send a netdev-add uevent to the new namespace */
7376         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7377         netdev_adjacent_add_links(dev);
7378
7379         /* Fixup kobjects */
7380         err = device_rename(&dev->dev, dev->name);
7381         WARN_ON(err);
7382
7383         /* Add the device back in the hashes */
7384         list_netdevice(dev);
7385
7386         /* Notify protocols, that a new device appeared. */
7387         call_netdevice_notifiers(NETDEV_REGISTER, dev);
7388
7389         /*
7390          *      Prevent userspace races by waiting until the network
7391          *      device is fully setup before sending notifications.
7392          */
7393         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7394
7395         synchronize_net();
7396         err = 0;
7397 out:
7398         return err;
7399 }
7400 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7401
7402 static int dev_cpu_callback(struct notifier_block *nfb,
7403                             unsigned long action,
7404                             void *ocpu)
7405 {
7406         struct sk_buff **list_skb;
7407         struct sk_buff *skb;
7408         unsigned int cpu, oldcpu = (unsigned long)ocpu;
7409         struct softnet_data *sd, *oldsd;
7410
7411         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
7412                 return NOTIFY_OK;
7413
7414         local_irq_disable();
7415         cpu = smp_processor_id();
7416         sd = &per_cpu(softnet_data, cpu);
7417         oldsd = &per_cpu(softnet_data, oldcpu);
7418
7419         /* Find end of our completion_queue. */
7420         list_skb = &sd->completion_queue;
7421         while (*list_skb)
7422                 list_skb = &(*list_skb)->next;
7423         /* Append completion queue from offline CPU. */
7424         *list_skb = oldsd->completion_queue;
7425         oldsd->completion_queue = NULL;
7426
7427         /* Append output queue from offline CPU. */
7428         if (oldsd->output_queue) {
7429                 *sd->output_queue_tailp = oldsd->output_queue;
7430                 sd->output_queue_tailp = oldsd->output_queue_tailp;
7431                 oldsd->output_queue = NULL;
7432                 oldsd->output_queue_tailp = &oldsd->output_queue;
7433         }
7434         /* Append NAPI poll list from offline CPU, with one exception :
7435          * process_backlog() must be called by cpu owning percpu backlog.
7436          * We properly handle process_queue & input_pkt_queue later.
7437          */
7438         while (!list_empty(&oldsd->poll_list)) {
7439                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7440                                                             struct napi_struct,
7441                                                             poll_list);
7442
7443                 list_del_init(&napi->poll_list);
7444                 if (napi->poll == process_backlog)
7445                         napi->state = 0;
7446                 else
7447                         ____napi_schedule(sd, napi);
7448         }
7449
7450         raise_softirq_irqoff(NET_TX_SOFTIRQ);
7451         local_irq_enable();
7452
7453         /* Process offline CPU's input_pkt_queue */
7454         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7455                 netif_rx_ni(skb);
7456                 input_queue_head_incr(oldsd);
7457         }
7458         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7459                 netif_rx_ni(skb);
7460                 input_queue_head_incr(oldsd);
7461         }
7462
7463         return NOTIFY_OK;
7464 }
7465
7466
7467 /**
7468  *      netdev_increment_features - increment feature set by one
7469  *      @all: current feature set
7470  *      @one: new feature set
7471  *      @mask: mask feature set
7472  *
7473  *      Computes a new feature set after adding a device with feature set
7474  *      @one to the master device with current feature set @all.  Will not
7475  *      enable anything that is off in @mask. Returns the new feature set.
7476  */
7477 netdev_features_t netdev_increment_features(netdev_features_t all,
7478         netdev_features_t one, netdev_features_t mask)
7479 {
7480         if (mask & NETIF_F_GEN_CSUM)
7481                 mask |= NETIF_F_ALL_CSUM;
7482         mask |= NETIF_F_VLAN_CHALLENGED;
7483
7484         all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask;
7485         all &= one | ~NETIF_F_ALL_FOR_ALL;
7486
7487         /* If one device supports hw checksumming, set for all. */
7488         if (all & NETIF_F_GEN_CSUM)
7489                 all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM);
7490
7491         return all;
7492 }
7493 EXPORT_SYMBOL(netdev_increment_features);
7494
7495 static struct hlist_head * __net_init netdev_create_hash(void)
7496 {
7497         int i;
7498         struct hlist_head *hash;
7499
7500         hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7501         if (hash != NULL)
7502                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7503                         INIT_HLIST_HEAD(&hash[i]);
7504
7505         return hash;
7506 }
7507
7508 /* Initialize per network namespace state */
7509 static int __net_init netdev_init(struct net *net)
7510 {
7511         if (net != &init_net)
7512                 INIT_LIST_HEAD(&net->dev_base_head);
7513
7514         net->dev_name_head = netdev_create_hash();
7515         if (net->dev_name_head == NULL)
7516                 goto err_name;
7517
7518         net->dev_index_head = netdev_create_hash();
7519         if (net->dev_index_head == NULL)
7520                 goto err_idx;
7521
7522         return 0;
7523
7524 err_idx:
7525         kfree(net->dev_name_head);
7526 err_name:
7527         return -ENOMEM;
7528 }
7529
7530 /**
7531  *      netdev_drivername - network driver for the device
7532  *      @dev: network device
7533  *
7534  *      Determine network driver for device.
7535  */
7536 const char *netdev_drivername(const struct net_device *dev)
7537 {
7538         const struct device_driver *driver;
7539         const struct device *parent;
7540         const char *empty = "";
7541
7542         parent = dev->dev.parent;
7543         if (!parent)
7544                 return empty;
7545
7546         driver = parent->driver;
7547         if (driver && driver->name)
7548                 return driver->name;
7549         return empty;
7550 }
7551
7552 static void __netdev_printk(const char *level, const struct net_device *dev,
7553                             struct va_format *vaf)
7554 {
7555         if (dev && dev->dev.parent) {
7556                 dev_printk_emit(level[1] - '0',
7557                                 dev->dev.parent,
7558                                 "%s %s %s%s: %pV",
7559                                 dev_driver_string(dev->dev.parent),
7560                                 dev_name(dev->dev.parent),
7561                                 netdev_name(dev), netdev_reg_state(dev),
7562                                 vaf);
7563         } else if (dev) {
7564                 printk("%s%s%s: %pV",
7565                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
7566         } else {
7567                 printk("%s(NULL net_device): %pV", level, vaf);
7568         }
7569 }
7570
7571 void netdev_printk(const char *level, const struct net_device *dev,
7572                    const char *format, ...)
7573 {
7574         struct va_format vaf;
7575         va_list args;
7576
7577         va_start(args, format);
7578
7579         vaf.fmt = format;
7580         vaf.va = &args;
7581
7582         __netdev_printk(level, dev, &vaf);
7583
7584         va_end(args);
7585 }
7586 EXPORT_SYMBOL(netdev_printk);
7587
7588 #define define_netdev_printk_level(func, level)                 \
7589 void func(const struct net_device *dev, const char *fmt, ...)   \
7590 {                                                               \
7591         struct va_format vaf;                                   \
7592         va_list args;                                           \
7593                                                                 \
7594         va_start(args, fmt);                                    \
7595                                                                 \
7596         vaf.fmt = fmt;                                          \
7597         vaf.va = &args;                                         \
7598                                                                 \
7599         __netdev_printk(level, dev, &vaf);                      \
7600                                                                 \
7601         va_end(args);                                           \
7602 }                                                               \
7603 EXPORT_SYMBOL(func);
7604
7605 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7606 define_netdev_printk_level(netdev_alert, KERN_ALERT);
7607 define_netdev_printk_level(netdev_crit, KERN_CRIT);
7608 define_netdev_printk_level(netdev_err, KERN_ERR);
7609 define_netdev_printk_level(netdev_warn, KERN_WARNING);
7610 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7611 define_netdev_printk_level(netdev_info, KERN_INFO);
7612
7613 static void __net_exit netdev_exit(struct net *net)
7614 {
7615         kfree(net->dev_name_head);
7616         kfree(net->dev_index_head);
7617 }
7618
7619 static struct pernet_operations __net_initdata netdev_net_ops = {
7620         .init = netdev_init,
7621         .exit = netdev_exit,
7622 };
7623
7624 static void __net_exit default_device_exit(struct net *net)
7625 {
7626         struct net_device *dev, *aux;
7627         /*
7628          * Push all migratable network devices back to the
7629          * initial network namespace
7630          */
7631         rtnl_lock();
7632         for_each_netdev_safe(net, dev, aux) {
7633                 int err;
7634                 char fb_name[IFNAMSIZ];
7635
7636                 /* Ignore unmoveable devices (i.e. loopback) */
7637                 if (dev->features & NETIF_F_NETNS_LOCAL)
7638                         continue;
7639
7640                 /* Leave virtual devices for the generic cleanup */
7641                 if (dev->rtnl_link_ops)
7642                         continue;
7643
7644                 /* Push remaining network devices to init_net */
7645                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7646                 err = dev_change_net_namespace(dev, &init_net, fb_name);
7647                 if (err) {
7648                         pr_emerg("%s: failed to move %s to init_net: %d\n",
7649                                  __func__, dev->name, err);
7650                         BUG();
7651                 }
7652         }
7653         rtnl_unlock();
7654 }
7655
7656 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7657 {
7658         /* Return with the rtnl_lock held when there are no network
7659          * devices unregistering in any network namespace in net_list.
7660          */
7661         struct net *net;
7662         bool unregistering;
7663         DEFINE_WAIT_FUNC(wait, woken_wake_function);
7664
7665         add_wait_queue(&netdev_unregistering_wq, &wait);
7666         for (;;) {
7667                 unregistering = false;
7668                 rtnl_lock();
7669                 list_for_each_entry(net, net_list, exit_list) {
7670                         if (net->dev_unreg_count > 0) {
7671                                 unregistering = true;
7672                                 break;
7673                         }
7674                 }
7675                 if (!unregistering)
7676                         break;
7677                 __rtnl_unlock();
7678
7679                 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
7680         }
7681         remove_wait_queue(&netdev_unregistering_wq, &wait);
7682 }
7683
7684 static void __net_exit default_device_exit_batch(struct list_head *net_list)
7685 {
7686         /* At exit all network devices most be removed from a network
7687          * namespace.  Do this in the reverse order of registration.
7688          * Do this across as many network namespaces as possible to
7689          * improve batching efficiency.
7690          */
7691         struct net_device *dev;
7692         struct net *net;
7693         LIST_HEAD(dev_kill_list);
7694
7695         /* To prevent network device cleanup code from dereferencing
7696          * loopback devices or network devices that have been freed
7697          * wait here for all pending unregistrations to complete,
7698          * before unregistring the loopback device and allowing the
7699          * network namespace be freed.
7700          *
7701          * The netdev todo list containing all network devices
7702          * unregistrations that happen in default_device_exit_batch
7703          * will run in the rtnl_unlock() at the end of
7704          * default_device_exit_batch.
7705          */
7706         rtnl_lock_unregistering(net_list);
7707         list_for_each_entry(net, net_list, exit_list) {
7708                 for_each_netdev_reverse(net, dev) {
7709                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
7710                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
7711                         else
7712                                 unregister_netdevice_queue(dev, &dev_kill_list);
7713                 }
7714         }
7715         unregister_netdevice_many(&dev_kill_list);
7716         rtnl_unlock();
7717 }
7718
7719 static struct pernet_operations __net_initdata default_device_ops = {
7720         .exit = default_device_exit,
7721         .exit_batch = default_device_exit_batch,
7722 };
7723
7724 /*
7725  *      Initialize the DEV module. At boot time this walks the device list and
7726  *      unhooks any devices that fail to initialise (normally hardware not
7727  *      present) and leaves us with a valid list of present and active devices.
7728  *
7729  */
7730
7731 /*
7732  *       This is called single threaded during boot, so no need
7733  *       to take the rtnl semaphore.
7734  */
7735 static int __init net_dev_init(void)
7736 {
7737         int i, rc = -ENOMEM;
7738
7739         BUG_ON(!dev_boot_phase);
7740
7741         if (dev_proc_init())
7742                 goto out;
7743
7744         if (netdev_kobject_init())
7745                 goto out;
7746
7747         INIT_LIST_HEAD(&ptype_all);
7748         for (i = 0; i < PTYPE_HASH_SIZE; i++)
7749                 INIT_LIST_HEAD(&ptype_base[i]);
7750
7751         INIT_LIST_HEAD(&offload_base);
7752
7753         if (register_pernet_subsys(&netdev_net_ops))
7754                 goto out;
7755
7756         /*
7757          *      Initialise the packet receive queues.
7758          */
7759
7760         for_each_possible_cpu(i) {
7761                 struct softnet_data *sd = &per_cpu(softnet_data, i);
7762
7763                 skb_queue_head_init(&sd->input_pkt_queue);
7764                 skb_queue_head_init(&sd->process_queue);
7765                 INIT_LIST_HEAD(&sd->poll_list);
7766                 sd->output_queue_tailp = &sd->output_queue;
7767 #ifdef CONFIG_RPS
7768                 sd->csd.func = rps_trigger_softirq;
7769                 sd->csd.info = sd;
7770                 sd->cpu = i;
7771 #endif
7772
7773                 sd->backlog.poll = process_backlog;
7774                 sd->backlog.weight = weight_p;
7775         }
7776
7777         dev_boot_phase = 0;
7778
7779         /* The loopback device is special if any other network devices
7780          * is present in a network namespace the loopback device must
7781          * be present. Since we now dynamically allocate and free the
7782          * loopback device ensure this invariant is maintained by
7783          * keeping the loopback device as the first device on the
7784          * list of network devices.  Ensuring the loopback devices
7785          * is the first device that appears and the last network device
7786          * that disappears.
7787          */
7788         if (register_pernet_device(&loopback_net_ops))
7789                 goto out;
7790
7791         if (register_pernet_device(&default_device_ops))
7792                 goto out;
7793
7794         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
7795         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
7796
7797         hotcpu_notifier(dev_cpu_callback, 0);
7798         dst_subsys_init();
7799         rc = 0;
7800 out:
7801         return rc;
7802 }
7803
7804 subsys_initcall(net_dev_init);