drivers/net/ethernet/intel/igb/igb_main.c

   1 /* Intel(R) Gigabit Ethernet Linux driver
   2  * Copyright(c) 2007-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, see <http://www.gnu.org/licenses/>.
  15  *
  16  * The full GNU General Public License is included in this distribution in
  17  * the file called "COPYING".
  18  *
  19  * Contact Information:
  20  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  21  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  22  */
  23
  24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  25
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <linux/init.h>
  29 #include <linux/bitops.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/pagemap.h>
  32 #include <linux/netdevice.h>
  33 #include <linux/ipv6.h>
  34 #include <linux/slab.h>
  35 #include <net/checksum.h>
  36 #include <net/ip6_checksum.h>
  37 #include <linux/net_tstamp.h>
  38 #include <linux/mii.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/if.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/pci.h>
  43 #include <linux/pci-aspm.h>
  44 #include <linux/delay.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/ip.h>
  47 #include <linux/tcp.h>
  48 #include <linux/sctp.h>
  49 #include <linux/if_ether.h>
  50 #include <linux/aer.h>
  51 #include <linux/prefetch.h>
  52 #include <linux/pm_runtime.h>
  53 #include <linux/etherdevice.h>
  54 #ifdef CONFIG_IGB_DCA
  55 #include <linux/dca.h>
  56 #endif
  57 #include <linux/i2c.h>
  58 #include "igb.h"
  59
  60 #define MAJ 5
  61 #define MIN 4
  62 #define BUILD 0
  63 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  64 __stringify(BUILD) "-k"
  65 char igb_driver_name[] = "igb";
  66 char igb_driver_version[] = DRV_VERSION;
  67 static const char igb_driver_string[] =
  68                                 "Intel(R) Gigabit Ethernet Network Driver";
  69 static const char igb_copyright[] =
  70                                 "Copyright (c) 2007-2014 Intel Corporation.";
  71
  72 static const struct e1000_info *igb_info_tbl[] = {
  73         [board_82575] = &e1000_82575_info,
  74 };
  75
  76 static const struct pci_device_id igb_pci_tbl[] = {
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
  99         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
 100         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
 101         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
 102         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
 103         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
 104         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
 105         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
 106         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
 107         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
 108         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
 109         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
 110         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
 111         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
 112         /* required last entry */
 113         {0, }
 114 };
 115
 116 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 117
 118 static int igb_setup_all_tx_resources(struct igb_adapter *);
 119 static int igb_setup_all_rx_resources(struct igb_adapter *);
 120 static void igb_free_all_tx_resources(struct igb_adapter *);
 121 static void igb_free_all_rx_resources(struct igb_adapter *);
 122 static void igb_setup_mrqc(struct igb_adapter *);
 123 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 124 static void igb_remove(struct pci_dev *pdev);
 125 static int igb_sw_init(struct igb_adapter *);
 126 int igb_open(struct net_device *);
 127 int igb_close(struct net_device *);
 128 static void igb_configure(struct igb_adapter *);
 129 static void igb_configure_tx(struct igb_adapter *);
 130 static void igb_configure_rx(struct igb_adapter *);
 131 static void igb_clean_all_tx_rings(struct igb_adapter *);
 132 static void igb_clean_all_rx_rings(struct igb_adapter *);
 133 static void igb_clean_tx_ring(struct igb_ring *);
 134 static void igb_clean_rx_ring(struct igb_ring *);
 135 static void igb_set_rx_mode(struct net_device *);
 136 static void igb_update_phy_info(unsigned long);
 137 static void igb_watchdog(unsigned long);
 138 static void igb_watchdog_task(struct work_struct *);
 139 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 140 static void igb_get_stats64(struct net_device *dev,
 141                             struct rtnl_link_stats64 *stats);
 142 static int igb_change_mtu(struct net_device *, int);
 143 static int igb_set_mac(struct net_device *, void *);
 144 static void igb_set_uta(struct igb_adapter *adapter, bool set);
 145 static irqreturn_t igb_intr(int irq, void *);
 146 static irqreturn_t igb_intr_msi(int irq, void *);
 147 static irqreturn_t igb_msix_other(int irq, void *);
 148 static irqreturn_t igb_msix_ring(int irq, void *);
 149 #ifdef CONFIG_IGB_DCA
 150 static void igb_update_dca(struct igb_q_vector *);
 151 static void igb_setup_dca(struct igb_adapter *);
 152 #endif /* CONFIG_IGB_DCA */
 153 static int igb_poll(struct napi_struct *, int);
 154 static bool igb_clean_tx_irq(struct igb_q_vector *, int);
 155 static int igb_clean_rx_irq(struct igb_q_vector *, int);
 156 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 157 static void igb_tx_timeout(struct net_device *);
 158 static void igb_reset_task(struct work_struct *);
 159 static void igb_vlan_mode(struct net_device *netdev,
 160                           netdev_features_t features);
 161 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
 162 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
 163 static void igb_restore_vlan(struct igb_adapter *);
 164 static void igb_rar_set_index(struct igb_adapter *, u32);
 165 static void igb_ping_all_vfs(struct igb_adapter *);
 166 static void igb_msg_task(struct igb_adapter *);
 167 static void igb_vmm_control(struct igb_adapter *);
 168 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 169 static void igb_flush_mac_table(struct igb_adapter *);
 170 static int igb_available_rars(struct igb_adapter *, u8);
 171 static void igb_set_default_mac_filter(struct igb_adapter *);
 172 static int igb_uc_sync(struct net_device *, const unsigned char *);
 173 static int igb_uc_unsync(struct net_device *, const unsigned char *);
 174 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 175 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 176 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 177                                int vf, u16 vlan, u8 qos, __be16 vlan_proto);
 178 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
 179 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
 180                                    bool setting);
 181 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 182                                  struct ifla_vf_info *ivi);
 183 static void igb_check_vf_rate_limit(struct igb_adapter *);
 184 static void igb_nfc_filter_exit(struct igb_adapter *adapter);
 185 static void igb_nfc_filter_restore(struct igb_adapter *adapter);
 186
 187 #ifdef CONFIG_PCI_IOV
 188 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 189 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
 190 static int igb_disable_sriov(struct pci_dev *dev);
 191 static int igb_pci_disable_sriov(struct pci_dev *dev);
 192 #endif
 193
 194 static int igb_suspend(struct device *);
 195 static int igb_resume(struct device *);
 196 static int igb_runtime_suspend(struct device *dev);
 197 static int igb_runtime_resume(struct device *dev);
 198 static int igb_runtime_idle(struct device *dev);
 199 static const struct dev_pm_ops igb_pm_ops = {
 200         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 201         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 202                         igb_runtime_idle)
 203 };
 204 static void igb_shutdown(struct pci_dev *);
 205 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
 206 #ifdef CONFIG_IGB_DCA
 207 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 208 static struct notifier_block dca_notifier = {
 209         .notifier_call  = igb_notify_dca,
 210         .next           = NULL,
 211         .priority       = 0
 212 };
 213 #endif
 214 #ifdef CONFIG_NET_POLL_CONTROLLER
 215 /* for netdump / net console */
 216 static void igb_netpoll(struct net_device *);
 217 #endif
 218 #ifdef CONFIG_PCI_IOV
 219 static unsigned int max_vfs;
 220 module_param(max_vfs, uint, 0);
 221 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
 222 #endif /* CONFIG_PCI_IOV */
 223
 224 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 225                      pci_channel_state_t);
 226 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 227 static void igb_io_resume(struct pci_dev *);
 228
 229 static const struct pci_error_handlers igb_err_handler = {
 230         .error_detected = igb_io_error_detected,
 231         .slot_reset = igb_io_slot_reset,
 232         .resume = igb_io_resume,
 233 };
 234
 235 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 236
 237 static struct pci_driver igb_driver = {
 238         .name     = igb_driver_name,
 239         .id_table = igb_pci_tbl,
 240         .probe    = igb_probe,
 241         .remove   = igb_remove,
 242 #ifdef CONFIG_PM
 243         .driver.pm = &igb_pm_ops,
 244 #endif
 245         .shutdown = igb_shutdown,
 246         .sriov_configure = igb_pci_sriov_configure,
 247         .err_handler = &igb_err_handler
 248 };
 249
 250 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 251 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 252 MODULE_LICENSE("GPL");
 253 MODULE_VERSION(DRV_VERSION);
 254
 255 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 256 static int debug = -1;
 257 module_param(debug, int, 0);
 258 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 259
 260 struct igb_reg_info {
 261         u32 ofs;
 262         char *name;
 263 };
 264
 265 static const struct igb_reg_info igb_reg_info_tbl[] = {
 266
 267         /* General Registers */
 268         {E1000_CTRL, "CTRL"},
 269         {E1000_STATUS, "STATUS"},
 270         {E1000_CTRL_EXT, "CTRL_EXT"},
 271
 272         /* Interrupt Registers */
 273         {E1000_ICR, "ICR"},
 274
 275         /* RX Registers */
 276         {E1000_RCTL, "RCTL"},
 277         {E1000_RDLEN(0), "RDLEN"},
 278         {E1000_RDH(0), "RDH"},
 279         {E1000_RDT(0), "RDT"},
 280         {E1000_RXDCTL(0), "RXDCTL"},
 281         {E1000_RDBAL(0), "RDBAL"},
 282         {E1000_RDBAH(0), "RDBAH"},
 283
 284         /* TX Registers */
 285         {E1000_TCTL, "TCTL"},
 286         {E1000_TDBAL(0), "TDBAL"},
 287         {E1000_TDBAH(0), "TDBAH"},
 288         {E1000_TDLEN(0), "TDLEN"},
 289         {E1000_TDH(0), "TDH"},
 290         {E1000_TDT(0), "TDT"},
 291         {E1000_TXDCTL(0), "TXDCTL"},
 292         {E1000_TDFH, "TDFH"},
 293         {E1000_TDFT, "TDFT"},
 294         {E1000_TDFHS, "TDFHS"},
 295         {E1000_TDFPC, "TDFPC"},
 296
 297         /* List Terminator */
 298         {}
 299 };
 300
 301 /* igb_regdump - register printout routine */
 302 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
 303 {
 304         int n = 0;
 305         char rname[16];
 306         u32 regs[8];
 307
 308         switch (reginfo->ofs) {
 309         case E1000_RDLEN(0):
 310                 for (n = 0; n < 4; n++)
 311                         regs[n] = rd32(E1000_RDLEN(n));
 312                 break;
 313         case E1000_RDH(0):
 314                 for (n = 0; n < 4; n++)
 315                         regs[n] = rd32(E1000_RDH(n));
 316                 break;
 317         case E1000_RDT(0):
 318                 for (n = 0; n < 4; n++)
 319                         regs[n] = rd32(E1000_RDT(n));
 320                 break;
 321         case E1000_RXDCTL(0):
 322                 for (n = 0; n < 4; n++)
 323                         regs[n] = rd32(E1000_RXDCTL(n));
 324                 break;
 325         case E1000_RDBAL(0):
 326                 for (n = 0; n < 4; n++)
 327                         regs[n] = rd32(E1000_RDBAL(n));
 328                 break;
 329         case E1000_RDBAH(0):
 330                 for (n = 0; n < 4; n++)
 331                         regs[n] = rd32(E1000_RDBAH(n));
 332                 break;
 333         case E1000_TDBAL(0):
 334                 for (n = 0; n < 4; n++)
 335                         regs[n] = rd32(E1000_RDBAL(n));
 336                 break;
 337         case E1000_TDBAH(0):
 338                 for (n = 0; n < 4; n++)
 339                         regs[n] = rd32(E1000_TDBAH(n));
 340                 break;
 341         case E1000_TDLEN(0):
 342                 for (n = 0; n < 4; n++)
 343                         regs[n] = rd32(E1000_TDLEN(n));
 344                 break;
 345         case E1000_TDH(0):
 346                 for (n = 0; n < 4; n++)
 347                         regs[n] = rd32(E1000_TDH(n));
 348                 break;
 349         case E1000_TDT(0):
 350                 for (n = 0; n < 4; n++)
 351                         regs[n] = rd32(E1000_TDT(n));
 352                 break;
 353         case E1000_TXDCTL(0):
 354                 for (n = 0; n < 4; n++)
 355                         regs[n] = rd32(E1000_TXDCTL(n));
 356                 break;
 357         default:
 358                 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
 359                 return;
 360         }
 361
 362         snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
 363         pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
 364                 regs[2], regs[3]);
 365 }
 366
 367 /* igb_dump - Print registers, Tx-rings and Rx-rings */
 368 static void igb_dump(struct igb_adapter *adapter)
 369 {
 370         struct net_device *netdev = adapter->netdev;
 371         struct e1000_hw *hw = &adapter->hw;
 372         struct igb_reg_info *reginfo;
 373         struct igb_ring *tx_ring;
 374         union e1000_adv_tx_desc *tx_desc;
 375         struct my_u0 { u64 a; u64 b; } *u0;
 376         struct igb_ring *rx_ring;
 377         union e1000_adv_rx_desc *rx_desc;
 378         u32 staterr;
 379         u16 i, n;
 380
 381         if (!netif_msg_hw(adapter))
 382                 return;
 383
 384         /* Print netdevice Info */
 385         if (netdev) {
 386                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 387                 pr_info("Device Name     state            trans_start\n");
 388                 pr_info("%-15s %016lX %016lX\n", netdev->name,
 389                         netdev->state, dev_trans_start(netdev));
 390         }
 391
 392         /* Print Registers */
 393         dev_info(&adapter->pdev->dev, "Register Dump\n");
 394         pr_info(" Register Name   Value\n");
 395         for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
 396              reginfo->name; reginfo++) {
 397                 igb_regdump(hw, reginfo);
 398         }
 399
 400         /* Print TX Ring Summary */
 401         if (!netdev || !netif_running(netdev))
 402                 goto exit;
 403
 404         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 405         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 406         for (n = 0; n < adapter->num_tx_queues; n++) {
 407                 struct igb_tx_buffer *buffer_info;
 408                 tx_ring = adapter->tx_ring[n];
 409                 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 410                 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
 411                         n, tx_ring->next_to_use, tx_ring->next_to_clean,
 412                         (u64)dma_unmap_addr(buffer_info, dma),
 413                         dma_unmap_len(buffer_info, len),
 414                         buffer_info->next_to_watch,
 415                         (u64)buffer_info->time_stamp);
 416         }
 417
 418         /* Print TX Rings */
 419         if (!netif_msg_tx_done(adapter))
 420                 goto rx_ring_summary;
 421
 422         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 423
 424         /* Transmit Descriptor Formats
 425          *
 426          * Advanced Transmit Descriptor
 427          *   +--------------------------------------------------------------+
 428          * 0 |         Buffer Address [63:0]                                |
 429          *   +--------------------------------------------------------------+
 430          * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
 431          *   +--------------------------------------------------------------+
 432          *   63      46 45    40 39 38 36 35 32 31   24             15       0
 433          */
 434
 435         for (n = 0; n < adapter->num_tx_queues; n++) {
 436                 tx_ring = adapter->tx_ring[n];
 437                 pr_info("------------------------------------\n");
 438                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 439                 pr_info("------------------------------------\n");
 440                 pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] [bi->dma       ] leng  ntw timestamp        bi->skb\n");
 441
 442                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 443                         const char *next_desc;
 444                         struct igb_tx_buffer *buffer_info;
 445                         tx_desc = IGB_TX_DESC(tx_ring, i);
 446                         buffer_info = &tx_ring->tx_buffer_info[i];
 447                         u0 = (struct my_u0 *)tx_desc;
 448                         if (i == tx_ring->next_to_use &&
 449                             i == tx_ring->next_to_clean)
 450                                 next_desc = " NTC/U";
 451                         else if (i == tx_ring->next_to_use)
 452                                 next_desc = " NTU";
 453                         else if (i == tx_ring->next_to_clean)
 454                                 next_desc = " NTC";
 455                         else
 456                                 next_desc = "";
 457
 458                         pr_info("T [0x%03X]    %016llX %016llX %016llX %04X  %p %016llX %p%s\n",
 459                                 i, le64_to_cpu(u0->a),
 460                                 le64_to_cpu(u0->b),
 461                                 (u64)dma_unmap_addr(buffer_info, dma),
 462                                 dma_unmap_len(buffer_info, len),
 463                                 buffer_info->next_to_watch,
 464                                 (u64)buffer_info->time_stamp,
 465                                 buffer_info->skb, next_desc);
 466
 467                         if (netif_msg_pktdata(adapter) && buffer_info->skb)
 468                                 print_hex_dump(KERN_INFO, "",
 469                                         DUMP_PREFIX_ADDRESS,
 470                                         16, 1, buffer_info->skb->data,
 471                                         dma_unmap_len(buffer_info, len),
 472                                         true);
 473                 }
 474         }
 475
 476         /* Print RX Rings Summary */
 477 rx_ring_summary:
 478         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 479         pr_info("Queue [NTU] [NTC]\n");
 480         for (n = 0; n < adapter->num_rx_queues; n++) {
 481                 rx_ring = adapter->rx_ring[n];
 482                 pr_info(" %5d %5X %5X\n",
 483                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 484         }
 485
 486         /* Print RX Rings */
 487         if (!netif_msg_rx_status(adapter))
 488                 goto exit;
 489
 490         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 491
 492         /* Advanced Receive Descriptor (Read) Format
 493          *    63                                           1        0
 494          *    +-----------------------------------------------------+
 495          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 496          *    +----------------------------------------------+------+
 497          *  8 |       Header Buffer Address [63:1]           |  DD  |
 498          *    +-----------------------------------------------------+
 499          *
 500          *
 501          * Advanced Receive Descriptor (Write-Back) Format
 502          *
 503          *   63       48 47    32 31  30      21 20 17 16   4 3     0
 504          *   +------------------------------------------------------+
 505          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 506          *   | Checksum   Ident  |   |           |    | Type | Type |
 507          *   +------------------------------------------------------+
 508          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 509          *   +------------------------------------------------------+
 510          *   63       48 47    32 31            20 19               0
 511          */
 512
 513         for (n = 0; n < adapter->num_rx_queues; n++) {
 514                 rx_ring = adapter->rx_ring[n];
 515                 pr_info("------------------------------------\n");
 516                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 517                 pr_info("------------------------------------\n");
 518                 pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] [bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
 519                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
 520
 521                 for (i = 0; i < rx_ring->count; i++) {
 522                         const char *next_desc;
 523                         struct igb_rx_buffer *buffer_info;
 524                         buffer_info = &rx_ring->rx_buffer_info[i];
 525                         rx_desc = IGB_RX_DESC(rx_ring, i);
 526                         u0 = (struct my_u0 *)rx_desc;
 527                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 528
 529                         if (i == rx_ring->next_to_use)
 530                                 next_desc = " NTU";
 531                         else if (i == rx_ring->next_to_clean)
 532                                 next_desc = " NTC";
 533                         else
 534                                 next_desc = "";
 535
 536                         if (staterr & E1000_RXD_STAT_DD) {
 537                                 /* Descriptor Done */
 538                                 pr_info("%s[0x%03X]     %016llX %016llX ---------------- %s\n",
 539                                         "RWB", i,
 540                                         le64_to_cpu(u0->a),
 541                                         le64_to_cpu(u0->b),
 542                                         next_desc);
 543                         } else {
 544                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %s\n",
 545                                         "R  ", i,
 546                                         le64_to_cpu(u0->a),
 547                                         le64_to_cpu(u0->b),
 548                                         (u64)buffer_info->dma,
 549                                         next_desc);
 550
 551                                 if (netif_msg_pktdata(adapter) &&
 552                                     buffer_info->dma && buffer_info->page) {
 553                                         print_hex_dump(KERN_INFO, "",
 554                                           DUMP_PREFIX_ADDRESS,
 555                                           16, 1,
 556                                           page_address(buffer_info->page) +
 557                                                       buffer_info->page_offset,
 558                                           igb_rx_bufsz(rx_ring), true);
 559                                 }
 560                         }
 561                 }
 562         }
 563
 564 exit:
 565         return;
 566 }
 567
 568 /**
 569  *  igb_get_i2c_data - Reads the I2C SDA data bit
 570  *  @hw: pointer to hardware structure
 571  *  @i2cctl: Current value of I2CCTL register
 572  *
 573  *  Returns the I2C data bit value
 574  **/
 575 static int igb_get_i2c_data(void *data)
 576 {
 577         struct igb_adapter *adapter = (struct igb_adapter *)data;
 578         struct e1000_hw *hw = &adapter->hw;
 579         s32 i2cctl = rd32(E1000_I2CPARAMS);
 580
 581         return !!(i2cctl & E1000_I2C_DATA_IN);
 582 }
 583
 584 /**
 585  *  igb_set_i2c_data - Sets the I2C data bit
 586  *  @data: pointer to hardware structure
 587  *  @state: I2C data value (0 or 1) to set
 588  *
 589  *  Sets the I2C data bit
 590  **/
 591 static void igb_set_i2c_data(void *data, int state)
 592 {
 593         struct igb_adapter *adapter = (struct igb_adapter *)data;
 594         struct e1000_hw *hw = &adapter->hw;
 595         s32 i2cctl = rd32(E1000_I2CPARAMS);
 596
 597         if (state)
 598                 i2cctl |= E1000_I2C_DATA_OUT;
 599         else
 600                 i2cctl &= ~E1000_I2C_DATA_OUT;
 601
 602         i2cctl &= ~E1000_I2C_DATA_OE_N;
 603         i2cctl |= E1000_I2C_CLK_OE_N;
 604         wr32(E1000_I2CPARAMS, i2cctl);
 605         wrfl();
 606
 607 }
 608
 609 /**
 610  *  igb_set_i2c_clk - Sets the I2C SCL clock
 611  *  @data: pointer to hardware structure
 612  *  @state: state to set clock
 613  *
 614  *  Sets the I2C clock line to state
 615  **/
 616 static void igb_set_i2c_clk(void *data, int state)
 617 {
 618         struct igb_adapter *adapter = (struct igb_adapter *)data;
 619         struct e1000_hw *hw = &adapter->hw;
 620         s32 i2cctl = rd32(E1000_I2CPARAMS);
 621
 622         if (state) {
 623                 i2cctl |= E1000_I2C_CLK_OUT;
 624                 i2cctl &= ~E1000_I2C_CLK_OE_N;
 625         } else {
 626                 i2cctl &= ~E1000_I2C_CLK_OUT;
 627                 i2cctl &= ~E1000_I2C_CLK_OE_N;
 628         }
 629         wr32(E1000_I2CPARAMS, i2cctl);
 630         wrfl();
 631 }
 632
 633 /**
 634  *  igb_get_i2c_clk - Gets the I2C SCL clock state
 635  *  @data: pointer to hardware structure
 636  *
 637  *  Gets the I2C clock state
 638  **/
 639 static int igb_get_i2c_clk(void *data)
 640 {
 641         struct igb_adapter *adapter = (struct igb_adapter *)data;
 642         struct e1000_hw *hw = &adapter->hw;
 643         s32 i2cctl = rd32(E1000_I2CPARAMS);
 644
 645         return !!(i2cctl & E1000_I2C_CLK_IN);
 646 }
 647
 648 static const struct i2c_algo_bit_data igb_i2c_algo = {
 649         .setsda         = igb_set_i2c_data,
 650         .setscl         = igb_set_i2c_clk,
 651         .getsda         = igb_get_i2c_data,
 652         .getscl         = igb_get_i2c_clk,
 653         .udelay         = 5,
 654         .timeout        = 20,
 655 };
 656
 657 /**
 658  *  igb_get_hw_dev - return device
 659  *  @hw: pointer to hardware structure
 660  *
 661  *  used by hardware layer to print debugging information
 662  **/
 663 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 664 {
 665         struct igb_adapter *adapter = hw->back;
 666         return adapter->netdev;
 667 }
 668
 669 /**
 670  *  igb_init_module - Driver Registration Routine
 671  *
 672  *  igb_init_module is the first routine called when the driver is
 673  *  loaded. All it does is register with the PCI subsystem.
 674  **/
 675 static int __init igb_init_module(void)
 676 {
 677         int ret;
 678
 679         pr_info("%s - version %s\n",
 680                igb_driver_string, igb_driver_version);
 681         pr_info("%s\n", igb_copyright);
 682
 683 #ifdef CONFIG_IGB_DCA
 684         dca_register_notify(&dca_notifier);
 685 #endif
 686         ret = pci_register_driver(&igb_driver);
 687         return ret;
 688 }
 689
 690 module_init(igb_init_module);
 691
 692 /**
 693  *  igb_exit_module - Driver Exit Cleanup Routine
 694  *
 695  *  igb_exit_module is called just before the driver is removed
 696  *  from memory.
 697  **/
 698 static void __exit igb_exit_module(void)
 699 {
 700 #ifdef CONFIG_IGB_DCA
 701         dca_unregister_notify(&dca_notifier);
 702 #endif
 703         pci_unregister_driver(&igb_driver);
 704 }
 705
 706 module_exit(igb_exit_module);
 707
 708 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 709 /**
 710  *  igb_cache_ring_register - Descriptor ring to register mapping
 711  *  @adapter: board private structure to initialize
 712  *
 713  *  Once we know the feature-set enabled for the device, we'll cache
 714  *  the register offset the descriptor ring is assigned to.
 715  **/
 716 static void igb_cache_ring_register(struct igb_adapter *adapter)
 717 {
 718         int i = 0, j = 0;
 719         u32 rbase_offset = adapter->vfs_allocated_count;
 720
 721         switch (adapter->hw.mac.type) {
 722         case e1000_82576:
 723                 /* The queues are allocated for virtualization such that VF 0
 724                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 725                  * In order to avoid collision we start at the first free queue
 726                  * and continue consuming queues in the same sequence
 727                  */
 728                 if (adapter->vfs_allocated_count) {
 729                         for (; i < adapter->rss_queues; i++)
 730                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 731                                                                Q_IDX_82576(i);
 732                 }
 733                 /* Fall through */
 734         case e1000_82575:
 735         case e1000_82580:
 736         case e1000_i350:
 737         case e1000_i354:
 738         case e1000_i210:
 739         case e1000_i211:
 740                 /* Fall through */
 741         default:
 742                 for (; i < adapter->num_rx_queues; i++)
 743                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 744                 for (; j < adapter->num_tx_queues; j++)
 745                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 746                 break;
 747         }
 748 }
 749
 750 u32 igb_rd32(struct e1000_hw *hw, u32 reg)
 751 {
 752         struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
 753         u8 __iomem *hw_addr = ACCESS_ONCE(hw->hw_addr);
 754         u32 value = 0;
 755
 756         if (E1000_REMOVED(hw_addr))
 757                 return ~value;
 758
 759         value = readl(&hw_addr[reg]);
 760
 761         /* reads should not return all F's */
 762         if (!(~value) && (!reg || !(~readl(hw_addr)))) {
 763                 struct net_device *netdev = igb->netdev;
 764                 hw->hw_addr = NULL;
 765                 netif_device_detach(netdev);
 766                 netdev_err(netdev, "PCIe link lost, device now detached\n");
 767         }
 768
 769         return value;
 770 }
 771
 772 /**
 773  *  igb_write_ivar - configure ivar for given MSI-X vector
 774  *  @hw: pointer to the HW structure
 775  *  @msix_vector: vector number we are allocating to a given ring
 776  *  @index: row index of IVAR register to write within IVAR table
 777  *  @offset: column offset of in IVAR, should be multiple of 8
 778  *
 779  *  This function is intended to handle the writing of the IVAR register
 780  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 781  *  each containing an cause allocation for an Rx and Tx ring, and a
 782  *  variable number of rows depending on the number of queues supported.
 783  **/
 784 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 785                            int index, int offset)
 786 {
 787         u32 ivar = array_rd32(E1000_IVAR0, index);
 788
 789         /* clear any bits that are currently set */
 790         ivar &= ~((u32)0xFF << offset);
 791
 792         /* write vector and valid bit */
 793         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 794
 795         array_wr32(E1000_IVAR0, index, ivar);
 796 }
 797
 798 #define IGB_N0_QUEUE -1
 799 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 800 {
 801         struct igb_adapter *adapter = q_vector->adapter;
 802         struct e1000_hw *hw = &adapter->hw;
 803         int rx_queue = IGB_N0_QUEUE;
 804         int tx_queue = IGB_N0_QUEUE;
 805         u32 msixbm = 0;
 806
 807         if (q_vector->rx.ring)
 808                 rx_queue = q_vector->rx.ring->reg_idx;
 809         if (q_vector->tx.ring)
 810                 tx_queue = q_vector->tx.ring->reg_idx;
 811
 812         switch (hw->mac.type) {
 813         case e1000_82575:
 814                 /* The 82575 assigns vectors using a bitmask, which matches the
 815                  * bitmask for the EICR/EIMS/EIMC registers.  To assign one
 816                  * or more queues to a vector, we write the appropriate bits
 817                  * into the MSIXBM register for that vector.
 818                  */
 819                 if (rx_queue > IGB_N0_QUEUE)
 820                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 821                 if (tx_queue > IGB_N0_QUEUE)
 822                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 823                 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
 824                         msixbm |= E1000_EIMS_OTHER;
 825                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 826                 q_vector->eims_value = msixbm;
 827                 break;
 828         case e1000_82576:
 829                 /* 82576 uses a table that essentially consists of 2 columns
 830                  * with 8 rows.  The ordering is column-major so we use the
 831                  * lower 3 bits as the row index, and the 4th bit as the
 832                  * column offset.
 833                  */
 834                 if (rx_queue > IGB_N0_QUEUE)
 835                         igb_write_ivar(hw, msix_vector,
 836                                        rx_queue & 0x7,
 837                                        (rx_queue & 0x8) << 1);
 838                 if (tx_queue > IGB_N0_QUEUE)
 839                         igb_write_ivar(hw, msix_vector,
 840                                        tx_queue & 0x7,
 841                                        ((tx_queue & 0x8) << 1) + 8);
 842                 q_vector->eims_value = BIT(msix_vector);
 843                 break;
 844         case e1000_82580:
 845         case e1000_i350:
 846         case e1000_i354:
 847         case e1000_i210:
 848         case e1000_i211:
 849                 /* On 82580 and newer adapters the scheme is similar to 82576
 850                  * however instead of ordering column-major we have things
 851                  * ordered row-major.  So we traverse the table by using
 852                  * bit 0 as the column offset, and the remaining bits as the
 853                  * row index.
 854                  */
 855                 if (rx_queue > IGB_N0_QUEUE)
 856                         igb_write_ivar(hw, msix_vector,
 857                                        rx_queue >> 1,
 858                                        (rx_queue & 0x1) << 4);
 859                 if (tx_queue > IGB_N0_QUEUE)
 860                         igb_write_ivar(hw, msix_vector,
 861                                        tx_queue >> 1,
 862                                        ((tx_queue & 0x1) << 4) + 8);
 863                 q_vector->eims_value = BIT(msix_vector);
 864                 break;
 865         default:
 866                 BUG();
 867                 break;
 868         }
 869
 870         /* add q_vector eims value to global eims_enable_mask */
 871         adapter->eims_enable_mask |= q_vector->eims_value;
 872
 873         /* configure q_vector to set itr on first interrupt */
 874         q_vector->set_itr = 1;
 875 }
 876
 877 /**
 878  *  igb_configure_msix - Configure MSI-X hardware
 879  *  @adapter: board private structure to initialize
 880  *
 881  *  igb_configure_msix sets up the hardware to properly
 882  *  generate MSI-X interrupts.
 883  **/
 884 static void igb_configure_msix(struct igb_adapter *adapter)
 885 {
 886         u32 tmp;
 887         int i, vector = 0;
 888         struct e1000_hw *hw = &adapter->hw;
 889
 890         adapter->eims_enable_mask = 0;
 891
 892         /* set vector for other causes, i.e. link changes */
 893         switch (hw->mac.type) {
 894         case e1000_82575:
 895                 tmp = rd32(E1000_CTRL_EXT);
 896                 /* enable MSI-X PBA support*/
 897                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 898
 899                 /* Auto-Mask interrupts upon ICR read. */
 900                 tmp |= E1000_CTRL_EXT_EIAME;
 901                 tmp |= E1000_CTRL_EXT_IRCA;
 902
 903                 wr32(E1000_CTRL_EXT, tmp);
 904
 905                 /* enable msix_other interrupt */
 906                 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
 907                 adapter->eims_other = E1000_EIMS_OTHER;
 908
 909                 break;
 910
 911         case e1000_82576:
 912         case e1000_82580:
 913         case e1000_i350:
 914         case e1000_i354:
 915         case e1000_i210:
 916         case e1000_i211:
 917                 /* Turn on MSI-X capability first, or our settings
 918                  * won't stick.  And it will take days to debug.
 919                  */
 920                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 921                      E1000_GPIE_PBA | E1000_GPIE_EIAME |
 922                      E1000_GPIE_NSICR);
 923
 924                 /* enable msix_other interrupt */
 925                 adapter->eims_other = BIT(vector);
 926                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 927
 928                 wr32(E1000_IVAR_MISC, tmp);
 929                 break;
 930         default:
 931                 /* do nothing, since nothing else supports MSI-X */
 932                 break;
 933         } /* switch (hw->mac.type) */
 934
 935         adapter->eims_enable_mask |= adapter->eims_other;
 936
 937         for (i = 0; i < adapter->num_q_vectors; i++)
 938                 igb_assign_vector(adapter->q_vector[i], vector++);
 939
 940         wrfl();
 941 }
 942
 943 /**
 944  *  igb_request_msix - Initialize MSI-X interrupts
 945  *  @adapter: board private structure to initialize
 946  *
 947  *  igb_request_msix allocates MSI-X vectors and requests interrupts from the
 948  *  kernel.
 949  **/
 950 static int igb_request_msix(struct igb_adapter *adapter)
 951 {
 952         unsigned int num_q_vectors = adapter->num_q_vectors;
 953         struct net_device *netdev = adapter->netdev;
 954         int i, err = 0, vector = 0, free_vector = 0;
 955
 956         err = request_irq(adapter->msix_entries[vector].vector,
 957                           igb_msix_other, 0, netdev->name, adapter);
 958         if (err)
 959                 goto err_out;
 960
 961         if (num_q_vectors > MAX_Q_VECTORS) {
 962                 num_q_vectors = MAX_Q_VECTORS;
 963                 dev_warn(&adapter->pdev->dev,
 964                          "The number of queue vectors (%d) is higher than max allowed (%d)\n",
 965                          adapter->num_q_vectors, MAX_Q_VECTORS);
 966         }
 967         for (i = 0; i < num_q_vectors; i++) {
 968                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 969
 970                 vector++;
 971
 972                 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
 973
 974                 if (q_vector->rx.ring && q_vector->tx.ring)
 975                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 976                                 q_vector->rx.ring->queue_index);
 977                 else if (q_vector->tx.ring)
 978                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 979                                 q_vector->tx.ring->queue_index);
 980                 else if (q_vector->rx.ring)
 981                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 982                                 q_vector->rx.ring->queue_index);
 983                 else
 984                         sprintf(q_vector->name, "%s-unused", netdev->name);
 985
 986                 err = request_irq(adapter->msix_entries[vector].vector,
 987                                   igb_msix_ring, 0, q_vector->name,
 988                                   q_vector);
 989                 if (err)
 990                         goto err_free;
 991         }
 992
 993         igb_configure_msix(adapter);
 994         return 0;
 995
 996 err_free:
 997         /* free already assigned IRQs */
 998         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
 999
1000         vector--;
1001         for (i = 0; i < vector; i++) {
1002                 free_irq(adapter->msix_entries[free_vector++].vector,
1003                          adapter->q_vector[i]);
1004         }
1005 err_out:
1006         return err;
1007 }
1008
1009 /**
1010  *  igb_free_q_vector - Free memory allocated for specific interrupt vector
1011  *  @adapter: board private structure to initialize
1012  *  @v_idx: Index of vector to be freed
1013  *
1014  *  This function frees the memory allocated to the q_vector.
1015  **/
1016 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
1017 {
1018         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1019
1020         adapter->q_vector[v_idx] = NULL;
1021
1022         /* igb_get_stats64() might access the rings on this vector,
1023          * we must wait a grace period before freeing it.
1024          */
1025         if (q_vector)
1026                 kfree_rcu(q_vector, rcu);
1027 }
1028
1029 /**
1030  *  igb_reset_q_vector - Reset config for interrupt vector
1031  *  @adapter: board private structure to initialize
1032  *  @v_idx: Index of vector to be reset
1033  *
1034  *  If NAPI is enabled it will delete any references to the
1035  *  NAPI struct. This is preparation for igb_free_q_vector.
1036  **/
1037 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1038 {
1039         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1040
1041         /* Coming from igb_set_interrupt_capability, the vectors are not yet
1042          * allocated. So, q_vector is NULL so we should stop here.
1043          */
1044         if (!q_vector)
1045                 return;
1046
1047         if (q_vector->tx.ring)
1048                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1049
1050         if (q_vector->rx.ring)
1051                 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1052
1053         netif_napi_del(&q_vector->napi);
1054
1055 }
1056
1057 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1058 {
1059         int v_idx = adapter->num_q_vectors;
1060
1061         if (adapter->flags & IGB_FLAG_HAS_MSIX)
1062                 pci_disable_msix(adapter->pdev);
1063         else if (adapter->flags & IGB_FLAG_HAS_MSI)
1064                 pci_disable_msi(adapter->pdev);
1065
1066         while (v_idx--)
1067                 igb_reset_q_vector(adapter, v_idx);
1068 }
1069
1070 /**
1071  *  igb_free_q_vectors - Free memory allocated for interrupt vectors
1072  *  @adapter: board private structure to initialize
1073  *
1074  *  This function frees the memory allocated to the q_vectors.  In addition if
1075  *  NAPI is enabled it will delete any references to the NAPI struct prior
1076  *  to freeing the q_vector.
1077  **/
1078 static void igb_free_q_vectors(struct igb_adapter *adapter)
1079 {
1080         int v_idx = adapter->num_q_vectors;
1081
1082         adapter->num_tx_queues = 0;
1083         adapter->num_rx_queues = 0;
1084         adapter->num_q_vectors = 0;
1085
1086         while (v_idx--) {
1087                 igb_reset_q_vector(adapter, v_idx);
1088                 igb_free_q_vector(adapter, v_idx);
1089         }
1090 }
1091
1092 /**
1093  *  igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1094  *  @adapter: board private structure to initialize
1095  *
1096  *  This function resets the device so that it has 0 Rx queues, Tx queues, and
1097  *  MSI-X interrupts allocated.
1098  */
1099 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1100 {
1101         igb_free_q_vectors(adapter);
1102         igb_reset_interrupt_capability(adapter);
1103 }
1104
1105 /**
1106  *  igb_set_interrupt_capability - set MSI or MSI-X if supported
1107  *  @adapter: board private structure to initialize
1108  *  @msix: boolean value of MSIX capability
1109  *
1110  *  Attempt to configure interrupts using the best available
1111  *  capabilities of the hardware and kernel.
1112  **/
1113 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1114 {
1115         int err;
1116         int numvecs, i;
1117
1118         if (!msix)
1119                 goto msi_only;
1120         adapter->flags |= IGB_FLAG_HAS_MSIX;
1121
1122         /* Number of supported queues. */
1123         adapter->num_rx_queues = adapter->rss_queues;
1124         if (adapter->vfs_allocated_count)
1125                 adapter->num_tx_queues = 1;
1126         else
1127                 adapter->num_tx_queues = adapter->rss_queues;
1128
1129         /* start with one vector for every Rx queue */
1130         numvecs = adapter->num_rx_queues;
1131
1132         /* if Tx handler is separate add 1 for every Tx queue */
1133         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1134                 numvecs += adapter->num_tx_queues;
1135
1136         /* store the number of vectors reserved for queues */
1137         adapter->num_q_vectors = numvecs;
1138
1139         /* add 1 vector for link status interrupts */
1140         numvecs++;
1141         for (i = 0; i < numvecs; i++)
1142                 adapter->msix_entries[i].entry = i;
1143
1144         err = pci_enable_msix_range(adapter->pdev,
1145                                     adapter->msix_entries,
1146                                     numvecs,
1147                                     numvecs);
1148         if (err > 0)
1149                 return;
1150
1151         igb_reset_interrupt_capability(adapter);
1152
1153         /* If we can't do MSI-X, try MSI */
1154 msi_only:
1155         adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1156 #ifdef CONFIG_PCI_IOV
1157         /* disable SR-IOV for non MSI-X configurations */
1158         if (adapter->vf_data) {
1159                 struct e1000_hw *hw = &adapter->hw;
1160                 /* disable iov and allow time for transactions to clear */
1161                 pci_disable_sriov(adapter->pdev);
1162                 msleep(500);
1163
1164                 kfree(adapter->vf_mac_list);
1165                 adapter->vf_mac_list = NULL;
1166                 kfree(adapter->vf_data);
1167                 adapter->vf_data = NULL;
1168                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1169                 wrfl();
1170                 msleep(100);
1171                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1172         }
1173 #endif
1174         adapter->vfs_allocated_count = 0;
1175         adapter->rss_queues = 1;
1176         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1177         adapter->num_rx_queues = 1;
1178         adapter->num_tx_queues = 1;
1179         adapter->num_q_vectors = 1;
1180         if (!pci_enable_msi(adapter->pdev))
1181                 adapter->flags |= IGB_FLAG_HAS_MSI;
1182 }
1183
1184 static void igb_add_ring(struct igb_ring *ring,
1185                          struct igb_ring_container *head)
1186 {
1187         head->ring = ring;
1188         head->count++;
1189 }
1190
1191 /**
1192  *  igb_alloc_q_vector - Allocate memory for a single interrupt vector
1193  *  @adapter: board private structure to initialize
1194  *  @v_count: q_vectors allocated on adapter, used for ring interleaving
1195  *  @v_idx: index of vector in adapter struct
1196  *  @txr_count: total number of Tx rings to allocate
1197  *  @txr_idx: index of first Tx ring to allocate
1198  *  @rxr_count: total number of Rx rings to allocate
1199  *  @rxr_idx: index of first Rx ring to allocate
1200  *
1201  *  We allocate one q_vector.  If allocation fails we return -ENOMEM.
1202  **/
1203 static int igb_alloc_q_vector(struct igb_adapter *adapter,
1204                               int v_count, int v_idx,
1205                               int txr_count, int txr_idx,
1206                               int rxr_count, int rxr_idx)
1207 {
1208         struct igb_q_vector *q_vector;
1209         struct igb_ring *ring;
1210         int ring_count, size;
1211
1212         /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1213         if (txr_count > 1 || rxr_count > 1)
1214                 return -ENOMEM;
1215
1216         ring_count = txr_count + rxr_count;
1217         size = sizeof(struct igb_q_vector) +
1218                (sizeof(struct igb_ring) * ring_count);
1219
1220         /* allocate q_vector and rings */
1221         q_vector = adapter->q_vector[v_idx];
1222         if (!q_vector) {
1223                 q_vector = kzalloc(size, GFP_KERNEL);
1224         } else if (size > ksize(q_vector)) {
1225                 struct igb_q_vector *new_q_vector;
1226
1227                 new_q_vector = kzalloc(size, GFP_KERNEL);
1228                 if (new_q_vector)
1229                         kfree_rcu(q_vector, rcu);
1230                 q_vector = new_q_vector;
1231         } else {
1232                 memset(q_vector, 0, size);
1233         }
1234         if (!q_vector)
1235                 return -ENOMEM;
1236
1237         /* initialize NAPI */
1238         netif_napi_add(adapter->netdev, &q_vector->napi,
1239                        igb_poll, 64);
1240
1241         /* tie q_vector and adapter together */
1242         adapter->q_vector[v_idx] = q_vector;
1243         q_vector->adapter = adapter;
1244
1245         /* initialize work limits */
1246         q_vector->tx.work_limit = adapter->tx_work_limit;
1247
1248         /* initialize ITR configuration */
1249         q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1250         q_vector->itr_val = IGB_START_ITR;
1251
1252         /* initialize pointer to rings */
1253         ring = q_vector->ring;
1254
1255         /* intialize ITR */
1256         if (rxr_count) {
1257                 /* rx or rx/tx vector */
1258                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1259                         q_vector->itr_val = adapter->rx_itr_setting;
1260         } else {
1261                 /* tx only vector */
1262                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1263                         q_vector->itr_val = adapter->tx_itr_setting;
1264         }
1265
1266         if (txr_count) {
1267                 /* assign generic ring traits */
1268                 ring->dev = &adapter->pdev->dev;
1269                 ring->netdev = adapter->netdev;
1270
1271                 /* configure backlink on ring */
1272                 ring->q_vector = q_vector;
1273
1274                 /* update q_vector Tx values */
1275                 igb_add_ring(ring, &q_vector->tx);
1276
1277                 /* For 82575, context index must be unique per ring. */
1278                 if (adapter->hw.mac.type == e1000_82575)
1279                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1280
1281                 /* apply Tx specific ring traits */
1282                 ring->count = adapter->tx_ring_count;
1283                 ring->queue_index = txr_idx;
1284
1285                 u64_stats_init(&ring->tx_syncp);
1286                 u64_stats_init(&ring->tx_syncp2);
1287
1288                 /* assign ring to adapter */
1289                 adapter->tx_ring[txr_idx] = ring;
1290
1291                 /* push pointer to next ring */
1292                 ring++;
1293         }
1294
1295         if (rxr_count) {
1296                 /* assign generic ring traits */
1297                 ring->dev = &adapter->pdev->dev;
1298                 ring->netdev = adapter->netdev;
1299
1300                 /* configure backlink on ring */
1301                 ring->q_vector = q_vector;
1302
1303                 /* update q_vector Rx values */
1304                 igb_add_ring(ring, &q_vector->rx);
1305
1306                 /* set flag indicating ring supports SCTP checksum offload */
1307                 if (adapter->hw.mac.type >= e1000_82576)
1308                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1309
1310                 /* On i350, i354, i210, and i211, loopback VLAN packets
1311                  * have the tag byte-swapped.
1312                  */
1313                 if (adapter->hw.mac.type >= e1000_i350)
1314                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1315
1316                 /* apply Rx specific ring traits */
1317                 ring->count = adapter->rx_ring_count;
1318                 ring->queue_index = rxr_idx;
1319
1320                 u64_stats_init(&ring->rx_syncp);
1321
1322                 /* assign ring to adapter */
1323                 adapter->rx_ring[rxr_idx] = ring;
1324         }
1325
1326         return 0;
1327 }
1328
1329
1330 /**
1331  *  igb_alloc_q_vectors - Allocate memory for interrupt vectors
1332  *  @adapter: board private structure to initialize
1333  *
1334  *  We allocate one q_vector per queue interrupt.  If allocation fails we
1335  *  return -ENOMEM.
1336  **/
1337 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1338 {
1339         int q_vectors = adapter->num_q_vectors;
1340         int rxr_remaining = adapter->num_rx_queues;
1341         int txr_remaining = adapter->num_tx_queues;
1342         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1343         int err;
1344
1345         if (q_vectors >= (rxr_remaining + txr_remaining)) {
1346                 for (; rxr_remaining; v_idx++) {
1347                         err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1348                                                  0, 0, 1, rxr_idx);
1349
1350                         if (err)
1351                                 goto err_out;
1352
1353                         /* update counts and index */
1354                         rxr_remaining--;
1355                         rxr_idx++;
1356                 }
1357         }
1358
1359         for (; v_idx < q_vectors; v_idx++) {
1360                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1361                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1362
1363                 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1364                                          tqpv, txr_idx, rqpv, rxr_idx);
1365
1366                 if (err)
1367                         goto err_out;
1368
1369                 /* update counts and index */
1370                 rxr_remaining -= rqpv;
1371                 txr_remaining -= tqpv;
1372                 rxr_idx++;
1373                 txr_idx++;
1374         }
1375
1376         return 0;
1377
1378 err_out:
1379         adapter->num_tx_queues = 0;
1380         adapter->num_rx_queues = 0;
1381         adapter->num_q_vectors = 0;
1382
1383         while (v_idx--)
1384                 igb_free_q_vector(adapter, v_idx);
1385
1386         return -ENOMEM;
1387 }
1388
1389 /**
1390  *  igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1391  *  @adapter: board private structure to initialize
1392  *  @msix: boolean value of MSIX capability
1393  *
1394  *  This function initializes the interrupts and allocates all of the queues.
1395  **/
1396 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1397 {
1398         struct pci_dev *pdev = adapter->pdev;
1399         int err;
1400
1401         igb_set_interrupt_capability(adapter, msix);
1402
1403         err = igb_alloc_q_vectors(adapter);
1404         if (err) {
1405                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1406                 goto err_alloc_q_vectors;
1407         }
1408
1409         igb_cache_ring_register(adapter);
1410
1411         return 0;
1412
1413 err_alloc_q_vectors:
1414         igb_reset_interrupt_capability(adapter);
1415         return err;
1416 }
1417
1418 /**
1419  *  igb_request_irq - initialize interrupts
1420  *  @adapter: board private structure to initialize
1421  *
1422  *  Attempts to configure interrupts using the best available
1423  *  capabilities of the hardware and kernel.
1424  **/
1425 static int igb_request_irq(struct igb_adapter *adapter)
1426 {
1427         struct net_device *netdev = adapter->netdev;
1428         struct pci_dev *pdev = adapter->pdev;
1429         int err = 0;
1430
1431         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1432                 err = igb_request_msix(adapter);
1433                 if (!err)
1434                         goto request_done;
1435                 /* fall back to MSI */
1436                 igb_free_all_tx_resources(adapter);
1437                 igb_free_all_rx_resources(adapter);
1438
1439                 igb_clear_interrupt_scheme(adapter);
1440                 err = igb_init_interrupt_scheme(adapter, false);
1441                 if (err)
1442                         goto request_done;
1443
1444                 igb_setup_all_tx_resources(adapter);
1445                 igb_setup_all_rx_resources(adapter);
1446                 igb_configure(adapter);
1447         }
1448
1449         igb_assign_vector(adapter->q_vector[0], 0);
1450
1451         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1452                 err = request_irq(pdev->irq, igb_intr_msi, 0,
1453                                   netdev->name, adapter);
1454                 if (!err)
1455                         goto request_done;
1456
1457                 /* fall back to legacy interrupts */
1458                 igb_reset_interrupt_capability(adapter);
1459                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1460         }
1461
1462         err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1463                           netdev->name, adapter);
1464
1465         if (err)
1466                 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1467                         err);
1468
1469 request_done:
1470         return err;
1471 }
1472
1473 static void igb_free_irq(struct igb_adapter *adapter)
1474 {
1475         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1476                 int vector = 0, i;
1477
1478                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1479
1480                 for (i = 0; i < adapter->num_q_vectors; i++)
1481                         free_irq(adapter->msix_entries[vector++].vector,
1482                                  adapter->q_vector[i]);
1483         } else {
1484                 free_irq(adapter->pdev->irq, adapter);
1485         }
1486 }
1487
1488 /**
1489  *  igb_irq_disable - Mask off interrupt generation on the NIC
1490  *  @adapter: board private structure
1491  **/
1492 static void igb_irq_disable(struct igb_adapter *adapter)
1493 {
1494         struct e1000_hw *hw = &adapter->hw;
1495
1496         /* we need to be careful when disabling interrupts.  The VFs are also
1497          * mapped into these registers and so clearing the bits can cause
1498          * issues on the VF drivers so we only need to clear what we set
1499          */
1500         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1501                 u32 regval = rd32(E1000_EIAM);
1502
1503                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1504                 wr32(E1000_EIMC, adapter->eims_enable_mask);
1505                 regval = rd32(E1000_EIAC);
1506                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1507         }
1508
1509         wr32(E1000_IAM, 0);
1510         wr32(E1000_IMC, ~0);
1511         wrfl();
1512         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1513                 int i;
1514
1515                 for (i = 0; i < adapter->num_q_vectors; i++)
1516                         synchronize_irq(adapter->msix_entries[i].vector);
1517         } else {
1518                 synchronize_irq(adapter->pdev->irq);
1519         }
1520 }
1521
1522 /**
1523  *  igb_irq_enable - Enable default interrupt generation settings
1524  *  @adapter: board private structure
1525  **/
1526 static void igb_irq_enable(struct igb_adapter *adapter)
1527 {
1528         struct e1000_hw *hw = &adapter->hw;
1529
1530         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1531                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1532                 u32 regval = rd32(E1000_EIAC);
1533
1534                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1535                 regval = rd32(E1000_EIAM);
1536                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1537                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1538                 if (adapter->vfs_allocated_count) {
1539                         wr32(E1000_MBVFIMR, 0xFF);
1540                         ims |= E1000_IMS_VMMB;
1541                 }
1542                 wr32(E1000_IMS, ims);
1543         } else {
1544                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1545                                 E1000_IMS_DRSTA);
1546                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1547                                 E1000_IMS_DRSTA);
1548         }
1549 }
1550
1551 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1552 {
1553         struct e1000_hw *hw = &adapter->hw;
1554         u16 pf_id = adapter->vfs_allocated_count;
1555         u16 vid = adapter->hw.mng_cookie.vlan_id;
1556         u16 old_vid = adapter->mng_vlan_id;
1557
1558         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1559                 /* add VID to filter table */
1560                 igb_vfta_set(hw, vid, pf_id, true, true);
1561                 adapter->mng_vlan_id = vid;
1562         } else {
1563                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1564         }
1565
1566         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1567             (vid != old_vid) &&
1568             !test_bit(old_vid, adapter->active_vlans)) {
1569                 /* remove VID from filter table */
1570                 igb_vfta_set(hw, vid, pf_id, false, true);
1571         }
1572 }
1573
1574 /**
1575  *  igb_release_hw_control - release control of the h/w to f/w
1576  *  @adapter: address of board private structure
1577  *
1578  *  igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1579  *  For ASF and Pass Through versions of f/w this means that the
1580  *  driver is no longer loaded.
1581  **/
1582 static void igb_release_hw_control(struct igb_adapter *adapter)
1583 {
1584         struct e1000_hw *hw = &adapter->hw;
1585         u32 ctrl_ext;
1586
1587         /* Let firmware take over control of h/w */
1588         ctrl_ext = rd32(E1000_CTRL_EXT);
1589         wr32(E1000_CTRL_EXT,
1590                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1591 }
1592
1593 /**
1594  *  igb_get_hw_control - get control of the h/w from f/w
1595  *  @adapter: address of board private structure
1596  *
1597  *  igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1598  *  For ASF and Pass Through versions of f/w this means that
1599  *  the driver is loaded.
1600  **/
1601 static void igb_get_hw_control(struct igb_adapter *adapter)
1602 {
1603         struct e1000_hw *hw = &adapter->hw;
1604         u32 ctrl_ext;
1605
1606         /* Let firmware know the driver has taken over */
1607         ctrl_ext = rd32(E1000_CTRL_EXT);
1608         wr32(E1000_CTRL_EXT,
1609                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1610 }
1611
1612 /**
1613  *  igb_configure - configure the hardware for RX and TX
1614  *  @adapter: private board structure
1615  **/
1616 static void igb_configure(struct igb_adapter *adapter)
1617 {
1618         struct net_device *netdev = adapter->netdev;
1619         int i;
1620
1621         igb_get_hw_control(adapter);
1622         igb_set_rx_mode(netdev);
1623
1624         igb_restore_vlan(adapter);
1625
1626         igb_setup_tctl(adapter);
1627         igb_setup_mrqc(adapter);
1628         igb_setup_rctl(adapter);
1629
1630         igb_nfc_filter_restore(adapter);
1631         igb_configure_tx(adapter);
1632         igb_configure_rx(adapter);
1633
1634         igb_rx_fifo_flush_82575(&adapter->hw);
1635
1636         /* call igb_desc_unused which always leaves
1637          * at least 1 descriptor unused to make sure
1638          * next_to_use != next_to_clean
1639          */
1640         for (i = 0; i < adapter->num_rx_queues; i++) {
1641                 struct igb_ring *ring = adapter->rx_ring[i];
1642                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1643         }
1644 }
1645
1646 /**
1647  *  igb_power_up_link - Power up the phy/serdes link
1648  *  @adapter: address of board private structure
1649  **/
1650 void igb_power_up_link(struct igb_adapter *adapter)
1651 {
1652         igb_reset_phy(&adapter->hw);
1653
1654         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1655                 igb_power_up_phy_copper(&adapter->hw);
1656         else
1657                 igb_power_up_serdes_link_82575(&adapter->hw);
1658
1659         igb_setup_link(&adapter->hw);
1660 }
1661
1662 /**
1663  *  igb_power_down_link - Power down the phy/serdes link
1664  *  @adapter: address of board private structure
1665  */
1666 static void igb_power_down_link(struct igb_adapter *adapter)
1667 {
1668         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1669                 igb_power_down_phy_copper_82575(&adapter->hw);
1670         else
1671                 igb_shutdown_serdes_link_82575(&adapter->hw);
1672 }
1673
1674 /**
1675  * Detect and switch function for Media Auto Sense
1676  * @adapter: address of the board private structure
1677  **/
1678 static void igb_check_swap_media(struct igb_adapter *adapter)
1679 {
1680         struct e1000_hw *hw = &adapter->hw;
1681         u32 ctrl_ext, connsw;
1682         bool swap_now = false;
1683
1684         ctrl_ext = rd32(E1000_CTRL_EXT);
1685         connsw = rd32(E1000_CONNSW);
1686
1687         /* need to live swap if current media is copper and we have fiber/serdes
1688          * to go to.
1689          */
1690
1691         if ((hw->phy.media_type == e1000_media_type_copper) &&
1692             (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
1693                 swap_now = true;
1694         } else if ((hw->phy.media_type != e1000_media_type_copper) &&
1695                    !(connsw & E1000_CONNSW_SERDESD)) {
1696                 /* copper signal takes time to appear */
1697                 if (adapter->copper_tries < 4) {
1698                         adapter->copper_tries++;
1699                         connsw |= E1000_CONNSW_AUTOSENSE_CONF;
1700                         wr32(E1000_CONNSW, connsw);
1701                         return;
1702                 } else {
1703                         adapter->copper_tries = 0;
1704                         if ((connsw & E1000_CONNSW_PHYSD) &&
1705                             (!(connsw & E1000_CONNSW_PHY_PDN))) {
1706                                 swap_now = true;
1707                                 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
1708                                 wr32(E1000_CONNSW, connsw);
1709                         }
1710                 }
1711         }
1712
1713         if (!swap_now)
1714                 return;
1715
1716         switch (hw->phy.media_type) {
1717         case e1000_media_type_copper:
1718                 netdev_info(adapter->netdev,
1719                         "MAS: changing media to fiber/serdes\n");
1720                 ctrl_ext |=
1721                         E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
1722                 adapter->flags |= IGB_FLAG_MEDIA_RESET;
1723                 adapter->copper_tries = 0;
1724                 break;
1725         case e1000_media_type_internal_serdes:
1726         case e1000_media_type_fiber:
1727                 netdev_info(adapter->netdev,
1728                         "MAS: changing media to copper\n");
1729                 ctrl_ext &=
1730                         ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
1731                 adapter->flags |= IGB_FLAG_MEDIA_RESET;
1732                 break;
1733         default:
1734                 /* shouldn't get here during regular operation */
1735                 netdev_err(adapter->netdev,
1736                         "AMS: Invalid media type found, returning\n");
1737                 break;
1738         }
1739         wr32(E1000_CTRL_EXT, ctrl_ext);
1740 }
1741
1742 /**
1743  *  igb_up - Open the interface and prepare it to handle traffic
1744  *  @adapter: board private structure
1745  **/
1746 int igb_up(struct igb_adapter *adapter)
1747 {
1748         struct e1000_hw *hw = &adapter->hw;
1749         int i;
1750
1751         /* hardware has been reset, we need to reload some things */
1752         igb_configure(adapter);
1753
1754         clear_bit(__IGB_DOWN, &adapter->state);
1755
1756         for (i = 0; i < adapter->num_q_vectors; i++)
1757                 napi_enable(&(adapter->q_vector[i]->napi));
1758
1759         if (adapter->flags & IGB_FLAG_HAS_MSIX)
1760                 igb_configure_msix(adapter);
1761         else
1762                 igb_assign_vector(adapter->q_vector[0], 0);
1763
1764         /* Clear any pending interrupts. */
1765         rd32(E1000_ICR);
1766         igb_irq_enable(adapter);
1767
1768         /* notify VFs that reset has been completed */
1769         if (adapter->vfs_allocated_count) {
1770                 u32 reg_data = rd32(E1000_CTRL_EXT);
1771
1772                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1773                 wr32(E1000_CTRL_EXT, reg_data);
1774         }
1775
1776         netif_tx_start_all_queues(adapter->netdev);
1777
1778         /* start the watchdog. */
1779         hw->mac.get_link_status = 1;
1780         schedule_work(&adapter->watchdog_task);
1781
1782         if ((adapter->flags & IGB_FLAG_EEE) &&
1783             (!hw->dev_spec._82575.eee_disable))
1784                 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
1785
1786         return 0;
1787 }
1788
1789 void igb_down(struct igb_adapter *adapter)
1790 {
1791         struct net_device *netdev = adapter->netdev;
1792         struct e1000_hw *hw = &adapter->hw;
1793         u32 tctl, rctl;
1794         int i;
1795
1796         /* signal that we're down so the interrupt handler does not
1797          * reschedule our watchdog timer
1798          */
1799         set_bit(__IGB_DOWN, &adapter->state);
1800
1801         /* disable receives in the hardware */
1802         rctl = rd32(E1000_RCTL);
1803         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1804         /* flush and sleep below */
1805
1806         igb_nfc_filter_exit(adapter);
1807
1808         netif_carrier_off(netdev);
1809         netif_tx_stop_all_queues(netdev);
1810
1811         /* disable transmits in the hardware */
1812         tctl = rd32(E1000_TCTL);
1813         tctl &= ~E1000_TCTL_EN;
1814         wr32(E1000_TCTL, tctl);
1815         /* flush both disables and wait for them to finish */
1816         wrfl();
1817         usleep_range(10000, 11000);
1818
1819         igb_irq_disable(adapter);
1820
1821         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
1822
1823         for (i = 0; i < adapter->num_q_vectors; i++) {
1824                 if (adapter->q_vector[i]) {
1825                         napi_synchronize(&adapter->q_vector[i]->napi);
1826                         napi_disable(&adapter->q_vector[i]->napi);
1827                 }
1828         }
1829
1830         del_timer_sync(&adapter->watchdog_timer);
1831         del_timer_sync(&adapter->phy_info_timer);
1832
1833         /* record the stats before reset*/
1834         spin_lock(&adapter->stats64_lock);
1835         igb_update_stats(adapter);
1836         spin_unlock(&adapter->stats64_lock);
1837
1838         adapter->link_speed = 0;
1839         adapter->link_duplex = 0;
1840
1841         if (!pci_channel_offline(adapter->pdev))
1842                 igb_reset(adapter);
1843
1844         /* clear VLAN promisc flag so VFTA will be updated if necessary */
1845         adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
1846
1847         igb_clean_all_tx_rings(adapter);
1848         igb_clean_all_rx_rings(adapter);
1849 #ifdef CONFIG_IGB_DCA
1850
1851         /* since we reset the hardware DCA settings were cleared */
1852         igb_setup_dca(adapter);
1853 #endif
1854 }
1855
1856 void igb_reinit_locked(struct igb_adapter *adapter)
1857 {
1858         WARN_ON(in_interrupt());
1859         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1860                 usleep_range(1000, 2000);
1861         igb_down(adapter);
1862         igb_up(adapter);
1863         clear_bit(__IGB_RESETTING, &adapter->state);
1864 }
1865
1866 /** igb_enable_mas - Media Autosense re-enable after swap
1867  *
1868  * @adapter: adapter struct
1869  **/
1870 static void igb_enable_mas(struct igb_adapter *adapter)
1871 {
1872         struct e1000_hw *hw = &adapter->hw;
1873         u32 connsw = rd32(E1000_CONNSW);
1874
1875         /* configure for SerDes media detect */
1876         if ((hw->phy.media_type == e1000_media_type_copper) &&
1877             (!(connsw & E1000_CONNSW_SERDESD))) {
1878                 connsw |= E1000_CONNSW_ENRGSRC;
1879                 connsw |= E1000_CONNSW_AUTOSENSE_EN;
1880                 wr32(E1000_CONNSW, connsw);
1881                 wrfl();
1882         }
1883 }
1884
1885 void igb_reset(struct igb_adapter *adapter)
1886 {
1887         struct pci_dev *pdev = adapter->pdev;
1888         struct e1000_hw *hw = &adapter->hw;
1889         struct e1000_mac_info *mac = &hw->mac;
1890         struct e1000_fc_info *fc = &hw->fc;
1891         u32 pba, hwm;
1892
1893         /* Repartition Pba for greater than 9k mtu
1894          * To take effect CTRL.RST is required.
1895          */
1896         switch (mac->type) {
1897         case e1000_i350:
1898         case e1000_i354:
1899         case e1000_82580:
1900                 pba = rd32(E1000_RXPBS);
1901                 pba = igb_rxpbs_adjust_82580(pba);
1902                 break;
1903         case e1000_82576:
1904                 pba = rd32(E1000_RXPBS);
1905                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1906                 break;
1907         case e1000_82575:
1908         case e1000_i210:
1909         case e1000_i211:
1910         default:
1911                 pba = E1000_PBA_34K;
1912                 break;
1913         }
1914
1915         if (mac->type == e1000_82575) {
1916                 u32 min_rx_space, min_tx_space, needed_tx_space;
1917
1918                 /* write Rx PBA so that hardware can report correct Tx PBA */
1919                 wr32(E1000_PBA, pba);
1920
1921                 /* To maintain wire speed transmits, the Tx FIFO should be
1922                  * large enough to accommodate two full transmit packets,
1923                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1924                  * the Rx FIFO should be large enough to accommodate at least
1925                  * one full receive packet and is similarly rounded up and
1926                  * expressed in KB.
1927                  */
1928                 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
1929
1930                 /* The Tx FIFO also stores 16 bytes of information about the Tx
1931                  * but don't include Ethernet FCS because hardware appends it.
1932                  * We only need to round down to the nearest 512 byte block
1933                  * count since the value we care about is 2 frames, not 1.
1934                  */
1935                 min_tx_space = adapter->max_frame_size;
1936                 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
1937                 min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
1938
1939                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1940                 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
1941
1942                 /* If current Tx allocation is less than the min Tx FIFO size,
1943                  * and the min Tx FIFO size is less than the current Rx FIFO
1944                  * allocation, take space away from current Rx allocation.
1945                  */
1946                 if (needed_tx_space < pba) {
1947                         pba -= needed_tx_space;
1948
1949                         /* if short on Rx space, Rx wins and must trump Tx
1950                          * adjustment
1951                          */
1952                         if (pba < min_rx_space)
1953                                 pba = min_rx_space;
1954                 }
1955
1956                 /* adjust PBA for jumbo frames */
1957                 wr32(E1000_PBA, pba);
1958         }
1959
1960         /* flow control settings
1961          * The high water mark must be low enough to fit one full frame
1962          * after transmitting the pause frame.  As such we must have enough
1963          * space to allow for us to complete our current transmit and then
1964          * receive the frame that is in progress from the link partner.
1965          * Set it to:
1966          * - the full Rx FIFO size minus one full Tx plus one full Rx frame
1967          */
1968         hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
1969
1970         fc->high_water = hwm & 0xFFFFFFF0;      /* 16-byte granularity */
1971         fc->low_water = fc->high_water - 16;
1972         fc->pause_time = 0xFFFF;
1973         fc->send_xon = 1;
1974         fc->current_mode = fc->requested_mode;
1975
1976         /* disable receive for all VFs and wait one second */
1977         if (adapter->vfs_allocated_count) {
1978                 int i;
1979
1980                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1981                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1982
1983                 /* ping all the active vfs to let them know we are going down */
1984                 igb_ping_all_vfs(adapter);
1985
1986                 /* disable transmits and receives */
1987                 wr32(E1000_VFRE, 0);
1988                 wr32(E1000_VFTE, 0);
1989         }
1990
1991         /* Allow time for pending master requests to run */
1992         hw->mac.ops.reset_hw(hw);
1993         wr32(E1000_WUC, 0);
1994
1995         if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
1996                 /* need to resetup here after media swap */
1997                 adapter->ei.get_invariants(hw);
1998                 adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
1999         }
2000         if ((mac->type == e1000_82575) &&
2001             (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2002                 igb_enable_mas(adapter);
2003         }
2004         if (hw->mac.ops.init_hw(hw))
2005                 dev_err(&pdev->dev, "Hardware Error\n");
2006
2007         /* RAR registers were cleared during init_hw, clear mac table */
2008         igb_flush_mac_table(adapter);
2009         __dev_uc_unsync(adapter->netdev, NULL);
2010
2011         /* Recover default RAR entry */
2012         igb_set_default_mac_filter(adapter);
2013
2014         /* Flow control settings reset on hardware reset, so guarantee flow
2015          * control is off when forcing speed.
2016          */
2017         if (!hw->mac.autoneg)
2018                 igb_force_mac_fc(hw);
2019
2020         igb_init_dmac(adapter, pba);
2021 #ifdef CONFIG_IGB_HWMON
2022         /* Re-initialize the thermal sensor on i350 devices. */
2023         if (!test_bit(__IGB_DOWN, &adapter->state)) {
2024                 if (mac->type == e1000_i350 && hw->bus.func == 0) {
2025                         /* If present, re-initialize the external thermal sensor
2026                          * interface.
2027                          */
2028                         if (adapter->ets)
2029                                 mac->ops.init_thermal_sensor_thresh(hw);
2030                 }
2031         }
2032 #endif
2033         /* Re-establish EEE setting */
2034         if (hw->phy.media_type == e1000_media_type_copper) {
2035                 switch (mac->type) {
2036                 case e1000_i350:
2037                 case e1000_i210:
2038                 case e1000_i211:
2039                         igb_set_eee_i350(hw, true, true);
2040                         break;
2041                 case e1000_i354:
2042                         igb_set_eee_i354(hw, true, true);
2043                         break;
2044                 default:
2045                         break;
2046                 }
2047         }
2048         if (!netif_running(adapter->netdev))
2049                 igb_power_down_link(adapter);
2050
2051         igb_update_mng_vlan(adapter);
2052
2053         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2054         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2055
2056         /* Re-enable PTP, where applicable. */
2057         if (adapter->ptp_flags & IGB_PTP_ENABLED)
2058                 igb_ptp_reset(adapter);
2059
2060         igb_get_phy_info(hw);
2061 }
2062
2063 static netdev_features_t igb_fix_features(struct net_device *netdev,
2064         netdev_features_t features)
2065 {
2066         /* Since there is no support for separate Rx/Tx vlan accel
2067          * enable/disable make sure Tx flag is always in same state as Rx.
2068          */
2069         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2070                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2071         else
2072                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2073
2074         return features;
2075 }
2076
2077 static int igb_set_features(struct net_device *netdev,
2078         netdev_features_t features)
2079 {
2080         netdev_features_t changed = netdev->features ^ features;
2081         struct igb_adapter *adapter = netdev_priv(netdev);
2082
2083         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2084                 igb_vlan_mode(netdev, features);
2085
2086         if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2087                 return 0;
2088
2089         if (!(features & NETIF_F_NTUPLE)) {
2090                 struct hlist_node *node2;
2091                 struct igb_nfc_filter *rule;
2092
2093                 spin_lock(&adapter->nfc_lock);
2094                 hlist_for_each_entry_safe(rule, node2,
2095                                           &adapter->nfc_filter_list, nfc_node) {
2096                         igb_erase_filter(adapter, rule);
2097                         hlist_del(&rule->nfc_node);
2098                         kfree(rule);
2099                 }
2100                 spin_unlock(&adapter->nfc_lock);
2101                 adapter->nfc_filter_count = 0;
2102         }
2103
2104         netdev->features = features;
2105
2106         if (netif_running(netdev))
2107                 igb_reinit_locked(adapter);
2108         else
2109                 igb_reset(adapter);
2110
2111         return 0;
2112 }
2113
2114 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2115                            struct net_device *dev,
2116                            const unsigned char *addr, u16 vid,
2117                            u16 flags)
2118 {
2119         /* guarantee we can provide a unique filter for the unicast address */
2120         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2121                 struct igb_adapter *adapter = netdev_priv(dev);
2122                 int vfn = adapter->vfs_allocated_count;
2123
2124                 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2125                         return -ENOMEM;
2126         }
2127
2128         return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2129 }
2130
2131 #define IGB_MAX_MAC_HDR_LEN     127
2132 #define IGB_MAX_NETWORK_HDR_LEN 511
2133
2134 static netdev_features_t
2135 igb_features_check(struct sk_buff *skb, struct net_device *dev,
2136                    netdev_features_t features)
2137 {
2138         unsigned int network_hdr_len, mac_hdr_len;
2139
2140         /* Make certain the headers can be described by a context descriptor */
2141         mac_hdr_len = skb_network_header(skb) - skb->data;
2142         if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2143                 return features & ~(NETIF_F_HW_CSUM |
2144                                     NETIF_F_SCTP_CRC |
2145                                     NETIF_F_HW_VLAN_CTAG_TX |
2146                                     NETIF_F_TSO |
2147                                     NETIF_F_TSO6);
2148
2149         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2150         if (unlikely(network_hdr_len >  IGB_MAX_NETWORK_HDR_LEN))
2151                 return features & ~(NETIF_F_HW_CSUM |
2152                                     NETIF_F_SCTP_CRC |
2153                                     NETIF_F_TSO |
2154                                     NETIF_F_TSO6);
2155
2156         /* We can only support IPV4 TSO in tunnels if we can mangle the
2157          * inner IP ID field, so strip TSO if MANGLEID is not supported.
2158          */
2159         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2160                 features &= ~NETIF_F_TSO;
2161
2162         return features;
2163 }
2164
2165 static const struct net_device_ops igb_netdev_ops = {
2166         .ndo_open               = igb_open,
2167         .ndo_stop               = igb_close,
2168         .ndo_start_xmit         = igb_xmit_frame,
2169         .ndo_get_stats64        = igb_get_stats64,
2170         .ndo_set_rx_mode        = igb_set_rx_mode,
2171         .ndo_set_mac_address    = igb_set_mac,
2172         .ndo_change_mtu         = igb_change_mtu,
2173         .ndo_do_ioctl           = igb_ioctl,
2174         .ndo_tx_timeout         = igb_tx_timeout,
2175         .ndo_validate_addr      = eth_validate_addr,
2176         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
2177         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
2178         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
2179         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
2180         .ndo_set_vf_rate        = igb_ndo_set_vf_bw,
2181         .ndo_set_vf_spoofchk    = igb_ndo_set_vf_spoofchk,
2182         .ndo_get_vf_config      = igb_ndo_get_vf_config,
2183 #ifdef CONFIG_NET_POLL_CONTROLLER
2184         .ndo_poll_controller    = igb_netpoll,
2185 #endif
2186         .ndo_fix_features       = igb_fix_features,
2187         .ndo_set_features       = igb_set_features,
2188         .ndo_fdb_add            = igb_ndo_fdb_add,
2189         .ndo_features_check     = igb_features_check,
2190 };
2191
2192 /**
2193  * igb_set_fw_version - Configure version string for ethtool
2194  * @adapter: adapter struct
2195  **/
2196 void igb_set_fw_version(struct igb_adapter *adapter)
2197 {
2198         struct e1000_hw *hw = &adapter->hw;
2199         struct e1000_fw_version fw;
2200
2201         igb_get_fw_version(hw, &fw);
2202
2203         switch (hw->mac.type) {
2204         case e1000_i210:
2205         case e1000_i211:
2206                 if (!(igb_get_flash_presence_i210(hw))) {
2207                         snprintf(adapter->fw_version,
2208                                  sizeof(adapter->fw_version),
2209                                  "%2d.%2d-%d",
2210                                  fw.invm_major, fw.invm_minor,
2211                                  fw.invm_img_type);
2212                         break;
2213                 }
2214                 /* fall through */
2215         default:
2216                 /* if option is rom valid, display its version too */
2217                 if (fw.or_valid) {
2218                         snprintf(adapter->fw_version,
2219                                  sizeof(adapter->fw_version),
2220                                  "%d.%d, 0x%08x, %d.%d.%d",
2221                                  fw.eep_major, fw.eep_minor, fw.etrack_id,
2222                                  fw.or_major, fw.or_build, fw.or_patch);
2223                 /* no option rom */
2224                 } else if (fw.etrack_id != 0X0000) {
2225                         snprintf(adapter->fw_version,
2226                             sizeof(adapter->fw_version),
2227                             "%d.%d, 0x%08x",
2228                             fw.eep_major, fw.eep_minor, fw.etrack_id);
2229                 } else {
2230                 snprintf(adapter->fw_version,
2231                     sizeof(adapter->fw_version),
2232                     "%d.%d.%d",
2233                     fw.eep_major, fw.eep_minor, fw.eep_build);
2234                 }
2235                 break;
2236         }
2237 }
2238
2239 /**
2240  * igb_init_mas - init Media Autosense feature if enabled in the NVM
2241  *
2242  * @adapter: adapter struct
2243  **/
2244 static void igb_init_mas(struct igb_adapter *adapter)
2245 {
2246         struct e1000_hw *hw = &adapter->hw;
2247         u16 eeprom_data;
2248
2249         hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
2250         switch (hw->bus.func) {
2251         case E1000_FUNC_0:
2252                 if (eeprom_data & IGB_MAS_ENABLE_0) {
2253                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2254                         netdev_info(adapter->netdev,
2255                                 "MAS: Enabling Media Autosense for port %d\n",
2256                                 hw->bus.func);
2257                 }
2258                 break;
2259         case E1000_FUNC_1:
2260                 if (eeprom_data & IGB_MAS_ENABLE_1) {
2261                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2262                         netdev_info(adapter->netdev,
2263                                 "MAS: Enabling Media Autosense for port %d\n",
2264                                 hw->bus.func);
2265                 }
2266                 break;
2267         case E1000_FUNC_2:
2268                 if (eeprom_data & IGB_MAS_ENABLE_2) {
2269                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2270                         netdev_info(adapter->netdev,
2271                                 "MAS: Enabling Media Autosense for port %d\n",
2272                                 hw->bus.func);
2273                 }
2274                 break;
2275         case E1000_FUNC_3:
2276                 if (eeprom_data & IGB_MAS_ENABLE_3) {
2277                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2278                         netdev_info(adapter->netdev,
2279                                 "MAS: Enabling Media Autosense for port %d\n",
2280                                 hw->bus.func);
2281                 }
2282                 break;
2283         default:
2284                 /* Shouldn't get here */
2285                 netdev_err(adapter->netdev,
2286                         "MAS: Invalid port configuration, returning\n");
2287                 break;
2288         }
2289 }
2290
2291 /**
2292  *  igb_init_i2c - Init I2C interface
2293  *  @adapter: pointer to adapter structure
2294  **/
2295 static s32 igb_init_i2c(struct igb_adapter *adapter)
2296 {
2297         s32 status = 0;
2298
2299         /* I2C interface supported on i350 devices */
2300         if (adapter->hw.mac.type != e1000_i350)
2301                 return 0;
2302
2303         /* Initialize the i2c bus which is controlled by the registers.
2304          * This bus will use the i2c_algo_bit structue that implements
2305          * the protocol through toggling of the 4 bits in the register.
2306          */
2307         adapter->i2c_adap.owner = THIS_MODULE;
2308         adapter->i2c_algo = igb_i2c_algo;
2309         adapter->i2c_algo.data = adapter;
2310         adapter->i2c_adap.algo_data = &adapter->i2c_algo;
2311         adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
2312         strlcpy(adapter->i2c_adap.name, "igb BB",
2313                 sizeof(adapter->i2c_adap.name));
2314         status = i2c_bit_add_bus(&adapter->i2c_adap);
2315         return status;
2316 }
2317
2318 /**
2319  *  igb_probe - Device Initialization Routine
2320  *  @pdev: PCI device information struct
2321  *  @ent: entry in igb_pci_tbl
2322  *
2323  *  Returns 0 on success, negative on failure
2324  *
2325  *  igb_probe initializes an adapter identified by a pci_dev structure.
2326  *  The OS initialization, configuring of the adapter private structure,
2327  *  and a hardware reset occur.
2328  **/
2329 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2330 {
2331         struct net_device *netdev;
2332         struct igb_adapter *adapter;
2333         struct e1000_hw *hw;
2334         u16 eeprom_data = 0;
2335         s32 ret_val;
2336         static int global_quad_port_a; /* global quad port a indication */
2337         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
2338         int err, pci_using_dac;
2339         u8 part_str[E1000_PBANUM_LENGTH];
2340
2341         /* Catch broken hardware that put the wrong VF device ID in
2342          * the PCIe SR-IOV capability.
2343          */
2344         if (pdev->is_virtfn) {
2345                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
2346                         pci_name(pdev), pdev->vendor, pdev->device);
2347                 return -EINVAL;
2348         }
2349
2350         err = pci_enable_device_mem(pdev);
2351         if (err)
2352                 return err;
2353
2354         pci_using_dac = 0;
2355         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2356         if (!err) {
2357                 pci_using_dac = 1;
2358         } else {
2359                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2360                 if (err) {
2361                         dev_err(&pdev->dev,
2362                                 "No usable DMA configuration, aborting\n");
2363                         goto err_dma;
2364                 }
2365         }
2366
2367         err = pci_request_mem_regions(pdev, igb_driver_name);
2368         if (err)
2369                 goto err_pci_reg;
2370
2371         pci_enable_pcie_error_reporting(pdev);
2372
2373         pci_set_master(pdev);
2374         pci_save_state(pdev);
2375
2376         err = -ENOMEM;
2377         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2378                                    IGB_MAX_TX_QUEUES);
2379         if (!netdev)
2380                 goto err_alloc_etherdev;
2381
2382         SET_NETDEV_DEV(netdev, &pdev->dev);
2383
2384         pci_set_drvdata(pdev, netdev);
2385         adapter = netdev_priv(netdev);
2386         adapter->netdev = netdev;
2387         adapter->pdev = pdev;
2388         hw = &adapter->hw;
2389         hw->back = adapter;
2390         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2391
2392         err = -EIO;
2393         adapter->io_addr = pci_iomap(pdev, 0, 0);
2394         if (!adapter->io_addr)
2395                 goto err_ioremap;
2396         /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
2397         hw->hw_addr = adapter->io_addr;
2398
2399         netdev->netdev_ops = &igb_netdev_ops;
2400         igb_set_ethtool_ops(netdev);
2401         netdev->watchdog_timeo = 5 * HZ;
2402
2403         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2404
2405         netdev->mem_start = pci_resource_start(pdev, 0);
2406         netdev->mem_end = pci_resource_end(pdev, 0);
2407
2408         /* PCI config space info */
2409         hw->vendor_id = pdev->vendor;
2410         hw->device_id = pdev->device;
2411         hw->revision_id = pdev->revision;
2412         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2413         hw->subsystem_device_id = pdev->subsystem_device;
2414
2415         /* Copy the default MAC, PHY and NVM function pointers */
2416         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
2417         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
2418         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
2419         /* Initialize skew-specific constants */
2420         err = ei->get_invariants(hw);
2421         if (err)
2422                 goto err_sw_init;
2423
2424         /* setup the private structure */
2425         err = igb_sw_init(adapter);
2426         if (err)
2427                 goto err_sw_init;
2428
2429         igb_get_bus_info_pcie(hw);
2430
2431         hw->phy.autoneg_wait_to_complete = false;
2432
2433         /* Copper options */
2434         if (hw->phy.media_type == e1000_media_type_copper) {
2435                 hw->phy.mdix = AUTO_ALL_MODES;
2436                 hw->phy.disable_polarity_correction = false;
2437                 hw->phy.ms_type = e1000_ms_hw_default;
2438         }
2439
2440         if (igb_check_reset_block(hw))
2441                 dev_info(&pdev->dev,
2442                         "PHY reset is blocked due to SOL/IDER session.\n");
2443
2444         /* features is initialized to 0 in allocation, it might have bits
2445          * set by igb_sw_init so we should use an or instead of an
2446          * assignment.
2447          */
2448         netdev->features |= NETIF_F_SG |
2449                             NETIF_F_TSO |
2450                             NETIF_F_TSO6 |
2451                             NETIF_F_RXHASH |
2452                             NETIF_F_RXCSUM |
2453                             NETIF_F_HW_CSUM;
2454
2455         if (hw->mac.type >= e1000_82576)
2456                 netdev->features |= NETIF_F_SCTP_CRC;
2457
2458 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2459                                   NETIF_F_GSO_GRE_CSUM | \
2460                                   NETIF_F_GSO_IPXIP4 | \
2461                                   NETIF_F_GSO_IPXIP6 | \
2462                                   NETIF_F_GSO_UDP_TUNNEL | \
2463                                   NETIF_F_GSO_UDP_TUNNEL_CSUM)
2464
2465         netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
2466         netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
2467
2468         /* copy netdev features into list of user selectable features */
2469         netdev->hw_features |= netdev->features |
2470                                NETIF_F_HW_VLAN_CTAG_RX |
2471                                NETIF_F_HW_VLAN_CTAG_TX |
2472                                NETIF_F_RXALL;
2473
2474         if (hw->mac.type >= e1000_i350)
2475                 netdev->hw_features |= NETIF_F_NTUPLE;
2476
2477         if (pci_using_dac)
2478                 netdev->features |= NETIF_F_HIGHDMA;
2479
2480         netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2481         netdev->mpls_features |= NETIF_F_HW_CSUM;
2482         netdev->hw_enc_features |= netdev->vlan_features;
2483
2484         /* set this bit last since it cannot be part of vlan_features */
2485         netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2486                             NETIF_F_HW_VLAN_CTAG_RX |
2487                             NETIF_F_HW_VLAN_CTAG_TX;
2488
2489         netdev->priv_flags |= IFF_SUPP_NOFCS;
2490
2491         netdev->priv_flags |= IFF_UNICAST_FLT;
2492
2493         /* MTU range: 68 - 9216 */
2494         netdev->min_mtu = ETH_MIN_MTU;
2495         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2496
2497         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2498
2499         /* before reading the NVM, reset the controller to put the device in a
2500          * known good starting state
2501          */
2502         hw->mac.ops.reset_hw(hw);
2503
2504         /* make sure the NVM is good , i211/i210 parts can have special NVM
2505          * that doesn't contain a checksum
2506          */
2507         switch (hw->mac.type) {
2508         case e1000_i210:
2509         case e1000_i211:
2510                 if (igb_get_flash_presence_i210(hw)) {
2511                         if (hw->nvm.ops.validate(hw) < 0) {
2512                                 dev_err(&pdev->dev,
2513                                         "The NVM Checksum Is Not Valid\n");
2514                                 err = -EIO;
2515                                 goto err_eeprom;
2516                         }
2517                 }
2518                 break;
2519         default:
2520                 if (hw->nvm.ops.validate(hw) < 0) {
2521                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2522                         err = -EIO;
2523                         goto err_eeprom;
2524                 }
2525                 break;
2526         }
2527
2528         if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
2529                 /* copy the MAC address out of the NVM */
2530                 if (hw->mac.ops.read_mac_addr(hw))
2531                         dev_err(&pdev->dev, "NVM Read Error\n");
2532         }
2533
2534         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2535
2536         if (!is_valid_ether_addr(netdev->dev_addr)) {
2537                 dev_err(&pdev->dev, "Invalid MAC Address\n");
2538                 err = -EIO;
2539                 goto err_eeprom;
2540         }
2541
2542         igb_set_default_mac_filter(adapter);
2543
2544         /* get firmware version for ethtool -i */
2545         igb_set_fw_version(adapter);
2546
2547         /* configure RXPBSIZE and TXPBSIZE */
2548         if (hw->mac.type == e1000_i210) {
2549                 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
2550                 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
2551         }
2552
2553         setup_timer(&adapter->watchdog_timer, igb_watchdog,
2554                     (unsigned long) adapter);
2555         setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2556                     (unsigned long) adapter);
2557
2558         INIT_WORK(&adapter->reset_task, igb_reset_task);
2559         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2560
2561         /* Initialize link properties that are user-changeable */
2562         adapter->fc_autoneg = true;
2563         hw->mac.autoneg = true;
2564         hw->phy.autoneg_advertised = 0x2f;
2565
2566         hw->fc.requested_mode = e1000_fc_default;
2567         hw->fc.current_mode = e1000_fc_default;
2568
2569         igb_validate_mdi_setting(hw);
2570
2571         /* By default, support wake on port A */
2572         if (hw->bus.func == 0)
2573                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2574
2575         /* Check the NVM for wake support on non-port A ports */
2576         if (hw->mac.type >= e1000_82580)
2577                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2578                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2579                                  &eeprom_data);
2580         else if (hw->bus.func == 1)
2581                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2582
2583         if (eeprom_data & IGB_EEPROM_APME)
2584                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2585
2586         /* now that we have the eeprom settings, apply the special cases where
2587          * the eeprom may be wrong or the board simply won't support wake on
2588          * lan on a particular port
2589          */
2590         switch (pdev->device) {
2591         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2592                 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2593                 break;
2594         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2595         case E1000_DEV_ID_82576_FIBER:
2596         case E1000_DEV_ID_82576_SERDES:
2597                 /* Wake events only supported on port A for dual fiber
2598                  * regardless of eeprom setting
2599                  */
2600                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2601                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2602                 break;
2603         case E1000_DEV_ID_82576_QUAD_COPPER:
2604         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2605                 /* if quad port adapter, disable WoL on all but port A */
2606                 if (global_quad_port_a != 0)
2607                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2608                 else
2609                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2610                 /* Reset for multiple quad port adapters */
2611                 if (++global_quad_port_a == 4)
2612                         global_quad_port_a = 0;
2613                 break;
2614         default:
2615                 /* If the device can't wake, don't set software support */
2616                 if (!device_can_wakeup(&adapter->pdev->dev))
2617                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2618         }
2619
2620         /* initialize the wol settings based on the eeprom settings */
2621         if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
2622                 adapter->wol |= E1000_WUFC_MAG;
2623
2624         /* Some vendors want WoL disabled by default, but still supported */
2625         if ((hw->mac.type == e1000_i350) &&
2626             (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
2627                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2628                 adapter->wol = 0;
2629         }
2630
2631         /* Some vendors want the ability to Use the EEPROM setting as
2632          * enable/disable only, and not for capability
2633          */
2634         if (((hw->mac.type == e1000_i350) ||
2635              (hw->mac.type == e1000_i354)) &&
2636             (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
2637                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2638                 adapter->wol = 0;
2639         }
2640         if (hw->mac.type == e1000_i350) {
2641                 if (((pdev->subsystem_device == 0x5001) ||
2642                      (pdev->subsystem_device == 0x5002)) &&
2643                                 (hw->bus.func == 0)) {
2644                         adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2645                         adapter->wol = 0;
2646                 }
2647                 if (pdev->subsystem_device == 0x1F52)
2648                         adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2649         }
2650
2651         device_set_wakeup_enable(&adapter->pdev->dev,
2652                                  adapter->flags & IGB_FLAG_WOL_SUPPORTED);
2653
2654         /* reset the hardware with the new settings */
2655         igb_reset(adapter);
2656
2657         /* Init the I2C interface */
2658         err = igb_init_i2c(adapter);
2659         if (err) {
2660                 dev_err(&pdev->dev, "failed to init i2c interface\n");
2661                 goto err_eeprom;
2662         }
2663
2664         /* let the f/w know that the h/w is now under the control of the
2665          * driver.
2666          */
2667         igb_get_hw_control(adapter);
2668
2669         strcpy(netdev->name, "eth%d");
2670         err = register_netdev(netdev);
2671         if (err)
2672                 goto err_register;
2673
2674         /* carrier off reporting is important to ethtool even BEFORE open */
2675         netif_carrier_off(netdev);
2676
2677 #ifdef CONFIG_IGB_DCA
2678         if (dca_add_requester(&pdev->dev) == 0) {
2679                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2680                 dev_info(&pdev->dev, "DCA enabled\n");
2681                 igb_setup_dca(adapter);
2682         }
2683
2684 #endif
2685 #ifdef CONFIG_IGB_HWMON
2686         /* Initialize the thermal sensor on i350 devices. */
2687         if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
2688                 u16 ets_word;
2689
2690                 /* Read the NVM to determine if this i350 device supports an
2691                  * external thermal sensor.
2692                  */
2693                 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
2694                 if (ets_word != 0x0000 && ets_word != 0xFFFF)
2695                         adapter->ets = true;
2696                 else
2697                         adapter->ets = false;
2698                 if (igb_sysfs_init(adapter))
2699                         dev_err(&pdev->dev,
2700                                 "failed to allocate sysfs resources\n");
2701         } else {
2702                 adapter->ets = false;
2703         }
2704 #endif
2705         /* Check if Media Autosense is enabled */
2706         adapter->ei = *ei;
2707         if (hw->dev_spec._82575.mas_capable)
2708                 igb_init_mas(adapter);
2709
2710         /* do hw tstamp init after resetting */
2711         igb_ptp_init(adapter);
2712
2713         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2714         /* print bus type/speed/width info, not applicable to i354 */
2715         if (hw->mac.type != e1000_i354) {
2716                 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2717                          netdev->name,
2718                          ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2719                           (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2720                            "unknown"),
2721                          ((hw->bus.width == e1000_bus_width_pcie_x4) ?
2722                           "Width x4" :
2723                           (hw->bus.width == e1000_bus_width_pcie_x2) ?
2724                           "Width x2" :
2725                           (hw->bus.width == e1000_bus_width_pcie_x1) ?
2726                           "Width x1" : "unknown"), netdev->dev_addr);
2727         }
2728
2729         if ((hw->mac.type >= e1000_i210 ||
2730              igb_get_flash_presence_i210(hw))) {
2731                 ret_val = igb_read_part_string(hw, part_str,
2732                                                E1000_PBANUM_LENGTH);
2733         } else {
2734                 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
2735         }
2736
2737         if (ret_val)
2738                 strcpy(part_str, "Unknown");
2739         dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2740         dev_info(&pdev->dev,
2741                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2742                 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
2743                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2744                 adapter->num_rx_queues, adapter->num_tx_queues);
2745         if (hw->phy.media_type == e1000_media_type_copper) {
2746                 switch (hw->mac.type) {
2747                 case e1000_i350:
2748                 case e1000_i210:
2749                 case e1000_i211:
2750                         /* Enable EEE for internal copper PHY devices */
2751                         err = igb_set_eee_i350(hw, true, true);
2752                         if ((!err) &&
2753                             (!hw->dev_spec._82575.eee_disable)) {
2754                                 adapter->eee_advert =
2755                                         MDIO_EEE_100TX | MDIO_EEE_1000T;
2756                                 adapter->flags |= IGB_FLAG_EEE;
2757                         }
2758                         break;
2759                 case e1000_i354:
2760                         if ((rd32(E1000_CTRL_EXT) &
2761                             E1000_CTRL_EXT_LINK_MODE_SGMII)) {
2762                                 err = igb_set_eee_i354(hw, true, true);
2763                                 if ((!err) &&
2764                                         (!hw->dev_spec._82575.eee_disable)) {
2765                                         adapter->eee_advert =
2766                                            MDIO_EEE_100TX | MDIO_EEE_1000T;
2767                                         adapter->flags |= IGB_FLAG_EEE;
2768                                 }
2769                         }
2770                         break;
2771                 default:
2772                         break;
2773                 }
2774         }
2775         pm_runtime_put_noidle(&pdev->dev);
2776         return 0;
2777
2778 err_register:
2779         igb_release_hw_control(adapter);
2780         memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
2781 err_eeprom:
2782         if (!igb_check_reset_block(hw))
2783                 igb_reset_phy(hw);
2784
2785         if (hw->flash_address)
2786                 iounmap(hw->flash_address);
2787 err_sw_init:
2788         kfree(adapter->mac_table);
2789         kfree(adapter->shadow_vfta);
2790         igb_clear_interrupt_scheme(adapter);
2791 #ifdef CONFIG_PCI_IOV
2792         igb_disable_sriov(pdev);
2793 #endif
2794         pci_iounmap(pdev, adapter->io_addr);
2795 err_ioremap:
2796         free_netdev(netdev);
2797 err_alloc_etherdev:
2798         pci_disable_pcie_error_reporting(pdev);
2799         pci_release_mem_regions(pdev);
2800 err_pci_reg:
2801 err_dma:
2802         pci_disable_device(pdev);
2803         return err;
2804 }
2805
2806 #ifdef CONFIG_PCI_IOV
2807 static int igb_disable_sriov(struct pci_dev *pdev)
2808 {
2809         struct net_device *netdev = pci_get_drvdata(pdev);
2810         struct igb_adapter *adapter = netdev_priv(netdev);
2811         struct e1000_hw *hw = &adapter->hw;
2812         unsigned long flags;
2813
2814         /* reclaim resources allocated to VFs */
2815         if (adapter->vf_data) {
2816                 /* disable iov and allow time for transactions to clear */
2817                 if (pci_vfs_assigned(pdev)) {
2818                         dev_warn(&pdev->dev,
2819                                  "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
2820                         return -EPERM;
2821                 } else {
2822                         pci_disable_sriov(pdev);
2823                         msleep(500);
2824                 }
2825                 spin_lock_irqsave(&adapter->vfs_lock, flags);
2826                 kfree(adapter->vf_mac_list);
2827                 adapter->vf_mac_list = NULL;
2828                 kfree(adapter->vf_data);
2829                 adapter->vf_data = NULL;
2830                 adapter->vfs_allocated_count = 0;
2831                 spin_unlock_irqrestore(&adapter->vfs_lock, flags);
2832                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2833                 wrfl();
2834                 msleep(100);
2835                 dev_info(&pdev->dev, "IOV Disabled\n");
2836
2837                 /* Re-enable DMA Coalescing flag since IOV is turned off */
2838                 adapter->flags |= IGB_FLAG_DMAC;
2839         }
2840
2841         return 0;
2842 }
2843
2844 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
2845 {
2846         struct net_device *netdev = pci_get_drvdata(pdev);
2847         struct igb_adapter *adapter = netdev_priv(netdev);
2848         int old_vfs = pci_num_vf(pdev);
2849         struct vf_mac_filter *mac_list;
2850         int err = 0;
2851         int num_vf_mac_filters, i;
2852
2853         if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
2854                 err = -EPERM;
2855                 goto out;
2856         }
2857         if (!num_vfs)
2858                 goto out;
2859
2860         if (old_vfs) {
2861                 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
2862                          old_vfs, max_vfs);
2863                 adapter->vfs_allocated_count = old_vfs;
2864         } else
2865                 adapter->vfs_allocated_count = num_vfs;
2866
2867         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2868                                 sizeof(struct vf_data_storage), GFP_KERNEL);
2869
2870         /* if allocation failed then we do not support SR-IOV */
2871         if (!adapter->vf_data) {
2872                 adapter->vfs_allocated_count = 0;
2873                 dev_err(&pdev->dev,
2874                         "Unable to allocate memory for VF Data Storage\n");
2875                 err = -ENOMEM;
2876                 goto out;
2877         }
2878
2879         /* Due to the limited number of RAR entries calculate potential
2880          * number of MAC filters available for the VFs. Reserve entries
2881          * for PF default MAC, PF MAC filters and at least one RAR entry
2882          * for each VF for VF MAC.
2883          */
2884         num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
2885                              (1 + IGB_PF_MAC_FILTERS_RESERVED +
2886                               adapter->vfs_allocated_count);
2887
2888         adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
2889                                        sizeof(struct vf_mac_filter),
2890                                        GFP_KERNEL);
2891
2892         mac_list = adapter->vf_mac_list;
2893         INIT_LIST_HEAD(&adapter->vf_macs.l);
2894
2895         if (adapter->vf_mac_list) {
2896                 /* Initialize list of VF MAC filters */
2897                 for (i = 0; i < num_vf_mac_filters; i++) {
2898                         mac_list->vf = -1;
2899                         mac_list->free = true;
2900                         list_add(&mac_list->l, &adapter->vf_macs.l);
2901                         mac_list++;
2902                 }
2903         } else {
2904                 /* If we could not allocate memory for the VF MAC filters
2905                  * we can continue without this feature but warn user.
2906                  */
2907                 dev_err(&pdev->dev,
2908                         "Unable to allocate memory for VF MAC filter list\n");
2909         }
2910
2911         /* only call pci_enable_sriov() if no VFs are allocated already */
2912         if (!old_vfs) {
2913                 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
2914                 if (err)
2915                         goto err_out;
2916         }
2917         dev_info(&pdev->dev, "%d VFs allocated\n",
2918                  adapter->vfs_allocated_count);
2919         for (i = 0; i < adapter->vfs_allocated_count; i++)
2920                 igb_vf_configure(adapter, i);
2921
2922         /* DMA Coalescing is not supported in IOV mode. */
2923         adapter->flags &= ~IGB_FLAG_DMAC;
2924         goto out;
2925
2926 err_out:
2927         kfree(adapter->vf_mac_list);
2928         adapter->vf_mac_list = NULL;
2929         kfree(adapter->vf_data);
2930         adapter->vf_data = NULL;
2931         adapter->vfs_allocated_count = 0;
2932 out:
2933         return err;
2934 }
2935
2936 #endif
2937 /**
2938  *  igb_remove_i2c - Cleanup  I2C interface
2939  *  @adapter: pointer to adapter structure
2940  **/
2941 static void igb_remove_i2c(struct igb_adapter *adapter)
2942 {
2943         /* free the adapter bus structure */
2944         i2c_del_adapter(&adapter->i2c_adap);
2945 }
2946
2947 /**
2948  *  igb_remove - Device Removal Routine
2949  *  @pdev: PCI device information struct
2950  *
2951  *  igb_remove is called by the PCI subsystem to alert the driver
2952  *  that it should release a PCI device.  The could be caused by a
2953  *  Hot-Plug event, or because the driver is going to be removed from
2954  *  memory.
2955  **/
2956 static void igb_remove(struct pci_dev *pdev)
2957 {
2958         struct net_device *netdev = pci_get_drvdata(pdev);
2959         struct igb_adapter *adapter = netdev_priv(netdev);
2960         struct e1000_hw *hw = &adapter->hw;
2961
2962         pm_runtime_get_noresume(&pdev->dev);
2963 #ifdef CONFIG_IGB_HWMON
2964         igb_sysfs_exit(adapter);
2965 #endif
2966         igb_remove_i2c(adapter);
2967         igb_ptp_stop(adapter);
2968         /* The watchdog timer may be rescheduled, so explicitly
2969          * disable watchdog from being rescheduled.
2970          */
2971         set_bit(__IGB_DOWN, &adapter->state);
2972         del_timer_sync(&adapter->watchdog_timer);
2973         del_timer_sync(&adapter->phy_info_timer);
2974
2975         cancel_work_sync(&adapter->reset_task);
2976         cancel_work_sync(&adapter->watchdog_task);
2977
2978 #ifdef CONFIG_IGB_DCA
2979         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2980                 dev_info(&pdev->dev, "DCA disabled\n");
2981                 dca_remove_requester(&pdev->dev);
2982                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2983                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2984         }
2985 #endif
2986
2987         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2988          * would have already happened in close and is redundant.
2989          */
2990         igb_release_hw_control(adapter);
2991
2992 #ifdef CONFIG_PCI_IOV
2993         igb_disable_sriov(pdev);
2994 #endif
2995
2996         unregister_netdev(netdev);
2997
2998         igb_clear_interrupt_scheme(adapter);
2999
3000         pci_iounmap(pdev, adapter->io_addr);
3001         if (hw->flash_address)
3002                 iounmap(hw->flash_address);
3003         pci_release_mem_regions(pdev);
3004
3005         kfree(adapter->mac_table);
3006         kfree(adapter->shadow_vfta);
3007         free_netdev(netdev);
3008
3009         pci_disable_pcie_error_reporting(pdev);
3010
3011         pci_disable_device(pdev);
3012 }
3013
3014 /**
3015  *  igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3016  *  @adapter: board private structure to initialize
3017  *
3018  *  This function initializes the vf specific data storage and then attempts to
3019  *  allocate the VFs.  The reason for ordering it this way is because it is much
3020  *  mor expensive time wise to disable SR-IOV than it is to allocate and free
3021  *  the memory for the VFs.
3022  **/
3023 static void igb_probe_vfs(struct igb_adapter *adapter)
3024 {
3025 #ifdef CONFIG_PCI_IOV
3026         struct pci_dev *pdev = adapter->pdev;
3027         struct e1000_hw *hw = &adapter->hw;
3028
3029         /* Virtualization features not supported on i210 and 82580 family. */
3030         if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) ||
3031             (hw->mac.type == e1000_82580))
3032                 return;
3033
3034         /* Of the below we really only want the effect of getting
3035          * IGB_FLAG_HAS_MSIX set (if available), without which
3036          * igb_enable_sriov() has no effect.
3037          */
3038         igb_set_interrupt_capability(adapter, true);
3039         igb_reset_interrupt_capability(adapter);
3040
3041         pci_sriov_set_totalvfs(pdev, 7);
3042         igb_enable_sriov(pdev, max_vfs);
3043
3044 #endif /* CONFIG_PCI_IOV */
3045 }
3046
3047 static void igb_init_queue_configuration(struct igb_adapter *adapter)
3048 {
3049         struct e1000_hw *hw = &adapter->hw;
3050         u32 max_rss_queues;
3051
3052         /* Determine the maximum number of RSS queues supported. */
3053         switch (hw->mac.type) {
3054         case e1000_i211:
3055                 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3056                 break;
3057         case e1000_82575:
3058         case e1000_i210:
3059                 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3060                 break;
3061         case e1000_i350:
3062                 /* I350 cannot do RSS and SR-IOV at the same time */
3063                 if (!!adapter->vfs_allocated_count) {
3064                         max_rss_queues = 1;
3065                         break;
3066                 }
3067                 /* fall through */
3068         case e1000_82576:
3069                 if (!!adapter->vfs_allocated_count) {
3070                         max_rss_queues = 2;
3071                         break;
3072                 }
3073                 /* fall through */
3074         case e1000_82580:
3075         case e1000_i354:
3076         default:
3077                 max_rss_queues = IGB_MAX_RX_QUEUES;
3078                 break;
3079         }
3080
3081         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
3082
3083         igb_set_flag_queue_pairs(adapter, max_rss_queues);
3084 }
3085
3086 void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
3087                               const u32 max_rss_queues)
3088 {
3089         struct e1000_hw *hw = &adapter->hw;
3090
3091         /* Determine if we need to pair queues. */
3092         switch (hw->mac.type) {
3093         case e1000_82575:
3094         case e1000_i211:
3095                 /* Device supports enough interrupts without queue pairing. */
3096                 break;
3097         case e1000_82576:
3098         case e1000_82580:
3099         case e1000_i350:
3100         case e1000_i354:
3101         case e1000_i210:
3102         default:
3103                 /* If rss_queues > half of max_rss_queues, pair the queues in
3104                  * order to conserve interrupts due to limited supply.
3105                  */
3106                 if (adapter->rss_queues > (max_rss_queues / 2))
3107                         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
3108                 else
3109                         adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
3110                 break;
3111         }
3112 }
3113
3114 /**
3115  *  igb_sw_init - Initialize general software structures (struct igb_adapter)
3116  *  @adapter: board private structure to initialize
3117  *
3118  *  igb_sw_init initializes the Adapter private data structure.
3119  *  Fields are initialized based on PCI device information and
3120  *  OS network device settings (MTU size).
3121  **/
3122 static int igb_sw_init(struct igb_adapter *adapter)
3123 {
3124         struct e1000_hw *hw = &adapter->hw;
3125         struct net_device *netdev = adapter->netdev;
3126         struct pci_dev *pdev = adapter->pdev;
3127
3128         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
3129
3130         /* set default ring sizes */
3131         adapter->tx_ring_count = IGB_DEFAULT_TXD;
3132         adapter->rx_ring_count = IGB_DEFAULT_RXD;
3133
3134         /* set default ITR values */
3135         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
3136         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
3137
3138         /* set default work limits */
3139         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
3140
3141         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
3142                                   VLAN_HLEN;
3143         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3144
3145         spin_lock_init(&adapter->nfc_lock);
3146         spin_lock_init(&adapter->stats64_lock);
3147
3148         /* init spinlock to avoid concurrency of VF resources */
3149         spin_lock_init(&adapter->vfs_lock);
3150 #ifdef CONFIG_PCI_IOV
3151         switch (hw->mac.type) {
3152         case e1000_82576:
3153         case e1000_i350:
3154                 if (max_vfs > 7) {
3155                         dev_warn(&pdev->dev,
3156                                  "Maximum of 7 VFs per PF, using max\n");
3157                         max_vfs = adapter->vfs_allocated_count = 7;
3158                 } else
3159                         adapter->vfs_allocated_count = max_vfs;
3160                 if (adapter->vfs_allocated_count)
3161                         dev_warn(&pdev->dev,
3162                                  "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
3163                 break;
3164         default:
3165                 break;
3166         }
3167 #endif /* CONFIG_PCI_IOV */
3168
3169         /* Assume MSI-X interrupts, will be checked during IRQ allocation */
3170         adapter->flags |= IGB_FLAG_HAS_MSIX;
3171
3172         adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
3173                                      hw->mac.rar_entry_count, GFP_ATOMIC);
3174         if (!adapter->mac_table)
3175                 return -ENOMEM;
3176
3177         igb_probe_vfs(adapter);
3178
3179         igb_init_queue_configuration(adapter);
3180
3181         /* Setup and initialize a copy of the hw vlan table array */
3182         adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
3183                                        GFP_ATOMIC);
3184         if (!adapter->shadow_vfta)
3185                 return -ENOMEM;
3186
3187         /* This call may decrease the number of queues */
3188         if (igb_init_interrupt_scheme(adapter, true)) {
3189                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
3190                 return -ENOMEM;
3191         }
3192
3193         /* Explicitly disable IRQ since the NIC can be in any state. */
3194         igb_irq_disable(adapter);
3195
3196         if (hw->mac.type >= e1000_i350)
3197                 adapter->flags &= ~IGB_FLAG_DMAC;
3198
3199         set_bit(__IGB_DOWN, &adapter->state);
3200         return 0;
3201 }
3202
3203 /**
3204  *  igb_open - Called when a network interface is made active
3205  *  @netdev: network interface device structure
3206  *
3207  *  Returns 0 on success, negative value on failure
3208  *
3209  *  The open entry point is called when a network interface is made
3210  *  active by the system (IFF_UP).  At this point all resources needed
3211  *  for transmit and receive operations are allocated, the interrupt
3212  *  handler is registered with the OS, the watchdog timer is started,
3213  *  and the stack is notified that the interface is ready.
3214  **/
3215 static int __igb_open(struct net_device *netdev, bool resuming)
3216 {
3217         struct igb_adapter *adapter = netdev_priv(netdev);
3218         struct e1000_hw *hw = &adapter->hw;
3219         struct pci_dev *pdev = adapter->pdev;
3220         int err;
3221         int i;
3222
3223         /* disallow open during test */
3224         if (test_bit(__IGB_TESTING, &adapter->state)) {
3225                 WARN_ON(resuming);
3226                 return -EBUSY;
3227         }
3228
3229         if (!resuming)
3230                 pm_runtime_get_sync(&pdev->dev);
3231
3232         netif_carrier_off(netdev);
3233
3234         /* allocate transmit descriptors */
3235         err = igb_setup_all_tx_resources(adapter);
3236         if (err)
3237                 goto err_setup_tx;
3238
3239         /* allocate receive descriptors */
3240         err = igb_setup_all_rx_resources(adapter);
3241         if (err)
3242                 goto err_setup_rx;
3243
3244         igb_power_up_link(adapter);
3245
3246         /* before we allocate an interrupt, we must be ready to handle it.
3247          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3248          * as soon as we call pci_request_irq, so we have to setup our
3249          * clean_rx handler before we do so.
3250          */
3251         igb_configure(adapter);
3252
3253         err = igb_request_irq(adapter);
3254         if (err)
3255                 goto err_req_irq;
3256
3257         /* Notify the stack of the actual queue counts. */
3258         err = netif_set_real_num_tx_queues(adapter->netdev,
3259                                            adapter->num_tx_queues);
3260         if (err)
3261                 goto err_set_queues;
3262
3263         err = netif_set_real_num_rx_queues(adapter->netdev,
3264                                            adapter->num_rx_queues);
3265         if (err)
3266                 goto err_set_queues;
3267
3268         /* From here on the code is the same as igb_up() */
3269         clear_bit(__IGB_DOWN, &adapter->state);
3270
3271         for (i = 0; i < adapter->num_q_vectors; i++)
3272                 napi_enable(&(adapter->q_vector[i]->napi));
3273
3274         /* Clear any pending interrupts. */
3275         rd32(E1000_ICR);
3276
3277         igb_irq_enable(adapter);
3278
3279         /* notify VFs that reset has been completed */
3280         if (adapter->vfs_allocated_count) {
3281                 u32 reg_data = rd32(E1000_CTRL_EXT);
3282
3283                 reg_data |= E1000_CTRL_EXT_PFRSTD;
3284                 wr32(E1000_CTRL_EXT, reg_data);
3285         }
3286
3287         netif_tx_start_all_queues(netdev);
3288
3289         if (!resuming)
3290                 pm_runtime_put(&pdev->dev);
3291
3292         /* start the watchdog. */
3293         hw->mac.get_link_status = 1;
3294         schedule_work(&adapter->watchdog_task);
3295
3296         return 0;
3297
3298 err_set_queues:
3299         igb_free_irq(adapter);
3300 err_req_irq:
3301         igb_release_hw_control(adapter);
3302         igb_power_down_link(adapter);
3303         igb_free_all_rx_resources(adapter);
3304 err_setup_rx:
3305         igb_free_all_tx_resources(adapter);
3306 err_setup_tx:
3307         igb_reset(adapter);
3308         if (!resuming)
3309                 pm_runtime_put(&pdev->dev);
3310
3311         return err;
3312 }
3313
3314 int igb_open(struct net_device *netdev)
3315 {
3316         return __igb_open(netdev, false);
3317 }
3318
3319 /**
3320  *  igb_close - Disables a network interface
3321  *  @netdev: network interface device structure
3322  *
3323  *  Returns 0, this is not allowed to fail
3324  *
3325  *  The close entry point is called when an interface is de-activated
3326  *  by the OS.  The hardware is still under the driver's control, but
3327  *  needs to be disabled.  A global MAC reset is issued to stop the
3328  *  hardware, and all transmit and receive resources are freed.
3329  **/
3330 static int __igb_close(struct net_device *netdev, bool suspending)
3331 {
3332         struct igb_adapter *adapter = netdev_priv(netdev);
3333         struct pci_dev *pdev = adapter->pdev;
3334
3335         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
3336
3337         if (!suspending)
3338                 pm_runtime_get_sync(&pdev->dev);
3339
3340         igb_down(adapter);
3341         igb_free_irq(adapter);
3342
3343         igb_free_all_tx_resources(adapter);
3344         igb_free_all_rx_resources(adapter);
3345
3346         if (!suspending)
3347                 pm_runtime_put_sync(&pdev->dev);
3348         return 0;
3349 }
3350
3351 int igb_close(struct net_device *netdev)
3352 {
3353         if (netif_device_present(netdev) || netdev->dismantle)
3354                 return __igb_close(netdev, false);
3355         return 0;
3356 }
3357
3358 /**
3359  *  igb_setup_tx_resources - allocate Tx resources (Descriptors)
3360  *  @tx_ring: tx descriptor ring (for a specific queue) to setup
3361  *
3362  *  Return 0 on success, negative on failure
3363  **/
3364 int igb_setup_tx_resources(struct igb_ring *tx_ring)
3365 {
3366         struct device *dev = tx_ring->dev;
3367         int size;
3368
3369         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3370
3371         tx_ring->tx_buffer_info = vmalloc(size);
3372         if (!tx_ring->tx_buffer_info)
3373                 goto err;
3374
3375         /* round up to nearest 4K */
3376         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
3377         tx_ring->size = ALIGN(tx_ring->size, 4096);
3378
3379         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3380                                            &tx_ring->dma, GFP_KERNEL);
3381         if (!tx_ring->desc)
3382                 goto err;
3383
3384         tx_ring->next_to_use = 0;
3385         tx_ring->next_to_clean = 0;
3386
3387         return 0;
3388
3389 err:
3390         vfree(tx_ring->tx_buffer_info);
3391         tx_ring->tx_buffer_info = NULL;
3392         dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
3393         return -ENOMEM;
3394 }
3395
3396 /**
3397  *  igb_setup_all_tx_resources - wrapper to allocate Tx resources
3398  *                               (Descriptors) for all queues
3399  *  @adapter: board private structure
3400  *
3401  *  Return 0 on success, negative on failure
3402  **/
3403 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
3404 {
3405         struct pci_dev *pdev = adapter->pdev;
3406         int i, err = 0;
3407
3408         for (i = 0; i < adapter->num_tx_queues; i++) {
3409                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
3410                 if (err) {
3411                         dev_err(&pdev->dev,
3412                                 "Allocation for Tx Queue %u failed\n", i);
3413                         for (i--; i >= 0; i--)
3414                                 igb_free_tx_resources(adapter->tx_ring[i]);
3415                         break;
3416                 }
3417         }
3418
3419         return err;
3420 }
3421
3422 /**
3423  *  igb_setup_tctl - configure the transmit control registers
3424  *  @adapter: Board private structure
3425  **/
3426 void igb_setup_tctl(struct igb_adapter *adapter)
3427 {
3428         struct e1000_hw *hw = &adapter->hw;
3429         u32 tctl;
3430
3431         /* disable queue 0 which is enabled by default on 82575 and 82576 */
3432         wr32(E1000_TXDCTL(0), 0);
3433
3434         /* Program the Transmit Control Register */
3435         tctl = rd32(E1000_TCTL);
3436         tctl &= ~E1000_TCTL_CT;
3437         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
3438                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
3439
3440         igb_config_collision_dist(hw);
3441
3442         /* Enable transmits */
3443         tctl |= E1000_TCTL_EN;
3444
3445         wr32(E1000_TCTL, tctl);
3446 }
3447
3448 /**
3449  *  igb_configure_tx_ring - Configure transmit ring after Reset
3450  *  @adapter: board private structure
3451  *  @ring: tx ring to configure
3452  *
3453  *  Configure a transmit ring after a reset.
3454  **/
3455 void igb_configure_tx_ring(struct igb_adapter *adapter,
3456                            struct igb_ring *ring)
3457 {
3458         struct e1000_hw *hw = &adapter->hw;
3459         u32 txdctl = 0;
3460         u64 tdba = ring->dma;
3461         int reg_idx = ring->reg_idx;
3462
3463         /* disable the queue */
3464         wr32(E1000_TXDCTL(reg_idx), 0);
3465         wrfl();
3466         mdelay(10);
3467
3468         wr32(E1000_TDLEN(reg_idx),
3469              ring->count * sizeof(union e1000_adv_tx_desc));
3470         wr32(E1000_TDBAL(reg_idx),
3471              tdba & 0x00000000ffffffffULL);
3472         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
3473
3474         ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
3475         wr32(E1000_TDH(reg_idx), 0);
3476         writel(0, ring->tail);
3477
3478         txdctl |= IGB_TX_PTHRESH;
3479         txdctl |= IGB_TX_HTHRESH << 8;
3480         txdctl |= IGB_TX_WTHRESH << 16;
3481
3482         /* reinitialize tx_buffer_info */
3483         memset(ring->tx_buffer_info, 0,
3484                sizeof(struct igb_tx_buffer) * ring->count);
3485
3486         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3487         wr32(E1000_TXDCTL(reg_idx), txdctl);
3488 }
3489
3490 /**
3491  *  igb_configure_tx - Configure transmit Unit after Reset
3492  *  @adapter: board private structure
3493  *
3494  *  Configure the Tx unit of the MAC after a reset.
3495  **/
3496 static void igb_configure_tx(struct igb_adapter *adapter)
3497 {
3498         int i;
3499
3500         for (i = 0; i < adapter->num_tx_queues; i++)
3501                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3502 }
3503
3504 /**
3505  *  igb_setup_rx_resources - allocate Rx resources (Descriptors)
3506  *  @rx_ring: Rx descriptor ring (for a specific queue) to setup
3507  *
3508  *  Returns 0 on success, negative on failure
3509  **/
3510 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3511 {
3512         struct device *dev = rx_ring->dev;
3513         int size;
3514
3515         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3516
3517         rx_ring->rx_buffer_info = vmalloc(size);
3518         if (!rx_ring->rx_buffer_info)
3519                 goto err;
3520
3521         /* Round up to nearest 4K */
3522         rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
3523         rx_ring->size = ALIGN(rx_ring->size, 4096);
3524
3525         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
3526                                            &rx_ring->dma, GFP_KERNEL);
3527         if (!rx_ring->desc)
3528                 goto err;
3529
3530         rx_ring->next_to_alloc = 0;
3531         rx_ring->next_to_clean = 0;
3532         rx_ring->next_to_use = 0;
3533
3534         return 0;
3535
3536 err:
3537         vfree(rx_ring->rx_buffer_info);
3538         rx_ring->rx_buffer_info = NULL;
3539         dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
3540         return -ENOMEM;
3541 }
3542
3543 /**
3544  *  igb_setup_all_rx_resources - wrapper to allocate Rx resources
3545  *                               (Descriptors) for all queues
3546  *  @adapter: board private structure
3547  *
3548  *  Return 0 on success, negative on failure
3549  **/
3550 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3551 {
3552         struct pci_dev *pdev = adapter->pdev;
3553         int i, err = 0;
3554
3555         for (i = 0; i < adapter->num_rx_queues; i++) {
3556                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3557                 if (err) {
3558                         dev_err(&pdev->dev,
3559                                 "Allocation for Rx Queue %u failed\n", i);
3560                         for (i--; i >= 0; i--)
3561                                 igb_free_rx_resources(adapter->rx_ring[i]);
3562                         break;
3563                 }
3564         }
3565
3566         return err;
3567 }
3568
3569 /**
3570  *  igb_setup_mrqc - configure the multiple receive queue control registers
3571  *  @adapter: Board private structure
3572  **/
3573 static void igb_setup_mrqc(struct igb_adapter *adapter)
3574 {
3575         struct e1000_hw *hw = &adapter->hw;
3576         u32 mrqc, rxcsum;
3577         u32 j, num_rx_queues;
3578         u32 rss_key[10];
3579
3580         netdev_rss_key_fill(rss_key, sizeof(rss_key));
3581         for (j = 0; j < 10; j++)
3582                 wr32(E1000_RSSRK(j), rss_key[j]);
3583
3584         num_rx_queues = adapter->rss_queues;
3585
3586         switch (hw->mac.type) {
3587         case e1000_82576:
3588                 /* 82576 supports 2 RSS queues for SR-IOV */
3589                 if (adapter->vfs_allocated_count)
3590                         num_rx_queues = 2;
3591                 break;
3592         default:
3593                 break;
3594         }
3595
3596         if (adapter->rss_indir_tbl_init != num_rx_queues) {
3597                 for (j = 0; j < IGB_RETA_SIZE; j++)
3598                         adapter->rss_indir_tbl[j] =
3599                         (j * num_rx_queues) / IGB_RETA_SIZE;
3600                 adapter->rss_indir_tbl_init = num_rx_queues;
3601         }
3602         igb_write_rss_indir_tbl(adapter);
3603
3604         /* Disable raw packet checksumming so that RSS hash is placed in
3605          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
3606          * offloads as they are enabled by default
3607          */
3608         rxcsum = rd32(E1000_RXCSUM);
3609         rxcsum |= E1000_RXCSUM_PCSD;
3610
3611         if (adapter->hw.mac.type >= e1000_82576)
3612                 /* Enable Receive Checksum Offload for SCTP */
3613                 rxcsum |= E1000_RXCSUM_CRCOFL;
3614
3615         /* Don't need to set TUOFL or IPOFL, they default to 1 */
3616         wr32(E1000_RXCSUM, rxcsum);
3617
3618         /* Generate RSS hash based on packet types, TCP/UDP
3619          * port numbers and/or IPv4/v6 src and dst addresses
3620          */
3621         mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
3622                E1000_MRQC_RSS_FIELD_IPV4_TCP |
3623                E1000_MRQC_RSS_FIELD_IPV6 |
3624                E1000_MRQC_RSS_FIELD_IPV6_TCP |
3625                E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3626
3627         if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
3628                 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
3629         if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
3630                 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
3631
3632         /* If VMDq is enabled then we set the appropriate mode for that, else
3633          * we default to RSS so that an RSS hash is calculated per packet even
3634          * if we are only using one queue
3635          */
3636         if (adapter->vfs_allocated_count) {
3637                 if (hw->mac.type > e1000_82575) {
3638                         /* Set the default pool for the PF's first queue */
3639                         u32 vtctl = rd32(E1000_VT_CTL);
3640
3641                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3642                                    E1000_VT_CTL_DISABLE_DEF_POOL);
3643                         vtctl |= adapter->vfs_allocated_count <<
3644                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3645                         wr32(E1000_VT_CTL, vtctl);
3646                 }
3647                 if (adapter->rss_queues > 1)
3648                         mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
3649                 else
3650                         mrqc |= E1000_MRQC_ENABLE_VMDQ;
3651         } else {
3652                 if (hw->mac.type != e1000_i211)
3653                         mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
3654         }
3655         igb_vmm_control(adapter);
3656
3657         wr32(E1000_MRQC, mrqc);
3658 }
3659
3660 /**
3661  *  igb_setup_rctl - configure the receive control registers
3662  *  @adapter: Board private structure
3663  **/
3664 void igb_setup_rctl(struct igb_adapter *adapter)
3665 {
3666         struct e1000_hw *hw = &adapter->hw;
3667         u32 rctl;
3668
3669         rctl = rd32(E1000_RCTL);
3670
3671         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3672         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3673
3674         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3675                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3676
3677         /* enable stripping of CRC. It's unlikely this will break BMC
3678          * redirection as it did with e1000. Newer features require
3679          * that the HW strips the CRC.
3680          */
3681         rctl |= E1000_RCTL_SECRC;
3682
3683         /* disable store bad packets and clear size bits. */
3684         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3685
3686         /* enable LPE to allow for reception of jumbo frames */
3687         rctl |= E1000_RCTL_LPE;
3688
3689         /* disable queue 0 to prevent tail write w/o re-config */
3690         wr32(E1000_RXDCTL(0), 0);
3691
3692         /* Attention!!!  For SR-IOV PF driver operations you must enable
3693          * queue drop for all VF and PF queues to prevent head of line blocking
3694          * if an un-trusted VF does not provide descriptors to hardware.
3695          */
3696         if (adapter->vfs_allocated_count) {
3697                 /* set all queue drop enable bits */
3698                 wr32(E1000_QDE, ALL_QUEUES);
3699         }
3700
3701         /* This is useful for sniffing bad packets. */
3702         if (adapter->netdev->features & NETIF_F_RXALL) {
3703                 /* UPE and MPE will be handled by normal PROMISC logic
3704                  * in e1000e_set_rx_mode
3705                  */
3706                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3707                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
3708                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3709
3710                 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
3711                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3712                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3713                  * and that breaks VLANs.
3714                  */
3715         }
3716
3717         wr32(E1000_RCTL, rctl);
3718 }
3719
3720 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3721                                    int vfn)
3722 {
3723         struct e1000_hw *hw = &adapter->hw;
3724         u32 vmolr;
3725
3726         if (size > MAX_JUMBO_FRAME_SIZE)
3727                 size = MAX_JUMBO_FRAME_SIZE;
3728
3729         vmolr = rd32(E1000_VMOLR(vfn));
3730         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3731         vmolr |= size | E1000_VMOLR_LPE;
3732         wr32(E1000_VMOLR(vfn), vmolr);
3733
3734         return 0;
3735 }
3736
3737 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3738                                          int vfn, bool enable)
3739 {
3740         struct e1000_hw *hw = &adapter->hw;
3741         u32 val, reg;
3742
3743         if (hw->mac.type < e1000_82576)
3744                 return;
3745
3746         if (hw->mac.type == e1000_i350)
3747                 reg = E1000_DVMOLR(vfn);
3748         else
3749                 reg = E1000_VMOLR(vfn);
3750
3751         val = rd32(reg);
3752         if (enable)
3753                 val |= E1000_VMOLR_STRVLAN;
3754         else
3755                 val &= ~(E1000_VMOLR_STRVLAN);
3756         wr32(reg, val);
3757 }
3758
3759 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3760                                  int vfn, bool aupe)
3761 {
3762         struct e1000_hw *hw = &adapter->hw;
3763         u32 vmolr;
3764
3765         /* This register exists only on 82576 and newer so if we are older then
3766          * we should exit and do nothing
3767          */
3768         if (hw->mac.type < e1000_82576)
3769                 return;
3770
3771         vmolr = rd32(E1000_VMOLR(vfn));
3772         if (aupe)
3773                 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3774         else
3775                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3776
3777         /* clear all bits that might not be set */
3778         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3779
3780         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3781                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3782         /* for VMDq only allow the VFs and pool 0 to accept broadcast and
3783          * multicast packets
3784          */
3785         if (vfn <= adapter->vfs_allocated_count)
3786                 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3787
3788         wr32(E1000_VMOLR(vfn), vmolr);
3789 }
3790
3791 /**
3792  *  igb_configure_rx_ring - Configure a receive ring after Reset
3793  *  @adapter: board private structure
3794  *  @ring: receive ring to be configured
3795  *
3796  *  Configure the Rx unit of the MAC after a reset.
3797  **/
3798 void igb_configure_rx_ring(struct igb_adapter *adapter,
3799                            struct igb_ring *ring)
3800 {
3801         struct e1000_hw *hw = &adapter->hw;
3802         union e1000_adv_rx_desc *rx_desc;
3803         u64 rdba = ring->dma;
3804         int reg_idx = ring->reg_idx;
3805         u32 srrctl = 0, rxdctl = 0;
3806
3807         /* disable the queue */
3808         wr32(E1000_RXDCTL(reg_idx), 0);
3809
3810         /* Set DMA base address registers */
3811         wr32(E1000_RDBAL(reg_idx),
3812              rdba & 0x00000000ffffffffULL);
3813         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3814         wr32(E1000_RDLEN(reg_idx),
3815              ring->count * sizeof(union e1000_adv_rx_desc));
3816
3817         /* initialize head and tail */
3818         ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
3819         wr32(E1000_RDH(reg_idx), 0);
3820         writel(0, ring->tail);
3821
3822         /* set descriptor configuration */
3823         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3824         if (ring_uses_large_buffer(ring))
3825                 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3826         else
3827                 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3828         srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3829         if (hw->mac.type >= e1000_82580)
3830                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3831         /* Only set Drop Enable if we are supporting multiple queues */
3832         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3833                 srrctl |= E1000_SRRCTL_DROP_EN;
3834
3835         wr32(E1000_SRRCTL(reg_idx), srrctl);
3836
3837         /* set filtering for VMDQ pools */
3838         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3839
3840         rxdctl |= IGB_RX_PTHRESH;
3841         rxdctl |= IGB_RX_HTHRESH << 8;
3842         rxdctl |= IGB_RX_WTHRESH << 16;
3843
3844         /* initialize rx_buffer_info */
3845         memset(ring->rx_buffer_info, 0,
3846                sizeof(struct igb_rx_buffer) * ring->count);
3847
3848         /* initialize Rx descriptor 0 */
3849         rx_desc = IGB_RX_DESC(ring, 0);
3850         rx_desc->wb.upper.length = 0;
3851
3852         /* enable receive descriptor fetching */
3853         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3854         wr32(E1000_RXDCTL(reg_idx), rxdctl);
3855 }
3856
3857 static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
3858                                   struct igb_ring *rx_ring)
3859 {
3860 #if (PAGE_SIZE < 8192)
3861         struct e1000_hw *hw = &adapter->hw;
3862 #endif
3863
3864         /* set build_skb and buffer size flags */
3865         clear_ring_build_skb_enabled(rx_ring);
3866         clear_ring_uses_large_buffer(rx_ring);
3867
3868         if (adapter->flags & IGB_FLAG_RX_LEGACY)
3869                 return;
3870
3871         set_ring_build_skb_enabled(rx_ring);
3872
3873 #if (PAGE_SIZE < 8192)
3874         if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB ||
3875             rd32(E1000_RCTL) & E1000_RCTL_SBP)
3876                 set_ring_uses_large_buffer(rx_ring);
3877 #endif
3878 }
3879
3880 /**
3881  *  igb_configure_rx - Configure receive Unit after Reset
3882  *  @adapter: board private structure
3883  *
3884  *  Configure the Rx unit of the MAC after a reset.
3885  **/
3886 static void igb_configure_rx(struct igb_adapter *adapter)
3887 {
3888         int i;
3889
3890         /* set the correct pool for the PF default MAC address in entry 0 */
3891         igb_set_default_mac_filter(adapter);
3892
3893         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3894          * the Base and Length of the Rx Descriptor Ring
3895          */
3896         for (i = 0; i < adapter->num_rx_queues; i++) {
3897                 struct igb_ring *rx_ring = adapter->rx_ring[i];
3898
3899                 igb_set_rx_buffer_len(adapter, rx_ring);
3900                 igb_configure_rx_ring(adapter, rx_ring);
3901         }
3902 }
3903
3904 /**
3905  *  igb_free_tx_resources - Free Tx Resources per Queue
3906  *  @tx_ring: Tx descriptor ring for a specific queue
3907  *
3908  *  Free all transmit software resources
3909  **/
3910 void igb_free_tx_resources(struct igb_ring *tx_ring)
3911 {
3912         igb_clean_tx_ring(tx_ring);
3913
3914         vfree(tx_ring->tx_buffer_info);
3915         tx_ring->tx_buffer_info = NULL;
3916
3917         /* if not set, then don't free */
3918         if (!tx_ring->desc)
3919                 return;
3920
3921         dma_free_coherent(tx_ring->dev, tx_ring->size,
3922                           tx_ring->desc, tx_ring->dma);
3923
3924         tx_ring->desc = NULL;
3925 }
3926
3927 /**
3928  *  igb_free_all_tx_resources - Free Tx Resources for All Queues
3929  *  @adapter: board private structure
3930  *
3931  *  Free all transmit software resources
3932  **/
3933 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3934 {
3935         int i;
3936
3937         for (i = 0; i < adapter->num_tx_queues; i++)
3938                 if (adapter->tx_ring[i])
3939                         igb_free_tx_resources(adapter->tx_ring[i]);
3940 }
3941
3942 /**
3943  *  igb_clean_tx_ring - Free Tx Buffers
3944  *  @tx_ring: ring to be cleaned
3945  **/
3946 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3947 {
3948         u16 i = tx_ring->next_to_clean;
3949         struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
3950
3951         while (i != tx_ring->next_to_use) {
3952                 union e1000_adv_tx_desc *eop_desc, *tx_desc;
3953
3954                 /* Free all the Tx ring sk_buffs */
3955                 dev_kfree_skb_any(tx_buffer->skb);
3956
3957                 /* unmap skb header data */
3958                 dma_unmap_single(tx_ring->dev,
3959                                  dma_unmap_addr(tx_buffer, dma),
3960                                  dma_unmap_len(tx_buffer, len),
3961                                  DMA_TO_DEVICE);
3962
3963                 /* check for eop_desc to determine the end of the packet */
3964                 eop_desc = tx_buffer->next_to_watch;
3965                 tx_desc = IGB_TX_DESC(tx_ring, i);
3966
3967                 /* unmap remaining buffers */
3968                 while (tx_desc != eop_desc) {
3969                         tx_buffer++;
3970                         tx_desc++;
3971                         i++;
3972                         if (unlikely(i == tx_ring->count)) {
3973                                 i = 0;
3974                                 tx_buffer = tx_ring->tx_buffer_info;
3975                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
3976                         }
3977
3978                         /* unmap any remaining paged data */
3979                         if (dma_unmap_len(tx_buffer, len))
3980                                 dma_unmap_page(tx_ring->dev,
3981                                                dma_unmap_addr(tx_buffer, dma),
3982                                                dma_unmap_len(tx_buffer, len),
3983                                                DMA_TO_DEVICE);
3984                 }
3985
3986                 tx_buffer->next_to_watch = NULL;
3987
3988                 /* move us one more past the eop_desc for start of next pkt */
3989                 tx_buffer++;
3990                 i++;
3991                 if (unlikely(i == tx_ring->count)) {
3992                         i = 0;
3993                         tx_buffer = tx_ring->tx_buffer_info;
3994                 }
3995         }
3996
3997         /* reset BQL for queue */
3998         netdev_tx_reset_queue(txring_txq(tx_ring));
3999
4000         /* reset next_to_use and next_to_clean */
4001         tx_ring->next_to_use = 0;
4002         tx_ring->next_to_clean = 0;
4003 }
4004
4005 /**
4006  *  igb_clean_all_tx_rings - Free Tx Buffers for all queues
4007  *  @adapter: board private structure
4008  **/
4009 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4010 {
4011         int i;
4012
4013         for (i = 0; i < adapter->num_tx_queues; i++)
4014                 if (adapter->tx_ring[i])
4015                         igb_clean_tx_ring(adapter->tx_ring[i]);
4016 }
4017
4018 /**
4019  *  igb_free_rx_resources - Free Rx Resources
4020  *  @rx_ring: ring to clean the resources from
4021  *
4022  *  Free all receive software resources
4023  **/
4024 void igb_free_rx_resources(struct igb_ring *rx_ring)
4025 {
4026         igb_clean_rx_ring(rx_ring);
4027
4028         vfree(rx_ring->rx_buffer_info);
4029         rx_ring->rx_buffer_info = NULL;
4030
4031         /* if not set, then don't free */
4032         if (!rx_ring->desc)
4033                 return;
4034
4035         dma_free_coherent(rx_ring->dev, rx_ring->size,
4036                           rx_ring->desc, rx_ring->dma);
4037
4038         rx_ring->desc = NULL;
4039 }
4040
4041 /**
4042  *  igb_free_all_rx_resources - Free Rx Resources for All Queues
4043  *  @adapter: board private structure
4044  *
4045  *  Free all receive software resources
4046  **/
4047 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
4048 {
4049         int i;
4050
4051         for (i = 0; i < adapter->num_rx_queues; i++)
4052                 if (adapter->rx_ring[i])
4053                         igb_free_rx_resources(adapter->rx_ring[i]);
4054 }
4055
4056 /**
4057  *  igb_clean_rx_ring - Free Rx Buffers per Queue
4058  *  @rx_ring: ring to free buffers from
4059  **/
4060 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
4061 {
4062         u16 i = rx_ring->next_to_clean;
4063
4064         if (rx_ring->skb)
4065                 dev_kfree_skb(rx_ring->skb);
4066         rx_ring->skb = NULL;
4067
4068         /* Free all the Rx ring sk_buffs */
4069         while (i != rx_ring->next_to_alloc) {
4070                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
4071
4072                 /* Invalidate cache lines that may have been written to by
4073                  * device so that we avoid corrupting memory.
4074                  */
4075                 dma_sync_single_range_for_cpu(rx_ring->dev,
4076                                               buffer_info->dma,
4077                                               buffer_info->page_offset,
4078                                               igb_rx_bufsz(rx_ring),
4079                                               DMA_FROM_DEVICE);
4080
4081                 /* free resources associated with mapping */
4082                 dma_unmap_page_attrs(rx_ring->dev,
4083                                      buffer_info->dma,
4084                                      igb_rx_pg_size(rx_ring),
4085                                      DMA_FROM_DEVICE,
4086                                      IGB_RX_DMA_ATTR);
4087                 __page_frag_cache_drain(buffer_info->page,
4088                                         buffer_info->pagecnt_bias);
4089
4090                 i++;
4091                 if (i == rx_ring->count)
4092                         i = 0;
4093         }
4094
4095         rx_ring->next_to_alloc = 0;
4096         rx_ring->next_to_clean = 0;
4097         rx_ring->next_to_use = 0;
4098 }
4099
4100 /**
4101  *  igb_clean_all_rx_rings - Free Rx Buffers for all queues
4102  *  @adapter: board private structure
4103  **/
4104 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
4105 {
4106         int i;
4107
4108         for (i = 0; i < adapter->num_rx_queues; i++)
4109                 if (adapter->rx_ring[i])
4110                         igb_clean_rx_ring(adapter->rx_ring[i]);
4111 }
4112
4113 /**
4114  *  igb_set_mac - Change the Ethernet Address of the NIC
4115  *  @netdev: network interface device structure
4116  *  @p: pointer to an address structure
4117  *
4118  *  Returns 0 on success, negative on failure
4119  **/
4120 static int igb_set_mac(struct net_device *netdev, void *p)
4121 {
4122         struct igb_adapter *adapter = netdev_priv(netdev);
4123         struct e1000_hw *hw = &adapter->hw;
4124         struct sockaddr *addr = p;
4125
4126         if (!is_valid_ether_addr(addr->sa_data))
4127                 return -EADDRNOTAVAIL;
4128
4129         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4130         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
4131
4132         /* set the correct pool for the new PF MAC address in entry 0 */
4133         igb_set_default_mac_filter(adapter);
4134
4135         return 0;
4136 }
4137
4138 /**
4139  *  igb_write_mc_addr_list - write multicast addresses to MTA
4140  *  @netdev: network interface device structure
4141  *
4142  *  Writes multicast address list to the MTA hash table.
4143  *  Returns: -ENOMEM on failure
4144  *           0 on no addresses written
4145  *           X on writing X addresses to MTA
4146  **/
4147 static int igb_write_mc_addr_list(struct net_device *netdev)
4148 {
4149         struct igb_adapter *adapter = netdev_priv(netdev);
4150         struct e1000_hw *hw = &adapter->hw;
4151         struct netdev_hw_addr *ha;
4152         u8  *mta_list;
4153         int i;
4154
4155         if (netdev_mc_empty(netdev)) {
4156                 /* nothing to program, so clear mc list */
4157                 igb_update_mc_addr_list(hw, NULL, 0);
4158                 igb_restore_vf_multicasts(adapter);
4159                 return 0;
4160         }
4161
4162         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
4163         if (!mta_list)
4164                 return -ENOMEM;
4165
4166         /* The shared function expects a packed array of only addresses. */
4167         i = 0;
4168         netdev_for_each_mc_addr(ha, netdev)
4169                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
4170
4171         igb_update_mc_addr_list(hw, mta_list, i);
4172         kfree(mta_list);
4173
4174         return netdev_mc_count(netdev);
4175 }
4176
4177 static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
4178 {
4179         struct e1000_hw *hw = &adapter->hw;
4180         u32 i, pf_id;
4181
4182         switch (hw->mac.type) {
4183         case e1000_i210:
4184         case e1000_i211:
4185         case e1000_i350:
4186                 /* VLAN filtering needed for VLAN prio filter */
4187                 if (adapter->netdev->features & NETIF_F_NTUPLE)
4188                         break;
4189                 /* fall through */
4190         case e1000_82576:
4191         case e1000_82580:
4192         case e1000_i354:
4193                 /* VLAN filtering needed for pool filtering */
4194                 if (adapter->vfs_allocated_count)
4195                         break;
4196                 /* fall through */
4197         default:
4198                 return 1;
4199         }
4200
4201         /* We are already in VLAN promisc, nothing to do */
4202         if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
4203                 return 0;
4204
4205         if (!adapter->vfs_allocated_count)
4206                 goto set_vfta;
4207
4208         /* Add PF to all active pools */
4209         pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
4210
4211         for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
4212                 u32 vlvf = rd32(E1000_VLVF(i));
4213
4214                 vlvf |= BIT(pf_id);
4215                 wr32(E1000_VLVF(i), vlvf);
4216         }
4217
4218 set_vfta:
4219         /* Set all bits in the VLAN filter table array */
4220         for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
4221                 hw->mac.ops.write_vfta(hw, i, ~0U);
4222
4223         /* Set flag so we don't redo unnecessary work */
4224         adapter->flags |= IGB_FLAG_VLAN_PROMISC;
4225
4226         return 0;
4227 }
4228
4229 #define VFTA_BLOCK_SIZE 8
4230 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
4231 {
4232         struct e1000_hw *hw = &adapter->hw;
4233         u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
4234         u32 vid_start = vfta_offset * 32;
4235         u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
4236         u32 i, vid, word, bits, pf_id;
4237
4238         /* guarantee that we don't scrub out management VLAN */
4239         vid = adapter->mng_vlan_id;
4240         if (vid >= vid_start && vid < vid_end)
4241                 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
4242
4243         if (!adapter->vfs_allocated_count)
4244                 goto set_vfta;
4245
4246         pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
4247
4248         for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
4249                 u32 vlvf = rd32(E1000_VLVF(i));
4250
4251                 /* pull VLAN ID from VLVF */
4252                 vid = vlvf & VLAN_VID_MASK;
4253
4254                 /* only concern ourselves with a certain range */
4255                 if (vid < vid_start || vid >= vid_end)
4256                         continue;
4257
4258                 if (vlvf & E1000_VLVF_VLANID_ENABLE) {
4259                         /* record VLAN ID in VFTA */
4260                         vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
4261
4262                         /* if PF is part of this then continue */
4263                         if (test_bit(vid, adapter->active_vlans))
4264                                 continue;
4265                 }
4266
4267                 /* remove PF from the pool */
4268                 bits = ~BIT(pf_id);
4269                 bits &= rd32(E1000_VLVF(i));
4270                 wr32(E1000_VLVF(i), bits);
4271         }
4272
4273 set_vfta:
4274         /* extract values from active_vlans and write back to VFTA */
4275         for (i = VFTA_BLOCK_SIZE; i--;) {
4276                 vid = (vfta_offset + i) * 32;
4277                 word = vid / BITS_PER_LONG;
4278                 bits = vid % BITS_PER_LONG;
4279
4280                 vfta[i] |= adapter->active_vlans[word] >> bits;
4281
4282                 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
4283         }
4284 }
4285
4286 static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
4287 {
4288         u32 i;
4289
4290         /* We are not in VLAN promisc, nothing to do */
4291         if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
4292                 return;
4293
4294         /* Set flag so we don't redo unnecessary work */
4295         adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
4296
4297         for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
4298                 igb_scrub_vfta(adapter, i);
4299 }
4300
4301 /**
4302  *  igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
4303  *  @netdev: network interface device structure
4304  *
4305  *  The set_rx_mode entry point is called whenever the unicast or multicast
4306  *  address lists or the network interface flags are updated.  This routine is
4307  *  responsible for configuring the hardware for proper unicast, multicast,
4308  *  promiscuous mode, and all-multi behavior.
4309  **/
4310 static void igb_set_rx_mode(struct net_device *netdev)
4311 {
4312         struct igb_adapter *adapter = netdev_priv(netdev);
4313         struct e1000_hw *hw = &adapter->hw;
4314         unsigned int vfn = adapter->vfs_allocated_count;
4315         u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
4316         int count;
4317
4318         /* Check for Promiscuous and All Multicast modes */
4319         if (netdev->flags & IFF_PROMISC) {
4320                 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
4321                 vmolr |= E1000_VMOLR_MPME;
4322
4323                 /* enable use of UTA filter to force packets to default pool */
4324                 if (hw->mac.type == e1000_82576)
4325                         vmolr |= E1000_VMOLR_ROPE;
4326         } else {
4327                 if (netdev->flags & IFF_ALLMULTI) {
4328                         rctl |= E1000_RCTL_MPE;
4329                         vmolr |= E1000_VMOLR_MPME;
4330                 } else {
4331                         /* Write addresses to the MTA, if the attempt fails
4332                          * then we should just turn on promiscuous mode so
4333                          * that we can at least receive multicast traffic
4334                          */
4335                         count = igb_write_mc_addr_list(netdev);
4336                         if (count < 0) {
4337                                 rctl |= E1000_RCTL_MPE;
4338                                 vmolr |= E1000_VMOLR_MPME;
4339                         } else if (count) {
4340                                 vmolr |= E1000_VMOLR_ROMPE;
4341                         }
4342                 }
4343         }
4344
4345         /* Write addresses to available RAR registers, if there is not
4346          * sufficient space to store all the addresses then enable
4347          * unicast promiscuous mode
4348          */
4349         if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
4350                 rctl |= E1000_RCTL_UPE;
4351                 vmolr |= E1000_VMOLR_ROPE;
4352         }
4353
4354         /* enable VLAN filtering by default */
4355         rctl |= E1000_RCTL_VFE;
4356
4357         /* disable VLAN filtering for modes that require it */
4358         if ((netdev->flags & IFF_PROMISC) ||
4359             (netdev->features & NETIF_F_RXALL)) {
4360                 /* if we fail to set all rules then just clear VFE */
4361                 if (igb_vlan_promisc_enable(adapter))
4362                         rctl &= ~E1000_RCTL_VFE;
4363         } else {
4364                 igb_vlan_promisc_disable(adapter);
4365         }
4366
4367         /* update state of unicast, multicast, and VLAN filtering modes */
4368         rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
4369                                      E1000_RCTL_VFE);
4370         wr32(E1000_RCTL, rctl);
4371
4372 #if (PAGE_SIZE < 8192)
4373         if (!adapter->vfs_allocated_count) {
4374                 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
4375                         rlpml = IGB_MAX_FRAME_BUILD_SKB;
4376         }
4377 #endif
4378         wr32(E1000_RLPML, rlpml);
4379
4380         /* In order to support SR-IOV and eventually VMDq it is necessary to set
4381          * the VMOLR to enable the appropriate modes.  Without this workaround
4382          * we will have issues with VLAN tag stripping not being done for frames
4383          * that are only arriving because we are the default pool
4384          */
4385         if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
4386                 return;
4387
4388         /* set UTA to appropriate mode */
4389         igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
4390
4391         vmolr |= rd32(E1000_VMOLR(vfn)) &
4392                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
4393
4394         /* enable Rx jumbo frames, restrict as needed to support build_skb */
4395         vmolr &= ~E1000_VMOLR_RLPML_MASK;
4396 #if (PAGE_SIZE < 8192)
4397         if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
4398                 vmolr |= IGB_MAX_FRAME_BUILD_SKB;
4399         else
4400 #endif
4401                 vmolr |= MAX_JUMBO_FRAME_SIZE;
4402         vmolr |= E1000_VMOLR_LPE;
4403
4404         wr32(E1000_VMOLR(vfn), vmolr);
4405
4406         igb_restore_vf_multicasts(adapter);
4407 }
4408
4409 static void igb_check_wvbr(struct igb_adapter *adapter)
4410 {
4411         struct e1000_hw *hw = &adapter->hw;
4412         u32 wvbr = 0;
4413
4414         switch (hw->mac.type) {
4415         case e1000_82576:
4416         case e1000_i350:
4417                 wvbr = rd32(E1000_WVBR);
4418                 if (!wvbr)
4419                         return;
4420                 break;
4421         default:
4422                 break;
4423         }
4424
4425         adapter->wvbr |= wvbr;
4426 }
4427
4428 #define IGB_STAGGERED_QUEUE_OFFSET 8
4429
4430 static void igb_spoof_check(struct igb_adapter *adapter)
4431 {
4432         int j;
4433
4434         if (!adapter->wvbr)
4435                 return;
4436
4437         for (j = 0; j < adapter->vfs_allocated_count; j++) {
4438                 if (adapter->wvbr & BIT(j) ||
4439                     adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
4440                         dev_warn(&adapter->pdev->dev,
4441                                 "Spoof event(s) detected on VF %d\n", j);
4442                         adapter->wvbr &=
4443                                 ~(BIT(j) |
4444                                   BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
4445                 }
4446         }
4447 }
4448
4449 /* Need to wait a few seconds after link up to get diagnostic information from
4450  * the phy
4451  */
4452 static void igb_update_phy_info(unsigned long data)
4453 {
4454         struct igb_adapter *adapter = (struct igb_adapter *) data;
4455         igb_get_phy_info(&adapter->hw);
4456 }
4457
4458 /**
4459  *  igb_has_link - check shared code for link and determine up/down
4460  *  @adapter: pointer to driver private info
4461  **/
4462 bool igb_has_link(struct igb_adapter *adapter)
4463 {
4464         struct e1000_hw *hw = &adapter->hw;
4465         bool link_active = false;
4466
4467         /* get_link_status is set on LSC (link status) interrupt or
4468          * rx sequence error interrupt.  get_link_status will stay
4469          * false until the e1000_check_for_link establishes link
4470          * for copper adapters ONLY
4471          */
4472         switch (hw->phy.media_type) {
4473         case e1000_media_type_copper:
4474                 if (!hw->mac.get_link_status)
4475                         return true;
4476         case e1000_media_type_internal_serdes:
4477                 hw->mac.ops.check_for_link(hw);
4478                 link_active = !hw->mac.get_link_status;
4479                 break;
4480         default:
4481         case e1000_media_type_unknown:
4482                 break;
4483         }
4484
4485         if (((hw->mac.type == e1000_i210) ||
4486              (hw->mac.type == e1000_i211)) &&
4487              (hw->phy.id == I210_I_PHY_ID)) {
4488                 if (!netif_carrier_ok(adapter->netdev)) {
4489                         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
4490                 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
4491                         adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
4492                         adapter->link_check_timeout = jiffies;
4493                 }
4494         }
4495
4496         return link_active;
4497 }
4498
4499 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
4500 {
4501         bool ret = false;
4502         u32 ctrl_ext, thstat;
4503
4504         /* check for thermal sensor event on i350 copper only */
4505         if (hw->mac.type == e1000_i350) {
4506                 thstat = rd32(E1000_THSTAT);
4507                 ctrl_ext = rd32(E1000_CTRL_EXT);
4508
4509                 if ((hw->phy.media_type == e1000_media_type_copper) &&
4510                     !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
4511                         ret = !!(thstat & event);
4512         }
4513
4514         return ret;
4515 }
4516
4517 /**
4518  *  igb_check_lvmmc - check for malformed packets received
4519  *  and indicated in LVMMC register
4520  *  @adapter: pointer to adapter
4521  **/
4522 static void igb_check_lvmmc(struct igb_adapter *adapter)
4523 {
4524         struct e1000_hw *hw = &adapter->hw;
4525         u32 lvmmc;
4526
4527         lvmmc = rd32(E1000_LVMMC);
4528         if (lvmmc) {
4529                 if (unlikely(net_ratelimit())) {
4530                         netdev_warn(adapter->netdev,
4531                                     "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
4532                                     lvmmc);
4533                 }
4534         }
4535 }
4536
4537 /**
4538  *  igb_watchdog - Timer Call-back
4539  *  @data: pointer to adapter cast into an unsigned long
4540  **/
4541 static void igb_watchdog(unsigned long data)
4542 {
4543         struct igb_adapter *adapter = (struct igb_adapter *)data;
4544         /* Do the rest outside of interrupt context */
4545         schedule_work(&adapter->watchdog_task);
4546 }
4547
4548 static void igb_watchdog_task(struct work_struct *work)
4549 {
4550         struct igb_adapter *adapter = container_of(work,
4551                                                    struct igb_adapter,
4552                                                    watchdog_task);
4553         struct e1000_hw *hw = &adapter->hw;
4554         struct e1000_phy_info *phy = &hw->phy;
4555         struct net_device *netdev = adapter->netdev;
4556         u32 link;
4557         int i;
4558         u32 connsw;
4559         u16 phy_data, retry_count = 20;
4560
4561         link = igb_has_link(adapter);
4562
4563         if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
4564                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
4565                         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
4566                 else
4567                         link = false;
4568         }
4569
4570         /* Force link down if we have fiber to swap to */
4571         if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
4572                 if (hw->phy.media_type == e1000_media_type_copper) {
4573                         connsw = rd32(E1000_CONNSW);
4574                         if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
4575                                 link = 0;
4576                 }
4577         }
4578         if (link) {
4579                 /* Perform a reset if the media type changed. */
4580                 if (hw->dev_spec._82575.media_changed) {
4581                         hw->dev_spec._82575.media_changed = false;
4582                         adapter->flags |= IGB_FLAG_MEDIA_RESET;
4583                         igb_reset(adapter);
4584                 }
4585                 /* Cancel scheduled suspend requests. */
4586                 pm_runtime_resume(netdev->dev.parent);
4587
4588                 if (!netif_carrier_ok(netdev)) {
4589                         u32 ctrl;
4590
4591                         hw->mac.ops.get_speed_and_duplex(hw,
4592                                                          &adapter->link_speed,
4593                                                          &adapter->link_duplex);
4594
4595                         ctrl = rd32(E1000_CTRL);
4596                         /* Links status message must follow this format */
4597                         netdev_info(netdev,
4598                                "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4599                                netdev->name,
4600                                adapter->link_speed,
4601                                adapter->link_duplex == FULL_DUPLEX ?
4602                                "Full" : "Half",
4603                                (ctrl & E1000_CTRL_TFCE) &&
4604                                (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
4605                                (ctrl & E1000_CTRL_RFCE) ?  "RX" :
4606                                (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
4607
4608                         /* disable EEE if enabled */
4609                         if ((adapter->flags & IGB_FLAG_EEE) &&
4610                                 (adapter->link_duplex == HALF_DUPLEX)) {
4611                                 dev_info(&adapter->pdev->dev,
4612                                 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
4613                                 adapter->hw.dev_spec._82575.eee_disable = true;
4614                                 adapter->flags &= ~IGB_FLAG_EEE;
4615                         }
4616
4617                         /* check if SmartSpeed worked */
4618                         igb_check_downshift(hw);
4619                         if (phy->speed_downgraded)
4620                                 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
4621
4622                         /* check for thermal sensor event */
4623                         if (igb_thermal_sensor_event(hw,
4624                             E1000_THSTAT_LINK_THROTTLE))
4625                                 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
4626
4627                         /* adjust timeout factor according to speed/duplex */
4628                         adapter->tx_timeout_factor = 1;
4629                         switch (adapter->link_speed) {
4630                         case SPEED_10:
4631                                 adapter->tx_timeout_factor = 14;
4632                                 break;
4633                         case SPEED_100:
4634                                 /* maybe add some timeout factor ? */
4635                                 break;
4636                         }
4637
4638                         if (adapter->link_speed != SPEED_1000 ||
4639                             !hw->phy.ops.read_reg)
4640                                 goto no_wait;
4641
4642                         /* wait for Remote receiver status OK */
4643 retry_read_status:
4644                         if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
4645                                               &phy_data)) {
4646                                 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
4647                                     retry_count) {
4648                                         msleep(100);
4649                                         retry_count--;
4650                                         goto retry_read_status;
4651                                 } else if (!retry_count) {
4652                                         dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
4653                                 }
4654                         } else {
4655                                 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
4656                         }
4657 no_wait:
4658                         netif_carrier_on(netdev);
4659
4660                         igb_ping_all_vfs(adapter);
4661                         igb_check_vf_rate_limit(adapter);
4662
4663                         /* link state has changed, schedule phy info update */
4664                         if (!test_bit(__IGB_DOWN, &adapter->state))
4665                                 mod_timer(&adapter->phy_info_timer,
4666                                           round_jiffies(jiffies + 2 * HZ));
4667                 }
4668         } else {
4669                 if (netif_carrier_ok(netdev)) {
4670                         adapter->link_speed = 0;
4671                         adapter->link_duplex = 0;
4672
4673                         /* check for thermal sensor event */
4674                         if (igb_thermal_sensor_event(hw,
4675                             E1000_THSTAT_PWR_DOWN)) {
4676                                 netdev_err(netdev, "The network adapter was stopped because it overheated\n");
4677                         }
4678
4679                         /* Links status message must follow this format */
4680                         netdev_info(netdev, "igb: %s NIC Link is Down\n",
4681                                netdev->name);
4682                         netif_carrier_off(netdev);
4683
4684                         igb_ping_all_vfs(adapter);
4685
4686                         /* link state has changed, schedule phy info update */
4687                         if (!test_bit(__IGB_DOWN, &adapter->state))
4688                                 mod_timer(&adapter->phy_info_timer,
4689                                           round_jiffies(jiffies + 2 * HZ));
4690
4691                         /* link is down, time to check for alternate media */
4692                         if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
4693                                 igb_check_swap_media(adapter);
4694                                 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
4695                                         schedule_work(&adapter->reset_task);
4696                                         /* return immediately */
4697                                         return;
4698                                 }
4699                         }
4700                         pm_schedule_suspend(netdev->dev.parent,
4701                                             MSEC_PER_SEC * 5);
4702
4703                 /* also check for alternate media here */
4704                 } else if (!netif_carrier_ok(netdev) &&
4705                            (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
4706                         igb_check_swap_media(adapter);
4707                         if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
4708                                 schedule_work(&adapter->reset_task);
4709                                 /* return immediately */
4710                                 return;
4711                         }
4712                 }
4713         }
4714
4715         spin_lock(&adapter->stats64_lock);
4716         igb_update_stats(adapter);
4717         spin_unlock(&adapter->stats64_lock);
4718
4719         for (i = 0; i < adapter->num_tx_queues; i++) {
4720                 struct igb_ring *tx_ring = adapter->tx_ring[i];
4721                 if (!netif_carrier_ok(netdev)) {
4722                         /* We've lost link, so the controller stops DMA,
4723                          * but we've got queued Tx work that's never going
4724                          * to get done, so reset controller to flush Tx.
4725                          * (Do the reset outside of interrupt context).
4726                          */
4727                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4728                                 adapter->tx_timeout_count++;
4729                                 schedule_work(&adapter->reset_task);
4730                                 /* return immediately since reset is imminent */
4731                                 return;
4732                         }
4733                 }
4734
4735                 /* Force detection of hung controller every watchdog period */
4736                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4737         }
4738
4739         /* Cause software interrupt to ensure Rx ring is cleaned */
4740         if (adapter->flags & IGB_FLAG_HAS_MSIX) {
4741                 u32 eics = 0;
4742
4743                 for (i = 0; i < adapter->num_q_vectors; i++)
4744                         eics |= adapter->q_vector[i]->eims_value;
4745                 wr32(E1000_EICS, eics);
4746         } else {
4747                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
4748         }
4749
4750         igb_spoof_check(adapter);
4751         igb_ptp_rx_hang(adapter);
4752         igb_ptp_tx_hang(adapter);
4753
4754         /* Check LVMMC register on i350/i354 only */
4755         if ((adapter->hw.mac.type == e1000_i350) ||
4756             (adapter->hw.mac.type == e1000_i354))
4757                 igb_check_lvmmc(adapter);
4758
4759         /* Reset the timer */
4760         if (!test_bit(__IGB_DOWN, &adapter->state)) {
4761                 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
4762                         mod_timer(&adapter->watchdog_timer,
4763                                   round_jiffies(jiffies +  HZ));
4764                 else
4765                         mod_timer(&adapter->watchdog_timer,
4766                                   round_jiffies(jiffies + 2 * HZ));
4767         }
4768 }
4769
4770 enum latency_range {
4771         lowest_latency = 0,
4772         low_latency = 1,
4773         bulk_latency = 2,
4774         latency_invalid = 255
4775 };
4776
4777 /**
4778  *  igb_update_ring_itr - update the dynamic ITR value based on packet size
4779  *  @q_vector: pointer to q_vector
4780  *
4781  *  Stores a new ITR value based on strictly on packet size.  This
4782  *  algorithm is less sophisticated than that used in igb_update_itr,
4783  *  due to the difficulty of synchronizing statistics across multiple
4784  *  receive rings.  The divisors and thresholds used by this function
4785  *  were determined based on theoretical maximum wire speed and testing
4786  *  data, in order to minimize response time while increasing bulk
4787  *  throughput.
4788  *  This functionality is controlled by ethtool's coalescing settings.
4789  *  NOTE:  This function is called only when operating in a multiqueue
4790  *         receive environment.
4791  **/
4792 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4793 {
4794         int new_val = q_vector->itr_val;
4795         int avg_wire_size = 0;
4796         struct igb_adapter *adapter = q_vector->adapter;
4797         unsigned int packets;
4798
4799         /* For non-gigabit speeds, just fix the interrupt rate at 4000
4800          * ints/sec - ITR timer value of 120 ticks.
4801          */
4802         if (adapter->link_speed != SPEED_1000) {
4803                 new_val = IGB_4K_ITR;
4804                 goto set_itr_val;
4805         }
4806
4807         packets = q_vector->rx.total_packets;
4808         if (packets)
4809                 avg_wire_size = q_vector->rx.total_bytes / packets;
4810
4811         packets = q_vector->tx.total_packets;
4812         if (packets)
4813                 avg_wire_size = max_t(u32, avg_wire_size,
4814                                       q_vector->tx.total_bytes / packets);
4815
4816         /* if avg_wire_size isn't set no work was done */
4817         if (!avg_wire_size)
4818                 goto clear_counts;
4819
4820         /* Add 24 bytes to size to account for CRC, preamble, and gap */
4821         avg_wire_size += 24;
4822
4823         /* Don't starve jumbo frames */
4824         avg_wire_size = min(avg_wire_size, 3000);
4825
4826         /* Give a little boost to mid-size frames */
4827         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4828                 new_val = avg_wire_size / 3;
4829         else
4830                 new_val = avg_wire_size / 2;
4831
4832         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4833         if (new_val < IGB_20K_ITR &&
4834             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4835              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4836                 new_val = IGB_20K_ITR;
4837
4838 set_itr_val:
4839         if (new_val != q_vector->itr_val) {
4840                 q_vector->itr_val = new_val;
4841                 q_vector->set_itr = 1;
4842         }
4843 clear_counts:
4844         q_vector->rx.total_bytes = 0;
4845         q_vector->rx.total_packets = 0;
4846         q_vector->tx.total_bytes = 0;
4847         q_vector->tx.total_packets = 0;
4848 }
4849
4850 /**
4851  *  igb_update_itr - update the dynamic ITR value based on statistics
4852  *  @q_vector: pointer to q_vector
4853  *  @ring_container: ring info to update the itr for
4854  *
4855  *  Stores a new ITR value based on packets and byte
4856  *  counts during the last interrupt.  The advantage of per interrupt
4857  *  computation is faster updates and more accurate ITR for the current
4858  *  traffic pattern.  Constants in this function were computed
4859  *  based on theoretical maximum wire speed and thresholds were set based
4860  *  on testing data as well as attempting to minimize response time
4861  *  while increasing bulk throughput.
4862  *  This functionality is controlled by ethtool's coalescing settings.
4863  *  NOTE:  These calculations are only valid when operating in a single-
4864  *         queue environment.
4865  **/
4866 static void igb_update_itr(struct igb_q_vector *q_vector,
4867                            struct igb_ring_container *ring_container)
4868 {
4869         unsigned int packets = ring_container->total_packets;
4870         unsigned int bytes = ring_container->total_bytes;
4871         u8 itrval = ring_container->itr;
4872
4873         /* no packets, exit with status unchanged */
4874         if (packets == 0)
4875                 return;
4876
4877         switch (itrval) {
4878         case lowest_latency:
4879                 /* handle TSO and jumbo frames */
4880                 if (bytes/packets > 8000)
4881                         itrval = bulk_latency;
4882                 else if ((packets < 5) && (bytes > 512))
4883                         itrval = low_latency;
4884                 break;
4885         case low_latency:  /* 50 usec aka 20000 ints/s */
4886                 if (bytes > 10000) {
4887                         /* this if handles the TSO accounting */
4888                         if (bytes/packets > 8000)
4889                                 itrval = bulk_latency;
4890                         else if ((packets < 10) || ((bytes/packets) > 1200))
4891                                 itrval = bulk_latency;
4892                         else if ((packets > 35))
4893                                 itrval = lowest_latency;
4894                 } else if (bytes/packets > 2000) {
4895                         itrval = bulk_latency;
4896                 } else if (packets <= 2 && bytes < 512) {
4897                         itrval = lowest_latency;
4898                 }
4899                 break;
4900         case bulk_latency: /* 250 usec aka 4000 ints/s */
4901                 if (bytes > 25000) {
4902                         if (packets > 35)
4903                                 itrval = low_latency;
4904                 } else if (bytes < 1500) {
4905                         itrval = low_latency;
4906                 }
4907                 break;
4908         }
4909
4910         /* clear work counters since we have the values we need */
4911         ring_container->total_bytes = 0;
4912         ring_container->total_packets = 0;
4913
4914         /* write updated itr to ring container */
4915         ring_container->itr = itrval;
4916 }
4917
4918 static void igb_set_itr(struct igb_q_vector *q_vector)
4919 {
4920         struct igb_adapter *adapter = q_vector->adapter;
4921         u32 new_itr = q_vector->itr_val;
4922         u8 current_itr = 0;
4923
4924         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4925         if (adapter->link_speed != SPEED_1000) {
4926                 current_itr = 0;
4927                 new_itr = IGB_4K_ITR;
4928                 goto set_itr_now;
4929         }
4930
4931         igb_update_itr(q_vector, &q_vector->tx);
4932         igb_update_itr(q_vector, &q_vector->rx);
4933
4934         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4935
4936         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4937         if (current_itr == lowest_latency &&
4938             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4939              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4940                 current_itr = low_latency;
4941
4942         switch (current_itr) {
4943         /* counts and packets in update_itr are dependent on these numbers */
4944         case lowest_latency:
4945                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4946                 break;
4947         case low_latency:
4948                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4949                 break;
4950         case bulk_latency:
4951                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4952                 break;
4953         default:
4954                 break;
4955         }
4956
4957 set_itr_now:
4958         if (new_itr != q_vector->itr_val) {
4959                 /* this attempts to bias the interrupt rate towards Bulk
4960                  * by adding intermediate steps when interrupt rate is
4961                  * increasing
4962                  */
4963                 new_itr = new_itr > q_vector->itr_val ?
4964                           max((new_itr * q_vector->itr_val) /
4965                           (new_itr + (q_vector->itr_val >> 2)),
4966                           new_itr) : new_itr;
4967                 /* Don't write the value here; it resets the adapter's
4968                  * internal timer, and causes us to delay far longer than
4969                  * we should between interrupts.  Instead, we write the ITR
4970                  * value at the beginning of the next interrupt so the timing
4971                  * ends up being correct.
4972                  */
4973                 q_vector->itr_val = new_itr;
4974                 q_vector->set_itr = 1;
4975         }
4976 }
4977
4978 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4979                             u32 type_tucmd, u32 mss_l4len_idx)
4980 {
4981         struct e1000_adv_tx_context_desc *context_desc;
4982         u16 i = tx_ring->next_to_use;
4983
4984         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4985
4986         i++;
4987         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4988
4989         /* set bits to identify this as an advanced context descriptor */
4990         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4991
4992         /* For 82575, context index must be unique per ring. */
4993         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4994                 mss_l4len_idx |= tx_ring->reg_idx << 4;
4995
4996         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
4997         context_desc->seqnum_seed       = 0;
4998         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
4999         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
5000 }
5001
5002 static int igb_tso(struct igb_ring *tx_ring,
5003                    struct igb_tx_buffer *first,
5004                    u8 *hdr_len)
5005 {
5006         u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5007         struct sk_buff *skb = first->skb;
5008         union {
5009                 struct iphdr *v4;
5010                 struct ipv6hdr *v6;
5011                 unsigned char *hdr;
5012         } ip;
5013         union {
5014                 struct tcphdr *tcp;
5015                 unsigned char *hdr;
5016         } l4;
5017         u32 paylen, l4_offset;
5018         int err;
5019
5020         if (skb->ip_summed != CHECKSUM_PARTIAL)
5021                 return 0;
5022
5023         if (!skb_is_gso(skb))
5024                 return 0;
5025
5026         err = skb_cow_head(skb, 0);
5027         if (err < 0)
5028                 return err;
5029
5030         ip.hdr = skb_network_header(skb);
5031         l4.hdr = skb_checksum_start(skb);
5032
5033         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5034         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
5035
5036         /* initialize outer IP header fields */
5037         if (ip.v4->version == 4) {
5038                 unsigned char *csum_start = skb_checksum_start(skb);
5039                 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
5040
5041                 /* IP header will have to cancel out any data that
5042                  * is not a part of the outer IP header
5043                  */
5044                 ip.v4->check = csum_fold(csum_partial(trans_start,
5045                                                       csum_start - trans_start,
5046                                                       0));
5047                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
5048
5049                 ip.v4->tot_len = 0;
5050                 first->tx_flags |= IGB_TX_FLAGS_TSO |
5051                                    IGB_TX_FLAGS_CSUM |
5052                                    IGB_TX_FLAGS_IPV4;
5053         } else {
5054                 ip.v6->payload_len = 0;
5055                 first->tx_flags |= IGB_TX_FLAGS_TSO |
5056                                    IGB_TX_FLAGS_CSUM;
5057         }
5058
5059         /* determine offset of inner transport header */
5060         l4_offset = l4.hdr - skb->data;
5061
5062         /* compute length of segmentation header */
5063         *hdr_len = (l4.tcp->doff * 4) + l4_offset;
5064
5065         /* remove payload length from inner checksum */
5066         paylen = skb->len - l4_offset;
5067         csum_replace_by_diff(&l4.tcp->check, htonl(paylen));
5068
5069         /* update gso size and bytecount with header size */
5070         first->gso_segs = skb_shinfo(skb)->gso_segs;
5071         first->bytecount += (first->gso_segs - 1) * *hdr_len;
5072
5073         /* MSS L4LEN IDX */
5074         mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
5075         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
5076
5077         /* VLAN MACLEN IPLEN */
5078         vlan_macip_lens = l4.hdr - ip.hdr;
5079         vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
5080         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5081
5082         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
5083
5084         return 1;
5085 }
5086
5087 static inline bool igb_ipv6_csum_is_sctp(struct sk_buff *skb)
5088 {
5089         unsigned int offset = 0;
5090
5091         ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
5092
5093         return offset == skb_checksum_start_offset(skb);
5094 }
5095
5096 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
5097 {
5098         struct sk_buff *skb = first->skb;
5099         u32 vlan_macip_lens = 0;
5100         u32 type_tucmd = 0;
5101
5102         if (skb->ip_summed != CHECKSUM_PARTIAL) {
5103 csum_failed:
5104                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
5105                         return;
5106                 goto no_csum;
5107         }
5108
5109         switch (skb->csum_offset) {
5110         case offsetof(struct tcphdr, check):
5111                 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
5112                 /* fall through */
5113         case offsetof(struct udphdr, check):
5114                 break;
5115         case offsetof(struct sctphdr, checksum):
5116                 /* validate that this is actually an SCTP request */
5117                 if (((first->protocol == htons(ETH_P_IP)) &&
5118                      (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
5119                     ((first->protocol == htons(ETH_P_IPV6)) &&
5120                      igb_ipv6_csum_is_sctp(skb))) {
5121                         type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
5122                         break;
5123                 }
5124         default:
5125                 skb_checksum_help(skb);
5126                 goto csum_failed;
5127         }
5128
5129         /* update TX checksum flag */
5130         first->tx_flags |= IGB_TX_FLAGS_CSUM;
5131         vlan_macip_lens = skb_checksum_start_offset(skb) -
5132                           skb_network_offset(skb);
5133 no_csum:
5134         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
5135         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5136
5137         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
5138 }
5139
5140 #define IGB_SET_FLAG(_input, _flag, _result) \
5141         ((_flag <= _result) ? \
5142          ((u32)(_input & _flag) * (_result / _flag)) : \
5143          ((u32)(_input & _flag) / (_flag / _result)))
5144
5145 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
5146 {
5147         /* set type for advanced descriptor with frame checksum insertion */
5148         u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
5149                        E1000_ADVTXD_DCMD_DEXT |
5150                        E1000_ADVTXD_DCMD_IFCS;
5151
5152         /* set HW vlan bit if vlan is present */
5153         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
5154                                  (E1000_ADVTXD_DCMD_VLE));
5155
5156         /* set segmentation bits for TSO */
5157         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
5158                                  (E1000_ADVTXD_DCMD_TSE));
5159
5160         /* set timestamp bit if present */
5161         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
5162                                  (E1000_ADVTXD_MAC_TSTAMP));
5163
5164         /* insert frame checksum */
5165         cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
5166
5167         return cmd_type;
5168 }
5169
5170 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
5171                                  union e1000_adv_tx_desc *tx_desc,
5172                                  u32 tx_flags, unsigned int paylen)
5173 {
5174         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
5175
5176         /* 82575 requires a unique index per ring */
5177         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5178                 olinfo_status |= tx_ring->reg_idx << 4;
5179
5180         /* insert L4 checksum */
5181         olinfo_status |= IGB_SET_FLAG(tx_flags,
5182                                       IGB_TX_FLAGS_CSUM,
5183                                       (E1000_TXD_POPTS_TXSM << 8));
5184
5185         /* insert IPv4 checksum */
5186         olinfo_status |= IGB_SET_FLAG(tx_flags,
5187                                       IGB_TX_FLAGS_IPV4,
5188                                       (E1000_TXD_POPTS_IXSM << 8));
5189
5190         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
5191 }
5192
5193 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5194 {
5195         struct net_device *netdev = tx_ring->netdev;
5196
5197         netif_stop_subqueue(netdev, tx_ring->queue_index);
5198
5199         /* Herbert's original patch had:
5200          *  smp_mb__after_netif_stop_queue();
5201          * but since that doesn't exist yet, just open code it.
5202          */
5203         smp_mb();
5204
5205         /* We need to check again in a case another CPU has just
5206          * made room available.
5207          */
5208         if (igb_desc_unused(tx_ring) < size)
5209                 return -EBUSY;
5210
5211         /* A reprieve! */
5212         netif_wake_subqueue(netdev, tx_ring->queue_index);
5213
5214         u64_stats_update_begin(&tx_ring->tx_syncp2);
5215         tx_ring->tx_stats.restart_queue2++;
5216         u64_stats_update_end(&tx_ring->tx_syncp2);
5217
5218         return 0;
5219 }
5220
5221 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5222 {
5223         if (igb_desc_unused(tx_ring) >= size)
5224                 return 0;
5225         return __igb_maybe_stop_tx(tx_ring, size);
5226 }
5227
5228 static int igb_tx_map(struct igb_ring *tx_ring,
5229                       struct igb_tx_buffer *first,
5230                       const u8 hdr_len)
5231 {
5232         struct sk_buff *skb = first->skb;
5233         struct igb_tx_buffer *tx_buffer;
5234         union e1000_adv_tx_desc *tx_desc;
5235         struct skb_frag_struct *frag;
5236         dma_addr_t dma;
5237         unsigned int data_len, size;
5238         u32 tx_flags = first->tx_flags;
5239         u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
5240         u16 i = tx_ring->next_to_use;
5241
5242         tx_desc = IGB_TX_DESC(tx_ring, i);
5243
5244         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
5245
5246         size = skb_headlen(skb);
5247         data_len = skb->data_len;
5248
5249         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
5250
5251         tx_buffer = first;
5252
5253         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
5254                 if (dma_mapping_error(tx_ring->dev, dma))
5255                         goto dma_error;
5256
5257                 /* record length, and DMA address */
5258                 dma_unmap_len_set(tx_buffer, len, size);
5259                 dma_unmap_addr_set(tx_buffer, dma, dma);
5260
5261                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
5262
5263                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
5264                         tx_desc->read.cmd_type_len =
5265                                 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
5266
5267                         i++;
5268                         tx_desc++;
5269                         if (i == tx_ring->count) {
5270                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
5271                                 i = 0;
5272                         }
5273                         tx_desc->read.olinfo_status = 0;
5274
5275                         dma += IGB_MAX_DATA_PER_TXD;
5276                         size -= IGB_MAX_DATA_PER_TXD;
5277
5278                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
5279                 }
5280
5281                 if (likely(!data_len))
5282                         break;
5283
5284                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
5285
5286                 i++;
5287                 tx_desc++;
5288                 if (i == tx_ring->count) {
5289                         tx_desc = IGB_TX_DESC(tx_ring, 0);
5290                         i = 0;
5291                 }
5292                 tx_desc->read.olinfo_status = 0;
5293
5294                 size = skb_frag_size(frag);
5295                 data_len -= size;
5296
5297                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
5298                                        size, DMA_TO_DEVICE);
5299
5300                 tx_buffer = &tx_ring->tx_buffer_info[i];
5301         }
5302
5303         /* write last descriptor with RS and EOP bits */
5304         cmd_type |= size | IGB_TXD_DCMD;
5305         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
5306
5307         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
5308
5309         /* set the timestamp */
5310         first->time_stamp = jiffies;
5311
5312         /* Force memory writes to complete before letting h/w know there
5313          * are new descriptors to fetch.  (Only applicable for weak-ordered
5314          * memory model archs, such as IA-64).
5315          *
5316          * We also need this memory barrier to make certain all of the
5317          * status bits have been updated before next_to_watch is written.
5318          */
5319         wmb();
5320
5321         /* set next_to_watch value indicating a packet is present */
5322         first->next_to_watch = tx_desc;
5323
5324         i++;
5325         if (i == tx_ring->count)
5326                 i = 0;
5327
5328         tx_ring->next_to_use = i;
5329
5330         /* Make sure there is space in the ring for the next send. */
5331         igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
5332
5333         if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
5334                 writel(i, tx_ring->tail);
5335
5336                 /* we need this if more than one processor can write to our tail
5337                  * at a time, it synchronizes IO on IA64/Altix systems
5338                  */
5339                 mmiowb();
5340         }
5341         return 0;
5342
5343 dma_error:
5344         dev_err(tx_ring->dev, "TX DMA map failed\n");
5345         tx_buffer = &tx_ring->tx_buffer_info[i];
5346
5347         /* clear dma mappings for failed tx_buffer_info map */
5348         while (tx_buffer != first) {
5349                 if (dma_unmap_len(tx_buffer, len))
5350                         dma_unmap_page(tx_ring->dev,
5351                                        dma_unmap_addr(tx_buffer, dma),
5352                                        dma_unmap_len(tx_buffer, len),
5353                                        DMA_TO_DEVICE);
5354                 dma_unmap_len_set(tx_buffer, len, 0);
5355
5356                 if (i-- == 0)
5357                         i += tx_ring->count;
5358                 tx_buffer = &tx_ring->tx_buffer_info[i];
5359         }
5360
5361         if (dma_unmap_len(tx_buffer, len))
5362                 dma_unmap_single(tx_ring->dev,
5363                                  dma_unmap_addr(tx_buffer, dma),
5364                                  dma_unmap_len(tx_buffer, len),
5365                                  DMA_TO_DEVICE);
5366         dma_unmap_len_set(tx_buffer, len, 0);
5367
5368         dev_kfree_skb_any(tx_buffer->skb);
5369         tx_buffer->skb = NULL;
5370
5371         tx_ring->next_to_use = i;
5372
5373         return -1;
5374 }
5375
5376 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
5377                                 struct igb_ring *tx_ring)
5378 {
5379         struct igb_tx_buffer *first;
5380         int tso;
5381         u32 tx_flags = 0;
5382         unsigned short f;
5383         u16 count = TXD_USE_COUNT(skb_headlen(skb));
5384         __be16 protocol = vlan_get_protocol(skb);
5385         u8 hdr_len = 0;
5386
5387         /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
5388          *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
5389          *       + 2 desc gap to keep tail from touching head,
5390          *       + 1 desc for context descriptor,
5391          * otherwise try next time
5392          */
5393         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
5394                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
5395
5396         if (igb_maybe_stop_tx(tx_ring, count + 3)) {
5397                 /* this is a hard error */
5398                 return NETDEV_TX_BUSY;
5399         }
5400
5401         /* record the location of the first descriptor for this packet */
5402         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
5403         first->skb = skb;
5404         first->bytecount = skb->len;
5405         first->gso_segs = 1;
5406
5407         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
5408                 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
5409
5410                 if (adapter->tstamp_config.tx_type & HWTSTAMP_TX_ON &&
5411                     !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
5412                                            &adapter->state)) {
5413                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5414                         tx_flags |= IGB_TX_FLAGS_TSTAMP;
5415
5416                         adapter->ptp_tx_skb = skb_get(skb);
5417                         adapter->ptp_tx_start = jiffies;
5418                         if (adapter->hw.mac.type == e1000_82576)
5419                                 schedule_work(&adapter->ptp_tx_work);
5420                 } else {
5421                         adapter->tx_hwtstamp_skipped++;
5422                 }
5423         }
5424
5425         skb_tx_timestamp(skb);
5426
5427         if (skb_vlan_tag_present(skb)) {
5428                 tx_flags |= IGB_TX_FLAGS_VLAN;
5429                 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
5430         }
5431
5432         /* record initial flags and protocol */
5433         first->tx_flags = tx_flags;
5434         first->protocol = protocol;
5435
5436         tso = igb_tso(tx_ring, first, &hdr_len);
5437         if (tso < 0)
5438                 goto out_drop;
5439         else if (!tso)
5440                 igb_tx_csum(tx_ring, first);
5441
5442         if (igb_tx_map(tx_ring, first, hdr_len))
5443                 goto cleanup_tx_tstamp;
5444
5445         return NETDEV_TX_OK;
5446
5447 out_drop:
5448         dev_kfree_skb_any(first->skb);
5449         first->skb = NULL;
5450 cleanup_tx_tstamp:
5451         if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
5452                 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
5453
5454                 dev_kfree_skb_any(adapter->ptp_tx_skb);
5455                 adapter->ptp_tx_skb = NULL;
5456                 if (adapter->hw.mac.type == e1000_82576)
5457                         cancel_work_sync(&adapter->ptp_tx_work);
5458                 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
5459         }
5460
5461         return NETDEV_TX_OK;
5462 }
5463
5464 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
5465                                                     struct sk_buff *skb)
5466 {
5467         unsigned int r_idx = skb->queue_mapping;
5468
5469         if (r_idx >= adapter->num_tx_queues)
5470                 r_idx = r_idx % adapter->num_tx_queues;
5471
5472         return adapter->tx_ring[r_idx];
5473 }
5474
5475 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
5476                                   struct net_device *netdev)
5477 {
5478         struct igb_adapter *adapter = netdev_priv(netdev);
5479
5480         /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
5481          * in order to meet this minimum size requirement.
5482          */
5483         if (skb_put_padto(skb, 17))
5484                 return NETDEV_TX_OK;
5485
5486         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
5487 }
5488
5489 /**
5490  *  igb_tx_timeout - Respond to a Tx Hang
5491  *  @netdev: network interface device structure
5492  **/
5493 static void igb_tx_timeout(struct net_device *netdev)
5494 {
5495         struct igb_adapter *adapter = netdev_priv(netdev);
5496         struct e1000_hw *hw = &adapter->hw;
5497
5498         /* Do the reset outside of interrupt context */
5499         adapter->tx_timeout_count++;
5500
5501         if (hw->mac.type >= e1000_82580)
5502                 hw->dev_spec._82575.global_device_reset = true;
5503
5504         schedule_work(&adapter->reset_task);
5505         wr32(E1000_EICS,
5506              (adapter->eims_enable_mask & ~adapter->eims_other));
5507 }
5508
5509 static void igb_reset_task(struct work_struct *work)
5510 {
5511         struct igb_adapter *adapter;
5512         adapter = container_of(work, struct igb_adapter, reset_task);
5513
5514         rtnl_lock();
5515         /* If we're already down or resetting, just bail */
5516         if (test_bit(__IGB_DOWN, &adapter->state) ||
5517             test_bit(__IGB_RESETTING, &adapter->state)) {
5518                 rtnl_unlock();
5519                 return;
5520         }
5521
5522         igb_dump(adapter);
5523         netdev_err(adapter->netdev, "Reset adapter\n");
5524         igb_reinit_locked(adapter);
5525         rtnl_unlock();
5526 }
5527
5528 /**
5529  *  igb_get_stats64 - Get System Network Statistics
5530  *  @netdev: network interface device structure
5531  *  @stats: rtnl_link_stats64 pointer
5532  **/
5533 static void igb_get_stats64(struct net_device *netdev,
5534                             struct rtnl_link_stats64 *stats)
5535 {
5536         struct igb_adapter *adapter = netdev_priv(netdev);
5537
5538         spin_lock(&adapter->stats64_lock);
5539         igb_update_stats(adapter);
5540         memcpy(stats, &adapter->stats64, sizeof(*stats));
5541         spin_unlock(&adapter->stats64_lock);
5542 }
5543
5544 /**
5545  *  igb_change_mtu - Change the Maximum Transfer Unit
5546  *  @netdev: network interface device structure
5547  *  @new_mtu: new value for maximum frame size
5548  *
5549  *  Returns 0 on success, negative on failure
5550  **/
5551 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5552 {
5553         struct igb_adapter *adapter = netdev_priv(netdev);
5554         struct pci_dev *pdev = adapter->pdev;
5555         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5556
5557         /* adjust max frame to be at least the size of a standard frame */
5558         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5559                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5560
5561         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5562                 usleep_range(1000, 2000);
5563
5564         /* igb_down has a dependency on max_frame_size */
5565         adapter->max_frame_size = max_frame;
5566
5567         if (netif_running(netdev))
5568                 igb_down(adapter);
5569
5570         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
5571                  netdev->mtu, new_mtu);
5572         netdev->mtu = new_mtu;
5573
5574         if (netif_running(netdev))
5575                 igb_up(adapter);
5576         else
5577                 igb_reset(adapter);
5578
5579         clear_bit(__IGB_RESETTING, &adapter->state);
5580
5581         return 0;
5582 }
5583
5584 /**
5585  *  igb_update_stats - Update the board statistics counters
5586  *  @adapter: board private structure
5587  **/
5588 void igb_update_stats(struct igb_adapter *adapter)
5589 {
5590         struct rtnl_link_stats64 *net_stats = &adapter->stats64;
5591         struct e1000_hw *hw = &adapter->hw;
5592         struct pci_dev *pdev = adapter->pdev;
5593         u32 reg, mpc;
5594         int i;
5595         u64 bytes, packets;
5596         unsigned int start;
5597         u64 _bytes, _packets;
5598
5599         /* Prevent stats update while adapter is being reset, or if the pci
5600          * connection is down.
5601          */
5602         if (adapter->link_speed == 0)
5603                 return;
5604         if (pci_channel_offline(pdev))
5605                 return;
5606
5607         bytes = 0;
5608         packets = 0;
5609
5610         rcu_read_lock();
5611         for (i = 0; i < adapter->num_rx_queues; i++) {
5612                 struct igb_ring *ring = adapter->rx_ring[i];
5613                 u32 rqdpc = rd32(E1000_RQDPC(i));
5614                 if (hw->mac.type >= e1000_i210)
5615                         wr32(E1000_RQDPC(i), 0);
5616
5617                 if (rqdpc) {
5618                         ring->rx_stats.drops += rqdpc;
5619                         net_stats->rx_fifo_errors += rqdpc;
5620                 }
5621
5622                 do {
5623                         start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
5624                         _bytes = ring->rx_stats.bytes;
5625                         _packets = ring->rx_stats.packets;
5626                 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
5627                 bytes += _bytes;
5628                 packets += _packets;
5629         }
5630
5631         net_stats->rx_bytes = bytes;
5632         net_stats->rx_packets = packets;
5633
5634         bytes = 0;
5635         packets = 0;
5636         for (i = 0; i < adapter->num_tx_queues; i++) {
5637                 struct igb_ring *ring = adapter->tx_ring[i];
5638                 do {
5639                         start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
5640                         _bytes = ring->tx_stats.bytes;
5641                         _packets = ring->tx_stats.packets;
5642                 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
5643                 bytes += _bytes;
5644                 packets += _packets;
5645         }
5646         net_stats->tx_bytes = bytes;
5647         net_stats->tx_packets = packets;
5648         rcu_read_unlock();
5649
5650         /* read stats registers */
5651         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
5652         adapter->stats.gprc += rd32(E1000_GPRC);
5653         adapter->stats.gorc += rd32(E1000_GORCL);
5654         rd32(E1000_GORCH); /* clear GORCL */
5655         adapter->stats.bprc += rd32(E1000_BPRC);
5656         adapter->stats.mprc += rd32(E1000_MPRC);
5657         adapter->stats.roc += rd32(E1000_ROC);
5658
5659         adapter->stats.prc64 += rd32(E1000_PRC64);
5660         adapter->stats.prc127 += rd32(E1000_PRC127);
5661         adapter->stats.prc255 += rd32(E1000_PRC255);
5662         adapter->stats.prc511 += rd32(E1000_PRC511);
5663         adapter->stats.prc1023 += rd32(E1000_PRC1023);
5664         adapter->stats.prc1522 += rd32(E1000_PRC1522);
5665         adapter->stats.symerrs += rd32(E1000_SYMERRS);
5666         adapter->stats.sec += rd32(E1000_SEC);
5667
5668         mpc = rd32(E1000_MPC);
5669         adapter->stats.mpc += mpc;
5670         net_stats->rx_fifo_errors += mpc;
5671         adapter->stats.scc += rd32(E1000_SCC);
5672         adapter->stats.ecol += rd32(E1000_ECOL);
5673         adapter->stats.mcc += rd32(E1000_MCC);
5674         adapter->stats.latecol += rd32(E1000_LATECOL);
5675         adapter->stats.dc += rd32(E1000_DC);
5676         adapter->stats.rlec += rd32(E1000_RLEC);
5677         adapter->stats.xonrxc += rd32(E1000_XONRXC);
5678         adapter->stats.xontxc += rd32(E1000_XONTXC);
5679         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
5680         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
5681         adapter->stats.fcruc += rd32(E1000_FCRUC);
5682         adapter->stats.gptc += rd32(E1000_GPTC);
5683         adapter->stats.gotc += rd32(E1000_GOTCL);
5684         rd32(E1000_GOTCH); /* clear GOTCL */
5685         adapter->stats.rnbc += rd32(E1000_RNBC);
5686         adapter->stats.ruc += rd32(E1000_RUC);
5687         adapter->stats.rfc += rd32(E1000_RFC);
5688         adapter->stats.rjc += rd32(E1000_RJC);
5689         adapter->stats.tor += rd32(E1000_TORH);
5690         adapter->stats.tot += rd32(E1000_TOTH);
5691         adapter->stats.tpr += rd32(E1000_TPR);
5692
5693         adapter->stats.ptc64 += rd32(E1000_PTC64);
5694         adapter->stats.ptc127 += rd32(E1000_PTC127);
5695         adapter->stats.ptc255 += rd32(E1000_PTC255);
5696         adapter->stats.ptc511 += rd32(E1000_PTC511);
5697         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
5698         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
5699
5700         adapter->stats.mptc += rd32(E1000_MPTC);
5701         adapter->stats.bptc += rd32(E1000_BPTC);
5702
5703         adapter->stats.tpt += rd32(E1000_TPT);
5704         adapter->stats.colc += rd32(E1000_COLC);
5705
5706         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
5707         /* read internal phy specific stats */
5708         reg = rd32(E1000_CTRL_EXT);
5709         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5710                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
5711
5712                 /* this stat has invalid values on i210/i211 */
5713                 if ((hw->mac.type != e1000_i210) &&
5714                     (hw->mac.type != e1000_i211))
5715                         adapter->stats.tncrs += rd32(E1000_TNCRS);
5716         }
5717
5718         adapter->stats.tsctc += rd32(E1000_TSCTC);
5719         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
5720
5721         adapter->stats.iac += rd32(E1000_IAC);
5722         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
5723         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
5724         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
5725         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
5726         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
5727         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
5728         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
5729         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
5730
5731         /* Fill out the OS statistics structure */
5732         net_stats->multicast = adapter->stats.mprc;
5733         net_stats->collisions = adapter->stats.colc;
5734
5735         /* Rx Errors */
5736
5737         /* RLEC on some newer hardware can be incorrect so build
5738          * our own version based on RUC and ROC
5739          */
5740         net_stats->rx_errors = adapter->stats.rxerrc +
5741                 adapter->stats.crcerrs + adapter->stats.algnerrc +
5742                 adapter->stats.ruc + adapter->stats.roc +
5743                 adapter->stats.cexterr;
5744         net_stats->rx_length_errors = adapter->stats.ruc +
5745                                       adapter->stats.roc;
5746         net_stats->rx_crc_errors = adapter->stats.crcerrs;
5747         net_stats->rx_frame_errors = adapter->stats.algnerrc;
5748         net_stats->rx_missed_errors = adapter->stats.mpc;
5749
5750         /* Tx Errors */
5751         net_stats->tx_errors = adapter->stats.ecol +
5752                                adapter->stats.latecol;
5753         net_stats->tx_aborted_errors = adapter->stats.ecol;
5754         net_stats->tx_window_errors = adapter->stats.latecol;
5755         net_stats->tx_carrier_errors = adapter->stats.tncrs;
5756
5757         /* Tx Dropped needs to be maintained elsewhere */
5758
5759         /* Management Stats */
5760         adapter->stats.mgptc += rd32(E1000_MGTPTC);
5761         adapter->stats.mgprc += rd32(E1000_MGTPRC);
5762         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
5763
5764         /* OS2BMC Stats */
5765         reg = rd32(E1000_MANC);
5766         if (reg & E1000_MANC_EN_BMC2OS) {
5767                 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
5768                 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
5769                 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
5770                 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
5771         }
5772 }
5773
5774 static void igb_tsync_interrupt(struct igb_adapter *adapter)
5775 {
5776         struct e1000_hw *hw = &adapter->hw;
5777         struct ptp_clock_event event;
5778         struct timespec64 ts;
5779         u32 ack = 0, tsauxc, sec, nsec, tsicr = rd32(E1000_TSICR);
5780
5781         if (tsicr & TSINTR_SYS_WRAP) {
5782                 event.type = PTP_CLOCK_PPS;
5783                 if (adapter->ptp_caps.pps)
5784                         ptp_clock_event(adapter->ptp_clock, &event);
5785                 ack |= TSINTR_SYS_WRAP;
5786         }
5787
5788         if (tsicr & E1000_TSICR_TXTS) {
5789                 /* retrieve hardware timestamp */
5790                 schedule_work(&adapter->ptp_tx_work);
5791                 ack |= E1000_TSICR_TXTS;
5792         }
5793
5794         if (tsicr & TSINTR_TT0) {
5795                 spin_lock(&adapter->tmreg_lock);
5796                 ts = timespec64_add(adapter->perout[0].start,
5797                                     adapter->perout[0].period);
5798                 /* u32 conversion of tv_sec is safe until y2106 */
5799                 wr32(E1000_TRGTTIML0, ts.tv_nsec);
5800                 wr32(E1000_TRGTTIMH0, (u32)ts.tv_sec);
5801                 tsauxc = rd32(E1000_TSAUXC);
5802                 tsauxc |= TSAUXC_EN_TT0;
5803                 wr32(E1000_TSAUXC, tsauxc);
5804                 adapter->perout[0].start = ts;
5805                 spin_unlock(&adapter->tmreg_lock);
5806                 ack |= TSINTR_TT0;
5807         }
5808
5809         if (tsicr & TSINTR_TT1) {
5810                 spin_lock(&adapter->tmreg_lock);
5811                 ts = timespec64_add(adapter->perout[1].start,
5812                                     adapter->perout[1].period);
5813                 wr32(E1000_TRGTTIML1, ts.tv_nsec);
5814                 wr32(E1000_TRGTTIMH1, (u32)ts.tv_sec);
5815                 tsauxc = rd32(E1000_TSAUXC);
5816                 tsauxc |= TSAUXC_EN_TT1;
5817                 wr32(E1000_TSAUXC, tsauxc);
5818                 adapter->perout[1].start = ts;
5819                 spin_unlock(&adapter->tmreg_lock);
5820                 ack |= TSINTR_TT1;
5821         }
5822
5823         if (tsicr & TSINTR_AUTT0) {
5824                 nsec = rd32(E1000_AUXSTMPL0);
5825                 sec  = rd32(E1000_AUXSTMPH0);
5826                 event.type = PTP_CLOCK_EXTTS;
5827                 event.index = 0;
5828                 event.timestamp = sec * 1000000000ULL + nsec;
5829                 ptp_clock_event(adapter->ptp_clock, &event);
5830                 ack |= TSINTR_AUTT0;
5831         }
5832
5833         if (tsicr & TSINTR_AUTT1) {
5834                 nsec = rd32(E1000_AUXSTMPL1);
5835                 sec  = rd32(E1000_AUXSTMPH1);
5836                 event.type = PTP_CLOCK_EXTTS;
5837                 event.index = 1;
5838                 event.timestamp = sec * 1000000000ULL + nsec;
5839                 ptp_clock_event(adapter->ptp_clock, &event);
5840                 ack |= TSINTR_AUTT1;
5841         }
5842
5843         /* acknowledge the interrupts */
5844         wr32(E1000_TSICR, ack);
5845 }
5846
5847 static irqreturn_t igb_msix_other(int irq, void *data)
5848 {
5849         struct igb_adapter *adapter = data;
5850         struct e1000_hw *hw = &adapter->hw;
5851         u32 icr = rd32(E1000_ICR);
5852         /* reading ICR causes bit 31 of EICR to be cleared */
5853
5854         if (icr & E1000_ICR_DRSTA)
5855                 schedule_work(&adapter->reset_task);
5856
5857         if (icr & E1000_ICR_DOUTSYNC) {
5858                 /* HW is reporting DMA is out of sync */
5859                 adapter->stats.doosync++;
5860                 /* The DMA Out of Sync is also indication of a spoof event
5861                  * in IOV mode. Check the Wrong VM Behavior register to
5862                  * see if it is really a spoof event.
5863                  */
5864                 igb_check_wvbr(adapter);
5865         }
5866
5867         /* Check for a mailbox event */
5868         if (icr & E1000_ICR_VMMB)
5869                 igb_msg_task(adapter);
5870
5871         if (icr & E1000_ICR_LSC) {
5872                 hw->mac.get_link_status = 1;
5873                 /* guard against interrupt when we're going down */
5874                 if (!test_bit(__IGB_DOWN, &adapter->state))
5875                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5876         }
5877
5878         if (icr & E1000_ICR_TS)
5879                 igb_tsync_interrupt(adapter);
5880
5881         wr32(E1000_EIMS, adapter->eims_other);
5882
5883         return IRQ_HANDLED;
5884 }
5885
5886 static void igb_write_itr(struct igb_q_vector *q_vector)
5887 {
5888         struct igb_adapter *adapter = q_vector->adapter;
5889         u32 itr_val = q_vector->itr_val & 0x7FFC;
5890
5891         if (!q_vector->set_itr)
5892                 return;
5893
5894         if (!itr_val)
5895                 itr_val = 0x4;
5896
5897         if (adapter->hw.mac.type == e1000_82575)
5898                 itr_val |= itr_val << 16;
5899         else
5900                 itr_val |= E1000_EITR_CNT_IGNR;
5901
5902         writel(itr_val, q_vector->itr_register);
5903         q_vector->set_itr = 0;
5904 }
5905
5906 static irqreturn_t igb_msix_ring(int irq, void *data)
5907 {
5908         struct igb_q_vector *q_vector = data;
5909
5910         /* Write the ITR value calculated from the previous interrupt. */
5911         igb_write_itr(q_vector);
5912
5913         napi_schedule(&q_vector->napi);
5914
5915         return IRQ_HANDLED;
5916 }
5917
5918 #ifdef CONFIG_IGB_DCA
5919 static void igb_update_tx_dca(struct igb_adapter *adapter,
5920                               struct igb_ring *tx_ring,
5921                               int cpu)
5922 {
5923         struct e1000_hw *hw = &adapter->hw;
5924         u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
5925
5926         if (hw->mac.type != e1000_82575)
5927                 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
5928
5929         /* We can enable relaxed ordering for reads, but not writes when
5930          * DCA is enabled.  This is due to a known issue in some chipsets
5931          * which will cause the DCA tag to be cleared.
5932          */
5933         txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
5934                   E1000_DCA_TXCTRL_DATA_RRO_EN |
5935                   E1000_DCA_TXCTRL_DESC_DCA_EN;
5936
5937         wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
5938 }
5939
5940 static void igb_update_rx_dca(struct igb_adapter *adapter,
5941                               struct igb_ring *rx_ring,
5942                               int cpu)
5943 {
5944         struct e1000_hw *hw = &adapter->hw;
5945         u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
5946
5947         if (hw->mac.type != e1000_82575)
5948                 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
5949
5950         /* We can enable relaxed ordering for reads, but not writes when
5951          * DCA is enabled.  This is due to a known issue in some chipsets
5952          * which will cause the DCA tag to be cleared.
5953          */
5954         rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
5955                   E1000_DCA_RXCTRL_DESC_DCA_EN;
5956
5957         wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
5958 }
5959
5960 static void igb_update_dca(struct igb_q_vector *q_vector)
5961 {
5962         struct igb_adapter *adapter = q_vector->adapter;
5963         int cpu = get_cpu();
5964
5965         if (q_vector->cpu == cpu)
5966                 goto out_no_update;
5967
5968         if (q_vector->tx.ring)
5969                 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
5970
5971         if (q_vector->rx.ring)
5972                 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
5973
5974         q_vector->cpu = cpu;
5975 out_no_update:
5976         put_cpu();
5977 }
5978
5979 static void igb_setup_dca(struct igb_adapter *adapter)
5980 {
5981         struct e1000_hw *hw = &adapter->hw;
5982         int i;
5983
5984         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
5985                 return;
5986
5987         /* Always use CB2 mode, difference is masked in the CB driver. */
5988         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
5989
5990         for (i = 0; i < adapter->num_q_vectors; i++) {
5991                 adapter->q_vector[i]->cpu = -1;
5992                 igb_update_dca(adapter->q_vector[i]);
5993         }
5994 }
5995
5996 static int __igb_notify_dca(struct device *dev, void *data)
5997 {
5998         struct net_device *netdev = dev_get_drvdata(dev);
5999         struct igb_adapter *adapter = netdev_priv(netdev);
6000         struct pci_dev *pdev = adapter->pdev;
6001         struct e1000_hw *hw = &adapter->hw;
6002         unsigned long event = *(unsigned long *)data;
6003
6004         switch (event) {
6005         case DCA_PROVIDER_ADD:
6006                 /* if already enabled, don't do it again */
6007                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
6008                         break;
6009                 if (dca_add_requester(dev) == 0) {
6010                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
6011                         dev_info(&pdev->dev, "DCA enabled\n");
6012                         igb_setup_dca(adapter);
6013                         break;
6014                 }
6015                 /* Fall Through since DCA is disabled. */
6016         case DCA_PROVIDER_REMOVE:
6017                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
6018                         /* without this a class_device is left
6019                          * hanging around in the sysfs model
6020                          */
6021                         dca_remove_requester(dev);
6022                         dev_info(&pdev->dev, "DCA disabled\n");
6023                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
6024                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
6025                 }
6026                 break;
6027         }
6028
6029         return 0;
6030 }
6031
6032 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
6033                           void *p)
6034 {
6035         int ret_val;
6036
6037         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
6038                                          __igb_notify_dca);
6039
6040         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
6041 }
6042 #endif /* CONFIG_IGB_DCA */
6043
6044 #ifdef CONFIG_PCI_IOV
6045 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
6046 {
6047         unsigned char mac_addr[ETH_ALEN];
6048
6049         eth_zero_addr(mac_addr);
6050         igb_set_vf_mac(adapter, vf, mac_addr);
6051
6052         /* By default spoof check is enabled for all VFs */
6053         adapter->vf_data[vf].spoofchk_enabled = true;
6054
6055         return 0;
6056 }
6057
6058 #endif
6059 static void igb_ping_all_vfs(struct igb_adapter *adapter)
6060 {
6061         struct e1000_hw *hw = &adapter->hw;
6062         u32 ping;
6063         int i;
6064
6065         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
6066                 ping = E1000_PF_CONTROL_MSG;
6067                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
6068                         ping |= E1000_VT_MSGTYPE_CTS;
6069                 igb_write_mbx(hw, &ping, 1, i);
6070         }
6071 }
6072
6073 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
6074 {
6075         struct e1000_hw *hw = &adapter->hw;
6076         u32 vmolr = rd32(E1000_VMOLR(vf));
6077         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6078
6079         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
6080                             IGB_VF_FLAG_MULTI_PROMISC);
6081         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6082
6083         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
6084                 vmolr |= E1000_VMOLR_MPME;
6085                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
6086                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
6087         } else {
6088                 /* if we have hashes and we are clearing a multicast promisc
6089                  * flag we need to write the hashes to the MTA as this step
6090                  * was previously skipped
6091                  */
6092                 if (vf_data->num_vf_mc_hashes > 30) {
6093                         vmolr |= E1000_VMOLR_MPME;
6094                 } else if (vf_data->num_vf_mc_hashes) {
6095                         int j;
6096
6097                         vmolr |= E1000_VMOLR_ROMPE;
6098                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6099                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
6100                 }
6101         }
6102
6103         wr32(E1000_VMOLR(vf), vmolr);
6104
6105         /* there are flags left unprocessed, likely not supported */
6106         if (*msgbuf & E1000_VT_MSGINFO_MASK)
6107                 return -EINVAL;
6108
6109         return 0;
6110 }
6111
6112 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
6113                                   u32 *msgbuf, u32 vf)
6114 {
6115         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
6116         u16 *hash_list = (u16 *)&msgbuf[1];
6117         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6118         int i;
6119
6120         /* salt away the number of multicast addresses assigned
6121          * to this VF for later use to restore when the PF multi cast
6122          * list changes
6123          */
6124         vf_data->num_vf_mc_hashes = n;
6125
6126         /* only up to 30 hash values supported */
6127         if (n > 30)
6128                 n = 30;
6129
6130         /* store the hashes for later use */
6131         for (i = 0; i < n; i++)
6132                 vf_data->vf_mc_hashes[i] = hash_list[i];
6133
6134         /* Flush and reset the mta with the new values */
6135         igb_set_rx_mode(adapter->netdev);
6136
6137         return 0;
6138 }
6139
6140 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
6141 {
6142         struct e1000_hw *hw = &adapter->hw;
6143         struct vf_data_storage *vf_data;
6144         int i, j;
6145
6146         for (i = 0; i < adapter->vfs_allocated_count; i++) {
6147                 u32 vmolr = rd32(E1000_VMOLR(i));
6148
6149                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6150
6151                 vf_data = &adapter->vf_data[i];
6152
6153                 if ((vf_data->num_vf_mc_hashes > 30) ||
6154                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
6155                         vmolr |= E1000_VMOLR_MPME;
6156                 } else if (vf_data->num_vf_mc_hashes) {
6157                         vmolr |= E1000_VMOLR_ROMPE;
6158                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6159                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
6160                 }
6161                 wr32(E1000_VMOLR(i), vmolr);
6162         }
6163 }
6164
6165 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
6166 {
6167         struct e1000_hw *hw = &adapter->hw;
6168         u32 pool_mask, vlvf_mask, i;
6169
6170         /* create mask for VF and other pools */
6171         pool_mask = E1000_VLVF_POOLSEL_MASK;
6172         vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
6173
6174         /* drop PF from pool bits */
6175         pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
6176                              adapter->vfs_allocated_count);
6177
6178         /* Find the vlan filter for this id */
6179         for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
6180                 u32 vlvf = rd32(E1000_VLVF(i));
6181                 u32 vfta_mask, vid, vfta;
6182
6183                 /* remove the vf from the pool */
6184                 if (!(vlvf & vlvf_mask))
6185                         continue;
6186
6187                 /* clear out bit from VLVF */
6188                 vlvf ^= vlvf_mask;
6189
6190                 /* if other pools are present, just remove ourselves */
6191                 if (vlvf & pool_mask)
6192                         goto update_vlvfb;
6193
6194                 /* if PF is present, leave VFTA */
6195                 if (vlvf & E1000_VLVF_POOLSEL_MASK)
6196                         goto update_vlvf;
6197
6198                 vid = vlvf & E1000_VLVF_VLANID_MASK;
6199                 vfta_mask = BIT(vid % 32);
6200
6201                 /* clear bit from VFTA */
6202                 vfta = adapter->shadow_vfta[vid / 32];
6203                 if (vfta & vfta_mask)
6204                         hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
6205 update_vlvf:
6206                 /* clear pool selection enable */
6207                 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
6208                         vlvf &= E1000_VLVF_POOLSEL_MASK;
6209                 else
6210                         vlvf = 0;
6211 update_vlvfb:
6212                 /* clear pool bits */
6213                 wr32(E1000_VLVF(i), vlvf);
6214         }
6215 }
6216
6217 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
6218 {
6219         u32 vlvf;
6220         int idx;
6221
6222         /* short cut the special case */
6223         if (vlan == 0)
6224                 return 0;
6225
6226         /* Search for the VLAN id in the VLVF entries */
6227         for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
6228                 vlvf = rd32(E1000_VLVF(idx));
6229                 if ((vlvf & VLAN_VID_MASK) == vlan)
6230                         break;
6231         }
6232
6233         return idx;
6234 }
6235
6236 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
6237 {
6238         struct e1000_hw *hw = &adapter->hw;
6239         u32 bits, pf_id;
6240         int idx;
6241
6242         idx = igb_find_vlvf_entry(hw, vid);
6243         if (!idx)
6244                 return;
6245
6246         /* See if any other pools are set for this VLAN filter
6247          * entry other than the PF.
6248          */
6249         pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
6250         bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
6251         bits &= rd32(E1000_VLVF(idx));
6252
6253         /* Disable the filter so this falls into the default pool. */
6254         if (!bits) {
6255                 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
6256                         wr32(E1000_VLVF(idx), BIT(pf_id));
6257                 else
6258                         wr32(E1000_VLVF(idx), 0);
6259         }
6260 }
6261
6262 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
6263                            bool add, u32 vf)
6264 {
6265         int pf_id = adapter->vfs_allocated_count;
6266         struct e1000_hw *hw = &adapter->hw;
6267         int err;
6268
6269         /* If VLAN overlaps with one the PF is currently monitoring make
6270          * sure that we are able to allocate a VLVF entry.  This may be
6271          * redundant but it guarantees PF will maintain visibility to
6272          * the VLAN.
6273          */
6274         if (add && test_bit(vid, adapter->active_vlans)) {
6275                 err = igb_vfta_set(hw, vid, pf_id, true, false);
6276                 if (err)
6277                         return err;
6278         }
6279
6280         err = igb_vfta_set(hw, vid, vf, add, false);
6281
6282         if (add && !err)
6283                 return err;
6284
6285         /* If we failed to add the VF VLAN or we are removing the VF VLAN
6286          * we may need to drop the PF pool bit in order to allow us to free
6287          * up the VLVF resources.
6288          */
6289         if (test_bit(vid, adapter->active_vlans) ||
6290             (adapter->flags & IGB_FLAG_VLAN_PROMISC))
6291                 igb_update_pf_vlvf(adapter, vid);
6292
6293         return err;
6294 }
6295
6296 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
6297 {
6298         struct e1000_hw *hw = &adapter->hw;
6299
6300         if (vid)
6301                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
6302         else
6303                 wr32(E1000_VMVIR(vf), 0);
6304 }
6305
6306 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
6307                                 u16 vlan, u8 qos)
6308 {
6309         int err;
6310
6311         err = igb_set_vf_vlan(adapter, vlan, true, vf);
6312         if (err)
6313                 return err;
6314
6315         igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
6316         igb_set_vmolr(adapter, vf, !vlan);
6317
6318         /* revoke access to previous VLAN */
6319         if (vlan != adapter->vf_data[vf].pf_vlan)
6320                 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
6321                                 false, vf);
6322
6323         adapter->vf_data[vf].pf_vlan = vlan;
6324         adapter->vf_data[vf].pf_qos = qos;
6325         igb_set_vf_vlan_strip(adapter, vf, true);
6326         dev_info(&adapter->pdev->dev,
6327                  "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
6328         if (test_bit(__IGB_DOWN, &adapter->state)) {
6329                 dev_warn(&adapter->pdev->dev,
6330                          "The VF VLAN has been set, but the PF device is not up.\n");
6331                 dev_warn(&adapter->pdev->dev,
6332                          "Bring the PF device up before attempting to use the VF device.\n");
6333         }
6334
6335         return err;
6336 }
6337
6338 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
6339 {
6340         /* Restore tagless access via VLAN 0 */
6341         igb_set_vf_vlan(adapter, 0, true, vf);
6342
6343         igb_set_vmvir(adapter, 0, vf);
6344         igb_set_vmolr(adapter, vf, true);
6345
6346         /* Remove any PF assigned VLAN */
6347         if (adapter->vf_data[vf].pf_vlan)
6348                 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
6349                                 false, vf);
6350
6351         adapter->vf_data[vf].pf_vlan = 0;
6352         adapter->vf_data[vf].pf_qos = 0;
6353         igb_set_vf_vlan_strip(adapter, vf, false);
6354
6355         return 0;
6356 }
6357
6358 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
6359                                u16 vlan, u8 qos, __be16 vlan_proto)
6360 {
6361         struct igb_adapter *adapter = netdev_priv(netdev);
6362
6363         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
6364                 return -EINVAL;
6365
6366         if (vlan_proto != htons(ETH_P_8021Q))
6367                 return -EPROTONOSUPPORT;
6368
6369         return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
6370                                igb_disable_port_vlan(adapter, vf);
6371 }
6372
6373 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
6374 {
6375         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
6376         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
6377         int ret;
6378
6379         if (adapter->vf_data[vf].pf_vlan)
6380                 return -1;
6381
6382         /* VLAN 0 is a special case, don't allow it to be removed */
6383         if (!vid && !add)
6384                 return 0;
6385
6386         ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
6387         if (!ret)
6388                 igb_set_vf_vlan_strip(adapter, vf, !!vid);
6389         return ret;
6390 }
6391
6392 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
6393 {
6394         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6395
6396         /* clear flags - except flag that indicates PF has set the MAC */
6397         vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
6398         vf_data->last_nack = jiffies;
6399
6400         /* reset vlans for device */
6401         igb_clear_vf_vfta(adapter, vf);
6402         igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
6403         igb_set_vmvir(adapter, vf_data->pf_vlan |
6404                                (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
6405         igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
6406         igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
6407
6408         /* reset multicast table array for vf */
6409         adapter->vf_data[vf].num_vf_mc_hashes = 0;
6410
6411         /* Flush and reset the mta with the new values */
6412         igb_set_rx_mode(adapter->netdev);
6413 }
6414
6415 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
6416 {
6417         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
6418
6419         /* clear mac address as we were hotplug removed/added */
6420         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
6421                 eth_zero_addr(vf_mac);
6422
6423         /* process remaining reset events */
6424         igb_vf_reset(adapter, vf);
6425 }
6426
6427 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
6428 {
6429         struct e1000_hw *hw = &adapter->hw;
6430         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
6431         u32 reg, msgbuf[3] = {};
6432         u8 *addr = (u8 *)(&msgbuf[1]);
6433
6434         /* process all the same items cleared in a function level reset */
6435         igb_vf_reset(adapter, vf);
6436
6437         /* set vf mac address */
6438         igb_set_vf_mac(adapter, vf, vf_mac);
6439
6440         /* enable transmit and receive for vf */
6441         reg = rd32(E1000_VFTE);
6442         wr32(E1000_VFTE, reg | BIT(vf));
6443         reg = rd32(E1000_VFRE);
6444         wr32(E1000_VFRE, reg | BIT(vf));
6445
6446         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6447
6448         /* reply to reset with ack and vf mac address */
6449         if (!is_zero_ether_addr(vf_mac)) {
6450                 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6451                 memcpy(addr, vf_mac, ETH_ALEN);
6452         } else {
6453                 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
6454         }
6455         igb_write_mbx(hw, msgbuf, 3, vf);
6456 }
6457
6458 static void igb_flush_mac_table(struct igb_adapter *adapter)
6459 {
6460         struct e1000_hw *hw = &adapter->hw;
6461         int i;
6462
6463         for (i = 0; i < hw->mac.rar_entry_count; i++) {
6464                 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
6465                 memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
6466                 adapter->mac_table[i].queue = 0;
6467                 igb_rar_set_index(adapter, i);
6468         }
6469 }
6470
6471 static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
6472 {
6473         struct e1000_hw *hw = &adapter->hw;
6474         /* do not count rar entries reserved for VFs MAC addresses */
6475         int rar_entries = hw->mac.rar_entry_count -
6476                           adapter->vfs_allocated_count;
6477         int i, count = 0;
6478
6479         for (i = 0; i < rar_entries; i++) {
6480                 /* do not count default entries */
6481                 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
6482                         continue;
6483
6484                 /* do not count "in use" entries for different queues */
6485                 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
6486                     (adapter->mac_table[i].queue != queue))
6487                         continue;
6488
6489                 count++;
6490         }
6491
6492         return count;
6493 }
6494
6495 /* Set default MAC address for the PF in the first RAR entry */
6496 static void igb_set_default_mac_filter(struct igb_adapter *adapter)
6497 {
6498         struct igb_mac_addr *mac_table = &adapter->mac_table[0];
6499
6500         ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
6501         mac_table->queue = adapter->vfs_allocated_count;
6502         mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
6503
6504         igb_rar_set_index(adapter, 0);
6505 }
6506
6507 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
6508                               const u8 queue)
6509 {
6510         struct e1000_hw *hw = &adapter->hw;
6511         int rar_entries = hw->mac.rar_entry_count -
6512                           adapter->vfs_allocated_count;
6513         int i;
6514
6515         if (is_zero_ether_addr(addr))
6516                 return -EINVAL;
6517
6518         /* Search for the first empty entry in the MAC table.
6519          * Do not touch entries at the end of the table reserved for the VF MAC
6520          * addresses.
6521          */
6522         for (i = 0; i < rar_entries; i++) {
6523                 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
6524                         continue;
6525
6526                 ether_addr_copy(adapter->mac_table[i].addr, addr);
6527                 adapter->mac_table[i].queue = queue;
6528                 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE;
6529
6530                 igb_rar_set_index(adapter, i);
6531                 return i;
6532         }
6533
6534         return -ENOSPC;
6535 }
6536
6537 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
6538                               const u8 queue)
6539 {
6540         struct e1000_hw *hw = &adapter->hw;
6541         int rar_entries = hw->mac.rar_entry_count -
6542                           adapter->vfs_allocated_count;
6543         int i;
6544
6545         if (is_zero_ether_addr(addr))
6546                 return -EINVAL;
6547
6548         /* Search for matching entry in the MAC table based on given address
6549          * and queue. Do not touch entries at the end of the table reserved
6550          * for the VF MAC addresses.
6551          */
6552         for (i = 0; i < rar_entries; i++) {
6553                 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
6554                         continue;
6555                 if (adapter->mac_table[i].queue != queue)
6556                         continue;
6557                 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
6558                         continue;
6559
6560                 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
6561                 memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
6562                 adapter->mac_table[i].queue = 0;
6563
6564                 igb_rar_set_index(adapter, i);
6565                 return 0;
6566         }
6567
6568         return -ENOENT;
6569 }
6570
6571 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
6572 {
6573         struct igb_adapter *adapter = netdev_priv(netdev);
6574         int ret;
6575
6576         ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
6577
6578         return min_t(int, ret, 0);
6579 }
6580
6581 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
6582 {
6583         struct igb_adapter *adapter = netdev_priv(netdev);
6584
6585         igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
6586
6587         return 0;
6588 }
6589
6590 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
6591                                  const u32 info, const u8 *addr)
6592 {
6593         struct pci_dev *pdev = adapter->pdev;
6594         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6595         struct list_head *pos;
6596         struct vf_mac_filter *entry = NULL;
6597         int ret = 0;
6598
6599         switch (info) {
6600         case E1000_VF_MAC_FILTER_CLR:
6601                 /* remove all unicast MAC filters related to the current VF */
6602                 list_for_each(pos, &adapter->vf_macs.l) {
6603                         entry = list_entry(pos, struct vf_mac_filter, l);
6604                         if (entry->vf == vf) {
6605                                 entry->vf = -1;
6606                                 entry->free = true;
6607                                 igb_del_mac_filter(adapter, entry->vf_mac, vf);
6608                         }
6609                 }
6610                 break;
6611         case E1000_VF_MAC_FILTER_ADD:
6612                 if (vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) {
6613                         dev_warn(&pdev->dev,
6614                                  "VF %d requested MAC filter but is administratively denied\n",
6615                                  vf);
6616                         return -EINVAL;
6617                 }
6618
6619                 if (!is_valid_ether_addr(addr)) {
6620                         dev_warn(&pdev->dev,
6621                                  "VF %d attempted to set invalid MAC filter\n",
6622                                  vf);
6623                         return -EINVAL;
6624                 }
6625
6626                 /* try to find empty slot in the list */
6627                 list_for_each(pos, &adapter->vf_macs.l) {
6628                         entry = list_entry(pos, struct vf_mac_filter, l);
6629                         if (entry->free)
6630                                 break;
6631                 }
6632
6633                 if (entry && entry->free) {
6634                         entry->free = false;
6635                         entry->vf = vf;
6636                         ether_addr_copy(entry->vf_mac, addr);
6637
6638                         ret = igb_add_mac_filter(adapter, addr, vf);
6639                         ret = min_t(int, ret, 0);
6640                 } else {
6641                         ret = -ENOSPC;
6642                 }
6643
6644                 if (ret == -ENOSPC)
6645                         dev_warn(&pdev->dev,
6646                                  "VF %d has requested MAC filter but there is no space for it\n",
6647                                  vf);
6648                 break;
6649         default:
6650                 ret = -EINVAL;
6651                 break;
6652         }
6653
6654         return ret;
6655 }
6656
6657 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6658 {
6659         struct pci_dev *pdev = adapter->pdev;
6660         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6661         u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
6662
6663         /* The VF MAC Address is stored in a packed array of bytes
6664          * starting at the second 32 bit word of the msg array
6665          */
6666         unsigned char *addr = (unsigned char *)&msg[1];
6667         int ret = 0;
6668
6669         if (!info) {
6670                 if (vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) {
6671                         dev_warn(&pdev->dev,
6672                                  "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
6673                                  vf);
6674                         return -EINVAL;
6675                 }
6676
6677                 if (!is_valid_ether_addr(addr)) {
6678                         dev_warn(&pdev->dev,
6679                                  "VF %d attempted to set invalid MAC\n",
6680                                  vf);
6681                         return -EINVAL;
6682                 }
6683
6684                 ret = igb_set_vf_mac(adapter, vf, addr);
6685         } else {
6686                 ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
6687         }
6688
6689         return ret;
6690 }
6691
6692 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6693 {
6694         struct e1000_hw *hw = &adapter->hw;
6695         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6696         u32 msg = E1000_VT_MSGTYPE_NACK;
6697
6698         /* if device isn't clear to send it shouldn't be reading either */
6699         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6700             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6701                 igb_write_mbx(hw, &msg, 1, vf);
6702                 vf_data->last_nack = jiffies;
6703         }
6704 }
6705
6706 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6707 {
6708         struct pci_dev *pdev = adapter->pdev;
6709         u32 msgbuf[E1000_VFMAILBOX_SIZE];
6710         struct e1000_hw *hw = &adapter->hw;
6711         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6712         s32 retval;
6713
6714         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
6715
6716         if (retval) {
6717                 /* if receive failed revoke VF CTS stats and restart init */
6718                 dev_err(&pdev->dev, "Error receiving message from VF\n");
6719                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
6720                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
6721                         goto unlock;
6722                 goto out;
6723         }
6724
6725         /* this is a message we already processed, do nothing */
6726         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6727                 goto unlock;
6728
6729         /* until the vf completes a reset it should not be
6730          * allowed to start any configuration.
6731          */
6732         if (msgbuf[0] == E1000_VF_RESET) {
6733                 /* unlocks mailbox */
6734                 igb_vf_reset_msg(adapter, vf);
6735                 return;
6736         }
6737
6738         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6739                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
6740                         goto unlock;
6741                 retval = -1;
6742                 goto out;
6743         }
6744
6745         switch ((msgbuf[0] & 0xFFFF)) {
6746         case E1000_VF_SET_MAC_ADDR:
6747                 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6748                 break;
6749         case E1000_VF_SET_PROMISC:
6750                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6751                 break;
6752         case E1000_VF_SET_MULTICAST:
6753                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6754                 break;
6755         case E1000_VF_SET_LPE:
6756                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6757                 break;
6758         case E1000_VF_SET_VLAN:
6759                 retval = -1;
6760                 if (vf_data->pf_vlan)
6761                         dev_warn(&pdev->dev,
6762                                  "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
6763                                  vf);
6764                 else
6765                         retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
6766                 break;
6767         default:
6768                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
6769                 retval = -1;
6770                 break;
6771         }
6772
6773         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6774 out:
6775         /* notify the VF of the results of what it sent us */
6776         if (retval)
6777                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6778         else
6779                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6780
6781         /* unlocks mailbox */
6782         igb_write_mbx(hw, msgbuf, 1, vf);
6783         return;
6784
6785 unlock:
6786         igb_unlock_mbx(hw, vf);
6787 }
6788
6789 static void igb_msg_task(struct igb_adapter *adapter)
6790 {
6791         struct e1000_hw *hw = &adapter->hw;
6792         unsigned long flags;
6793         u32 vf;
6794
6795         spin_lock_irqsave(&adapter->vfs_lock, flags);
6796         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6797                 /* process any reset requests */
6798                 if (!igb_check_for_rst(hw, vf))
6799                         igb_vf_reset_event(adapter, vf);
6800
6801                 /* process any messages pending */
6802                 if (!igb_check_for_msg(hw, vf))
6803                         igb_rcv_msg_from_vf(adapter, vf);
6804
6805                 /* process any acks */
6806                 if (!igb_check_for_ack(hw, vf))
6807                         igb_rcv_ack_from_vf(adapter, vf);
6808         }
6809         spin_unlock_irqrestore(&adapter->vfs_lock, flags);
6810 }
6811
6812 /**
6813  *  igb_set_uta - Set unicast filter table address
6814  *  @adapter: board private structure
6815  *  @set: boolean indicating if we are setting or clearing bits
6816  *
6817  *  The unicast table address is a register array of 32-bit registers.
6818  *  The table is meant to be used in a way similar to how the MTA is used
6819  *  however due to certain limitations in the hardware it is necessary to
6820  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6821  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
6822  **/
6823 static void igb_set_uta(struct igb_adapter *adapter, bool set)
6824 {
6825         struct e1000_hw *hw = &adapter->hw;
6826         u32 uta = set ? ~0 : 0;
6827         int i;
6828
6829         /* we only need to do this if VMDq is enabled */
6830         if (!adapter->vfs_allocated_count)
6831                 return;
6832
6833         for (i = hw->mac.uta_reg_count; i--;)
6834                 array_wr32(E1000_UTA, i, uta);
6835 }
6836
6837 /**
6838  *  igb_intr_msi - Interrupt Handler
6839  *  @irq: interrupt number
6840  *  @data: pointer to a network interface device structure
6841  **/
6842 static irqreturn_t igb_intr_msi(int irq, void *data)
6843 {
6844         struct igb_adapter *adapter = data;
6845         struct igb_q_vector *q_vector = adapter->q_vector[0];
6846         struct e1000_hw *hw = &adapter->hw;
6847         /* read ICR disables interrupts using IAM */
6848         u32 icr = rd32(E1000_ICR);
6849
6850         igb_write_itr(q_vector);
6851
6852         if (icr & E1000_ICR_DRSTA)
6853                 schedule_work(&adapter->reset_task);
6854
6855         if (icr & E1000_ICR_DOUTSYNC) {
6856                 /* HW is reporting DMA is out of sync */
6857                 adapter->stats.doosync++;
6858         }
6859
6860         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6861                 hw->mac.get_link_status = 1;
6862                 if (!test_bit(__IGB_DOWN, &adapter->state))
6863                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6864         }
6865
6866         if (icr & E1000_ICR_TS)
6867                 igb_tsync_interrupt(adapter);
6868
6869         napi_schedule(&q_vector->napi);
6870
6871         return IRQ_HANDLED;
6872 }
6873
6874 /**
6875  *  igb_intr - Legacy Interrupt Handler
6876  *  @irq: interrupt number
6877  *  @data: pointer to a network interface device structure
6878  **/
6879 static irqreturn_t igb_intr(int irq, void *data)
6880 {
6881         struct igb_adapter *adapter = data;
6882         struct igb_q_vector *q_vector = adapter->q_vector[0];
6883         struct e1000_hw *hw = &adapter->hw;
6884         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
6885          * need for the IMC write
6886          */
6887         u32 icr = rd32(E1000_ICR);
6888
6889         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6890          * not set, then the adapter didn't send an interrupt
6891          */
6892         if (!(icr & E1000_ICR_INT_ASSERTED))
6893                 return IRQ_NONE;
6894
6895         igb_write_itr(q_vector);
6896
6897         if (icr & E1000_ICR_DRSTA)
6898                 schedule_work(&adapter->reset_task);
6899
6900         if (icr & E1000_ICR_DOUTSYNC) {
6901                 /* HW is reporting DMA is out of sync */
6902                 adapter->stats.doosync++;
6903         }
6904
6905         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6906                 hw->mac.get_link_status = 1;
6907                 /* guard against interrupt when we're going down */
6908                 if (!test_bit(__IGB_DOWN, &adapter->state))
6909                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6910         }
6911
6912         if (icr & E1000_ICR_TS)
6913                 igb_tsync_interrupt(adapter);
6914
6915         napi_schedule(&q_vector->napi);
6916
6917         return IRQ_HANDLED;
6918 }
6919
6920 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6921 {
6922         struct igb_adapter *adapter = q_vector->adapter;
6923         struct e1000_hw *hw = &adapter->hw;
6924
6925         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6926             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6927                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6928                         igb_set_itr(q_vector);
6929                 else
6930                         igb_update_ring_itr(q_vector);
6931         }
6932
6933         if (!test_bit(__IGB_DOWN, &adapter->state)) {
6934                 if (adapter->flags & IGB_FLAG_HAS_MSIX)
6935                         wr32(E1000_EIMS, q_vector->eims_value);
6936                 else
6937                         igb_irq_enable(adapter);
6938         }
6939 }
6940
6941 /**
6942  *  igb_poll - NAPI Rx polling callback
6943  *  @napi: napi polling structure
6944  *  @budget: count of how many packets we should handle
6945  **/
6946 static int igb_poll(struct napi_struct *napi, int budget)
6947 {
6948         struct igb_q_vector *q_vector = container_of(napi,
6949                                                      struct igb_q_vector,
6950                                                      napi);
6951         bool clean_complete = true;
6952         int work_done = 0;
6953
6954 #ifdef CONFIG_IGB_DCA
6955         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
6956                 igb_update_dca(q_vector);
6957 #endif
6958         if (q_vector->tx.ring)
6959                 clean_complete = igb_clean_tx_irq(q_vector, budget);
6960
6961         if (q_vector->rx.ring) {
6962                 int cleaned = igb_clean_rx_irq(q_vector, budget);
6963
6964                 work_done += cleaned;
6965                 if (cleaned >= budget)
6966                         clean_complete = false;
6967         }
6968
6969         /* If all work not completed, return budget and keep polling */
6970         if (!clean_complete)
6971                 return budget;
6972
6973         /* If not enough Rx work done, exit the polling mode */
6974         napi_complete_done(napi, work_done);
6975         igb_ring_irq_enable(q_vector);
6976
6977         return 0;
6978 }
6979
6980 /**
6981  *  igb_clean_tx_irq - Reclaim resources after transmit completes
6982  *  @q_vector: pointer to q_vector containing needed info
6983  *  @napi_budget: Used to determine if we are in netpoll
6984  *
6985  *  returns true if ring is completely cleaned
6986  **/
6987 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
6988 {
6989         struct igb_adapter *adapter = q_vector->adapter;
6990         struct igb_ring *tx_ring = q_vector->tx.ring;
6991         struct igb_tx_buffer *tx_buffer;
6992         union e1000_adv_tx_desc *tx_desc;
6993         unsigned int total_bytes = 0, total_packets = 0;
6994         unsigned int budget = q_vector->tx.work_limit;
6995         unsigned int i = tx_ring->next_to_clean;
6996
6997         if (test_bit(__IGB_DOWN, &adapter->state))
6998                 return true;
6999
7000         tx_buffer = &tx_ring->tx_buffer_info[i];
7001         tx_desc = IGB_TX_DESC(tx_ring, i);
7002         i -= tx_ring->count;
7003
7004         do {
7005                 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
7006
7007                 /* if next_to_watch is not set then there is no work pending */
7008                 if (!eop_desc)
7009                         break;
7010
7011                 /* prevent any other reads prior to eop_desc */
7012                 smp_rmb();
7013
7014                 /* if DD is not set pending work has not been completed */
7015                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
7016                         break;
7017
7018                 /* clear next_to_watch to prevent false hangs */
7019                 tx_buffer->next_to_watch = NULL;
7020
7021                 /* update the statistics for this packet */
7022                 total_bytes += tx_buffer->bytecount;
7023                 total_packets += tx_buffer->gso_segs;
7024
7025                 /* free the skb */
7026                 napi_consume_skb(tx_buffer->skb, napi_budget);
7027
7028                 /* unmap skb header data */
7029                 dma_unmap_single(tx_ring->dev,
7030                                  dma_unmap_addr(tx_buffer, dma),
7031                                  dma_unmap_len(tx_buffer, len),
7032                                  DMA_TO_DEVICE);
7033
7034                 /* clear tx_buffer data */
7035                 dma_unmap_len_set(tx_buffer, len, 0);
7036
7037                 /* clear last DMA location and unmap remaining buffers */
7038                 while (tx_desc != eop_desc) {
7039                         tx_buffer++;
7040                         tx_desc++;
7041                         i++;
7042                         if (unlikely(!i)) {
7043                                 i -= tx_ring->count;
7044                                 tx_buffer = tx_ring->tx_buffer_info;
7045                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
7046                         }
7047
7048                         /* unmap any remaining paged data */
7049                         if (dma_unmap_len(tx_buffer, len)) {
7050                                 dma_unmap_page(tx_ring->dev,
7051                                                dma_unmap_addr(tx_buffer, dma),
7052                                                dma_unmap_len(tx_buffer, len),
7053                                                DMA_TO_DEVICE);
7054                                 dma_unmap_len_set(tx_buffer, len, 0);
7055                         }
7056                 }
7057
7058                 /* move us one more past the eop_desc for start of next pkt */
7059                 tx_buffer++;
7060                 tx_desc++;
7061                 i++;
7062                 if (unlikely(!i)) {
7063                         i -= tx_ring->count;
7064                         tx_buffer = tx_ring->tx_buffer_info;
7065                         tx_desc = IGB_TX_DESC(tx_ring, 0);
7066                 }
7067
7068                 /* issue prefetch for next Tx descriptor */
7069                 prefetch(tx_desc);
7070
7071                 /* update budget accounting */
7072                 budget--;
7073         } while (likely(budget));
7074
7075         netdev_tx_completed_queue(txring_txq(tx_ring),
7076                                   total_packets, total_bytes);
7077         i += tx_ring->count;
7078         tx_ring->next_to_clean = i;
7079         u64_stats_update_begin(&tx_ring->tx_syncp);
7080         tx_ring->tx_stats.bytes += total_bytes;
7081         tx_ring->tx_stats.packets += total_packets;
7082         u64_stats_update_end(&tx_ring->tx_syncp);
7083         q_vector->tx.total_bytes += total_bytes;
7084         q_vector->tx.total_packets += total_packets;
7085
7086         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
7087                 struct e1000_hw *hw = &adapter->hw;
7088
7089                 /* Detect a transmit hang in hardware, this serializes the
7090                  * check with the clearing of time_stamp and movement of i
7091                  */
7092                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
7093                 if (tx_buffer->next_to_watch &&
7094                     time_after(jiffies, tx_buffer->time_stamp +
7095                                (adapter->tx_timeout_factor * HZ)) &&
7096                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
7097
7098                         /* detected Tx unit hang */
7099                         dev_err(tx_ring->dev,
7100                                 "Detected Tx Unit Hang\n"
7101                                 "  Tx Queue             <%d>\n"
7102                                 "  TDH                  <%x>\n"
7103                                 "  TDT                  <%x>\n"
7104                                 "  next_to_use          <%x>\n"
7105                                 "  next_to_clean        <%x>\n"
7106                                 "buffer_info[next_to_clean]\n"
7107                                 "  time_stamp           <%lx>\n"
7108                                 "  next_to_watch        <%p>\n"
7109                                 "  jiffies              <%lx>\n"
7110                                 "  desc.status          <%x>\n",
7111                                 tx_ring->queue_index,
7112                                 rd32(E1000_TDH(tx_ring->reg_idx)),
7113                                 readl(tx_ring->tail),
7114                                 tx_ring->next_to_use,
7115                                 tx_ring->next_to_clean,
7116                                 tx_buffer->time_stamp,
7117                                 tx_buffer->next_to_watch,
7118                                 jiffies,
7119                                 tx_buffer->next_to_watch->wb.status);
7120                         netif_stop_subqueue(tx_ring->netdev,
7121                                             tx_ring->queue_index);
7122
7123                         /* we are about to reset, no point in enabling stuff */
7124                         return true;
7125                 }
7126         }
7127
7128 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7129         if (unlikely(total_packets &&
7130             netif_carrier_ok(tx_ring->netdev) &&
7131             igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
7132                 /* Make sure that anybody stopping the queue after this
7133                  * sees the new next_to_clean.
7134                  */
7135                 smp_mb();
7136                 if (__netif_subqueue_stopped(tx_ring->netdev,
7137                                              tx_ring->queue_index) &&
7138                     !(test_bit(__IGB_DOWN, &adapter->state))) {
7139                         netif_wake_subqueue(tx_ring->netdev,
7140                                             tx_ring->queue_index);
7141
7142                         u64_stats_update_begin(&tx_ring->tx_syncp);
7143                         tx_ring->tx_stats.restart_queue++;
7144                         u64_stats_update_end(&tx_ring->tx_syncp);
7145                 }
7146         }
7147
7148         return !!budget;
7149 }
7150
7151 /**
7152  *  igb_reuse_rx_page - page flip buffer and store it back on the ring
7153  *  @rx_ring: rx descriptor ring to store buffers on
7154  *  @old_buff: donor buffer to have page reused
7155  *
7156  *  Synchronizes page for reuse by the adapter
7157  **/
7158 static void igb_reuse_rx_page(struct igb_ring *rx_ring,
7159                               struct igb_rx_buffer *old_buff)
7160 {
7161         struct igb_rx_buffer *new_buff;
7162         u16 nta = rx_ring->next_to_alloc;
7163
7164         new_buff = &rx_ring->rx_buffer_info[nta];
7165
7166         /* update, and store next to alloc */
7167         nta++;
7168         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
7169
7170         /* Transfer page from old buffer to new buffer.
7171          * Move each member individually to avoid possible store
7172          * forwarding stalls.
7173          */
7174         new_buff->dma           = old_buff->dma;
7175         new_buff->page          = old_buff->page;
7176         new_buff->page_offset   = old_buff->page_offset;
7177         new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
7178 }
7179
7180 static inline bool igb_page_is_reserved(struct page *page)
7181 {
7182         return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
7183 }
7184
7185 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer)
7186 {
7187         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
7188         struct page *page = rx_buffer->page;
7189
7190         /* avoid re-using remote pages */
7191         if (unlikely(igb_page_is_reserved(page)))
7192                 return false;
7193
7194 #if (PAGE_SIZE < 8192)
7195         /* if we are only owner of page we can reuse it */
7196         if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
7197                 return false;
7198 #else
7199 #define IGB_LAST_OFFSET \
7200         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
7201
7202         if (rx_buffer->page_offset > IGB_LAST_OFFSET)
7203                 return false;
7204 #endif
7205
7206         /* If we have drained the page fragment pool we need to update
7207          * the pagecnt_bias and page count so that we fully restock the
7208          * number of references the driver holds.
7209          */
7210         if (unlikely(!pagecnt_bias)) {
7211                 page_ref_add(page, USHRT_MAX);
7212                 rx_buffer->pagecnt_bias = USHRT_MAX;
7213         }
7214
7215         return true;
7216 }
7217
7218 /**
7219  *  igb_add_rx_frag - Add contents of Rx buffer to sk_buff
7220  *  @rx_ring: rx descriptor ring to transact packets on
7221  *  @rx_buffer: buffer containing page to add
7222  *  @skb: sk_buff to place the data into
7223  *  @size: size of buffer to be added
7224  *
7225  *  This function will add the data contained in rx_buffer->page to the skb.
7226  **/
7227 static void igb_add_rx_frag(struct igb_ring *rx_ring,
7228                             struct igb_rx_buffer *rx_buffer,
7229                             struct sk_buff *skb,
7230                             unsigned int size)
7231 {
7232 #if (PAGE_SIZE < 8192)
7233         unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
7234 #else
7235         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
7236                                 SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
7237                                 SKB_DATA_ALIGN(size);
7238 #endif
7239         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
7240                         rx_buffer->page_offset, size, truesize);
7241 #if (PAGE_SIZE < 8192)
7242         rx_buffer->page_offset ^= truesize;
7243 #else
7244         rx_buffer->page_offset += truesize;
7245 #endif
7246 }
7247
7248 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
7249                                          struct igb_rx_buffer *rx_buffer,
7250                                          union e1000_adv_rx_desc *rx_desc,
7251                                          unsigned int size)
7252 {
7253         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
7254 #if (PAGE_SIZE < 8192)
7255         unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
7256 #else
7257         unsigned int truesize = SKB_DATA_ALIGN(size);
7258 #endif
7259         unsigned int headlen;
7260         struct sk_buff *skb;
7261
7262         /* prefetch first cache line of first page */
7263         prefetch(va);
7264 #if L1_CACHE_BYTES < 128
7265         prefetch(va + L1_CACHE_BYTES);
7266 #endif
7267
7268         /* allocate a skb to store the frags */
7269         skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
7270         if (unlikely(!skb))
7271                 return NULL;
7272
7273         if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {
7274                 igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
7275                 va += IGB_TS_HDR_LEN;
7276                 size -= IGB_TS_HDR_LEN;
7277         }
7278
7279         /* Determine available headroom for copy */
7280         headlen = size;
7281         if (headlen > IGB_RX_HDR_LEN)
7282                 headlen = eth_get_headlen(va, IGB_RX_HDR_LEN);
7283
7284         /* align pull length to size of long to optimize memcpy performance */
7285         memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
7286
7287         /* update all of the pointers */
7288         size -= headlen;
7289         if (size) {
7290                 skb_add_rx_frag(skb, 0, rx_buffer->page,
7291                                 (va + headlen) - page_address(rx_buffer->page),
7292                                 size, truesize);
7293 #if (PAGE_SIZE < 8192)
7294                 rx_buffer->page_offset ^= truesize;
7295 #else
7296                 rx_buffer->page_offset += truesize;
7297 #endif
7298         } else {
7299                 rx_buffer->pagecnt_bias++;
7300         }
7301
7302         return skb;
7303 }
7304
7305 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
7306                                      struct igb_rx_buffer *rx_buffer,
7307                                      union e1000_adv_rx_desc *rx_desc,
7308                                      unsigned int size)
7309 {
7310         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
7311 #if (PAGE_SIZE < 8192)
7312         unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
7313 #else
7314         unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
7315                                 SKB_DATA_ALIGN(IGB_SKB_PAD + size);
7316 #endif
7317         struct sk_buff *skb;
7318
7319         /* prefetch first cache line of first page */
7320         prefetch(va);
7321 #if L1_CACHE_BYTES < 128
7322         prefetch(va + L1_CACHE_BYTES);
7323 #endif
7324
7325         /* build an skb around the page buffer */
7326         skb = build_skb(va - IGB_SKB_PAD, truesize);
7327         if (unlikely(!skb))
7328                 return NULL;
7329
7330         /* update pointers within the skb to store the data */
7331         skb_reserve(skb, IGB_SKB_PAD);
7332         __skb_put(skb, size);
7333
7334         /* pull timestamp out of packet data */
7335         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
7336                 igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb);
7337                 __skb_pull(skb, IGB_TS_HDR_LEN);
7338         }
7339
7340         /* update buffer offset */
7341 #if (PAGE_SIZE < 8192)
7342         rx_buffer->page_offset ^= truesize;
7343 #else
7344         rx_buffer->page_offset += truesize;
7345 #endif
7346
7347         return skb;
7348 }
7349
7350 static inline void igb_rx_checksum(struct igb_ring *ring,
7351                                    union e1000_adv_rx_desc *rx_desc,
7352                                    struct sk_buff *skb)
7353 {
7354         skb_checksum_none_assert(skb);
7355
7356         /* Ignore Checksum bit is set */
7357         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
7358                 return;
7359
7360         /* Rx checksum disabled via ethtool */
7361         if (!(ring->netdev->features & NETIF_F_RXCSUM))
7362                 return;
7363
7364         /* TCP/UDP checksum error bit is set */
7365         if (igb_test_staterr(rx_desc,
7366                              E1000_RXDEXT_STATERR_TCPE |
7367                              E1000_RXDEXT_STATERR_IPE)) {
7368                 /* work around errata with sctp packets where the TCPE aka
7369                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
7370                  * packets, (aka let the stack check the crc32c)
7371                  */
7372                 if (!((skb->len == 60) &&
7373                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
7374                         u64_stats_update_begin(&ring->rx_syncp);
7375                         ring->rx_stats.csum_err++;
7376                         u64_stats_update_end(&ring->rx_syncp);
7377                 }
7378                 /* let the stack verify checksum errors */
7379                 return;
7380         }
7381         /* It must be a TCP or UDP packet with a valid checksum */
7382         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
7383                                       E1000_RXD_STAT_UDPCS))
7384                 skb->ip_summed = CHECKSUM_UNNECESSARY;
7385
7386         dev_dbg(ring->dev, "cksum success: bits %08X\n",
7387                 le32_to_cpu(rx_desc->wb.upper.status_error));
7388 }
7389
7390 static inline void igb_rx_hash(struct igb_ring *ring,
7391                                union e1000_adv_rx_desc *rx_desc,
7392                                struct sk_buff *skb)
7393 {
7394         if (ring->netdev->features & NETIF_F_RXHASH)
7395                 skb_set_hash(skb,
7396                              le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
7397                              PKT_HASH_TYPE_L3);
7398 }
7399
7400 /**
7401  *  igb_is_non_eop - process handling of non-EOP buffers
7402  *  @rx_ring: Rx ring being processed
7403  *  @rx_desc: Rx descriptor for current buffer
7404  *  @skb: current socket buffer containing buffer in progress
7405  *
7406  *  This function updates next to clean.  If the buffer is an EOP buffer
7407  *  this function exits returning false, otherwise it will place the
7408  *  sk_buff in the next buffer to be chained and return true indicating
7409  *  that this is in fact a non-EOP buffer.
7410  **/
7411 static bool igb_is_non_eop(struct igb_ring *rx_ring,
7412                            union e1000_adv_rx_desc *rx_desc)
7413 {
7414         u32 ntc = rx_ring->next_to_clean + 1;
7415
7416         /* fetch, update, and store next to clean */
7417         ntc = (ntc < rx_ring->count) ? ntc : 0;
7418         rx_ring->next_to_clean = ntc;
7419
7420         prefetch(IGB_RX_DESC(rx_ring, ntc));
7421
7422         if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
7423                 return false;
7424
7425         return true;
7426 }
7427
7428 /**
7429  *  igb_cleanup_headers - Correct corrupted or empty headers
7430  *  @rx_ring: rx descriptor ring packet is being transacted on
7431  *  @rx_desc: pointer to the EOP Rx descriptor
7432  *  @skb: pointer to current skb being fixed
7433  *
7434  *  Address the case where we are pulling data in on pages only
7435  *  and as such no data is present in the skb header.
7436  *
7437  *  In addition if skb is not at least 60 bytes we need to pad it so that
7438  *  it is large enough to qualify as a valid Ethernet frame.
7439  *
7440  *  Returns true if an error was encountered and skb was freed.
7441  **/
7442 static bool igb_cleanup_headers(struct igb_ring *rx_ring,
7443                                 union e1000_adv_rx_desc *rx_desc,
7444                                 struct sk_buff *skb)
7445 {
7446         if (unlikely((igb_test_staterr(rx_desc,
7447                                        E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
7448                 struct net_device *netdev = rx_ring->netdev;
7449                 if (!(netdev->features & NETIF_F_RXALL)) {
7450                         dev_kfree_skb_any(skb);
7451                         return true;
7452                 }
7453         }
7454
7455         /* if eth_skb_pad returns an error the skb was freed */
7456         if (eth_skb_pad(skb))
7457                 return true;
7458
7459         return false;
7460 }
7461
7462 /**
7463  *  igb_process_skb_fields - Populate skb header fields from Rx descriptor
7464  *  @rx_ring: rx descriptor ring packet is being transacted on
7465  *  @rx_desc: pointer to the EOP Rx descriptor
7466  *  @skb: pointer to current skb being populated
7467  *
7468  *  This function checks the ring, descriptor, and packet information in
7469  *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
7470  *  other fields within the skb.
7471  **/
7472 static void igb_process_skb_fields(struct igb_ring *rx_ring,
7473                                    union e1000_adv_rx_desc *rx_desc,
7474                                    struct sk_buff *skb)
7475 {
7476         struct net_device *dev = rx_ring->netdev;
7477
7478         igb_rx_hash(rx_ring, rx_desc, skb);
7479
7480         igb_rx_checksum(rx_ring, rx_desc, skb);
7481
7482         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
7483             !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
7484                 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
7485
7486         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
7487             igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
7488                 u16 vid;
7489
7490                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
7491                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
7492                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
7493                 else
7494                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
7495
7496                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
7497         }
7498
7499         skb_record_rx_queue(skb, rx_ring->queue_index);
7500
7501         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
7502 }
7503
7504 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
7505                                                const unsigned int size)
7506 {
7507         struct igb_rx_buffer *rx_buffer;
7508
7509         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
7510         prefetchw(rx_buffer->page);
7511
7512         /* we are reusing so sync this buffer for CPU use */
7513         dma_sync_single_range_for_cpu(rx_ring->dev,
7514                                       rx_buffer->dma,
7515                                       rx_buffer->page_offset,
7516                                       size,
7517                                       DMA_FROM_DEVICE);
7518
7519         rx_buffer->pagecnt_bias--;
7520
7521         return rx_buffer;
7522 }
7523
7524 static void igb_put_rx_buffer(struct igb_ring *rx_ring,
7525                               struct igb_rx_buffer *rx_buffer)
7526 {
7527         if (igb_can_reuse_rx_page(rx_buffer)) {
7528                 /* hand second half of page back to the ring */
7529                 igb_reuse_rx_page(rx_ring, rx_buffer);
7530         } else {
7531                 /* We are not reusing the buffer so unmap it and free
7532                  * any references we are holding to it
7533                  */
7534                 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
7535                                      igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
7536                                      IGB_RX_DMA_ATTR);
7537                 __page_frag_cache_drain(rx_buffer->page,
7538                                         rx_buffer->pagecnt_bias);
7539         }
7540
7541         /* clear contents of rx_buffer */
7542         rx_buffer->page = NULL;
7543 }
7544
7545 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
7546 {
7547         struct igb_ring *rx_ring = q_vector->rx.ring;
7548         struct sk_buff *skb = rx_ring->skb;
7549         unsigned int total_bytes = 0, total_packets = 0;
7550         u16 cleaned_count = igb_desc_unused(rx_ring);
7551
7552         while (likely(total_packets < budget)) {
7553                 union e1000_adv_rx_desc *rx_desc;
7554                 struct igb_rx_buffer *rx_buffer;
7555                 unsigned int size;
7556
7557                 /* return some buffers to hardware, one at a time is too slow */
7558                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
7559                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
7560                         cleaned_count = 0;
7561                 }
7562
7563                 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
7564                 size = le16_to_cpu(rx_desc->wb.upper.length);
7565                 if (!size)
7566                         break;
7567
7568                 /* This memory barrier is needed to keep us from reading
7569                  * any other fields out of the rx_desc until we know the
7570                  * descriptor has been written back
7571                  */
7572                 dma_rmb();
7573
7574                 rx_buffer = igb_get_rx_buffer(rx_ring, size);
7575
7576                 /* retrieve a buffer from the ring */
7577                 if (skb)
7578                         igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
7579                 else if (ring_uses_build_skb(rx_ring))
7580                         skb = igb_build_skb(rx_ring, rx_buffer, rx_desc, size);
7581                 else
7582                         skb = igb_construct_skb(rx_ring, rx_buffer,
7583                                                 rx_desc, size);
7584
7585                 /* exit if we failed to retrieve a buffer */
7586                 if (!skb) {
7587                         rx_ring->rx_stats.alloc_failed++;
7588                         rx_buffer->pagecnt_bias++;
7589                         break;
7590                 }
7591
7592                 igb_put_rx_buffer(rx_ring, rx_buffer);
7593                 cleaned_count++;
7594
7595                 /* fetch next buffer in frame if non-eop */
7596                 if (igb_is_non_eop(rx_ring, rx_desc))
7597                         continue;
7598
7599                 /* verify the packet layout is correct */
7600                 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
7601                         skb = NULL;
7602                         continue;
7603                 }
7604
7605                 /* probably a little skewed due to removing CRC */
7606                 total_bytes += skb->len;
7607
7608                 /* populate checksum, timestamp, VLAN, and protocol */
7609                 igb_process_skb_fields(rx_ring, rx_desc, skb);
7610
7611                 napi_gro_receive(&q_vector->napi, skb);
7612
7613                 /* reset skb pointer */
7614                 skb = NULL;
7615
7616                 /* update budget accounting */
7617                 total_packets++;
7618         }
7619
7620         /* place incomplete frames back on ring for completion */
7621         rx_ring->skb = skb;
7622
7623         u64_stats_update_begin(&rx_ring->rx_syncp);
7624         rx_ring->rx_stats.packets += total_packets;
7625         rx_ring->rx_stats.bytes += total_bytes;
7626         u64_stats_update_end(&rx_ring->rx_syncp);
7627         q_vector->rx.total_packets += total_packets;
7628         q_vector->rx.total_bytes += total_bytes;
7629
7630         if (cleaned_count)
7631                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7632
7633         return total_packets;
7634 }
7635
7636 static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring)
7637 {
7638         return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
7639 }
7640
7641 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7642                                   struct igb_rx_buffer *bi)
7643 {
7644         struct page *page = bi->page;
7645         dma_addr_t dma;
7646
7647         /* since we are recycling buffers we should seldom need to alloc */
7648         if (likely(page))
7649                 return true;
7650
7651         /* alloc new page for storage */
7652         page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
7653         if (unlikely(!page)) {
7654                 rx_ring->rx_stats.alloc_failed++;
7655                 return false;
7656         }
7657
7658         /* map page for use */
7659         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
7660                                  igb_rx_pg_size(rx_ring),
7661                                  DMA_FROM_DEVICE,
7662                                  IGB_RX_DMA_ATTR);
7663
7664         /* if mapping failed free memory back to system since
7665          * there isn't much point in holding memory we can't use
7666          */
7667         if (dma_mapping_error(rx_ring->dev, dma)) {
7668                 __free_pages(page, igb_rx_pg_order(rx_ring));
7669
7670                 rx_ring->rx_stats.alloc_failed++;
7671                 return false;
7672         }
7673
7674         bi->dma = dma;
7675         bi->page = page;
7676         bi->page_offset = igb_rx_offset(rx_ring);
7677         bi->pagecnt_bias = 1;
7678
7679         return true;
7680 }
7681
7682 /**
7683  *  igb_alloc_rx_buffers - Replace used receive buffers; packet split
7684  *  @adapter: address of board private structure
7685  **/
7686 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7687 {
7688         union e1000_adv_rx_desc *rx_desc;
7689         struct igb_rx_buffer *bi;
7690         u16 i = rx_ring->next_to_use;
7691         u16 bufsz;
7692
7693         /* nothing to do */
7694         if (!cleaned_count)
7695                 return;
7696
7697         rx_desc = IGB_RX_DESC(rx_ring, i);
7698         bi = &rx_ring->rx_buffer_info[i];
7699         i -= rx_ring->count;
7700
7701         bufsz = igb_rx_bufsz(rx_ring);
7702
7703         do {
7704                 if (!igb_alloc_mapped_page(rx_ring, bi))
7705                         break;
7706
7707                 /* sync the buffer for use by the device */
7708                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
7709                                                  bi->page_offset, bufsz,
7710                                                  DMA_FROM_DEVICE);
7711
7712                 /* Refresh the desc even if buffer_addrs didn't change
7713                  * because each write-back erases this info.
7714                  */
7715                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
7716
7717                 rx_desc++;
7718                 bi++;
7719                 i++;
7720                 if (unlikely(!i)) {
7721                         rx_desc = IGB_RX_DESC(rx_ring, 0);
7722                         bi = rx_ring->rx_buffer_info;
7723                         i -= rx_ring->count;
7724                 }
7725
7726                 /* clear the length for the next_to_use descriptor */
7727                 rx_desc->wb.upper.length = 0;
7728
7729                 cleaned_count--;
7730         } while (cleaned_count);
7731
7732         i += rx_ring->count;
7733
7734         if (rx_ring->next_to_use != i) {
7735                 /* record the next descriptor to use */
7736                 rx_ring->next_to_use = i;
7737
7738                 /* update next to alloc since we have filled the ring */
7739                 rx_ring->next_to_alloc = i;
7740
7741                 /* Force memory writes to complete before letting h/w
7742                  * know there are new descriptors to fetch.  (Only
7743                  * applicable for weak-ordered memory model archs,
7744                  * such as IA-64).
7745                  */
7746                 wmb();
7747                 writel(i, rx_ring->tail);
7748         }
7749 }
7750
7751 /**
7752  * igb_mii_ioctl -
7753  * @netdev:
7754  * @ifreq:
7755  * @cmd:
7756  **/
7757 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7758 {
7759         struct igb_adapter *adapter = netdev_priv(netdev);
7760         struct mii_ioctl_data *data = if_mii(ifr);
7761
7762         if (adapter->hw.phy.media_type != e1000_media_type_copper)
7763                 return -EOPNOTSUPP;
7764
7765         switch (cmd) {
7766         case SIOCGMIIPHY:
7767                 data->phy_id = adapter->hw.phy.addr;
7768                 break;
7769         case SIOCGMIIREG:
7770                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7771                                      &data->val_out))
7772                         return -EIO;
7773                 break;
7774         case SIOCSMIIREG:
7775         default:
7776                 return -EOPNOTSUPP;
7777         }
7778         return 0;
7779 }
7780
7781 /**
7782  * igb_ioctl -
7783  * @netdev:
7784  * @ifreq:
7785  * @cmd:
7786  **/
7787 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7788 {
7789         switch (cmd) {
7790         case SIOCGMIIPHY:
7791         case SIOCGMIIREG:
7792         case SIOCSMIIREG:
7793                 return igb_mii_ioctl(netdev, ifr, cmd);
7794         case SIOCGHWTSTAMP:
7795                 return igb_ptp_get_ts_config(netdev, ifr);
7796         case SIOCSHWTSTAMP:
7797                 return igb_ptp_set_ts_config(netdev, ifr);
7798         default:
7799                 return -EOPNOTSUPP;
7800         }
7801 }
7802
7803 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
7804 {
7805         struct igb_adapter *adapter = hw->back;
7806
7807         pci_read_config_word(adapter->pdev, reg, value);
7808 }
7809
7810 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
7811 {
7812         struct igb_adapter *adapter = hw->back;
7813
7814         pci_write_config_word(adapter->pdev, reg, *value);
7815 }
7816
7817 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7818 {
7819         struct igb_adapter *adapter = hw->back;
7820
7821         if (pcie_capability_read_word(adapter->pdev, reg, value))
7822                 return -E1000_ERR_CONFIG;
7823
7824         return 0;
7825 }
7826
7827 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7828 {
7829         struct igb_adapter *adapter = hw->back;
7830
7831         if (pcie_capability_write_word(adapter->pdev, reg, *value))
7832                 return -E1000_ERR_CONFIG;
7833
7834         return 0;
7835 }
7836
7837 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
7838 {
7839         struct igb_adapter *adapter = netdev_priv(netdev);
7840         struct e1000_hw *hw = &adapter->hw;
7841         u32 ctrl, rctl;
7842         bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
7843
7844         if (enable) {
7845                 /* enable VLAN tag insert/strip */
7846                 ctrl = rd32(E1000_CTRL);
7847                 ctrl |= E1000_CTRL_VME;
7848                 wr32(E1000_CTRL, ctrl);
7849
7850                 /* Disable CFI check */
7851                 rctl = rd32(E1000_RCTL);
7852                 rctl &= ~E1000_RCTL_CFIEN;
7853                 wr32(E1000_RCTL, rctl);
7854         } else {
7855                 /* disable VLAN tag insert/strip */
7856                 ctrl = rd32(E1000_CTRL);
7857                 ctrl &= ~E1000_CTRL_VME;
7858                 wr32(E1000_CTRL, ctrl);
7859         }
7860
7861         igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
7862 }
7863
7864 static int igb_vlan_rx_add_vid(struct net_device *netdev,
7865                                __be16 proto, u16 vid)
7866 {
7867         struct igb_adapter *adapter = netdev_priv(netdev);
7868         struct e1000_hw *hw = &adapter->hw;
7869         int pf_id = adapter->vfs_allocated_count;
7870
7871         /* add the filter since PF can receive vlans w/o entry in vlvf */
7872         if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
7873                 igb_vfta_set(hw, vid, pf_id, true, !!vid);
7874
7875         set_bit(vid, adapter->active_vlans);
7876
7877         return 0;
7878 }
7879
7880 static int igb_vlan_rx_kill_vid(struct net_device *netdev,
7881                                 __be16 proto, u16 vid)
7882 {
7883         struct igb_adapter *adapter = netdev_priv(netdev);
7884         int pf_id = adapter->vfs_allocated_count;
7885         struct e1000_hw *hw = &adapter->hw;
7886
7887         /* remove VID from filter table */
7888         if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
7889                 igb_vfta_set(hw, vid, pf_id, false, true);
7890
7891         clear_bit(vid, adapter->active_vlans);
7892
7893         return 0;
7894 }
7895
7896 static void igb_restore_vlan(struct igb_adapter *adapter)
7897 {
7898         u16 vid = 1;
7899
7900         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
7901         igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
7902
7903         for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
7904                 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7905 }
7906
7907 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
7908 {
7909         struct pci_dev *pdev = adapter->pdev;
7910         struct e1000_mac_info *mac = &adapter->hw.mac;
7911
7912         mac->autoneg = 0;
7913
7914         /* Make sure dplx is at most 1 bit and lsb of speed is not set
7915          * for the switch() below to work
7916          */
7917         if ((spd & 1) || (dplx & ~1))
7918                 goto err_inval;
7919
7920         /* Fiber NIC's only allow 1000 gbps Full duplex
7921          * and 100Mbps Full duplex for 100baseFx sfp
7922          */
7923         if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
7924                 switch (spd + dplx) {
7925                 case SPEED_10 + DUPLEX_HALF:
7926                 case SPEED_10 + DUPLEX_FULL:
7927                 case SPEED_100 + DUPLEX_HALF:
7928                         goto err_inval;
7929                 default:
7930                         break;
7931                 }
7932         }
7933
7934         switch (spd + dplx) {
7935         case SPEED_10 + DUPLEX_HALF:
7936                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
7937                 break;
7938         case SPEED_10 + DUPLEX_FULL:
7939                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
7940                 break;
7941         case SPEED_100 + DUPLEX_HALF:
7942                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
7943                 break;
7944         case SPEED_100 + DUPLEX_FULL:
7945                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
7946                 break;
7947         case SPEED_1000 + DUPLEX_FULL:
7948                 mac->autoneg = 1;
7949                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
7950                 break;
7951         case SPEED_1000 + DUPLEX_HALF: /* not supported */
7952         default:
7953                 goto err_inval;
7954         }
7955
7956         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
7957         adapter->hw.phy.mdix = AUTO_ALL_MODES;
7958
7959         return 0;
7960
7961 err_inval:
7962         dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
7963         return -EINVAL;
7964 }
7965
7966 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
7967                           bool runtime)
7968 {
7969         struct net_device *netdev = pci_get_drvdata(pdev);
7970         struct igb_adapter *adapter = netdev_priv(netdev);
7971         struct e1000_hw *hw = &adapter->hw;
7972         u32 ctrl, rctl, status;
7973         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7974         bool wake;
7975
7976         rtnl_lock();
7977         netif_device_detach(netdev);
7978
7979         if (netif_running(netdev))
7980                 __igb_close(netdev, true);
7981
7982         igb_ptp_suspend(adapter);
7983
7984         igb_clear_interrupt_scheme(adapter);
7985         rtnl_unlock();
7986
7987         status = rd32(E1000_STATUS);
7988         if (status & E1000_STATUS_LU)
7989                 wufc &= ~E1000_WUFC_LNKC;
7990
7991         if (wufc) {
7992                 igb_setup_rctl(adapter);
7993                 igb_set_rx_mode(netdev);
7994
7995                 /* turn on all-multi mode if wake on multicast is enabled */
7996                 if (wufc & E1000_WUFC_MC) {
7997                         rctl = rd32(E1000_RCTL);
7998                         rctl |= E1000_RCTL_MPE;
7999                         wr32(E1000_RCTL, rctl);
8000                 }
8001
8002                 ctrl = rd32(E1000_CTRL);
8003                 ctrl |= E1000_CTRL_ADVD3WUC;
8004                 wr32(E1000_CTRL, ctrl);
8005
8006                 /* Allow time for pending master requests to run */
8007                 igb_disable_pcie_master(hw);
8008
8009                 wr32(E1000_WUC, E1000_WUC_PME_EN);
8010                 wr32(E1000_WUFC, wufc);
8011         } else {
8012                 wr32(E1000_WUC, 0);
8013                 wr32(E1000_WUFC, 0);
8014         }
8015
8016         wake = wufc || adapter->en_mng_pt;
8017         if (!wake)
8018                 igb_power_down_link(adapter);
8019         else
8020                 igb_power_up_link(adapter);
8021
8022         if (enable_wake)
8023                 *enable_wake = wake;
8024
8025         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
8026          * would have already happened in close and is redundant.
8027          */
8028         igb_release_hw_control(adapter);
8029
8030         pci_disable_device(pdev);
8031
8032         return 0;
8033 }
8034
8035 static void igb_deliver_wake_packet(struct net_device *netdev)
8036 {
8037         struct igb_adapter *adapter = netdev_priv(netdev);
8038         struct e1000_hw *hw = &adapter->hw;
8039         struct sk_buff *skb;
8040         u32 wupl;
8041
8042         wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
8043
8044         /* WUPM stores only the first 128 bytes of the wake packet.
8045          * Read the packet only if we have the whole thing.
8046          */
8047         if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
8048                 return;
8049
8050         skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
8051         if (!skb)
8052                 return;
8053
8054         skb_put(skb, wupl);
8055
8056         /* Ensure reads are 32-bit aligned */
8057         wupl = roundup(wupl, 4);
8058
8059         memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
8060
8061         skb->protocol = eth_type_trans(skb, netdev);
8062         netif_rx(skb);
8063 }
8064
8065 static int __maybe_unused igb_suspend(struct device *dev)
8066 {
8067         return __igb_shutdown(to_pci_dev(dev), NULL, 0);
8068 }
8069
8070 static int __maybe_unused igb_resume(struct device *dev)
8071 {
8072         struct pci_dev *pdev = to_pci_dev(dev);
8073         struct net_device *netdev = pci_get_drvdata(pdev);
8074         struct igb_adapter *adapter = netdev_priv(netdev);
8075         struct e1000_hw *hw = &adapter->hw;
8076         u32 err, val;
8077
8078         pci_set_power_state(pdev, PCI_D0);
8079         pci_restore_state(pdev);
8080         pci_save_state(pdev);
8081
8082         if (!pci_device_is_present(pdev))
8083                 return -ENODEV;
8084         err = pci_enable_device_mem(pdev);
8085         if (err) {
8086                 dev_err(&pdev->dev,
8087                         "igb: Cannot enable PCI device from suspend\n");
8088                 return err;
8089         }
8090         pci_set_master(pdev);
8091
8092         pci_enable_wake(pdev, PCI_D3hot, 0);
8093         pci_enable_wake(pdev, PCI_D3cold, 0);
8094
8095         if (igb_init_interrupt_scheme(adapter, true)) {
8096                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
8097                 return -ENOMEM;
8098         }
8099
8100         igb_reset(adapter);
8101
8102         /* let the f/w know that the h/w is now under the control of the
8103          * driver.
8104          */
8105         igb_get_hw_control(adapter);
8106
8107         val = rd32(E1000_WUS);
8108         if (val & WAKE_PKT_WUS)
8109                 igb_deliver_wake_packet(netdev);
8110
8111         wr32(E1000_WUS, ~0);
8112
8113         rtnl_lock();
8114         if (!err && netif_running(netdev))
8115                 err = __igb_open(netdev, true);
8116
8117         if (!err)
8118                 netif_device_attach(netdev);
8119         rtnl_unlock();
8120
8121         return err;
8122 }
8123
8124 static int __maybe_unused igb_runtime_idle(struct device *dev)
8125 {
8126         struct pci_dev *pdev = to_pci_dev(dev);
8127         struct net_device *netdev = pci_get_drvdata(pdev);
8128         struct igb_adapter *adapter = netdev_priv(netdev);
8129
8130         if (!igb_has_link(adapter))
8131                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8132
8133         return -EBUSY;
8134 }
8135
8136 static int __maybe_unused igb_runtime_suspend(struct device *dev)
8137 {
8138         return __igb_shutdown(to_pci_dev(dev), NULL, 1);
8139 }
8140
8141 static int __maybe_unused igb_runtime_resume(struct device *dev)
8142 {
8143         return igb_resume(dev);
8144 }
8145
8146 static void igb_shutdown(struct pci_dev *pdev)
8147 {
8148         bool wake;
8149
8150         __igb_shutdown(pdev, &wake, 0);
8151
8152         if (system_state == SYSTEM_POWER_OFF) {
8153                 pci_wake_from_d3(pdev, wake);
8154                 pci_set_power_state(pdev, PCI_D3hot);
8155         }
8156 }
8157
8158 #ifdef CONFIG_PCI_IOV
8159 static int igb_sriov_reinit(struct pci_dev *dev)
8160 {
8161         struct net_device *netdev = pci_get_drvdata(dev);
8162         struct igb_adapter *adapter = netdev_priv(netdev);
8163         struct pci_dev *pdev = adapter->pdev;
8164
8165         rtnl_lock();
8166
8167         if (netif_running(netdev))
8168                 igb_close(netdev);
8169         else
8170                 igb_reset(adapter);
8171
8172         igb_clear_interrupt_scheme(adapter);
8173
8174         igb_init_queue_configuration(adapter);
8175
8176         if (igb_init_interrupt_scheme(adapter, true)) {
8177                 rtnl_unlock();
8178                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
8179                 return -ENOMEM;
8180         }
8181
8182         if (netif_running(netdev))
8183                 igb_open(netdev);
8184
8185         rtnl_unlock();
8186
8187         return 0;
8188 }
8189
8190 static int igb_pci_disable_sriov(struct pci_dev *dev)
8191 {
8192         int err = igb_disable_sriov(dev);
8193
8194         if (!err)
8195                 err = igb_sriov_reinit(dev);
8196
8197         return err;
8198 }
8199
8200 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
8201 {
8202         int err = igb_enable_sriov(dev, num_vfs);
8203
8204         if (err)
8205                 goto out;
8206
8207         err = igb_sriov_reinit(dev);
8208         if (!err)
8209                 return num_vfs;
8210
8211 out:
8212         return err;
8213 }
8214
8215 #endif
8216 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
8217 {
8218 #ifdef CONFIG_PCI_IOV
8219         if (num_vfs == 0)
8220                 return igb_pci_disable_sriov(dev);
8221         else
8222                 return igb_pci_enable_sriov(dev, num_vfs);
8223 #endif
8224         return 0;
8225 }
8226
8227 #ifdef CONFIG_NET_POLL_CONTROLLER
8228 /* Polling 'interrupt' - used by things like netconsole to send skbs
8229  * without having to re-enable interrupts. It's not called while
8230  * the interrupt routine is executing.
8231  */
8232 static void igb_netpoll(struct net_device *netdev)
8233 {
8234         struct igb_adapter *adapter = netdev_priv(netdev);
8235         struct e1000_hw *hw = &adapter->hw;
8236         struct igb_q_vector *q_vector;
8237         int i;
8238
8239         for (i = 0; i < adapter->num_q_vectors; i++) {
8240                 q_vector = adapter->q_vector[i];
8241                 if (adapter->flags & IGB_FLAG_HAS_MSIX)
8242                         wr32(E1000_EIMC, q_vector->eims_value);
8243                 else
8244                         igb_irq_disable(adapter);
8245                 napi_schedule(&q_vector->napi);
8246         }
8247 }
8248 #endif /* CONFIG_NET_POLL_CONTROLLER */
8249
8250 /**
8251  *  igb_io_error_detected - called when PCI error is detected
8252  *  @pdev: Pointer to PCI device
8253  *  @state: The current pci connection state
8254  *
8255  *  This function is called after a PCI bus error affecting
8256  *  this device has been detected.
8257  **/
8258 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
8259                                               pci_channel_state_t state)
8260 {
8261         struct net_device *netdev = pci_get_drvdata(pdev);
8262         struct igb_adapter *adapter = netdev_priv(netdev);
8263
8264         if (state == pci_channel_io_normal) {
8265                 dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n");
8266                 return PCI_ERS_RESULT_CAN_RECOVER;
8267         }
8268
8269         netif_device_detach(netdev);
8270
8271         if (state == pci_channel_io_perm_failure)
8272                 return PCI_ERS_RESULT_DISCONNECT;
8273
8274         if (netif_running(netdev))
8275                 igb_down(adapter);
8276         pci_disable_device(pdev);
8277
8278         /* Request a slot slot reset. */
8279         return PCI_ERS_RESULT_NEED_RESET;
8280 }
8281
8282 /**
8283  *  igb_io_slot_reset - called after the pci bus has been reset.
8284  *  @pdev: Pointer to PCI device
8285  *
8286  *  Restart the card from scratch, as if from a cold-boot. Implementation
8287  *  resembles the first-half of the igb_resume routine.
8288  **/
8289 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
8290 {
8291         struct net_device *netdev = pci_get_drvdata(pdev);
8292         struct igb_adapter *adapter = netdev_priv(netdev);
8293         struct e1000_hw *hw = &adapter->hw;
8294         pci_ers_result_t result;
8295         int err;
8296
8297         if (pci_enable_device_mem(pdev)) {
8298                 dev_err(&pdev->dev,
8299                         "Cannot re-enable PCI device after reset.\n");
8300                 result = PCI_ERS_RESULT_DISCONNECT;
8301         } else {
8302                 pci_set_master(pdev);
8303                 pci_restore_state(pdev);
8304                 pci_save_state(pdev);
8305
8306                 pci_enable_wake(pdev, PCI_D3hot, 0);
8307                 pci_enable_wake(pdev, PCI_D3cold, 0);
8308
8309                 /* In case of PCI error, adapter lose its HW address
8310                  * so we should re-assign it here.
8311                  */
8312                 hw->hw_addr = adapter->io_addr;
8313
8314                 igb_reset(adapter);
8315                 wr32(E1000_WUS, ~0);
8316                 result = PCI_ERS_RESULT_RECOVERED;
8317         }
8318
8319         err = pci_cleanup_aer_uncorrect_error_status(pdev);
8320         if (err) {
8321                 dev_err(&pdev->dev,
8322                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8323                         err);
8324                 /* non-fatal, continue */
8325         }
8326
8327         return result;
8328 }
8329
8330 /**
8331  *  igb_io_resume - called when traffic can start flowing again.
8332  *  @pdev: Pointer to PCI device
8333  *
8334  *  This callback is called when the error recovery driver tells us that
8335  *  its OK to resume normal operation. Implementation resembles the
8336  *  second-half of the igb_resume routine.
8337  */
8338 static void igb_io_resume(struct pci_dev *pdev)
8339 {
8340         struct net_device *netdev = pci_get_drvdata(pdev);
8341         struct igb_adapter *adapter = netdev_priv(netdev);
8342
8343         if (netif_running(netdev)) {
8344                 if (igb_up(adapter)) {
8345                         dev_err(&pdev->dev, "igb_up failed after reset\n");
8346                         return;
8347                 }
8348         }
8349
8350         netif_device_attach(netdev);
8351
8352         /* let the f/w know that the h/w is now under the control of the
8353          * driver.
8354          */
8355         igb_get_hw_control(adapter);
8356 }
8357
8358 /**
8359  *  igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
8360  *  @adapter: Pointer to adapter structure
8361  *  @index: Index of the RAR entry which need to be synced with MAC table
8362  **/
8363 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
8364 {
8365         struct e1000_hw *hw = &adapter->hw;
8366         u32 rar_low, rar_high;
8367         u8 *addr = adapter->mac_table[index].addr;
8368
8369         /* HW expects these to be in network order when they are plugged
8370          * into the registers which are little endian.  In order to guarantee
8371          * that ordering we need to do an leXX_to_cpup here in order to be
8372          * ready for the byteswap that occurs with writel
8373          */
8374         rar_low = le32_to_cpup((__le32 *)(addr));
8375         rar_high = le16_to_cpup((__le16 *)(addr + 4));
8376
8377         /* Indicate to hardware the Address is Valid. */
8378         if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
8379                 if (is_valid_ether_addr(addr))
8380                         rar_high |= E1000_RAH_AV;
8381
8382                 switch (hw->mac.type) {
8383                 case e1000_82575:
8384                 case e1000_i210:
8385                         rar_high |= E1000_RAH_POOL_1 *
8386                                     adapter->mac_table[index].queue;
8387                         break;
8388                 default:
8389                         rar_high |= E1000_RAH_POOL_1 <<
8390                                     adapter->mac_table[index].queue;
8391                         break;
8392                 }
8393         }
8394
8395         wr32(E1000_RAL(index), rar_low);
8396         wrfl();
8397         wr32(E1000_RAH(index), rar_high);
8398         wrfl();
8399 }
8400
8401 static int igb_set_vf_mac(struct igb_adapter *adapter,
8402                           int vf, unsigned char *mac_addr)
8403 {
8404         struct e1000_hw *hw = &adapter->hw;
8405         /* VF MAC addresses start at end of receive addresses and moves
8406          * towards the first, as a result a collision should not be possible
8407          */
8408         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
8409         unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
8410
8411         ether_addr_copy(vf_mac_addr, mac_addr);
8412         ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
8413         adapter->mac_table[rar_entry].queue = vf;
8414         adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
8415         igb_rar_set_index(adapter, rar_entry);
8416
8417         return 0;
8418 }
8419
8420 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
8421 {
8422         struct igb_adapter *adapter = netdev_priv(netdev);
8423
8424         if (vf >= adapter->vfs_allocated_count)
8425                 return -EINVAL;
8426
8427         /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
8428          * flag and allows to overwrite the MAC via VF netdev.  This
8429          * is necessary to allow libvirt a way to restore the original
8430          * MAC after unbinding vfio-pci and reloading igbvf after shutting
8431          * down a VM.
8432          */
8433         if (is_zero_ether_addr(mac)) {
8434                 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
8435                 dev_info(&adapter->pdev->dev,
8436                          "remove administratively set MAC on VF %d\n",
8437                          vf);
8438         } else if (is_valid_ether_addr(mac)) {
8439                 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
8440                 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
8441                          mac, vf);
8442                 dev_info(&adapter->pdev->dev,
8443                          "Reload the VF driver to make this change effective.");
8444                 /* Generate additional warning if PF is down */
8445                 if (test_bit(__IGB_DOWN, &adapter->state)) {
8446                         dev_warn(&adapter->pdev->dev,
8447                                  "The VF MAC address has been set, but the PF device is not up.\n");
8448                         dev_warn(&adapter->pdev->dev,
8449                                  "Bring the PF device up before attempting to use the VF device.\n");
8450                 }
8451         } else {
8452                 return -EINVAL;
8453         }
8454         return igb_set_vf_mac(adapter, vf, mac);
8455 }
8456
8457 static int igb_link_mbps(int internal_link_speed)
8458 {
8459         switch (internal_link_speed) {
8460         case SPEED_100:
8461                 return 100;
8462         case SPEED_1000:
8463                 return 1000;
8464         default:
8465                 return 0;
8466         }
8467 }
8468
8469 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
8470                                   int link_speed)
8471 {
8472         int rf_dec, rf_int;
8473         u32 bcnrc_val;
8474
8475         if (tx_rate != 0) {
8476                 /* Calculate the rate factor values to set */
8477                 rf_int = link_speed / tx_rate;
8478                 rf_dec = (link_speed - (rf_int * tx_rate));
8479                 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
8480                          tx_rate;
8481
8482                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
8483                 bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
8484                               E1000_RTTBCNRC_RF_INT_MASK);
8485                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
8486         } else {
8487                 bcnrc_val = 0;
8488         }
8489
8490         wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
8491         /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8492          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8493          */
8494         wr32(E1000_RTTBCNRM, 0x14);
8495         wr32(E1000_RTTBCNRC, bcnrc_val);
8496 }
8497
8498 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
8499 {
8500         int actual_link_speed, i;
8501         bool reset_rate = false;
8502
8503         /* VF TX rate limit was not set or not supported */
8504         if ((adapter->vf_rate_link_speed == 0) ||
8505             (adapter->hw.mac.type != e1000_82576))
8506                 return;
8507
8508         actual_link_speed = igb_link_mbps(adapter->link_speed);
8509         if (actual_link_speed != adapter->vf_rate_link_speed) {
8510                 reset_rate = true;
8511                 adapter->vf_rate_link_speed = 0;
8512                 dev_info(&adapter->pdev->dev,
8513                          "Link speed has been changed. VF Transmit rate is disabled\n");
8514         }
8515
8516         for (i = 0; i < adapter->vfs_allocated_count; i++) {
8517                 if (reset_rate)
8518                         adapter->vf_data[i].tx_rate = 0;
8519
8520                 igb_set_vf_rate_limit(&adapter->hw, i,
8521                                       adapter->vf_data[i].tx_rate,
8522                                       actual_link_speed);
8523         }
8524 }
8525
8526 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
8527                              int min_tx_rate, int max_tx_rate)
8528 {
8529         struct igb_adapter *adapter = netdev_priv(netdev);
8530         struct e1000_hw *hw = &adapter->hw;
8531         int actual_link_speed;
8532
8533         if (hw->mac.type != e1000_82576)
8534                 return -EOPNOTSUPP;
8535
8536         if (min_tx_rate)
8537                 return -EINVAL;
8538
8539         actual_link_speed = igb_link_mbps(adapter->link_speed);
8540         if ((vf >= adapter->vfs_allocated_count) ||
8541             (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
8542             (max_tx_rate < 0) ||
8543             (max_tx_rate > actual_link_speed))
8544                 return -EINVAL;
8545
8546         adapter->vf_rate_link_speed = actual_link_speed;
8547         adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
8548         igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
8549
8550         return 0;
8551 }
8552
8553 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
8554                                    bool setting)
8555 {
8556         struct igb_adapter *adapter = netdev_priv(netdev);
8557         struct e1000_hw *hw = &adapter->hw;
8558         u32 reg_val, reg_offset;
8559
8560         if (!adapter->vfs_allocated_count)
8561                 return -EOPNOTSUPP;
8562
8563         if (vf >= adapter->vfs_allocated_count)
8564                 return -EINVAL;
8565
8566         reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
8567         reg_val = rd32(reg_offset);
8568         if (setting)
8569                 reg_val |= (BIT(vf) |
8570                             BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
8571         else
8572                 reg_val &= ~(BIT(vf) |
8573                              BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
8574         wr32(reg_offset, reg_val);
8575
8576         adapter->vf_data[vf].spoofchk_enabled = setting;
8577         return 0;
8578 }
8579
8580 static int igb_ndo_get_vf_config(struct net_device *netdev,
8581                                  int vf, struct ifla_vf_info *ivi)
8582 {
8583         struct igb_adapter *adapter = netdev_priv(netdev);
8584         if (vf >= adapter->vfs_allocated_count)
8585                 return -EINVAL;
8586         ivi->vf = vf;
8587         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
8588         ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
8589         ivi->min_tx_rate = 0;
8590         ivi->vlan = adapter->vf_data[vf].pf_vlan;
8591         ivi->qos = adapter->vf_data[vf].pf_qos;
8592         ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
8593         return 0;
8594 }
8595
8596 static void igb_vmm_control(struct igb_adapter *adapter)
8597 {
8598         struct e1000_hw *hw = &adapter->hw;
8599         u32 reg;
8600
8601         switch (hw->mac.type) {
8602         case e1000_82575:
8603         case e1000_i210:
8604         case e1000_i211:
8605         case e1000_i354:
8606         default:
8607                 /* replication is not supported for 82575 */
8608                 return;
8609         case e1000_82576:
8610                 /* notify HW that the MAC is adding vlan tags */
8611                 reg = rd32(E1000_DTXCTL);
8612                 reg |= E1000_DTXCTL_VLAN_ADDED;
8613                 wr32(E1000_DTXCTL, reg);
8614                 /* Fall through */
8615         case e1000_82580:
8616                 /* enable replication vlan tag stripping */
8617                 reg = rd32(E1000_RPLOLR);
8618                 reg |= E1000_RPLOLR_STRVLAN;
8619                 wr32(E1000_RPLOLR, reg);
8620                 /* Fall through */
8621         case e1000_i350:
8622                 /* none of the above registers are supported by i350 */
8623                 break;
8624         }
8625
8626         if (adapter->vfs_allocated_count) {
8627                 igb_vmdq_set_loopback_pf(hw, true);
8628                 igb_vmdq_set_replication_pf(hw, true);
8629                 igb_vmdq_set_anti_spoofing_pf(hw, true,
8630                                               adapter->vfs_allocated_count);
8631         } else {
8632                 igb_vmdq_set_loopback_pf(hw, false);
8633                 igb_vmdq_set_replication_pf(hw, false);
8634         }
8635 }
8636
8637 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
8638 {
8639         struct e1000_hw *hw = &adapter->hw;
8640         u32 dmac_thr;
8641         u16 hwm;
8642         u32 reg;
8643
8644         if (hw->mac.type > e1000_82580) {
8645                 if (adapter->flags & IGB_FLAG_DMAC) {
8646                         /* force threshold to 0. */
8647                         wr32(E1000_DMCTXTH, 0);
8648
8649                         /* DMA Coalescing high water mark needs to be greater
8650                          * than the Rx threshold. Set hwm to PBA - max frame
8651                          * size in 16B units, capping it at PBA - 6KB.
8652                          */
8653                         hwm = 64 * (pba - 6);
8654                         reg = rd32(E1000_FCRTC);
8655                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
8656                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
8657                                 & E1000_FCRTC_RTH_COAL_MASK);
8658                         wr32(E1000_FCRTC, reg);
8659
8660                         /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
8661                          * frame size, capping it at PBA - 10KB.
8662                          */
8663                         dmac_thr = pba - 10;
8664                         reg = rd32(E1000_DMACR);
8665                         reg &= ~E1000_DMACR_DMACTHR_MASK;
8666                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
8667                                 & E1000_DMACR_DMACTHR_MASK);
8668
8669                         /* transition to L0x or L1 if available..*/
8670                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
8671
8672                         /* watchdog timer= +-1000 usec in 32usec intervals */
8673                         reg |= (1000 >> 5);
8674
8675                         /* Disable BMC-to-OS Watchdog Enable */
8676                         if (hw->mac.type != e1000_i354)
8677                                 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
8678                         wr32(E1000_DMACR, reg);
8679
8680                         /* no lower threshold to disable
8681                          * coalescing(smart fifb)-UTRESH=0
8682                          */
8683                         wr32(E1000_DMCRTRH, 0);
8684
8685                         reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
8686
8687                         wr32(E1000_DMCTLX, reg);
8688
8689                         /* free space in tx packet buffer to wake from
8690                          * DMA coal
8691                          */
8692                         wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
8693                              (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
8694                 }
8695
8696                 if (hw->mac.type >= e1000_i210 ||
8697                     (adapter->flags & IGB_FLAG_DMAC)) {
8698                         reg = rd32(E1000_PCIEMISC);
8699                         reg |= E1000_PCIEMISC_LX_DECISION;
8700                         wr32(E1000_PCIEMISC, reg);
8701                 } /* endif adapter->dmac is not disabled */
8702         } else if (hw->mac.type == e1000_82580) {
8703                 u32 reg = rd32(E1000_PCIEMISC);
8704
8705                 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
8706                 wr32(E1000_DMACR, 0);
8707         }
8708 }
8709
8710 /**
8711  *  igb_read_i2c_byte - Reads 8 bit word over I2C
8712  *  @hw: pointer to hardware structure
8713  *  @byte_offset: byte offset to read
8714  *  @dev_addr: device address
8715  *  @data: value read
8716  *
8717  *  Performs byte read operation over I2C interface at
8718  *  a specified device address.
8719  **/
8720 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
8721                       u8 dev_addr, u8 *data)
8722 {
8723         struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
8724         struct i2c_client *this_client = adapter->i2c_client;
8725         s32 status;
8726         u16 swfw_mask = 0;
8727
8728         if (!this_client)
8729                 return E1000_ERR_I2C;
8730
8731         swfw_mask = E1000_SWFW_PHY0_SM;
8732
8733         if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
8734                 return E1000_ERR_SWFW_SYNC;
8735
8736         status = i2c_smbus_read_byte_data(this_client, byte_offset);
8737         hw->mac.ops.release_swfw_sync(hw, swfw_mask);
8738
8739         if (status < 0)
8740                 return E1000_ERR_I2C;
8741         else {
8742                 *data = status;
8743                 return 0;
8744         }
8745 }
8746
8747 /**
8748  *  igb_write_i2c_byte - Writes 8 bit word over I2C
8749  *  @hw: pointer to hardware structure
8750  *  @byte_offset: byte offset to write
8751  *  @dev_addr: device address
8752  *  @data: value to write
8753  *
8754  *  Performs byte write operation over I2C interface at
8755  *  a specified device address.
8756  **/
8757 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
8758                        u8 dev_addr, u8 data)
8759 {
8760         struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
8761         struct i2c_client *this_client = adapter->i2c_client;
8762         s32 status;
8763         u16 swfw_mask = E1000_SWFW_PHY0_SM;
8764
8765         if (!this_client)
8766                 return E1000_ERR_I2C;
8767
8768         if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
8769                 return E1000_ERR_SWFW_SYNC;
8770         status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
8771         hw->mac.ops.release_swfw_sync(hw, swfw_mask);
8772
8773         if (status)
8774                 return E1000_ERR_I2C;
8775         else
8776                 return 0;
8777
8778 }
8779
8780 int igb_reinit_queues(struct igb_adapter *adapter)
8781 {
8782         struct net_device *netdev = adapter->netdev;
8783         struct pci_dev *pdev = adapter->pdev;
8784         int err = 0;
8785
8786         if (netif_running(netdev))
8787                 igb_close(netdev);
8788
8789         igb_reset_interrupt_capability(adapter);
8790
8791         if (igb_init_interrupt_scheme(adapter, true)) {
8792                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
8793                 return -ENOMEM;
8794         }
8795
8796         if (netif_running(netdev))
8797                 err = igb_open(netdev);
8798
8799         return err;
8800 }
8801
8802 static void igb_nfc_filter_exit(struct igb_adapter *adapter)
8803 {
8804         struct igb_nfc_filter *rule;
8805
8806         spin_lock(&adapter->nfc_lock);
8807
8808         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
8809                 igb_erase_filter(adapter, rule);
8810
8811         spin_unlock(&adapter->nfc_lock);
8812 }
8813
8814 static void igb_nfc_filter_restore(struct igb_adapter *adapter)
8815 {
8816         struct igb_nfc_filter *rule;
8817
8818         spin_lock(&adapter->nfc_lock);
8819
8820         hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
8821                 igb_add_filter(adapter, rule);
8822
8823         spin_unlock(&adapter->nfc_lock);
8824 }
8825 /* igb_main.c */