drivers/net/ethernet/intel/e1000/e1000_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 1999 - 2006 Intel Corporation. */
   3
   4 #include "e1000.h"
   5 #include <net/ip6_checksum.h>
   6 #include <linux/io.h>
   7 #include <linux/prefetch.h>
   8 #include <linux/bitops.h>
   9 #include <linux/if_vlan.h>
  10
  11 char e1000_driver_name[] = "e1000";
  12 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  13 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  14
  15 /* e1000_pci_tbl - PCI Device ID Table
  16  *
  17  * Last entry must be all 0s
  18  *
  19  * Macro expands to...
  20  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  21  */
  22 static const struct pci_device_id e1000_pci_tbl[] = {
  23         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  24         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  25         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  26         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  27         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  28         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  29         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  30         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  31         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  32         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  33         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  34         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  35         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  36         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  37         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  38         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  39         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  40         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  41         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  42         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  43         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  44         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  45         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  46         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  47         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  48         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  49         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  50         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  53         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  54         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  55         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  56         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  57         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  58         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  59         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  60         /* required last entry */
  61         {0,}
  62 };
  63
  64 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  65
  66 int e1000_up(struct e1000_adapter *adapter);
  67 void e1000_down(struct e1000_adapter *adapter);
  68 void e1000_reinit_locked(struct e1000_adapter *adapter);
  69 void e1000_reset(struct e1000_adapter *adapter);
  70 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  71 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  72 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
  73 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
  74 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
  75                                     struct e1000_tx_ring *txdr);
  76 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
  77                                     struct e1000_rx_ring *rxdr);
  78 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
  79                                     struct e1000_tx_ring *tx_ring);
  80 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
  81                                     struct e1000_rx_ring *rx_ring);
  82 void e1000_update_stats(struct e1000_adapter *adapter);
  83
  84 static int e1000_init_module(void);
  85 static void e1000_exit_module(void);
  86 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  87 static void e1000_remove(struct pci_dev *pdev);
  88 static int e1000_alloc_queues(struct e1000_adapter *adapter);
  89 static int e1000_sw_init(struct e1000_adapter *adapter);
  90 int e1000_open(struct net_device *netdev);
  91 int e1000_close(struct net_device *netdev);
  92 static void e1000_configure_tx(struct e1000_adapter *adapter);
  93 static void e1000_configure_rx(struct e1000_adapter *adapter);
  94 static void e1000_setup_rctl(struct e1000_adapter *adapter);
  95 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
  96 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
  97 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
  98                                 struct e1000_tx_ring *tx_ring);
  99 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 100                                 struct e1000_rx_ring *rx_ring);
 101 static void e1000_set_rx_mode(struct net_device *netdev);
 102 static void e1000_update_phy_info_task(struct work_struct *work);
 103 static void e1000_watchdog(struct work_struct *work);
 104 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 105 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 106                                     struct net_device *netdev);
 107 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 108 static int e1000_set_mac(struct net_device *netdev, void *p);
 109 static irqreturn_t e1000_intr(int irq, void *data);
 110 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 111                                struct e1000_tx_ring *tx_ring);
 112 static int e1000_clean(struct napi_struct *napi, int budget);
 113 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 114                                struct e1000_rx_ring *rx_ring,
 115                                int *work_done, int work_to_do);
 116 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 117                                      struct e1000_rx_ring *rx_ring,
 118                                      int *work_done, int work_to_do);
 119 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
 120                                          struct e1000_rx_ring *rx_ring,
 121                                          int cleaned_count)
 122 {
 123 }
 124 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 125                                    struct e1000_rx_ring *rx_ring,
 126                                    int cleaned_count);
 127 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 128                                          struct e1000_rx_ring *rx_ring,
 129                                          int cleaned_count);
 130 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 131 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 132                            int cmd);
 133 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 134 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 135 static void e1000_tx_timeout(struct net_device *dev, unsigned int txqueue);
 136 static void e1000_reset_task(struct work_struct *work);
 137 static void e1000_smartspeed(struct e1000_adapter *adapter);
 138 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 139                                        struct sk_buff *skb);
 140
 141 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 142 static void e1000_vlan_mode(struct net_device *netdev,
 143                             netdev_features_t features);
 144 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 145                                      bool filter_on);
 146 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 147                                  __be16 proto, u16 vid);
 148 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 149                                   __be16 proto, u16 vid);
 150 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 151
 152 static int __maybe_unused e1000_suspend(struct device *dev);
 153 static int __maybe_unused e1000_resume(struct device *dev);
 154 static void e1000_shutdown(struct pci_dev *pdev);
 155
 156 #ifdef CONFIG_NET_POLL_CONTROLLER
 157 /* for netdump / net console */
 158 static void e1000_netpoll (struct net_device *netdev);
 159 #endif
 160
 161 #define COPYBREAK_DEFAULT 256
 162 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 163 module_param(copybreak, uint, 0644);
 164 MODULE_PARM_DESC(copybreak,
 165         "Maximum size of packet that is copied to a new buffer on receive");
 166
 167 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 168                                                 pci_channel_state_t state);
 169 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 170 static void e1000_io_resume(struct pci_dev *pdev);
 171
 172 static const struct pci_error_handlers e1000_err_handler = {
 173         .error_detected = e1000_io_error_detected,
 174         .slot_reset = e1000_io_slot_reset,
 175         .resume = e1000_io_resume,
 176 };
 177
 178 static SIMPLE_DEV_PM_OPS(e1000_pm_ops, e1000_suspend, e1000_resume);
 179
 180 static struct pci_driver e1000_driver = {
 181         .name     = e1000_driver_name,
 182         .id_table = e1000_pci_tbl,
 183         .probe    = e1000_probe,
 184         .remove   = e1000_remove,
 185         .driver = {
 186                 .pm = &e1000_pm_ops,
 187         },
 188         .shutdown = e1000_shutdown,
 189         .err_handler = &e1000_err_handler
 190 };
 191
 192 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 193 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 194 MODULE_LICENSE("GPL v2");
 195
 196 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 197 static int debug = -1;
 198 module_param(debug, int, 0);
 199 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 200
 201 /**
 202  * e1000_get_hw_dev - helper function for getting netdev
 203  * @hw: pointer to HW struct
 204  *
 205  * return device used by hardware layer to print debugging information
 206  *
 207  **/
 208 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 209 {
 210         struct e1000_adapter *adapter = hw->back;
 211         return adapter->netdev;
 212 }
 213
 214 /**
 215  * e1000_init_module - Driver Registration Routine
 216  *
 217  * e1000_init_module is the first routine called when the driver is
 218  * loaded. All it does is register with the PCI subsystem.
 219  **/
 220 static int __init e1000_init_module(void)
 221 {
 222         int ret;
 223         pr_info("%s\n", e1000_driver_string);
 224
 225         pr_info("%s\n", e1000_copyright);
 226
 227         ret = pci_register_driver(&e1000_driver);
 228         if (copybreak != COPYBREAK_DEFAULT) {
 229                 if (copybreak == 0)
 230                         pr_info("copybreak disabled\n");
 231                 else
 232                         pr_info("copybreak enabled for "
 233                                    "packets <= %u bytes\n", copybreak);
 234         }
 235         return ret;
 236 }
 237
 238 module_init(e1000_init_module);
 239
 240 /**
 241  * e1000_exit_module - Driver Exit Cleanup Routine
 242  *
 243  * e1000_exit_module is called just before the driver is removed
 244  * from memory.
 245  **/
 246 static void __exit e1000_exit_module(void)
 247 {
 248         pci_unregister_driver(&e1000_driver);
 249 }
 250
 251 module_exit(e1000_exit_module);
 252
 253 static int e1000_request_irq(struct e1000_adapter *adapter)
 254 {
 255         struct net_device *netdev = adapter->netdev;
 256         irq_handler_t handler = e1000_intr;
 257         int irq_flags = IRQF_SHARED;
 258         int err;
 259
 260         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 261                           netdev);
 262         if (err) {
 263                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 264         }
 265
 266         return err;
 267 }
 268
 269 static void e1000_free_irq(struct e1000_adapter *adapter)
 270 {
 271         struct net_device *netdev = adapter->netdev;
 272
 273         free_irq(adapter->pdev->irq, netdev);
 274 }
 275
 276 /**
 277  * e1000_irq_disable - Mask off interrupt generation on the NIC
 278  * @adapter: board private structure
 279  **/
 280 static void e1000_irq_disable(struct e1000_adapter *adapter)
 281 {
 282         struct e1000_hw *hw = &adapter->hw;
 283
 284         ew32(IMC, ~0);
 285         E1000_WRITE_FLUSH();
 286         synchronize_irq(adapter->pdev->irq);
 287 }
 288
 289 /**
 290  * e1000_irq_enable - Enable default interrupt generation settings
 291  * @adapter: board private structure
 292  **/
 293 static void e1000_irq_enable(struct e1000_adapter *adapter)
 294 {
 295         struct e1000_hw *hw = &adapter->hw;
 296
 297         ew32(IMS, IMS_ENABLE_MASK);
 298         E1000_WRITE_FLUSH();
 299 }
 300
 301 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 302 {
 303         struct e1000_hw *hw = &adapter->hw;
 304         struct net_device *netdev = adapter->netdev;
 305         u16 vid = hw->mng_cookie.vlan_id;
 306         u16 old_vid = adapter->mng_vlan_id;
 307
 308         if (!e1000_vlan_used(adapter))
 309                 return;
 310
 311         if (!test_bit(vid, adapter->active_vlans)) {
 312                 if (hw->mng_cookie.status &
 313                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 314                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 315                         adapter->mng_vlan_id = vid;
 316                 } else {
 317                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 318                 }
 319                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 320                     (vid != old_vid) &&
 321                     !test_bit(old_vid, adapter->active_vlans))
 322                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 323                                                old_vid);
 324         } else {
 325                 adapter->mng_vlan_id = vid;
 326         }
 327 }
 328
 329 static void e1000_init_manageability(struct e1000_adapter *adapter)
 330 {
 331         struct e1000_hw *hw = &adapter->hw;
 332
 333         if (adapter->en_mng_pt) {
 334                 u32 manc = er32(MANC);
 335
 336                 /* disable hardware interception of ARP */
 337                 manc &= ~(E1000_MANC_ARP_EN);
 338
 339                 ew32(MANC, manc);
 340         }
 341 }
 342
 343 static void e1000_release_manageability(struct e1000_adapter *adapter)
 344 {
 345         struct e1000_hw *hw = &adapter->hw;
 346
 347         if (adapter->en_mng_pt) {
 348                 u32 manc = er32(MANC);
 349
 350                 /* re-enable hardware interception of ARP */
 351                 manc |= E1000_MANC_ARP_EN;
 352
 353                 ew32(MANC, manc);
 354         }
 355 }
 356
 357 /**
 358  * e1000_configure - configure the hardware for RX and TX
 359  * @adapter: private board structure
 360  **/
 361 static void e1000_configure(struct e1000_adapter *adapter)
 362 {
 363         struct net_device *netdev = adapter->netdev;
 364         int i;
 365
 366         e1000_set_rx_mode(netdev);
 367
 368         e1000_restore_vlan(adapter);
 369         e1000_init_manageability(adapter);
 370
 371         e1000_configure_tx(adapter);
 372         e1000_setup_rctl(adapter);
 373         e1000_configure_rx(adapter);
 374         /* call E1000_DESC_UNUSED which always leaves
 375          * at least 1 descriptor unused to make sure
 376          * next_to_use != next_to_clean
 377          */
 378         for (i = 0; i < adapter->num_rx_queues; i++) {
 379                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 380                 adapter->alloc_rx_buf(adapter, ring,
 381                                       E1000_DESC_UNUSED(ring));
 382         }
 383 }
 384
 385 int e1000_up(struct e1000_adapter *adapter)
 386 {
 387         struct e1000_hw *hw = &adapter->hw;
 388
 389         /* hardware has been reset, we need to reload some things */
 390         e1000_configure(adapter);
 391
 392         clear_bit(__E1000_DOWN, &adapter->flags);
 393
 394         napi_enable(&adapter->napi);
 395
 396         e1000_irq_enable(adapter);
 397
 398         netif_wake_queue(adapter->netdev);
 399
 400         /* fire a link change interrupt to start the watchdog */
 401         ew32(ICS, E1000_ICS_LSC);
 402         return 0;
 403 }
 404
 405 /**
 406  * e1000_power_up_phy - restore link in case the phy was powered down
 407  * @adapter: address of board private structure
 408  *
 409  * The phy may be powered down to save power and turn off link when the
 410  * driver is unloaded and wake on lan is not enabled (among others)
 411  * *** this routine MUST be followed by a call to e1000_reset ***
 412  **/
 413 void e1000_power_up_phy(struct e1000_adapter *adapter)
 414 {
 415         struct e1000_hw *hw = &adapter->hw;
 416         u16 mii_reg = 0;
 417
 418         /* Just clear the power down bit to wake the phy back up */
 419         if (hw->media_type == e1000_media_type_copper) {
 420                 /* according to the manual, the phy will retain its
 421                  * settings across a power-down/up cycle
 422                  */
 423                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 424                 mii_reg &= ~MII_CR_POWER_DOWN;
 425                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 426         }
 427 }
 428
 429 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 430 {
 431         struct e1000_hw *hw = &adapter->hw;
 432
 433         /* Power down the PHY so no link is implied when interface is down *
 434          * The PHY cannot be powered down if any of the following is true *
 435          * (a) WoL is enabled
 436          * (b) AMT is active
 437          * (c) SoL/IDER session is active
 438          */
 439         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 440            hw->media_type == e1000_media_type_copper) {
 441                 u16 mii_reg = 0;
 442
 443                 switch (hw->mac_type) {
 444                 case e1000_82540:
 445                 case e1000_82545:
 446                 case e1000_82545_rev_3:
 447                 case e1000_82546:
 448                 case e1000_ce4100:
 449                 case e1000_82546_rev_3:
 450                 case e1000_82541:
 451                 case e1000_82541_rev_2:
 452                 case e1000_82547:
 453                 case e1000_82547_rev_2:
 454                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 455                                 goto out;
 456                         break;
 457                 default:
 458                         goto out;
 459                 }
 460                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 461                 mii_reg |= MII_CR_POWER_DOWN;
 462                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 463                 msleep(1);
 464         }
 465 out:
 466         return;
 467 }
 468
 469 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 470 {
 471         set_bit(__E1000_DOWN, &adapter->flags);
 472
 473         cancel_delayed_work_sync(&adapter->watchdog_task);
 474
 475         /*
 476          * Since the watchdog task can reschedule other tasks, we should cancel
 477          * it first, otherwise we can run into the situation when a work is
 478          * still running after the adapter has been turned down.
 479          */
 480
 481         cancel_delayed_work_sync(&adapter->phy_info_task);
 482         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 483
 484         /* Only kill reset task if adapter is not resetting */
 485         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 486                 cancel_work_sync(&adapter->reset_task);
 487 }
 488
 489 void e1000_down(struct e1000_adapter *adapter)
 490 {
 491         struct e1000_hw *hw = &adapter->hw;
 492         struct net_device *netdev = adapter->netdev;
 493         u32 rctl, tctl;
 494
 495         /* disable receives in the hardware */
 496         rctl = er32(RCTL);
 497         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 498         /* flush and sleep below */
 499
 500         netif_tx_disable(netdev);
 501
 502         /* disable transmits in the hardware */
 503         tctl = er32(TCTL);
 504         tctl &= ~E1000_TCTL_EN;
 505         ew32(TCTL, tctl);
 506         /* flush both disables and wait for them to finish */
 507         E1000_WRITE_FLUSH();
 508         msleep(10);
 509
 510         /* Set the carrier off after transmits have been disabled in the
 511          * hardware, to avoid race conditions with e1000_watchdog() (which
 512          * may be running concurrently to us, checking for the carrier
 513          * bit to decide whether it should enable transmits again). Such
 514          * a race condition would result into transmission being disabled
 515          * in the hardware until the next IFF_DOWN+IFF_UP cycle.
 516          */
 517         netif_carrier_off(netdev);
 518
 519         napi_disable(&adapter->napi);
 520
 521         e1000_irq_disable(adapter);
 522
 523         /* Setting DOWN must be after irq_disable to prevent
 524          * a screaming interrupt.  Setting DOWN also prevents
 525          * tasks from rescheduling.
 526          */
 527         e1000_down_and_stop(adapter);
 528
 529         adapter->link_speed = 0;
 530         adapter->link_duplex = 0;
 531
 532         e1000_reset(adapter);
 533         e1000_clean_all_tx_rings(adapter);
 534         e1000_clean_all_rx_rings(adapter);
 535 }
 536
 537 void e1000_reinit_locked(struct e1000_adapter *adapter)
 538 {
 539         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 540                 msleep(1);
 541
 542         /* only run the task if not already down */
 543         if (!test_bit(__E1000_DOWN, &adapter->flags)) {
 544                 e1000_down(adapter);
 545                 e1000_up(adapter);
 546         }
 547
 548         clear_bit(__E1000_RESETTING, &adapter->flags);
 549 }
 550
 551 void e1000_reset(struct e1000_adapter *adapter)
 552 {
 553         struct e1000_hw *hw = &adapter->hw;
 554         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 555         bool legacy_pba_adjust = false;
 556         u16 hwm;
 557
 558         /* Repartition Pba for greater than 9k mtu
 559          * To take effect CTRL.RST is required.
 560          */
 561
 562         switch (hw->mac_type) {
 563         case e1000_82542_rev2_0:
 564         case e1000_82542_rev2_1:
 565         case e1000_82543:
 566         case e1000_82544:
 567         case e1000_82540:
 568         case e1000_82541:
 569         case e1000_82541_rev_2:
 570                 legacy_pba_adjust = true;
 571                 pba = E1000_PBA_48K;
 572                 break;
 573         case e1000_82545:
 574         case e1000_82545_rev_3:
 575         case e1000_82546:
 576         case e1000_ce4100:
 577         case e1000_82546_rev_3:
 578                 pba = E1000_PBA_48K;
 579                 break;
 580         case e1000_82547:
 581         case e1000_82547_rev_2:
 582                 legacy_pba_adjust = true;
 583                 pba = E1000_PBA_30K;
 584                 break;
 585         case e1000_undefined:
 586         case e1000_num_macs:
 587                 break;
 588         }
 589
 590         if (legacy_pba_adjust) {
 591                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 592                         pba -= 8; /* allocate more FIFO for Tx */
 593
 594                 if (hw->mac_type == e1000_82547) {
 595                         adapter->tx_fifo_head = 0;
 596                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 597                         adapter->tx_fifo_size =
 598                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 599                         atomic_set(&adapter->tx_fifo_stall, 0);
 600                 }
 601         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 602                 /* adjust PBA for jumbo frames */
 603                 ew32(PBA, pba);
 604
 605                 /* To maintain wire speed transmits, the Tx FIFO should be
 606                  * large enough to accommodate two full transmit packets,
 607                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 608                  * the Rx FIFO should be large enough to accommodate at least
 609                  * one full receive packet and is similarly rounded up and
 610                  * expressed in KB.
 611                  */
 612                 pba = er32(PBA);
 613                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 614                 tx_space = pba >> 16;
 615                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 616                 pba &= 0xffff;
 617                 /* the Tx fifo also stores 16 bytes of information about the Tx
 618                  * but don't include ethernet FCS because hardware appends it
 619                  */
 620                 min_tx_space = (hw->max_frame_size +
 621                                 sizeof(struct e1000_tx_desc) -
 622                                 ETH_FCS_LEN) * 2;
 623                 min_tx_space = ALIGN(min_tx_space, 1024);
 624                 min_tx_space >>= 10;
 625                 /* software strips receive CRC, so leave room for it */
 626                 min_rx_space = hw->max_frame_size;
 627                 min_rx_space = ALIGN(min_rx_space, 1024);
 628                 min_rx_space >>= 10;
 629
 630                 /* If current Tx allocation is less than the min Tx FIFO size,
 631                  * and the min Tx FIFO size is less than the current Rx FIFO
 632                  * allocation, take space away from current Rx allocation
 633                  */
 634                 if (tx_space < min_tx_space &&
 635                     ((min_tx_space - tx_space) < pba)) {
 636                         pba = pba - (min_tx_space - tx_space);
 637
 638                         /* PCI/PCIx hardware has PBA alignment constraints */
 639                         switch (hw->mac_type) {
 640                         case e1000_82545 ... e1000_82546_rev_3:
 641                                 pba &= ~(E1000_PBA_8K - 1);
 642                                 break;
 643                         default:
 644                                 break;
 645                         }
 646
 647                         /* if short on Rx space, Rx wins and must trump Tx
 648                          * adjustment or use Early Receive if available
 649                          */
 650                         if (pba < min_rx_space)
 651                                 pba = min_rx_space;
 652                 }
 653         }
 654
 655         ew32(PBA, pba);
 656
 657         /* flow control settings:
 658          * The high water mark must be low enough to fit one full frame
 659          * (or the size used for early receive) above it in the Rx FIFO.
 660          * Set it to the lower of:
 661          * - 90% of the Rx FIFO size, and
 662          * - the full Rx FIFO size minus the early receive size (for parts
 663          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 664          * - the full Rx FIFO size minus one full frame
 665          */
 666         hwm = min(((pba << 10) * 9 / 10),
 667                   ((pba << 10) - hw->max_frame_size));
 668
 669         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 670         hw->fc_low_water = hw->fc_high_water - 8;
 671         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 672         hw->fc_send_xon = 1;
 673         hw->fc = hw->original_fc;
 674
 675         /* Allow time for pending master requests to run */
 676         e1000_reset_hw(hw);
 677         if (hw->mac_type >= e1000_82544)
 678                 ew32(WUC, 0);
 679
 680         if (e1000_init_hw(hw))
 681                 e_dev_err("Hardware Error\n");
 682         e1000_update_mng_vlan(adapter);
 683
 684         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 685         if (hw->mac_type >= e1000_82544 &&
 686             hw->autoneg == 1 &&
 687             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 688                 u32 ctrl = er32(CTRL);
 689                 /* clear phy power management bit if we are in gig only mode,
 690                  * which if enabled will attempt negotiation to 100Mb, which
 691                  * can cause a loss of link at power off or driver unload
 692                  */
 693                 ctrl &= ~E1000_CTRL_SWDPIN3;
 694                 ew32(CTRL, ctrl);
 695         }
 696
 697         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 698         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 699
 700         e1000_reset_adaptive(hw);
 701         e1000_phy_get_info(hw, &adapter->phy_info);
 702
 703         e1000_release_manageability(adapter);
 704 }
 705
 706 /* Dump the eeprom for users having checksum issues */
 707 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 708 {
 709         struct net_device *netdev = adapter->netdev;
 710         struct ethtool_eeprom eeprom;
 711         const struct ethtool_ops *ops = netdev->ethtool_ops;
 712         u8 *data;
 713         int i;
 714         u16 csum_old, csum_new = 0;
 715
 716         eeprom.len = ops->get_eeprom_len(netdev);
 717         eeprom.offset = 0;
 718
 719         data = kmalloc(eeprom.len, GFP_KERNEL);
 720         if (!data)
 721                 return;
 722
 723         ops->get_eeprom(netdev, &eeprom, data);
 724
 725         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 726                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 727         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 728                 csum_new += data[i] + (data[i + 1] << 8);
 729         csum_new = EEPROM_SUM - csum_new;
 730
 731         pr_err("/*********************/\n");
 732         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 733         pr_err("Calculated              : 0x%04x\n", csum_new);
 734
 735         pr_err("Offset    Values\n");
 736         pr_err("========  ======\n");
 737         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 738
 739         pr_err("Include this output when contacting your support provider.\n");
 740         pr_err("This is not a software error! Something bad happened to\n");
 741         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 742         pr_err("result in further problems, possibly loss of data,\n");
 743         pr_err("corruption or system hangs!\n");
 744         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 745         pr_err("which is invalid and requires you to set the proper MAC\n");
 746         pr_err("address manually before continuing to enable this network\n");
 747         pr_err("device. Please inspect the EEPROM dump and report the\n");
 748         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 749         pr_err("/*********************/\n");
 750
 751         kfree(data);
 752 }
 753
 754 /**
 755  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 756  * @pdev: PCI device information struct
 757  *
 758  * Return true if an adapter needs ioport resources
 759  **/
 760 static int e1000_is_need_ioport(struct pci_dev *pdev)
 761 {
 762         switch (pdev->device) {
 763         case E1000_DEV_ID_82540EM:
 764         case E1000_DEV_ID_82540EM_LOM:
 765         case E1000_DEV_ID_82540EP:
 766         case E1000_DEV_ID_82540EP_LOM:
 767         case E1000_DEV_ID_82540EP_LP:
 768         case E1000_DEV_ID_82541EI:
 769         case E1000_DEV_ID_82541EI_MOBILE:
 770         case E1000_DEV_ID_82541ER:
 771         case E1000_DEV_ID_82541ER_LOM:
 772         case E1000_DEV_ID_82541GI:
 773         case E1000_DEV_ID_82541GI_LF:
 774         case E1000_DEV_ID_82541GI_MOBILE:
 775         case E1000_DEV_ID_82544EI_COPPER:
 776         case E1000_DEV_ID_82544EI_FIBER:
 777         case E1000_DEV_ID_82544GC_COPPER:
 778         case E1000_DEV_ID_82544GC_LOM:
 779         case E1000_DEV_ID_82545EM_COPPER:
 780         case E1000_DEV_ID_82545EM_FIBER:
 781         case E1000_DEV_ID_82546EB_COPPER:
 782         case E1000_DEV_ID_82546EB_FIBER:
 783         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 784                 return true;
 785         default:
 786                 return false;
 787         }
 788 }
 789
 790 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 791         netdev_features_t features)
 792 {
 793         /* Since there is no support for separate Rx/Tx vlan accel
 794          * enable/disable make sure Tx flag is always in same state as Rx.
 795          */
 796         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 797                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 798         else
 799                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 800
 801         return features;
 802 }
 803
 804 static int e1000_set_features(struct net_device *netdev,
 805         netdev_features_t features)
 806 {
 807         struct e1000_adapter *adapter = netdev_priv(netdev);
 808         netdev_features_t changed = features ^ netdev->features;
 809
 810         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 811                 e1000_vlan_mode(netdev, features);
 812
 813         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 814                 return 0;
 815
 816         netdev->features = features;
 817         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 818
 819         if (netif_running(netdev))
 820                 e1000_reinit_locked(adapter);
 821         else
 822                 e1000_reset(adapter);
 823
 824         return 1;
 825 }
 826
 827 static const struct net_device_ops e1000_netdev_ops = {
 828         .ndo_open               = e1000_open,
 829         .ndo_stop               = e1000_close,
 830         .ndo_start_xmit         = e1000_xmit_frame,
 831         .ndo_set_rx_mode        = e1000_set_rx_mode,
 832         .ndo_set_mac_address    = e1000_set_mac,
 833         .ndo_tx_timeout         = e1000_tx_timeout,
 834         .ndo_change_mtu         = e1000_change_mtu,
 835         .ndo_do_ioctl           = e1000_ioctl,
 836         .ndo_validate_addr      = eth_validate_addr,
 837         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 838         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 839 #ifdef CONFIG_NET_POLL_CONTROLLER
 840         .ndo_poll_controller    = e1000_netpoll,
 841 #endif
 842         .ndo_fix_features       = e1000_fix_features,
 843         .ndo_set_features       = e1000_set_features,
 844 };
 845
 846 /**
 847  * e1000_init_hw_struct - initialize members of hw struct
 848  * @adapter: board private struct
 849  * @hw: structure used by e1000_hw.c
 850  *
 851  * Factors out initialization of the e1000_hw struct to its own function
 852  * that can be called very early at init (just after struct allocation).
 853  * Fields are initialized based on PCI device information and
 854  * OS network device settings (MTU size).
 855  * Returns negative error codes if MAC type setup fails.
 856  */
 857 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 858                                 struct e1000_hw *hw)
 859 {
 860         struct pci_dev *pdev = adapter->pdev;
 861
 862         /* PCI config space info */
 863         hw->vendor_id = pdev->vendor;
 864         hw->device_id = pdev->device;
 865         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 866         hw->subsystem_id = pdev->subsystem_device;
 867         hw->revision_id = pdev->revision;
 868
 869         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 870
 871         hw->max_frame_size = adapter->netdev->mtu +
 872                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 873         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 874
 875         /* identify the MAC */
 876         if (e1000_set_mac_type(hw)) {
 877                 e_err(probe, "Unknown MAC Type\n");
 878                 return -EIO;
 879         }
 880
 881         switch (hw->mac_type) {
 882         default:
 883                 break;
 884         case e1000_82541:
 885         case e1000_82547:
 886         case e1000_82541_rev_2:
 887         case e1000_82547_rev_2:
 888                 hw->phy_init_script = 1;
 889                 break;
 890         }
 891
 892         e1000_set_media_type(hw);
 893         e1000_get_bus_info(hw);
 894
 895         hw->wait_autoneg_complete = false;
 896         hw->tbi_compatibility_en = true;
 897         hw->adaptive_ifs = true;
 898
 899         /* Copper options */
 900
 901         if (hw->media_type == e1000_media_type_copper) {
 902                 hw->mdix = AUTO_ALL_MODES;
 903                 hw->disable_polarity_correction = false;
 904                 hw->master_slave = E1000_MASTER_SLAVE;
 905         }
 906
 907         return 0;
 908 }
 909
 910 /**
 911  * e1000_probe - Device Initialization Routine
 912  * @pdev: PCI device information struct
 913  * @ent: entry in e1000_pci_tbl
 914  *
 915  * Returns 0 on success, negative on failure
 916  *
 917  * e1000_probe initializes an adapter identified by a pci_dev structure.
 918  * The OS initialization, configuring of the adapter private structure,
 919  * and a hardware reset occur.
 920  **/
 921 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 922 {
 923         struct net_device *netdev;
 924         struct e1000_adapter *adapter = NULL;
 925         struct e1000_hw *hw;
 926
 927         static int cards_found;
 928         static int global_quad_port_a; /* global ksp3 port a indication */
 929         int i, err, pci_using_dac;
 930         u16 eeprom_data = 0;
 931         u16 tmp = 0;
 932         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 933         int bars, need_ioport;
 934         bool disable_dev = false;
 935
 936         /* do not allocate ioport bars when not needed */
 937         need_ioport = e1000_is_need_ioport(pdev);
 938         if (need_ioport) {
 939                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 940                 err = pci_enable_device(pdev);
 941         } else {
 942                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 943                 err = pci_enable_device_mem(pdev);
 944         }
 945         if (err)
 946                 return err;
 947
 948         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 949         if (err)
 950                 goto err_pci_reg;
 951
 952         pci_set_master(pdev);
 953         err = pci_save_state(pdev);
 954         if (err)
 955                 goto err_alloc_etherdev;
 956
 957         err = -ENOMEM;
 958         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 959         if (!netdev)
 960                 goto err_alloc_etherdev;
 961
 962         SET_NETDEV_DEV(netdev, &pdev->dev);
 963
 964         pci_set_drvdata(pdev, netdev);
 965         adapter = netdev_priv(netdev);
 966         adapter->netdev = netdev;
 967         adapter->pdev = pdev;
 968         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 969         adapter->bars = bars;
 970         adapter->need_ioport = need_ioport;
 971
 972         hw = &adapter->hw;
 973         hw->back = adapter;
 974
 975         err = -EIO;
 976         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
 977         if (!hw->hw_addr)
 978                 goto err_ioremap;
 979
 980         if (adapter->need_ioport) {
 981                 for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) {
 982                         if (pci_resource_len(pdev, i) == 0)
 983                                 continue;
 984                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 985                                 hw->io_base = pci_resource_start(pdev, i);
 986                                 break;
 987                         }
 988                 }
 989         }
 990
 991         /* make ready for any if (hw->...) below */
 992         err = e1000_init_hw_struct(adapter, hw);
 993         if (err)
 994                 goto err_sw_init;
 995
 996         /* there is a workaround being applied below that limits
 997          * 64-bit DMA addresses to 64-bit hardware.  There are some
 998          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
 999          */
1000         pci_using_dac = 0;
1001         if ((hw->bus_type == e1000_bus_type_pcix) &&
1002             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1003                 pci_using_dac = 1;
1004         } else {
1005                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1006                 if (err) {
1007                         pr_err("No usable DMA config, aborting\n");
1008                         goto err_dma;
1009                 }
1010         }
1011
1012         netdev->netdev_ops = &e1000_netdev_ops;
1013         e1000_set_ethtool_ops(netdev);
1014         netdev->watchdog_timeo = 5 * HZ;
1015         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1016
1017         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1018
1019         adapter->bd_number = cards_found;
1020
1021         /* setup the private structure */
1022
1023         err = e1000_sw_init(adapter);
1024         if (err)
1025                 goto err_sw_init;
1026
1027         err = -EIO;
1028         if (hw->mac_type == e1000_ce4100) {
1029                 hw->ce4100_gbe_mdio_base_virt =
1030                                         ioremap(pci_resource_start(pdev, BAR_1),
1031                                                 pci_resource_len(pdev, BAR_1));
1032
1033                 if (!hw->ce4100_gbe_mdio_base_virt)
1034                         goto err_mdio_ioremap;
1035         }
1036
1037         if (hw->mac_type >= e1000_82543) {
1038                 netdev->hw_features = NETIF_F_SG |
1039                                    NETIF_F_HW_CSUM |
1040                                    NETIF_F_HW_VLAN_CTAG_RX;
1041                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1042                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1043         }
1044
1045         if ((hw->mac_type >= e1000_82544) &&
1046            (hw->mac_type != e1000_82547))
1047                 netdev->hw_features |= NETIF_F_TSO;
1048
1049         netdev->priv_flags |= IFF_SUPP_NOFCS;
1050
1051         netdev->features |= netdev->hw_features;
1052         netdev->hw_features |= (NETIF_F_RXCSUM |
1053                                 NETIF_F_RXALL |
1054                                 NETIF_F_RXFCS);
1055
1056         if (pci_using_dac) {
1057                 netdev->features |= NETIF_F_HIGHDMA;
1058                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1059         }
1060
1061         netdev->vlan_features |= (NETIF_F_TSO |
1062                                   NETIF_F_HW_CSUM |
1063                                   NETIF_F_SG);
1064
1065         /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1066         if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1067             hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1068                 netdev->priv_flags |= IFF_UNICAST_FLT;
1069
1070         /* MTU range: 46 - 16110 */
1071         netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1072         netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1073
1074         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1075
1076         /* initialize eeprom parameters */
1077         if (e1000_init_eeprom_params(hw)) {
1078                 e_err(probe, "EEPROM initialization failed\n");
1079                 goto err_eeprom;
1080         }
1081
1082         /* before reading the EEPROM, reset the controller to
1083          * put the device in a known good starting state
1084          */
1085
1086         e1000_reset_hw(hw);
1087
1088         /* make sure the EEPROM is good */
1089         if (e1000_validate_eeprom_checksum(hw) < 0) {
1090                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1091                 e1000_dump_eeprom(adapter);
1092                 /* set MAC address to all zeroes to invalidate and temporary
1093                  * disable this device for the user. This blocks regular
1094                  * traffic while still permitting ethtool ioctls from reaching
1095                  * the hardware as well as allowing the user to run the
1096                  * interface after manually setting a hw addr using
1097                  * `ip set address`
1098                  */
1099                 memset(hw->mac_addr, 0, netdev->addr_len);
1100         } else {
1101                 /* copy the MAC address out of the EEPROM */
1102                 if (e1000_read_mac_addr(hw))
1103                         e_err(probe, "EEPROM Read Error\n");
1104         }
1105         /* don't block initialization here due to bad MAC address */
1106         memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1107
1108         if (!is_valid_ether_addr(netdev->dev_addr))
1109                 e_err(probe, "Invalid MAC Address\n");
1110
1111
1112         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1113         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1114                           e1000_82547_tx_fifo_stall_task);
1115         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1116         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1117
1118         e1000_check_options(adapter);
1119
1120         /* Initial Wake on LAN setting
1121          * If APM wake is enabled in the EEPROM,
1122          * enable the ACPI Magic Packet filter
1123          */
1124
1125         switch (hw->mac_type) {
1126         case e1000_82542_rev2_0:
1127         case e1000_82542_rev2_1:
1128         case e1000_82543:
1129                 break;
1130         case e1000_82544:
1131                 e1000_read_eeprom(hw,
1132                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1133                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1134                 break;
1135         case e1000_82546:
1136         case e1000_82546_rev_3:
1137                 if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1138                         e1000_read_eeprom(hw,
1139                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1140                         break;
1141                 }
1142                 fallthrough;
1143         default:
1144                 e1000_read_eeprom(hw,
1145                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1146                 break;
1147         }
1148         if (eeprom_data & eeprom_apme_mask)
1149                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1150
1151         /* now that we have the eeprom settings, apply the special cases
1152          * where the eeprom may be wrong or the board simply won't support
1153          * wake on lan on a particular port
1154          */
1155         switch (pdev->device) {
1156         case E1000_DEV_ID_82546GB_PCIE:
1157                 adapter->eeprom_wol = 0;
1158                 break;
1159         case E1000_DEV_ID_82546EB_FIBER:
1160         case E1000_DEV_ID_82546GB_FIBER:
1161                 /* Wake events only supported on port A for dual fiber
1162                  * regardless of eeprom setting
1163                  */
1164                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1165                         adapter->eeprom_wol = 0;
1166                 break;
1167         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1168                 /* if quad port adapter, disable WoL on all but port A */
1169                 if (global_quad_port_a != 0)
1170                         adapter->eeprom_wol = 0;
1171                 else
1172                         adapter->quad_port_a = true;
1173                 /* Reset for multiple quad port adapters */
1174                 if (++global_quad_port_a == 4)
1175                         global_quad_port_a = 0;
1176                 break;
1177         }
1178
1179         /* initialize the wol settings based on the eeprom settings */
1180         adapter->wol = adapter->eeprom_wol;
1181         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1182
1183         /* Auto detect PHY address */
1184         if (hw->mac_type == e1000_ce4100) {
1185                 for (i = 0; i < 32; i++) {
1186                         hw->phy_addr = i;
1187                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1188
1189                         if (tmp != 0 && tmp != 0xFF)
1190                                 break;
1191                 }
1192
1193                 if (i >= 32)
1194                         goto err_eeprom;
1195         }
1196
1197         /* reset the hardware with the new settings */
1198         e1000_reset(adapter);
1199
1200         strcpy(netdev->name, "eth%d");
1201         err = register_netdev(netdev);
1202         if (err)
1203                 goto err_register;
1204
1205         e1000_vlan_filter_on_off(adapter, false);
1206
1207         /* print bus type/speed/width info */
1208         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1209                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1210                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1211                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1212                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1213                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1214                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1215                netdev->dev_addr);
1216
1217         /* carrier off reporting is important to ethtool even BEFORE open */
1218         netif_carrier_off(netdev);
1219
1220         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1221
1222         cards_found++;
1223         return 0;
1224
1225 err_register:
1226 err_eeprom:
1227         e1000_phy_hw_reset(hw);
1228
1229         if (hw->flash_address)
1230                 iounmap(hw->flash_address);
1231         kfree(adapter->tx_ring);
1232         kfree(adapter->rx_ring);
1233 err_dma:
1234 err_sw_init:
1235 err_mdio_ioremap:
1236         iounmap(hw->ce4100_gbe_mdio_base_virt);
1237         iounmap(hw->hw_addr);
1238 err_ioremap:
1239         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1240         free_netdev(netdev);
1241 err_alloc_etherdev:
1242         pci_release_selected_regions(pdev, bars);
1243 err_pci_reg:
1244         if (!adapter || disable_dev)
1245                 pci_disable_device(pdev);
1246         return err;
1247 }
1248
1249 /**
1250  * e1000_remove - Device Removal Routine
1251  * @pdev: PCI device information struct
1252  *
1253  * e1000_remove is called by the PCI subsystem to alert the driver
1254  * that it should release a PCI device. That could be caused by a
1255  * Hot-Plug event, or because the driver is going to be removed from
1256  * memory.
1257  **/
1258 static void e1000_remove(struct pci_dev *pdev)
1259 {
1260         struct net_device *netdev = pci_get_drvdata(pdev);
1261         struct e1000_adapter *adapter = netdev_priv(netdev);
1262         struct e1000_hw *hw = &adapter->hw;
1263         bool disable_dev;
1264
1265         e1000_down_and_stop(adapter);
1266         e1000_release_manageability(adapter);
1267
1268         unregister_netdev(netdev);
1269
1270         e1000_phy_hw_reset(hw);
1271
1272         kfree(adapter->tx_ring);
1273         kfree(adapter->rx_ring);
1274
1275         if (hw->mac_type == e1000_ce4100)
1276                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1277         iounmap(hw->hw_addr);
1278         if (hw->flash_address)
1279                 iounmap(hw->flash_address);
1280         pci_release_selected_regions(pdev, adapter->bars);
1281
1282         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1283         free_netdev(netdev);
1284
1285         if (disable_dev)
1286                 pci_disable_device(pdev);
1287 }
1288
1289 /**
1290  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1291  * @adapter: board private structure to initialize
1292  *
1293  * e1000_sw_init initializes the Adapter private data structure.
1294  * e1000_init_hw_struct MUST be called before this function
1295  **/
1296 static int e1000_sw_init(struct e1000_adapter *adapter)
1297 {
1298         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1299
1300         adapter->num_tx_queues = 1;
1301         adapter->num_rx_queues = 1;
1302
1303         if (e1000_alloc_queues(adapter)) {
1304                 e_err(probe, "Unable to allocate memory for queues\n");
1305                 return -ENOMEM;
1306         }
1307
1308         /* Explicitly disable IRQ since the NIC can be in any state. */
1309         e1000_irq_disable(adapter);
1310
1311         spin_lock_init(&adapter->stats_lock);
1312
1313         set_bit(__E1000_DOWN, &adapter->flags);
1314
1315         return 0;
1316 }
1317
1318 /**
1319  * e1000_alloc_queues - Allocate memory for all rings
1320  * @adapter: board private structure to initialize
1321  *
1322  * We allocate one ring per queue at run-time since we don't know the
1323  * number of queues at compile-time.
1324  **/
1325 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1326 {
1327         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1328                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1329         if (!adapter->tx_ring)
1330                 return -ENOMEM;
1331
1332         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1333                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1334         if (!adapter->rx_ring) {
1335                 kfree(adapter->tx_ring);
1336                 return -ENOMEM;
1337         }
1338
1339         return E1000_SUCCESS;
1340 }
1341
1342 /**
1343  * e1000_open - Called when a network interface is made active
1344  * @netdev: network interface device structure
1345  *
1346  * Returns 0 on success, negative value on failure
1347  *
1348  * The open entry point is called when a network interface is made
1349  * active by the system (IFF_UP).  At this point all resources needed
1350  * for transmit and receive operations are allocated, the interrupt
1351  * handler is registered with the OS, the watchdog task is started,
1352  * and the stack is notified that the interface is ready.
1353  **/
1354 int e1000_open(struct net_device *netdev)
1355 {
1356         struct e1000_adapter *adapter = netdev_priv(netdev);
1357         struct e1000_hw *hw = &adapter->hw;
1358         int err;
1359
1360         /* disallow open during test */
1361         if (test_bit(__E1000_TESTING, &adapter->flags))
1362                 return -EBUSY;
1363
1364         netif_carrier_off(netdev);
1365
1366         /* allocate transmit descriptors */
1367         err = e1000_setup_all_tx_resources(adapter);
1368         if (err)
1369                 goto err_setup_tx;
1370
1371         /* allocate receive descriptors */
1372         err = e1000_setup_all_rx_resources(adapter);
1373         if (err)
1374                 goto err_setup_rx;
1375
1376         e1000_power_up_phy(adapter);
1377
1378         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1379         if ((hw->mng_cookie.status &
1380                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1381                 e1000_update_mng_vlan(adapter);
1382         }
1383
1384         /* before we allocate an interrupt, we must be ready to handle it.
1385          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1386          * as soon as we call pci_request_irq, so we have to setup our
1387          * clean_rx handler before we do so.
1388          */
1389         e1000_configure(adapter);
1390
1391         err = e1000_request_irq(adapter);
1392         if (err)
1393                 goto err_req_irq;
1394
1395         /* From here on the code is the same as e1000_up() */
1396         clear_bit(__E1000_DOWN, &adapter->flags);
1397
1398         napi_enable(&adapter->napi);
1399
1400         e1000_irq_enable(adapter);
1401
1402         netif_start_queue(netdev);
1403
1404         /* fire a link status change interrupt to start the watchdog */
1405         ew32(ICS, E1000_ICS_LSC);
1406
1407         return E1000_SUCCESS;
1408
1409 err_req_irq:
1410         e1000_power_down_phy(adapter);
1411         e1000_free_all_rx_resources(adapter);
1412 err_setup_rx:
1413         e1000_free_all_tx_resources(adapter);
1414 err_setup_tx:
1415         e1000_reset(adapter);
1416
1417         return err;
1418 }
1419
1420 /**
1421  * e1000_close - Disables a network interface
1422  * @netdev: network interface device structure
1423  *
1424  * Returns 0, this is not allowed to fail
1425  *
1426  * The close entry point is called when an interface is de-activated
1427  * by the OS.  The hardware is still under the drivers control, but
1428  * needs to be disabled.  A global MAC reset is issued to stop the
1429  * hardware, and all transmit and receive resources are freed.
1430  **/
1431 int e1000_close(struct net_device *netdev)
1432 {
1433         struct e1000_adapter *adapter = netdev_priv(netdev);
1434         struct e1000_hw *hw = &adapter->hw;
1435         int count = E1000_CHECK_RESET_COUNT;
1436
1437         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags) && count--)
1438                 usleep_range(10000, 20000);
1439
1440         WARN_ON(count < 0);
1441
1442         /* signal that we're down so that the reset task will no longer run */
1443         set_bit(__E1000_DOWN, &adapter->flags);
1444         clear_bit(__E1000_RESETTING, &adapter->flags);
1445
1446         e1000_down(adapter);
1447         e1000_power_down_phy(adapter);
1448         e1000_free_irq(adapter);
1449
1450         e1000_free_all_tx_resources(adapter);
1451         e1000_free_all_rx_resources(adapter);
1452
1453         /* kill manageability vlan ID if supported, but not if a vlan with
1454          * the same ID is registered on the host OS (let 8021q kill it)
1455          */
1456         if ((hw->mng_cookie.status &
1457              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1458             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1459                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1460                                        adapter->mng_vlan_id);
1461         }
1462
1463         return 0;
1464 }
1465
1466 /**
1467  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1468  * @adapter: address of board private structure
1469  * @start: address of beginning of memory
1470  * @len: length of memory
1471  **/
1472 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1473                                   unsigned long len)
1474 {
1475         struct e1000_hw *hw = &adapter->hw;
1476         unsigned long begin = (unsigned long)start;
1477         unsigned long end = begin + len;
1478
1479         /* First rev 82545 and 82546 need to not allow any memory
1480          * write location to cross 64k boundary due to errata 23
1481          */
1482         if (hw->mac_type == e1000_82545 ||
1483             hw->mac_type == e1000_ce4100 ||
1484             hw->mac_type == e1000_82546) {
1485                 return ((begin ^ (end - 1)) >> 16) == 0;
1486         }
1487
1488         return true;
1489 }
1490
1491 /**
1492  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1493  * @adapter: board private structure
1494  * @txdr:    tx descriptor ring (for a specific queue) to setup
1495  *
1496  * Return 0 on success, negative on failure
1497  **/
1498 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1499                                     struct e1000_tx_ring *txdr)
1500 {
1501         struct pci_dev *pdev = adapter->pdev;
1502         int size;
1503
1504         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1505         txdr->buffer_info = vzalloc(size);
1506         if (!txdr->buffer_info)
1507                 return -ENOMEM;
1508
1509         /* round up to nearest 4K */
1510
1511         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1512         txdr->size = ALIGN(txdr->size, 4096);
1513
1514         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1515                                         GFP_KERNEL);
1516         if (!txdr->desc) {
1517 setup_tx_desc_die:
1518                 vfree(txdr->buffer_info);
1519                 return -ENOMEM;
1520         }
1521
1522         /* Fix for errata 23, can't cross 64kB boundary */
1523         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1524                 void *olddesc = txdr->desc;
1525                 dma_addr_t olddma = txdr->dma;
1526                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1527                       txdr->size, txdr->desc);
1528                 /* Try again, without freeing the previous */
1529                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1530                                                 &txdr->dma, GFP_KERNEL);
1531                 /* Failed allocation, critical failure */
1532                 if (!txdr->desc) {
1533                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1534                                           olddma);
1535                         goto setup_tx_desc_die;
1536                 }
1537
1538                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1539                         /* give up */
1540                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1541                                           txdr->dma);
1542                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1543                                           olddma);
1544                         e_err(probe, "Unable to allocate aligned memory "
1545                               "for the transmit descriptor ring\n");
1546                         vfree(txdr->buffer_info);
1547                         return -ENOMEM;
1548                 } else {
1549                         /* Free old allocation, new allocation was successful */
1550                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1551                                           olddma);
1552                 }
1553         }
1554         memset(txdr->desc, 0, txdr->size);
1555
1556         txdr->next_to_use = 0;
1557         txdr->next_to_clean = 0;
1558
1559         return 0;
1560 }
1561
1562 /**
1563  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1564  *                                (Descriptors) for all queues
1565  * @adapter: board private structure
1566  *
1567  * Return 0 on success, negative on failure
1568  **/
1569 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1570 {
1571         int i, err = 0;
1572
1573         for (i = 0; i < adapter->num_tx_queues; i++) {
1574                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1575                 if (err) {
1576                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1577                         for (i-- ; i >= 0; i--)
1578                                 e1000_free_tx_resources(adapter,
1579                                                         &adapter->tx_ring[i]);
1580                         break;
1581                 }
1582         }
1583
1584         return err;
1585 }
1586
1587 /**
1588  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1589  * @adapter: board private structure
1590  *
1591  * Configure the Tx unit of the MAC after a reset.
1592  **/
1593 static void e1000_configure_tx(struct e1000_adapter *adapter)
1594 {
1595         u64 tdba;
1596         struct e1000_hw *hw = &adapter->hw;
1597         u32 tdlen, tctl, tipg;
1598         u32 ipgr1, ipgr2;
1599
1600         /* Setup the HW Tx Head and Tail descriptor pointers */
1601
1602         switch (adapter->num_tx_queues) {
1603         case 1:
1604         default:
1605                 tdba = adapter->tx_ring[0].dma;
1606                 tdlen = adapter->tx_ring[0].count *
1607                         sizeof(struct e1000_tx_desc);
1608                 ew32(TDLEN, tdlen);
1609                 ew32(TDBAH, (tdba >> 32));
1610                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1611                 ew32(TDT, 0);
1612                 ew32(TDH, 0);
1613                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1614                                            E1000_TDH : E1000_82542_TDH);
1615                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1616                                            E1000_TDT : E1000_82542_TDT);
1617                 break;
1618         }
1619
1620         /* Set the default values for the Tx Inter Packet Gap timer */
1621         if ((hw->media_type == e1000_media_type_fiber ||
1622              hw->media_type == e1000_media_type_internal_serdes))
1623                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1624         else
1625                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1626
1627         switch (hw->mac_type) {
1628         case e1000_82542_rev2_0:
1629         case e1000_82542_rev2_1:
1630                 tipg = DEFAULT_82542_TIPG_IPGT;
1631                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1632                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1633                 break;
1634         default:
1635                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1636                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1637                 break;
1638         }
1639         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1640         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1641         ew32(TIPG, tipg);
1642
1643         /* Set the Tx Interrupt Delay register */
1644
1645         ew32(TIDV, adapter->tx_int_delay);
1646         if (hw->mac_type >= e1000_82540)
1647                 ew32(TADV, adapter->tx_abs_int_delay);
1648
1649         /* Program the Transmit Control Register */
1650
1651         tctl = er32(TCTL);
1652         tctl &= ~E1000_TCTL_CT;
1653         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1654                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1655
1656         e1000_config_collision_dist(hw);
1657
1658         /* Setup Transmit Descriptor Settings for eop descriptor */
1659         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1660
1661         /* only set IDE if we are delaying interrupts using the timers */
1662         if (adapter->tx_int_delay)
1663                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1664
1665         if (hw->mac_type < e1000_82543)
1666                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1667         else
1668                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1669
1670         /* Cache if we're 82544 running in PCI-X because we'll
1671          * need this to apply a workaround later in the send path.
1672          */
1673         if (hw->mac_type == e1000_82544 &&
1674             hw->bus_type == e1000_bus_type_pcix)
1675                 adapter->pcix_82544 = true;
1676
1677         ew32(TCTL, tctl);
1678
1679 }
1680
1681 /**
1682  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1683  * @adapter: board private structure
1684  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1685  *
1686  * Returns 0 on success, negative on failure
1687  **/
1688 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1689                                     struct e1000_rx_ring *rxdr)
1690 {
1691         struct pci_dev *pdev = adapter->pdev;
1692         int size, desc_len;
1693
1694         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1695         rxdr->buffer_info = vzalloc(size);
1696         if (!rxdr->buffer_info)
1697                 return -ENOMEM;
1698
1699         desc_len = sizeof(struct e1000_rx_desc);
1700
1701         /* Round up to nearest 4K */
1702
1703         rxdr->size = rxdr->count * desc_len;
1704         rxdr->size = ALIGN(rxdr->size, 4096);
1705
1706         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1707                                         GFP_KERNEL);
1708         if (!rxdr->desc) {
1709 setup_rx_desc_die:
1710                 vfree(rxdr->buffer_info);
1711                 return -ENOMEM;
1712         }
1713
1714         /* Fix for errata 23, can't cross 64kB boundary */
1715         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1716                 void *olddesc = rxdr->desc;
1717                 dma_addr_t olddma = rxdr->dma;
1718                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1719                       rxdr->size, rxdr->desc);
1720                 /* Try again, without freeing the previous */
1721                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1722                                                 &rxdr->dma, GFP_KERNEL);
1723                 /* Failed allocation, critical failure */
1724                 if (!rxdr->desc) {
1725                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1726                                           olddma);
1727                         goto setup_rx_desc_die;
1728                 }
1729
1730                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1731                         /* give up */
1732                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1733                                           rxdr->dma);
1734                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1735                                           olddma);
1736                         e_err(probe, "Unable to allocate aligned memory for "
1737                               "the Rx descriptor ring\n");
1738                         goto setup_rx_desc_die;
1739                 } else {
1740                         /* Free old allocation, new allocation was successful */
1741                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1742                                           olddma);
1743                 }
1744         }
1745         memset(rxdr->desc, 0, rxdr->size);
1746
1747         rxdr->next_to_clean = 0;
1748         rxdr->next_to_use = 0;
1749         rxdr->rx_skb_top = NULL;
1750
1751         return 0;
1752 }
1753
1754 /**
1755  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1756  *                                (Descriptors) for all queues
1757  * @adapter: board private structure
1758  *
1759  * Return 0 on success, negative on failure
1760  **/
1761 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1762 {
1763         int i, err = 0;
1764
1765         for (i = 0; i < adapter->num_rx_queues; i++) {
1766                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1767                 if (err) {
1768                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1769                         for (i-- ; i >= 0; i--)
1770                                 e1000_free_rx_resources(adapter,
1771                                                         &adapter->rx_ring[i]);
1772                         break;
1773                 }
1774         }
1775
1776         return err;
1777 }
1778
1779 /**
1780  * e1000_setup_rctl - configure the receive control registers
1781  * @adapter: Board private structure
1782  **/
1783 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1784 {
1785         struct e1000_hw *hw = &adapter->hw;
1786         u32 rctl;
1787
1788         rctl = er32(RCTL);
1789
1790         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1791
1792         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1793                 E1000_RCTL_RDMTS_HALF |
1794                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1795
1796         if (hw->tbi_compatibility_on == 1)
1797                 rctl |= E1000_RCTL_SBP;
1798         else
1799                 rctl &= ~E1000_RCTL_SBP;
1800
1801         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1802                 rctl &= ~E1000_RCTL_LPE;
1803         else
1804                 rctl |= E1000_RCTL_LPE;
1805
1806         /* Setup buffer sizes */
1807         rctl &= ~E1000_RCTL_SZ_4096;
1808         rctl |= E1000_RCTL_BSEX;
1809         switch (adapter->rx_buffer_len) {
1810         case E1000_RXBUFFER_2048:
1811         default:
1812                 rctl |= E1000_RCTL_SZ_2048;
1813                 rctl &= ~E1000_RCTL_BSEX;
1814                 break;
1815         case E1000_RXBUFFER_4096:
1816                 rctl |= E1000_RCTL_SZ_4096;
1817                 break;
1818         case E1000_RXBUFFER_8192:
1819                 rctl |= E1000_RCTL_SZ_8192;
1820                 break;
1821         case E1000_RXBUFFER_16384:
1822                 rctl |= E1000_RCTL_SZ_16384;
1823                 break;
1824         }
1825
1826         /* This is useful for sniffing bad packets. */
1827         if (adapter->netdev->features & NETIF_F_RXALL) {
1828                 /* UPE and MPE will be handled by normal PROMISC logic
1829                  * in e1000e_set_rx_mode
1830                  */
1831                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1832                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1833                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1834
1835                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1836                           E1000_RCTL_DPF | /* Allow filtered pause */
1837                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1838                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1839                  * and that breaks VLANs.
1840                  */
1841         }
1842
1843         ew32(RCTL, rctl);
1844 }
1845
1846 /**
1847  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1848  * @adapter: board private structure
1849  *
1850  * Configure the Rx unit of the MAC after a reset.
1851  **/
1852 static void e1000_configure_rx(struct e1000_adapter *adapter)
1853 {
1854         u64 rdba;
1855         struct e1000_hw *hw = &adapter->hw;
1856         u32 rdlen, rctl, rxcsum;
1857
1858         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1859                 rdlen = adapter->rx_ring[0].count *
1860                         sizeof(struct e1000_rx_desc);
1861                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1862                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1863         } else {
1864                 rdlen = adapter->rx_ring[0].count *
1865                         sizeof(struct e1000_rx_desc);
1866                 adapter->clean_rx = e1000_clean_rx_irq;
1867                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1868         }
1869
1870         /* disable receives while setting up the descriptors */
1871         rctl = er32(RCTL);
1872         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1873
1874         /* set the Receive Delay Timer Register */
1875         ew32(RDTR, adapter->rx_int_delay);
1876
1877         if (hw->mac_type >= e1000_82540) {
1878                 ew32(RADV, adapter->rx_abs_int_delay);
1879                 if (adapter->itr_setting != 0)
1880                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1881         }
1882
1883         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1884          * the Base and Length of the Rx Descriptor Ring
1885          */
1886         switch (adapter->num_rx_queues) {
1887         case 1:
1888         default:
1889                 rdba = adapter->rx_ring[0].dma;
1890                 ew32(RDLEN, rdlen);
1891                 ew32(RDBAH, (rdba >> 32));
1892                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1893                 ew32(RDT, 0);
1894                 ew32(RDH, 0);
1895                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1896                                            E1000_RDH : E1000_82542_RDH);
1897                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1898                                            E1000_RDT : E1000_82542_RDT);
1899                 break;
1900         }
1901
1902         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1903         if (hw->mac_type >= e1000_82543) {
1904                 rxcsum = er32(RXCSUM);
1905                 if (adapter->rx_csum)
1906                         rxcsum |= E1000_RXCSUM_TUOFL;
1907                 else
1908                         /* don't need to clear IPPCSE as it defaults to 0 */
1909                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1910                 ew32(RXCSUM, rxcsum);
1911         }
1912
1913         /* Enable Receives */
1914         ew32(RCTL, rctl | E1000_RCTL_EN);
1915 }
1916
1917 /**
1918  * e1000_free_tx_resources - Free Tx Resources per Queue
1919  * @adapter: board private structure
1920  * @tx_ring: Tx descriptor ring for a specific queue
1921  *
1922  * Free all transmit software resources
1923  **/
1924 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1925                                     struct e1000_tx_ring *tx_ring)
1926 {
1927         struct pci_dev *pdev = adapter->pdev;
1928
1929         e1000_clean_tx_ring(adapter, tx_ring);
1930
1931         vfree(tx_ring->buffer_info);
1932         tx_ring->buffer_info = NULL;
1933
1934         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1935                           tx_ring->dma);
1936
1937         tx_ring->desc = NULL;
1938 }
1939
1940 /**
1941  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1942  * @adapter: board private structure
1943  *
1944  * Free all transmit software resources
1945  **/
1946 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1947 {
1948         int i;
1949
1950         for (i = 0; i < adapter->num_tx_queues; i++)
1951                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1952 }
1953
1954 static void
1955 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1956                                  struct e1000_tx_buffer *buffer_info)
1957 {
1958         if (buffer_info->dma) {
1959                 if (buffer_info->mapped_as_page)
1960                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1961                                        buffer_info->length, DMA_TO_DEVICE);
1962                 else
1963                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1964                                          buffer_info->length,
1965                                          DMA_TO_DEVICE);
1966                 buffer_info->dma = 0;
1967         }
1968         if (buffer_info->skb) {
1969                 dev_kfree_skb_any(buffer_info->skb);
1970                 buffer_info->skb = NULL;
1971         }
1972         buffer_info->time_stamp = 0;
1973         /* buffer_info must be completely set up in the transmit path */
1974 }
1975
1976 /**
1977  * e1000_clean_tx_ring - Free Tx Buffers
1978  * @adapter: board private structure
1979  * @tx_ring: ring to be cleaned
1980  **/
1981 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1982                                 struct e1000_tx_ring *tx_ring)
1983 {
1984         struct e1000_hw *hw = &adapter->hw;
1985         struct e1000_tx_buffer *buffer_info;
1986         unsigned long size;
1987         unsigned int i;
1988
1989         /* Free all the Tx ring sk_buffs */
1990
1991         for (i = 0; i < tx_ring->count; i++) {
1992                 buffer_info = &tx_ring->buffer_info[i];
1993                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1994         }
1995
1996         netdev_reset_queue(adapter->netdev);
1997         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1998         memset(tx_ring->buffer_info, 0, size);
1999
2000         /* Zero out the descriptor ring */
2001
2002         memset(tx_ring->desc, 0, tx_ring->size);
2003
2004         tx_ring->next_to_use = 0;
2005         tx_ring->next_to_clean = 0;
2006         tx_ring->last_tx_tso = false;
2007
2008         writel(0, hw->hw_addr + tx_ring->tdh);
2009         writel(0, hw->hw_addr + tx_ring->tdt);
2010 }
2011
2012 /**
2013  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2014  * @adapter: board private structure
2015  **/
2016 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2017 {
2018         int i;
2019
2020         for (i = 0; i < adapter->num_tx_queues; i++)
2021                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2022 }
2023
2024 /**
2025  * e1000_free_rx_resources - Free Rx Resources
2026  * @adapter: board private structure
2027  * @rx_ring: ring to clean the resources from
2028  *
2029  * Free all receive software resources
2030  **/
2031 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2032                                     struct e1000_rx_ring *rx_ring)
2033 {
2034         struct pci_dev *pdev = adapter->pdev;
2035
2036         e1000_clean_rx_ring(adapter, rx_ring);
2037
2038         vfree(rx_ring->buffer_info);
2039         rx_ring->buffer_info = NULL;
2040
2041         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2042                           rx_ring->dma);
2043
2044         rx_ring->desc = NULL;
2045 }
2046
2047 /**
2048  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2049  * @adapter: board private structure
2050  *
2051  * Free all receive software resources
2052  **/
2053 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2054 {
2055         int i;
2056
2057         for (i = 0; i < adapter->num_rx_queues; i++)
2058                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2059 }
2060
2061 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2062 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2063 {
2064         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2065                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2066 }
2067
2068 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2069 {
2070         unsigned int len = e1000_frag_len(a);
2071         u8 *data = netdev_alloc_frag(len);
2072
2073         if (likely(data))
2074                 data += E1000_HEADROOM;
2075         return data;
2076 }
2077
2078 /**
2079  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2080  * @adapter: board private structure
2081  * @rx_ring: ring to free buffers from
2082  **/
2083 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2084                                 struct e1000_rx_ring *rx_ring)
2085 {
2086         struct e1000_hw *hw = &adapter->hw;
2087         struct e1000_rx_buffer *buffer_info;
2088         struct pci_dev *pdev = adapter->pdev;
2089         unsigned long size;
2090         unsigned int i;
2091
2092         /* Free all the Rx netfrags */
2093         for (i = 0; i < rx_ring->count; i++) {
2094                 buffer_info = &rx_ring->buffer_info[i];
2095                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2096                         if (buffer_info->dma)
2097                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2098                                                  adapter->rx_buffer_len,
2099                                                  DMA_FROM_DEVICE);
2100                         if (buffer_info->rxbuf.data) {
2101                                 skb_free_frag(buffer_info->rxbuf.data);
2102                                 buffer_info->rxbuf.data = NULL;
2103                         }
2104                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2105                         if (buffer_info->dma)
2106                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2107                                                adapter->rx_buffer_len,
2108                                                DMA_FROM_DEVICE);
2109                         if (buffer_info->rxbuf.page) {
2110                                 put_page(buffer_info->rxbuf.page);
2111                                 buffer_info->rxbuf.page = NULL;
2112                         }
2113                 }
2114
2115                 buffer_info->dma = 0;
2116         }
2117
2118         /* there also may be some cached data from a chained receive */
2119         napi_free_frags(&adapter->napi);
2120         rx_ring->rx_skb_top = NULL;
2121
2122         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2123         memset(rx_ring->buffer_info, 0, size);
2124
2125         /* Zero out the descriptor ring */
2126         memset(rx_ring->desc, 0, rx_ring->size);
2127
2128         rx_ring->next_to_clean = 0;
2129         rx_ring->next_to_use = 0;
2130
2131         writel(0, hw->hw_addr + rx_ring->rdh);
2132         writel(0, hw->hw_addr + rx_ring->rdt);
2133 }
2134
2135 /**
2136  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2137  * @adapter: board private structure
2138  **/
2139 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2140 {
2141         int i;
2142
2143         for (i = 0; i < adapter->num_rx_queues; i++)
2144                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2145 }
2146
2147 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2148  * and memory write and invalidate disabled for certain operations
2149  */
2150 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2151 {
2152         struct e1000_hw *hw = &adapter->hw;
2153         struct net_device *netdev = adapter->netdev;
2154         u32 rctl;
2155
2156         e1000_pci_clear_mwi(hw);
2157
2158         rctl = er32(RCTL);
2159         rctl |= E1000_RCTL_RST;
2160         ew32(RCTL, rctl);
2161         E1000_WRITE_FLUSH();
2162         mdelay(5);
2163
2164         if (netif_running(netdev))
2165                 e1000_clean_all_rx_rings(adapter);
2166 }
2167
2168 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2169 {
2170         struct e1000_hw *hw = &adapter->hw;
2171         struct net_device *netdev = adapter->netdev;
2172         u32 rctl;
2173
2174         rctl = er32(RCTL);
2175         rctl &= ~E1000_RCTL_RST;
2176         ew32(RCTL, rctl);
2177         E1000_WRITE_FLUSH();
2178         mdelay(5);
2179
2180         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2181                 e1000_pci_set_mwi(hw);
2182
2183         if (netif_running(netdev)) {
2184                 /* No need to loop, because 82542 supports only 1 queue */
2185                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2186                 e1000_configure_rx(adapter);
2187                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2188         }
2189 }
2190
2191 /**
2192  * e1000_set_mac - Change the Ethernet Address of the NIC
2193  * @netdev: network interface device structure
2194  * @p: pointer to an address structure
2195  *
2196  * Returns 0 on success, negative on failure
2197  **/
2198 static int e1000_set_mac(struct net_device *netdev, void *p)
2199 {
2200         struct e1000_adapter *adapter = netdev_priv(netdev);
2201         struct e1000_hw *hw = &adapter->hw;
2202         struct sockaddr *addr = p;
2203
2204         if (!is_valid_ether_addr(addr->sa_data))
2205                 return -EADDRNOTAVAIL;
2206
2207         /* 82542 2.0 needs to be in reset to write receive address registers */
2208
2209         if (hw->mac_type == e1000_82542_rev2_0)
2210                 e1000_enter_82542_rst(adapter);
2211
2212         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2213         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2214
2215         e1000_rar_set(hw, hw->mac_addr, 0);
2216
2217         if (hw->mac_type == e1000_82542_rev2_0)
2218                 e1000_leave_82542_rst(adapter);
2219
2220         return 0;
2221 }
2222
2223 /**
2224  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2225  * @netdev: network interface device structure
2226  *
2227  * The set_rx_mode entry point is called whenever the unicast or multicast
2228  * address lists or the network interface flags are updated. This routine is
2229  * responsible for configuring the hardware for proper unicast, multicast,
2230  * promiscuous mode, and all-multi behavior.
2231  **/
2232 static void e1000_set_rx_mode(struct net_device *netdev)
2233 {
2234         struct e1000_adapter *adapter = netdev_priv(netdev);
2235         struct e1000_hw *hw = &adapter->hw;
2236         struct netdev_hw_addr *ha;
2237         bool use_uc = false;
2238         u32 rctl;
2239         u32 hash_value;
2240         int i, rar_entries = E1000_RAR_ENTRIES;
2241         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2242         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2243
2244         if (!mcarray)
2245                 return;
2246
2247         /* Check for Promiscuous and All Multicast modes */
2248
2249         rctl = er32(RCTL);
2250
2251         if (netdev->flags & IFF_PROMISC) {
2252                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2253                 rctl &= ~E1000_RCTL_VFE;
2254         } else {
2255                 if (netdev->flags & IFF_ALLMULTI)
2256                         rctl |= E1000_RCTL_MPE;
2257                 else
2258                         rctl &= ~E1000_RCTL_MPE;
2259                 /* Enable VLAN filter if there is a VLAN */
2260                 if (e1000_vlan_used(adapter))
2261                         rctl |= E1000_RCTL_VFE;
2262         }
2263
2264         if (netdev_uc_count(netdev) > rar_entries - 1) {
2265                 rctl |= E1000_RCTL_UPE;
2266         } else if (!(netdev->flags & IFF_PROMISC)) {
2267                 rctl &= ~E1000_RCTL_UPE;
2268                 use_uc = true;
2269         }
2270
2271         ew32(RCTL, rctl);
2272
2273         /* 82542 2.0 needs to be in reset to write receive address registers */
2274
2275         if (hw->mac_type == e1000_82542_rev2_0)
2276                 e1000_enter_82542_rst(adapter);
2277
2278         /* load the first 14 addresses into the exact filters 1-14. Unicast
2279          * addresses take precedence to avoid disabling unicast filtering
2280          * when possible.
2281          *
2282          * RAR 0 is used for the station MAC address
2283          * if there are not 14 addresses, go ahead and clear the filters
2284          */
2285         i = 1;
2286         if (use_uc)
2287                 netdev_for_each_uc_addr(ha, netdev) {
2288                         if (i == rar_entries)
2289                                 break;
2290                         e1000_rar_set(hw, ha->addr, i++);
2291                 }
2292
2293         netdev_for_each_mc_addr(ha, netdev) {
2294                 if (i == rar_entries) {
2295                         /* load any remaining addresses into the hash table */
2296                         u32 hash_reg, hash_bit, mta;
2297                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2298                         hash_reg = (hash_value >> 5) & 0x7F;
2299                         hash_bit = hash_value & 0x1F;
2300                         mta = (1 << hash_bit);
2301                         mcarray[hash_reg] |= mta;
2302                 } else {
2303                         e1000_rar_set(hw, ha->addr, i++);
2304                 }
2305         }
2306
2307         for (; i < rar_entries; i++) {
2308                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2309                 E1000_WRITE_FLUSH();
2310                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2311                 E1000_WRITE_FLUSH();
2312         }
2313
2314         /* write the hash table completely, write from bottom to avoid
2315          * both stupid write combining chipsets, and flushing each write
2316          */
2317         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2318                 /* If we are on an 82544 has an errata where writing odd
2319                  * offsets overwrites the previous even offset, but writing
2320                  * backwards over the range solves the issue by always
2321                  * writing the odd offset first
2322                  */
2323                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2324         }
2325         E1000_WRITE_FLUSH();
2326
2327         if (hw->mac_type == e1000_82542_rev2_0)
2328                 e1000_leave_82542_rst(adapter);
2329
2330         kfree(mcarray);
2331 }
2332
2333 /**
2334  * e1000_update_phy_info_task - get phy info
2335  * @work: work struct contained inside adapter struct
2336  *
2337  * Need to wait a few seconds after link up to get diagnostic information from
2338  * the phy
2339  */
2340 static void e1000_update_phy_info_task(struct work_struct *work)
2341 {
2342         struct e1000_adapter *adapter = container_of(work,
2343                                                      struct e1000_adapter,
2344                                                      phy_info_task.work);
2345
2346         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2347 }
2348
2349 /**
2350  * e1000_82547_tx_fifo_stall_task - task to complete work
2351  * @work: work struct contained inside adapter struct
2352  **/
2353 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2354 {
2355         struct e1000_adapter *adapter = container_of(work,
2356                                                      struct e1000_adapter,
2357                                                      fifo_stall_task.work);
2358         struct e1000_hw *hw = &adapter->hw;
2359         struct net_device *netdev = adapter->netdev;
2360         u32 tctl;
2361
2362         if (atomic_read(&adapter->tx_fifo_stall)) {
2363                 if ((er32(TDT) == er32(TDH)) &&
2364                    (er32(TDFT) == er32(TDFH)) &&
2365                    (er32(TDFTS) == er32(TDFHS))) {
2366                         tctl = er32(TCTL);
2367                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2368                         ew32(TDFT, adapter->tx_head_addr);
2369                         ew32(TDFH, adapter->tx_head_addr);
2370                         ew32(TDFTS, adapter->tx_head_addr);
2371                         ew32(TDFHS, adapter->tx_head_addr);
2372                         ew32(TCTL, tctl);
2373                         E1000_WRITE_FLUSH();
2374
2375                         adapter->tx_fifo_head = 0;
2376                         atomic_set(&adapter->tx_fifo_stall, 0);
2377                         netif_wake_queue(netdev);
2378                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2379                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2380                 }
2381         }
2382 }
2383
2384 bool e1000_has_link(struct e1000_adapter *adapter)
2385 {
2386         struct e1000_hw *hw = &adapter->hw;
2387         bool link_active = false;
2388
2389         /* get_link_status is set on LSC (link status) interrupt or rx
2390          * sequence error interrupt (except on intel ce4100).
2391          * get_link_status will stay false until the
2392          * e1000_check_for_link establishes link for copper adapters
2393          * ONLY
2394          */
2395         switch (hw->media_type) {
2396         case e1000_media_type_copper:
2397                 if (hw->mac_type == e1000_ce4100)
2398                         hw->get_link_status = 1;
2399                 if (hw->get_link_status) {
2400                         e1000_check_for_link(hw);
2401                         link_active = !hw->get_link_status;
2402                 } else {
2403                         link_active = true;
2404                 }
2405                 break;
2406         case e1000_media_type_fiber:
2407                 e1000_check_for_link(hw);
2408                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2409                 break;
2410         case e1000_media_type_internal_serdes:
2411                 e1000_check_for_link(hw);
2412                 link_active = hw->serdes_has_link;
2413                 break;
2414         default:
2415                 break;
2416         }
2417
2418         return link_active;
2419 }
2420
2421 /**
2422  * e1000_watchdog - work function
2423  * @work: work struct contained inside adapter struct
2424  **/
2425 static void e1000_watchdog(struct work_struct *work)
2426 {
2427         struct e1000_adapter *adapter = container_of(work,
2428                                                      struct e1000_adapter,
2429                                                      watchdog_task.work);
2430         struct e1000_hw *hw = &adapter->hw;
2431         struct net_device *netdev = adapter->netdev;
2432         struct e1000_tx_ring *txdr = adapter->tx_ring;
2433         u32 link, tctl;
2434
2435         link = e1000_has_link(adapter);
2436         if ((netif_carrier_ok(netdev)) && link)
2437                 goto link_up;
2438
2439         if (link) {
2440                 if (!netif_carrier_ok(netdev)) {
2441                         u32 ctrl;
2442                         /* update snapshot of PHY registers on LSC */
2443                         e1000_get_speed_and_duplex(hw,
2444                                                    &adapter->link_speed,
2445                                                    &adapter->link_duplex);
2446
2447                         ctrl = er32(CTRL);
2448                         pr_info("%s NIC Link is Up %d Mbps %s, "
2449                                 "Flow Control: %s\n",
2450                                 netdev->name,
2451                                 adapter->link_speed,
2452                                 adapter->link_duplex == FULL_DUPLEX ?
2453                                 "Full Duplex" : "Half Duplex",
2454                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2455                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2456                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2457                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2458
2459                         /* adjust timeout factor according to speed/duplex */
2460                         adapter->tx_timeout_factor = 1;
2461                         switch (adapter->link_speed) {
2462                         case SPEED_10:
2463                                 adapter->tx_timeout_factor = 16;
2464                                 break;
2465                         case SPEED_100:
2466                                 /* maybe add some timeout factor ? */
2467                                 break;
2468                         }
2469
2470                         /* enable transmits in the hardware */
2471                         tctl = er32(TCTL);
2472                         tctl |= E1000_TCTL_EN;
2473                         ew32(TCTL, tctl);
2474
2475                         netif_carrier_on(netdev);
2476                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2477                                 schedule_delayed_work(&adapter->phy_info_task,
2478                                                       2 * HZ);
2479                         adapter->smartspeed = 0;
2480                 }
2481         } else {
2482                 if (netif_carrier_ok(netdev)) {
2483                         adapter->link_speed = 0;
2484                         adapter->link_duplex = 0;
2485                         pr_info("%s NIC Link is Down\n",
2486                                 netdev->name);
2487                         netif_carrier_off(netdev);
2488
2489                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2490                                 schedule_delayed_work(&adapter->phy_info_task,
2491                                                       2 * HZ);
2492                 }
2493
2494                 e1000_smartspeed(adapter);
2495         }
2496
2497 link_up:
2498         e1000_update_stats(adapter);
2499
2500         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2501         adapter->tpt_old = adapter->stats.tpt;
2502         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2503         adapter->colc_old = adapter->stats.colc;
2504
2505         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2506         adapter->gorcl_old = adapter->stats.gorcl;
2507         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2508         adapter->gotcl_old = adapter->stats.gotcl;
2509
2510         e1000_update_adaptive(hw);
2511
2512         if (!netif_carrier_ok(netdev)) {
2513                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2514                         /* We've lost link, so the controller stops DMA,
2515                          * but we've got queued Tx work that's never going
2516                          * to get done, so reset controller to flush Tx.
2517                          * (Do the reset outside of interrupt context).
2518                          */
2519                         adapter->tx_timeout_count++;
2520                         schedule_work(&adapter->reset_task);
2521                         /* exit immediately since reset is imminent */
2522                         return;
2523                 }
2524         }
2525
2526         /* Simple mode for Interrupt Throttle Rate (ITR) */
2527         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2528                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2529                  * Total asymmetrical Tx or Rx gets ITR=8000;
2530                  * everyone else is between 2000-8000.
2531                  */
2532                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2533                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2534                             adapter->gotcl - adapter->gorcl :
2535                             adapter->gorcl - adapter->gotcl) / 10000;
2536                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2537
2538                 ew32(ITR, 1000000000 / (itr * 256));
2539         }
2540
2541         /* Cause software interrupt to ensure rx ring is cleaned */
2542         ew32(ICS, E1000_ICS_RXDMT0);
2543
2544         /* Force detection of hung controller every watchdog period */
2545         adapter->detect_tx_hung = true;
2546
2547         /* Reschedule the task */
2548         if (!test_bit(__E1000_DOWN, &adapter->flags))
2549                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2550 }
2551
2552 enum latency_range {
2553         lowest_latency = 0,
2554         low_latency = 1,
2555         bulk_latency = 2,
2556         latency_invalid = 255
2557 };
2558
2559 /**
2560  * e1000_update_itr - update the dynamic ITR value based on statistics
2561  * @adapter: pointer to adapter
2562  * @itr_setting: current adapter->itr
2563  * @packets: the number of packets during this measurement interval
2564  * @bytes: the number of bytes during this measurement interval
2565  *
2566  *      Stores a new ITR value based on packets and byte
2567  *      counts during the last interrupt.  The advantage of per interrupt
2568  *      computation is faster updates and more accurate ITR for the current
2569  *      traffic pattern.  Constants in this function were computed
2570  *      based on theoretical maximum wire speed and thresholds were set based
2571  *      on testing data as well as attempting to minimize response time
2572  *      while increasing bulk throughput.
2573  *      this functionality is controlled by the InterruptThrottleRate module
2574  *      parameter (see e1000_param.c)
2575  **/
2576 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2577                                      u16 itr_setting, int packets, int bytes)
2578 {
2579         unsigned int retval = itr_setting;
2580         struct e1000_hw *hw = &adapter->hw;
2581
2582         if (unlikely(hw->mac_type < e1000_82540))
2583                 goto update_itr_done;
2584
2585         if (packets == 0)
2586                 goto update_itr_done;
2587
2588         switch (itr_setting) {
2589         case lowest_latency:
2590                 /* jumbo frames get bulk treatment*/
2591                 if (bytes/packets > 8000)
2592                         retval = bulk_latency;
2593                 else if ((packets < 5) && (bytes > 512))
2594                         retval = low_latency;
2595                 break;
2596         case low_latency:  /* 50 usec aka 20000 ints/s */
2597                 if (bytes > 10000) {
2598                         /* jumbo frames need bulk latency setting */
2599                         if (bytes/packets > 8000)
2600                                 retval = bulk_latency;
2601                         else if ((packets < 10) || ((bytes/packets) > 1200))
2602                                 retval = bulk_latency;
2603                         else if ((packets > 35))
2604                                 retval = lowest_latency;
2605                 } else if (bytes/packets > 2000)
2606                         retval = bulk_latency;
2607                 else if (packets <= 2 && bytes < 512)
2608                         retval = lowest_latency;
2609                 break;
2610         case bulk_latency: /* 250 usec aka 4000 ints/s */
2611                 if (bytes > 25000) {
2612                         if (packets > 35)
2613                                 retval = low_latency;
2614                 } else if (bytes < 6000) {
2615                         retval = low_latency;
2616                 }
2617                 break;
2618         }
2619
2620 update_itr_done:
2621         return retval;
2622 }
2623
2624 static void e1000_set_itr(struct e1000_adapter *adapter)
2625 {
2626         struct e1000_hw *hw = &adapter->hw;
2627         u16 current_itr;
2628         u32 new_itr = adapter->itr;
2629
2630         if (unlikely(hw->mac_type < e1000_82540))
2631                 return;
2632
2633         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2634         if (unlikely(adapter->link_speed != SPEED_1000)) {
2635                 current_itr = 0;
2636                 new_itr = 4000;
2637                 goto set_itr_now;
2638         }
2639
2640         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2641                                            adapter->total_tx_packets,
2642                                            adapter->total_tx_bytes);
2643         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2644         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2645                 adapter->tx_itr = low_latency;
2646
2647         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2648                                            adapter->total_rx_packets,
2649                                            adapter->total_rx_bytes);
2650         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2651         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2652                 adapter->rx_itr = low_latency;
2653
2654         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2655
2656         switch (current_itr) {
2657         /* counts and packets in update_itr are dependent on these numbers */
2658         case lowest_latency:
2659                 new_itr = 70000;
2660                 break;
2661         case low_latency:
2662                 new_itr = 20000; /* aka hwitr = ~200 */
2663                 break;
2664         case bulk_latency:
2665                 new_itr = 4000;
2666                 break;
2667         default:
2668                 break;
2669         }
2670
2671 set_itr_now:
2672         if (new_itr != adapter->itr) {
2673                 /* this attempts to bias the interrupt rate towards Bulk
2674                  * by adding intermediate steps when interrupt rate is
2675                  * increasing
2676                  */
2677                 new_itr = new_itr > adapter->itr ?
2678                           min(adapter->itr + (new_itr >> 2), new_itr) :
2679                           new_itr;
2680                 adapter->itr = new_itr;
2681                 ew32(ITR, 1000000000 / (new_itr * 256));
2682         }
2683 }
2684
2685 #define E1000_TX_FLAGS_CSUM             0x00000001
2686 #define E1000_TX_FLAGS_VLAN             0x00000002
2687 #define E1000_TX_FLAGS_TSO              0x00000004
2688 #define E1000_TX_FLAGS_IPV4             0x00000008
2689 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2690 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2691 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2692
2693 static int e1000_tso(struct e1000_adapter *adapter,
2694                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2695                      __be16 protocol)
2696 {
2697         struct e1000_context_desc *context_desc;
2698         struct e1000_tx_buffer *buffer_info;
2699         unsigned int i;
2700         u32 cmd_length = 0;
2701         u16 ipcse = 0, tucse, mss;
2702         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2703
2704         if (skb_is_gso(skb)) {
2705                 int err;
2706
2707                 err = skb_cow_head(skb, 0);
2708                 if (err < 0)
2709                         return err;
2710
2711                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2712                 mss = skb_shinfo(skb)->gso_size;
2713                 if (protocol == htons(ETH_P_IP)) {
2714                         struct iphdr *iph = ip_hdr(skb);
2715                         iph->tot_len = 0;
2716                         iph->check = 0;
2717                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2718                                                                  iph->daddr, 0,
2719                                                                  IPPROTO_TCP,
2720                                                                  0);
2721                         cmd_length = E1000_TXD_CMD_IP;
2722                         ipcse = skb_transport_offset(skb) - 1;
2723                 } else if (skb_is_gso_v6(skb)) {
2724                         tcp_v6_gso_csum_prep(skb);
2725                         ipcse = 0;
2726                 }
2727                 ipcss = skb_network_offset(skb);
2728                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2729                 tucss = skb_transport_offset(skb);
2730                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2731                 tucse = 0;
2732
2733                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2734                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2735
2736                 i = tx_ring->next_to_use;
2737                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2738                 buffer_info = &tx_ring->buffer_info[i];
2739
2740                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2741                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2742                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2743                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2744                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2745                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2746                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2747                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2748                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2749
2750                 buffer_info->time_stamp = jiffies;
2751                 buffer_info->next_to_watch = i;
2752
2753                 if (++i == tx_ring->count)
2754                         i = 0;
2755
2756                 tx_ring->next_to_use = i;
2757
2758                 return true;
2759         }
2760         return false;
2761 }
2762
2763 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2764                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2765                           __be16 protocol)
2766 {
2767         struct e1000_context_desc *context_desc;
2768         struct e1000_tx_buffer *buffer_info;
2769         unsigned int i;
2770         u8 css;
2771         u32 cmd_len = E1000_TXD_CMD_DEXT;
2772
2773         if (skb->ip_summed != CHECKSUM_PARTIAL)
2774                 return false;
2775
2776         switch (protocol) {
2777         case cpu_to_be16(ETH_P_IP):
2778                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2779                         cmd_len |= E1000_TXD_CMD_TCP;
2780                 break;
2781         case cpu_to_be16(ETH_P_IPV6):
2782                 /* XXX not handling all IPV6 headers */
2783                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2784                         cmd_len |= E1000_TXD_CMD_TCP;
2785                 break;
2786         default:
2787                 if (unlikely(net_ratelimit()))
2788                         e_warn(drv, "checksum_partial proto=%x!\n",
2789                                skb->protocol);
2790                 break;
2791         }
2792
2793         css = skb_checksum_start_offset(skb);
2794
2795         i = tx_ring->next_to_use;
2796         buffer_info = &tx_ring->buffer_info[i];
2797         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2798
2799         context_desc->lower_setup.ip_config = 0;
2800         context_desc->upper_setup.tcp_fields.tucss = css;
2801         context_desc->upper_setup.tcp_fields.tucso =
2802                 css + skb->csum_offset;
2803         context_desc->upper_setup.tcp_fields.tucse = 0;
2804         context_desc->tcp_seg_setup.data = 0;
2805         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2806
2807         buffer_info->time_stamp = jiffies;
2808         buffer_info->next_to_watch = i;
2809
2810         if (unlikely(++i == tx_ring->count))
2811                 i = 0;
2812
2813         tx_ring->next_to_use = i;
2814
2815         return true;
2816 }
2817
2818 #define E1000_MAX_TXD_PWR       12
2819 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2820
2821 static int e1000_tx_map(struct e1000_adapter *adapter,
2822                         struct e1000_tx_ring *tx_ring,
2823                         struct sk_buff *skb, unsigned int first,
2824                         unsigned int max_per_txd, unsigned int nr_frags,
2825                         unsigned int mss)
2826 {
2827         struct e1000_hw *hw = &adapter->hw;
2828         struct pci_dev *pdev = adapter->pdev;
2829         struct e1000_tx_buffer *buffer_info;
2830         unsigned int len = skb_headlen(skb);
2831         unsigned int offset = 0, size, count = 0, i;
2832         unsigned int f, bytecount, segs;
2833
2834         i = tx_ring->next_to_use;
2835
2836         while (len) {
2837                 buffer_info = &tx_ring->buffer_info[i];
2838                 size = min(len, max_per_txd);
2839                 /* Workaround for Controller erratum --
2840                  * descriptor for non-tso packet in a linear SKB that follows a
2841                  * tso gets written back prematurely before the data is fully
2842                  * DMA'd to the controller
2843                  */
2844                 if (!skb->data_len && tx_ring->last_tx_tso &&
2845                     !skb_is_gso(skb)) {
2846                         tx_ring->last_tx_tso = false;
2847                         size -= 4;
2848                 }
2849
2850                 /* Workaround for premature desc write-backs
2851                  * in TSO mode.  Append 4-byte sentinel desc
2852                  */
2853                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2854                         size -= 4;
2855                 /* work-around for errata 10 and it applies
2856                  * to all controllers in PCI-X mode
2857                  * The fix is to make sure that the first descriptor of a
2858                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2859                  */
2860                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2861                              (size > 2015) && count == 0))
2862                         size = 2015;
2863
2864                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2865                  * terminating buffers within evenly-aligned dwords.
2866                  */
2867                 if (unlikely(adapter->pcix_82544 &&
2868                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2869                    size > 4))
2870                         size -= 4;
2871
2872                 buffer_info->length = size;
2873                 /* set time_stamp *before* dma to help avoid a possible race */
2874                 buffer_info->time_stamp = jiffies;
2875                 buffer_info->mapped_as_page = false;
2876                 buffer_info->dma = dma_map_single(&pdev->dev,
2877                                                   skb->data + offset,
2878                                                   size, DMA_TO_DEVICE);
2879                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2880                         goto dma_error;
2881                 buffer_info->next_to_watch = i;
2882
2883                 len -= size;
2884                 offset += size;
2885                 count++;
2886                 if (len) {
2887                         i++;
2888                         if (unlikely(i == tx_ring->count))
2889                                 i = 0;
2890                 }
2891         }
2892
2893         for (f = 0; f < nr_frags; f++) {
2894                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2895
2896                 len = skb_frag_size(frag);
2897                 offset = 0;
2898
2899                 while (len) {
2900                         unsigned long bufend;
2901                         i++;
2902                         if (unlikely(i == tx_ring->count))
2903                                 i = 0;
2904
2905                         buffer_info = &tx_ring->buffer_info[i];
2906                         size = min(len, max_per_txd);
2907                         /* Workaround for premature desc write-backs
2908                          * in TSO mode.  Append 4-byte sentinel desc
2909                          */
2910                         if (unlikely(mss && f == (nr_frags-1) &&
2911                             size == len && size > 8))
2912                                 size -= 4;
2913                         /* Workaround for potential 82544 hang in PCI-X.
2914                          * Avoid terminating buffers within evenly-aligned
2915                          * dwords.
2916                          */
2917                         bufend = (unsigned long)
2918                                 page_to_phys(skb_frag_page(frag));
2919                         bufend += offset + size - 1;
2920                         if (unlikely(adapter->pcix_82544 &&
2921                                      !(bufend & 4) &&
2922                                      size > 4))
2923                                 size -= 4;
2924
2925                         buffer_info->length = size;
2926                         buffer_info->time_stamp = jiffies;
2927                         buffer_info->mapped_as_page = true;
2928                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2929                                                 offset, size, DMA_TO_DEVICE);
2930                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2931                                 goto dma_error;
2932                         buffer_info->next_to_watch = i;
2933
2934                         len -= size;
2935                         offset += size;
2936                         count++;
2937                 }
2938         }
2939
2940         segs = skb_shinfo(skb)->gso_segs ?: 1;
2941         /* multiply data chunks by size of headers */
2942         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2943
2944         tx_ring->buffer_info[i].skb = skb;
2945         tx_ring->buffer_info[i].segs = segs;
2946         tx_ring->buffer_info[i].bytecount = bytecount;
2947         tx_ring->buffer_info[first].next_to_watch = i;
2948
2949         return count;
2950
2951 dma_error:
2952         dev_err(&pdev->dev, "TX DMA map failed\n");
2953         buffer_info->dma = 0;
2954         if (count)
2955                 count--;
2956
2957         while (count--) {
2958                 if (i == 0)
2959                         i += tx_ring->count;
2960                 i--;
2961                 buffer_info = &tx_ring->buffer_info[i];
2962                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2963         }
2964
2965         return 0;
2966 }
2967
2968 static void e1000_tx_queue(struct e1000_adapter *adapter,
2969                            struct e1000_tx_ring *tx_ring, int tx_flags,
2970                            int count)
2971 {
2972         struct e1000_tx_desc *tx_desc = NULL;
2973         struct e1000_tx_buffer *buffer_info;
2974         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2975         unsigned int i;
2976
2977         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2978                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2979                              E1000_TXD_CMD_TSE;
2980                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2981
2982                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2983                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2984         }
2985
2986         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2987                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2988                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2989         }
2990
2991         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2992                 txd_lower |= E1000_TXD_CMD_VLE;
2993                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2994         }
2995
2996         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
2997                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
2998
2999         i = tx_ring->next_to_use;
3000
3001         while (count--) {
3002                 buffer_info = &tx_ring->buffer_info[i];
3003                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3004                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3005                 tx_desc->lower.data =
3006                         cpu_to_le32(txd_lower | buffer_info->length);
3007                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3008                 if (unlikely(++i == tx_ring->count))
3009                         i = 0;
3010         }
3011
3012         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3013
3014         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3015         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3016                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3017
3018         /* Force memory writes to complete before letting h/w
3019          * know there are new descriptors to fetch.  (Only
3020          * applicable for weak-ordered memory model archs,
3021          * such as IA-64).
3022          */
3023         dma_wmb();
3024
3025         tx_ring->next_to_use = i;
3026 }
3027
3028 /* 82547 workaround to avoid controller hang in half-duplex environment.
3029  * The workaround is to avoid queuing a large packet that would span
3030  * the internal Tx FIFO ring boundary by notifying the stack to resend
3031  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3032  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3033  * to the beginning of the Tx FIFO.
3034  */
3035
3036 #define E1000_FIFO_HDR                  0x10
3037 #define E1000_82547_PAD_LEN             0x3E0
3038
3039 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3040                                        struct sk_buff *skb)
3041 {
3042         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3043         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3044
3045         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3046
3047         if (adapter->link_duplex != HALF_DUPLEX)
3048                 goto no_fifo_stall_required;
3049
3050         if (atomic_read(&adapter->tx_fifo_stall))
3051                 return 1;
3052
3053         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3054                 atomic_set(&adapter->tx_fifo_stall, 1);
3055                 return 1;
3056         }
3057
3058 no_fifo_stall_required:
3059         adapter->tx_fifo_head += skb_fifo_len;
3060         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3061                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3062         return 0;
3063 }
3064
3065 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3066 {
3067         struct e1000_adapter *adapter = netdev_priv(netdev);
3068         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3069
3070         netif_stop_queue(netdev);
3071         /* Herbert's original patch had:
3072          *  smp_mb__after_netif_stop_queue();
3073          * but since that doesn't exist yet, just open code it.
3074          */
3075         smp_mb();
3076
3077         /* We need to check again in a case another CPU has just
3078          * made room available.
3079          */
3080         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3081                 return -EBUSY;
3082
3083         /* A reprieve! */
3084         netif_start_queue(netdev);
3085         ++adapter->restart_queue;
3086         return 0;
3087 }
3088
3089 static int e1000_maybe_stop_tx(struct net_device *netdev,
3090                                struct e1000_tx_ring *tx_ring, int size)
3091 {
3092         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3093                 return 0;
3094         return __e1000_maybe_stop_tx(netdev, size);
3095 }
3096
3097 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3098 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3099                                     struct net_device *netdev)
3100 {
3101         struct e1000_adapter *adapter = netdev_priv(netdev);
3102         struct e1000_hw *hw = &adapter->hw;
3103         struct e1000_tx_ring *tx_ring;
3104         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3105         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3106         unsigned int tx_flags = 0;
3107         unsigned int len = skb_headlen(skb);
3108         unsigned int nr_frags;
3109         unsigned int mss;
3110         int count = 0;
3111         int tso;
3112         unsigned int f;
3113         __be16 protocol = vlan_get_protocol(skb);
3114
3115         /* This goes back to the question of how to logically map a Tx queue
3116          * to a flow.  Right now, performance is impacted slightly negatively
3117          * if using multiple Tx queues.  If the stack breaks away from a
3118          * single qdisc implementation, we can look at this again.
3119          */
3120         tx_ring = adapter->tx_ring;
3121
3122         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3123          * packets may get corrupted during padding by HW.
3124          * To WA this issue, pad all small packets manually.
3125          */
3126         if (eth_skb_pad(skb))
3127                 return NETDEV_TX_OK;
3128
3129         mss = skb_shinfo(skb)->gso_size;
3130         /* The controller does a simple calculation to
3131          * make sure there is enough room in the FIFO before
3132          * initiating the DMA for each buffer.  The calc is:
3133          * 4 = ceil(buffer len/mss).  To make sure we don't
3134          * overrun the FIFO, adjust the max buffer len if mss
3135          * drops.
3136          */
3137         if (mss) {
3138                 u8 hdr_len;
3139                 max_per_txd = min(mss << 2, max_per_txd);
3140                 max_txd_pwr = fls(max_per_txd) - 1;
3141
3142                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3143                 if (skb->data_len && hdr_len == len) {
3144                         switch (hw->mac_type) {
3145                         case e1000_82544: {
3146                                 unsigned int pull_size;
3147
3148                                 /* Make sure we have room to chop off 4 bytes,
3149                                  * and that the end alignment will work out to
3150                                  * this hardware's requirements
3151                                  * NOTE: this is a TSO only workaround
3152                                  * if end byte alignment not correct move us
3153                                  * into the next dword
3154                                  */
3155                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3156                                     & 4)
3157                                         break;
3158                                 pull_size = min((unsigned int)4, skb->data_len);
3159                                 if (!__pskb_pull_tail(skb, pull_size)) {
3160                                         e_err(drv, "__pskb_pull_tail "
3161                                               "failed.\n");
3162                                         dev_kfree_skb_any(skb);
3163                                         return NETDEV_TX_OK;
3164                                 }
3165                                 len = skb_headlen(skb);
3166                                 break;
3167                         }
3168                         default:
3169                                 /* do nothing */
3170                                 break;
3171                         }
3172                 }
3173         }
3174
3175         /* reserve a descriptor for the offload context */
3176         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3177                 count++;
3178         count++;
3179
3180         /* Controller Erratum workaround */
3181         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3182                 count++;
3183
3184         count += TXD_USE_COUNT(len, max_txd_pwr);
3185
3186         if (adapter->pcix_82544)
3187                 count++;
3188
3189         /* work-around for errata 10 and it applies to all controllers
3190          * in PCI-X mode, so add one more descriptor to the count
3191          */
3192         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3193                         (len > 2015)))
3194                 count++;
3195
3196         nr_frags = skb_shinfo(skb)->nr_frags;
3197         for (f = 0; f < nr_frags; f++)
3198                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3199                                        max_txd_pwr);
3200         if (adapter->pcix_82544)
3201                 count += nr_frags;
3202
3203         /* need: count + 2 desc gap to keep tail from touching
3204          * head, otherwise try next time
3205          */
3206         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3207                 return NETDEV_TX_BUSY;
3208
3209         if (unlikely((hw->mac_type == e1000_82547) &&
3210                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3211                 netif_stop_queue(netdev);
3212                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3213                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3214                 return NETDEV_TX_BUSY;
3215         }
3216
3217         if (skb_vlan_tag_present(skb)) {
3218                 tx_flags |= E1000_TX_FLAGS_VLAN;
3219                 tx_flags |= (skb_vlan_tag_get(skb) <<
3220                              E1000_TX_FLAGS_VLAN_SHIFT);
3221         }
3222
3223         first = tx_ring->next_to_use;
3224
3225         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3226         if (tso < 0) {
3227                 dev_kfree_skb_any(skb);
3228                 return NETDEV_TX_OK;
3229         }
3230
3231         if (likely(tso)) {
3232                 if (likely(hw->mac_type != e1000_82544))
3233                         tx_ring->last_tx_tso = true;
3234                 tx_flags |= E1000_TX_FLAGS_TSO;
3235         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3236                 tx_flags |= E1000_TX_FLAGS_CSUM;
3237
3238         if (protocol == htons(ETH_P_IP))
3239                 tx_flags |= E1000_TX_FLAGS_IPV4;
3240
3241         if (unlikely(skb->no_fcs))
3242                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3243
3244         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3245                              nr_frags, mss);
3246
3247         if (count) {
3248                 /* The descriptors needed is higher than other Intel drivers
3249                  * due to a number of workarounds.  The breakdown is below:
3250                  * Data descriptors: MAX_SKB_FRAGS + 1
3251                  * Context Descriptor: 1
3252                  * Keep head from touching tail: 2
3253                  * Workarounds: 3
3254                  */
3255                 int desc_needed = MAX_SKB_FRAGS + 7;
3256
3257                 netdev_sent_queue(netdev, skb->len);
3258                 skb_tx_timestamp(skb);
3259
3260                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3261
3262                 /* 82544 potentially requires twice as many data descriptors
3263                  * in order to guarantee buffers don't end on evenly-aligned
3264                  * dwords
3265                  */
3266                 if (adapter->pcix_82544)
3267                         desc_needed += MAX_SKB_FRAGS + 1;
3268
3269                 /* Make sure there is space in the ring for the next send. */
3270                 e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3271
3272                 if (!netdev_xmit_more() ||
3273                     netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3274                         writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3275                 }
3276         } else {
3277                 dev_kfree_skb_any(skb);
3278                 tx_ring->buffer_info[first].time_stamp = 0;
3279                 tx_ring->next_to_use = first;
3280         }
3281
3282         return NETDEV_TX_OK;
3283 }
3284
3285 #define NUM_REGS 38 /* 1 based count */
3286 static void e1000_regdump(struct e1000_adapter *adapter)
3287 {
3288         struct e1000_hw *hw = &adapter->hw;
3289         u32 regs[NUM_REGS];
3290         u32 *regs_buff = regs;
3291         int i = 0;
3292
3293         static const char * const reg_name[] = {
3294                 "CTRL",  "STATUS",
3295                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3296                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3297                 "TIDV", "TXDCTL", "TADV", "TARC0",
3298                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3299                 "TXDCTL1", "TARC1",
3300                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3301                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3302                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3303         };
3304
3305         regs_buff[0]  = er32(CTRL);
3306         regs_buff[1]  = er32(STATUS);
3307
3308         regs_buff[2]  = er32(RCTL);
3309         regs_buff[3]  = er32(RDLEN);
3310         regs_buff[4]  = er32(RDH);
3311         regs_buff[5]  = er32(RDT);
3312         regs_buff[6]  = er32(RDTR);
3313
3314         regs_buff[7]  = er32(TCTL);
3315         regs_buff[8]  = er32(TDBAL);
3316         regs_buff[9]  = er32(TDBAH);
3317         regs_buff[10] = er32(TDLEN);
3318         regs_buff[11] = er32(TDH);
3319         regs_buff[12] = er32(TDT);
3320         regs_buff[13] = er32(TIDV);
3321         regs_buff[14] = er32(TXDCTL);
3322         regs_buff[15] = er32(TADV);
3323         regs_buff[16] = er32(TARC0);
3324
3325         regs_buff[17] = er32(TDBAL1);
3326         regs_buff[18] = er32(TDBAH1);
3327         regs_buff[19] = er32(TDLEN1);
3328         regs_buff[20] = er32(TDH1);
3329         regs_buff[21] = er32(TDT1);
3330         regs_buff[22] = er32(TXDCTL1);
3331         regs_buff[23] = er32(TARC1);
3332         regs_buff[24] = er32(CTRL_EXT);
3333         regs_buff[25] = er32(ERT);
3334         regs_buff[26] = er32(RDBAL0);
3335         regs_buff[27] = er32(RDBAH0);
3336         regs_buff[28] = er32(TDFH);
3337         regs_buff[29] = er32(TDFT);
3338         regs_buff[30] = er32(TDFHS);
3339         regs_buff[31] = er32(TDFTS);
3340         regs_buff[32] = er32(TDFPC);
3341         regs_buff[33] = er32(RDFH);
3342         regs_buff[34] = er32(RDFT);
3343         regs_buff[35] = er32(RDFHS);
3344         regs_buff[36] = er32(RDFTS);
3345         regs_buff[37] = er32(RDFPC);
3346
3347         pr_info("Register dump\n");
3348         for (i = 0; i < NUM_REGS; i++)
3349                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3350 }
3351
3352 /*
3353  * e1000_dump: Print registers, tx ring and rx ring
3354  */
3355 static void e1000_dump(struct e1000_adapter *adapter)
3356 {
3357         /* this code doesn't handle multiple rings */
3358         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3359         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3360         int i;
3361
3362         if (!netif_msg_hw(adapter))
3363                 return;
3364
3365         /* Print Registers */
3366         e1000_regdump(adapter);
3367
3368         /* transmit dump */
3369         pr_info("TX Desc ring0 dump\n");
3370
3371         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3372          *
3373          * Legacy Transmit Descriptor
3374          *   +--------------------------------------------------------------+
3375          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3376          *   +--------------------------------------------------------------+
3377          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3378          *   +--------------------------------------------------------------+
3379          *   63       48 47        36 35    32 31     24 23    16 15        0
3380          *
3381          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3382          *   63      48 47    40 39       32 31             16 15    8 7      0
3383          *   +----------------------------------------------------------------+
3384          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3385          *   +----------------------------------------------------------------+
3386          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3387          *   +----------------------------------------------------------------+
3388          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3389          *
3390          * Extended Data Descriptor (DTYP=0x1)
3391          *   +----------------------------------------------------------------+
3392          * 0 |                     Buffer Address [63:0]                      |
3393          *   +----------------------------------------------------------------+
3394          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3395          *   +----------------------------------------------------------------+
3396          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3397          */
3398         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3399         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3400
3401         if (!netif_msg_tx_done(adapter))
3402                 goto rx_ring_summary;
3403
3404         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3405                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3406                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3407                 struct my_u { __le64 a; __le64 b; };
3408                 struct my_u *u = (struct my_u *)tx_desc;
3409                 const char *type;
3410
3411                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3412                         type = "NTC/U";
3413                 else if (i == tx_ring->next_to_use)
3414                         type = "NTU";
3415                 else if (i == tx_ring->next_to_clean)
3416                         type = "NTC";
3417                 else
3418                         type = "";
3419
3420                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3421                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3422                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3423                         (u64)buffer_info->dma, buffer_info->length,
3424                         buffer_info->next_to_watch,
3425                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3426         }
3427
3428 rx_ring_summary:
3429         /* receive dump */
3430         pr_info("\nRX Desc ring dump\n");
3431
3432         /* Legacy Receive Descriptor Format
3433          *
3434          * +-----------------------------------------------------+
3435          * |                Buffer Address [63:0]                |
3436          * +-----------------------------------------------------+
3437          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3438          * +-----------------------------------------------------+
3439          * 63       48 47    40 39      32 31         16 15      0
3440          */
3441         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3442
3443         if (!netif_msg_rx_status(adapter))
3444                 goto exit;
3445
3446         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3447                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3448                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3449                 struct my_u { __le64 a; __le64 b; };
3450                 struct my_u *u = (struct my_u *)rx_desc;
3451                 const char *type;
3452
3453                 if (i == rx_ring->next_to_use)
3454                         type = "NTU";
3455                 else if (i == rx_ring->next_to_clean)
3456                         type = "NTC";
3457                 else
3458                         type = "";
3459
3460                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3461                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3462                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3463         } /* for */
3464
3465         /* dump the descriptor caches */
3466         /* rx */
3467         pr_info("Rx descriptor cache in 64bit format\n");
3468         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3469                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3470                         i,
3471                         readl(adapter->hw.hw_addr + i+4),
3472                         readl(adapter->hw.hw_addr + i),
3473                         readl(adapter->hw.hw_addr + i+12),
3474                         readl(adapter->hw.hw_addr + i+8));
3475         }
3476         /* tx */
3477         pr_info("Tx descriptor cache in 64bit format\n");
3478         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3479                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3480                         i,
3481                         readl(adapter->hw.hw_addr + i+4),
3482                         readl(adapter->hw.hw_addr + i),
3483                         readl(adapter->hw.hw_addr + i+12),
3484                         readl(adapter->hw.hw_addr + i+8));
3485         }
3486 exit:
3487         return;
3488 }
3489
3490 /**
3491  * e1000_tx_timeout - Respond to a Tx Hang
3492  * @netdev: network interface device structure
3493  * @txqueue: number of the Tx queue that hung (unused)
3494  **/
3495 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
3496 {
3497         struct e1000_adapter *adapter = netdev_priv(netdev);
3498
3499         /* Do the reset outside of interrupt context */
3500         adapter->tx_timeout_count++;
3501         schedule_work(&adapter->reset_task);
3502 }
3503
3504 static void e1000_reset_task(struct work_struct *work)
3505 {
3506         struct e1000_adapter *adapter =
3507                 container_of(work, struct e1000_adapter, reset_task);
3508
3509         e_err(drv, "Reset adapter\n");
3510         e1000_reinit_locked(adapter);
3511 }
3512
3513 /**
3514  * e1000_change_mtu - Change the Maximum Transfer Unit
3515  * @netdev: network interface device structure
3516  * @new_mtu: new value for maximum frame size
3517  *
3518  * Returns 0 on success, negative on failure
3519  **/
3520 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3521 {
3522         struct e1000_adapter *adapter = netdev_priv(netdev);
3523         struct e1000_hw *hw = &adapter->hw;
3524         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3525
3526         /* Adapter-specific max frame size limits. */
3527         switch (hw->mac_type) {
3528         case e1000_undefined ... e1000_82542_rev2_1:
3529                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3530                         e_err(probe, "Jumbo Frames not supported.\n");
3531                         return -EINVAL;
3532                 }
3533                 break;
3534         default:
3535                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3536                 break;
3537         }
3538
3539         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3540                 msleep(1);
3541         /* e1000_down has a dependency on max_frame_size */
3542         hw->max_frame_size = max_frame;
3543         if (netif_running(netdev)) {
3544                 /* prevent buffers from being reallocated */
3545                 adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3546                 e1000_down(adapter);
3547         }
3548
3549         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3550          * means we reserve 2 more, this pushes us to allocate from the next
3551          * larger slab size.
3552          * i.e. RXBUFFER_2048 --> size-4096 slab
3553          * however with the new *_jumbo_rx* routines, jumbo receives will use
3554          * fragmented skbs
3555          */
3556
3557         if (max_frame <= E1000_RXBUFFER_2048)
3558                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3559         else
3560 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3561                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3562 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3563                 adapter->rx_buffer_len = PAGE_SIZE;
3564 #endif
3565
3566         /* adjust allocation if LPE protects us, and we aren't using SBP */
3567         if (!hw->tbi_compatibility_on &&
3568             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3569              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3570                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3571
3572         netdev_dbg(netdev, "changing MTU from %d to %d\n",
3573                    netdev->mtu, new_mtu);
3574         netdev->mtu = new_mtu;
3575
3576         if (netif_running(netdev))
3577                 e1000_up(adapter);
3578         else
3579                 e1000_reset(adapter);
3580
3581         clear_bit(__E1000_RESETTING, &adapter->flags);
3582
3583         return 0;
3584 }
3585
3586 /**
3587  * e1000_update_stats - Update the board statistics counters
3588  * @adapter: board private structure
3589  **/
3590 void e1000_update_stats(struct e1000_adapter *adapter)
3591 {
3592         struct net_device *netdev = adapter->netdev;
3593         struct e1000_hw *hw = &adapter->hw;
3594         struct pci_dev *pdev = adapter->pdev;
3595         unsigned long flags;
3596         u16 phy_tmp;
3597
3598 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3599
3600         /* Prevent stats update while adapter is being reset, or if the pci
3601          * connection is down.
3602          */
3603         if (adapter->link_speed == 0)
3604                 return;
3605         if (pci_channel_offline(pdev))
3606                 return;
3607
3608         spin_lock_irqsave(&adapter->stats_lock, flags);
3609
3610         /* these counters are modified from e1000_tbi_adjust_stats,
3611          * called from the interrupt context, so they must only
3612          * be written while holding adapter->stats_lock
3613          */
3614
3615         adapter->stats.crcerrs += er32(CRCERRS);
3616         adapter->stats.gprc += er32(GPRC);
3617         adapter->stats.gorcl += er32(GORCL);
3618         adapter->stats.gorch += er32(GORCH);
3619         adapter->stats.bprc += er32(BPRC);
3620         adapter->stats.mprc += er32(MPRC);
3621         adapter->stats.roc += er32(ROC);
3622
3623         adapter->stats.prc64 += er32(PRC64);
3624         adapter->stats.prc127 += er32(PRC127);
3625         adapter->stats.prc255 += er32(PRC255);
3626         adapter->stats.prc511 += er32(PRC511);
3627         adapter->stats.prc1023 += er32(PRC1023);
3628         adapter->stats.prc1522 += er32(PRC1522);
3629
3630         adapter->stats.symerrs += er32(SYMERRS);
3631         adapter->stats.mpc += er32(MPC);
3632         adapter->stats.scc += er32(SCC);
3633         adapter->stats.ecol += er32(ECOL);
3634         adapter->stats.mcc += er32(MCC);
3635         adapter->stats.latecol += er32(LATECOL);
3636         adapter->stats.dc += er32(DC);
3637         adapter->stats.sec += er32(SEC);
3638         adapter->stats.rlec += er32(RLEC);
3639         adapter->stats.xonrxc += er32(XONRXC);
3640         adapter->stats.xontxc += er32(XONTXC);
3641         adapter->stats.xoffrxc += er32(XOFFRXC);
3642         adapter->stats.xofftxc += er32(XOFFTXC);
3643         adapter->stats.fcruc += er32(FCRUC);
3644         adapter->stats.gptc += er32(GPTC);
3645         adapter->stats.gotcl += er32(GOTCL);
3646         adapter->stats.gotch += er32(GOTCH);
3647         adapter->stats.rnbc += er32(RNBC);
3648         adapter->stats.ruc += er32(RUC);
3649         adapter->stats.rfc += er32(RFC);
3650         adapter->stats.rjc += er32(RJC);
3651         adapter->stats.torl += er32(TORL);
3652         adapter->stats.torh += er32(TORH);
3653         adapter->stats.totl += er32(TOTL);
3654         adapter->stats.toth += er32(TOTH);
3655         adapter->stats.tpr += er32(TPR);
3656
3657         adapter->stats.ptc64 += er32(PTC64);
3658         adapter->stats.ptc127 += er32(PTC127);
3659         adapter->stats.ptc255 += er32(PTC255);
3660         adapter->stats.ptc511 += er32(PTC511);
3661         adapter->stats.ptc1023 += er32(PTC1023);
3662         adapter->stats.ptc1522 += er32(PTC1522);
3663
3664         adapter->stats.mptc += er32(MPTC);
3665         adapter->stats.bptc += er32(BPTC);
3666
3667         /* used for adaptive IFS */
3668
3669         hw->tx_packet_delta = er32(TPT);
3670         adapter->stats.tpt += hw->tx_packet_delta;
3671         hw->collision_delta = er32(COLC);
3672         adapter->stats.colc += hw->collision_delta;
3673
3674         if (hw->mac_type >= e1000_82543) {
3675                 adapter->stats.algnerrc += er32(ALGNERRC);
3676                 adapter->stats.rxerrc += er32(RXERRC);
3677                 adapter->stats.tncrs += er32(TNCRS);
3678                 adapter->stats.cexterr += er32(CEXTERR);
3679                 adapter->stats.tsctc += er32(TSCTC);
3680                 adapter->stats.tsctfc += er32(TSCTFC);
3681         }
3682
3683         /* Fill out the OS statistics structure */
3684         netdev->stats.multicast = adapter->stats.mprc;
3685         netdev->stats.collisions = adapter->stats.colc;
3686
3687         /* Rx Errors */
3688
3689         /* RLEC on some newer hardware can be incorrect so build
3690          * our own version based on RUC and ROC
3691          */
3692         netdev->stats.rx_errors = adapter->stats.rxerrc +
3693                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3694                 adapter->stats.ruc + adapter->stats.roc +
3695                 adapter->stats.cexterr;
3696         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3697         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3698         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3699         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3700         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3701
3702         /* Tx Errors */
3703         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3704         netdev->stats.tx_errors = adapter->stats.txerrc;
3705         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3706         netdev->stats.tx_window_errors = adapter->stats.latecol;
3707         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3708         if (hw->bad_tx_carr_stats_fd &&
3709             adapter->link_duplex == FULL_DUPLEX) {
3710                 netdev->stats.tx_carrier_errors = 0;
3711                 adapter->stats.tncrs = 0;
3712         }
3713
3714         /* Tx Dropped needs to be maintained elsewhere */
3715
3716         /* Phy Stats */
3717         if (hw->media_type == e1000_media_type_copper) {
3718                 if ((adapter->link_speed == SPEED_1000) &&
3719                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3720                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3721                         adapter->phy_stats.idle_errors += phy_tmp;
3722                 }
3723
3724                 if ((hw->mac_type <= e1000_82546) &&
3725                    (hw->phy_type == e1000_phy_m88) &&
3726                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3727                         adapter->phy_stats.receive_errors += phy_tmp;
3728         }
3729
3730         /* Management Stats */
3731         if (hw->has_smbus) {
3732                 adapter->stats.mgptc += er32(MGTPTC);
3733                 adapter->stats.mgprc += er32(MGTPRC);
3734                 adapter->stats.mgpdc += er32(MGTPDC);
3735         }
3736
3737         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3738 }
3739
3740 /**
3741  * e1000_intr - Interrupt Handler
3742  * @irq: interrupt number
3743  * @data: pointer to a network interface device structure
3744  **/
3745 static irqreturn_t e1000_intr(int irq, void *data)
3746 {
3747         struct net_device *netdev = data;
3748         struct e1000_adapter *adapter = netdev_priv(netdev);
3749         struct e1000_hw *hw = &adapter->hw;
3750         u32 icr = er32(ICR);
3751
3752         if (unlikely((!icr)))
3753                 return IRQ_NONE;  /* Not our interrupt */
3754
3755         /* we might have caused the interrupt, but the above
3756          * read cleared it, and just in case the driver is
3757          * down there is nothing to do so return handled
3758          */
3759         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3760                 return IRQ_HANDLED;
3761
3762         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3763                 hw->get_link_status = 1;
3764                 /* guard against interrupt when we're going down */
3765                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3766                         schedule_delayed_work(&adapter->watchdog_task, 1);
3767         }
3768
3769         /* disable interrupts, without the synchronize_irq bit */
3770         ew32(IMC, ~0);
3771         E1000_WRITE_FLUSH();
3772
3773         if (likely(napi_schedule_prep(&adapter->napi))) {
3774                 adapter->total_tx_bytes = 0;
3775                 adapter->total_tx_packets = 0;
3776                 adapter->total_rx_bytes = 0;
3777                 adapter->total_rx_packets = 0;
3778                 __napi_schedule(&adapter->napi);
3779         } else {
3780                 /* this really should not happen! if it does it is basically a
3781                  * bug, but not a hard error, so enable ints and continue
3782                  */
3783                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3784                         e1000_irq_enable(adapter);
3785         }
3786
3787         return IRQ_HANDLED;
3788 }
3789
3790 /**
3791  * e1000_clean - NAPI Rx polling callback
3792  * @napi: napi struct containing references to driver info
3793  * @budget: budget given to driver for receive packets
3794  **/
3795 static int e1000_clean(struct napi_struct *napi, int budget)
3796 {
3797         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3798                                                      napi);
3799         int tx_clean_complete = 0, work_done = 0;
3800
3801         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3802
3803         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3804
3805         if (!tx_clean_complete || work_done == budget)
3806                 return budget;
3807
3808         /* Exit the polling mode, but don't re-enable interrupts if stack might
3809          * poll us due to busy-polling
3810          */
3811         if (likely(napi_complete_done(napi, work_done))) {
3812                 if (likely(adapter->itr_setting & 3))
3813                         e1000_set_itr(adapter);
3814                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3815                         e1000_irq_enable(adapter);
3816         }
3817
3818         return work_done;
3819 }
3820
3821 /**
3822  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3823  * @adapter: board private structure
3824  * @tx_ring: ring to clean
3825  **/
3826 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3827                                struct e1000_tx_ring *tx_ring)
3828 {
3829         struct e1000_hw *hw = &adapter->hw;
3830         struct net_device *netdev = adapter->netdev;
3831         struct e1000_tx_desc *tx_desc, *eop_desc;
3832         struct e1000_tx_buffer *buffer_info;
3833         unsigned int i, eop;
3834         unsigned int count = 0;
3835         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3836         unsigned int bytes_compl = 0, pkts_compl = 0;
3837
3838         i = tx_ring->next_to_clean;
3839         eop = tx_ring->buffer_info[i].next_to_watch;
3840         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3841
3842         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3843                (count < tx_ring->count)) {
3844                 bool cleaned = false;
3845                 dma_rmb();      /* read buffer_info after eop_desc */
3846                 for ( ; !cleaned; count++) {
3847                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3848                         buffer_info = &tx_ring->buffer_info[i];
3849                         cleaned = (i == eop);
3850
3851                         if (cleaned) {
3852                                 total_tx_packets += buffer_info->segs;
3853                                 total_tx_bytes += buffer_info->bytecount;
3854                                 if (buffer_info->skb) {
3855                                         bytes_compl += buffer_info->skb->len;
3856                                         pkts_compl++;
3857                                 }
3858
3859                         }
3860                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3861                         tx_desc->upper.data = 0;
3862
3863                         if (unlikely(++i == tx_ring->count))
3864                                 i = 0;
3865                 }
3866
3867                 eop = tx_ring->buffer_info[i].next_to_watch;
3868                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3869         }
3870
3871         /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3872          * which will reuse the cleaned buffers.
3873          */
3874         smp_store_release(&tx_ring->next_to_clean, i);
3875
3876         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3877
3878 #define TX_WAKE_THRESHOLD 32
3879         if (unlikely(count && netif_carrier_ok(netdev) &&
3880                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3881                 /* Make sure that anybody stopping the queue after this
3882                  * sees the new next_to_clean.
3883                  */
3884                 smp_mb();
3885
3886                 if (netif_queue_stopped(netdev) &&
3887                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3888                         netif_wake_queue(netdev);
3889                         ++adapter->restart_queue;
3890                 }
3891         }
3892
3893         if (adapter->detect_tx_hung) {
3894                 /* Detect a transmit hang in hardware, this serializes the
3895                  * check with the clearing of time_stamp and movement of i
3896                  */
3897                 adapter->detect_tx_hung = false;
3898                 if (tx_ring->buffer_info[eop].time_stamp &&
3899                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3900                                (adapter->tx_timeout_factor * HZ)) &&
3901                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3902
3903                         /* detected Tx unit hang */
3904                         e_err(drv, "Detected Tx Unit Hang\n"
3905                               "  Tx Queue             <%lu>\n"
3906                               "  TDH                  <%x>\n"
3907                               "  TDT                  <%x>\n"
3908                               "  next_to_use          <%x>\n"
3909                               "  next_to_clean        <%x>\n"
3910                               "buffer_info[next_to_clean]\n"
3911                               "  time_stamp           <%lx>\n"
3912                               "  next_to_watch        <%x>\n"
3913                               "  jiffies              <%lx>\n"
3914                               "  next_to_watch.status <%x>\n",
3915                                 (unsigned long)(tx_ring - adapter->tx_ring),
3916                                 readl(hw->hw_addr + tx_ring->tdh),
3917                                 readl(hw->hw_addr + tx_ring->tdt),
3918                                 tx_ring->next_to_use,
3919                                 tx_ring->next_to_clean,
3920                                 tx_ring->buffer_info[eop].time_stamp,
3921                                 eop,
3922                                 jiffies,
3923                                 eop_desc->upper.fields.status);
3924                         e1000_dump(adapter);
3925                         netif_stop_queue(netdev);
3926                 }
3927         }
3928         adapter->total_tx_bytes += total_tx_bytes;
3929         adapter->total_tx_packets += total_tx_packets;
3930         netdev->stats.tx_bytes += total_tx_bytes;
3931         netdev->stats.tx_packets += total_tx_packets;
3932         return count < tx_ring->count;
3933 }
3934
3935 /**
3936  * e1000_rx_checksum - Receive Checksum Offload for 82543
3937  * @adapter:     board private structure
3938  * @status_err:  receive descriptor status and error fields
3939  * @csum:        receive descriptor csum field
3940  * @skb:         socket buffer with received data
3941  **/
3942 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3943                               u32 csum, struct sk_buff *skb)
3944 {
3945         struct e1000_hw *hw = &adapter->hw;
3946         u16 status = (u16)status_err;
3947         u8 errors = (u8)(status_err >> 24);
3948
3949         skb_checksum_none_assert(skb);
3950
3951         /* 82543 or newer only */
3952         if (unlikely(hw->mac_type < e1000_82543))
3953                 return;
3954         /* Ignore Checksum bit is set */
3955         if (unlikely(status & E1000_RXD_STAT_IXSM))
3956                 return;
3957         /* TCP/UDP checksum error bit is set */
3958         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3959                 /* let the stack verify checksum errors */
3960                 adapter->hw_csum_err++;
3961                 return;
3962         }
3963         /* TCP/UDP Checksum has not been calculated */
3964         if (!(status & E1000_RXD_STAT_TCPCS))
3965                 return;
3966
3967         /* It must be a TCP or UDP packet with a valid checksum */
3968         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3969                 /* TCP checksum is good */
3970                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3971         }
3972         adapter->hw_csum_good++;
3973 }
3974
3975 /**
3976  * e1000_consume_page - helper function for jumbo Rx path
3977  * @bi: software descriptor shadow data
3978  * @skb: skb being modified
3979  * @length: length of data being added
3980  **/
3981 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3982                                u16 length)
3983 {
3984         bi->rxbuf.page = NULL;
3985         skb->len += length;
3986         skb->data_len += length;
3987         skb->truesize += PAGE_SIZE;
3988 }
3989
3990 /**
3991  * e1000_receive_skb - helper function to handle rx indications
3992  * @adapter: board private structure
3993  * @status: descriptor status field as written by hardware
3994  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3995  * @skb: pointer to sk_buff to be indicated to stack
3996  */
3997 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3998                               __le16 vlan, struct sk_buff *skb)
3999 {
4000         skb->protocol = eth_type_trans(skb, adapter->netdev);
4001
4002         if (status & E1000_RXD_STAT_VP) {
4003                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4004
4005                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4006         }
4007         napi_gro_receive(&adapter->napi, skb);
4008 }
4009
4010 /**
4011  * e1000_tbi_adjust_stats
4012  * @hw: Struct containing variables accessed by shared code
4013  * @stats: point to stats struct
4014  * @frame_len: The length of the frame in question
4015  * @mac_addr: The Ethernet destination address of the frame in question
4016  *
4017  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4018  */
4019 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4020                                    struct e1000_hw_stats *stats,
4021                                    u32 frame_len, const u8 *mac_addr)
4022 {
4023         u64 carry_bit;
4024
4025         /* First adjust the frame length. */
4026         frame_len--;
4027         /* We need to adjust the statistics counters, since the hardware
4028          * counters overcount this packet as a CRC error and undercount
4029          * the packet as a good packet
4030          */
4031         /* This packet should not be counted as a CRC error. */
4032         stats->crcerrs--;
4033         /* This packet does count as a Good Packet Received. */
4034         stats->gprc++;
4035
4036         /* Adjust the Good Octets received counters */
4037         carry_bit = 0x80000000 & stats->gorcl;
4038         stats->gorcl += frame_len;
4039         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4040          * Received Count) was one before the addition,
4041          * AND it is zero after, then we lost the carry out,
4042          * need to add one to Gorch (Good Octets Received Count High).
4043          * This could be simplified if all environments supported
4044          * 64-bit integers.
4045          */
4046         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4047                 stats->gorch++;
4048         /* Is this a broadcast or multicast?  Check broadcast first,
4049          * since the test for a multicast frame will test positive on
4050          * a broadcast frame.
4051          */
4052         if (is_broadcast_ether_addr(mac_addr))
4053                 stats->bprc++;
4054         else if (is_multicast_ether_addr(mac_addr))
4055                 stats->mprc++;
4056
4057         if (frame_len == hw->max_frame_size) {
4058                 /* In this case, the hardware has overcounted the number of
4059                  * oversize frames.
4060                  */
4061                 if (stats->roc > 0)
4062                         stats->roc--;
4063         }
4064
4065         /* Adjust the bin counters when the extra byte put the frame in the
4066          * wrong bin. Remember that the frame_len was adjusted above.
4067          */
4068         if (frame_len == 64) {
4069                 stats->prc64++;
4070                 stats->prc127--;
4071         } else if (frame_len == 127) {
4072                 stats->prc127++;
4073                 stats->prc255--;
4074         } else if (frame_len == 255) {
4075                 stats->prc255++;
4076                 stats->prc511--;
4077         } else if (frame_len == 511) {
4078                 stats->prc511++;
4079                 stats->prc1023--;
4080         } else if (frame_len == 1023) {
4081                 stats->prc1023++;
4082                 stats->prc1522--;
4083         } else if (frame_len == 1522) {
4084                 stats->prc1522++;
4085         }
4086 }
4087
4088 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4089                                     u8 status, u8 errors,
4090                                     u32 length, const u8 *data)
4091 {
4092         struct e1000_hw *hw = &adapter->hw;
4093         u8 last_byte = *(data + length - 1);
4094
4095         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4096                 unsigned long irq_flags;
4097
4098                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4099                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4100                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4101
4102                 return true;
4103         }
4104
4105         return false;
4106 }
4107
4108 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4109                                           unsigned int bufsz)
4110 {
4111         struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4112
4113         if (unlikely(!skb))
4114                 adapter->alloc_rx_buff_failed++;
4115         return skb;
4116 }
4117
4118 /**
4119  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4120  * @adapter: board private structure
4121  * @rx_ring: ring to clean
4122  * @work_done: amount of napi work completed this call
4123  * @work_to_do: max amount of work allowed for this call to do
4124  *
4125  * the return value indicates whether actual cleaning was done, there
4126  * is no guarantee that everything was cleaned
4127  */
4128 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4129                                      struct e1000_rx_ring *rx_ring,
4130                                      int *work_done, int work_to_do)
4131 {
4132         struct net_device *netdev = adapter->netdev;
4133         struct pci_dev *pdev = adapter->pdev;
4134         struct e1000_rx_desc *rx_desc, *next_rxd;
4135         struct e1000_rx_buffer *buffer_info, *next_buffer;
4136         u32 length;
4137         unsigned int i;
4138         int cleaned_count = 0;
4139         bool cleaned = false;
4140         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4141
4142         i = rx_ring->next_to_clean;
4143         rx_desc = E1000_RX_DESC(*rx_ring, i);
4144         buffer_info = &rx_ring->buffer_info[i];
4145
4146         while (rx_desc->status & E1000_RXD_STAT_DD) {
4147                 struct sk_buff *skb;
4148                 u8 status;
4149
4150                 if (*work_done >= work_to_do)
4151                         break;
4152                 (*work_done)++;
4153                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4154
4155                 status = rx_desc->status;
4156
4157                 if (++i == rx_ring->count)
4158                         i = 0;
4159
4160                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4161                 prefetch(next_rxd);
4162
4163                 next_buffer = &rx_ring->buffer_info[i];
4164
4165                 cleaned = true;
4166                 cleaned_count++;
4167                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4168                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4169                 buffer_info->dma = 0;
4170
4171                 length = le16_to_cpu(rx_desc->length);
4172
4173                 /* errors is only valid for DD + EOP descriptors */
4174                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4175                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4176                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4177
4178                         if (e1000_tbi_should_accept(adapter, status,
4179                                                     rx_desc->errors,
4180                                                     length, mapped)) {
4181                                 length--;
4182                         } else if (netdev->features & NETIF_F_RXALL) {
4183                                 goto process_skb;
4184                         } else {
4185                                 /* an error means any chain goes out the window
4186                                  * too
4187                                  */
4188                                 dev_kfree_skb(rx_ring->rx_skb_top);
4189                                 rx_ring->rx_skb_top = NULL;
4190                                 goto next_desc;
4191                         }
4192                 }
4193
4194 #define rxtop rx_ring->rx_skb_top
4195 process_skb:
4196                 if (!(status & E1000_RXD_STAT_EOP)) {
4197                         /* this descriptor is only the beginning (or middle) */
4198                         if (!rxtop) {
4199                                 /* this is the beginning of a chain */
4200                                 rxtop = napi_get_frags(&adapter->napi);
4201                                 if (!rxtop)
4202                                         break;
4203
4204                                 skb_fill_page_desc(rxtop, 0,
4205                                                    buffer_info->rxbuf.page,
4206                                                    0, length);
4207                         } else {
4208                                 /* this is the middle of a chain */
4209                                 skb_fill_page_desc(rxtop,
4210                                     skb_shinfo(rxtop)->nr_frags,
4211                                     buffer_info->rxbuf.page, 0, length);
4212                         }
4213                         e1000_consume_page(buffer_info, rxtop, length);
4214                         goto next_desc;
4215                 } else {
4216                         if (rxtop) {
4217                                 /* end of the chain */
4218                                 skb_fill_page_desc(rxtop,
4219                                     skb_shinfo(rxtop)->nr_frags,
4220                                     buffer_info->rxbuf.page, 0, length);
4221                                 skb = rxtop;
4222                                 rxtop = NULL;
4223                                 e1000_consume_page(buffer_info, skb, length);
4224                         } else {
4225                                 struct page *p;
4226                                 /* no chain, got EOP, this buf is the packet
4227                                  * copybreak to save the put_page/alloc_page
4228                                  */
4229                                 p = buffer_info->rxbuf.page;
4230                                 if (length <= copybreak) {
4231                                         u8 *vaddr;
4232
4233                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4234                                                 length -= 4;
4235                                         skb = e1000_alloc_rx_skb(adapter,
4236                                                                  length);
4237                                         if (!skb)
4238                                                 break;
4239
4240                                         vaddr = kmap_atomic(p);
4241                                         memcpy(skb_tail_pointer(skb), vaddr,
4242                                                length);
4243                                         kunmap_atomic(vaddr);
4244                                         /* re-use the page, so don't erase
4245                                          * buffer_info->rxbuf.page
4246                                          */
4247                                         skb_put(skb, length);
4248                                         e1000_rx_checksum(adapter,
4249                                                           status | rx_desc->errors << 24,
4250                                                           le16_to_cpu(rx_desc->csum), skb);
4251
4252                                         total_rx_bytes += skb->len;
4253                                         total_rx_packets++;
4254
4255                                         e1000_receive_skb(adapter, status,
4256                                                           rx_desc->special, skb);
4257                                         goto next_desc;
4258                                 } else {
4259                                         skb = napi_get_frags(&adapter->napi);
4260                                         if (!skb) {
4261                                                 adapter->alloc_rx_buff_failed++;
4262                                                 break;
4263                                         }
4264                                         skb_fill_page_desc(skb, 0, p, 0,
4265                                                            length);
4266                                         e1000_consume_page(buffer_info, skb,
4267                                                            length);
4268                                 }
4269                         }
4270                 }
4271
4272                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4273                 e1000_rx_checksum(adapter,
4274                                   (u32)(status) |
4275                                   ((u32)(rx_desc->errors) << 24),
4276                                   le16_to_cpu(rx_desc->csum), skb);
4277
4278                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4279                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4280                         pskb_trim(skb, skb->len - 4);
4281                 total_rx_packets++;
4282
4283                 if (status & E1000_RXD_STAT_VP) {
4284                         __le16 vlan = rx_desc->special;
4285                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4286
4287                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4288                 }
4289
4290                 napi_gro_frags(&adapter->napi);
4291
4292 next_desc:
4293                 rx_desc->status = 0;
4294
4295                 /* return some buffers to hardware, one at a time is too slow */
4296                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4297                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4298                         cleaned_count = 0;
4299                 }
4300
4301                 /* use prefetched values */
4302                 rx_desc = next_rxd;
4303                 buffer_info = next_buffer;
4304         }
4305         rx_ring->next_to_clean = i;
4306
4307         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4308         if (cleaned_count)
4309                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4310
4311         adapter->total_rx_packets += total_rx_packets;
4312         adapter->total_rx_bytes += total_rx_bytes;
4313         netdev->stats.rx_bytes += total_rx_bytes;
4314         netdev->stats.rx_packets += total_rx_packets;
4315         return cleaned;
4316 }
4317
4318 /* this should improve performance for small packets with large amounts
4319  * of reassembly being done in the stack
4320  */
4321 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4322                                        struct e1000_rx_buffer *buffer_info,
4323                                        u32 length, const void *data)
4324 {
4325         struct sk_buff *skb;
4326
4327         if (length > copybreak)
4328                 return NULL;
4329
4330         skb = e1000_alloc_rx_skb(adapter, length);
4331         if (!skb)
4332                 return NULL;
4333
4334         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4335                                 length, DMA_FROM_DEVICE);
4336
4337         skb_put_data(skb, data, length);
4338
4339         return skb;
4340 }
4341
4342 /**
4343  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4344  * @adapter: board private structure
4345  * @rx_ring: ring to clean
4346  * @work_done: amount of napi work completed this call
4347  * @work_to_do: max amount of work allowed for this call to do
4348  */
4349 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4350                                struct e1000_rx_ring *rx_ring,
4351                                int *work_done, int work_to_do)
4352 {
4353         struct net_device *netdev = adapter->netdev;
4354         struct pci_dev *pdev = adapter->pdev;
4355         struct e1000_rx_desc *rx_desc, *next_rxd;
4356         struct e1000_rx_buffer *buffer_info, *next_buffer;
4357         u32 length;
4358         unsigned int i;
4359         int cleaned_count = 0;
4360         bool cleaned = false;
4361         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4362
4363         i = rx_ring->next_to_clean;
4364         rx_desc = E1000_RX_DESC(*rx_ring, i);
4365         buffer_info = &rx_ring->buffer_info[i];
4366
4367         while (rx_desc->status & E1000_RXD_STAT_DD) {
4368                 struct sk_buff *skb;
4369                 u8 *data;
4370                 u8 status;
4371
4372                 if (*work_done >= work_to_do)
4373                         break;
4374                 (*work_done)++;
4375                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4376
4377                 status = rx_desc->status;
4378                 length = le16_to_cpu(rx_desc->length);
4379
4380                 data = buffer_info->rxbuf.data;
4381                 prefetch(data);
4382                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4383                 if (!skb) {
4384                         unsigned int frag_len = e1000_frag_len(adapter);
4385
4386                         skb = build_skb(data - E1000_HEADROOM, frag_len);
4387                         if (!skb) {
4388                                 adapter->alloc_rx_buff_failed++;
4389                                 break;
4390                         }
4391
4392                         skb_reserve(skb, E1000_HEADROOM);
4393                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4394                                          adapter->rx_buffer_len,
4395                                          DMA_FROM_DEVICE);
4396                         buffer_info->dma = 0;
4397                         buffer_info->rxbuf.data = NULL;
4398                 }
4399
4400                 if (++i == rx_ring->count)
4401                         i = 0;
4402
4403                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4404                 prefetch(next_rxd);
4405
4406                 next_buffer = &rx_ring->buffer_info[i];
4407
4408                 cleaned = true;
4409                 cleaned_count++;
4410
4411                 /* !EOP means multiple descriptors were used to store a single
4412                  * packet, if thats the case we need to toss it.  In fact, we
4413                  * to toss every packet with the EOP bit clear and the next
4414                  * frame that _does_ have the EOP bit set, as it is by
4415                  * definition only a frame fragment
4416                  */
4417                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4418                         adapter->discarding = true;
4419
4420                 if (adapter->discarding) {
4421                         /* All receives must fit into a single buffer */
4422                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4423                         dev_kfree_skb(skb);
4424                         if (status & E1000_RXD_STAT_EOP)
4425                                 adapter->discarding = false;
4426                         goto next_desc;
4427                 }
4428
4429                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4430                         if (e1000_tbi_should_accept(adapter, status,
4431                                                     rx_desc->errors,
4432                                                     length, data)) {
4433                                 length--;
4434                         } else if (netdev->features & NETIF_F_RXALL) {
4435                                 goto process_skb;
4436                         } else {
4437                                 dev_kfree_skb(skb);
4438                                 goto next_desc;
4439                         }
4440                 }
4441
4442 process_skb:
4443                 total_rx_bytes += (length - 4); /* don't count FCS */
4444                 total_rx_packets++;
4445
4446                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4447                         /* adjust length to remove Ethernet CRC, this must be
4448                          * done after the TBI_ACCEPT workaround above
4449                          */
4450                         length -= 4;
4451
4452                 if (buffer_info->rxbuf.data == NULL)
4453                         skb_put(skb, length);
4454                 else /* copybreak skb */
4455                         skb_trim(skb, length);
4456
4457                 /* Receive Checksum Offload */
4458                 e1000_rx_checksum(adapter,
4459                                   (u32)(status) |
4460                                   ((u32)(rx_desc->errors) << 24),
4461                                   le16_to_cpu(rx_desc->csum), skb);
4462
4463                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4464
4465 next_desc:
4466                 rx_desc->status = 0;
4467
4468                 /* return some buffers to hardware, one at a time is too slow */
4469                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4470                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4471                         cleaned_count = 0;
4472                 }
4473
4474                 /* use prefetched values */
4475                 rx_desc = next_rxd;
4476                 buffer_info = next_buffer;
4477         }
4478         rx_ring->next_to_clean = i;
4479
4480         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4481         if (cleaned_count)
4482                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4483
4484         adapter->total_rx_packets += total_rx_packets;
4485         adapter->total_rx_bytes += total_rx_bytes;
4486         netdev->stats.rx_bytes += total_rx_bytes;
4487         netdev->stats.rx_packets += total_rx_packets;
4488         return cleaned;
4489 }
4490
4491 /**
4492  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4493  * @adapter: address of board private structure
4494  * @rx_ring: pointer to receive ring structure
4495  * @cleaned_count: number of buffers to allocate this pass
4496  **/
4497 static void
4498 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4499                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4500 {
4501         struct pci_dev *pdev = adapter->pdev;
4502         struct e1000_rx_desc *rx_desc;
4503         struct e1000_rx_buffer *buffer_info;
4504         unsigned int i;
4505
4506         i = rx_ring->next_to_use;
4507         buffer_info = &rx_ring->buffer_info[i];
4508
4509         while (cleaned_count--) {
4510                 /* allocate a new page if necessary */
4511                 if (!buffer_info->rxbuf.page) {
4512                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4513                         if (unlikely(!buffer_info->rxbuf.page)) {
4514                                 adapter->alloc_rx_buff_failed++;
4515                                 break;
4516                         }
4517                 }
4518
4519                 if (!buffer_info->dma) {
4520                         buffer_info->dma = dma_map_page(&pdev->dev,
4521                                                         buffer_info->rxbuf.page, 0,
4522                                                         adapter->rx_buffer_len,
4523                                                         DMA_FROM_DEVICE);
4524                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4525                                 put_page(buffer_info->rxbuf.page);
4526                                 buffer_info->rxbuf.page = NULL;
4527                                 buffer_info->dma = 0;
4528                                 adapter->alloc_rx_buff_failed++;
4529                                 break;
4530                         }
4531                 }
4532
4533                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4534                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4535
4536                 if (unlikely(++i == rx_ring->count))
4537                         i = 0;
4538                 buffer_info = &rx_ring->buffer_info[i];
4539         }
4540
4541         if (likely(rx_ring->next_to_use != i)) {
4542                 rx_ring->next_to_use = i;
4543                 if (unlikely(i-- == 0))
4544                         i = (rx_ring->count - 1);
4545
4546                 /* Force memory writes to complete before letting h/w
4547                  * know there are new descriptors to fetch.  (Only
4548                  * applicable for weak-ordered memory model archs,
4549                  * such as IA-64).
4550                  */
4551                 dma_wmb();
4552                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4553         }
4554 }
4555
4556 /**
4557  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4558  * @adapter: address of board private structure
4559  * @rx_ring: pointer to ring struct
4560  * @cleaned_count: number of new Rx buffers to try to allocate
4561  **/
4562 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4563                                    struct e1000_rx_ring *rx_ring,
4564                                    int cleaned_count)
4565 {
4566         struct e1000_hw *hw = &adapter->hw;
4567         struct pci_dev *pdev = adapter->pdev;
4568         struct e1000_rx_desc *rx_desc;
4569         struct e1000_rx_buffer *buffer_info;
4570         unsigned int i;
4571         unsigned int bufsz = adapter->rx_buffer_len;
4572
4573         i = rx_ring->next_to_use;
4574         buffer_info = &rx_ring->buffer_info[i];
4575
4576         while (cleaned_count--) {
4577                 void *data;
4578
4579                 if (buffer_info->rxbuf.data)
4580                         goto skip;
4581
4582                 data = e1000_alloc_frag(adapter);
4583                 if (!data) {
4584                         /* Better luck next round */
4585                         adapter->alloc_rx_buff_failed++;
4586                         break;
4587                 }
4588
4589                 /* Fix for errata 23, can't cross 64kB boundary */
4590                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4591                         void *olddata = data;
4592                         e_err(rx_err, "skb align check failed: %u bytes at "
4593                               "%p\n", bufsz, data);
4594                         /* Try again, without freeing the previous */
4595                         data = e1000_alloc_frag(adapter);
4596                         /* Failed allocation, critical failure */
4597                         if (!data) {
4598                                 skb_free_frag(olddata);
4599                                 adapter->alloc_rx_buff_failed++;
4600                                 break;
4601                         }
4602
4603                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4604                                 /* give up */
4605                                 skb_free_frag(data);
4606                                 skb_free_frag(olddata);
4607                                 adapter->alloc_rx_buff_failed++;
4608                                 break;
4609                         }
4610
4611                         /* Use new allocation */
4612                         skb_free_frag(olddata);
4613                 }
4614                 buffer_info->dma = dma_map_single(&pdev->dev,
4615                                                   data,
4616                                                   adapter->rx_buffer_len,
4617                                                   DMA_FROM_DEVICE);
4618                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4619                         skb_free_frag(data);
4620                         buffer_info->dma = 0;
4621                         adapter->alloc_rx_buff_failed++;
4622                         break;
4623                 }
4624
4625                 /* XXX if it was allocated cleanly it will never map to a
4626                  * boundary crossing
4627                  */
4628
4629                 /* Fix for errata 23, can't cross 64kB boundary */
4630                 if (!e1000_check_64k_bound(adapter,
4631                                         (void *)(unsigned long)buffer_info->dma,
4632                                         adapter->rx_buffer_len)) {
4633                         e_err(rx_err, "dma align check failed: %u bytes at "
4634                               "%p\n", adapter->rx_buffer_len,
4635                               (void *)(unsigned long)buffer_info->dma);
4636
4637                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4638                                          adapter->rx_buffer_len,
4639                                          DMA_FROM_DEVICE);
4640
4641                         skb_free_frag(data);
4642                         buffer_info->rxbuf.data = NULL;
4643                         buffer_info->dma = 0;
4644
4645                         adapter->alloc_rx_buff_failed++;
4646                         break;
4647                 }
4648                 buffer_info->rxbuf.data = data;
4649  skip:
4650                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4651                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4652
4653                 if (unlikely(++i == rx_ring->count))
4654                         i = 0;
4655                 buffer_info = &rx_ring->buffer_info[i];
4656         }
4657
4658         if (likely(rx_ring->next_to_use != i)) {
4659                 rx_ring->next_to_use = i;
4660                 if (unlikely(i-- == 0))
4661                         i = (rx_ring->count - 1);
4662
4663                 /* Force memory writes to complete before letting h/w
4664                  * know there are new descriptors to fetch.  (Only
4665                  * applicable for weak-ordered memory model archs,
4666                  * such as IA-64).
4667                  */
4668                 dma_wmb();
4669                 writel(i, hw->hw_addr + rx_ring->rdt);
4670         }
4671 }
4672
4673 /**
4674  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4675  * @adapter: address of board private structure
4676  **/
4677 static void e1000_smartspeed(struct e1000_adapter *adapter)
4678 {
4679         struct e1000_hw *hw = &adapter->hw;
4680         u16 phy_status;
4681         u16 phy_ctrl;
4682
4683         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4684            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4685                 return;
4686
4687         if (adapter->smartspeed == 0) {
4688                 /* If Master/Slave config fault is asserted twice,
4689                  * we assume back-to-back
4690                  */
4691                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4692                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4693                         return;
4694                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4695                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4696                         return;
4697                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4698                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4699                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4700                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4701                                             phy_ctrl);
4702                         adapter->smartspeed++;
4703                         if (!e1000_phy_setup_autoneg(hw) &&
4704                            !e1000_read_phy_reg(hw, PHY_CTRL,
4705                                                &phy_ctrl)) {
4706                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4707                                              MII_CR_RESTART_AUTO_NEG);
4708                                 e1000_write_phy_reg(hw, PHY_CTRL,
4709                                                     phy_ctrl);
4710                         }
4711                 }
4712                 return;
4713         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4714                 /* If still no link, perhaps using 2/3 pair cable */
4715                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4716                 phy_ctrl |= CR_1000T_MS_ENABLE;
4717                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4718                 if (!e1000_phy_setup_autoneg(hw) &&
4719                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4720                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4721                                      MII_CR_RESTART_AUTO_NEG);
4722                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4723                 }
4724         }
4725         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4726         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4727                 adapter->smartspeed = 0;
4728 }
4729
4730 /**
4731  * e1000_ioctl - handle ioctl calls
4732  * @netdev: pointer to our netdev
4733  * @ifr: pointer to interface request structure
4734  * @cmd: ioctl data
4735  **/
4736 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4737 {
4738         switch (cmd) {
4739         case SIOCGMIIPHY:
4740         case SIOCGMIIREG:
4741         case SIOCSMIIREG:
4742                 return e1000_mii_ioctl(netdev, ifr, cmd);
4743         default:
4744                 return -EOPNOTSUPP;
4745         }
4746 }
4747
4748 /**
4749  * e1000_mii_ioctl -
4750  * @netdev: pointer to our netdev
4751  * @ifr: pointer to interface request structure
4752  * @cmd: ioctl data
4753  **/
4754 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4755                            int cmd)
4756 {
4757         struct e1000_adapter *adapter = netdev_priv(netdev);
4758         struct e1000_hw *hw = &adapter->hw;
4759         struct mii_ioctl_data *data = if_mii(ifr);
4760         int retval;
4761         u16 mii_reg;
4762         unsigned long flags;
4763
4764         if (hw->media_type != e1000_media_type_copper)
4765                 return -EOPNOTSUPP;
4766
4767         switch (cmd) {
4768         case SIOCGMIIPHY:
4769                 data->phy_id = hw->phy_addr;
4770                 break;
4771         case SIOCGMIIREG:
4772                 spin_lock_irqsave(&adapter->stats_lock, flags);
4773                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4774                                    &data->val_out)) {
4775                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4776                         return -EIO;
4777                 }
4778                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4779                 break;
4780         case SIOCSMIIREG:
4781                 if (data->reg_num & ~(0x1F))
4782                         return -EFAULT;
4783                 mii_reg = data->val_in;
4784                 spin_lock_irqsave(&adapter->stats_lock, flags);
4785                 if (e1000_write_phy_reg(hw, data->reg_num,
4786                                         mii_reg)) {
4787                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4788                         return -EIO;
4789                 }
4790                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4791                 if (hw->media_type == e1000_media_type_copper) {
4792                         switch (data->reg_num) {
4793                         case PHY_CTRL:
4794                                 if (mii_reg & MII_CR_POWER_DOWN)
4795                                         break;
4796                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4797                                         hw->autoneg = 1;
4798                                         hw->autoneg_advertised = 0x2F;
4799                                 } else {
4800                                         u32 speed;
4801                                         if (mii_reg & 0x40)
4802                                                 speed = SPEED_1000;
4803                                         else if (mii_reg & 0x2000)
4804                                                 speed = SPEED_100;
4805                                         else
4806                                                 speed = SPEED_10;
4807                                         retval = e1000_set_spd_dplx(
4808                                                 adapter, speed,
4809                                                 ((mii_reg & 0x100)
4810                                                  ? DUPLEX_FULL :
4811                                                  DUPLEX_HALF));
4812                                         if (retval)
4813                                                 return retval;
4814                                 }
4815                                 if (netif_running(adapter->netdev))
4816                                         e1000_reinit_locked(adapter);
4817                                 else
4818                                         e1000_reset(adapter);
4819                                 break;
4820                         case M88E1000_PHY_SPEC_CTRL:
4821                         case M88E1000_EXT_PHY_SPEC_CTRL:
4822                                 if (e1000_phy_reset(hw))
4823                                         return -EIO;
4824                                 break;
4825                         }
4826                 } else {
4827                         switch (data->reg_num) {
4828                         case PHY_CTRL:
4829                                 if (mii_reg & MII_CR_POWER_DOWN)
4830                                         break;
4831                                 if (netif_running(adapter->netdev))
4832                                         e1000_reinit_locked(adapter);
4833                                 else
4834                                         e1000_reset(adapter);
4835                                 break;
4836                         }
4837                 }
4838                 break;
4839         default:
4840                 return -EOPNOTSUPP;
4841         }
4842         return E1000_SUCCESS;
4843 }
4844
4845 void e1000_pci_set_mwi(struct e1000_hw *hw)
4846 {
4847         struct e1000_adapter *adapter = hw->back;
4848         int ret_val = pci_set_mwi(adapter->pdev);
4849
4850         if (ret_val)
4851                 e_err(probe, "Error in setting MWI\n");
4852 }
4853
4854 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4855 {
4856         struct e1000_adapter *adapter = hw->back;
4857
4858         pci_clear_mwi(adapter->pdev);
4859 }
4860
4861 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4862 {
4863         struct e1000_adapter *adapter = hw->back;
4864         return pcix_get_mmrbc(adapter->pdev);
4865 }
4866
4867 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4868 {
4869         struct e1000_adapter *adapter = hw->back;
4870         pcix_set_mmrbc(adapter->pdev, mmrbc);
4871 }
4872
4873 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4874 {
4875         outl(value, port);
4876 }
4877
4878 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4879 {
4880         u16 vid;
4881
4882         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4883                 return true;
4884         return false;
4885 }
4886
4887 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4888                               netdev_features_t features)
4889 {
4890         struct e1000_hw *hw = &adapter->hw;
4891         u32 ctrl;
4892
4893         ctrl = er32(CTRL);
4894         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4895                 /* enable VLAN tag insert/strip */
4896                 ctrl |= E1000_CTRL_VME;
4897         } else {
4898                 /* disable VLAN tag insert/strip */
4899                 ctrl &= ~E1000_CTRL_VME;
4900         }
4901         ew32(CTRL, ctrl);
4902 }
4903 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4904                                      bool filter_on)
4905 {
4906         struct e1000_hw *hw = &adapter->hw;
4907         u32 rctl;
4908
4909         if (!test_bit(__E1000_DOWN, &adapter->flags))
4910                 e1000_irq_disable(adapter);
4911
4912         __e1000_vlan_mode(adapter, adapter->netdev->features);
4913         if (filter_on) {
4914                 /* enable VLAN receive filtering */
4915                 rctl = er32(RCTL);
4916                 rctl &= ~E1000_RCTL_CFIEN;
4917                 if (!(adapter->netdev->flags & IFF_PROMISC))
4918                         rctl |= E1000_RCTL_VFE;
4919                 ew32(RCTL, rctl);
4920                 e1000_update_mng_vlan(adapter);
4921         } else {
4922                 /* disable VLAN receive filtering */
4923                 rctl = er32(RCTL);
4924                 rctl &= ~E1000_RCTL_VFE;
4925                 ew32(RCTL, rctl);
4926         }
4927
4928         if (!test_bit(__E1000_DOWN, &adapter->flags))
4929                 e1000_irq_enable(adapter);
4930 }
4931
4932 static void e1000_vlan_mode(struct net_device *netdev,
4933                             netdev_features_t features)
4934 {
4935         struct e1000_adapter *adapter = netdev_priv(netdev);
4936
4937         if (!test_bit(__E1000_DOWN, &adapter->flags))
4938                 e1000_irq_disable(adapter);
4939
4940         __e1000_vlan_mode(adapter, features);
4941
4942         if (!test_bit(__E1000_DOWN, &adapter->flags))
4943                 e1000_irq_enable(adapter);
4944 }
4945
4946 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4947                                  __be16 proto, u16 vid)
4948 {
4949         struct e1000_adapter *adapter = netdev_priv(netdev);
4950         struct e1000_hw *hw = &adapter->hw;
4951         u32 vfta, index;
4952
4953         if ((hw->mng_cookie.status &
4954              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4955             (vid == adapter->mng_vlan_id))
4956                 return 0;
4957
4958         if (!e1000_vlan_used(adapter))
4959                 e1000_vlan_filter_on_off(adapter, true);
4960
4961         /* add VID to filter table */
4962         index = (vid >> 5) & 0x7F;
4963         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4964         vfta |= (1 << (vid & 0x1F));
4965         e1000_write_vfta(hw, index, vfta);
4966
4967         set_bit(vid, adapter->active_vlans);
4968
4969         return 0;
4970 }
4971
4972 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4973                                   __be16 proto, u16 vid)
4974 {
4975         struct e1000_adapter *adapter = netdev_priv(netdev);
4976         struct e1000_hw *hw = &adapter->hw;
4977         u32 vfta, index;
4978
4979         if (!test_bit(__E1000_DOWN, &adapter->flags))
4980                 e1000_irq_disable(adapter);
4981         if (!test_bit(__E1000_DOWN, &adapter->flags))
4982                 e1000_irq_enable(adapter);
4983
4984         /* remove VID from filter table */
4985         index = (vid >> 5) & 0x7F;
4986         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4987         vfta &= ~(1 << (vid & 0x1F));
4988         e1000_write_vfta(hw, index, vfta);
4989
4990         clear_bit(vid, adapter->active_vlans);
4991
4992         if (!e1000_vlan_used(adapter))
4993                 e1000_vlan_filter_on_off(adapter, false);
4994
4995         return 0;
4996 }
4997
4998 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4999 {
5000         u16 vid;
5001
5002         if (!e1000_vlan_used(adapter))
5003                 return;
5004
5005         e1000_vlan_filter_on_off(adapter, true);
5006         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5007                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5008 }
5009
5010 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5011 {
5012         struct e1000_hw *hw = &adapter->hw;
5013
5014         hw->autoneg = 0;
5015
5016         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5017          * for the switch() below to work
5018          */
5019         if ((spd & 1) || (dplx & ~1))
5020                 goto err_inval;
5021
5022         /* Fiber NICs only allow 1000 gbps Full duplex */
5023         if ((hw->media_type == e1000_media_type_fiber) &&
5024             spd != SPEED_1000 &&
5025             dplx != DUPLEX_FULL)
5026                 goto err_inval;
5027
5028         switch (spd + dplx) {
5029         case SPEED_10 + DUPLEX_HALF:
5030                 hw->forced_speed_duplex = e1000_10_half;
5031                 break;
5032         case SPEED_10 + DUPLEX_FULL:
5033                 hw->forced_speed_duplex = e1000_10_full;
5034                 break;
5035         case SPEED_100 + DUPLEX_HALF:
5036                 hw->forced_speed_duplex = e1000_100_half;
5037                 break;
5038         case SPEED_100 + DUPLEX_FULL:
5039                 hw->forced_speed_duplex = e1000_100_full;
5040                 break;
5041         case SPEED_1000 + DUPLEX_FULL:
5042                 hw->autoneg = 1;
5043                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5044                 break;
5045         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5046         default:
5047                 goto err_inval;
5048         }
5049
5050         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5051         hw->mdix = AUTO_ALL_MODES;
5052
5053         return 0;
5054
5055 err_inval:
5056         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5057         return -EINVAL;
5058 }
5059
5060 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5061 {
5062         struct net_device *netdev = pci_get_drvdata(pdev);
5063         struct e1000_adapter *adapter = netdev_priv(netdev);
5064         struct e1000_hw *hw = &adapter->hw;
5065         u32 ctrl, ctrl_ext, rctl, status;
5066         u32 wufc = adapter->wol;
5067
5068         netif_device_detach(netdev);
5069
5070         if (netif_running(netdev)) {
5071                 int count = E1000_CHECK_RESET_COUNT;
5072
5073                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5074                         usleep_range(10000, 20000);
5075
5076                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5077                 e1000_down(adapter);
5078         }
5079
5080         status = er32(STATUS);
5081         if (status & E1000_STATUS_LU)
5082                 wufc &= ~E1000_WUFC_LNKC;
5083
5084         if (wufc) {
5085                 e1000_setup_rctl(adapter);
5086                 e1000_set_rx_mode(netdev);
5087
5088                 rctl = er32(RCTL);
5089
5090                 /* turn on all-multi mode if wake on multicast is enabled */
5091                 if (wufc & E1000_WUFC_MC)
5092                         rctl |= E1000_RCTL_MPE;
5093
5094                 /* enable receives in the hardware */
5095                 ew32(RCTL, rctl | E1000_RCTL_EN);
5096
5097                 if (hw->mac_type >= e1000_82540) {
5098                         ctrl = er32(CTRL);
5099                         /* advertise wake from D3Cold */
5100                         #define E1000_CTRL_ADVD3WUC 0x00100000
5101                         /* phy power management enable */
5102                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5103                         ctrl |= E1000_CTRL_ADVD3WUC |
5104                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5105                         ew32(CTRL, ctrl);
5106                 }
5107
5108                 if (hw->media_type == e1000_media_type_fiber ||
5109                     hw->media_type == e1000_media_type_internal_serdes) {
5110                         /* keep the laser running in D3 */
5111                         ctrl_ext = er32(CTRL_EXT);
5112                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5113                         ew32(CTRL_EXT, ctrl_ext);
5114                 }
5115
5116                 ew32(WUC, E1000_WUC_PME_EN);
5117                 ew32(WUFC, wufc);
5118         } else {
5119                 ew32(WUC, 0);
5120                 ew32(WUFC, 0);
5121         }
5122
5123         e1000_release_manageability(adapter);
5124
5125         *enable_wake = !!wufc;
5126
5127         /* make sure adapter isn't asleep if manageability is enabled */
5128         if (adapter->en_mng_pt)
5129                 *enable_wake = true;
5130
5131         if (netif_running(netdev))
5132                 e1000_free_irq(adapter);
5133
5134         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5135                 pci_disable_device(pdev);
5136
5137         return 0;
5138 }
5139
5140 static int __maybe_unused e1000_suspend(struct device *dev)
5141 {
5142         int retval;
5143         struct pci_dev *pdev = to_pci_dev(dev);
5144         bool wake;
5145
5146         retval = __e1000_shutdown(pdev, &wake);
5147         device_set_wakeup_enable(dev, wake);
5148
5149         return retval;
5150 }
5151
5152 static int __maybe_unused e1000_resume(struct device *dev)
5153 {
5154         struct pci_dev *pdev = to_pci_dev(dev);
5155         struct net_device *netdev = pci_get_drvdata(pdev);
5156         struct e1000_adapter *adapter = netdev_priv(netdev);
5157         struct e1000_hw *hw = &adapter->hw;
5158         u32 err;
5159
5160         if (adapter->need_ioport)
5161                 err = pci_enable_device(pdev);
5162         else
5163                 err = pci_enable_device_mem(pdev);
5164         if (err) {
5165                 pr_err("Cannot enable PCI device from suspend\n");
5166                 return err;
5167         }
5168
5169         /* flush memory to make sure state is correct */
5170         smp_mb__before_atomic();
5171         clear_bit(__E1000_DISABLED, &adapter->flags);
5172         pci_set_master(pdev);
5173
5174         pci_enable_wake(pdev, PCI_D3hot, 0);
5175         pci_enable_wake(pdev, PCI_D3cold, 0);
5176
5177         if (netif_running(netdev)) {
5178                 err = e1000_request_irq(adapter);
5179                 if (err)
5180                         return err;
5181         }
5182
5183         e1000_power_up_phy(adapter);
5184         e1000_reset(adapter);
5185         ew32(WUS, ~0);
5186
5187         e1000_init_manageability(adapter);
5188
5189         if (netif_running(netdev))
5190                 e1000_up(adapter);
5191
5192         netif_device_attach(netdev);
5193
5194         return 0;
5195 }
5196
5197 static void e1000_shutdown(struct pci_dev *pdev)
5198 {
5199         bool wake;
5200
5201         __e1000_shutdown(pdev, &wake);
5202
5203         if (system_state == SYSTEM_POWER_OFF) {
5204                 pci_wake_from_d3(pdev, wake);
5205                 pci_set_power_state(pdev, PCI_D3hot);
5206         }
5207 }
5208
5209 #ifdef CONFIG_NET_POLL_CONTROLLER
5210 /* Polling 'interrupt' - used by things like netconsole to send skbs
5211  * without having to re-enable interrupts. It's not called while
5212  * the interrupt routine is executing.
5213  */
5214 static void e1000_netpoll(struct net_device *netdev)
5215 {
5216         struct e1000_adapter *adapter = netdev_priv(netdev);
5217
5218         if (disable_hardirq(adapter->pdev->irq))
5219                 e1000_intr(adapter->pdev->irq, netdev);
5220         enable_irq(adapter->pdev->irq);
5221 }
5222 #endif
5223
5224 /**
5225  * e1000_io_error_detected - called when PCI error is detected
5226  * @pdev: Pointer to PCI device
5227  * @state: The current pci connection state
5228  *
5229  * This function is called after a PCI bus error affecting
5230  * this device has been detected.
5231  */
5232 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5233                                                 pci_channel_state_t state)
5234 {
5235         struct net_device *netdev = pci_get_drvdata(pdev);
5236         struct e1000_adapter *adapter = netdev_priv(netdev);
5237
5238         netif_device_detach(netdev);
5239
5240         if (state == pci_channel_io_perm_failure)
5241                 return PCI_ERS_RESULT_DISCONNECT;
5242
5243         if (netif_running(netdev))
5244                 e1000_down(adapter);
5245
5246         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5247                 pci_disable_device(pdev);
5248
5249         /* Request a slot slot reset. */
5250         return PCI_ERS_RESULT_NEED_RESET;
5251 }
5252
5253 /**
5254  * e1000_io_slot_reset - called after the pci bus has been reset.
5255  * @pdev: Pointer to PCI device
5256  *
5257  * Restart the card from scratch, as if from a cold-boot. Implementation
5258  * resembles the first-half of the e1000_resume routine.
5259  */
5260 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5261 {
5262         struct net_device *netdev = pci_get_drvdata(pdev);
5263         struct e1000_adapter *adapter = netdev_priv(netdev);
5264         struct e1000_hw *hw = &adapter->hw;
5265         int err;
5266
5267         if (adapter->need_ioport)
5268                 err = pci_enable_device(pdev);
5269         else
5270                 err = pci_enable_device_mem(pdev);
5271         if (err) {
5272                 pr_err("Cannot re-enable PCI device after reset.\n");
5273                 return PCI_ERS_RESULT_DISCONNECT;
5274         }
5275
5276         /* flush memory to make sure state is correct */
5277         smp_mb__before_atomic();
5278         clear_bit(__E1000_DISABLED, &adapter->flags);
5279         pci_set_master(pdev);
5280
5281         pci_enable_wake(pdev, PCI_D3hot, 0);
5282         pci_enable_wake(pdev, PCI_D3cold, 0);
5283
5284         e1000_reset(adapter);
5285         ew32(WUS, ~0);
5286
5287         return PCI_ERS_RESULT_RECOVERED;
5288 }
5289
5290 /**
5291  * e1000_io_resume - called when traffic can start flowing again.
5292  * @pdev: Pointer to PCI device
5293  *
5294  * This callback is called when the error recovery driver tells us that
5295  * its OK to resume normal operation. Implementation resembles the
5296  * second-half of the e1000_resume routine.
5297  */
5298 static void e1000_io_resume(struct pci_dev *pdev)
5299 {
5300         struct net_device *netdev = pci_get_drvdata(pdev);
5301         struct e1000_adapter *adapter = netdev_priv(netdev);
5302
5303         e1000_init_manageability(adapter);
5304
5305         if (netif_running(netdev)) {
5306                 if (e1000_up(adapter)) {
5307                         pr_info("can't bring device back up after reset\n");
5308                         return;
5309                 }
5310         }
5311
5312         netif_device_attach(netdev);
5313 }
5314
5315 /* e1000_main.c */