drivers/net/ethernet/intel/igc/igc_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2018 Intel Corporation */
   3
   4 #include <linux/module.h>
   5 #include <linux/types.h>
   6 #include <linux/if_vlan.h>
   7 #include <linux/aer.h>
   8 #include <linux/tcp.h>
   9 #include <linux/udp.h>
  10 #include <linux/ip.h>
  11 #include <linux/pm_runtime.h>
  12 #include <linux/pci.h>
  13 #include <net/pkt_sched.h>
  14
  15 #include <net/ipv6.h>
  16
  17 #include "igc.h"
  18 #include "igc_hw.h"
  19 #include "igc_tsn.h"
  20
  21 #define DRV_SUMMARY     "Intel(R) 2.5G Ethernet Linux Driver"
  22
  23 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  24
  25 static int debug = -1;
  26
  27 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  28 MODULE_DESCRIPTION(DRV_SUMMARY);
  29 MODULE_LICENSE("GPL v2");
  30 module_param(debug, int, 0);
  31 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  32
  33 char igc_driver_name[] = "igc";
  34 static const char igc_driver_string[] = DRV_SUMMARY;
  35 static const char igc_copyright[] =
  36         "Copyright(c) 2018 Intel Corporation.";
  37
  38 static const struct igc_info *igc_info_tbl[] = {
  39         [board_base] = &igc_base_info,
  40 };
  41
  42 static const struct pci_device_id igc_pci_tbl[] = {
  43         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
  44         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
  45         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
  46         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
  47         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
  48         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
  49         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
  50         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
  51         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
  52         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
  53         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
  54         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
  55         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
  56         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
  57         /* required last entry */
  58         {0, }
  59 };
  60
  61 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
  62
  63 enum latency_range {
  64         lowest_latency = 0,
  65         low_latency = 1,
  66         bulk_latency = 2,
  67         latency_invalid = 255
  68 };
  69
  70 void igc_reset(struct igc_adapter *adapter)
  71 {
  72         struct net_device *dev = adapter->netdev;
  73         struct igc_hw *hw = &adapter->hw;
  74         struct igc_fc_info *fc = &hw->fc;
  75         u32 pba, hwm;
  76
  77         /* Repartition PBA for greater than 9k MTU if required */
  78         pba = IGC_PBA_34K;
  79
  80         /* flow control settings
  81          * The high water mark must be low enough to fit one full frame
  82          * after transmitting the pause frame.  As such we must have enough
  83          * space to allow for us to complete our current transmit and then
  84          * receive the frame that is in progress from the link partner.
  85          * Set it to:
  86          * - the full Rx FIFO size minus one full Tx plus one full Rx frame
  87          */
  88         hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
  89
  90         fc->high_water = hwm & 0xFFFFFFF0;      /* 16-byte granularity */
  91         fc->low_water = fc->high_water - 16;
  92         fc->pause_time = 0xFFFF;
  93         fc->send_xon = 1;
  94         fc->current_mode = fc->requested_mode;
  95
  96         hw->mac.ops.reset_hw(hw);
  97
  98         if (hw->mac.ops.init_hw(hw))
  99                 netdev_err(dev, "Error on hardware initialization\n");
 100
 101         /* Re-establish EEE setting */
 102         igc_set_eee_i225(hw, true, true, true);
 103
 104         if (!netif_running(adapter->netdev))
 105                 igc_power_down_phy_copper_base(&adapter->hw);
 106
 107         /* Re-enable PTP, where applicable. */
 108         igc_ptp_reset(adapter);
 109
 110         /* Re-enable TSN offloading, where applicable. */
 111         igc_tsn_offload_apply(adapter);
 112
 113         igc_get_phy_info(hw);
 114 }
 115
 116 /**
 117  * igc_power_up_link - Power up the phy link
 118  * @adapter: address of board private structure
 119  */
 120 static void igc_power_up_link(struct igc_adapter *adapter)
 121 {
 122         igc_reset_phy(&adapter->hw);
 123
 124         igc_power_up_phy_copper(&adapter->hw);
 125
 126         igc_setup_link(&adapter->hw);
 127 }
 128
 129 /**
 130  * igc_release_hw_control - release control of the h/w to f/w
 131  * @adapter: address of board private structure
 132  *
 133  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 134  * For ASF and Pass Through versions of f/w this means that the
 135  * driver is no longer loaded.
 136  */
 137 static void igc_release_hw_control(struct igc_adapter *adapter)
 138 {
 139         struct igc_hw *hw = &adapter->hw;
 140         u32 ctrl_ext;
 141
 142         if (!pci_device_is_present(adapter->pdev))
 143                 return;
 144
 145         /* Let firmware take over control of h/w */
 146         ctrl_ext = rd32(IGC_CTRL_EXT);
 147         wr32(IGC_CTRL_EXT,
 148              ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 149 }
 150
 151 /**
 152  * igc_get_hw_control - get control of the h/w from f/w
 153  * @adapter: address of board private structure
 154  *
 155  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 156  * For ASF and Pass Through versions of f/w this means that
 157  * the driver is loaded.
 158  */
 159 static void igc_get_hw_control(struct igc_adapter *adapter)
 160 {
 161         struct igc_hw *hw = &adapter->hw;
 162         u32 ctrl_ext;
 163
 164         /* Let firmware know the driver has taken over */
 165         ctrl_ext = rd32(IGC_CTRL_EXT);
 166         wr32(IGC_CTRL_EXT,
 167              ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 168 }
 169
 170 /**
 171  * igc_clean_tx_ring - Free Tx Buffers
 172  * @tx_ring: ring to be cleaned
 173  */
 174 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 175 {
 176         u16 i = tx_ring->next_to_clean;
 177         struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 178
 179         while (i != tx_ring->next_to_use) {
 180                 union igc_adv_tx_desc *eop_desc, *tx_desc;
 181
 182                 /* Free all the Tx ring sk_buffs */
 183                 dev_kfree_skb_any(tx_buffer->skb);
 184
 185                 /* unmap skb header data */
 186                 dma_unmap_single(tx_ring->dev,
 187                                  dma_unmap_addr(tx_buffer, dma),
 188                                  dma_unmap_len(tx_buffer, len),
 189                                  DMA_TO_DEVICE);
 190
 191                 /* check for eop_desc to determine the end of the packet */
 192                 eop_desc = tx_buffer->next_to_watch;
 193                 tx_desc = IGC_TX_DESC(tx_ring, i);
 194
 195                 /* unmap remaining buffers */
 196                 while (tx_desc != eop_desc) {
 197                         tx_buffer++;
 198                         tx_desc++;
 199                         i++;
 200                         if (unlikely(i == tx_ring->count)) {
 201                                 i = 0;
 202                                 tx_buffer = tx_ring->tx_buffer_info;
 203                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 204                         }
 205
 206                         /* unmap any remaining paged data */
 207                         if (dma_unmap_len(tx_buffer, len))
 208                                 dma_unmap_page(tx_ring->dev,
 209                                                dma_unmap_addr(tx_buffer, dma),
 210                                                dma_unmap_len(tx_buffer, len),
 211                                                DMA_TO_DEVICE);
 212                 }
 213
 214                 tx_buffer->next_to_watch = NULL;
 215
 216                 /* move us one more past the eop_desc for start of next pkt */
 217                 tx_buffer++;
 218                 i++;
 219                 if (unlikely(i == tx_ring->count)) {
 220                         i = 0;
 221                         tx_buffer = tx_ring->tx_buffer_info;
 222                 }
 223         }
 224
 225         /* reset BQL for queue */
 226         netdev_tx_reset_queue(txring_txq(tx_ring));
 227
 228         /* reset next_to_use and next_to_clean */
 229         tx_ring->next_to_use = 0;
 230         tx_ring->next_to_clean = 0;
 231 }
 232
 233 /**
 234  * igc_free_tx_resources - Free Tx Resources per Queue
 235  * @tx_ring: Tx descriptor ring for a specific queue
 236  *
 237  * Free all transmit software resources
 238  */
 239 void igc_free_tx_resources(struct igc_ring *tx_ring)
 240 {
 241         igc_clean_tx_ring(tx_ring);
 242
 243         vfree(tx_ring->tx_buffer_info);
 244         tx_ring->tx_buffer_info = NULL;
 245
 246         /* if not set, then don't free */
 247         if (!tx_ring->desc)
 248                 return;
 249
 250         dma_free_coherent(tx_ring->dev, tx_ring->size,
 251                           tx_ring->desc, tx_ring->dma);
 252
 253         tx_ring->desc = NULL;
 254 }
 255
 256 /**
 257  * igc_free_all_tx_resources - Free Tx Resources for All Queues
 258  * @adapter: board private structure
 259  *
 260  * Free all transmit software resources
 261  */
 262 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
 263 {
 264         int i;
 265
 266         for (i = 0; i < adapter->num_tx_queues; i++)
 267                 igc_free_tx_resources(adapter->tx_ring[i]);
 268 }
 269
 270 /**
 271  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
 272  * @adapter: board private structure
 273  */
 274 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
 275 {
 276         int i;
 277
 278         for (i = 0; i < adapter->num_tx_queues; i++)
 279                 if (adapter->tx_ring[i])
 280                         igc_clean_tx_ring(adapter->tx_ring[i]);
 281 }
 282
 283 /**
 284  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
 285  * @tx_ring: tx descriptor ring (for a specific queue) to setup
 286  *
 287  * Return 0 on success, negative on failure
 288  */
 289 int igc_setup_tx_resources(struct igc_ring *tx_ring)
 290 {
 291         struct net_device *ndev = tx_ring->netdev;
 292         struct device *dev = tx_ring->dev;
 293         int size = 0;
 294
 295         size = sizeof(struct igc_tx_buffer) * tx_ring->count;
 296         tx_ring->tx_buffer_info = vzalloc(size);
 297         if (!tx_ring->tx_buffer_info)
 298                 goto err;
 299
 300         /* round up to nearest 4K */
 301         tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
 302         tx_ring->size = ALIGN(tx_ring->size, 4096);
 303
 304         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 305                                            &tx_ring->dma, GFP_KERNEL);
 306
 307         if (!tx_ring->desc)
 308                 goto err;
 309
 310         tx_ring->next_to_use = 0;
 311         tx_ring->next_to_clean = 0;
 312
 313         return 0;
 314
 315 err:
 316         vfree(tx_ring->tx_buffer_info);
 317         netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
 318         return -ENOMEM;
 319 }
 320
 321 /**
 322  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
 323  * @adapter: board private structure
 324  *
 325  * Return 0 on success, negative on failure
 326  */
 327 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
 328 {
 329         struct net_device *dev = adapter->netdev;
 330         int i, err = 0;
 331
 332         for (i = 0; i < adapter->num_tx_queues; i++) {
 333                 err = igc_setup_tx_resources(adapter->tx_ring[i]);
 334                 if (err) {
 335                         netdev_err(dev, "Error on Tx queue %u setup\n", i);
 336                         for (i--; i >= 0; i--)
 337                                 igc_free_tx_resources(adapter->tx_ring[i]);
 338                         break;
 339                 }
 340         }
 341
 342         return err;
 343 }
 344
 345 /**
 346  * igc_clean_rx_ring - Free Rx Buffers per Queue
 347  * @rx_ring: ring to free buffers from
 348  */
 349 static void igc_clean_rx_ring(struct igc_ring *rx_ring)
 350 {
 351         u16 i = rx_ring->next_to_clean;
 352
 353         dev_kfree_skb(rx_ring->skb);
 354         rx_ring->skb = NULL;
 355
 356         /* Free all the Rx ring sk_buffs */
 357         while (i != rx_ring->next_to_alloc) {
 358                 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
 359
 360                 /* Invalidate cache lines that may have been written to by
 361                  * device so that we avoid corrupting memory.
 362                  */
 363                 dma_sync_single_range_for_cpu(rx_ring->dev,
 364                                               buffer_info->dma,
 365                                               buffer_info->page_offset,
 366                                               igc_rx_bufsz(rx_ring),
 367                                               DMA_FROM_DEVICE);
 368
 369                 /* free resources associated with mapping */
 370                 dma_unmap_page_attrs(rx_ring->dev,
 371                                      buffer_info->dma,
 372                                      igc_rx_pg_size(rx_ring),
 373                                      DMA_FROM_DEVICE,
 374                                      IGC_RX_DMA_ATTR);
 375                 __page_frag_cache_drain(buffer_info->page,
 376                                         buffer_info->pagecnt_bias);
 377
 378                 i++;
 379                 if (i == rx_ring->count)
 380                         i = 0;
 381         }
 382
 383         rx_ring->next_to_alloc = 0;
 384         rx_ring->next_to_clean = 0;
 385         rx_ring->next_to_use = 0;
 386 }
 387
 388 /**
 389  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
 390  * @adapter: board private structure
 391  */
 392 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
 393 {
 394         int i;
 395
 396         for (i = 0; i < adapter->num_rx_queues; i++)
 397                 if (adapter->rx_ring[i])
 398                         igc_clean_rx_ring(adapter->rx_ring[i]);
 399 }
 400
 401 /**
 402  * igc_free_rx_resources - Free Rx Resources
 403  * @rx_ring: ring to clean the resources from
 404  *
 405  * Free all receive software resources
 406  */
 407 void igc_free_rx_resources(struct igc_ring *rx_ring)
 408 {
 409         igc_clean_rx_ring(rx_ring);
 410
 411         vfree(rx_ring->rx_buffer_info);
 412         rx_ring->rx_buffer_info = NULL;
 413
 414         /* if not set, then don't free */
 415         if (!rx_ring->desc)
 416                 return;
 417
 418         dma_free_coherent(rx_ring->dev, rx_ring->size,
 419                           rx_ring->desc, rx_ring->dma);
 420
 421         rx_ring->desc = NULL;
 422 }
 423
 424 /**
 425  * igc_free_all_rx_resources - Free Rx Resources for All Queues
 426  * @adapter: board private structure
 427  *
 428  * Free all receive software resources
 429  */
 430 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
 431 {
 432         int i;
 433
 434         for (i = 0; i < adapter->num_rx_queues; i++)
 435                 igc_free_rx_resources(adapter->rx_ring[i]);
 436 }
 437
 438 /**
 439  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
 440  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 441  *
 442  * Returns 0 on success, negative on failure
 443  */
 444 int igc_setup_rx_resources(struct igc_ring *rx_ring)
 445 {
 446         struct net_device *ndev = rx_ring->netdev;
 447         struct device *dev = rx_ring->dev;
 448         int size, desc_len;
 449
 450         size = sizeof(struct igc_rx_buffer) * rx_ring->count;
 451         rx_ring->rx_buffer_info = vzalloc(size);
 452         if (!rx_ring->rx_buffer_info)
 453                 goto err;
 454
 455         desc_len = sizeof(union igc_adv_rx_desc);
 456
 457         /* Round up to nearest 4K */
 458         rx_ring->size = rx_ring->count * desc_len;
 459         rx_ring->size = ALIGN(rx_ring->size, 4096);
 460
 461         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
 462                                            &rx_ring->dma, GFP_KERNEL);
 463
 464         if (!rx_ring->desc)
 465                 goto err;
 466
 467         rx_ring->next_to_alloc = 0;
 468         rx_ring->next_to_clean = 0;
 469         rx_ring->next_to_use = 0;
 470
 471         return 0;
 472
 473 err:
 474         vfree(rx_ring->rx_buffer_info);
 475         rx_ring->rx_buffer_info = NULL;
 476         netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
 477         return -ENOMEM;
 478 }
 479
 480 /**
 481  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
 482  *                                (Descriptors) for all queues
 483  * @adapter: board private structure
 484  *
 485  * Return 0 on success, negative on failure
 486  */
 487 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
 488 {
 489         struct net_device *dev = adapter->netdev;
 490         int i, err = 0;
 491
 492         for (i = 0; i < adapter->num_rx_queues; i++) {
 493                 err = igc_setup_rx_resources(adapter->rx_ring[i]);
 494                 if (err) {
 495                         netdev_err(dev, "Error on Rx queue %u setup\n", i);
 496                         for (i--; i >= 0; i--)
 497                                 igc_free_rx_resources(adapter->rx_ring[i]);
 498                         break;
 499                 }
 500         }
 501
 502         return err;
 503 }
 504
 505 /**
 506  * igc_configure_rx_ring - Configure a receive ring after Reset
 507  * @adapter: board private structure
 508  * @ring: receive ring to be configured
 509  *
 510  * Configure the Rx unit of the MAC after a reset.
 511  */
 512 static void igc_configure_rx_ring(struct igc_adapter *adapter,
 513                                   struct igc_ring *ring)
 514 {
 515         struct igc_hw *hw = &adapter->hw;
 516         union igc_adv_rx_desc *rx_desc;
 517         int reg_idx = ring->reg_idx;
 518         u32 srrctl = 0, rxdctl = 0;
 519         u64 rdba = ring->dma;
 520
 521         /* disable the queue */
 522         wr32(IGC_RXDCTL(reg_idx), 0);
 523
 524         /* Set DMA base address registers */
 525         wr32(IGC_RDBAL(reg_idx),
 526              rdba & 0x00000000ffffffffULL);
 527         wr32(IGC_RDBAH(reg_idx), rdba >> 32);
 528         wr32(IGC_RDLEN(reg_idx),
 529              ring->count * sizeof(union igc_adv_rx_desc));
 530
 531         /* initialize head and tail */
 532         ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
 533         wr32(IGC_RDH(reg_idx), 0);
 534         writel(0, ring->tail);
 535
 536         /* reset next-to- use/clean to place SW in sync with hardware */
 537         ring->next_to_clean = 0;
 538         ring->next_to_use = 0;
 539
 540         /* set descriptor configuration */
 541         srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
 542         if (ring_uses_large_buffer(ring))
 543                 srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 544         else
 545                 srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 546         srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
 547
 548         wr32(IGC_SRRCTL(reg_idx), srrctl);
 549
 550         rxdctl |= IGC_RX_PTHRESH;
 551         rxdctl |= IGC_RX_HTHRESH << 8;
 552         rxdctl |= IGC_RX_WTHRESH << 16;
 553
 554         /* initialize rx_buffer_info */
 555         memset(ring->rx_buffer_info, 0,
 556                sizeof(struct igc_rx_buffer) * ring->count);
 557
 558         /* initialize Rx descriptor 0 */
 559         rx_desc = IGC_RX_DESC(ring, 0);
 560         rx_desc->wb.upper.length = 0;
 561
 562         /* enable receive descriptor fetching */
 563         rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
 564
 565         wr32(IGC_RXDCTL(reg_idx), rxdctl);
 566 }
 567
 568 /**
 569  * igc_configure_rx - Configure receive Unit after Reset
 570  * @adapter: board private structure
 571  *
 572  * Configure the Rx unit of the MAC after a reset.
 573  */
 574 static void igc_configure_rx(struct igc_adapter *adapter)
 575 {
 576         int i;
 577
 578         /* Setup the HW Rx Head and Tail Descriptor Pointers and
 579          * the Base and Length of the Rx Descriptor Ring
 580          */
 581         for (i = 0; i < adapter->num_rx_queues; i++)
 582                 igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
 583 }
 584
 585 /**
 586  * igc_configure_tx_ring - Configure transmit ring after Reset
 587  * @adapter: board private structure
 588  * @ring: tx ring to configure
 589  *
 590  * Configure a transmit ring after a reset.
 591  */
 592 static void igc_configure_tx_ring(struct igc_adapter *adapter,
 593                                   struct igc_ring *ring)
 594 {
 595         struct igc_hw *hw = &adapter->hw;
 596         int reg_idx = ring->reg_idx;
 597         u64 tdba = ring->dma;
 598         u32 txdctl = 0;
 599
 600         /* disable the queue */
 601         wr32(IGC_TXDCTL(reg_idx), 0);
 602         wrfl();
 603
 604         wr32(IGC_TDLEN(reg_idx),
 605              ring->count * sizeof(union igc_adv_tx_desc));
 606         wr32(IGC_TDBAL(reg_idx),
 607              tdba & 0x00000000ffffffffULL);
 608         wr32(IGC_TDBAH(reg_idx), tdba >> 32);
 609
 610         ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
 611         wr32(IGC_TDH(reg_idx), 0);
 612         writel(0, ring->tail);
 613
 614         txdctl |= IGC_TX_PTHRESH;
 615         txdctl |= IGC_TX_HTHRESH << 8;
 616         txdctl |= IGC_TX_WTHRESH << 16;
 617
 618         txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
 619         wr32(IGC_TXDCTL(reg_idx), txdctl);
 620 }
 621
 622 /**
 623  * igc_configure_tx - Configure transmit Unit after Reset
 624  * @adapter: board private structure
 625  *
 626  * Configure the Tx unit of the MAC after a reset.
 627  */
 628 static void igc_configure_tx(struct igc_adapter *adapter)
 629 {
 630         int i;
 631
 632         for (i = 0; i < adapter->num_tx_queues; i++)
 633                 igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
 634 }
 635
 636 /**
 637  * igc_setup_mrqc - configure the multiple receive queue control registers
 638  * @adapter: Board private structure
 639  */
 640 static void igc_setup_mrqc(struct igc_adapter *adapter)
 641 {
 642         struct igc_hw *hw = &adapter->hw;
 643         u32 j, num_rx_queues;
 644         u32 mrqc, rxcsum;
 645         u32 rss_key[10];
 646
 647         netdev_rss_key_fill(rss_key, sizeof(rss_key));
 648         for (j = 0; j < 10; j++)
 649                 wr32(IGC_RSSRK(j), rss_key[j]);
 650
 651         num_rx_queues = adapter->rss_queues;
 652
 653         if (adapter->rss_indir_tbl_init != num_rx_queues) {
 654                 for (j = 0; j < IGC_RETA_SIZE; j++)
 655                         adapter->rss_indir_tbl[j] =
 656                         (j * num_rx_queues) / IGC_RETA_SIZE;
 657                 adapter->rss_indir_tbl_init = num_rx_queues;
 658         }
 659         igc_write_rss_indir_tbl(adapter);
 660
 661         /* Disable raw packet checksumming so that RSS hash is placed in
 662          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
 663          * offloads as they are enabled by default
 664          */
 665         rxcsum = rd32(IGC_RXCSUM);
 666         rxcsum |= IGC_RXCSUM_PCSD;
 667
 668         /* Enable Receive Checksum Offload for SCTP */
 669         rxcsum |= IGC_RXCSUM_CRCOFL;
 670
 671         /* Don't need to set TUOFL or IPOFL, they default to 1 */
 672         wr32(IGC_RXCSUM, rxcsum);
 673
 674         /* Generate RSS hash based on packet types, TCP/UDP
 675          * port numbers and/or IPv4/v6 src and dst addresses
 676          */
 677         mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
 678                IGC_MRQC_RSS_FIELD_IPV4_TCP |
 679                IGC_MRQC_RSS_FIELD_IPV6 |
 680                IGC_MRQC_RSS_FIELD_IPV6_TCP |
 681                IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
 682
 683         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
 684                 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
 685         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
 686                 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
 687
 688         mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
 689
 690         wr32(IGC_MRQC, mrqc);
 691 }
 692
 693 /**
 694  * igc_setup_rctl - configure the receive control registers
 695  * @adapter: Board private structure
 696  */
 697 static void igc_setup_rctl(struct igc_adapter *adapter)
 698 {
 699         struct igc_hw *hw = &adapter->hw;
 700         u32 rctl;
 701
 702         rctl = rd32(IGC_RCTL);
 703
 704         rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
 705         rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
 706
 707         rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
 708                 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
 709
 710         /* enable stripping of CRC. Newer features require
 711          * that the HW strips the CRC.
 712          */
 713         rctl |= IGC_RCTL_SECRC;
 714
 715         /* disable store bad packets and clear size bits. */
 716         rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
 717
 718         /* enable LPE to allow for reception of jumbo frames */
 719         rctl |= IGC_RCTL_LPE;
 720
 721         /* disable queue 0 to prevent tail write w/o re-config */
 722         wr32(IGC_RXDCTL(0), 0);
 723
 724         /* This is useful for sniffing bad packets. */
 725         if (adapter->netdev->features & NETIF_F_RXALL) {
 726                 /* UPE and MPE will be handled by normal PROMISC logic
 727                  * in set_rx_mode
 728                  */
 729                 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
 730                          IGC_RCTL_BAM | /* RX All Bcast Pkts */
 731                          IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
 732
 733                 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
 734                           IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
 735         }
 736
 737         wr32(IGC_RCTL, rctl);
 738 }
 739
 740 /**
 741  * igc_setup_tctl - configure the transmit control registers
 742  * @adapter: Board private structure
 743  */
 744 static void igc_setup_tctl(struct igc_adapter *adapter)
 745 {
 746         struct igc_hw *hw = &adapter->hw;
 747         u32 tctl;
 748
 749         /* disable queue 0 which icould be enabled by default */
 750         wr32(IGC_TXDCTL(0), 0);
 751
 752         /* Program the Transmit Control Register */
 753         tctl = rd32(IGC_TCTL);
 754         tctl &= ~IGC_TCTL_CT;
 755         tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
 756                 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
 757
 758         /* Enable transmits */
 759         tctl |= IGC_TCTL_EN;
 760
 761         wr32(IGC_TCTL, tctl);
 762 }
 763
 764 /**
 765  * igc_set_mac_filter_hw() - Set MAC address filter in hardware
 766  * @adapter: Pointer to adapter where the filter should be set
 767  * @index: Filter index
 768  * @type: MAC address filter type (source or destination)
 769  * @addr: MAC address
 770  * @queue: If non-negative, queue assignment feature is enabled and frames
 771  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
 772  *         assignment is disabled.
 773  */
 774 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
 775                                   enum igc_mac_filter_type type,
 776                                   const u8 *addr, int queue)
 777 {
 778         struct net_device *dev = adapter->netdev;
 779         struct igc_hw *hw = &adapter->hw;
 780         u32 ral, rah;
 781
 782         if (WARN_ON(index >= hw->mac.rar_entry_count))
 783                 return;
 784
 785         ral = le32_to_cpup((__le32 *)(addr));
 786         rah = le16_to_cpup((__le16 *)(addr + 4));
 787
 788         if (type == IGC_MAC_FILTER_TYPE_SRC) {
 789                 rah &= ~IGC_RAH_ASEL_MASK;
 790                 rah |= IGC_RAH_ASEL_SRC_ADDR;
 791         }
 792
 793         if (queue >= 0) {
 794                 rah &= ~IGC_RAH_QSEL_MASK;
 795                 rah |= (queue << IGC_RAH_QSEL_SHIFT);
 796                 rah |= IGC_RAH_QSEL_ENABLE;
 797         }
 798
 799         rah |= IGC_RAH_AV;
 800
 801         wr32(IGC_RAL(index), ral);
 802         wr32(IGC_RAH(index), rah);
 803
 804         netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
 805 }
 806
 807 /**
 808  * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
 809  * @adapter: Pointer to adapter where the filter should be cleared
 810  * @index: Filter index
 811  */
 812 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
 813 {
 814         struct net_device *dev = adapter->netdev;
 815         struct igc_hw *hw = &adapter->hw;
 816
 817         if (WARN_ON(index >= hw->mac.rar_entry_count))
 818                 return;
 819
 820         wr32(IGC_RAL(index), 0);
 821         wr32(IGC_RAH(index), 0);
 822
 823         netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
 824 }
 825
 826 /* Set default MAC address for the PF in the first RAR entry */
 827 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
 828 {
 829         struct net_device *dev = adapter->netdev;
 830         u8 *addr = adapter->hw.mac.addr;
 831
 832         netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
 833
 834         igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
 835 }
 836
 837 /**
 838  * igc_set_mac - Change the Ethernet Address of the NIC
 839  * @netdev: network interface device structure
 840  * @p: pointer to an address structure
 841  *
 842  * Returns 0 on success, negative on failure
 843  */
 844 static int igc_set_mac(struct net_device *netdev, void *p)
 845 {
 846         struct igc_adapter *adapter = netdev_priv(netdev);
 847         struct igc_hw *hw = &adapter->hw;
 848         struct sockaddr *addr = p;
 849
 850         if (!is_valid_ether_addr(addr->sa_data))
 851                 return -EADDRNOTAVAIL;
 852
 853         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 854         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 855
 856         /* set the correct pool for the new PF MAC address in entry 0 */
 857         igc_set_default_mac_filter(adapter);
 858
 859         return 0;
 860 }
 861
 862 /**
 863  *  igc_write_mc_addr_list - write multicast addresses to MTA
 864  *  @netdev: network interface device structure
 865  *
 866  *  Writes multicast address list to the MTA hash table.
 867  *  Returns: -ENOMEM on failure
 868  *           0 on no addresses written
 869  *           X on writing X addresses to MTA
 870  **/
 871 static int igc_write_mc_addr_list(struct net_device *netdev)
 872 {
 873         struct igc_adapter *adapter = netdev_priv(netdev);
 874         struct igc_hw *hw = &adapter->hw;
 875         struct netdev_hw_addr *ha;
 876         u8  *mta_list;
 877         int i;
 878
 879         if (netdev_mc_empty(netdev)) {
 880                 /* nothing to program, so clear mc list */
 881                 igc_update_mc_addr_list(hw, NULL, 0);
 882                 return 0;
 883         }
 884
 885         mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
 886         if (!mta_list)
 887                 return -ENOMEM;
 888
 889         /* The shared function expects a packed array of only addresses. */
 890         i = 0;
 891         netdev_for_each_mc_addr(ha, netdev)
 892                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
 893
 894         igc_update_mc_addr_list(hw, mta_list, i);
 895         kfree(mta_list);
 896
 897         return netdev_mc_count(netdev);
 898 }
 899
 900 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
 901                                 bool *first_flag, bool *insert_empty)
 902 {
 903         struct igc_adapter *adapter = netdev_priv(ring->netdev);
 904         ktime_t cycle_time = adapter->cycle_time;
 905         ktime_t base_time = adapter->base_time;
 906         ktime_t now = ktime_get_clocktai();
 907         ktime_t baset_est, end_of_cycle;
 908         s32 launchtime;
 909         s64 n;
 910
 911         n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
 912
 913         baset_est = ktime_add_ns(base_time, cycle_time * (n));
 914         end_of_cycle = ktime_add_ns(baset_est, cycle_time);
 915
 916         if (ktime_compare(txtime, end_of_cycle) >= 0) {
 917                 if (baset_est != ring->last_ff_cycle) {
 918                         *first_flag = true;
 919                         ring->last_ff_cycle = baset_est;
 920
 921                         if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0)
 922                                 *insert_empty = true;
 923                 }
 924         }
 925
 926         /* Introducing a window at end of cycle on which packets
 927          * potentially not honor launchtime. Window of 5us chosen
 928          * considering software update the tail pointer and packets
 929          * are dma'ed to packet buffer.
 930          */
 931         if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
 932                 netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
 933                             txtime);
 934
 935         ring->last_tx_cycle = end_of_cycle;
 936
 937         launchtime = ktime_sub_ns(txtime, baset_est);
 938         if (launchtime > 0)
 939                 div_s64_rem(launchtime, cycle_time, &launchtime);
 940         else
 941                 launchtime = 0;
 942
 943         return cpu_to_le32(launchtime);
 944 }
 945
 946 static int igc_init_empty_frame(struct igc_ring *ring,
 947                                 struct igc_tx_buffer *buffer,
 948                                 struct sk_buff *skb)
 949 {
 950         unsigned int size;
 951         dma_addr_t dma;
 952
 953         size = skb_headlen(skb);
 954
 955         dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
 956         if (dma_mapping_error(ring->dev, dma)) {
 957                 netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
 958                 return -ENOMEM;
 959         }
 960
 961         buffer->skb = skb;
 962         buffer->protocol = 0;
 963         buffer->bytecount = skb->len;
 964         buffer->gso_segs = 1;
 965         buffer->time_stamp = jiffies;
 966         dma_unmap_len_set(buffer, len, skb->len);
 967         dma_unmap_addr_set(buffer, dma, dma);
 968
 969         return 0;
 970 }
 971
 972 static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
 973                                         struct sk_buff *skb,
 974                                         struct igc_tx_buffer *first)
 975 {
 976         union igc_adv_tx_desc *desc;
 977         u32 cmd_type, olinfo_status;
 978         int err;
 979
 980         if (!igc_desc_unused(ring))
 981                 return -EBUSY;
 982
 983         err = igc_init_empty_frame(ring, first, skb);
 984         if (err)
 985                 return err;
 986
 987         cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
 988                    IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
 989                    first->bytecount;
 990         olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
 991
 992         desc = IGC_TX_DESC(ring, ring->next_to_use);
 993         desc->read.cmd_type_len = cpu_to_le32(cmd_type);
 994         desc->read.olinfo_status = cpu_to_le32(olinfo_status);
 995         desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
 996
 997         netdev_tx_sent_queue(txring_txq(ring), skb->len);
 998
 999         first->next_to_watch = desc;
1000
1001         ring->next_to_use++;
1002         if (ring->next_to_use == ring->count)
1003                 ring->next_to_use = 0;
1004
1005         return 0;
1006 }
1007
1008 #define IGC_EMPTY_FRAME_SIZE 60
1009
1010 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1011                             __le32 launch_time, bool first_flag,
1012                             u32 vlan_macip_lens, u32 type_tucmd,
1013                             u32 mss_l4len_idx)
1014 {
1015         struct igc_adv_tx_context_desc *context_desc;
1016         u16 i = tx_ring->next_to_use;
1017
1018         context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1019
1020         i++;
1021         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1022
1023         /* set bits to identify this as an advanced context descriptor */
1024         type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1025
1026         /* For i225, context index must be unique per ring. */
1027         if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1028                 mss_l4len_idx |= tx_ring->reg_idx << 4;
1029
1030         if (first_flag)
1031                 mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1032
1033         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
1034         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
1035         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
1036         context_desc->launch_time       = launch_time;
1037 }
1038
1039 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1040                         __le32 launch_time, bool first_flag)
1041 {
1042         struct sk_buff *skb = first->skb;
1043         u32 vlan_macip_lens = 0;
1044         u32 type_tucmd = 0;
1045
1046         if (skb->ip_summed != CHECKSUM_PARTIAL) {
1047 csum_failed:
1048                 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1049                     !tx_ring->launchtime_enable)
1050                         return;
1051                 goto no_csum;
1052         }
1053
1054         switch (skb->csum_offset) {
1055         case offsetof(struct tcphdr, check):
1056                 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1057                 fallthrough;
1058         case offsetof(struct udphdr, check):
1059                 break;
1060         case offsetof(struct sctphdr, checksum):
1061                 /* validate that this is actually an SCTP request */
1062                 if (skb_csum_is_sctp(skb)) {
1063                         type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1064                         break;
1065                 }
1066                 fallthrough;
1067         default:
1068                 skb_checksum_help(skb);
1069                 goto csum_failed;
1070         }
1071
1072         /* update TX checksum flag */
1073         first->tx_flags |= IGC_TX_FLAGS_CSUM;
1074         vlan_macip_lens = skb_checksum_start_offset(skb) -
1075                           skb_network_offset(skb);
1076 no_csum:
1077         vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1078         vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1079
1080         igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1081                         vlan_macip_lens, type_tucmd, 0);
1082 }
1083
1084 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1085 {
1086         struct net_device *netdev = tx_ring->netdev;
1087
1088         netif_stop_subqueue(netdev, tx_ring->queue_index);
1089
1090         /* memory barriier comment */
1091         smp_mb();
1092
1093         /* We need to check again in a case another CPU has just
1094          * made room available.
1095          */
1096         if (igc_desc_unused(tx_ring) < size)
1097                 return -EBUSY;
1098
1099         /* A reprieve! */
1100         netif_wake_subqueue(netdev, tx_ring->queue_index);
1101
1102         u64_stats_update_begin(&tx_ring->tx_syncp2);
1103         tx_ring->tx_stats.restart_queue2++;
1104         u64_stats_update_end(&tx_ring->tx_syncp2);
1105
1106         return 0;
1107 }
1108
1109 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1110 {
1111         if (igc_desc_unused(tx_ring) >= size)
1112                 return 0;
1113         return __igc_maybe_stop_tx(tx_ring, size);
1114 }
1115
1116 #define IGC_SET_FLAG(_input, _flag, _result) \
1117         (((_flag) <= (_result)) ?                               \
1118          ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :  \
1119          ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1120
1121 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1122 {
1123         /* set type for advanced descriptor with frame checksum insertion */
1124         u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1125                        IGC_ADVTXD_DCMD_DEXT |
1126                        IGC_ADVTXD_DCMD_IFCS;
1127
1128         /* set segmentation bits for TSO */
1129         cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1130                                  (IGC_ADVTXD_DCMD_TSE));
1131
1132         /* set timestamp bit if present */
1133         cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1134                                  (IGC_ADVTXD_MAC_TSTAMP));
1135
1136         return cmd_type;
1137 }
1138
1139 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1140                                  union igc_adv_tx_desc *tx_desc,
1141                                  u32 tx_flags, unsigned int paylen)
1142 {
1143         u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1144
1145         /* insert L4 checksum */
1146         olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
1147                           ((IGC_TXD_POPTS_TXSM << 8) /
1148                           IGC_TX_FLAGS_CSUM);
1149
1150         /* insert IPv4 checksum */
1151         olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
1152                           (((IGC_TXD_POPTS_IXSM << 8)) /
1153                           IGC_TX_FLAGS_IPV4);
1154
1155         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1156 }
1157
1158 static int igc_tx_map(struct igc_ring *tx_ring,
1159                       struct igc_tx_buffer *first,
1160                       const u8 hdr_len)
1161 {
1162         struct sk_buff *skb = first->skb;
1163         struct igc_tx_buffer *tx_buffer;
1164         union igc_adv_tx_desc *tx_desc;
1165         u32 tx_flags = first->tx_flags;
1166         skb_frag_t *frag;
1167         u16 i = tx_ring->next_to_use;
1168         unsigned int data_len, size;
1169         dma_addr_t dma;
1170         u32 cmd_type = igc_tx_cmd_type(skb, tx_flags);
1171
1172         tx_desc = IGC_TX_DESC(tx_ring, i);
1173
1174         igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1175
1176         size = skb_headlen(skb);
1177         data_len = skb->data_len;
1178
1179         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1180
1181         tx_buffer = first;
1182
1183         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1184                 if (dma_mapping_error(tx_ring->dev, dma))
1185                         goto dma_error;
1186
1187                 /* record length, and DMA address */
1188                 dma_unmap_len_set(tx_buffer, len, size);
1189                 dma_unmap_addr_set(tx_buffer, dma, dma);
1190
1191                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1192
1193                 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1194                         tx_desc->read.cmd_type_len =
1195                                 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1196
1197                         i++;
1198                         tx_desc++;
1199                         if (i == tx_ring->count) {
1200                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
1201                                 i = 0;
1202                         }
1203                         tx_desc->read.olinfo_status = 0;
1204
1205                         dma += IGC_MAX_DATA_PER_TXD;
1206                         size -= IGC_MAX_DATA_PER_TXD;
1207
1208                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
1209                 }
1210
1211                 if (likely(!data_len))
1212                         break;
1213
1214                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1215
1216                 i++;
1217                 tx_desc++;
1218                 if (i == tx_ring->count) {
1219                         tx_desc = IGC_TX_DESC(tx_ring, 0);
1220                         i = 0;
1221                 }
1222                 tx_desc->read.olinfo_status = 0;
1223
1224                 size = skb_frag_size(frag);
1225                 data_len -= size;
1226
1227                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1228                                        size, DMA_TO_DEVICE);
1229
1230                 tx_buffer = &tx_ring->tx_buffer_info[i];
1231         }
1232
1233         /* write last descriptor with RS and EOP bits */
1234         cmd_type |= size | IGC_TXD_DCMD;
1235         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1236
1237         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1238
1239         /* set the timestamp */
1240         first->time_stamp = jiffies;
1241
1242         skb_tx_timestamp(skb);
1243
1244         /* Force memory writes to complete before letting h/w know there
1245          * are new descriptors to fetch.  (Only applicable for weak-ordered
1246          * memory model archs, such as IA-64).
1247          *
1248          * We also need this memory barrier to make certain all of the
1249          * status bits have been updated before next_to_watch is written.
1250          */
1251         wmb();
1252
1253         /* set next_to_watch value indicating a packet is present */
1254         first->next_to_watch = tx_desc;
1255
1256         i++;
1257         if (i == tx_ring->count)
1258                 i = 0;
1259
1260         tx_ring->next_to_use = i;
1261
1262         /* Make sure there is space in the ring for the next send. */
1263         igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1264
1265         if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1266                 writel(i, tx_ring->tail);
1267         }
1268
1269         return 0;
1270 dma_error:
1271         netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1272         tx_buffer = &tx_ring->tx_buffer_info[i];
1273
1274         /* clear dma mappings for failed tx_buffer_info map */
1275         while (tx_buffer != first) {
1276                 if (dma_unmap_len(tx_buffer, len))
1277                         dma_unmap_page(tx_ring->dev,
1278                                        dma_unmap_addr(tx_buffer, dma),
1279                                        dma_unmap_len(tx_buffer, len),
1280                                        DMA_TO_DEVICE);
1281                 dma_unmap_len_set(tx_buffer, len, 0);
1282
1283                 if (i-- == 0)
1284                         i += tx_ring->count;
1285                 tx_buffer = &tx_ring->tx_buffer_info[i];
1286         }
1287
1288         if (dma_unmap_len(tx_buffer, len))
1289                 dma_unmap_single(tx_ring->dev,
1290                                  dma_unmap_addr(tx_buffer, dma),
1291                                  dma_unmap_len(tx_buffer, len),
1292                                  DMA_TO_DEVICE);
1293         dma_unmap_len_set(tx_buffer, len, 0);
1294
1295         dev_kfree_skb_any(tx_buffer->skb);
1296         tx_buffer->skb = NULL;
1297
1298         tx_ring->next_to_use = i;
1299
1300         return -1;
1301 }
1302
1303 static int igc_tso(struct igc_ring *tx_ring,
1304                    struct igc_tx_buffer *first,
1305                    __le32 launch_time, bool first_flag,
1306                    u8 *hdr_len)
1307 {
1308         u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1309         struct sk_buff *skb = first->skb;
1310         union {
1311                 struct iphdr *v4;
1312                 struct ipv6hdr *v6;
1313                 unsigned char *hdr;
1314         } ip;
1315         union {
1316                 struct tcphdr *tcp;
1317                 struct udphdr *udp;
1318                 unsigned char *hdr;
1319         } l4;
1320         u32 paylen, l4_offset;
1321         int err;
1322
1323         if (skb->ip_summed != CHECKSUM_PARTIAL)
1324                 return 0;
1325
1326         if (!skb_is_gso(skb))
1327                 return 0;
1328
1329         err = skb_cow_head(skb, 0);
1330         if (err < 0)
1331                 return err;
1332
1333         ip.hdr = skb_network_header(skb);
1334         l4.hdr = skb_checksum_start(skb);
1335
1336         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1337         type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1338
1339         /* initialize outer IP header fields */
1340         if (ip.v4->version == 4) {
1341                 unsigned char *csum_start = skb_checksum_start(skb);
1342                 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1343
1344                 /* IP header will have to cancel out any data that
1345                  * is not a part of the outer IP header
1346                  */
1347                 ip.v4->check = csum_fold(csum_partial(trans_start,
1348                                                       csum_start - trans_start,
1349                                                       0));
1350                 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1351
1352                 ip.v4->tot_len = 0;
1353                 first->tx_flags |= IGC_TX_FLAGS_TSO |
1354                                    IGC_TX_FLAGS_CSUM |
1355                                    IGC_TX_FLAGS_IPV4;
1356         } else {
1357                 ip.v6->payload_len = 0;
1358                 first->tx_flags |= IGC_TX_FLAGS_TSO |
1359                                    IGC_TX_FLAGS_CSUM;
1360         }
1361
1362         /* determine offset of inner transport header */
1363         l4_offset = l4.hdr - skb->data;
1364
1365         /* remove payload length from inner checksum */
1366         paylen = skb->len - l4_offset;
1367         if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1368                 /* compute length of segmentation header */
1369                 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1370                 csum_replace_by_diff(&l4.tcp->check,
1371                                      (__force __wsum)htonl(paylen));
1372         } else {
1373                 /* compute length of segmentation header */
1374                 *hdr_len = sizeof(*l4.udp) + l4_offset;
1375                 csum_replace_by_diff(&l4.udp->check,
1376                                      (__force __wsum)htonl(paylen));
1377         }
1378
1379         /* update gso size and bytecount with header size */
1380         first->gso_segs = skb_shinfo(skb)->gso_segs;
1381         first->bytecount += (first->gso_segs - 1) * *hdr_len;
1382
1383         /* MSS L4LEN IDX */
1384         mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1385         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1386
1387         /* VLAN MACLEN IPLEN */
1388         vlan_macip_lens = l4.hdr - ip.hdr;
1389         vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1390         vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1391
1392         igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1393                         vlan_macip_lens, type_tucmd, mss_l4len_idx);
1394
1395         return 1;
1396 }
1397
1398 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1399                                        struct igc_ring *tx_ring)
1400 {
1401         bool first_flag = false, insert_empty = false;
1402         u16 count = TXD_USE_COUNT(skb_headlen(skb));
1403         __be16 protocol = vlan_get_protocol(skb);
1404         struct igc_tx_buffer *first;
1405         __le32 launch_time = 0;
1406         u32 tx_flags = 0;
1407         unsigned short f;
1408         ktime_t txtime;
1409         u8 hdr_len = 0;
1410         int tso = 0;
1411
1412         /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1413          *      + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1414          *      + 2 desc gap to keep tail from touching head,
1415          *      + 1 desc for context descriptor,
1416          * otherwise try next time
1417          */
1418         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1419                 count += TXD_USE_COUNT(skb_frag_size(
1420                                                 &skb_shinfo(skb)->frags[f]));
1421
1422         if (igc_maybe_stop_tx(tx_ring, count + 5)) {
1423                 /* this is a hard error */
1424                 return NETDEV_TX_BUSY;
1425         }
1426
1427         if (!tx_ring->launchtime_enable)
1428                 goto done;
1429
1430         txtime = skb->tstamp;
1431         skb->tstamp = ktime_set(0, 0);
1432         launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1433
1434         if (insert_empty) {
1435                 struct igc_tx_buffer *empty_info;
1436                 struct sk_buff *empty;
1437                 void *data;
1438
1439                 empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1440                 empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1441                 if (!empty)
1442                         goto done;
1443
1444                 data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1445                 memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1446
1447                 igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1448
1449                 if (igc_init_tx_empty_descriptor(tx_ring,
1450                                                  empty,
1451                                                  empty_info) < 0)
1452                         dev_kfree_skb_any(empty);
1453         }
1454
1455 done:
1456         /* record the location of the first descriptor for this packet */
1457         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1458         first->skb = skb;
1459         first->bytecount = skb->len;
1460         first->gso_segs = 1;
1461
1462         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1463                 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1464
1465                 /* FIXME: add support for retrieving timestamps from
1466                  * the other timer registers before skipping the
1467                  * timestamping request.
1468                  */
1469                 unsigned long flags;
1470
1471                 spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
1472                 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && !adapter->ptp_tx_skb) {
1473                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1474                         tx_flags |= IGC_TX_FLAGS_TSTAMP;
1475
1476                         adapter->ptp_tx_skb = skb_get(skb);
1477                         adapter->ptp_tx_start = jiffies;
1478                 } else {
1479                         adapter->tx_hwtstamp_skipped++;
1480                 }
1481
1482                 spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
1483         }
1484
1485         /* record initial flags and protocol */
1486         first->tx_flags = tx_flags;
1487         first->protocol = protocol;
1488
1489         tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1490         if (tso < 0)
1491                 goto out_drop;
1492         else if (!tso)
1493                 igc_tx_csum(tx_ring, first, launch_time, first_flag);
1494
1495         igc_tx_map(tx_ring, first, hdr_len);
1496
1497         return NETDEV_TX_OK;
1498
1499 out_drop:
1500         dev_kfree_skb_any(first->skb);
1501         first->skb = NULL;
1502
1503         return NETDEV_TX_OK;
1504 }
1505
1506 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1507                                                     struct sk_buff *skb)
1508 {
1509         unsigned int r_idx = skb->queue_mapping;
1510
1511         if (r_idx >= adapter->num_tx_queues)
1512                 r_idx = r_idx % adapter->num_tx_queues;
1513
1514         return adapter->tx_ring[r_idx];
1515 }
1516
1517 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1518                                   struct net_device *netdev)
1519 {
1520         struct igc_adapter *adapter = netdev_priv(netdev);
1521
1522         /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1523          * in order to meet this minimum size requirement.
1524          */
1525         if (skb->len < 17) {
1526                 if (skb_padto(skb, 17))
1527                         return NETDEV_TX_OK;
1528                 skb->len = 17;
1529         }
1530
1531         return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1532 }
1533
1534 static void igc_rx_checksum(struct igc_ring *ring,
1535                             union igc_adv_rx_desc *rx_desc,
1536                             struct sk_buff *skb)
1537 {
1538         skb_checksum_none_assert(skb);
1539
1540         /* Ignore Checksum bit is set */
1541         if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1542                 return;
1543
1544         /* Rx checksum disabled via ethtool */
1545         if (!(ring->netdev->features & NETIF_F_RXCSUM))
1546                 return;
1547
1548         /* TCP/UDP checksum error bit is set */
1549         if (igc_test_staterr(rx_desc,
1550                              IGC_RXDEXT_STATERR_L4E |
1551                              IGC_RXDEXT_STATERR_IPE)) {
1552                 /* work around errata with sctp packets where the TCPE aka
1553                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1554                  * packets (aka let the stack check the crc32c)
1555                  */
1556                 if (!(skb->len == 60 &&
1557                       test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1558                         u64_stats_update_begin(&ring->rx_syncp);
1559                         ring->rx_stats.csum_err++;
1560                         u64_stats_update_end(&ring->rx_syncp);
1561                 }
1562                 /* let the stack verify checksum errors */
1563                 return;
1564         }
1565         /* It must be a TCP or UDP packet with a valid checksum */
1566         if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1567                                       IGC_RXD_STAT_UDPCS))
1568                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1569
1570         netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1571                    le32_to_cpu(rx_desc->wb.upper.status_error));
1572 }
1573
1574 /* Mapping HW RSS Type to enum pkt_hash_types */
1575 static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = {
1576         [IGC_RSS_TYPE_NO_HASH]          = PKT_HASH_TYPE_L2,
1577         [IGC_RSS_TYPE_HASH_TCP_IPV4]    = PKT_HASH_TYPE_L4,
1578         [IGC_RSS_TYPE_HASH_IPV4]        = PKT_HASH_TYPE_L3,
1579         [IGC_RSS_TYPE_HASH_TCP_IPV6]    = PKT_HASH_TYPE_L4,
1580         [IGC_RSS_TYPE_HASH_IPV6_EX]     = PKT_HASH_TYPE_L3,
1581         [IGC_RSS_TYPE_HASH_IPV6]        = PKT_HASH_TYPE_L3,
1582         [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4,
1583         [IGC_RSS_TYPE_HASH_UDP_IPV4]    = PKT_HASH_TYPE_L4,
1584         [IGC_RSS_TYPE_HASH_UDP_IPV6]    = PKT_HASH_TYPE_L4,
1585         [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4,
1586         [10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW  */
1587         [11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask   */
1588         [12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisons       */
1589         [13] = PKT_HASH_TYPE_NONE,
1590         [14] = PKT_HASH_TYPE_NONE,
1591         [15] = PKT_HASH_TYPE_NONE,
1592 };
1593
1594 static inline void igc_rx_hash(struct igc_ring *ring,
1595                                union igc_adv_rx_desc *rx_desc,
1596                                struct sk_buff *skb)
1597 {
1598         if (ring->netdev->features & NETIF_F_RXHASH) {
1599                 u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1600                 u32 rss_type = igc_rss_type(rx_desc);
1601
1602                 skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]);
1603         }
1604 }
1605
1606 /**
1607  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1608  * @rx_ring: rx descriptor ring packet is being transacted on
1609  * @rx_desc: pointer to the EOP Rx descriptor
1610  * @skb: pointer to current skb being populated
1611  *
1612  * This function checks the ring, descriptor, and packet information in order
1613  * to populate the hash, checksum, VLAN, protocol, and other fields within the
1614  * skb.
1615  */
1616 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1617                                    union igc_adv_rx_desc *rx_desc,
1618                                    struct sk_buff *skb)
1619 {
1620         igc_rx_hash(rx_ring, rx_desc, skb);
1621
1622         igc_rx_checksum(rx_ring, rx_desc, skb);
1623
1624         skb_record_rx_queue(skb, rx_ring->queue_index);
1625
1626         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1627 }
1628
1629 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1630                                                const unsigned int size)
1631 {
1632         struct igc_rx_buffer *rx_buffer;
1633
1634         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1635         prefetchw(rx_buffer->page);
1636
1637         /* we are reusing so sync this buffer for CPU use */
1638         dma_sync_single_range_for_cpu(rx_ring->dev,
1639                                       rx_buffer->dma,
1640                                       rx_buffer->page_offset,
1641                                       size,
1642                                       DMA_FROM_DEVICE);
1643
1644         rx_buffer->pagecnt_bias--;
1645
1646         return rx_buffer;
1647 }
1648
1649 /**
1650  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1651  * @rx_ring: rx descriptor ring to transact packets on
1652  * @rx_buffer: buffer containing page to add
1653  * @skb: sk_buff to place the data into
1654  * @size: size of buffer to be added
1655  *
1656  * This function will add the data contained in rx_buffer->page to the skb.
1657  */
1658 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1659                             struct igc_rx_buffer *rx_buffer,
1660                             struct sk_buff *skb,
1661                             unsigned int size)
1662 {
1663 #if (PAGE_SIZE < 8192)
1664         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1665
1666         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1667                         rx_buffer->page_offset, size, truesize);
1668         rx_buffer->page_offset ^= truesize;
1669 #else
1670         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
1671                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1672                                 SKB_DATA_ALIGN(size);
1673         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1674                         rx_buffer->page_offset, size, truesize);
1675         rx_buffer->page_offset += truesize;
1676 #endif
1677 }
1678
1679 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1680                                      struct igc_rx_buffer *rx_buffer,
1681                                      union igc_adv_rx_desc *rx_desc,
1682                                      unsigned int size)
1683 {
1684         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1685 #if (PAGE_SIZE < 8192)
1686         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1687 #else
1688         unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1689                                 SKB_DATA_ALIGN(IGC_SKB_PAD + size);
1690 #endif
1691         struct sk_buff *skb;
1692
1693         /* prefetch first cache line of first page */
1694         net_prefetch(va);
1695
1696         /* build an skb around the page buffer */
1697         skb = build_skb(va - IGC_SKB_PAD, truesize);
1698         if (unlikely(!skb))
1699                 return NULL;
1700
1701         /* update pointers within the skb to store the data */
1702         skb_reserve(skb, IGC_SKB_PAD);
1703         __skb_put(skb, size);
1704
1705         /* update buffer offset */
1706 #if (PAGE_SIZE < 8192)
1707         rx_buffer->page_offset ^= truesize;
1708 #else
1709         rx_buffer->page_offset += truesize;
1710 #endif
1711
1712         return skb;
1713 }
1714
1715 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1716                                          struct igc_rx_buffer *rx_buffer,
1717                                          union igc_adv_rx_desc *rx_desc,
1718                                          unsigned int size)
1719 {
1720         void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1721 #if (PAGE_SIZE < 8192)
1722         unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
1723 #else
1724         unsigned int truesize = SKB_DATA_ALIGN(size);
1725 #endif
1726         unsigned int headlen;
1727         struct sk_buff *skb;
1728
1729         /* prefetch first cache line of first page */
1730         net_prefetch(va);
1731
1732         /* allocate a skb to store the frags */
1733         skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);
1734         if (unlikely(!skb))
1735                 return NULL;
1736
1737         if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) {
1738                 igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
1739                 va += IGC_TS_HDR_LEN;
1740                 size -= IGC_TS_HDR_LEN;
1741         }
1742
1743         /* Determine available headroom for copy */
1744         headlen = size;
1745         if (headlen > IGC_RX_HDR_LEN)
1746                 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1747
1748         /* align pull length to size of long to optimize memcpy performance */
1749         memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1750
1751         /* update all of the pointers */
1752         size -= headlen;
1753         if (size) {
1754                 skb_add_rx_frag(skb, 0, rx_buffer->page,
1755                                 (va + headlen) - page_address(rx_buffer->page),
1756                                 size, truesize);
1757 #if (PAGE_SIZE < 8192)
1758                 rx_buffer->page_offset ^= truesize;
1759 #else
1760                 rx_buffer->page_offset += truesize;
1761 #endif
1762         } else {
1763                 rx_buffer->pagecnt_bias++;
1764         }
1765
1766         return skb;
1767 }
1768
1769 /**
1770  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1771  * @rx_ring: rx descriptor ring to store buffers on
1772  * @old_buff: donor buffer to have page reused
1773  *
1774  * Synchronizes page for reuse by the adapter
1775  */
1776 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1777                               struct igc_rx_buffer *old_buff)
1778 {
1779         u16 nta = rx_ring->next_to_alloc;
1780         struct igc_rx_buffer *new_buff;
1781
1782         new_buff = &rx_ring->rx_buffer_info[nta];
1783
1784         /* update, and store next to alloc */
1785         nta++;
1786         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1787
1788         /* Transfer page from old buffer to new buffer.
1789          * Move each member individually to avoid possible store
1790          * forwarding stalls.
1791          */
1792         new_buff->dma           = old_buff->dma;
1793         new_buff->page          = old_buff->page;
1794         new_buff->page_offset   = old_buff->page_offset;
1795         new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
1796 }
1797
1798 static inline bool igc_page_is_reserved(struct page *page)
1799 {
1800         return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
1801 }
1802
1803 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer)
1804 {
1805         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1806         struct page *page = rx_buffer->page;
1807
1808         /* avoid re-using remote pages */
1809         if (unlikely(igc_page_is_reserved(page)))
1810                 return false;
1811
1812 #if (PAGE_SIZE < 8192)
1813         /* if we are only owner of page we can reuse it */
1814         if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
1815                 return false;
1816 #else
1817 #define IGC_LAST_OFFSET \
1818         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1819
1820         if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1821                 return false;
1822 #endif
1823
1824         /* If we have drained the page fragment pool we need to update
1825          * the pagecnt_bias and page count so that we fully restock the
1826          * number of references the driver holds.
1827          */
1828         if (unlikely(!pagecnt_bias)) {
1829                 page_ref_add(page, USHRT_MAX);
1830                 rx_buffer->pagecnt_bias = USHRT_MAX;
1831         }
1832
1833         return true;
1834 }
1835
1836 /**
1837  * igc_is_non_eop - process handling of non-EOP buffers
1838  * @rx_ring: Rx ring being processed
1839  * @rx_desc: Rx descriptor for current buffer
1840  *
1841  * This function updates next to clean.  If the buffer is an EOP buffer
1842  * this function exits returning false, otherwise it will place the
1843  * sk_buff in the next buffer to be chained and return true indicating
1844  * that this is in fact a non-EOP buffer.
1845  */
1846 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1847                            union igc_adv_rx_desc *rx_desc)
1848 {
1849         u32 ntc = rx_ring->next_to_clean + 1;
1850
1851         /* fetch, update, and store next to clean */
1852         ntc = (ntc < rx_ring->count) ? ntc : 0;
1853         rx_ring->next_to_clean = ntc;
1854
1855         prefetch(IGC_RX_DESC(rx_ring, ntc));
1856
1857         if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
1858                 return false;
1859
1860         return true;
1861 }
1862
1863 /**
1864  * igc_cleanup_headers - Correct corrupted or empty headers
1865  * @rx_ring: rx descriptor ring packet is being transacted on
1866  * @rx_desc: pointer to the EOP Rx descriptor
1867  * @skb: pointer to current skb being fixed
1868  *
1869  * Address the case where we are pulling data in on pages only
1870  * and as such no data is present in the skb header.
1871  *
1872  * In addition if skb is not at least 60 bytes we need to pad it so that
1873  * it is large enough to qualify as a valid Ethernet frame.
1874  *
1875  * Returns true if an error was encountered and skb was freed.
1876  */
1877 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
1878                                 union igc_adv_rx_desc *rx_desc,
1879                                 struct sk_buff *skb)
1880 {
1881         if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
1882                 struct net_device *netdev = rx_ring->netdev;
1883
1884                 if (!(netdev->features & NETIF_F_RXALL)) {
1885                         dev_kfree_skb_any(skb);
1886                         return true;
1887                 }
1888         }
1889
1890         /* if eth_skb_pad returns an error the skb was freed */
1891         if (eth_skb_pad(skb))
1892                 return true;
1893
1894         return false;
1895 }
1896
1897 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
1898                               struct igc_rx_buffer *rx_buffer)
1899 {
1900         if (igc_can_reuse_rx_page(rx_buffer)) {
1901                 /* hand second half of page back to the ring */
1902                 igc_reuse_rx_page(rx_ring, rx_buffer);
1903         } else {
1904                 /* We are not reusing the buffer so unmap it and free
1905                  * any references we are holding to it
1906                  */
1907                 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1908                                      igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
1909                                      IGC_RX_DMA_ATTR);
1910                 __page_frag_cache_drain(rx_buffer->page,
1911                                         rx_buffer->pagecnt_bias);
1912         }
1913
1914         /* clear contents of rx_buffer */
1915         rx_buffer->page = NULL;
1916 }
1917
1918 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
1919 {
1920         return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
1921 }
1922
1923 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
1924                                   struct igc_rx_buffer *bi)
1925 {
1926         struct page *page = bi->page;
1927         dma_addr_t dma;
1928
1929         /* since we are recycling buffers we should seldom need to alloc */
1930         if (likely(page))
1931                 return true;
1932
1933         /* alloc new page for storage */
1934         page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
1935         if (unlikely(!page)) {
1936                 rx_ring->rx_stats.alloc_failed++;
1937                 return false;
1938         }
1939
1940         /* map page for use */
1941         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
1942                                  igc_rx_pg_size(rx_ring),
1943                                  DMA_FROM_DEVICE,
1944                                  IGC_RX_DMA_ATTR);
1945
1946         /* if mapping failed free memory back to system since
1947          * there isn't much point in holding memory we can't use
1948          */
1949         if (dma_mapping_error(rx_ring->dev, dma)) {
1950                 __free_page(page);
1951
1952                 rx_ring->rx_stats.alloc_failed++;
1953                 return false;
1954         }
1955
1956         bi->dma = dma;
1957         bi->page = page;
1958         bi->page_offset = igc_rx_offset(rx_ring);
1959         bi->pagecnt_bias = 1;
1960
1961         return true;
1962 }
1963
1964 /**
1965  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
1966  * @rx_ring: rx descriptor ring
1967  * @cleaned_count: number of buffers to clean
1968  */
1969 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
1970 {
1971         union igc_adv_rx_desc *rx_desc;
1972         u16 i = rx_ring->next_to_use;
1973         struct igc_rx_buffer *bi;
1974         u16 bufsz;
1975
1976         /* nothing to do */
1977         if (!cleaned_count)
1978                 return;
1979
1980         rx_desc = IGC_RX_DESC(rx_ring, i);
1981         bi = &rx_ring->rx_buffer_info[i];
1982         i -= rx_ring->count;
1983
1984         bufsz = igc_rx_bufsz(rx_ring);
1985
1986         do {
1987                 if (!igc_alloc_mapped_page(rx_ring, bi))
1988                         break;
1989
1990                 /* sync the buffer for use by the device */
1991                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
1992                                                  bi->page_offset, bufsz,
1993                                                  DMA_FROM_DEVICE);
1994
1995                 /* Refresh the desc even if buffer_addrs didn't change
1996                  * because each write-back erases this info.
1997                  */
1998                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1999
2000                 rx_desc++;
2001                 bi++;
2002                 i++;
2003                 if (unlikely(!i)) {
2004                         rx_desc = IGC_RX_DESC(rx_ring, 0);
2005                         bi = rx_ring->rx_buffer_info;
2006                         i -= rx_ring->count;
2007                 }
2008
2009                 /* clear the length for the next_to_use descriptor */
2010                 rx_desc->wb.upper.length = 0;
2011
2012                 cleaned_count--;
2013         } while (cleaned_count);
2014
2015         i += rx_ring->count;
2016
2017         if (rx_ring->next_to_use != i) {
2018                 /* record the next descriptor to use */
2019                 rx_ring->next_to_use = i;
2020
2021                 /* update next to alloc since we have filled the ring */
2022                 rx_ring->next_to_alloc = i;
2023
2024                 /* Force memory writes to complete before letting h/w
2025                  * know there are new descriptors to fetch.  (Only
2026                  * applicable for weak-ordered memory model archs,
2027                  * such as IA-64).
2028                  */
2029                 wmb();
2030                 writel(i, rx_ring->tail);
2031         }
2032 }
2033
2034 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2035 {
2036         unsigned int total_bytes = 0, total_packets = 0;
2037         struct igc_ring *rx_ring = q_vector->rx.ring;
2038         struct sk_buff *skb = rx_ring->skb;
2039         u16 cleaned_count = igc_desc_unused(rx_ring);
2040
2041         while (likely(total_packets < budget)) {
2042                 union igc_adv_rx_desc *rx_desc;
2043                 struct igc_rx_buffer *rx_buffer;
2044                 unsigned int size;
2045
2046                 /* return some buffers to hardware, one at a time is too slow */
2047                 if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2048                         igc_alloc_rx_buffers(rx_ring, cleaned_count);
2049                         cleaned_count = 0;
2050                 }
2051
2052                 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2053                 size = le16_to_cpu(rx_desc->wb.upper.length);
2054                 if (!size)
2055                         break;
2056
2057                 /* This memory barrier is needed to keep us from reading
2058                  * any other fields out of the rx_desc until we know the
2059                  * descriptor has been written back
2060                  */
2061                 dma_rmb();
2062
2063                 rx_buffer = igc_get_rx_buffer(rx_ring, size);
2064
2065                 /* retrieve a buffer from the ring */
2066                 if (skb)
2067                         igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2068                 else if (ring_uses_build_skb(rx_ring))
2069                         skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);
2070                 else
2071                         skb = igc_construct_skb(rx_ring, rx_buffer,
2072                                                 rx_desc, size);
2073
2074                 /* exit if we failed to retrieve a buffer */
2075                 if (!skb) {
2076                         rx_ring->rx_stats.alloc_failed++;
2077                         rx_buffer->pagecnt_bias++;
2078                         break;
2079                 }
2080
2081                 igc_put_rx_buffer(rx_ring, rx_buffer);
2082                 cleaned_count++;
2083
2084                 /* fetch next buffer in frame if non-eop */
2085                 if (igc_is_non_eop(rx_ring, rx_desc))
2086                         continue;
2087
2088                 /* verify the packet layout is correct */
2089                 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2090                         skb = NULL;
2091                         continue;
2092                 }
2093
2094                 /* probably a little skewed due to removing CRC */
2095                 total_bytes += skb->len;
2096
2097                 /* populate checksum, VLAN, and protocol */
2098                 igc_process_skb_fields(rx_ring, rx_desc, skb);
2099
2100                 napi_gro_receive(&q_vector->napi, skb);
2101
2102                 /* reset skb pointer */
2103                 skb = NULL;
2104
2105                 /* update budget accounting */
2106                 total_packets++;
2107         }
2108
2109         /* place incomplete frames back on ring for completion */
2110         rx_ring->skb = skb;
2111
2112         u64_stats_update_begin(&rx_ring->rx_syncp);
2113         rx_ring->rx_stats.packets += total_packets;
2114         rx_ring->rx_stats.bytes += total_bytes;
2115         u64_stats_update_end(&rx_ring->rx_syncp);
2116         q_vector->rx.total_packets += total_packets;
2117         q_vector->rx.total_bytes += total_bytes;
2118
2119         if (cleaned_count)
2120                 igc_alloc_rx_buffers(rx_ring, cleaned_count);
2121
2122         return total_packets;
2123 }
2124
2125 /**
2126  * igc_clean_tx_irq - Reclaim resources after transmit completes
2127  * @q_vector: pointer to q_vector containing needed info
2128  * @napi_budget: Used to determine if we are in netpoll
2129  *
2130  * returns true if ring is completely cleaned
2131  */
2132 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
2133 {
2134         struct igc_adapter *adapter = q_vector->adapter;
2135         unsigned int total_bytes = 0, total_packets = 0;
2136         unsigned int budget = q_vector->tx.work_limit;
2137         struct igc_ring *tx_ring = q_vector->tx.ring;
2138         unsigned int i = tx_ring->next_to_clean;
2139         struct igc_tx_buffer *tx_buffer;
2140         union igc_adv_tx_desc *tx_desc;
2141
2142         if (test_bit(__IGC_DOWN, &adapter->state))
2143                 return true;
2144
2145         tx_buffer = &tx_ring->tx_buffer_info[i];
2146         tx_desc = IGC_TX_DESC(tx_ring, i);
2147         i -= tx_ring->count;
2148
2149         do {
2150                 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
2151
2152                 /* if next_to_watch is not set then there is no work pending */
2153                 if (!eop_desc)
2154                         break;
2155
2156                 /* prevent any other reads prior to eop_desc */
2157                 smp_rmb();
2158
2159                 /* if DD is not set pending work has not been completed */
2160                 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
2161                         break;
2162
2163                 /* clear next_to_watch to prevent false hangs */
2164                 tx_buffer->next_to_watch = NULL;
2165
2166                 /* update the statistics for this packet */
2167                 total_bytes += tx_buffer->bytecount;
2168                 total_packets += tx_buffer->gso_segs;
2169
2170                 /* free the skb */
2171                 napi_consume_skb(tx_buffer->skb, napi_budget);
2172
2173                 /* unmap skb header data */
2174                 dma_unmap_single(tx_ring->dev,
2175                                  dma_unmap_addr(tx_buffer, dma),
2176                                  dma_unmap_len(tx_buffer, len),
2177                                  DMA_TO_DEVICE);
2178
2179                 /* clear tx_buffer data */
2180                 dma_unmap_len_set(tx_buffer, len, 0);
2181
2182                 /* clear last DMA location and unmap remaining buffers */
2183                 while (tx_desc != eop_desc) {
2184                         tx_buffer++;
2185                         tx_desc++;
2186                         i++;
2187                         if (unlikely(!i)) {
2188                                 i -= tx_ring->count;
2189                                 tx_buffer = tx_ring->tx_buffer_info;
2190                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
2191                         }
2192
2193                         /* unmap any remaining paged data */
2194                         if (dma_unmap_len(tx_buffer, len)) {
2195                                 dma_unmap_page(tx_ring->dev,
2196                                                dma_unmap_addr(tx_buffer, dma),
2197                                                dma_unmap_len(tx_buffer, len),
2198                                                DMA_TO_DEVICE);
2199                                 dma_unmap_len_set(tx_buffer, len, 0);
2200                         }
2201                 }
2202
2203                 /* move us one more past the eop_desc for start of next pkt */
2204                 tx_buffer++;
2205                 tx_desc++;
2206                 i++;
2207                 if (unlikely(!i)) {
2208                         i -= tx_ring->count;
2209                         tx_buffer = tx_ring->tx_buffer_info;
2210                         tx_desc = IGC_TX_DESC(tx_ring, 0);
2211                 }
2212
2213                 /* issue prefetch for next Tx descriptor */
2214                 prefetch(tx_desc);
2215
2216                 /* update budget accounting */
2217                 budget--;
2218         } while (likely(budget));
2219
2220         netdev_tx_completed_queue(txring_txq(tx_ring),
2221                                   total_packets, total_bytes);
2222
2223         i += tx_ring->count;
2224         tx_ring->next_to_clean = i;
2225         u64_stats_update_begin(&tx_ring->tx_syncp);
2226         tx_ring->tx_stats.bytes += total_bytes;
2227         tx_ring->tx_stats.packets += total_packets;
2228         u64_stats_update_end(&tx_ring->tx_syncp);
2229         q_vector->tx.total_bytes += total_bytes;
2230         q_vector->tx.total_packets += total_packets;
2231
2232         if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
2233                 struct igc_hw *hw = &adapter->hw;
2234
2235                 /* Detect a transmit hang in hardware, this serializes the
2236                  * check with the clearing of time_stamp and movement of i
2237                  */
2238                 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2239                 if (tx_buffer->next_to_watch &&
2240                     time_after(jiffies, tx_buffer->time_stamp +
2241                     (adapter->tx_timeout_factor * HZ)) &&
2242                     !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
2243                         /* detected Tx unit hang */
2244                         netdev_err(tx_ring->netdev,
2245                                    "Detected Tx Unit Hang\n"
2246                                    "  Tx Queue             <%d>\n"
2247                                    "  TDH                  <%x>\n"
2248                                    "  TDT                  <%x>\n"
2249                                    "  next_to_use          <%x>\n"
2250                                    "  next_to_clean        <%x>\n"
2251                                    "buffer_info[next_to_clean]\n"
2252                                    "  time_stamp           <%lx>\n"
2253                                    "  next_to_watch        <%p>\n"
2254                                    "  jiffies              <%lx>\n"
2255                                    "  desc.status          <%x>\n",
2256                                    tx_ring->queue_index,
2257                                    rd32(IGC_TDH(tx_ring->reg_idx)),
2258                                    readl(tx_ring->tail),
2259                                    tx_ring->next_to_use,
2260                                    tx_ring->next_to_clean,
2261                                    tx_buffer->time_stamp,
2262                                    tx_buffer->next_to_watch,
2263                                    jiffies,
2264                                    tx_buffer->next_to_watch->wb.status);
2265                         netif_stop_subqueue(tx_ring->netdev,
2266                                             tx_ring->queue_index);
2267
2268                         /* we are about to reset, no point in enabling stuff */
2269                         return true;
2270                 }
2271         }
2272
2273 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
2274         if (unlikely(total_packets &&
2275                      netif_carrier_ok(tx_ring->netdev) &&
2276                      igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
2277                 /* Make sure that anybody stopping the queue after this
2278                  * sees the new next_to_clean.
2279                  */
2280                 smp_mb();
2281                 if (__netif_subqueue_stopped(tx_ring->netdev,
2282                                              tx_ring->queue_index) &&
2283                     !(test_bit(__IGC_DOWN, &adapter->state))) {
2284                         netif_wake_subqueue(tx_ring->netdev,
2285                                             tx_ring->queue_index);
2286
2287                         u64_stats_update_begin(&tx_ring->tx_syncp);
2288                         tx_ring->tx_stats.restart_queue++;
2289                         u64_stats_update_end(&tx_ring->tx_syncp);
2290                 }
2291         }
2292
2293         return !!budget;
2294 }
2295
2296 static int igc_find_mac_filter(struct igc_adapter *adapter,
2297                                enum igc_mac_filter_type type, const u8 *addr)
2298 {
2299         struct igc_hw *hw = &adapter->hw;
2300         int max_entries = hw->mac.rar_entry_count;
2301         u32 ral, rah;
2302         int i;
2303
2304         for (i = 0; i < max_entries; i++) {
2305                 ral = rd32(IGC_RAL(i));
2306                 rah = rd32(IGC_RAH(i));
2307
2308                 if (!(rah & IGC_RAH_AV))
2309                         continue;
2310                 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
2311                         continue;
2312                 if ((rah & IGC_RAH_RAH_MASK) !=
2313                     le16_to_cpup((__le16 *)(addr + 4)))
2314                         continue;
2315                 if (ral != le32_to_cpup((__le32 *)(addr)))
2316                         continue;
2317
2318                 return i;
2319         }
2320
2321         return -1;
2322 }
2323
2324 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
2325 {
2326         struct igc_hw *hw = &adapter->hw;
2327         int max_entries = hw->mac.rar_entry_count;
2328         u32 rah;
2329         int i;
2330
2331         for (i = 0; i < max_entries; i++) {
2332                 rah = rd32(IGC_RAH(i));
2333
2334                 if (!(rah & IGC_RAH_AV))
2335                         return i;
2336         }
2337
2338         return -1;
2339 }
2340
2341 /**
2342  * igc_add_mac_filter() - Add MAC address filter
2343  * @adapter: Pointer to adapter where the filter should be added
2344  * @type: MAC address filter type (source or destination)
2345  * @addr: MAC address
2346  * @queue: If non-negative, queue assignment feature is enabled and frames
2347  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
2348  *         assignment is disabled.
2349  *
2350  * Return: 0 in case of success, negative errno code otherwise.
2351  */
2352 static int igc_add_mac_filter(struct igc_adapter *adapter,
2353                               enum igc_mac_filter_type type, const u8 *addr,
2354                               int queue)
2355 {
2356         struct net_device *dev = adapter->netdev;
2357         int index;
2358
2359         index = igc_find_mac_filter(adapter, type, addr);
2360         if (index >= 0)
2361                 goto update_filter;
2362
2363         index = igc_get_avail_mac_filter_slot(adapter);
2364         if (index < 0)
2365                 return -ENOSPC;
2366
2367         netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
2368                    index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
2369                    addr, queue);
2370
2371 update_filter:
2372         igc_set_mac_filter_hw(adapter, index, type, addr, queue);
2373         return 0;
2374 }
2375
2376 /**
2377  * igc_del_mac_filter() - Delete MAC address filter
2378  * @adapter: Pointer to adapter where the filter should be deleted from
2379  * @type: MAC address filter type (source or destination)
2380  * @addr: MAC address
2381  */
2382 static void igc_del_mac_filter(struct igc_adapter *adapter,
2383                                enum igc_mac_filter_type type, const u8 *addr)
2384 {
2385         struct net_device *dev = adapter->netdev;
2386         int index;
2387
2388         index = igc_find_mac_filter(adapter, type, addr);
2389         if (index < 0)
2390                 return;
2391
2392         if (index == 0) {
2393                 /* If this is the default filter, we don't actually delete it.
2394                  * We just reset to its default value i.e. disable queue
2395                  * assignment.
2396                  */
2397                 netdev_dbg(dev, "Disable default MAC filter queue assignment");
2398
2399                 igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
2400         } else {
2401                 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
2402                            index,
2403                            type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
2404                            addr);
2405
2406                 igc_clear_mac_filter_hw(adapter, index);
2407         }
2408 }
2409
2410 /**
2411  * igc_add_vlan_prio_filter() - Add VLAN priority filter
2412  * @adapter: Pointer to adapter where the filter should be added
2413  * @prio: VLAN priority value
2414  * @queue: Queue number which matching frames are assigned to
2415  *
2416  * Return: 0 in case of success, negative errno code otherwise.
2417  */
2418 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
2419                                     int queue)
2420 {
2421         struct net_device *dev = adapter->netdev;
2422         struct igc_hw *hw = &adapter->hw;
2423         u32 vlanpqf;
2424
2425         vlanpqf = rd32(IGC_VLANPQF);
2426
2427         if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
2428                 netdev_dbg(dev, "VLAN priority filter already in use\n");
2429                 return -EEXIST;
2430         }
2431
2432         vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
2433         vlanpqf |= IGC_VLANPQF_VALID(prio);
2434
2435         wr32(IGC_VLANPQF, vlanpqf);
2436
2437         netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
2438                    prio, queue);
2439         return 0;
2440 }
2441
2442 /**
2443  * igc_del_vlan_prio_filter() - Delete VLAN priority filter
2444  * @adapter: Pointer to adapter where the filter should be deleted from
2445  * @prio: VLAN priority value
2446  */
2447 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
2448 {
2449         struct igc_hw *hw = &adapter->hw;
2450         u32 vlanpqf;
2451
2452         vlanpqf = rd32(IGC_VLANPQF);
2453
2454         vlanpqf &= ~IGC_VLANPQF_VALID(prio);
2455         vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
2456
2457         wr32(IGC_VLANPQF, vlanpqf);
2458
2459         netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
2460                    prio);
2461 }
2462
2463 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
2464 {
2465         struct igc_hw *hw = &adapter->hw;
2466         int i;
2467
2468         for (i = 0; i < MAX_ETYPE_FILTER; i++) {
2469                 u32 etqf = rd32(IGC_ETQF(i));
2470
2471                 if (!(etqf & IGC_ETQF_FILTER_ENABLE))
2472                         return i;
2473         }
2474
2475         return -1;
2476 }
2477
2478 /**
2479  * igc_add_etype_filter() - Add ethertype filter
2480  * @adapter: Pointer to adapter where the filter should be added
2481  * @etype: Ethertype value
2482  * @queue: If non-negative, queue assignment feature is enabled and frames
2483  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
2484  *         assignment is disabled.
2485  *
2486  * Return: 0 in case of success, negative errno code otherwise.
2487  */
2488 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
2489                                 int queue)
2490 {
2491         struct igc_hw *hw = &adapter->hw;
2492         int index;
2493         u32 etqf;
2494
2495         index = igc_get_avail_etype_filter_slot(adapter);
2496         if (index < 0)
2497                 return -ENOSPC;
2498
2499         etqf = rd32(IGC_ETQF(index));
2500
2501         etqf &= ~IGC_ETQF_ETYPE_MASK;
2502         etqf |= etype;
2503
2504         if (queue >= 0) {
2505                 etqf &= ~IGC_ETQF_QUEUE_MASK;
2506                 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
2507                 etqf |= IGC_ETQF_QUEUE_ENABLE;
2508         }
2509
2510         etqf |= IGC_ETQF_FILTER_ENABLE;
2511
2512         wr32(IGC_ETQF(index), etqf);
2513
2514         netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
2515                    etype, queue);
2516         return 0;
2517 }
2518
2519 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
2520 {
2521         struct igc_hw *hw = &adapter->hw;
2522         int i;
2523
2524         for (i = 0; i < MAX_ETYPE_FILTER; i++) {
2525                 u32 etqf = rd32(IGC_ETQF(i));
2526
2527                 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
2528                         return i;
2529         }
2530
2531         return -1;
2532 }
2533
2534 /**
2535  * igc_del_etype_filter() - Delete ethertype filter
2536  * @adapter: Pointer to adapter where the filter should be deleted from
2537  * @etype: Ethertype value
2538  */
2539 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
2540 {
2541         struct igc_hw *hw = &adapter->hw;
2542         int index;
2543
2544         index = igc_find_etype_filter(adapter, etype);
2545         if (index < 0)
2546                 return;
2547
2548         wr32(IGC_ETQF(index), 0);
2549
2550         netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
2551                    etype);
2552 }
2553
2554 static int igc_enable_nfc_rule(struct igc_adapter *adapter,
2555                                const struct igc_nfc_rule *rule)
2556 {
2557         int err;
2558
2559         if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
2560                 err = igc_add_etype_filter(adapter, rule->filter.etype,
2561                                            rule->action);
2562                 if (err)
2563                         return err;
2564         }
2565
2566         if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
2567                 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
2568                                          rule->filter.src_addr, rule->action);
2569                 if (err)
2570                         return err;
2571         }
2572
2573         if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
2574                 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
2575                                          rule->filter.dst_addr, rule->action);
2576                 if (err)
2577                         return err;
2578         }
2579
2580         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
2581                 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
2582                            VLAN_PRIO_SHIFT;
2583
2584                 err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
2585                 if (err)
2586                         return err;
2587         }
2588
2589         return 0;
2590 }
2591
2592 static void igc_disable_nfc_rule(struct igc_adapter *adapter,
2593                                  const struct igc_nfc_rule *rule)
2594 {
2595         if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
2596                 igc_del_etype_filter(adapter, rule->filter.etype);
2597
2598         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
2599                 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
2600                            VLAN_PRIO_SHIFT;
2601
2602                 igc_del_vlan_prio_filter(adapter, prio);
2603         }
2604
2605         if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
2606                 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
2607                                    rule->filter.src_addr);
2608
2609         if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
2610                 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
2611                                    rule->filter.dst_addr);
2612 }
2613
2614 /**
2615  * igc_get_nfc_rule() - Get NFC rule
2616  * @adapter: Pointer to adapter
2617  * @location: Rule location
2618  *
2619  * Context: Expects adapter->nfc_rule_lock to be held by caller.
2620  *
2621  * Return: Pointer to NFC rule at @location. If not found, NULL.
2622  */
2623 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
2624                                       u32 location)
2625 {
2626         struct igc_nfc_rule *rule;
2627
2628         list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
2629                 if (rule->location == location)
2630                         return rule;
2631                 if (rule->location > location)
2632                         break;
2633         }
2634
2635         return NULL;
2636 }
2637
2638 /**
2639  * igc_del_nfc_rule() - Delete NFC rule
2640  * @adapter: Pointer to adapter
2641  * @rule: Pointer to rule to be deleted
2642  *
2643  * Disable NFC rule in hardware and delete it from adapter.
2644  *
2645  * Context: Expects adapter->nfc_rule_lock to be held by caller.
2646  */
2647 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
2648 {
2649         igc_disable_nfc_rule(adapter, rule);
2650
2651         list_del(&rule->list);
2652         adapter->nfc_rule_count--;
2653
2654         kfree(rule);
2655 }
2656
2657 static void igc_flush_nfc_rules(struct igc_adapter *adapter)
2658 {
2659         struct igc_nfc_rule *rule, *tmp;
2660
2661         mutex_lock(&adapter->nfc_rule_lock);
2662
2663         list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
2664                 igc_del_nfc_rule(adapter, rule);
2665
2666         mutex_unlock(&adapter->nfc_rule_lock);
2667 }
2668
2669 /**
2670  * igc_add_nfc_rule() - Add NFC rule
2671  * @adapter: Pointer to adapter
2672  * @rule: Pointer to rule to be added
2673  *
2674  * Enable NFC rule in hardware and add it to adapter.
2675  *
2676  * Context: Expects adapter->nfc_rule_lock to be held by caller.
2677  *
2678  * Return: 0 on success, negative errno on failure.
2679  */
2680 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
2681 {
2682         struct igc_nfc_rule *pred, *cur;
2683         int err;
2684
2685         err = igc_enable_nfc_rule(adapter, rule);
2686         if (err)
2687                 return err;
2688
2689         pred = NULL;
2690         list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
2691                 if (cur->location >= rule->location)
2692                         break;
2693                 pred = cur;
2694         }
2695
2696         list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
2697         adapter->nfc_rule_count++;
2698         return 0;
2699 }
2700
2701 static void igc_restore_nfc_rules(struct igc_adapter *adapter)
2702 {
2703         struct igc_nfc_rule *rule;
2704
2705         mutex_lock(&adapter->nfc_rule_lock);
2706
2707         list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
2708                 igc_enable_nfc_rule(adapter, rule);
2709
2710         mutex_unlock(&adapter->nfc_rule_lock);
2711 }
2712
2713 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
2714 {
2715         struct igc_adapter *adapter = netdev_priv(netdev);
2716
2717         return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
2718 }
2719
2720 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
2721 {
2722         struct igc_adapter *adapter = netdev_priv(netdev);
2723
2724         igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
2725         return 0;
2726 }
2727
2728 /**
2729  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2730  * @netdev: network interface device structure
2731  *
2732  * The set_rx_mode entry point is called whenever the unicast or multicast
2733  * address lists or the network interface flags are updated.  This routine is
2734  * responsible for configuring the hardware for proper unicast, multicast,
2735  * promiscuous mode, and all-multi behavior.
2736  */
2737 static void igc_set_rx_mode(struct net_device *netdev)
2738 {
2739         struct igc_adapter *adapter = netdev_priv(netdev);
2740         struct igc_hw *hw = &adapter->hw;
2741         u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
2742         int count;
2743
2744         /* Check for Promiscuous and All Multicast modes */
2745         if (netdev->flags & IFF_PROMISC) {
2746                 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
2747         } else {
2748                 if (netdev->flags & IFF_ALLMULTI) {
2749                         rctl |= IGC_RCTL_MPE;
2750                 } else {
2751                         /* Write addresses to the MTA, if the attempt fails
2752                          * then we should just turn on promiscuous mode so
2753                          * that we can at least receive multicast traffic
2754                          */
2755                         count = igc_write_mc_addr_list(netdev);
2756                         if (count < 0)
2757                                 rctl |= IGC_RCTL_MPE;
2758                 }
2759         }
2760
2761         /* Write addresses to available RAR registers, if there is not
2762          * sufficient space to store all the addresses then enable
2763          * unicast promiscuous mode
2764          */
2765         if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
2766                 rctl |= IGC_RCTL_UPE;
2767
2768         /* update state of unicast and multicast */
2769         rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
2770         wr32(IGC_RCTL, rctl);
2771
2772 #if (PAGE_SIZE < 8192)
2773         if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
2774                 rlpml = IGC_MAX_FRAME_BUILD_SKB;
2775 #endif
2776         wr32(IGC_RLPML, rlpml);
2777 }
2778
2779 /**
2780  * igc_configure - configure the hardware for RX and TX
2781  * @adapter: private board structure
2782  */
2783 static void igc_configure(struct igc_adapter *adapter)
2784 {
2785         struct net_device *netdev = adapter->netdev;
2786         int i = 0;
2787
2788         igc_get_hw_control(adapter);
2789         igc_set_rx_mode(netdev);
2790
2791         igc_setup_tctl(adapter);
2792         igc_setup_mrqc(adapter);
2793         igc_setup_rctl(adapter);
2794
2795         igc_set_default_mac_filter(adapter);
2796         igc_restore_nfc_rules(adapter);
2797
2798         igc_configure_tx(adapter);
2799         igc_configure_rx(adapter);
2800
2801         igc_rx_fifo_flush_base(&adapter->hw);
2802
2803         /* call igc_desc_unused which always leaves
2804          * at least 1 descriptor unused to make sure
2805          * next_to_use != next_to_clean
2806          */
2807         for (i = 0; i < adapter->num_rx_queues; i++) {
2808                 struct igc_ring *ring = adapter->rx_ring[i];
2809
2810                 igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
2811         }
2812 }
2813
2814 /**
2815  * igc_write_ivar - configure ivar for given MSI-X vector
2816  * @hw: pointer to the HW structure
2817  * @msix_vector: vector number we are allocating to a given ring
2818  * @index: row index of IVAR register to write within IVAR table
2819  * @offset: column offset of in IVAR, should be multiple of 8
2820  *
2821  * The IVAR table consists of 2 columns,
2822  * each containing an cause allocation for an Rx and Tx ring, and a
2823  * variable number of rows depending on the number of queues supported.
2824  */
2825 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
2826                            int index, int offset)
2827 {
2828         u32 ivar = array_rd32(IGC_IVAR0, index);
2829
2830         /* clear any bits that are currently set */
2831         ivar &= ~((u32)0xFF << offset);
2832
2833         /* write vector and valid bit */
2834         ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
2835
2836         array_wr32(IGC_IVAR0, index, ivar);
2837 }
2838
2839 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
2840 {
2841         struct igc_adapter *adapter = q_vector->adapter;
2842         struct igc_hw *hw = &adapter->hw;
2843         int rx_queue = IGC_N0_QUEUE;
2844         int tx_queue = IGC_N0_QUEUE;
2845
2846         if (q_vector->rx.ring)
2847                 rx_queue = q_vector->rx.ring->reg_idx;
2848         if (q_vector->tx.ring)
2849                 tx_queue = q_vector->tx.ring->reg_idx;
2850
2851         switch (hw->mac.type) {
2852         case igc_i225:
2853                 if (rx_queue > IGC_N0_QUEUE)
2854                         igc_write_ivar(hw, msix_vector,
2855                                        rx_queue >> 1,
2856                                        (rx_queue & 0x1) << 4);
2857                 if (tx_queue > IGC_N0_QUEUE)
2858                         igc_write_ivar(hw, msix_vector,
2859                                        tx_queue >> 1,
2860                                        ((tx_queue & 0x1) << 4) + 8);
2861                 q_vector->eims_value = BIT(msix_vector);
2862                 break;
2863         default:
2864                 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
2865                 break;
2866         }
2867
2868         /* add q_vector eims value to global eims_enable_mask */
2869         adapter->eims_enable_mask |= q_vector->eims_value;
2870
2871         /* configure q_vector to set itr on first interrupt */
2872         q_vector->set_itr = 1;
2873 }
2874
2875 /**
2876  * igc_configure_msix - Configure MSI-X hardware
2877  * @adapter: Pointer to adapter structure
2878  *
2879  * igc_configure_msix sets up the hardware to properly
2880  * generate MSI-X interrupts.
2881  */
2882 static void igc_configure_msix(struct igc_adapter *adapter)
2883 {
2884         struct igc_hw *hw = &adapter->hw;
2885         int i, vector = 0;
2886         u32 tmp;
2887
2888         adapter->eims_enable_mask = 0;
2889
2890         /* set vector for other causes, i.e. link changes */
2891         switch (hw->mac.type) {
2892         case igc_i225:
2893                 /* Turn on MSI-X capability first, or our settings
2894                  * won't stick.  And it will take days to debug.
2895                  */
2896                 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
2897                      IGC_GPIE_PBA | IGC_GPIE_EIAME |
2898                      IGC_GPIE_NSICR);
2899
2900                 /* enable msix_other interrupt */
2901                 adapter->eims_other = BIT(vector);
2902                 tmp = (vector++ | IGC_IVAR_VALID) << 8;
2903
2904                 wr32(IGC_IVAR_MISC, tmp);
2905                 break;
2906         default:
2907                 /* do nothing, since nothing else supports MSI-X */
2908                 break;
2909         } /* switch (hw->mac.type) */
2910
2911         adapter->eims_enable_mask |= adapter->eims_other;
2912
2913         for (i = 0; i < adapter->num_q_vectors; i++)
2914                 igc_assign_vector(adapter->q_vector[i], vector++);
2915
2916         wrfl();
2917 }
2918
2919 /**
2920  * igc_irq_enable - Enable default interrupt generation settings
2921  * @adapter: board private structure
2922  */
2923 static void igc_irq_enable(struct igc_adapter *adapter)
2924 {
2925         struct igc_hw *hw = &adapter->hw;
2926
2927         if (adapter->msix_entries) {
2928                 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
2929                 u32 regval = rd32(IGC_EIAC);
2930
2931                 wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
2932                 regval = rd32(IGC_EIAM);
2933                 wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
2934                 wr32(IGC_EIMS, adapter->eims_enable_mask);
2935                 wr32(IGC_IMS, ims);
2936         } else {
2937                 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
2938                 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
2939         }
2940 }
2941
2942 /**
2943  * igc_irq_disable - Mask off interrupt generation on the NIC
2944  * @adapter: board private structure
2945  */
2946 static void igc_irq_disable(struct igc_adapter *adapter)
2947 {
2948         struct igc_hw *hw = &adapter->hw;
2949
2950         if (adapter->msix_entries) {
2951                 u32 regval = rd32(IGC_EIAM);
2952
2953                 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
2954                 wr32(IGC_EIMC, adapter->eims_enable_mask);
2955                 regval = rd32(IGC_EIAC);
2956                 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
2957         }
2958
2959         wr32(IGC_IAM, 0);
2960         wr32(IGC_IMC, ~0);
2961         wrfl();
2962
2963         if (adapter->msix_entries) {
2964                 int vector = 0, i;
2965
2966                 synchronize_irq(adapter->msix_entries[vector++].vector);
2967
2968                 for (i = 0; i < adapter->num_q_vectors; i++)
2969                         synchronize_irq(adapter->msix_entries[vector++].vector);
2970         } else {
2971                 synchronize_irq(adapter->pdev->irq);
2972         }
2973 }
2974
2975 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
2976                               const u32 max_rss_queues)
2977 {
2978         /* Determine if we need to pair queues. */
2979         /* If rss_queues > half of max_rss_queues, pair the queues in
2980          * order to conserve interrupts due to limited supply.
2981          */
2982         if (adapter->rss_queues > (max_rss_queues / 2))
2983                 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
2984         else
2985                 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
2986 }
2987
2988 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
2989 {
2990         return IGC_MAX_RX_QUEUES;
2991 }
2992
2993 static void igc_init_queue_configuration(struct igc_adapter *adapter)
2994 {
2995         u32 max_rss_queues;
2996
2997         max_rss_queues = igc_get_max_rss_queues(adapter);
2998         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2999
3000         igc_set_flag_queue_pairs(adapter, max_rss_queues);
3001 }
3002
3003 /**
3004  * igc_reset_q_vector - Reset config for interrupt vector
3005  * @adapter: board private structure to initialize
3006  * @v_idx: Index of vector to be reset
3007  *
3008  * If NAPI is enabled it will delete any references to the
3009  * NAPI struct. This is preparation for igc_free_q_vector.
3010  */
3011 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
3012 {
3013         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
3014
3015         /* if we're coming from igc_set_interrupt_capability, the vectors are
3016          * not yet allocated
3017          */
3018         if (!q_vector)
3019                 return;
3020
3021         if (q_vector->tx.ring)
3022                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
3023
3024         if (q_vector->rx.ring)
3025                 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
3026
3027         netif_napi_del(&q_vector->napi);
3028 }
3029
3030 /**
3031  * igc_free_q_vector - Free memory allocated for specific interrupt vector
3032  * @adapter: board private structure to initialize
3033  * @v_idx: Index of vector to be freed
3034  *
3035  * This function frees the memory allocated to the q_vector.
3036  */
3037 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
3038 {
3039         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
3040
3041         adapter->q_vector[v_idx] = NULL;
3042
3043         /* igc_get_stats64() might access the rings on this vector,
3044          * we must wait a grace period before freeing it.
3045          */
3046         if (q_vector)
3047                 kfree_rcu(q_vector, rcu);
3048 }
3049
3050 /**
3051  * igc_free_q_vectors - Free memory allocated for interrupt vectors
3052  * @adapter: board private structure to initialize
3053  *
3054  * This function frees the memory allocated to the q_vectors.  In addition if
3055  * NAPI is enabled it will delete any references to the NAPI struct prior
3056  * to freeing the q_vector.
3057  */
3058 static void igc_free_q_vectors(struct igc_adapter *adapter)
3059 {
3060         int v_idx = adapter->num_q_vectors;
3061
3062         adapter->num_tx_queues = 0;
3063         adapter->num_rx_queues = 0;
3064         adapter->num_q_vectors = 0;
3065
3066         while (v_idx--) {
3067                 igc_reset_q_vector(adapter, v_idx);
3068                 igc_free_q_vector(adapter, v_idx);
3069         }
3070 }
3071
3072 /**
3073  * igc_update_itr - update the dynamic ITR value based on statistics
3074  * @q_vector: pointer to q_vector
3075  * @ring_container: ring info to update the itr for
3076  *
3077  * Stores a new ITR value based on packets and byte
3078  * counts during the last interrupt.  The advantage of per interrupt
3079  * computation is faster updates and more accurate ITR for the current
3080  * traffic pattern.  Constants in this function were computed
3081  * based on theoretical maximum wire speed and thresholds were set based
3082  * on testing data as well as attempting to minimize response time
3083  * while increasing bulk throughput.
3084  * NOTE: These calculations are only valid when operating in a single-
3085  * queue environment.
3086  */
3087 static void igc_update_itr(struct igc_q_vector *q_vector,
3088                            struct igc_ring_container *ring_container)
3089 {
3090         unsigned int packets = ring_container->total_packets;
3091         unsigned int bytes = ring_container->total_bytes;
3092         u8 itrval = ring_container->itr;
3093
3094         /* no packets, exit with status unchanged */
3095         if (packets == 0)
3096                 return;
3097
3098         switch (itrval) {
3099         case lowest_latency:
3100                 /* handle TSO and jumbo frames */
3101                 if (bytes / packets > 8000)
3102                         itrval = bulk_latency;
3103                 else if ((packets < 5) && (bytes > 512))
3104                         itrval = low_latency;
3105                 break;
3106         case low_latency:  /* 50 usec aka 20000 ints/s */
3107                 if (bytes > 10000) {
3108                         /* this if handles the TSO accounting */
3109                         if (bytes / packets > 8000)
3110                                 itrval = bulk_latency;
3111                         else if ((packets < 10) || ((bytes / packets) > 1200))
3112                                 itrval = bulk_latency;
3113                         else if ((packets > 35))
3114                                 itrval = lowest_latency;
3115                 } else if (bytes / packets > 2000) {
3116                         itrval = bulk_latency;
3117                 } else if (packets <= 2 && bytes < 512) {
3118                         itrval = lowest_latency;
3119                 }
3120                 break;
3121         case bulk_latency: /* 250 usec aka 4000 ints/s */
3122                 if (bytes > 25000) {
3123                         if (packets > 35)
3124                                 itrval = low_latency;
3125                 } else if (bytes < 1500) {
3126                         itrval = low_latency;
3127                 }
3128                 break;
3129         }
3130
3131         /* clear work counters since we have the values we need */
3132         ring_container->total_bytes = 0;
3133         ring_container->total_packets = 0;
3134
3135         /* write updated itr to ring container */
3136         ring_container->itr = itrval;
3137 }
3138
3139 static void igc_set_itr(struct igc_q_vector *q_vector)
3140 {
3141         struct igc_adapter *adapter = q_vector->adapter;
3142         u32 new_itr = q_vector->itr_val;
3143         u8 current_itr = 0;
3144
3145         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3146         switch (adapter->link_speed) {
3147         case SPEED_10:
3148         case SPEED_100:
3149                 current_itr = 0;
3150                 new_itr = IGC_4K_ITR;
3151                 goto set_itr_now;
3152         default:
3153                 break;
3154         }
3155
3156         igc_update_itr(q_vector, &q_vector->tx);
3157         igc_update_itr(q_vector, &q_vector->rx);
3158
3159         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3160
3161         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3162         if (current_itr == lowest_latency &&
3163             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3164             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3165                 current_itr = low_latency;
3166
3167         switch (current_itr) {
3168         /* counts and packets in update_itr are dependent on these numbers */
3169         case lowest_latency:
3170                 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
3171                 break;
3172         case low_latency:
3173                 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
3174                 break;
3175         case bulk_latency:
3176                 new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
3177                 break;
3178         default:
3179                 break;
3180         }
3181
3182 set_itr_now:
3183         if (new_itr != q_vector->itr_val) {
3184                 /* this attempts to bias the interrupt rate towards Bulk
3185                  * by adding intermediate steps when interrupt rate is
3186                  * increasing
3187                  */
3188                 new_itr = new_itr > q_vector->itr_val ?
3189                           max((new_itr * q_vector->itr_val) /
3190                           (new_itr + (q_vector->itr_val >> 2)),
3191                           new_itr) : new_itr;
3192                 /* Don't write the value here; it resets the adapter's
3193                  * internal timer, and causes us to delay far longer than
3194                  * we should between interrupts.  Instead, we write the ITR
3195                  * value at the beginning of the next interrupt so the timing
3196                  * ends up being correct.
3197                  */
3198                 q_vector->itr_val = new_itr;
3199                 q_vector->set_itr = 1;
3200         }
3201 }
3202
3203 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
3204 {
3205         int v_idx = adapter->num_q_vectors;
3206
3207         if (adapter->msix_entries) {
3208                 pci_disable_msix(adapter->pdev);
3209                 kfree(adapter->msix_entries);
3210                 adapter->msix_entries = NULL;
3211         } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
3212                 pci_disable_msi(adapter->pdev);
3213         }
3214
3215         while (v_idx--)
3216                 igc_reset_q_vector(adapter, v_idx);
3217 }
3218
3219 /**
3220  * igc_set_interrupt_capability - set MSI or MSI-X if supported
3221  * @adapter: Pointer to adapter structure
3222  * @msix: boolean value for MSI-X capability
3223  *
3224  * Attempt to configure interrupts using the best available
3225  * capabilities of the hardware and kernel.
3226  */
3227 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
3228                                          bool msix)
3229 {
3230         int numvecs, i;
3231         int err;
3232
3233         if (!msix)
3234                 goto msi_only;
3235         adapter->flags |= IGC_FLAG_HAS_MSIX;
3236
3237         /* Number of supported queues. */
3238         adapter->num_rx_queues = adapter->rss_queues;
3239
3240         adapter->num_tx_queues = adapter->rss_queues;
3241
3242         /* start with one vector for every Rx queue */
3243         numvecs = adapter->num_rx_queues;
3244
3245         /* if Tx handler is separate add 1 for every Tx queue */
3246         if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
3247                 numvecs += adapter->num_tx_queues;
3248
3249         /* store the number of vectors reserved for queues */
3250         adapter->num_q_vectors = numvecs;
3251
3252         /* add 1 vector for link status interrupts */
3253         numvecs++;
3254
3255         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
3256                                         GFP_KERNEL);
3257
3258         if (!adapter->msix_entries)
3259                 return;
3260
3261         /* populate entry values */
3262         for (i = 0; i < numvecs; i++)
3263                 adapter->msix_entries[i].entry = i;
3264
3265         err = pci_enable_msix_range(adapter->pdev,
3266                                     adapter->msix_entries,
3267                                     numvecs,
3268                                     numvecs);
3269         if (err > 0)
3270                 return;
3271
3272         kfree(adapter->msix_entries);
3273         adapter->msix_entries = NULL;
3274
3275         igc_reset_interrupt_capability(adapter);
3276
3277 msi_only:
3278         adapter->flags &= ~IGC_FLAG_HAS_MSIX;
3279
3280         adapter->rss_queues = 1;
3281         adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
3282         adapter->num_rx_queues = 1;
3283         adapter->num_tx_queues = 1;
3284         adapter->num_q_vectors = 1;
3285         if (!pci_enable_msi(adapter->pdev))
3286                 adapter->flags |= IGC_FLAG_HAS_MSI;
3287 }
3288
3289 /**
3290  * igc_update_ring_itr - update the dynamic ITR value based on packet size
3291  * @q_vector: pointer to q_vector
3292  *
3293  * Stores a new ITR value based on strictly on packet size.  This
3294  * algorithm is less sophisticated than that used in igc_update_itr,
3295  * due to the difficulty of synchronizing statistics across multiple
3296  * receive rings.  The divisors and thresholds used by this function
3297  * were determined based on theoretical maximum wire speed and testing
3298  * data, in order to minimize response time while increasing bulk
3299  * throughput.
3300  * NOTE: This function is called only when operating in a multiqueue
3301  * receive environment.
3302  */
3303 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
3304 {
3305         struct igc_adapter *adapter = q_vector->adapter;
3306         int new_val = q_vector->itr_val;
3307         int avg_wire_size = 0;
3308         unsigned int packets;
3309
3310         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3311          * ints/sec - ITR timer value of 120 ticks.
3312          */
3313         switch (adapter->link_speed) {
3314         case SPEED_10:
3315         case SPEED_100:
3316                 new_val = IGC_4K_ITR;
3317                 goto set_itr_val;
3318         default:
3319                 break;
3320         }
3321
3322         packets = q_vector->rx.total_packets;
3323         if (packets)
3324                 avg_wire_size = q_vector->rx.total_bytes / packets;
3325
3326         packets = q_vector->tx.total_packets;
3327         if (packets)
3328                 avg_wire_size = max_t(u32, avg_wire_size,
3329                                       q_vector->tx.total_bytes / packets);
3330
3331         /* if avg_wire_size isn't set no work was done */
3332         if (!avg_wire_size)
3333                 goto clear_counts;
3334
3335         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3336         avg_wire_size += 24;
3337
3338         /* Don't starve jumbo frames */
3339         avg_wire_size = min(avg_wire_size, 3000);
3340
3341         /* Give a little boost to mid-size frames */
3342         if (avg_wire_size > 300 && avg_wire_size < 1200)
3343                 new_val = avg_wire_size / 3;
3344         else
3345                 new_val = avg_wire_size / 2;
3346
3347         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3348         if (new_val < IGC_20K_ITR &&
3349             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3350             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3351                 new_val = IGC_20K_ITR;
3352
3353 set_itr_val:
3354         if (new_val != q_vector->itr_val) {
3355                 q_vector->itr_val = new_val;
3356                 q_vector->set_itr = 1;
3357         }
3358 clear_counts:
3359         q_vector->rx.total_bytes = 0;
3360         q_vector->rx.total_packets = 0;
3361         q_vector->tx.total_bytes = 0;
3362         q_vector->tx.total_packets = 0;
3363 }
3364
3365 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
3366 {
3367         struct igc_adapter *adapter = q_vector->adapter;
3368         struct igc_hw *hw = &adapter->hw;
3369
3370         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
3371             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
3372                 if (adapter->num_q_vectors == 1)
3373                         igc_set_itr(q_vector);
3374                 else
3375                         igc_update_ring_itr(q_vector);
3376         }
3377
3378         if (!test_bit(__IGC_DOWN, &adapter->state)) {
3379                 if (adapter->msix_entries)
3380                         wr32(IGC_EIMS, q_vector->eims_value);
3381                 else
3382                         igc_irq_enable(adapter);
3383         }
3384 }
3385
3386 static void igc_add_ring(struct igc_ring *ring,
3387                          struct igc_ring_container *head)
3388 {
3389         head->ring = ring;
3390         head->count++;
3391 }
3392
3393 /**
3394  * igc_cache_ring_register - Descriptor ring to register mapping
3395  * @adapter: board private structure to initialize
3396  *
3397  * Once we know the feature-set enabled for the device, we'll cache
3398  * the register offset the descriptor ring is assigned to.
3399  */
3400 static void igc_cache_ring_register(struct igc_adapter *adapter)
3401 {
3402         int i = 0, j = 0;
3403
3404         switch (adapter->hw.mac.type) {
3405         case igc_i225:
3406         default:
3407                 for (; i < adapter->num_rx_queues; i++)
3408                         adapter->rx_ring[i]->reg_idx = i;
3409                 for (; j < adapter->num_tx_queues; j++)
3410                         adapter->tx_ring[j]->reg_idx = j;
3411                 break;
3412         }
3413 }
3414
3415 /**
3416  * igc_poll - NAPI Rx polling callback
3417  * @napi: napi polling structure
3418  * @budget: count of how many packets we should handle
3419  */
3420 static int igc_poll(struct napi_struct *napi, int budget)
3421 {
3422         struct igc_q_vector *q_vector = container_of(napi,
3423                                                      struct igc_q_vector,
3424                                                      napi);
3425         bool clean_complete = true;
3426         int work_done = 0;
3427
3428         if (q_vector->tx.ring)
3429                 clean_complete = igc_clean_tx_irq(q_vector, budget);
3430
3431         if (q_vector->rx.ring) {
3432                 int cleaned = igc_clean_rx_irq(q_vector, budget);
3433
3434                 work_done += cleaned;
3435                 if (cleaned >= budget)
3436                         clean_complete = false;
3437         }
3438
3439         /* If all work not completed, return budget and keep polling */
3440         if (!clean_complete)
3441                 return budget;
3442
3443         /* Exit the polling mode, but don't re-enable interrupts if stack might
3444          * poll us due to busy-polling
3445          */
3446         if (likely(napi_complete_done(napi, work_done)))
3447                 igc_ring_irq_enable(q_vector);
3448
3449         return min(work_done, budget - 1);
3450 }
3451
3452 /**
3453  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
3454  * @adapter: board private structure to initialize
3455  * @v_count: q_vectors allocated on adapter, used for ring interleaving
3456  * @v_idx: index of vector in adapter struct
3457  * @txr_count: total number of Tx rings to allocate
3458  * @txr_idx: index of first Tx ring to allocate
3459  * @rxr_count: total number of Rx rings to allocate
3460  * @rxr_idx: index of first Rx ring to allocate
3461  *
3462  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
3463  */
3464 static int igc_alloc_q_vector(struct igc_adapter *adapter,
3465                               unsigned int v_count, unsigned int v_idx,
3466                               unsigned int txr_count, unsigned int txr_idx,
3467                               unsigned int rxr_count, unsigned int rxr_idx)
3468 {
3469         struct igc_q_vector *q_vector;
3470         struct igc_ring *ring;
3471         int ring_count;
3472
3473         /* igc only supports 1 Tx and/or 1 Rx queue per vector */
3474         if (txr_count > 1 || rxr_count > 1)
3475                 return -ENOMEM;
3476
3477         ring_count = txr_count + rxr_count;
3478
3479         /* allocate q_vector and rings */
3480         q_vector = adapter->q_vector[v_idx];
3481         if (!q_vector)
3482                 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
3483                                    GFP_KERNEL);
3484         else
3485                 memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
3486         if (!q_vector)
3487                 return -ENOMEM;
3488
3489         /* initialize NAPI */
3490         netif_napi_add(adapter->netdev, &q_vector->napi,
3491                        igc_poll, 64);
3492
3493         /* tie q_vector and adapter together */
3494         adapter->q_vector[v_idx] = q_vector;
3495         q_vector->adapter = adapter;
3496
3497         /* initialize work limits */
3498         q_vector->tx.work_limit = adapter->tx_work_limit;
3499
3500         /* initialize ITR configuration */
3501         q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
3502         q_vector->itr_val = IGC_START_ITR;
3503
3504         /* initialize pointer to rings */
3505         ring = q_vector->ring;
3506
3507         /* initialize ITR */
3508         if (rxr_count) {
3509                 /* rx or rx/tx vector */
3510                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
3511                         q_vector->itr_val = adapter->rx_itr_setting;
3512         } else {
3513                 /* tx only vector */
3514                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
3515                         q_vector->itr_val = adapter->tx_itr_setting;
3516         }
3517
3518         if (txr_count) {
3519                 /* assign generic ring traits */
3520                 ring->dev = &adapter->pdev->dev;
3521                 ring->netdev = adapter->netdev;
3522
3523                 /* configure backlink on ring */
3524                 ring->q_vector = q_vector;
3525
3526                 /* update q_vector Tx values */
3527                 igc_add_ring(ring, &q_vector->tx);
3528
3529                 /* apply Tx specific ring traits */
3530                 ring->count = adapter->tx_ring_count;
3531                 ring->queue_index = txr_idx;
3532
3533                 /* assign ring to adapter */
3534                 adapter->tx_ring[txr_idx] = ring;
3535
3536                 /* push pointer to next ring */
3537                 ring++;
3538         }
3539
3540         if (rxr_count) {
3541                 /* assign generic ring traits */
3542                 ring->dev = &adapter->pdev->dev;
3543                 ring->netdev = adapter->netdev;
3544
3545                 /* configure backlink on ring */
3546                 ring->q_vector = q_vector;
3547
3548                 /* update q_vector Rx values */
3549                 igc_add_ring(ring, &q_vector->rx);
3550
3551                 /* apply Rx specific ring traits */
3552                 ring->count = adapter->rx_ring_count;
3553                 ring->queue_index = rxr_idx;
3554
3555                 /* assign ring to adapter */
3556                 adapter->rx_ring[rxr_idx] = ring;
3557         }
3558
3559         return 0;
3560 }
3561
3562 /**
3563  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
3564  * @adapter: board private structure to initialize
3565  *
3566  * We allocate one q_vector per queue interrupt.  If allocation fails we
3567  * return -ENOMEM.
3568  */
3569 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
3570 {
3571         int rxr_remaining = adapter->num_rx_queues;
3572         int txr_remaining = adapter->num_tx_queues;
3573         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
3574         int q_vectors = adapter->num_q_vectors;
3575         int err;
3576
3577         if (q_vectors >= (rxr_remaining + txr_remaining)) {
3578                 for (; rxr_remaining; v_idx++) {
3579                         err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3580                                                  0, 0, 1, rxr_idx);
3581
3582                         if (err)
3583                                 goto err_out;
3584
3585                         /* update counts and index */
3586                         rxr_remaining--;
3587                         rxr_idx++;
3588                 }
3589         }
3590
3591         for (; v_idx < q_vectors; v_idx++) {
3592                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
3593                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
3594
3595                 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
3596                                          tqpv, txr_idx, rqpv, rxr_idx);
3597
3598                 if (err)
3599                         goto err_out;
3600
3601                 /* update counts and index */
3602                 rxr_remaining -= rqpv;
3603                 txr_remaining -= tqpv;
3604                 rxr_idx++;
3605                 txr_idx++;
3606         }
3607
3608         return 0;
3609
3610 err_out:
3611         adapter->num_tx_queues = 0;
3612         adapter->num_rx_queues = 0;
3613         adapter->num_q_vectors = 0;
3614
3615         while (v_idx--)
3616                 igc_free_q_vector(adapter, v_idx);
3617
3618         return -ENOMEM;
3619 }
3620
3621 /**
3622  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
3623  * @adapter: Pointer to adapter structure
3624  * @msix: boolean for MSI-X capability
3625  *
3626  * This function initializes the interrupts and allocates all of the queues.
3627  */
3628 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
3629 {
3630         struct net_device *dev = adapter->netdev;
3631         int err = 0;
3632
3633         igc_set_interrupt_capability(adapter, msix);
3634
3635         err = igc_alloc_q_vectors(adapter);
3636         if (err) {
3637                 netdev_err(dev, "Unable to allocate memory for vectors\n");
3638                 goto err_alloc_q_vectors;
3639         }
3640
3641         igc_cache_ring_register(adapter);
3642
3643         return 0;
3644
3645 err_alloc_q_vectors:
3646         igc_reset_interrupt_capability(adapter);
3647         return err;
3648 }
3649
3650 /**
3651  * igc_sw_init - Initialize general software structures (struct igc_adapter)
3652  * @adapter: board private structure to initialize
3653  *
3654  * igc_sw_init initializes the Adapter private data structure.
3655  * Fields are initialized based on PCI device information and
3656  * OS network device settings (MTU size).
3657  */
3658 static int igc_sw_init(struct igc_adapter *adapter)
3659 {
3660         struct net_device *netdev = adapter->netdev;
3661         struct pci_dev *pdev = adapter->pdev;
3662         struct igc_hw *hw = &adapter->hw;
3663
3664         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
3665
3666         /* set default ring sizes */
3667         adapter->tx_ring_count = IGC_DEFAULT_TXD;
3668         adapter->rx_ring_count = IGC_DEFAULT_RXD;
3669
3670         /* set default ITR values */
3671         adapter->rx_itr_setting = IGC_DEFAULT_ITR;
3672         adapter->tx_itr_setting = IGC_DEFAULT_ITR;
3673
3674         /* set default work limits */
3675         adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
3676
3677         /* adjust max frame to be at least the size of a standard frame */
3678         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
3679                                 VLAN_HLEN;
3680         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3681
3682         mutex_init(&adapter->nfc_rule_lock);
3683         INIT_LIST_HEAD(&adapter->nfc_rule_list);
3684         adapter->nfc_rule_count = 0;
3685
3686         spin_lock_init(&adapter->stats64_lock);
3687         /* Assume MSI-X interrupts, will be checked during IRQ allocation */
3688         adapter->flags |= IGC_FLAG_HAS_MSIX;
3689
3690         igc_init_queue_configuration(adapter);
3691
3692         /* This call may decrease the number of queues */
3693         if (igc_init_interrupt_scheme(adapter, true)) {
3694                 netdev_err(netdev, "Unable to allocate memory for queues\n");
3695                 return -ENOMEM;
3696         }
3697
3698         /* Explicitly disable IRQ since the NIC can be in any state. */
3699         igc_irq_disable(adapter);
3700
3701         set_bit(__IGC_DOWN, &adapter->state);
3702
3703         return 0;
3704 }
3705
3706 /**
3707  * igc_up - Open the interface and prepare it to handle traffic
3708  * @adapter: board private structure
3709  */
3710 void igc_up(struct igc_adapter *adapter)
3711 {
3712         struct igc_hw *hw = &adapter->hw;
3713         int i = 0;
3714
3715         /* hardware has been reset, we need to reload some things */
3716         igc_configure(adapter);
3717
3718         clear_bit(__IGC_DOWN, &adapter->state);
3719
3720         for (i = 0; i < adapter->num_q_vectors; i++)
3721                 napi_enable(&adapter->q_vector[i]->napi);
3722
3723         if (adapter->msix_entries)
3724                 igc_configure_msix(adapter);
3725         else
3726                 igc_assign_vector(adapter->q_vector[0], 0);
3727
3728         /* Clear any pending interrupts. */
3729         rd32(IGC_ICR);
3730         igc_irq_enable(adapter);
3731
3732         netif_tx_start_all_queues(adapter->netdev);
3733
3734         /* start the watchdog. */
3735         hw->mac.get_link_status = 1;
3736         schedule_work(&adapter->watchdog_task);
3737 }
3738
3739 /**
3740  * igc_update_stats - Update the board statistics counters
3741  * @adapter: board private structure
3742  */
3743 void igc_update_stats(struct igc_adapter *adapter)
3744 {
3745         struct rtnl_link_stats64 *net_stats = &adapter->stats64;
3746         struct pci_dev *pdev = adapter->pdev;
3747         struct igc_hw *hw = &adapter->hw;
3748         u64 _bytes, _packets;
3749         u64 bytes, packets;
3750         unsigned int start;
3751         u32 mpc;
3752         int i;
3753
3754         /* Prevent stats update while adapter is being reset, or if the pci
3755          * connection is down.
3756          */
3757         if (adapter->link_speed == 0)
3758                 return;
3759         if (pci_channel_offline(pdev))
3760                 return;
3761
3762         packets = 0;
3763         bytes = 0;
3764
3765         rcu_read_lock();
3766         for (i = 0; i < adapter->num_rx_queues; i++) {
3767                 struct igc_ring *ring = adapter->rx_ring[i];
3768                 u32 rqdpc = rd32(IGC_RQDPC(i));
3769
3770                 if (hw->mac.type >= igc_i225)
3771                         wr32(IGC_RQDPC(i), 0);
3772
3773                 if (rqdpc) {
3774                         ring->rx_stats.drops += rqdpc;
3775                         net_stats->rx_fifo_errors += rqdpc;
3776                 }
3777
3778                 do {
3779                         start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
3780                         _bytes = ring->rx_stats.bytes;
3781                         _packets = ring->rx_stats.packets;
3782                 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
3783                 bytes += _bytes;
3784                 packets += _packets;
3785         }
3786
3787         net_stats->rx_bytes = bytes;
3788         net_stats->rx_packets = packets;
3789
3790         packets = 0;
3791         bytes = 0;
3792         for (i = 0; i < adapter->num_tx_queues; i++) {
3793                 struct igc_ring *ring = adapter->tx_ring[i];
3794
3795                 do {
3796                         start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
3797                         _bytes = ring->tx_stats.bytes;
3798                         _packets = ring->tx_stats.packets;
3799                 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
3800                 bytes += _bytes;
3801                 packets += _packets;
3802         }
3803         net_stats->tx_bytes = bytes;
3804         net_stats->tx_packets = packets;
3805         rcu_read_unlock();
3806
3807         /* read stats registers */
3808         adapter->stats.crcerrs += rd32(IGC_CRCERRS);
3809         adapter->stats.gprc += rd32(IGC_GPRC);
3810         adapter->stats.gorc += rd32(IGC_GORCL);
3811         rd32(IGC_GORCH); /* clear GORCL */
3812         adapter->stats.bprc += rd32(IGC_BPRC);
3813         adapter->stats.mprc += rd32(IGC_MPRC);
3814         adapter->stats.roc += rd32(IGC_ROC);
3815
3816         adapter->stats.prc64 += rd32(IGC_PRC64);
3817         adapter->stats.prc127 += rd32(IGC_PRC127);
3818         adapter->stats.prc255 += rd32(IGC_PRC255);
3819         adapter->stats.prc511 += rd32(IGC_PRC511);
3820         adapter->stats.prc1023 += rd32(IGC_PRC1023);
3821         adapter->stats.prc1522 += rd32(IGC_PRC1522);
3822         adapter->stats.tlpic += rd32(IGC_TLPIC);
3823         adapter->stats.rlpic += rd32(IGC_RLPIC);
3824
3825         mpc = rd32(IGC_MPC);
3826         adapter->stats.mpc += mpc;
3827         net_stats->rx_fifo_errors += mpc;
3828         adapter->stats.scc += rd32(IGC_SCC);
3829         adapter->stats.ecol += rd32(IGC_ECOL);
3830         adapter->stats.mcc += rd32(IGC_MCC);
3831         adapter->stats.latecol += rd32(IGC_LATECOL);
3832         adapter->stats.dc += rd32(IGC_DC);
3833         adapter->stats.rlec += rd32(IGC_RLEC);
3834         adapter->stats.xonrxc += rd32(IGC_XONRXC);
3835         adapter->stats.xontxc += rd32(IGC_XONTXC);
3836         adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
3837         adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
3838         adapter->stats.fcruc += rd32(IGC_FCRUC);
3839         adapter->stats.gptc += rd32(IGC_GPTC);
3840         adapter->stats.gotc += rd32(IGC_GOTCL);
3841         rd32(IGC_GOTCH); /* clear GOTCL */
3842         adapter->stats.rnbc += rd32(IGC_RNBC);
3843         adapter->stats.ruc += rd32(IGC_RUC);
3844         adapter->stats.rfc += rd32(IGC_RFC);
3845         adapter->stats.rjc += rd32(IGC_RJC);
3846         adapter->stats.tor += rd32(IGC_TORH);
3847         adapter->stats.tot += rd32(IGC_TOTH);
3848         adapter->stats.tpr += rd32(IGC_TPR);
3849
3850         adapter->stats.ptc64 += rd32(IGC_PTC64);
3851         adapter->stats.ptc127 += rd32(IGC_PTC127);
3852         adapter->stats.ptc255 += rd32(IGC_PTC255);
3853         adapter->stats.ptc511 += rd32(IGC_PTC511);
3854         adapter->stats.ptc1023 += rd32(IGC_PTC1023);
3855         adapter->stats.ptc1522 += rd32(IGC_PTC1522);
3856
3857         adapter->stats.mptc += rd32(IGC_MPTC);
3858         adapter->stats.bptc += rd32(IGC_BPTC);
3859
3860         adapter->stats.tpt += rd32(IGC_TPT);
3861         adapter->stats.colc += rd32(IGC_COLC);
3862         adapter->stats.colc += rd32(IGC_RERC);
3863
3864         adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
3865
3866         adapter->stats.tsctc += rd32(IGC_TSCTC);
3867
3868         adapter->stats.iac += rd32(IGC_IAC);
3869
3870         /* Fill out the OS statistics structure */
3871         net_stats->multicast = adapter->stats.mprc;
3872         net_stats->collisions = adapter->stats.colc;
3873
3874         /* Rx Errors */
3875
3876         /* RLEC on some newer hardware can be incorrect so build
3877          * our own version based on RUC and ROC
3878          */
3879         net_stats->rx_errors = adapter->stats.rxerrc +
3880                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3881                 adapter->stats.ruc + adapter->stats.roc +
3882                 adapter->stats.cexterr;
3883         net_stats->rx_length_errors = adapter->stats.ruc +
3884                                       adapter->stats.roc;
3885         net_stats->rx_crc_errors = adapter->stats.crcerrs;
3886         net_stats->rx_frame_errors = adapter->stats.algnerrc;
3887         net_stats->rx_missed_errors = adapter->stats.mpc;
3888
3889         /* Tx Errors */
3890         net_stats->tx_errors = adapter->stats.ecol +
3891                                adapter->stats.latecol;
3892         net_stats->tx_aborted_errors = adapter->stats.ecol;
3893         net_stats->tx_window_errors = adapter->stats.latecol;
3894         net_stats->tx_carrier_errors = adapter->stats.tncrs;
3895
3896         /* Tx Dropped needs to be maintained elsewhere */
3897
3898         /* Management Stats */
3899         adapter->stats.mgptc += rd32(IGC_MGTPTC);
3900         adapter->stats.mgprc += rd32(IGC_MGTPRC);
3901         adapter->stats.mgpdc += rd32(IGC_MGTPDC);
3902 }
3903
3904 /**
3905  * igc_down - Close the interface
3906  * @adapter: board private structure
3907  */
3908 void igc_down(struct igc_adapter *adapter)
3909 {
3910         struct net_device *netdev = adapter->netdev;
3911         struct igc_hw *hw = &adapter->hw;
3912         u32 tctl, rctl;
3913         int i = 0;
3914
3915         set_bit(__IGC_DOWN, &adapter->state);
3916
3917         igc_ptp_suspend(adapter);
3918
3919         if (pci_device_is_present(adapter->pdev)) {
3920                 /* disable receives in the hardware */
3921                 rctl = rd32(IGC_RCTL);
3922                 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
3923                 /* flush and sleep below */
3924         }
3925         /* set trans_start so we don't get spurious watchdogs during reset */
3926         netif_trans_update(netdev);
3927
3928         netif_carrier_off(netdev);
3929         netif_tx_stop_all_queues(netdev);
3930
3931         if (pci_device_is_present(adapter->pdev)) {
3932                 /* disable transmits in the hardware */
3933                 tctl = rd32(IGC_TCTL);
3934                 tctl &= ~IGC_TCTL_EN;
3935                 wr32(IGC_TCTL, tctl);
3936                 /* flush both disables and wait for them to finish */
3937                 wrfl();
3938                 usleep_range(10000, 20000);
3939
3940                 igc_irq_disable(adapter);
3941         }
3942
3943         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
3944
3945         for (i = 0; i < adapter->num_q_vectors; i++) {
3946                 if (adapter->q_vector[i]) {
3947                         napi_synchronize(&adapter->q_vector[i]->napi);
3948                         napi_disable(&adapter->q_vector[i]->napi);
3949                 }
3950         }
3951
3952         del_timer_sync(&adapter->watchdog_timer);
3953         del_timer_sync(&adapter->phy_info_timer);
3954
3955         /* record the stats before reset*/
3956         spin_lock(&adapter->stats64_lock);
3957         igc_update_stats(adapter);
3958         spin_unlock(&adapter->stats64_lock);
3959
3960         adapter->link_speed = 0;
3961         adapter->link_duplex = 0;
3962
3963         if (!pci_channel_offline(adapter->pdev))
3964                 igc_reset(adapter);
3965
3966         /* clear VLAN promisc flag so VFTA will be updated if necessary */
3967         adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
3968
3969         igc_clean_all_tx_rings(adapter);
3970         igc_clean_all_rx_rings(adapter);
3971 }
3972
3973 void igc_reinit_locked(struct igc_adapter *adapter)
3974 {
3975         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
3976                 usleep_range(1000, 2000);
3977         igc_down(adapter);
3978         igc_up(adapter);
3979         clear_bit(__IGC_RESETTING, &adapter->state);
3980 }
3981
3982 static void igc_reset_task(struct work_struct *work)
3983 {
3984         struct igc_adapter *adapter;
3985
3986         adapter = container_of(work, struct igc_adapter, reset_task);
3987
3988         rtnl_lock();
3989         /* If we're already down or resetting, just bail */
3990         if (test_bit(__IGC_DOWN, &adapter->state) ||
3991             test_bit(__IGC_RESETTING, &adapter->state)) {
3992                 rtnl_unlock();
3993                 return;
3994         }
3995
3996         igc_rings_dump(adapter);
3997         igc_regs_dump(adapter);
3998         netdev_err(adapter->netdev, "Reset adapter\n");
3999         igc_reinit_locked(adapter);
4000         rtnl_unlock();
4001 }
4002
4003 /**
4004  * igc_change_mtu - Change the Maximum Transfer Unit
4005  * @netdev: network interface device structure
4006  * @new_mtu: new value for maximum frame size
4007  *
4008  * Returns 0 on success, negative on failure
4009  */
4010 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
4011 {
4012         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4013         struct igc_adapter *adapter = netdev_priv(netdev);
4014
4015         /* adjust max frame to be at least the size of a standard frame */
4016         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
4017                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
4018
4019         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
4020                 usleep_range(1000, 2000);
4021
4022         /* igc_down has a dependency on max_frame_size */
4023         adapter->max_frame_size = max_frame;
4024
4025         if (netif_running(netdev))
4026                 igc_down(adapter);
4027
4028         netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
4029         netdev->mtu = new_mtu;
4030
4031         if (netif_running(netdev))
4032                 igc_up(adapter);
4033         else
4034                 igc_reset(adapter);
4035
4036         clear_bit(__IGC_RESETTING, &adapter->state);
4037
4038         return 0;
4039 }
4040
4041 /**
4042  * igc_get_stats64 - Get System Network Statistics
4043  * @netdev: network interface device structure
4044  * @stats: rtnl_link_stats64 pointer
4045  *
4046  * Returns the address of the device statistics structure.
4047  * The statistics are updated here and also from the timer callback.
4048  */
4049 static void igc_get_stats64(struct net_device *netdev,
4050                             struct rtnl_link_stats64 *stats)
4051 {
4052         struct igc_adapter *adapter = netdev_priv(netdev);
4053
4054         spin_lock(&adapter->stats64_lock);
4055         if (!test_bit(__IGC_RESETTING, &adapter->state))
4056                 igc_update_stats(adapter);
4057         memcpy(stats, &adapter->stats64, sizeof(*stats));
4058         spin_unlock(&adapter->stats64_lock);
4059 }
4060
4061 static netdev_features_t igc_fix_features(struct net_device *netdev,
4062                                           netdev_features_t features)
4063 {
4064         /* Since there is no support for separate Rx/Tx vlan accel
4065          * enable/disable make sure Tx flag is always in same state as Rx.
4066          */
4067         if (features & NETIF_F_HW_VLAN_CTAG_RX)
4068                 features |= NETIF_F_HW_VLAN_CTAG_TX;
4069         else
4070                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4071
4072         return features;
4073 }
4074
4075 static int igc_set_features(struct net_device *netdev,
4076                             netdev_features_t features)
4077 {
4078         netdev_features_t changed = netdev->features ^ features;
4079         struct igc_adapter *adapter = netdev_priv(netdev);
4080
4081         /* Add VLAN support */
4082         if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
4083                 return 0;
4084
4085         if (!(features & NETIF_F_NTUPLE))
4086                 igc_flush_nfc_rules(adapter);
4087
4088         netdev->features = features;
4089
4090         if (netif_running(netdev))
4091                 igc_reinit_locked(adapter);
4092         else
4093                 igc_reset(adapter);
4094
4095         return 1;
4096 }
4097
4098 static netdev_features_t
4099 igc_features_check(struct sk_buff *skb, struct net_device *dev,
4100                    netdev_features_t features)
4101 {
4102         unsigned int network_hdr_len, mac_hdr_len;
4103
4104         /* Make certain the headers can be described by a context descriptor */
4105         mac_hdr_len = skb_network_header(skb) - skb->data;
4106         if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
4107                 return features & ~(NETIF_F_HW_CSUM |
4108                                     NETIF_F_SCTP_CRC |
4109                                     NETIF_F_HW_VLAN_CTAG_TX |
4110                                     NETIF_F_TSO |
4111                                     NETIF_F_TSO6);
4112
4113         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
4114         if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
4115                 return features & ~(NETIF_F_HW_CSUM |
4116                                     NETIF_F_SCTP_CRC |
4117                                     NETIF_F_TSO |
4118                                     NETIF_F_TSO6);
4119
4120         /* We can only support IPv4 TSO in tunnels if we can mangle the
4121          * inner IP ID field, so strip TSO if MANGLEID is not supported.
4122          */
4123         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
4124                 features &= ~NETIF_F_TSO;
4125
4126         return features;
4127 }
4128
4129 static void igc_tsync_interrupt(struct igc_adapter *adapter)
4130 {
4131         struct igc_hw *hw = &adapter->hw;
4132         u32 tsicr = rd32(IGC_TSICR);
4133         u32 ack = 0;
4134
4135         if (tsicr & IGC_TSICR_TXTS) {
4136                 /* retrieve hardware timestamp */
4137                 schedule_work(&adapter->ptp_tx_work);
4138                 ack |= IGC_TSICR_TXTS;
4139         }
4140
4141         /* acknowledge the interrupts */
4142         wr32(IGC_TSICR, ack);
4143 }
4144
4145 /**
4146  * igc_msix_other - msix other interrupt handler
4147  * @irq: interrupt number
4148  * @data: pointer to a q_vector
4149  */
4150 static irqreturn_t igc_msix_other(int irq, void *data)
4151 {
4152         struct igc_adapter *adapter = data;
4153         struct igc_hw *hw = &adapter->hw;
4154         u32 icr = rd32(IGC_ICR);
4155
4156         /* reading ICR causes bit 31 of EICR to be cleared */
4157         if (icr & IGC_ICR_DRSTA)
4158                 schedule_work(&adapter->reset_task);
4159
4160         if (icr & IGC_ICR_DOUTSYNC) {
4161                 /* HW is reporting DMA is out of sync */
4162                 adapter->stats.doosync++;
4163         }
4164
4165         if (icr & IGC_ICR_LSC) {
4166                 hw->mac.get_link_status = 1;
4167                 /* guard against interrupt when we're going down */
4168                 if (!test_bit(__IGC_DOWN, &adapter->state))
4169                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4170         }
4171
4172         if (icr & IGC_ICR_TS)
4173                 igc_tsync_interrupt(adapter);
4174
4175         wr32(IGC_EIMS, adapter->eims_other);
4176
4177         return IRQ_HANDLED;
4178 }
4179
4180 static void igc_write_itr(struct igc_q_vector *q_vector)
4181 {
4182         u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
4183
4184         if (!q_vector->set_itr)
4185                 return;
4186
4187         if (!itr_val)
4188                 itr_val = IGC_ITR_VAL_MASK;
4189
4190         itr_val |= IGC_EITR_CNT_IGNR;
4191
4192         writel(itr_val, q_vector->itr_register);
4193         q_vector->set_itr = 0;
4194 }
4195
4196 static irqreturn_t igc_msix_ring(int irq, void *data)
4197 {
4198         struct igc_q_vector *q_vector = data;
4199
4200         /* Write the ITR value calculated from the previous interrupt. */
4201         igc_write_itr(q_vector);
4202
4203         napi_schedule(&q_vector->napi);
4204
4205         return IRQ_HANDLED;
4206 }
4207
4208 /**
4209  * igc_request_msix - Initialize MSI-X interrupts
4210  * @adapter: Pointer to adapter structure
4211  *
4212  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
4213  * kernel.
4214  */
4215 static int igc_request_msix(struct igc_adapter *adapter)
4216 {
4217         unsigned int num_q_vectors = adapter->num_q_vectors;
4218         int i = 0, err = 0, vector = 0, free_vector = 0;
4219         struct net_device *netdev = adapter->netdev;
4220
4221         err = request_irq(adapter->msix_entries[vector].vector,
4222                           &igc_msix_other, 0, netdev->name, adapter);
4223         if (err)
4224                 goto err_out;
4225
4226         if (num_q_vectors > MAX_Q_VECTORS) {
4227                 num_q_vectors = MAX_Q_VECTORS;
4228                 dev_warn(&adapter->pdev->dev,
4229                          "The number of queue vectors (%d) is higher than max allowed (%d)\n",
4230                          adapter->num_q_vectors, MAX_Q_VECTORS);
4231         }
4232         for (i = 0; i < num_q_vectors; i++) {
4233                 struct igc_q_vector *q_vector = adapter->q_vector[i];
4234
4235                 vector++;
4236
4237                 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
4238
4239                 if (q_vector->rx.ring && q_vector->tx.ring)
4240                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
4241                                 q_vector->rx.ring->queue_index);
4242                 else if (q_vector->tx.ring)
4243                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
4244                                 q_vector->tx.ring->queue_index);
4245                 else if (q_vector->rx.ring)
4246                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
4247                                 q_vector->rx.ring->queue_index);
4248                 else
4249                         sprintf(q_vector->name, "%s-unused", netdev->name);
4250
4251                 err = request_irq(adapter->msix_entries[vector].vector,
4252                                   igc_msix_ring, 0, q_vector->name,
4253                                   q_vector);
4254                 if (err)
4255                         goto err_free;
4256         }
4257
4258         igc_configure_msix(adapter);
4259         return 0;
4260
4261 err_free:
4262         /* free already assigned IRQs */
4263         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
4264
4265         vector--;
4266         for (i = 0; i < vector; i++) {
4267                 free_irq(adapter->msix_entries[free_vector++].vector,
4268                          adapter->q_vector[i]);
4269         }
4270 err_out:
4271         return err;
4272 }
4273
4274 /**
4275  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
4276  * @adapter: Pointer to adapter structure
4277  *
4278  * This function resets the device so that it has 0 rx queues, tx queues, and
4279  * MSI-X interrupts allocated.
4280  */
4281 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
4282 {
4283         igc_free_q_vectors(adapter);
4284         igc_reset_interrupt_capability(adapter);
4285 }
4286
4287 /* Need to wait a few seconds after link up to get diagnostic information from
4288  * the phy
4289  */
4290 static void igc_update_phy_info(struct timer_list *t)
4291 {
4292         struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
4293
4294         igc_get_phy_info(&adapter->hw);
4295 }
4296
4297 /**
4298  * igc_has_link - check shared code for link and determine up/down
4299  * @adapter: pointer to driver private info
4300  */
4301 bool igc_has_link(struct igc_adapter *adapter)
4302 {
4303         struct igc_hw *hw = &adapter->hw;
4304         bool link_active = false;
4305
4306         /* get_link_status is set on LSC (link status) interrupt or
4307          * rx sequence error interrupt.  get_link_status will stay
4308          * false until the igc_check_for_link establishes link
4309          * for copper adapters ONLY
4310          */
4311         if (!hw->mac.get_link_status)
4312                 return true;
4313         hw->mac.ops.check_for_link(hw);
4314         link_active = !hw->mac.get_link_status;
4315
4316         if (hw->mac.type == igc_i225) {
4317                 if (!netif_carrier_ok(adapter->netdev)) {
4318                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
4319                 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
4320                         adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
4321                         adapter->link_check_timeout = jiffies;
4322                 }
4323         }
4324
4325         return link_active;
4326 }
4327
4328 /**
4329  * igc_watchdog - Timer Call-back
4330  * @t: timer for the watchdog
4331  */
4332 static void igc_watchdog(struct timer_list *t)
4333 {
4334         struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
4335         /* Do the rest outside of interrupt context */
4336         schedule_work(&adapter->watchdog_task);
4337 }
4338
4339 static void igc_watchdog_task(struct work_struct *work)
4340 {
4341         struct igc_adapter *adapter = container_of(work,
4342                                                    struct igc_adapter,
4343                                                    watchdog_task);
4344         struct net_device *netdev = adapter->netdev;
4345         struct igc_hw *hw = &adapter->hw;
4346         struct igc_phy_info *phy = &hw->phy;
4347         u16 phy_data, retry_count = 20;
4348         u32 link;
4349         int i;
4350
4351         link = igc_has_link(adapter);
4352
4353         if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
4354                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
4355                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
4356                 else
4357                         link = false;
4358         }
4359
4360         if (link) {
4361                 /* Cancel scheduled suspend requests. */
4362                 pm_runtime_resume(netdev->dev.parent);
4363
4364                 if (!netif_carrier_ok(netdev)) {
4365                         u32 ctrl;
4366
4367                         hw->mac.ops.get_speed_and_duplex(hw,
4368                                                          &adapter->link_speed,
4369                                                          &adapter->link_duplex);
4370
4371                         ctrl = rd32(IGC_CTRL);
4372                         /* Link status message must follow this format */
4373                         netdev_info(netdev,
4374                                     "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4375                                     adapter->link_speed,
4376                                     adapter->link_duplex == FULL_DUPLEX ?
4377                                     "Full" : "Half",
4378                                     (ctrl & IGC_CTRL_TFCE) &&
4379                                     (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
4380                                     (ctrl & IGC_CTRL_RFCE) ?  "RX" :
4381                                     (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
4382
4383                         /* disable EEE if enabled */
4384                         if ((adapter->flags & IGC_FLAG_EEE) &&
4385                             adapter->link_duplex == HALF_DUPLEX) {
4386                                 netdev_info(netdev,
4387                                             "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
4388                                 adapter->hw.dev_spec._base.eee_enable = false;
4389                                 adapter->flags &= ~IGC_FLAG_EEE;
4390                         }
4391
4392                         /* check if SmartSpeed worked */
4393                         igc_check_downshift(hw);
4394                         if (phy->speed_downgraded)
4395                                 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
4396
4397                         /* adjust timeout factor according to speed/duplex */
4398                         adapter->tx_timeout_factor = 1;
4399                         switch (adapter->link_speed) {
4400                         case SPEED_10:
4401                                 adapter->tx_timeout_factor = 14;
4402                                 break;
4403                         case SPEED_100:
4404                                 /* maybe add some timeout factor ? */
4405                                 break;
4406                         }
4407
4408                         if (adapter->link_speed != SPEED_1000)
4409                                 goto no_wait;
4410
4411                         /* wait for Remote receiver status OK */
4412 retry_read_status:
4413                         if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
4414                                               &phy_data)) {
4415                                 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
4416                                     retry_count) {
4417                                         msleep(100);
4418                                         retry_count--;
4419                                         goto retry_read_status;
4420                                 } else if (!retry_count) {
4421                                         netdev_err(netdev, "exceed max 2 second\n");
4422                                 }
4423                         } else {
4424                                 netdev_err(netdev, "read 1000Base-T Status Reg\n");
4425                         }
4426 no_wait:
4427                         netif_carrier_on(netdev);
4428
4429                         /* link state has changed, schedule phy info update */
4430                         if (!test_bit(__IGC_DOWN, &adapter->state))
4431                                 mod_timer(&adapter->phy_info_timer,
4432                                           round_jiffies(jiffies + 2 * HZ));
4433                 }
4434         } else {
4435                 if (netif_carrier_ok(netdev)) {
4436                         adapter->link_speed = 0;
4437                         adapter->link_duplex = 0;
4438
4439                         /* Links status message must follow this format */
4440                         netdev_info(netdev, "NIC Link is Down\n");
4441                         netif_carrier_off(netdev);
4442
4443                         /* link state has changed, schedule phy info update */
4444                         if (!test_bit(__IGC_DOWN, &adapter->state))
4445                                 mod_timer(&adapter->phy_info_timer,
4446                                           round_jiffies(jiffies + 2 * HZ));
4447
4448                         /* link is down, time to check for alternate media */
4449                         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
4450                                 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
4451                                         schedule_work(&adapter->reset_task);
4452                                         /* return immediately */
4453                                         return;
4454                                 }
4455                         }
4456                         pm_schedule_suspend(netdev->dev.parent,
4457                                             MSEC_PER_SEC * 5);
4458
4459                 /* also check for alternate media here */
4460                 } else if (!netif_carrier_ok(netdev) &&
4461                            (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
4462                         if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
4463                                 schedule_work(&adapter->reset_task);
4464                                 /* return immediately */
4465                                 return;
4466                         }
4467                 }
4468         }
4469
4470         spin_lock(&adapter->stats64_lock);
4471         igc_update_stats(adapter);
4472         spin_unlock(&adapter->stats64_lock);
4473
4474         for (i = 0; i < adapter->num_tx_queues; i++) {
4475                 struct igc_ring *tx_ring = adapter->tx_ring[i];
4476
4477                 if (!netif_carrier_ok(netdev)) {
4478                         /* We've lost link, so the controller stops DMA,
4479                          * but we've got queued Tx work that's never going
4480                          * to get done, so reset controller to flush Tx.
4481                          * (Do the reset outside of interrupt context).
4482                          */
4483                         if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
4484                                 adapter->tx_timeout_count++;
4485                                 schedule_work(&adapter->reset_task);
4486                                 /* return immediately since reset is imminent */
4487                                 return;
4488                         }
4489                 }
4490
4491                 /* Force detection of hung controller every watchdog period */
4492                 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4493         }
4494
4495         /* Cause software interrupt to ensure Rx ring is cleaned */
4496         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
4497                 u32 eics = 0;
4498
4499                 for (i = 0; i < adapter->num_q_vectors; i++)
4500                         eics |= adapter->q_vector[i]->eims_value;
4501                 wr32(IGC_EICS, eics);
4502         } else {
4503                 wr32(IGC_ICS, IGC_ICS_RXDMT0);
4504         }
4505
4506         igc_ptp_tx_hang(adapter);
4507
4508         /* Reset the timer */
4509         if (!test_bit(__IGC_DOWN, &adapter->state)) {
4510                 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
4511                         mod_timer(&adapter->watchdog_timer,
4512                                   round_jiffies(jiffies +  HZ));
4513                 else
4514                         mod_timer(&adapter->watchdog_timer,
4515                                   round_jiffies(jiffies + 2 * HZ));
4516         }
4517 }
4518
4519 /**
4520  * igc_intr_msi - Interrupt Handler
4521  * @irq: interrupt number
4522  * @data: pointer to a network interface device structure
4523  */
4524 static irqreturn_t igc_intr_msi(int irq, void *data)
4525 {
4526         struct igc_adapter *adapter = data;
4527         struct igc_q_vector *q_vector = adapter->q_vector[0];
4528         struct igc_hw *hw = &adapter->hw;
4529         /* read ICR disables interrupts using IAM */
4530         u32 icr = rd32(IGC_ICR);
4531
4532         igc_write_itr(q_vector);
4533
4534         if (icr & IGC_ICR_DRSTA)
4535                 schedule_work(&adapter->reset_task);
4536
4537         if (icr & IGC_ICR_DOUTSYNC) {
4538                 /* HW is reporting DMA is out of sync */
4539                 adapter->stats.doosync++;
4540         }
4541
4542         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
4543                 hw->mac.get_link_status = 1;
4544                 if (!test_bit(__IGC_DOWN, &adapter->state))
4545                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4546         }
4547
4548         if (icr & IGC_ICR_TS)
4549                 igc_tsync_interrupt(adapter);
4550
4551         napi_schedule(&q_vector->napi);
4552
4553         return IRQ_HANDLED;
4554 }
4555
4556 /**
4557  * igc_intr - Legacy Interrupt Handler
4558  * @irq: interrupt number
4559  * @data: pointer to a network interface device structure
4560  */
4561 static irqreturn_t igc_intr(int irq, void *data)
4562 {
4563         struct igc_adapter *adapter = data;
4564         struct igc_q_vector *q_vector = adapter->q_vector[0];
4565         struct igc_hw *hw = &adapter->hw;
4566         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
4567          * need for the IMC write
4568          */
4569         u32 icr = rd32(IGC_ICR);
4570
4571         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4572          * not set, then the adapter didn't send an interrupt
4573          */
4574         if (!(icr & IGC_ICR_INT_ASSERTED))
4575                 return IRQ_NONE;
4576
4577         igc_write_itr(q_vector);
4578
4579         if (icr & IGC_ICR_DRSTA)
4580                 schedule_work(&adapter->reset_task);
4581
4582         if (icr & IGC_ICR_DOUTSYNC) {
4583                 /* HW is reporting DMA is out of sync */
4584                 adapter->stats.doosync++;
4585         }
4586
4587         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
4588                 hw->mac.get_link_status = 1;
4589                 /* guard against interrupt when we're going down */
4590                 if (!test_bit(__IGC_DOWN, &adapter->state))
4591                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4592         }
4593
4594         if (icr & IGC_ICR_TS)
4595                 igc_tsync_interrupt(adapter);
4596
4597         napi_schedule(&q_vector->napi);
4598
4599         return IRQ_HANDLED;
4600 }
4601
4602 static void igc_free_irq(struct igc_adapter *adapter)
4603 {
4604         if (adapter->msix_entries) {
4605                 int vector = 0, i;
4606
4607                 free_irq(adapter->msix_entries[vector++].vector, adapter);
4608
4609                 for (i = 0; i < adapter->num_q_vectors; i++)
4610                         free_irq(adapter->msix_entries[vector++].vector,
4611                                  adapter->q_vector[i]);
4612         } else {
4613                 free_irq(adapter->pdev->irq, adapter);
4614         }
4615 }
4616
4617 /**
4618  * igc_request_irq - initialize interrupts
4619  * @adapter: Pointer to adapter structure
4620  *
4621  * Attempts to configure interrupts using the best available
4622  * capabilities of the hardware and kernel.
4623  */
4624 static int igc_request_irq(struct igc_adapter *adapter)
4625 {
4626         struct net_device *netdev = adapter->netdev;
4627         struct pci_dev *pdev = adapter->pdev;
4628         int err = 0;
4629
4630         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
4631                 err = igc_request_msix(adapter);
4632                 if (!err)
4633                         goto request_done;
4634                 /* fall back to MSI */
4635                 igc_free_all_tx_resources(adapter);
4636                 igc_free_all_rx_resources(adapter);
4637
4638                 igc_clear_interrupt_scheme(adapter);
4639                 err = igc_init_interrupt_scheme(adapter, false);
4640                 if (err)
4641                         goto request_done;
4642                 igc_setup_all_tx_resources(adapter);
4643                 igc_setup_all_rx_resources(adapter);
4644                 igc_configure(adapter);
4645         }
4646
4647         igc_assign_vector(adapter->q_vector[0], 0);
4648
4649         if (adapter->flags & IGC_FLAG_HAS_MSI) {
4650                 err = request_irq(pdev->irq, &igc_intr_msi, 0,
4651                                   netdev->name, adapter);
4652                 if (!err)
4653                         goto request_done;
4654
4655                 /* fall back to legacy interrupts */
4656                 igc_reset_interrupt_capability(adapter);
4657                 adapter->flags &= ~IGC_FLAG_HAS_MSI;
4658         }
4659
4660         err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
4661                           netdev->name, adapter);
4662
4663         if (err)
4664                 netdev_err(netdev, "Error %d getting interrupt\n", err);
4665
4666 request_done:
4667         return err;
4668 }
4669
4670 /**
4671  * __igc_open - Called when a network interface is made active
4672  * @netdev: network interface device structure
4673  * @resuming: boolean indicating if the device is resuming
4674  *
4675  * Returns 0 on success, negative value on failure
4676  *
4677  * The open entry point is called when a network interface is made
4678  * active by the system (IFF_UP).  At this point all resources needed
4679  * for transmit and receive operations are allocated, the interrupt
4680  * handler is registered with the OS, the watchdog timer is started,
4681  * and the stack is notified that the interface is ready.
4682  */
4683 static int __igc_open(struct net_device *netdev, bool resuming)
4684 {
4685         struct igc_adapter *adapter = netdev_priv(netdev);
4686         struct pci_dev *pdev = adapter->pdev;
4687         struct igc_hw *hw = &adapter->hw;
4688         int err = 0;
4689         int i = 0;
4690
4691         /* disallow open during test */
4692
4693         if (test_bit(__IGC_TESTING, &adapter->state)) {
4694                 WARN_ON(resuming);
4695                 return -EBUSY;
4696         }
4697
4698         if (!resuming)
4699                 pm_runtime_get_sync(&pdev->dev);
4700
4701         netif_carrier_off(netdev);
4702
4703         /* allocate transmit descriptors */
4704         err = igc_setup_all_tx_resources(adapter);
4705         if (err)
4706                 goto err_setup_tx;
4707
4708         /* allocate receive descriptors */
4709         err = igc_setup_all_rx_resources(adapter);
4710         if (err)
4711                 goto err_setup_rx;
4712
4713         igc_power_up_link(adapter);
4714
4715         igc_configure(adapter);
4716
4717         err = igc_request_irq(adapter);
4718         if (err)
4719                 goto err_req_irq;
4720
4721         /* Notify the stack of the actual queue counts. */
4722         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
4723         if (err)
4724                 goto err_set_queues;
4725
4726         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
4727         if (err)
4728                 goto err_set_queues;
4729
4730         clear_bit(__IGC_DOWN, &adapter->state);
4731
4732         for (i = 0; i < adapter->num_q_vectors; i++)
4733                 napi_enable(&adapter->q_vector[i]->napi);
4734
4735         /* Clear any pending interrupts. */
4736         rd32(IGC_ICR);
4737         igc_irq_enable(adapter);
4738
4739         if (!resuming)
4740                 pm_runtime_put(&pdev->dev);
4741
4742         netif_tx_start_all_queues(netdev);
4743
4744         /* start the watchdog. */
4745         hw->mac.get_link_status = 1;
4746         schedule_work(&adapter->watchdog_task);
4747
4748         return IGC_SUCCESS;
4749
4750 err_set_queues:
4751         igc_free_irq(adapter);
4752 err_req_irq:
4753         igc_release_hw_control(adapter);
4754         igc_power_down_phy_copper_base(&adapter->hw);
4755         igc_free_all_rx_resources(adapter);
4756 err_setup_rx:
4757         igc_free_all_tx_resources(adapter);
4758 err_setup_tx:
4759         igc_reset(adapter);
4760         if (!resuming)
4761                 pm_runtime_put(&pdev->dev);
4762
4763         return err;
4764 }
4765
4766 int igc_open(struct net_device *netdev)
4767 {
4768         return __igc_open(netdev, false);
4769 }
4770
4771 /**
4772  * __igc_close - Disables a network interface
4773  * @netdev: network interface device structure
4774  * @suspending: boolean indicating the device is suspending
4775  *
4776  * Returns 0, this is not allowed to fail
4777  *
4778  * The close entry point is called when an interface is de-activated
4779  * by the OS.  The hardware is still under the driver's control, but
4780  * needs to be disabled.  A global MAC reset is issued to stop the
4781  * hardware, and all transmit and receive resources are freed.
4782  */
4783 static int __igc_close(struct net_device *netdev, bool suspending)
4784 {
4785         struct igc_adapter *adapter = netdev_priv(netdev);
4786         struct pci_dev *pdev = adapter->pdev;
4787
4788         WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
4789
4790         if (!suspending)
4791                 pm_runtime_get_sync(&pdev->dev);
4792
4793         igc_down(adapter);
4794
4795         igc_release_hw_control(adapter);
4796
4797         igc_free_irq(adapter);
4798
4799         igc_free_all_tx_resources(adapter);
4800         igc_free_all_rx_resources(adapter);
4801
4802         if (!suspending)
4803                 pm_runtime_put_sync(&pdev->dev);
4804
4805         return 0;
4806 }
4807
4808 int igc_close(struct net_device *netdev)
4809 {
4810         if (netif_device_present(netdev) || netdev->dismantle)
4811                 return __igc_close(netdev, false);
4812         return 0;
4813 }
4814
4815 /**
4816  * igc_ioctl - Access the hwtstamp interface
4817  * @netdev: network interface device structure
4818  * @ifr: interface request data
4819  * @cmd: ioctl command
4820  **/
4821 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4822 {
4823         switch (cmd) {
4824         case SIOCGHWTSTAMP:
4825                 return igc_ptp_get_ts_config(netdev, ifr);
4826         case SIOCSHWTSTAMP:
4827                 return igc_ptp_set_ts_config(netdev, ifr);
4828         default:
4829                 return -EOPNOTSUPP;
4830         }
4831 }
4832
4833 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
4834                                       bool enable)
4835 {
4836         struct igc_ring *ring;
4837         int i;
4838
4839         if (queue < 0 || queue >= adapter->num_tx_queues)
4840                 return -EINVAL;
4841
4842         ring = adapter->tx_ring[queue];
4843         ring->launchtime_enable = enable;
4844
4845         if (adapter->base_time)
4846                 return 0;
4847
4848         adapter->cycle_time = NSEC_PER_SEC;
4849
4850         for (i = 0; i < adapter->num_tx_queues; i++) {
4851                 ring = adapter->tx_ring[i];
4852                 ring->start_time = 0;
4853                 ring->end_time = NSEC_PER_SEC;
4854         }
4855
4856         return 0;
4857 }
4858
4859 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
4860 {
4861         struct timespec64 b;
4862
4863         b = ktime_to_timespec64(base_time);
4864
4865         return timespec64_compare(now, &b) > 0;
4866 }
4867
4868 static bool validate_schedule(struct igc_adapter *adapter,
4869                               const struct tc_taprio_qopt_offload *qopt)
4870 {
4871         int queue_uses[IGC_MAX_TX_QUEUES] = { };
4872         struct timespec64 now;
4873         size_t n;
4874
4875         if (qopt->cycle_time_extension)
4876                 return false;
4877
4878         igc_ptp_read(adapter, &now);
4879
4880         /* If we program the controller's BASET registers with a time
4881          * in the future, it will hold all the packets until that
4882          * time, causing a lot of TX Hangs, so to avoid that, we
4883          * reject schedules that would start in the future.
4884          */
4885         if (!is_base_time_past(qopt->base_time, &now))
4886                 return false;
4887
4888         for (n = 0; n < qopt->num_entries; n++) {
4889                 const struct tc_taprio_sched_entry *e, *prev;
4890                 int i;
4891
4892                 prev = n ? &qopt->entries[n - 1] : NULL;
4893                 e = &qopt->entries[n];
4894
4895                 /* i225 only supports "global" frame preemption
4896                  * settings.
4897                  */
4898                 if (e->command != TC_TAPRIO_CMD_SET_GATES)
4899                         return false;
4900
4901                 for (i = 0; i < adapter->num_tx_queues; i++)
4902                         if (e->gate_mask & BIT(i)) {
4903                                 queue_uses[i]++;
4904
4905                                 /* There are limitations: A single queue cannot
4906                                  * be opened and closed multiple times per cycle
4907                                  * unless the gate stays open. Check for it.
4908                                  */
4909                                 if (queue_uses[i] > 1 &&
4910                                     !(prev->gate_mask & BIT(i)))
4911                                         return false;
4912                         }
4913         }
4914
4915         return true;
4916 }
4917
4918 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
4919                                      struct tc_etf_qopt_offload *qopt)
4920 {
4921         struct igc_hw *hw = &adapter->hw;
4922         int err;
4923
4924         if (hw->mac.type != igc_i225)
4925                 return -EOPNOTSUPP;
4926
4927         err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
4928         if (err)
4929                 return err;
4930
4931         return igc_tsn_offload_apply(adapter);
4932 }
4933
4934 static int igc_save_qbv_schedule(struct igc_adapter *adapter,
4935                                  struct tc_taprio_qopt_offload *qopt)
4936 {
4937         bool queue_configured[IGC_MAX_TX_QUEUES] = { };
4938         u32 start_time = 0, end_time = 0;
4939         size_t n;
4940         int i;
4941
4942         if (!qopt->enable) {
4943                 adapter->base_time = 0;
4944                 return 0;
4945         }
4946
4947         if (qopt->base_time < 0)
4948                 return -ERANGE;
4949
4950         if (adapter->base_time)
4951                 return -EALREADY;
4952
4953         if (!validate_schedule(adapter, qopt))
4954                 return -EINVAL;
4955
4956         adapter->cycle_time = qopt->cycle_time;
4957         adapter->base_time = qopt->base_time;
4958
4959         for (n = 0; n < qopt->num_entries; n++) {
4960                 struct tc_taprio_sched_entry *e = &qopt->entries[n];
4961
4962                 end_time += e->interval;
4963
4964                 /* If any of the conditions below are true, we need to manually
4965                  * control the end time of the cycle.
4966                  * 1. Qbv users can specify a cycle time that is not equal
4967                  * to the total GCL intervals. Hence, recalculation is
4968                  * necessary here to exclude the time interval that
4969                  * exceeds the cycle time.
4970                  * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
4971                  * once the end of the list is reached, it will switch
4972                  * to the END_OF_CYCLE state and leave the gates in the
4973                  * same state until the next cycle is started.
4974                  */
4975                 if (end_time > adapter->cycle_time ||
4976                     n + 1 == qopt->num_entries)
4977                         end_time = adapter->cycle_time;
4978
4979                 for (i = 0; i < adapter->num_tx_queues; i++) {
4980                         struct igc_ring *ring = adapter->tx_ring[i];
4981
4982                         if (!(e->gate_mask & BIT(i)))
4983                                 continue;
4984
4985                         /* Check whether a queue stays open for more than one
4986                          * entry. If so, keep the start and advance the end
4987                          * time.
4988                          */
4989                         if (!queue_configured[i])
4990                                 ring->start_time = start_time;
4991                         ring->end_time = end_time;
4992
4993                         queue_configured[i] = true;
4994                 }
4995
4996                 start_time += e->interval;
4997         }
4998
4999         /* Check whether a queue gets configured.
5000          * If not, set the start and end time to be end time.
5001          */
5002         for (i = 0; i < adapter->num_tx_queues; i++) {
5003                 if (!queue_configured[i]) {
5004                         struct igc_ring *ring = adapter->tx_ring[i];
5005
5006                         ring->start_time = end_time;
5007                         ring->end_time = end_time;
5008                 }
5009         }
5010
5011         return 0;
5012 }
5013
5014 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
5015                                          struct tc_taprio_qopt_offload *qopt)
5016 {
5017         struct igc_hw *hw = &adapter->hw;
5018         int err;
5019
5020         if (hw->mac.type != igc_i225)
5021                 return -EOPNOTSUPP;
5022
5023         err = igc_save_qbv_schedule(adapter, qopt);
5024         if (err)
5025                 return err;
5026
5027         return igc_tsn_offload_apply(adapter);
5028 }
5029
5030 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
5031                         void *type_data)
5032 {
5033         struct igc_adapter *adapter = netdev_priv(dev);
5034
5035         switch (type) {
5036         case TC_SETUP_QDISC_TAPRIO:
5037                 return igc_tsn_enable_qbv_scheduling(adapter, type_data);
5038
5039         case TC_SETUP_QDISC_ETF:
5040                 return igc_tsn_enable_launchtime(adapter, type_data);
5041
5042         default:
5043                 return -EOPNOTSUPP;
5044         }
5045 }
5046
5047 static const struct net_device_ops igc_netdev_ops = {
5048         .ndo_open               = igc_open,
5049         .ndo_stop               = igc_close,
5050         .ndo_start_xmit         = igc_xmit_frame,
5051         .ndo_set_rx_mode        = igc_set_rx_mode,
5052         .ndo_set_mac_address    = igc_set_mac,
5053         .ndo_change_mtu         = igc_change_mtu,
5054         .ndo_get_stats64        = igc_get_stats64,
5055         .ndo_fix_features       = igc_fix_features,
5056         .ndo_set_features       = igc_set_features,
5057         .ndo_features_check     = igc_features_check,
5058         .ndo_do_ioctl           = igc_ioctl,
5059         .ndo_setup_tc           = igc_setup_tc,
5060 };
5061
5062 /* PCIe configuration access */
5063 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
5064 {
5065         struct igc_adapter *adapter = hw->back;
5066
5067         pci_read_config_word(adapter->pdev, reg, value);
5068 }
5069
5070 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
5071 {
5072         struct igc_adapter *adapter = hw->back;
5073
5074         pci_write_config_word(adapter->pdev, reg, *value);
5075 }
5076
5077 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
5078 {
5079         struct igc_adapter *adapter = hw->back;
5080
5081         if (!pci_is_pcie(adapter->pdev))
5082                 return -IGC_ERR_CONFIG;
5083
5084         pcie_capability_read_word(adapter->pdev, reg, value);
5085
5086         return IGC_SUCCESS;
5087 }
5088
5089 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
5090 {
5091         struct igc_adapter *adapter = hw->back;
5092
5093         if (!pci_is_pcie(adapter->pdev))
5094                 return -IGC_ERR_CONFIG;
5095
5096         pcie_capability_write_word(adapter->pdev, reg, *value);
5097
5098         return IGC_SUCCESS;
5099 }
5100
5101 u32 igc_rd32(struct igc_hw *hw, u32 reg)
5102 {
5103         struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
5104         u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
5105         u32 value = 0;
5106
5107         if (IGC_REMOVED(hw_addr))
5108                 return ~value;
5109
5110         value = readl(&hw_addr[reg]);
5111
5112         /* reads should not return all F's */
5113         if (!(~value) && (!reg || !(~readl(hw_addr)))) {
5114                 struct net_device *netdev = igc->netdev;
5115
5116                 hw->hw_addr = NULL;
5117                 netif_device_detach(netdev);
5118                 netdev_err(netdev, "PCIe link lost, device now detached\n");
5119                 WARN(pci_device_is_present(igc->pdev),
5120                      "igc: Failed to read reg 0x%x!\n", reg);
5121         }
5122
5123         return value;
5124 }
5125
5126 int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx)
5127 {
5128         struct igc_mac_info *mac = &adapter->hw.mac;
5129
5130         mac->autoneg = 0;
5131
5132         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5133          * for the switch() below to work
5134          */
5135         if ((spd & 1) || (dplx & ~1))
5136                 goto err_inval;
5137
5138         switch (spd + dplx) {
5139         case SPEED_10 + DUPLEX_HALF:
5140                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5141                 break;
5142         case SPEED_10 + DUPLEX_FULL:
5143                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5144                 break;
5145         case SPEED_100 + DUPLEX_HALF:
5146                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5147                 break;
5148         case SPEED_100 + DUPLEX_FULL:
5149                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5150                 break;
5151         case SPEED_1000 + DUPLEX_FULL:
5152                 mac->autoneg = 1;
5153                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5154                 break;
5155         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5156                 goto err_inval;
5157         case SPEED_2500 + DUPLEX_FULL:
5158                 mac->autoneg = 1;
5159                 adapter->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL;
5160                 break;
5161         case SPEED_2500 + DUPLEX_HALF: /* not supported */
5162         default:
5163                 goto err_inval;
5164         }
5165
5166         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5167         adapter->hw.phy.mdix = AUTO_ALL_MODES;
5168
5169         return 0;
5170
5171 err_inval:
5172         netdev_err(adapter->netdev, "Unsupported Speed/Duplex configuration\n");
5173         return -EINVAL;
5174 }
5175
5176 /**
5177  * igc_probe - Device Initialization Routine
5178  * @pdev: PCI device information struct
5179  * @ent: entry in igc_pci_tbl
5180  *
5181  * Returns 0 on success, negative on failure
5182  *
5183  * igc_probe initializes an adapter identified by a pci_dev structure.
5184  * The OS initialization, configuring the adapter private structure,
5185  * and a hardware reset occur.
5186  */
5187 static int igc_probe(struct pci_dev *pdev,
5188                      const struct pci_device_id *ent)
5189 {
5190         struct igc_adapter *adapter;
5191         struct net_device *netdev;
5192         struct igc_hw *hw;
5193         const struct igc_info *ei = igc_info_tbl[ent->driver_data];
5194         int err, pci_using_dac;
5195
5196         err = pci_enable_device_mem(pdev);
5197         if (err)
5198                 return err;
5199
5200         pci_using_dac = 0;
5201         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5202         if (!err) {
5203                 pci_using_dac = 1;
5204         } else {
5205                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5206                 if (err) {
5207                         dev_err(&pdev->dev,
5208                                 "No usable DMA configuration, aborting\n");
5209                         goto err_dma;
5210                 }
5211         }
5212
5213         err = pci_request_mem_regions(pdev, igc_driver_name);
5214         if (err)
5215                 goto err_pci_reg;
5216
5217         pci_enable_pcie_error_reporting(pdev);
5218
5219         err = pci_enable_ptm(pdev, NULL);
5220         if (err < 0)
5221                 dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
5222
5223         pci_set_master(pdev);
5224
5225         err = -ENOMEM;
5226         netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
5227                                    IGC_MAX_TX_QUEUES);
5228
5229         if (!netdev)
5230                 goto err_alloc_etherdev;
5231
5232         SET_NETDEV_DEV(netdev, &pdev->dev);
5233
5234         pci_set_drvdata(pdev, netdev);
5235         adapter = netdev_priv(netdev);
5236         adapter->netdev = netdev;
5237         adapter->pdev = pdev;
5238         hw = &adapter->hw;
5239         hw->back = adapter;
5240         adapter->port_num = hw->bus.func;
5241         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
5242
5243         err = pci_save_state(pdev);
5244         if (err)
5245                 goto err_ioremap;
5246
5247         err = -EIO;
5248         adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
5249                                    pci_resource_len(pdev, 0));
5250         if (!adapter->io_addr)
5251                 goto err_ioremap;
5252
5253         /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
5254         hw->hw_addr = adapter->io_addr;
5255
5256         netdev->netdev_ops = &igc_netdev_ops;
5257         igc_ethtool_set_ops(netdev);
5258         netdev->watchdog_timeo = 5 * HZ;
5259
5260         netdev->mem_start = pci_resource_start(pdev, 0);
5261         netdev->mem_end = pci_resource_end(pdev, 0);
5262
5263         /* PCI config space info */
5264         hw->vendor_id = pdev->vendor;
5265         hw->device_id = pdev->device;
5266         hw->revision_id = pdev->revision;
5267         hw->subsystem_vendor_id = pdev->subsystem_vendor;
5268         hw->subsystem_device_id = pdev->subsystem_device;
5269
5270         /* Copy the default MAC and PHY function pointers */
5271         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
5272         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
5273
5274         /* Initialize skew-specific constants */
5275         err = ei->get_invariants(hw);
5276         if (err)
5277                 goto err_sw_init;
5278
5279         /* Add supported features to the features list*/
5280         netdev->features |= NETIF_F_SG;
5281         netdev->features |= NETIF_F_TSO;
5282         netdev->features |= NETIF_F_TSO6;
5283         netdev->features |= NETIF_F_TSO_ECN;
5284         netdev->features |= NETIF_F_RXHASH;
5285         netdev->features |= NETIF_F_RXCSUM;
5286         netdev->features |= NETIF_F_HW_CSUM;
5287         netdev->features |= NETIF_F_SCTP_CRC;
5288         netdev->features |= NETIF_F_HW_TC;
5289
5290 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
5291                                   NETIF_F_GSO_GRE_CSUM | \
5292                                   NETIF_F_GSO_IPXIP4 | \
5293                                   NETIF_F_GSO_IPXIP6 | \
5294                                   NETIF_F_GSO_UDP_TUNNEL | \
5295                                   NETIF_F_GSO_UDP_TUNNEL_CSUM)
5296
5297         netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
5298         netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
5299
5300         /* setup the private structure */
5301         err = igc_sw_init(adapter);
5302         if (err)
5303                 goto err_sw_init;
5304
5305         /* copy netdev features into list of user selectable features */
5306         netdev->hw_features |= NETIF_F_NTUPLE;
5307         netdev->hw_features |= netdev->features;
5308
5309         if (pci_using_dac)
5310                 netdev->features |= NETIF_F_HIGHDMA;
5311
5312         /* MTU range: 68 - 9216 */
5313         netdev->min_mtu = ETH_MIN_MTU;
5314         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
5315
5316         /* before reading the NVM, reset the controller to put the device in a
5317          * known good starting state
5318          */
5319         hw->mac.ops.reset_hw(hw);
5320
5321         if (igc_get_flash_presence_i225(hw)) {
5322                 if (hw->nvm.ops.validate(hw) < 0) {
5323                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
5324                         err = -EIO;
5325                         goto err_eeprom;
5326                 }
5327         }
5328
5329         if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
5330                 /* copy the MAC address out of the NVM */
5331                 if (hw->mac.ops.read_mac_addr(hw))
5332                         dev_err(&pdev->dev, "NVM Read Error\n");
5333         }
5334
5335         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
5336
5337         if (!is_valid_ether_addr(netdev->dev_addr)) {
5338                 dev_err(&pdev->dev, "Invalid MAC Address\n");
5339                 err = -EIO;
5340                 goto err_eeprom;
5341         }
5342
5343         /* configure RXPBSIZE and TXPBSIZE */
5344         wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
5345         wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
5346
5347         timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
5348         timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
5349
5350         INIT_WORK(&adapter->reset_task, igc_reset_task);
5351         INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
5352
5353         /* Initialize link properties that are user-changeable */
5354         adapter->fc_autoneg = true;
5355         hw->mac.autoneg = true;
5356         hw->phy.autoneg_advertised = 0xaf;
5357
5358         hw->fc.requested_mode = igc_fc_default;
5359         hw->fc.current_mode = igc_fc_default;
5360
5361         /* By default, support wake on port A */
5362         adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
5363
5364         /* initialize the wol settings based on the eeprom settings */
5365         if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
5366                 adapter->wol |= IGC_WUFC_MAG;
5367
5368         device_set_wakeup_enable(&adapter->pdev->dev,
5369                                  adapter->flags & IGC_FLAG_WOL_SUPPORTED);
5370
5371         igc_ptp_init(adapter);
5372
5373         /* reset the hardware with the new settings */
5374         igc_reset(adapter);
5375
5376         /* let the f/w know that the h/w is now under the control of the
5377          * driver.
5378          */
5379         igc_get_hw_control(adapter);
5380
5381         strncpy(netdev->name, "eth%d", IFNAMSIZ);
5382         err = register_netdev(netdev);
5383         if (err)
5384                 goto err_register;
5385
5386          /* carrier off reporting is important to ethtool even BEFORE open */
5387         netif_carrier_off(netdev);
5388
5389         /* Check if Media Autosense is enabled */
5390         adapter->ei = *ei;
5391
5392         /* print pcie link status and MAC address */
5393         pcie_print_link_status(pdev);
5394         netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
5395
5396         dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
5397         /* Disable EEE for internal PHY devices */
5398         hw->dev_spec._base.eee_enable = false;
5399         adapter->flags &= ~IGC_FLAG_EEE;
5400         igc_set_eee_i225(hw, false, false, false);
5401
5402         pm_runtime_put_noidle(&pdev->dev);
5403
5404         return 0;
5405
5406 err_register:
5407         igc_release_hw_control(adapter);
5408 err_eeprom:
5409         if (!igc_check_reset_block(hw))
5410                 igc_reset_phy(hw);
5411 err_sw_init:
5412         igc_clear_interrupt_scheme(adapter);
5413         iounmap(adapter->io_addr);
5414 err_ioremap:
5415         free_netdev(netdev);
5416 err_alloc_etherdev:
5417         pci_disable_pcie_error_reporting(pdev);
5418         pci_release_mem_regions(pdev);
5419 err_pci_reg:
5420 err_dma:
5421         pci_disable_device(pdev);
5422         return err;
5423 }
5424
5425 /**
5426  * igc_remove - Device Removal Routine
5427  * @pdev: PCI device information struct
5428  *
5429  * igc_remove is called by the PCI subsystem to alert the driver
5430  * that it should release a PCI device.  This could be caused by a
5431  * Hot-Plug event, or because the driver is going to be removed from
5432  * memory.
5433  */
5434 static void igc_remove(struct pci_dev *pdev)
5435 {
5436         struct net_device *netdev = pci_get_drvdata(pdev);
5437         struct igc_adapter *adapter = netdev_priv(netdev);
5438
5439         pm_runtime_get_noresume(&pdev->dev);
5440
5441         igc_flush_nfc_rules(adapter);
5442
5443         igc_ptp_stop(adapter);
5444
5445         set_bit(__IGC_DOWN, &adapter->state);
5446
5447         del_timer_sync(&adapter->watchdog_timer);
5448         del_timer_sync(&adapter->phy_info_timer);
5449
5450         cancel_work_sync(&adapter->reset_task);
5451         cancel_work_sync(&adapter->watchdog_task);
5452
5453         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5454          * would have already happened in close and is redundant.
5455          */
5456         igc_release_hw_control(adapter);
5457         unregister_netdev(netdev);
5458
5459         igc_clear_interrupt_scheme(adapter);
5460         pci_iounmap(pdev, adapter->io_addr);
5461         pci_release_mem_regions(pdev);
5462
5463         free_netdev(netdev);
5464
5465         pci_disable_pcie_error_reporting(pdev);
5466
5467         pci_disable_device(pdev);
5468 }
5469
5470 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
5471                           bool runtime)
5472 {
5473         struct net_device *netdev = pci_get_drvdata(pdev);
5474         struct igc_adapter *adapter = netdev_priv(netdev);
5475         u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
5476         struct igc_hw *hw = &adapter->hw;
5477         u32 ctrl, rctl, status;
5478         bool wake;
5479
5480         rtnl_lock();
5481         netif_device_detach(netdev);
5482
5483         if (netif_running(netdev))
5484                 __igc_close(netdev, true);
5485
5486         igc_ptp_suspend(adapter);
5487
5488         igc_clear_interrupt_scheme(adapter);
5489         rtnl_unlock();
5490
5491         status = rd32(IGC_STATUS);
5492         if (status & IGC_STATUS_LU)
5493                 wufc &= ~IGC_WUFC_LNKC;
5494
5495         if (wufc) {
5496                 igc_setup_rctl(adapter);
5497                 igc_set_rx_mode(netdev);
5498
5499                 /* turn on all-multi mode if wake on multicast is enabled */
5500                 if (wufc & IGC_WUFC_MC) {
5501                         rctl = rd32(IGC_RCTL);
5502                         rctl |= IGC_RCTL_MPE;
5503                         wr32(IGC_RCTL, rctl);
5504                 }
5505
5506                 ctrl = rd32(IGC_CTRL);
5507                 ctrl |= IGC_CTRL_ADVD3WUC;
5508                 wr32(IGC_CTRL, ctrl);
5509
5510                 /* Allow time for pending master requests to run */
5511                 igc_disable_pcie_master(hw);
5512
5513                 wr32(IGC_WUC, IGC_WUC_PME_EN);
5514                 wr32(IGC_WUFC, wufc);
5515         } else {
5516                 wr32(IGC_WUC, 0);
5517                 wr32(IGC_WUFC, 0);
5518         }
5519
5520         wake = wufc || adapter->en_mng_pt;
5521         if (!wake)
5522                 igc_power_down_phy_copper_base(&adapter->hw);
5523         else
5524                 igc_power_up_link(adapter);
5525
5526         if (enable_wake)
5527                 *enable_wake = wake;
5528
5529         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5530          * would have already happened in close and is redundant.
5531          */
5532         igc_release_hw_control(adapter);
5533
5534         pci_disable_device(pdev);
5535
5536         return 0;
5537 }
5538
5539 #ifdef CONFIG_PM
5540 static int __maybe_unused igc_runtime_suspend(struct device *dev)
5541 {
5542         return __igc_shutdown(to_pci_dev(dev), NULL, 1);
5543 }
5544
5545 static void igc_deliver_wake_packet(struct net_device *netdev)
5546 {
5547         struct igc_adapter *adapter = netdev_priv(netdev);
5548         struct igc_hw *hw = &adapter->hw;
5549         struct sk_buff *skb;
5550         u32 wupl;
5551
5552         wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
5553
5554         /* WUPM stores only the first 128 bytes of the wake packet.
5555          * Read the packet only if we have the whole thing.
5556          */
5557         if (wupl == 0 || wupl > IGC_WUPM_BYTES)
5558                 return;
5559
5560         skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
5561         if (!skb)
5562                 return;
5563
5564         skb_put(skb, wupl);
5565
5566         /* Ensure reads are 32-bit aligned */
5567         wupl = roundup(wupl, 4);
5568
5569         memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
5570
5571         skb->protocol = eth_type_trans(skb, netdev);
5572         netif_rx(skb);
5573 }
5574
5575 static int __maybe_unused igc_resume(struct device *dev)
5576 {
5577         struct pci_dev *pdev = to_pci_dev(dev);
5578         struct net_device *netdev = pci_get_drvdata(pdev);
5579         struct igc_adapter *adapter = netdev_priv(netdev);
5580         struct igc_hw *hw = &adapter->hw;
5581         u32 err, val;
5582
5583         pci_set_power_state(pdev, PCI_D0);
5584         pci_restore_state(pdev);
5585         pci_save_state(pdev);
5586
5587         if (!pci_device_is_present(pdev))
5588                 return -ENODEV;
5589         err = pci_enable_device_mem(pdev);
5590         if (err) {
5591                 netdev_err(netdev, "Cannot enable PCI device from suspend\n");
5592                 return err;
5593         }
5594         pci_set_master(pdev);
5595
5596         pci_enable_wake(pdev, PCI_D3hot, 0);
5597         pci_enable_wake(pdev, PCI_D3cold, 0);
5598
5599         if (igc_init_interrupt_scheme(adapter, true)) {
5600                 netdev_err(netdev, "Unable to allocate memory for queues\n");
5601                 return -ENOMEM;
5602         }
5603
5604         igc_reset(adapter);
5605
5606         /* let the f/w know that the h/w is now under the control of the
5607          * driver.
5608          */
5609         igc_get_hw_control(adapter);
5610
5611         val = rd32(IGC_WUS);
5612         if (val & WAKE_PKT_WUS)
5613                 igc_deliver_wake_packet(netdev);
5614
5615         wr32(IGC_WUS, ~0);
5616
5617         rtnl_lock();
5618         if (!err && netif_running(netdev))
5619                 err = __igc_open(netdev, true);
5620
5621         if (!err)
5622                 netif_device_attach(netdev);
5623         rtnl_unlock();
5624
5625         return err;
5626 }
5627
5628 static int __maybe_unused igc_runtime_resume(struct device *dev)
5629 {
5630         return igc_resume(dev);
5631 }
5632
5633 static int __maybe_unused igc_suspend(struct device *dev)
5634 {
5635         return __igc_shutdown(to_pci_dev(dev), NULL, 0);
5636 }
5637
5638 static int __maybe_unused igc_runtime_idle(struct device *dev)
5639 {
5640         struct net_device *netdev = dev_get_drvdata(dev);
5641         struct igc_adapter *adapter = netdev_priv(netdev);
5642
5643         if (!igc_has_link(adapter))
5644                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
5645
5646         return -EBUSY;
5647 }
5648 #endif /* CONFIG_PM */
5649
5650 static void igc_shutdown(struct pci_dev *pdev)
5651 {
5652         bool wake;
5653
5654         __igc_shutdown(pdev, &wake, 0);
5655
5656         if (system_state == SYSTEM_POWER_OFF) {
5657                 pci_wake_from_d3(pdev, wake);
5658                 pci_set_power_state(pdev, PCI_D3hot);
5659         }
5660 }
5661
5662 /**
5663  *  igc_io_error_detected - called when PCI error is detected
5664  *  @pdev: Pointer to PCI device
5665  *  @state: The current PCI connection state
5666  *
5667  *  This function is called after a PCI bus error affecting
5668  *  this device has been detected.
5669  **/
5670 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
5671                                               pci_channel_state_t state)
5672 {
5673         struct net_device *netdev = pci_get_drvdata(pdev);
5674         struct igc_adapter *adapter = netdev_priv(netdev);
5675
5676         netif_device_detach(netdev);
5677
5678         if (state == pci_channel_io_perm_failure)
5679                 return PCI_ERS_RESULT_DISCONNECT;
5680
5681         if (netif_running(netdev))
5682                 igc_down(adapter);
5683         pci_disable_device(pdev);
5684
5685         /* Request a slot reset. */
5686         return PCI_ERS_RESULT_NEED_RESET;
5687 }
5688
5689 /**
5690  *  igc_io_slot_reset - called after the PCI bus has been reset.
5691  *  @pdev: Pointer to PCI device
5692  *
5693  *  Restart the card from scratch, as if from a cold-boot. Implementation
5694  *  resembles the first-half of the igc_resume routine.
5695  **/
5696 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
5697 {
5698         struct net_device *netdev = pci_get_drvdata(pdev);
5699         struct igc_adapter *adapter = netdev_priv(netdev);
5700         struct igc_hw *hw = &adapter->hw;
5701         pci_ers_result_t result;
5702
5703         if (pci_enable_device_mem(pdev)) {
5704                 netdev_err(netdev, "Could not re-enable PCI device after reset\n");
5705                 result = PCI_ERS_RESULT_DISCONNECT;
5706         } else {
5707                 pci_set_master(pdev);
5708                 pci_restore_state(pdev);
5709                 pci_save_state(pdev);
5710
5711                 pci_enable_wake(pdev, PCI_D3hot, 0);
5712                 pci_enable_wake(pdev, PCI_D3cold, 0);
5713
5714                 /* In case of PCI error, adapter loses its HW address
5715                  * so we should re-assign it here.
5716                  */
5717                 hw->hw_addr = adapter->io_addr;
5718
5719                 igc_reset(adapter);
5720                 wr32(IGC_WUS, ~0);
5721                 result = PCI_ERS_RESULT_RECOVERED;
5722         }
5723
5724         return result;
5725 }
5726
5727 /**
5728  *  igc_io_resume - called when traffic can start to flow again.
5729  *  @pdev: Pointer to PCI device
5730  *
5731  *  This callback is called when the error recovery driver tells us that
5732  *  its OK to resume normal operation. Implementation resembles the
5733  *  second-half of the igc_resume routine.
5734  */
5735 static void igc_io_resume(struct pci_dev *pdev)
5736 {
5737         struct net_device *netdev = pci_get_drvdata(pdev);
5738         struct igc_adapter *adapter = netdev_priv(netdev);
5739
5740         rtnl_lock();
5741         if (netif_running(netdev)) {
5742                 if (igc_open(netdev)) {
5743                         netdev_err(netdev, "igc_open failed after reset\n");
5744                         return;
5745                 }
5746         }
5747
5748         netif_device_attach(netdev);
5749
5750         /* let the f/w know that the h/w is now under the control of the
5751          * driver.
5752          */
5753         igc_get_hw_control(adapter);
5754         rtnl_unlock();
5755 }
5756
5757 static const struct pci_error_handlers igc_err_handler = {
5758         .error_detected = igc_io_error_detected,
5759         .slot_reset = igc_io_slot_reset,
5760         .resume = igc_io_resume,
5761 };
5762
5763 #ifdef CONFIG_PM
5764 static const struct dev_pm_ops igc_pm_ops = {
5765         SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
5766         SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
5767                            igc_runtime_idle)
5768 };
5769 #endif
5770
5771 static struct pci_driver igc_driver = {
5772         .name     = igc_driver_name,
5773         .id_table = igc_pci_tbl,
5774         .probe    = igc_probe,
5775         .remove   = igc_remove,
5776 #ifdef CONFIG_PM
5777         .driver.pm = &igc_pm_ops,
5778 #endif
5779         .shutdown = igc_shutdown,
5780         .err_handler = &igc_err_handler,
5781 };
5782
5783 /**
5784  * igc_reinit_queues - return error
5785  * @adapter: pointer to adapter structure
5786  */
5787 int igc_reinit_queues(struct igc_adapter *adapter)
5788 {
5789         struct net_device *netdev = adapter->netdev;
5790         int err = 0;
5791
5792         if (netif_running(netdev))
5793                 igc_close(netdev);
5794
5795         igc_reset_interrupt_capability(adapter);
5796
5797         if (igc_init_interrupt_scheme(adapter, true)) {
5798                 netdev_err(netdev, "Unable to allocate memory for queues\n");
5799                 return -ENOMEM;
5800         }
5801
5802         if (netif_running(netdev))
5803                 err = igc_open(netdev);
5804
5805         return err;
5806 }
5807
5808 /**
5809  * igc_get_hw_dev - return device
5810  * @hw: pointer to hardware structure
5811  *
5812  * used by hardware layer to print debugging information
5813  */
5814 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
5815 {
5816         struct igc_adapter *adapter = hw->back;
5817
5818         return adapter->netdev;
5819 }
5820
5821 /**
5822  * igc_init_module - Driver Registration Routine
5823  *
5824  * igc_init_module is the first routine called when the driver is
5825  * loaded. All it does is register with the PCI subsystem.
5826  */
5827 static int __init igc_init_module(void)
5828 {
5829         int ret;
5830
5831         pr_info("%s\n", igc_driver_string);
5832         pr_info("%s\n", igc_copyright);
5833
5834         ret = pci_register_driver(&igc_driver);
5835         return ret;
5836 }
5837
5838 module_init(igc_init_module);
5839
5840 /**
5841  * igc_exit_module - Driver Exit Cleanup Routine
5842  *
5843  * igc_exit_module is called just before the driver is removed
5844  * from memory.
5845  */
5846 static void __exit igc_exit_module(void)
5847 {
5848         pci_unregister_driver(&igc_driver);
5849 }
5850
5851 module_exit(igc_exit_module);
5852 /* igc_main.c */