drivers/net/ethernet/intel/igbvf/netdev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2009 - 2018 Intel Corporation. */
   3
   4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/delay.h>
   8 #include <linux/ethtool.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/init.h>
  11 #include <linux/ipv6.h>
  12 #include <linux/mii.h>
  13 #include <linux/module.h>
  14 #include <linux/netdevice.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/pci.h>
  17 #include <linux/prefetch.h>
  18 #include <linux/sctp.h>
  19 #include <linux/slab.h>
  20 #include <linux/tcp.h>
  21 #include <linux/types.h>
  22 #include <linux/vmalloc.h>
  23 #include <net/checksum.h>
  24 #include <net/ip6_checksum.h>
  25 #include "igbvf.h"
  26
  27 char igbvf_driver_name[] = "igbvf";
  28 static const char igbvf_driver_string[] =
  29                   "Intel(R) Gigabit Virtual Function Network Driver";
  30 static const char igbvf_copyright[] =
  31                   "Copyright (c) 2009 - 2012 Intel Corporation.";
  32
  33 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  34 static int debug = -1;
  35 module_param(debug, int, 0);
  36 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  37
  38 static int igbvf_poll(struct napi_struct *napi, int budget);
  39 static void igbvf_reset(struct igbvf_adapter *);
  40 static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
  41 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
  42
  43 static struct igbvf_info igbvf_vf_info = {
  44         .mac            = e1000_vfadapt,
  45         .flags          = 0,
  46         .pba            = 10,
  47         .init_ops       = e1000_init_function_pointers_vf,
  48 };
  49
  50 static struct igbvf_info igbvf_i350_vf_info = {
  51         .mac            = e1000_vfadapt_i350,
  52         .flags          = 0,
  53         .pba            = 10,
  54         .init_ops       = e1000_init_function_pointers_vf,
  55 };
  56
  57 static const struct igbvf_info *igbvf_info_tbl[] = {
  58         [board_vf]      = &igbvf_vf_info,
  59         [board_i350_vf] = &igbvf_i350_vf_info,
  60 };
  61
  62 /**
  63  * igbvf_desc_unused - calculate if we have unused descriptors
  64  * @ring: address of receive ring structure
  65  **/
  66 static int igbvf_desc_unused(struct igbvf_ring *ring)
  67 {
  68         if (ring->next_to_clean > ring->next_to_use)
  69                 return ring->next_to_clean - ring->next_to_use - 1;
  70
  71         return ring->count + ring->next_to_clean - ring->next_to_use - 1;
  72 }
  73
  74 /**
  75  * igbvf_receive_skb - helper function to handle Rx indications
  76  * @adapter: board private structure
  77  * @netdev: pointer to netdev struct
  78  * @skb: skb to indicate to stack
  79  * @status: descriptor status field as written by hardware
  80  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
  81  * @skb: pointer to sk_buff to be indicated to stack
  82  **/
  83 static void igbvf_receive_skb(struct igbvf_adapter *adapter,
  84                               struct net_device *netdev,
  85                               struct sk_buff *skb,
  86                               u32 status, __le16 vlan)
  87 {
  88         u16 vid;
  89
  90         if (status & E1000_RXD_STAT_VP) {
  91                 if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
  92                     (status & E1000_RXDEXT_STATERR_LB))
  93                         vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
  94                 else
  95                         vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
  96                 if (test_bit(vid, adapter->active_vlans))
  97                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
  98         }
  99
 100         napi_gro_receive(&adapter->rx_ring->napi, skb);
 101 }
 102
 103 static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
 104                                          u32 status_err, struct sk_buff *skb)
 105 {
 106         skb_checksum_none_assert(skb);
 107
 108         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
 109         if ((status_err & E1000_RXD_STAT_IXSM) ||
 110             (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
 111                 return;
 112
 113         /* TCP/UDP checksum error bit is set */
 114         if (status_err &
 115             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
 116                 /* let the stack verify checksum errors */
 117                 adapter->hw_csum_err++;
 118                 return;
 119         }
 120
 121         /* It must be a TCP or UDP packet with a valid checksum */
 122         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
 123                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 124
 125         adapter->hw_csum_good++;
 126 }
 127
 128 /**
 129  * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
 130  * @rx_ring: address of ring structure to repopulate
 131  * @cleaned_count: number of buffers to repopulate
 132  **/
 133 static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
 134                                    int cleaned_count)
 135 {
 136         struct igbvf_adapter *adapter = rx_ring->adapter;
 137         struct net_device *netdev = adapter->netdev;
 138         struct pci_dev *pdev = adapter->pdev;
 139         union e1000_adv_rx_desc *rx_desc;
 140         struct igbvf_buffer *buffer_info;
 141         struct sk_buff *skb;
 142         unsigned int i;
 143         int bufsz;
 144
 145         i = rx_ring->next_to_use;
 146         buffer_info = &rx_ring->buffer_info[i];
 147
 148         if (adapter->rx_ps_hdr_size)
 149                 bufsz = adapter->rx_ps_hdr_size;
 150         else
 151                 bufsz = adapter->rx_buffer_len;
 152
 153         while (cleaned_count--) {
 154                 rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 155
 156                 if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
 157                         if (!buffer_info->page) {
 158                                 buffer_info->page = alloc_page(GFP_ATOMIC);
 159                                 if (!buffer_info->page) {
 160                                         adapter->alloc_rx_buff_failed++;
 161                                         goto no_buffers;
 162                                 }
 163                                 buffer_info->page_offset = 0;
 164                         } else {
 165                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
 166                         }
 167                         buffer_info->page_dma =
 168                                 dma_map_page(&pdev->dev, buffer_info->page,
 169                                              buffer_info->page_offset,
 170                                              PAGE_SIZE / 2,
 171                                              DMA_FROM_DEVICE);
 172                         if (dma_mapping_error(&pdev->dev,
 173                                               buffer_info->page_dma)) {
 174                                 __free_page(buffer_info->page);
 175                                 buffer_info->page = NULL;
 176                                 dev_err(&pdev->dev, "RX DMA map failed\n");
 177                                 break;
 178                         }
 179                 }
 180
 181                 if (!buffer_info->skb) {
 182                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
 183                         if (!skb) {
 184                                 adapter->alloc_rx_buff_failed++;
 185                                 goto no_buffers;
 186                         }
 187
 188                         buffer_info->skb = skb;
 189                         buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 190                                                           bufsz,
 191                                                           DMA_FROM_DEVICE);
 192                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 193                                 dev_kfree_skb(buffer_info->skb);
 194                                 buffer_info->skb = NULL;
 195                                 dev_err(&pdev->dev, "RX DMA map failed\n");
 196                                 goto no_buffers;
 197                         }
 198                 }
 199                 /* Refresh the desc even if buffer_addrs didn't change because
 200                  * each write-back erases this info.
 201                  */
 202                 if (adapter->rx_ps_hdr_size) {
 203                         rx_desc->read.pkt_addr =
 204                              cpu_to_le64(buffer_info->page_dma);
 205                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
 206                 } else {
 207                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
 208                         rx_desc->read.hdr_addr = 0;
 209                 }
 210
 211                 i++;
 212                 if (i == rx_ring->count)
 213                         i = 0;
 214                 buffer_info = &rx_ring->buffer_info[i];
 215         }
 216
 217 no_buffers:
 218         if (rx_ring->next_to_use != i) {
 219                 rx_ring->next_to_use = i;
 220                 if (i == 0)
 221                         i = (rx_ring->count - 1);
 222                 else
 223                         i--;
 224
 225                 /* Force memory writes to complete before letting h/w
 226                  * know there are new descriptors to fetch.  (Only
 227                  * applicable for weak-ordered memory model archs,
 228                  * such as IA-64).
 229                 */
 230                 wmb();
 231                 writel(i, adapter->hw.hw_addr + rx_ring->tail);
 232         }
 233 }
 234
 235 /**
 236  * igbvf_clean_rx_irq - Send received data up the network stack; legacy
 237  * @adapter: board private structure
 238  * @work_done: output parameter used to indicate completed work
 239  * @work_to_do: input parameter setting limit of work
 240  *
 241  * the return value indicates whether actual cleaning was done, there
 242  * is no guarantee that everything was cleaned
 243  **/
 244 static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
 245                                int *work_done, int work_to_do)
 246 {
 247         struct igbvf_ring *rx_ring = adapter->rx_ring;
 248         struct net_device *netdev = adapter->netdev;
 249         struct pci_dev *pdev = adapter->pdev;
 250         union e1000_adv_rx_desc *rx_desc, *next_rxd;
 251         struct igbvf_buffer *buffer_info, *next_buffer;
 252         struct sk_buff *skb;
 253         bool cleaned = false;
 254         int cleaned_count = 0;
 255         unsigned int total_bytes = 0, total_packets = 0;
 256         unsigned int i;
 257         u32 length, hlen, staterr;
 258
 259         i = rx_ring->next_to_clean;
 260         rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 261         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 262
 263         while (staterr & E1000_RXD_STAT_DD) {
 264                 if (*work_done >= work_to_do)
 265                         break;
 266                 (*work_done)++;
 267                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 268
 269                 buffer_info = &rx_ring->buffer_info[i];
 270
 271                 /* HW will not DMA in data larger than the given buffer, even
 272                  * if it parses the (NFS, of course) header to be larger.  In
 273                  * that case, it fills the header buffer and spills the rest
 274                  * into the page.
 275                  */
 276                 hlen = le16_get_bits(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,
 277                                      E1000_RXDADV_HDRBUFLEN_MASK);
 278                 if (hlen > adapter->rx_ps_hdr_size)
 279                         hlen = adapter->rx_ps_hdr_size;
 280
 281                 length = le16_to_cpu(rx_desc->wb.upper.length);
 282                 cleaned = true;
 283                 cleaned_count++;
 284
 285                 skb = buffer_info->skb;
 286                 prefetch(skb->data - NET_IP_ALIGN);
 287                 buffer_info->skb = NULL;
 288                 if (!adapter->rx_ps_hdr_size) {
 289                         dma_unmap_single(&pdev->dev, buffer_info->dma,
 290                                          adapter->rx_buffer_len,
 291                                          DMA_FROM_DEVICE);
 292                         buffer_info->dma = 0;
 293                         skb_put(skb, length);
 294                         goto send_up;
 295                 }
 296
 297                 if (!skb_shinfo(skb)->nr_frags) {
 298                         dma_unmap_single(&pdev->dev, buffer_info->dma,
 299                                          adapter->rx_ps_hdr_size,
 300                                          DMA_FROM_DEVICE);
 301                         buffer_info->dma = 0;
 302                         skb_put(skb, hlen);
 303                 }
 304
 305                 if (length) {
 306                         dma_unmap_page(&pdev->dev, buffer_info->page_dma,
 307                                        PAGE_SIZE / 2,
 308                                        DMA_FROM_DEVICE);
 309                         buffer_info->page_dma = 0;
 310
 311                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 312                                            buffer_info->page,
 313                                            buffer_info->page_offset,
 314                                            length);
 315
 316                         if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
 317                             (page_count(buffer_info->page) != 1))
 318                                 buffer_info->page = NULL;
 319                         else
 320                                 get_page(buffer_info->page);
 321
 322                         skb->len += length;
 323                         skb->data_len += length;
 324                         skb->truesize += PAGE_SIZE / 2;
 325                 }
 326 send_up:
 327                 i++;
 328                 if (i == rx_ring->count)
 329                         i = 0;
 330                 next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
 331                 prefetch(next_rxd);
 332                 next_buffer = &rx_ring->buffer_info[i];
 333
 334                 if (!(staterr & E1000_RXD_STAT_EOP)) {
 335                         buffer_info->skb = next_buffer->skb;
 336                         buffer_info->dma = next_buffer->dma;
 337                         next_buffer->skb = skb;
 338                         next_buffer->dma = 0;
 339                         goto next_desc;
 340                 }
 341
 342                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
 343                         dev_kfree_skb_irq(skb);
 344                         goto next_desc;
 345                 }
 346
 347                 total_bytes += skb->len;
 348                 total_packets++;
 349
 350                 igbvf_rx_checksum_adv(adapter, staterr, skb);
 351
 352                 skb->protocol = eth_type_trans(skb, netdev);
 353
 354                 igbvf_receive_skb(adapter, netdev, skb, staterr,
 355                                   rx_desc->wb.upper.vlan);
 356
 357 next_desc:
 358                 rx_desc->wb.upper.status_error = 0;
 359
 360                 /* return some buffers to hardware, one at a time is too slow */
 361                 if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
 362                         igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 363                         cleaned_count = 0;
 364                 }
 365
 366                 /* use prefetched values */
 367                 rx_desc = next_rxd;
 368                 buffer_info = next_buffer;
 369
 370                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 371         }
 372
 373         rx_ring->next_to_clean = i;
 374         cleaned_count = igbvf_desc_unused(rx_ring);
 375
 376         if (cleaned_count)
 377                 igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 378
 379         adapter->total_rx_packets += total_packets;
 380         adapter->total_rx_bytes += total_bytes;
 381         netdev->stats.rx_bytes += total_bytes;
 382         netdev->stats.rx_packets += total_packets;
 383         return cleaned;
 384 }
 385
 386 static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
 387                             struct igbvf_buffer *buffer_info)
 388 {
 389         if (buffer_info->dma) {
 390                 if (buffer_info->mapped_as_page)
 391                         dma_unmap_page(&adapter->pdev->dev,
 392                                        buffer_info->dma,
 393                                        buffer_info->length,
 394                                        DMA_TO_DEVICE);
 395                 else
 396                         dma_unmap_single(&adapter->pdev->dev,
 397                                          buffer_info->dma,
 398                                          buffer_info->length,
 399                                          DMA_TO_DEVICE);
 400                 buffer_info->dma = 0;
 401         }
 402         if (buffer_info->skb) {
 403                 dev_kfree_skb_any(buffer_info->skb);
 404                 buffer_info->skb = NULL;
 405         }
 406         buffer_info->time_stamp = 0;
 407 }
 408
 409 /**
 410  * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
 411  * @adapter: board private structure
 412  * @tx_ring: ring being initialized
 413  *
 414  * Return 0 on success, negative on failure
 415  **/
 416 int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
 417                              struct igbvf_ring *tx_ring)
 418 {
 419         struct pci_dev *pdev = adapter->pdev;
 420         int size;
 421
 422         size = sizeof(struct igbvf_buffer) * tx_ring->count;
 423         tx_ring->buffer_info = vzalloc(size);
 424         if (!tx_ring->buffer_info)
 425                 goto err;
 426
 427         /* round up to nearest 4K */
 428         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
 429         tx_ring->size = ALIGN(tx_ring->size, 4096);
 430
 431         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
 432                                            &tx_ring->dma, GFP_KERNEL);
 433         if (!tx_ring->desc)
 434                 goto err;
 435
 436         tx_ring->adapter = adapter;
 437         tx_ring->next_to_use = 0;
 438         tx_ring->next_to_clean = 0;
 439
 440         return 0;
 441 err:
 442         vfree(tx_ring->buffer_info);
 443         dev_err(&adapter->pdev->dev,
 444                 "Unable to allocate memory for the transmit descriptor ring\n");
 445         return -ENOMEM;
 446 }
 447
 448 /**
 449  * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
 450  * @adapter: board private structure
 451  * @rx_ring: ring being initialized
 452  *
 453  * Returns 0 on success, negative on failure
 454  **/
 455 int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
 456                              struct igbvf_ring *rx_ring)
 457 {
 458         struct pci_dev *pdev = adapter->pdev;
 459         int size, desc_len;
 460
 461         size = sizeof(struct igbvf_buffer) * rx_ring->count;
 462         rx_ring->buffer_info = vzalloc(size);
 463         if (!rx_ring->buffer_info)
 464                 goto err;
 465
 466         desc_len = sizeof(union e1000_adv_rx_desc);
 467
 468         /* Round up to nearest 4K */
 469         rx_ring->size = rx_ring->count * desc_len;
 470         rx_ring->size = ALIGN(rx_ring->size, 4096);
 471
 472         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
 473                                            &rx_ring->dma, GFP_KERNEL);
 474         if (!rx_ring->desc)
 475                 goto err;
 476
 477         rx_ring->next_to_clean = 0;
 478         rx_ring->next_to_use = 0;
 479
 480         rx_ring->adapter = adapter;
 481
 482         return 0;
 483
 484 err:
 485         vfree(rx_ring->buffer_info);
 486         rx_ring->buffer_info = NULL;
 487         dev_err(&adapter->pdev->dev,
 488                 "Unable to allocate memory for the receive descriptor ring\n");
 489         return -ENOMEM;
 490 }
 491
 492 /**
 493  * igbvf_clean_tx_ring - Free Tx Buffers
 494  * @tx_ring: ring to be cleaned
 495  **/
 496 static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
 497 {
 498         struct igbvf_adapter *adapter = tx_ring->adapter;
 499         struct igbvf_buffer *buffer_info;
 500         unsigned long size;
 501         unsigned int i;
 502
 503         if (!tx_ring->buffer_info)
 504                 return;
 505
 506         /* Free all the Tx ring sk_buffs */
 507         for (i = 0; i < tx_ring->count; i++) {
 508                 buffer_info = &tx_ring->buffer_info[i];
 509                 igbvf_put_txbuf(adapter, buffer_info);
 510         }
 511
 512         size = sizeof(struct igbvf_buffer) * tx_ring->count;
 513         memset(tx_ring->buffer_info, 0, size);
 514
 515         /* Zero out the descriptor ring */
 516         memset(tx_ring->desc, 0, tx_ring->size);
 517
 518         tx_ring->next_to_use = 0;
 519         tx_ring->next_to_clean = 0;
 520
 521         writel(0, adapter->hw.hw_addr + tx_ring->head);
 522         writel(0, adapter->hw.hw_addr + tx_ring->tail);
 523 }
 524
 525 /**
 526  * igbvf_free_tx_resources - Free Tx Resources per Queue
 527  * @tx_ring: ring to free resources from
 528  *
 529  * Free all transmit software resources
 530  **/
 531 void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
 532 {
 533         struct pci_dev *pdev = tx_ring->adapter->pdev;
 534
 535         igbvf_clean_tx_ring(tx_ring);
 536
 537         vfree(tx_ring->buffer_info);
 538         tx_ring->buffer_info = NULL;
 539
 540         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
 541                           tx_ring->dma);
 542
 543         tx_ring->desc = NULL;
 544 }
 545
 546 /**
 547  * igbvf_clean_rx_ring - Free Rx Buffers per Queue
 548  * @rx_ring: ring structure pointer to free buffers from
 549  **/
 550 static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
 551 {
 552         struct igbvf_adapter *adapter = rx_ring->adapter;
 553         struct igbvf_buffer *buffer_info;
 554         struct pci_dev *pdev = adapter->pdev;
 555         unsigned long size;
 556         unsigned int i;
 557
 558         if (!rx_ring->buffer_info)
 559                 return;
 560
 561         /* Free all the Rx ring sk_buffs */
 562         for (i = 0; i < rx_ring->count; i++) {
 563                 buffer_info = &rx_ring->buffer_info[i];
 564                 if (buffer_info->dma) {
 565                         if (adapter->rx_ps_hdr_size) {
 566                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
 567                                                  adapter->rx_ps_hdr_size,
 568                                                  DMA_FROM_DEVICE);
 569                         } else {
 570                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
 571                                                  adapter->rx_buffer_len,
 572                                                  DMA_FROM_DEVICE);
 573                         }
 574                         buffer_info->dma = 0;
 575                 }
 576
 577                 if (buffer_info->skb) {
 578                         dev_kfree_skb(buffer_info->skb);
 579                         buffer_info->skb = NULL;
 580                 }
 581
 582                 if (buffer_info->page) {
 583                         if (buffer_info->page_dma)
 584                                 dma_unmap_page(&pdev->dev,
 585                                                buffer_info->page_dma,
 586                                                PAGE_SIZE / 2,
 587                                                DMA_FROM_DEVICE);
 588                         put_page(buffer_info->page);
 589                         buffer_info->page = NULL;
 590                         buffer_info->page_dma = 0;
 591                         buffer_info->page_offset = 0;
 592                 }
 593         }
 594
 595         size = sizeof(struct igbvf_buffer) * rx_ring->count;
 596         memset(rx_ring->buffer_info, 0, size);
 597
 598         /* Zero out the descriptor ring */
 599         memset(rx_ring->desc, 0, rx_ring->size);
 600
 601         rx_ring->next_to_clean = 0;
 602         rx_ring->next_to_use = 0;
 603
 604         writel(0, adapter->hw.hw_addr + rx_ring->head);
 605         writel(0, adapter->hw.hw_addr + rx_ring->tail);
 606 }
 607
 608 /**
 609  * igbvf_free_rx_resources - Free Rx Resources
 610  * @rx_ring: ring to clean the resources from
 611  *
 612  * Free all receive software resources
 613  **/
 614
 615 void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
 616 {
 617         struct pci_dev *pdev = rx_ring->adapter->pdev;
 618
 619         igbvf_clean_rx_ring(rx_ring);
 620
 621         vfree(rx_ring->buffer_info);
 622         rx_ring->buffer_info = NULL;
 623
 624         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
 625                           rx_ring->dma);
 626         rx_ring->desc = NULL;
 627 }
 628
 629 /**
 630  * igbvf_update_itr - update the dynamic ITR value based on statistics
 631  * @adapter: pointer to adapter
 632  * @itr_setting: current adapter->itr
 633  * @packets: the number of packets during this measurement interval
 634  * @bytes: the number of bytes during this measurement interval
 635  *
 636  * Stores a new ITR value based on packets and byte counts during the last
 637  * interrupt.  The advantage of per interrupt computation is faster updates
 638  * and more accurate ITR for the current traffic pattern.  Constants in this
 639  * function were computed based on theoretical maximum wire speed and thresholds
 640  * were set based on testing data as well as attempting to minimize response
 641  * time while increasing bulk throughput.
 642  **/
 643 static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
 644                                            enum latency_range itr_setting,
 645                                            int packets, int bytes)
 646 {
 647         enum latency_range retval = itr_setting;
 648
 649         if (packets == 0)
 650                 goto update_itr_done;
 651
 652         switch (itr_setting) {
 653         case lowest_latency:
 654                 /* handle TSO and jumbo frames */
 655                 if (bytes/packets > 8000)
 656                         retval = bulk_latency;
 657                 else if ((packets < 5) && (bytes > 512))
 658                         retval = low_latency;
 659                 break;
 660         case low_latency:  /* 50 usec aka 20000 ints/s */
 661                 if (bytes > 10000) {
 662                         /* this if handles the TSO accounting */
 663                         if (bytes/packets > 8000)
 664                                 retval = bulk_latency;
 665                         else if ((packets < 10) || ((bytes/packets) > 1200))
 666                                 retval = bulk_latency;
 667                         else if ((packets > 35))
 668                                 retval = lowest_latency;
 669                 } else if (bytes/packets > 2000) {
 670                         retval = bulk_latency;
 671                 } else if (packets <= 2 && bytes < 512) {
 672                         retval = lowest_latency;
 673                 }
 674                 break;
 675         case bulk_latency: /* 250 usec aka 4000 ints/s */
 676                 if (bytes > 25000) {
 677                         if (packets > 35)
 678                                 retval = low_latency;
 679                 } else if (bytes < 6000) {
 680                         retval = low_latency;
 681                 }
 682                 break;
 683         default:
 684                 break;
 685         }
 686
 687 update_itr_done:
 688         return retval;
 689 }
 690
 691 static int igbvf_range_to_itr(enum latency_range current_range)
 692 {
 693         int new_itr;
 694
 695         switch (current_range) {
 696         /* counts and packets in update_itr are dependent on these numbers */
 697         case lowest_latency:
 698                 new_itr = IGBVF_70K_ITR;
 699                 break;
 700         case low_latency:
 701                 new_itr = IGBVF_20K_ITR;
 702                 break;
 703         case bulk_latency:
 704                 new_itr = IGBVF_4K_ITR;
 705                 break;
 706         default:
 707                 new_itr = IGBVF_START_ITR;
 708                 break;
 709         }
 710         return new_itr;
 711 }
 712
 713 static void igbvf_set_itr(struct igbvf_adapter *adapter)
 714 {
 715         u32 new_itr;
 716
 717         adapter->tx_ring->itr_range =
 718                         igbvf_update_itr(adapter,
 719                                          adapter->tx_ring->itr_val,
 720                                          adapter->total_tx_packets,
 721                                          adapter->total_tx_bytes);
 722
 723         /* conservative mode (itr 3) eliminates the lowest_latency setting */
 724         if (adapter->requested_itr == 3 &&
 725             adapter->tx_ring->itr_range == lowest_latency)
 726                 adapter->tx_ring->itr_range = low_latency;
 727
 728         new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
 729
 730         if (new_itr != adapter->tx_ring->itr_val) {
 731                 u32 current_itr = adapter->tx_ring->itr_val;
 732                 /* this attempts to bias the interrupt rate towards Bulk
 733                  * by adding intermediate steps when interrupt rate is
 734                  * increasing
 735                  */
 736                 new_itr = new_itr > current_itr ?
 737                           min(current_itr + (new_itr >> 2), new_itr) :
 738                           new_itr;
 739                 adapter->tx_ring->itr_val = new_itr;
 740
 741                 adapter->tx_ring->set_itr = 1;
 742         }
 743
 744         adapter->rx_ring->itr_range =
 745                         igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
 746                                          adapter->total_rx_packets,
 747                                          adapter->total_rx_bytes);
 748         if (adapter->requested_itr == 3 &&
 749             adapter->rx_ring->itr_range == lowest_latency)
 750                 adapter->rx_ring->itr_range = low_latency;
 751
 752         new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
 753
 754         if (new_itr != adapter->rx_ring->itr_val) {
 755                 u32 current_itr = adapter->rx_ring->itr_val;
 756
 757                 new_itr = new_itr > current_itr ?
 758                           min(current_itr + (new_itr >> 2), new_itr) :
 759                           new_itr;
 760                 adapter->rx_ring->itr_val = new_itr;
 761
 762                 adapter->rx_ring->set_itr = 1;
 763         }
 764 }
 765
 766 /**
 767  * igbvf_clean_tx_irq - Reclaim resources after transmit completes
 768  * @tx_ring: ring structure to clean descriptors from
 769  *
 770  * returns true if ring is completely cleaned
 771  **/
 772 static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
 773 {
 774         struct igbvf_adapter *adapter = tx_ring->adapter;
 775         struct net_device *netdev = adapter->netdev;
 776         struct igbvf_buffer *buffer_info;
 777         struct sk_buff *skb;
 778         union e1000_adv_tx_desc *tx_desc, *eop_desc;
 779         unsigned int total_bytes = 0, total_packets = 0;
 780         unsigned int i, count = 0;
 781         bool cleaned = false;
 782
 783         i = tx_ring->next_to_clean;
 784         buffer_info = &tx_ring->buffer_info[i];
 785         eop_desc = buffer_info->next_to_watch;
 786
 787         do {
 788                 /* if next_to_watch is not set then there is no work pending */
 789                 if (!eop_desc)
 790                         break;
 791
 792                 /* prevent any other reads prior to eop_desc */
 793                 smp_rmb();
 794
 795                 /* if DD is not set pending work has not been completed */
 796                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
 797                         break;
 798
 799                 /* clear next_to_watch to prevent false hangs */
 800                 buffer_info->next_to_watch = NULL;
 801
 802                 for (cleaned = false; !cleaned; count++) {
 803                         tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
 804                         cleaned = (tx_desc == eop_desc);
 805                         skb = buffer_info->skb;
 806
 807                         if (skb) {
 808                                 unsigned int segs, bytecount;
 809
 810                                 /* gso_segs is currently only valid for tcp */
 811                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
 812                                 /* multiply data chunks by size of headers */
 813                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
 814                                             skb->len;
 815                                 total_packets += segs;
 816                                 total_bytes += bytecount;
 817                         }
 818
 819                         igbvf_put_txbuf(adapter, buffer_info);
 820                         tx_desc->wb.status = 0;
 821
 822                         i++;
 823                         if (i == tx_ring->count)
 824                                 i = 0;
 825
 826                         buffer_info = &tx_ring->buffer_info[i];
 827                 }
 828
 829                 eop_desc = buffer_info->next_to_watch;
 830         } while (count < tx_ring->count);
 831
 832         tx_ring->next_to_clean = i;
 833
 834         if (unlikely(count && netif_carrier_ok(netdev) &&
 835             igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
 836                 /* Make sure that anybody stopping the queue after this
 837                  * sees the new next_to_clean.
 838                  */
 839                 smp_mb();
 840                 if (netif_queue_stopped(netdev) &&
 841                     !(test_bit(__IGBVF_DOWN, &adapter->state))) {
 842                         netif_wake_queue(netdev);
 843                         ++adapter->restart_queue;
 844                 }
 845         }
 846
 847         netdev->stats.tx_bytes += total_bytes;
 848         netdev->stats.tx_packets += total_packets;
 849         return count < tx_ring->count;
 850 }
 851
 852 static irqreturn_t igbvf_msix_other(int irq, void *data)
 853 {
 854         struct net_device *netdev = data;
 855         struct igbvf_adapter *adapter = netdev_priv(netdev);
 856         struct e1000_hw *hw = &adapter->hw;
 857
 858         adapter->int_counter1++;
 859
 860         hw->mac.get_link_status = 1;
 861         if (!test_bit(__IGBVF_DOWN, &adapter->state))
 862                 mod_timer(&adapter->watchdog_timer, jiffies + 1);
 863
 864         ew32(EIMS, adapter->eims_other);
 865
 866         return IRQ_HANDLED;
 867 }
 868
 869 static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
 870 {
 871         struct net_device *netdev = data;
 872         struct igbvf_adapter *adapter = netdev_priv(netdev);
 873         struct e1000_hw *hw = &adapter->hw;
 874         struct igbvf_ring *tx_ring = adapter->tx_ring;
 875
 876         if (tx_ring->set_itr) {
 877                 writel(tx_ring->itr_val,
 878                        adapter->hw.hw_addr + tx_ring->itr_register);
 879                 adapter->tx_ring->set_itr = 0;
 880         }
 881
 882         adapter->total_tx_bytes = 0;
 883         adapter->total_tx_packets = 0;
 884
 885         /* auto mask will automatically re-enable the interrupt when we write
 886          * EICS
 887          */
 888         if (!igbvf_clean_tx_irq(tx_ring))
 889                 /* Ring was not completely cleaned, so fire another interrupt */
 890                 ew32(EICS, tx_ring->eims_value);
 891         else
 892                 ew32(EIMS, tx_ring->eims_value);
 893
 894         return IRQ_HANDLED;
 895 }
 896
 897 static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
 898 {
 899         struct net_device *netdev = data;
 900         struct igbvf_adapter *adapter = netdev_priv(netdev);
 901
 902         adapter->int_counter0++;
 903
 904         /* Write the ITR value calculated at the end of the
 905          * previous interrupt.
 906          */
 907         if (adapter->rx_ring->set_itr) {
 908                 writel(adapter->rx_ring->itr_val,
 909                        adapter->hw.hw_addr + adapter->rx_ring->itr_register);
 910                 adapter->rx_ring->set_itr = 0;
 911         }
 912
 913         if (napi_schedule_prep(&adapter->rx_ring->napi)) {
 914                 adapter->total_rx_bytes = 0;
 915                 adapter->total_rx_packets = 0;
 916                 __napi_schedule(&adapter->rx_ring->napi);
 917         }
 918
 919         return IRQ_HANDLED;
 920 }
 921
 922 #define IGBVF_NO_QUEUE -1
 923
 924 static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
 925                                 int tx_queue, int msix_vector)
 926 {
 927         struct e1000_hw *hw = &adapter->hw;
 928         u32 ivar, index;
 929
 930         /* 82576 uses a table-based method for assigning vectors.
 931          * Each queue has a single entry in the table to which we write
 932          * a vector number along with a "valid" bit.  Sadly, the layout
 933          * of the table is somewhat counterintuitive.
 934          */
 935         if (rx_queue > IGBVF_NO_QUEUE) {
 936                 index = (rx_queue >> 1);
 937                 ivar = array_er32(IVAR0, index);
 938                 if (rx_queue & 0x1) {
 939                         /* vector goes into third byte of register */
 940                         ivar = ivar & 0xFF00FFFF;
 941                         ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 942                 } else {
 943                         /* vector goes into low byte of register */
 944                         ivar = ivar & 0xFFFFFF00;
 945                         ivar |= msix_vector | E1000_IVAR_VALID;
 946                 }
 947                 adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
 948                 array_ew32(IVAR0, index, ivar);
 949         }
 950         if (tx_queue > IGBVF_NO_QUEUE) {
 951                 index = (tx_queue >> 1);
 952                 ivar = array_er32(IVAR0, index);
 953                 if (tx_queue & 0x1) {
 954                         /* vector goes into high byte of register */
 955                         ivar = ivar & 0x00FFFFFF;
 956                         ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 957                 } else {
 958                         /* vector goes into second byte of register */
 959                         ivar = ivar & 0xFFFF00FF;
 960                         ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 961                 }
 962                 adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
 963                 array_ew32(IVAR0, index, ivar);
 964         }
 965 }
 966
 967 /**
 968  * igbvf_configure_msix - Configure MSI-X hardware
 969  * @adapter: board private structure
 970  *
 971  * igbvf_configure_msix sets up the hardware to properly
 972  * generate MSI-X interrupts.
 973  **/
 974 static void igbvf_configure_msix(struct igbvf_adapter *adapter)
 975 {
 976         u32 tmp;
 977         struct e1000_hw *hw = &adapter->hw;
 978         struct igbvf_ring *tx_ring = adapter->tx_ring;
 979         struct igbvf_ring *rx_ring = adapter->rx_ring;
 980         int vector = 0;
 981
 982         adapter->eims_enable_mask = 0;
 983
 984         igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
 985         adapter->eims_enable_mask |= tx_ring->eims_value;
 986         writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
 987         igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
 988         adapter->eims_enable_mask |= rx_ring->eims_value;
 989         writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
 990
 991         /* set vector for other causes, i.e. link changes */
 992
 993         tmp = (vector++ | E1000_IVAR_VALID);
 994
 995         ew32(IVAR_MISC, tmp);
 996
 997         adapter->eims_enable_mask = GENMASK(vector - 1, 0);
 998         adapter->eims_other = BIT(vector - 1);
 999         e1e_flush();
1000 }
1001
1002 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
1003 {
1004         if (adapter->msix_entries) {
1005                 pci_disable_msix(adapter->pdev);
1006                 kfree(adapter->msix_entries);
1007                 adapter->msix_entries = NULL;
1008         }
1009 }
1010
1011 /**
1012  * igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1013  * @adapter: board private structure
1014  *
1015  * Attempt to configure interrupts using the best available
1016  * capabilities of the hardware and kernel.
1017  **/
1018 static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1019 {
1020         int err = -ENOMEM;
1021         int i;
1022
1023         /* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1024         adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),
1025                                         GFP_KERNEL);
1026         if (adapter->msix_entries) {
1027                 for (i = 0; i < 3; i++)
1028                         adapter->msix_entries[i].entry = i;
1029
1030                 err = pci_enable_msix_range(adapter->pdev,
1031                                             adapter->msix_entries, 3, 3);
1032         }
1033
1034         if (err < 0) {
1035                 /* MSI-X failed */
1036                 dev_err(&adapter->pdev->dev,
1037                         "Failed to initialize MSI-X interrupts.\n");
1038                 igbvf_reset_interrupt_capability(adapter);
1039         }
1040 }
1041
1042 /**
1043  * igbvf_request_msix - Initialize MSI-X interrupts
1044  * @adapter: board private structure
1045  *
1046  * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1047  * kernel.
1048  **/
1049 static int igbvf_request_msix(struct igbvf_adapter *adapter)
1050 {
1051         struct net_device *netdev = adapter->netdev;
1052         int err = 0, vector = 0;
1053
1054         if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1055                 sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);
1056                 sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);
1057         } else {
1058                 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1059                 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1060         }
1061
1062         err = request_irq(adapter->msix_entries[vector].vector,
1063                           igbvf_intr_msix_tx, 0, adapter->tx_ring->name,
1064                           netdev);
1065         if (err)
1066                 goto out;
1067
1068         adapter->tx_ring->itr_register = E1000_EITR(vector);
1069         adapter->tx_ring->itr_val = adapter->current_itr;
1070         vector++;
1071
1072         err = request_irq(adapter->msix_entries[vector].vector,
1073                           igbvf_intr_msix_rx, 0, adapter->rx_ring->name,
1074                           netdev);
1075         if (err)
1076                 goto free_irq_tx;
1077
1078         adapter->rx_ring->itr_register = E1000_EITR(vector);
1079         adapter->rx_ring->itr_val = adapter->current_itr;
1080         vector++;
1081
1082         err = request_irq(adapter->msix_entries[vector].vector,
1083                           igbvf_msix_other, 0, netdev->name, netdev);
1084         if (err)
1085                 goto free_irq_rx;
1086
1087         igbvf_configure_msix(adapter);
1088         return 0;
1089 free_irq_rx:
1090         free_irq(adapter->msix_entries[--vector].vector, netdev);
1091 free_irq_tx:
1092         free_irq(adapter->msix_entries[--vector].vector, netdev);
1093 out:
1094         return err;
1095 }
1096
1097 /**
1098  * igbvf_alloc_queues - Allocate memory for all rings
1099  * @adapter: board private structure to initialize
1100  **/
1101 static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1102 {
1103         struct net_device *netdev = adapter->netdev;
1104
1105         adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1106         if (!adapter->tx_ring)
1107                 return -ENOMEM;
1108
1109         adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1110         if (!adapter->rx_ring) {
1111                 kfree(adapter->tx_ring);
1112                 return -ENOMEM;
1113         }
1114
1115         netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
1116
1117         return 0;
1118 }
1119
1120 /**
1121  * igbvf_request_irq - initialize interrupts
1122  * @adapter: board private structure
1123  *
1124  * Attempts to configure interrupts using the best available
1125  * capabilities of the hardware and kernel.
1126  **/
1127 static int igbvf_request_irq(struct igbvf_adapter *adapter)
1128 {
1129         int err = -1;
1130
1131         /* igbvf supports msi-x only */
1132         if (adapter->msix_entries)
1133                 err = igbvf_request_msix(adapter);
1134
1135         if (!err)
1136                 return err;
1137
1138         dev_err(&adapter->pdev->dev,
1139                 "Unable to allocate interrupt, Error: %d\n", err);
1140
1141         return err;
1142 }
1143
1144 static void igbvf_free_irq(struct igbvf_adapter *adapter)
1145 {
1146         struct net_device *netdev = adapter->netdev;
1147         int vector;
1148
1149         if (adapter->msix_entries) {
1150                 for (vector = 0; vector < 3; vector++)
1151                         free_irq(adapter->msix_entries[vector].vector, netdev);
1152         }
1153 }
1154
1155 /**
1156  * igbvf_irq_disable - Mask off interrupt generation on the NIC
1157  * @adapter: board private structure
1158  **/
1159 static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1160 {
1161         struct e1000_hw *hw = &adapter->hw;
1162
1163         ew32(EIMC, ~0);
1164
1165         if (adapter->msix_entries)
1166                 ew32(EIAC, 0);
1167 }
1168
1169 /**
1170  * igbvf_irq_enable - Enable default interrupt generation settings
1171  * @adapter: board private structure
1172  **/
1173 static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1174 {
1175         struct e1000_hw *hw = &adapter->hw;
1176
1177         ew32(EIAC, adapter->eims_enable_mask);
1178         ew32(EIAM, adapter->eims_enable_mask);
1179         ew32(EIMS, adapter->eims_enable_mask);
1180 }
1181
1182 /**
1183  * igbvf_poll - NAPI Rx polling callback
1184  * @napi: struct associated with this polling callback
1185  * @budget: amount of packets driver is allowed to process this poll
1186  **/
1187 static int igbvf_poll(struct napi_struct *napi, int budget)
1188 {
1189         struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1190         struct igbvf_adapter *adapter = rx_ring->adapter;
1191         struct e1000_hw *hw = &adapter->hw;
1192         int work_done = 0;
1193
1194         igbvf_clean_rx_irq(adapter, &work_done, budget);
1195
1196         if (work_done == budget)
1197                 return budget;
1198
1199         /* Exit the polling mode, but don't re-enable interrupts if stack might
1200          * poll us due to busy-polling
1201          */
1202         if (likely(napi_complete_done(napi, work_done))) {
1203                 if (adapter->requested_itr & 3)
1204                         igbvf_set_itr(adapter);
1205
1206                 if (!test_bit(__IGBVF_DOWN, &adapter->state))
1207                         ew32(EIMS, adapter->rx_ring->eims_value);
1208         }
1209
1210         return work_done;
1211 }
1212
1213 /**
1214  * igbvf_set_rlpml - set receive large packet maximum length
1215  * @adapter: board private structure
1216  *
1217  * Configure the maximum size of packets that will be received
1218  */
1219 static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1220 {
1221         int max_frame_size;
1222         struct e1000_hw *hw = &adapter->hw;
1223
1224         max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1225
1226         spin_lock_bh(&hw->mbx_lock);
1227
1228         e1000_rlpml_set_vf(hw, max_frame_size);
1229
1230         spin_unlock_bh(&hw->mbx_lock);
1231 }
1232
1233 static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1234                                  __be16 proto, u16 vid)
1235 {
1236         struct igbvf_adapter *adapter = netdev_priv(netdev);
1237         struct e1000_hw *hw = &adapter->hw;
1238
1239         spin_lock_bh(&hw->mbx_lock);
1240
1241         if (hw->mac.ops.set_vfta(hw, vid, true)) {
1242                 dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);
1243                 spin_unlock_bh(&hw->mbx_lock);
1244                 return -EINVAL;
1245         }
1246
1247         spin_unlock_bh(&hw->mbx_lock);
1248
1249         set_bit(vid, adapter->active_vlans);
1250         return 0;
1251 }
1252
1253 static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1254                                   __be16 proto, u16 vid)
1255 {
1256         struct igbvf_adapter *adapter = netdev_priv(netdev);
1257         struct e1000_hw *hw = &adapter->hw;
1258
1259         spin_lock_bh(&hw->mbx_lock);
1260
1261         if (hw->mac.ops.set_vfta(hw, vid, false)) {
1262                 dev_err(&adapter->pdev->dev,
1263                         "Failed to remove vlan id %d\n", vid);
1264                 spin_unlock_bh(&hw->mbx_lock);
1265                 return -EINVAL;
1266         }
1267
1268         spin_unlock_bh(&hw->mbx_lock);
1269
1270         clear_bit(vid, adapter->active_vlans);
1271         return 0;
1272 }
1273
1274 static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1275 {
1276         u16 vid;
1277
1278         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1279                 igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
1280 }
1281
1282 /**
1283  * igbvf_configure_tx - Configure Transmit Unit after Reset
1284  * @adapter: board private structure
1285  *
1286  * Configure the Tx unit of the MAC after a reset.
1287  **/
1288 static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1289 {
1290         struct e1000_hw *hw = &adapter->hw;
1291         struct igbvf_ring *tx_ring = adapter->tx_ring;
1292         u64 tdba;
1293         u32 txdctl, dca_txctrl;
1294
1295         /* disable transmits */
1296         txdctl = er32(TXDCTL(0));
1297         ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1298         e1e_flush();
1299         msleep(10);
1300
1301         /* Setup the HW Tx Head and Tail descriptor pointers */
1302         ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1303         tdba = tx_ring->dma;
1304         ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1305         ew32(TDBAH(0), (tdba >> 32));
1306         ew32(TDH(0), 0);
1307         ew32(TDT(0), 0);
1308         tx_ring->head = E1000_TDH(0);
1309         tx_ring->tail = E1000_TDT(0);
1310
1311         /* Turn off Relaxed Ordering on head write-backs.  The writebacks
1312          * MUST be delivered in order or it will completely screw up
1313          * our bookkeeping.
1314          */
1315         dca_txctrl = er32(DCA_TXCTRL(0));
1316         dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1317         ew32(DCA_TXCTRL(0), dca_txctrl);
1318
1319         /* enable transmits */
1320         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1321         ew32(TXDCTL(0), txdctl);
1322
1323         /* Setup Transmit Descriptor Settings for eop descriptor */
1324         adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1325
1326         /* enable Report Status bit */
1327         adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1328 }
1329
1330 /**
1331  * igbvf_setup_srrctl - configure the receive control registers
1332  * @adapter: Board private structure
1333  **/
1334 static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1335 {
1336         struct e1000_hw *hw = &adapter->hw;
1337         u32 srrctl = 0;
1338
1339         srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1340                     E1000_SRRCTL_BSIZEHDR_MASK |
1341                     E1000_SRRCTL_BSIZEPKT_MASK);
1342
1343         /* Enable queue drop to avoid head of line blocking */
1344         srrctl |= E1000_SRRCTL_DROP_EN;
1345
1346         /* Setup buffer sizes */
1347         srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1348                   E1000_SRRCTL_BSIZEPKT_SHIFT;
1349
1350         if (adapter->rx_buffer_len < 2048) {
1351                 adapter->rx_ps_hdr_size = 0;
1352                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1353         } else {
1354                 adapter->rx_ps_hdr_size = 128;
1355                 srrctl |= adapter->rx_ps_hdr_size <<
1356                           E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1357                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1358         }
1359
1360         ew32(SRRCTL(0), srrctl);
1361 }
1362
1363 /**
1364  * igbvf_configure_rx - Configure Receive Unit after Reset
1365  * @adapter: board private structure
1366  *
1367  * Configure the Rx unit of the MAC after a reset.
1368  **/
1369 static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1370 {
1371         struct e1000_hw *hw = &adapter->hw;
1372         struct igbvf_ring *rx_ring = adapter->rx_ring;
1373         u64 rdba;
1374         u32 rxdctl;
1375
1376         /* disable receives */
1377         rxdctl = er32(RXDCTL(0));
1378         ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1379         e1e_flush();
1380         msleep(10);
1381
1382         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1383          * the Base and Length of the Rx Descriptor Ring
1384          */
1385         rdba = rx_ring->dma;
1386         ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1387         ew32(RDBAH(0), (rdba >> 32));
1388         ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1389         rx_ring->head = E1000_RDH(0);
1390         rx_ring->tail = E1000_RDT(0);
1391         ew32(RDH(0), 0);
1392         ew32(RDT(0), 0);
1393
1394         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1395         rxdctl &= 0xFFF00000;
1396         rxdctl |= IGBVF_RX_PTHRESH;
1397         rxdctl |= IGBVF_RX_HTHRESH << 8;
1398         rxdctl |= IGBVF_RX_WTHRESH << 16;
1399
1400         igbvf_set_rlpml(adapter);
1401
1402         /* enable receives */
1403         ew32(RXDCTL(0), rxdctl);
1404 }
1405
1406 /**
1407  * igbvf_set_multi - Multicast and Promiscuous mode set
1408  * @netdev: network interface device structure
1409  *
1410  * The set_multi entry point is called whenever the multicast address
1411  * list or the network interface flags are updated.  This routine is
1412  * responsible for configuring the hardware for proper multicast,
1413  * promiscuous mode, and all-multi behavior.
1414  **/
1415 static void igbvf_set_multi(struct net_device *netdev)
1416 {
1417         struct igbvf_adapter *adapter = netdev_priv(netdev);
1418         struct e1000_hw *hw = &adapter->hw;
1419         struct netdev_hw_addr *ha;
1420         u8  *mta_list = NULL;
1421         int i;
1422
1423         if (!netdev_mc_empty(netdev)) {
1424                 mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1425                                          GFP_ATOMIC);
1426                 if (!mta_list)
1427                         return;
1428         }
1429
1430         /* prepare a packed array of only addresses. */
1431         i = 0;
1432         netdev_for_each_mc_addr(ha, netdev)
1433                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1434
1435         spin_lock_bh(&hw->mbx_lock);
1436
1437         hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1438
1439         spin_unlock_bh(&hw->mbx_lock);
1440         kfree(mta_list);
1441 }
1442
1443 /**
1444  * igbvf_set_uni - Configure unicast MAC filters
1445  * @netdev: network interface device structure
1446  *
1447  * This routine is responsible for configuring the hardware for proper
1448  * unicast filters.
1449  **/
1450 static int igbvf_set_uni(struct net_device *netdev)
1451 {
1452         struct igbvf_adapter *adapter = netdev_priv(netdev);
1453         struct e1000_hw *hw = &adapter->hw;
1454
1455         if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1456                 pr_err("Too many unicast filters - No Space\n");
1457                 return -ENOSPC;
1458         }
1459
1460         spin_lock_bh(&hw->mbx_lock);
1461
1462         /* Clear all unicast MAC filters */
1463         hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1464
1465         spin_unlock_bh(&hw->mbx_lock);
1466
1467         if (!netdev_uc_empty(netdev)) {
1468                 struct netdev_hw_addr *ha;
1469
1470                 /* Add MAC filters one by one */
1471                 netdev_for_each_uc_addr(ha, netdev) {
1472                         spin_lock_bh(&hw->mbx_lock);
1473
1474                         hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1475                                                 ha->addr);
1476
1477                         spin_unlock_bh(&hw->mbx_lock);
1478                         udelay(200);
1479                 }
1480         }
1481
1482         return 0;
1483 }
1484
1485 static void igbvf_set_rx_mode(struct net_device *netdev)
1486 {
1487         igbvf_set_multi(netdev);
1488         igbvf_set_uni(netdev);
1489 }
1490
1491 /**
1492  * igbvf_configure - configure the hardware for Rx and Tx
1493  * @adapter: private board structure
1494  **/
1495 static void igbvf_configure(struct igbvf_adapter *adapter)
1496 {
1497         igbvf_set_rx_mode(adapter->netdev);
1498
1499         igbvf_restore_vlan(adapter);
1500
1501         igbvf_configure_tx(adapter);
1502         igbvf_setup_srrctl(adapter);
1503         igbvf_configure_rx(adapter);
1504         igbvf_alloc_rx_buffers(adapter->rx_ring,
1505                                igbvf_desc_unused(adapter->rx_ring));
1506 }
1507
1508 /* igbvf_reset - bring the hardware into a known good state
1509  * @adapter: private board structure
1510  *
1511  * This function boots the hardware and enables some settings that
1512  * require a configuration cycle of the hardware - those cannot be
1513  * set/changed during runtime. After reset the device needs to be
1514  * properly configured for Rx, Tx etc.
1515  */
1516 static void igbvf_reset(struct igbvf_adapter *adapter)
1517 {
1518         struct e1000_mac_info *mac = &adapter->hw.mac;
1519         struct net_device *netdev = adapter->netdev;
1520         struct e1000_hw *hw = &adapter->hw;
1521
1522         spin_lock_bh(&hw->mbx_lock);
1523
1524         /* Allow time for pending master requests to run */
1525         if (mac->ops.reset_hw(hw))
1526                 dev_info(&adapter->pdev->dev, "PF still resetting\n");
1527
1528         mac->ops.init_hw(hw);
1529
1530         spin_unlock_bh(&hw->mbx_lock);
1531
1532         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1533                 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
1534                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1535                        netdev->addr_len);
1536         }
1537
1538         adapter->last_reset = jiffies;
1539 }
1540
1541 int igbvf_up(struct igbvf_adapter *adapter)
1542 {
1543         struct e1000_hw *hw = &adapter->hw;
1544
1545         /* hardware has been reset, we need to reload some things */
1546         igbvf_configure(adapter);
1547
1548         clear_bit(__IGBVF_DOWN, &adapter->state);
1549
1550         napi_enable(&adapter->rx_ring->napi);
1551         if (adapter->msix_entries)
1552                 igbvf_configure_msix(adapter);
1553
1554         /* Clear any pending interrupts. */
1555         er32(EICR);
1556         igbvf_irq_enable(adapter);
1557
1558         /* start the watchdog */
1559         hw->mac.get_link_status = 1;
1560         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1561
1562         return 0;
1563 }
1564
1565 void igbvf_down(struct igbvf_adapter *adapter)
1566 {
1567         struct net_device *netdev = adapter->netdev;
1568         struct e1000_hw *hw = &adapter->hw;
1569         u32 rxdctl, txdctl;
1570
1571         /* signal that we're down so the interrupt handler does not
1572          * reschedule our watchdog timer
1573          */
1574         set_bit(__IGBVF_DOWN, &adapter->state);
1575
1576         /* disable receives in the hardware */
1577         rxdctl = er32(RXDCTL(0));
1578         ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1579
1580         netif_carrier_off(netdev);
1581         netif_stop_queue(netdev);
1582
1583         /* disable transmits in the hardware */
1584         txdctl = er32(TXDCTL(0));
1585         ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1586
1587         /* flush both disables and wait for them to finish */
1588         e1e_flush();
1589         msleep(10);
1590
1591         napi_disable(&adapter->rx_ring->napi);
1592
1593         igbvf_irq_disable(adapter);
1594
1595         del_timer_sync(&adapter->watchdog_timer);
1596
1597         /* record the stats before reset*/
1598         igbvf_update_stats(adapter);
1599
1600         adapter->link_speed = 0;
1601         adapter->link_duplex = 0;
1602
1603         igbvf_reset(adapter);
1604         igbvf_clean_tx_ring(adapter->tx_ring);
1605         igbvf_clean_rx_ring(adapter->rx_ring);
1606 }
1607
1608 void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1609 {
1610         might_sleep();
1611         while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
1612                 usleep_range(1000, 2000);
1613         igbvf_down(adapter);
1614         igbvf_up(adapter);
1615         clear_bit(__IGBVF_RESETTING, &adapter->state);
1616 }
1617
1618 /**
1619  * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1620  * @adapter: board private structure to initialize
1621  *
1622  * igbvf_sw_init initializes the Adapter private data structure.
1623  * Fields are initialized based on PCI device information and
1624  * OS network device settings (MTU size).
1625  **/
1626 static int igbvf_sw_init(struct igbvf_adapter *adapter)
1627 {
1628         struct net_device *netdev = adapter->netdev;
1629         s32 rc;
1630
1631         adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1632         adapter->rx_ps_hdr_size = 0;
1633         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1634         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1635
1636         adapter->tx_int_delay = 8;
1637         adapter->tx_abs_int_delay = 32;
1638         adapter->rx_int_delay = 0;
1639         adapter->rx_abs_int_delay = 8;
1640         adapter->requested_itr = 3;
1641         adapter->current_itr = IGBVF_START_ITR;
1642
1643         /* Set various function pointers */
1644         adapter->ei->init_ops(&adapter->hw);
1645
1646         rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1647         if (rc)
1648                 return rc;
1649
1650         rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1651         if (rc)
1652                 return rc;
1653
1654         igbvf_set_interrupt_capability(adapter);
1655
1656         if (igbvf_alloc_queues(adapter))
1657                 return -ENOMEM;
1658
1659         spin_lock_init(&adapter->tx_queue_lock);
1660
1661         /* Explicitly disable IRQ since the NIC can be in any state. */
1662         igbvf_irq_disable(adapter);
1663
1664         spin_lock_init(&adapter->stats_lock);
1665         spin_lock_init(&adapter->hw.mbx_lock);
1666
1667         set_bit(__IGBVF_DOWN, &adapter->state);
1668         return 0;
1669 }
1670
1671 static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1672 {
1673         struct e1000_hw *hw = &adapter->hw;
1674
1675         adapter->stats.last_gprc = er32(VFGPRC);
1676         adapter->stats.last_gorc = er32(VFGORC);
1677         adapter->stats.last_gptc = er32(VFGPTC);
1678         adapter->stats.last_gotc = er32(VFGOTC);
1679         adapter->stats.last_mprc = er32(VFMPRC);
1680         adapter->stats.last_gotlbc = er32(VFGOTLBC);
1681         adapter->stats.last_gptlbc = er32(VFGPTLBC);
1682         adapter->stats.last_gorlbc = er32(VFGORLBC);
1683         adapter->stats.last_gprlbc = er32(VFGPRLBC);
1684
1685         adapter->stats.base_gprc = er32(VFGPRC);
1686         adapter->stats.base_gorc = er32(VFGORC);
1687         adapter->stats.base_gptc = er32(VFGPTC);
1688         adapter->stats.base_gotc = er32(VFGOTC);
1689         adapter->stats.base_mprc = er32(VFMPRC);
1690         adapter->stats.base_gotlbc = er32(VFGOTLBC);
1691         adapter->stats.base_gptlbc = er32(VFGPTLBC);
1692         adapter->stats.base_gorlbc = er32(VFGORLBC);
1693         adapter->stats.base_gprlbc = er32(VFGPRLBC);
1694 }
1695
1696 /**
1697  * igbvf_open - Called when a network interface is made active
1698  * @netdev: network interface device structure
1699  *
1700  * Returns 0 on success, negative value on failure
1701  *
1702  * The open entry point is called when a network interface is made
1703  * active by the system (IFF_UP).  At this point all resources needed
1704  * for transmit and receive operations are allocated, the interrupt
1705  * handler is registered with the OS, the watchdog timer is started,
1706  * and the stack is notified that the interface is ready.
1707  **/
1708 static int igbvf_open(struct net_device *netdev)
1709 {
1710         struct igbvf_adapter *adapter = netdev_priv(netdev);
1711         struct e1000_hw *hw = &adapter->hw;
1712         int err;
1713
1714         /* disallow open during test */
1715         if (test_bit(__IGBVF_TESTING, &adapter->state))
1716                 return -EBUSY;
1717
1718         /* allocate transmit descriptors */
1719         err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);
1720         if (err)
1721                 goto err_setup_tx;
1722
1723         /* allocate receive descriptors */
1724         err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);
1725         if (err)
1726                 goto err_setup_rx;
1727
1728         /* before we allocate an interrupt, we must be ready to handle it.
1729          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1730          * as soon as we call pci_request_irq, so we have to setup our
1731          * clean_rx handler before we do so.
1732          */
1733         igbvf_configure(adapter);
1734
1735         err = igbvf_request_irq(adapter);
1736         if (err)
1737                 goto err_req_irq;
1738
1739         /* From here on the code is the same as igbvf_up() */
1740         clear_bit(__IGBVF_DOWN, &adapter->state);
1741
1742         napi_enable(&adapter->rx_ring->napi);
1743
1744         /* clear any pending interrupts */
1745         er32(EICR);
1746
1747         igbvf_irq_enable(adapter);
1748
1749         /* start the watchdog */
1750         hw->mac.get_link_status = 1;
1751         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1752
1753         return 0;
1754
1755 err_req_irq:
1756         igbvf_free_rx_resources(adapter->rx_ring);
1757 err_setup_rx:
1758         igbvf_free_tx_resources(adapter->tx_ring);
1759 err_setup_tx:
1760         igbvf_reset(adapter);
1761
1762         return err;
1763 }
1764
1765 /**
1766  * igbvf_close - Disables a network interface
1767  * @netdev: network interface device structure
1768  *
1769  * Returns 0, this is not allowed to fail
1770  *
1771  * The close entry point is called when an interface is de-activated
1772  * by the OS.  The hardware is still under the drivers control, but
1773  * needs to be disabled.  A global MAC reset is issued to stop the
1774  * hardware, and all transmit and receive resources are freed.
1775  **/
1776 static int igbvf_close(struct net_device *netdev)
1777 {
1778         struct igbvf_adapter *adapter = netdev_priv(netdev);
1779
1780         WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1781         igbvf_down(adapter);
1782
1783         igbvf_free_irq(adapter);
1784
1785         igbvf_free_tx_resources(adapter->tx_ring);
1786         igbvf_free_rx_resources(adapter->rx_ring);
1787
1788         return 0;
1789 }
1790
1791 /**
1792  * igbvf_set_mac - Change the Ethernet Address of the NIC
1793  * @netdev: network interface device structure
1794  * @p: pointer to an address structure
1795  *
1796  * Returns 0 on success, negative on failure
1797  **/
1798 static int igbvf_set_mac(struct net_device *netdev, void *p)
1799 {
1800         struct igbvf_adapter *adapter = netdev_priv(netdev);
1801         struct e1000_hw *hw = &adapter->hw;
1802         struct sockaddr *addr = p;
1803
1804         if (!is_valid_ether_addr(addr->sa_data))
1805                 return -EADDRNOTAVAIL;
1806
1807         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1808
1809         spin_lock_bh(&hw->mbx_lock);
1810
1811         hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1812
1813         spin_unlock_bh(&hw->mbx_lock);
1814
1815         if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
1816                 return -EADDRNOTAVAIL;
1817
1818         eth_hw_addr_set(netdev, addr->sa_data);
1819
1820         return 0;
1821 }
1822
1823 #define UPDATE_VF_COUNTER(reg, name) \
1824 { \
1825         u32 current_counter = er32(reg); \
1826         if (current_counter < adapter->stats.last_##name) \
1827                 adapter->stats.name += 0x100000000LL; \
1828         adapter->stats.last_##name = current_counter; \
1829         adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1830         adapter->stats.name |= current_counter; \
1831 }
1832
1833 /**
1834  * igbvf_update_stats - Update the board statistics counters
1835  * @adapter: board private structure
1836 **/
1837 void igbvf_update_stats(struct igbvf_adapter *adapter)
1838 {
1839         struct e1000_hw *hw = &adapter->hw;
1840         struct pci_dev *pdev = adapter->pdev;
1841
1842         /* Prevent stats update while adapter is being reset, link is down
1843          * or if the pci connection is down.
1844          */
1845         if (adapter->link_speed == 0)
1846                 return;
1847
1848         if (test_bit(__IGBVF_RESETTING, &adapter->state))
1849                 return;
1850
1851         if (pci_channel_offline(pdev))
1852                 return;
1853
1854         UPDATE_VF_COUNTER(VFGPRC, gprc);
1855         UPDATE_VF_COUNTER(VFGORC, gorc);
1856         UPDATE_VF_COUNTER(VFGPTC, gptc);
1857         UPDATE_VF_COUNTER(VFGOTC, gotc);
1858         UPDATE_VF_COUNTER(VFMPRC, mprc);
1859         UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1860         UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1861         UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1862         UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1863
1864         /* Fill out the OS statistics structure */
1865         adapter->netdev->stats.multicast = adapter->stats.mprc;
1866 }
1867
1868 static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1869 {
1870         dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1871                  adapter->link_speed,
1872                  adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1873 }
1874
1875 static bool igbvf_has_link(struct igbvf_adapter *adapter)
1876 {
1877         struct e1000_hw *hw = &adapter->hw;
1878         s32 ret_val = E1000_SUCCESS;
1879         bool link_active;
1880
1881         /* If interface is down, stay link down */
1882         if (test_bit(__IGBVF_DOWN, &adapter->state))
1883                 return false;
1884
1885         spin_lock_bh(&hw->mbx_lock);
1886
1887         ret_val = hw->mac.ops.check_for_link(hw);
1888
1889         spin_unlock_bh(&hw->mbx_lock);
1890
1891         link_active = !hw->mac.get_link_status;
1892
1893         /* if check for link returns error we will need to reset */
1894         if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1895                 schedule_work(&adapter->reset_task);
1896
1897         return link_active;
1898 }
1899
1900 /**
1901  * igbvf_watchdog - Timer Call-back
1902  * @t: timer list pointer containing private struct
1903  **/
1904 static void igbvf_watchdog(struct timer_list *t)
1905 {
1906         struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);
1907
1908         /* Do the rest outside of interrupt context */
1909         schedule_work(&adapter->watchdog_task);
1910 }
1911
1912 static void igbvf_watchdog_task(struct work_struct *work)
1913 {
1914         struct igbvf_adapter *adapter = container_of(work,
1915                                                      struct igbvf_adapter,
1916                                                      watchdog_task);
1917         struct net_device *netdev = adapter->netdev;
1918         struct e1000_mac_info *mac = &adapter->hw.mac;
1919         struct igbvf_ring *tx_ring = adapter->tx_ring;
1920         struct e1000_hw *hw = &adapter->hw;
1921         u32 link;
1922         int tx_pending = 0;
1923
1924         link = igbvf_has_link(adapter);
1925
1926         if (link) {
1927                 if (!netif_carrier_ok(netdev)) {
1928                         mac->ops.get_link_up_info(&adapter->hw,
1929                                                   &adapter->link_speed,
1930                                                   &adapter->link_duplex);
1931                         igbvf_print_link_info(adapter);
1932
1933                         netif_carrier_on(netdev);
1934                         netif_wake_queue(netdev);
1935                 }
1936         } else {
1937                 if (netif_carrier_ok(netdev)) {
1938                         adapter->link_speed = 0;
1939                         adapter->link_duplex = 0;
1940                         dev_info(&adapter->pdev->dev, "Link is Down\n");
1941                         netif_carrier_off(netdev);
1942                         netif_stop_queue(netdev);
1943                 }
1944         }
1945
1946         if (netif_carrier_ok(netdev)) {
1947                 igbvf_update_stats(adapter);
1948         } else {
1949                 tx_pending = (igbvf_desc_unused(tx_ring) + 1 <
1950                               tx_ring->count);
1951                 if (tx_pending) {
1952                         /* We've lost link, so the controller stops DMA,
1953                          * but we've got queued Tx work that's never going
1954                          * to get done, so reset controller to flush Tx.
1955                          * (Do the reset outside of interrupt context).
1956                          */
1957                         adapter->tx_timeout_count++;
1958                         schedule_work(&adapter->reset_task);
1959                 }
1960         }
1961
1962         /* Cause software interrupt to ensure Rx ring is cleaned */
1963         ew32(EICS, adapter->rx_ring->eims_value);
1964
1965         /* Reset the timer */
1966         if (!test_bit(__IGBVF_DOWN, &adapter->state))
1967                 mod_timer(&adapter->watchdog_timer,
1968                           round_jiffies(jiffies + (2 * HZ)));
1969 }
1970
1971 #define IGBVF_TX_FLAGS_CSUM             0x00000001
1972 #define IGBVF_TX_FLAGS_VLAN             0x00000002
1973 #define IGBVF_TX_FLAGS_TSO              0x00000004
1974 #define IGBVF_TX_FLAGS_IPV4             0x00000008
1975 #define IGBVF_TX_FLAGS_VLAN_MASK        0xffff0000
1976 #define IGBVF_TX_FLAGS_VLAN_SHIFT       16
1977
1978 static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1979                               u32 type_tucmd, u32 mss_l4len_idx)
1980 {
1981         struct e1000_adv_tx_context_desc *context_desc;
1982         struct igbvf_buffer *buffer_info;
1983         u16 i = tx_ring->next_to_use;
1984
1985         context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1986         buffer_info = &tx_ring->buffer_info[i];
1987
1988         i++;
1989         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1990
1991         /* set bits to identify this as an advanced context descriptor */
1992         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1993
1994         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
1995         context_desc->seqnum_seed       = 0;
1996         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
1997         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
1998
1999         buffer_info->time_stamp = jiffies;
2000         buffer_info->dma = 0;
2001 }
2002
2003 static int igbvf_tso(struct igbvf_ring *tx_ring,
2004                      struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2005 {
2006         u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
2007         union {
2008                 struct iphdr *v4;
2009                 struct ipv6hdr *v6;
2010                 unsigned char *hdr;
2011         } ip;
2012         union {
2013                 struct tcphdr *tcp;
2014                 unsigned char *hdr;
2015         } l4;
2016         u32 paylen, l4_offset;
2017         int err;
2018
2019         if (skb->ip_summed != CHECKSUM_PARTIAL)
2020                 return 0;
2021
2022         if (!skb_is_gso(skb))
2023                 return 0;
2024
2025         err = skb_cow_head(skb, 0);
2026         if (err < 0)
2027                 return err;
2028
2029         ip.hdr = skb_network_header(skb);
2030         l4.hdr = skb_checksum_start(skb);
2031
2032         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2033         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2034
2035         /* initialize outer IP header fields */
2036         if (ip.v4->version == 4) {
2037                 unsigned char *csum_start = skb_checksum_start(skb);
2038                 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2039
2040                 /* IP header will have to cancel out any data that
2041                  * is not a part of the outer IP header
2042                  */
2043                 ip.v4->check = csum_fold(csum_partial(trans_start,
2044                                                       csum_start - trans_start,
2045                                                       0));
2046                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2047
2048                 ip.v4->tot_len = 0;
2049         } else {
2050                 ip.v6->payload_len = 0;
2051         }
2052
2053         /* determine offset of inner transport header */
2054         l4_offset = l4.hdr - skb->data;
2055
2056         /* compute length of segmentation header */
2057         *hdr_len = (l4.tcp->doff * 4) + l4_offset;
2058
2059         /* remove payload length from inner checksum */
2060         paylen = skb->len - l4_offset;
2061         csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
2062
2063         /* MSS L4LEN IDX */
2064         mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2065         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2066
2067         /* VLAN MACLEN IPLEN */
2068         vlan_macip_lens = l4.hdr - ip.hdr;
2069         vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2070         vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2071
2072         igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2073
2074         return 1;
2075 }
2076
2077 static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2078                           u32 tx_flags, __be16 protocol)
2079 {
2080         u32 vlan_macip_lens = 0;
2081         u32 type_tucmd = 0;
2082
2083         if (skb->ip_summed != CHECKSUM_PARTIAL) {
2084 csum_failed:
2085                 if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2086                         return false;
2087                 goto no_csum;
2088         }
2089
2090         switch (skb->csum_offset) {
2091         case offsetof(struct tcphdr, check):
2092                 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2093                 fallthrough;
2094         case offsetof(struct udphdr, check):
2095                 break;
2096         case offsetof(struct sctphdr, checksum):
2097                 /* validate that this is actually an SCTP request */
2098                 if (skb_csum_is_sctp(skb)) {
2099                         type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2100                         break;
2101                 }
2102                 fallthrough;
2103         default:
2104                 skb_checksum_help(skb);
2105                 goto csum_failed;
2106         }
2107
2108         vlan_macip_lens = skb_checksum_start_offset(skb) -
2109                           skb_network_offset(skb);
2110 no_csum:
2111         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2112         vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2113
2114         igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
2115         return true;
2116 }
2117
2118 static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2119 {
2120         struct igbvf_adapter *adapter = netdev_priv(netdev);
2121
2122         /* there is enough descriptors then we don't need to worry  */
2123         if (igbvf_desc_unused(adapter->tx_ring) >= size)
2124                 return 0;
2125
2126         netif_stop_queue(netdev);
2127
2128         /* Herbert's original patch had:
2129          *  smp_mb__after_netif_stop_queue();
2130          * but since that doesn't exist yet, just open code it.
2131          */
2132         smp_mb();
2133
2134         /* We need to check again just in case room has been made available */
2135         if (igbvf_desc_unused(adapter->tx_ring) < size)
2136                 return -EBUSY;
2137
2138         netif_wake_queue(netdev);
2139
2140         ++adapter->restart_queue;
2141         return 0;
2142 }
2143
2144 #define IGBVF_MAX_TXD_PWR       16
2145 #define IGBVF_MAX_DATA_PER_TXD  (1u << IGBVF_MAX_TXD_PWR)
2146
2147 static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2148                                    struct igbvf_ring *tx_ring,
2149                                    struct sk_buff *skb)
2150 {
2151         struct igbvf_buffer *buffer_info;
2152         struct pci_dev *pdev = adapter->pdev;
2153         unsigned int len = skb_headlen(skb);
2154         unsigned int count = 0, i;
2155         unsigned int f;
2156
2157         i = tx_ring->next_to_use;
2158
2159         buffer_info = &tx_ring->buffer_info[i];
2160         BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2161         buffer_info->length = len;
2162         /* set time_stamp *before* dma to help avoid a possible race */
2163         buffer_info->time_stamp = jiffies;
2164         buffer_info->mapped_as_page = false;
2165         buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2166                                           DMA_TO_DEVICE);
2167         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2168                 goto dma_error;
2169
2170         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2171                 const skb_frag_t *frag;
2172
2173                 count++;
2174                 i++;
2175                 if (i == tx_ring->count)
2176                         i = 0;
2177
2178                 frag = &skb_shinfo(skb)->frags[f];
2179                 len = skb_frag_size(frag);
2180
2181                 buffer_info = &tx_ring->buffer_info[i];
2182                 BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2183                 buffer_info->length = len;
2184                 buffer_info->time_stamp = jiffies;
2185                 buffer_info->mapped_as_page = true;
2186                 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,
2187                                                     DMA_TO_DEVICE);
2188                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2189                         goto dma_error;
2190         }
2191
2192         tx_ring->buffer_info[i].skb = skb;
2193
2194         return ++count;
2195
2196 dma_error:
2197         dev_err(&pdev->dev, "TX DMA map failed\n");
2198
2199         /* clear timestamp and dma mappings for failed buffer_info mapping */
2200         buffer_info->dma = 0;
2201         buffer_info->time_stamp = 0;
2202         buffer_info->length = 0;
2203         buffer_info->mapped_as_page = false;
2204         if (count)
2205                 count--;
2206
2207         /* clear timestamp and dma mappings for remaining portion of packet */
2208         while (count--) {
2209                 if (i == 0)
2210                         i += tx_ring->count;
2211                 i--;
2212                 buffer_info = &tx_ring->buffer_info[i];
2213                 igbvf_put_txbuf(adapter, buffer_info);
2214         }
2215
2216         return 0;
2217 }
2218
2219 static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2220                                       struct igbvf_ring *tx_ring,
2221                                       int tx_flags, int count,
2222                                       unsigned int first, u32 paylen,
2223                                       u8 hdr_len)
2224 {
2225         union e1000_adv_tx_desc *tx_desc = NULL;
2226         struct igbvf_buffer *buffer_info;
2227         u32 olinfo_status = 0, cmd_type_len;
2228         unsigned int i;
2229
2230         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2231                         E1000_ADVTXD_DCMD_DEXT);
2232
2233         if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2234                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2235
2236         if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2237                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2238
2239                 /* insert tcp checksum */
2240                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2241
2242                 /* insert ip checksum */
2243                 if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2244                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2245
2246         } else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2247                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2248         }
2249
2250         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2251
2252         i = tx_ring->next_to_use;
2253         while (count--) {
2254                 buffer_info = &tx_ring->buffer_info[i];
2255                 tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2256                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2257                 tx_desc->read.cmd_type_len =
2258                          cpu_to_le32(cmd_type_len | buffer_info->length);
2259                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2260                 i++;
2261                 if (i == tx_ring->count)
2262                         i = 0;
2263         }
2264
2265         tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2266         /* Force memory writes to complete before letting h/w
2267          * know there are new descriptors to fetch.  (Only
2268          * applicable for weak-ordered memory model archs,
2269          * such as IA-64).
2270          */
2271         wmb();
2272
2273         tx_ring->buffer_info[first].next_to_watch = tx_desc;
2274         tx_ring->next_to_use = i;
2275         writel(i, adapter->hw.hw_addr + tx_ring->tail);
2276 }
2277
2278 static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2279                                              struct net_device *netdev,
2280                                              struct igbvf_ring *tx_ring)
2281 {
2282         struct igbvf_adapter *adapter = netdev_priv(netdev);
2283         unsigned int first, tx_flags = 0;
2284         u8 hdr_len = 0;
2285         int count = 0;
2286         int tso = 0;
2287         __be16 protocol = vlan_get_protocol(skb);
2288
2289         if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2290                 dev_kfree_skb_any(skb);
2291                 return NETDEV_TX_OK;
2292         }
2293
2294         if (skb->len <= 0) {
2295                 dev_kfree_skb_any(skb);
2296                 return NETDEV_TX_OK;
2297         }
2298
2299         /* need: count + 4 desc gap to keep tail from touching
2300          *       + 2 desc gap to keep tail from touching head,
2301          *       + 1 desc for skb->data,
2302          *       + 1 desc for context descriptor,
2303          * head, otherwise try next time
2304          */
2305         if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2306                 /* this is a hard error */
2307                 return NETDEV_TX_BUSY;
2308         }
2309
2310         if (skb_vlan_tag_present(skb)) {
2311                 tx_flags |= IGBVF_TX_FLAGS_VLAN;
2312                 tx_flags |= (skb_vlan_tag_get(skb) <<
2313                              IGBVF_TX_FLAGS_VLAN_SHIFT);
2314         }
2315
2316         if (protocol == htons(ETH_P_IP))
2317                 tx_flags |= IGBVF_TX_FLAGS_IPV4;
2318
2319         first = tx_ring->next_to_use;
2320
2321         tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);
2322         if (unlikely(tso < 0)) {
2323                 dev_kfree_skb_any(skb);
2324                 return NETDEV_TX_OK;
2325         }
2326
2327         if (tso)
2328                 tx_flags |= IGBVF_TX_FLAGS_TSO;
2329         else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2330                  (skb->ip_summed == CHECKSUM_PARTIAL))
2331                 tx_flags |= IGBVF_TX_FLAGS_CSUM;
2332
2333         /* count reflects descriptors mapped, if 0 then mapping error
2334          * has occurred and we need to rewind the descriptor queue
2335          */
2336         count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2337
2338         if (count) {
2339                 igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2340                                    first, skb->len, hdr_len);
2341                 /* Make sure there is space in the ring for the next send. */
2342                 igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2343         } else {
2344                 dev_kfree_skb_any(skb);
2345                 tx_ring->buffer_info[first].time_stamp = 0;
2346                 tx_ring->next_to_use = first;
2347         }
2348
2349         return NETDEV_TX_OK;
2350 }
2351
2352 static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2353                                     struct net_device *netdev)
2354 {
2355         struct igbvf_adapter *adapter = netdev_priv(netdev);
2356         struct igbvf_ring *tx_ring;
2357
2358         if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2359                 dev_kfree_skb_any(skb);
2360                 return NETDEV_TX_OK;
2361         }
2362
2363         tx_ring = &adapter->tx_ring[0];
2364
2365         return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2366 }
2367
2368 /**
2369  * igbvf_tx_timeout - Respond to a Tx Hang
2370  * @netdev: network interface device structure
2371  * @txqueue: queue timing out (unused)
2372  **/
2373 static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2374 {
2375         struct igbvf_adapter *adapter = netdev_priv(netdev);
2376
2377         /* Do the reset outside of interrupt context */
2378         adapter->tx_timeout_count++;
2379         schedule_work(&adapter->reset_task);
2380 }
2381
2382 static void igbvf_reset_task(struct work_struct *work)
2383 {
2384         struct igbvf_adapter *adapter;
2385
2386         adapter = container_of(work, struct igbvf_adapter, reset_task);
2387
2388         igbvf_reinit_locked(adapter);
2389 }
2390
2391 /**
2392  * igbvf_change_mtu - Change the Maximum Transfer Unit
2393  * @netdev: network interface device structure
2394  * @new_mtu: new value for maximum frame size
2395  *
2396  * Returns 0 on success, negative on failure
2397  **/
2398 static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2399 {
2400         struct igbvf_adapter *adapter = netdev_priv(netdev);
2401         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2402
2403         while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2404                 usleep_range(1000, 2000);
2405         /* igbvf_down has a dependency on max_frame_size */
2406         adapter->max_frame_size = max_frame;
2407         if (netif_running(netdev))
2408                 igbvf_down(adapter);
2409
2410         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2411          * means we reserve 2 more, this pushes us to allocate from the next
2412          * larger slab size.
2413          * i.e. RXBUFFER_2048 --> size-4096 slab
2414          * However with the new *_jumbo_rx* routines, jumbo receives will use
2415          * fragmented skbs
2416          */
2417
2418         if (max_frame <= 1024)
2419                 adapter->rx_buffer_len = 1024;
2420         else if (max_frame <= 2048)
2421                 adapter->rx_buffer_len = 2048;
2422         else
2423 #if (PAGE_SIZE / 2) > 16384
2424                 adapter->rx_buffer_len = 16384;
2425 #else
2426                 adapter->rx_buffer_len = PAGE_SIZE / 2;
2427 #endif
2428
2429         /* adjust allocation if LPE protects us, and we aren't using SBP */
2430         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2431             (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2432                 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2433                                          ETH_FCS_LEN;
2434
2435         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2436                    netdev->mtu, new_mtu);
2437         netdev->mtu = new_mtu;
2438
2439         if (netif_running(netdev))
2440                 igbvf_up(adapter);
2441         else
2442                 igbvf_reset(adapter);
2443
2444         clear_bit(__IGBVF_RESETTING, &adapter->state);
2445
2446         return 0;
2447 }
2448
2449 static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2450 {
2451         switch (cmd) {
2452         default:
2453                 return -EOPNOTSUPP;
2454         }
2455 }
2456
2457 static int igbvf_suspend(struct device *dev_d)
2458 {
2459         struct net_device *netdev = dev_get_drvdata(dev_d);
2460         struct igbvf_adapter *adapter = netdev_priv(netdev);
2461
2462         netif_device_detach(netdev);
2463
2464         if (netif_running(netdev)) {
2465                 WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2466                 igbvf_down(adapter);
2467                 igbvf_free_irq(adapter);
2468         }
2469
2470         return 0;
2471 }
2472
2473 static int __maybe_unused igbvf_resume(struct device *dev_d)
2474 {
2475         struct pci_dev *pdev = to_pci_dev(dev_d);
2476         struct net_device *netdev = pci_get_drvdata(pdev);
2477         struct igbvf_adapter *adapter = netdev_priv(netdev);
2478         u32 err;
2479
2480         pci_set_master(pdev);
2481
2482         if (netif_running(netdev)) {
2483                 err = igbvf_request_irq(adapter);
2484                 if (err)
2485                         return err;
2486         }
2487
2488         igbvf_reset(adapter);
2489
2490         if (netif_running(netdev))
2491                 igbvf_up(adapter);
2492
2493         netif_device_attach(netdev);
2494
2495         return 0;
2496 }
2497
2498 static void igbvf_shutdown(struct pci_dev *pdev)
2499 {
2500         igbvf_suspend(&pdev->dev);
2501 }
2502
2503 #ifdef CONFIG_NET_POLL_CONTROLLER
2504 /* Polling 'interrupt' - used by things like netconsole to send skbs
2505  * without having to re-enable interrupts. It's not called while
2506  * the interrupt routine is executing.
2507  */
2508 static void igbvf_netpoll(struct net_device *netdev)
2509 {
2510         struct igbvf_adapter *adapter = netdev_priv(netdev);
2511
2512         disable_irq(adapter->pdev->irq);
2513
2514         igbvf_clean_tx_irq(adapter->tx_ring);
2515
2516         enable_irq(adapter->pdev->irq);
2517 }
2518 #endif
2519
2520 /**
2521  * igbvf_io_error_detected - called when PCI error is detected
2522  * @pdev: Pointer to PCI device
2523  * @state: The current pci connection state
2524  *
2525  * This function is called after a PCI bus error affecting
2526  * this device has been detected.
2527  */
2528 static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2529                                                 pci_channel_state_t state)
2530 {
2531         struct net_device *netdev = pci_get_drvdata(pdev);
2532         struct igbvf_adapter *adapter = netdev_priv(netdev);
2533
2534         netif_device_detach(netdev);
2535
2536         if (state == pci_channel_io_perm_failure)
2537                 return PCI_ERS_RESULT_DISCONNECT;
2538
2539         if (netif_running(netdev))
2540                 igbvf_down(adapter);
2541         pci_disable_device(pdev);
2542
2543         /* Request a slot reset. */
2544         return PCI_ERS_RESULT_NEED_RESET;
2545 }
2546
2547 /**
2548  * igbvf_io_slot_reset - called after the pci bus has been reset.
2549  * @pdev: Pointer to PCI device
2550  *
2551  * Restart the card from scratch, as if from a cold-boot. Implementation
2552  * resembles the first-half of the igbvf_resume routine.
2553  */
2554 static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2555 {
2556         struct net_device *netdev = pci_get_drvdata(pdev);
2557         struct igbvf_adapter *adapter = netdev_priv(netdev);
2558
2559         if (pci_enable_device_mem(pdev)) {
2560                 dev_err(&pdev->dev,
2561                         "Cannot re-enable PCI device after reset.\n");
2562                 return PCI_ERS_RESULT_DISCONNECT;
2563         }
2564         pci_set_master(pdev);
2565
2566         igbvf_reset(adapter);
2567
2568         return PCI_ERS_RESULT_RECOVERED;
2569 }
2570
2571 /**
2572  * igbvf_io_resume - called when traffic can start flowing again.
2573  * @pdev: Pointer to PCI device
2574  *
2575  * This callback is called when the error recovery driver tells us that
2576  * its OK to resume normal operation. Implementation resembles the
2577  * second-half of the igbvf_resume routine.
2578  */
2579 static void igbvf_io_resume(struct pci_dev *pdev)
2580 {
2581         struct net_device *netdev = pci_get_drvdata(pdev);
2582         struct igbvf_adapter *adapter = netdev_priv(netdev);
2583
2584         if (netif_running(netdev)) {
2585                 if (igbvf_up(adapter)) {
2586                         dev_err(&pdev->dev,
2587                                 "can't bring device back up after reset\n");
2588                         return;
2589                 }
2590         }
2591
2592         netif_device_attach(netdev);
2593 }
2594
2595 /**
2596  * igbvf_io_prepare - prepare device driver for PCI reset
2597  * @pdev: PCI device information struct
2598  */
2599 static void igbvf_io_prepare(struct pci_dev *pdev)
2600 {
2601         struct net_device *netdev = pci_get_drvdata(pdev);
2602         struct igbvf_adapter *adapter = netdev_priv(netdev);
2603
2604         while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2605                 usleep_range(1000, 2000);
2606         igbvf_down(adapter);
2607 }
2608
2609 /**
2610  * igbvf_io_reset_done - PCI reset done, device driver reset can begin
2611  * @pdev: PCI device information struct
2612  */
2613 static void igbvf_io_reset_done(struct pci_dev *pdev)
2614 {
2615         struct net_device *netdev = pci_get_drvdata(pdev);
2616         struct igbvf_adapter *adapter = netdev_priv(netdev);
2617
2618         igbvf_up(adapter);
2619         clear_bit(__IGBVF_RESETTING, &adapter->state);
2620 }
2621
2622 static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2623 {
2624         struct e1000_hw *hw = &adapter->hw;
2625         struct net_device *netdev = adapter->netdev;
2626         struct pci_dev *pdev = adapter->pdev;
2627
2628         if (hw->mac.type == e1000_vfadapt_i350)
2629                 dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2630         else
2631                 dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2632         dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2633 }
2634
2635 static int igbvf_set_features(struct net_device *netdev,
2636                               netdev_features_t features)
2637 {
2638         struct igbvf_adapter *adapter = netdev_priv(netdev);
2639
2640         if (features & NETIF_F_RXCSUM)
2641                 adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2642         else
2643                 adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2644
2645         return 0;
2646 }
2647
2648 #define IGBVF_MAX_MAC_HDR_LEN           127
2649 #define IGBVF_MAX_NETWORK_HDR_LEN       511
2650
2651 static netdev_features_t
2652 igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2653                      netdev_features_t features)
2654 {
2655         unsigned int network_hdr_len, mac_hdr_len;
2656
2657         /* Make certain the headers can be described by a context descriptor */
2658         mac_hdr_len = skb_network_header(skb) - skb->data;
2659         if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2660                 return features & ~(NETIF_F_HW_CSUM |
2661                                     NETIF_F_SCTP_CRC |
2662                                     NETIF_F_HW_VLAN_CTAG_TX |
2663                                     NETIF_F_TSO |
2664                                     NETIF_F_TSO6);
2665
2666         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2667         if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))
2668                 return features & ~(NETIF_F_HW_CSUM |
2669                                     NETIF_F_SCTP_CRC |
2670                                     NETIF_F_TSO |
2671                                     NETIF_F_TSO6);
2672
2673         /* We can only support IPV4 TSO in tunnels if we can mangle the
2674          * inner IP ID field, so strip TSO if MANGLEID is not supported.
2675          */
2676         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2677                 features &= ~NETIF_F_TSO;
2678
2679         return features;
2680 }
2681
2682 static const struct net_device_ops igbvf_netdev_ops = {
2683         .ndo_open               = igbvf_open,
2684         .ndo_stop               = igbvf_close,
2685         .ndo_start_xmit         = igbvf_xmit_frame,
2686         .ndo_set_rx_mode        = igbvf_set_rx_mode,
2687         .ndo_set_mac_address    = igbvf_set_mac,
2688         .ndo_change_mtu         = igbvf_change_mtu,
2689         .ndo_eth_ioctl          = igbvf_ioctl,
2690         .ndo_tx_timeout         = igbvf_tx_timeout,
2691         .ndo_vlan_rx_add_vid    = igbvf_vlan_rx_add_vid,
2692         .ndo_vlan_rx_kill_vid   = igbvf_vlan_rx_kill_vid,
2693 #ifdef CONFIG_NET_POLL_CONTROLLER
2694         .ndo_poll_controller    = igbvf_netpoll,
2695 #endif
2696         .ndo_set_features       = igbvf_set_features,
2697         .ndo_features_check     = igbvf_features_check,
2698 };
2699
2700 /**
2701  * igbvf_probe - Device Initialization Routine
2702  * @pdev: PCI device information struct
2703  * @ent: entry in igbvf_pci_tbl
2704  *
2705  * Returns 0 on success, negative on failure
2706  *
2707  * igbvf_probe initializes an adapter identified by a pci_dev structure.
2708  * The OS initialization, configuring of the adapter private structure,
2709  * and a hardware reset occur.
2710  **/
2711 static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2712 {
2713         struct net_device *netdev;
2714         struct igbvf_adapter *adapter;
2715         struct e1000_hw *hw;
2716         const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
2717         static int cards_found;
2718         int err;
2719
2720         err = pci_enable_device_mem(pdev);
2721         if (err)
2722                 return err;
2723
2724         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2725         if (err) {
2726                 dev_err(&pdev->dev,
2727                         "No usable DMA configuration, aborting\n");
2728                 goto err_dma;
2729         }
2730
2731         err = pci_request_regions(pdev, igbvf_driver_name);
2732         if (err)
2733                 goto err_pci_reg;
2734
2735         pci_set_master(pdev);
2736
2737         err = -ENOMEM;
2738         netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2739         if (!netdev)
2740                 goto err_alloc_etherdev;
2741
2742         SET_NETDEV_DEV(netdev, &pdev->dev);
2743
2744         pci_set_drvdata(pdev, netdev);
2745         adapter = netdev_priv(netdev);
2746         hw = &adapter->hw;
2747         adapter->netdev = netdev;
2748         adapter->pdev = pdev;
2749         adapter->ei = ei;
2750         adapter->pba = ei->pba;
2751         adapter->flags = ei->flags;
2752         adapter->hw.back = adapter;
2753         adapter->hw.mac.type = ei->mac;
2754         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2755
2756         /* PCI config space info */
2757
2758         hw->vendor_id = pdev->vendor;
2759         hw->device_id = pdev->device;
2760         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2761         hw->subsystem_device_id = pdev->subsystem_device;
2762         hw->revision_id = pdev->revision;
2763
2764         err = -EIO;
2765         adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2766                                       pci_resource_len(pdev, 0));
2767
2768         if (!adapter->hw.hw_addr)
2769                 goto err_ioremap;
2770
2771         if (ei->get_variants) {
2772                 err = ei->get_variants(adapter);
2773                 if (err)
2774                         goto err_get_variants;
2775         }
2776
2777         /* setup adapter struct */
2778         err = igbvf_sw_init(adapter);
2779         if (err)
2780                 goto err_sw_init;
2781
2782         /* construct the net_device struct */
2783         netdev->netdev_ops = &igbvf_netdev_ops;
2784
2785         igbvf_set_ethtool_ops(netdev);
2786         netdev->watchdog_timeo = 5 * HZ;
2787         strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
2788
2789         adapter->bd_number = cards_found++;
2790
2791         netdev->hw_features = NETIF_F_SG |
2792                               NETIF_F_TSO |
2793                               NETIF_F_TSO6 |
2794                               NETIF_F_RXCSUM |
2795                               NETIF_F_HW_CSUM |
2796                               NETIF_F_SCTP_CRC;
2797
2798 #define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2799                                     NETIF_F_GSO_GRE_CSUM | \
2800                                     NETIF_F_GSO_IPXIP4 | \
2801                                     NETIF_F_GSO_IPXIP6 | \
2802                                     NETIF_F_GSO_UDP_TUNNEL | \
2803                                     NETIF_F_GSO_UDP_TUNNEL_CSUM)
2804
2805         netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2806         netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2807                                IGBVF_GSO_PARTIAL_FEATURES;
2808
2809         netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
2810
2811         netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2812         netdev->mpls_features |= NETIF_F_HW_CSUM;
2813         netdev->hw_enc_features |= netdev->vlan_features;
2814
2815         /* set this bit last since it cannot be part of vlan_features */
2816         netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2817                             NETIF_F_HW_VLAN_CTAG_RX |
2818                             NETIF_F_HW_VLAN_CTAG_TX;
2819
2820         /* MTU range: 68 - 9216 */
2821         netdev->min_mtu = ETH_MIN_MTU;
2822         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2823
2824         spin_lock_bh(&hw->mbx_lock);
2825
2826         /*reset the controller to put the device in a known good state */
2827         err = hw->mac.ops.reset_hw(hw);
2828         if (err) {
2829                 dev_info(&pdev->dev,
2830                          "PF still in reset state. Is the PF interface up?\n");
2831         } else {
2832                 err = hw->mac.ops.read_mac_addr(hw);
2833                 if (err)
2834                         dev_info(&pdev->dev, "Error reading MAC address.\n");
2835                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2836                         dev_info(&pdev->dev,
2837                                  "MAC address not assigned by administrator.\n");
2838                 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2839         }
2840
2841         spin_unlock_bh(&hw->mbx_lock);
2842
2843         if (!is_valid_ether_addr(netdev->dev_addr)) {
2844                 dev_info(&pdev->dev, "Assigning random MAC address.\n");
2845                 eth_hw_addr_random(netdev);
2846                 memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2847                        netdev->addr_len);
2848         }
2849
2850         timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2851
2852         INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2853         INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2854
2855         /* ring size defaults */
2856         adapter->rx_ring->count = 1024;
2857         adapter->tx_ring->count = 1024;
2858
2859         /* reset the hardware with the new settings */
2860         igbvf_reset(adapter);
2861
2862         /* set hardware-specific flags */
2863         if (adapter->hw.mac.type == e1000_vfadapt_i350)
2864                 adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2865
2866         strcpy(netdev->name, "eth%d");
2867         err = register_netdev(netdev);
2868         if (err)
2869                 goto err_hw_init;
2870
2871         /* tell the stack to leave us alone until igbvf_open() is called */
2872         netif_carrier_off(netdev);
2873         netif_stop_queue(netdev);
2874
2875         igbvf_print_device_info(adapter);
2876
2877         igbvf_initialize_last_counter_stats(adapter);
2878
2879         return 0;
2880
2881 err_hw_init:
2882         netif_napi_del(&adapter->rx_ring->napi);
2883         kfree(adapter->tx_ring);
2884         kfree(adapter->rx_ring);
2885 err_sw_init:
2886         igbvf_reset_interrupt_capability(adapter);
2887 err_get_variants:
2888         iounmap(adapter->hw.hw_addr);
2889 err_ioremap:
2890         free_netdev(netdev);
2891 err_alloc_etherdev:
2892         pci_release_regions(pdev);
2893 err_pci_reg:
2894 err_dma:
2895         pci_disable_device(pdev);
2896         return err;
2897 }
2898
2899 /**
2900  * igbvf_remove - Device Removal Routine
2901  * @pdev: PCI device information struct
2902  *
2903  * igbvf_remove is called by the PCI subsystem to alert the driver
2904  * that it should release a PCI device.  The could be caused by a
2905  * Hot-Plug event, or because the driver is going to be removed from
2906  * memory.
2907  **/
2908 static void igbvf_remove(struct pci_dev *pdev)
2909 {
2910         struct net_device *netdev = pci_get_drvdata(pdev);
2911         struct igbvf_adapter *adapter = netdev_priv(netdev);
2912         struct e1000_hw *hw = &adapter->hw;
2913
2914         /* The watchdog timer may be rescheduled, so explicitly
2915          * disable it from being rescheduled.
2916          */
2917         set_bit(__IGBVF_DOWN, &adapter->state);
2918         del_timer_sync(&adapter->watchdog_timer);
2919
2920         cancel_work_sync(&adapter->reset_task);
2921         cancel_work_sync(&adapter->watchdog_task);
2922
2923         unregister_netdev(netdev);
2924
2925         igbvf_reset_interrupt_capability(adapter);
2926
2927         /* it is important to delete the NAPI struct prior to freeing the
2928          * Rx ring so that you do not end up with null pointer refs
2929          */
2930         netif_napi_del(&adapter->rx_ring->napi);
2931         kfree(adapter->tx_ring);
2932         kfree(adapter->rx_ring);
2933
2934         iounmap(hw->hw_addr);
2935         if (hw->flash_address)
2936                 iounmap(hw->flash_address);
2937         pci_release_regions(pdev);
2938
2939         free_netdev(netdev);
2940
2941         pci_disable_device(pdev);
2942 }
2943
2944 /* PCI Error Recovery (ERS) */
2945 static const struct pci_error_handlers igbvf_err_handler = {
2946         .error_detected = igbvf_io_error_detected,
2947         .slot_reset = igbvf_io_slot_reset,
2948         .resume = igbvf_io_resume,
2949         .reset_prepare = igbvf_io_prepare,
2950         .reset_done = igbvf_io_reset_done,
2951 };
2952
2953 static const struct pci_device_id igbvf_pci_tbl[] = {
2954         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2955         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2956         { } /* terminate list */
2957 };
2958 MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2959
2960 static SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2961
2962 /* PCI Device API Driver */
2963 static struct pci_driver igbvf_driver = {
2964         .name           = igbvf_driver_name,
2965         .id_table       = igbvf_pci_tbl,
2966         .probe          = igbvf_probe,
2967         .remove         = igbvf_remove,
2968         .driver.pm      = &igbvf_pm_ops,
2969         .shutdown       = igbvf_shutdown,
2970         .err_handler    = &igbvf_err_handler
2971 };
2972
2973 /**
2974  * igbvf_init_module - Driver Registration Routine
2975  *
2976  * igbvf_init_module is the first routine called when the driver is
2977  * loaded. All it does is register with the PCI subsystem.
2978  **/
2979 static int __init igbvf_init_module(void)
2980 {
2981         int ret;
2982
2983         pr_info("%s\n", igbvf_driver_string);
2984         pr_info("%s\n", igbvf_copyright);
2985
2986         ret = pci_register_driver(&igbvf_driver);
2987
2988         return ret;
2989 }
2990 module_init(igbvf_init_module);
2991
2992 /**
2993  * igbvf_exit_module - Driver Exit Cleanup Routine
2994  *
2995  * igbvf_exit_module is called just before the driver is removed
2996  * from memory.
2997  **/
2998 static void __exit igbvf_exit_module(void)
2999 {
3000         pci_unregister_driver(&igbvf_driver);
3001 }
3002 module_exit(igbvf_exit_module);
3003
3004 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
3005 MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
3006 MODULE_LICENSE("GPL v2");
3007
3008 /* netdev.c */