drivers/net/ethernet/synopsys/dwc-xlgmac-net.c

   1 /* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver
   2  *
   3  * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)
   4  *
   5  * This program is dual-licensed; you may select either version 2 of
   6  * the GNU General Public License ("GPL") or BSD license ("BSD").
   7  *
   8  * This Synopsys DWC XLGMAC software driver and associated documentation
   9  * (hereinafter the "Software") is an unsupported proprietary work of
  10  * Synopsys, Inc. unless otherwise expressly agreed to in writing between
  11  * Synopsys and you. The Software IS NOT an item of Licensed Software or a
  12  * Licensed Product under any End User Software License Agreement or
  13  * Agreement for Licensed Products with Synopsys or any supplement thereto.
  14  * Synopsys is a registered trademark of Synopsys, Inc. Other names included
  15  * in the SOFTWARE may be the trademarks of their respective owners.
  16  */
  17
  18 #include <linux/netdevice.h>
  19 #include <linux/tcp.h>
  20 #include <linux/interrupt.h>
  21
  22 #include "dwc-xlgmac.h"
  23 #include "dwc-xlgmac-reg.h"
  24
  25 static int xlgmac_one_poll(struct napi_struct *, int);
  26 static int xlgmac_all_poll(struct napi_struct *, int);
  27
  28 static inline unsigned int xlgmac_tx_avail_desc(struct xlgmac_ring *ring)
  29 {
  30         return (ring->dma_desc_count - (ring->cur - ring->dirty));
  31 }
  32
  33 static inline unsigned int xlgmac_rx_dirty_desc(struct xlgmac_ring *ring)
  34 {
  35         return (ring->cur - ring->dirty);
  36 }
  37
  38 static int xlgmac_maybe_stop_tx_queue(
  39                         struct xlgmac_channel *channel,
  40                         struct xlgmac_ring *ring,
  41                         unsigned int count)
  42 {
  43         struct xlgmac_pdata *pdata = channel->pdata;
  44
  45         if (count > xlgmac_tx_avail_desc(ring)) {
  46                 netif_info(pdata, drv, pdata->netdev,
  47                            "Tx queue stopped, not enough descriptors available\n");
  48                 netif_stop_subqueue(pdata->netdev, channel->queue_index);
  49                 ring->tx.queue_stopped = 1;
  50
  51                 /* If we haven't notified the hardware because of xmit_more
  52                  * support, tell it now
  53                  */
  54                 if (ring->tx.xmit_more)
  55                         pdata->hw_ops.tx_start_xmit(channel, ring);
  56
  57                 return NETDEV_TX_BUSY;
  58         }
  59
  60         return 0;
  61 }
  62
  63 static void xlgmac_prep_vlan(struct sk_buff *skb,
  64                              struct xlgmac_pkt_info *pkt_info)
  65 {
  66         if (skb_vlan_tag_present(skb))
  67                 pkt_info->vlan_ctag = skb_vlan_tag_get(skb);
  68 }
  69
  70 static int xlgmac_prep_tso(struct sk_buff *skb,
  71                            struct xlgmac_pkt_info *pkt_info)
  72 {
  73         int ret;
  74
  75         if (!XLGMAC_GET_REG_BITS(pkt_info->attributes,
  76                                  TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
  77                                  TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN))
  78                 return 0;
  79
  80         ret = skb_cow_head(skb, 0);
  81         if (ret)
  82                 return ret;
  83
  84         pkt_info->header_len = skb_tcp_all_headers(skb);
  85         pkt_info->tcp_header_len = tcp_hdrlen(skb);
  86         pkt_info->tcp_payload_len = skb->len - pkt_info->header_len;
  87         pkt_info->mss = skb_shinfo(skb)->gso_size;
  88
  89         XLGMAC_PR("header_len=%u\n", pkt_info->header_len);
  90         XLGMAC_PR("tcp_header_len=%u, tcp_payload_len=%u\n",
  91                   pkt_info->tcp_header_len, pkt_info->tcp_payload_len);
  92         XLGMAC_PR("mss=%u\n", pkt_info->mss);
  93
  94         /* Update the number of packets that will ultimately be transmitted
  95          * along with the extra bytes for each extra packet
  96          */
  97         pkt_info->tx_packets = skb_shinfo(skb)->gso_segs;
  98         pkt_info->tx_bytes += (pkt_info->tx_packets - 1) * pkt_info->header_len;
  99
 100         return 0;
 101 }
 102
 103 static int xlgmac_is_tso(struct sk_buff *skb)
 104 {
 105         if (skb->ip_summed != CHECKSUM_PARTIAL)
 106                 return 0;
 107
 108         if (!skb_is_gso(skb))
 109                 return 0;
 110
 111         return 1;
 112 }
 113
 114 static void xlgmac_prep_tx_pkt(struct xlgmac_pdata *pdata,
 115                                struct xlgmac_ring *ring,
 116                                struct sk_buff *skb,
 117                                struct xlgmac_pkt_info *pkt_info)
 118 {
 119         skb_frag_t *frag;
 120         unsigned int context_desc;
 121         unsigned int len;
 122         unsigned int i;
 123
 124         pkt_info->skb = skb;
 125
 126         context_desc = 0;
 127         pkt_info->desc_count = 0;
 128
 129         pkt_info->tx_packets = 1;
 130         pkt_info->tx_bytes = skb->len;
 131
 132         if (xlgmac_is_tso(skb)) {
 133                 /* TSO requires an extra descriptor if mss is different */
 134                 if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) {
 135                         context_desc = 1;
 136                         pkt_info->desc_count++;
 137                 }
 138
 139                 /* TSO requires an extra descriptor for TSO header */
 140                 pkt_info->desc_count++;
 141
 142                 pkt_info->attributes = XLGMAC_SET_REG_BITS(
 143                                         pkt_info->attributes,
 144                                         TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,
 145                                         TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN,
 146                                         1);
 147                 pkt_info->attributes = XLGMAC_SET_REG_BITS(
 148                                         pkt_info->attributes,
 149                                         TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,
 150                                         TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,
 151                                         1);
 152         } else if (skb->ip_summed == CHECKSUM_PARTIAL)
 153                 pkt_info->attributes = XLGMAC_SET_REG_BITS(
 154                                         pkt_info->attributes,
 155                                         TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,
 156                                         TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,
 157                                         1);
 158
 159         if (skb_vlan_tag_present(skb)) {
 160                 /* VLAN requires an extra descriptor if tag is different */
 161                 if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)
 162                         /* We can share with the TSO context descriptor */
 163                         if (!context_desc) {
 164                                 context_desc = 1;
 165                                 pkt_info->desc_count++;
 166                         }
 167
 168                 pkt_info->attributes = XLGMAC_SET_REG_BITS(
 169                                         pkt_info->attributes,
 170                                         TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
 171                                         TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN,
 172                                         1);
 173         }
 174
 175         for (len = skb_headlen(skb); len;) {
 176                 pkt_info->desc_count++;
 177                 len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);
 178         }
 179
 180         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 181                 frag = &skb_shinfo(skb)->frags[i];
 182                 for (len = skb_frag_size(frag); len; ) {
 183                         pkt_info->desc_count++;
 184                         len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);
 185                 }
 186         }
 187 }
 188
 189 static int xlgmac_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu)
 190 {
 191         unsigned int rx_buf_size;
 192
 193         if (mtu > XLGMAC_JUMBO_PACKET_MTU) {
 194                 netdev_alert(netdev, "MTU exceeds maximum supported value\n");
 195                 return -EINVAL;
 196         }
 197
 198         rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 199         rx_buf_size = clamp_val(rx_buf_size, XLGMAC_RX_MIN_BUF_SIZE, PAGE_SIZE);
 200
 201         rx_buf_size = (rx_buf_size + XLGMAC_RX_BUF_ALIGN - 1) &
 202                       ~(XLGMAC_RX_BUF_ALIGN - 1);
 203
 204         return rx_buf_size;
 205 }
 206
 207 static void xlgmac_enable_rx_tx_ints(struct xlgmac_pdata *pdata)
 208 {
 209         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 210         struct xlgmac_channel *channel;
 211         enum xlgmac_int int_id;
 212         unsigned int i;
 213
 214         channel = pdata->channel_head;
 215         for (i = 0; i < pdata->channel_count; i++, channel++) {
 216                 if (channel->tx_ring && channel->rx_ring)
 217                         int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;
 218                 else if (channel->tx_ring)
 219                         int_id = XLGMAC_INT_DMA_CH_SR_TI;
 220                 else if (channel->rx_ring)
 221                         int_id = XLGMAC_INT_DMA_CH_SR_RI;
 222                 else
 223                         continue;
 224
 225                 hw_ops->enable_int(channel, int_id);
 226         }
 227 }
 228
 229 static void xlgmac_disable_rx_tx_ints(struct xlgmac_pdata *pdata)
 230 {
 231         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 232         struct xlgmac_channel *channel;
 233         enum xlgmac_int int_id;
 234         unsigned int i;
 235
 236         channel = pdata->channel_head;
 237         for (i = 0; i < pdata->channel_count; i++, channel++) {
 238                 if (channel->tx_ring && channel->rx_ring)
 239                         int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;
 240                 else if (channel->tx_ring)
 241                         int_id = XLGMAC_INT_DMA_CH_SR_TI;
 242                 else if (channel->rx_ring)
 243                         int_id = XLGMAC_INT_DMA_CH_SR_RI;
 244                 else
 245                         continue;
 246
 247                 hw_ops->disable_int(channel, int_id);
 248         }
 249 }
 250
 251 static irqreturn_t xlgmac_isr(int irq, void *data)
 252 {
 253         unsigned int dma_isr, dma_ch_isr, mac_isr;
 254         struct xlgmac_pdata *pdata = data;
 255         struct xlgmac_channel *channel;
 256         struct xlgmac_hw_ops *hw_ops;
 257         unsigned int i, ti, ri;
 258
 259         hw_ops = &pdata->hw_ops;
 260
 261         /* The DMA interrupt status register also reports MAC and MTL
 262          * interrupts. So for polling mode, we just need to check for
 263          * this register to be non-zero
 264          */
 265         dma_isr = readl(pdata->mac_regs + DMA_ISR);
 266         if (!dma_isr)
 267                 return IRQ_HANDLED;
 268
 269         netif_dbg(pdata, intr, pdata->netdev, "DMA_ISR=%#010x\n", dma_isr);
 270
 271         for (i = 0; i < pdata->channel_count; i++) {
 272                 if (!(dma_isr & (1 << i)))
 273                         continue;
 274
 275                 channel = pdata->channel_head + i;
 276
 277                 dma_ch_isr = readl(XLGMAC_DMA_REG(channel, DMA_CH_SR));
 278                 netif_dbg(pdata, intr, pdata->netdev, "DMA_CH%u_ISR=%#010x\n",
 279                           i, dma_ch_isr);
 280
 281                 /* The TI or RI interrupt bits may still be set even if using
 282                  * per channel DMA interrupts. Check to be sure those are not
 283                  * enabled before using the private data napi structure.
 284                  */
 285                 ti = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TI_POS,
 286                                          DMA_CH_SR_TI_LEN);
 287                 ri = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RI_POS,
 288                                          DMA_CH_SR_RI_LEN);
 289                 if (!pdata->per_channel_irq && (ti || ri)) {
 290                         if (napi_schedule_prep(&pdata->napi)) {
 291                                 /* Disable Tx and Rx interrupts */
 292                                 xlgmac_disable_rx_tx_ints(pdata);
 293
 294                                 pdata->stats.napi_poll_isr++;
 295                                 /* Turn on polling */
 296                                 __napi_schedule_irqoff(&pdata->napi);
 297                         }
 298                 }
 299
 300                 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TPS_POS,
 301                                         DMA_CH_SR_TPS_LEN))
 302                         pdata->stats.tx_process_stopped++;
 303
 304                 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RPS_POS,
 305                                         DMA_CH_SR_RPS_LEN))
 306                         pdata->stats.rx_process_stopped++;
 307
 308                 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TBU_POS,
 309                                         DMA_CH_SR_TBU_LEN))
 310                         pdata->stats.tx_buffer_unavailable++;
 311
 312                 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RBU_POS,
 313                                         DMA_CH_SR_RBU_LEN))
 314                         pdata->stats.rx_buffer_unavailable++;
 315
 316                 /* Restart the device on a Fatal Bus Error */
 317                 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_FBE_POS,
 318                                         DMA_CH_SR_FBE_LEN)) {
 319                         pdata->stats.fatal_bus_error++;
 320                         schedule_work(&pdata->restart_work);
 321                 }
 322
 323                 /* Clear all interrupt signals */
 324                 writel(dma_ch_isr, XLGMAC_DMA_REG(channel, DMA_CH_SR));
 325         }
 326
 327         if (XLGMAC_GET_REG_BITS(dma_isr, DMA_ISR_MACIS_POS,
 328                                 DMA_ISR_MACIS_LEN)) {
 329                 mac_isr = readl(pdata->mac_regs + MAC_ISR);
 330
 331                 if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCTXIS_POS,
 332                                         MAC_ISR_MMCTXIS_LEN))
 333                         hw_ops->tx_mmc_int(pdata);
 334
 335                 if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCRXIS_POS,
 336                                         MAC_ISR_MMCRXIS_LEN))
 337                         hw_ops->rx_mmc_int(pdata);
 338         }
 339
 340         return IRQ_HANDLED;
 341 }
 342
 343 static irqreturn_t xlgmac_dma_isr(int irq, void *data)
 344 {
 345         struct xlgmac_channel *channel = data;
 346
 347         /* Per channel DMA interrupts are enabled, so we use the per
 348          * channel napi structure and not the private data napi structure
 349          */
 350         if (napi_schedule_prep(&channel->napi)) {
 351                 /* Disable Tx and Rx interrupts */
 352                 disable_irq_nosync(channel->dma_irq);
 353
 354                 /* Turn on polling */
 355                 __napi_schedule_irqoff(&channel->napi);
 356         }
 357
 358         return IRQ_HANDLED;
 359 }
 360
 361 static void xlgmac_tx_timer(struct timer_list *t)
 362 {
 363         struct xlgmac_channel *channel = from_timer(channel, t, tx_timer);
 364         struct xlgmac_pdata *pdata = channel->pdata;
 365         struct napi_struct *napi;
 366
 367         napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
 368
 369         if (napi_schedule_prep(napi)) {
 370                 /* Disable Tx and Rx interrupts */
 371                 if (pdata->per_channel_irq)
 372                         disable_irq_nosync(channel->dma_irq);
 373                 else
 374                         xlgmac_disable_rx_tx_ints(pdata);
 375
 376                 pdata->stats.napi_poll_txtimer++;
 377                 /* Turn on polling */
 378                 __napi_schedule(napi);
 379         }
 380
 381         channel->tx_timer_active = 0;
 382 }
 383
 384 static void xlgmac_init_timers(struct xlgmac_pdata *pdata)
 385 {
 386         struct xlgmac_channel *channel;
 387         unsigned int i;
 388
 389         channel = pdata->channel_head;
 390         for (i = 0; i < pdata->channel_count; i++, channel++) {
 391                 if (!channel->tx_ring)
 392                         break;
 393
 394                 timer_setup(&channel->tx_timer, xlgmac_tx_timer, 0);
 395         }
 396 }
 397
 398 static void xlgmac_stop_timers(struct xlgmac_pdata *pdata)
 399 {
 400         struct xlgmac_channel *channel;
 401         unsigned int i;
 402
 403         channel = pdata->channel_head;
 404         for (i = 0; i < pdata->channel_count; i++, channel++) {
 405                 if (!channel->tx_ring)
 406                         break;
 407
 408                 del_timer_sync(&channel->tx_timer);
 409         }
 410 }
 411
 412 static void xlgmac_napi_enable(struct xlgmac_pdata *pdata, unsigned int add)
 413 {
 414         struct xlgmac_channel *channel;
 415         unsigned int i;
 416
 417         if (pdata->per_channel_irq) {
 418                 channel = pdata->channel_head;
 419                 for (i = 0; i < pdata->channel_count; i++, channel++) {
 420                         if (add)
 421                                 netif_napi_add(pdata->netdev, &channel->napi,
 422                                                xlgmac_one_poll);
 423
 424                         napi_enable(&channel->napi);
 425                 }
 426         } else {
 427                 if (add)
 428                         netif_napi_add(pdata->netdev, &pdata->napi,
 429                                        xlgmac_all_poll);
 430
 431                 napi_enable(&pdata->napi);
 432         }
 433 }
 434
 435 static void xlgmac_napi_disable(struct xlgmac_pdata *pdata, unsigned int del)
 436 {
 437         struct xlgmac_channel *channel;
 438         unsigned int i;
 439
 440         if (pdata->per_channel_irq) {
 441                 channel = pdata->channel_head;
 442                 for (i = 0; i < pdata->channel_count; i++, channel++) {
 443                         napi_disable(&channel->napi);
 444
 445                         if (del)
 446                                 netif_napi_del(&channel->napi);
 447                 }
 448         } else {
 449                 napi_disable(&pdata->napi);
 450
 451                 if (del)
 452                         netif_napi_del(&pdata->napi);
 453         }
 454 }
 455
 456 static int xlgmac_request_irqs(struct xlgmac_pdata *pdata)
 457 {
 458         struct net_device *netdev = pdata->netdev;
 459         struct xlgmac_channel *channel;
 460         unsigned int i;
 461         int ret;
 462
 463         ret = devm_request_irq(pdata->dev, pdata->dev_irq, xlgmac_isr,
 464                                IRQF_SHARED, netdev->name, pdata);
 465         if (ret) {
 466                 netdev_alert(netdev, "error requesting irq %d\n",
 467                              pdata->dev_irq);
 468                 return ret;
 469         }
 470
 471         if (!pdata->per_channel_irq)
 472                 return 0;
 473
 474         channel = pdata->channel_head;
 475         for (i = 0; i < pdata->channel_count; i++, channel++) {
 476                 snprintf(channel->dma_irq_name,
 477                          sizeof(channel->dma_irq_name) - 1,
 478                          "%s-TxRx-%u", netdev_name(netdev),
 479                          channel->queue_index);
 480
 481                 ret = devm_request_irq(pdata->dev, channel->dma_irq,
 482                                        xlgmac_dma_isr, 0,
 483                                        channel->dma_irq_name, channel);
 484                 if (ret) {
 485                         netdev_alert(netdev, "error requesting irq %d\n",
 486                                      channel->dma_irq);
 487                         goto err_irq;
 488                 }
 489         }
 490
 491         return 0;
 492
 493 err_irq:
 494         /* Using an unsigned int, 'i' will go to UINT_MAX and exit */
 495         for (i--, channel--; i < pdata->channel_count; i--, channel--)
 496                 devm_free_irq(pdata->dev, channel->dma_irq, channel);
 497
 498         devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
 499
 500         return ret;
 501 }
 502
 503 static void xlgmac_free_irqs(struct xlgmac_pdata *pdata)
 504 {
 505         struct xlgmac_channel *channel;
 506         unsigned int i;
 507
 508         devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
 509
 510         if (!pdata->per_channel_irq)
 511                 return;
 512
 513         channel = pdata->channel_head;
 514         for (i = 0; i < pdata->channel_count; i++, channel++)
 515                 devm_free_irq(pdata->dev, channel->dma_irq, channel);
 516 }
 517
 518 static void xlgmac_free_tx_data(struct xlgmac_pdata *pdata)
 519 {
 520         struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;
 521         struct xlgmac_desc_data *desc_data;
 522         struct xlgmac_channel *channel;
 523         struct xlgmac_ring *ring;
 524         unsigned int i, j;
 525
 526         channel = pdata->channel_head;
 527         for (i = 0; i < pdata->channel_count; i++, channel++) {
 528                 ring = channel->tx_ring;
 529                 if (!ring)
 530                         break;
 531
 532                 for (j = 0; j < ring->dma_desc_count; j++) {
 533                         desc_data = XLGMAC_GET_DESC_DATA(ring, j);
 534                         desc_ops->unmap_desc_data(pdata, desc_data);
 535                 }
 536         }
 537 }
 538
 539 static void xlgmac_free_rx_data(struct xlgmac_pdata *pdata)
 540 {
 541         struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;
 542         struct xlgmac_desc_data *desc_data;
 543         struct xlgmac_channel *channel;
 544         struct xlgmac_ring *ring;
 545         unsigned int i, j;
 546
 547         channel = pdata->channel_head;
 548         for (i = 0; i < pdata->channel_count; i++, channel++) {
 549                 ring = channel->rx_ring;
 550                 if (!ring)
 551                         break;
 552
 553                 for (j = 0; j < ring->dma_desc_count; j++) {
 554                         desc_data = XLGMAC_GET_DESC_DATA(ring, j);
 555                         desc_ops->unmap_desc_data(pdata, desc_data);
 556                 }
 557         }
 558 }
 559
 560 static int xlgmac_start(struct xlgmac_pdata *pdata)
 561 {
 562         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 563         struct net_device *netdev = pdata->netdev;
 564         int ret;
 565
 566         hw_ops->init(pdata);
 567         xlgmac_napi_enable(pdata, 1);
 568
 569         ret = xlgmac_request_irqs(pdata);
 570         if (ret)
 571                 goto err_napi;
 572
 573         hw_ops->enable_tx(pdata);
 574         hw_ops->enable_rx(pdata);
 575         netif_tx_start_all_queues(netdev);
 576
 577         return 0;
 578
 579 err_napi:
 580         xlgmac_napi_disable(pdata, 1);
 581         hw_ops->exit(pdata);
 582
 583         return ret;
 584 }
 585
 586 static void xlgmac_stop(struct xlgmac_pdata *pdata)
 587 {
 588         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 589         struct net_device *netdev = pdata->netdev;
 590         struct xlgmac_channel *channel;
 591         struct netdev_queue *txq;
 592         unsigned int i;
 593
 594         netif_tx_stop_all_queues(netdev);
 595         xlgmac_stop_timers(pdata);
 596         hw_ops->disable_tx(pdata);
 597         hw_ops->disable_rx(pdata);
 598         xlgmac_free_irqs(pdata);
 599         xlgmac_napi_disable(pdata, 1);
 600         hw_ops->exit(pdata);
 601
 602         channel = pdata->channel_head;
 603         for (i = 0; i < pdata->channel_count; i++, channel++) {
 604                 if (!channel->tx_ring)
 605                         continue;
 606
 607                 txq = netdev_get_tx_queue(netdev, channel->queue_index);
 608                 netdev_tx_reset_queue(txq);
 609         }
 610 }
 611
 612 static void xlgmac_restart_dev(struct xlgmac_pdata *pdata)
 613 {
 614         /* If not running, "restart" will happen on open */
 615         if (!netif_running(pdata->netdev))
 616                 return;
 617
 618         xlgmac_stop(pdata);
 619
 620         xlgmac_free_tx_data(pdata);
 621         xlgmac_free_rx_data(pdata);
 622
 623         xlgmac_start(pdata);
 624 }
 625
 626 static void xlgmac_restart(struct work_struct *work)
 627 {
 628         struct xlgmac_pdata *pdata = container_of(work,
 629                                                    struct xlgmac_pdata,
 630                                                    restart_work);
 631
 632         rtnl_lock();
 633
 634         xlgmac_restart_dev(pdata);
 635
 636         rtnl_unlock();
 637 }
 638
 639 static int xlgmac_open(struct net_device *netdev)
 640 {
 641         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 642         struct xlgmac_desc_ops *desc_ops;
 643         int ret;
 644
 645         desc_ops = &pdata->desc_ops;
 646
 647         /* TODO: Initialize the phy */
 648
 649         /* Calculate the Rx buffer size before allocating rings */
 650         ret = xlgmac_calc_rx_buf_size(netdev, netdev->mtu);
 651         if (ret < 0)
 652                 return ret;
 653         pdata->rx_buf_size = ret;
 654
 655         /* Allocate the channels and rings */
 656         ret = desc_ops->alloc_channels_and_rings(pdata);
 657         if (ret)
 658                 return ret;
 659
 660         INIT_WORK(&pdata->restart_work, xlgmac_restart);
 661         xlgmac_init_timers(pdata);
 662
 663         ret = xlgmac_start(pdata);
 664         if (ret)
 665                 goto err_channels_and_rings;
 666
 667         return 0;
 668
 669 err_channels_and_rings:
 670         desc_ops->free_channels_and_rings(pdata);
 671
 672         return ret;
 673 }
 674
 675 static int xlgmac_close(struct net_device *netdev)
 676 {
 677         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 678         struct xlgmac_desc_ops *desc_ops;
 679
 680         desc_ops = &pdata->desc_ops;
 681
 682         /* Stop the device */
 683         xlgmac_stop(pdata);
 684
 685         /* Free the channels and rings */
 686         desc_ops->free_channels_and_rings(pdata);
 687
 688         return 0;
 689 }
 690
 691 static void xlgmac_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 692 {
 693         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 694
 695         netdev_warn(netdev, "tx timeout, device restarting\n");
 696         schedule_work(&pdata->restart_work);
 697 }
 698
 699 static netdev_tx_t xlgmac_xmit(struct sk_buff *skb, struct net_device *netdev)
 700 {
 701         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 702         struct xlgmac_pkt_info *tx_pkt_info;
 703         struct xlgmac_desc_ops *desc_ops;
 704         struct xlgmac_channel *channel;
 705         struct xlgmac_hw_ops *hw_ops;
 706         struct netdev_queue *txq;
 707         struct xlgmac_ring *ring;
 708         int ret;
 709
 710         desc_ops = &pdata->desc_ops;
 711         hw_ops = &pdata->hw_ops;
 712
 713         XLGMAC_PR("skb->len = %d\n", skb->len);
 714
 715         channel = pdata->channel_head + skb->queue_mapping;
 716         txq = netdev_get_tx_queue(netdev, channel->queue_index);
 717         ring = channel->tx_ring;
 718         tx_pkt_info = &ring->pkt_info;
 719
 720         if (skb->len == 0) {
 721                 netif_err(pdata, tx_err, netdev,
 722                           "empty skb received from stack\n");
 723                 dev_kfree_skb_any(skb);
 724                 return NETDEV_TX_OK;
 725         }
 726
 727         /* Prepare preliminary packet info for TX */
 728         memset(tx_pkt_info, 0, sizeof(*tx_pkt_info));
 729         xlgmac_prep_tx_pkt(pdata, ring, skb, tx_pkt_info);
 730
 731         /* Check that there are enough descriptors available */
 732         ret = xlgmac_maybe_stop_tx_queue(channel, ring,
 733                                          tx_pkt_info->desc_count);
 734         if (ret)
 735                 return ret;
 736
 737         ret = xlgmac_prep_tso(skb, tx_pkt_info);
 738         if (ret) {
 739                 netif_err(pdata, tx_err, netdev,
 740                           "error processing TSO packet\n");
 741                 dev_kfree_skb_any(skb);
 742                 return ret;
 743         }
 744         xlgmac_prep_vlan(skb, tx_pkt_info);
 745
 746         if (!desc_ops->map_tx_skb(channel, skb)) {
 747                 dev_kfree_skb_any(skb);
 748                 return NETDEV_TX_OK;
 749         }
 750
 751         /* Report on the actual number of bytes (to be) sent */
 752         netdev_tx_sent_queue(txq, tx_pkt_info->tx_bytes);
 753
 754         /* Configure required descriptor fields for transmission */
 755         hw_ops->dev_xmit(channel);
 756
 757         if (netif_msg_pktdata(pdata))
 758                 xlgmac_print_pkt(netdev, skb, true);
 759
 760         /* Stop the queue in advance if there may not be enough descriptors */
 761         xlgmac_maybe_stop_tx_queue(channel, ring, XLGMAC_TX_MAX_DESC_NR);
 762
 763         return NETDEV_TX_OK;
 764 }
 765
 766 static void xlgmac_get_stats64(struct net_device *netdev,
 767                                struct rtnl_link_stats64 *s)
 768 {
 769         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 770         struct xlgmac_stats *pstats = &pdata->stats;
 771
 772         pdata->hw_ops.read_mmc_stats(pdata);
 773
 774         s->rx_packets = pstats->rxframecount_gb;
 775         s->rx_bytes = pstats->rxoctetcount_gb;
 776         s->rx_errors = pstats->rxframecount_gb -
 777                        pstats->rxbroadcastframes_g -
 778                        pstats->rxmulticastframes_g -
 779                        pstats->rxunicastframes_g;
 780         s->multicast = pstats->rxmulticastframes_g;
 781         s->rx_length_errors = pstats->rxlengtherror;
 782         s->rx_crc_errors = pstats->rxcrcerror;
 783         s->rx_fifo_errors = pstats->rxfifooverflow;
 784
 785         s->tx_packets = pstats->txframecount_gb;
 786         s->tx_bytes = pstats->txoctetcount_gb;
 787         s->tx_errors = pstats->txframecount_gb - pstats->txframecount_g;
 788         s->tx_dropped = netdev->stats.tx_dropped;
 789 }
 790
 791 static int xlgmac_set_mac_address(struct net_device *netdev, void *addr)
 792 {
 793         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 794         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 795         struct sockaddr *saddr = addr;
 796
 797         if (!is_valid_ether_addr(saddr->sa_data))
 798                 return -EADDRNOTAVAIL;
 799
 800         eth_hw_addr_set(netdev, saddr->sa_data);
 801
 802         hw_ops->set_mac_address(pdata, netdev->dev_addr);
 803
 804         return 0;
 805 }
 806
 807 static int xlgmac_ioctl(struct net_device *netdev,
 808                         struct ifreq *ifreq, int cmd)
 809 {
 810         if (!netif_running(netdev))
 811                 return -ENODEV;
 812
 813         return 0;
 814 }
 815
 816 static int xlgmac_change_mtu(struct net_device *netdev, int mtu)
 817 {
 818         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 819         int ret;
 820
 821         ret = xlgmac_calc_rx_buf_size(netdev, mtu);
 822         if (ret < 0)
 823                 return ret;
 824
 825         pdata->rx_buf_size = ret;
 826         netdev->mtu = mtu;
 827
 828         xlgmac_restart_dev(pdata);
 829
 830         return 0;
 831 }
 832
 833 static int xlgmac_vlan_rx_add_vid(struct net_device *netdev,
 834                                   __be16 proto,
 835                                   u16 vid)
 836 {
 837         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 838         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 839
 840         set_bit(vid, pdata->active_vlans);
 841         hw_ops->update_vlan_hash_table(pdata);
 842
 843         return 0;
 844 }
 845
 846 static int xlgmac_vlan_rx_kill_vid(struct net_device *netdev,
 847                                    __be16 proto,
 848                                    u16 vid)
 849 {
 850         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 851         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 852
 853         clear_bit(vid, pdata->active_vlans);
 854         hw_ops->update_vlan_hash_table(pdata);
 855
 856         return 0;
 857 }
 858
 859 #ifdef CONFIG_NET_POLL_CONTROLLER
 860 static void xlgmac_poll_controller(struct net_device *netdev)
 861 {
 862         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 863         struct xlgmac_channel *channel;
 864         unsigned int i;
 865
 866         if (pdata->per_channel_irq) {
 867                 channel = pdata->channel_head;
 868                 for (i = 0; i < pdata->channel_count; i++, channel++)
 869                         xlgmac_dma_isr(channel->dma_irq, channel);
 870         } else {
 871                 disable_irq(pdata->dev_irq);
 872                 xlgmac_isr(pdata->dev_irq, pdata);
 873                 enable_irq(pdata->dev_irq);
 874         }
 875 }
 876 #endif /* CONFIG_NET_POLL_CONTROLLER */
 877
 878 static int xlgmac_set_features(struct net_device *netdev,
 879                                netdev_features_t features)
 880 {
 881         netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
 882         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 883         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 884         int ret = 0;
 885
 886         rxhash = pdata->netdev_features & NETIF_F_RXHASH;
 887         rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
 888         rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
 889         rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;
 890
 891         if ((features & NETIF_F_RXHASH) && !rxhash)
 892                 ret = hw_ops->enable_rss(pdata);
 893         else if (!(features & NETIF_F_RXHASH) && rxhash)
 894                 ret = hw_ops->disable_rss(pdata);
 895         if (ret)
 896                 return ret;
 897
 898         if ((features & NETIF_F_RXCSUM) && !rxcsum)
 899                 hw_ops->enable_rx_csum(pdata);
 900         else if (!(features & NETIF_F_RXCSUM) && rxcsum)
 901                 hw_ops->disable_rx_csum(pdata);
 902
 903         if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan)
 904                 hw_ops->enable_rx_vlan_stripping(pdata);
 905         else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && rxvlan)
 906                 hw_ops->disable_rx_vlan_stripping(pdata);
 907
 908         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && !rxvlan_filter)
 909                 hw_ops->enable_rx_vlan_filtering(pdata);
 910         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)
 911                 hw_ops->disable_rx_vlan_filtering(pdata);
 912
 913         pdata->netdev_features = features;
 914
 915         return 0;
 916 }
 917
 918 static void xlgmac_set_rx_mode(struct net_device *netdev)
 919 {
 920         struct xlgmac_pdata *pdata = netdev_priv(netdev);
 921         struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;
 922
 923         hw_ops->config_rx_mode(pdata);
 924 }
 925
 926 static const struct net_device_ops xlgmac_netdev_ops = {
 927         .ndo_open               = xlgmac_open,
 928         .ndo_stop               = xlgmac_close,
 929         .ndo_start_xmit         = xlgmac_xmit,
 930         .ndo_tx_timeout         = xlgmac_tx_timeout,
 931         .ndo_get_stats64        = xlgmac_get_stats64,
 932         .ndo_change_mtu         = xlgmac_change_mtu,
 933         .ndo_set_mac_address    = xlgmac_set_mac_address,
 934         .ndo_validate_addr      = eth_validate_addr,
 935         .ndo_eth_ioctl          = xlgmac_ioctl,
 936         .ndo_vlan_rx_add_vid    = xlgmac_vlan_rx_add_vid,
 937         .ndo_vlan_rx_kill_vid   = xlgmac_vlan_rx_kill_vid,
 938 #ifdef CONFIG_NET_POLL_CONTROLLER
 939         .ndo_poll_controller    = xlgmac_poll_controller,
 940 #endif
 941         .ndo_set_features       = xlgmac_set_features,
 942         .ndo_set_rx_mode        = xlgmac_set_rx_mode,
 943 };
 944
 945 const struct net_device_ops *xlgmac_get_netdev_ops(void)
 946 {
 947         return &xlgmac_netdev_ops;
 948 }
 949
 950 static void xlgmac_rx_refresh(struct xlgmac_channel *channel)
 951 {
 952         struct xlgmac_pdata *pdata = channel->pdata;
 953         struct xlgmac_ring *ring = channel->rx_ring;
 954         struct xlgmac_desc_data *desc_data;
 955         struct xlgmac_desc_ops *desc_ops;
 956         struct xlgmac_hw_ops *hw_ops;
 957
 958         desc_ops = &pdata->desc_ops;
 959         hw_ops = &pdata->hw_ops;
 960
 961         while (ring->dirty != ring->cur) {
 962                 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);
 963
 964                 /* Reset desc_data values */
 965                 desc_ops->unmap_desc_data(pdata, desc_data);
 966
 967                 if (desc_ops->map_rx_buffer(pdata, ring, desc_data))
 968                         break;
 969
 970                 hw_ops->rx_desc_reset(pdata, desc_data, ring->dirty);
 971
 972                 ring->dirty++;
 973         }
 974
 975         /* Make sure everything is written before the register write */
 976         wmb();
 977
 978         /* Update the Rx Tail Pointer Register with address of
 979          * the last cleaned entry
 980          */
 981         desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty - 1);
 982         writel(lower_32_bits(desc_data->dma_desc_addr),
 983                XLGMAC_DMA_REG(channel, DMA_CH_RDTR_LO));
 984 }
 985
 986 static struct sk_buff *xlgmac_create_skb(struct xlgmac_pdata *pdata,
 987                                          struct napi_struct *napi,
 988                                          struct xlgmac_desc_data *desc_data,
 989                                          unsigned int len)
 990 {
 991         unsigned int copy_len;
 992         struct sk_buff *skb;
 993         u8 *packet;
 994
 995         skb = napi_alloc_skb(napi, desc_data->rx.hdr.dma_len);
 996         if (!skb)
 997                 return NULL;
 998
 999         /* Start with the header buffer which may contain just the header
1000          * or the header plus data
1001          */
1002         dma_sync_single_range_for_cpu(pdata->dev, desc_data->rx.hdr.dma_base,
1003                                       desc_data->rx.hdr.dma_off,
1004                                       desc_data->rx.hdr.dma_len,
1005                                       DMA_FROM_DEVICE);
1006
1007         packet = page_address(desc_data->rx.hdr.pa.pages) +
1008                  desc_data->rx.hdr.pa.pages_offset;
1009         copy_len = (desc_data->rx.hdr_len) ? desc_data->rx.hdr_len : len;
1010         copy_len = min(desc_data->rx.hdr.dma_len, copy_len);
1011         skb_copy_to_linear_data(skb, packet, copy_len);
1012         skb_put(skb, copy_len);
1013
1014         len -= copy_len;
1015         if (len) {
1016                 /* Add the remaining data as a frag */
1017                 dma_sync_single_range_for_cpu(pdata->dev,
1018                                               desc_data->rx.buf.dma_base,
1019                                               desc_data->rx.buf.dma_off,
1020                                               desc_data->rx.buf.dma_len,
1021                                               DMA_FROM_DEVICE);
1022
1023                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1024                                 desc_data->rx.buf.pa.pages,
1025                                 desc_data->rx.buf.pa.pages_offset,
1026                                 len, desc_data->rx.buf.dma_len);
1027                 desc_data->rx.buf.pa.pages = NULL;
1028         }
1029
1030         return skb;
1031 }
1032
1033 static int xlgmac_tx_poll(struct xlgmac_channel *channel)
1034 {
1035         struct xlgmac_pdata *pdata = channel->pdata;
1036         struct xlgmac_ring *ring = channel->tx_ring;
1037         struct net_device *netdev = pdata->netdev;
1038         unsigned int tx_packets = 0, tx_bytes = 0;
1039         struct xlgmac_desc_data *desc_data;
1040         struct xlgmac_dma_desc *dma_desc;
1041         struct xlgmac_desc_ops *desc_ops;
1042         struct xlgmac_hw_ops *hw_ops;
1043         struct netdev_queue *txq;
1044         int processed = 0;
1045         unsigned int cur;
1046
1047         desc_ops = &pdata->desc_ops;
1048         hw_ops = &pdata->hw_ops;
1049
1050         /* Nothing to do if there isn't a Tx ring for this channel */
1051         if (!ring)
1052                 return 0;
1053
1054         cur = ring->cur;
1055
1056         /* Be sure we get ring->cur before accessing descriptor data */
1057         smp_rmb();
1058
1059         txq = netdev_get_tx_queue(netdev, channel->queue_index);
1060
1061         while ((processed < XLGMAC_TX_DESC_MAX_PROC) &&
1062                (ring->dirty != cur)) {
1063                 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);
1064                 dma_desc = desc_data->dma_desc;
1065
1066                 if (!hw_ops->tx_complete(dma_desc))
1067                         break;
1068
1069                 /* Make sure descriptor fields are read after reading
1070                  * the OWN bit
1071                  */
1072                 dma_rmb();
1073
1074                 if (netif_msg_tx_done(pdata))
1075                         xlgmac_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);
1076
1077                 if (hw_ops->is_last_desc(dma_desc)) {
1078                         tx_packets += desc_data->tx.packets;
1079                         tx_bytes += desc_data->tx.bytes;
1080                 }
1081
1082                 /* Free the SKB and reset the descriptor for re-use */
1083                 desc_ops->unmap_desc_data(pdata, desc_data);
1084                 hw_ops->tx_desc_reset(desc_data);
1085
1086                 processed++;
1087                 ring->dirty++;
1088         }
1089
1090         if (!processed)
1091                 return 0;
1092
1093         netdev_tx_completed_queue(txq, tx_packets, tx_bytes);
1094
1095         if ((ring->tx.queue_stopped == 1) &&
1096             (xlgmac_tx_avail_desc(ring) > XLGMAC_TX_DESC_MIN_FREE)) {
1097                 ring->tx.queue_stopped = 0;
1098                 netif_tx_wake_queue(txq);
1099         }
1100
1101         XLGMAC_PR("processed=%d\n", processed);
1102
1103         return processed;
1104 }
1105
1106 static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget)
1107 {
1108         struct xlgmac_pdata *pdata = channel->pdata;
1109         struct xlgmac_ring *ring = channel->rx_ring;
1110         struct net_device *netdev = pdata->netdev;
1111         unsigned int len, dma_desc_len, max_len;
1112         unsigned int context_next, context;
1113         struct xlgmac_desc_data *desc_data;
1114         struct xlgmac_pkt_info *pkt_info;
1115         unsigned int incomplete, error;
1116         struct xlgmac_hw_ops *hw_ops;
1117         unsigned int received = 0;
1118         struct napi_struct *napi;
1119         struct sk_buff *skb;
1120         int packet_count = 0;
1121
1122         hw_ops = &pdata->hw_ops;
1123
1124         /* Nothing to do if there isn't a Rx ring for this channel */
1125         if (!ring)
1126                 return 0;
1127
1128         incomplete = 0;
1129         context_next = 0;
1130
1131         napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
1132
1133         desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
1134         pkt_info = &ring->pkt_info;
1135         while (packet_count < budget) {
1136                 /* First time in loop see if we need to restore state */
1137                 if (!received && desc_data->state_saved) {
1138                         skb = desc_data->state.skb;
1139                         error = desc_data->state.error;
1140                         len = desc_data->state.len;
1141                 } else {
1142                         memset(pkt_info, 0, sizeof(*pkt_info));
1143                         skb = NULL;
1144                         error = 0;
1145                         len = 0;
1146                 }
1147
1148 read_again:
1149                 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
1150
1151                 if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY)
1152                         xlgmac_rx_refresh(channel);
1153
1154                 if (hw_ops->dev_read(channel))
1155                         break;
1156
1157                 received++;
1158                 ring->cur++;
1159
1160                 incomplete = XLGMAC_GET_REG_BITS(
1161                                         pkt_info->attributes,
1162                                         RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,
1163                                         RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN);
1164                 context_next = XLGMAC_GET_REG_BITS(
1165                                         pkt_info->attributes,
1166                                         RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,
1167                                         RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN);
1168                 context = XLGMAC_GET_REG_BITS(
1169                                         pkt_info->attributes,
1170                                         RX_PACKET_ATTRIBUTES_CONTEXT_POS,
1171                                         RX_PACKET_ATTRIBUTES_CONTEXT_LEN);
1172
1173                 /* Earlier error, just drain the remaining data */
1174                 if ((incomplete || context_next) && error)
1175                         goto read_again;
1176
1177                 if (error || pkt_info->errors) {
1178                         if (pkt_info->errors)
1179                                 netif_err(pdata, rx_err, netdev,
1180                                           "error in received packet\n");
1181                         dev_kfree_skb(skb);
1182                         goto next_packet;
1183                 }
1184
1185                 if (!context) {
1186                         /* Length is cumulative, get this descriptor's length */
1187                         dma_desc_len = desc_data->rx.len - len;
1188                         len += dma_desc_len;
1189
1190                         if (dma_desc_len && !skb) {
1191                                 skb = xlgmac_create_skb(pdata, napi, desc_data,
1192                                                         dma_desc_len);
1193                                 if (!skb)
1194                                         error = 1;
1195                         } else if (dma_desc_len) {
1196                                 dma_sync_single_range_for_cpu(
1197                                                 pdata->dev,
1198                                                 desc_data->rx.buf.dma_base,
1199                                                 desc_data->rx.buf.dma_off,
1200                                                 desc_data->rx.buf.dma_len,
1201                                                 DMA_FROM_DEVICE);
1202
1203                                 skb_add_rx_frag(
1204                                         skb, skb_shinfo(skb)->nr_frags,
1205                                         desc_data->rx.buf.pa.pages,
1206                                         desc_data->rx.buf.pa.pages_offset,
1207                                         dma_desc_len,
1208                                         desc_data->rx.buf.dma_len);
1209                                 desc_data->rx.buf.pa.pages = NULL;
1210                         }
1211                 }
1212
1213                 if (incomplete || context_next)
1214                         goto read_again;
1215
1216                 if (!skb)
1217                         goto next_packet;
1218
1219                 /* Be sure we don't exceed the configured MTU */
1220                 max_len = netdev->mtu + ETH_HLEN;
1221                 if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1222                     (skb->protocol == htons(ETH_P_8021Q)))
1223                         max_len += VLAN_HLEN;
1224
1225                 if (skb->len > max_len) {
1226                         netif_err(pdata, rx_err, netdev,
1227                                   "packet length exceeds configured MTU\n");
1228                         dev_kfree_skb(skb);
1229                         goto next_packet;
1230                 }
1231
1232                 if (netif_msg_pktdata(pdata))
1233                         xlgmac_print_pkt(netdev, skb, false);
1234
1235                 skb_checksum_none_assert(skb);
1236                 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
1237                                         RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,
1238                                     RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN))
1239                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1240
1241                 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
1242                                         RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,
1243                                     RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN)) {
1244                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1245                                                pkt_info->vlan_ctag);
1246                         pdata->stats.rx_vlan_packets++;
1247                 }
1248
1249                 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,
1250                                         RX_PACKET_ATTRIBUTES_RSS_HASH_POS,
1251                                     RX_PACKET_ATTRIBUTES_RSS_HASH_LEN))
1252                         skb_set_hash(skb, pkt_info->rss_hash,
1253                                      pkt_info->rss_hash_type);
1254
1255                 skb->dev = netdev;
1256                 skb->protocol = eth_type_trans(skb, netdev);
1257                 skb_record_rx_queue(skb, channel->queue_index);
1258
1259                 napi_gro_receive(napi, skb);
1260
1261 next_packet:
1262                 packet_count++;
1263         }
1264
1265         /* Check if we need to save state before leaving */
1266         if (received && (incomplete || context_next)) {
1267                 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);
1268                 desc_data->state_saved = 1;
1269                 desc_data->state.skb = skb;
1270                 desc_data->state.len = len;
1271                 desc_data->state.error = error;
1272         }
1273
1274         XLGMAC_PR("packet_count = %d\n", packet_count);
1275
1276         return packet_count;
1277 }
1278
1279 static int xlgmac_one_poll(struct napi_struct *napi, int budget)
1280 {
1281         struct xlgmac_channel *channel = container_of(napi,
1282                                                 struct xlgmac_channel,
1283                                                 napi);
1284         int processed = 0;
1285
1286         XLGMAC_PR("budget=%d\n", budget);
1287
1288         /* Cleanup Tx ring first */
1289         xlgmac_tx_poll(channel);
1290
1291         /* Process Rx ring next */
1292         processed = xlgmac_rx_poll(channel, budget);
1293
1294         /* If we processed everything, we are done */
1295         if (processed < budget) {
1296                 /* Turn off polling */
1297                 napi_complete_done(napi, processed);
1298
1299                 /* Enable Tx and Rx interrupts */
1300                 enable_irq(channel->dma_irq);
1301         }
1302
1303         XLGMAC_PR("received = %d\n", processed);
1304
1305         return processed;
1306 }
1307
1308 static int xlgmac_all_poll(struct napi_struct *napi, int budget)
1309 {
1310         struct xlgmac_pdata *pdata = container_of(napi,
1311                                                    struct xlgmac_pdata,
1312                                                    napi);
1313         struct xlgmac_channel *channel;
1314         int processed, last_processed;
1315         int ring_budget;
1316         unsigned int i;
1317
1318         XLGMAC_PR("budget=%d\n", budget);
1319
1320         processed = 0;
1321         ring_budget = budget / pdata->rx_ring_count;
1322         do {
1323                 last_processed = processed;
1324
1325                 channel = pdata->channel_head;
1326                 for (i = 0; i < pdata->channel_count; i++, channel++) {
1327                         /* Cleanup Tx ring first */
1328                         xlgmac_tx_poll(channel);
1329
1330                         /* Process Rx ring next */
1331                         if (ring_budget > (budget - processed))
1332                                 ring_budget = budget - processed;
1333                         processed += xlgmac_rx_poll(channel, ring_budget);
1334                 }
1335         } while ((processed < budget) && (processed != last_processed));
1336
1337         /* If we processed everything, we are done */
1338         if (processed < budget) {
1339                 /* Turn off polling */
1340                 napi_complete_done(napi, processed);
1341
1342                 /* Enable Tx and Rx interrupts */
1343                 xlgmac_enable_rx_tx_ints(pdata);
1344         }
1345
1346         XLGMAC_PR("received = %d\n", processed);
1347
1348         return processed;
1349 }