GNU Linux-libre 4.19.207-gnu1
[releases.git] / drivers / net / ethernet / samsung / sxgbe / sxgbe_main.c
1 /* 10G controller driver for Samsung SoCs
2  *
3  * Copyright (C) 2013 Samsung Electronics Co., Ltd.
4  *              http://www.samsung.com
5  *
6  * Author: Siva Reddy Kallam <siva.kallam@samsung.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 as
10  * published by the Free Software Foundation.
11  */
12
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14
15 #include <linux/clk.h>
16 #include <linux/crc32.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/etherdevice.h>
19 #include <linux/ethtool.h>
20 #include <linux/if.h>
21 #include <linux/if_ether.h>
22 #include <linux/if_vlan.h>
23 #include <linux/init.h>
24 #include <linux/interrupt.h>
25 #include <linux/ip.h>
26 #include <linux/kernel.h>
27 #include <linux/mii.h>
28 #include <linux/module.h>
29 #include <linux/net_tstamp.h>
30 #include <linux/netdevice.h>
31 #include <linux/phy.h>
32 #include <linux/platform_device.h>
33 #include <linux/prefetch.h>
34 #include <linux/skbuff.h>
35 #include <linux/slab.h>
36 #include <linux/tcp.h>
37 #include <linux/sxgbe_platform.h>
38
39 #include "sxgbe_common.h"
40 #include "sxgbe_desc.h"
41 #include "sxgbe_dma.h"
42 #include "sxgbe_mtl.h"
43 #include "sxgbe_reg.h"
44
45 #define SXGBE_ALIGN(x)  L1_CACHE_ALIGN(x)
46 #define JUMBO_LEN       9000
47
48 /* Module parameters */
49 #define TX_TIMEO        5000
50 #define DMA_TX_SIZE     512
51 #define DMA_RX_SIZE     1024
52 #define TC_DEFAULT      64
53 #define DMA_BUFFER_SIZE BUF_SIZE_2KiB
54 /* The default timer value as per the sxgbe specification 1 sec(1000 ms) */
55 #define SXGBE_DEFAULT_LPI_TIMER 1000
56
57 static int debug = -1;
58 static int eee_timer = SXGBE_DEFAULT_LPI_TIMER;
59
60 module_param(eee_timer, int, 0644);
61
62 module_param(debug, int, 0644);
63 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
64                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
65                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
66
67 static irqreturn_t sxgbe_common_interrupt(int irq, void *dev_id);
68 static irqreturn_t sxgbe_tx_interrupt(int irq, void *dev_id);
69 static irqreturn_t sxgbe_rx_interrupt(int irq, void *dev_id);
70
71 #define SXGBE_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
72
73 #define SXGBE_LPI_TIMER(x) (jiffies + msecs_to_jiffies(x))
74
75 /**
76  * sxgbe_verify_args - verify the driver parameters.
77  * Description: it verifies if some wrong parameter is passed to the driver.
78  * Note that wrong parameters are replaced with the default values.
79  */
80 static void sxgbe_verify_args(void)
81 {
82         if (unlikely(eee_timer < 0))
83                 eee_timer = SXGBE_DEFAULT_LPI_TIMER;
84 }
85
86 static void sxgbe_enable_eee_mode(const struct sxgbe_priv_data *priv)
87 {
88         /* Check and enter in LPI mode */
89         if (!priv->tx_path_in_lpi_mode)
90                 priv->hw->mac->set_eee_mode(priv->ioaddr);
91 }
92
93 void sxgbe_disable_eee_mode(struct sxgbe_priv_data * const priv)
94 {
95         /* Exit and disable EEE in case of we are are in LPI state. */
96         priv->hw->mac->reset_eee_mode(priv->ioaddr);
97         del_timer_sync(&priv->eee_ctrl_timer);
98         priv->tx_path_in_lpi_mode = false;
99 }
100
101 /**
102  * sxgbe_eee_ctrl_timer
103  * @arg : data hook
104  * Description:
105  *  If there is no data transfer and if we are not in LPI state,
106  *  then MAC Transmitter can be moved to LPI state.
107  */
108 static void sxgbe_eee_ctrl_timer(struct timer_list *t)
109 {
110         struct sxgbe_priv_data *priv = from_timer(priv, t, eee_ctrl_timer);
111
112         sxgbe_enable_eee_mode(priv);
113         mod_timer(&priv->eee_ctrl_timer, SXGBE_LPI_TIMER(eee_timer));
114 }
115
116 /**
117  * sxgbe_eee_init
118  * @priv: private device pointer
119  * Description:
120  *  If the EEE support has been enabled while configuring the driver,
121  *  if the GMAC actually supports the EEE (from the HW cap reg) and the
122  *  phy can also manage EEE, so enable the LPI state and start the timer
123  *  to verify if the tx path can enter in LPI state.
124  */
125 bool sxgbe_eee_init(struct sxgbe_priv_data * const priv)
126 {
127         struct net_device *ndev = priv->dev;
128         bool ret = false;
129
130         /* MAC core supports the EEE feature. */
131         if (priv->hw_cap.eee) {
132                 /* Check if the PHY supports EEE */
133                 if (phy_init_eee(ndev->phydev, 1))
134                         return false;
135
136                 priv->eee_active = 1;
137                 timer_setup(&priv->eee_ctrl_timer, sxgbe_eee_ctrl_timer, 0);
138                 priv->eee_ctrl_timer.expires = SXGBE_LPI_TIMER(eee_timer);
139                 add_timer(&priv->eee_ctrl_timer);
140
141                 priv->hw->mac->set_eee_timer(priv->ioaddr,
142                                              SXGBE_DEFAULT_LPI_TIMER,
143                                              priv->tx_lpi_timer);
144
145                 pr_info("Energy-Efficient Ethernet initialized\n");
146
147                 ret = true;
148         }
149
150         return ret;
151 }
152
153 static void sxgbe_eee_adjust(const struct sxgbe_priv_data *priv)
154 {
155         struct net_device *ndev = priv->dev;
156
157         /* When the EEE has been already initialised we have to
158          * modify the PLS bit in the LPI ctrl & status reg according
159          * to the PHY link status. For this reason.
160          */
161         if (priv->eee_enabled)
162                 priv->hw->mac->set_eee_pls(priv->ioaddr, ndev->phydev->link);
163 }
164
165 /**
166  * sxgbe_clk_csr_set - dynamically set the MDC clock
167  * @priv: driver private structure
168  * Description: this is to dynamically set the MDC clock according to the csr
169  * clock input.
170  */
171 static void sxgbe_clk_csr_set(struct sxgbe_priv_data *priv)
172 {
173         u32 clk_rate = clk_get_rate(priv->sxgbe_clk);
174
175         /* assign the proper divider, this will be used during
176          * mdio communication
177          */
178         if (clk_rate < SXGBE_CSR_F_150M)
179                 priv->clk_csr = SXGBE_CSR_100_150M;
180         else if (clk_rate <= SXGBE_CSR_F_250M)
181                 priv->clk_csr = SXGBE_CSR_150_250M;
182         else if (clk_rate <= SXGBE_CSR_F_300M)
183                 priv->clk_csr = SXGBE_CSR_250_300M;
184         else if (clk_rate <= SXGBE_CSR_F_350M)
185                 priv->clk_csr = SXGBE_CSR_300_350M;
186         else if (clk_rate <= SXGBE_CSR_F_400M)
187                 priv->clk_csr = SXGBE_CSR_350_400M;
188         else if (clk_rate <= SXGBE_CSR_F_500M)
189                 priv->clk_csr = SXGBE_CSR_400_500M;
190 }
191
192 /* minimum number of free TX descriptors required to wake up TX process */
193 #define SXGBE_TX_THRESH(x)      (x->dma_tx_size/4)
194
195 static inline u32 sxgbe_tx_avail(struct sxgbe_tx_queue *queue, int tx_qsize)
196 {
197         return queue->dirty_tx + tx_qsize - queue->cur_tx - 1;
198 }
199
200 /**
201  * sxgbe_adjust_link
202  * @dev: net device structure
203  * Description: it adjusts the link parameters.
204  */
205 static void sxgbe_adjust_link(struct net_device *dev)
206 {
207         struct sxgbe_priv_data *priv = netdev_priv(dev);
208         struct phy_device *phydev = dev->phydev;
209         u8 new_state = 0;
210         u8 speed = 0xff;
211
212         if (!phydev)
213                 return;
214
215         /* SXGBE is not supporting auto-negotiation and
216          * half duplex mode. so, not handling duplex change
217          * in this function. only handling speed and link status
218          */
219         if (phydev->link) {
220                 if (phydev->speed != priv->speed) {
221                         new_state = 1;
222                         switch (phydev->speed) {
223                         case SPEED_10000:
224                                 speed = SXGBE_SPEED_10G;
225                                 break;
226                         case SPEED_2500:
227                                 speed = SXGBE_SPEED_2_5G;
228                                 break;
229                         case SPEED_1000:
230                                 speed = SXGBE_SPEED_1G;
231                                 break;
232                         default:
233                                 netif_err(priv, link, dev,
234                                           "Speed (%d) not supported\n",
235                                           phydev->speed);
236                         }
237
238                         priv->speed = phydev->speed;
239                         priv->hw->mac->set_speed(priv->ioaddr, speed);
240                 }
241
242                 if (!priv->oldlink) {
243                         new_state = 1;
244                         priv->oldlink = 1;
245                 }
246         } else if (priv->oldlink) {
247                 new_state = 1;
248                 priv->oldlink = 0;
249                 priv->speed = SPEED_UNKNOWN;
250         }
251
252         if (new_state & netif_msg_link(priv))
253                 phy_print_status(phydev);
254
255         /* Alter the MAC settings for EEE */
256         sxgbe_eee_adjust(priv);
257 }
258
259 /**
260  * sxgbe_init_phy - PHY initialization
261  * @dev: net device structure
262  * Description: it initializes the driver's PHY state, and attaches the PHY
263  * to the mac driver.
264  *  Return value:
265  *  0 on success
266  */
267 static int sxgbe_init_phy(struct net_device *ndev)
268 {
269         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
270         char bus_id[MII_BUS_ID_SIZE];
271         struct phy_device *phydev;
272         struct sxgbe_priv_data *priv = netdev_priv(ndev);
273         int phy_iface = priv->plat->interface;
274
275         /* assign default link status */
276         priv->oldlink = 0;
277         priv->speed = SPEED_UNKNOWN;
278         priv->oldduplex = DUPLEX_UNKNOWN;
279
280         if (priv->plat->phy_bus_name)
281                 snprintf(bus_id, MII_BUS_ID_SIZE, "%s-%x",
282                          priv->plat->phy_bus_name, priv->plat->bus_id);
283         else
284                 snprintf(bus_id, MII_BUS_ID_SIZE, "sxgbe-%x",
285                          priv->plat->bus_id);
286
287         snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
288                  priv->plat->phy_addr);
289         netdev_dbg(ndev, "%s: trying to attach to %s\n", __func__, phy_id_fmt);
290
291         phydev = phy_connect(ndev, phy_id_fmt, &sxgbe_adjust_link, phy_iface);
292
293         if (IS_ERR(phydev)) {
294                 netdev_err(ndev, "Could not attach to PHY\n");
295                 return PTR_ERR(phydev);
296         }
297
298         /* Stop Advertising 1000BASE Capability if interface is not GMII */
299         if ((phy_iface == PHY_INTERFACE_MODE_MII) ||
300             (phy_iface == PHY_INTERFACE_MODE_RMII))
301                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
302                                          SUPPORTED_1000baseT_Full);
303         if (phydev->phy_id == 0) {
304                 phy_disconnect(phydev);
305                 return -ENODEV;
306         }
307
308         netdev_dbg(ndev, "%s: attached to PHY (UID 0x%x) Link = %d\n",
309                    __func__, phydev->phy_id, phydev->link);
310
311         return 0;
312 }
313
314 /**
315  * sxgbe_clear_descriptors: clear descriptors
316  * @priv: driver private structure
317  * Description: this function is called to clear the tx and rx descriptors
318  * in case of both basic and extended descriptors are used.
319  */
320 static void sxgbe_clear_descriptors(struct sxgbe_priv_data *priv)
321 {
322         int i, j;
323         unsigned int txsize = priv->dma_tx_size;
324         unsigned int rxsize = priv->dma_rx_size;
325
326         /* Clear the Rx/Tx descriptors */
327         for (j = 0; j < SXGBE_RX_QUEUES; j++) {
328                 for (i = 0; i < rxsize; i++)
329                         priv->hw->desc->init_rx_desc(&priv->rxq[j]->dma_rx[i],
330                                                      priv->use_riwt, priv->mode,
331                                                      (i == rxsize - 1));
332         }
333
334         for (j = 0; j < SXGBE_TX_QUEUES; j++) {
335                 for (i = 0; i < txsize; i++)
336                         priv->hw->desc->init_tx_desc(&priv->txq[j]->dma_tx[i]);
337         }
338 }
339
340 static int sxgbe_init_rx_buffers(struct net_device *dev,
341                                  struct sxgbe_rx_norm_desc *p, int i,
342                                  unsigned int dma_buf_sz,
343                                  struct sxgbe_rx_queue *rx_ring)
344 {
345         struct sxgbe_priv_data *priv = netdev_priv(dev);
346         struct sk_buff *skb;
347
348         skb = __netdev_alloc_skb_ip_align(dev, dma_buf_sz, GFP_KERNEL);
349         if (!skb)
350                 return -ENOMEM;
351
352         rx_ring->rx_skbuff[i] = skb;
353         rx_ring->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
354                                                    dma_buf_sz, DMA_FROM_DEVICE);
355
356         if (dma_mapping_error(priv->device, rx_ring->rx_skbuff_dma[i])) {
357                 netdev_err(dev, "%s: DMA mapping error\n", __func__);
358                 dev_kfree_skb_any(skb);
359                 return -EINVAL;
360         }
361
362         p->rdes23.rx_rd_des23.buf2_addr = rx_ring->rx_skbuff_dma[i];
363
364         return 0;
365 }
366
367 /**
368  * sxgbe_free_rx_buffers - free what sxgbe_init_rx_buffers() allocated
369  * @dev: net device structure
370  * @rx_ring: ring to be freed
371  * @rx_rsize: ring size
372  * Description:  this function initializes the DMA RX descriptor
373  */
374 static void sxgbe_free_rx_buffers(struct net_device *dev,
375                                   struct sxgbe_rx_norm_desc *p, int i,
376                                   unsigned int dma_buf_sz,
377                                   struct sxgbe_rx_queue *rx_ring)
378 {
379         struct sxgbe_priv_data *priv = netdev_priv(dev);
380
381         kfree_skb(rx_ring->rx_skbuff[i]);
382         dma_unmap_single(priv->device, rx_ring->rx_skbuff_dma[i],
383                          dma_buf_sz, DMA_FROM_DEVICE);
384 }
385
386 /**
387  * init_tx_ring - init the TX descriptor ring
388  * @dev: net device structure
389  * @tx_ring: ring to be initialised
390  * @tx_rsize: ring size
391  * Description:  this function initializes the DMA TX descriptor
392  */
393 static int init_tx_ring(struct device *dev, u8 queue_no,
394                         struct sxgbe_tx_queue *tx_ring, int tx_rsize)
395 {
396         /* TX ring is not allcoated */
397         if (!tx_ring) {
398                 dev_err(dev, "No memory for TX queue of SXGBE\n");
399                 return -ENOMEM;
400         }
401
402         /* allocate memory for TX descriptors */
403         tx_ring->dma_tx = dma_zalloc_coherent(dev,
404                                               tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
405                                               &tx_ring->dma_tx_phy, GFP_KERNEL);
406         if (!tx_ring->dma_tx)
407                 return -ENOMEM;
408
409         /* allocate memory for TX skbuff array */
410         tx_ring->tx_skbuff_dma = devm_kcalloc(dev, tx_rsize,
411                                               sizeof(dma_addr_t), GFP_KERNEL);
412         if (!tx_ring->tx_skbuff_dma)
413                 goto dmamem_err;
414
415         tx_ring->tx_skbuff = devm_kcalloc(dev, tx_rsize,
416                                           sizeof(struct sk_buff *), GFP_KERNEL);
417
418         if (!tx_ring->tx_skbuff)
419                 goto dmamem_err;
420
421         /* assign queue number */
422         tx_ring->queue_no = queue_no;
423
424         /* initialise counters */
425         tx_ring->dirty_tx = 0;
426         tx_ring->cur_tx = 0;
427
428         return 0;
429
430 dmamem_err:
431         dma_free_coherent(dev, tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
432                           tx_ring->dma_tx, tx_ring->dma_tx_phy);
433         return -ENOMEM;
434 }
435
436 /**
437  * free_rx_ring - free the RX descriptor ring
438  * @dev: net device structure
439  * @rx_ring: ring to be initialised
440  * @rx_rsize: ring size
441  * Description:  this function initializes the DMA RX descriptor
442  */
443 static void free_rx_ring(struct device *dev, struct sxgbe_rx_queue *rx_ring,
444                          int rx_rsize)
445 {
446         dma_free_coherent(dev, rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
447                           rx_ring->dma_rx, rx_ring->dma_rx_phy);
448         kfree(rx_ring->rx_skbuff_dma);
449         kfree(rx_ring->rx_skbuff);
450 }
451
452 /**
453  * init_rx_ring - init the RX descriptor ring
454  * @dev: net device structure
455  * @rx_ring: ring to be initialised
456  * @rx_rsize: ring size
457  * Description:  this function initializes the DMA RX descriptor
458  */
459 static int init_rx_ring(struct net_device *dev, u8 queue_no,
460                         struct sxgbe_rx_queue *rx_ring, int rx_rsize)
461 {
462         struct sxgbe_priv_data *priv = netdev_priv(dev);
463         int desc_index;
464         unsigned int bfsize = 0;
465         unsigned int ret = 0;
466
467         /* Set the max buffer size according to the MTU. */
468         bfsize = ALIGN(dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN, 8);
469
470         netif_dbg(priv, probe, dev, "%s: bfsize %d\n", __func__, bfsize);
471
472         /* RX ring is not allcoated */
473         if (rx_ring == NULL) {
474                 netdev_err(dev, "No memory for RX queue\n");
475                 return -ENOMEM;
476         }
477
478         /* assign queue number */
479         rx_ring->queue_no = queue_no;
480
481         /* allocate memory for RX descriptors */
482         rx_ring->dma_rx = dma_zalloc_coherent(priv->device,
483                                               rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
484                                               &rx_ring->dma_rx_phy, GFP_KERNEL);
485
486         if (rx_ring->dma_rx == NULL)
487                 return -ENOMEM;
488
489         /* allocate memory for RX skbuff array */
490         rx_ring->rx_skbuff_dma = kmalloc_array(rx_rsize,
491                                                sizeof(dma_addr_t), GFP_KERNEL);
492         if (!rx_ring->rx_skbuff_dma) {
493                 ret = -ENOMEM;
494                 goto err_free_dma_rx;
495         }
496
497         rx_ring->rx_skbuff = kmalloc_array(rx_rsize,
498                                            sizeof(struct sk_buff *), GFP_KERNEL);
499         if (!rx_ring->rx_skbuff) {
500                 ret = -ENOMEM;
501                 goto err_free_skbuff_dma;
502         }
503
504         /* initialise the buffers */
505         for (desc_index = 0; desc_index < rx_rsize; desc_index++) {
506                 struct sxgbe_rx_norm_desc *p;
507                 p = rx_ring->dma_rx + desc_index;
508                 ret = sxgbe_init_rx_buffers(dev, p, desc_index,
509                                             bfsize, rx_ring);
510                 if (ret)
511                         goto err_free_rx_buffers;
512         }
513
514         /* initialise counters */
515         rx_ring->cur_rx = 0;
516         rx_ring->dirty_rx = (unsigned int)(desc_index - rx_rsize);
517         priv->dma_buf_sz = bfsize;
518
519         return 0;
520
521 err_free_rx_buffers:
522         while (--desc_index >= 0) {
523                 struct sxgbe_rx_norm_desc *p;
524
525                 p = rx_ring->dma_rx + desc_index;
526                 sxgbe_free_rx_buffers(dev, p, desc_index, bfsize, rx_ring);
527         }
528         kfree(rx_ring->rx_skbuff);
529 err_free_skbuff_dma:
530         kfree(rx_ring->rx_skbuff_dma);
531 err_free_dma_rx:
532         dma_free_coherent(priv->device,
533                           rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
534                           rx_ring->dma_rx, rx_ring->dma_rx_phy);
535
536         return ret;
537 }
538 /**
539  * free_tx_ring - free the TX descriptor ring
540  * @dev: net device structure
541  * @tx_ring: ring to be initialised
542  * @tx_rsize: ring size
543  * Description:  this function initializes the DMA TX descriptor
544  */
545 static void free_tx_ring(struct device *dev, struct sxgbe_tx_queue *tx_ring,
546                          int tx_rsize)
547 {
548         dma_free_coherent(dev, tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
549                           tx_ring->dma_tx, tx_ring->dma_tx_phy);
550 }
551
552 /**
553  * init_dma_desc_rings - init the RX/TX descriptor rings
554  * @dev: net device structure
555  * Description:  this function initializes the DMA RX/TX descriptors
556  * and allocates the socket buffers. It suppors the chained and ring
557  * modes.
558  */
559 static int init_dma_desc_rings(struct net_device *netd)
560 {
561         int queue_num, ret;
562         struct sxgbe_priv_data *priv = netdev_priv(netd);
563         int tx_rsize = priv->dma_tx_size;
564         int rx_rsize = priv->dma_rx_size;
565
566         /* Allocate memory for queue structures and TX descs */
567         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
568                 ret = init_tx_ring(priv->device, queue_num,
569                                    priv->txq[queue_num], tx_rsize);
570                 if (ret) {
571                         dev_err(&netd->dev, "TX DMA ring allocation failed!\n");
572                         goto txalloc_err;
573                 }
574
575                 /* save private pointer in each ring this
576                  * pointer is needed during cleaing TX queue
577                  */
578                 priv->txq[queue_num]->priv_ptr = priv;
579         }
580
581         /* Allocate memory for queue structures and RX descs */
582         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
583                 ret = init_rx_ring(netd, queue_num,
584                                    priv->rxq[queue_num], rx_rsize);
585                 if (ret) {
586                         netdev_err(netd, "RX DMA ring allocation failed!!\n");
587                         goto rxalloc_err;
588                 }
589
590                 /* save private pointer in each ring this
591                  * pointer is needed during cleaing TX queue
592                  */
593                 priv->rxq[queue_num]->priv_ptr = priv;
594         }
595
596         sxgbe_clear_descriptors(priv);
597
598         return 0;
599
600 txalloc_err:
601         while (queue_num--)
602                 free_tx_ring(priv->device, priv->txq[queue_num], tx_rsize);
603         return ret;
604
605 rxalloc_err:
606         while (queue_num--)
607                 free_rx_ring(priv->device, priv->rxq[queue_num], rx_rsize);
608         return ret;
609 }
610
611 static void tx_free_ring_skbufs(struct sxgbe_tx_queue *txqueue)
612 {
613         int dma_desc;
614         struct sxgbe_priv_data *priv = txqueue->priv_ptr;
615         int tx_rsize = priv->dma_tx_size;
616
617         for (dma_desc = 0; dma_desc < tx_rsize; dma_desc++) {
618                 struct sxgbe_tx_norm_desc *tdesc = txqueue->dma_tx + dma_desc;
619
620                 if (txqueue->tx_skbuff_dma[dma_desc])
621                         dma_unmap_single(priv->device,
622                                          txqueue->tx_skbuff_dma[dma_desc],
623                                          priv->hw->desc->get_tx_len(tdesc),
624                                          DMA_TO_DEVICE);
625
626                 dev_kfree_skb_any(txqueue->tx_skbuff[dma_desc]);
627                 txqueue->tx_skbuff[dma_desc] = NULL;
628                 txqueue->tx_skbuff_dma[dma_desc] = 0;
629         }
630 }
631
632
633 static void dma_free_tx_skbufs(struct sxgbe_priv_data *priv)
634 {
635         int queue_num;
636
637         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
638                 struct sxgbe_tx_queue *tqueue = priv->txq[queue_num];
639                 tx_free_ring_skbufs(tqueue);
640         }
641 }
642
643 static void free_dma_desc_resources(struct sxgbe_priv_data *priv)
644 {
645         int queue_num;
646         int tx_rsize = priv->dma_tx_size;
647         int rx_rsize = priv->dma_rx_size;
648
649         /* Release the DMA TX buffers */
650         dma_free_tx_skbufs(priv);
651
652         /* Release the TX ring memory also */
653         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
654                 free_tx_ring(priv->device, priv->txq[queue_num], tx_rsize);
655         }
656
657         /* Release the RX ring memory also */
658         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
659                 free_rx_ring(priv->device, priv->rxq[queue_num], rx_rsize);
660         }
661 }
662
663 static int txring_mem_alloc(struct sxgbe_priv_data *priv)
664 {
665         int queue_num;
666
667         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
668                 priv->txq[queue_num] = devm_kmalloc(priv->device,
669                                                     sizeof(struct sxgbe_tx_queue), GFP_KERNEL);
670                 if (!priv->txq[queue_num])
671                         return -ENOMEM;
672         }
673
674         return 0;
675 }
676
677 static int rxring_mem_alloc(struct sxgbe_priv_data *priv)
678 {
679         int queue_num;
680
681         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
682                 priv->rxq[queue_num] = devm_kmalloc(priv->device,
683                                                     sizeof(struct sxgbe_rx_queue), GFP_KERNEL);
684                 if (!priv->rxq[queue_num])
685                         return -ENOMEM;
686         }
687
688         return 0;
689 }
690
691 /**
692  *  sxgbe_mtl_operation_mode - HW MTL operation mode
693  *  @priv: driver private structure
694  *  Description: it sets the MTL operation mode: tx/rx MTL thresholds
695  *  or Store-And-Forward capability.
696  */
697 static void sxgbe_mtl_operation_mode(struct sxgbe_priv_data *priv)
698 {
699         int queue_num;
700
701         /* TX/RX threshold control */
702         if (likely(priv->plat->force_sf_dma_mode)) {
703                 /* set TC mode for TX QUEUES */
704                 SXGBE_FOR_EACH_QUEUE(priv->hw_cap.tx_mtl_queues, queue_num)
705                         priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr, queue_num,
706                                                        SXGBE_MTL_SFMODE);
707                 priv->tx_tc = SXGBE_MTL_SFMODE;
708
709                 /* set TC mode for RX QUEUES */
710                 SXGBE_FOR_EACH_QUEUE(priv->hw_cap.rx_mtl_queues, queue_num)
711                         priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr, queue_num,
712                                                        SXGBE_MTL_SFMODE);
713                 priv->rx_tc = SXGBE_MTL_SFMODE;
714         } else if (unlikely(priv->plat->force_thresh_dma_mode)) {
715                 /* set TC mode for TX QUEUES */
716                 SXGBE_FOR_EACH_QUEUE(priv->hw_cap.tx_mtl_queues, queue_num)
717                         priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr, queue_num,
718                                                        priv->tx_tc);
719                 /* set TC mode for RX QUEUES */
720                 SXGBE_FOR_EACH_QUEUE(priv->hw_cap.rx_mtl_queues, queue_num)
721                         priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr, queue_num,
722                                                        priv->rx_tc);
723         } else {
724                 pr_err("ERROR: %s: Invalid TX threshold mode\n", __func__);
725         }
726 }
727
728 /**
729  * sxgbe_tx_queue_clean:
730  * @priv: driver private structure
731  * Description: it reclaims resources after transmission completes.
732  */
733 static void sxgbe_tx_queue_clean(struct sxgbe_tx_queue *tqueue)
734 {
735         struct sxgbe_priv_data *priv = tqueue->priv_ptr;
736         unsigned int tx_rsize = priv->dma_tx_size;
737         struct netdev_queue *dev_txq;
738         u8 queue_no = tqueue->queue_no;
739
740         dev_txq = netdev_get_tx_queue(priv->dev, queue_no);
741
742         __netif_tx_lock(dev_txq, smp_processor_id());
743
744         priv->xstats.tx_clean++;
745         while (tqueue->dirty_tx != tqueue->cur_tx) {
746                 unsigned int entry = tqueue->dirty_tx % tx_rsize;
747                 struct sk_buff *skb = tqueue->tx_skbuff[entry];
748                 struct sxgbe_tx_norm_desc *p;
749
750                 p = tqueue->dma_tx + entry;
751
752                 /* Check if the descriptor is owned by the DMA. */
753                 if (priv->hw->desc->get_tx_owner(p))
754                         break;
755
756                 if (netif_msg_tx_done(priv))
757                         pr_debug("%s: curr %d, dirty %d\n",
758                                  __func__, tqueue->cur_tx, tqueue->dirty_tx);
759
760                 if (likely(tqueue->tx_skbuff_dma[entry])) {
761                         dma_unmap_single(priv->device,
762                                          tqueue->tx_skbuff_dma[entry],
763                                          priv->hw->desc->get_tx_len(p),
764                                          DMA_TO_DEVICE);
765                         tqueue->tx_skbuff_dma[entry] = 0;
766                 }
767
768                 if (likely(skb)) {
769                         dev_kfree_skb(skb);
770                         tqueue->tx_skbuff[entry] = NULL;
771                 }
772
773                 priv->hw->desc->release_tx_desc(p);
774
775                 tqueue->dirty_tx++;
776         }
777
778         /* wake up queue */
779         if (unlikely(netif_tx_queue_stopped(dev_txq) &&
780             sxgbe_tx_avail(tqueue, tx_rsize) > SXGBE_TX_THRESH(priv))) {
781                 if (netif_msg_tx_done(priv))
782                         pr_debug("%s: restart transmit\n", __func__);
783                 netif_tx_wake_queue(dev_txq);
784         }
785
786         __netif_tx_unlock(dev_txq);
787 }
788
789 /**
790  * sxgbe_tx_clean:
791  * @priv: driver private structure
792  * Description: it reclaims resources after transmission completes.
793  */
794 static void sxgbe_tx_all_clean(struct sxgbe_priv_data * const priv)
795 {
796         u8 queue_num;
797
798         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
799                 struct sxgbe_tx_queue *tqueue = priv->txq[queue_num];
800
801                 sxgbe_tx_queue_clean(tqueue);
802         }
803
804         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
805                 sxgbe_enable_eee_mode(priv);
806                 mod_timer(&priv->eee_ctrl_timer, SXGBE_LPI_TIMER(eee_timer));
807         }
808 }
809
810 /**
811  * sxgbe_restart_tx_queue: irq tx error mng function
812  * @priv: driver private structure
813  * Description: it cleans the descriptors and restarts the transmission
814  * in case of errors.
815  */
816 static void sxgbe_restart_tx_queue(struct sxgbe_priv_data *priv, int queue_num)
817 {
818         struct sxgbe_tx_queue *tx_ring = priv->txq[queue_num];
819         struct netdev_queue *dev_txq = netdev_get_tx_queue(priv->dev,
820                                                            queue_num);
821
822         /* stop the queue */
823         netif_tx_stop_queue(dev_txq);
824
825         /* stop the tx dma */
826         priv->hw->dma->stop_tx_queue(priv->ioaddr, queue_num);
827
828         /* free the skbuffs of the ring */
829         tx_free_ring_skbufs(tx_ring);
830
831         /* initialise counters */
832         tx_ring->cur_tx = 0;
833         tx_ring->dirty_tx = 0;
834
835         /* start the tx dma */
836         priv->hw->dma->start_tx_queue(priv->ioaddr, queue_num);
837
838         priv->dev->stats.tx_errors++;
839
840         /* wakeup the queue */
841         netif_tx_wake_queue(dev_txq);
842 }
843
844 /**
845  * sxgbe_reset_all_tx_queues: irq tx error mng function
846  * @priv: driver private structure
847  * Description: it cleans all the descriptors and
848  * restarts the transmission on all queues in case of errors.
849  */
850 static void sxgbe_reset_all_tx_queues(struct sxgbe_priv_data *priv)
851 {
852         int queue_num;
853
854         /* On TX timeout of net device, resetting of all queues
855          * may not be proper way, revisit this later if needed
856          */
857         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
858                 sxgbe_restart_tx_queue(priv, queue_num);
859 }
860
861 /**
862  * sxgbe_get_hw_features: get XMAC capabilities from the HW cap. register.
863  * @priv: driver private structure
864  * Description:
865  *  new GMAC chip generations have a new register to indicate the
866  *  presence of the optional feature/functions.
867  *  This can be also used to override the value passed through the
868  *  platform and necessary for old MAC10/100 and GMAC chips.
869  */
870 static int sxgbe_get_hw_features(struct sxgbe_priv_data * const priv)
871 {
872         int rval = 0;
873         struct sxgbe_hw_features *features = &priv->hw_cap;
874
875         /* Read First Capability Register CAP[0] */
876         rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 0);
877         if (rval) {
878                 features->pmt_remote_wake_up =
879                         SXGBE_HW_FEAT_PMT_TEMOTE_WOP(rval);
880                 features->pmt_magic_frame = SXGBE_HW_FEAT_PMT_MAGIC_PKT(rval);
881                 features->atime_stamp = SXGBE_HW_FEAT_IEEE1500_2008(rval);
882                 features->tx_csum_offload =
883                         SXGBE_HW_FEAT_TX_CSUM_OFFLOAD(rval);
884                 features->rx_csum_offload =
885                         SXGBE_HW_FEAT_RX_CSUM_OFFLOAD(rval);
886                 features->multi_macaddr = SXGBE_HW_FEAT_MACADDR_COUNT(rval);
887                 features->tstamp_srcselect = SXGBE_HW_FEAT_TSTMAP_SRC(rval);
888                 features->sa_vlan_insert = SXGBE_HW_FEAT_SRCADDR_VLAN(rval);
889                 features->eee = SXGBE_HW_FEAT_EEE(rval);
890         }
891
892         /* Read First Capability Register CAP[1] */
893         rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 1);
894         if (rval) {
895                 features->rxfifo_size = SXGBE_HW_FEAT_RX_FIFO_SIZE(rval);
896                 features->txfifo_size = SXGBE_HW_FEAT_TX_FIFO_SIZE(rval);
897                 features->atstmap_hword = SXGBE_HW_FEAT_TX_FIFO_SIZE(rval);
898                 features->dcb_enable = SXGBE_HW_FEAT_DCB(rval);
899                 features->splithead_enable = SXGBE_HW_FEAT_SPLIT_HDR(rval);
900                 features->tcpseg_offload = SXGBE_HW_FEAT_TSO(rval);
901                 features->debug_mem = SXGBE_HW_FEAT_DEBUG_MEM_IFACE(rval);
902                 features->rss_enable = SXGBE_HW_FEAT_RSS(rval);
903                 features->hash_tsize = SXGBE_HW_FEAT_HASH_TABLE_SIZE(rval);
904                 features->l3l4_filer_size = SXGBE_HW_FEAT_L3L4_FILTER_NUM(rval);
905         }
906
907         /* Read First Capability Register CAP[2] */
908         rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 2);
909         if (rval) {
910                 features->rx_mtl_queues = SXGBE_HW_FEAT_RX_MTL_QUEUES(rval);
911                 features->tx_mtl_queues = SXGBE_HW_FEAT_TX_MTL_QUEUES(rval);
912                 features->rx_dma_channels = SXGBE_HW_FEAT_RX_DMA_CHANNELS(rval);
913                 features->tx_dma_channels = SXGBE_HW_FEAT_TX_DMA_CHANNELS(rval);
914                 features->pps_output_count = SXGBE_HW_FEAT_PPS_OUTPUTS(rval);
915                 features->aux_input_count = SXGBE_HW_FEAT_AUX_SNAPSHOTS(rval);
916         }
917
918         return rval;
919 }
920
921 /**
922  * sxgbe_check_ether_addr: check if the MAC addr is valid
923  * @priv: driver private structure
924  * Description:
925  * it is to verify if the MAC address is valid, in case of failures it
926  * generates a random MAC address
927  */
928 static void sxgbe_check_ether_addr(struct sxgbe_priv_data *priv)
929 {
930         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
931                 priv->hw->mac->get_umac_addr((void __iomem *)
932                                              priv->ioaddr,
933                                              priv->dev->dev_addr, 0);
934                 if (!is_valid_ether_addr(priv->dev->dev_addr))
935                         eth_hw_addr_random(priv->dev);
936         }
937         dev_info(priv->device, "device MAC address %pM\n",
938                  priv->dev->dev_addr);
939 }
940
941 /**
942  * sxgbe_init_dma_engine: DMA init.
943  * @priv: driver private structure
944  * Description:
945  * It inits the DMA invoking the specific SXGBE callback.
946  * Some DMA parameters can be passed from the platform;
947  * in case of these are not passed a default is kept for the MAC or GMAC.
948  */
949 static int sxgbe_init_dma_engine(struct sxgbe_priv_data *priv)
950 {
951         int pbl = DEFAULT_DMA_PBL, fixed_burst = 0, burst_map = 0;
952         int queue_num;
953
954         if (priv->plat->dma_cfg) {
955                 pbl = priv->plat->dma_cfg->pbl;
956                 fixed_burst = priv->plat->dma_cfg->fixed_burst;
957                 burst_map = priv->plat->dma_cfg->burst_map;
958         }
959
960         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
961                 priv->hw->dma->cha_init(priv->ioaddr, queue_num,
962                                         fixed_burst, pbl,
963                                         (priv->txq[queue_num])->dma_tx_phy,
964                                         (priv->rxq[queue_num])->dma_rx_phy,
965                                         priv->dma_tx_size, priv->dma_rx_size);
966
967         return priv->hw->dma->init(priv->ioaddr, fixed_burst, burst_map);
968 }
969
970 /**
971  * sxgbe_init_mtl_engine: MTL init.
972  * @priv: driver private structure
973  * Description:
974  * It inits the MTL invoking the specific SXGBE callback.
975  */
976 static void sxgbe_init_mtl_engine(struct sxgbe_priv_data *priv)
977 {
978         int queue_num;
979
980         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
981                 priv->hw->mtl->mtl_set_txfifosize(priv->ioaddr, queue_num,
982                                                   priv->hw_cap.tx_mtl_qsize);
983                 priv->hw->mtl->mtl_enable_txqueue(priv->ioaddr, queue_num);
984         }
985 }
986
987 /**
988  * sxgbe_disable_mtl_engine: MTL disable.
989  * @priv: driver private structure
990  * Description:
991  * It disables the MTL queues by invoking the specific SXGBE callback.
992  */
993 static void sxgbe_disable_mtl_engine(struct sxgbe_priv_data *priv)
994 {
995         int queue_num;
996
997         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
998                 priv->hw->mtl->mtl_disable_txqueue(priv->ioaddr, queue_num);
999 }
1000
1001
1002 /**
1003  * sxgbe_tx_timer: mitigation sw timer for tx.
1004  * @t: timer pointer
1005  * Description:
1006  * This is the timer handler to directly invoke the sxgbe_tx_clean.
1007  */
1008 static void sxgbe_tx_timer(struct timer_list *t)
1009 {
1010         struct sxgbe_tx_queue *p = from_timer(p, t, txtimer);
1011         sxgbe_tx_queue_clean(p);
1012 }
1013
1014 /**
1015  * sxgbe_init_tx_coalesce: init tx mitigation options.
1016  * @priv: driver private structure
1017  * Description:
1018  * This inits the transmit coalesce parameters: i.e. timer rate,
1019  * timer handler and default threshold used for enabling the
1020  * interrupt on completion bit.
1021  */
1022 static void sxgbe_tx_init_coalesce(struct sxgbe_priv_data *priv)
1023 {
1024         u8 queue_num;
1025
1026         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1027                 struct sxgbe_tx_queue *p = priv->txq[queue_num];
1028                 p->tx_coal_frames =  SXGBE_TX_FRAMES;
1029                 p->tx_coal_timer = SXGBE_COAL_TX_TIMER;
1030                 timer_setup(&p->txtimer, sxgbe_tx_timer, 0);
1031                 p->txtimer.expires = SXGBE_COAL_TIMER(p->tx_coal_timer);
1032                 add_timer(&p->txtimer);
1033         }
1034 }
1035
1036 static void sxgbe_tx_del_timer(struct sxgbe_priv_data *priv)
1037 {
1038         u8 queue_num;
1039
1040         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1041                 struct sxgbe_tx_queue *p = priv->txq[queue_num];
1042                 del_timer_sync(&p->txtimer);
1043         }
1044 }
1045
1046 /**
1047  *  sxgbe_open - open entry point of the driver
1048  *  @dev : pointer to the device structure.
1049  *  Description:
1050  *  This function is the open entry point of the driver.
1051  *  Return value:
1052  *  0 on success and an appropriate (-)ve integer as defined in errno.h
1053  *  file on failure.
1054  */
1055 static int sxgbe_open(struct net_device *dev)
1056 {
1057         struct sxgbe_priv_data *priv = netdev_priv(dev);
1058         int ret, queue_num;
1059
1060         clk_prepare_enable(priv->sxgbe_clk);
1061
1062         sxgbe_check_ether_addr(priv);
1063
1064         /* Init the phy */
1065         ret = sxgbe_init_phy(dev);
1066         if (ret) {
1067                 netdev_err(dev, "%s: Cannot attach to PHY (error: %d)\n",
1068                            __func__, ret);
1069                 goto phy_error;
1070         }
1071
1072         /* Create and initialize the TX/RX descriptors chains. */
1073         priv->dma_tx_size = SXGBE_ALIGN(DMA_TX_SIZE);
1074         priv->dma_rx_size = SXGBE_ALIGN(DMA_RX_SIZE);
1075         priv->dma_buf_sz = SXGBE_ALIGN(DMA_BUFFER_SIZE);
1076         priv->tx_tc = TC_DEFAULT;
1077         priv->rx_tc = TC_DEFAULT;
1078         init_dma_desc_rings(dev);
1079
1080         /* DMA initialization and SW reset */
1081         ret = sxgbe_init_dma_engine(priv);
1082         if (ret < 0) {
1083                 netdev_err(dev, "%s: DMA initialization failed\n", __func__);
1084                 goto init_error;
1085         }
1086
1087         /*  MTL initialization */
1088         sxgbe_init_mtl_engine(priv);
1089
1090         /* Copy the MAC addr into the HW  */
1091         priv->hw->mac->set_umac_addr(priv->ioaddr, dev->dev_addr, 0);
1092
1093         /* Initialize the MAC Core */
1094         priv->hw->mac->core_init(priv->ioaddr);
1095         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
1096                 priv->hw->mac->enable_rxqueue(priv->ioaddr, queue_num);
1097         }
1098
1099         /* Request the IRQ lines */
1100         ret = devm_request_irq(priv->device, priv->irq, sxgbe_common_interrupt,
1101                                IRQF_SHARED, dev->name, dev);
1102         if (unlikely(ret < 0)) {
1103                 netdev_err(dev, "%s: ERROR: allocating the IRQ %d (error: %d)\n",
1104                            __func__, priv->irq, ret);
1105                 goto init_error;
1106         }
1107
1108         /* If the LPI irq is different from the mac irq
1109          * register a dedicated handler
1110          */
1111         if (priv->lpi_irq != dev->irq) {
1112                 ret = devm_request_irq(priv->device, priv->lpi_irq,
1113                                        sxgbe_common_interrupt,
1114                                        IRQF_SHARED, dev->name, dev);
1115                 if (unlikely(ret < 0)) {
1116                         netdev_err(dev, "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
1117                                    __func__, priv->lpi_irq, ret);
1118                         goto init_error;
1119                 }
1120         }
1121
1122         /* Request TX DMA irq lines */
1123         SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1124                 ret = devm_request_irq(priv->device,
1125                                        (priv->txq[queue_num])->irq_no,
1126                                        sxgbe_tx_interrupt, 0,
1127                                        dev->name, priv->txq[queue_num]);
1128                 if (unlikely(ret < 0)) {
1129                         netdev_err(dev, "%s: ERROR: allocating TX IRQ %d (error: %d)\n",
1130                                    __func__, priv->irq, ret);
1131                         goto init_error;
1132                 }
1133         }
1134
1135         /* Request RX DMA irq lines */
1136         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
1137                 ret = devm_request_irq(priv->device,
1138                                        (priv->rxq[queue_num])->irq_no,
1139                                        sxgbe_rx_interrupt, 0,
1140                                        dev->name, priv->rxq[queue_num]);
1141                 if (unlikely(ret < 0)) {
1142                         netdev_err(dev, "%s: ERROR: allocating TX IRQ %d (error: %d)\n",
1143                                    __func__, priv->irq, ret);
1144                         goto init_error;
1145                 }
1146         }
1147
1148         /* Enable the MAC Rx/Tx */
1149         priv->hw->mac->enable_tx(priv->ioaddr, true);
1150         priv->hw->mac->enable_rx(priv->ioaddr, true);
1151
1152         /* Set the HW DMA mode and the COE */
1153         sxgbe_mtl_operation_mode(priv);
1154
1155         /* Extra statistics */
1156         memset(&priv->xstats, 0, sizeof(struct sxgbe_extra_stats));
1157
1158         priv->xstats.tx_threshold = priv->tx_tc;
1159         priv->xstats.rx_threshold = priv->rx_tc;
1160
1161         /* Start the ball rolling... */
1162         netdev_dbg(dev, "DMA RX/TX processes started...\n");
1163         priv->hw->dma->start_tx(priv->ioaddr, SXGBE_TX_QUEUES);
1164         priv->hw->dma->start_rx(priv->ioaddr, SXGBE_RX_QUEUES);
1165
1166         if (dev->phydev)
1167                 phy_start(dev->phydev);
1168
1169         /* initialise TX coalesce parameters */
1170         sxgbe_tx_init_coalesce(priv);
1171
1172         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
1173                 priv->rx_riwt = SXGBE_MAX_DMA_RIWT;
1174                 priv->hw->dma->rx_watchdog(priv->ioaddr, SXGBE_MAX_DMA_RIWT);
1175         }
1176
1177         priv->tx_lpi_timer = SXGBE_DEFAULT_LPI_TIMER;
1178         priv->eee_enabled = sxgbe_eee_init(priv);
1179
1180         napi_enable(&priv->napi);
1181         netif_start_queue(dev);
1182
1183         return 0;
1184
1185 init_error:
1186         free_dma_desc_resources(priv);
1187         if (dev->phydev)
1188                 phy_disconnect(dev->phydev);
1189 phy_error:
1190         clk_disable_unprepare(priv->sxgbe_clk);
1191
1192         return ret;
1193 }
1194
1195 /**
1196  *  sxgbe_release - close entry point of the driver
1197  *  @dev : device pointer.
1198  *  Description:
1199  *  This is the stop entry point of the driver.
1200  */
1201 static int sxgbe_release(struct net_device *dev)
1202 {
1203         struct sxgbe_priv_data *priv = netdev_priv(dev);
1204
1205         if (priv->eee_enabled)
1206                 del_timer_sync(&priv->eee_ctrl_timer);
1207
1208         /* Stop and disconnect the PHY */
1209         if (dev->phydev) {
1210                 phy_stop(dev->phydev);
1211                 phy_disconnect(dev->phydev);
1212         }
1213
1214         netif_tx_stop_all_queues(dev);
1215
1216         napi_disable(&priv->napi);
1217
1218         /* delete TX timers */
1219         sxgbe_tx_del_timer(priv);
1220
1221         /* Stop TX/RX DMA and clear the descriptors */
1222         priv->hw->dma->stop_tx(priv->ioaddr, SXGBE_TX_QUEUES);
1223         priv->hw->dma->stop_rx(priv->ioaddr, SXGBE_RX_QUEUES);
1224
1225         /* disable MTL queue */
1226         sxgbe_disable_mtl_engine(priv);
1227
1228         /* Release and free the Rx/Tx resources */
1229         free_dma_desc_resources(priv);
1230
1231         /* Disable the MAC Rx/Tx */
1232         priv->hw->mac->enable_tx(priv->ioaddr, false);
1233         priv->hw->mac->enable_rx(priv->ioaddr, false);
1234
1235         clk_disable_unprepare(priv->sxgbe_clk);
1236
1237         return 0;
1238 }
1239 /* Prepare first Tx descriptor for doing TSO operation */
1240 static void sxgbe_tso_prepare(struct sxgbe_priv_data *priv,
1241                               struct sxgbe_tx_norm_desc *first_desc,
1242                               struct sk_buff *skb)
1243 {
1244         unsigned int total_hdr_len, tcp_hdr_len;
1245
1246         /* Write first Tx descriptor with appropriate value */
1247         tcp_hdr_len = tcp_hdrlen(skb);
1248         total_hdr_len = skb_transport_offset(skb) + tcp_hdr_len;
1249
1250         first_desc->tdes01 = dma_map_single(priv->device, skb->data,
1251                                             total_hdr_len, DMA_TO_DEVICE);
1252         if (dma_mapping_error(priv->device, first_desc->tdes01))
1253                 pr_err("%s: TX dma mapping failed!!\n", __func__);
1254
1255         first_desc->tdes23.tx_rd_des23.first_desc = 1;
1256         priv->hw->desc->tx_desc_enable_tse(first_desc, 1, total_hdr_len,
1257                                            tcp_hdr_len,
1258                                            skb->len - total_hdr_len);
1259 }
1260
1261 /**
1262  *  sxgbe_xmit: Tx entry point of the driver
1263  *  @skb : the socket buffer
1264  *  @dev : device pointer
1265  *  Description : this is the tx entry point of the driver.
1266  *  It programs the chain or the ring and supports oversized frames
1267  *  and SG feature.
1268  */
1269 static netdev_tx_t sxgbe_xmit(struct sk_buff *skb, struct net_device *dev)
1270 {
1271         unsigned int entry, frag_num;
1272         int cksum_flag = 0;
1273         struct netdev_queue *dev_txq;
1274         unsigned txq_index = skb_get_queue_mapping(skb);
1275         struct sxgbe_priv_data *priv = netdev_priv(dev);
1276         unsigned int tx_rsize = priv->dma_tx_size;
1277         struct sxgbe_tx_queue *tqueue = priv->txq[txq_index];
1278         struct sxgbe_tx_norm_desc *tx_desc, *first_desc;
1279         struct sxgbe_tx_ctxt_desc *ctxt_desc = NULL;
1280         int nr_frags = skb_shinfo(skb)->nr_frags;
1281         int no_pagedlen = skb_headlen(skb);
1282         int is_jumbo = 0;
1283         u16 cur_mss = skb_shinfo(skb)->gso_size;
1284         u32 ctxt_desc_req = 0;
1285
1286         /* get the TX queue handle */
1287         dev_txq = netdev_get_tx_queue(dev, txq_index);
1288
1289         if (unlikely(skb_is_gso(skb) && tqueue->prev_mss != cur_mss))
1290                 ctxt_desc_req = 1;
1291
1292         if (unlikely(skb_vlan_tag_present(skb) ||
1293                      ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1294                       tqueue->hwts_tx_en)))
1295                 ctxt_desc_req = 1;
1296
1297         if (priv->tx_path_in_lpi_mode)
1298                 sxgbe_disable_eee_mode(priv);
1299
1300         if (unlikely(sxgbe_tx_avail(tqueue, tx_rsize) < nr_frags + 1)) {
1301                 if (!netif_tx_queue_stopped(dev_txq)) {
1302                         netif_tx_stop_queue(dev_txq);
1303                         netdev_err(dev, "%s: Tx Ring is full when %d queue is awake\n",
1304                                    __func__, txq_index);
1305                 }
1306                 return NETDEV_TX_BUSY;
1307         }
1308
1309         entry = tqueue->cur_tx % tx_rsize;
1310         tx_desc = tqueue->dma_tx + entry;
1311
1312         first_desc = tx_desc;
1313         if (ctxt_desc_req)
1314                 ctxt_desc = (struct sxgbe_tx_ctxt_desc *)first_desc;
1315
1316         /* save the skb address */
1317         tqueue->tx_skbuff[entry] = skb;
1318
1319         if (!is_jumbo) {
1320                 if (likely(skb_is_gso(skb))) {
1321                         /* TSO support */
1322                         if (unlikely(tqueue->prev_mss != cur_mss)) {
1323                                 priv->hw->desc->tx_ctxt_desc_set_mss(
1324                                                 ctxt_desc, cur_mss);
1325                                 priv->hw->desc->tx_ctxt_desc_set_tcmssv(
1326                                                 ctxt_desc);
1327                                 priv->hw->desc->tx_ctxt_desc_reset_ostc(
1328                                                 ctxt_desc);
1329                                 priv->hw->desc->tx_ctxt_desc_set_ctxt(
1330                                                 ctxt_desc);
1331                                 priv->hw->desc->tx_ctxt_desc_set_owner(
1332                                                 ctxt_desc);
1333
1334                                 entry = (++tqueue->cur_tx) % tx_rsize;
1335                                 first_desc = tqueue->dma_tx + entry;
1336
1337                                 tqueue->prev_mss = cur_mss;
1338                         }
1339                         sxgbe_tso_prepare(priv, first_desc, skb);
1340                 } else {
1341                         tx_desc->tdes01 = dma_map_single(priv->device,
1342                                                          skb->data, no_pagedlen, DMA_TO_DEVICE);
1343                         if (dma_mapping_error(priv->device, tx_desc->tdes01))
1344                                 netdev_err(dev, "%s: TX dma mapping failed!!\n",
1345                                            __func__);
1346
1347                         priv->hw->desc->prepare_tx_desc(tx_desc, 1, no_pagedlen,
1348                                                         no_pagedlen, cksum_flag);
1349                 }
1350         }
1351
1352         for (frag_num = 0; frag_num < nr_frags; frag_num++) {
1353                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_num];
1354                 int len = skb_frag_size(frag);
1355
1356                 entry = (++tqueue->cur_tx) % tx_rsize;
1357                 tx_desc = tqueue->dma_tx + entry;
1358                 tx_desc->tdes01 = skb_frag_dma_map(priv->device, frag, 0, len,
1359                                                    DMA_TO_DEVICE);
1360
1361                 tqueue->tx_skbuff_dma[entry] = tx_desc->tdes01;
1362                 tqueue->tx_skbuff[entry] = NULL;
1363
1364                 /* prepare the descriptor */
1365                 priv->hw->desc->prepare_tx_desc(tx_desc, 0, len,
1366                                                 len, cksum_flag);
1367                 /* memory barrier to flush descriptor */
1368                 wmb();
1369
1370                 /* set the owner */
1371                 priv->hw->desc->set_tx_owner(tx_desc);
1372         }
1373
1374         /* close the descriptors */
1375         priv->hw->desc->close_tx_desc(tx_desc);
1376
1377         /* memory barrier to flush descriptor */
1378         wmb();
1379
1380         tqueue->tx_count_frames += nr_frags + 1;
1381         if (tqueue->tx_count_frames > tqueue->tx_coal_frames) {
1382                 priv->hw->desc->clear_tx_ic(tx_desc);
1383                 priv->xstats.tx_reset_ic_bit++;
1384                 mod_timer(&tqueue->txtimer,
1385                           SXGBE_COAL_TIMER(tqueue->tx_coal_timer));
1386         } else {
1387                 tqueue->tx_count_frames = 0;
1388         }
1389
1390         /* set owner for first desc */
1391         priv->hw->desc->set_tx_owner(first_desc);
1392
1393         /* memory barrier to flush descriptor */
1394         wmb();
1395
1396         tqueue->cur_tx++;
1397
1398         /* display current ring */
1399         netif_dbg(priv, pktdata, dev, "%s: curr %d dirty=%d entry=%d, first=%p, nfrags=%d\n",
1400                   __func__, tqueue->cur_tx % tx_rsize,
1401                   tqueue->dirty_tx % tx_rsize, entry,
1402                   first_desc, nr_frags);
1403
1404         if (unlikely(sxgbe_tx_avail(tqueue, tx_rsize) <= (MAX_SKB_FRAGS + 1))) {
1405                 netif_dbg(priv, hw, dev, "%s: stop transmitted packets\n",
1406                           __func__);
1407                 netif_tx_stop_queue(dev_txq);
1408         }
1409
1410         dev->stats.tx_bytes += skb->len;
1411
1412         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1413                      tqueue->hwts_tx_en)) {
1414                 /* declare that device is doing timestamping */
1415                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1416                 priv->hw->desc->tx_enable_tstamp(first_desc);
1417         }
1418
1419         skb_tx_timestamp(skb);
1420
1421         priv->hw->dma->enable_dma_transmission(priv->ioaddr, txq_index);
1422
1423         return NETDEV_TX_OK;
1424 }
1425
1426 /**
1427  * sxgbe_rx_refill: refill used skb preallocated buffers
1428  * @priv: driver private structure
1429  * Description : this is to reallocate the skb for the reception process
1430  * that is based on zero-copy.
1431  */
1432 static void sxgbe_rx_refill(struct sxgbe_priv_data *priv)
1433 {
1434         unsigned int rxsize = priv->dma_rx_size;
1435         int bfsize = priv->dma_buf_sz;
1436         u8 qnum = priv->cur_rx_qnum;
1437
1438         for (; priv->rxq[qnum]->cur_rx - priv->rxq[qnum]->dirty_rx > 0;
1439              priv->rxq[qnum]->dirty_rx++) {
1440                 unsigned int entry = priv->rxq[qnum]->dirty_rx % rxsize;
1441                 struct sxgbe_rx_norm_desc *p;
1442
1443                 p = priv->rxq[qnum]->dma_rx + entry;
1444
1445                 if (likely(priv->rxq[qnum]->rx_skbuff[entry] == NULL)) {
1446                         struct sk_buff *skb;
1447
1448                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
1449
1450                         if (unlikely(skb == NULL))
1451                                 break;
1452
1453                         priv->rxq[qnum]->rx_skbuff[entry] = skb;
1454                         priv->rxq[qnum]->rx_skbuff_dma[entry] =
1455                                 dma_map_single(priv->device, skb->data, bfsize,
1456                                                DMA_FROM_DEVICE);
1457
1458                         p->rdes23.rx_rd_des23.buf2_addr =
1459                                 priv->rxq[qnum]->rx_skbuff_dma[entry];
1460                 }
1461
1462                 /* Added memory barrier for RX descriptor modification */
1463                 wmb();
1464                 priv->hw->desc->set_rx_owner(p);
1465                 priv->hw->desc->set_rx_int_on_com(p);
1466                 /* Added memory barrier for RX descriptor modification */
1467                 wmb();
1468         }
1469 }
1470
1471 /**
1472  * sxgbe_rx: receive the frames from the remote host
1473  * @priv: driver private structure
1474  * @limit: napi bugget.
1475  * Description :  this the function called by the napi poll method.
1476  * It gets all the frames inside the ring.
1477  */
1478 static int sxgbe_rx(struct sxgbe_priv_data *priv, int limit)
1479 {
1480         u8 qnum = priv->cur_rx_qnum;
1481         unsigned int rxsize = priv->dma_rx_size;
1482         unsigned int entry = priv->rxq[qnum]->cur_rx;
1483         unsigned int next_entry = 0;
1484         unsigned int count = 0;
1485         int checksum;
1486         int status;
1487
1488         while (count < limit) {
1489                 struct sxgbe_rx_norm_desc *p;
1490                 struct sk_buff *skb;
1491                 int frame_len;
1492
1493                 p = priv->rxq[qnum]->dma_rx + entry;
1494
1495                 if (priv->hw->desc->get_rx_owner(p))
1496                         break;
1497
1498                 count++;
1499
1500                 next_entry = (++priv->rxq[qnum]->cur_rx) % rxsize;
1501                 prefetch(priv->rxq[qnum]->dma_rx + next_entry);
1502
1503                 /* Read the status of the incoming frame and also get checksum
1504                  * value based on whether it is enabled in SXGBE hardware or
1505                  * not.
1506                  */
1507                 status = priv->hw->desc->rx_wbstatus(p, &priv->xstats,
1508                                                      &checksum);
1509                 if (unlikely(status < 0)) {
1510                         entry = next_entry;
1511                         continue;
1512                 }
1513                 if (unlikely(!priv->rxcsum_insertion))
1514                         checksum = CHECKSUM_NONE;
1515
1516                 skb = priv->rxq[qnum]->rx_skbuff[entry];
1517
1518                 if (unlikely(!skb))
1519                         netdev_err(priv->dev, "rx descriptor is not consistent\n");
1520
1521                 prefetch(skb->data - NET_IP_ALIGN);
1522                 priv->rxq[qnum]->rx_skbuff[entry] = NULL;
1523
1524                 frame_len = priv->hw->desc->get_rx_frame_len(p);
1525
1526                 skb_put(skb, frame_len);
1527
1528                 skb->ip_summed = checksum;
1529                 if (checksum == CHECKSUM_NONE)
1530                         netif_receive_skb(skb);
1531                 else
1532                         napi_gro_receive(&priv->napi, skb);
1533
1534                 entry = next_entry;
1535         }
1536
1537         sxgbe_rx_refill(priv);
1538
1539         return count;
1540 }
1541
1542 /**
1543  *  sxgbe_poll - sxgbe poll method (NAPI)
1544  *  @napi : pointer to the napi structure.
1545  *  @budget : maximum number of packets that the current CPU can receive from
1546  *            all interfaces.
1547  *  Description :
1548  *  To look at the incoming frames and clear the tx resources.
1549  */
1550 static int sxgbe_poll(struct napi_struct *napi, int budget)
1551 {
1552         struct sxgbe_priv_data *priv = container_of(napi,
1553                                                     struct sxgbe_priv_data, napi);
1554         int work_done = 0;
1555         u8 qnum = priv->cur_rx_qnum;
1556
1557         priv->xstats.napi_poll++;
1558         /* first, clean the tx queues */
1559         sxgbe_tx_all_clean(priv);
1560
1561         work_done = sxgbe_rx(priv, budget);
1562         if (work_done < budget) {
1563                 napi_complete_done(napi, work_done);
1564                 priv->hw->dma->enable_dma_irq(priv->ioaddr, qnum);
1565         }
1566
1567         return work_done;
1568 }
1569
1570 /**
1571  *  sxgbe_tx_timeout
1572  *  @dev : Pointer to net device structure
1573  *  Description: this function is called when a packet transmission fails to
1574  *   complete within a reasonable time. The driver will mark the error in the
1575  *   netdev structure and arrange for the device to be reset to a sane state
1576  *   in order to transmit a new packet.
1577  */
1578 static void sxgbe_tx_timeout(struct net_device *dev)
1579 {
1580         struct sxgbe_priv_data *priv = netdev_priv(dev);
1581
1582         sxgbe_reset_all_tx_queues(priv);
1583 }
1584
1585 /**
1586  *  sxgbe_common_interrupt - main ISR
1587  *  @irq: interrupt number.
1588  *  @dev_id: to pass the net device pointer.
1589  *  Description: this is the main driver interrupt service routine.
1590  *  It calls the DMA ISR and also the core ISR to manage PMT, MMC, LPI
1591  *  interrupts.
1592  */
1593 static irqreturn_t sxgbe_common_interrupt(int irq, void *dev_id)
1594 {
1595         struct net_device *netdev = (struct net_device *)dev_id;
1596         struct sxgbe_priv_data *priv = netdev_priv(netdev);
1597         int status;
1598
1599         status = priv->hw->mac->host_irq_status(priv->ioaddr, &priv->xstats);
1600         /* For LPI we need to save the tx status */
1601         if (status & TX_ENTRY_LPI_MODE) {
1602                 priv->xstats.tx_lpi_entry_n++;
1603                 priv->tx_path_in_lpi_mode = true;
1604         }
1605         if (status & TX_EXIT_LPI_MODE) {
1606                 priv->xstats.tx_lpi_exit_n++;
1607                 priv->tx_path_in_lpi_mode = false;
1608         }
1609         if (status & RX_ENTRY_LPI_MODE)
1610                 priv->xstats.rx_lpi_entry_n++;
1611         if (status & RX_EXIT_LPI_MODE)
1612                 priv->xstats.rx_lpi_exit_n++;
1613
1614         return IRQ_HANDLED;
1615 }
1616
1617 /**
1618  *  sxgbe_tx_interrupt - TX DMA ISR
1619  *  @irq: interrupt number.
1620  *  @dev_id: to pass the net device pointer.
1621  *  Description: this is the tx dma interrupt service routine.
1622  */
1623 static irqreturn_t sxgbe_tx_interrupt(int irq, void *dev_id)
1624 {
1625         int status;
1626         struct sxgbe_tx_queue *txq = (struct sxgbe_tx_queue *)dev_id;
1627         struct sxgbe_priv_data *priv = txq->priv_ptr;
1628
1629         /* get the channel status */
1630         status = priv->hw->dma->tx_dma_int_status(priv->ioaddr, txq->queue_no,
1631                                                   &priv->xstats);
1632         /* check for normal path */
1633         if (likely((status & handle_tx)))
1634                 napi_schedule(&priv->napi);
1635
1636         /* check for unrecoverable error */
1637         if (unlikely((status & tx_hard_error)))
1638                 sxgbe_restart_tx_queue(priv, txq->queue_no);
1639
1640         /* check for TC configuration change */
1641         if (unlikely((status & tx_bump_tc) &&
1642                      (priv->tx_tc != SXGBE_MTL_SFMODE) &&
1643                      (priv->tx_tc < 512))) {
1644                 /* step of TX TC is 32 till 128, otherwise 64 */
1645                 priv->tx_tc += (priv->tx_tc < 128) ? 32 : 64;
1646                 priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr,
1647                                                txq->queue_no, priv->tx_tc);
1648                 priv->xstats.tx_threshold = priv->tx_tc;
1649         }
1650
1651         return IRQ_HANDLED;
1652 }
1653
1654 /**
1655  *  sxgbe_rx_interrupt - RX DMA ISR
1656  *  @irq: interrupt number.
1657  *  @dev_id: to pass the net device pointer.
1658  *  Description: this is the rx dma interrupt service routine.
1659  */
1660 static irqreturn_t sxgbe_rx_interrupt(int irq, void *dev_id)
1661 {
1662         int status;
1663         struct sxgbe_rx_queue *rxq = (struct sxgbe_rx_queue *)dev_id;
1664         struct sxgbe_priv_data *priv = rxq->priv_ptr;
1665
1666         /* get the channel status */
1667         status = priv->hw->dma->rx_dma_int_status(priv->ioaddr, rxq->queue_no,
1668                                                   &priv->xstats);
1669
1670         if (likely((status & handle_rx) && (napi_schedule_prep(&priv->napi)))) {
1671                 priv->hw->dma->disable_dma_irq(priv->ioaddr, rxq->queue_no);
1672                 __napi_schedule(&priv->napi);
1673         }
1674
1675         /* check for TC configuration change */
1676         if (unlikely((status & rx_bump_tc) &&
1677                      (priv->rx_tc != SXGBE_MTL_SFMODE) &&
1678                      (priv->rx_tc < 128))) {
1679                 /* step of TC is 32 */
1680                 priv->rx_tc += 32;
1681                 priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr,
1682                                                rxq->queue_no, priv->rx_tc);
1683                 priv->xstats.rx_threshold = priv->rx_tc;
1684         }
1685
1686         return IRQ_HANDLED;
1687 }
1688
1689 static inline u64 sxgbe_get_stat64(void __iomem *ioaddr, int reg_lo, int reg_hi)
1690 {
1691         u64 val = readl(ioaddr + reg_lo);
1692
1693         val |= ((u64)readl(ioaddr + reg_hi)) << 32;
1694
1695         return val;
1696 }
1697
1698
1699 /*  sxgbe_get_stats64 - entry point to see statistical information of device
1700  *  @dev : device pointer.
1701  *  @stats : pointer to hold all the statistical information of device.
1702  *  Description:
1703  *  This function is a driver entry point whenever ifconfig command gets
1704  *  executed to see device statistics. Statistics are number of
1705  *  bytes sent or received, errors occurred etc.
1706  */
1707 static void sxgbe_get_stats64(struct net_device *dev,
1708                               struct rtnl_link_stats64 *stats)
1709 {
1710         struct sxgbe_priv_data *priv = netdev_priv(dev);
1711         void __iomem *ioaddr = priv->ioaddr;
1712         u64 count;
1713
1714         spin_lock(&priv->stats_lock);
1715         /* Freeze the counter registers before reading value otherwise it may
1716          * get updated by hardware while we are reading them
1717          */
1718         writel(SXGBE_MMC_CTRL_CNT_FRZ, ioaddr + SXGBE_MMC_CTL_REG);
1719
1720         stats->rx_bytes = sxgbe_get_stat64(ioaddr,
1721                                            SXGBE_MMC_RXOCTETLO_GCNT_REG,
1722                                            SXGBE_MMC_RXOCTETHI_GCNT_REG);
1723
1724         stats->rx_packets = sxgbe_get_stat64(ioaddr,
1725                                              SXGBE_MMC_RXFRAMELO_GBCNT_REG,
1726                                              SXGBE_MMC_RXFRAMEHI_GBCNT_REG);
1727
1728         stats->multicast = sxgbe_get_stat64(ioaddr,
1729                                             SXGBE_MMC_RXMULTILO_GCNT_REG,
1730                                             SXGBE_MMC_RXMULTIHI_GCNT_REG);
1731
1732         stats->rx_crc_errors = sxgbe_get_stat64(ioaddr,
1733                                                 SXGBE_MMC_RXCRCERRLO_REG,
1734                                                 SXGBE_MMC_RXCRCERRHI_REG);
1735
1736         stats->rx_length_errors = sxgbe_get_stat64(ioaddr,
1737                                                   SXGBE_MMC_RXLENERRLO_REG,
1738                                                   SXGBE_MMC_RXLENERRHI_REG);
1739
1740         stats->rx_missed_errors = sxgbe_get_stat64(ioaddr,
1741                                                    SXGBE_MMC_RXFIFOOVERFLOWLO_GBCNT_REG,
1742                                                    SXGBE_MMC_RXFIFOOVERFLOWHI_GBCNT_REG);
1743
1744         stats->tx_bytes = sxgbe_get_stat64(ioaddr,
1745                                            SXGBE_MMC_TXOCTETLO_GCNT_REG,
1746                                            SXGBE_MMC_TXOCTETHI_GCNT_REG);
1747
1748         count = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXFRAMELO_GBCNT_REG,
1749                                  SXGBE_MMC_TXFRAMEHI_GBCNT_REG);
1750
1751         stats->tx_errors = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXFRAMELO_GCNT_REG,
1752                                             SXGBE_MMC_TXFRAMEHI_GCNT_REG);
1753         stats->tx_errors = count - stats->tx_errors;
1754         stats->tx_packets = count;
1755         stats->tx_fifo_errors = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXUFLWLO_GBCNT_REG,
1756                                                  SXGBE_MMC_TXUFLWHI_GBCNT_REG);
1757         writel(0, ioaddr + SXGBE_MMC_CTL_REG);
1758         spin_unlock(&priv->stats_lock);
1759 }
1760
1761 /*  sxgbe_set_features - entry point to set offload features of the device.
1762  *  @dev : device pointer.
1763  *  @features : features which are required to be set.
1764  *  Description:
1765  *  This function is a driver entry point and called by Linux kernel whenever
1766  *  any device features are set or reset by user.
1767  *  Return value:
1768  *  This function returns 0 after setting or resetting device features.
1769  */
1770 static int sxgbe_set_features(struct net_device *dev,
1771                               netdev_features_t features)
1772 {
1773         struct sxgbe_priv_data *priv = netdev_priv(dev);
1774         netdev_features_t changed = dev->features ^ features;
1775
1776         if (changed & NETIF_F_RXCSUM) {
1777                 if (features & NETIF_F_RXCSUM) {
1778                         priv->hw->mac->enable_rx_csum(priv->ioaddr);
1779                         priv->rxcsum_insertion = true;
1780                 } else {
1781                         priv->hw->mac->disable_rx_csum(priv->ioaddr);
1782                         priv->rxcsum_insertion = false;
1783                 }
1784         }
1785
1786         return 0;
1787 }
1788
1789 /*  sxgbe_change_mtu - entry point to change MTU size for the device.
1790  *  @dev : device pointer.
1791  *  @new_mtu : the new MTU size for the device.
1792  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
1793  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
1794  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
1795  *  Return value:
1796  *  0 on success and an appropriate (-)ve integer as defined in errno.h
1797  *  file on failure.
1798  */
1799 static int sxgbe_change_mtu(struct net_device *dev, int new_mtu)
1800 {
1801         dev->mtu = new_mtu;
1802
1803         if (!netif_running(dev))
1804                 return 0;
1805
1806         /* Recevice ring buffer size is needed to be set based on MTU. If MTU is
1807          * changed then reinitilisation of the receive ring buffers need to be
1808          * done. Hence bring interface down and bring interface back up
1809          */
1810         sxgbe_release(dev);
1811         return sxgbe_open(dev);
1812 }
1813
1814 static void sxgbe_set_umac_addr(void __iomem *ioaddr, unsigned char *addr,
1815                                 unsigned int reg_n)
1816 {
1817         unsigned long data;
1818
1819         data = (addr[5] << 8) | addr[4];
1820         /* For MAC Addr registers se have to set the Address Enable (AE)
1821          * bit that has no effect on the High Reg 0 where the bit 31 (MO)
1822          * is RO.
1823          */
1824         writel(data | SXGBE_HI_REG_AE, ioaddr + SXGBE_ADDR_HIGH(reg_n));
1825         data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
1826         writel(data, ioaddr + SXGBE_ADDR_LOW(reg_n));
1827 }
1828
1829 /**
1830  * sxgbe_set_rx_mode - entry point for setting different receive mode of
1831  * a device. unicast, multicast addressing
1832  * @dev : pointer to the device structure
1833  * Description:
1834  * This function is a driver entry point which gets called by the kernel
1835  * whenever different receive mode like unicast, multicast and promiscuous
1836  * must be enabled/disabled.
1837  * Return value:
1838  * void.
1839  */
1840 static void sxgbe_set_rx_mode(struct net_device *dev)
1841 {
1842         struct sxgbe_priv_data *priv = netdev_priv(dev);
1843         void __iomem *ioaddr = (void __iomem *)priv->ioaddr;
1844         unsigned int value = 0;
1845         u32 mc_filter[2];
1846         struct netdev_hw_addr *ha;
1847         int reg = 1;
1848
1849         netdev_dbg(dev, "%s: # mcasts %d, # unicast %d\n",
1850                    __func__, netdev_mc_count(dev), netdev_uc_count(dev));
1851
1852         if (dev->flags & IFF_PROMISC) {
1853                 value = SXGBE_FRAME_FILTER_PR;
1854
1855         } else if ((netdev_mc_count(dev) > SXGBE_HASH_TABLE_SIZE) ||
1856                    (dev->flags & IFF_ALLMULTI)) {
1857                 value = SXGBE_FRAME_FILTER_PM;  /* pass all multi */
1858                 writel(0xffffffff, ioaddr + SXGBE_HASH_HIGH);
1859                 writel(0xffffffff, ioaddr + SXGBE_HASH_LOW);
1860
1861         } else if (!netdev_mc_empty(dev)) {
1862                 /* Hash filter for multicast */
1863                 value = SXGBE_FRAME_FILTER_HMC;
1864
1865                 memset(mc_filter, 0, sizeof(mc_filter));
1866                 netdev_for_each_mc_addr(ha, dev) {
1867                         /* The upper 6 bits of the calculated CRC are used to
1868                          * index the contens of the hash table
1869                          */
1870                         int bit_nr = bitrev32(~crc32_le(~0, ha->addr, 6)) >> 26;
1871
1872                         /* The most significant bit determines the register to
1873                          * use (H/L) while the other 5 bits determine the bit
1874                          * within the register.
1875                          */
1876                         mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
1877                 }
1878                 writel(mc_filter[0], ioaddr + SXGBE_HASH_LOW);
1879                 writel(mc_filter[1], ioaddr + SXGBE_HASH_HIGH);
1880         }
1881
1882         /* Handle multiple unicast addresses (perfect filtering) */
1883         if (netdev_uc_count(dev) > SXGBE_MAX_PERFECT_ADDRESSES)
1884                 /* Switch to promiscuous mode if more than 16 addrs
1885                  * are required
1886                  */
1887                 value |= SXGBE_FRAME_FILTER_PR;
1888         else {
1889                 netdev_for_each_uc_addr(ha, dev) {
1890                         sxgbe_set_umac_addr(ioaddr, ha->addr, reg);
1891                         reg++;
1892                 }
1893         }
1894 #ifdef FRAME_FILTER_DEBUG
1895         /* Enable Receive all mode (to debug filtering_fail errors) */
1896         value |= SXGBE_FRAME_FILTER_RA;
1897 #endif
1898         writel(value, ioaddr + SXGBE_FRAME_FILTER);
1899
1900         netdev_dbg(dev, "Filter: 0x%08x\n\tHash: HI 0x%08x, LO 0x%08x\n",
1901                    readl(ioaddr + SXGBE_FRAME_FILTER),
1902                    readl(ioaddr + SXGBE_HASH_HIGH),
1903                    readl(ioaddr + SXGBE_HASH_LOW));
1904 }
1905
1906 #ifdef CONFIG_NET_POLL_CONTROLLER
1907 /**
1908  * sxgbe_poll_controller - entry point for polling receive by device
1909  * @dev : pointer to the device structure
1910  * Description:
1911  * This function is used by NETCONSOLE and other diagnostic tools
1912  * to allow network I/O with interrupts disabled.
1913  * Return value:
1914  * Void.
1915  */
1916 static void sxgbe_poll_controller(struct net_device *dev)
1917 {
1918         struct sxgbe_priv_data *priv = netdev_priv(dev);
1919
1920         disable_irq(priv->irq);
1921         sxgbe_rx_interrupt(priv->irq, dev);
1922         enable_irq(priv->irq);
1923 }
1924 #endif
1925
1926 /*  sxgbe_ioctl - Entry point for the Ioctl
1927  *  @dev: Device pointer.
1928  *  @rq: An IOCTL specefic structure, that can contain a pointer to
1929  *  a proprietary structure used to pass information to the driver.
1930  *  @cmd: IOCTL command
1931  *  Description:
1932  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
1933  */
1934 static int sxgbe_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1935 {
1936         int ret = -EOPNOTSUPP;
1937
1938         if (!netif_running(dev))
1939                 return -EINVAL;
1940
1941         switch (cmd) {
1942         case SIOCGMIIPHY:
1943         case SIOCGMIIREG:
1944         case SIOCSMIIREG:
1945                 if (!dev->phydev)
1946                         return -EINVAL;
1947                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
1948                 break;
1949         default:
1950                 break;
1951         }
1952
1953         return ret;
1954 }
1955
1956 static const struct net_device_ops sxgbe_netdev_ops = {
1957         .ndo_open               = sxgbe_open,
1958         .ndo_start_xmit         = sxgbe_xmit,
1959         .ndo_stop               = sxgbe_release,
1960         .ndo_get_stats64        = sxgbe_get_stats64,
1961         .ndo_change_mtu         = sxgbe_change_mtu,
1962         .ndo_set_features       = sxgbe_set_features,
1963         .ndo_set_rx_mode        = sxgbe_set_rx_mode,
1964         .ndo_tx_timeout         = sxgbe_tx_timeout,
1965         .ndo_do_ioctl           = sxgbe_ioctl,
1966 #ifdef CONFIG_NET_POLL_CONTROLLER
1967         .ndo_poll_controller    = sxgbe_poll_controller,
1968 #endif
1969         .ndo_set_mac_address    = eth_mac_addr,
1970 };
1971
1972 /* Get the hardware ops */
1973 static void sxgbe_get_ops(struct sxgbe_ops * const ops_ptr)
1974 {
1975         ops_ptr->mac            = sxgbe_get_core_ops();
1976         ops_ptr->desc           = sxgbe_get_desc_ops();
1977         ops_ptr->dma            = sxgbe_get_dma_ops();
1978         ops_ptr->mtl            = sxgbe_get_mtl_ops();
1979
1980         /* set the MDIO communication Address/Data regisers */
1981         ops_ptr->mii.addr       = SXGBE_MDIO_SCMD_ADD_REG;
1982         ops_ptr->mii.data       = SXGBE_MDIO_SCMD_DATA_REG;
1983
1984         /* Assigning the default link settings
1985          * no SXGBE defined default values to be set in registers,
1986          * so assigning as 0 for port and duplex
1987          */
1988         ops_ptr->link.port      = 0;
1989         ops_ptr->link.duplex    = 0;
1990         ops_ptr->link.speed     = SXGBE_SPEED_10G;
1991 }
1992
1993 /**
1994  *  sxgbe_hw_init - Init the GMAC device
1995  *  @priv: driver private structure
1996  *  Description: this function checks the HW capability
1997  *  (if supported) and sets the driver's features.
1998  */
1999 static int sxgbe_hw_init(struct sxgbe_priv_data * const priv)
2000 {
2001         u32 ctrl_ids;
2002
2003         priv->hw = kmalloc(sizeof(*priv->hw), GFP_KERNEL);
2004         if(!priv->hw)
2005                 return -ENOMEM;
2006
2007         /* get the hardware ops */
2008         sxgbe_get_ops(priv->hw);
2009
2010         /* get the controller id */
2011         ctrl_ids = priv->hw->mac->get_controller_version(priv->ioaddr);
2012         priv->hw->ctrl_uid = (ctrl_ids & 0x00ff0000) >> 16;
2013         priv->hw->ctrl_id = (ctrl_ids & 0x000000ff);
2014         pr_info("user ID: 0x%x, Controller ID: 0x%x\n",
2015                 priv->hw->ctrl_uid, priv->hw->ctrl_id);
2016
2017         /* get the H/W features */
2018         if (!sxgbe_get_hw_features(priv))
2019                 pr_info("Hardware features not found\n");
2020
2021         if (priv->hw_cap.tx_csum_offload)
2022                 pr_info("TX Checksum offload supported\n");
2023
2024         if (priv->hw_cap.rx_csum_offload)
2025                 pr_info("RX Checksum offload supported\n");
2026
2027         return 0;
2028 }
2029
2030 static int sxgbe_sw_reset(void __iomem *addr)
2031 {
2032         int retry_count = 10;
2033
2034         writel(SXGBE_DMA_SOFT_RESET, addr + SXGBE_DMA_MODE_REG);
2035         while (retry_count--) {
2036                 if (!(readl(addr + SXGBE_DMA_MODE_REG) &
2037                       SXGBE_DMA_SOFT_RESET))
2038                         break;
2039                 mdelay(10);
2040         }
2041
2042         if (retry_count < 0)
2043                 return -EBUSY;
2044
2045         return 0;
2046 }
2047
2048 /**
2049  * sxgbe_drv_probe
2050  * @device: device pointer
2051  * @plat_dat: platform data pointer
2052  * @addr: iobase memory address
2053  * Description: this is the main probe function used to
2054  * call the alloc_etherdev, allocate the priv structure.
2055  */
2056 struct sxgbe_priv_data *sxgbe_drv_probe(struct device *device,
2057                                         struct sxgbe_plat_data *plat_dat,
2058                                         void __iomem *addr)
2059 {
2060         struct sxgbe_priv_data *priv;
2061         struct net_device *ndev;
2062         int ret;
2063         u8 queue_num;
2064
2065         ndev = alloc_etherdev_mqs(sizeof(struct sxgbe_priv_data),
2066                                   SXGBE_TX_QUEUES, SXGBE_RX_QUEUES);
2067         if (!ndev)
2068                 return NULL;
2069
2070         SET_NETDEV_DEV(ndev, device);
2071
2072         priv = netdev_priv(ndev);
2073         priv->device = device;
2074         priv->dev = ndev;
2075
2076         sxgbe_set_ethtool_ops(ndev);
2077         priv->plat = plat_dat;
2078         priv->ioaddr = addr;
2079
2080         ret = sxgbe_sw_reset(priv->ioaddr);
2081         if (ret)
2082                 goto error_free_netdev;
2083
2084         /* Verify driver arguments */
2085         sxgbe_verify_args();
2086
2087         /* Init MAC and get the capabilities */
2088         ret = sxgbe_hw_init(priv);
2089         if (ret)
2090                 goto error_free_netdev;
2091
2092         /* allocate memory resources for Descriptor rings */
2093         ret = txring_mem_alloc(priv);
2094         if (ret)
2095                 goto error_free_hw;
2096
2097         ret = rxring_mem_alloc(priv);
2098         if (ret)
2099                 goto error_free_hw;
2100
2101         ndev->netdev_ops = &sxgbe_netdev_ops;
2102
2103         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2104                 NETIF_F_RXCSUM | NETIF_F_TSO | NETIF_F_TSO6 |
2105                 NETIF_F_GRO;
2106         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
2107         ndev->watchdog_timeo = msecs_to_jiffies(TX_TIMEO);
2108
2109         /* assign filtering support */
2110         ndev->priv_flags |= IFF_UNICAST_FLT;
2111
2112         /* MTU range: 68 - 9000 */
2113         ndev->min_mtu = MIN_MTU;
2114         ndev->max_mtu = MAX_MTU;
2115
2116         priv->msg_enable = netif_msg_init(debug, default_msg_level);
2117
2118         /* Enable TCP segmentation offload for all DMA channels */
2119         if (priv->hw_cap.tcpseg_offload) {
2120                 SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
2121                         priv->hw->dma->enable_tso(priv->ioaddr, queue_num);
2122                 }
2123         }
2124
2125         /* Enable Rx checksum offload */
2126         if (priv->hw_cap.rx_csum_offload) {
2127                 priv->hw->mac->enable_rx_csum(priv->ioaddr);
2128                 priv->rxcsum_insertion = true;
2129         }
2130
2131         /* Initialise pause frame settings */
2132         priv->rx_pause = 1;
2133         priv->tx_pause = 1;
2134
2135         /* Rx Watchdog is available, enable depend on platform data */
2136         if (!priv->plat->riwt_off) {
2137                 priv->use_riwt = 1;
2138                 pr_info("Enable RX Mitigation via HW Watchdog Timer\n");
2139         }
2140
2141         netif_napi_add(ndev, &priv->napi, sxgbe_poll, 64);
2142
2143         spin_lock_init(&priv->stats_lock);
2144
2145         priv->sxgbe_clk = clk_get(priv->device, SXGBE_RESOURCE_NAME);
2146         if (IS_ERR(priv->sxgbe_clk)) {
2147                 netdev_warn(ndev, "%s: warning: cannot get CSR clock\n",
2148                             __func__);
2149                 goto error_napi_del;
2150         }
2151
2152         /* If a specific clk_csr value is passed from the platform
2153          * this means that the CSR Clock Range selection cannot be
2154          * changed at run-time and it is fixed. Viceversa the driver'll try to
2155          * set the MDC clock dynamically according to the csr actual
2156          * clock input.
2157          */
2158         if (!priv->plat->clk_csr)
2159                 sxgbe_clk_csr_set(priv);
2160         else
2161                 priv->clk_csr = priv->plat->clk_csr;
2162
2163         /* MDIO bus Registration */
2164         ret = sxgbe_mdio_register(ndev);
2165         if (ret < 0) {
2166                 netdev_dbg(ndev, "%s: MDIO bus (id: %d) registration failed\n",
2167                            __func__, priv->plat->bus_id);
2168                 goto error_clk_put;
2169         }
2170
2171         ret = register_netdev(ndev);
2172         if (ret) {
2173                 pr_err("%s: ERROR %i registering the device\n", __func__, ret);
2174                 goto error_mdio_unregister;
2175         }
2176
2177         sxgbe_check_ether_addr(priv);
2178
2179         return priv;
2180
2181 error_mdio_unregister:
2182         sxgbe_mdio_unregister(ndev);
2183 error_clk_put:
2184         clk_put(priv->sxgbe_clk);
2185 error_napi_del:
2186         netif_napi_del(&priv->napi);
2187 error_free_hw:
2188         kfree(priv->hw);
2189 error_free_netdev:
2190         free_netdev(ndev);
2191
2192         return NULL;
2193 }
2194
2195 /**
2196  * sxgbe_drv_remove
2197  * @ndev: net device pointer
2198  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
2199  * changes the link status, releases the DMA descriptor rings.
2200  */
2201 int sxgbe_drv_remove(struct net_device *ndev)
2202 {
2203         struct sxgbe_priv_data *priv = netdev_priv(ndev);
2204         u8 queue_num;
2205
2206         netdev_info(ndev, "%s: removing driver\n", __func__);
2207
2208         SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
2209                 priv->hw->mac->disable_rxqueue(priv->ioaddr, queue_num);
2210         }
2211
2212         priv->hw->dma->stop_rx(priv->ioaddr, SXGBE_RX_QUEUES);
2213         priv->hw->dma->stop_tx(priv->ioaddr, SXGBE_TX_QUEUES);
2214
2215         priv->hw->mac->enable_tx(priv->ioaddr, false);
2216         priv->hw->mac->enable_rx(priv->ioaddr, false);
2217
2218         unregister_netdev(ndev);
2219
2220         sxgbe_mdio_unregister(ndev);
2221
2222         clk_put(priv->sxgbe_clk);
2223
2224         netif_napi_del(&priv->napi);
2225
2226         kfree(priv->hw);
2227
2228         free_netdev(ndev);
2229
2230         return 0;
2231 }
2232
2233 #ifdef CONFIG_PM
2234 int sxgbe_suspend(struct net_device *ndev)
2235 {
2236         return 0;
2237 }
2238
2239 int sxgbe_resume(struct net_device *ndev)
2240 {
2241         return 0;
2242 }
2243
2244 int sxgbe_freeze(struct net_device *ndev)
2245 {
2246         return -ENOSYS;
2247 }
2248
2249 int sxgbe_restore(struct net_device *ndev)
2250 {
2251         return -ENOSYS;
2252 }
2253 #endif /* CONFIG_PM */
2254
2255 /* Driver is configured as Platform driver */
2256 static int __init sxgbe_init(void)
2257 {
2258         int ret;
2259
2260         ret = sxgbe_register_platform();
2261         if (ret)
2262                 goto err;
2263         return 0;
2264 err:
2265         pr_err("driver registration failed\n");
2266         return ret;
2267 }
2268
2269 static void __exit sxgbe_exit(void)
2270 {
2271         sxgbe_unregister_platform();
2272 }
2273
2274 module_init(sxgbe_init);
2275 module_exit(sxgbe_exit);
2276
2277 #ifndef MODULE
2278 static int __init sxgbe_cmdline_opt(char *str)
2279 {
2280         char *opt;
2281
2282         if (!str || !*str)
2283                 return -EINVAL;
2284         while ((opt = strsep(&str, ",")) != NULL) {
2285                 if (!strncmp(opt, "eee_timer:", 10)) {
2286                         if (kstrtoint(opt + 10, 0, &eee_timer))
2287                                 goto err;
2288                 }
2289         }
2290         return 0;
2291
2292 err:
2293         pr_err("%s: ERROR broken module parameter conversion\n", __func__);
2294         return -EINVAL;
2295 }
2296
2297 __setup("sxgbeeth=", sxgbe_cmdline_opt);
2298 #endif /* MODULE */
2299
2300
2301
2302 MODULE_DESCRIPTION("SAMSUNG 10G/2.5G/1G Ethernet PLATFORM driver");
2303
2304 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
2305 MODULE_PARM_DESC(eee_timer, "EEE-LPI Default LS timer value");
2306
2307 MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@samsung.com>");
2308 MODULE_AUTHOR("ByungHo An <bh74.an@samsung.com>");
2309 MODULE_AUTHOR("Girish K S <ks.giri@samsung.com>");
2310 MODULE_AUTHOR("Vipul Pandya <vipul.pandya@samsung.com>");
2311
2312 MODULE_LICENSE("GPL");