drivers/net/ethernet/ti/cpsw.c

   1 /*
   2  * Texas Instruments Ethernet Switch Driver
   3  *
   4  * Copyright (C) 2012 Texas Instruments
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation version 2.
   9  *
  10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
  11  * kind, whether express or implied; without even the implied warranty
  12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/io.h>
  18 #include <linux/clk.h>
  19 #include <linux/timer.h>
  20 #include <linux/module.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/irqreturn.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/if_ether.h>
  25 #include <linux/etherdevice.h>
  26 #include <linux/netdevice.h>
  27 #include <linux/net_tstamp.h>
  28 #include <linux/phy.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/delay.h>
  31 #include <linux/pm_runtime.h>
  32 #include <linux/gpio.h>
  33 #include <linux/of.h>
  34 #include <linux/of_mdio.h>
  35 #include <linux/of_net.h>
  36 #include <linux/of_device.h>
  37 #include <linux/if_vlan.h>
  38
  39 #include <linux/pinctrl/consumer.h>
  40
  41 #include "cpsw.h"
  42 #include "cpsw_ale.h"
  43 #include "cpts.h"
  44 #include "davinci_cpdma.h"
  45
  46 #define CPSW_DEBUG      (NETIF_MSG_HW           | NETIF_MSG_WOL         | \
  47                          NETIF_MSG_DRV          | NETIF_MSG_LINK        | \
  48                          NETIF_MSG_IFUP         | NETIF_MSG_INTR        | \
  49                          NETIF_MSG_PROBE        | NETIF_MSG_TIMER       | \
  50                          NETIF_MSG_IFDOWN       | NETIF_MSG_RX_ERR      | \
  51                          NETIF_MSG_TX_ERR       | NETIF_MSG_TX_DONE     | \
  52                          NETIF_MSG_PKTDATA      | NETIF_MSG_TX_QUEUED   | \
  53                          NETIF_MSG_RX_STATUS)
  54
  55 #define cpsw_info(priv, type, format, ...)              \
  56 do {                                                            \
  57         if (netif_msg_##type(priv) && net_ratelimit())          \
  58                 dev_info(priv->dev, format, ## __VA_ARGS__);    \
  59 } while (0)
  60
  61 #define cpsw_err(priv, type, format, ...)               \
  62 do {                                                            \
  63         if (netif_msg_##type(priv) && net_ratelimit())          \
  64                 dev_err(priv->dev, format, ## __VA_ARGS__);     \
  65 } while (0)
  66
  67 #define cpsw_dbg(priv, type, format, ...)               \
  68 do {                                                            \
  69         if (netif_msg_##type(priv) && net_ratelimit())          \
  70                 dev_dbg(priv->dev, format, ## __VA_ARGS__);     \
  71 } while (0)
  72
  73 #define cpsw_notice(priv, type, format, ...)            \
  74 do {                                                            \
  75         if (netif_msg_##type(priv) && net_ratelimit())          \
  76                 dev_notice(priv->dev, format, ## __VA_ARGS__);  \
  77 } while (0)
  78
  79 #define ALE_ALL_PORTS           0x7
  80
  81 #define CPSW_MAJOR_VERSION(reg)         (reg >> 8 & 0x7)
  82 #define CPSW_MINOR_VERSION(reg)         (reg & 0xff)
  83 #define CPSW_RTL_VERSION(reg)           ((reg >> 11) & 0x1f)
  84
  85 #define CPSW_VERSION_1          0x19010a
  86 #define CPSW_VERSION_2          0x19010c
  87 #define CPSW_VERSION_3          0x19010f
  88 #define CPSW_VERSION_4          0x190112
  89
  90 #define HOST_PORT_NUM           0
  91 #define SLIVER_SIZE             0x40
  92
  93 #define CPSW1_HOST_PORT_OFFSET  0x028
  94 #define CPSW1_SLAVE_OFFSET      0x050
  95 #define CPSW1_SLAVE_SIZE        0x040
  96 #define CPSW1_CPDMA_OFFSET      0x100
  97 #define CPSW1_STATERAM_OFFSET   0x200
  98 #define CPSW1_HW_STATS          0x400
  99 #define CPSW1_CPTS_OFFSET       0x500
 100 #define CPSW1_ALE_OFFSET        0x600
 101 #define CPSW1_SLIVER_OFFSET     0x700
 102
 103 #define CPSW2_HOST_PORT_OFFSET  0x108
 104 #define CPSW2_SLAVE_OFFSET      0x200
 105 #define CPSW2_SLAVE_SIZE        0x100
 106 #define CPSW2_CPDMA_OFFSET      0x800
 107 #define CPSW2_HW_STATS          0x900
 108 #define CPSW2_STATERAM_OFFSET   0xa00
 109 #define CPSW2_CPTS_OFFSET       0xc00
 110 #define CPSW2_ALE_OFFSET        0xd00
 111 #define CPSW2_SLIVER_OFFSET     0xd80
 112 #define CPSW2_BD_OFFSET         0x2000
 113
 114 #define CPDMA_RXTHRESH          0x0c0
 115 #define CPDMA_RXFREE            0x0e0
 116 #define CPDMA_TXHDP             0x00
 117 #define CPDMA_RXHDP             0x20
 118 #define CPDMA_TXCP              0x40
 119 #define CPDMA_RXCP              0x60
 120
 121 #define CPSW_POLL_WEIGHT        64
 122 #define CPSW_MIN_PACKET_SIZE    60
 123 #define CPSW_MAX_PACKET_SIZE    (1500 + 14 + 4 + 4)
 124
 125 #define RX_PRIORITY_MAPPING     0x76543210
 126 #define TX_PRIORITY_MAPPING     0x33221100
 127 #define CPDMA_TX_PRIORITY_MAP   0x76543210
 128
 129 #define CPSW_VLAN_AWARE         BIT(1)
 130 #define CPSW_ALE_VLAN_AWARE     1
 131
 132 #define CPSW_FIFO_NORMAL_MODE           (0 << 16)
 133 #define CPSW_FIFO_DUAL_MAC_MODE         (1 << 16)
 134 #define CPSW_FIFO_RATE_LIMIT_MODE       (2 << 16)
 135
 136 #define CPSW_INTPACEEN          (0x3f << 16)
 137 #define CPSW_INTPRESCALE_MASK   (0x7FF << 0)
 138 #define CPSW_CMINTMAX_CNT       63
 139 #define CPSW_CMINTMIN_CNT       2
 140 #define CPSW_CMINTMAX_INTVL     (1000 / CPSW_CMINTMIN_CNT)
 141 #define CPSW_CMINTMIN_INTVL     ((1000 / CPSW_CMINTMAX_CNT) + 1)
 142
 143 #define cpsw_slave_index(cpsw, priv)                            \
 144                 ((cpsw->data.dual_emac) ? priv->emac_port :     \
 145                 cpsw->data.active_slave)
 146 #define IRQ_NUM                 2
 147 #define CPSW_MAX_QUEUES         8
 148
 149 static int debug_level;
 150 module_param(debug_level, int, 0);
 151 MODULE_PARM_DESC(debug_level, "cpsw debug level (NETIF_MSG bits)");
 152
 153 static int ale_ageout = 10;
 154 module_param(ale_ageout, int, 0);
 155 MODULE_PARM_DESC(ale_ageout, "cpsw ale ageout interval (seconds)");
 156
 157 static int rx_packet_max = CPSW_MAX_PACKET_SIZE;
 158 module_param(rx_packet_max, int, 0);
 159 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
 160
 161 struct cpsw_wr_regs {
 162         u32     id_ver;
 163         u32     soft_reset;
 164         u32     control;
 165         u32     int_control;
 166         u32     rx_thresh_en;
 167         u32     rx_en;
 168         u32     tx_en;
 169         u32     misc_en;
 170         u32     mem_allign1[8];
 171         u32     rx_thresh_stat;
 172         u32     rx_stat;
 173         u32     tx_stat;
 174         u32     misc_stat;
 175         u32     mem_allign2[8];
 176         u32     rx_imax;
 177         u32     tx_imax;
 178
 179 };
 180
 181 struct cpsw_ss_regs {
 182         u32     id_ver;
 183         u32     control;
 184         u32     soft_reset;
 185         u32     stat_port_en;
 186         u32     ptype;
 187         u32     soft_idle;
 188         u32     thru_rate;
 189         u32     gap_thresh;
 190         u32     tx_start_wds;
 191         u32     flow_control;
 192         u32     vlan_ltype;
 193         u32     ts_ltype;
 194         u32     dlr_ltype;
 195 };
 196
 197 /* CPSW_PORT_V1 */
 198 #define CPSW1_MAX_BLKS      0x00 /* Maximum FIFO Blocks */
 199 #define CPSW1_BLK_CNT       0x04 /* FIFO Block Usage Count (Read Only) */
 200 #define CPSW1_TX_IN_CTL     0x08 /* Transmit FIFO Control */
 201 #define CPSW1_PORT_VLAN     0x0c /* VLAN Register */
 202 #define CPSW1_TX_PRI_MAP    0x10 /* Tx Header Priority to Switch Pri Mapping */
 203 #define CPSW1_TS_CTL        0x14 /* Time Sync Control */
 204 #define CPSW1_TS_SEQ_LTYPE  0x18 /* Time Sync Sequence ID Offset and Msg Type */
 205 #define CPSW1_TS_VLAN       0x1c /* Time Sync VLAN1 and VLAN2 */
 206
 207 /* CPSW_PORT_V2 */
 208 #define CPSW2_CONTROL       0x00 /* Control Register */
 209 #define CPSW2_MAX_BLKS      0x08 /* Maximum FIFO Blocks */
 210 #define CPSW2_BLK_CNT       0x0c /* FIFO Block Usage Count (Read Only) */
 211 #define CPSW2_TX_IN_CTL     0x10 /* Transmit FIFO Control */
 212 #define CPSW2_PORT_VLAN     0x14 /* VLAN Register */
 213 #define CPSW2_TX_PRI_MAP    0x18 /* Tx Header Priority to Switch Pri Mapping */
 214 #define CPSW2_TS_SEQ_MTYPE  0x1c /* Time Sync Sequence ID Offset and Msg Type */
 215
 216 /* CPSW_PORT_V1 and V2 */
 217 #define SA_LO               0x20 /* CPGMAC_SL Source Address Low */
 218 #define SA_HI               0x24 /* CPGMAC_SL Source Address High */
 219 #define SEND_PERCENT        0x28 /* Transmit Queue Send Percentages */
 220
 221 /* CPSW_PORT_V2 only */
 222 #define RX_DSCP_PRI_MAP0    0x30 /* Rx DSCP Priority to Rx Packet Mapping */
 223 #define RX_DSCP_PRI_MAP1    0x34 /* Rx DSCP Priority to Rx Packet Mapping */
 224 #define RX_DSCP_PRI_MAP2    0x38 /* Rx DSCP Priority to Rx Packet Mapping */
 225 #define RX_DSCP_PRI_MAP3    0x3c /* Rx DSCP Priority to Rx Packet Mapping */
 226 #define RX_DSCP_PRI_MAP4    0x40 /* Rx DSCP Priority to Rx Packet Mapping */
 227 #define RX_DSCP_PRI_MAP5    0x44 /* Rx DSCP Priority to Rx Packet Mapping */
 228 #define RX_DSCP_PRI_MAP6    0x48 /* Rx DSCP Priority to Rx Packet Mapping */
 229 #define RX_DSCP_PRI_MAP7    0x4c /* Rx DSCP Priority to Rx Packet Mapping */
 230
 231 /* Bit definitions for the CPSW2_CONTROL register */
 232 #define PASS_PRI_TAGGED     (1<<24) /* Pass Priority Tagged */
 233 #define VLAN_LTYPE2_EN      (1<<21) /* VLAN LTYPE 2 enable */
 234 #define VLAN_LTYPE1_EN      (1<<20) /* VLAN LTYPE 1 enable */
 235 #define DSCP_PRI_EN         (1<<16) /* DSCP Priority Enable */
 236 #define TS_320              (1<<14) /* Time Sync Dest Port 320 enable */
 237 #define TS_319              (1<<13) /* Time Sync Dest Port 319 enable */
 238 #define TS_132              (1<<12) /* Time Sync Dest IP Addr 132 enable */
 239 #define TS_131              (1<<11) /* Time Sync Dest IP Addr 131 enable */
 240 #define TS_130              (1<<10) /* Time Sync Dest IP Addr 130 enable */
 241 #define TS_129              (1<<9)  /* Time Sync Dest IP Addr 129 enable */
 242 #define TS_TTL_NONZERO      (1<<8)  /* Time Sync Time To Live Non-zero enable */
 243 #define TS_ANNEX_F_EN       (1<<6)  /* Time Sync Annex F enable */
 244 #define TS_ANNEX_D_EN       (1<<4)  /* Time Sync Annex D enable */
 245 #define TS_LTYPE2_EN        (1<<3)  /* Time Sync LTYPE 2 enable */
 246 #define TS_LTYPE1_EN        (1<<2)  /* Time Sync LTYPE 1 enable */
 247 #define TS_TX_EN            (1<<1)  /* Time Sync Transmit Enable */
 248 #define TS_RX_EN            (1<<0)  /* Time Sync Receive Enable */
 249
 250 #define CTRL_V2_TS_BITS \
 251         (TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
 252          TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
 253
 254 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
 255 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
 256 #define CTRL_V2_RX_TS_BITS  (CTRL_V2_TS_BITS | TS_RX_EN)
 257
 258
 259 #define CTRL_V3_TS_BITS \
 260         (TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
 261          TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
 262          TS_LTYPE1_EN)
 263
 264 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
 265 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
 266 #define CTRL_V3_RX_TS_BITS  (CTRL_V3_TS_BITS | TS_RX_EN)
 267
 268 /* Bit definitions for the CPSW2_TS_SEQ_MTYPE register */
 269 #define TS_SEQ_ID_OFFSET_SHIFT   (16)    /* Time Sync Sequence ID Offset */
 270 #define TS_SEQ_ID_OFFSET_MASK    (0x3f)
 271 #define TS_MSG_TYPE_EN_SHIFT     (0)     /* Time Sync Message Type Enable */
 272 #define TS_MSG_TYPE_EN_MASK      (0xffff)
 273
 274 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
 275 #define EVENT_MSG_BITS ((1<<0) | (1<<1) | (1<<2) | (1<<3))
 276
 277 /* Bit definitions for the CPSW1_TS_CTL register */
 278 #define CPSW_V1_TS_RX_EN                BIT(0)
 279 #define CPSW_V1_TS_TX_EN                BIT(4)
 280 #define CPSW_V1_MSG_TYPE_OFS            16
 281
 282 /* Bit definitions for the CPSW1_TS_SEQ_LTYPE register */
 283 #define CPSW_V1_SEQ_ID_OFS_SHIFT        16
 284
 285 #define CPSW_MAX_BLKS_TX                15
 286 #define CPSW_MAX_BLKS_TX_SHIFT          4
 287 #define CPSW_MAX_BLKS_RX                5
 288
 289 struct cpsw_host_regs {
 290         u32     max_blks;
 291         u32     blk_cnt;
 292         u32     tx_in_ctl;
 293         u32     port_vlan;
 294         u32     tx_pri_map;
 295         u32     cpdma_tx_pri_map;
 296         u32     cpdma_rx_chan_map;
 297 };
 298
 299 struct cpsw_sliver_regs {
 300         u32     id_ver;
 301         u32     mac_control;
 302         u32     mac_status;
 303         u32     soft_reset;
 304         u32     rx_maxlen;
 305         u32     __reserved_0;
 306         u32     rx_pause;
 307         u32     tx_pause;
 308         u32     __reserved_1;
 309         u32     rx_pri_map;
 310 };
 311
 312 struct cpsw_hw_stats {
 313         u32     rxgoodframes;
 314         u32     rxbroadcastframes;
 315         u32     rxmulticastframes;
 316         u32     rxpauseframes;
 317         u32     rxcrcerrors;
 318         u32     rxaligncodeerrors;
 319         u32     rxoversizedframes;
 320         u32     rxjabberframes;
 321         u32     rxundersizedframes;
 322         u32     rxfragments;
 323         u32     __pad_0[2];
 324         u32     rxoctets;
 325         u32     txgoodframes;
 326         u32     txbroadcastframes;
 327         u32     txmulticastframes;
 328         u32     txpauseframes;
 329         u32     txdeferredframes;
 330         u32     txcollisionframes;
 331         u32     txsinglecollframes;
 332         u32     txmultcollframes;
 333         u32     txexcessivecollisions;
 334         u32     txlatecollisions;
 335         u32     txunderrun;
 336         u32     txcarriersenseerrors;
 337         u32     txoctets;
 338         u32     octetframes64;
 339         u32     octetframes65t127;
 340         u32     octetframes128t255;
 341         u32     octetframes256t511;
 342         u32     octetframes512t1023;
 343         u32     octetframes1024tup;
 344         u32     netoctets;
 345         u32     rxsofoverruns;
 346         u32     rxmofoverruns;
 347         u32     rxdmaoverruns;
 348 };
 349
 350 struct cpsw_slave {
 351         void __iomem                    *regs;
 352         struct cpsw_sliver_regs __iomem *sliver;
 353         int                             slave_num;
 354         u32                             mac_control;
 355         struct cpsw_slave_data          *data;
 356         struct phy_device               *phy;
 357         struct net_device               *ndev;
 358         u32                             port_vlan;
 359         u32                             open_stat;
 360 };
 361
 362 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
 363 {
 364         return __raw_readl(slave->regs + offset);
 365 }
 366
 367 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
 368 {
 369         __raw_writel(val, slave->regs + offset);
 370 }
 371
 372 struct cpsw_common {
 373         struct device                   *dev;
 374         struct cpsw_platform_data       data;
 375         struct napi_struct              napi_rx;
 376         struct napi_struct              napi_tx;
 377         struct cpsw_ss_regs __iomem     *regs;
 378         struct cpsw_wr_regs __iomem     *wr_regs;
 379         u8 __iomem                      *hw_stats;
 380         struct cpsw_host_regs __iomem   *host_port_regs;
 381         u32                             version;
 382         u32                             coal_intvl;
 383         u32                             bus_freq_mhz;
 384         int                             rx_packet_max;
 385         struct cpsw_slave               *slaves;
 386         struct cpdma_ctlr               *dma;
 387         struct cpdma_chan               *txch[CPSW_MAX_QUEUES];
 388         struct cpdma_chan               *rxch[CPSW_MAX_QUEUES];
 389         struct cpsw_ale                 *ale;
 390         bool                            quirk_irq;
 391         bool                            rx_irq_disabled;
 392         bool                            tx_irq_disabled;
 393         u32 irqs_table[IRQ_NUM];
 394         struct cpts                     *cpts;
 395         int                             rx_ch_num, tx_ch_num;
 396 };
 397
 398 struct cpsw_priv {
 399         struct net_device               *ndev;
 400         struct device                   *dev;
 401         u32                             msg_enable;
 402         u8                              mac_addr[ETH_ALEN];
 403         bool                            rx_pause;
 404         bool                            tx_pause;
 405         u32 emac_port;
 406         struct cpsw_common *cpsw;
 407 };
 408
 409 struct cpsw_stats {
 410         char stat_string[ETH_GSTRING_LEN];
 411         int type;
 412         int sizeof_stat;
 413         int stat_offset;
 414 };
 415
 416 enum {
 417         CPSW_STATS,
 418         CPDMA_RX_STATS,
 419         CPDMA_TX_STATS,
 420 };
 421
 422 #define CPSW_STAT(m)            CPSW_STATS,                             \
 423                                 sizeof(((struct cpsw_hw_stats *)0)->m), \
 424                                 offsetof(struct cpsw_hw_stats, m)
 425 #define CPDMA_RX_STAT(m)        CPDMA_RX_STATS,                            \
 426                                 sizeof(((struct cpdma_chan_stats *)0)->m), \
 427                                 offsetof(struct cpdma_chan_stats, m)
 428 #define CPDMA_TX_STAT(m)        CPDMA_TX_STATS,                            \
 429                                 sizeof(((struct cpdma_chan_stats *)0)->m), \
 430                                 offsetof(struct cpdma_chan_stats, m)
 431
 432 static const struct cpsw_stats cpsw_gstrings_stats[] = {
 433         { "Good Rx Frames", CPSW_STAT(rxgoodframes) },
 434         { "Broadcast Rx Frames", CPSW_STAT(rxbroadcastframes) },
 435         { "Multicast Rx Frames", CPSW_STAT(rxmulticastframes) },
 436         { "Pause Rx Frames", CPSW_STAT(rxpauseframes) },
 437         { "Rx CRC Errors", CPSW_STAT(rxcrcerrors) },
 438         { "Rx Align/Code Errors", CPSW_STAT(rxaligncodeerrors) },
 439         { "Oversize Rx Frames", CPSW_STAT(rxoversizedframes) },
 440         { "Rx Jabbers", CPSW_STAT(rxjabberframes) },
 441         { "Undersize (Short) Rx Frames", CPSW_STAT(rxundersizedframes) },
 442         { "Rx Fragments", CPSW_STAT(rxfragments) },
 443         { "Rx Octets", CPSW_STAT(rxoctets) },
 444         { "Good Tx Frames", CPSW_STAT(txgoodframes) },
 445         { "Broadcast Tx Frames", CPSW_STAT(txbroadcastframes) },
 446         { "Multicast Tx Frames", CPSW_STAT(txmulticastframes) },
 447         { "Pause Tx Frames", CPSW_STAT(txpauseframes) },
 448         { "Deferred Tx Frames", CPSW_STAT(txdeferredframes) },
 449         { "Collisions", CPSW_STAT(txcollisionframes) },
 450         { "Single Collision Tx Frames", CPSW_STAT(txsinglecollframes) },
 451         { "Multiple Collision Tx Frames", CPSW_STAT(txmultcollframes) },
 452         { "Excessive Collisions", CPSW_STAT(txexcessivecollisions) },
 453         { "Late Collisions", CPSW_STAT(txlatecollisions) },
 454         { "Tx Underrun", CPSW_STAT(txunderrun) },
 455         { "Carrier Sense Errors", CPSW_STAT(txcarriersenseerrors) },
 456         { "Tx Octets", CPSW_STAT(txoctets) },
 457         { "Rx + Tx 64 Octet Frames", CPSW_STAT(octetframes64) },
 458         { "Rx + Tx 65-127 Octet Frames", CPSW_STAT(octetframes65t127) },
 459         { "Rx + Tx 128-255 Octet Frames", CPSW_STAT(octetframes128t255) },
 460         { "Rx + Tx 256-511 Octet Frames", CPSW_STAT(octetframes256t511) },
 461         { "Rx + Tx 512-1023 Octet Frames", CPSW_STAT(octetframes512t1023) },
 462         { "Rx + Tx 1024-Up Octet Frames", CPSW_STAT(octetframes1024tup) },
 463         { "Net Octets", CPSW_STAT(netoctets) },
 464         { "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
 465         { "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
 466         { "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
 467 };
 468
 469 static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
 470         { "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
 471         { "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
 472         { "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
 473         { "misqueued", CPDMA_RX_STAT(misqueued) },
 474         { "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
 475         { "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
 476         { "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
 477         { "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
 478         { "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
 479         { "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
 480         { "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
 481         { "requeue", CPDMA_RX_STAT(requeue) },
 482         { "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
 483 };
 484
 485 #define CPSW_STATS_COMMON_LEN   ARRAY_SIZE(cpsw_gstrings_stats)
 486 #define CPSW_STATS_CH_LEN       ARRAY_SIZE(cpsw_gstrings_ch_stats)
 487
 488 #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
 489 #define napi_to_cpsw(napi)      container_of(napi, struct cpsw_common, napi)
 490 #define for_each_slave(priv, func, arg...)                              \
 491         do {                                                            \
 492                 struct cpsw_slave *slave;                               \
 493                 struct cpsw_common *cpsw = (priv)->cpsw;                \
 494                 int n;                                                  \
 495                 if (cpsw->data.dual_emac)                               \
 496                         (func)((cpsw)->slaves + priv->emac_port, ##arg);\
 497                 else                                                    \
 498                         for (n = cpsw->data.slaves,                     \
 499                                         slave = cpsw->slaves;           \
 500                                         n; n--)                         \
 501                                 (func)(slave++, ##arg);                 \
 502         } while (0)
 503
 504 #define cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb)         \
 505         do {                                                            \
 506                 if (!cpsw->data.dual_emac)                              \
 507                         break;                                          \
 508                 if (CPDMA_RX_SOURCE_PORT(status) == 1) {                \
 509                         ndev = cpsw->slaves[0].ndev;                    \
 510                         skb->dev = ndev;                                \
 511                 } else if (CPDMA_RX_SOURCE_PORT(status) == 2) {         \
 512                         ndev = cpsw->slaves[1].ndev;                    \
 513                         skb->dev = ndev;                                \
 514                 }                                                       \
 515         } while (0)
 516 #define cpsw_add_mcast(cpsw, priv, addr)                                \
 517         do {                                                            \
 518                 if (cpsw->data.dual_emac) {                             \
 519                         struct cpsw_slave *slave = cpsw->slaves +       \
 520                                                 priv->emac_port;        \
 521                         int slave_port = cpsw_get_slave_port(           \
 522                                                 slave->slave_num);      \
 523                         cpsw_ale_add_mcast(cpsw->ale, addr,             \
 524                                 1 << slave_port | ALE_PORT_HOST,        \
 525                                 ALE_VLAN, slave->port_vlan, 0);         \
 526                 } else {                                                \
 527                         cpsw_ale_add_mcast(cpsw->ale, addr,             \
 528                                 ALE_ALL_PORTS,                          \
 529                                 0, 0, 0);                               \
 530                 }                                                       \
 531         } while (0)
 532
 533 static inline int cpsw_get_slave_port(u32 slave_num)
 534 {
 535         return slave_num + 1;
 536 }
 537
 538 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
 539 {
 540         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 541         struct cpsw_ale *ale = cpsw->ale;
 542         int i;
 543
 544         if (cpsw->data.dual_emac) {
 545                 bool flag = false;
 546
 547                 /* Enabling promiscuous mode for one interface will be
 548                  * common for both the interface as the interface shares
 549                  * the same hardware resource.
 550                  */
 551                 for (i = 0; i < cpsw->data.slaves; i++)
 552                         if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
 553                                 flag = true;
 554
 555                 if (!enable && flag) {
 556                         enable = true;
 557                         dev_err(&ndev->dev, "promiscuity not disabled as the other interface is still in promiscuity mode\n");
 558                 }
 559
 560                 if (enable) {
 561                         /* Enable Bypass */
 562                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 1);
 563
 564                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
 565                 } else {
 566                         /* Disable Bypass */
 567                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 0);
 568                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
 569                 }
 570         } else {
 571                 if (enable) {
 572                         unsigned long timeout = jiffies + HZ;
 573
 574                         /* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
 575                         for (i = 0; i <= cpsw->data.slaves; i++) {
 576                                 cpsw_ale_control_set(ale, i,
 577                                                      ALE_PORT_NOLEARN, 1);
 578                                 cpsw_ale_control_set(ale, i,
 579                                                      ALE_PORT_NO_SA_UPDATE, 1);
 580                         }
 581
 582                         /* Clear All Untouched entries */
 583                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
 584                         do {
 585                                 cpu_relax();
 586                                 if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
 587                                         break;
 588                         } while (time_after(timeout, jiffies));
 589                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
 590
 591                         /* Clear all mcast from ALE */
 592                         cpsw_ale_flush_multicast(ale, ALE_ALL_PORTS, -1);
 593                         __dev_mc_unsync(ndev, NULL);
 594
 595                         /* Flood All Unicast Packets to Host port */
 596                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
 597                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
 598                 } else {
 599                         /* Don't Flood All Unicast Packets to Host port */
 600                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
 601
 602                         /* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
 603                         for (i = 0; i <= cpsw->data.slaves; i++) {
 604                                 cpsw_ale_control_set(ale, i,
 605                                                      ALE_PORT_NOLEARN, 0);
 606                                 cpsw_ale_control_set(ale, i,
 607                                                      ALE_PORT_NO_SA_UPDATE, 0);
 608                         }
 609                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
 610                 }
 611         }
 612 }
 613
 614 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
 615 {
 616         struct cpsw_priv *priv = netdev_priv(ndev);
 617         struct cpsw_common *cpsw = priv->cpsw;
 618         int vid;
 619
 620         if (cpsw->data.dual_emac)
 621                 vid = cpsw->slaves[priv->emac_port].port_vlan;
 622         else
 623                 vid = cpsw->data.default_vlan;
 624
 625         if (ndev->flags & IFF_PROMISC) {
 626                 /* Enable promiscuous mode */
 627                 cpsw_set_promiscious(ndev, true);
 628                 cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
 629                 return;
 630         } else {
 631                 /* Disable promiscuous mode */
 632                 cpsw_set_promiscious(ndev, false);
 633         }
 634
 635         /* Restore allmulti on vlans if necessary */
 636         cpsw_ale_set_allmulti(cpsw->ale, priv->ndev->flags & IFF_ALLMULTI);
 637
 638         /* Clear all mcast from ALE */
 639         cpsw_ale_flush_multicast(cpsw->ale, ALE_ALL_PORTS, vid);
 640
 641         if (!netdev_mc_empty(ndev)) {
 642                 struct netdev_hw_addr *ha;
 643
 644                 /* program multicast address list into ALE register */
 645                 netdev_for_each_mc_addr(ha, ndev) {
 646                         cpsw_add_mcast(cpsw, priv, (u8 *)ha->addr);
 647                 }
 648         }
 649 }
 650
 651 static void cpsw_intr_enable(struct cpsw_common *cpsw)
 652 {
 653         __raw_writel(0xFF, &cpsw->wr_regs->tx_en);
 654         __raw_writel(0xFF, &cpsw->wr_regs->rx_en);
 655
 656         cpdma_ctlr_int_ctrl(cpsw->dma, true);
 657         return;
 658 }
 659
 660 static void cpsw_intr_disable(struct cpsw_common *cpsw)
 661 {
 662         __raw_writel(0, &cpsw->wr_regs->tx_en);
 663         __raw_writel(0, &cpsw->wr_regs->rx_en);
 664
 665         cpdma_ctlr_int_ctrl(cpsw->dma, false);
 666         return;
 667 }
 668
 669 static void cpsw_tx_handler(void *token, int len, int status)
 670 {
 671         struct netdev_queue     *txq;
 672         struct sk_buff          *skb = token;
 673         struct net_device       *ndev = skb->dev;
 674         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
 675
 676         /* Check whether the queue is stopped due to stalled tx dma, if the
 677          * queue is stopped then start the queue as we have free desc for tx
 678          */
 679         txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
 680         if (unlikely(netif_tx_queue_stopped(txq)))
 681                 netif_tx_wake_queue(txq);
 682
 683         cpts_tx_timestamp(cpsw->cpts, skb);
 684         ndev->stats.tx_packets++;
 685         ndev->stats.tx_bytes += len;
 686         dev_kfree_skb_any(skb);
 687 }
 688
 689 static void cpsw_rx_handler(void *token, int len, int status)
 690 {
 691         struct cpdma_chan       *ch;
 692         struct sk_buff          *skb = token;
 693         struct sk_buff          *new_skb;
 694         struct net_device       *ndev = skb->dev;
 695         int                     ret = 0;
 696         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
 697
 698         cpsw_dual_emac_src_port_detect(cpsw, status, ndev, skb);
 699
 700         if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
 701                 bool ndev_status = false;
 702                 struct cpsw_slave *slave = cpsw->slaves;
 703                 int n;
 704
 705                 if (cpsw->data.dual_emac) {
 706                         /* In dual emac mode check for all interfaces */
 707                         for (n = cpsw->data.slaves; n; n--, slave++)
 708                                 if (netif_running(slave->ndev))
 709                                         ndev_status = true;
 710                 }
 711
 712                 if (ndev_status && (status >= 0)) {
 713                         /* The packet received is for the interface which
 714                          * is already down and the other interface is up
 715                          * and running, instead of freeing which results
 716                          * in reducing of the number of rx descriptor in
 717                          * DMA engine, requeue skb back to cpdma.
 718                          */
 719                         new_skb = skb;
 720                         goto requeue;
 721                 }
 722
 723                 /* the interface is going down, skbs are purged */
 724                 dev_kfree_skb_any(skb);
 725                 return;
 726         }
 727
 728         new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
 729         if (new_skb) {
 730                 skb_copy_queue_mapping(new_skb, skb);
 731                 skb_put(skb, len);
 732                 cpts_rx_timestamp(cpsw->cpts, skb);
 733                 skb->protocol = eth_type_trans(skb, ndev);
 734                 netif_receive_skb(skb);
 735                 ndev->stats.rx_bytes += len;
 736                 ndev->stats.rx_packets++;
 737                 kmemleak_not_leak(new_skb);
 738         } else {
 739                 ndev->stats.rx_dropped++;
 740                 new_skb = skb;
 741         }
 742
 743 requeue:
 744         if (netif_dormant(ndev)) {
 745                 dev_kfree_skb_any(new_skb);
 746                 return;
 747         }
 748
 749         ch = cpsw->rxch[skb_get_queue_mapping(new_skb)];
 750         ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
 751                                 skb_tailroom(new_skb), 0);
 752         if (WARN_ON(ret < 0))
 753                 dev_kfree_skb_any(new_skb);
 754 }
 755
 756 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
 757 {
 758         struct cpsw_common *cpsw = dev_id;
 759
 760         writel(0, &cpsw->wr_regs->tx_en);
 761         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
 762
 763         if (cpsw->quirk_irq) {
 764                 disable_irq_nosync(cpsw->irqs_table[1]);
 765                 cpsw->tx_irq_disabled = true;
 766         }
 767
 768         napi_schedule(&cpsw->napi_tx);
 769         return IRQ_HANDLED;
 770 }
 771
 772 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
 773 {
 774         struct cpsw_common *cpsw = dev_id;
 775
 776         writel(0, &cpsw->wr_regs->rx_en);
 777         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
 778
 779         if (cpsw->quirk_irq) {
 780                 disable_irq_nosync(cpsw->irqs_table[0]);
 781                 cpsw->rx_irq_disabled = true;
 782         }
 783
 784         napi_schedule(&cpsw->napi_rx);
 785         return IRQ_HANDLED;
 786 }
 787
 788 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
 789 {
 790         u32                     ch_map;
 791         int                     num_tx, ch;
 792         struct cpsw_common      *cpsw = napi_to_cpsw(napi_tx);
 793
 794         /* process every unprocessed channel */
 795         ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
 796         for (ch = 0, num_tx = 0; num_tx < budget; ch_map >>= 1, ch++) {
 797                 if (!ch_map) {
 798                         ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
 799                         if (!ch_map)
 800                                 break;
 801
 802                         ch = 0;
 803                 }
 804
 805                 if (!(ch_map & 0x01))
 806                         continue;
 807
 808                 num_tx += cpdma_chan_process(cpsw->txch[ch], budget - num_tx);
 809         }
 810
 811         if (num_tx < budget) {
 812                 napi_complete(napi_tx);
 813                 writel(0xff, &cpsw->wr_regs->tx_en);
 814                 if (cpsw->quirk_irq && cpsw->tx_irq_disabled) {
 815                         cpsw->tx_irq_disabled = false;
 816                         enable_irq(cpsw->irqs_table[1]);
 817                 }
 818         }
 819
 820         return num_tx;
 821 }
 822
 823 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
 824 {
 825         u32                     ch_map;
 826         int                     num_rx, ch;
 827         struct cpsw_common      *cpsw = napi_to_cpsw(napi_rx);
 828
 829         /* process every unprocessed channel */
 830         ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
 831         for (ch = 0, num_rx = 0; num_rx < budget; ch_map >>= 1, ch++) {
 832                 if (!ch_map) {
 833                         ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
 834                         if (!ch_map)
 835                                 break;
 836
 837                         ch = 0;
 838                 }
 839
 840                 if (!(ch_map & 0x01))
 841                         continue;
 842
 843                 num_rx += cpdma_chan_process(cpsw->rxch[ch], budget - num_rx);
 844         }
 845
 846         if (num_rx < budget) {
 847                 napi_complete(napi_rx);
 848                 writel(0xff, &cpsw->wr_regs->rx_en);
 849                 if (cpsw->quirk_irq && cpsw->rx_irq_disabled) {
 850                         cpsw->rx_irq_disabled = false;
 851                         enable_irq(cpsw->irqs_table[0]);
 852                 }
 853         }
 854
 855         return num_rx;
 856 }
 857
 858 static inline void soft_reset(const char *module, void __iomem *reg)
 859 {
 860         unsigned long timeout = jiffies + HZ;
 861
 862         __raw_writel(1, reg);
 863         do {
 864                 cpu_relax();
 865         } while ((__raw_readl(reg) & 1) && time_after(timeout, jiffies));
 866
 867         WARN(__raw_readl(reg) & 1, "failed to soft-reset %s\n", module);
 868 }
 869
 870 #define mac_hi(mac)     (((mac)[0] << 0) | ((mac)[1] << 8) |    \
 871                          ((mac)[2] << 16) | ((mac)[3] << 24))
 872 #define mac_lo(mac)     (((mac)[4] << 0) | ((mac)[5] << 8))
 873
 874 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
 875                                struct cpsw_priv *priv)
 876 {
 877         slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
 878         slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
 879 }
 880
 881 static void _cpsw_adjust_link(struct cpsw_slave *slave,
 882                               struct cpsw_priv *priv, bool *link)
 883 {
 884         struct phy_device       *phy = slave->phy;
 885         u32                     mac_control = 0;
 886         u32                     slave_port;
 887         struct cpsw_common *cpsw = priv->cpsw;
 888
 889         if (!phy)
 890                 return;
 891
 892         slave_port = cpsw_get_slave_port(slave->slave_num);
 893
 894         if (phy->link) {
 895                 mac_control = cpsw->data.mac_control;
 896
 897                 /* enable forwarding */
 898                 cpsw_ale_control_set(cpsw->ale, slave_port,
 899                                      ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
 900
 901                 if (phy->speed == 1000)
 902                         mac_control |= BIT(7);  /* GIGABITEN    */
 903                 if (phy->duplex)
 904                         mac_control |= BIT(0);  /* FULLDUPLEXEN */
 905
 906                 /* set speed_in input in case RMII mode is used in 100Mbps */
 907                 if (phy->speed == 100)
 908                         mac_control |= BIT(15);
 909                 /* in band mode only works in 10Mbps RGMII mode */
 910                 else if ((phy->speed == 10) && phy_interface_is_rgmii(phy))
 911                         mac_control |= BIT(18); /* In Band mode */
 912
 913                 if (priv->rx_pause)
 914                         mac_control |= BIT(3);
 915
 916                 if (priv->tx_pause)
 917                         mac_control |= BIT(4);
 918
 919                 *link = true;
 920         } else {
 921                 mac_control = 0;
 922                 /* disable forwarding */
 923                 cpsw_ale_control_set(cpsw->ale, slave_port,
 924                                      ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
 925         }
 926
 927         if (mac_control != slave->mac_control) {
 928                 phy_print_status(phy);
 929                 __raw_writel(mac_control, &slave->sliver->mac_control);
 930         }
 931
 932         slave->mac_control = mac_control;
 933 }
 934
 935 static void cpsw_adjust_link(struct net_device *ndev)
 936 {
 937         struct cpsw_priv        *priv = netdev_priv(ndev);
 938         bool                    link = false;
 939
 940         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
 941
 942         if (link) {
 943                 netif_carrier_on(ndev);
 944                 if (netif_running(ndev))
 945                         netif_tx_wake_all_queues(ndev);
 946         } else {
 947                 netif_carrier_off(ndev);
 948                 netif_tx_stop_all_queues(ndev);
 949         }
 950 }
 951
 952 static int cpsw_get_coalesce(struct net_device *ndev,
 953                                 struct ethtool_coalesce *coal)
 954 {
 955         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 956
 957         coal->rx_coalesce_usecs = cpsw->coal_intvl;
 958         return 0;
 959 }
 960
 961 static int cpsw_set_coalesce(struct net_device *ndev,
 962                                 struct ethtool_coalesce *coal)
 963 {
 964         struct cpsw_priv *priv = netdev_priv(ndev);
 965         u32 int_ctrl;
 966         u32 num_interrupts = 0;
 967         u32 prescale = 0;
 968         u32 addnl_dvdr = 1;
 969         u32 coal_intvl = 0;
 970         struct cpsw_common *cpsw = priv->cpsw;
 971
 972         coal_intvl = coal->rx_coalesce_usecs;
 973
 974         int_ctrl =  readl(&cpsw->wr_regs->int_control);
 975         prescale = cpsw->bus_freq_mhz * 4;
 976
 977         if (!coal->rx_coalesce_usecs) {
 978                 int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
 979                 goto update_return;
 980         }
 981
 982         if (coal_intvl < CPSW_CMINTMIN_INTVL)
 983                 coal_intvl = CPSW_CMINTMIN_INTVL;
 984
 985         if (coal_intvl > CPSW_CMINTMAX_INTVL) {
 986                 /* Interrupt pacer works with 4us Pulse, we can
 987                  * throttle further by dilating the 4us pulse.
 988                  */
 989                 addnl_dvdr = CPSW_INTPRESCALE_MASK / prescale;
 990
 991                 if (addnl_dvdr > 1) {
 992                         prescale *= addnl_dvdr;
 993                         if (coal_intvl > (CPSW_CMINTMAX_INTVL * addnl_dvdr))
 994                                 coal_intvl = (CPSW_CMINTMAX_INTVL
 995                                                 * addnl_dvdr);
 996                 } else {
 997                         addnl_dvdr = 1;
 998                         coal_intvl = CPSW_CMINTMAX_INTVL;
 999                 }
1000         }
1001
1002         num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
1003         writel(num_interrupts, &cpsw->wr_regs->rx_imax);
1004         writel(num_interrupts, &cpsw->wr_regs->tx_imax);
1005
1006         int_ctrl |= CPSW_INTPACEEN;
1007         int_ctrl &= (~CPSW_INTPRESCALE_MASK);
1008         int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
1009
1010 update_return:
1011         writel(int_ctrl, &cpsw->wr_regs->int_control);
1012
1013         cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
1014         cpsw->coal_intvl = coal_intvl;
1015
1016         return 0;
1017 }
1018
1019 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
1020 {
1021         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1022
1023         switch (sset) {
1024         case ETH_SS_STATS:
1025                 return (CPSW_STATS_COMMON_LEN +
1026                        (cpsw->rx_ch_num + cpsw->tx_ch_num) *
1027                        CPSW_STATS_CH_LEN);
1028         default:
1029                 return -EOPNOTSUPP;
1030         }
1031 }
1032
1033 static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
1034 {
1035         int ch_stats_len;
1036         int line;
1037         int i;
1038
1039         ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
1040         for (i = 0; i < ch_stats_len; i++) {
1041                 line = i % CPSW_STATS_CH_LEN;
1042                 snprintf(*p, ETH_GSTRING_LEN,
1043                          "%s DMA chan %d: %s", rx_dir ? "Rx" : "Tx",
1044                          i / CPSW_STATS_CH_LEN,
1045                          cpsw_gstrings_ch_stats[line].stat_string);
1046                 *p += ETH_GSTRING_LEN;
1047         }
1048 }
1049
1050 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1051 {
1052         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1053         u8 *p = data;
1054         int i;
1055
1056         switch (stringset) {
1057         case ETH_SS_STATS:
1058                 for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
1059                         memcpy(p, cpsw_gstrings_stats[i].stat_string,
1060                                ETH_GSTRING_LEN);
1061                         p += ETH_GSTRING_LEN;
1062                 }
1063
1064                 cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
1065                 cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
1066                 break;
1067         }
1068 }
1069
1070 static void cpsw_get_ethtool_stats(struct net_device *ndev,
1071                                     struct ethtool_stats *stats, u64 *data)
1072 {
1073         u8 *p;
1074         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1075         struct cpdma_chan_stats ch_stats;
1076         int i, l, ch;
1077
1078         /* Collect Davinci CPDMA stats for Rx and Tx Channel */
1079         for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
1080                 data[l] = readl(cpsw->hw_stats +
1081                                 cpsw_gstrings_stats[l].stat_offset);
1082
1083         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1084                 cpdma_chan_get_stats(cpsw->rxch[ch], &ch_stats);
1085                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1086                         p = (u8 *)&ch_stats +
1087                                 cpsw_gstrings_ch_stats[i].stat_offset;
1088                         data[l] = *(u32 *)p;
1089                 }
1090         }
1091
1092         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1093                 cpdma_chan_get_stats(cpsw->txch[ch], &ch_stats);
1094                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1095                         p = (u8 *)&ch_stats +
1096                                 cpsw_gstrings_ch_stats[i].stat_offset;
1097                         data[l] = *(u32 *)p;
1098                 }
1099         }
1100 }
1101
1102 static int cpsw_common_res_usage_state(struct cpsw_common *cpsw)
1103 {
1104         u32 i;
1105         u32 usage_count = 0;
1106
1107         if (!cpsw->data.dual_emac)
1108                 return 0;
1109
1110         for (i = 0; i < cpsw->data.slaves; i++)
1111                 if (cpsw->slaves[i].open_stat)
1112                         usage_count++;
1113
1114         return usage_count;
1115 }
1116
1117 static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
1118                                         struct sk_buff *skb,
1119                                         struct cpdma_chan *txch)
1120 {
1121         struct cpsw_common *cpsw = priv->cpsw;
1122
1123         return cpdma_chan_submit(txch, skb, skb->data, skb->len,
1124                                  priv->emac_port + cpsw->data.dual_emac);
1125 }
1126
1127 static inline void cpsw_add_dual_emac_def_ale_entries(
1128                 struct cpsw_priv *priv, struct cpsw_slave *slave,
1129                 u32 slave_port)
1130 {
1131         struct cpsw_common *cpsw = priv->cpsw;
1132         u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
1133
1134         if (cpsw->version == CPSW_VERSION_1)
1135                 slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
1136         else
1137                 slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
1138         cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
1139                           port_mask, port_mask, 0);
1140         cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1141                            port_mask, ALE_VLAN, slave->port_vlan, 0);
1142         cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1143                            HOST_PORT_NUM, ALE_VLAN |
1144                            ALE_SECURE, slave->port_vlan);
1145         cpsw_ale_control_set(cpsw->ale, slave_port,
1146                              ALE_PORT_DROP_UNKNOWN_VLAN, 1);
1147 }
1148
1149 static void soft_reset_slave(struct cpsw_slave *slave)
1150 {
1151         char name[32];
1152
1153         snprintf(name, sizeof(name), "slave-%d", slave->slave_num);
1154         soft_reset(name, &slave->sliver->soft_reset);
1155 }
1156
1157 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
1158 {
1159         u32 slave_port;
1160         struct cpsw_common *cpsw = priv->cpsw;
1161
1162         soft_reset_slave(slave);
1163
1164         /* setup priority mapping */
1165         __raw_writel(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
1166
1167         switch (cpsw->version) {
1168         case CPSW_VERSION_1:
1169                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
1170                 /* Increase RX FIFO size to 5 for supporting fullduplex
1171                  * flow control mode
1172                  */
1173                 slave_write(slave,
1174                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1175                             CPSW_MAX_BLKS_RX, CPSW1_MAX_BLKS);
1176                 break;
1177         case CPSW_VERSION_2:
1178         case CPSW_VERSION_3:
1179         case CPSW_VERSION_4:
1180                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW2_TX_PRI_MAP);
1181                 /* Increase RX FIFO size to 5 for supporting fullduplex
1182                  * flow control mode
1183                  */
1184                 slave_write(slave,
1185                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1186                             CPSW_MAX_BLKS_RX, CPSW2_MAX_BLKS);
1187                 break;
1188         }
1189
1190         /* setup max packet size, and mac address */
1191         __raw_writel(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
1192         cpsw_set_slave_mac(slave, priv);
1193
1194         slave->mac_control = 0; /* no link yet */
1195
1196         slave_port = cpsw_get_slave_port(slave->slave_num);
1197
1198         if (cpsw->data.dual_emac)
1199                 cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
1200         else
1201                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1202                                    1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
1203
1204         if (slave->data->phy_node) {
1205                 slave->phy = of_phy_connect(priv->ndev, slave->data->phy_node,
1206                                  &cpsw_adjust_link, 0, slave->data->phy_if);
1207                 if (!slave->phy) {
1208                         dev_err(priv->dev, "phy \"%s\" not found on slave %d\n",
1209                                 slave->data->phy_node->full_name,
1210                                 slave->slave_num);
1211                         return;
1212                 }
1213         } else {
1214                 slave->phy = phy_connect(priv->ndev, slave->data->phy_id,
1215                                  &cpsw_adjust_link, slave->data->phy_if);
1216                 if (IS_ERR(slave->phy)) {
1217                         dev_err(priv->dev,
1218                                 "phy \"%s\" not found on slave %d, err %ld\n",
1219                                 slave->data->phy_id, slave->slave_num,
1220                                 PTR_ERR(slave->phy));
1221                         slave->phy = NULL;
1222                         return;
1223                 }
1224         }
1225
1226         phy_attached_info(slave->phy);
1227
1228         phy_start(slave->phy);
1229
1230         /* Configure GMII_SEL register */
1231         cpsw_phy_sel(cpsw->dev, slave->phy->interface, slave->slave_num);
1232 }
1233
1234 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
1235 {
1236         struct cpsw_common *cpsw = priv->cpsw;
1237         const int vlan = cpsw->data.default_vlan;
1238         u32 reg;
1239         int i;
1240         int unreg_mcast_mask;
1241
1242         reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
1243                CPSW2_PORT_VLAN;
1244
1245         writel(vlan, &cpsw->host_port_regs->port_vlan);
1246
1247         for (i = 0; i < cpsw->data.slaves; i++)
1248                 slave_write(cpsw->slaves + i, vlan, reg);
1249
1250         if (priv->ndev->flags & IFF_ALLMULTI)
1251                 unreg_mcast_mask = ALE_ALL_PORTS;
1252         else
1253                 unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1254
1255         cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
1256                           ALE_ALL_PORTS, ALE_ALL_PORTS,
1257                           unreg_mcast_mask);
1258 }
1259
1260 static void cpsw_init_host_port(struct cpsw_priv *priv)
1261 {
1262         u32 fifo_mode;
1263         u32 control_reg;
1264         struct cpsw_common *cpsw = priv->cpsw;
1265
1266         /* soft reset the controller and initialize ale */
1267         soft_reset("cpsw", &cpsw->regs->soft_reset);
1268         cpsw_ale_start(cpsw->ale);
1269
1270         /* switch to vlan unaware mode */
1271         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
1272                              CPSW_ALE_VLAN_AWARE);
1273         control_reg = readl(&cpsw->regs->control);
1274         control_reg |= CPSW_VLAN_AWARE;
1275         writel(control_reg, &cpsw->regs->control);
1276         fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
1277                      CPSW_FIFO_NORMAL_MODE;
1278         writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
1279
1280         /* setup host port priority mapping */
1281         __raw_writel(CPDMA_TX_PRIORITY_MAP,
1282                      &cpsw->host_port_regs->cpdma_tx_pri_map);
1283         __raw_writel(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
1284
1285         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
1286                              ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1287
1288         if (!cpsw->data.dual_emac) {
1289                 cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1290                                    0, 0);
1291                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1292                                    ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
1293         }
1294 }
1295
1296 static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1297 {
1298         struct cpsw_common *cpsw = priv->cpsw;
1299         struct sk_buff *skb;
1300         int ch_buf_num;
1301         int ch, i, ret;
1302
1303         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1304                 ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxch[ch]);
1305                 for (i = 0; i < ch_buf_num; i++) {
1306                         skb = __netdev_alloc_skb_ip_align(priv->ndev,
1307                                                           cpsw->rx_packet_max,
1308                                                           GFP_KERNEL);
1309                         if (!skb) {
1310                                 cpsw_err(priv, ifup, "cannot allocate skb\n");
1311                                 return -ENOMEM;
1312                         }
1313
1314                         skb_set_queue_mapping(skb, ch);
1315                         ret = cpdma_chan_submit(cpsw->rxch[ch], skb, skb->data,
1316                                                 skb_tailroom(skb), 0);
1317                         if (ret < 0) {
1318                                 cpsw_err(priv, ifup,
1319                                          "cannot submit skb to channel %d rx, error %d\n",
1320                                          ch, ret);
1321                                 kfree_skb(skb);
1322                                 return ret;
1323                         }
1324                         kmemleak_not_leak(skb);
1325                 }
1326
1327                 cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1328                           ch, ch_buf_num);
1329         }
1330
1331         return 0;
1332 }
1333
1334 static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
1335 {
1336         u32 slave_port;
1337
1338         slave_port = cpsw_get_slave_port(slave->slave_num);
1339
1340         if (!slave->phy)
1341                 return;
1342         phy_stop(slave->phy);
1343         phy_disconnect(slave->phy);
1344         slave->phy = NULL;
1345         cpsw_ale_control_set(cpsw->ale, slave_port,
1346                              ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1347         soft_reset_slave(slave);
1348 }
1349
1350 static int cpsw_ndo_open(struct net_device *ndev)
1351 {
1352         struct cpsw_priv *priv = netdev_priv(ndev);
1353         struct cpsw_common *cpsw = priv->cpsw;
1354         int ret;
1355         u32 reg;
1356
1357         ret = pm_runtime_get_sync(cpsw->dev);
1358         if (ret < 0) {
1359                 pm_runtime_put_noidle(cpsw->dev);
1360                 return ret;
1361         }
1362
1363         if (!cpsw_common_res_usage_state(cpsw))
1364                 cpsw_intr_disable(cpsw);
1365         netif_carrier_off(ndev);
1366
1367         /* Notify the stack of the actual queue counts. */
1368         ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
1369         if (ret) {
1370                 dev_err(priv->dev, "cannot set real number of tx queues\n");
1371                 goto err_cleanup;
1372         }
1373
1374         ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
1375         if (ret) {
1376                 dev_err(priv->dev, "cannot set real number of rx queues\n");
1377                 goto err_cleanup;
1378         }
1379
1380         reg = cpsw->version;
1381
1382         dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
1383                  CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
1384                  CPSW_RTL_VERSION(reg));
1385
1386         /* initialize host and slave ports */
1387         if (!cpsw_common_res_usage_state(cpsw))
1388                 cpsw_init_host_port(priv);
1389         for_each_slave(priv, cpsw_slave_open, priv);
1390
1391         /* Add default VLAN */
1392         if (!cpsw->data.dual_emac)
1393                 cpsw_add_default_vlan(priv);
1394         else
1395                 cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
1396                                   ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
1397
1398         if (!cpsw_common_res_usage_state(cpsw)) {
1399                 /* setup tx dma to fixed prio and zero offset */
1400                 cpdma_control_set(cpsw->dma, CPDMA_TX_PRIO_FIXED, 1);
1401                 cpdma_control_set(cpsw->dma, CPDMA_RX_BUFFER_OFFSET, 0);
1402
1403                 /* disable priority elevation */
1404                 __raw_writel(0, &cpsw->regs->ptype);
1405
1406                 /* enable statistics collection only on all ports */
1407                 __raw_writel(0x7, &cpsw->regs->stat_port_en);
1408
1409                 /* Enable internal fifo flow control */
1410                 writel(0x7, &cpsw->regs->flow_control);
1411
1412                 napi_enable(&cpsw->napi_rx);
1413                 napi_enable(&cpsw->napi_tx);
1414
1415                 if (cpsw->tx_irq_disabled) {
1416                         cpsw->tx_irq_disabled = false;
1417                         enable_irq(cpsw->irqs_table[1]);
1418                 }
1419
1420                 if (cpsw->rx_irq_disabled) {
1421                         cpsw->rx_irq_disabled = false;
1422                         enable_irq(cpsw->irqs_table[0]);
1423                 }
1424
1425                 ret = cpsw_fill_rx_channels(priv);
1426                 if (ret < 0)
1427                         goto err_cleanup;
1428
1429                 if (cpts_register(cpsw->dev, cpsw->cpts,
1430                                   cpsw->data.cpts_clock_mult,
1431                                   cpsw->data.cpts_clock_shift))
1432                         dev_err(priv->dev, "error registering cpts device\n");
1433
1434         }
1435
1436         /* Enable Interrupt pacing if configured */
1437         if (cpsw->coal_intvl != 0) {
1438                 struct ethtool_coalesce coal;
1439
1440                 coal.rx_coalesce_usecs = cpsw->coal_intvl;
1441                 cpsw_set_coalesce(ndev, &coal);
1442         }
1443
1444         cpdma_ctlr_start(cpsw->dma);
1445         cpsw_intr_enable(cpsw);
1446
1447         if (cpsw->data.dual_emac)
1448                 cpsw->slaves[priv->emac_port].open_stat = true;
1449
1450         netif_tx_start_all_queues(ndev);
1451
1452         return 0;
1453
1454 err_cleanup:
1455         cpdma_ctlr_stop(cpsw->dma);
1456         for_each_slave(priv, cpsw_slave_stop, cpsw);
1457         pm_runtime_put_sync(cpsw->dev);
1458         netif_carrier_off(priv->ndev);
1459         return ret;
1460 }
1461
1462 static int cpsw_ndo_stop(struct net_device *ndev)
1463 {
1464         struct cpsw_priv *priv = netdev_priv(ndev);
1465         struct cpsw_common *cpsw = priv->cpsw;
1466
1467         cpsw_info(priv, ifdown, "shutting down cpsw device\n");
1468         netif_tx_stop_all_queues(priv->ndev);
1469         netif_carrier_off(priv->ndev);
1470
1471         if (cpsw_common_res_usage_state(cpsw) <= 1) {
1472                 napi_disable(&cpsw->napi_rx);
1473                 napi_disable(&cpsw->napi_tx);
1474                 cpts_unregister(cpsw->cpts);
1475                 cpsw_intr_disable(cpsw);
1476                 cpdma_ctlr_stop(cpsw->dma);
1477                 cpsw_ale_stop(cpsw->ale);
1478         }
1479         for_each_slave(priv, cpsw_slave_stop, cpsw);
1480         pm_runtime_put_sync(cpsw->dev);
1481         if (cpsw->data.dual_emac)
1482                 cpsw->slaves[priv->emac_port].open_stat = false;
1483         return 0;
1484 }
1485
1486 static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
1487                                        struct net_device *ndev)
1488 {
1489         struct cpsw_priv *priv = netdev_priv(ndev);
1490         struct cpsw_common *cpsw = priv->cpsw;
1491         struct netdev_queue *txq;
1492         struct cpdma_chan *txch;
1493         int ret, q_idx;
1494
1495         netif_trans_update(ndev);
1496
1497         if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
1498                 cpsw_err(priv, tx_err, "packet pad failed\n");
1499                 ndev->stats.tx_dropped++;
1500                 return NETDEV_TX_OK;
1501         }
1502
1503         if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
1504                                 cpsw->cpts->tx_enable)
1505                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1506
1507         skb_tx_timestamp(skb);
1508
1509         q_idx = skb_get_queue_mapping(skb);
1510         if (q_idx >= cpsw->tx_ch_num)
1511                 q_idx = q_idx % cpsw->tx_ch_num;
1512
1513         txch = cpsw->txch[q_idx];
1514         ret = cpsw_tx_packet_submit(priv, skb, txch);
1515         if (unlikely(ret != 0)) {
1516                 cpsw_err(priv, tx_err, "desc submit failed\n");
1517                 goto fail;
1518         }
1519
1520         /* If there is no more tx desc left free then we need to
1521          * tell the kernel to stop sending us tx frames.
1522          */
1523         if (unlikely(!cpdma_check_free_tx_desc(txch))) {
1524                 txq = netdev_get_tx_queue(ndev, q_idx);
1525                 netif_tx_stop_queue(txq);
1526         }
1527
1528         return NETDEV_TX_OK;
1529 fail:
1530         ndev->stats.tx_dropped++;
1531         txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
1532         netif_tx_stop_queue(txq);
1533         return NETDEV_TX_BUSY;
1534 }
1535
1536 #ifdef CONFIG_TI_CPTS
1537
1538 static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
1539 {
1540         struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
1541         u32 ts_en, seq_id;
1542
1543         if (!cpsw->cpts->tx_enable && !cpsw->cpts->rx_enable) {
1544                 slave_write(slave, 0, CPSW1_TS_CTL);
1545                 return;
1546         }
1547
1548         seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
1549         ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
1550
1551         if (cpsw->cpts->tx_enable)
1552                 ts_en |= CPSW_V1_TS_TX_EN;
1553
1554         if (cpsw->cpts->rx_enable)
1555                 ts_en |= CPSW_V1_TS_RX_EN;
1556
1557         slave_write(slave, ts_en, CPSW1_TS_CTL);
1558         slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
1559 }
1560
1561 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
1562 {
1563         struct cpsw_slave *slave;
1564         struct cpsw_common *cpsw = priv->cpsw;
1565         u32 ctrl, mtype;
1566
1567         if (cpsw->data.dual_emac)
1568                 slave = &cpsw->slaves[priv->emac_port];
1569         else
1570                 slave = &cpsw->slaves[cpsw->data.active_slave];
1571
1572         ctrl = slave_read(slave, CPSW2_CONTROL);
1573         switch (cpsw->version) {
1574         case CPSW_VERSION_2:
1575                 ctrl &= ~CTRL_V2_ALL_TS_MASK;
1576
1577                 if (cpsw->cpts->tx_enable)
1578                         ctrl |= CTRL_V2_TX_TS_BITS;
1579
1580                 if (cpsw->cpts->rx_enable)
1581                         ctrl |= CTRL_V2_RX_TS_BITS;
1582                 break;
1583         case CPSW_VERSION_3:
1584         default:
1585                 ctrl &= ~CTRL_V3_ALL_TS_MASK;
1586
1587                 if (cpsw->cpts->tx_enable)
1588                         ctrl |= CTRL_V3_TX_TS_BITS;
1589
1590                 if (cpsw->cpts->rx_enable)
1591                         ctrl |= CTRL_V3_RX_TS_BITS;
1592                 break;
1593         }
1594
1595         mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
1596
1597         slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
1598         slave_write(slave, ctrl, CPSW2_CONTROL);
1599         __raw_writel(ETH_P_1588, &cpsw->regs->ts_ltype);
1600 }
1601
1602 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1603 {
1604         struct cpsw_priv *priv = netdev_priv(dev);
1605         struct hwtstamp_config cfg;
1606         struct cpsw_common *cpsw = priv->cpsw;
1607         struct cpts *cpts = cpsw->cpts;
1608
1609         if (cpsw->version != CPSW_VERSION_1 &&
1610             cpsw->version != CPSW_VERSION_2 &&
1611             cpsw->version != CPSW_VERSION_3)
1612                 return -EOPNOTSUPP;
1613
1614         if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1615                 return -EFAULT;
1616
1617         /* reserved for future extensions */
1618         if (cfg.flags)
1619                 return -EINVAL;
1620
1621         if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
1622                 return -ERANGE;
1623
1624         switch (cfg.rx_filter) {
1625         case HWTSTAMP_FILTER_NONE:
1626                 cpts->rx_enable = 0;
1627                 break;
1628         case HWTSTAMP_FILTER_ALL:
1629         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1630         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1631         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1632                 return -ERANGE;
1633         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1634         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1635         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1636         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1637         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1638         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1639         case HWTSTAMP_FILTER_PTP_V2_EVENT:
1640         case HWTSTAMP_FILTER_PTP_V2_SYNC:
1641         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1642                 cpts->rx_enable = 1;
1643                 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1644                 break;
1645         default:
1646                 return -ERANGE;
1647         }
1648
1649         cpts->tx_enable = cfg.tx_type == HWTSTAMP_TX_ON;
1650
1651         switch (cpsw->version) {
1652         case CPSW_VERSION_1:
1653                 cpsw_hwtstamp_v1(cpsw);
1654                 break;
1655         case CPSW_VERSION_2:
1656         case CPSW_VERSION_3:
1657                 cpsw_hwtstamp_v2(priv);
1658                 break;
1659         default:
1660                 WARN_ON(1);
1661         }
1662
1663         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1664 }
1665
1666 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1667 {
1668         struct cpsw_common *cpsw = ndev_to_cpsw(dev);
1669         struct cpts *cpts = cpsw->cpts;
1670         struct hwtstamp_config cfg;
1671
1672         if (cpsw->version != CPSW_VERSION_1 &&
1673             cpsw->version != CPSW_VERSION_2 &&
1674             cpsw->version != CPSW_VERSION_3)
1675                 return -EOPNOTSUPP;
1676
1677         cfg.flags = 0;
1678         cfg.tx_type = cpts->tx_enable ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1679         cfg.rx_filter = (cpts->rx_enable ?
1680                          HWTSTAMP_FILTER_PTP_V2_EVENT : HWTSTAMP_FILTER_NONE);
1681
1682         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1683 }
1684
1685 #endif /*CONFIG_TI_CPTS*/
1686
1687 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
1688 {
1689         struct cpsw_priv *priv = netdev_priv(dev);
1690         struct cpsw_common *cpsw = priv->cpsw;
1691         int slave_no = cpsw_slave_index(cpsw, priv);
1692
1693         if (!netif_running(dev))
1694                 return -EINVAL;
1695
1696         switch (cmd) {
1697 #ifdef CONFIG_TI_CPTS
1698         case SIOCSHWTSTAMP:
1699                 return cpsw_hwtstamp_set(dev, req);
1700         case SIOCGHWTSTAMP:
1701                 return cpsw_hwtstamp_get(dev, req);
1702 #endif
1703         }
1704
1705         if (!cpsw->slaves[slave_no].phy)
1706                 return -EOPNOTSUPP;
1707         return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
1708 }
1709
1710 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
1711 {
1712         struct cpsw_priv *priv = netdev_priv(ndev);
1713         struct cpsw_common *cpsw = priv->cpsw;
1714         int ch;
1715
1716         cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
1717         ndev->stats.tx_errors++;
1718         cpsw_intr_disable(cpsw);
1719         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1720                 cpdma_chan_stop(cpsw->txch[ch]);
1721                 cpdma_chan_start(cpsw->txch[ch]);
1722         }
1723
1724         cpsw_intr_enable(cpsw);
1725 }
1726
1727 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
1728 {
1729         struct cpsw_priv *priv = netdev_priv(ndev);
1730         struct sockaddr *addr = (struct sockaddr *)p;
1731         struct cpsw_common *cpsw = priv->cpsw;
1732         int flags = 0;
1733         u16 vid = 0;
1734         int ret;
1735
1736         if (!is_valid_ether_addr(addr->sa_data))
1737                 return -EADDRNOTAVAIL;
1738
1739         ret = pm_runtime_get_sync(cpsw->dev);
1740         if (ret < 0) {
1741                 pm_runtime_put_noidle(cpsw->dev);
1742                 return ret;
1743         }
1744
1745         if (cpsw->data.dual_emac) {
1746                 vid = cpsw->slaves[priv->emac_port].port_vlan;
1747                 flags = ALE_VLAN;
1748         }
1749
1750         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1751                            flags, vid);
1752         cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
1753                            flags, vid);
1754
1755         memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
1756         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
1757         for_each_slave(priv, cpsw_set_slave_mac, priv);
1758
1759         pm_runtime_put(cpsw->dev);
1760
1761         return 0;
1762 }
1763
1764 #ifdef CONFIG_NET_POLL_CONTROLLER
1765 static void cpsw_ndo_poll_controller(struct net_device *ndev)
1766 {
1767         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1768
1769         cpsw_intr_disable(cpsw);
1770         cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
1771         cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
1772         cpsw_intr_enable(cpsw);
1773 }
1774 #endif
1775
1776 static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
1777                                 unsigned short vid)
1778 {
1779         int ret;
1780         int unreg_mcast_mask = 0;
1781         u32 port_mask;
1782         struct cpsw_common *cpsw = priv->cpsw;
1783
1784         if (cpsw->data.dual_emac) {
1785                 port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
1786
1787                 if (priv->ndev->flags & IFF_ALLMULTI)
1788                         unreg_mcast_mask = port_mask;
1789         } else {
1790                 port_mask = ALE_ALL_PORTS;
1791
1792                 if (priv->ndev->flags & IFF_ALLMULTI)
1793                         unreg_mcast_mask = ALE_ALL_PORTS;
1794                 else
1795                         unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1796         }
1797
1798         ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
1799                                 unreg_mcast_mask);
1800         if (ret != 0)
1801                 return ret;
1802
1803         ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1804                                  HOST_PORT_NUM, ALE_VLAN, vid);
1805         if (ret != 0)
1806                 goto clean_vid;
1807
1808         ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1809                                  port_mask, ALE_VLAN, vid, 0);
1810         if (ret != 0)
1811                 goto clean_vlan_ucast;
1812         return 0;
1813
1814 clean_vlan_ucast:
1815         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
1816                            HOST_PORT_NUM, ALE_VLAN, vid);
1817 clean_vid:
1818         cpsw_ale_del_vlan(cpsw->ale, vid, 0);
1819         return ret;
1820 }
1821
1822 static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
1823                                     __be16 proto, u16 vid)
1824 {
1825         struct cpsw_priv *priv = netdev_priv(ndev);
1826         struct cpsw_common *cpsw = priv->cpsw;
1827         int ret;
1828
1829         if (vid == cpsw->data.default_vlan)
1830                 return 0;
1831
1832         ret = pm_runtime_get_sync(cpsw->dev);
1833         if (ret < 0) {
1834                 pm_runtime_put_noidle(cpsw->dev);
1835                 return ret;
1836         }
1837
1838         if (cpsw->data.dual_emac) {
1839                 /* In dual EMAC, reserved VLAN id should not be used for
1840                  * creating VLAN interfaces as this can break the dual
1841                  * EMAC port separation
1842                  */
1843                 int i;
1844
1845                 for (i = 0; i < cpsw->data.slaves; i++) {
1846                         if (vid == cpsw->slaves[i].port_vlan)
1847                                 return -EINVAL;
1848                 }
1849         }
1850
1851         dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
1852         ret = cpsw_add_vlan_ale_entry(priv, vid);
1853
1854         pm_runtime_put(cpsw->dev);
1855         return ret;
1856 }
1857
1858 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
1859                                      __be16 proto, u16 vid)
1860 {
1861         struct cpsw_priv *priv = netdev_priv(ndev);
1862         struct cpsw_common *cpsw = priv->cpsw;
1863         int ret;
1864
1865         if (vid == cpsw->data.default_vlan)
1866                 return 0;
1867
1868         ret = pm_runtime_get_sync(cpsw->dev);
1869         if (ret < 0) {
1870                 pm_runtime_put_noidle(cpsw->dev);
1871                 return ret;
1872         }
1873
1874         if (cpsw->data.dual_emac) {
1875                 int i;
1876
1877                 for (i = 0; i < cpsw->data.slaves; i++) {
1878                         if (vid == cpsw->slaves[i].port_vlan)
1879                                 return -EINVAL;
1880                 }
1881         }
1882
1883         dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
1884         ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
1885         if (ret != 0)
1886                 return ret;
1887
1888         ret = cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
1889                                  HOST_PORT_NUM, ALE_VLAN, vid);
1890         if (ret != 0)
1891                 return ret;
1892
1893         ret = cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
1894                                  0, ALE_VLAN, vid);
1895         pm_runtime_put(cpsw->dev);
1896         return ret;
1897 }
1898
1899 static const struct net_device_ops cpsw_netdev_ops = {
1900         .ndo_open               = cpsw_ndo_open,
1901         .ndo_stop               = cpsw_ndo_stop,
1902         .ndo_start_xmit         = cpsw_ndo_start_xmit,
1903         .ndo_set_mac_address    = cpsw_ndo_set_mac_address,
1904         .ndo_do_ioctl           = cpsw_ndo_ioctl,
1905         .ndo_validate_addr      = eth_validate_addr,
1906         .ndo_change_mtu         = eth_change_mtu,
1907         .ndo_tx_timeout         = cpsw_ndo_tx_timeout,
1908         .ndo_set_rx_mode        = cpsw_ndo_set_rx_mode,
1909 #ifdef CONFIG_NET_POLL_CONTROLLER
1910         .ndo_poll_controller    = cpsw_ndo_poll_controller,
1911 #endif
1912         .ndo_vlan_rx_add_vid    = cpsw_ndo_vlan_rx_add_vid,
1913         .ndo_vlan_rx_kill_vid   = cpsw_ndo_vlan_rx_kill_vid,
1914 };
1915
1916 static int cpsw_get_regs_len(struct net_device *ndev)
1917 {
1918         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1919
1920         return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
1921 }
1922
1923 static void cpsw_get_regs(struct net_device *ndev,
1924                           struct ethtool_regs *regs, void *p)
1925 {
1926         u32 *reg = p;
1927         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1928
1929         /* update CPSW IP version */
1930         regs->version = cpsw->version;
1931
1932         cpsw_ale_dump(cpsw->ale, reg);
1933 }
1934
1935 static void cpsw_get_drvinfo(struct net_device *ndev,
1936                              struct ethtool_drvinfo *info)
1937 {
1938         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1939         struct platform_device  *pdev = to_platform_device(cpsw->dev);
1940
1941         strlcpy(info->driver, "cpsw", sizeof(info->driver));
1942         strlcpy(info->version, "1.0", sizeof(info->version));
1943         strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
1944 }
1945
1946 static u32 cpsw_get_msglevel(struct net_device *ndev)
1947 {
1948         struct cpsw_priv *priv = netdev_priv(ndev);
1949         return priv->msg_enable;
1950 }
1951
1952 static void cpsw_set_msglevel(struct net_device *ndev, u32 value)
1953 {
1954         struct cpsw_priv *priv = netdev_priv(ndev);
1955         priv->msg_enable = value;
1956 }
1957
1958 static int cpsw_get_ts_info(struct net_device *ndev,
1959                             struct ethtool_ts_info *info)
1960 {
1961 #ifdef CONFIG_TI_CPTS
1962         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1963
1964         info->so_timestamping =
1965                 SOF_TIMESTAMPING_TX_HARDWARE |
1966                 SOF_TIMESTAMPING_TX_SOFTWARE |
1967                 SOF_TIMESTAMPING_RX_HARDWARE |
1968                 SOF_TIMESTAMPING_RX_SOFTWARE |
1969                 SOF_TIMESTAMPING_SOFTWARE |
1970                 SOF_TIMESTAMPING_RAW_HARDWARE;
1971         info->phc_index = cpsw->cpts->phc_index;
1972         info->tx_types =
1973                 (1 << HWTSTAMP_TX_OFF) |
1974                 (1 << HWTSTAMP_TX_ON);
1975         info->rx_filters =
1976                 (1 << HWTSTAMP_FILTER_NONE) |
1977                 (1 << HWTSTAMP_FILTER_PTP_V2_EVENT);
1978 #else
1979         info->so_timestamping =
1980                 SOF_TIMESTAMPING_TX_SOFTWARE |
1981                 SOF_TIMESTAMPING_RX_SOFTWARE |
1982                 SOF_TIMESTAMPING_SOFTWARE;
1983         info->phc_index = -1;
1984         info->tx_types = 0;
1985         info->rx_filters = 0;
1986 #endif
1987         return 0;
1988 }
1989
1990 static int cpsw_get_settings(struct net_device *ndev,
1991                              struct ethtool_cmd *ecmd)
1992 {
1993         struct cpsw_priv *priv = netdev_priv(ndev);
1994         struct cpsw_common *cpsw = priv->cpsw;
1995         int slave_no = cpsw_slave_index(cpsw, priv);
1996
1997         if (cpsw->slaves[slave_no].phy)
1998                 return phy_ethtool_gset(cpsw->slaves[slave_no].phy, ecmd);
1999         else
2000                 return -EOPNOTSUPP;
2001 }
2002
2003 static int cpsw_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
2004 {
2005         struct cpsw_priv *priv = netdev_priv(ndev);
2006         struct cpsw_common *cpsw = priv->cpsw;
2007         int slave_no = cpsw_slave_index(cpsw, priv);
2008
2009         if (cpsw->slaves[slave_no].phy)
2010                 return phy_ethtool_sset(cpsw->slaves[slave_no].phy, ecmd);
2011         else
2012                 return -EOPNOTSUPP;
2013 }
2014
2015 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2016 {
2017         struct cpsw_priv *priv = netdev_priv(ndev);
2018         struct cpsw_common *cpsw = priv->cpsw;
2019         int slave_no = cpsw_slave_index(cpsw, priv);
2020
2021         wol->supported = 0;
2022         wol->wolopts = 0;
2023
2024         if (cpsw->slaves[slave_no].phy)
2025                 phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
2026 }
2027
2028 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2029 {
2030         struct cpsw_priv *priv = netdev_priv(ndev);
2031         struct cpsw_common *cpsw = priv->cpsw;
2032         int slave_no = cpsw_slave_index(cpsw, priv);
2033
2034         if (cpsw->slaves[slave_no].phy)
2035                 return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
2036         else
2037                 return -EOPNOTSUPP;
2038 }
2039
2040 static void cpsw_get_pauseparam(struct net_device *ndev,
2041                                 struct ethtool_pauseparam *pause)
2042 {
2043         struct cpsw_priv *priv = netdev_priv(ndev);
2044
2045         pause->autoneg = AUTONEG_DISABLE;
2046         pause->rx_pause = priv->rx_pause ? true : false;
2047         pause->tx_pause = priv->tx_pause ? true : false;
2048 }
2049
2050 static int cpsw_set_pauseparam(struct net_device *ndev,
2051                                struct ethtool_pauseparam *pause)
2052 {
2053         struct cpsw_priv *priv = netdev_priv(ndev);
2054         bool link;
2055
2056         priv->rx_pause = pause->rx_pause ? true : false;
2057         priv->tx_pause = pause->tx_pause ? true : false;
2058
2059         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
2060         return 0;
2061 }
2062
2063 static int cpsw_ethtool_op_begin(struct net_device *ndev)
2064 {
2065         struct cpsw_priv *priv = netdev_priv(ndev);
2066         struct cpsw_common *cpsw = priv->cpsw;
2067         int ret;
2068
2069         ret = pm_runtime_get_sync(cpsw->dev);
2070         if (ret < 0) {
2071                 cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
2072                 pm_runtime_put_noidle(cpsw->dev);
2073         }
2074
2075         return ret;
2076 }
2077
2078 static void cpsw_ethtool_op_complete(struct net_device *ndev)
2079 {
2080         struct cpsw_priv *priv = netdev_priv(ndev);
2081         int ret;
2082
2083         ret = pm_runtime_put(priv->cpsw->dev);
2084         if (ret < 0)
2085                 cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
2086 }
2087
2088 static void cpsw_get_channels(struct net_device *ndev,
2089                               struct ethtool_channels *ch)
2090 {
2091         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2092
2093         ch->max_combined = 0;
2094         ch->max_rx = CPSW_MAX_QUEUES;
2095         ch->max_tx = CPSW_MAX_QUEUES;
2096         ch->max_other = 0;
2097         ch->other_count = 0;
2098         ch->rx_count = cpsw->rx_ch_num;
2099         ch->tx_count = cpsw->tx_ch_num;
2100         ch->combined_count = 0;
2101 }
2102
2103 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
2104                                   struct ethtool_channels *ch)
2105 {
2106         if (ch->combined_count)
2107                 return -EINVAL;
2108
2109         /* verify we have at least one channel in each direction */
2110         if (!ch->rx_count || !ch->tx_count)
2111                 return -EINVAL;
2112
2113         if (ch->rx_count > cpsw->data.channels ||
2114             ch->tx_count > cpsw->data.channels)
2115                 return -EINVAL;
2116
2117         return 0;
2118 }
2119
2120 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
2121 {
2122         int (*poll)(struct napi_struct *, int);
2123         struct cpsw_common *cpsw = priv->cpsw;
2124         void (*handler)(void *, int, int);
2125         struct cpdma_chan **chan;
2126         int ret, *ch;
2127
2128         if (rx) {
2129                 ch = &cpsw->rx_ch_num;
2130                 chan = cpsw->rxch;
2131                 handler = cpsw_rx_handler;
2132                 poll = cpsw_rx_poll;
2133         } else {
2134                 ch = &cpsw->tx_ch_num;
2135                 chan = cpsw->txch;
2136                 handler = cpsw_tx_handler;
2137                 poll = cpsw_tx_poll;
2138         }
2139
2140         while (*ch < ch_num) {
2141                 chan[*ch] = cpdma_chan_create(cpsw->dma, *ch, handler, rx);
2142
2143                 if (IS_ERR(chan[*ch]))
2144                         return PTR_ERR(chan[*ch]);
2145
2146                 if (!chan[*ch])
2147                         return -EINVAL;
2148
2149                 cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
2150                           (rx ? "rx" : "tx"));
2151                 (*ch)++;
2152         }
2153
2154         while (*ch > ch_num) {
2155                 (*ch)--;
2156
2157                 ret = cpdma_chan_destroy(chan[*ch]);
2158                 if (ret)
2159                         return ret;
2160
2161                 cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
2162                           (rx ? "rx" : "tx"));
2163         }
2164
2165         return 0;
2166 }
2167
2168 static int cpsw_update_channels(struct cpsw_priv *priv,
2169                                 struct ethtool_channels *ch)
2170 {
2171         int ret;
2172
2173         ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
2174         if (ret)
2175                 return ret;
2176
2177         ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
2178         if (ret)
2179                 return ret;
2180
2181         return 0;
2182 }
2183
2184 static int cpsw_set_channels(struct net_device *ndev,
2185                              struct ethtool_channels *chs)
2186 {
2187         struct cpsw_priv *priv = netdev_priv(ndev);
2188         struct cpsw_common *cpsw = priv->cpsw;
2189         struct cpsw_slave *slave;
2190         int i, ret;
2191
2192         ret = cpsw_check_ch_settings(cpsw, chs);
2193         if (ret < 0)
2194                 return ret;
2195
2196         /* Disable NAPI scheduling */
2197         cpsw_intr_disable(cpsw);
2198
2199         /* Stop all transmit queues for every network device.
2200          * Disable re-using rx descriptors with dormant_on.
2201          */
2202         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2203                 if (!(slave->ndev && netif_running(slave->ndev)))
2204                         continue;
2205
2206                 netif_tx_stop_all_queues(slave->ndev);
2207                 netif_dormant_on(slave->ndev);
2208         }
2209
2210         /* Handle rest of tx packets and stop cpdma channels */
2211         cpdma_ctlr_stop(cpsw->dma);
2212         ret = cpsw_update_channels(priv, chs);
2213         if (ret)
2214                 goto err;
2215
2216         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2217                 if (!(slave->ndev && netif_running(slave->ndev)))
2218                         continue;
2219
2220                 /* Inform stack about new count of queues */
2221                 ret = netif_set_real_num_tx_queues(slave->ndev,
2222                                                    cpsw->tx_ch_num);
2223                 if (ret) {
2224                         dev_err(priv->dev, "cannot set real number of tx queues\n");
2225                         goto err;
2226                 }
2227
2228                 ret = netif_set_real_num_rx_queues(slave->ndev,
2229                                                    cpsw->rx_ch_num);
2230                 if (ret) {
2231                         dev_err(priv->dev, "cannot set real number of rx queues\n");
2232                         goto err;
2233                 }
2234
2235                 /* Enable rx packets handling */
2236                 netif_dormant_off(slave->ndev);
2237         }
2238
2239         if (cpsw_common_res_usage_state(cpsw)) {
2240                 ret = cpsw_fill_rx_channels(priv);
2241                 if (ret)
2242                         goto err;
2243
2244                 /* After this receive is started */
2245                 cpdma_ctlr_start(cpsw->dma);
2246                 cpsw_intr_enable(cpsw);
2247         }
2248
2249         /* Resume transmit for every affected interface */
2250         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2251                 if (!(slave->ndev && netif_running(slave->ndev)))
2252                         continue;
2253                 netif_tx_start_all_queues(slave->ndev);
2254         }
2255         return 0;
2256 err:
2257         dev_err(priv->dev, "cannot update channels number, closing device\n");
2258         dev_close(ndev);
2259         return ret;
2260 }
2261
2262 static const struct ethtool_ops cpsw_ethtool_ops = {
2263         .get_drvinfo    = cpsw_get_drvinfo,
2264         .get_msglevel   = cpsw_get_msglevel,
2265         .set_msglevel   = cpsw_set_msglevel,
2266         .get_link       = ethtool_op_get_link,
2267         .get_ts_info    = cpsw_get_ts_info,
2268         .get_settings   = cpsw_get_settings,
2269         .set_settings   = cpsw_set_settings,
2270         .get_coalesce   = cpsw_get_coalesce,
2271         .set_coalesce   = cpsw_set_coalesce,
2272         .get_sset_count         = cpsw_get_sset_count,
2273         .get_strings            = cpsw_get_strings,
2274         .get_ethtool_stats      = cpsw_get_ethtool_stats,
2275         .get_pauseparam         = cpsw_get_pauseparam,
2276         .set_pauseparam         = cpsw_set_pauseparam,
2277         .get_wol        = cpsw_get_wol,
2278         .set_wol        = cpsw_set_wol,
2279         .get_regs_len   = cpsw_get_regs_len,
2280         .get_regs       = cpsw_get_regs,
2281         .begin          = cpsw_ethtool_op_begin,
2282         .complete       = cpsw_ethtool_op_complete,
2283         .get_channels   = cpsw_get_channels,
2284         .set_channels   = cpsw_set_channels,
2285 };
2286
2287 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
2288                             u32 slave_reg_ofs, u32 sliver_reg_ofs)
2289 {
2290         void __iomem            *regs = cpsw->regs;
2291         int                     slave_num = slave->slave_num;
2292         struct cpsw_slave_data  *data = cpsw->data.slave_data + slave_num;
2293
2294         slave->data     = data;
2295         slave->regs     = regs + slave_reg_ofs;
2296         slave->sliver   = regs + sliver_reg_ofs;
2297         slave->port_vlan = data->dual_emac_res_vlan;
2298 }
2299
2300 static int cpsw_probe_dt(struct cpsw_platform_data *data,
2301                          struct platform_device *pdev)
2302 {
2303         struct device_node *node = pdev->dev.of_node;
2304         struct device_node *slave_node;
2305         int i = 0, ret;
2306         u32 prop;
2307
2308         if (!node)
2309                 return -EINVAL;
2310
2311         if (of_property_read_u32(node, "slaves", &prop)) {
2312                 dev_err(&pdev->dev, "Missing slaves property in the DT.\n");
2313                 return -EINVAL;
2314         }
2315         data->slaves = prop;
2316
2317         if (of_property_read_u32(node, "active_slave", &prop)) {
2318                 dev_err(&pdev->dev, "Missing active_slave property in the DT.\n");
2319                 return -EINVAL;
2320         }
2321         data->active_slave = prop;
2322
2323         if (of_property_read_u32(node, "cpts_clock_mult", &prop)) {
2324                 dev_err(&pdev->dev, "Missing cpts_clock_mult property in the DT.\n");
2325                 return -EINVAL;
2326         }
2327         data->cpts_clock_mult = prop;
2328
2329         if (of_property_read_u32(node, "cpts_clock_shift", &prop)) {
2330                 dev_err(&pdev->dev, "Missing cpts_clock_shift property in the DT.\n");
2331                 return -EINVAL;
2332         }
2333         data->cpts_clock_shift = prop;
2334
2335         data->slave_data = devm_kzalloc(&pdev->dev, data->slaves
2336                                         * sizeof(struct cpsw_slave_data),
2337                                         GFP_KERNEL);
2338         if (!data->slave_data)
2339                 return -ENOMEM;
2340
2341         if (of_property_read_u32(node, "cpdma_channels", &prop)) {
2342                 dev_err(&pdev->dev, "Missing cpdma_channels property in the DT.\n");
2343                 return -EINVAL;
2344         }
2345         data->channels = prop;
2346
2347         if (of_property_read_u32(node, "ale_entries", &prop)) {
2348                 dev_err(&pdev->dev, "Missing ale_entries property in the DT.\n");
2349                 return -EINVAL;
2350         }
2351         data->ale_entries = prop;
2352
2353         if (of_property_read_u32(node, "bd_ram_size", &prop)) {
2354                 dev_err(&pdev->dev, "Missing bd_ram_size property in the DT.\n");
2355                 return -EINVAL;
2356         }
2357         data->bd_ram_size = prop;
2358
2359         if (of_property_read_u32(node, "mac_control", &prop)) {
2360                 dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
2361                 return -EINVAL;
2362         }
2363         data->mac_control = prop;
2364
2365         if (of_property_read_bool(node, "dual_emac"))
2366                 data->dual_emac = 1;
2367
2368         /*
2369          * Populate all the child nodes here...
2370          */
2371         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2372         /* We do not want to force this, as in some cases may not have child */
2373         if (ret)
2374                 dev_warn(&pdev->dev, "Doesn't have any child node\n");
2375
2376         for_each_available_child_of_node(node, slave_node) {
2377                 struct cpsw_slave_data *slave_data = data->slave_data + i;
2378                 const void *mac_addr = NULL;
2379                 int lenp;
2380                 const __be32 *parp;
2381
2382                 /* This is no slave child node, continue */
2383                 if (strcmp(slave_node->name, "slave"))
2384                         continue;
2385
2386                 slave_data->phy_node = of_parse_phandle(slave_node,
2387                                                         "phy-handle", 0);
2388                 parp = of_get_property(slave_node, "phy_id", &lenp);
2389                 if (slave_data->phy_node) {
2390                         dev_dbg(&pdev->dev,
2391                                 "slave[%d] using phy-handle=\"%s\"\n",
2392                                 i, slave_data->phy_node->full_name);
2393                 } else if (of_phy_is_fixed_link(slave_node)) {
2394                         /* In the case of a fixed PHY, the DT node associated
2395                          * to the PHY is the Ethernet MAC DT node.
2396                          */
2397                         ret = of_phy_register_fixed_link(slave_node);
2398                         if (ret) {
2399                                 if (ret != -EPROBE_DEFER)
2400                                         dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret);
2401                                 return ret;
2402                         }
2403                         slave_data->phy_node = of_node_get(slave_node);
2404                 } else if (parp) {
2405                         u32 phyid;
2406                         struct device_node *mdio_node;
2407                         struct platform_device *mdio;
2408
2409                         if (lenp != (sizeof(__be32) * 2)) {
2410                                 dev_err(&pdev->dev, "Invalid slave[%d] phy_id property\n", i);
2411                                 goto no_phy_slave;
2412                         }
2413                         mdio_node = of_find_node_by_phandle(be32_to_cpup(parp));
2414                         phyid = be32_to_cpup(parp+1);
2415                         mdio = of_find_device_by_node(mdio_node);
2416                         of_node_put(mdio_node);
2417                         if (!mdio) {
2418                                 dev_err(&pdev->dev, "Missing mdio platform device\n");
2419                                 return -EINVAL;
2420                         }
2421                         snprintf(slave_data->phy_id, sizeof(slave_data->phy_id),
2422                                  PHY_ID_FMT, mdio->name, phyid);
2423                         put_device(&mdio->dev);
2424                 } else {
2425                         dev_err(&pdev->dev,
2426                                 "No slave[%d] phy_id, phy-handle, or fixed-link property\n",
2427                                 i);
2428                         goto no_phy_slave;
2429                 }
2430                 slave_data->phy_if = of_get_phy_mode(slave_node);
2431                 if (slave_data->phy_if < 0) {
2432                         dev_err(&pdev->dev, "Missing or malformed slave[%d] phy-mode property\n",
2433                                 i);
2434                         return slave_data->phy_if;
2435                 }
2436
2437 no_phy_slave:
2438                 mac_addr = of_get_mac_address(slave_node);
2439                 if (mac_addr) {
2440                         memcpy(slave_data->mac_addr, mac_addr, ETH_ALEN);
2441                 } else {
2442                         ret = ti_cm_get_macid(&pdev->dev, i,
2443                                               slave_data->mac_addr);
2444                         if (ret)
2445                                 return ret;
2446                 }
2447                 if (data->dual_emac) {
2448                         if (of_property_read_u32(slave_node, "dual_emac_res_vlan",
2449                                                  &prop)) {
2450                                 dev_err(&pdev->dev, "Missing dual_emac_res_vlan in DT.\n");
2451                                 slave_data->dual_emac_res_vlan = i+1;
2452                                 dev_err(&pdev->dev, "Using %d as Reserved VLAN for %d slave\n",
2453                                         slave_data->dual_emac_res_vlan, i);
2454                         } else {
2455                                 slave_data->dual_emac_res_vlan = prop;
2456                         }
2457                 }
2458
2459                 i++;
2460                 if (i == data->slaves)
2461                         break;
2462         }
2463
2464         return 0;
2465 }
2466
2467 static void cpsw_remove_dt(struct platform_device *pdev)
2468 {
2469         struct net_device *ndev = platform_get_drvdata(pdev);
2470         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2471         struct cpsw_platform_data *data = &cpsw->data;
2472         struct device_node *node = pdev->dev.of_node;
2473         struct device_node *slave_node;
2474         int i = 0;
2475
2476         for_each_available_child_of_node(node, slave_node) {
2477                 struct cpsw_slave_data *slave_data = &data->slave_data[i];
2478
2479                 if (strcmp(slave_node->name, "slave"))
2480                         continue;
2481
2482                 if (of_phy_is_fixed_link(slave_node))
2483                         of_phy_deregister_fixed_link(slave_node);
2484
2485                 of_node_put(slave_data->phy_node);
2486
2487                 i++;
2488                 if (i == data->slaves)
2489                         break;
2490         }
2491
2492         of_platform_depopulate(&pdev->dev);
2493 }
2494
2495 static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
2496 {
2497         struct cpsw_common              *cpsw = priv->cpsw;
2498         struct cpsw_platform_data       *data = &cpsw->data;
2499         struct net_device               *ndev;
2500         struct cpsw_priv                *priv_sl2;
2501         int ret = 0;
2502
2503         ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2504         if (!ndev) {
2505                 dev_err(cpsw->dev, "cpsw: error allocating net_device\n");
2506                 return -ENOMEM;
2507         }
2508
2509         priv_sl2 = netdev_priv(ndev);
2510         priv_sl2->cpsw = cpsw;
2511         priv_sl2->ndev = ndev;
2512         priv_sl2->dev  = &ndev->dev;
2513         priv_sl2->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2514
2515         if (is_valid_ether_addr(data->slave_data[1].mac_addr)) {
2516                 memcpy(priv_sl2->mac_addr, data->slave_data[1].mac_addr,
2517                         ETH_ALEN);
2518                 dev_info(cpsw->dev, "cpsw: Detected MACID = %pM\n",
2519                          priv_sl2->mac_addr);
2520         } else {
2521                 random_ether_addr(priv_sl2->mac_addr);
2522                 dev_info(cpsw->dev, "cpsw: Random MACID = %pM\n",
2523                          priv_sl2->mac_addr);
2524         }
2525         memcpy(ndev->dev_addr, priv_sl2->mac_addr, ETH_ALEN);
2526
2527         priv_sl2->emac_port = 1;
2528         cpsw->slaves[1].ndev = ndev;
2529         ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2530
2531         ndev->netdev_ops = &cpsw_netdev_ops;
2532         ndev->ethtool_ops = &cpsw_ethtool_ops;
2533
2534         /* register the network device */
2535         SET_NETDEV_DEV(ndev, cpsw->dev);
2536         ret = register_netdev(ndev);
2537         if (ret) {
2538                 dev_err(cpsw->dev, "cpsw: error registering net device\n");
2539                 free_netdev(ndev);
2540                 ret = -ENODEV;
2541         }
2542
2543         return ret;
2544 }
2545
2546 #define CPSW_QUIRK_IRQ          BIT(0)
2547
2548 static struct platform_device_id cpsw_devtype[] = {
2549         {
2550                 /* keep it for existing comaptibles */
2551                 .name = "cpsw",
2552                 .driver_data = CPSW_QUIRK_IRQ,
2553         }, {
2554                 .name = "am335x-cpsw",
2555                 .driver_data = CPSW_QUIRK_IRQ,
2556         }, {
2557                 .name = "am4372-cpsw",
2558                 .driver_data = 0,
2559         }, {
2560                 .name = "dra7-cpsw",
2561                 .driver_data = 0,
2562         }, {
2563                 /* sentinel */
2564         }
2565 };
2566 MODULE_DEVICE_TABLE(platform, cpsw_devtype);
2567
2568 enum ti_cpsw_type {
2569         CPSW = 0,
2570         AM335X_CPSW,
2571         AM4372_CPSW,
2572         DRA7_CPSW,
2573 };
2574
2575 static const struct of_device_id cpsw_of_mtable[] = {
2576         { .compatible = "ti,cpsw", .data = &cpsw_devtype[CPSW], },
2577         { .compatible = "ti,am335x-cpsw", .data = &cpsw_devtype[AM335X_CPSW], },
2578         { .compatible = "ti,am4372-cpsw", .data = &cpsw_devtype[AM4372_CPSW], },
2579         { .compatible = "ti,dra7-cpsw", .data = &cpsw_devtype[DRA7_CPSW], },
2580         { /* sentinel */ },
2581 };
2582 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
2583
2584 static int cpsw_probe(struct platform_device *pdev)
2585 {
2586         struct clk                      *clk;
2587         struct cpsw_platform_data       *data;
2588         struct net_device               *ndev;
2589         struct cpsw_priv                *priv;
2590         struct cpdma_params             dma_params;
2591         struct cpsw_ale_params          ale_params;
2592         void __iomem                    *ss_regs;
2593         struct resource                 *res, *ss_res;
2594         const struct of_device_id       *of_id;
2595         struct gpio_descs               *mode;
2596         u32 slave_offset, sliver_offset, slave_size;
2597         struct cpsw_common              *cpsw;
2598         int ret = 0, i;
2599         int irq;
2600
2601         cpsw = devm_kzalloc(&pdev->dev, sizeof(struct cpsw_common), GFP_KERNEL);
2602         if (!cpsw)
2603                 return -ENOMEM;
2604
2605         cpsw->dev = &pdev->dev;
2606
2607         ndev = alloc_etherdev_mq(sizeof(struct cpsw_priv), CPSW_MAX_QUEUES);
2608         if (!ndev) {
2609                 dev_err(&pdev->dev, "error allocating net_device\n");
2610                 return -ENOMEM;
2611         }
2612
2613         platform_set_drvdata(pdev, ndev);
2614         priv = netdev_priv(ndev);
2615         priv->cpsw = cpsw;
2616         priv->ndev = ndev;
2617         priv->dev  = &ndev->dev;
2618         priv->msg_enable = netif_msg_init(debug_level, CPSW_DEBUG);
2619         cpsw->rx_packet_max = max(rx_packet_max, 128);
2620         cpsw->cpts = devm_kzalloc(&pdev->dev, sizeof(struct cpts), GFP_KERNEL);
2621         if (!cpsw->cpts) {
2622                 dev_err(&pdev->dev, "error allocating cpts\n");
2623                 ret = -ENOMEM;
2624                 goto clean_ndev_ret;
2625         }
2626
2627         mode = devm_gpiod_get_array_optional(&pdev->dev, "mode", GPIOD_OUT_LOW);
2628         if (IS_ERR(mode)) {
2629                 ret = PTR_ERR(mode);
2630                 dev_err(&pdev->dev, "gpio request failed, ret %d\n", ret);
2631                 goto clean_ndev_ret;
2632         }
2633
2634         /*
2635          * This may be required here for child devices.
2636          */
2637         pm_runtime_enable(&pdev->dev);
2638
2639         /* Select default pin state */
2640         pinctrl_pm_select_default_state(&pdev->dev);
2641
2642         /* Need to enable clocks with runtime PM api to access module
2643          * registers
2644          */
2645         ret = pm_runtime_get_sync(&pdev->dev);
2646         if (ret < 0) {
2647                 pm_runtime_put_noidle(&pdev->dev);
2648                 goto clean_runtime_disable_ret;
2649         }
2650
2651         ret = cpsw_probe_dt(&cpsw->data, pdev);
2652         if (ret)
2653                 goto clean_dt_ret;
2654
2655         data = &cpsw->data;
2656         cpsw->rx_ch_num = 1;
2657         cpsw->tx_ch_num = 1;
2658
2659         if (is_valid_ether_addr(data->slave_data[0].mac_addr)) {
2660                 memcpy(priv->mac_addr, data->slave_data[0].mac_addr, ETH_ALEN);
2661                 dev_info(&pdev->dev, "Detected MACID = %pM\n", priv->mac_addr);
2662         } else {
2663                 eth_random_addr(priv->mac_addr);
2664                 dev_info(&pdev->dev, "Random MACID = %pM\n", priv->mac_addr);
2665         }
2666
2667         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
2668
2669         cpsw->slaves = devm_kzalloc(&pdev->dev,
2670                                     sizeof(struct cpsw_slave) * data->slaves,
2671                                     GFP_KERNEL);
2672         if (!cpsw->slaves) {
2673                 ret = -ENOMEM;
2674                 goto clean_dt_ret;
2675         }
2676         for (i = 0; i < data->slaves; i++)
2677                 cpsw->slaves[i].slave_num = i;
2678
2679         cpsw->slaves[0].ndev = ndev;
2680         priv->emac_port = 0;
2681
2682         clk = devm_clk_get(&pdev->dev, "fck");
2683         if (IS_ERR(clk)) {
2684                 dev_err(priv->dev, "fck is not found\n");
2685                 ret = -ENODEV;
2686                 goto clean_dt_ret;
2687         }
2688         cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000;
2689
2690         ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2691         ss_regs = devm_ioremap_resource(&pdev->dev, ss_res);
2692         if (IS_ERR(ss_regs)) {
2693                 ret = PTR_ERR(ss_regs);
2694                 goto clean_dt_ret;
2695         }
2696         cpsw->regs = ss_regs;
2697
2698         cpsw->version = readl(&cpsw->regs->id_ver);
2699
2700         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2701         cpsw->wr_regs = devm_ioremap_resource(&pdev->dev, res);
2702         if (IS_ERR(cpsw->wr_regs)) {
2703                 ret = PTR_ERR(cpsw->wr_regs);
2704                 goto clean_dt_ret;
2705         }
2706
2707         memset(&dma_params, 0, sizeof(dma_params));
2708         memset(&ale_params, 0, sizeof(ale_params));
2709
2710         switch (cpsw->version) {
2711         case CPSW_VERSION_1:
2712                 cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
2713                 cpsw->cpts->reg      = ss_regs + CPSW1_CPTS_OFFSET;
2714                 cpsw->hw_stats       = ss_regs + CPSW1_HW_STATS;
2715                 dma_params.dmaregs   = ss_regs + CPSW1_CPDMA_OFFSET;
2716                 dma_params.txhdp     = ss_regs + CPSW1_STATERAM_OFFSET;
2717                 ale_params.ale_regs  = ss_regs + CPSW1_ALE_OFFSET;
2718                 slave_offset         = CPSW1_SLAVE_OFFSET;
2719                 slave_size           = CPSW1_SLAVE_SIZE;
2720                 sliver_offset        = CPSW1_SLIVER_OFFSET;
2721                 dma_params.desc_mem_phys = 0;
2722                 break;
2723         case CPSW_VERSION_2:
2724         case CPSW_VERSION_3:
2725         case CPSW_VERSION_4:
2726                 cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
2727                 cpsw->cpts->reg      = ss_regs + CPSW2_CPTS_OFFSET;
2728                 cpsw->hw_stats       = ss_regs + CPSW2_HW_STATS;
2729                 dma_params.dmaregs   = ss_regs + CPSW2_CPDMA_OFFSET;
2730                 dma_params.txhdp     = ss_regs + CPSW2_STATERAM_OFFSET;
2731                 ale_params.ale_regs  = ss_regs + CPSW2_ALE_OFFSET;
2732                 slave_offset         = CPSW2_SLAVE_OFFSET;
2733                 slave_size           = CPSW2_SLAVE_SIZE;
2734                 sliver_offset        = CPSW2_SLIVER_OFFSET;
2735                 dma_params.desc_mem_phys =
2736                         (u32 __force) ss_res->start + CPSW2_BD_OFFSET;
2737                 break;
2738         default:
2739                 dev_err(priv->dev, "unknown version 0x%08x\n", cpsw->version);
2740                 ret = -ENODEV;
2741                 goto clean_dt_ret;
2742         }
2743         for (i = 0; i < cpsw->data.slaves; i++) {
2744                 struct cpsw_slave *slave = &cpsw->slaves[i];
2745
2746                 cpsw_slave_init(slave, cpsw, slave_offset, sliver_offset);
2747                 slave_offset  += slave_size;
2748                 sliver_offset += SLIVER_SIZE;
2749         }
2750
2751         dma_params.dev          = &pdev->dev;
2752         dma_params.rxthresh     = dma_params.dmaregs + CPDMA_RXTHRESH;
2753         dma_params.rxfree       = dma_params.dmaregs + CPDMA_RXFREE;
2754         dma_params.rxhdp        = dma_params.txhdp + CPDMA_RXHDP;
2755         dma_params.txcp         = dma_params.txhdp + CPDMA_TXCP;
2756         dma_params.rxcp         = dma_params.txhdp + CPDMA_RXCP;
2757
2758         dma_params.num_chan             = data->channels;
2759         dma_params.has_soft_reset       = true;
2760         dma_params.min_packet_size      = CPSW_MIN_PACKET_SIZE;
2761         dma_params.desc_mem_size        = data->bd_ram_size;
2762         dma_params.desc_align           = 16;
2763         dma_params.has_ext_regs         = true;
2764         dma_params.desc_hw_addr         = dma_params.desc_mem_phys;
2765
2766         cpsw->dma = cpdma_ctlr_create(&dma_params);
2767         if (!cpsw->dma) {
2768                 dev_err(priv->dev, "error initializing dma\n");
2769                 ret = -ENOMEM;
2770                 goto clean_dt_ret;
2771         }
2772
2773         cpsw->txch[0] = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
2774         cpsw->rxch[0] = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
2775         if (WARN_ON(!cpsw->rxch[0] || !cpsw->txch[0])) {
2776                 dev_err(priv->dev, "error initializing dma channels\n");
2777                 ret = -ENOMEM;
2778                 goto clean_dma_ret;
2779         }
2780
2781         ale_params.dev                  = &ndev->dev;
2782         ale_params.ale_ageout           = ale_ageout;
2783         ale_params.ale_entries          = data->ale_entries;
2784         ale_params.ale_ports            = data->slaves;
2785
2786         cpsw->ale = cpsw_ale_create(&ale_params);
2787         if (!cpsw->ale) {
2788                 dev_err(priv->dev, "error initializing ale engine\n");
2789                 ret = -ENODEV;
2790                 goto clean_dma_ret;
2791         }
2792
2793         ndev->irq = platform_get_irq(pdev, 1);
2794         if (ndev->irq < 0) {
2795                 dev_err(priv->dev, "error getting irq resource\n");
2796                 ret = ndev->irq;
2797                 goto clean_ale_ret;
2798         }
2799
2800         of_id = of_match_device(cpsw_of_mtable, &pdev->dev);
2801         if (of_id) {
2802                 pdev->id_entry = of_id->data;
2803                 if (pdev->id_entry->driver_data)
2804                         cpsw->quirk_irq = true;
2805         }
2806
2807         /* Grab RX and TX IRQs. Note that we also have RX_THRESHOLD and
2808          * MISC IRQs which are always kept disabled with this driver so
2809          * we will not request them.
2810          *
2811          * If anyone wants to implement support for those, make sure to
2812          * first request and append them to irqs_table array.
2813          */
2814
2815         /* RX IRQ */
2816         irq = platform_get_irq(pdev, 1);
2817         if (irq < 0) {
2818                 ret = irq;
2819                 goto clean_ale_ret;
2820         }
2821
2822         cpsw->irqs_table[0] = irq;
2823         ret = devm_request_irq(&pdev->dev, irq, cpsw_rx_interrupt,
2824                                0, dev_name(&pdev->dev), cpsw);
2825         if (ret < 0) {
2826                 dev_err(priv->dev, "error attaching irq (%d)\n", ret);
2827                 goto clean_ale_ret;
2828         }
2829
2830         /* TX IRQ */
2831         irq = platform_get_irq(pdev, 2);
2832         if (irq < 0) {
2833                 ret = irq;
2834                 goto clean_ale_ret;
2835         }
2836
2837         cpsw->irqs_table[1] = irq;
2838         ret = devm_request_irq(&pdev->dev, irq, cpsw_tx_interrupt,
2839                                0, dev_name(&pdev->dev), cpsw);
2840         if (ret < 0) {
2841                 dev_err(priv->dev, "error attaching irq (%d)\n", ret);
2842                 goto clean_ale_ret;
2843         }
2844
2845         ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2846
2847         ndev->netdev_ops = &cpsw_netdev_ops;
2848         ndev->ethtool_ops = &cpsw_ethtool_ops;
2849         netif_napi_add(ndev, &cpsw->napi_rx, cpsw_rx_poll, CPSW_POLL_WEIGHT);
2850         netif_tx_napi_add(ndev, &cpsw->napi_tx, cpsw_tx_poll, CPSW_POLL_WEIGHT);
2851
2852         /* register the network device */
2853         SET_NETDEV_DEV(ndev, &pdev->dev);
2854         ret = register_netdev(ndev);
2855         if (ret) {
2856                 dev_err(priv->dev, "error registering net device\n");
2857                 ret = -ENODEV;
2858                 goto clean_ale_ret;
2859         }
2860
2861         cpsw_notice(priv, probe, "initialized device (regs %pa, irq %d)\n",
2862                     &ss_res->start, ndev->irq);
2863
2864         if (cpsw->data.dual_emac) {
2865                 ret = cpsw_probe_dual_emac(priv);
2866                 if (ret) {
2867                         cpsw_err(priv, probe, "error probe slave 2 emac interface\n");
2868                         goto clean_unregister_netdev_ret;
2869                 }
2870         }
2871
2872         pm_runtime_put(&pdev->dev);
2873
2874         return 0;
2875
2876 clean_unregister_netdev_ret:
2877         unregister_netdev(ndev);
2878 clean_ale_ret:
2879         cpsw_ale_destroy(cpsw->ale);
2880 clean_dma_ret:
2881         cpdma_ctlr_destroy(cpsw->dma);
2882 clean_dt_ret:
2883         cpsw_remove_dt(pdev);
2884         pm_runtime_put_sync(&pdev->dev);
2885 clean_runtime_disable_ret:
2886         pm_runtime_disable(&pdev->dev);
2887 clean_ndev_ret:
2888         free_netdev(priv->ndev);
2889         return ret;
2890 }
2891
2892 static int cpsw_remove(struct platform_device *pdev)
2893 {
2894         struct net_device *ndev = platform_get_drvdata(pdev);
2895         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2896         int ret;
2897
2898         ret = pm_runtime_get_sync(&pdev->dev);
2899         if (ret < 0) {
2900                 pm_runtime_put_noidle(&pdev->dev);
2901                 return ret;
2902         }
2903
2904         if (cpsw->data.dual_emac)
2905                 unregister_netdev(cpsw->slaves[1].ndev);
2906         unregister_netdev(ndev);
2907
2908         cpsw_ale_destroy(cpsw->ale);
2909         cpdma_ctlr_destroy(cpsw->dma);
2910         cpsw_remove_dt(pdev);
2911         pm_runtime_put_sync(&pdev->dev);
2912         pm_runtime_disable(&pdev->dev);
2913         if (cpsw->data.dual_emac)
2914                 free_netdev(cpsw->slaves[1].ndev);
2915         free_netdev(ndev);
2916         return 0;
2917 }
2918
2919 #ifdef CONFIG_PM_SLEEP
2920 static int cpsw_suspend(struct device *dev)
2921 {
2922         struct platform_device  *pdev = to_platform_device(dev);
2923         struct net_device       *ndev = platform_get_drvdata(pdev);
2924         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
2925
2926         if (cpsw->data.dual_emac) {
2927                 int i;
2928
2929                 for (i = 0; i < cpsw->data.slaves; i++) {
2930                         if (netif_running(cpsw->slaves[i].ndev))
2931                                 cpsw_ndo_stop(cpsw->slaves[i].ndev);
2932                 }
2933         } else {
2934                 if (netif_running(ndev))
2935                         cpsw_ndo_stop(ndev);
2936         }
2937
2938         /* Select sleep pin state */
2939         pinctrl_pm_select_sleep_state(dev);
2940
2941         return 0;
2942 }
2943
2944 static int cpsw_resume(struct device *dev)
2945 {
2946         struct platform_device  *pdev = to_platform_device(dev);
2947         struct net_device       *ndev = platform_get_drvdata(pdev);
2948         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
2949
2950         /* Select default pin state */
2951         pinctrl_pm_select_default_state(dev);
2952
2953         /* shut up ASSERT_RTNL() warning in netif_set_real_num_tx/rx_queues */
2954         rtnl_lock();
2955         if (cpsw->data.dual_emac) {
2956                 int i;
2957
2958                 for (i = 0; i < cpsw->data.slaves; i++) {
2959                         if (netif_running(cpsw->slaves[i].ndev))
2960                                 cpsw_ndo_open(cpsw->slaves[i].ndev);
2961                 }
2962         } else {
2963                 if (netif_running(ndev))
2964                         cpsw_ndo_open(ndev);
2965         }
2966         rtnl_unlock();
2967
2968         return 0;
2969 }
2970 #endif
2971
2972 static SIMPLE_DEV_PM_OPS(cpsw_pm_ops, cpsw_suspend, cpsw_resume);
2973
2974 static struct platform_driver cpsw_driver = {
2975         .driver = {
2976                 .name    = "cpsw",
2977                 .pm      = &cpsw_pm_ops,
2978                 .of_match_table = cpsw_of_mtable,
2979         },
2980         .probe = cpsw_probe,
2981         .remove = cpsw_remove,
2982 };
2983
2984 module_platform_driver(cpsw_driver);
2985
2986 MODULE_LICENSE("GPL");
2987 MODULE_AUTHOR("Cyril Chemparathy <cyril@ti.com>");
2988 MODULE_AUTHOR("Mugunthan V N <mugunthanvnm@ti.com>");
2989 MODULE_DESCRIPTION("TI CPSW Ethernet driver");