drivers/net/ethernet/apm/xgene/xgene_enet_main.c

   1 /* Applied Micro X-Gene SoC Ethernet Driver
   2  *
   3  * Copyright (c) 2014, Applied Micro Circuits Corporation
   4  * Authors: Iyappan Subramanian <isubramanian@apm.com>
   5  *          Ravi Patel <rapatel@apm.com>
   6  *          Keyur Chudgar <kchudgar@apm.com>
   7  *
   8  * This program is free software; you can redistribute  it and/or modify it
   9  * under  the terms of  the GNU General  Public License as published by the
  10  * Free Software Foundation;  either version 2 of the  License, or (at your
  11  * option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 #include <linux/gpio.h>
  23 #include "xgene_enet_main.h"
  24 #include "xgene_enet_hw.h"
  25 #include "xgene_enet_sgmac.h"
  26 #include "xgene_enet_xgmac.h"
  27
  28 #define RES_ENET_CSR    0
  29 #define RES_RING_CSR    1
  30 #define RES_RING_CMD    2
  31
  32 static void xgene_enet_init_bufpool(struct xgene_enet_desc_ring *buf_pool)
  33 {
  34         struct xgene_enet_raw_desc16 *raw_desc;
  35         int i;
  36
  37         if (!buf_pool)
  38                 return;
  39
  40         for (i = 0; i < buf_pool->slots; i++) {
  41                 raw_desc = &buf_pool->raw_desc16[i];
  42
  43                 /* Hardware expects descriptor in little endian format */
  44                 raw_desc->m0 = cpu_to_le64(i |
  45                                 SET_VAL(FPQNUM, buf_pool->dst_ring_num) |
  46                                 SET_VAL(STASH, 3));
  47         }
  48 }
  49
  50 static u16 xgene_enet_get_data_len(u64 bufdatalen)
  51 {
  52         u16 hw_len, mask;
  53
  54         hw_len = GET_VAL(BUFDATALEN, bufdatalen);
  55
  56         if (unlikely(hw_len == 0x7800)) {
  57                 return 0;
  58         } else if (!(hw_len & BIT(14))) {
  59                 mask = GENMASK(13, 0);
  60                 return (hw_len & mask) ? (hw_len & mask) : SIZE_16K;
  61         } else if (!(hw_len & GENMASK(13, 12))) {
  62                 mask = GENMASK(11, 0);
  63                 return (hw_len & mask) ? (hw_len & mask) : SIZE_4K;
  64         } else {
  65                 mask = GENMASK(11, 0);
  66                 return (hw_len & mask) ? (hw_len & mask) : SIZE_2K;
  67         }
  68 }
  69
  70 static u16 xgene_enet_set_data_len(u32 size)
  71 {
  72         u16 hw_len;
  73
  74         hw_len =  (size == SIZE_4K) ? BIT(14) : 0;
  75
  76         return hw_len;
  77 }
  78
  79 static int xgene_enet_refill_pagepool(struct xgene_enet_desc_ring *buf_pool,
  80                                       u32 nbuf)
  81 {
  82         struct xgene_enet_raw_desc16 *raw_desc;
  83         struct xgene_enet_pdata *pdata;
  84         struct net_device *ndev;
  85         dma_addr_t dma_addr;
  86         struct device *dev;
  87         struct page *page;
  88         u32 slots, tail;
  89         u16 hw_len;
  90         int i;
  91
  92         if (unlikely(!buf_pool))
  93                 return 0;
  94
  95         ndev = buf_pool->ndev;
  96         pdata = netdev_priv(ndev);
  97         dev = ndev_to_dev(ndev);
  98         slots = buf_pool->slots - 1;
  99         tail = buf_pool->tail;
 100
 101         for (i = 0; i < nbuf; i++) {
 102                 raw_desc = &buf_pool->raw_desc16[tail];
 103
 104                 page = dev_alloc_page();
 105                 if (unlikely(!page))
 106                         return -ENOMEM;
 107
 108                 dma_addr = dma_map_page(dev, page, 0,
 109                                         PAGE_SIZE, DMA_FROM_DEVICE);
 110                 if (unlikely(dma_mapping_error(dev, dma_addr))) {
 111                         put_page(page);
 112                         return -ENOMEM;
 113                 }
 114
 115                 hw_len = xgene_enet_set_data_len(PAGE_SIZE);
 116                 raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 117                                            SET_VAL(BUFDATALEN, hw_len) |
 118                                            SET_BIT(COHERENT));
 119
 120                 buf_pool->frag_page[tail] = page;
 121                 tail = (tail + 1) & slots;
 122         }
 123
 124         pdata->ring_ops->wr_cmd(buf_pool, nbuf);
 125         buf_pool->tail = tail;
 126
 127         return 0;
 128 }
 129
 130 static int xgene_enet_refill_bufpool(struct xgene_enet_desc_ring *buf_pool,
 131                                      u32 nbuf)
 132 {
 133         struct sk_buff *skb;
 134         struct xgene_enet_raw_desc16 *raw_desc;
 135         struct xgene_enet_pdata *pdata;
 136         struct net_device *ndev;
 137         struct device *dev;
 138         dma_addr_t dma_addr;
 139         u32 tail = buf_pool->tail;
 140         u32 slots = buf_pool->slots - 1;
 141         u16 bufdatalen, len;
 142         int i;
 143
 144         ndev = buf_pool->ndev;
 145         dev = ndev_to_dev(buf_pool->ndev);
 146         pdata = netdev_priv(ndev);
 147
 148         bufdatalen = BUF_LEN_CODE_2K | (SKB_BUFFER_SIZE & GENMASK(11, 0));
 149         len = XGENE_ENET_STD_MTU;
 150
 151         for (i = 0; i < nbuf; i++) {
 152                 raw_desc = &buf_pool->raw_desc16[tail];
 153
 154                 skb = netdev_alloc_skb_ip_align(ndev, len);
 155                 if (unlikely(!skb))
 156                         return -ENOMEM;
 157
 158                 dma_addr = dma_map_single(dev, skb->data, len, DMA_FROM_DEVICE);
 159                 if (dma_mapping_error(dev, dma_addr)) {
 160                         netdev_err(ndev, "DMA mapping error\n");
 161                         dev_kfree_skb_any(skb);
 162                         return -EINVAL;
 163                 }
 164
 165                 buf_pool->rx_skb[tail] = skb;
 166
 167                 raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 168                                            SET_VAL(BUFDATALEN, bufdatalen) |
 169                                            SET_BIT(COHERENT));
 170                 tail = (tail + 1) & slots;
 171         }
 172
 173         pdata->ring_ops->wr_cmd(buf_pool, nbuf);
 174         buf_pool->tail = tail;
 175
 176         return 0;
 177 }
 178
 179 static u8 xgene_enet_hdr_len(const void *data)
 180 {
 181         const struct ethhdr *eth = data;
 182
 183         return (eth->h_proto == htons(ETH_P_8021Q)) ? VLAN_ETH_HLEN : ETH_HLEN;
 184 }
 185
 186 static void xgene_enet_delete_bufpool(struct xgene_enet_desc_ring *buf_pool)
 187 {
 188         struct device *dev = ndev_to_dev(buf_pool->ndev);
 189         struct xgene_enet_raw_desc16 *raw_desc;
 190         dma_addr_t dma_addr;
 191         int i;
 192
 193         /* Free up the buffers held by hardware */
 194         for (i = 0; i < buf_pool->slots; i++) {
 195                 if (buf_pool->rx_skb[i]) {
 196                         dev_kfree_skb_any(buf_pool->rx_skb[i]);
 197
 198                         raw_desc = &buf_pool->raw_desc16[i];
 199                         dma_addr = GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1));
 200                         dma_unmap_single(dev, dma_addr, XGENE_ENET_MAX_MTU,
 201                                          DMA_FROM_DEVICE);
 202                 }
 203         }
 204 }
 205
 206 static void xgene_enet_delete_pagepool(struct xgene_enet_desc_ring *buf_pool)
 207 {
 208         struct device *dev = ndev_to_dev(buf_pool->ndev);
 209         dma_addr_t dma_addr;
 210         struct page *page;
 211         int i;
 212
 213         /* Free up the buffers held by hardware */
 214         for (i = 0; i < buf_pool->slots; i++) {
 215                 page = buf_pool->frag_page[i];
 216                 if (page) {
 217                         dma_addr = buf_pool->frag_dma_addr[i];
 218                         dma_unmap_page(dev, dma_addr, PAGE_SIZE,
 219                                        DMA_FROM_DEVICE);
 220                         put_page(page);
 221                 }
 222         }
 223 }
 224
 225 static irqreturn_t xgene_enet_rx_irq(const int irq, void *data)
 226 {
 227         struct xgene_enet_desc_ring *rx_ring = data;
 228
 229         if (napi_schedule_prep(&rx_ring->napi)) {
 230                 disable_irq_nosync(irq);
 231                 __napi_schedule(&rx_ring->napi);
 232         }
 233
 234         return IRQ_HANDLED;
 235 }
 236
 237 static int xgene_enet_tx_completion(struct xgene_enet_desc_ring *cp_ring,
 238                                     struct xgene_enet_raw_desc *raw_desc)
 239 {
 240         struct xgene_enet_pdata *pdata = netdev_priv(cp_ring->ndev);
 241         struct sk_buff *skb;
 242         struct device *dev;
 243         skb_frag_t *frag;
 244         dma_addr_t *frag_dma_addr;
 245         u16 skb_index;
 246         u8 mss_index;
 247         u8 status;
 248         int i;
 249
 250         skb_index = GET_VAL(USERINFO, le64_to_cpu(raw_desc->m0));
 251         skb = cp_ring->cp_skb[skb_index];
 252         frag_dma_addr = &cp_ring->frag_dma_addr[skb_index * MAX_SKB_FRAGS];
 253
 254         dev = ndev_to_dev(cp_ring->ndev);
 255         dma_unmap_single(dev, GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1)),
 256                          skb_headlen(skb),
 257                          DMA_TO_DEVICE);
 258
 259         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 260                 frag = &skb_shinfo(skb)->frags[i];
 261                 dma_unmap_page(dev, frag_dma_addr[i], skb_frag_size(frag),
 262                                DMA_TO_DEVICE);
 263         }
 264
 265         if (GET_BIT(ET, le64_to_cpu(raw_desc->m3))) {
 266                 mss_index = GET_VAL(MSS, le64_to_cpu(raw_desc->m3));
 267                 spin_lock(&pdata->mss_lock);
 268                 pdata->mss_refcnt[mss_index]--;
 269                 spin_unlock(&pdata->mss_lock);
 270         }
 271
 272         /* Checking for error */
 273         status = GET_VAL(LERR, le64_to_cpu(raw_desc->m0));
 274         if (unlikely(status > 2)) {
 275                 cp_ring->tx_dropped++;
 276                 cp_ring->tx_errors++;
 277         }
 278
 279         if (likely(skb)) {
 280                 dev_kfree_skb_any(skb);
 281         } else {
 282                 netdev_err(cp_ring->ndev, "completion skb is NULL\n");
 283         }
 284
 285         return 0;
 286 }
 287
 288 static int xgene_enet_setup_mss(struct net_device *ndev, u32 mss)
 289 {
 290         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 291         int mss_index = -EBUSY;
 292         int i;
 293
 294         spin_lock(&pdata->mss_lock);
 295
 296         /* Reuse the slot if MSS matches */
 297         for (i = 0; mss_index < 0 && i < NUM_MSS_REG; i++) {
 298                 if (pdata->mss[i] == mss) {
 299                         pdata->mss_refcnt[i]++;
 300                         mss_index = i;
 301                 }
 302         }
 303
 304         /* Overwrite the slot with ref_count = 0 */
 305         for (i = 0; mss_index < 0 && i < NUM_MSS_REG; i++) {
 306                 if (!pdata->mss_refcnt[i]) {
 307                         pdata->mss_refcnt[i]++;
 308                         pdata->mac_ops->set_mss(pdata, mss, i);
 309                         pdata->mss[i] = mss;
 310                         mss_index = i;
 311                 }
 312         }
 313
 314         spin_unlock(&pdata->mss_lock);
 315
 316         return mss_index;
 317 }
 318
 319 static int xgene_enet_work_msg(struct sk_buff *skb, u64 *hopinfo)
 320 {
 321         struct net_device *ndev = skb->dev;
 322         struct iphdr *iph;
 323         u8 l3hlen = 0, l4hlen = 0;
 324         u8 ethhdr, proto = 0, csum_enable = 0;
 325         u32 hdr_len, mss = 0;
 326         u32 i, len, nr_frags;
 327         int mss_index;
 328
 329         ethhdr = xgene_enet_hdr_len(skb->data);
 330
 331         if (unlikely(skb->protocol != htons(ETH_P_IP)) &&
 332             unlikely(skb->protocol != htons(ETH_P_8021Q)))
 333                 goto out;
 334
 335         if (unlikely(!(skb->dev->features & NETIF_F_IP_CSUM)))
 336                 goto out;
 337
 338         iph = ip_hdr(skb);
 339         if (unlikely(ip_is_fragment(iph)))
 340                 goto out;
 341
 342         if (likely(iph->protocol == IPPROTO_TCP)) {
 343                 l4hlen = tcp_hdrlen(skb) >> 2;
 344                 csum_enable = 1;
 345                 proto = TSO_IPPROTO_TCP;
 346                 if (ndev->features & NETIF_F_TSO) {
 347                         hdr_len = ethhdr + ip_hdrlen(skb) + tcp_hdrlen(skb);
 348                         mss = skb_shinfo(skb)->gso_size;
 349
 350                         if (skb_is_nonlinear(skb)) {
 351                                 len = skb_headlen(skb);
 352                                 nr_frags = skb_shinfo(skb)->nr_frags;
 353
 354                                 for (i = 0; i < 2 && i < nr_frags; i++)
 355                                         len += skb_shinfo(skb)->frags[i].size;
 356
 357                                 /* HW requires header must reside in 3 buffer */
 358                                 if (unlikely(hdr_len > len)) {
 359                                         if (skb_linearize(skb))
 360                                                 return 0;
 361                                 }
 362                         }
 363
 364                         if (!mss || ((skb->len - hdr_len) <= mss))
 365                                 goto out;
 366
 367                         mss_index = xgene_enet_setup_mss(ndev, mss);
 368                         if (unlikely(mss_index < 0))
 369                                 return -EBUSY;
 370
 371                         *hopinfo |= SET_BIT(ET) | SET_VAL(MSS, mss_index);
 372                 }
 373         } else if (iph->protocol == IPPROTO_UDP) {
 374                 l4hlen = UDP_HDR_SIZE;
 375                 csum_enable = 1;
 376         }
 377 out:
 378         l3hlen = ip_hdrlen(skb) >> 2;
 379         *hopinfo |= SET_VAL(TCPHDR, l4hlen) |
 380                     SET_VAL(IPHDR, l3hlen) |
 381                     SET_VAL(ETHHDR, ethhdr) |
 382                     SET_VAL(EC, csum_enable) |
 383                     SET_VAL(IS, proto) |
 384                     SET_BIT(IC) |
 385                     SET_BIT(TYPE_ETH_WORK_MESSAGE);
 386
 387         return 0;
 388 }
 389
 390 static u16 xgene_enet_encode_len(u16 len)
 391 {
 392         return (len == BUFLEN_16K) ? 0 : len;
 393 }
 394
 395 static void xgene_set_addr_len(__le64 *desc, u32 idx, dma_addr_t addr, u32 len)
 396 {
 397         desc[idx ^ 1] = cpu_to_le64(SET_VAL(DATAADDR, addr) |
 398                                     SET_VAL(BUFDATALEN, len));
 399 }
 400
 401 static __le64 *xgene_enet_get_exp_bufs(struct xgene_enet_desc_ring *ring)
 402 {
 403         __le64 *exp_bufs;
 404
 405         exp_bufs = &ring->exp_bufs[ring->exp_buf_tail * MAX_EXP_BUFFS];
 406         memset(exp_bufs, 0, sizeof(__le64) * MAX_EXP_BUFFS);
 407         ring->exp_buf_tail = (ring->exp_buf_tail + 1) & ((ring->slots / 2) - 1);
 408
 409         return exp_bufs;
 410 }
 411
 412 static dma_addr_t *xgene_get_frag_dma_array(struct xgene_enet_desc_ring *ring)
 413 {
 414         return &ring->cp_ring->frag_dma_addr[ring->tail * MAX_SKB_FRAGS];
 415 }
 416
 417 static int xgene_enet_setup_tx_desc(struct xgene_enet_desc_ring *tx_ring,
 418                                     struct sk_buff *skb)
 419 {
 420         struct device *dev = ndev_to_dev(tx_ring->ndev);
 421         struct xgene_enet_pdata *pdata = netdev_priv(tx_ring->ndev);
 422         struct xgene_enet_raw_desc *raw_desc;
 423         __le64 *exp_desc = NULL, *exp_bufs = NULL;
 424         dma_addr_t dma_addr, pbuf_addr, *frag_dma_addr;
 425         skb_frag_t *frag;
 426         u16 tail = tx_ring->tail;
 427         u64 hopinfo = 0;
 428         u32 len, hw_len;
 429         u8 ll = 0, nv = 0, idx = 0;
 430         bool split = false;
 431         u32 size, offset, ell_bytes = 0;
 432         u32 i, fidx, nr_frags, count = 1;
 433         int ret;
 434
 435         raw_desc = &tx_ring->raw_desc[tail];
 436         tail = (tail + 1) & (tx_ring->slots - 1);
 437         memset(raw_desc, 0, sizeof(struct xgene_enet_raw_desc));
 438
 439         ret = xgene_enet_work_msg(skb, &hopinfo);
 440         if (ret)
 441                 return ret;
 442
 443         raw_desc->m3 = cpu_to_le64(SET_VAL(HENQNUM, tx_ring->dst_ring_num) |
 444                                    hopinfo);
 445
 446         len = skb_headlen(skb);
 447         hw_len = xgene_enet_encode_len(len);
 448
 449         dma_addr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 450         if (dma_mapping_error(dev, dma_addr)) {
 451                 netdev_err(tx_ring->ndev, "DMA mapping error\n");
 452                 return -EINVAL;
 453         }
 454
 455         /* Hardware expects descriptor in little endian format */
 456         raw_desc->m1 = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 457                                    SET_VAL(BUFDATALEN, hw_len) |
 458                                    SET_BIT(COHERENT));
 459
 460         if (!skb_is_nonlinear(skb))
 461                 goto out;
 462
 463         /* scatter gather */
 464         nv = 1;
 465         exp_desc = (void *)&tx_ring->raw_desc[tail];
 466         tail = (tail + 1) & (tx_ring->slots - 1);
 467         memset(exp_desc, 0, sizeof(struct xgene_enet_raw_desc));
 468
 469         nr_frags = skb_shinfo(skb)->nr_frags;
 470         for (i = nr_frags; i < 4 ; i++)
 471                 exp_desc[i ^ 1] = cpu_to_le64(LAST_BUFFER);
 472
 473         frag_dma_addr = xgene_get_frag_dma_array(tx_ring);
 474
 475         for (i = 0, fidx = 0; split || (fidx < nr_frags); i++) {
 476                 if (!split) {
 477                         frag = &skb_shinfo(skb)->frags[fidx];
 478                         size = skb_frag_size(frag);
 479                         offset = 0;
 480
 481                         pbuf_addr = skb_frag_dma_map(dev, frag, 0, size,
 482                                                      DMA_TO_DEVICE);
 483                         if (dma_mapping_error(dev, pbuf_addr))
 484                                 return -EINVAL;
 485
 486                         frag_dma_addr[fidx] = pbuf_addr;
 487                         fidx++;
 488
 489                         if (size > BUFLEN_16K)
 490                                 split = true;
 491                 }
 492
 493                 if (size > BUFLEN_16K) {
 494                         len = BUFLEN_16K;
 495                         size -= BUFLEN_16K;
 496                 } else {
 497                         len = size;
 498                         split = false;
 499                 }
 500
 501                 dma_addr = pbuf_addr + offset;
 502                 hw_len = xgene_enet_encode_len(len);
 503
 504                 switch (i) {
 505                 case 0:
 506                 case 1:
 507                 case 2:
 508                         xgene_set_addr_len(exp_desc, i, dma_addr, hw_len);
 509                         break;
 510                 case 3:
 511                         if (split || (fidx != nr_frags)) {
 512                                 exp_bufs = xgene_enet_get_exp_bufs(tx_ring);
 513                                 xgene_set_addr_len(exp_bufs, idx, dma_addr,
 514                                                    hw_len);
 515                                 idx++;
 516                                 ell_bytes += len;
 517                         } else {
 518                                 xgene_set_addr_len(exp_desc, i, dma_addr,
 519                                                    hw_len);
 520                         }
 521                         break;
 522                 default:
 523                         xgene_set_addr_len(exp_bufs, idx, dma_addr, hw_len);
 524                         idx++;
 525                         ell_bytes += len;
 526                         break;
 527                 }
 528
 529                 if (split)
 530                         offset += BUFLEN_16K;
 531         }
 532         count++;
 533
 534         if (idx) {
 535                 ll = 1;
 536                 dma_addr = dma_map_single(dev, exp_bufs,
 537                                           sizeof(u64) * MAX_EXP_BUFFS,
 538                                           DMA_TO_DEVICE);
 539                 if (dma_mapping_error(dev, dma_addr)) {
 540                         dev_kfree_skb_any(skb);
 541                         return -EINVAL;
 542                 }
 543                 i = ell_bytes >> LL_BYTES_LSB_LEN;
 544                 exp_desc[2] = cpu_to_le64(SET_VAL(DATAADDR, dma_addr) |
 545                                           SET_VAL(LL_BYTES_MSB, i) |
 546                                           SET_VAL(LL_LEN, idx));
 547                 raw_desc->m2 = cpu_to_le64(SET_VAL(LL_BYTES_LSB, ell_bytes));
 548         }
 549
 550 out:
 551         raw_desc->m0 = cpu_to_le64(SET_VAL(LL, ll) | SET_VAL(NV, nv) |
 552                                    SET_VAL(USERINFO, tx_ring->tail));
 553         tx_ring->cp_ring->cp_skb[tx_ring->tail] = skb;
 554         pdata->tx_level[tx_ring->cp_ring->index] += count;
 555         tx_ring->tail = tail;
 556
 557         return count;
 558 }
 559
 560 static netdev_tx_t xgene_enet_start_xmit(struct sk_buff *skb,
 561                                          struct net_device *ndev)
 562 {
 563         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 564         struct xgene_enet_desc_ring *tx_ring;
 565         int index = skb->queue_mapping;
 566         u32 tx_level = pdata->tx_level[index];
 567         int count;
 568
 569         tx_ring = pdata->tx_ring[index];
 570         if (tx_level < pdata->txc_level[index])
 571                 tx_level += ((typeof(pdata->tx_level[index]))~0U);
 572
 573         if ((tx_level - pdata->txc_level[index]) > pdata->tx_qcnt_hi) {
 574                 netif_stop_subqueue(ndev, index);
 575                 return NETDEV_TX_BUSY;
 576         }
 577
 578         if (skb_padto(skb, XGENE_MIN_ENET_FRAME_SIZE))
 579                 return NETDEV_TX_OK;
 580
 581         count = xgene_enet_setup_tx_desc(tx_ring, skb);
 582         if (count == -EBUSY)
 583                 return NETDEV_TX_BUSY;
 584
 585         if (count <= 0) {
 586                 dev_kfree_skb_any(skb);
 587                 return NETDEV_TX_OK;
 588         }
 589
 590         skb_tx_timestamp(skb);
 591
 592         tx_ring->tx_packets++;
 593         tx_ring->tx_bytes += skb->len;
 594
 595         pdata->ring_ops->wr_cmd(tx_ring, count);
 596         return NETDEV_TX_OK;
 597 }
 598
 599 static void xgene_enet_rx_csum(struct sk_buff *skb)
 600 {
 601         struct net_device *ndev = skb->dev;
 602         struct iphdr *iph = ip_hdr(skb);
 603
 604         if (!(ndev->features & NETIF_F_RXCSUM))
 605                 return;
 606
 607         if (skb->protocol != htons(ETH_P_IP))
 608                 return;
 609
 610         if (ip_is_fragment(iph))
 611                 return;
 612
 613         if (iph->protocol != IPPROTO_TCP && iph->protocol != IPPROTO_UDP)
 614                 return;
 615
 616         skb->ip_summed = CHECKSUM_UNNECESSARY;
 617 }
 618
 619 static void xgene_enet_free_pagepool(struct xgene_enet_desc_ring *buf_pool,
 620                                      struct xgene_enet_raw_desc *raw_desc,
 621                                      struct xgene_enet_raw_desc *exp_desc)
 622 {
 623         __le64 *desc = (void *)exp_desc;
 624         dma_addr_t dma_addr;
 625         struct device *dev;
 626         struct page *page;
 627         u16 slots, head;
 628         u32 frag_size;
 629         int i;
 630
 631         if (!buf_pool || !raw_desc || !exp_desc ||
 632             (!GET_VAL(NV, le64_to_cpu(raw_desc->m0))))
 633                 return;
 634
 635         dev = ndev_to_dev(buf_pool->ndev);
 636         slots = buf_pool->slots - 1;
 637         head = buf_pool->head;
 638
 639         for (i = 0; i < 4; i++) {
 640                 frag_size = xgene_enet_get_data_len(le64_to_cpu(desc[i ^ 1]));
 641                 if (!frag_size)
 642                         break;
 643
 644                 dma_addr = GET_VAL(DATAADDR, le64_to_cpu(desc[i ^ 1]));
 645                 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
 646
 647                 page = buf_pool->frag_page[head];
 648                 put_page(page);
 649
 650                 buf_pool->frag_page[head] = NULL;
 651                 head = (head + 1) & slots;
 652         }
 653         buf_pool->head = head;
 654 }
 655
 656 /* Errata 10GE_10 and ENET_15 - Fix duplicated HW statistic counters */
 657 static bool xgene_enet_errata_10GE_10(struct sk_buff *skb, u32 len, u8 status)
 658 {
 659         if (status == INGRESS_CRC &&
 660             len >= (ETHER_STD_PACKET + 1) &&
 661             len <= (ETHER_STD_PACKET + 4) &&
 662             skb->protocol == htons(ETH_P_8021Q))
 663                 return true;
 664
 665         return false;
 666 }
 667
 668 /* Errata 10GE_8 and ENET_11 - allow packet with length <=64B */
 669 static bool xgene_enet_errata_10GE_8(struct sk_buff *skb, u32 len, u8 status)
 670 {
 671         if (status == INGRESS_PKT_LEN && len == ETHER_MIN_PACKET) {
 672                 if (ntohs(eth_hdr(skb)->h_proto) < 46)
 673                         return true;
 674         }
 675
 676         return false;
 677 }
 678
 679 static int xgene_enet_rx_frame(struct xgene_enet_desc_ring *rx_ring,
 680                                struct xgene_enet_raw_desc *raw_desc,
 681                                struct xgene_enet_raw_desc *exp_desc)
 682 {
 683         struct xgene_enet_desc_ring *buf_pool, *page_pool;
 684         u32 datalen, frag_size, skb_index;
 685         struct xgene_enet_pdata *pdata;
 686         struct net_device *ndev;
 687         dma_addr_t dma_addr;
 688         struct sk_buff *skb;
 689         struct device *dev;
 690         struct page *page;
 691         u16 slots, head;
 692         int i, ret = 0;
 693         __le64 *desc;
 694         u8 status;
 695         bool nv;
 696
 697         ndev = rx_ring->ndev;
 698         pdata = netdev_priv(ndev);
 699         dev = ndev_to_dev(rx_ring->ndev);
 700         buf_pool = rx_ring->buf_pool;
 701         page_pool = rx_ring->page_pool;
 702
 703         dma_unmap_single(dev, GET_VAL(DATAADDR, le64_to_cpu(raw_desc->m1)),
 704                          XGENE_ENET_STD_MTU, DMA_FROM_DEVICE);
 705         skb_index = GET_VAL(USERINFO, le64_to_cpu(raw_desc->m0));
 706         skb = buf_pool->rx_skb[skb_index];
 707         buf_pool->rx_skb[skb_index] = NULL;
 708
 709         datalen = xgene_enet_get_data_len(le64_to_cpu(raw_desc->m1));
 710
 711         /* strip off CRC as HW isn't doing this */
 712         nv = GET_VAL(NV, le64_to_cpu(raw_desc->m0));
 713         if (!nv)
 714                 datalen -= 4;
 715
 716         skb_put(skb, datalen);
 717         prefetch(skb->data - NET_IP_ALIGN);
 718         skb->protocol = eth_type_trans(skb, ndev);
 719
 720         /* checking for error */
 721         status = (GET_VAL(ELERR, le64_to_cpu(raw_desc->m0)) << LERR_LEN) |
 722                   GET_VAL(LERR, le64_to_cpu(raw_desc->m0));
 723         if (unlikely(status)) {
 724                 if (xgene_enet_errata_10GE_8(skb, datalen, status)) {
 725                         pdata->false_rflr++;
 726                 } else if (xgene_enet_errata_10GE_10(skb, datalen, status)) {
 727                         pdata->vlan_rjbr++;
 728                 } else {
 729                         dev_kfree_skb_any(skb);
 730                         xgene_enet_free_pagepool(page_pool, raw_desc, exp_desc);
 731                         xgene_enet_parse_error(rx_ring, status);
 732                         rx_ring->rx_dropped++;
 733                         goto out;
 734                 }
 735         }
 736
 737         if (!nv)
 738                 goto skip_jumbo;
 739
 740         slots = page_pool->slots - 1;
 741         head = page_pool->head;
 742         desc = (void *)exp_desc;
 743
 744         for (i = 0; i < 4; i++) {
 745                 frag_size = xgene_enet_get_data_len(le64_to_cpu(desc[i ^ 1]));
 746                 if (!frag_size)
 747                         break;
 748
 749                 dma_addr = GET_VAL(DATAADDR, le64_to_cpu(desc[i ^ 1]));
 750                 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
 751
 752                 page = page_pool->frag_page[head];
 753                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 0,
 754                                 frag_size, PAGE_SIZE);
 755
 756                 datalen += frag_size;
 757
 758                 page_pool->frag_page[head] = NULL;
 759                 head = (head + 1) & slots;
 760         }
 761
 762         page_pool->head = head;
 763         rx_ring->npagepool -= skb_shinfo(skb)->nr_frags;
 764
 765 skip_jumbo:
 766         skb_checksum_none_assert(skb);
 767         xgene_enet_rx_csum(skb);
 768
 769         rx_ring->rx_packets++;
 770         rx_ring->rx_bytes += datalen;
 771         napi_gro_receive(&rx_ring->napi, skb);
 772
 773 out:
 774         if (rx_ring->npagepool <= 0) {
 775                 ret = xgene_enet_refill_pagepool(page_pool, NUM_NXTBUFPOOL);
 776                 rx_ring->npagepool = NUM_NXTBUFPOOL;
 777                 if (ret)
 778                         return ret;
 779         }
 780
 781         if (--rx_ring->nbufpool == 0) {
 782                 ret = xgene_enet_refill_bufpool(buf_pool, NUM_BUFPOOL);
 783                 rx_ring->nbufpool = NUM_BUFPOOL;
 784         }
 785
 786         return ret;
 787 }
 788
 789 static bool is_rx_desc(struct xgene_enet_raw_desc *raw_desc)
 790 {
 791         return GET_VAL(FPQNUM, le64_to_cpu(raw_desc->m0)) ? true : false;
 792 }
 793
 794 static int xgene_enet_process_ring(struct xgene_enet_desc_ring *ring,
 795                                    int budget)
 796 {
 797         struct net_device *ndev = ring->ndev;
 798         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 799         struct xgene_enet_raw_desc *raw_desc, *exp_desc;
 800         u16 head = ring->head;
 801         u16 slots = ring->slots - 1;
 802         int ret, desc_count, count = 0, processed = 0;
 803         bool is_completion;
 804
 805         do {
 806                 raw_desc = &ring->raw_desc[head];
 807                 desc_count = 0;
 808                 is_completion = false;
 809                 exp_desc = NULL;
 810                 if (unlikely(xgene_enet_is_desc_slot_empty(raw_desc)))
 811                         break;
 812
 813                 /* read fpqnum field after dataaddr field */
 814                 dma_rmb();
 815                 if (GET_BIT(NV, le64_to_cpu(raw_desc->m0))) {
 816                         head = (head + 1) & slots;
 817                         exp_desc = &ring->raw_desc[head];
 818
 819                         if (unlikely(xgene_enet_is_desc_slot_empty(exp_desc))) {
 820                                 head = (head - 1) & slots;
 821                                 break;
 822                         }
 823                         dma_rmb();
 824                         count++;
 825                         desc_count++;
 826                 }
 827                 if (is_rx_desc(raw_desc)) {
 828                         ret = xgene_enet_rx_frame(ring, raw_desc, exp_desc);
 829                 } else {
 830                         ret = xgene_enet_tx_completion(ring, raw_desc);
 831                         is_completion = true;
 832                 }
 833                 xgene_enet_mark_desc_slot_empty(raw_desc);
 834                 if (exp_desc)
 835                         xgene_enet_mark_desc_slot_empty(exp_desc);
 836
 837                 head = (head + 1) & slots;
 838                 count++;
 839                 desc_count++;
 840                 processed++;
 841                 if (is_completion)
 842                         pdata->txc_level[ring->index] += desc_count;
 843
 844                 if (ret)
 845                         break;
 846         } while (--budget);
 847
 848         if (likely(count)) {
 849                 pdata->ring_ops->wr_cmd(ring, -count);
 850                 ring->head = head;
 851
 852                 if (__netif_subqueue_stopped(ndev, ring->index))
 853                         netif_start_subqueue(ndev, ring->index);
 854         }
 855
 856         return processed;
 857 }
 858
 859 static int xgene_enet_napi(struct napi_struct *napi, const int budget)
 860 {
 861         struct xgene_enet_desc_ring *ring;
 862         int processed;
 863
 864         ring = container_of(napi, struct xgene_enet_desc_ring, napi);
 865         processed = xgene_enet_process_ring(ring, budget);
 866
 867         if (processed != budget) {
 868                 napi_complete_done(napi, processed);
 869                 enable_irq(ring->irq);
 870         }
 871
 872         return processed;
 873 }
 874
 875 static void xgene_enet_timeout(struct net_device *ndev)
 876 {
 877         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 878         struct netdev_queue *txq;
 879         int i;
 880
 881         pdata->mac_ops->reset(pdata);
 882
 883         for (i = 0; i < pdata->txq_cnt; i++) {
 884                 txq = netdev_get_tx_queue(ndev, i);
 885                 txq->trans_start = jiffies;
 886                 netif_tx_start_queue(txq);
 887         }
 888 }
 889
 890 static void xgene_enet_set_irq_name(struct net_device *ndev)
 891 {
 892         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 893         struct xgene_enet_desc_ring *ring;
 894         int i;
 895
 896         for (i = 0; i < pdata->rxq_cnt; i++) {
 897                 ring = pdata->rx_ring[i];
 898                 if (!pdata->cq_cnt) {
 899                         snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-rx-txc",
 900                                  ndev->name);
 901                 } else {
 902                         snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-rx-%d",
 903                                  ndev->name, i);
 904                 }
 905         }
 906
 907         for (i = 0; i < pdata->cq_cnt; i++) {
 908                 ring = pdata->tx_ring[i]->cp_ring;
 909                 snprintf(ring->irq_name, IRQ_ID_SIZE, "%s-txc-%d",
 910                          ndev->name, i);
 911         }
 912 }
 913
 914 static int xgene_enet_register_irq(struct net_device *ndev)
 915 {
 916         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
 917         struct device *dev = ndev_to_dev(ndev);
 918         struct xgene_enet_desc_ring *ring;
 919         int ret = 0, i;
 920
 921         xgene_enet_set_irq_name(ndev);
 922         for (i = 0; i < pdata->rxq_cnt; i++) {
 923                 ring = pdata->rx_ring[i];
 924                 irq_set_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 925                 ret = devm_request_irq(dev, ring->irq, xgene_enet_rx_irq,
 926                                        0, ring->irq_name, ring);
 927                 if (ret) {
 928                         netdev_err(ndev, "Failed to request irq %s\n",
 929                                    ring->irq_name);
 930                 }
 931         }
 932
 933         for (i = 0; i < pdata->cq_cnt; i++) {
 934                 ring = pdata->tx_ring[i]->cp_ring;
 935                 irq_set_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 936                 ret = devm_request_irq(dev, ring->irq, xgene_enet_rx_irq,
 937                                        0, ring->irq_name, ring);
 938                 if (ret) {
 939                         netdev_err(ndev, "Failed to request irq %s\n",
 940                                    ring->irq_name);
 941                 }
 942         }
 943
 944         return ret;
 945 }
 946
 947 static void xgene_enet_free_irq(struct net_device *ndev)
 948 {
 949         struct xgene_enet_pdata *pdata;
 950         struct xgene_enet_desc_ring *ring;
 951         struct device *dev;
 952         int i;
 953
 954         pdata = netdev_priv(ndev);
 955         dev = ndev_to_dev(ndev);
 956
 957         for (i = 0; i < pdata->rxq_cnt; i++) {
 958                 ring = pdata->rx_ring[i];
 959                 irq_clear_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 960                 devm_free_irq(dev, ring->irq, ring);
 961         }
 962
 963         for (i = 0; i < pdata->cq_cnt; i++) {
 964                 ring = pdata->tx_ring[i]->cp_ring;
 965                 irq_clear_status_flags(ring->irq, IRQ_DISABLE_UNLAZY);
 966                 devm_free_irq(dev, ring->irq, ring);
 967         }
 968 }
 969
 970 static void xgene_enet_napi_enable(struct xgene_enet_pdata *pdata)
 971 {
 972         struct napi_struct *napi;
 973         int i;
 974
 975         for (i = 0; i < pdata->rxq_cnt; i++) {
 976                 napi = &pdata->rx_ring[i]->napi;
 977                 napi_enable(napi);
 978         }
 979
 980         for (i = 0; i < pdata->cq_cnt; i++) {
 981                 napi = &pdata->tx_ring[i]->cp_ring->napi;
 982                 napi_enable(napi);
 983         }
 984 }
 985
 986 static void xgene_enet_napi_disable(struct xgene_enet_pdata *pdata)
 987 {
 988         struct napi_struct *napi;
 989         int i;
 990
 991         for (i = 0; i < pdata->rxq_cnt; i++) {
 992                 napi = &pdata->rx_ring[i]->napi;
 993                 napi_disable(napi);
 994         }
 995
 996         for (i = 0; i < pdata->cq_cnt; i++) {
 997                 napi = &pdata->tx_ring[i]->cp_ring->napi;
 998                 napi_disable(napi);
 999         }
1000 }
1001
1002 static int xgene_enet_open(struct net_device *ndev)
1003 {
1004         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1005         const struct xgene_mac_ops *mac_ops = pdata->mac_ops;
1006         int ret;
1007
1008         ret = netif_set_real_num_tx_queues(ndev, pdata->txq_cnt);
1009         if (ret)
1010                 return ret;
1011
1012         ret = netif_set_real_num_rx_queues(ndev, pdata->rxq_cnt);
1013         if (ret)
1014                 return ret;
1015
1016         xgene_enet_napi_enable(pdata);
1017         ret = xgene_enet_register_irq(ndev);
1018         if (ret) {
1019                 xgene_enet_napi_disable(pdata);
1020                 return ret;
1021         }
1022
1023         if (ndev->phydev) {
1024                 phy_start(ndev->phydev);
1025         } else {
1026                 schedule_delayed_work(&pdata->link_work, PHY_POLL_LINK_OFF);
1027                 netif_carrier_off(ndev);
1028         }
1029
1030         mac_ops->tx_enable(pdata);
1031         mac_ops->rx_enable(pdata);
1032         netif_tx_start_all_queues(ndev);
1033
1034         return ret;
1035 }
1036
1037 static int xgene_enet_close(struct net_device *ndev)
1038 {
1039         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1040         const struct xgene_mac_ops *mac_ops = pdata->mac_ops;
1041         int i;
1042
1043         netif_tx_stop_all_queues(ndev);
1044         mac_ops->tx_disable(pdata);
1045         mac_ops->rx_disable(pdata);
1046
1047         if (ndev->phydev)
1048                 phy_stop(ndev->phydev);
1049         else
1050                 cancel_delayed_work_sync(&pdata->link_work);
1051
1052         xgene_enet_free_irq(ndev);
1053         xgene_enet_napi_disable(pdata);
1054         for (i = 0; i < pdata->rxq_cnt; i++)
1055                 xgene_enet_process_ring(pdata->rx_ring[i], -1);
1056
1057         return 0;
1058 }
1059 static void xgene_enet_delete_ring(struct xgene_enet_desc_ring *ring)
1060 {
1061         struct xgene_enet_pdata *pdata;
1062         struct device *dev;
1063
1064         pdata = netdev_priv(ring->ndev);
1065         dev = ndev_to_dev(ring->ndev);
1066
1067         pdata->ring_ops->clear(ring);
1068         dmam_free_coherent(dev, ring->size, ring->desc_addr, ring->dma);
1069 }
1070
1071 static void xgene_enet_delete_desc_rings(struct xgene_enet_pdata *pdata)
1072 {
1073         struct xgene_enet_desc_ring *buf_pool, *page_pool;
1074         struct xgene_enet_desc_ring *ring;
1075         int i;
1076
1077         for (i = 0; i < pdata->txq_cnt; i++) {
1078                 ring = pdata->tx_ring[i];
1079                 if (ring) {
1080                         xgene_enet_delete_ring(ring);
1081                         pdata->port_ops->clear(pdata, ring);
1082                         if (pdata->cq_cnt)
1083                                 xgene_enet_delete_ring(ring->cp_ring);
1084                         pdata->tx_ring[i] = NULL;
1085                 }
1086
1087         }
1088
1089         for (i = 0; i < pdata->rxq_cnt; i++) {
1090                 ring = pdata->rx_ring[i];
1091                 if (ring) {
1092                         page_pool = ring->page_pool;
1093                         if (page_pool) {
1094                                 xgene_enet_delete_pagepool(page_pool);
1095                                 xgene_enet_delete_ring(page_pool);
1096                                 pdata->port_ops->clear(pdata, page_pool);
1097                         }
1098
1099                         buf_pool = ring->buf_pool;
1100                         xgene_enet_delete_bufpool(buf_pool);
1101                         xgene_enet_delete_ring(buf_pool);
1102                         pdata->port_ops->clear(pdata, buf_pool);
1103
1104                         xgene_enet_delete_ring(ring);
1105                         pdata->rx_ring[i] = NULL;
1106                 }
1107
1108         }
1109 }
1110
1111 static int xgene_enet_get_ring_size(struct device *dev,
1112                                     enum xgene_enet_ring_cfgsize cfgsize)
1113 {
1114         int size = -EINVAL;
1115
1116         switch (cfgsize) {
1117         case RING_CFGSIZE_512B:
1118                 size = 0x200;
1119                 break;
1120         case RING_CFGSIZE_2KB:
1121                 size = 0x800;
1122                 break;
1123         case RING_CFGSIZE_16KB:
1124                 size = 0x4000;
1125                 break;
1126         case RING_CFGSIZE_64KB:
1127                 size = 0x10000;
1128                 break;
1129         case RING_CFGSIZE_512KB:
1130                 size = 0x80000;
1131                 break;
1132         default:
1133                 dev_err(dev, "Unsupported cfg ring size %d\n", cfgsize);
1134                 break;
1135         }
1136
1137         return size;
1138 }
1139
1140 static void xgene_enet_free_desc_ring(struct xgene_enet_desc_ring *ring)
1141 {
1142         struct xgene_enet_pdata *pdata;
1143         struct device *dev;
1144
1145         if (!ring)
1146                 return;
1147
1148         dev = ndev_to_dev(ring->ndev);
1149         pdata = netdev_priv(ring->ndev);
1150
1151         if (ring->desc_addr) {
1152                 pdata->ring_ops->clear(ring);
1153                 dmam_free_coherent(dev, ring->size, ring->desc_addr, ring->dma);
1154         }
1155         devm_kfree(dev, ring);
1156 }
1157
1158 static void xgene_enet_free_desc_rings(struct xgene_enet_pdata *pdata)
1159 {
1160         struct xgene_enet_desc_ring *page_pool;
1161         struct device *dev = &pdata->pdev->dev;
1162         struct xgene_enet_desc_ring *ring;
1163         void *p;
1164         int i;
1165
1166         for (i = 0; i < pdata->txq_cnt; i++) {
1167                 ring = pdata->tx_ring[i];
1168                 if (ring) {
1169                         if (ring->cp_ring && ring->cp_ring->cp_skb)
1170                                 devm_kfree(dev, ring->cp_ring->cp_skb);
1171
1172                         if (ring->cp_ring && pdata->cq_cnt)
1173                                 xgene_enet_free_desc_ring(ring->cp_ring);
1174
1175                         xgene_enet_free_desc_ring(ring);
1176                 }
1177
1178         }
1179
1180         for (i = 0; i < pdata->rxq_cnt; i++) {
1181                 ring = pdata->rx_ring[i];
1182                 if (ring) {
1183                         if (ring->buf_pool) {
1184                                 if (ring->buf_pool->rx_skb)
1185                                         devm_kfree(dev, ring->buf_pool->rx_skb);
1186
1187                                 xgene_enet_free_desc_ring(ring->buf_pool);
1188                         }
1189
1190                         page_pool = ring->page_pool;
1191                         if (page_pool) {
1192                                 p = page_pool->frag_page;
1193                                 if (p)
1194                                         devm_kfree(dev, p);
1195
1196                                 p = page_pool->frag_dma_addr;
1197                                 if (p)
1198                                         devm_kfree(dev, p);
1199                         }
1200
1201                         xgene_enet_free_desc_ring(ring);
1202                 }
1203         }
1204 }
1205
1206 static bool is_irq_mbox_required(struct xgene_enet_pdata *pdata,
1207                                  struct xgene_enet_desc_ring *ring)
1208 {
1209         if ((pdata->enet_id == XGENE_ENET2) &&
1210             (xgene_enet_ring_owner(ring->id) == RING_OWNER_CPU)) {
1211                 return true;
1212         }
1213
1214         return false;
1215 }
1216
1217 static void __iomem *xgene_enet_ring_cmd_base(struct xgene_enet_pdata *pdata,
1218                                               struct xgene_enet_desc_ring *ring)
1219 {
1220         u8 num_ring_id_shift = pdata->ring_ops->num_ring_id_shift;
1221
1222         return pdata->ring_cmd_addr + (ring->num << num_ring_id_shift);
1223 }
1224
1225 static struct xgene_enet_desc_ring *xgene_enet_create_desc_ring(
1226                         struct net_device *ndev, u32 ring_num,
1227                         enum xgene_enet_ring_cfgsize cfgsize, u32 ring_id)
1228 {
1229         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1230         struct device *dev = ndev_to_dev(ndev);
1231         struct xgene_enet_desc_ring *ring;
1232         void *irq_mbox_addr;
1233         int size;
1234
1235         size = xgene_enet_get_ring_size(dev, cfgsize);
1236         if (size < 0)
1237                 return NULL;
1238
1239         ring = devm_kzalloc(dev, sizeof(struct xgene_enet_desc_ring),
1240                             GFP_KERNEL);
1241         if (!ring)
1242                 return NULL;
1243
1244         ring->ndev = ndev;
1245         ring->num = ring_num;
1246         ring->cfgsize = cfgsize;
1247         ring->id = ring_id;
1248
1249         ring->desc_addr = dmam_alloc_coherent(dev, size, &ring->dma,
1250                                               GFP_KERNEL | __GFP_ZERO);
1251         if (!ring->desc_addr) {
1252                 devm_kfree(dev, ring);
1253                 return NULL;
1254         }
1255         ring->size = size;
1256
1257         if (is_irq_mbox_required(pdata, ring)) {
1258                 irq_mbox_addr = dmam_alloc_coherent(dev, INTR_MBOX_SIZE,
1259                                                     &ring->irq_mbox_dma,
1260                                                     GFP_KERNEL | __GFP_ZERO);
1261                 if (!irq_mbox_addr) {
1262                         dmam_free_coherent(dev, size, ring->desc_addr,
1263                                            ring->dma);
1264                         devm_kfree(dev, ring);
1265                         return NULL;
1266                 }
1267                 ring->irq_mbox_addr = irq_mbox_addr;
1268         }
1269
1270         ring->cmd_base = xgene_enet_ring_cmd_base(pdata, ring);
1271         ring->cmd = ring->cmd_base + INC_DEC_CMD_ADDR;
1272         ring = pdata->ring_ops->setup(ring);
1273         netdev_dbg(ndev, "ring info: num=%d  size=%d  id=%d  slots=%d\n",
1274                    ring->num, ring->size, ring->id, ring->slots);
1275
1276         return ring;
1277 }
1278
1279 static u16 xgene_enet_get_ring_id(enum xgene_ring_owner owner, u8 bufnum)
1280 {
1281         return (owner << 6) | (bufnum & GENMASK(5, 0));
1282 }
1283
1284 static enum xgene_ring_owner xgene_derive_ring_owner(struct xgene_enet_pdata *p)
1285 {
1286         enum xgene_ring_owner owner;
1287
1288         if (p->enet_id == XGENE_ENET1) {
1289                 switch (p->phy_mode) {
1290                 case PHY_INTERFACE_MODE_SGMII:
1291                         owner = RING_OWNER_ETH0;
1292                         break;
1293                 default:
1294                         owner = (!p->port_id) ? RING_OWNER_ETH0 :
1295                                                 RING_OWNER_ETH1;
1296                         break;
1297                 }
1298         } else {
1299                 owner = (!p->port_id) ? RING_OWNER_ETH0 : RING_OWNER_ETH1;
1300         }
1301
1302         return owner;
1303 }
1304
1305 static u8 xgene_start_cpu_bufnum(struct xgene_enet_pdata *pdata)
1306 {
1307         struct device *dev = &pdata->pdev->dev;
1308         u32 cpu_bufnum;
1309         int ret;
1310
1311         ret = device_property_read_u32(dev, "channel", &cpu_bufnum);
1312
1313         return (!ret) ? cpu_bufnum : pdata->cpu_bufnum;
1314 }
1315
1316 static int xgene_enet_create_desc_rings(struct net_device *ndev)
1317 {
1318         struct xgene_enet_desc_ring *rx_ring, *tx_ring, *cp_ring;
1319         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1320         struct xgene_enet_desc_ring *page_pool = NULL;
1321         struct xgene_enet_desc_ring *buf_pool = NULL;
1322         struct device *dev = ndev_to_dev(ndev);
1323         u8 eth_bufnum = pdata->eth_bufnum;
1324         u8 bp_bufnum = pdata->bp_bufnum;
1325         u16 ring_num = pdata->ring_num;
1326         enum xgene_ring_owner owner;
1327         dma_addr_t dma_exp_bufs;
1328         u16 ring_id, slots;
1329         __le64 *exp_bufs;
1330         int i, ret, size;
1331         u8 cpu_bufnum;
1332
1333         cpu_bufnum = xgene_start_cpu_bufnum(pdata);
1334
1335         for (i = 0; i < pdata->rxq_cnt; i++) {
1336                 /* allocate rx descriptor ring */
1337                 owner = xgene_derive_ring_owner(pdata);
1338                 ring_id = xgene_enet_get_ring_id(RING_OWNER_CPU, cpu_bufnum++);
1339                 rx_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1340                                                       RING_CFGSIZE_16KB,
1341                                                       ring_id);
1342                 if (!rx_ring) {
1343                         ret = -ENOMEM;
1344                         goto err;
1345                 }
1346
1347                 /* allocate buffer pool for receiving packets */
1348                 owner = xgene_derive_ring_owner(pdata);
1349                 ring_id = xgene_enet_get_ring_id(owner, bp_bufnum++);
1350                 buf_pool = xgene_enet_create_desc_ring(ndev, ring_num++,
1351                                                        RING_CFGSIZE_16KB,
1352                                                        ring_id);
1353                 if (!buf_pool) {
1354                         ret = -ENOMEM;
1355                         goto err;
1356                 }
1357
1358                 rx_ring->nbufpool = NUM_BUFPOOL;
1359                 rx_ring->npagepool = NUM_NXTBUFPOOL;
1360                 rx_ring->irq = pdata->irqs[i];
1361                 buf_pool->rx_skb = devm_kcalloc(dev, buf_pool->slots,
1362                                                 sizeof(struct sk_buff *),
1363                                                 GFP_KERNEL);
1364                 if (!buf_pool->rx_skb) {
1365                         ret = -ENOMEM;
1366                         goto err;
1367                 }
1368
1369                 buf_pool->dst_ring_num = xgene_enet_dst_ring_num(buf_pool);
1370                 rx_ring->buf_pool = buf_pool;
1371                 pdata->rx_ring[i] = rx_ring;
1372
1373                 if ((pdata->enet_id == XGENE_ENET1 &&  pdata->rxq_cnt > 4) ||
1374                     (pdata->enet_id == XGENE_ENET2 &&  pdata->rxq_cnt > 16)) {
1375                         break;
1376                 }
1377
1378                 /* allocate next buffer pool for jumbo packets */
1379                 owner = xgene_derive_ring_owner(pdata);
1380                 ring_id = xgene_enet_get_ring_id(owner, bp_bufnum++);
1381                 page_pool = xgene_enet_create_desc_ring(ndev, ring_num++,
1382                                                         RING_CFGSIZE_16KB,
1383                                                         ring_id);
1384                 if (!page_pool) {
1385                         ret = -ENOMEM;
1386                         goto err;
1387                 }
1388
1389                 slots = page_pool->slots;
1390                 page_pool->frag_page = devm_kcalloc(dev, slots,
1391                                                     sizeof(struct page *),
1392                                                     GFP_KERNEL);
1393                 if (!page_pool->frag_page) {
1394                         ret = -ENOMEM;
1395                         goto err;
1396                 }
1397
1398                 page_pool->frag_dma_addr = devm_kcalloc(dev, slots,
1399                                                         sizeof(dma_addr_t),
1400                                                         GFP_KERNEL);
1401                 if (!page_pool->frag_dma_addr) {
1402                         ret = -ENOMEM;
1403                         goto err;
1404                 }
1405
1406                 page_pool->dst_ring_num = xgene_enet_dst_ring_num(page_pool);
1407                 rx_ring->page_pool = page_pool;
1408         }
1409
1410         for (i = 0; i < pdata->txq_cnt; i++) {
1411                 /* allocate tx descriptor ring */
1412                 owner = xgene_derive_ring_owner(pdata);
1413                 ring_id = xgene_enet_get_ring_id(owner, eth_bufnum++);
1414                 tx_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1415                                                       RING_CFGSIZE_16KB,
1416                                                       ring_id);
1417                 if (!tx_ring) {
1418                         ret = -ENOMEM;
1419                         goto err;
1420                 }
1421
1422                 size = (tx_ring->slots / 2) * sizeof(__le64) * MAX_EXP_BUFFS;
1423                 exp_bufs = dmam_alloc_coherent(dev, size, &dma_exp_bufs,
1424                                                GFP_KERNEL | __GFP_ZERO);
1425                 if (!exp_bufs) {
1426                         ret = -ENOMEM;
1427                         goto err;
1428                 }
1429                 tx_ring->exp_bufs = exp_bufs;
1430
1431                 pdata->tx_ring[i] = tx_ring;
1432
1433                 if (!pdata->cq_cnt) {
1434                         cp_ring = pdata->rx_ring[i];
1435                 } else {
1436                         /* allocate tx completion descriptor ring */
1437                         ring_id = xgene_enet_get_ring_id(RING_OWNER_CPU,
1438                                                          cpu_bufnum++);
1439                         cp_ring = xgene_enet_create_desc_ring(ndev, ring_num++,
1440                                                               RING_CFGSIZE_16KB,
1441                                                               ring_id);
1442                         if (!cp_ring) {
1443                                 ret = -ENOMEM;
1444                                 goto err;
1445                         }
1446
1447                         cp_ring->irq = pdata->irqs[pdata->rxq_cnt + i];
1448                         cp_ring->index = i;
1449                 }
1450
1451                 cp_ring->cp_skb = devm_kcalloc(dev, tx_ring->slots,
1452                                                sizeof(struct sk_buff *),
1453                                                GFP_KERNEL);
1454                 if (!cp_ring->cp_skb) {
1455                         ret = -ENOMEM;
1456                         goto err;
1457                 }
1458
1459                 size = sizeof(dma_addr_t) * MAX_SKB_FRAGS;
1460                 cp_ring->frag_dma_addr = devm_kcalloc(dev, tx_ring->slots,
1461                                                       size, GFP_KERNEL);
1462                 if (!cp_ring->frag_dma_addr) {
1463                         devm_kfree(dev, cp_ring->cp_skb);
1464                         ret = -ENOMEM;
1465                         goto err;
1466                 }
1467
1468                 tx_ring->cp_ring = cp_ring;
1469                 tx_ring->dst_ring_num = xgene_enet_dst_ring_num(cp_ring);
1470         }
1471
1472         if (pdata->ring_ops->coalesce)
1473                 pdata->ring_ops->coalesce(pdata->tx_ring[0]);
1474         pdata->tx_qcnt_hi = pdata->tx_ring[0]->slots - 128;
1475
1476         return 0;
1477
1478 err:
1479         xgene_enet_free_desc_rings(pdata);
1480         return ret;
1481 }
1482
1483 static void xgene_enet_get_stats64(
1484                         struct net_device *ndev,
1485                         struct rtnl_link_stats64 *stats)
1486 {
1487         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1488         struct xgene_enet_desc_ring *ring;
1489         int i;
1490
1491         for (i = 0; i < pdata->txq_cnt; i++) {
1492                 ring = pdata->tx_ring[i];
1493                 if (ring) {
1494                         stats->tx_packets += ring->tx_packets;
1495                         stats->tx_bytes += ring->tx_bytes;
1496                         stats->tx_dropped += ring->tx_dropped;
1497                         stats->tx_errors += ring->tx_errors;
1498                 }
1499         }
1500
1501         for (i = 0; i < pdata->rxq_cnt; i++) {
1502                 ring = pdata->rx_ring[i];
1503                 if (ring) {
1504                         stats->rx_packets += ring->rx_packets;
1505                         stats->rx_bytes += ring->rx_bytes;
1506                         stats->rx_dropped += ring->rx_dropped;
1507                         stats->rx_errors += ring->rx_errors +
1508                                 ring->rx_length_errors +
1509                                 ring->rx_crc_errors +
1510                                 ring->rx_frame_errors +
1511                                 ring->rx_fifo_errors;
1512                         stats->rx_length_errors += ring->rx_length_errors;
1513                         stats->rx_crc_errors += ring->rx_crc_errors;
1514                         stats->rx_frame_errors += ring->rx_frame_errors;
1515                         stats->rx_fifo_errors += ring->rx_fifo_errors;
1516                 }
1517         }
1518 }
1519
1520 static int xgene_enet_set_mac_address(struct net_device *ndev, void *addr)
1521 {
1522         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1523         int ret;
1524
1525         ret = eth_mac_addr(ndev, addr);
1526         if (ret)
1527                 return ret;
1528         pdata->mac_ops->set_mac_addr(pdata);
1529
1530         return ret;
1531 }
1532
1533 static int xgene_change_mtu(struct net_device *ndev, int new_mtu)
1534 {
1535         struct xgene_enet_pdata *pdata = netdev_priv(ndev);
1536         int frame_size;
1537
1538         if (!netif_running(ndev))
1539                 return 0;
1540
1541         frame_size = (new_mtu > ETH_DATA_LEN) ? (new_mtu + 18) : 0x600;
1542
1543         xgene_enet_close(ndev);
1544         ndev->mtu = new_mtu;
1545         pdata->mac_ops->set_framesize(pdata, frame_size);
1546         xgene_enet_open(ndev);
1547
1548         return 0;
1549 }
1550
1551 static const struct net_device_ops xgene_ndev_ops = {
1552         .ndo_open = xgene_enet_open,
1553         .ndo_stop = xgene_enet_close,
1554         .ndo_start_xmit = xgene_enet_start_xmit,
1555         .ndo_tx_timeout = xgene_enet_timeout,
1556         .ndo_get_stats64 = xgene_enet_get_stats64,
1557         .ndo_change_mtu = xgene_change_mtu,
1558         .ndo_set_mac_address = xgene_enet_set_mac_address,
1559 };
1560
1561 #ifdef CONFIG_ACPI
1562 static void xgene_get_port_id_acpi(struct device *dev,
1563                                   struct xgene_enet_pdata *pdata)
1564 {
1565         acpi_status status;
1566         u64 temp;
1567
1568         status = acpi_evaluate_integer(ACPI_HANDLE(dev), "_SUN", NULL, &temp);
1569         if (ACPI_FAILURE(status)) {
1570                 pdata->port_id = 0;
1571         } else {
1572                 pdata->port_id = temp;
1573         }
1574
1575         return;
1576 }
1577 #endif
1578
1579 static void xgene_get_port_id_dt(struct device *dev, struct xgene_enet_pdata *pdata)
1580 {
1581         u32 id = 0;
1582
1583         of_property_read_u32(dev->of_node, "port-id", &id);
1584
1585         pdata->port_id = id & BIT(0);
1586
1587         return;
1588 }
1589
1590 static int xgene_get_tx_delay(struct xgene_enet_pdata *pdata)
1591 {
1592         struct device *dev = &pdata->pdev->dev;
1593         int delay, ret;
1594
1595         ret = device_property_read_u32(dev, "tx-delay", &delay);
1596         if (ret) {
1597                 pdata->tx_delay = 4;
1598                 return 0;
1599         }
1600
1601         if (delay < 0 || delay > 7) {
1602                 dev_err(dev, "Invalid tx-delay specified\n");
1603                 return -EINVAL;
1604         }
1605
1606         pdata->tx_delay = delay;
1607
1608         return 0;
1609 }
1610
1611 static int xgene_get_rx_delay(struct xgene_enet_pdata *pdata)
1612 {
1613         struct device *dev = &pdata->pdev->dev;
1614         int delay, ret;
1615
1616         ret = device_property_read_u32(dev, "rx-delay", &delay);
1617         if (ret) {
1618                 pdata->rx_delay = 2;
1619                 return 0;
1620         }
1621
1622         if (delay < 0 || delay > 7) {
1623                 dev_err(dev, "Invalid rx-delay specified\n");
1624                 return -EINVAL;
1625         }
1626
1627         pdata->rx_delay = delay;
1628
1629         return 0;
1630 }
1631
1632 static int xgene_enet_get_irqs(struct xgene_enet_pdata *pdata)
1633 {
1634         struct platform_device *pdev = pdata->pdev;
1635         struct device *dev = &pdev->dev;
1636         int i, ret, max_irqs;
1637
1638         if (phy_interface_mode_is_rgmii(pdata->phy_mode))
1639                 max_irqs = 1;
1640         else if (pdata->phy_mode == PHY_INTERFACE_MODE_SGMII)
1641                 max_irqs = 2;
1642         else
1643                 max_irqs = XGENE_MAX_ENET_IRQ;
1644
1645         for (i = 0; i < max_irqs; i++) {
1646                 ret = platform_get_irq(pdev, i);
1647                 if (ret <= 0) {
1648                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1649                                 max_irqs = i;
1650                                 pdata->rxq_cnt = max_irqs / 2;
1651                                 pdata->txq_cnt = max_irqs / 2;
1652                                 pdata->cq_cnt = max_irqs / 2;
1653                                 break;
1654                         }
1655                         dev_err(dev, "Unable to get ENET IRQ\n");
1656                         ret = ret ? : -ENXIO;
1657                         return ret;
1658                 }
1659                 pdata->irqs[i] = ret;
1660         }
1661
1662         return 0;
1663 }
1664
1665 static void xgene_enet_check_phy_handle(struct xgene_enet_pdata *pdata)
1666 {
1667         int ret;
1668
1669         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII)
1670                 return;
1671
1672         if (!IS_ENABLED(CONFIG_MDIO_XGENE))
1673                 return;
1674
1675         ret = xgene_enet_phy_connect(pdata->ndev);
1676         if (!ret)
1677                 pdata->mdio_driver = true;
1678 }
1679
1680 static void xgene_enet_gpiod_get(struct xgene_enet_pdata *pdata)
1681 {
1682         struct device *dev = &pdata->pdev->dev;
1683
1684         pdata->sfp_gpio_en = false;
1685         if (pdata->phy_mode != PHY_INTERFACE_MODE_XGMII ||
1686             (!device_property_present(dev, "sfp-gpios") &&
1687              !device_property_present(dev, "rxlos-gpios")))
1688                 return;
1689
1690         pdata->sfp_gpio_en = true;
1691         pdata->sfp_rdy = gpiod_get(dev, "rxlos", GPIOD_IN);
1692         if (IS_ERR(pdata->sfp_rdy))
1693                 pdata->sfp_rdy = gpiod_get(dev, "sfp", GPIOD_IN);
1694 }
1695
1696 static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
1697 {
1698         struct platform_device *pdev;
1699         struct net_device *ndev;
1700         struct device *dev;
1701         struct resource *res;
1702         void __iomem *base_addr;
1703         u32 offset;
1704         int ret = 0;
1705
1706         pdev = pdata->pdev;
1707         dev = &pdev->dev;
1708         ndev = pdata->ndev;
1709
1710         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_ENET_CSR);
1711         if (!res) {
1712                 dev_err(dev, "Resource enet_csr not defined\n");
1713                 return -ENODEV;
1714         }
1715         pdata->base_addr = devm_ioremap(dev, res->start, resource_size(res));
1716         if (!pdata->base_addr) {
1717                 dev_err(dev, "Unable to retrieve ENET Port CSR region\n");
1718                 return -ENOMEM;
1719         }
1720
1721         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_RING_CSR);
1722         if (!res) {
1723                 dev_err(dev, "Resource ring_csr not defined\n");
1724                 return -ENODEV;
1725         }
1726         pdata->ring_csr_addr = devm_ioremap(dev, res->start,
1727                                                         resource_size(res));
1728         if (!pdata->ring_csr_addr) {
1729                 dev_err(dev, "Unable to retrieve ENET Ring CSR region\n");
1730                 return -ENOMEM;
1731         }
1732
1733         res = platform_get_resource(pdev, IORESOURCE_MEM, RES_RING_CMD);
1734         if (!res) {
1735                 dev_err(dev, "Resource ring_cmd not defined\n");
1736                 return -ENODEV;
1737         }
1738         pdata->ring_cmd_addr = devm_ioremap(dev, res->start,
1739                                                         resource_size(res));
1740         if (!pdata->ring_cmd_addr) {
1741                 dev_err(dev, "Unable to retrieve ENET Ring command region\n");
1742                 return -ENOMEM;
1743         }
1744
1745         if (dev->of_node)
1746                 xgene_get_port_id_dt(dev, pdata);
1747 #ifdef CONFIG_ACPI
1748         else
1749                 xgene_get_port_id_acpi(dev, pdata);
1750 #endif
1751
1752         if (!device_get_mac_address(dev, ndev->dev_addr, ETH_ALEN))
1753                 eth_hw_addr_random(ndev);
1754
1755         memcpy(ndev->perm_addr, ndev->dev_addr, ndev->addr_len);
1756
1757         pdata->phy_mode = device_get_phy_mode(dev);
1758         if (pdata->phy_mode < 0) {
1759                 dev_err(dev, "Unable to get phy-connection-type\n");
1760                 return pdata->phy_mode;
1761         }
1762         if (!phy_interface_mode_is_rgmii(pdata->phy_mode) &&
1763             pdata->phy_mode != PHY_INTERFACE_MODE_SGMII &&
1764             pdata->phy_mode != PHY_INTERFACE_MODE_XGMII) {
1765                 dev_err(dev, "Incorrect phy-connection-type specified\n");
1766                 return -ENODEV;
1767         }
1768
1769         ret = xgene_get_tx_delay(pdata);
1770         if (ret)
1771                 return ret;
1772
1773         ret = xgene_get_rx_delay(pdata);
1774         if (ret)
1775                 return ret;
1776
1777         ret = xgene_enet_get_irqs(pdata);
1778         if (ret)
1779                 return ret;
1780
1781         xgene_enet_gpiod_get(pdata);
1782
1783         pdata->clk = devm_clk_get(&pdev->dev, NULL);
1784         if (IS_ERR(pdata->clk)) {
1785                 if (pdata->phy_mode != PHY_INTERFACE_MODE_SGMII) {
1786                         /* Abort if the clock is defined but couldn't be
1787                          * retrived. Always abort if the clock is missing on
1788                          * DT system as the driver can't cope with this case.
1789                          */
1790                         if (PTR_ERR(pdata->clk) != -ENOENT || dev->of_node)
1791                                 return PTR_ERR(pdata->clk);
1792                         /* Firmware may have set up the clock already. */
1793                         dev_info(dev, "clocks have been setup already\n");
1794                 }
1795         }
1796
1797         if (pdata->phy_mode != PHY_INTERFACE_MODE_XGMII)
1798                 base_addr = pdata->base_addr - (pdata->port_id * MAC_OFFSET);
1799         else
1800                 base_addr = pdata->base_addr;
1801         pdata->eth_csr_addr = base_addr + BLOCK_ETH_CSR_OFFSET;
1802         pdata->cle.base = base_addr + BLOCK_ETH_CLE_CSR_OFFSET;
1803         pdata->eth_ring_if_addr = base_addr + BLOCK_ETH_RING_IF_OFFSET;
1804         pdata->eth_diag_csr_addr = base_addr + BLOCK_ETH_DIAG_CSR_OFFSET;
1805         if (phy_interface_mode_is_rgmii(pdata->phy_mode) ||
1806             pdata->phy_mode == PHY_INTERFACE_MODE_SGMII) {
1807                 pdata->mcx_mac_addr = pdata->base_addr + BLOCK_ETH_MAC_OFFSET;
1808                 pdata->mcx_stats_addr =
1809                         pdata->base_addr + BLOCK_ETH_STATS_OFFSET;
1810                 offset = (pdata->enet_id == XGENE_ENET1) ?
1811                           BLOCK_ETH_MAC_CSR_OFFSET :
1812                           X2_BLOCK_ETH_MAC_CSR_OFFSET;
1813                 pdata->mcx_mac_csr_addr = base_addr + offset;
1814         } else {
1815                 pdata->mcx_mac_addr = base_addr + BLOCK_AXG_MAC_OFFSET;
1816                 pdata->mcx_stats_addr = base_addr + BLOCK_AXG_STATS_OFFSET;
1817                 pdata->mcx_mac_csr_addr = base_addr + BLOCK_AXG_MAC_CSR_OFFSET;
1818                 pdata->pcs_addr = base_addr + BLOCK_PCS_OFFSET;
1819         }
1820         pdata->rx_buff_cnt = NUM_PKT_BUF;
1821
1822         return 0;
1823 }
1824
1825 static int xgene_enet_init_hw(struct xgene_enet_pdata *pdata)
1826 {
1827         struct xgene_enet_cle *enet_cle = &pdata->cle;
1828         struct xgene_enet_desc_ring *page_pool;
1829         struct net_device *ndev = pdata->ndev;
1830         struct xgene_enet_desc_ring *buf_pool;
1831         u16 dst_ring_num, ring_id;
1832         int i, ret;
1833         u32 count;
1834
1835         ret = pdata->port_ops->reset(pdata);
1836         if (ret)
1837                 return ret;
1838
1839         ret = xgene_enet_create_desc_rings(ndev);
1840         if (ret) {
1841                 netdev_err(ndev, "Error in ring configuration\n");
1842                 return ret;
1843         }
1844
1845         /* setup buffer pool */
1846         for (i = 0; i < pdata->rxq_cnt; i++) {
1847                 buf_pool = pdata->rx_ring[i]->buf_pool;
1848                 xgene_enet_init_bufpool(buf_pool);
1849                 page_pool = pdata->rx_ring[i]->page_pool;
1850                 xgene_enet_init_bufpool(page_pool);
1851
1852                 count = pdata->rx_buff_cnt;
1853                 ret = xgene_enet_refill_bufpool(buf_pool, count);
1854                 if (ret)
1855                         goto err;
1856
1857                 ret = xgene_enet_refill_pagepool(page_pool, count);
1858                 if (ret)
1859                         goto err;
1860
1861         }
1862
1863         dst_ring_num = xgene_enet_dst_ring_num(pdata->rx_ring[0]);
1864         buf_pool = pdata->rx_ring[0]->buf_pool;
1865         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1866                 /* Initialize and Enable  PreClassifier Tree */
1867                 enet_cle->max_nodes = 512;
1868                 enet_cle->max_dbptrs = 1024;
1869                 enet_cle->parsers = 3;
1870                 enet_cle->active_parser = PARSER_ALL;
1871                 enet_cle->ptree.start_node = 0;
1872                 enet_cle->ptree.start_dbptr = 0;
1873                 enet_cle->jump_bytes = 8;
1874                 ret = pdata->cle_ops->cle_init(pdata);
1875                 if (ret) {
1876                         netdev_err(ndev, "Preclass Tree init error\n");
1877                         goto err;
1878                 }
1879
1880         } else {
1881                 dst_ring_num = xgene_enet_dst_ring_num(pdata->rx_ring[0]);
1882                 buf_pool = pdata->rx_ring[0]->buf_pool;
1883                 page_pool = pdata->rx_ring[0]->page_pool;
1884                 ring_id = (page_pool) ? page_pool->id : 0;
1885                 pdata->port_ops->cle_bypass(pdata, dst_ring_num,
1886                                             buf_pool->id, ring_id);
1887         }
1888
1889         ndev->max_mtu = XGENE_ENET_MAX_MTU;
1890         pdata->phy_speed = SPEED_UNKNOWN;
1891         pdata->mac_ops->init(pdata);
1892
1893         return ret;
1894
1895 err:
1896         xgene_enet_delete_desc_rings(pdata);
1897         return ret;
1898 }
1899
1900 static void xgene_enet_setup_ops(struct xgene_enet_pdata *pdata)
1901 {
1902         switch (pdata->phy_mode) {
1903         case PHY_INTERFACE_MODE_RGMII:
1904         case PHY_INTERFACE_MODE_RGMII_ID:
1905         case PHY_INTERFACE_MODE_RGMII_RXID:
1906         case PHY_INTERFACE_MODE_RGMII_TXID:
1907                 pdata->mac_ops = &xgene_gmac_ops;
1908                 pdata->port_ops = &xgene_gport_ops;
1909                 pdata->rm = RM3;
1910                 pdata->rxq_cnt = 1;
1911                 pdata->txq_cnt = 1;
1912                 pdata->cq_cnt = 0;
1913                 break;
1914         case PHY_INTERFACE_MODE_SGMII:
1915                 pdata->mac_ops = &xgene_sgmac_ops;
1916                 pdata->port_ops = &xgene_sgport_ops;
1917                 pdata->rm = RM1;
1918                 pdata->rxq_cnt = 1;
1919                 pdata->txq_cnt = 1;
1920                 pdata->cq_cnt = 1;
1921                 break;
1922         default:
1923                 pdata->mac_ops = &xgene_xgmac_ops;
1924                 pdata->port_ops = &xgene_xgport_ops;
1925                 pdata->cle_ops = &xgene_cle3in_ops;
1926                 pdata->rm = RM0;
1927                 if (!pdata->rxq_cnt) {
1928                         pdata->rxq_cnt = XGENE_NUM_RX_RING;
1929                         pdata->txq_cnt = XGENE_NUM_TX_RING;
1930                         pdata->cq_cnt = XGENE_NUM_TXC_RING;
1931                 }
1932                 break;
1933         }
1934
1935         if (pdata->enet_id == XGENE_ENET1) {
1936                 switch (pdata->port_id) {
1937                 case 0:
1938                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1939                                 pdata->cpu_bufnum = X2_START_CPU_BUFNUM_0;
1940                                 pdata->eth_bufnum = X2_START_ETH_BUFNUM_0;
1941                                 pdata->bp_bufnum = X2_START_BP_BUFNUM_0;
1942                                 pdata->ring_num = START_RING_NUM_0;
1943                         } else {
1944                                 pdata->cpu_bufnum = START_CPU_BUFNUM_0;
1945                                 pdata->eth_bufnum = START_ETH_BUFNUM_0;
1946                                 pdata->bp_bufnum = START_BP_BUFNUM_0;
1947                                 pdata->ring_num = START_RING_NUM_0;
1948                         }
1949                         break;
1950                 case 1:
1951                         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
1952                                 pdata->cpu_bufnum = XG_START_CPU_BUFNUM_1;
1953                                 pdata->eth_bufnum = XG_START_ETH_BUFNUM_1;
1954                                 pdata->bp_bufnum = XG_START_BP_BUFNUM_1;
1955                                 pdata->ring_num = XG_START_RING_NUM_1;
1956                         } else {
1957                                 pdata->cpu_bufnum = START_CPU_BUFNUM_1;
1958                                 pdata->eth_bufnum = START_ETH_BUFNUM_1;
1959                                 pdata->bp_bufnum = START_BP_BUFNUM_1;
1960                                 pdata->ring_num = START_RING_NUM_1;
1961                         }
1962                         break;
1963                 default:
1964                         break;
1965                 }
1966                 pdata->ring_ops = &xgene_ring1_ops;
1967         } else {
1968                 switch (pdata->port_id) {
1969                 case 0:
1970                         pdata->cpu_bufnum = X2_START_CPU_BUFNUM_0;
1971                         pdata->eth_bufnum = X2_START_ETH_BUFNUM_0;
1972                         pdata->bp_bufnum = X2_START_BP_BUFNUM_0;
1973                         pdata->ring_num = X2_START_RING_NUM_0;
1974                         break;
1975                 case 1:
1976                         pdata->cpu_bufnum = X2_START_CPU_BUFNUM_1;
1977                         pdata->eth_bufnum = X2_START_ETH_BUFNUM_1;
1978                         pdata->bp_bufnum = X2_START_BP_BUFNUM_1;
1979                         pdata->ring_num = X2_START_RING_NUM_1;
1980                         break;
1981                 default:
1982                         break;
1983                 }
1984                 pdata->rm = RM0;
1985                 pdata->ring_ops = &xgene_ring2_ops;
1986         }
1987 }
1988
1989 static void xgene_enet_napi_add(struct xgene_enet_pdata *pdata)
1990 {
1991         struct napi_struct *napi;
1992         int i;
1993
1994         for (i = 0; i < pdata->rxq_cnt; i++) {
1995                 napi = &pdata->rx_ring[i]->napi;
1996                 netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
1997                                NAPI_POLL_WEIGHT);
1998         }
1999
2000         for (i = 0; i < pdata->cq_cnt; i++) {
2001                 napi = &pdata->tx_ring[i]->cp_ring->napi;
2002                 netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
2003                                NAPI_POLL_WEIGHT);
2004         }
2005 }
2006
2007 #ifdef CONFIG_ACPI
2008 static const struct acpi_device_id xgene_enet_acpi_match[] = {
2009         { "APMC0D05", XGENE_ENET1},
2010         { "APMC0D30", XGENE_ENET1},
2011         { "APMC0D31", XGENE_ENET1},
2012         { "APMC0D3F", XGENE_ENET1},
2013         { "APMC0D26", XGENE_ENET2},
2014         { "APMC0D25", XGENE_ENET2},
2015         { }
2016 };
2017 MODULE_DEVICE_TABLE(acpi, xgene_enet_acpi_match);
2018 #endif
2019
2020 static const struct of_device_id xgene_enet_of_match[] = {
2021         {.compatible = "apm,xgene-enet",    .data = (void *)XGENE_ENET1},
2022         {.compatible = "apm,xgene1-sgenet", .data = (void *)XGENE_ENET1},
2023         {.compatible = "apm,xgene1-xgenet", .data = (void *)XGENE_ENET1},
2024         {.compatible = "apm,xgene2-sgenet", .data = (void *)XGENE_ENET2},
2025         {.compatible = "apm,xgene2-xgenet", .data = (void *)XGENE_ENET2},
2026         {},
2027 };
2028
2029 MODULE_DEVICE_TABLE(of, xgene_enet_of_match);
2030
2031 static int xgene_enet_probe(struct platform_device *pdev)
2032 {
2033         struct net_device *ndev;
2034         struct xgene_enet_pdata *pdata;
2035         struct device *dev = &pdev->dev;
2036         void (*link_state)(struct work_struct *);
2037         const struct of_device_id *of_id;
2038         int ret;
2039
2040         ndev = alloc_etherdev_mqs(sizeof(struct xgene_enet_pdata),
2041                                   XGENE_NUM_TX_RING, XGENE_NUM_RX_RING);
2042         if (!ndev)
2043                 return -ENOMEM;
2044
2045         pdata = netdev_priv(ndev);
2046
2047         pdata->pdev = pdev;
2048         pdata->ndev = ndev;
2049         SET_NETDEV_DEV(ndev, dev);
2050         platform_set_drvdata(pdev, pdata);
2051         ndev->netdev_ops = &xgene_ndev_ops;
2052         xgene_enet_set_ethtool_ops(ndev);
2053         ndev->features |= NETIF_F_IP_CSUM |
2054                           NETIF_F_GSO |
2055                           NETIF_F_GRO |
2056                           NETIF_F_SG;
2057
2058         of_id = of_match_device(xgene_enet_of_match, &pdev->dev);
2059         if (of_id) {
2060                 pdata->enet_id = (enum xgene_enet_id)of_id->data;
2061         }
2062 #ifdef CONFIG_ACPI
2063         else {
2064                 const struct acpi_device_id *acpi_id;
2065
2066                 acpi_id = acpi_match_device(xgene_enet_acpi_match, &pdev->dev);
2067                 if (acpi_id)
2068                         pdata->enet_id = (enum xgene_enet_id) acpi_id->driver_data;
2069         }
2070 #endif
2071         if (!pdata->enet_id) {
2072                 ret = -ENODEV;
2073                 goto err;
2074         }
2075
2076         ret = xgene_enet_get_resources(pdata);
2077         if (ret)
2078                 goto err;
2079
2080         xgene_enet_setup_ops(pdata);
2081         spin_lock_init(&pdata->mac_lock);
2082
2083         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
2084                 ndev->features |= NETIF_F_TSO | NETIF_F_RXCSUM;
2085                 spin_lock_init(&pdata->mss_lock);
2086         }
2087         ndev->hw_features = ndev->features;
2088
2089         ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
2090         if (ret) {
2091                 netdev_err(ndev, "No usable DMA configuration\n");
2092                 goto err;
2093         }
2094
2095         xgene_enet_check_phy_handle(pdata);
2096
2097         ret = xgene_enet_init_hw(pdata);
2098         if (ret)
2099                 goto err2;
2100
2101         link_state = pdata->mac_ops->link_state;
2102         if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
2103                 INIT_DELAYED_WORK(&pdata->link_work, link_state);
2104         } else if (!pdata->mdio_driver) {
2105                 if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2106                         ret = xgene_enet_mdio_config(pdata);
2107                 else
2108                         INIT_DELAYED_WORK(&pdata->link_work, link_state);
2109
2110                 if (ret)
2111                         goto err1;
2112         }
2113
2114         spin_lock_init(&pdata->stats_lock);
2115         ret = xgene_extd_stats_init(pdata);
2116         if (ret)
2117                 goto err1;
2118
2119         xgene_enet_napi_add(pdata);
2120         ret = register_netdev(ndev);
2121         if (ret) {
2122                 netdev_err(ndev, "Failed to register netdev\n");
2123                 goto err1;
2124         }
2125
2126         return 0;
2127
2128 err1:
2129         /*
2130          * If necessary, free_netdev() will call netif_napi_del() and undo
2131          * the effects of xgene_enet_napi_add()'s calls to netif_napi_add().
2132          */
2133
2134         xgene_enet_delete_desc_rings(pdata);
2135
2136 err2:
2137         if (pdata->mdio_driver)
2138                 xgene_enet_phy_disconnect(pdata);
2139         else if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2140                 xgene_enet_mdio_remove(pdata);
2141 err:
2142         free_netdev(ndev);
2143         return ret;
2144 }
2145
2146 static int xgene_enet_remove(struct platform_device *pdev)
2147 {
2148         struct xgene_enet_pdata *pdata;
2149         struct net_device *ndev;
2150
2151         pdata = platform_get_drvdata(pdev);
2152         ndev = pdata->ndev;
2153
2154         rtnl_lock();
2155         if (netif_running(ndev))
2156                 dev_close(ndev);
2157         rtnl_unlock();
2158
2159         if (pdata->mdio_driver)
2160                 xgene_enet_phy_disconnect(pdata);
2161         else if (phy_interface_mode_is_rgmii(pdata->phy_mode))
2162                 xgene_enet_mdio_remove(pdata);
2163
2164         unregister_netdev(ndev);
2165         xgene_enet_delete_desc_rings(pdata);
2166         pdata->port_ops->shutdown(pdata);
2167         free_netdev(ndev);
2168
2169         return 0;
2170 }
2171
2172 static void xgene_enet_shutdown(struct platform_device *pdev)
2173 {
2174         struct xgene_enet_pdata *pdata;
2175
2176         pdata = platform_get_drvdata(pdev);
2177         if (!pdata)
2178                 return;
2179
2180         if (!pdata->ndev)
2181                 return;
2182
2183         xgene_enet_remove(pdev);
2184 }
2185
2186 static struct platform_driver xgene_enet_driver = {
2187         .driver = {
2188                    .name = "xgene-enet",
2189                    .of_match_table = of_match_ptr(xgene_enet_of_match),
2190                    .acpi_match_table = ACPI_PTR(xgene_enet_acpi_match),
2191         },
2192         .probe = xgene_enet_probe,
2193         .remove = xgene_enet_remove,
2194         .shutdown = xgene_enet_shutdown,
2195 };
2196
2197 module_platform_driver(xgene_enet_driver);
2198
2199 MODULE_DESCRIPTION("APM X-Gene SoC Ethernet driver");
2200 MODULE_VERSION(XGENE_DRV_VERSION);
2201 MODULE_AUTHOR("Iyappan Subramanian <isubramanian@apm.com>");
2202 MODULE_AUTHOR("Keyur Chudgar <kchudgar@apm.com>");
2203 MODULE_LICENSE("GPL");