drivers/staging/mt7621-dma/hsdma-mt7621.c

   1 /*
   2  *  Copyright (C) 2015, Michael Lee <igvtee@gmail.com>
   3  *  MTK HSDMA support
   4  *
   5  *  This program is free software; you can redistribute it and/or modify it
   6  *  under  the terms of the GNU General  Public License as published by the
   7  *  Free Software Foundation;  either version 2 of the License, or (at your
   8  *  option) any later version.
   9  *
  10  */
  11
  12 #include <linux/dmaengine.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/err.h>
  15 #include <linux/init.h>
  16 #include <linux/list.h>
  17 #include <linux/module.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/slab.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/irq.h>
  22 #include <linux/of_dma.h>
  23 #include <linux/reset.h>
  24 #include <linux/of_device.h>
  25
  26 #include "virt-dma.h"
  27
  28 #define HSDMA_BASE_OFFSET               0x800
  29
  30 #define HSDMA_REG_TX_BASE               0x00
  31 #define HSDMA_REG_TX_CNT                0x04
  32 #define HSDMA_REG_TX_CTX                0x08
  33 #define HSDMA_REG_TX_DTX                0x0c
  34 #define HSDMA_REG_RX_BASE               0x100
  35 #define HSDMA_REG_RX_CNT                0x104
  36 #define HSDMA_REG_RX_CRX                0x108
  37 #define HSDMA_REG_RX_DRX                0x10c
  38 #define HSDMA_REG_INFO                  0x200
  39 #define HSDMA_REG_GLO_CFG               0x204
  40 #define HSDMA_REG_RST_CFG               0x208
  41 #define HSDMA_REG_DELAY_INT             0x20c
  42 #define HSDMA_REG_FREEQ_THRES           0x210
  43 #define HSDMA_REG_INT_STATUS            0x220
  44 #define HSDMA_REG_INT_MASK              0x228
  45 #define HSDMA_REG_SCH_Q01               0x280
  46 #define HSDMA_REG_SCH_Q23               0x284
  47
  48 #define HSDMA_DESCS_MAX                 0xfff
  49 #define HSDMA_DESCS_NUM                 8
  50 #define HSDMA_DESCS_MASK                (HSDMA_DESCS_NUM - 1)
  51 #define HSDMA_NEXT_DESC(x)              (((x) + 1) & HSDMA_DESCS_MASK)
  52
  53 /* HSDMA_REG_INFO */
  54 #define HSDMA_INFO_INDEX_MASK           0xf
  55 #define HSDMA_INFO_INDEX_SHIFT          24
  56 #define HSDMA_INFO_BASE_MASK            0xff
  57 #define HSDMA_INFO_BASE_SHIFT           16
  58 #define HSDMA_INFO_RX_MASK              0xff
  59 #define HSDMA_INFO_RX_SHIFT             8
  60 #define HSDMA_INFO_TX_MASK              0xff
  61 #define HSDMA_INFO_TX_SHIFT             0
  62
  63 /* HSDMA_REG_GLO_CFG */
  64 #define HSDMA_GLO_TX_2B_OFFSET          BIT(31)
  65 #define HSDMA_GLO_CLK_GATE              BIT(30)
  66 #define HSDMA_GLO_BYTE_SWAP             BIT(29)
  67 #define HSDMA_GLO_MULTI_DMA             BIT(10)
  68 #define HSDMA_GLO_TWO_BUF               BIT(9)
  69 #define HSDMA_GLO_32B_DESC              BIT(8)
  70 #define HSDMA_GLO_BIG_ENDIAN            BIT(7)
  71 #define HSDMA_GLO_TX_DONE               BIT(6)
  72 #define HSDMA_GLO_BT_MASK               0x3
  73 #define HSDMA_GLO_BT_SHIFT              4
  74 #define HSDMA_GLO_RX_BUSY               BIT(3)
  75 #define HSDMA_GLO_RX_DMA                BIT(2)
  76 #define HSDMA_GLO_TX_BUSY               BIT(1)
  77 #define HSDMA_GLO_TX_DMA                BIT(0)
  78
  79 #define HSDMA_BT_SIZE_16BYTES           (0 << HSDMA_GLO_BT_SHIFT)
  80 #define HSDMA_BT_SIZE_32BYTES           (1 << HSDMA_GLO_BT_SHIFT)
  81 #define HSDMA_BT_SIZE_64BYTES           (2 << HSDMA_GLO_BT_SHIFT)
  82 #define HSDMA_BT_SIZE_128BYTES          (3 << HSDMA_GLO_BT_SHIFT)
  83
  84 #define HSDMA_GLO_DEFAULT               (HSDMA_GLO_MULTI_DMA | \
  85                 HSDMA_GLO_RX_DMA | HSDMA_GLO_TX_DMA | HSDMA_BT_SIZE_32BYTES)
  86
  87 /* HSDMA_REG_RST_CFG */
  88 #define HSDMA_RST_RX_SHIFT              16
  89 #define HSDMA_RST_TX_SHIFT              0
  90
  91 /* HSDMA_REG_DELAY_INT */
  92 #define HSDMA_DELAY_INT_EN              BIT(15)
  93 #define HSDMA_DELAY_PEND_OFFSET         8
  94 #define HSDMA_DELAY_TIME_OFFSET         0
  95 #define HSDMA_DELAY_TX_OFFSET           16
  96 #define HSDMA_DELAY_RX_OFFSET           0
  97
  98 #define HSDMA_DELAY_INIT(x)             (HSDMA_DELAY_INT_EN | \
  99                 ((x) << HSDMA_DELAY_PEND_OFFSET))
 100 #define HSDMA_DELAY(x)                  ((HSDMA_DELAY_INIT(x) << \
 101                 HSDMA_DELAY_TX_OFFSET) | HSDMA_DELAY_INIT(x))
 102
 103 /* HSDMA_REG_INT_STATUS */
 104 #define HSDMA_INT_DELAY_RX_COH          BIT(31)
 105 #define HSDMA_INT_DELAY_RX_INT          BIT(30)
 106 #define HSDMA_INT_DELAY_TX_COH          BIT(29)
 107 #define HSDMA_INT_DELAY_TX_INT          BIT(28)
 108 #define HSDMA_INT_RX_MASK               0x3
 109 #define HSDMA_INT_RX_SHIFT              16
 110 #define HSDMA_INT_RX_Q0                 BIT(16)
 111 #define HSDMA_INT_TX_MASK               0xf
 112 #define HSDMA_INT_TX_SHIFT              0
 113 #define HSDMA_INT_TX_Q0                 BIT(0)
 114
 115 /* tx/rx dma desc flags */
 116 #define HSDMA_PLEN_MASK                 0x3fff
 117 #define HSDMA_DESC_DONE                 BIT(31)
 118 #define HSDMA_DESC_LS0                  BIT(30)
 119 #define HSDMA_DESC_PLEN0(_x)            (((_x) & HSDMA_PLEN_MASK) << 16)
 120 #define HSDMA_DESC_TAG                  BIT(15)
 121 #define HSDMA_DESC_LS1                  BIT(14)
 122 #define HSDMA_DESC_PLEN1(_x)            ((_x) & HSDMA_PLEN_MASK)
 123
 124 /* align 4 bytes */
 125 #define HSDMA_ALIGN_SIZE                3
 126 /* align size 128bytes */
 127 #define HSDMA_MAX_PLEN                  0x3f80
 128
 129 struct hsdma_desc {
 130         u32 addr0;
 131         u32 flags;
 132         u32 addr1;
 133         u32 unused;
 134 };
 135
 136 struct mtk_hsdma_sg {
 137         dma_addr_t src_addr;
 138         dma_addr_t dst_addr;
 139         u32 len;
 140 };
 141
 142 struct mtk_hsdma_desc {
 143         struct virt_dma_desc vdesc;
 144         unsigned int num_sgs;
 145         struct mtk_hsdma_sg sg[1];
 146 };
 147
 148 struct mtk_hsdma_chan {
 149         struct virt_dma_chan vchan;
 150         unsigned int id;
 151         dma_addr_t desc_addr;
 152         int tx_idx;
 153         int rx_idx;
 154         struct hsdma_desc *tx_ring;
 155         struct hsdma_desc *rx_ring;
 156         struct mtk_hsdma_desc *desc;
 157         unsigned int next_sg;
 158 };
 159
 160 struct mtk_hsdam_engine {
 161         struct dma_device ddev;
 162         struct device_dma_parameters dma_parms;
 163         void __iomem *base;
 164         struct tasklet_struct task;
 165         volatile unsigned long chan_issued;
 166
 167         struct mtk_hsdma_chan chan[1];
 168 };
 169
 170 static inline struct mtk_hsdam_engine *mtk_hsdma_chan_get_dev(
 171                 struct mtk_hsdma_chan *chan)
 172 {
 173         return container_of(chan->vchan.chan.device, struct mtk_hsdam_engine,
 174                         ddev);
 175 }
 176
 177 static inline struct mtk_hsdma_chan *to_mtk_hsdma_chan(struct dma_chan *c)
 178 {
 179         return container_of(c, struct mtk_hsdma_chan, vchan.chan);
 180 }
 181
 182 static inline struct mtk_hsdma_desc *to_mtk_hsdma_desc(
 183                 struct virt_dma_desc *vdesc)
 184 {
 185         return container_of(vdesc, struct mtk_hsdma_desc, vdesc);
 186 }
 187
 188 static inline u32 mtk_hsdma_read(struct mtk_hsdam_engine *hsdma, u32 reg)
 189 {
 190         return readl(hsdma->base + reg);
 191 }
 192
 193 static inline void mtk_hsdma_write(struct mtk_hsdam_engine *hsdma,
 194                                    unsigned reg, u32 val)
 195 {
 196         writel(val, hsdma->base + reg);
 197 }
 198
 199 static void mtk_hsdma_reset_chan(struct mtk_hsdam_engine *hsdma,
 200                                  struct mtk_hsdma_chan *chan)
 201 {
 202         chan->tx_idx = 0;
 203         chan->rx_idx = HSDMA_DESCS_NUM - 1;
 204
 205         mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx);
 206         mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx);
 207
 208         mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG,
 209                         0x1 << (chan->id + HSDMA_RST_TX_SHIFT));
 210         mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG,
 211                         0x1 << (chan->id + HSDMA_RST_RX_SHIFT));
 212 }
 213
 214 static void hsdma_dump_reg(struct mtk_hsdam_engine *hsdma)
 215 {
 216         dev_dbg(hsdma->ddev.dev, "tbase %08x, tcnt %08x, " \
 217                         "tctx %08x, tdtx: %08x, rbase %08x, " \
 218                         "rcnt %08x, rctx %08x, rdtx %08x\n",
 219                         mtk_hsdma_read(hsdma, HSDMA_REG_TX_BASE),
 220                         mtk_hsdma_read(hsdma, HSDMA_REG_TX_CNT),
 221                         mtk_hsdma_read(hsdma, HSDMA_REG_TX_CTX),
 222                         mtk_hsdma_read(hsdma, HSDMA_REG_TX_DTX),
 223                         mtk_hsdma_read(hsdma, HSDMA_REG_RX_BASE),
 224                         mtk_hsdma_read(hsdma, HSDMA_REG_RX_CNT),
 225                         mtk_hsdma_read(hsdma, HSDMA_REG_RX_CRX),
 226                         mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX));
 227
 228         dev_dbg(hsdma->ddev.dev, "info %08x, glo %08x, delay %08x, " \
 229                         "intr_stat %08x, intr_mask %08x\n",
 230                         mtk_hsdma_read(hsdma, HSDMA_REG_INFO),
 231                         mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG),
 232                         mtk_hsdma_read(hsdma, HSDMA_REG_DELAY_INT),
 233                         mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS),
 234                         mtk_hsdma_read(hsdma, HSDMA_REG_INT_MASK));
 235 }
 236
 237 static void hsdma_dump_desc(struct mtk_hsdam_engine *hsdma,
 238                             struct mtk_hsdma_chan *chan)
 239 {
 240         struct hsdma_desc *tx_desc;
 241         struct hsdma_desc *rx_desc;
 242         int i;
 243
 244         dev_dbg(hsdma->ddev.dev, "tx idx: %d, rx idx: %d\n",
 245                         chan->tx_idx, chan->rx_idx);
 246
 247         for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 248                 tx_desc = &chan->tx_ring[i];
 249                 rx_desc = &chan->rx_ring[i];
 250
 251                 dev_dbg(hsdma->ddev.dev, "%d tx addr0: %08x, flags %08x, " \
 252                                 "tx addr1: %08x, rx addr0 %08x, flags %08x\n",
 253                                 i, tx_desc->addr0, tx_desc->flags, \
 254                                 tx_desc->addr1, rx_desc->addr0, rx_desc->flags);
 255         }
 256 }
 257
 258 static void mtk_hsdma_reset(struct mtk_hsdam_engine *hsdma,
 259                             struct mtk_hsdma_chan *chan)
 260 {
 261         int i;
 262
 263         /* disable dma */
 264         mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0);
 265
 266         /* disable intr */
 267         mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0);
 268
 269         /* init desc value */
 270         for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 271                 chan->tx_ring[i].addr0 = 0;
 272                 chan->tx_ring[i].flags = HSDMA_DESC_LS0 |
 273                         HSDMA_DESC_DONE;
 274         }
 275         for (i = 0; i < HSDMA_DESCS_NUM; i++) {
 276                 chan->rx_ring[i].addr0 = 0;
 277                 chan->rx_ring[i].flags = 0;
 278         }
 279
 280         /* reset */
 281         mtk_hsdma_reset_chan(hsdma, chan);
 282
 283         /* enable intr */
 284         mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0);
 285
 286         /* enable dma */
 287         mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT);
 288 }
 289
 290 static int mtk_hsdma_terminate_all(struct dma_chan *c)
 291 {
 292         struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 293         struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan);
 294         unsigned long timeout;
 295         LIST_HEAD(head);
 296
 297         spin_lock_bh(&chan->vchan.lock);
 298         chan->desc = NULL;
 299         clear_bit(chan->id, &hsdma->chan_issued);
 300         vchan_get_all_descriptors(&chan->vchan, &head);
 301         spin_unlock_bh(&chan->vchan.lock);
 302
 303         vchan_dma_desc_free_list(&chan->vchan, &head);
 304
 305         /* wait dma transfer complete */
 306         timeout = jiffies + msecs_to_jiffies(2000);
 307         while (mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG) &
 308                         (HSDMA_GLO_RX_BUSY | HSDMA_GLO_TX_BUSY)) {
 309                 if (time_after_eq(jiffies, timeout)) {
 310                         hsdma_dump_desc(hsdma, chan);
 311                         mtk_hsdma_reset(hsdma, chan);
 312                         dev_err(hsdma->ddev.dev, "timeout, reset it\n");
 313                         break;
 314                 }
 315                 cpu_relax();
 316         }
 317
 318         return 0;
 319 }
 320
 321 static int mtk_hsdma_start_transfer(struct mtk_hsdam_engine *hsdma,
 322                                     struct mtk_hsdma_chan *chan)
 323 {
 324         dma_addr_t src, dst;
 325         size_t len, tlen;
 326         struct hsdma_desc *tx_desc, *rx_desc;
 327         struct mtk_hsdma_sg *sg;
 328         unsigned int i;
 329         int rx_idx;
 330
 331         sg = &chan->desc->sg[0];
 332         len = sg->len;
 333         chan->desc->num_sgs = DIV_ROUND_UP(len, HSDMA_MAX_PLEN);
 334
 335         /* tx desc */
 336         src = sg->src_addr;
 337         for (i = 0; i < chan->desc->num_sgs; i++) {
 338                 tx_desc = &chan->tx_ring[chan->tx_idx];
 339
 340                 if (len > HSDMA_MAX_PLEN)
 341                         tlen = HSDMA_MAX_PLEN;
 342                 else
 343                         tlen = len;
 344
 345                 if (i & 0x1) {
 346                         tx_desc->addr1 = src;
 347                         tx_desc->flags |= HSDMA_DESC_PLEN1(tlen);
 348                 } else {
 349                         tx_desc->addr0 = src;
 350                         tx_desc->flags = HSDMA_DESC_PLEN0(tlen);
 351
 352                         /* update index */
 353                         chan->tx_idx = HSDMA_NEXT_DESC(chan->tx_idx);
 354                 }
 355
 356                 src += tlen;
 357                 len -= tlen;
 358         }
 359         if (i & 0x1)
 360                 tx_desc->flags |= HSDMA_DESC_LS0;
 361         else
 362                 tx_desc->flags |= HSDMA_DESC_LS1;
 363
 364         /* rx desc */
 365         rx_idx = HSDMA_NEXT_DESC(chan->rx_idx);
 366         len = sg->len;
 367         dst = sg->dst_addr;
 368         for (i = 0; i < chan->desc->num_sgs; i++) {
 369                 rx_desc = &chan->rx_ring[rx_idx];
 370                 if (len > HSDMA_MAX_PLEN)
 371                         tlen = HSDMA_MAX_PLEN;
 372                 else
 373                         tlen = len;
 374
 375                 rx_desc->addr0 = dst;
 376                 rx_desc->flags = HSDMA_DESC_PLEN0(tlen);
 377
 378                 dst += tlen;
 379                 len -= tlen;
 380
 381                 /* update index */
 382                 rx_idx = HSDMA_NEXT_DESC(rx_idx);
 383         }
 384
 385         /* make sure desc and index all up to date */
 386         wmb();
 387         mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx);
 388
 389         return 0;
 390 }
 391
 392 static int gdma_next_desc(struct mtk_hsdma_chan *chan)
 393 {
 394         struct virt_dma_desc *vdesc;
 395
 396         vdesc = vchan_next_desc(&chan->vchan);
 397         if (!vdesc) {
 398                 chan->desc = NULL;
 399                 return 0;
 400         }
 401         chan->desc = to_mtk_hsdma_desc(vdesc);
 402         chan->next_sg = 0;
 403
 404         return 1;
 405 }
 406
 407 static void mtk_hsdma_chan_done(struct mtk_hsdam_engine *hsdma,
 408                                 struct mtk_hsdma_chan *chan)
 409 {
 410         struct mtk_hsdma_desc *desc;
 411         int chan_issued;
 412
 413         chan_issued = 0;
 414         spin_lock_bh(&chan->vchan.lock);
 415         desc = chan->desc;
 416         if (likely(desc)) {
 417                 if (chan->next_sg == desc->num_sgs) {
 418                         list_del(&desc->vdesc.node);
 419                         vchan_cookie_complete(&desc->vdesc);
 420                         chan_issued = gdma_next_desc(chan);
 421                 }
 422         } else
 423                 dev_dbg(hsdma->ddev.dev, "no desc to complete\n");
 424
 425         if (chan_issued)
 426                 set_bit(chan->id, &hsdma->chan_issued);
 427         spin_unlock_bh(&chan->vchan.lock);
 428 }
 429
 430 static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
 431 {
 432         struct mtk_hsdam_engine *hsdma = devid;
 433         u32 status;
 434
 435         status = mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS);
 436         if (unlikely(!status))
 437                 return IRQ_NONE;
 438
 439         if (likely(status & HSDMA_INT_RX_Q0))
 440                 tasklet_schedule(&hsdma->task);
 441         else
 442                 dev_dbg(hsdma->ddev.dev, "unhandle irq status %08x\n",
 443                         status);
 444         /* clean intr bits */
 445         mtk_hsdma_write(hsdma, HSDMA_REG_INT_STATUS, status);
 446
 447         return IRQ_HANDLED;
 448 }
 449
 450 static void mtk_hsdma_issue_pending(struct dma_chan *c)
 451 {
 452         struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 453         struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan);
 454
 455         spin_lock_bh(&chan->vchan.lock);
 456         if (vchan_issue_pending(&chan->vchan) && !chan->desc) {
 457                 if (gdma_next_desc(chan)) {
 458                         set_bit(chan->id, &hsdma->chan_issued);
 459                         tasklet_schedule(&hsdma->task);
 460                 } else
 461                         dev_dbg(hsdma->ddev.dev, "no desc to issue\n");
 462         }
 463         spin_unlock_bh(&chan->vchan.lock);
 464 }
 465
 466 static struct dma_async_tx_descriptor *mtk_hsdma_prep_dma_memcpy(
 467                 struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
 468                 size_t len, unsigned long flags)
 469 {
 470         struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c);
 471         struct mtk_hsdma_desc *desc;
 472
 473         if (len <= 0)
 474                 return NULL;
 475
 476         desc = kzalloc(sizeof(struct mtk_hsdma_desc), GFP_ATOMIC);
 477         if (!desc) {
 478                 dev_err(c->device->dev, "alloc memcpy decs error\n");
 479                 return NULL;
 480         }
 481
 482         desc->sg[0].src_addr = src;
 483         desc->sg[0].dst_addr = dest;
 484         desc->sg[0].len = len;
 485
 486         return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 487 }
 488
 489 static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
 490                                            dma_cookie_t cookie,
 491                                            struct dma_tx_state *state)
 492 {
 493         return dma_cookie_status(c, cookie, state);
 494 }
 495
 496 static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
 497 {
 498         vchan_free_chan_resources(to_virt_chan(c));
 499 }
 500
 501 static void mtk_hsdma_desc_free(struct virt_dma_desc *vdesc)
 502 {
 503         kfree(container_of(vdesc, struct mtk_hsdma_desc, vdesc));
 504 }
 505
 506 static void mtk_hsdma_tx(struct mtk_hsdam_engine *hsdma)
 507 {
 508         struct mtk_hsdma_chan *chan;
 509
 510         if (test_and_clear_bit(0, &hsdma->chan_issued)) {
 511                 chan = &hsdma->chan[0];
 512                 if (chan->desc)
 513                         mtk_hsdma_start_transfer(hsdma, chan);
 514                 else
 515                         dev_dbg(hsdma->ddev.dev, "chan 0 no desc to issue\n");
 516         }
 517 }
 518
 519 static void mtk_hsdma_rx(struct mtk_hsdam_engine *hsdma)
 520 {
 521         struct mtk_hsdma_chan *chan;
 522         int next_idx, drx_idx, cnt;
 523
 524         chan = &hsdma->chan[0];
 525         next_idx = HSDMA_NEXT_DESC(chan->rx_idx);
 526         drx_idx = mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX);
 527
 528         cnt = (drx_idx - next_idx) & HSDMA_DESCS_MASK;
 529         if (!cnt)
 530                 return;
 531
 532         chan->next_sg += cnt;
 533         chan->rx_idx = (chan->rx_idx + cnt) & HSDMA_DESCS_MASK;
 534
 535         /* update rx crx */
 536         wmb();
 537         mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx);
 538
 539         mtk_hsdma_chan_done(hsdma, chan);
 540 }
 541
 542 static void mtk_hsdma_tasklet(unsigned long arg)
 543 {
 544         struct mtk_hsdam_engine *hsdma = (struct mtk_hsdam_engine *)arg;
 545
 546         mtk_hsdma_rx(hsdma);
 547         mtk_hsdma_tx(hsdma);
 548 }
 549
 550 static int mtk_hsdam_alloc_desc(struct mtk_hsdam_engine *hsdma,
 551                                 struct mtk_hsdma_chan *chan)
 552 {
 553         int i;
 554
 555         chan->tx_ring = dma_alloc_coherent(hsdma->ddev.dev,
 556                         2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring),
 557                         &chan->desc_addr, GFP_ATOMIC | __GFP_ZERO);
 558         if (!chan->tx_ring)
 559                 goto no_mem;
 560
 561         chan->rx_ring = &chan->tx_ring[HSDMA_DESCS_NUM];
 562
 563         /* init tx ring value */
 564         for (i = 0; i < HSDMA_DESCS_NUM; i++)
 565                 chan->tx_ring[i].flags = HSDMA_DESC_LS0 | HSDMA_DESC_DONE;
 566
 567         return 0;
 568 no_mem:
 569         return -ENOMEM;
 570 }
 571
 572 static void mtk_hsdam_free_desc(struct mtk_hsdam_engine *hsdma,
 573                                 struct mtk_hsdma_chan *chan)
 574 {
 575         if (chan->tx_ring) {
 576                 dma_free_coherent(hsdma->ddev.dev,
 577                                 2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring),
 578                                 chan->tx_ring, chan->desc_addr);
 579                 chan->tx_ring = NULL;
 580                 chan->rx_ring = NULL;
 581         }
 582 }
 583
 584 static int mtk_hsdma_init(struct mtk_hsdam_engine *hsdma)
 585 {
 586         struct mtk_hsdma_chan *chan;
 587         int ret;
 588         u32 reg;
 589
 590         /* init desc */
 591         chan = &hsdma->chan[0];
 592         ret = mtk_hsdam_alloc_desc(hsdma, chan);
 593         if (ret)
 594                 return ret;
 595
 596         /* tx */
 597         mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, chan->desc_addr);
 598         mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, HSDMA_DESCS_NUM);
 599         /* rx */
 600         mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, chan->desc_addr +
 601                         (sizeof(struct hsdma_desc) * HSDMA_DESCS_NUM));
 602         mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, HSDMA_DESCS_NUM);
 603         /* reset */
 604         mtk_hsdma_reset_chan(hsdma, chan);
 605
 606         /* enable rx intr */
 607         mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0);
 608
 609         /* enable dma */
 610         mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT);
 611
 612         /* hardware info */
 613         reg = mtk_hsdma_read(hsdma, HSDMA_REG_INFO);
 614         dev_info(hsdma->ddev.dev, "rx: %d, tx: %d\n",
 615                  (reg >> HSDMA_INFO_RX_SHIFT) & HSDMA_INFO_RX_MASK,
 616                  (reg >> HSDMA_INFO_TX_SHIFT) & HSDMA_INFO_TX_MASK);
 617
 618         hsdma_dump_reg(hsdma);
 619
 620         return ret;
 621 }
 622
 623 static void mtk_hsdma_uninit(struct mtk_hsdam_engine *hsdma)
 624 {
 625         struct mtk_hsdma_chan *chan;
 626
 627         /* disable dma */
 628         mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0);
 629
 630         /* disable intr */
 631         mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0);
 632
 633         /* free desc */
 634         chan = &hsdma->chan[0];
 635         mtk_hsdam_free_desc(hsdma, chan);
 636
 637         /* tx */
 638         mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, 0);
 639         mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, 0);
 640         /* rx */
 641         mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, 0);
 642         mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, 0);
 643         /* reset */
 644         mtk_hsdma_reset_chan(hsdma, chan);
 645 }
 646
 647 static const struct of_device_id mtk_hsdma_of_match[] = {
 648         { .compatible = "mediatek,mt7621-hsdma" },
 649         { },
 650 };
 651
 652 static int mtk_hsdma_probe(struct platform_device *pdev)
 653 {
 654         const struct of_device_id *match;
 655         struct mtk_hsdma_chan *chan;
 656         struct mtk_hsdam_engine *hsdma;
 657         struct dma_device *dd;
 658         struct resource *res;
 659         int ret;
 660         int irq;
 661         void __iomem *base;
 662
 663         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 664         if (ret)
 665                 return ret;
 666
 667         match = of_match_device(mtk_hsdma_of_match, &pdev->dev);
 668         if (!match)
 669                 return -EINVAL;
 670
 671         hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);
 672         if (!hsdma) {
 673                 dev_err(&pdev->dev, "alloc dma device failed\n");
 674                 return -EINVAL;
 675         }
 676
 677         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 678         base = devm_ioremap_resource(&pdev->dev, res);
 679         if (IS_ERR(base))
 680                 return PTR_ERR(base);
 681         hsdma->base = base + HSDMA_BASE_OFFSET;
 682         tasklet_init(&hsdma->task, mtk_hsdma_tasklet, (unsigned long)hsdma);
 683
 684         irq = platform_get_irq(pdev, 0);
 685         if (irq < 0) {
 686                 dev_err(&pdev->dev, "failed to get irq\n");
 687                 return -EINVAL;
 688         }
 689         ret = devm_request_irq(&pdev->dev, irq, mtk_hsdma_irq,
 690                                0, dev_name(&pdev->dev), hsdma);
 691         if (ret) {
 692                 dev_err(&pdev->dev, "failed to request irq\n");
 693                 return ret;
 694         }
 695
 696         device_reset(&pdev->dev);
 697
 698         dd = &hsdma->ddev;
 699         dma_cap_set(DMA_MEMCPY, dd->cap_mask);
 700         dd->copy_align = HSDMA_ALIGN_SIZE;
 701         dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
 702         dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
 703         dd->device_terminate_all = mtk_hsdma_terminate_all;
 704         dd->device_tx_status = mtk_hsdma_tx_status;
 705         dd->device_issue_pending = mtk_hsdma_issue_pending;
 706         dd->dev = &pdev->dev;
 707         dd->dev->dma_parms = &hsdma->dma_parms;
 708         dma_set_max_seg_size(dd->dev, HSDMA_MAX_PLEN);
 709         INIT_LIST_HEAD(&dd->channels);
 710
 711         chan = &hsdma->chan[0];
 712         chan->id = 0;
 713         chan->vchan.desc_free = mtk_hsdma_desc_free;
 714         vchan_init(&chan->vchan, dd);
 715
 716         /* init hardware */
 717         ret = mtk_hsdma_init(hsdma);
 718         if (ret) {
 719                 dev_err(&pdev->dev, "failed to alloc ring descs\n");
 720                 return ret;
 721         }
 722
 723         ret = dma_async_device_register(dd);
 724         if (ret) {
 725                 dev_err(&pdev->dev, "failed to register dma device\n");
 726                 goto err_uninit_hsdma;
 727         }
 728
 729         ret = of_dma_controller_register(pdev->dev.of_node,
 730                                          of_dma_xlate_by_chan_id, hsdma);
 731         if (ret) {
 732                 dev_err(&pdev->dev, "failed to register of dma controller\n");
 733                 goto err_unregister;
 734         }
 735
 736         platform_set_drvdata(pdev, hsdma);
 737
 738         return 0;
 739
 740 err_unregister:
 741         dma_async_device_unregister(dd);
 742 err_uninit_hsdma:
 743         mtk_hsdma_uninit(hsdma);
 744         return ret;
 745 }
 746
 747 static int mtk_hsdma_remove(struct platform_device *pdev)
 748 {
 749         struct mtk_hsdam_engine *hsdma = platform_get_drvdata(pdev);
 750
 751         mtk_hsdma_uninit(hsdma);
 752
 753         of_dma_controller_free(pdev->dev.of_node);
 754         dma_async_device_unregister(&hsdma->ddev);
 755
 756         return 0;
 757 }
 758
 759 static struct platform_driver mtk_hsdma_driver = {
 760         .probe = mtk_hsdma_probe,
 761         .remove = mtk_hsdma_remove,
 762         .driver = {
 763                 .name = KBUILD_MODNAME,
 764                 .of_match_table = mtk_hsdma_of_match,
 765         },
 766 };
 767 module_platform_driver(mtk_hsdma_driver);
 768
 769 MODULE_AUTHOR("Michael Lee <igvtee@gmail.com>");
 770 MODULE_DESCRIPTION("MTK HSDMA driver");
 771 MODULE_LICENSE("GPL v2");