drivers/dma/ti/edma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * TI EDMA DMA engine driver
   4  *
   5  * Copyright 2012 Texas Instruments
   6  */
   7
   8 #include <linux/dmaengine.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/bitmap.h>
  11 #include <linux/err.h>
  12 #include <linux/init.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/list.h>
  15 #include <linux/module.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/slab.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/of.h>
  20 #include <linux/of_dma.h>
  21 #include <linux/of_irq.h>
  22 #include <linux/of_address.h>
  23 #include <linux/of_device.h>
  24 #include <linux/pm_runtime.h>
  25
  26 #include <linux/platform_data/edma.h>
  27
  28 #include "../dmaengine.h"
  29 #include "../virt-dma.h"
  30
  31 /* Offsets matching "struct edmacc_param" */
  32 #define PARM_OPT                0x00
  33 #define PARM_SRC                0x04
  34 #define PARM_A_B_CNT            0x08
  35 #define PARM_DST                0x0c
  36 #define PARM_SRC_DST_BIDX       0x10
  37 #define PARM_LINK_BCNTRLD       0x14
  38 #define PARM_SRC_DST_CIDX       0x18
  39 #define PARM_CCNT               0x1c
  40
  41 #define PARM_SIZE               0x20
  42
  43 /* Offsets for EDMA CC global channel registers and their shadows */
  44 #define SH_ER                   0x00    /* 64 bits */
  45 #define SH_ECR                  0x08    /* 64 bits */
  46 #define SH_ESR                  0x10    /* 64 bits */
  47 #define SH_CER                  0x18    /* 64 bits */
  48 #define SH_EER                  0x20    /* 64 bits */
  49 #define SH_EECR                 0x28    /* 64 bits */
  50 #define SH_EESR                 0x30    /* 64 bits */
  51 #define SH_SER                  0x38    /* 64 bits */
  52 #define SH_SECR                 0x40    /* 64 bits */
  53 #define SH_IER                  0x50    /* 64 bits */
  54 #define SH_IECR                 0x58    /* 64 bits */
  55 #define SH_IESR                 0x60    /* 64 bits */
  56 #define SH_IPR                  0x68    /* 64 bits */
  57 #define SH_ICR                  0x70    /* 64 bits */
  58 #define SH_IEVAL                0x78
  59 #define SH_QER                  0x80
  60 #define SH_QEER                 0x84
  61 #define SH_QEECR                0x88
  62 #define SH_QEESR                0x8c
  63 #define SH_QSER                 0x90
  64 #define SH_QSECR                0x94
  65 #define SH_SIZE                 0x200
  66
  67 /* Offsets for EDMA CC global registers */
  68 #define EDMA_REV                0x0000
  69 #define EDMA_CCCFG              0x0004
  70 #define EDMA_QCHMAP             0x0200  /* 8 registers */
  71 #define EDMA_DMAQNUM            0x0240  /* 8 registers (4 on OMAP-L1xx) */
  72 #define EDMA_QDMAQNUM           0x0260
  73 #define EDMA_QUETCMAP           0x0280
  74 #define EDMA_QUEPRI             0x0284
  75 #define EDMA_EMR                0x0300  /* 64 bits */
  76 #define EDMA_EMCR               0x0308  /* 64 bits */
  77 #define EDMA_QEMR               0x0310
  78 #define EDMA_QEMCR              0x0314
  79 #define EDMA_CCERR              0x0318
  80 #define EDMA_CCERRCLR           0x031c
  81 #define EDMA_EEVAL              0x0320
  82 #define EDMA_DRAE               0x0340  /* 4 x 64 bits*/
  83 #define EDMA_QRAE               0x0380  /* 4 registers */
  84 #define EDMA_QUEEVTENTRY        0x0400  /* 2 x 16 registers */
  85 #define EDMA_QSTAT              0x0600  /* 2 registers */
  86 #define EDMA_QWMTHRA            0x0620
  87 #define EDMA_QWMTHRB            0x0624
  88 #define EDMA_CCSTAT             0x0640
  89
  90 #define EDMA_M                  0x1000  /* global channel registers */
  91 #define EDMA_ECR                0x1008
  92 #define EDMA_ECRH               0x100C
  93 #define EDMA_SHADOW0            0x2000  /* 4 shadow regions */
  94 #define EDMA_PARM               0x4000  /* PaRAM entries */
  95
  96 #define PARM_OFFSET(param_no)   (EDMA_PARM + ((param_no) << 5))
  97
  98 #define EDMA_DCHMAP             0x0100  /* 64 registers */
  99
 100 /* CCCFG register */
 101 #define GET_NUM_DMACH(x)        (x & 0x7) /* bits 0-2 */
 102 #define GET_NUM_QDMACH(x)       ((x & 0x70) >> 4) /* bits 4-6 */
 103 #define GET_NUM_PAENTRY(x)      ((x & 0x7000) >> 12) /* bits 12-14 */
 104 #define GET_NUM_EVQUE(x)        ((x & 0x70000) >> 16) /* bits 16-18 */
 105 #define GET_NUM_REGN(x)         ((x & 0x300000) >> 20) /* bits 20-21 */
 106 #define CHMAP_EXIST             BIT(24)
 107
 108 /* CCSTAT register */
 109 #define EDMA_CCSTAT_ACTV        BIT(4)
 110
 111 /*
 112  * Max of 20 segments per channel to conserve PaRAM slots
 113  * Also note that MAX_NR_SG should be at least the no.of periods
 114  * that are required for ASoC, otherwise DMA prep calls will
 115  * fail. Today davinci-pcm is the only user of this driver and
 116  * requires at least 17 slots, so we setup the default to 20.
 117  */
 118 #define MAX_NR_SG               20
 119 #define EDMA_MAX_SLOTS          MAX_NR_SG
 120 #define EDMA_DESCRIPTORS        16
 121
 122 #define EDMA_CHANNEL_ANY                -1      /* for edma_alloc_channel() */
 123 #define EDMA_SLOT_ANY                   -1      /* for edma_alloc_slot() */
 124 #define EDMA_CONT_PARAMS_ANY             1001
 125 #define EDMA_CONT_PARAMS_FIXED_EXACT     1002
 126 #define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
 127
 128 /*
 129  * 64bit array registers are split into two 32bit registers:
 130  * reg0: channel/event 0-31
 131  * reg1: channel/event 32-63
 132  *
 133  * bit 5 in the channel number tells the array index (0/1)
 134  * bit 0-4 (0x1f) is the bit offset within the register
 135  */
 136 #define EDMA_REG_ARRAY_INDEX(channel)   ((channel) >> 5)
 137 #define EDMA_CHANNEL_BIT(channel)       (BIT((channel) & 0x1f))
 138
 139 /* PaRAM slots are laid out like this */
 140 struct edmacc_param {
 141         u32 opt;
 142         u32 src;
 143         u32 a_b_cnt;
 144         u32 dst;
 145         u32 src_dst_bidx;
 146         u32 link_bcntrld;
 147         u32 src_dst_cidx;
 148         u32 ccnt;
 149 } __packed;
 150
 151 /* fields in edmacc_param.opt */
 152 #define SAM             BIT(0)
 153 #define DAM             BIT(1)
 154 #define SYNCDIM         BIT(2)
 155 #define STATIC          BIT(3)
 156 #define EDMA_FWID       (0x07 << 8)
 157 #define TCCMODE         BIT(11)
 158 #define EDMA_TCC(t)     ((t) << 12)
 159 #define TCINTEN         BIT(20)
 160 #define ITCINTEN        BIT(21)
 161 #define TCCHEN          BIT(22)
 162 #define ITCCHEN         BIT(23)
 163
 164 struct edma_pset {
 165         u32                             len;
 166         dma_addr_t                      addr;
 167         struct edmacc_param             param;
 168 };
 169
 170 struct edma_desc {
 171         struct virt_dma_desc            vdesc;
 172         struct list_head                node;
 173         enum dma_transfer_direction     direction;
 174         int                             cyclic;
 175         bool                            polled;
 176         int                             absync;
 177         int                             pset_nr;
 178         struct edma_chan                *echan;
 179         int                             processed;
 180
 181         /*
 182          * The following 4 elements are used for residue accounting.
 183          *
 184          * - processed_stat: the number of SG elements we have traversed
 185          * so far to cover accounting. This is updated directly to processed
 186          * during edma_callback and is always <= processed, because processed
 187          * refers to the number of pending transfer (programmed to EDMA
 188          * controller), where as processed_stat tracks number of transfers
 189          * accounted for so far.
 190          *
 191          * - residue: The amount of bytes we have left to transfer for this desc
 192          *
 193          * - residue_stat: The residue in bytes of data we have covered
 194          * so far for accounting. This is updated directly to residue
 195          * during callbacks to keep it current.
 196          *
 197          * - sg_len: Tracks the length of the current intermediate transfer,
 198          * this is required to update the residue during intermediate transfer
 199          * completion callback.
 200          */
 201         int                             processed_stat;
 202         u32                             sg_len;
 203         u32                             residue;
 204         u32                             residue_stat;
 205
 206         struct edma_pset                pset[];
 207 };
 208
 209 struct edma_cc;
 210
 211 struct edma_tc {
 212         struct device_node              *node;
 213         u16                             id;
 214 };
 215
 216 struct edma_chan {
 217         struct virt_dma_chan            vchan;
 218         struct list_head                node;
 219         struct edma_desc                *edesc;
 220         struct edma_cc                  *ecc;
 221         struct edma_tc                  *tc;
 222         int                             ch_num;
 223         bool                            alloced;
 224         bool                            hw_triggered;
 225         int                             slot[EDMA_MAX_SLOTS];
 226         int                             missed;
 227         struct dma_slave_config         cfg;
 228 };
 229
 230 struct edma_cc {
 231         struct device                   *dev;
 232         struct edma_soc_info            *info;
 233         void __iomem                    *base;
 234         int                             id;
 235         bool                            legacy_mode;
 236
 237         /* eDMA3 resource information */
 238         unsigned                        num_channels;
 239         unsigned                        num_qchannels;
 240         unsigned                        num_region;
 241         unsigned                        num_slots;
 242         unsigned                        num_tc;
 243         bool                            chmap_exist;
 244         enum dma_event_q                default_queue;
 245
 246         unsigned int                    ccint;
 247         unsigned int                    ccerrint;
 248
 249         /*
 250          * The slot_inuse bit for each PaRAM slot is clear unless the slot is
 251          * in use by Linux or if it is allocated to be used by DSP.
 252          */
 253         unsigned long *slot_inuse;
 254
 255         /*
 256          * For tracking reserved channels used by DSP.
 257          * If the bit is cleared, the channel is allocated to be used by DSP
 258          * and Linux must not touch it.
 259          */
 260         unsigned long *channels_mask;
 261
 262         struct dma_device               dma_slave;
 263         struct dma_device               *dma_memcpy;
 264         struct edma_chan                *slave_chans;
 265         struct edma_tc                  *tc_list;
 266         int                             dummy_slot;
 267 };
 268
 269 /* dummy param set used to (re)initialize parameter RAM slots */
 270 static const struct edmacc_param dummy_paramset = {
 271         .link_bcntrld = 0xffff,
 272         .ccnt = 1,
 273 };
 274
 275 #define EDMA_BINDING_LEGACY     0
 276 #define EDMA_BINDING_TPCC       1
 277 static const u32 edma_binding_type[] = {
 278         [EDMA_BINDING_LEGACY] = EDMA_BINDING_LEGACY,
 279         [EDMA_BINDING_TPCC] = EDMA_BINDING_TPCC,
 280 };
 281
 282 static const struct of_device_id edma_of_ids[] = {
 283         {
 284                 .compatible = "ti,edma3",
 285                 .data = &edma_binding_type[EDMA_BINDING_LEGACY],
 286         },
 287         {
 288                 .compatible = "ti,edma3-tpcc",
 289                 .data = &edma_binding_type[EDMA_BINDING_TPCC],
 290         },
 291         {}
 292 };
 293 MODULE_DEVICE_TABLE(of, edma_of_ids);
 294
 295 static const struct of_device_id edma_tptc_of_ids[] = {
 296         { .compatible = "ti,edma3-tptc", },
 297         {}
 298 };
 299 MODULE_DEVICE_TABLE(of, edma_tptc_of_ids);
 300
 301 static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
 302 {
 303         return (unsigned int)__raw_readl(ecc->base + offset);
 304 }
 305
 306 static inline void edma_write(struct edma_cc *ecc, int offset, int val)
 307 {
 308         __raw_writel(val, ecc->base + offset);
 309 }
 310
 311 static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
 312                                unsigned or)
 313 {
 314         unsigned val = edma_read(ecc, offset);
 315
 316         val &= and;
 317         val |= or;
 318         edma_write(ecc, offset, val);
 319 }
 320
 321 static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
 322 {
 323         unsigned val = edma_read(ecc, offset);
 324
 325         val &= and;
 326         edma_write(ecc, offset, val);
 327 }
 328
 329 static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
 330 {
 331         unsigned val = edma_read(ecc, offset);
 332
 333         val |= or;
 334         edma_write(ecc, offset, val);
 335 }
 336
 337 static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
 338                                            int i)
 339 {
 340         return edma_read(ecc, offset + (i << 2));
 341 }
 342
 343 static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
 344                                     unsigned val)
 345 {
 346         edma_write(ecc, offset + (i << 2), val);
 347 }
 348
 349 static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
 350                                      unsigned and, unsigned or)
 351 {
 352         edma_modify(ecc, offset + (i << 2), and, or);
 353 }
 354
 355 static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
 356                                   unsigned or)
 357 {
 358         edma_or(ecc, offset + ((i * 2 + j) << 2), or);
 359 }
 360
 361 static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
 362                                      int j, unsigned val)
 363 {
 364         edma_write(ecc, offset + ((i * 2 + j) << 2), val);
 365 }
 366
 367 static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
 368                                                    int offset, int i)
 369 {
 370         return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
 371 }
 372
 373 static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
 374                                       unsigned val)
 375 {
 376         edma_write(ecc, EDMA_SHADOW0 + offset, val);
 377 }
 378
 379 static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
 380                                             int i, unsigned val)
 381 {
 382         edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
 383 }
 384
 385 static inline void edma_param_modify(struct edma_cc *ecc, int offset,
 386                                      int param_no, unsigned and, unsigned or)
 387 {
 388         edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
 389 }
 390
 391 static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
 392                                           int priority)
 393 {
 394         int bit = queue_no * 4;
 395
 396         edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
 397 }
 398
 399 static void edma_set_chmap(struct edma_chan *echan, int slot)
 400 {
 401         struct edma_cc *ecc = echan->ecc;
 402         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 403
 404         if (ecc->chmap_exist) {
 405                 slot = EDMA_CHAN_SLOT(slot);
 406                 edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
 407         }
 408 }
 409
 410 static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
 411 {
 412         struct edma_cc *ecc = echan->ecc;
 413         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 414         int idx = EDMA_REG_ARRAY_INDEX(channel);
 415         int ch_bit = EDMA_CHANNEL_BIT(channel);
 416
 417         if (enable) {
 418                 edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 419                 edma_shadow0_write_array(ecc, SH_IESR, idx, ch_bit);
 420         } else {
 421                 edma_shadow0_write_array(ecc, SH_IECR, idx, ch_bit);
 422         }
 423 }
 424
 425 /*
 426  * paRAM slot management functions
 427  */
 428 static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
 429                             const struct edmacc_param *param)
 430 {
 431         slot = EDMA_CHAN_SLOT(slot);
 432         if (slot >= ecc->num_slots)
 433                 return;
 434         memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
 435 }
 436
 437 static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
 438                            struct edmacc_param *param)
 439 {
 440         slot = EDMA_CHAN_SLOT(slot);
 441         if (slot >= ecc->num_slots)
 442                 return -EINVAL;
 443         memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
 444
 445         return 0;
 446 }
 447
 448 /**
 449  * edma_alloc_slot - allocate DMA parameter RAM
 450  * @ecc: pointer to edma_cc struct
 451  * @slot: specific slot to allocate; negative for "any unused slot"
 452  *
 453  * This allocates a parameter RAM slot, initializing it to hold a
 454  * dummy transfer.  Slots allocated using this routine have not been
 455  * mapped to a hardware DMA channel, and will normally be used by
 456  * linking to them from a slot associated with a DMA channel.
 457  *
 458  * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
 459  * slots may be allocated on behalf of DSP firmware.
 460  *
 461  * Returns the number of the slot, else negative errno.
 462  */
 463 static int edma_alloc_slot(struct edma_cc *ecc, int slot)
 464 {
 465         if (slot >= 0) {
 466                 slot = EDMA_CHAN_SLOT(slot);
 467                 /* Requesting entry paRAM slot for a HW triggered channel. */
 468                 if (ecc->chmap_exist && slot < ecc->num_channels)
 469                         slot = EDMA_SLOT_ANY;
 470         }
 471
 472         if (slot < 0) {
 473                 if (ecc->chmap_exist)
 474                         slot = 0;
 475                 else
 476                         slot = ecc->num_channels;
 477                 for (;;) {
 478                         slot = find_next_zero_bit(ecc->slot_inuse,
 479                                                   ecc->num_slots,
 480                                                   slot);
 481                         if (slot == ecc->num_slots)
 482                                 return -ENOMEM;
 483                         if (!test_and_set_bit(slot, ecc->slot_inuse))
 484                                 break;
 485                 }
 486         } else if (slot >= ecc->num_slots) {
 487                 return -EINVAL;
 488         } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
 489                 return -EBUSY;
 490         }
 491
 492         edma_write_slot(ecc, slot, &dummy_paramset);
 493
 494         return EDMA_CTLR_CHAN(ecc->id, slot);
 495 }
 496
 497 static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
 498 {
 499         slot = EDMA_CHAN_SLOT(slot);
 500         if (slot >= ecc->num_slots)
 501                 return;
 502
 503         edma_write_slot(ecc, slot, &dummy_paramset);
 504         clear_bit(slot, ecc->slot_inuse);
 505 }
 506
 507 /**
 508  * edma_link - link one parameter RAM slot to another
 509  * @ecc: pointer to edma_cc struct
 510  * @from: parameter RAM slot originating the link
 511  * @to: parameter RAM slot which is the link target
 512  *
 513  * The originating slot should not be part of any active DMA transfer.
 514  */
 515 static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
 516 {
 517         if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
 518                 dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
 519
 520         from = EDMA_CHAN_SLOT(from);
 521         to = EDMA_CHAN_SLOT(to);
 522         if (from >= ecc->num_slots || to >= ecc->num_slots)
 523                 return;
 524
 525         edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
 526                           PARM_OFFSET(to));
 527 }
 528
 529 /**
 530  * edma_get_position - returns the current transfer point
 531  * @ecc: pointer to edma_cc struct
 532  * @slot: parameter RAM slot being examined
 533  * @dst:  true selects the dest position, false the source
 534  *
 535  * Returns the position of the current active slot
 536  */
 537 static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
 538                                     bool dst)
 539 {
 540         u32 offs;
 541
 542         slot = EDMA_CHAN_SLOT(slot);
 543         offs = PARM_OFFSET(slot);
 544         offs += dst ? PARM_DST : PARM_SRC;
 545
 546         return edma_read(ecc, offs);
 547 }
 548
 549 /*
 550  * Channels with event associations will be triggered by their hardware
 551  * events, and channels without such associations will be triggered by
 552  * software.  (At this writing there is no interface for using software
 553  * triggers except with channels that don't support hardware triggers.)
 554  */
 555 static void edma_start(struct edma_chan *echan)
 556 {
 557         struct edma_cc *ecc = echan->ecc;
 558         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 559         int idx = EDMA_REG_ARRAY_INDEX(channel);
 560         int ch_bit = EDMA_CHANNEL_BIT(channel);
 561
 562         if (!echan->hw_triggered) {
 563                 /* EDMA channels without event association */
 564                 dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 565                         edma_shadow0_read_array(ecc, SH_ESR, idx));
 566                 edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 567         } else {
 568                 /* EDMA channel with event association */
 569                 dev_dbg(ecc->dev, "ER%d %08x\n", idx,
 570                         edma_shadow0_read_array(ecc, SH_ER, idx));
 571                 /* Clear any pending event or error */
 572                 edma_write_array(ecc, EDMA_ECR, idx, ch_bit);
 573                 edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 574                 /* Clear any SER */
 575                 edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 576                 edma_shadow0_write_array(ecc, SH_EESR, idx, ch_bit);
 577                 dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 578                         edma_shadow0_read_array(ecc, SH_EER, idx));
 579         }
 580 }
 581
 582 static void edma_stop(struct edma_chan *echan)
 583 {
 584         struct edma_cc *ecc = echan->ecc;
 585         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 586         int idx = EDMA_REG_ARRAY_INDEX(channel);
 587         int ch_bit = EDMA_CHANNEL_BIT(channel);
 588
 589         edma_shadow0_write_array(ecc, SH_EECR, idx, ch_bit);
 590         edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 591         edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 592         edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 593
 594         /* clear possibly pending completion interrupt */
 595         edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
 596
 597         dev_dbg(ecc->dev, "EER%d %08x\n", idx,
 598                 edma_shadow0_read_array(ecc, SH_EER, idx));
 599
 600         /* REVISIT:  consider guarding against inappropriate event
 601          * chaining by overwriting with dummy_paramset.
 602          */
 603 }
 604
 605 /*
 606  * Temporarily disable EDMA hardware events on the specified channel,
 607  * preventing them from triggering new transfers
 608  */
 609 static void edma_pause(struct edma_chan *echan)
 610 {
 611         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 612
 613         edma_shadow0_write_array(echan->ecc, SH_EECR,
 614                                  EDMA_REG_ARRAY_INDEX(channel),
 615                                  EDMA_CHANNEL_BIT(channel));
 616 }
 617
 618 /* Re-enable EDMA hardware events on the specified channel.  */
 619 static void edma_resume(struct edma_chan *echan)
 620 {
 621         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 622
 623         edma_shadow0_write_array(echan->ecc, SH_EESR,
 624                                  EDMA_REG_ARRAY_INDEX(channel),
 625                                  EDMA_CHANNEL_BIT(channel));
 626 }
 627
 628 static void edma_trigger_channel(struct edma_chan *echan)
 629 {
 630         struct edma_cc *ecc = echan->ecc;
 631         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 632         int idx = EDMA_REG_ARRAY_INDEX(channel);
 633         int ch_bit = EDMA_CHANNEL_BIT(channel);
 634
 635         edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
 636
 637         dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
 638                 edma_shadow0_read_array(ecc, SH_ESR, idx));
 639 }
 640
 641 static void edma_clean_channel(struct edma_chan *echan)
 642 {
 643         struct edma_cc *ecc = echan->ecc;
 644         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 645         int idx = EDMA_REG_ARRAY_INDEX(channel);
 646         int ch_bit = EDMA_CHANNEL_BIT(channel);
 647
 648         dev_dbg(ecc->dev, "EMR%d %08x\n", idx,
 649                 edma_read_array(ecc, EDMA_EMR, idx));
 650         edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
 651         /* Clear the corresponding EMR bits */
 652         edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
 653         /* Clear any SER */
 654         edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
 655         edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
 656 }
 657
 658 /* Move channel to a specific event queue */
 659 static void edma_assign_channel_eventq(struct edma_chan *echan,
 660                                        enum dma_event_q eventq_no)
 661 {
 662         struct edma_cc *ecc = echan->ecc;
 663         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 664         int bit = (channel & 0x7) * 4;
 665
 666         /* default to low priority queue */
 667         if (eventq_no == EVENTQ_DEFAULT)
 668                 eventq_no = ecc->default_queue;
 669         if (eventq_no >= ecc->num_tc)
 670                 return;
 671
 672         eventq_no &= 7;
 673         edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
 674                           eventq_no << bit);
 675 }
 676
 677 static int edma_alloc_channel(struct edma_chan *echan,
 678                               enum dma_event_q eventq_no)
 679 {
 680         struct edma_cc *ecc = echan->ecc;
 681         int channel = EDMA_CHAN_SLOT(echan->ch_num);
 682
 683         if (!test_bit(echan->ch_num, ecc->channels_mask)) {
 684                 dev_err(ecc->dev, "Channel%d is reserved, can not be used!\n",
 685                         echan->ch_num);
 686                 return -EINVAL;
 687         }
 688
 689         /* ensure access through shadow region 0 */
 690         edma_or_array2(ecc, EDMA_DRAE, 0, EDMA_REG_ARRAY_INDEX(channel),
 691                        EDMA_CHANNEL_BIT(channel));
 692
 693         /* ensure no events are pending */
 694         edma_stop(echan);
 695
 696         edma_setup_interrupt(echan, true);
 697
 698         edma_assign_channel_eventq(echan, eventq_no);
 699
 700         return 0;
 701 }
 702
 703 static void edma_free_channel(struct edma_chan *echan)
 704 {
 705         /* ensure no events are pending */
 706         edma_stop(echan);
 707         /* REVISIT should probably take out of shadow region 0 */
 708         edma_setup_interrupt(echan, false);
 709 }
 710
 711 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
 712 {
 713         return container_of(c, struct edma_chan, vchan.chan);
 714 }
 715
 716 static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
 717 {
 718         return container_of(tx, struct edma_desc, vdesc.tx);
 719 }
 720
 721 static void edma_desc_free(struct virt_dma_desc *vdesc)
 722 {
 723         kfree(container_of(vdesc, struct edma_desc, vdesc));
 724 }
 725
 726 /* Dispatch a queued descriptor to the controller (caller holds lock) */
 727 static void edma_execute(struct edma_chan *echan)
 728 {
 729         struct edma_cc *ecc = echan->ecc;
 730         struct virt_dma_desc *vdesc;
 731         struct edma_desc *edesc;
 732         struct device *dev = echan->vchan.chan.device->dev;
 733         int i, j, left, nslots;
 734
 735         if (!echan->edesc) {
 736                 /* Setup is needed for the first transfer */
 737                 vdesc = vchan_next_desc(&echan->vchan);
 738                 if (!vdesc)
 739                         return;
 740                 list_del(&vdesc->node);
 741                 echan->edesc = to_edma_desc(&vdesc->tx);
 742         }
 743
 744         edesc = echan->edesc;
 745
 746         /* Find out how many left */
 747         left = edesc->pset_nr - edesc->processed;
 748         nslots = min(MAX_NR_SG, left);
 749         edesc->sg_len = 0;
 750
 751         /* Write descriptor PaRAM set(s) */
 752         for (i = 0; i < nslots; i++) {
 753                 j = i + edesc->processed;
 754                 edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
 755                 edesc->sg_len += edesc->pset[j].len;
 756                 dev_vdbg(dev,
 757                          "\n pset[%d]:\n"
 758                          "  chnum\t%d\n"
 759                          "  slot\t%d\n"
 760                          "  opt\t%08x\n"
 761                          "  src\t%08x\n"
 762                          "  dst\t%08x\n"
 763                          "  abcnt\t%08x\n"
 764                          "  ccnt\t%08x\n"
 765                          "  bidx\t%08x\n"
 766                          "  cidx\t%08x\n"
 767                          "  lkrld\t%08x\n",
 768                          j, echan->ch_num, echan->slot[i],
 769                          edesc->pset[j].param.opt,
 770                          edesc->pset[j].param.src,
 771                          edesc->pset[j].param.dst,
 772                          edesc->pset[j].param.a_b_cnt,
 773                          edesc->pset[j].param.ccnt,
 774                          edesc->pset[j].param.src_dst_bidx,
 775                          edesc->pset[j].param.src_dst_cidx,
 776                          edesc->pset[j].param.link_bcntrld);
 777                 /* Link to the previous slot if not the last set */
 778                 if (i != (nslots - 1))
 779                         edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
 780         }
 781
 782         edesc->processed += nslots;
 783
 784         /*
 785          * If this is either the last set in a set of SG-list transactions
 786          * then setup a link to the dummy slot, this results in all future
 787          * events being absorbed and that's OK because we're done
 788          */
 789         if (edesc->processed == edesc->pset_nr) {
 790                 if (edesc->cyclic)
 791                         edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
 792                 else
 793                         edma_link(ecc, echan->slot[nslots - 1],
 794                                   echan->ecc->dummy_slot);
 795         }
 796
 797         if (echan->missed) {
 798                 /*
 799                  * This happens due to setup times between intermediate
 800                  * transfers in long SG lists which have to be broken up into
 801                  * transfers of MAX_NR_SG
 802                  */
 803                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
 804                 edma_clean_channel(echan);
 805                 edma_stop(echan);
 806                 edma_start(echan);
 807                 edma_trigger_channel(echan);
 808                 echan->missed = 0;
 809         } else if (edesc->processed <= MAX_NR_SG) {
 810                 dev_dbg(dev, "first transfer starting on channel %d\n",
 811                         echan->ch_num);
 812                 edma_start(echan);
 813         } else {
 814                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
 815                         echan->ch_num, edesc->processed);
 816                 edma_resume(echan);
 817         }
 818 }
 819
 820 static int edma_terminate_all(struct dma_chan *chan)
 821 {
 822         struct edma_chan *echan = to_edma_chan(chan);
 823         unsigned long flags;
 824         LIST_HEAD(head);
 825
 826         spin_lock_irqsave(&echan->vchan.lock, flags);
 827
 828         /*
 829          * Stop DMA activity: we assume the callback will not be called
 830          * after edma_dma() returns (even if it does, it will see
 831          * echan->edesc is NULL and exit.)
 832          */
 833         if (echan->edesc) {
 834                 edma_stop(echan);
 835                 /* Move the cyclic channel back to default queue */
 836                 if (!echan->tc && echan->edesc->cyclic)
 837                         edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
 838
 839                 vchan_terminate_vdesc(&echan->edesc->vdesc);
 840                 echan->edesc = NULL;
 841         }
 842
 843         vchan_get_all_descriptors(&echan->vchan, &head);
 844         spin_unlock_irqrestore(&echan->vchan.lock, flags);
 845         vchan_dma_desc_free_list(&echan->vchan, &head);
 846
 847         return 0;
 848 }
 849
 850 static void edma_synchronize(struct dma_chan *chan)
 851 {
 852         struct edma_chan *echan = to_edma_chan(chan);
 853
 854         vchan_synchronize(&echan->vchan);
 855 }
 856
 857 static int edma_slave_config(struct dma_chan *chan,
 858         struct dma_slave_config *cfg)
 859 {
 860         struct edma_chan *echan = to_edma_chan(chan);
 861
 862         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
 863             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
 864                 return -EINVAL;
 865
 866         if (cfg->src_maxburst > chan->device->max_burst ||
 867             cfg->dst_maxburst > chan->device->max_burst)
 868                 return -EINVAL;
 869
 870         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
 871
 872         return 0;
 873 }
 874
 875 static int edma_dma_pause(struct dma_chan *chan)
 876 {
 877         struct edma_chan *echan = to_edma_chan(chan);
 878
 879         if (!echan->edesc)
 880                 return -EINVAL;
 881
 882         edma_pause(echan);
 883         return 0;
 884 }
 885
 886 static int edma_dma_resume(struct dma_chan *chan)
 887 {
 888         struct edma_chan *echan = to_edma_chan(chan);
 889
 890         edma_resume(echan);
 891         return 0;
 892 }
 893
 894 /*
 895  * A PaRAM set configuration abstraction used by other modes
 896  * @chan: Channel who's PaRAM set we're configuring
 897  * @pset: PaRAM set to initialize and setup.
 898  * @src_addr: Source address of the DMA
 899  * @dst_addr: Destination address of the DMA
 900  * @burst: In units of dev_width, how much to send
 901  * @dev_width: How much is the dev_width
 902  * @dma_length: Total length of the DMA transfer
 903  * @direction: Direction of the transfer
 904  */
 905 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
 906                             dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
 907                             unsigned int acnt, unsigned int dma_length,
 908                             enum dma_transfer_direction direction)
 909 {
 910         struct edma_chan *echan = to_edma_chan(chan);
 911         struct device *dev = chan->device->dev;
 912         struct edmacc_param *param = &epset->param;
 913         int bcnt, ccnt, cidx;
 914         int src_bidx, dst_bidx, src_cidx, dst_cidx;
 915         int absync;
 916
 917         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
 918         if (!burst)
 919                 burst = 1;
 920         /*
 921          * If the maxburst is equal to the fifo width, use
 922          * A-synced transfers. This allows for large contiguous
 923          * buffer transfers using only one PaRAM set.
 924          */
 925         if (burst == 1) {
 926                 /*
 927                  * For the A-sync case, bcnt and ccnt are the remainder
 928                  * and quotient respectively of the division of:
 929                  * (dma_length / acnt) by (SZ_64K -1). This is so
 930                  * that in case bcnt over flows, we have ccnt to use.
 931                  * Note: In A-sync transfer only, bcntrld is used, but it
 932                  * only applies for sg_dma_len(sg) >= SZ_64K.
 933                  * In this case, the best way adopted is- bccnt for the
 934                  * first frame will be the remainder below. Then for
 935                  * every successive frame, bcnt will be SZ_64K-1. This
 936                  * is assured as bcntrld = 0xffff in end of function.
 937                  */
 938                 absync = false;
 939                 ccnt = dma_length / acnt / (SZ_64K - 1);
 940                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
 941                 /*
 942                  * If bcnt is non-zero, we have a remainder and hence an
 943                  * extra frame to transfer, so increment ccnt.
 944                  */
 945                 if (bcnt)
 946                         ccnt++;
 947                 else
 948                         bcnt = SZ_64K - 1;
 949                 cidx = acnt;
 950         } else {
 951                 /*
 952                  * If maxburst is greater than the fifo address_width,
 953                  * use AB-synced transfers where A count is the fifo
 954                  * address_width and B count is the maxburst. In this
 955                  * case, we are limited to transfers of C count frames
 956                  * of (address_width * maxburst) where C count is limited
 957                  * to SZ_64K-1. This places an upper bound on the length
 958                  * of an SG segment that can be handled.
 959                  */
 960                 absync = true;
 961                 bcnt = burst;
 962                 ccnt = dma_length / (acnt * bcnt);
 963                 if (ccnt > (SZ_64K - 1)) {
 964                         dev_err(dev, "Exceeded max SG segment size\n");
 965                         return -EINVAL;
 966                 }
 967                 cidx = acnt * bcnt;
 968         }
 969
 970         epset->len = dma_length;
 971
 972         if (direction == DMA_MEM_TO_DEV) {
 973                 src_bidx = acnt;
 974                 src_cidx = cidx;
 975                 dst_bidx = 0;
 976                 dst_cidx = 0;
 977                 epset->addr = src_addr;
 978         } else if (direction == DMA_DEV_TO_MEM)  {
 979                 src_bidx = 0;
 980                 src_cidx = 0;
 981                 dst_bidx = acnt;
 982                 dst_cidx = cidx;
 983                 epset->addr = dst_addr;
 984         } else if (direction == DMA_MEM_TO_MEM)  {
 985                 src_bidx = acnt;
 986                 src_cidx = cidx;
 987                 dst_bidx = acnt;
 988                 dst_cidx = cidx;
 989                 epset->addr = src_addr;
 990         } else {
 991                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
 992                 return -EINVAL;
 993         }
 994
 995         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
 996         /* Configure A or AB synchronized transfers */
 997         if (absync)
 998                 param->opt |= SYNCDIM;
 999
1000         param->src = src_addr;
1001         param->dst = dst_addr;
1002
1003         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1004         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
1005
1006         param->a_b_cnt = bcnt << 16 | acnt;
1007         param->ccnt = ccnt;
1008         /*
1009          * Only time when (bcntrld) auto reload is required is for
1010          * A-sync case, and in this case, a requirement of reload value
1011          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
1012          * and then later will be populated by edma_execute.
1013          */
1014         param->link_bcntrld = 0xffffffff;
1015         return absync;
1016 }
1017
1018 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
1019         struct dma_chan *chan, struct scatterlist *sgl,
1020         unsigned int sg_len, enum dma_transfer_direction direction,
1021         unsigned long tx_flags, void *context)
1022 {
1023         struct edma_chan *echan = to_edma_chan(chan);
1024         struct device *dev = chan->device->dev;
1025         struct edma_desc *edesc;
1026         dma_addr_t src_addr = 0, dst_addr = 0;
1027         enum dma_slave_buswidth dev_width;
1028         u32 burst;
1029         struct scatterlist *sg;
1030         int i, nslots, ret;
1031
1032         if (unlikely(!echan || !sgl || !sg_len))
1033                 return NULL;
1034
1035         if (direction == DMA_DEV_TO_MEM) {
1036                 src_addr = echan->cfg.src_addr;
1037                 dev_width = echan->cfg.src_addr_width;
1038                 burst = echan->cfg.src_maxburst;
1039         } else if (direction == DMA_MEM_TO_DEV) {
1040                 dst_addr = echan->cfg.dst_addr;
1041                 dev_width = echan->cfg.dst_addr_width;
1042                 burst = echan->cfg.dst_maxburst;
1043         } else {
1044                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1045                 return NULL;
1046         }
1047
1048         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1049                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1050                 return NULL;
1051         }
1052
1053         edesc = kzalloc(struct_size(edesc, pset, sg_len), GFP_ATOMIC);
1054         if (!edesc)
1055                 return NULL;
1056
1057         edesc->pset_nr = sg_len;
1058         edesc->residue = 0;
1059         edesc->direction = direction;
1060         edesc->echan = echan;
1061
1062         /* Allocate a PaRAM slot, if needed */
1063         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
1064
1065         for (i = 0; i < nslots; i++) {
1066                 if (echan->slot[i] < 0) {
1067                         echan->slot[i] =
1068                                 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1069                         if (echan->slot[i] < 0) {
1070                                 kfree(edesc);
1071                                 dev_err(dev, "%s: Failed to allocate slot\n",
1072                                         __func__);
1073                                 return NULL;
1074                         }
1075                 }
1076         }
1077
1078         /* Configure PaRAM sets for each SG */
1079         for_each_sg(sgl, sg, sg_len, i) {
1080                 /* Get address for each SG */
1081                 if (direction == DMA_DEV_TO_MEM)
1082                         dst_addr = sg_dma_address(sg);
1083                 else
1084                         src_addr = sg_dma_address(sg);
1085
1086                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1087                                        dst_addr, burst, dev_width,
1088                                        sg_dma_len(sg), direction);
1089                 if (ret < 0) {
1090                         kfree(edesc);
1091                         return NULL;
1092                 }
1093
1094                 edesc->absync = ret;
1095                 edesc->residue += sg_dma_len(sg);
1096
1097                 if (i == sg_len - 1)
1098                         /* Enable completion interrupt */
1099                         edesc->pset[i].param.opt |= TCINTEN;
1100                 else if (!((i+1) % MAX_NR_SG))
1101                         /*
1102                          * Enable early completion interrupt for the
1103                          * intermediateset. In this case the driver will be
1104                          * notified when the paRAM set is submitted to TC. This
1105                          * will allow more time to set up the next set of slots.
1106                          */
1107                         edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);
1108         }
1109         edesc->residue_stat = edesc->residue;
1110
1111         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1112 }
1113
1114 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
1115         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1116         size_t len, unsigned long tx_flags)
1117 {
1118         int ret, nslots;
1119         struct edma_desc *edesc;
1120         struct device *dev = chan->device->dev;
1121         struct edma_chan *echan = to_edma_chan(chan);
1122         unsigned int width, pset_len, array_size;
1123
1124         if (unlikely(!echan || !len))
1125                 return NULL;
1126
1127         /* Align the array size (acnt block) with the transfer properties */
1128         switch (__ffs((src | dest | len))) {
1129         case 0:
1130                 array_size = SZ_32K - 1;
1131                 break;
1132         case 1:
1133                 array_size = SZ_32K - 2;
1134                 break;
1135         default:
1136                 array_size = SZ_32K - 4;
1137                 break;
1138         }
1139
1140         if (len < SZ_64K) {
1141                 /*
1142                  * Transfer size less than 64K can be handled with one paRAM
1143                  * slot and with one burst.
1144                  * ACNT = length
1145                  */
1146                 width = len;
1147                 pset_len = len;
1148                 nslots = 1;
1149         } else {
1150                 /*
1151                  * Transfer size bigger than 64K will be handled with maximum of
1152                  * two paRAM slots.
1153                  * slot1: (full_length / 32767) times 32767 bytes bursts.
1154                  *        ACNT = 32767, length1: (full_length / 32767) * 32767
1155                  * slot2: the remaining amount of data after slot1.
1156                  *        ACNT = full_length - length1, length2 = ACNT
1157                  *
1158                  * When the full_length is a multiple of 32767 one slot can be
1159                  * used to complete the transfer.
1160                  */
1161                 width = array_size;
1162                 pset_len = rounddown(len, width);
1163                 /* One slot is enough for lengths multiple of (SZ_32K -1) */
1164                 if (unlikely(pset_len == len))
1165                         nslots = 1;
1166                 else
1167                         nslots = 2;
1168         }
1169
1170         edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1171         if (!edesc)
1172                 return NULL;
1173
1174         edesc->pset_nr = nslots;
1175         edesc->residue = edesc->residue_stat = len;
1176         edesc->direction = DMA_MEM_TO_MEM;
1177         edesc->echan = echan;
1178
1179         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
1180                                width, pset_len, DMA_MEM_TO_MEM);
1181         if (ret < 0) {
1182                 kfree(edesc);
1183                 return NULL;
1184         }
1185
1186         edesc->absync = ret;
1187
1188         edesc->pset[0].param.opt |= ITCCHEN;
1189         if (nslots == 1) {
1190                 /* Enable transfer complete interrupt if requested */
1191                 if (tx_flags & DMA_PREP_INTERRUPT)
1192                         edesc->pset[0].param.opt |= TCINTEN;
1193         } else {
1194                 /* Enable transfer complete chaining for the first slot */
1195                 edesc->pset[0].param.opt |= TCCHEN;
1196
1197                 if (echan->slot[1] < 0) {
1198                         echan->slot[1] = edma_alloc_slot(echan->ecc,
1199                                                          EDMA_SLOT_ANY);
1200                         if (echan->slot[1] < 0) {
1201                                 kfree(edesc);
1202                                 dev_err(dev, "%s: Failed to allocate slot\n",
1203                                         __func__);
1204                                 return NULL;
1205                         }
1206                 }
1207                 dest += pset_len;
1208                 src += pset_len;
1209                 pset_len = width = len % array_size;
1210
1211                 ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,
1212                                        width, pset_len, DMA_MEM_TO_MEM);
1213                 if (ret < 0) {
1214                         kfree(edesc);
1215                         return NULL;
1216                 }
1217
1218                 edesc->pset[1].param.opt |= ITCCHEN;
1219                 /* Enable transfer complete interrupt if requested */
1220                 if (tx_flags & DMA_PREP_INTERRUPT)
1221                         edesc->pset[1].param.opt |= TCINTEN;
1222         }
1223
1224         if (!(tx_flags & DMA_PREP_INTERRUPT))
1225                 edesc->polled = true;
1226
1227         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1228 }
1229
1230 static struct dma_async_tx_descriptor *
1231 edma_prep_dma_interleaved(struct dma_chan *chan,
1232                           struct dma_interleaved_template *xt,
1233                           unsigned long tx_flags)
1234 {
1235         struct device *dev = chan->device->dev;
1236         struct edma_chan *echan = to_edma_chan(chan);
1237         struct edmacc_param *param;
1238         struct edma_desc *edesc;
1239         size_t src_icg, dst_icg;
1240         int src_bidx, dst_bidx;
1241
1242         /* Slave mode is not supported */
1243         if (is_slave_direction(xt->dir))
1244                 return NULL;
1245
1246         if (xt->frame_size != 1 || xt->numf == 0)
1247                 return NULL;
1248
1249         if (xt->sgl[0].size > SZ_64K || xt->numf > SZ_64K)
1250                 return NULL;
1251
1252         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1253         if (src_icg) {
1254                 src_bidx = src_icg + xt->sgl[0].size;
1255         } else if (xt->src_inc) {
1256                 src_bidx = xt->sgl[0].size;
1257         } else {
1258                 dev_err(dev, "%s: SRC constant addressing is not supported\n",
1259                         __func__);
1260                 return NULL;
1261         }
1262
1263         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1264         if (dst_icg) {
1265                 dst_bidx = dst_icg + xt->sgl[0].size;
1266         } else if (xt->dst_inc) {
1267                 dst_bidx = xt->sgl[0].size;
1268         } else {
1269                 dev_err(dev, "%s: DST constant addressing is not supported\n",
1270                         __func__);
1271                 return NULL;
1272         }
1273
1274         if (src_bidx > SZ_64K || dst_bidx > SZ_64K)
1275                 return NULL;
1276
1277         edesc = kzalloc(struct_size(edesc, pset, 1), GFP_ATOMIC);
1278         if (!edesc)
1279                 return NULL;
1280
1281         edesc->direction = DMA_MEM_TO_MEM;
1282         edesc->echan = echan;
1283         edesc->pset_nr = 1;
1284
1285         param = &edesc->pset[0].param;
1286
1287         param->src = xt->src_start;
1288         param->dst = xt->dst_start;
1289         param->a_b_cnt = xt->numf << 16 | xt->sgl[0].size;
1290         param->ccnt = 1;
1291         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1292         param->src_dst_cidx = 0;
1293
1294         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
1295         param->opt |= ITCCHEN;
1296         /* Enable transfer complete interrupt if requested */
1297         if (tx_flags & DMA_PREP_INTERRUPT)
1298                 param->opt |= TCINTEN;
1299         else
1300                 edesc->polled = true;
1301
1302         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1303 }
1304
1305 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
1306         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1307         size_t period_len, enum dma_transfer_direction direction,
1308         unsigned long tx_flags)
1309 {
1310         struct edma_chan *echan = to_edma_chan(chan);
1311         struct device *dev = chan->device->dev;
1312         struct edma_desc *edesc;
1313         dma_addr_t src_addr, dst_addr;
1314         enum dma_slave_buswidth dev_width;
1315         bool use_intermediate = false;
1316         u32 burst;
1317         int i, ret, nslots;
1318
1319         if (unlikely(!echan || !buf_len || !period_len))
1320                 return NULL;
1321
1322         if (direction == DMA_DEV_TO_MEM) {
1323                 src_addr = echan->cfg.src_addr;
1324                 dst_addr = buf_addr;
1325                 dev_width = echan->cfg.src_addr_width;
1326                 burst = echan->cfg.src_maxburst;
1327         } else if (direction == DMA_MEM_TO_DEV) {
1328                 src_addr = buf_addr;
1329                 dst_addr = echan->cfg.dst_addr;
1330                 dev_width = echan->cfg.dst_addr_width;
1331                 burst = echan->cfg.dst_maxburst;
1332         } else {
1333                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
1334                 return NULL;
1335         }
1336
1337         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
1338                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
1339                 return NULL;
1340         }
1341
1342         if (unlikely(buf_len % period_len)) {
1343                 dev_err(dev, "Period should be multiple of Buffer length\n");
1344                 return NULL;
1345         }
1346
1347         nslots = (buf_len / period_len) + 1;
1348
1349         /*
1350          * Cyclic DMA users such as audio cannot tolerate delays introduced
1351          * by cases where the number of periods is more than the maximum
1352          * number of SGs the EDMA driver can handle at a time. For DMA types
1353          * such as Slave SGs, such delays are tolerable and synchronized,
1354          * but the synchronization is difficult to achieve with Cyclic and
1355          * cannot be guaranteed, so we error out early.
1356          */
1357         if (nslots > MAX_NR_SG) {
1358                 /*
1359                  * If the burst and period sizes are the same, we can put
1360                  * the full buffer into a single period and activate
1361                  * intermediate interrupts. This will produce interrupts
1362                  * after each burst, which is also after each desired period.
1363                  */
1364                 if (burst == period_len) {
1365                         period_len = buf_len;
1366                         nslots = 2;
1367                         use_intermediate = true;
1368                 } else {
1369                         return NULL;
1370                 }
1371         }
1372
1373         edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);
1374         if (!edesc)
1375                 return NULL;
1376
1377         edesc->cyclic = 1;
1378         edesc->pset_nr = nslots;
1379         edesc->residue = edesc->residue_stat = buf_len;
1380         edesc->direction = direction;
1381         edesc->echan = echan;
1382
1383         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
1384                 __func__, echan->ch_num, nslots, period_len, buf_len);
1385
1386         for (i = 0; i < nslots; i++) {
1387                 /* Allocate a PaRAM slot, if needed */
1388                 if (echan->slot[i] < 0) {
1389                         echan->slot[i] =
1390                                 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1391                         if (echan->slot[i] < 0) {
1392                                 kfree(edesc);
1393                                 dev_err(dev, "%s: Failed to allocate slot\n",
1394                                         __func__);
1395                                 return NULL;
1396                         }
1397                 }
1398
1399                 if (i == nslots - 1) {
1400                         memcpy(&edesc->pset[i], &edesc->pset[0],
1401                                sizeof(edesc->pset[0]));
1402                         break;
1403                 }
1404
1405                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1406                                        dst_addr, burst, dev_width, period_len,
1407                                        direction);
1408                 if (ret < 0) {
1409                         kfree(edesc);
1410                         return NULL;
1411                 }
1412
1413                 if (direction == DMA_DEV_TO_MEM)
1414                         dst_addr += period_len;
1415                 else
1416                         src_addr += period_len;
1417
1418                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
1419                 dev_vdbg(dev,
1420                         "\n pset[%d]:\n"
1421                         "  chnum\t%d\n"
1422                         "  slot\t%d\n"
1423                         "  opt\t%08x\n"
1424                         "  src\t%08x\n"
1425                         "  dst\t%08x\n"
1426                         "  abcnt\t%08x\n"
1427                         "  ccnt\t%08x\n"
1428                         "  bidx\t%08x\n"
1429                         "  cidx\t%08x\n"
1430                         "  lkrld\t%08x\n",
1431                         i, echan->ch_num, echan->slot[i],
1432                         edesc->pset[i].param.opt,
1433                         edesc->pset[i].param.src,
1434                         edesc->pset[i].param.dst,
1435                         edesc->pset[i].param.a_b_cnt,
1436                         edesc->pset[i].param.ccnt,
1437                         edesc->pset[i].param.src_dst_bidx,
1438                         edesc->pset[i].param.src_dst_cidx,
1439                         edesc->pset[i].param.link_bcntrld);
1440
1441                 edesc->absync = ret;
1442
1443                 /*
1444                  * Enable period interrupt only if it is requested
1445                  */
1446                 if (tx_flags & DMA_PREP_INTERRUPT) {
1447                         edesc->pset[i].param.opt |= TCINTEN;
1448
1449                         /* Also enable intermediate interrupts if necessary */
1450                         if (use_intermediate)
1451                                 edesc->pset[i].param.opt |= ITCINTEN;
1452                 }
1453         }
1454
1455         /* Place the cyclic channel to highest priority queue */
1456         if (!echan->tc)
1457                 edma_assign_channel_eventq(echan, EVENTQ_0);
1458
1459         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1460 }
1461
1462 static void edma_completion_handler(struct edma_chan *echan)
1463 {
1464         struct device *dev = echan->vchan.chan.device->dev;
1465         struct edma_desc *edesc;
1466
1467         spin_lock(&echan->vchan.lock);
1468         edesc = echan->edesc;
1469         if (edesc) {
1470                 if (edesc->cyclic) {
1471                         vchan_cyclic_callback(&edesc->vdesc);
1472                         spin_unlock(&echan->vchan.lock);
1473                         return;
1474                 } else if (edesc->processed == edesc->pset_nr) {
1475                         edesc->residue = 0;
1476                         edma_stop(echan);
1477                         vchan_cookie_complete(&edesc->vdesc);
1478                         echan->edesc = NULL;
1479
1480                         dev_dbg(dev, "Transfer completed on channel %d\n",
1481                                 echan->ch_num);
1482                 } else {
1483                         dev_dbg(dev, "Sub transfer completed on channel %d\n",
1484                                 echan->ch_num);
1485
1486                         edma_pause(echan);
1487
1488                         /* Update statistics for tx_status */
1489                         edesc->residue -= edesc->sg_len;
1490                         edesc->residue_stat = edesc->residue;
1491                         edesc->processed_stat = edesc->processed;
1492                 }
1493                 edma_execute(echan);
1494         }
1495
1496         spin_unlock(&echan->vchan.lock);
1497 }
1498
1499 /* eDMA interrupt handler */
1500 static irqreturn_t dma_irq_handler(int irq, void *data)
1501 {
1502         struct edma_cc *ecc = data;
1503         int ctlr;
1504         u32 sh_ier;
1505         u32 sh_ipr;
1506         u32 bank;
1507
1508         ctlr = ecc->id;
1509         if (ctlr < 0)
1510                 return IRQ_NONE;
1511
1512         dev_vdbg(ecc->dev, "dma_irq_handler\n");
1513
1514         sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1515         if (!sh_ipr) {
1516                 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1517                 if (!sh_ipr)
1518                         return IRQ_NONE;
1519                 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1520                 bank = 1;
1521         } else {
1522                 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1523                 bank = 0;
1524         }
1525
1526         do {
1527                 u32 slot;
1528                 u32 channel;
1529
1530                 slot = __ffs(sh_ipr);
1531                 sh_ipr &= ~(BIT(slot));
1532
1533                 if (sh_ier & BIT(slot)) {
1534                         channel = (bank << 5) | slot;
1535                         /* Clear the corresponding IPR bits */
1536                         edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1537                         edma_completion_handler(&ecc->slave_chans[channel]);
1538                 }
1539         } while (sh_ipr);
1540
1541         edma_shadow0_write(ecc, SH_IEVAL, 1);
1542         return IRQ_HANDLED;
1543 }
1544
1545 static void edma_error_handler(struct edma_chan *echan)
1546 {
1547         struct edma_cc *ecc = echan->ecc;
1548         struct device *dev = echan->vchan.chan.device->dev;
1549         struct edmacc_param p;
1550         int err;
1551
1552         if (!echan->edesc)
1553                 return;
1554
1555         spin_lock(&echan->vchan.lock);
1556
1557         err = edma_read_slot(ecc, echan->slot[0], &p);
1558
1559         /*
1560          * Issue later based on missed flag which will be sure
1561          * to happen as:
1562          * (1) we finished transmitting an intermediate slot and
1563          *     edma_execute is coming up.
1564          * (2) or we finished current transfer and issue will
1565          *     call edma_execute.
1566          *
1567          * Important note: issuing can be dangerous here and
1568          * lead to some nasty recursion when we are in a NULL
1569          * slot. So we avoid doing so and set the missed flag.
1570          */
1571         if (err || (p.a_b_cnt == 0 && p.ccnt == 0)) {
1572                 dev_dbg(dev, "Error on null slot, setting miss\n");
1573                 echan->missed = 1;
1574         } else {
1575                 /*
1576                  * The slot is already programmed but the event got
1577                  * missed, so its safe to issue it here.
1578                  */
1579                 dev_dbg(dev, "Missed event, TRIGGERING\n");
1580                 edma_clean_channel(echan);
1581                 edma_stop(echan);
1582                 edma_start(echan);
1583                 edma_trigger_channel(echan);
1584         }
1585         spin_unlock(&echan->vchan.lock);
1586 }
1587
1588 static inline bool edma_error_pending(struct edma_cc *ecc)
1589 {
1590         if (edma_read_array(ecc, EDMA_EMR, 0) ||
1591             edma_read_array(ecc, EDMA_EMR, 1) ||
1592             edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1593                 return true;
1594
1595         return false;
1596 }
1597
1598 /* eDMA error interrupt handler */
1599 static irqreturn_t dma_ccerr_handler(int irq, void *data)
1600 {
1601         struct edma_cc *ecc = data;
1602         int i, j;
1603         int ctlr;
1604         unsigned int cnt = 0;
1605         unsigned int val;
1606
1607         ctlr = ecc->id;
1608         if (ctlr < 0)
1609                 return IRQ_NONE;
1610
1611         dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1612
1613         if (!edma_error_pending(ecc)) {
1614                 /*
1615                  * The registers indicate no pending error event but the irq
1616                  * handler has been called.
1617                  * Ask eDMA to re-evaluate the error registers.
1618                  */
1619                 dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
1620                         __func__);
1621                 edma_write(ecc, EDMA_EEVAL, 1);
1622                 return IRQ_NONE;
1623         }
1624
1625         while (1) {
1626                 /* Event missed register(s) */
1627                 for (j = 0; j < 2; j++) {
1628                         unsigned long emr;
1629
1630                         val = edma_read_array(ecc, EDMA_EMR, j);
1631                         if (!val)
1632                                 continue;
1633
1634                         dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
1635                         emr = val;
1636                         for_each_set_bit(i, &emr, 32) {
1637                                 int k = (j << 5) + i;
1638
1639                                 /* Clear the corresponding EMR bits */
1640                                 edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
1641                                 /* Clear any SER */
1642                                 edma_shadow0_write_array(ecc, SH_SECR, j,
1643                                                          BIT(i));
1644                                 edma_error_handler(&ecc->slave_chans[k]);
1645                         }
1646                 }
1647
1648                 val = edma_read(ecc, EDMA_QEMR);
1649                 if (val) {
1650                         dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
1651                         /* Not reported, just clear the interrupt reason. */
1652                         edma_write(ecc, EDMA_QEMCR, val);
1653                         edma_shadow0_write(ecc, SH_QSECR, val);
1654                 }
1655
1656                 val = edma_read(ecc, EDMA_CCERR);
1657                 if (val) {
1658                         dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
1659                         /* Not reported, just clear the interrupt reason. */
1660                         edma_write(ecc, EDMA_CCERRCLR, val);
1661                 }
1662
1663                 if (!edma_error_pending(ecc))
1664                         break;
1665                 cnt++;
1666                 if (cnt > 10)
1667                         break;
1668         }
1669         edma_write(ecc, EDMA_EEVAL, 1);
1670         return IRQ_HANDLED;
1671 }
1672
1673 /* Alloc channel resources */
1674 static int edma_alloc_chan_resources(struct dma_chan *chan)
1675 {
1676         struct edma_chan *echan = to_edma_chan(chan);
1677         struct edma_cc *ecc = echan->ecc;
1678         struct device *dev = ecc->dev;
1679         enum dma_event_q eventq_no = EVENTQ_DEFAULT;
1680         int ret;
1681
1682         if (echan->tc) {
1683                 eventq_no = echan->tc->id;
1684         } else if (ecc->tc_list) {
1685                 /* memcpy channel */
1686                 echan->tc = &ecc->tc_list[ecc->info->default_queue];
1687                 eventq_no = echan->tc->id;
1688         }
1689
1690         ret = edma_alloc_channel(echan, eventq_no);
1691         if (ret)
1692                 return ret;
1693
1694         echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
1695         if (echan->slot[0] < 0) {
1696                 dev_err(dev, "Entry slot allocation failed for channel %u\n",
1697                         EDMA_CHAN_SLOT(echan->ch_num));
1698                 ret = echan->slot[0];
1699                 goto err_slot;
1700         }
1701
1702         /* Set up channel -> slot mapping for the entry slot */
1703         edma_set_chmap(echan, echan->slot[0]);
1704         echan->alloced = true;
1705
1706         dev_dbg(dev, "Got eDMA channel %d for virt channel %d (%s trigger)\n",
1707                 EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id,
1708                 echan->hw_triggered ? "HW" : "SW");
1709
1710         return 0;
1711
1712 err_slot:
1713         edma_free_channel(echan);
1714         return ret;
1715 }
1716
1717 /* Free channel resources */
1718 static void edma_free_chan_resources(struct dma_chan *chan)
1719 {
1720         struct edma_chan *echan = to_edma_chan(chan);
1721         struct device *dev = echan->ecc->dev;
1722         int i;
1723
1724         /* Terminate transfers */
1725         edma_stop(echan);
1726
1727         vchan_free_chan_resources(&echan->vchan);
1728
1729         /* Free EDMA PaRAM slots */
1730         for (i = 0; i < EDMA_MAX_SLOTS; i++) {
1731                 if (echan->slot[i] >= 0) {
1732                         edma_free_slot(echan->ecc, echan->slot[i]);
1733                         echan->slot[i] = -1;
1734                 }
1735         }
1736
1737         /* Set entry slot to the dummy slot */
1738         edma_set_chmap(echan, echan->ecc->dummy_slot);
1739
1740         /* Free EDMA channel */
1741         if (echan->alloced) {
1742                 edma_free_channel(echan);
1743                 echan->alloced = false;
1744         }
1745
1746         echan->tc = NULL;
1747         echan->hw_triggered = false;
1748
1749         dev_dbg(dev, "Free eDMA channel %d for virt channel %d\n",
1750                 EDMA_CHAN_SLOT(echan->ch_num), chan->chan_id);
1751 }
1752
1753 /* Send pending descriptor to hardware */
1754 static void edma_issue_pending(struct dma_chan *chan)
1755 {
1756         struct edma_chan *echan = to_edma_chan(chan);
1757         unsigned long flags;
1758
1759         spin_lock_irqsave(&echan->vchan.lock, flags);
1760         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
1761                 edma_execute(echan);
1762         spin_unlock_irqrestore(&echan->vchan.lock, flags);
1763 }
1764
1765 /*
1766  * This limit exists to avoid a possible infinite loop when waiting for proof
1767  * that a particular transfer is completed. This limit can be hit if there
1768  * are large bursts to/from slow devices or the CPU is never able to catch
1769  * the DMA hardware idle. On an AM335x transferring 48 bytes from the UART
1770  * RX-FIFO, as many as 55 loops have been seen.
1771  */
1772 #define EDMA_MAX_TR_WAIT_LOOPS 1000
1773
1774 static u32 edma_residue(struct edma_desc *edesc)
1775 {
1776         bool dst = edesc->direction == DMA_DEV_TO_MEM;
1777         int loop_count = EDMA_MAX_TR_WAIT_LOOPS;
1778         struct edma_chan *echan = edesc->echan;
1779         struct edma_pset *pset = edesc->pset;
1780         dma_addr_t done, pos, pos_old;
1781         int channel = EDMA_CHAN_SLOT(echan->ch_num);
1782         int idx = EDMA_REG_ARRAY_INDEX(channel);
1783         int ch_bit = EDMA_CHANNEL_BIT(channel);
1784         int event_reg;
1785         int i;
1786
1787         /*
1788          * We always read the dst/src position from the first RamPar
1789          * pset. That's the one which is active now.
1790          */
1791         pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1792
1793         /*
1794          * "pos" may represent a transfer request that is still being
1795          * processed by the EDMACC or EDMATC. We will busy wait until
1796          * any one of the situations occurs:
1797          *   1. while and event is pending for the channel
1798          *   2. a position updated
1799          *   3. we hit the loop limit
1800          */
1801         if (is_slave_direction(edesc->direction))
1802                 event_reg = SH_ER;
1803         else
1804                 event_reg = SH_ESR;
1805
1806         pos_old = pos;
1807         while (edma_shadow0_read_array(echan->ecc, event_reg, idx) & ch_bit) {
1808                 pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1809                 if (pos != pos_old)
1810                         break;
1811
1812                 if (!--loop_count) {
1813                         dev_dbg_ratelimited(echan->vchan.chan.device->dev,
1814                                 "%s: timeout waiting for PaRAM update\n",
1815                                 __func__);
1816                         break;
1817                 }
1818
1819                 cpu_relax();
1820         }
1821
1822         /*
1823          * Cyclic is simple. Just subtract pset[0].addr from pos.
1824          *
1825          * We never update edesc->residue in the cyclic case, so we
1826          * can tell the remaining room to the end of the circular
1827          * buffer.
1828          */
1829         if (edesc->cyclic) {
1830                 done = pos - pset->addr;
1831                 edesc->residue_stat = edesc->residue - done;
1832                 return edesc->residue_stat;
1833         }
1834
1835         /*
1836          * If the position is 0, then EDMA loaded the closing dummy slot, the
1837          * transfer is completed
1838          */
1839         if (!pos)
1840                 return 0;
1841         /*
1842          * For SG operation we catch up with the last processed
1843          * status.
1844          */
1845         pset += edesc->processed_stat;
1846
1847         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
1848                 /*
1849                  * If we are inside this pset address range, we know
1850                  * this is the active one. Get the current delta and
1851                  * stop walking the psets.
1852                  */
1853                 if (pos >= pset->addr && pos < pset->addr + pset->len)
1854                         return edesc->residue_stat - (pos - pset->addr);
1855
1856                 /* Otherwise mark it done and update residue_stat. */
1857                 edesc->processed_stat++;
1858                 edesc->residue_stat -= pset->len;
1859         }
1860         return edesc->residue_stat;
1861 }
1862
1863 /* Check request completion status */
1864 static enum dma_status edma_tx_status(struct dma_chan *chan,
1865                                       dma_cookie_t cookie,
1866                                       struct dma_tx_state *txstate)
1867 {
1868         struct edma_chan *echan = to_edma_chan(chan);
1869         struct dma_tx_state txstate_tmp;
1870         enum dma_status ret;
1871         unsigned long flags;
1872
1873         ret = dma_cookie_status(chan, cookie, txstate);
1874
1875         if (ret == DMA_COMPLETE)
1876                 return ret;
1877
1878         /* Provide a dummy dma_tx_state for completion checking */
1879         if (!txstate)
1880                 txstate = &txstate_tmp;
1881
1882         spin_lock_irqsave(&echan->vchan.lock, flags);
1883         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {
1884                 txstate->residue = edma_residue(echan->edesc);
1885         } else {
1886                 struct virt_dma_desc *vdesc = vchan_find_desc(&echan->vchan,
1887                                                               cookie);
1888
1889                 if (vdesc)
1890                         txstate->residue = to_edma_desc(&vdesc->tx)->residue;
1891                 else
1892                         txstate->residue = 0;
1893         }
1894
1895         /*
1896          * Mark the cookie completed if the residue is 0 for non cyclic
1897          * transfers
1898          */
1899         if (ret != DMA_COMPLETE && !txstate->residue &&
1900             echan->edesc && echan->edesc->polled &&
1901             echan->edesc->vdesc.tx.cookie == cookie) {
1902                 edma_stop(echan);
1903                 vchan_cookie_complete(&echan->edesc->vdesc);
1904                 echan->edesc = NULL;
1905                 edma_execute(echan);
1906                 ret = DMA_COMPLETE;
1907         }
1908
1909         spin_unlock_irqrestore(&echan->vchan.lock, flags);
1910
1911         return ret;
1912 }
1913
1914 static bool edma_is_memcpy_channel(int ch_num, s32 *memcpy_channels)
1915 {
1916         if (!memcpy_channels)
1917                 return false;
1918         while (*memcpy_channels != -1) {
1919                 if (*memcpy_channels == ch_num)
1920                         return true;
1921                 memcpy_channels++;
1922         }
1923         return false;
1924 }
1925
1926 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1927                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1928                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
1929                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1930
1931 static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
1932 {
1933         struct dma_device *s_ddev = &ecc->dma_slave;
1934         struct dma_device *m_ddev = NULL;
1935         s32 *memcpy_channels = ecc->info->memcpy_channels;
1936         int i, j;
1937
1938         dma_cap_zero(s_ddev->cap_mask);
1939         dma_cap_set(DMA_SLAVE, s_ddev->cap_mask);
1940         dma_cap_set(DMA_CYCLIC, s_ddev->cap_mask);
1941         if (ecc->legacy_mode && !memcpy_channels) {
1942                 dev_warn(ecc->dev,
1943                          "Legacy memcpy is enabled, things might not work\n");
1944
1945                 dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask);
1946                 dma_cap_set(DMA_INTERLEAVE, s_ddev->cap_mask);
1947                 s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1948                 s_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved;
1949                 s_ddev->directions = BIT(DMA_MEM_TO_MEM);
1950         }
1951
1952         s_ddev->device_prep_slave_sg = edma_prep_slave_sg;
1953         s_ddev->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1954         s_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1955         s_ddev->device_free_chan_resources = edma_free_chan_resources;
1956         s_ddev->device_issue_pending = edma_issue_pending;
1957         s_ddev->device_tx_status = edma_tx_status;
1958         s_ddev->device_config = edma_slave_config;
1959         s_ddev->device_pause = edma_dma_pause;
1960         s_ddev->device_resume = edma_dma_resume;
1961         s_ddev->device_terminate_all = edma_terminate_all;
1962         s_ddev->device_synchronize = edma_synchronize;
1963
1964         s_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1965         s_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1966         s_ddev->directions |= (BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV));
1967         s_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1968         s_ddev->max_burst = SZ_32K - 1; /* CIDX: 16bit signed */
1969
1970         s_ddev->dev = ecc->dev;
1971         INIT_LIST_HEAD(&s_ddev->channels);
1972
1973         if (memcpy_channels) {
1974                 m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
1975                 if (!m_ddev) {
1976                         dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
1977                         memcpy_channels = NULL;
1978                         goto ch_setup;
1979                 }
1980                 ecc->dma_memcpy = m_ddev;
1981
1982                 dma_cap_zero(m_ddev->cap_mask);
1983                 dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask);
1984                 dma_cap_set(DMA_INTERLEAVE, m_ddev->cap_mask);
1985
1986                 m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1987                 m_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved;
1988                 m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources;
1989                 m_ddev->device_free_chan_resources = edma_free_chan_resources;
1990                 m_ddev->device_issue_pending = edma_issue_pending;
1991                 m_ddev->device_tx_status = edma_tx_status;
1992                 m_ddev->device_config = edma_slave_config;
1993                 m_ddev->device_pause = edma_dma_pause;
1994                 m_ddev->device_resume = edma_dma_resume;
1995                 m_ddev->device_terminate_all = edma_terminate_all;
1996                 m_ddev->device_synchronize = edma_synchronize;
1997
1998                 m_ddev->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1999                 m_ddev->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
2000                 m_ddev->directions = BIT(DMA_MEM_TO_MEM);
2001                 m_ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
2002
2003                 m_ddev->dev = ecc->dev;
2004                 INIT_LIST_HEAD(&m_ddev->channels);
2005         } else if (!ecc->legacy_mode) {
2006                 dev_info(ecc->dev, "memcpy is disabled\n");
2007         }
2008
2009 ch_setup:
2010         for (i = 0; i < ecc->num_channels; i++) {
2011                 struct edma_chan *echan = &ecc->slave_chans[i];
2012                 echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
2013                 echan->ecc = ecc;
2014                 echan->vchan.desc_free = edma_desc_free;
2015
2016                 if (m_ddev && edma_is_memcpy_channel(i, memcpy_channels))
2017                         vchan_init(&echan->vchan, m_ddev);
2018                 else
2019                         vchan_init(&echan->vchan, s_ddev);
2020
2021                 INIT_LIST_HEAD(&echan->node);
2022                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
2023                         echan->slot[j] = -1;
2024         }
2025 }
2026
2027 static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
2028                               struct edma_cc *ecc)
2029 {
2030         int i;
2031         u32 value, cccfg;
2032         s8 (*queue_priority_map)[2];
2033
2034         /* Decode the eDMA3 configuration from CCCFG register */
2035         cccfg = edma_read(ecc, EDMA_CCCFG);
2036
2037         value = GET_NUM_REGN(cccfg);
2038         ecc->num_region = BIT(value);
2039
2040         value = GET_NUM_DMACH(cccfg);
2041         ecc->num_channels = BIT(value + 1);
2042
2043         value = GET_NUM_QDMACH(cccfg);
2044         ecc->num_qchannels = value * 2;
2045
2046         value = GET_NUM_PAENTRY(cccfg);
2047         ecc->num_slots = BIT(value + 4);
2048
2049         value = GET_NUM_EVQUE(cccfg);
2050         ecc->num_tc = value + 1;
2051
2052         ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
2053
2054         dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
2055         dev_dbg(dev, "num_region: %u\n", ecc->num_region);
2056         dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
2057         dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
2058         dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
2059         dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
2060         dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
2061
2062         /* Nothing need to be done if queue priority is provided */
2063         if (pdata->queue_priority_mapping)
2064                 return 0;
2065
2066         /*
2067          * Configure TC/queue priority as follows:
2068          * Q0 - priority 0
2069          * Q1 - priority 1
2070          * Q2 - priority 2
2071          * ...
2072          * The meaning of priority numbers: 0 highest priority, 7 lowest
2073          * priority. So Q0 is the highest priority queue and the last queue has
2074          * the lowest priority.
2075          */
2076         queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
2077                                           GFP_KERNEL);
2078         if (!queue_priority_map)
2079                 return -ENOMEM;
2080
2081         for (i = 0; i < ecc->num_tc; i++) {
2082                 queue_priority_map[i][0] = i;
2083                 queue_priority_map[i][1] = i;
2084         }
2085         queue_priority_map[i][0] = -1;
2086         queue_priority_map[i][1] = -1;
2087
2088         pdata->queue_priority_mapping = queue_priority_map;
2089         /* Default queue has the lowest priority */
2090         pdata->default_queue = i - 1;
2091
2092         return 0;
2093 }
2094
2095 #if IS_ENABLED(CONFIG_OF)
2096 static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
2097                                size_t sz)
2098 {
2099         const char pname[] = "ti,edma-xbar-event-map";
2100         struct resource res;
2101         void __iomem *xbar;
2102         s16 (*xbar_chans)[2];
2103         size_t nelm = sz / sizeof(s16);
2104         u32 shift, offset, mux;
2105         int ret, i;
2106
2107         xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
2108         if (!xbar_chans)
2109                 return -ENOMEM;
2110
2111         ret = of_address_to_resource(dev->of_node, 1, &res);
2112         if (ret)
2113                 return -ENOMEM;
2114
2115         xbar = devm_ioremap(dev, res.start, resource_size(&res));
2116         if (!xbar)
2117                 return -ENOMEM;
2118
2119         ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
2120                                          nelm);
2121         if (ret)
2122                 return -EIO;
2123
2124         /* Invalidate last entry for the other user of this mess */
2125         nelm >>= 1;
2126         xbar_chans[nelm][0] = -1;
2127         xbar_chans[nelm][1] = -1;
2128
2129         for (i = 0; i < nelm; i++) {
2130                 shift = (xbar_chans[i][1] & 0x03) << 3;
2131                 offset = xbar_chans[i][1] & 0xfffffffc;
2132                 mux = readl(xbar + offset);
2133                 mux &= ~(0xff << shift);
2134                 mux |= xbar_chans[i][0] << shift;
2135                 writel(mux, (xbar + offset));
2136         }
2137
2138         pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
2139         return 0;
2140 }
2141
2142 static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2143                                                      bool legacy_mode)
2144 {
2145         struct edma_soc_info *info;
2146         struct property *prop;
2147         int sz, ret;
2148
2149         info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
2150         if (!info)
2151                 return ERR_PTR(-ENOMEM);
2152
2153         if (legacy_mode) {
2154                 prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map",
2155                                         &sz);
2156                 if (prop) {
2157                         ret = edma_xbar_event_map(dev, info, sz);
2158                         if (ret)
2159                                 return ERR_PTR(ret);
2160                 }
2161                 return info;
2162         }
2163
2164         /* Get the list of channels allocated to be used for memcpy */
2165         prop = of_find_property(dev->of_node, "ti,edma-memcpy-channels", &sz);
2166         if (prop) {
2167                 const char pname[] = "ti,edma-memcpy-channels";
2168                 size_t nelm = sz / sizeof(s32);
2169                 s32 *memcpy_ch;
2170
2171                 memcpy_ch = devm_kcalloc(dev, nelm + 1, sizeof(s32),
2172                                          GFP_KERNEL);
2173                 if (!memcpy_ch)
2174                         return ERR_PTR(-ENOMEM);
2175
2176                 ret = of_property_read_u32_array(dev->of_node, pname,
2177                                                  (u32 *)memcpy_ch, nelm);
2178                 if (ret)
2179                         return ERR_PTR(ret);
2180
2181                 memcpy_ch[nelm] = -1;
2182                 info->memcpy_channels = memcpy_ch;
2183         }
2184
2185         prop = of_find_property(dev->of_node, "ti,edma-reserved-slot-ranges",
2186                                 &sz);
2187         if (prop) {
2188                 const char pname[] = "ti,edma-reserved-slot-ranges";
2189                 u32 (*tmp)[2];
2190                 s16 (*rsv_slots)[2];
2191                 size_t nelm = sz / sizeof(*tmp);
2192                 struct edma_rsv_info *rsv_info;
2193                 int i;
2194
2195                 if (!nelm)
2196                         return info;
2197
2198                 tmp = kcalloc(nelm, sizeof(*tmp), GFP_KERNEL);
2199                 if (!tmp)
2200                         return ERR_PTR(-ENOMEM);
2201
2202                 rsv_info = devm_kzalloc(dev, sizeof(*rsv_info), GFP_KERNEL);
2203                 if (!rsv_info) {
2204                         kfree(tmp);
2205                         return ERR_PTR(-ENOMEM);
2206                 }
2207
2208                 rsv_slots = devm_kcalloc(dev, nelm + 1, sizeof(*rsv_slots),
2209                                          GFP_KERNEL);
2210                 if (!rsv_slots) {
2211                         kfree(tmp);
2212                         return ERR_PTR(-ENOMEM);
2213                 }
2214
2215                 ret = of_property_read_u32_array(dev->of_node, pname,
2216                                                  (u32 *)tmp, nelm * 2);
2217                 if (ret) {
2218                         kfree(tmp);
2219                         return ERR_PTR(ret);
2220                 }
2221
2222                 for (i = 0; i < nelm; i++) {
2223                         rsv_slots[i][0] = tmp[i][0];
2224                         rsv_slots[i][1] = tmp[i][1];
2225                 }
2226                 rsv_slots[nelm][0] = -1;
2227                 rsv_slots[nelm][1] = -1;
2228
2229                 info->rsv = rsv_info;
2230                 info->rsv->rsv_slots = (const s16 (*)[2])rsv_slots;
2231
2232                 kfree(tmp);
2233         }
2234
2235         return info;
2236 }
2237
2238 static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2239                                       struct of_dma *ofdma)
2240 {
2241         struct edma_cc *ecc = ofdma->of_dma_data;
2242         struct dma_chan *chan = NULL;
2243         struct edma_chan *echan;
2244         int i;
2245
2246         if (!ecc || dma_spec->args_count < 1)
2247                 return NULL;
2248
2249         for (i = 0; i < ecc->num_channels; i++) {
2250                 echan = &ecc->slave_chans[i];
2251                 if (echan->ch_num == dma_spec->args[0]) {
2252                         chan = &echan->vchan.chan;
2253                         break;
2254                 }
2255         }
2256
2257         if (!chan)
2258                 return NULL;
2259
2260         if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
2261                 goto out;
2262
2263         if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
2264             dma_spec->args[1] < echan->ecc->num_tc) {
2265                 echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
2266                 goto out;
2267         }
2268
2269         return NULL;
2270 out:
2271         /* The channel is going to be used as HW synchronized */
2272         echan->hw_triggered = true;
2273         return dma_get_slave_channel(chan);
2274 }
2275 #else
2276 static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev,
2277                                                      bool legacy_mode)
2278 {
2279         return ERR_PTR(-EINVAL);
2280 }
2281
2282 static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec,
2283                                       struct of_dma *ofdma)
2284 {
2285         return NULL;
2286 }
2287 #endif
2288
2289 static bool edma_filter_fn(struct dma_chan *chan, void *param);
2290
2291 static int edma_probe(struct platform_device *pdev)
2292 {
2293         struct edma_soc_info    *info = pdev->dev.platform_data;
2294         s8                      (*queue_priority_mapping)[2];
2295         const s16               (*reserved)[2];
2296         int                     i, irq;
2297         char                    *irq_name;
2298         struct resource         *mem;
2299         struct device_node      *node = pdev->dev.of_node;
2300         struct device           *dev = &pdev->dev;
2301         struct edma_cc          *ecc;
2302         bool                    legacy_mode = true;
2303         int ret;
2304
2305         if (node) {
2306                 const struct of_device_id *match;
2307
2308                 match = of_match_node(edma_of_ids, node);
2309                 if (match && (*(u32 *)match->data) == EDMA_BINDING_TPCC)
2310                         legacy_mode = false;
2311
2312                 info = edma_setup_info_from_dt(dev, legacy_mode);
2313                 if (IS_ERR(info)) {
2314                         dev_err(dev, "failed to get DT data\n");
2315                         return PTR_ERR(info);
2316                 }
2317         }
2318
2319         if (!info)
2320                 return -ENODEV;
2321
2322         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
2323         if (ret)
2324                 return ret;
2325
2326         ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
2327         if (!ecc)
2328                 return -ENOMEM;
2329
2330         ecc->dev = dev;
2331         ecc->id = pdev->id;
2332         ecc->legacy_mode = legacy_mode;
2333         /* When booting with DT the pdev->id is -1 */
2334         if (ecc->id < 0)
2335                 ecc->id = 0;
2336
2337         mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
2338         if (!mem) {
2339                 dev_dbg(dev, "mem resource not found, using index 0\n");
2340                 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2341                 if (!mem) {
2342                         dev_err(dev, "no mem resource?\n");
2343                         return -ENODEV;
2344                 }
2345         }
2346         ecc->base = devm_ioremap_resource(dev, mem);
2347         if (IS_ERR(ecc->base))
2348                 return PTR_ERR(ecc->base);
2349
2350         platform_set_drvdata(pdev, ecc);
2351
2352         pm_runtime_enable(dev);
2353         ret = pm_runtime_get_sync(dev);
2354         if (ret < 0) {
2355                 dev_err(dev, "pm_runtime_get_sync() failed\n");
2356                 pm_runtime_disable(dev);
2357                 return ret;
2358         }
2359
2360         /* Get eDMA3 configuration from IP */
2361         ret = edma_setup_from_hw(dev, info, ecc);
2362         if (ret)
2363                 goto err_disable_pm;
2364
2365         /* Allocate memory based on the information we got from the IP */
2366         ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2367                                         sizeof(*ecc->slave_chans), GFP_KERNEL);
2368
2369         ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
2370                                        sizeof(unsigned long), GFP_KERNEL);
2371
2372         ecc->channels_mask = devm_kcalloc(dev,
2373                                            BITS_TO_LONGS(ecc->num_channels),
2374                                            sizeof(unsigned long), GFP_KERNEL);
2375         if (!ecc->slave_chans || !ecc->slot_inuse || !ecc->channels_mask) {
2376                 ret = -ENOMEM;
2377                 goto err_disable_pm;
2378         }
2379
2380         /* Mark all channels available initially */
2381         bitmap_fill(ecc->channels_mask, ecc->num_channels);
2382
2383         ecc->default_queue = info->default_queue;
2384
2385         if (info->rsv) {
2386                 /* Set the reserved slots in inuse list */
2387                 reserved = info->rsv->rsv_slots;
2388                 if (reserved) {
2389                         for (i = 0; reserved[i][0] != -1; i++)
2390                                 bitmap_set(ecc->slot_inuse, reserved[i][0],
2391                                            reserved[i][1]);
2392                 }
2393
2394                 /* Clear channels not usable for Linux */
2395                 reserved = info->rsv->rsv_chans;
2396                 if (reserved) {
2397                         for (i = 0; reserved[i][0] != -1; i++)
2398                                 bitmap_clear(ecc->channels_mask, reserved[i][0],
2399                                              reserved[i][1]);
2400                 }
2401         }
2402
2403         for (i = 0; i < ecc->num_slots; i++) {
2404                 /* Reset only unused - not reserved - paRAM slots */
2405                 if (!test_bit(i, ecc->slot_inuse))
2406                         edma_write_slot(ecc, i, &dummy_paramset);
2407         }
2408
2409         irq = platform_get_irq_byname(pdev, "edma3_ccint");
2410         if (irq < 0 && node)
2411                 irq = irq_of_parse_and_map(node, 0);
2412
2413         if (irq >= 0) {
2414                 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
2415                                           dev_name(dev));
2416                 ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
2417                                        ecc);
2418                 if (ret) {
2419                         dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
2420                         goto err_disable_pm;
2421                 }
2422                 ecc->ccint = irq;
2423         }
2424
2425         irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
2426         if (irq < 0 && node)
2427                 irq = irq_of_parse_and_map(node, 2);
2428
2429         if (irq >= 0) {
2430                 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
2431                                           dev_name(dev));
2432                 ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
2433                                        ecc);
2434                 if (ret) {
2435                         dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
2436                         goto err_disable_pm;
2437                 }
2438                 ecc->ccerrint = irq;
2439         }
2440
2441         ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
2442         if (ecc->dummy_slot < 0) {
2443                 dev_err(dev, "Can't allocate PaRAM dummy slot\n");
2444                 ret = ecc->dummy_slot;
2445                 goto err_disable_pm;
2446         }
2447
2448         queue_priority_mapping = info->queue_priority_mapping;
2449
2450         if (!ecc->legacy_mode) {
2451                 int lowest_priority = 0;
2452                 unsigned int array_max;
2453                 struct of_phandle_args tc_args;
2454
2455                 ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
2456                                             sizeof(*ecc->tc_list), GFP_KERNEL);
2457                 if (!ecc->tc_list) {
2458                         ret = -ENOMEM;
2459                         goto err_reg1;
2460                 }
2461
2462                 for (i = 0;; i++) {
2463                         ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs",
2464                                                                1, i, &tc_args);
2465                         if (ret || i == ecc->num_tc)
2466                                 break;
2467
2468                         ecc->tc_list[i].node = tc_args.np;
2469                         ecc->tc_list[i].id = i;
2470                         queue_priority_mapping[i][1] = tc_args.args[0];
2471                         if (queue_priority_mapping[i][1] > lowest_priority) {
2472                                 lowest_priority = queue_priority_mapping[i][1];
2473                                 info->default_queue = i;
2474                         }
2475                 }
2476
2477                 /* See if we have optional dma-channel-mask array */
2478                 array_max = DIV_ROUND_UP(ecc->num_channels, BITS_PER_TYPE(u32));
2479                 ret = of_property_read_variable_u32_array(node,
2480                                                 "dma-channel-mask",
2481                                                 (u32 *)ecc->channels_mask,
2482                                                 1, array_max);
2483                 if (ret > 0 && ret != array_max)
2484                         dev_warn(dev, "dma-channel-mask is not complete.\n");
2485                 else if (ret == -EOVERFLOW || ret == -ENODATA)
2486                         dev_warn(dev,
2487                                  "dma-channel-mask is out of range or empty\n");
2488         }
2489
2490         /* Event queue priority mapping */
2491         for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2492                 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2493                                               queue_priority_mapping[i][1]);
2494
2495         edma_write_array2(ecc, EDMA_DRAE, 0, 0, 0x0);
2496         edma_write_array2(ecc, EDMA_DRAE, 0, 1, 0x0);
2497         edma_write_array(ecc, EDMA_QRAE, 0, 0x0);
2498
2499         ecc->info = info;
2500
2501         /* Init the dma device and channels */
2502         edma_dma_init(ecc, legacy_mode);
2503
2504         for (i = 0; i < ecc->num_channels; i++) {
2505                 /* Do not touch reserved channels */
2506                 if (!test_bit(i, ecc->channels_mask))
2507                         continue;
2508
2509                 /* Assign all channels to the default queue */
2510                 edma_assign_channel_eventq(&ecc->slave_chans[i],
2511                                            info->default_queue);
2512                 /* Set entry slot to the dummy slot */
2513                 edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
2514         }
2515
2516         ecc->dma_slave.filter.map = info->slave_map;
2517         ecc->dma_slave.filter.mapcnt = info->slavecnt;
2518         ecc->dma_slave.filter.fn = edma_filter_fn;
2519
2520         ret = dma_async_device_register(&ecc->dma_slave);
2521         if (ret) {
2522                 dev_err(dev, "slave ddev registration failed (%d)\n", ret);
2523                 goto err_reg1;
2524         }
2525
2526         if (ecc->dma_memcpy) {
2527                 ret = dma_async_device_register(ecc->dma_memcpy);
2528                 if (ret) {
2529                         dev_err(dev, "memcpy ddev registration failed (%d)\n",
2530                                 ret);
2531                         dma_async_device_unregister(&ecc->dma_slave);
2532                         goto err_reg1;
2533                 }
2534         }
2535
2536         if (node)
2537                 of_dma_controller_register(node, of_edma_xlate, ecc);
2538
2539         dev_info(dev, "TI EDMA DMA engine driver\n");
2540
2541         return 0;
2542
2543 err_reg1:
2544         edma_free_slot(ecc, ecc->dummy_slot);
2545 err_disable_pm:
2546         pm_runtime_put_sync(dev);
2547         pm_runtime_disable(dev);
2548         return ret;
2549 }
2550
2551 static void edma_cleanupp_vchan(struct dma_device *dmadev)
2552 {
2553         struct edma_chan *echan, *_echan;
2554
2555         list_for_each_entry_safe(echan, _echan,
2556                         &dmadev->channels, vchan.chan.device_node) {
2557                 list_del(&echan->vchan.chan.device_node);
2558                 tasklet_kill(&echan->vchan.task);
2559         }
2560 }
2561
2562 static int edma_remove(struct platform_device *pdev)
2563 {
2564         struct device *dev = &pdev->dev;
2565         struct edma_cc *ecc = dev_get_drvdata(dev);
2566
2567         devm_free_irq(dev, ecc->ccint, ecc);
2568         devm_free_irq(dev, ecc->ccerrint, ecc);
2569
2570         edma_cleanupp_vchan(&ecc->dma_slave);
2571
2572         if (dev->of_node)
2573                 of_dma_controller_free(dev->of_node);
2574         dma_async_device_unregister(&ecc->dma_slave);
2575         if (ecc->dma_memcpy)
2576                 dma_async_device_unregister(ecc->dma_memcpy);
2577         edma_free_slot(ecc, ecc->dummy_slot);
2578         pm_runtime_put_sync(dev);
2579         pm_runtime_disable(dev);
2580
2581         return 0;
2582 }
2583
2584 #ifdef CONFIG_PM_SLEEP
2585 static int edma_pm_suspend(struct device *dev)
2586 {
2587         struct edma_cc *ecc = dev_get_drvdata(dev);
2588         struct edma_chan *echan = ecc->slave_chans;
2589         int i;
2590
2591         for (i = 0; i < ecc->num_channels; i++) {
2592                 if (echan[i].alloced)
2593                         edma_setup_interrupt(&echan[i], false);
2594         }
2595
2596         return 0;
2597 }
2598
2599 static int edma_pm_resume(struct device *dev)
2600 {
2601         struct edma_cc *ecc = dev_get_drvdata(dev);
2602         struct edma_chan *echan = ecc->slave_chans;
2603         int i;
2604         s8 (*queue_priority_mapping)[2];
2605
2606         /* re initialize dummy slot to dummy param set */
2607         edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
2608
2609         queue_priority_mapping = ecc->info->queue_priority_mapping;
2610
2611         /* Event queue priority mapping */
2612         for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2613                 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2614                                               queue_priority_mapping[i][1]);
2615
2616         for (i = 0; i < ecc->num_channels; i++) {
2617                 if (echan[i].alloced) {
2618                         /* ensure access through shadow region 0 */
2619                         edma_or_array2(ecc, EDMA_DRAE, 0,
2620                                        EDMA_REG_ARRAY_INDEX(i),
2621                                        EDMA_CHANNEL_BIT(i));
2622
2623                         edma_setup_interrupt(&echan[i], true);
2624
2625                         /* Set up channel -> slot mapping for the entry slot */
2626                         edma_set_chmap(&echan[i], echan[i].slot[0]);
2627                 }
2628         }
2629
2630         return 0;
2631 }
2632 #endif
2633
2634 static const struct dev_pm_ops edma_pm_ops = {
2635         SET_LATE_SYSTEM_SLEEP_PM_OPS(edma_pm_suspend, edma_pm_resume)
2636 };
2637
2638 static struct platform_driver edma_driver = {
2639         .probe          = edma_probe,
2640         .remove         = edma_remove,
2641         .driver = {
2642                 .name   = "edma",
2643                 .pm     = &edma_pm_ops,
2644                 .of_match_table = edma_of_ids,
2645         },
2646 };
2647
2648 static int edma_tptc_probe(struct platform_device *pdev)
2649 {
2650         pm_runtime_enable(&pdev->dev);
2651         return pm_runtime_get_sync(&pdev->dev);
2652 }
2653
2654 static struct platform_driver edma_tptc_driver = {
2655         .probe          = edma_tptc_probe,
2656         .driver = {
2657                 .name   = "edma3-tptc",
2658                 .of_match_table = edma_tptc_of_ids,
2659         },
2660 };
2661
2662 static bool edma_filter_fn(struct dma_chan *chan, void *param)
2663 {
2664         bool match = false;
2665
2666         if (chan->device->dev->driver == &edma_driver.driver) {
2667                 struct edma_chan *echan = to_edma_chan(chan);
2668                 unsigned ch_req = *(unsigned *)param;
2669                 if (ch_req == echan->ch_num) {
2670                         /* The channel is going to be used as HW synchronized */
2671                         echan->hw_triggered = true;
2672                         match = true;
2673                 }
2674         }
2675         return match;
2676 }
2677
2678 static int edma_init(void)
2679 {
2680         int ret;
2681
2682         ret = platform_driver_register(&edma_tptc_driver);
2683         if (ret)
2684                 return ret;
2685
2686         return platform_driver_register(&edma_driver);
2687 }
2688 subsys_initcall(edma_init);
2689
2690 static void __exit edma_exit(void)
2691 {
2692         platform_driver_unregister(&edma_driver);
2693         platform_driver_unregister(&edma_tptc_driver);
2694 }
2695 module_exit(edma_exit);
2696
2697 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
2698 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
2699 MODULE_LICENSE("GPL v2");