drivers/dma/dw-edma/dw-edma-v0-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
   4  * Synopsys DesignWare eDMA v0 core
   5  *
   6  * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
   7  */
   8
   9 #include <linux/bitfield.h>
  10 #include <linux/irqreturn.h>
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 #include "dw-edma-core.h"
  14 #include "dw-edma-v0-core.h"
  15 #include "dw-edma-v0-regs.h"
  16 #include "dw-edma-v0-debugfs.h"
  17
  18 enum dw_edma_control {
  19         DW_EDMA_V0_CB                                   = BIT(0),
  20         DW_EDMA_V0_TCB                                  = BIT(1),
  21         DW_EDMA_V0_LLP                                  = BIT(2),
  22         DW_EDMA_V0_LIE                                  = BIT(3),
  23         DW_EDMA_V0_RIE                                  = BIT(4),
  24         DW_EDMA_V0_CCS                                  = BIT(8),
  25         DW_EDMA_V0_LLE                                  = BIT(9),
  26 };
  27
  28 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
  29 {
  30         return dw->chip->reg_base;
  31 }
  32
  33 #define SET_32(dw, name, value)                         \
  34         writel(value, &(__dw_regs(dw)->name))
  35
  36 #define GET_32(dw, name)                                \
  37         readl(&(__dw_regs(dw)->name))
  38
  39 #define SET_RW_32(dw, dir, name, value)                 \
  40         do {                                            \
  41                 if ((dir) == EDMA_DIR_WRITE)            \
  42                         SET_32(dw, wr_##name, value);   \
  43                 else                                    \
  44                         SET_32(dw, rd_##name, value);   \
  45         } while (0)
  46
  47 #define GET_RW_32(dw, dir, name)                        \
  48         ((dir) == EDMA_DIR_WRITE                        \
  49           ? GET_32(dw, wr_##name)                       \
  50           : GET_32(dw, rd_##name))
  51
  52 #define SET_BOTH_32(dw, name, value)                    \
  53         do {                                            \
  54                 SET_32(dw, wr_##name, value);           \
  55                 SET_32(dw, rd_##name, value);           \
  56         } while (0)
  57
  58 #define SET_64(dw, name, value)                         \
  59         writeq(value, &(__dw_regs(dw)->name))
  60
  61 #define GET_64(dw, name)                                \
  62         readq(&(__dw_regs(dw)->name))
  63
  64 #define SET_RW_64(dw, dir, name, value)                 \
  65         do {                                            \
  66                 if ((dir) == EDMA_DIR_WRITE)            \
  67                         SET_64(dw, wr_##name, value);   \
  68                 else                                    \
  69                         SET_64(dw, rd_##name, value);   \
  70         } while (0)
  71
  72 #define GET_RW_64(dw, dir, name)                        \
  73         ((dir) == EDMA_DIR_WRITE                        \
  74           ? GET_64(dw, wr_##name)                       \
  75           : GET_64(dw, rd_##name))
  76
  77 #define SET_BOTH_64(dw, name, value)                    \
  78         do {                                            \
  79                 SET_64(dw, wr_##name, value);           \
  80                 SET_64(dw, rd_##name, value);           \
  81         } while (0)
  82
  83 #define SET_COMPAT(dw, name, value)                     \
  84         writel(value, &(__dw_regs(dw)->type.unroll.name))
  85
  86 #define SET_RW_COMPAT(dw, dir, name, value)             \
  87         do {                                            \
  88                 if ((dir) == EDMA_DIR_WRITE)            \
  89                         SET_COMPAT(dw, wr_##name, value); \
  90                 else                                    \
  91                         SET_COMPAT(dw, rd_##name, value); \
  92         } while (0)
  93
  94 static inline struct dw_edma_v0_ch_regs __iomem *
  95 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
  96 {
  97         if (dw->chip->mf == EDMA_MF_EDMA_LEGACY)
  98                 return &(__dw_regs(dw)->type.legacy.ch);
  99
 100         if (dir == EDMA_DIR_WRITE)
 101                 return &__dw_regs(dw)->type.unroll.ch[ch].wr;
 102
 103         return &__dw_regs(dw)->type.unroll.ch[ch].rd;
 104 }
 105
 106 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
 107                              u32 value, void __iomem *addr)
 108 {
 109         if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
 110                 u32 viewport_sel;
 111                 unsigned long flags;
 112
 113                 raw_spin_lock_irqsave(&dw->lock, flags);
 114
 115                 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
 116                 if (dir == EDMA_DIR_READ)
 117                         viewport_sel |= BIT(31);
 118
 119                 writel(viewport_sel,
 120                        &(__dw_regs(dw)->type.legacy.viewport_sel));
 121                 writel(value, addr);
 122
 123                 raw_spin_unlock_irqrestore(&dw->lock, flags);
 124         } else {
 125                 writel(value, addr);
 126         }
 127 }
 128
 129 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
 130                            const void __iomem *addr)
 131 {
 132         u32 value;
 133
 134         if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
 135                 u32 viewport_sel;
 136                 unsigned long flags;
 137
 138                 raw_spin_lock_irqsave(&dw->lock, flags);
 139
 140                 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
 141                 if (dir == EDMA_DIR_READ)
 142                         viewport_sel |= BIT(31);
 143
 144                 writel(viewport_sel,
 145                        &(__dw_regs(dw)->type.legacy.viewport_sel));
 146                 value = readl(addr);
 147
 148                 raw_spin_unlock_irqrestore(&dw->lock, flags);
 149         } else {
 150                 value = readl(addr);
 151         }
 152
 153         return value;
 154 }
 155
 156 #define SET_CH_32(dw, dir, ch, name, value) \
 157         writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
 158
 159 #define GET_CH_32(dw, dir, ch, name) \
 160         readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
 161
 162 /* eDMA management callbacks */
 163 static void dw_edma_v0_core_off(struct dw_edma *dw)
 164 {
 165         SET_BOTH_32(dw, int_mask,
 166                     EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
 167         SET_BOTH_32(dw, int_clear,
 168                     EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
 169         SET_BOTH_32(dw, engine_en, 0);
 170 }
 171
 172 static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
 173 {
 174         u32 num_ch;
 175
 176         if (dir == EDMA_DIR_WRITE)
 177                 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,
 178                                    GET_32(dw, ctrl));
 179         else
 180                 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,
 181                                    GET_32(dw, ctrl));
 182
 183         if (num_ch > EDMA_V0_MAX_NR_CH)
 184                 num_ch = EDMA_V0_MAX_NR_CH;
 185
 186         return (u16)num_ch;
 187 }
 188
 189 static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
 190 {
 191         struct dw_edma *dw = chan->dw;
 192         u32 tmp;
 193
 194         tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
 195                         GET_CH_32(dw, chan->dir, chan->id, ch_control1));
 196
 197         if (tmp == 1)
 198                 return DMA_IN_PROGRESS;
 199         else if (tmp == 3)
 200                 return DMA_COMPLETE;
 201         else
 202                 return DMA_ERROR;
 203 }
 204
 205 static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
 206 {
 207         struct dw_edma *dw = chan->dw;
 208
 209         SET_RW_32(dw, chan->dir, int_clear,
 210                   FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
 211 }
 212
 213 static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
 214 {
 215         struct dw_edma *dw = chan->dw;
 216
 217         SET_RW_32(dw, chan->dir, int_clear,
 218                   FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
 219 }
 220
 221 static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
 222 {
 223         return FIELD_GET(EDMA_V0_DONE_INT_MASK,
 224                          GET_RW_32(dw, dir, int_status));
 225 }
 226
 227 static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
 228 {
 229         return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
 230                          GET_RW_32(dw, dir, int_status));
 231 }
 232
 233 static irqreturn_t
 234 dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 235                            dw_edma_handler_t done, dw_edma_handler_t abort)
 236 {
 237         struct dw_edma *dw = dw_irq->dw;
 238         unsigned long total, pos, val;
 239         irqreturn_t ret = IRQ_NONE;
 240         struct dw_edma_chan *chan;
 241         unsigned long off;
 242         u32 mask;
 243
 244         if (dir == EDMA_DIR_WRITE) {
 245                 total = dw->wr_ch_cnt;
 246                 off = 0;
 247                 mask = dw_irq->wr_mask;
 248         } else {
 249                 total = dw->rd_ch_cnt;
 250                 off = dw->wr_ch_cnt;
 251                 mask = dw_irq->rd_mask;
 252         }
 253
 254         val = dw_edma_v0_core_status_done_int(dw, dir);
 255         val &= mask;
 256         for_each_set_bit(pos, &val, total) {
 257                 chan = &dw->chan[pos + off];
 258
 259                 dw_edma_v0_core_clear_done_int(chan);
 260                 done(chan);
 261
 262                 ret = IRQ_HANDLED;
 263         }
 264
 265         val = dw_edma_v0_core_status_abort_int(dw, dir);
 266         val &= mask;
 267         for_each_set_bit(pos, &val, total) {
 268                 chan = &dw->chan[pos + off];
 269
 270                 dw_edma_v0_core_clear_abort_int(chan);
 271                 abort(chan);
 272
 273                 ret = IRQ_HANDLED;
 274         }
 275
 276         return ret;
 277 }
 278
 279 static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
 280                                      u32 control, u32 size, u64 sar, u64 dar)
 281 {
 282         ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
 283
 284         if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 285                 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
 286
 287                 lli->control = control;
 288                 lli->transfer_size = size;
 289                 lli->sar.reg = sar;
 290                 lli->dar.reg = dar;
 291         } else {
 292                 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
 293
 294                 writel(control, &lli->control);
 295                 writel(size, &lli->transfer_size);
 296                 writeq(sar, &lli->sar.reg);
 297                 writeq(dar, &lli->dar.reg);
 298         }
 299 }
 300
 301 static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
 302                                      int i, u32 control, u64 pointer)
 303 {
 304         ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
 305
 306         if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 307                 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
 308
 309                 llp->control = control;
 310                 llp->llp.reg = pointer;
 311         } else {
 312                 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
 313
 314                 writel(control, &llp->control);
 315                 writeq(pointer, &llp->llp.reg);
 316         }
 317 }
 318
 319 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
 320 {
 321         struct dw_edma_burst *child;
 322         struct dw_edma_chan *chan = chunk->chan;
 323         u32 control = 0, i = 0;
 324         int j;
 325
 326         if (chunk->cb)
 327                 control = DW_EDMA_V0_CB;
 328
 329         j = chunk->bursts_alloc;
 330         list_for_each_entry(child, &chunk->burst->list, list) {
 331                 j--;
 332                 if (!j) {
 333                         control |= DW_EDMA_V0_LIE;
 334                         if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 335                                 control |= DW_EDMA_V0_RIE;
 336                 }
 337
 338                 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
 339                                          child->sar, child->dar);
 340         }
 341
 342         control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
 343         if (!chunk->cb)
 344                 control |= DW_EDMA_V0_CB;
 345
 346         dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
 347 }
 348
 349 static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 350 {
 351         /*
 352          * In case of remote eDMA engine setup, the DW PCIe RP/EP internal
 353          * configuration registers and application memory are normally accessed
 354          * over different buses. Ensure LL-data reaches the memory before the
 355          * doorbell register is toggled by issuing the dummy-read from the remote
 356          * LL memory in a hope that the MRd TLP will return only after the
 357          * last MWr TLP is completed
 358          */
 359         if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 360                 readl(chunk->ll_region.vaddr.io);
 361 }
 362
 363 static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
 364 {
 365         struct dw_edma_chan *chan = chunk->chan;
 366         struct dw_edma *dw = chan->dw;
 367         u32 tmp;
 368
 369         dw_edma_v0_core_write_chunk(chunk);
 370
 371         if (first) {
 372                 /* Enable engine */
 373                 SET_RW_32(dw, chan->dir, engine_en, BIT(0));
 374                 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
 375                         switch (chan->id) {
 376                         case 0:
 377                                 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
 378                                               BIT(0));
 379                                 break;
 380                         case 1:
 381                                 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
 382                                               BIT(0));
 383                                 break;
 384                         case 2:
 385                                 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
 386                                               BIT(0));
 387                                 break;
 388                         case 3:
 389                                 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
 390                                               BIT(0));
 391                                 break;
 392                         case 4:
 393                                 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
 394                                               BIT(0));
 395                                 break;
 396                         case 5:
 397                                 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
 398                                               BIT(0));
 399                                 break;
 400                         case 6:
 401                                 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
 402                                               BIT(0));
 403                                 break;
 404                         case 7:
 405                                 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
 406                                               BIT(0));
 407                                 break;
 408                         }
 409                 }
 410                 /* Interrupt unmask - done, abort */
 411                 tmp = GET_RW_32(dw, chan->dir, int_mask);
 412                 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
 413                 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
 414                 SET_RW_32(dw, chan->dir, int_mask, tmp);
 415                 /* Linked list error */
 416                 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
 417                 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
 418                 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
 419                 /* Channel control */
 420                 SET_CH_32(dw, chan->dir, chan->id, ch_control1,
 421                           (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
 422                 /* Linked list */
 423                 /* llp is not aligned on 64bit -> keep 32bit accesses */
 424                 SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
 425                           lower_32_bits(chunk->ll_region.paddr));
 426                 SET_CH_32(dw, chan->dir, chan->id, llp.msb,
 427                           upper_32_bits(chunk->ll_region.paddr));
 428         }
 429
 430         dw_edma_v0_sync_ll_data(chunk);
 431
 432         /* Doorbell */
 433         SET_RW_32(dw, chan->dir, doorbell,
 434                   FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
 435 }
 436
 437 static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
 438 {
 439         struct dw_edma *dw = chan->dw;
 440         u32 tmp = 0;
 441
 442         /* MSI done addr - low, high */
 443         SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);
 444         SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);
 445         /* MSI abort addr - low, high */
 446         SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);
 447         SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);
 448         /* MSI data - low, high */
 449         switch (chan->id) {
 450         case 0:
 451         case 1:
 452                 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);
 453                 break;
 454
 455         case 2:
 456         case 3:
 457                 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);
 458                 break;
 459
 460         case 4:
 461         case 5:
 462                 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);
 463                 break;
 464
 465         case 6:
 466         case 7:
 467                 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);
 468                 break;
 469         }
 470
 471         if (chan->id & BIT(0)) {
 472                 /* Channel odd {1, 3, 5, 7} */
 473                 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;
 474                 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,
 475                                   chan->msi.data);
 476         } else {
 477                 /* Channel even {0, 2, 4, 6} */
 478                 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;
 479                 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,
 480                                   chan->msi.data);
 481         }
 482
 483         switch (chan->id) {
 484         case 0:
 485         case 1:
 486                 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);
 487                 break;
 488
 489         case 2:
 490         case 3:
 491                 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);
 492                 break;
 493
 494         case 4:
 495         case 5:
 496                 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);
 497                 break;
 498
 499         case 6:
 500         case 7:
 501                 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);
 502                 break;
 503         }
 504 }
 505
 506 /* eDMA debugfs callbacks */
 507 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 508 {
 509         dw_edma_v0_debugfs_on(dw);
 510 }
 511
 512 static const struct dw_edma_core_ops dw_edma_v0_core = {
 513         .off = dw_edma_v0_core_off,
 514         .ch_count = dw_edma_v0_core_ch_count,
 515         .ch_status = dw_edma_v0_core_ch_status,
 516         .handle_int = dw_edma_v0_core_handle_int,
 517         .start = dw_edma_v0_core_start,
 518         .ch_config = dw_edma_v0_core_ch_config,
 519         .debugfs_on = dw_edma_v0_core_debugfs_on,
 520 };
 521
 522 void dw_edma_v0_core_register(struct dw_edma *dw)
 523 {
 524         dw->core = &dw_edma_v0_core;
 525 }