drivers/dma/dw-edma/dw-hdma-v0-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2023 Cai Huoqing
   4  * Synopsys DesignWare HDMA v0 core
   5  */
   6
   7 #include <linux/bitfield.h>
   8 #include <linux/irqreturn.h>
   9 #include <linux/io-64-nonatomic-lo-hi.h>
  10
  11 #include "dw-edma-core.h"
  12 #include "dw-hdma-v0-core.h"
  13 #include "dw-hdma-v0-regs.h"
  14 #include "dw-hdma-v0-debugfs.h"
  15
  16 enum dw_hdma_control {
  17         DW_HDMA_V0_CB                                   = BIT(0),
  18         DW_HDMA_V0_TCB                                  = BIT(1),
  19         DW_HDMA_V0_LLP                                  = BIT(2),
  20         DW_HDMA_V0_LIE                                  = BIT(3),
  21         DW_HDMA_V0_RIE                                  = BIT(4),
  22         DW_HDMA_V0_CCS                                  = BIT(8),
  23         DW_HDMA_V0_LLE                                  = BIT(9),
  24 };
  25
  26 static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
  27 {
  28         return dw->chip->reg_base;
  29 }
  30
  31 static inline struct dw_hdma_v0_ch_regs __iomem *
  32 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
  33 {
  34         if (dir == EDMA_DIR_WRITE)
  35                 return &(__dw_regs(dw)->ch[ch].wr);
  36         else
  37                 return &(__dw_regs(dw)->ch[ch].rd);
  38 }
  39
  40 #define SET_CH_32(dw, dir, ch, name, value) \
  41         writel(value, &(__dw_ch_regs(dw, dir, ch)->name))
  42
  43 #define GET_CH_32(dw, dir, ch, name) \
  44         readl(&(__dw_ch_regs(dw, dir, ch)->name))
  45
  46 #define SET_BOTH_CH_32(dw, ch, name, value) \
  47         do {                                    \
  48                 writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name));   \
  49                 writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name));    \
  50         } while (0)
  51
  52 /* HDMA management callbacks */
  53 static void dw_hdma_v0_core_off(struct dw_edma *dw)
  54 {
  55         int id;
  56
  57         for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
  58                 SET_BOTH_CH_32(dw, id, int_setup,
  59                                HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
  60                 SET_BOTH_CH_32(dw, id, int_clear,
  61                                HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
  62                 SET_BOTH_CH_32(dw, id, ch_en, 0);
  63         }
  64 }
  65
  66 static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
  67 {
  68         /*
  69          * The HDMA IP have no way to know the number of hardware channels
  70          * available, we set it to maximum channels and let the platform
  71          * set the right number of channels.
  72          */
  73         return HDMA_V0_MAX_NR_CH;
  74 }
  75
  76 static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
  77 {
  78         struct dw_edma *dw = chan->dw;
  79         u32 tmp;
  80
  81         tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
  82                         GET_CH_32(dw, chan->id, chan->dir, ch_stat));
  83
  84         if (tmp == 1)
  85                 return DMA_IN_PROGRESS;
  86         else if (tmp == 3)
  87                 return DMA_COMPLETE;
  88         else
  89                 return DMA_ERROR;
  90 }
  91
  92 static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
  93 {
  94         struct dw_edma *dw = chan->dw;
  95
  96         SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
  97 }
  98
  99 static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
 100 {
 101         struct dw_edma *dw = chan->dw;
 102
 103         SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK);
 104 }
 105
 106 static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
 107 {
 108         struct dw_edma *dw = chan->dw;
 109
 110         return GET_CH_32(dw, chan->dir, chan->id, int_stat);
 111 }
 112
 113 static irqreturn_t
 114 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 115                            dw_edma_handler_t done, dw_edma_handler_t abort)
 116 {
 117         struct dw_edma *dw = dw_irq->dw;
 118         unsigned long total, pos, val;
 119         irqreturn_t ret = IRQ_NONE;
 120         struct dw_edma_chan *chan;
 121         unsigned long off, mask;
 122
 123         if (dir == EDMA_DIR_WRITE) {
 124                 total = dw->wr_ch_cnt;
 125                 off = 0;
 126                 mask = dw_irq->wr_mask;
 127         } else {
 128                 total = dw->rd_ch_cnt;
 129                 off = dw->wr_ch_cnt;
 130                 mask = dw_irq->rd_mask;
 131         }
 132
 133         for_each_set_bit(pos, &mask, total) {
 134                 chan = &dw->chan[pos + off];
 135
 136                 val = dw_hdma_v0_core_status_int(chan);
 137                 if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
 138                         dw_hdma_v0_core_clear_done_int(chan);
 139                         done(chan);
 140
 141                         ret = IRQ_HANDLED;
 142                 }
 143
 144                 if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
 145                         dw_hdma_v0_core_clear_abort_int(chan);
 146                         abort(chan);
 147
 148                         ret = IRQ_HANDLED;
 149                 }
 150         }
 151
 152         return ret;
 153 }
 154
 155 static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
 156                                      u32 control, u32 size, u64 sar, u64 dar)
 157 {
 158         ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
 159
 160         if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 161                 struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
 162
 163                 lli->control = control;
 164                 lli->transfer_size = size;
 165                 lli->sar.reg = sar;
 166                 lli->dar.reg = dar;
 167         } else {
 168                 struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
 169
 170                 writel(control, &lli->control);
 171                 writel(size, &lli->transfer_size);
 172                 writeq(sar, &lli->sar.reg);
 173                 writeq(dar, &lli->dar.reg);
 174         }
 175 }
 176
 177 static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
 178                                      int i, u32 control, u64 pointer)
 179 {
 180         ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
 181
 182         if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 183                 struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
 184
 185                 llp->control = control;
 186                 llp->llp.reg = pointer;
 187         } else {
 188                 struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
 189
 190                 writel(control, &llp->control);
 191                 writeq(pointer, &llp->llp.reg);
 192         }
 193 }
 194
 195 static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
 196 {
 197         struct dw_edma_burst *child;
 198         struct dw_edma_chan *chan = chunk->chan;
 199         u32 control = 0, i = 0;
 200         int j;
 201
 202         if (chunk->cb)
 203                 control = DW_HDMA_V0_CB;
 204
 205         j = chunk->bursts_alloc;
 206         list_for_each_entry(child, &chunk->burst->list, list) {
 207                 j--;
 208                 if (!j) {
 209                         control |= DW_HDMA_V0_LIE;
 210                         if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 211                                 control |= DW_HDMA_V0_RIE;
 212                 }
 213
 214                 dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
 215                                          child->sar, child->dar);
 216         }
 217
 218         control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
 219         if (!chunk->cb)
 220                 control |= DW_HDMA_V0_CB;
 221
 222         dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
 223 }
 224
 225 static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 226 {
 227         /*
 228          * In case of remote HDMA engine setup, the DW PCIe RP/EP internal
 229          * configuration registers and application memory are normally accessed
 230          * over different buses. Ensure LL-data reaches the memory before the
 231          * doorbell register is toggled by issuing the dummy-read from the remote
 232          * LL memory in a hope that the MRd TLP will return only after the
 233          * last MWr TLP is completed
 234          */
 235         if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 236                 readl(chunk->ll_region.vaddr.io);
 237 }
 238
 239 static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
 240 {
 241         struct dw_edma_chan *chan = chunk->chan;
 242         struct dw_edma *dw = chan->dw;
 243         u32 tmp;
 244
 245         dw_hdma_v0_core_write_chunk(chunk);
 246
 247         if (first) {
 248                 /* Enable engine */
 249                 SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
 250                 /* Interrupt enable&unmask - done, abort */
 251                 tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup) |
 252                       HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK |
 253                       HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
 254                 if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 255                         tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
 256                 SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
 257                 /* Channel control */
 258                 SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
 259                 /* Linked list */
 260                 /* llp is not aligned on 64bit -> keep 32bit accesses */
 261                 SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
 262                           lower_32_bits(chunk->ll_region.paddr));
 263                 SET_CH_32(dw, chan->dir, chan->id, llp.msb,
 264                           upper_32_bits(chunk->ll_region.paddr));
 265         }
 266         /* Set consumer cycle */
 267         SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
 268                   HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
 269
 270         dw_hdma_v0_sync_ll_data(chunk);
 271
 272         /* Doorbell */
 273         SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
 274 }
 275
 276 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
 277 {
 278         struct dw_edma *dw = chan->dw;
 279
 280         /* MSI done addr - low, high */
 281         SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
 282         SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
 283         /* MSI abort addr - low, high */
 284         SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
 285         SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
 286         /* config MSI data */
 287         SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
 288 }
 289
 290 /* HDMA debugfs callbacks */
 291 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 292 {
 293         dw_hdma_v0_debugfs_on(dw);
 294 }
 295
 296 static const struct dw_edma_core_ops dw_hdma_v0_core = {
 297         .off = dw_hdma_v0_core_off,
 298         .ch_count = dw_hdma_v0_core_ch_count,
 299         .ch_status = dw_hdma_v0_core_ch_status,
 300         .handle_int = dw_hdma_v0_core_handle_int,
 301         .start = dw_hdma_v0_core_start,
 302         .ch_config = dw_hdma_v0_core_ch_config,
 303         .debugfs_on = dw_hdma_v0_core_debugfs_on,
 304 };
 305
 306 void dw_hdma_v0_core_register(struct dw_edma *dw)
 307 {
 308         dw->core = &dw_hdma_v0_core;
 309 }