drivers/dma/dma-axi-dmac.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Driver for the Analog Devices AXI-DMAC core
   4  *
   5  * Copyright 2013-2019 Analog Devices Inc.
   6  *  Author: Lars-Peter Clausen <lars@metafoo.de>
   7  */
   8
   9 #include <linux/bitfield.h>
  10 #include <linux/clk.h>
  11 #include <linux/device.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/err.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/io.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/of_dma.h>
  21 #include <linux/of_address.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/regmap.h>
  24 #include <linux/slab.h>
  25 #include <linux/fpga/adi-axi-common.h>
  26
  27 #include <dt-bindings/dma/axi-dmac.h>
  28
  29 #include "dmaengine.h"
  30 #include "virt-dma.h"
  31
  32 /*
  33  * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
  34  * various instantiation parameters which decided the exact feature set support
  35  * by the core.
  36  *
  37  * Each channel of the core has a source interface and a destination interface.
  38  * The number of channels and the type of the channel interfaces is selected at
  39  * configuration time. A interface can either be a connected to a central memory
  40  * interconnect, which allows access to system memory, or it can be connected to
  41  * a dedicated bus which is directly connected to a data port on a peripheral.
  42  * Given that those are configuration options of the core that are selected when
  43  * it is instantiated this means that they can not be changed by software at
  44  * runtime. By extension this means that each channel is uni-directional. It can
  45  * either be device to memory or memory to device, but not both. Also since the
  46  * device side is a dedicated data bus only connected to a single peripheral
  47  * there is no address than can or needs to be configured for the device side.
  48  */
  49
  50 #define AXI_DMAC_REG_INTERFACE_DESC     0x10
  51 #define   AXI_DMAC_DMA_SRC_TYPE_MSK     GENMASK(13, 12)
  52 #define   AXI_DMAC_DMA_SRC_TYPE_GET(x)  FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x)
  53 #define   AXI_DMAC_DMA_SRC_WIDTH_MSK    GENMASK(11, 8)
  54 #define   AXI_DMAC_DMA_SRC_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x)
  55 #define   AXI_DMAC_DMA_DST_TYPE_MSK     GENMASK(5, 4)
  56 #define   AXI_DMAC_DMA_DST_TYPE_GET(x)  FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x)
  57 #define   AXI_DMAC_DMA_DST_WIDTH_MSK    GENMASK(3, 0)
  58 #define   AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x)
  59 #define AXI_DMAC_REG_COHERENCY_DESC     0x14
  60 #define   AXI_DMAC_DST_COHERENT_MSK     BIT(0)
  61 #define   AXI_DMAC_DST_COHERENT_GET(x)  FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x)
  62
  63 #define AXI_DMAC_REG_IRQ_MASK           0x80
  64 #define AXI_DMAC_REG_IRQ_PENDING        0x84
  65 #define AXI_DMAC_REG_IRQ_SOURCE         0x88
  66
  67 #define AXI_DMAC_REG_CTRL               0x400
  68 #define AXI_DMAC_REG_TRANSFER_ID        0x404
  69 #define AXI_DMAC_REG_START_TRANSFER     0x408
  70 #define AXI_DMAC_REG_FLAGS              0x40c
  71 #define AXI_DMAC_REG_DEST_ADDRESS       0x410
  72 #define AXI_DMAC_REG_SRC_ADDRESS        0x414
  73 #define AXI_DMAC_REG_X_LENGTH           0x418
  74 #define AXI_DMAC_REG_Y_LENGTH           0x41c
  75 #define AXI_DMAC_REG_DEST_STRIDE        0x420
  76 #define AXI_DMAC_REG_SRC_STRIDE         0x424
  77 #define AXI_DMAC_REG_TRANSFER_DONE      0x428
  78 #define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
  79 #define AXI_DMAC_REG_STATUS             0x430
  80 #define AXI_DMAC_REG_CURRENT_SRC_ADDR   0x434
  81 #define AXI_DMAC_REG_CURRENT_DEST_ADDR  0x438
  82 #define AXI_DMAC_REG_PARTIAL_XFER_LEN   0x44c
  83 #define AXI_DMAC_REG_PARTIAL_XFER_ID    0x450
  84 #define AXI_DMAC_REG_CURRENT_SG_ID      0x454
  85 #define AXI_DMAC_REG_SG_ADDRESS         0x47c
  86 #define AXI_DMAC_REG_SG_ADDRESS_HIGH    0x4bc
  87
  88 #define AXI_DMAC_CTRL_ENABLE            BIT(0)
  89 #define AXI_DMAC_CTRL_PAUSE             BIT(1)
  90 #define AXI_DMAC_CTRL_ENABLE_SG         BIT(2)
  91
  92 #define AXI_DMAC_IRQ_SOT                BIT(0)
  93 #define AXI_DMAC_IRQ_EOT                BIT(1)
  94
  95 #define AXI_DMAC_FLAG_CYCLIC            BIT(0)
  96 #define AXI_DMAC_FLAG_LAST              BIT(1)
  97 #define AXI_DMAC_FLAG_PARTIAL_REPORT    BIT(2)
  98
  99 #define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
 100
 101 /* The maximum ID allocated by the hardware is 31 */
 102 #define AXI_DMAC_SG_UNUSED 32U
 103
 104 /* Flags for axi_dmac_hw_desc.flags */
 105 #define AXI_DMAC_HW_FLAG_LAST           BIT(0)
 106 #define AXI_DMAC_HW_FLAG_IRQ            BIT(1)
 107
 108 struct axi_dmac_hw_desc {
 109         u32 flags;
 110         u32 id;
 111         u64 dest_addr;
 112         u64 src_addr;
 113         u64 next_sg_addr;
 114         u32 y_len;
 115         u32 x_len;
 116         u32 src_stride;
 117         u32 dst_stride;
 118         u64 __pad[2];
 119 };
 120
 121 struct axi_dmac_sg {
 122         unsigned int partial_len;
 123         bool schedule_when_free;
 124
 125         struct axi_dmac_hw_desc *hw;
 126         dma_addr_t hw_phys;
 127 };
 128
 129 struct axi_dmac_desc {
 130         struct virt_dma_desc vdesc;
 131         struct axi_dmac_chan *chan;
 132
 133         bool cyclic;
 134         bool have_partial_xfer;
 135
 136         unsigned int num_submitted;
 137         unsigned int num_completed;
 138         unsigned int num_sgs;
 139         struct axi_dmac_sg sg[] __counted_by(num_sgs);
 140 };
 141
 142 struct axi_dmac_chan {
 143         struct virt_dma_chan vchan;
 144
 145         struct axi_dmac_desc *next_desc;
 146         struct list_head active_descs;
 147         enum dma_transfer_direction direction;
 148
 149         unsigned int src_width;
 150         unsigned int dest_width;
 151         unsigned int src_type;
 152         unsigned int dest_type;
 153
 154         unsigned int max_length;
 155         unsigned int address_align_mask;
 156         unsigned int length_align_mask;
 157
 158         bool hw_partial_xfer;
 159         bool hw_cyclic;
 160         bool hw_2d;
 161         bool hw_sg;
 162 };
 163
 164 struct axi_dmac {
 165         void __iomem *base;
 166         int irq;
 167
 168         struct clk *clk;
 169
 170         struct dma_device dma_dev;
 171         struct axi_dmac_chan chan;
 172 };
 173
 174 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)
 175 {
 176         return container_of(chan->vchan.chan.device, struct axi_dmac,
 177                 dma_dev);
 178 }
 179
 180 static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c)
 181 {
 182         return container_of(c, struct axi_dmac_chan, vchan.chan);
 183 }
 184
 185 static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc)
 186 {
 187         return container_of(vdesc, struct axi_dmac_desc, vdesc);
 188 }
 189
 190 static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg,
 191         unsigned int val)
 192 {
 193         writel(val, axi_dmac->base + reg);
 194 }
 195
 196 static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg)
 197 {
 198         return readl(axi_dmac->base + reg);
 199 }
 200
 201 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)
 202 {
 203         return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 204 }
 205
 206 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
 207 {
 208         return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 209 }
 210
 211 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
 212 {
 213         if (len == 0)
 214                 return false;
 215         if ((len & chan->length_align_mask) != 0) /* Not aligned */
 216                 return false;
 217         return true;
 218 }
 219
 220 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
 221 {
 222         if ((addr & chan->address_align_mask) != 0) /* Not aligned */
 223                 return false;
 224         return true;
 225 }
 226
 227 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 228 {
 229         struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 230         struct virt_dma_desc *vdesc;
 231         struct axi_dmac_desc *desc;
 232         struct axi_dmac_sg *sg;
 233         unsigned int flags = 0;
 234         unsigned int val;
 235
 236         if (!chan->hw_sg) {
 237                 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
 238                 if (val) /* Queue is full, wait for the next SOT IRQ */
 239                         return;
 240         }
 241
 242         desc = chan->next_desc;
 243
 244         if (!desc) {
 245                 vdesc = vchan_next_desc(&chan->vchan);
 246                 if (!vdesc)
 247                         return;
 248                 list_move_tail(&vdesc->node, &chan->active_descs);
 249                 desc = to_axi_dmac_desc(vdesc);
 250         }
 251         sg = &desc->sg[desc->num_submitted];
 252
 253         /* Already queued in cyclic mode. Wait for it to finish */
 254         if (sg->hw->id != AXI_DMAC_SG_UNUSED) {
 255                 sg->schedule_when_free = true;
 256                 return;
 257         }
 258
 259         if (chan->hw_sg) {
 260                 chan->next_desc = NULL;
 261         } else if (++desc->num_submitted == desc->num_sgs ||
 262                    desc->have_partial_xfer) {
 263                 if (desc->cyclic)
 264                         desc->num_submitted = 0; /* Start again */
 265                 else
 266                         chan->next_desc = NULL;
 267                 flags |= AXI_DMAC_FLAG_LAST;
 268         } else {
 269                 chan->next_desc = desc;
 270         }
 271
 272         sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
 273
 274         if (!chan->hw_sg) {
 275                 if (axi_dmac_dest_is_mem(chan)) {
 276                         axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr);
 277                         axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride);
 278                 }
 279
 280                 if (axi_dmac_src_is_mem(chan)) {
 281                         axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr);
 282                         axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride);
 283                 }
 284         }
 285
 286         /*
 287          * If the hardware supports cyclic transfers and there is no callback to
 288          * call, enable hw cyclic mode to avoid unnecessary interrupts.
 289          */
 290         if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) {
 291                 if (chan->hw_sg)
 292                         desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ;
 293                 else if (desc->num_sgs == 1)
 294                         flags |= AXI_DMAC_FLAG_CYCLIC;
 295         }
 296
 297         if (chan->hw_partial_xfer)
 298                 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
 299
 300         if (chan->hw_sg) {
 301                 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys);
 302                 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH,
 303                                (u64)sg->hw_phys >> 32);
 304         } else {
 305                 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len);
 306                 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len);
 307         }
 308         axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
 309         axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
 310 }
 311
 312 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
 313 {
 314         return list_first_entry_or_null(&chan->active_descs,
 315                 struct axi_dmac_desc, vdesc.node);
 316 }
 317
 318 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
 319         struct axi_dmac_sg *sg)
 320 {
 321         if (chan->hw_2d)
 322                 return (sg->hw->x_len + 1) * (sg->hw->y_len + 1);
 323         else
 324                 return (sg->hw->x_len + 1);
 325 }
 326
 327 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan)
 328 {
 329         struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 330         struct axi_dmac_desc *desc;
 331         struct axi_dmac_sg *sg;
 332         u32 xfer_done, len, id, i;
 333         bool found_sg;
 334
 335         do {
 336                 len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);
 337                 id  = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID);
 338
 339                 found_sg = false;
 340                 list_for_each_entry(desc, &chan->active_descs, vdesc.node) {
 341                         for (i = 0; i < desc->num_sgs; i++) {
 342                                 sg = &desc->sg[i];
 343                                 if (sg->hw->id == AXI_DMAC_SG_UNUSED)
 344                                         continue;
 345                                 if (sg->hw->id == id) {
 346                                         desc->have_partial_xfer = true;
 347                                         sg->partial_len = len;
 348                                         found_sg = true;
 349                                         break;
 350                                 }
 351                         }
 352                         if (found_sg)
 353                                 break;
 354                 }
 355
 356                 if (found_sg) {
 357                         dev_dbg(dmac->dma_dev.dev,
 358                                 "Found partial segment id=%u, len=%u\n",
 359                                 id, len);
 360                 } else {
 361                         dev_warn(dmac->dma_dev.dev,
 362                                  "Not found partial segment id=%u, len=%u\n",
 363                                  id, len);
 364                 }
 365
 366                 /* Check if we have any more partial transfers */
 367                 xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 368                 xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE);
 369
 370         } while (!xfer_done);
 371 }
 372
 373 static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
 374         struct axi_dmac_desc *active)
 375 {
 376         struct dmaengine_result *rslt = &active->vdesc.tx_result;
 377         unsigned int start = active->num_completed - 1;
 378         struct axi_dmac_sg *sg;
 379         unsigned int i, total;
 380
 381         rslt->result = DMA_TRANS_NOERROR;
 382         rslt->residue = 0;
 383
 384         if (chan->hw_sg)
 385                 return;
 386
 387         /*
 388          * We get here if the last completed segment is partial, which
 389          * means we can compute the residue from that segment onwards
 390          */
 391         for (i = start; i < active->num_sgs; i++) {
 392                 sg = &active->sg[i];
 393                 total = axi_dmac_total_sg_bytes(chan, sg);
 394                 rslt->residue += (total - sg->partial_len);
 395         }
 396 }
 397
 398 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 399         unsigned int completed_transfers)
 400 {
 401         struct axi_dmac_desc *active;
 402         struct axi_dmac_sg *sg;
 403         bool start_next = false;
 404
 405         active = axi_dmac_active_desc(chan);
 406         if (!active)
 407                 return false;
 408
 409         if (chan->hw_partial_xfer &&
 410             (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
 411                 axi_dmac_dequeue_partial_xfers(chan);
 412
 413         if (chan->hw_sg) {
 414                 if (active->cyclic) {
 415                         vchan_cyclic_callback(&active->vdesc);
 416                 } else {
 417                         list_del(&active->vdesc.node);
 418                         vchan_cookie_complete(&active->vdesc);
 419                         active = axi_dmac_active_desc(chan);
 420                         start_next = !!active;
 421                 }
 422         } else {
 423                 do {
 424                         sg = &active->sg[active->num_completed];
 425                         if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
 426                                 break;
 427                         if (!(BIT(sg->hw->id) & completed_transfers))
 428                                 break;
 429                         active->num_completed++;
 430                         sg->hw->id = AXI_DMAC_SG_UNUSED;
 431                         if (sg->schedule_when_free) {
 432                                 sg->schedule_when_free = false;
 433                                 start_next = true;
 434                         }
 435
 436                         if (sg->partial_len)
 437                                 axi_dmac_compute_residue(chan, active);
 438
 439                         if (active->cyclic)
 440                                 vchan_cyclic_callback(&active->vdesc);
 441
 442                         if (active->num_completed == active->num_sgs ||
 443                             sg->partial_len) {
 444                                 if (active->cyclic) {
 445                                         active->num_completed = 0; /* wrap around */
 446                                 } else {
 447                                         list_del(&active->vdesc.node);
 448                                         vchan_cookie_complete(&active->vdesc);
 449                                         active = axi_dmac_active_desc(chan);
 450                                 }
 451                         }
 452                 } while (active);
 453         }
 454
 455         return start_next;
 456 }
 457
 458 static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid)
 459 {
 460         struct axi_dmac *dmac = devid;
 461         unsigned int pending;
 462         bool start_next = false;
 463
 464         pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING);
 465         if (!pending)
 466                 return IRQ_NONE;
 467
 468         axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending);
 469
 470         spin_lock(&dmac->chan.vchan.lock);
 471         /* One or more transfers have finished */
 472         if (pending & AXI_DMAC_IRQ_EOT) {
 473                 unsigned int completed;
 474
 475                 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 476                 start_next = axi_dmac_transfer_done(&dmac->chan, completed);
 477         }
 478         /* Space has become available in the descriptor queue */
 479         if ((pending & AXI_DMAC_IRQ_SOT) || start_next)
 480                 axi_dmac_start_transfer(&dmac->chan);
 481         spin_unlock(&dmac->chan.vchan.lock);
 482
 483         return IRQ_HANDLED;
 484 }
 485
 486 static int axi_dmac_terminate_all(struct dma_chan *c)
 487 {
 488         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 489         struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 490         unsigned long flags;
 491         LIST_HEAD(head);
 492
 493         spin_lock_irqsave(&chan->vchan.lock, flags);
 494         axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
 495         chan->next_desc = NULL;
 496         vchan_get_all_descriptors(&chan->vchan, &head);
 497         list_splice_tail_init(&chan->active_descs, &head);
 498         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 499
 500         vchan_dma_desc_free_list(&chan->vchan, &head);
 501
 502         return 0;
 503 }
 504
 505 static void axi_dmac_synchronize(struct dma_chan *c)
 506 {
 507         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 508
 509         vchan_synchronize(&chan->vchan);
 510 }
 511
 512 static void axi_dmac_issue_pending(struct dma_chan *c)
 513 {
 514         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 515         struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 516         unsigned long flags;
 517         u32 ctrl = AXI_DMAC_CTRL_ENABLE;
 518
 519         if (chan->hw_sg)
 520                 ctrl |= AXI_DMAC_CTRL_ENABLE_SG;
 521
 522         axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl);
 523
 524         spin_lock_irqsave(&chan->vchan.lock, flags);
 525         if (vchan_issue_pending(&chan->vchan))
 526                 axi_dmac_start_transfer(chan);
 527         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 528 }
 529
 530 static struct axi_dmac_desc *
 531 axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
 532 {
 533         struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 534         struct device *dev = dmac->dma_dev.dev;
 535         struct axi_dmac_hw_desc *hws;
 536         struct axi_dmac_desc *desc;
 537         dma_addr_t hw_phys;
 538         unsigned int i;
 539
 540         desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT);
 541         if (!desc)
 542                 return NULL;
 543         desc->num_sgs = num_sgs;
 544         desc->chan = chan;
 545
 546         hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)),
 547                                 &hw_phys, GFP_ATOMIC);
 548         if (!hws) {
 549                 kfree(desc);
 550                 return NULL;
 551         }
 552
 553         for (i = 0; i < num_sgs; i++) {
 554                 desc->sg[i].hw = &hws[i];
 555                 desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);
 556
 557                 hws[i].id = AXI_DMAC_SG_UNUSED;
 558                 hws[i].flags = 0;
 559
 560                 /* Link hardware descriptors */
 561                 hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
 562         }
 563
 564         /* The last hardware descriptor will trigger an interrupt */
 565         desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;
 566
 567         return desc;
 568 }
 569
 570 static void axi_dmac_free_desc(struct axi_dmac_desc *desc)
 571 {
 572         struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan);
 573         struct device *dev = dmac->dma_dev.dev;
 574         struct axi_dmac_hw_desc *hw = desc->sg[0].hw;
 575         dma_addr_t hw_phys = desc->sg[0].hw_phys;
 576
 577         dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)),
 578                           hw, hw_phys);
 579         kfree(desc);
 580 }
 581
 582 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
 583         enum dma_transfer_direction direction, dma_addr_t addr,
 584         unsigned int num_periods, unsigned int period_len,
 585         struct axi_dmac_sg *sg)
 586 {
 587         unsigned int num_segments, i;
 588         unsigned int segment_size;
 589         unsigned int len;
 590
 591         /* Split into multiple equally sized segments if necessary */
 592         num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 593         segment_size = DIV_ROUND_UP(period_len, num_segments);
 594         /* Take care of alignment */
 595         segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;
 596
 597         for (i = 0; i < num_periods; i++) {
 598                 for (len = period_len; len > segment_size; sg++) {
 599                         if (direction == DMA_DEV_TO_MEM)
 600                                 sg->hw->dest_addr = addr;
 601                         else
 602                                 sg->hw->src_addr = addr;
 603                         sg->hw->x_len = segment_size - 1;
 604                         sg->hw->y_len = 0;
 605                         sg->hw->flags = 0;
 606                         addr += segment_size;
 607                         len -= segment_size;
 608                 }
 609
 610                 if (direction == DMA_DEV_TO_MEM)
 611                         sg->hw->dest_addr = addr;
 612                 else
 613                         sg->hw->src_addr = addr;
 614                 sg->hw->x_len = len - 1;
 615                 sg->hw->y_len = 0;
 616                 sg++;
 617                 addr += len;
 618         }
 619
 620         return sg;
 621 }
 622
 623 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 624         struct dma_chan *c, struct scatterlist *sgl,
 625         unsigned int sg_len, enum dma_transfer_direction direction,
 626         unsigned long flags, void *context)
 627 {
 628         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 629         struct axi_dmac_desc *desc;
 630         struct axi_dmac_sg *dsg;
 631         struct scatterlist *sg;
 632         unsigned int num_sgs;
 633         unsigned int i;
 634
 635         if (direction != chan->direction)
 636                 return NULL;
 637
 638         num_sgs = 0;
 639         for_each_sg(sgl, sg, sg_len, i)
 640                 num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length);
 641
 642         desc = axi_dmac_alloc_desc(chan, num_sgs);
 643         if (!desc)
 644                 return NULL;
 645
 646         dsg = desc->sg;
 647
 648         for_each_sg(sgl, sg, sg_len, i) {
 649                 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
 650                     !axi_dmac_check_len(chan, sg_dma_len(sg))) {
 651                         axi_dmac_free_desc(desc);
 652                         return NULL;
 653                 }
 654
 655                 dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1,
 656                         sg_dma_len(sg), dsg);
 657         }
 658
 659         desc->cyclic = false;
 660
 661         return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 662 }
 663
 664 static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
 665         struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
 666         size_t period_len, enum dma_transfer_direction direction,
 667         unsigned long flags)
 668 {
 669         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 670         struct axi_dmac_desc *desc;
 671         unsigned int num_periods, num_segments, num_sgs;
 672
 673         if (direction != chan->direction)
 674                 return NULL;
 675
 676         if (!axi_dmac_check_len(chan, buf_len) ||
 677             !axi_dmac_check_addr(chan, buf_addr))
 678                 return NULL;
 679
 680         if (period_len == 0 || buf_len % period_len)
 681                 return NULL;
 682
 683         num_periods = buf_len / period_len;
 684         num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 685         num_sgs = num_periods * num_segments;
 686
 687         desc = axi_dmac_alloc_desc(chan, num_sgs);
 688         if (!desc)
 689                 return NULL;
 690
 691         /* Chain the last descriptor to the first, and remove its "last" flag */
 692         desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys;
 693         desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST;
 694
 695         axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods,
 696                 period_len, desc->sg);
 697
 698         desc->cyclic = true;
 699
 700         return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 701 }
 702
 703 static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
 704         struct dma_chan *c, struct dma_interleaved_template *xt,
 705         unsigned long flags)
 706 {
 707         struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 708         struct axi_dmac_desc *desc;
 709         size_t dst_icg, src_icg;
 710
 711         if (xt->frame_size != 1)
 712                 return NULL;
 713
 714         if (xt->dir != chan->direction)
 715                 return NULL;
 716
 717         if (axi_dmac_src_is_mem(chan)) {
 718                 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))
 719                         return NULL;
 720         }
 721
 722         if (axi_dmac_dest_is_mem(chan)) {
 723                 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))
 724                         return NULL;
 725         }
 726
 727         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
 728         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
 729
 730         if (chan->hw_2d) {
 731                 if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
 732                     xt->numf == 0)
 733                         return NULL;
 734                 if (xt->sgl[0].size + dst_icg > chan->max_length ||
 735                     xt->sgl[0].size + src_icg > chan->max_length)
 736                         return NULL;
 737         } else {
 738                 if (dst_icg != 0 || src_icg != 0)
 739                         return NULL;
 740                 if (chan->max_length / xt->sgl[0].size < xt->numf)
 741                         return NULL;
 742                 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))
 743                         return NULL;
 744         }
 745
 746         desc = axi_dmac_alloc_desc(chan, 1);
 747         if (!desc)
 748                 return NULL;
 749
 750         if (axi_dmac_src_is_mem(chan)) {
 751                 desc->sg[0].hw->src_addr = xt->src_start;
 752                 desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg;
 753         }
 754
 755         if (axi_dmac_dest_is_mem(chan)) {
 756                 desc->sg[0].hw->dest_addr = xt->dst_start;
 757                 desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg;
 758         }
 759
 760         if (chan->hw_2d) {
 761                 desc->sg[0].hw->x_len = xt->sgl[0].size - 1;
 762                 desc->sg[0].hw->y_len = xt->numf - 1;
 763         } else {
 764                 desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1;
 765                 desc->sg[0].hw->y_len = 0;
 766         }
 767
 768         if (flags & DMA_CYCLIC)
 769                 desc->cyclic = true;
 770
 771         return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 772 }
 773
 774 static void axi_dmac_free_chan_resources(struct dma_chan *c)
 775 {
 776         vchan_free_chan_resources(to_virt_chan(c));
 777 }
 778
 779 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
 780 {
 781         axi_dmac_free_desc(to_axi_dmac_desc(vdesc));
 782 }
 783
 784 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
 785 {
 786         switch (reg) {
 787         case AXI_DMAC_REG_IRQ_MASK:
 788         case AXI_DMAC_REG_IRQ_SOURCE:
 789         case AXI_DMAC_REG_IRQ_PENDING:
 790         case AXI_DMAC_REG_CTRL:
 791         case AXI_DMAC_REG_TRANSFER_ID:
 792         case AXI_DMAC_REG_START_TRANSFER:
 793         case AXI_DMAC_REG_FLAGS:
 794         case AXI_DMAC_REG_DEST_ADDRESS:
 795         case AXI_DMAC_REG_SRC_ADDRESS:
 796         case AXI_DMAC_REG_X_LENGTH:
 797         case AXI_DMAC_REG_Y_LENGTH:
 798         case AXI_DMAC_REG_DEST_STRIDE:
 799         case AXI_DMAC_REG_SRC_STRIDE:
 800         case AXI_DMAC_REG_TRANSFER_DONE:
 801         case AXI_DMAC_REG_ACTIVE_TRANSFER_ID:
 802         case AXI_DMAC_REG_STATUS:
 803         case AXI_DMAC_REG_CURRENT_SRC_ADDR:
 804         case AXI_DMAC_REG_CURRENT_DEST_ADDR:
 805         case AXI_DMAC_REG_PARTIAL_XFER_LEN:
 806         case AXI_DMAC_REG_PARTIAL_XFER_ID:
 807         case AXI_DMAC_REG_CURRENT_SG_ID:
 808         case AXI_DMAC_REG_SG_ADDRESS:
 809         case AXI_DMAC_REG_SG_ADDRESS_HIGH:
 810                 return true;
 811         default:
 812                 return false;
 813         }
 814 }
 815
 816 static const struct regmap_config axi_dmac_regmap_config = {
 817         .reg_bits = 32,
 818         .val_bits = 32,
 819         .reg_stride = 4,
 820         .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID,
 821         .readable_reg = axi_dmac_regmap_rdwr,
 822         .writeable_reg = axi_dmac_regmap_rdwr,
 823 };
 824
 825 static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan)
 826 {
 827         chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;
 828
 829         if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 830                 chan->direction = DMA_MEM_TO_MEM;
 831         else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 832                 chan->direction = DMA_MEM_TO_DEV;
 833         else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))
 834                 chan->direction = DMA_DEV_TO_MEM;
 835         else
 836                 chan->direction = DMA_DEV_TO_DEV;
 837 }
 838
 839 /*
 840  * The configuration stored in the devicetree matches the configuration
 841  * parameters of the peripheral instance and allows the driver to know which
 842  * features are implemented and how it should behave.
 843  */
 844 static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
 845         struct axi_dmac_chan *chan)
 846 {
 847         u32 val;
 848         int ret;
 849
 850         ret = of_property_read_u32(of_chan, "reg", &val);
 851         if (ret)
 852                 return ret;
 853
 854         /* We only support 1 channel for now */
 855         if (val != 0)
 856                 return -EINVAL;
 857
 858         ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);
 859         if (ret)
 860                 return ret;
 861         if (val > AXI_DMAC_BUS_TYPE_FIFO)
 862                 return -EINVAL;
 863         chan->src_type = val;
 864
 865         ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);
 866         if (ret)
 867                 return ret;
 868         if (val > AXI_DMAC_BUS_TYPE_FIFO)
 869                 return -EINVAL;
 870         chan->dest_type = val;
 871
 872         ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);
 873         if (ret)
 874                 return ret;
 875         chan->src_width = val / 8;
 876
 877         ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);
 878         if (ret)
 879                 return ret;
 880         chan->dest_width = val / 8;
 881
 882         axi_dmac_adjust_chan_params(chan);
 883
 884         return 0;
 885 }
 886
 887 static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac)
 888 {
 889         struct device_node *of_channels, *of_chan;
 890         int ret;
 891
 892         of_channels = of_get_child_by_name(dev->of_node, "adi,channels");
 893         if (of_channels == NULL)
 894                 return -ENODEV;
 895
 896         for_each_child_of_node(of_channels, of_chan) {
 897                 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);
 898                 if (ret) {
 899                         of_node_put(of_chan);
 900                         of_node_put(of_channels);
 901                         return -EINVAL;
 902                 }
 903         }
 904         of_node_put(of_channels);
 905
 906         return 0;
 907 }
 908
 909 static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac)
 910 {
 911         struct axi_dmac_chan *chan = &dmac->chan;
 912         unsigned int val, desc;
 913
 914         desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC);
 915         if (desc == 0) {
 916                 dev_err(dev, "DMA interface register reads zero\n");
 917                 return -EFAULT;
 918         }
 919
 920         val = AXI_DMAC_DMA_SRC_TYPE_GET(desc);
 921         if (val > AXI_DMAC_BUS_TYPE_FIFO) {
 922                 dev_err(dev, "Invalid source bus type read: %d\n", val);
 923                 return -EINVAL;
 924         }
 925         chan->src_type = val;
 926
 927         val = AXI_DMAC_DMA_DST_TYPE_GET(desc);
 928         if (val > AXI_DMAC_BUS_TYPE_FIFO) {
 929                 dev_err(dev, "Invalid destination bus type read: %d\n", val);
 930                 return -EINVAL;
 931         }
 932         chan->dest_type = val;
 933
 934         val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc);
 935         if (val == 0) {
 936                 dev_err(dev, "Source bus width is zero\n");
 937                 return -EINVAL;
 938         }
 939         /* widths are stored in log2 */
 940         chan->src_width = 1 << val;
 941
 942         val = AXI_DMAC_DMA_DST_WIDTH_GET(desc);
 943         if (val == 0) {
 944                 dev_err(dev, "Destination bus width is zero\n");
 945                 return -EINVAL;
 946         }
 947         chan->dest_width = 1 << val;
 948
 949         axi_dmac_adjust_chan_params(chan);
 950
 951         return 0;
 952 }
 953
 954 static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
 955 {
 956         struct axi_dmac_chan *chan = &dmac->chan;
 957
 958         axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
 959         if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
 960                 chan->hw_cyclic = true;
 961
 962         axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff);
 963         if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS))
 964                 chan->hw_sg = true;
 965
 966         axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
 967         if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
 968                 chan->hw_2d = true;
 969
 970         axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
 971         chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
 972         if (chan->max_length != UINT_MAX)
 973                 chan->max_length++;
 974
 975         axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff);
 976         if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 &&
 977             chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
 978                 dev_err(dmac->dma_dev.dev,
 979                         "Destination memory-mapped interface not supported.");
 980                 return -ENODEV;
 981         }
 982
 983         axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff);
 984         if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 &&
 985             chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
 986                 dev_err(dmac->dma_dev.dev,
 987                         "Source memory-mapped interface not supported.");
 988                 return -ENODEV;
 989         }
 990
 991         if (version >= ADI_AXI_PCORE_VER(4, 2, 'a'))
 992                 chan->hw_partial_xfer = true;
 993
 994         if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) {
 995                 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00);
 996                 chan->length_align_mask =
 997                         axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
 998         } else {
 999                 chan->length_align_mask = chan->address_align_mask;
1000         }
1001
1002         return 0;
1003 }
1004
1005 static int axi_dmac_probe(struct platform_device *pdev)
1006 {
1007         struct dma_device *dma_dev;
1008         struct axi_dmac *dmac;
1009         struct regmap *regmap;
1010         unsigned int version;
1011         u32 irq_mask = 0;
1012         int ret;
1013
1014         dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
1015         if (!dmac)
1016                 return -ENOMEM;
1017
1018         dmac->irq = platform_get_irq(pdev, 0);
1019         if (dmac->irq < 0)
1020                 return dmac->irq;
1021         if (dmac->irq == 0)
1022                 return -EINVAL;
1023
1024         dmac->base = devm_platform_ioremap_resource(pdev, 0);
1025         if (IS_ERR(dmac->base))
1026                 return PTR_ERR(dmac->base);
1027
1028         dmac->clk = devm_clk_get(&pdev->dev, NULL);
1029         if (IS_ERR(dmac->clk))
1030                 return PTR_ERR(dmac->clk);
1031
1032         ret = clk_prepare_enable(dmac->clk);
1033         if (ret < 0)
1034                 return ret;
1035
1036         version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
1037
1038         if (version >= ADI_AXI_PCORE_VER(4, 3, 'a'))
1039                 ret = axi_dmac_read_chan_config(&pdev->dev, dmac);
1040         else
1041                 ret = axi_dmac_parse_dt(&pdev->dev, dmac);
1042
1043         if (ret < 0)
1044                 goto err_clk_disable;
1045
1046         INIT_LIST_HEAD(&dmac->chan.active_descs);
1047
1048         dma_set_max_seg_size(&pdev->dev, UINT_MAX);
1049
1050         dma_dev = &dmac->dma_dev;
1051         dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1052         dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1053         dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
1054         dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
1055         dma_dev->device_tx_status = dma_cookie_status;
1056         dma_dev->device_issue_pending = axi_dmac_issue_pending;
1057         dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;
1058         dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;
1059         dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;
1060         dma_dev->device_terminate_all = axi_dmac_terminate_all;
1061         dma_dev->device_synchronize = axi_dmac_synchronize;
1062         dma_dev->dev = &pdev->dev;
1063         dma_dev->src_addr_widths = BIT(dmac->chan.src_width);
1064         dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
1065         dma_dev->directions = BIT(dmac->chan.direction);
1066         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1067         dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */
1068         INIT_LIST_HEAD(&dma_dev->channels);
1069
1070         dmac->chan.vchan.desc_free = axi_dmac_desc_free;
1071         vchan_init(&dmac->chan.vchan, dma_dev);
1072
1073         ret = axi_dmac_detect_caps(dmac, version);
1074         if (ret)
1075                 goto err_clk_disable;
1076
1077         dma_dev->copy_align = (dmac->chan.address_align_mask + 1);
1078
1079         if (dmac->chan.hw_sg)
1080                 irq_mask |= AXI_DMAC_IRQ_SOT;
1081
1082         axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask);
1083
1084         if (of_dma_is_coherent(pdev->dev.of_node)) {
1085                 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC);
1086
1087                 if (version < ADI_AXI_PCORE_VER(4, 4, 'a') ||
1088                     !AXI_DMAC_DST_COHERENT_GET(ret)) {
1089                         dev_err(dmac->dma_dev.dev,
1090                                 "Coherent DMA not supported in hardware");
1091                         ret = -EINVAL;
1092                         goto err_clk_disable;
1093                 }
1094         }
1095
1096         ret = dma_async_device_register(dma_dev);
1097         if (ret)
1098                 goto err_clk_disable;
1099
1100         ret = of_dma_controller_register(pdev->dev.of_node,
1101                 of_dma_xlate_by_chan_id, dma_dev);
1102         if (ret)
1103                 goto err_unregister_device;
1104
1105         ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED,
1106                 dev_name(&pdev->dev), dmac);
1107         if (ret)
1108                 goto err_unregister_of;
1109
1110         platform_set_drvdata(pdev, dmac);
1111
1112         regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,
1113                  &axi_dmac_regmap_config);
1114         if (IS_ERR(regmap)) {
1115                 ret = PTR_ERR(regmap);
1116                 goto err_free_irq;
1117         }
1118
1119         return 0;
1120
1121 err_free_irq:
1122         free_irq(dmac->irq, dmac);
1123 err_unregister_of:
1124         of_dma_controller_free(pdev->dev.of_node);
1125 err_unregister_device:
1126         dma_async_device_unregister(&dmac->dma_dev);
1127 err_clk_disable:
1128         clk_disable_unprepare(dmac->clk);
1129
1130         return ret;
1131 }
1132
1133 static void axi_dmac_remove(struct platform_device *pdev)
1134 {
1135         struct axi_dmac *dmac = platform_get_drvdata(pdev);
1136
1137         of_dma_controller_free(pdev->dev.of_node);
1138         free_irq(dmac->irq, dmac);
1139         tasklet_kill(&dmac->chan.vchan.task);
1140         dma_async_device_unregister(&dmac->dma_dev);
1141         clk_disable_unprepare(dmac->clk);
1142 }
1143
1144 static const struct of_device_id axi_dmac_of_match_table[] = {
1145         { .compatible = "adi,axi-dmac-1.00.a" },
1146         { },
1147 };
1148 MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table);
1149
1150 static struct platform_driver axi_dmac_driver = {
1151         .driver = {
1152                 .name = "dma-axi-dmac",
1153                 .of_match_table = axi_dmac_of_match_table,
1154         },
1155         .probe = axi_dmac_probe,
1156         .remove_new = axi_dmac_remove,
1157 };
1158 module_platform_driver(axi_dmac_driver);
1159
1160 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
1161 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");
1162 MODULE_LICENSE("GPL v2");