drivers/dma/xilinx/xdma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * DMA driver for Xilinx DMA/Bridge Subsystem
   4  *
   5  * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved.
   6  * Copyright (C) 2022, Advanced Micro Devices, Inc.
   7  */
   8
   9 /*
  10  * The DMA/Bridge Subsystem for PCI Express allows for the movement of data
  11  * between Host memory and the DMA subsystem. It does this by operating on
  12  * 'descriptors' that contain information about the source, destination and
  13  * amount of data to transfer. These direct memory transfers can be both in
  14  * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA can be
  15  * configured to have a single AXI4 Master interface shared by all channels
  16  * or one AXI4-Stream interface for each channel enabled. Memory transfers are
  17  * specified on a per-channel basis in descriptor linked lists, which the DMA
  18  * fetches from host memory and processes. Events such as descriptor completion
  19  * and errors are signaled using interrupts. The core also provides up to 16
  20  * user interrupt wires that generate interrupts to the host.
  21  */
  22
  23 #include <linux/mod_devicetable.h>
  24 #include <linux/bitfield.h>
  25 #include <linux/dmapool.h>
  26 #include <linux/regmap.h>
  27 #include <linux/dmaengine.h>
  28 #include <linux/dma/amd_xdma.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/platform_data/amd_xdma.h>
  31 #include <linux/dma-mapping.h>
  32 #include <linux/pci.h>
  33 #include "../virt-dma.h"
  34 #include "xdma-regs.h"
  35
  36 /* mmio regmap config for all XDMA registers */
  37 static const struct regmap_config xdma_regmap_config = {
  38         .reg_bits = 32,
  39         .val_bits = 32,
  40         .reg_stride = 4,
  41         .max_register = XDMA_REG_SPACE_LEN,
  42 };
  43
  44 /**
  45  * struct xdma_desc_block - Descriptor block
  46  * @virt_addr: Virtual address of block start
  47  * @dma_addr: DMA address of block start
  48  */
  49 struct xdma_desc_block {
  50         void            *virt_addr;
  51         dma_addr_t      dma_addr;
  52 };
  53
  54 /**
  55  * struct xdma_chan - Driver specific DMA channel structure
  56  * @vchan: Virtual channel
  57  * @xdev_hdl: Pointer to DMA device structure
  58  * @base: Offset of channel registers
  59  * @desc_pool: Descriptor pool
  60  * @busy: Busy flag of the channel
  61  * @dir: Transferring direction of the channel
  62  * @cfg: Transferring config of the channel
  63  * @irq: IRQ assigned to the channel
  64  */
  65 struct xdma_chan {
  66         struct virt_dma_chan            vchan;
  67         void                            *xdev_hdl;
  68         u32                             base;
  69         struct dma_pool                 *desc_pool;
  70         bool                            busy;
  71         enum dma_transfer_direction     dir;
  72         struct dma_slave_config         cfg;
  73         u32                             irq;
  74         struct completion               last_interrupt;
  75         bool                            stop_requested;
  76 };
  77
  78 /**
  79  * struct xdma_desc - DMA desc structure
  80  * @vdesc: Virtual DMA descriptor
  81  * @chan: DMA channel pointer
  82  * @dir: Transferring direction of the request
  83  * @desc_blocks: Hardware descriptor blocks
  84  * @dblk_num: Number of hardware descriptor blocks
  85  * @desc_num: Number of hardware descriptors
  86  * @completed_desc_num: Completed hardware descriptors
  87  * @cyclic: Cyclic transfer vs. scatter-gather
  88  * @interleaved_dma: Interleaved DMA transfer
  89  * @periods: Number of periods in the cyclic transfer
  90  * @period_size: Size of a period in bytes in cyclic transfers
  91  * @frames_left: Number of frames left in interleaved DMA transfer
  92  * @error: tx error flag
  93  */
  94 struct xdma_desc {
  95         struct virt_dma_desc            vdesc;
  96         struct xdma_chan                *chan;
  97         enum dma_transfer_direction     dir;
  98         struct xdma_desc_block          *desc_blocks;
  99         u32                             dblk_num;
 100         u32                             desc_num;
 101         u32                             completed_desc_num;
 102         bool                            cyclic;
 103         bool                            interleaved_dma;
 104         u32                             periods;
 105         u32                             period_size;
 106         u32                             frames_left;
 107         bool                            error;
 108 };
 109
 110 #define XDMA_DEV_STATUS_REG_DMA         BIT(0)
 111 #define XDMA_DEV_STATUS_INIT_MSIX       BIT(1)
 112
 113 /**
 114  * struct xdma_device - DMA device structure
 115  * @pdev: Platform device pointer
 116  * @dma_dev: DMA device structure
 117  * @rmap: MMIO regmap for DMA registers
 118  * @h2c_chans: Host to Card channels
 119  * @c2h_chans: Card to Host channels
 120  * @h2c_chan_num: Number of H2C channels
 121  * @c2h_chan_num: Number of C2H channels
 122  * @irq_start: Start IRQ assigned to device
 123  * @irq_num: Number of IRQ assigned to device
 124  * @status: Initialization status
 125  */
 126 struct xdma_device {
 127         struct platform_device  *pdev;
 128         struct dma_device       dma_dev;
 129         struct regmap           *rmap;
 130         struct xdma_chan        *h2c_chans;
 131         struct xdma_chan        *c2h_chans;
 132         u32                     h2c_chan_num;
 133         u32                     c2h_chan_num;
 134         u32                     irq_start;
 135         u32                     irq_num;
 136         u32                     status;
 137 };
 138
 139 #define xdma_err(xdev, fmt, args...)                                    \
 140         dev_err(&(xdev)->pdev->dev, fmt, ##args)
 141 #define XDMA_CHAN_NUM(_xd) ({                                           \
 142         typeof(_xd) (xd) = (_xd);                                       \
 143         ((xd)->h2c_chan_num + (xd)->c2h_chan_num); })
 144
 145 /* Get the last desc in a desc block */
 146 static inline void *xdma_blk_last_desc(struct xdma_desc_block *block)
 147 {
 148         return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * XDMA_DESC_SIZE;
 149 }
 150
 151 /**
 152  * xdma_link_sg_desc_blocks - Link SG descriptor blocks for DMA transfer
 153  * @sw_desc: Tx descriptor pointer
 154  */
 155 static void xdma_link_sg_desc_blocks(struct xdma_desc *sw_desc)
 156 {
 157         struct xdma_desc_block *block;
 158         u32 last_blk_desc, desc_control;
 159         struct xdma_hw_desc *desc;
 160         int i;
 161
 162         desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0);
 163         for (i = 1; i < sw_desc->dblk_num; i++) {
 164                 block = &sw_desc->desc_blocks[i - 1];
 165                 desc = xdma_blk_last_desc(block);
 166
 167                 if (!(i & XDMA_DESC_BLOCK_MASK)) {
 168                         desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
 169                         continue;
 170                 }
 171                 desc->control = cpu_to_le32(desc_control);
 172                 desc->next_desc = cpu_to_le64(block[1].dma_addr);
 173         }
 174
 175         /* update the last block */
 176         last_blk_desc = (sw_desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
 177         if (((sw_desc->dblk_num - 1) & XDMA_DESC_BLOCK_MASK) > 0) {
 178                 block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2];
 179                 desc = xdma_blk_last_desc(block);
 180                 desc_control = XDMA_DESC_CONTROL(last_blk_desc + 1, 0);
 181                 desc->control = cpu_to_le32(desc_control);
 182         }
 183
 184         block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1];
 185         desc = block->virt_addr + last_blk_desc * XDMA_DESC_SIZE;
 186         desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST);
 187 }
 188
 189 /**
 190  * xdma_link_cyclic_desc_blocks - Link cyclic descriptor blocks for DMA transfer
 191  * @sw_desc: Tx descriptor pointer
 192  */
 193 static void xdma_link_cyclic_desc_blocks(struct xdma_desc *sw_desc)
 194 {
 195         struct xdma_desc_block *block;
 196         struct xdma_hw_desc *desc;
 197         int i;
 198
 199         block = sw_desc->desc_blocks;
 200         for (i = 0; i < sw_desc->desc_num - 1; i++) {
 201                 desc = block->virt_addr + i * XDMA_DESC_SIZE;
 202                 desc->next_desc = cpu_to_le64(block->dma_addr + ((i + 1) * XDMA_DESC_SIZE));
 203         }
 204         desc = block->virt_addr + i * XDMA_DESC_SIZE;
 205         desc->next_desc = cpu_to_le64(block->dma_addr);
 206 }
 207
 208 static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan)
 209 {
 210         return container_of(chan, struct xdma_chan, vchan.chan);
 211 }
 212
 213 static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc *vdesc)
 214 {
 215         return container_of(vdesc, struct xdma_desc, vdesc);
 216 }
 217
 218 /**
 219  * xdma_channel_init - Initialize DMA channel registers
 220  * @chan: DMA channel pointer
 221  */
 222 static int xdma_channel_init(struct xdma_chan *chan)
 223 {
 224         struct xdma_device *xdev = chan->xdev_hdl;
 225         int ret;
 226
 227         ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_CONTROL_W1C,
 228                            CHAN_CTRL_NON_INCR_ADDR);
 229         if (ret)
 230                 return ret;
 231
 232         ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_INTR_ENABLE,
 233                            CHAN_IM_ALL);
 234         if (ret)
 235                 return ret;
 236
 237         return 0;
 238 }
 239
 240 /**
 241  * xdma_free_desc - Free descriptor
 242  * @vdesc: Virtual DMA descriptor
 243  */
 244 static void xdma_free_desc(struct virt_dma_desc *vdesc)
 245 {
 246         struct xdma_desc *sw_desc;
 247         int i;
 248
 249         sw_desc = to_xdma_desc(vdesc);
 250         for (i = 0; i < sw_desc->dblk_num; i++) {
 251                 if (!sw_desc->desc_blocks[i].virt_addr)
 252                         break;
 253                 dma_pool_free(sw_desc->chan->desc_pool,
 254                               sw_desc->desc_blocks[i].virt_addr,
 255                               sw_desc->desc_blocks[i].dma_addr);
 256         }
 257         kfree(sw_desc->desc_blocks);
 258         kfree(sw_desc);
 259 }
 260
 261 /**
 262  * xdma_alloc_desc - Allocate descriptor
 263  * @chan: DMA channel pointer
 264  * @desc_num: Number of hardware descriptors
 265  * @cyclic: Whether this is a cyclic transfer
 266  */
 267 static struct xdma_desc *
 268 xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num, bool cyclic)
 269 {
 270         struct xdma_desc *sw_desc;
 271         struct xdma_hw_desc *desc;
 272         dma_addr_t dma_addr;
 273         u32 dblk_num;
 274         u32 control;
 275         void *addr;
 276         int i, j;
 277
 278         sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT);
 279         if (!sw_desc)
 280                 return NULL;
 281
 282         sw_desc->chan = chan;
 283         sw_desc->desc_num = desc_num;
 284         sw_desc->cyclic = cyclic;
 285         sw_desc->error = false;
 286         dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT);
 287         sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks),
 288                                        GFP_NOWAIT);
 289         if (!sw_desc->desc_blocks)
 290                 goto failed;
 291
 292         if (cyclic)
 293                 control = XDMA_DESC_CONTROL_CYCLIC;
 294         else
 295                 control = XDMA_DESC_CONTROL(1, 0);
 296
 297         sw_desc->dblk_num = dblk_num;
 298         for (i = 0; i < sw_desc->dblk_num; i++) {
 299                 addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr);
 300                 if (!addr)
 301                         goto failed;
 302
 303                 sw_desc->desc_blocks[i].virt_addr = addr;
 304                 sw_desc->desc_blocks[i].dma_addr = dma_addr;
 305                 for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++)
 306                         desc[j].control = cpu_to_le32(control);
 307         }
 308
 309         if (cyclic)
 310                 xdma_link_cyclic_desc_blocks(sw_desc);
 311         else
 312                 xdma_link_sg_desc_blocks(sw_desc);
 313
 314         return sw_desc;
 315
 316 failed:
 317         xdma_free_desc(&sw_desc->vdesc);
 318         return NULL;
 319 }
 320
 321 /**
 322  * xdma_xfer_start - Start DMA transfer
 323  * @xchan: DMA channel pointer
 324  */
 325 static int xdma_xfer_start(struct xdma_chan *xchan)
 326 {
 327         struct virt_dma_desc *vd = vchan_next_desc(&xchan->vchan);
 328         struct xdma_device *xdev = xchan->xdev_hdl;
 329         struct xdma_desc_block *block;
 330         u32 val, completed_blocks;
 331         struct xdma_desc *desc;
 332         int ret;
 333
 334         /*
 335          * check if there is not any submitted descriptor or channel is busy.
 336          * vchan lock should be held where this function is called.
 337          */
 338         if (!vd || xchan->busy)
 339                 return -EINVAL;
 340
 341         /* clear run stop bit to get ready for transfer */
 342         ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C,
 343                            CHAN_CTRL_RUN_STOP);
 344         if (ret)
 345                 return ret;
 346
 347         desc = to_xdma_desc(vd);
 348         if (desc->dir != xchan->dir) {
 349                 xdma_err(xdev, "incorrect request direction");
 350                 return -EINVAL;
 351         }
 352
 353         /* set DMA engine to the first descriptor block */
 354         completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT;
 355         block = &desc->desc_blocks[completed_blocks];
 356         val = lower_32_bits(block->dma_addr);
 357         ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_LO, val);
 358         if (ret)
 359                 return ret;
 360
 361         val = upper_32_bits(block->dma_addr);
 362         ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_HI, val);
 363         if (ret)
 364                 return ret;
 365
 366         if (completed_blocks + 1 == desc->dblk_num)
 367                 val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK;
 368         else
 369                 val = XDMA_DESC_ADJACENT - 1;
 370         ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_ADJ, val);
 371         if (ret)
 372                 return ret;
 373
 374         /* kick off DMA transfer */
 375         ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL,
 376                            CHAN_CTRL_START);
 377         if (ret)
 378                 return ret;
 379
 380         xchan->busy = true;
 381         xchan->stop_requested = false;
 382         reinit_completion(&xchan->last_interrupt);
 383
 384         return 0;
 385 }
 386
 387 /**
 388  * xdma_xfer_stop - Stop DMA transfer
 389  * @xchan: DMA channel pointer
 390  */
 391 static int xdma_xfer_stop(struct xdma_chan *xchan)
 392 {
 393         int ret;
 394         struct xdma_device *xdev = xchan->xdev_hdl;
 395
 396         /* clear run stop bit to prevent any further auto-triggering */
 397         ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C,
 398                            CHAN_CTRL_RUN_STOP);
 399         if (ret)
 400                 return ret;
 401         return ret;
 402 }
 403
 404 /**
 405  * xdma_alloc_channels - Detect and allocate DMA channels
 406  * @xdev: DMA device pointer
 407  * @dir: Channel direction
 408  */
 409 static int xdma_alloc_channels(struct xdma_device *xdev,
 410                                enum dma_transfer_direction dir)
 411 {
 412         struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev);
 413         struct xdma_chan **chans, *xchan;
 414         u32 base, identifier, target;
 415         u32 *chan_num;
 416         int i, j, ret;
 417
 418         if (dir == DMA_MEM_TO_DEV) {
 419                 base = XDMA_CHAN_H2C_OFFSET;
 420                 target = XDMA_CHAN_H2C_TARGET;
 421                 chans = &xdev->h2c_chans;
 422                 chan_num = &xdev->h2c_chan_num;
 423         } else if (dir == DMA_DEV_TO_MEM) {
 424                 base = XDMA_CHAN_C2H_OFFSET;
 425                 target = XDMA_CHAN_C2H_TARGET;
 426                 chans = &xdev->c2h_chans;
 427                 chan_num = &xdev->c2h_chan_num;
 428         } else {
 429                 xdma_err(xdev, "invalid direction specified");
 430                 return -EINVAL;
 431         }
 432
 433         /* detect number of available DMA channels */
 434         for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) {
 435                 ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE,
 436                                   &identifier);
 437                 if (ret)
 438                         return ret;
 439
 440                 /* check if it is available DMA channel */
 441                 if (XDMA_CHAN_CHECK_TARGET(identifier, target))
 442                         (*chan_num)++;
 443         }
 444
 445         if (!*chan_num) {
 446                 xdma_err(xdev, "does not probe any channel");
 447                 return -EINVAL;
 448         }
 449
 450         *chans = devm_kcalloc(&xdev->pdev->dev, *chan_num, sizeof(**chans),
 451                               GFP_KERNEL);
 452         if (!*chans)
 453                 return -ENOMEM;
 454
 455         for (i = 0, j = 0; i < pdata->max_dma_channels; i++) {
 456                 ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE,
 457                                   &identifier);
 458                 if (ret)
 459                         return ret;
 460
 461                 if (!XDMA_CHAN_CHECK_TARGET(identifier, target))
 462                         continue;
 463
 464                 if (j == *chan_num) {
 465                         xdma_err(xdev, "invalid channel number");
 466                         return -EIO;
 467                 }
 468
 469                 /* init channel structure and hardware */
 470                 xchan = &(*chans)[j];
 471                 xchan->xdev_hdl = xdev;
 472                 xchan->base = base + i * XDMA_CHAN_STRIDE;
 473                 xchan->dir = dir;
 474                 xchan->stop_requested = false;
 475                 init_completion(&xchan->last_interrupt);
 476
 477                 ret = xdma_channel_init(xchan);
 478                 if (ret)
 479                         return ret;
 480                 xchan->vchan.desc_free = xdma_free_desc;
 481                 vchan_init(&xchan->vchan, &xdev->dma_dev);
 482
 483                 j++;
 484         }
 485
 486         dev_info(&xdev->pdev->dev, "configured %d %s channels", j,
 487                  (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H");
 488
 489         return 0;
 490 }
 491
 492 /**
 493  * xdma_issue_pending - Issue pending transactions
 494  * @chan: DMA channel pointer
 495  */
 496 static void xdma_issue_pending(struct dma_chan *chan)
 497 {
 498         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 499         unsigned long flags;
 500
 501         spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
 502         if (vchan_issue_pending(&xdma_chan->vchan))
 503                 xdma_xfer_start(xdma_chan);
 504         spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
 505 }
 506
 507 /**
 508  * xdma_terminate_all - Terminate all transactions
 509  * @chan: DMA channel pointer
 510  */
 511 static int xdma_terminate_all(struct dma_chan *chan)
 512 {
 513         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 514         struct virt_dma_desc *vd;
 515         unsigned long flags;
 516         LIST_HEAD(head);
 517
 518         xdma_xfer_stop(xdma_chan);
 519
 520         spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
 521
 522         xdma_chan->busy = false;
 523         xdma_chan->stop_requested = true;
 524         vd = vchan_next_desc(&xdma_chan->vchan);
 525         if (vd) {
 526                 list_del(&vd->node);
 527                 dma_cookie_complete(&vd->tx);
 528                 vchan_terminate_vdesc(vd);
 529         }
 530         vchan_get_all_descriptors(&xdma_chan->vchan, &head);
 531         list_splice_tail(&head, &xdma_chan->vchan.desc_terminated);
 532
 533         spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
 534
 535         return 0;
 536 }
 537
 538 /**
 539  * xdma_synchronize - Synchronize terminated transactions
 540  * @chan: DMA channel pointer
 541  */
 542 static void xdma_synchronize(struct dma_chan *chan)
 543 {
 544         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 545         struct xdma_device *xdev = xdma_chan->xdev_hdl;
 546         int st = 0;
 547
 548         /* If the engine continues running, wait for the last interrupt */
 549         regmap_read(xdev->rmap, xdma_chan->base + XDMA_CHAN_STATUS, &st);
 550         if (st & XDMA_CHAN_STATUS_BUSY)
 551                 wait_for_completion_timeout(&xdma_chan->last_interrupt, msecs_to_jiffies(1000));
 552
 553         vchan_synchronize(&xdma_chan->vchan);
 554 }
 555
 556 /**
 557  * xdma_fill_descs() - Fill hardware descriptors for one contiguous memory chunk.
 558  *                     More than one descriptor will be used if the size is bigger
 559  *                     than XDMA_DESC_BLEN_MAX.
 560  * @sw_desc: Descriptor container
 561  * @src_addr: First value for the ->src_addr field
 562  * @dst_addr: First value for the ->dst_addr field
 563  * @size: Size of the contiguous memory block
 564  * @filled_descs_num: Index of the first descriptor to take care of in @sw_desc
 565  */
 566 static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr,
 567                                   u64 dst_addr, u32 size, u32 filled_descs_num)
 568 {
 569         u32 left = size, len, desc_num = filled_descs_num;
 570         struct xdma_desc_block *dblk;
 571         struct xdma_hw_desc *desc;
 572
 573         dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT);
 574         desc = dblk->virt_addr;
 575         desc += desc_num & XDMA_DESC_ADJACENT_MASK;
 576         do {
 577                 len = min_t(u32, left, XDMA_DESC_BLEN_MAX);
 578                 /* set hardware descriptor */
 579                 desc->bytes = cpu_to_le32(len);
 580                 desc->src_addr = cpu_to_le64(src_addr);
 581                 desc->dst_addr = cpu_to_le64(dst_addr);
 582                 if (!(++desc_num & XDMA_DESC_ADJACENT_MASK))
 583                         desc = (++dblk)->virt_addr;
 584                 else
 585                         desc++;
 586
 587                 src_addr += len;
 588                 dst_addr += len;
 589                 left -= len;
 590         } while (left);
 591
 592         return desc_num - filled_descs_num;
 593 }
 594
 595 /**
 596  * xdma_prep_device_sg - prepare a descriptor for a DMA transaction
 597  * @chan: DMA channel pointer
 598  * @sgl: Transfer scatter gather list
 599  * @sg_len: Length of scatter gather list
 600  * @dir: Transfer direction
 601  * @flags: transfer ack flags
 602  * @context: APP words of the descriptor
 603  */
 604 static struct dma_async_tx_descriptor *
 605 xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
 606                     unsigned int sg_len, enum dma_transfer_direction dir,
 607                     unsigned long flags, void *context)
 608 {
 609         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 610         struct dma_async_tx_descriptor *tx_desc;
 611         struct xdma_desc *sw_desc;
 612         u32 desc_num = 0, i;
 613         u64 addr, dev_addr, *src, *dst;
 614         struct scatterlist *sg;
 615
 616         for_each_sg(sgl, sg, sg_len, i)
 617                 desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX);
 618
 619         sw_desc = xdma_alloc_desc(xdma_chan, desc_num, false);
 620         if (!sw_desc)
 621                 return NULL;
 622         sw_desc->dir = dir;
 623         sw_desc->cyclic = false;
 624         sw_desc->interleaved_dma = false;
 625
 626         if (dir == DMA_MEM_TO_DEV) {
 627                 dev_addr = xdma_chan->cfg.dst_addr;
 628                 src = &addr;
 629                 dst = &dev_addr;
 630         } else {
 631                 dev_addr = xdma_chan->cfg.src_addr;
 632                 src = &dev_addr;
 633                 dst = &addr;
 634         }
 635
 636         desc_num = 0;
 637         for_each_sg(sgl, sg, sg_len, i) {
 638                 addr = sg_dma_address(sg);
 639                 desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num);
 640                 dev_addr += sg_dma_len(sg);
 641         }
 642
 643         tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
 644         if (!tx_desc)
 645                 goto failed;
 646
 647         return tx_desc;
 648
 649 failed:
 650         xdma_free_desc(&sw_desc->vdesc);
 651
 652         return NULL;
 653 }
 654
 655 /**
 656  * xdma_prep_dma_cyclic - prepare for cyclic DMA transactions
 657  * @chan: DMA channel pointer
 658  * @address: Device DMA address to access
 659  * @size: Total length to transfer
 660  * @period_size: Period size to use for each transfer
 661  * @dir: Transfer direction
 662  * @flags: Transfer ack flags
 663  */
 664 static struct dma_async_tx_descriptor *
 665 xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address,
 666                      size_t size, size_t period_size,
 667                      enum dma_transfer_direction dir,
 668                      unsigned long flags)
 669 {
 670         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 671         struct xdma_device *xdev = xdma_chan->xdev_hdl;
 672         unsigned int periods = size / period_size;
 673         struct dma_async_tx_descriptor *tx_desc;
 674         struct xdma_desc *sw_desc;
 675         u64 addr, dev_addr, *src, *dst;
 676         u32 desc_num;
 677         unsigned int i;
 678
 679         /*
 680          * Simplify the whole logic by preventing an abnormally high number of
 681          * periods and periods size.
 682          */
 683         if (period_size > XDMA_DESC_BLEN_MAX) {
 684                 xdma_err(xdev, "period size limited to %lu bytes\n", XDMA_DESC_BLEN_MAX);
 685                 return NULL;
 686         }
 687
 688         if (periods > XDMA_DESC_ADJACENT) {
 689                 xdma_err(xdev, "number of periods limited to %u\n", XDMA_DESC_ADJACENT);
 690                 return NULL;
 691         }
 692
 693         sw_desc = xdma_alloc_desc(xdma_chan, periods, true);
 694         if (!sw_desc)
 695                 return NULL;
 696
 697         sw_desc->periods = periods;
 698         sw_desc->period_size = period_size;
 699         sw_desc->dir = dir;
 700         sw_desc->interleaved_dma = false;
 701
 702         addr = address;
 703         if (dir == DMA_MEM_TO_DEV) {
 704                 dev_addr = xdma_chan->cfg.dst_addr;
 705                 src = &addr;
 706                 dst = &dev_addr;
 707         } else {
 708                 dev_addr = xdma_chan->cfg.src_addr;
 709                 src = &dev_addr;
 710                 dst = &addr;
 711         }
 712
 713         desc_num = 0;
 714         for (i = 0; i < periods; i++) {
 715                 desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num);
 716                 addr += period_size;
 717         }
 718
 719         tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags);
 720         if (!tx_desc)
 721                 goto failed;
 722
 723         return tx_desc;
 724
 725 failed:
 726         xdma_free_desc(&sw_desc->vdesc);
 727
 728         return NULL;
 729 }
 730
 731 /**
 732  * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers
 733  * @chan: DMA channel
 734  * @xt: DMA transfer template
 735  * @flags: tx flags
 736  */
 737 static struct dma_async_tx_descriptor *
 738 xdma_prep_interleaved_dma(struct dma_chan *chan,
 739                           struct dma_interleaved_template *xt,
 740                           unsigned long flags)
 741 {
 742         int i;
 743         u32 desc_num = 0, period_size = 0;
 744         struct dma_async_tx_descriptor *tx_desc;
 745         struct xdma_chan *xchan = to_xdma_chan(chan);
 746         struct xdma_desc *sw_desc;
 747         u64 src_addr, dst_addr;
 748
 749         for (i = 0; i < xt->frame_size; ++i)
 750                 desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX);
 751
 752         sw_desc = xdma_alloc_desc(xchan, desc_num, false);
 753         if (!sw_desc)
 754                 return NULL;
 755         sw_desc->dir = xt->dir;
 756         sw_desc->interleaved_dma = true;
 757         sw_desc->cyclic = flags & DMA_PREP_REPEAT;
 758         sw_desc->frames_left = xt->numf;
 759         sw_desc->periods = xt->numf;
 760
 761         desc_num = 0;
 762         src_addr = xt->src_start;
 763         dst_addr = xt->dst_start;
 764         for (i = 0; i < xt->frame_size; ++i) {
 765                 desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num);
 766                 src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + (xt->src_inc ?
 767                                                               xt->sgl[i].size : 0);
 768                 dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + (xt->dst_inc ?
 769                                                               xt->sgl[i].size : 0);
 770                 period_size += xt->sgl[i].size;
 771         }
 772         sw_desc->period_size = period_size;
 773
 774         tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags);
 775         if (tx_desc)
 776                 return tx_desc;
 777
 778         xdma_free_desc(&sw_desc->vdesc);
 779         return NULL;
 780 }
 781
 782 /**
 783  * xdma_device_config - Configure the DMA channel
 784  * @chan: DMA channel
 785  * @cfg: channel configuration
 786  */
 787 static int xdma_device_config(struct dma_chan *chan,
 788                               struct dma_slave_config *cfg)
 789 {
 790         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 791
 792         memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg));
 793
 794         return 0;
 795 }
 796
 797 /**
 798  * xdma_free_chan_resources - Free channel resources
 799  * @chan: DMA channel
 800  */
 801 static void xdma_free_chan_resources(struct dma_chan *chan)
 802 {
 803         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 804
 805         vchan_free_chan_resources(&xdma_chan->vchan);
 806         dma_pool_destroy(xdma_chan->desc_pool);
 807         xdma_chan->desc_pool = NULL;
 808 }
 809
 810 /**
 811  * xdma_alloc_chan_resources - Allocate channel resources
 812  * @chan: DMA channel
 813  */
 814 static int xdma_alloc_chan_resources(struct dma_chan *chan)
 815 {
 816         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 817         struct xdma_device *xdev = xdma_chan->xdev_hdl;
 818         struct device *dev = xdev->dma_dev.dev;
 819
 820         while (dev && !dev_is_pci(dev))
 821                 dev = dev->parent;
 822         if (!dev) {
 823                 xdma_err(xdev, "unable to find pci device");
 824                 return -EINVAL;
 825         }
 826
 827         xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE,
 828                                                XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY);
 829         if (!xdma_chan->desc_pool) {
 830                 xdma_err(xdev, "unable to allocate descriptor pool");
 831                 return -ENOMEM;
 832         }
 833
 834         return 0;
 835 }
 836
 837 static enum dma_status xdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 838                                       struct dma_tx_state *state)
 839 {
 840         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
 841         struct xdma_desc *desc = NULL;
 842         struct virt_dma_desc *vd;
 843         enum dma_status ret;
 844         unsigned long flags;
 845         unsigned int period_idx;
 846         u32 residue = 0;
 847
 848         ret = dma_cookie_status(chan, cookie, state);
 849         if (ret == DMA_COMPLETE)
 850                 return ret;
 851
 852         spin_lock_irqsave(&xdma_chan->vchan.lock, flags);
 853
 854         vd = vchan_find_desc(&xdma_chan->vchan, cookie);
 855         if (!vd)
 856                 goto out;
 857
 858         desc = to_xdma_desc(vd);
 859         if (desc->error) {
 860                 ret = DMA_ERROR;
 861         } else if (desc->cyclic) {
 862                 period_idx = desc->completed_desc_num % desc->periods;
 863                 residue = (desc->periods - period_idx) * desc->period_size;
 864                 dma_set_residue(state, residue);
 865         }
 866 out:
 867         spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags);
 868
 869         return ret;
 870 }
 871
 872 /**
 873  * xdma_channel_isr - XDMA channel interrupt handler
 874  * @irq: IRQ number
 875  * @dev_id: Pointer to the DMA channel structure
 876  */
 877 static irqreturn_t xdma_channel_isr(int irq, void *dev_id)
 878 {
 879         struct xdma_chan *xchan = dev_id;
 880         u32 complete_desc_num = 0;
 881         struct xdma_device *xdev = xchan->xdev_hdl;
 882         struct virt_dma_desc *vd, *next_vd;
 883         struct xdma_desc *desc;
 884         int ret;
 885         u32 st;
 886         bool repeat_tx;
 887
 888         if (xchan->stop_requested)
 889                 complete(&xchan->last_interrupt);
 890
 891         spin_lock(&xchan->vchan.lock);
 892
 893         /* get submitted request */
 894         vd = vchan_next_desc(&xchan->vchan);
 895         if (!vd)
 896                 goto out;
 897
 898         /* Clear-on-read the status register */
 899         ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st);
 900         if (ret)
 901                 goto out;
 902
 903         desc = to_xdma_desc(vd);
 904
 905         st &= XDMA_CHAN_STATUS_MASK;
 906         if ((st & XDMA_CHAN_ERROR_MASK) ||
 907             !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) {
 908                 desc->error = true;
 909                 xdma_err(xdev, "channel error, status register value: 0x%x", st);
 910                 goto out;
 911         }
 912
 913         ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC,
 914                           &complete_desc_num);
 915         if (ret)
 916                 goto out;
 917
 918         if (desc->interleaved_dma) {
 919                 xchan->busy = false;
 920                 desc->completed_desc_num += complete_desc_num;
 921                 if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) {
 922                         xdma_xfer_start(xchan);
 923                         goto out;
 924                 }
 925
 926                 /* last desc of any frame */
 927                 desc->frames_left--;
 928                 if (desc->frames_left)
 929                         goto out;
 930
 931                 /* last desc of the last frame  */
 932                 repeat_tx = vd->tx.flags & DMA_PREP_REPEAT;
 933                 next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node);
 934                 if (next_vd)
 935                         repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT);
 936                 if (repeat_tx) {
 937                         desc->frames_left = desc->periods;
 938                         desc->completed_desc_num = 0;
 939                         vchan_cyclic_callback(vd);
 940                 } else {
 941                         list_del(&vd->node);
 942                         vchan_cookie_complete(vd);
 943                 }
 944                 /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */
 945                 xdma_xfer_start(xchan);
 946         } else if (!desc->cyclic) {
 947                 xchan->busy = false;
 948                 desc->completed_desc_num += complete_desc_num;
 949
 950                 /* if all data blocks are transferred, remove and complete the request */
 951                 if (desc->completed_desc_num == desc->desc_num) {
 952                         list_del(&vd->node);
 953                         vchan_cookie_complete(vd);
 954                         goto out;
 955                 }
 956
 957                 if (desc->completed_desc_num > desc->desc_num ||
 958                     complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT)
 959                         goto out;
 960
 961                 /* transfer the rest of data */
 962                 xdma_xfer_start(xchan);
 963         } else {
 964                 desc->completed_desc_num = complete_desc_num;
 965                 vchan_cyclic_callback(vd);
 966         }
 967
 968 out:
 969         spin_unlock(&xchan->vchan.lock);
 970         return IRQ_HANDLED;
 971 }
 972
 973 /**
 974  * xdma_irq_fini - Uninitialize IRQ
 975  * @xdev: DMA device pointer
 976  */
 977 static void xdma_irq_fini(struct xdma_device *xdev)
 978 {
 979         int i;
 980
 981         /* disable interrupt */
 982         regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1C, ~0);
 983
 984         /* free irq handler */
 985         for (i = 0; i < xdev->h2c_chan_num; i++)
 986                 free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
 987
 988         for (i = 0; i < xdev->c2h_chan_num; i++)
 989                 free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]);
 990 }
 991
 992 /**
 993  * xdma_set_vector_reg - configure hardware IRQ registers
 994  * @xdev: DMA device pointer
 995  * @vec_tbl_start: Start of IRQ registers
 996  * @irq_start: Start of IRQ
 997  * @irq_num: Number of IRQ
 998  */
 999 static int xdma_set_vector_reg(struct xdma_device *xdev, u32 vec_tbl_start,
1000                                u32 irq_start, u32 irq_num)
1001 {
1002         u32 shift, i, val = 0;
1003         int ret;
1004
1005         /* Each IRQ register is 32 bit and contains 4 IRQs */
1006         while (irq_num > 0) {
1007                 for (i = 0; i < 4; i++) {
1008                         shift = XDMA_IRQ_VEC_SHIFT * i;
1009                         val |= irq_start << shift;
1010                         irq_start++;
1011                         irq_num--;
1012                         if (!irq_num)
1013                                 break;
1014                 }
1015
1016                 /* write IRQ register */
1017                 ret = regmap_write(xdev->rmap, vec_tbl_start, val);
1018                 if (ret)
1019                         return ret;
1020                 vec_tbl_start += sizeof(u32);
1021                 val = 0;
1022         }
1023
1024         return 0;
1025 }
1026
1027 /**
1028  * xdma_irq_init - initialize IRQs
1029  * @xdev: DMA device pointer
1030  */
1031 static int xdma_irq_init(struct xdma_device *xdev)
1032 {
1033         u32 irq = xdev->irq_start;
1034         u32 user_irq_start;
1035         int i, j, ret;
1036
1037         /* return failure if there are not enough IRQs */
1038         if (xdev->irq_num < XDMA_CHAN_NUM(xdev)) {
1039                 xdma_err(xdev, "not enough irq");
1040                 return -EINVAL;
1041         }
1042
1043         /* setup H2C interrupt handler */
1044         for (i = 0; i < xdev->h2c_chan_num; i++) {
1045                 ret = request_irq(irq, xdma_channel_isr, 0,
1046                                   "xdma-h2c-channel", &xdev->h2c_chans[i]);
1047                 if (ret) {
1048                         xdma_err(xdev, "H2C channel%d request irq%d failed: %d",
1049                                  i, irq, ret);
1050                         goto failed_init_h2c;
1051                 }
1052                 xdev->h2c_chans[i].irq = irq;
1053                 irq++;
1054         }
1055
1056         /* setup C2H interrupt handler */
1057         for (j = 0; j < xdev->c2h_chan_num; j++) {
1058                 ret = request_irq(irq, xdma_channel_isr, 0,
1059                                   "xdma-c2h-channel", &xdev->c2h_chans[j]);
1060                 if (ret) {
1061                         xdma_err(xdev, "C2H channel%d request irq%d failed: %d",
1062                                  j, irq, ret);
1063                         goto failed_init_c2h;
1064                 }
1065                 xdev->c2h_chans[j].irq = irq;
1066                 irq++;
1067         }
1068
1069         /* config hardware IRQ registers */
1070         ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0,
1071                                   XDMA_CHAN_NUM(xdev));
1072         if (ret) {
1073                 xdma_err(xdev, "failed to set channel vectors: %d", ret);
1074                 goto failed_init_c2h;
1075         }
1076
1077         /* config user IRQ registers if needed */
1078         user_irq_start = XDMA_CHAN_NUM(xdev);
1079         if (xdev->irq_num > user_irq_start) {
1080                 ret = xdma_set_vector_reg(xdev, XDMA_IRQ_USER_VEC_NUM,
1081                                           user_irq_start,
1082                                           xdev->irq_num - user_irq_start);
1083                 if (ret) {
1084                         xdma_err(xdev, "failed to set user vectors: %d", ret);
1085                         goto failed_init_c2h;
1086                 }
1087         }
1088
1089         /* enable interrupt */
1090         ret = regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1S, ~0);
1091         if (ret)
1092                 goto failed_init_c2h;
1093
1094         return 0;
1095
1096 failed_init_c2h:
1097         while (j--)
1098                 free_irq(xdev->c2h_chans[j].irq, &xdev->c2h_chans[j]);
1099 failed_init_h2c:
1100         while (i--)
1101                 free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]);
1102
1103         return ret;
1104 }
1105
1106 static bool xdma_filter_fn(struct dma_chan *chan, void *param)
1107 {
1108         struct xdma_chan *xdma_chan = to_xdma_chan(chan);
1109         struct xdma_chan_info *chan_info = param;
1110
1111         return chan_info->dir == xdma_chan->dir;
1112 }
1113
1114 /**
1115  * xdma_disable_user_irq - Disable user interrupt
1116  * @pdev: Pointer to the platform_device structure
1117  * @irq_num: System IRQ number
1118  */
1119 void xdma_disable_user_irq(struct platform_device *pdev, u32 irq_num)
1120 {
1121         struct xdma_device *xdev = platform_get_drvdata(pdev);
1122         u32 index;
1123
1124         index = irq_num - xdev->irq_start;
1125         if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) {
1126                 xdma_err(xdev, "invalid user irq number");
1127                 return;
1128         }
1129         index -= XDMA_CHAN_NUM(xdev);
1130
1131         regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1C, 1 << index);
1132 }
1133 EXPORT_SYMBOL(xdma_disable_user_irq);
1134
1135 /**
1136  * xdma_enable_user_irq - Enable user logic interrupt
1137  * @pdev: Pointer to the platform_device structure
1138  * @irq_num: System IRQ number
1139  */
1140 int xdma_enable_user_irq(struct platform_device *pdev, u32 irq_num)
1141 {
1142         struct xdma_device *xdev = platform_get_drvdata(pdev);
1143         u32 index;
1144         int ret;
1145
1146         index = irq_num - xdev->irq_start;
1147         if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) {
1148                 xdma_err(xdev, "invalid user irq number");
1149                 return -EINVAL;
1150         }
1151         index -= XDMA_CHAN_NUM(xdev);
1152
1153         ret = regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1S, 1 << index);
1154         if (ret)
1155                 return ret;
1156
1157         return 0;
1158 }
1159 EXPORT_SYMBOL(xdma_enable_user_irq);
1160
1161 /**
1162  * xdma_get_user_irq - Get system IRQ number
1163  * @pdev: Pointer to the platform_device structure
1164  * @user_irq_index: User logic IRQ wire index
1165  *
1166  * Return: The system IRQ number allocated for the given wire index.
1167  */
1168 int xdma_get_user_irq(struct platform_device *pdev, u32 user_irq_index)
1169 {
1170         struct xdma_device *xdev = platform_get_drvdata(pdev);
1171
1172         if (XDMA_CHAN_NUM(xdev) + user_irq_index >= xdev->irq_num) {
1173                 xdma_err(xdev, "invalid user irq index");
1174                 return -EINVAL;
1175         }
1176
1177         return xdev->irq_start + XDMA_CHAN_NUM(xdev) + user_irq_index;
1178 }
1179 EXPORT_SYMBOL(xdma_get_user_irq);
1180
1181 /**
1182  * xdma_remove - Driver remove function
1183  * @pdev: Pointer to the platform_device structure
1184  */
1185 static void xdma_remove(struct platform_device *pdev)
1186 {
1187         struct xdma_device *xdev = platform_get_drvdata(pdev);
1188
1189         if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX)
1190                 xdma_irq_fini(xdev);
1191
1192         if (xdev->status & XDMA_DEV_STATUS_REG_DMA)
1193                 dma_async_device_unregister(&xdev->dma_dev);
1194 }
1195
1196 /**
1197  * xdma_probe - Driver probe function
1198  * @pdev: Pointer to the platform_device structure
1199  */
1200 static int xdma_probe(struct platform_device *pdev)
1201 {
1202         struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev);
1203         struct xdma_device *xdev;
1204         void __iomem *reg_base;
1205         struct resource *res;
1206         int ret = -ENODEV;
1207
1208         if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) {
1209                 dev_err(&pdev->dev, "invalid max dma channels %d",
1210                         pdata->max_dma_channels);
1211                 return -EINVAL;
1212         }
1213
1214         xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
1215         if (!xdev)
1216                 return -ENOMEM;
1217
1218         platform_set_drvdata(pdev, xdev);
1219         xdev->pdev = pdev;
1220
1221         res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1222         if (!res) {
1223                 xdma_err(xdev, "failed to get irq resource");
1224                 goto failed;
1225         }
1226         xdev->irq_start = res->start;
1227         xdev->irq_num = resource_size(res);
1228
1229         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1230         if (!res) {
1231                 xdma_err(xdev, "failed to get io resource");
1232                 goto failed;
1233         }
1234
1235         reg_base = devm_ioremap_resource(&pdev->dev, res);
1236         if (IS_ERR(reg_base)) {
1237                 xdma_err(xdev, "ioremap failed");
1238                 goto failed;
1239         }
1240
1241         xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base,
1242                                            &xdma_regmap_config);
1243         if (!xdev->rmap) {
1244                 xdma_err(xdev, "config regmap failed: %d", ret);
1245                 goto failed;
1246         }
1247         INIT_LIST_HEAD(&xdev->dma_dev.channels);
1248
1249         ret = xdma_alloc_channels(xdev, DMA_MEM_TO_DEV);
1250         if (ret) {
1251                 xdma_err(xdev, "config H2C channels failed: %d", ret);
1252                 goto failed;
1253         }
1254
1255         ret = xdma_alloc_channels(xdev, DMA_DEV_TO_MEM);
1256         if (ret) {
1257                 xdma_err(xdev, "config C2H channels failed: %d", ret);
1258                 goto failed;
1259         }
1260
1261         dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask);
1262         dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask);
1263         dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask);
1264         dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask);
1265         dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask);
1266         dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask);
1267
1268         xdev->dma_dev.dev = &pdev->dev;
1269         xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1270         xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources;
1271         xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources;
1272         xdev->dma_dev.device_tx_status = xdma_tx_status;
1273         xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg;
1274         xdev->dma_dev.device_config = xdma_device_config;
1275         xdev->dma_dev.device_issue_pending = xdma_issue_pending;
1276         xdev->dma_dev.device_terminate_all = xdma_terminate_all;
1277         xdev->dma_dev.device_synchronize = xdma_synchronize;
1278         xdev->dma_dev.filter.map = pdata->device_map;
1279         xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt;
1280         xdev->dma_dev.filter.fn = xdma_filter_fn;
1281         xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic;
1282         xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma;
1283
1284         ret = dma_async_device_register(&xdev->dma_dev);
1285         if (ret) {
1286                 xdma_err(xdev, "failed to register Xilinx XDMA: %d", ret);
1287                 goto failed;
1288         }
1289         xdev->status |= XDMA_DEV_STATUS_REG_DMA;
1290
1291         ret = xdma_irq_init(xdev);
1292         if (ret) {
1293                 xdma_err(xdev, "failed to init msix: %d", ret);
1294                 goto failed;
1295         }
1296         xdev->status |= XDMA_DEV_STATUS_INIT_MSIX;
1297
1298         return 0;
1299
1300 failed:
1301         xdma_remove(pdev);
1302
1303         return ret;
1304 }
1305
1306 static const struct platform_device_id xdma_id_table[] = {
1307         { "xdma", 0},
1308         { },
1309 };
1310
1311 static struct platform_driver xdma_driver = {
1312         .driver         = {
1313                 .name = "xdma",
1314         },
1315         .id_table       = xdma_id_table,
1316         .probe          = xdma_probe,
1317         .remove_new     = xdma_remove,
1318 };
1319
1320 module_platform_driver(xdma_driver);
1321
1322 MODULE_DESCRIPTION("AMD XDMA driver");
1323 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
1324 MODULE_LICENSE("GPL");