sf-pdma/sf-pdma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * SiFive FU540 Platform DMA driver
   4  * Copyright (C) 2019 SiFive
   5  *
   6  * Based partially on:
   7  * - drivers/dma/fsl-edma.c
   8  * - drivers/dma/dw-edma/
   9  * - drivers/dma/pxa-dma.c
  10  *
  11  * See the following sources for further documentation:
  12  * - Chapter 12 "Platform DMA Engine (PDMA)" of
  13  *   SiFive FU540-C000 v1.0
  14  *   https://static.dev.sifive.com/FU540-C000-v1.0.pdf
  15  */
  16 #include <linux/module.h>
  17 #include <linux/device.h>
  18 #include <linux/kernel.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/mod_devicetable.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/of.h>
  23 #include <linux/slab.h>
  24
  25 #include "sf-pdma.h"
  26
  27 #ifndef readq
  28 static inline unsigned long long readq(void __iomem *addr)
  29 {
  30         return readl(addr) | (((unsigned long long)readl(addr + 4)) << 32LL);
  31 }
  32 #endif
  33
  34 #ifndef writeq
  35 static inline void writeq(unsigned long long v, void __iomem *addr)
  36 {
  37         writel(lower_32_bits(v), addr);
  38         writel(upper_32_bits(v), addr + 4);
  39 }
  40 #endif
  41
  42 static inline struct sf_pdma_chan *to_sf_pdma_chan(struct dma_chan *dchan)
  43 {
  44         return container_of(dchan, struct sf_pdma_chan, vchan.chan);
  45 }
  46
  47 static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd)
  48 {
  49         return container_of(vd, struct sf_pdma_desc, vdesc);
  50 }
  51
  52 static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan)
  53 {
  54         struct sf_pdma_desc *desc;
  55
  56         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
  57         if (!desc)
  58                 return NULL;
  59
  60         desc->chan = chan;
  61
  62         return desc;
  63 }
  64
  65 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc,
  66                               u64 dst, u64 src, u64 size)
  67 {
  68         desc->xfer_type = PDMA_FULL_SPEED;
  69         desc->xfer_size = size;
  70         desc->dst_addr = dst;
  71         desc->src_addr = src;
  72 }
  73
  74 static void sf_pdma_disclaim_chan(struct sf_pdma_chan *chan)
  75 {
  76         struct pdma_regs *regs = &chan->regs;
  77
  78         writel(PDMA_CLEAR_CTRL, regs->ctrl);
  79 }
  80
  81 static struct dma_async_tx_descriptor *
  82 sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src,
  83                         size_t len, unsigned long flags)
  84 {
  85         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
  86         struct sf_pdma_desc *desc;
  87         unsigned long iflags;
  88
  89         if (chan && (!len || !dest || !src)) {
  90                 dev_err(chan->pdma->dma_dev.dev,
  91                         "Please check dma len, dest, src!\n");
  92                 return NULL;
  93         }
  94
  95         desc = sf_pdma_alloc_desc(chan);
  96         if (!desc)
  97                 return NULL;
  98
  99         desc->dirn = DMA_MEM_TO_MEM;
 100         desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 101
 102         spin_lock_irqsave(&chan->vchan.lock, iflags);
 103         sf_pdma_fill_desc(desc, dest, src, len);
 104         spin_unlock_irqrestore(&chan->vchan.lock, iflags);
 105
 106         return desc->async_tx;
 107 }
 108
 109 static int sf_pdma_slave_config(struct dma_chan *dchan,
 110                                 struct dma_slave_config *cfg)
 111 {
 112         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 113
 114         memcpy(&chan->cfg, cfg, sizeof(*cfg));
 115
 116         return 0;
 117 }
 118
 119 static int sf_pdma_alloc_chan_resources(struct dma_chan *dchan)
 120 {
 121         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 122         struct pdma_regs *regs = &chan->regs;
 123
 124         dma_cookie_init(dchan);
 125         writel(PDMA_CLAIM_MASK, regs->ctrl);
 126
 127         return 0;
 128 }
 129
 130 static void sf_pdma_disable_request(struct sf_pdma_chan *chan)
 131 {
 132         struct pdma_regs *regs = &chan->regs;
 133
 134         writel(readl(regs->ctrl) & ~PDMA_RUN_MASK, regs->ctrl);
 135 }
 136
 137 static void sf_pdma_free_chan_resources(struct dma_chan *dchan)
 138 {
 139         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 140         unsigned long flags;
 141         LIST_HEAD(head);
 142
 143         spin_lock_irqsave(&chan->vchan.lock, flags);
 144         sf_pdma_disable_request(chan);
 145         kfree(chan->desc);
 146         chan->desc = NULL;
 147         vchan_get_all_descriptors(&chan->vchan, &head);
 148         sf_pdma_disclaim_chan(chan);
 149         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 150         vchan_dma_desc_free_list(&chan->vchan, &head);
 151 }
 152
 153 static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan,
 154                                    dma_cookie_t cookie)
 155 {
 156         struct virt_dma_desc *vd = NULL;
 157         struct pdma_regs *regs = &chan->regs;
 158         unsigned long flags;
 159         u64 residue = 0;
 160         struct sf_pdma_desc *desc;
 161         struct dma_async_tx_descriptor *tx = NULL;
 162
 163         spin_lock_irqsave(&chan->vchan.lock, flags);
 164
 165         list_for_each_entry(vd, &chan->vchan.desc_submitted, node)
 166                 if (vd->tx.cookie == cookie)
 167                         tx = &vd->tx;
 168
 169         if (!tx)
 170                 goto out;
 171
 172         if (cookie == tx->chan->completed_cookie)
 173                 goto out;
 174
 175         if (cookie == tx->cookie) {
 176                 residue = readq(regs->residue);
 177         } else {
 178                 vd = vchan_find_desc(&chan->vchan, cookie);
 179                 if (!vd)
 180                         goto out;
 181
 182                 desc = to_sf_pdma_desc(vd);
 183                 residue = desc->xfer_size;
 184         }
 185
 186 out:
 187         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 188         return residue;
 189 }
 190
 191 static enum dma_status
 192 sf_pdma_tx_status(struct dma_chan *dchan,
 193                   dma_cookie_t cookie,
 194                   struct dma_tx_state *txstate)
 195 {
 196         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 197         enum dma_status status;
 198
 199         status = dma_cookie_status(dchan, cookie, txstate);
 200
 201         if (txstate && status != DMA_ERROR)
 202                 dma_set_residue(txstate, sf_pdma_desc_residue(chan, cookie));
 203
 204         return status;
 205 }
 206
 207 static int sf_pdma_terminate_all(struct dma_chan *dchan)
 208 {
 209         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 210         unsigned long flags;
 211         LIST_HEAD(head);
 212
 213         spin_lock_irqsave(&chan->vchan.lock, flags);
 214         sf_pdma_disable_request(chan);
 215         kfree(chan->desc);
 216         chan->desc = NULL;
 217         chan->xfer_err = false;
 218         vchan_get_all_descriptors(&chan->vchan, &head);
 219         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 220         vchan_dma_desc_free_list(&chan->vchan, &head);
 221
 222         return 0;
 223 }
 224
 225 static void sf_pdma_enable_request(struct sf_pdma_chan *chan)
 226 {
 227         struct pdma_regs *regs = &chan->regs;
 228         u32 v;
 229
 230         v = PDMA_CLAIM_MASK |
 231                 PDMA_ENABLE_DONE_INT_MASK |
 232                 PDMA_ENABLE_ERR_INT_MASK |
 233                 PDMA_RUN_MASK;
 234
 235         writel(v, regs->ctrl);
 236 }
 237
 238 static struct sf_pdma_desc *sf_pdma_get_first_pending_desc(struct sf_pdma_chan *chan)
 239 {
 240         struct virt_dma_chan *vchan = &chan->vchan;
 241         struct virt_dma_desc *vdesc;
 242
 243         if (list_empty(&vchan->desc_issued))
 244                 return NULL;
 245
 246         vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node);
 247
 248         return container_of(vdesc, struct sf_pdma_desc, vdesc);
 249 }
 250
 251 static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan)
 252 {
 253         struct sf_pdma_desc *desc = chan->desc;
 254         struct pdma_regs *regs = &chan->regs;
 255
 256         if (!desc) {
 257                 dev_err(chan->pdma->dma_dev.dev, "NULL desc.\n");
 258                 return;
 259         }
 260
 261         writel(desc->xfer_type, regs->xfer_type);
 262         writeq(desc->xfer_size, regs->xfer_size);
 263         writeq(desc->dst_addr, regs->dst_addr);
 264         writeq(desc->src_addr, regs->src_addr);
 265
 266         chan->desc = desc;
 267         chan->status = DMA_IN_PROGRESS;
 268         sf_pdma_enable_request(chan);
 269 }
 270
 271 static void sf_pdma_issue_pending(struct dma_chan *dchan)
 272 {
 273         struct sf_pdma_chan *chan = to_sf_pdma_chan(dchan);
 274         unsigned long flags;
 275
 276         spin_lock_irqsave(&chan->vchan.lock, flags);
 277
 278         if (!chan->desc && vchan_issue_pending(&chan->vchan)) {
 279                 /* vchan_issue_pending has made a check that desc in not NULL */
 280                 chan->desc = sf_pdma_get_first_pending_desc(chan);
 281                 sf_pdma_xfer_desc(chan);
 282         }
 283
 284         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 285 }
 286
 287 static void sf_pdma_free_desc(struct virt_dma_desc *vdesc)
 288 {
 289         struct sf_pdma_desc *desc;
 290
 291         desc = to_sf_pdma_desc(vdesc);
 292         kfree(desc);
 293 }
 294
 295 static void sf_pdma_donebh_tasklet(struct tasklet_struct *t)
 296 {
 297         struct sf_pdma_chan *chan = from_tasklet(chan, t, done_tasklet);
 298         unsigned long flags;
 299
 300         spin_lock_irqsave(&chan->lock, flags);
 301         if (chan->xfer_err) {
 302                 chan->retries = MAX_RETRY;
 303                 chan->status = DMA_COMPLETE;
 304                 chan->xfer_err = false;
 305         }
 306         spin_unlock_irqrestore(&chan->lock, flags);
 307
 308         spin_lock_irqsave(&chan->vchan.lock, flags);
 309         list_del(&chan->desc->vdesc.node);
 310         vchan_cookie_complete(&chan->desc->vdesc);
 311
 312         chan->desc = sf_pdma_get_first_pending_desc(chan);
 313         if (chan->desc)
 314                 sf_pdma_xfer_desc(chan);
 315
 316         spin_unlock_irqrestore(&chan->vchan.lock, flags);
 317 }
 318
 319 static void sf_pdma_errbh_tasklet(struct tasklet_struct *t)
 320 {
 321         struct sf_pdma_chan *chan = from_tasklet(chan, t, err_tasklet);
 322         struct sf_pdma_desc *desc = chan->desc;
 323         unsigned long flags;
 324
 325         spin_lock_irqsave(&chan->lock, flags);
 326         if (chan->retries <= 0) {
 327                 /* fail to recover */
 328                 spin_unlock_irqrestore(&chan->lock, flags);
 329                 dmaengine_desc_get_callback_invoke(desc->async_tx, NULL);
 330         } else {
 331                 /* retry */
 332                 chan->retries--;
 333                 chan->xfer_err = true;
 334                 chan->status = DMA_ERROR;
 335
 336                 sf_pdma_enable_request(chan);
 337                 spin_unlock_irqrestore(&chan->lock, flags);
 338         }
 339 }
 340
 341 static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id)
 342 {
 343         struct sf_pdma_chan *chan = dev_id;
 344         struct pdma_regs *regs = &chan->regs;
 345         u64 residue;
 346
 347         spin_lock(&chan->vchan.lock);
 348         writel((readl(regs->ctrl)) & ~PDMA_DONE_STATUS_MASK, regs->ctrl);
 349         residue = readq(regs->residue);
 350
 351         if (!residue) {
 352                 tasklet_hi_schedule(&chan->done_tasklet);
 353         } else {
 354                 /* submit next trascatioin if possible */
 355                 struct sf_pdma_desc *desc = chan->desc;
 356
 357                 desc->src_addr += desc->xfer_size - residue;
 358                 desc->dst_addr += desc->xfer_size - residue;
 359                 desc->xfer_size = residue;
 360
 361                 sf_pdma_xfer_desc(chan);
 362         }
 363
 364         spin_unlock(&chan->vchan.lock);
 365
 366         return IRQ_HANDLED;
 367 }
 368
 369 static irqreturn_t sf_pdma_err_isr(int irq, void *dev_id)
 370 {
 371         struct sf_pdma_chan *chan = dev_id;
 372         struct pdma_regs *regs = &chan->regs;
 373
 374         spin_lock(&chan->lock);
 375         writel((readl(regs->ctrl)) & ~PDMA_ERR_STATUS_MASK, regs->ctrl);
 376         spin_unlock(&chan->lock);
 377
 378         tasklet_schedule(&chan->err_tasklet);
 379
 380         return IRQ_HANDLED;
 381 }
 382
 383 /**
 384  * sf_pdma_irq_init() - Init PDMA IRQ Handlers
 385  * @pdev: pointer of platform_device
 386  * @pdma: pointer of PDMA engine. Caller should check NULL
 387  *
 388  * Initialize DONE and ERROR interrupt handler for 4 channels. Caller should
 389  * make sure the pointer passed in are non-NULL. This function should be called
 390  * only one time during the device probe.
 391  *
 392  * Context: Any context.
 393  *
 394  * Return:
 395  * * 0          - OK to init all IRQ handlers
 396  * * -EINVAL    - Fail to request IRQ
 397  */
 398 static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma)
 399 {
 400         int irq, r, i;
 401         struct sf_pdma_chan *chan;
 402
 403         for (i = 0; i < pdma->n_chans; i++) {
 404                 chan = &pdma->chans[i];
 405
 406                 irq = platform_get_irq(pdev, i * 2);
 407                 if (irq < 0)
 408                         return -EINVAL;
 409
 410                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0,
 411                                      dev_name(&pdev->dev), (void *)chan);
 412                 if (r) {
 413                         dev_err(&pdev->dev, "Fail to attach done ISR: %d\n", r);
 414                         return -EINVAL;
 415                 }
 416
 417                 chan->txirq = irq;
 418
 419                 irq = platform_get_irq(pdev, (i * 2) + 1);
 420                 if (irq < 0)
 421                         return -EINVAL;
 422
 423                 r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0,
 424                                      dev_name(&pdev->dev), (void *)chan);
 425                 if (r) {
 426                         dev_err(&pdev->dev, "Fail to attach err ISR: %d\n", r);
 427                         return -EINVAL;
 428                 }
 429
 430                 chan->errirq = irq;
 431         }
 432
 433         return 0;
 434 }
 435
 436 /**
 437  * sf_pdma_setup_chans() - Init settings of each channel
 438  * @pdma: pointer of PDMA engine. Caller should check NULL
 439  *
 440  * Initialize all data structure and register base. Caller should make sure
 441  * the pointer passed in are non-NULL. This function should be called only
 442  * one time during the device probe.
 443  *
 444  * Context: Any context.
 445  *
 446  * Return: none
 447  */
 448 static void sf_pdma_setup_chans(struct sf_pdma *pdma)
 449 {
 450         int i;
 451         struct sf_pdma_chan *chan;
 452
 453         INIT_LIST_HEAD(&pdma->dma_dev.channels);
 454
 455         for (i = 0; i < pdma->n_chans; i++) {
 456                 chan = &pdma->chans[i];
 457
 458                 chan->regs.ctrl =
 459                         SF_PDMA_REG_BASE(i) + PDMA_CTRL;
 460                 chan->regs.xfer_type =
 461                         SF_PDMA_REG_BASE(i) + PDMA_XFER_TYPE;
 462                 chan->regs.xfer_size =
 463                         SF_PDMA_REG_BASE(i) + PDMA_XFER_SIZE;
 464                 chan->regs.dst_addr =
 465                         SF_PDMA_REG_BASE(i) + PDMA_DST_ADDR;
 466                 chan->regs.src_addr =
 467                         SF_PDMA_REG_BASE(i) + PDMA_SRC_ADDR;
 468                 chan->regs.act_type =
 469                         SF_PDMA_REG_BASE(i) + PDMA_ACT_TYPE;
 470                 chan->regs.residue =
 471                         SF_PDMA_REG_BASE(i) + PDMA_REMAINING_BYTE;
 472                 chan->regs.cur_dst_addr =
 473                         SF_PDMA_REG_BASE(i) + PDMA_CUR_DST_ADDR;
 474                 chan->regs.cur_src_addr =
 475                         SF_PDMA_REG_BASE(i) + PDMA_CUR_SRC_ADDR;
 476
 477                 chan->pdma = pdma;
 478                 chan->pm_state = RUNNING;
 479                 chan->slave_id = i;
 480                 chan->xfer_err = false;
 481                 spin_lock_init(&chan->lock);
 482
 483                 chan->vchan.desc_free = sf_pdma_free_desc;
 484                 vchan_init(&chan->vchan, &pdma->dma_dev);
 485
 486                 writel(PDMA_CLEAR_CTRL, chan->regs.ctrl);
 487
 488                 tasklet_setup(&chan->done_tasklet, sf_pdma_donebh_tasklet);
 489                 tasklet_setup(&chan->err_tasklet, sf_pdma_errbh_tasklet);
 490         }
 491 }
 492
 493 static int sf_pdma_probe(struct platform_device *pdev)
 494 {
 495         struct sf_pdma *pdma;
 496         int ret, n_chans;
 497         const enum dma_slave_buswidth widths =
 498                 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
 499                 DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES |
 500                 DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES |
 501                 DMA_SLAVE_BUSWIDTH_64_BYTES;
 502
 503         ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans);
 504         if (ret) {
 505                 /* backwards-compatibility for no dma-channels property */
 506                 dev_dbg(&pdev->dev, "set number of channels to default value: 4\n");
 507                 n_chans = PDMA_MAX_NR_CH;
 508         } else if (n_chans > PDMA_MAX_NR_CH) {
 509                 dev_err(&pdev->dev, "the number of channels exceeds the maximum\n");
 510                 return -EINVAL;
 511         }
 512
 513         pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans),
 514                             GFP_KERNEL);
 515         if (!pdma)
 516                 return -ENOMEM;
 517
 518         pdma->n_chans = n_chans;
 519
 520         pdma->membase = devm_platform_ioremap_resource(pdev, 0);
 521         if (IS_ERR(pdma->membase))
 522                 return PTR_ERR(pdma->membase);
 523
 524         ret = sf_pdma_irq_init(pdev, pdma);
 525         if (ret)
 526                 return ret;
 527
 528         sf_pdma_setup_chans(pdma);
 529
 530         pdma->dma_dev.dev = &pdev->dev;
 531
 532         /* Setup capability */
 533         dma_cap_set(DMA_MEMCPY, pdma->dma_dev.cap_mask);
 534         pdma->dma_dev.copy_align = 2;
 535         pdma->dma_dev.src_addr_widths = widths;
 536         pdma->dma_dev.dst_addr_widths = widths;
 537         pdma->dma_dev.directions = BIT(DMA_MEM_TO_MEM);
 538         pdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 539         pdma->dma_dev.descriptor_reuse = true;
 540
 541         /* Setup DMA APIs */
 542         pdma->dma_dev.device_alloc_chan_resources =
 543                 sf_pdma_alloc_chan_resources;
 544         pdma->dma_dev.device_free_chan_resources =
 545                 sf_pdma_free_chan_resources;
 546         pdma->dma_dev.device_tx_status = sf_pdma_tx_status;
 547         pdma->dma_dev.device_prep_dma_memcpy = sf_pdma_prep_dma_memcpy;
 548         pdma->dma_dev.device_config = sf_pdma_slave_config;
 549         pdma->dma_dev.device_terminate_all = sf_pdma_terminate_all;
 550         pdma->dma_dev.device_issue_pending = sf_pdma_issue_pending;
 551
 552         platform_set_drvdata(pdev, pdma);
 553
 554         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 555         if (ret)
 556                 dev_warn(&pdev->dev,
 557                          "Failed to set DMA mask. Fall back to default.\n");
 558
 559         ret = dma_async_device_register(&pdma->dma_dev);
 560         if (ret) {
 561                 dev_err(&pdev->dev,
 562                         "Can't register SiFive Platform DMA. (%d)\n", ret);
 563                 return ret;
 564         }
 565
 566         return 0;
 567 }
 568
 569 static void sf_pdma_remove(struct platform_device *pdev)
 570 {
 571         struct sf_pdma *pdma = platform_get_drvdata(pdev);
 572         struct sf_pdma_chan *ch;
 573         int i;
 574
 575         for (i = 0; i < pdma->n_chans; i++) {
 576                 ch = &pdma->chans[i];
 577
 578                 devm_free_irq(&pdev->dev, ch->txirq, ch);
 579                 devm_free_irq(&pdev->dev, ch->errirq, ch);
 580                 list_del(&ch->vchan.chan.device_node);
 581                 tasklet_kill(&ch->vchan.task);
 582                 tasklet_kill(&ch->done_tasklet);
 583                 tasklet_kill(&ch->err_tasklet);
 584         }
 585
 586         dma_async_device_unregister(&pdma->dma_dev);
 587 }
 588
 589 static const struct of_device_id sf_pdma_dt_ids[] = {
 590         { .compatible = "sifive,fu540-c000-pdma" },
 591         { .compatible = "sifive,pdma0" },
 592         {},
 593 };
 594 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);
 595
 596 static struct platform_driver sf_pdma_driver = {
 597         .probe          = sf_pdma_probe,
 598         .remove_new     = sf_pdma_remove,
 599         .driver         = {
 600                 .name   = "sf-pdma",
 601                 .of_match_table = sf_pdma_dt_ids,
 602         },
 603 };
 604
 605 static int __init sf_pdma_init(void)
 606 {
 607         return platform_driver_register(&sf_pdma_driver);
 608 }
 609
 610 static void __exit sf_pdma_exit(void)
 611 {
 612         platform_driver_unregister(&sf_pdma_driver);
 613 }
 614
 615 /* do early init */
 616 subsys_initcall(sf_pdma_init);
 617 module_exit(sf_pdma_exit);
 618
 619 MODULE_LICENSE("GPL v2");
 620 MODULE_DESCRIPTION("SiFive Platform DMA driver");
 621 MODULE_AUTHOR("Green Wan <green.wan@sifive.com>");