drivers/dma/hisi_dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2019 HiSilicon Limited. */
   3 #include <linux/bitfield.h>
   4 #include <linux/dmaengine.h>
   5 #include <linux/init.h>
   6 #include <linux/iopoll.h>
   7 #include <linux/module.h>
   8 #include <linux/pci.h>
   9 #include <linux/spinlock.h>
  10 #include "virt-dma.h"
  11
  12 #define HISI_DMA_SQ_BASE_L              0x0
  13 #define HISI_DMA_SQ_BASE_H              0x4
  14 #define HISI_DMA_SQ_DEPTH               0x8
  15 #define HISI_DMA_SQ_TAIL_PTR            0xc
  16 #define HISI_DMA_CQ_BASE_L              0x10
  17 #define HISI_DMA_CQ_BASE_H              0x14
  18 #define HISI_DMA_CQ_DEPTH               0x18
  19 #define HISI_DMA_CQ_HEAD_PTR            0x1c
  20 #define HISI_DMA_CTRL0                  0x20
  21 #define HISI_DMA_CTRL0_QUEUE_EN_S       0
  22 #define HISI_DMA_CTRL0_QUEUE_PAUSE_S    4
  23 #define HISI_DMA_CTRL1                  0x24
  24 #define HISI_DMA_CTRL1_QUEUE_RESET_S    0
  25 #define HISI_DMA_Q_FSM_STS              0x30
  26 #define HISI_DMA_FSM_STS_MASK           GENMASK(3, 0)
  27 #define HISI_DMA_INT_STS                0x40
  28 #define HISI_DMA_INT_STS_MASK           GENMASK(12, 0)
  29 #define HISI_DMA_INT_MSK                0x44
  30 #define HISI_DMA_MODE                   0x217c
  31 #define HISI_DMA_OFFSET                 0x100
  32
  33 #define HISI_DMA_MSI_NUM                32
  34 #define HISI_DMA_CHAN_NUM               30
  35 #define HISI_DMA_Q_DEPTH_VAL            1024
  36
  37 #define PCI_BAR_2                       2
  38
  39 enum hisi_dma_mode {
  40         EP = 0,
  41         RC,
  42 };
  43
  44 enum hisi_dma_chan_status {
  45         DISABLE = -1,
  46         IDLE = 0,
  47         RUN,
  48         CPL,
  49         PAUSE,
  50         HALT,
  51         ABORT,
  52         WAIT,
  53         BUFFCLR,
  54 };
  55
  56 struct hisi_dma_sqe {
  57         __le32 dw0;
  58 #define OPCODE_MASK                     GENMASK(3, 0)
  59 #define OPCODE_SMALL_PACKAGE            0x1
  60 #define OPCODE_M2M                      0x4
  61 #define LOCAL_IRQ_EN                    BIT(8)
  62 #define ATTR_SRC_MASK                   GENMASK(14, 12)
  63         __le32 dw1;
  64         __le32 dw2;
  65 #define ATTR_DST_MASK                   GENMASK(26, 24)
  66         __le32 length;
  67         __le64 src_addr;
  68         __le64 dst_addr;
  69 };
  70
  71 struct hisi_dma_cqe {
  72         __le32 rsv0;
  73         __le32 rsv1;
  74         __le16 sq_head;
  75         __le16 rsv2;
  76         __le16 rsv3;
  77         __le16 w0;
  78 #define STATUS_MASK                     GENMASK(15, 1)
  79 #define STATUS_SUCC                     0x0
  80 #define VALID_BIT                       BIT(0)
  81 };
  82
  83 struct hisi_dma_desc {
  84         struct virt_dma_desc vd;
  85         struct hisi_dma_sqe sqe;
  86 };
  87
  88 struct hisi_dma_chan {
  89         struct virt_dma_chan vc;
  90         struct hisi_dma_dev *hdma_dev;
  91         struct hisi_dma_sqe *sq;
  92         struct hisi_dma_cqe *cq;
  93         dma_addr_t sq_dma;
  94         dma_addr_t cq_dma;
  95         u32 sq_tail;
  96         u32 cq_head;
  97         u32 qp_num;
  98         enum hisi_dma_chan_status status;
  99         struct hisi_dma_desc *desc;
 100 };
 101
 102 struct hisi_dma_dev {
 103         struct pci_dev *pdev;
 104         void __iomem *base;
 105         struct dma_device dma_dev;
 106         u32 chan_num;
 107         u32 chan_depth;
 108         struct hisi_dma_chan chan[];
 109 };
 110
 111 static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
 112 {
 113         return container_of(c, struct hisi_dma_chan, vc.chan);
 114 }
 115
 116 static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
 117 {
 118         return container_of(vd, struct hisi_dma_desc, vd);
 119 }
 120
 121 static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
 122                                        u32 val)
 123 {
 124         writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
 125 }
 126
 127 static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
 128 {
 129         u32 tmp;
 130
 131         tmp = readl_relaxed(addr);
 132         tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
 133         writel_relaxed(tmp, addr);
 134 }
 135
 136 static void hisi_dma_free_irq_vectors(void *data)
 137 {
 138         pci_free_irq_vectors(data);
 139 }
 140
 141 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
 142                                bool pause)
 143 {
 144         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
 145                              HISI_DMA_OFFSET;
 146
 147         hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
 148 }
 149
 150 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
 151                                 bool enable)
 152 {
 153         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
 154                              HISI_DMA_OFFSET;
 155
 156         hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
 157 }
 158
 159 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
 160 {
 161         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
 162                             HISI_DMA_INT_STS_MASK);
 163 }
 164
 165 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
 166 {
 167         void __iomem *base = hdma_dev->base;
 168
 169         hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
 170                             HISI_DMA_INT_STS_MASK);
 171         hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
 172 }
 173
 174 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
 175 {
 176         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
 177                              HISI_DMA_OFFSET;
 178
 179         hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
 180 }
 181
 182 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
 183 {
 184         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
 185         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
 186 }
 187
 188 static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
 189                                               bool disable)
 190 {
 191         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
 192         u32 index = chan->qp_num, tmp;
 193         int ret;
 194
 195         hisi_dma_pause_dma(hdma_dev, index, true);
 196         hisi_dma_enable_dma(hdma_dev, index, false);
 197         hisi_dma_mask_irq(hdma_dev, index);
 198
 199         ret = readl_relaxed_poll_timeout(hdma_dev->base +
 200                 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
 201                 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
 202         if (ret) {
 203                 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
 204                 WARN_ON(1);
 205         }
 206
 207         hisi_dma_do_reset(hdma_dev, index);
 208         hisi_dma_reset_qp_point(hdma_dev, index);
 209         hisi_dma_pause_dma(hdma_dev, index, false);
 210
 211         if (!disable) {
 212                 hisi_dma_enable_dma(hdma_dev, index, true);
 213                 hisi_dma_unmask_irq(hdma_dev, index);
 214         }
 215
 216         ret = readl_relaxed_poll_timeout(hdma_dev->base +
 217                 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
 218                 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
 219         if (ret) {
 220                 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
 221                 WARN_ON(1);
 222         }
 223 }
 224
 225 static void hisi_dma_free_chan_resources(struct dma_chan *c)
 226 {
 227         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
 228         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
 229
 230         hisi_dma_reset_or_disable_hw_chan(chan, false);
 231         vchan_free_chan_resources(&chan->vc);
 232
 233         memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
 234         memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
 235         chan->sq_tail = 0;
 236         chan->cq_head = 0;
 237         chan->status = DISABLE;
 238 }
 239
 240 static void hisi_dma_desc_free(struct virt_dma_desc *vd)
 241 {
 242         kfree(to_hisi_dma_desc(vd));
 243 }
 244
 245 static struct dma_async_tx_descriptor *
 246 hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
 247                          size_t len, unsigned long flags)
 248 {
 249         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
 250         struct hisi_dma_desc *desc;
 251
 252         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
 253         if (!desc)
 254                 return NULL;
 255
 256         desc->sqe.length = cpu_to_le32(len);
 257         desc->sqe.src_addr = cpu_to_le64(src);
 258         desc->sqe.dst_addr = cpu_to_le64(dst);
 259
 260         return vchan_tx_prep(&chan->vc, &desc->vd, flags);
 261 }
 262
 263 static enum dma_status
 264 hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
 265                    struct dma_tx_state *txstate)
 266 {
 267         return dma_cookie_status(c, cookie, txstate);
 268 }
 269
 270 static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
 271 {
 272         struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
 273         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
 274         struct hisi_dma_desc *desc;
 275         struct virt_dma_desc *vd;
 276
 277         vd = vchan_next_desc(&chan->vc);
 278         if (!vd) {
 279                 chan->desc = NULL;
 280                 return;
 281         }
 282         list_del(&vd->node);
 283         desc = to_hisi_dma_desc(vd);
 284         chan->desc = desc;
 285
 286         memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
 287
 288         /* update other field in sqe */
 289         sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
 290         sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
 291
 292         /* make sure data has been updated in sqe */
 293         wmb();
 294
 295         /* update sq tail, point to new sqe position */
 296         chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
 297
 298         /* update sq_tail to trigger a new task */
 299         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
 300                             chan->sq_tail);
 301 }
 302
 303 static void hisi_dma_issue_pending(struct dma_chan *c)
 304 {
 305         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
 306         unsigned long flags;
 307
 308         spin_lock_irqsave(&chan->vc.lock, flags);
 309
 310         if (vchan_issue_pending(&chan->vc) && !chan->desc)
 311                 hisi_dma_start_transfer(chan);
 312
 313         spin_unlock_irqrestore(&chan->vc.lock, flags);
 314 }
 315
 316 static int hisi_dma_terminate_all(struct dma_chan *c)
 317 {
 318         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
 319         unsigned long flags;
 320         LIST_HEAD(head);
 321
 322         spin_lock_irqsave(&chan->vc.lock, flags);
 323
 324         hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
 325         if (chan->desc) {
 326                 vchan_terminate_vdesc(&chan->desc->vd);
 327                 chan->desc = NULL;
 328         }
 329
 330         vchan_get_all_descriptors(&chan->vc, &head);
 331
 332         spin_unlock_irqrestore(&chan->vc.lock, flags);
 333
 334         vchan_dma_desc_free_list(&chan->vc, &head);
 335         hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
 336
 337         return 0;
 338 }
 339
 340 static void hisi_dma_synchronize(struct dma_chan *c)
 341 {
 342         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
 343
 344         vchan_synchronize(&chan->vc);
 345 }
 346
 347 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
 348 {
 349         size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
 350         size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
 351         struct device *dev = &hdma_dev->pdev->dev;
 352         struct hisi_dma_chan *chan;
 353         int i;
 354
 355         for (i = 0; i < hdma_dev->chan_num; i++) {
 356                 chan = &hdma_dev->chan[i];
 357                 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
 358                                                GFP_KERNEL);
 359                 if (!chan->sq)
 360                         return -ENOMEM;
 361
 362                 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
 363                                                GFP_KERNEL);
 364                 if (!chan->cq)
 365                         return -ENOMEM;
 366         }
 367
 368         return 0;
 369 }
 370
 371 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
 372 {
 373         struct hisi_dma_chan *chan = &hdma_dev->chan[index];
 374         u32 hw_depth = hdma_dev->chan_depth - 1;
 375         void __iomem *base = hdma_dev->base;
 376
 377         /* set sq, cq base */
 378         hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
 379                             lower_32_bits(chan->sq_dma));
 380         hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
 381                             upper_32_bits(chan->sq_dma));
 382         hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
 383                             lower_32_bits(chan->cq_dma));
 384         hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
 385                             upper_32_bits(chan->cq_dma));
 386
 387         /* set sq, cq depth */
 388         hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
 389         hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
 390
 391         /* init sq tail and cq head */
 392         hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
 393         hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
 394 }
 395
 396 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
 397 {
 398         hisi_dma_init_hw_qp(hdma_dev, qp_index);
 399         hisi_dma_unmask_irq(hdma_dev, qp_index);
 400         hisi_dma_enable_dma(hdma_dev, qp_index, true);
 401 }
 402
 403 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
 404 {
 405         hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
 406 }
 407
 408 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
 409 {
 410         int i;
 411
 412         for (i = 0; i < hdma_dev->chan_num; i++) {
 413                 hdma_dev->chan[i].qp_num = i;
 414                 hdma_dev->chan[i].hdma_dev = hdma_dev;
 415                 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
 416                 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
 417                 hisi_dma_enable_qp(hdma_dev, i);
 418         }
 419 }
 420
 421 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
 422 {
 423         int i;
 424
 425         for (i = 0; i < hdma_dev->chan_num; i++) {
 426                 hisi_dma_disable_qp(hdma_dev, i);
 427                 tasklet_kill(&hdma_dev->chan[i].vc.task);
 428         }
 429 }
 430
 431 static irqreturn_t hisi_dma_irq(int irq, void *data)
 432 {
 433         struct hisi_dma_chan *chan = data;
 434         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
 435         struct hisi_dma_desc *desc;
 436         struct hisi_dma_cqe *cqe;
 437         unsigned long flags;
 438
 439         spin_lock_irqsave(&chan->vc.lock, flags);
 440
 441         desc = chan->desc;
 442         cqe = chan->cq + chan->cq_head;
 443         if (desc) {
 444                 chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
 445                 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR,
 446                                     chan->qp_num, chan->cq_head);
 447                 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
 448                         vchan_cookie_complete(&desc->vd);
 449                         hisi_dma_start_transfer(chan);
 450                 } else {
 451                         dev_err(&hdma_dev->pdev->dev, "task error!\n");
 452                 }
 453         }
 454
 455         spin_unlock_irqrestore(&chan->vc.lock, flags);
 456
 457         return IRQ_HANDLED;
 458 }
 459
 460 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
 461 {
 462         struct pci_dev *pdev = hdma_dev->pdev;
 463         int i, ret;
 464
 465         for (i = 0; i < hdma_dev->chan_num; i++) {
 466                 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
 467                                        hisi_dma_irq, IRQF_SHARED, "hisi_dma",
 468                                        &hdma_dev->chan[i]);
 469                 if (ret)
 470                         return ret;
 471         }
 472
 473         return 0;
 474 }
 475
 476 /* This function enables all hw channels in a device */
 477 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
 478 {
 479         int ret;
 480
 481         ret = hisi_dma_alloc_qps_mem(hdma_dev);
 482         if (ret) {
 483                 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
 484                 return ret;
 485         }
 486
 487         ret = hisi_dma_request_qps_irq(hdma_dev);
 488         if (ret) {
 489                 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
 490                 return ret;
 491         }
 492
 493         hisi_dma_enable_qps(hdma_dev);
 494
 495         return 0;
 496 }
 497
 498 static void hisi_dma_disable_hw_channels(void *data)
 499 {
 500         hisi_dma_disable_qps(data);
 501 }
 502
 503 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
 504                               enum hisi_dma_mode mode)
 505 {
 506         writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
 507 }
 508
 509 static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 510 {
 511         struct device *dev = &pdev->dev;
 512         struct hisi_dma_dev *hdma_dev;
 513         struct dma_device *dma_dev;
 514         int ret;
 515
 516         ret = pcim_enable_device(pdev);
 517         if (ret) {
 518                 dev_err(dev, "failed to enable device mem!\n");
 519                 return ret;
 520         }
 521
 522         ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
 523         if (ret) {
 524                 dev_err(dev, "failed to remap I/O region!\n");
 525                 return ret;
 526         }
 527
 528         ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 529         if (ret)
 530                 return ret;
 531
 532         ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 533         if (ret)
 534                 return ret;
 535
 536         hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
 537         if (!hdma_dev)
 538                 return -EINVAL;
 539
 540         hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
 541         hdma_dev->pdev = pdev;
 542         hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
 543         hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
 544
 545         pci_set_drvdata(pdev, hdma_dev);
 546         pci_set_master(pdev);
 547
 548         ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
 549                                     PCI_IRQ_MSI);
 550         if (ret < 0) {
 551                 dev_err(dev, "Failed to allocate MSI vectors!\n");
 552                 return ret;
 553         }
 554
 555         ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
 556         if (ret)
 557                 return ret;
 558
 559         dma_dev = &hdma_dev->dma_dev;
 560         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 561         dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
 562         dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
 563         dma_dev->device_tx_status = hisi_dma_tx_status;
 564         dma_dev->device_issue_pending = hisi_dma_issue_pending;
 565         dma_dev->device_terminate_all = hisi_dma_terminate_all;
 566         dma_dev->device_synchronize = hisi_dma_synchronize;
 567         dma_dev->directions = BIT(DMA_MEM_TO_MEM);
 568         dma_dev->dev = dev;
 569         INIT_LIST_HEAD(&dma_dev->channels);
 570
 571         hisi_dma_set_mode(hdma_dev, RC);
 572
 573         ret = hisi_dma_enable_hw_channels(hdma_dev);
 574         if (ret < 0) {
 575                 dev_err(dev, "failed to enable hw channel!\n");
 576                 return ret;
 577         }
 578
 579         ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
 580                                        hdma_dev);
 581         if (ret)
 582                 return ret;
 583
 584         ret = dmaenginem_async_device_register(dma_dev);
 585         if (ret < 0)
 586                 dev_err(dev, "failed to register device!\n");
 587
 588         return ret;
 589 }
 590
 591 static const struct pci_device_id hisi_dma_pci_tbl[] = {
 592         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
 593         { 0, }
 594 };
 595
 596 static struct pci_driver hisi_dma_pci_driver = {
 597         .name           = "hisi_dma",
 598         .id_table       = hisi_dma_pci_tbl,
 599         .probe          = hisi_dma_probe,
 600 };
 601
 602 module_pci_driver(hisi_dma_pci_driver);
 603
 604 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
 605 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
 606 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
 607 MODULE_LICENSE("GPL v2");
 608 MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);