1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019 HiSilicon Limited. */
3 #include <linux/bitfield.h>
4 #include <linux/dmaengine.h>
5 #include <linux/init.h>
6 #include <linux/iopoll.h>
7 #include <linux/module.h>
9 #include <linux/spinlock.h>
12 #define HISI_DMA_SQ_BASE_L 0x0
13 #define HISI_DMA_SQ_BASE_H 0x4
14 #define HISI_DMA_SQ_DEPTH 0x8
15 #define HISI_DMA_SQ_TAIL_PTR 0xc
16 #define HISI_DMA_CQ_BASE_L 0x10
17 #define HISI_DMA_CQ_BASE_H 0x14
18 #define HISI_DMA_CQ_DEPTH 0x18
19 #define HISI_DMA_CQ_HEAD_PTR 0x1c
20 #define HISI_DMA_CTRL0 0x20
21 #define HISI_DMA_CTRL0_QUEUE_EN_S 0
22 #define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
23 #define HISI_DMA_CTRL1 0x24
24 #define HISI_DMA_CTRL1_QUEUE_RESET_S 0
25 #define HISI_DMA_Q_FSM_STS 0x30
26 #define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
27 #define HISI_DMA_INT_STS 0x40
28 #define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
29 #define HISI_DMA_INT_MSK 0x44
30 #define HISI_DMA_MODE 0x217c
31 #define HISI_DMA_OFFSET 0x100
33 #define HISI_DMA_MSI_NUM 32
34 #define HISI_DMA_CHAN_NUM 30
35 #define HISI_DMA_Q_DEPTH_VAL 1024
44 enum hisi_dma_chan_status {
58 #define OPCODE_MASK GENMASK(3, 0)
59 #define OPCODE_SMALL_PACKAGE 0x1
60 #define OPCODE_M2M 0x4
61 #define LOCAL_IRQ_EN BIT(8)
62 #define ATTR_SRC_MASK GENMASK(14, 12)
65 #define ATTR_DST_MASK GENMASK(26, 24)
78 #define STATUS_MASK GENMASK(15, 1)
79 #define STATUS_SUCC 0x0
80 #define VALID_BIT BIT(0)
83 struct hisi_dma_desc {
84 struct virt_dma_desc vd;
85 struct hisi_dma_sqe sqe;
88 struct hisi_dma_chan {
89 struct virt_dma_chan vc;
90 struct hisi_dma_dev *hdma_dev;
91 struct hisi_dma_sqe *sq;
92 struct hisi_dma_cqe *cq;
98 enum hisi_dma_chan_status status;
99 struct hisi_dma_desc *desc;
102 struct hisi_dma_dev {
103 struct pci_dev *pdev;
105 struct dma_device dma_dev;
108 struct hisi_dma_chan chan[];
111 static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
113 return container_of(c, struct hisi_dma_chan, vc.chan);
116 static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
118 return container_of(vd, struct hisi_dma_desc, vd);
121 static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
124 writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
127 static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
131 tmp = readl_relaxed(addr);
132 tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
133 writel_relaxed(tmp, addr);
136 static void hisi_dma_free_irq_vectors(void *data)
138 pci_free_irq_vectors(data);
141 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
144 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
147 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
150 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
153 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
156 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
159 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
161 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
162 HISI_DMA_INT_STS_MASK);
165 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
167 void __iomem *base = hdma_dev->base;
169 hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
170 HISI_DMA_INT_STS_MASK);
171 hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
174 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
176 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
179 hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
182 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
184 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
185 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
188 static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
191 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
192 u32 index = chan->qp_num, tmp;
195 hisi_dma_pause_dma(hdma_dev, index, true);
196 hisi_dma_enable_dma(hdma_dev, index, false);
197 hisi_dma_mask_irq(hdma_dev, index);
199 ret = readl_relaxed_poll_timeout(hdma_dev->base +
200 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
201 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
203 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
207 hisi_dma_do_reset(hdma_dev, index);
208 hisi_dma_reset_qp_point(hdma_dev, index);
209 hisi_dma_pause_dma(hdma_dev, index, false);
212 hisi_dma_enable_dma(hdma_dev, index, true);
213 hisi_dma_unmask_irq(hdma_dev, index);
216 ret = readl_relaxed_poll_timeout(hdma_dev->base +
217 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
218 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
220 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
225 static void hisi_dma_free_chan_resources(struct dma_chan *c)
227 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
228 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
230 hisi_dma_reset_or_disable_hw_chan(chan, false);
231 vchan_free_chan_resources(&chan->vc);
233 memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
234 memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
237 chan->status = DISABLE;
240 static void hisi_dma_desc_free(struct virt_dma_desc *vd)
242 kfree(to_hisi_dma_desc(vd));
245 static struct dma_async_tx_descriptor *
246 hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
247 size_t len, unsigned long flags)
249 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
250 struct hisi_dma_desc *desc;
252 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
256 desc->sqe.length = cpu_to_le32(len);
257 desc->sqe.src_addr = cpu_to_le64(src);
258 desc->sqe.dst_addr = cpu_to_le64(dst);
260 return vchan_tx_prep(&chan->vc, &desc->vd, flags);
263 static enum dma_status
264 hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
265 struct dma_tx_state *txstate)
267 return dma_cookie_status(c, cookie, txstate);
270 static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
272 struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
273 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
274 struct hisi_dma_desc *desc;
275 struct virt_dma_desc *vd;
277 vd = vchan_next_desc(&chan->vc);
283 desc = to_hisi_dma_desc(vd);
286 memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
288 /* update other field in sqe */
289 sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
290 sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
292 /* make sure data has been updated in sqe */
295 /* update sq tail, point to new sqe position */
296 chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
298 /* update sq_tail to trigger a new task */
299 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
303 static void hisi_dma_issue_pending(struct dma_chan *c)
305 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
308 spin_lock_irqsave(&chan->vc.lock, flags);
310 if (vchan_issue_pending(&chan->vc) && !chan->desc)
311 hisi_dma_start_transfer(chan);
313 spin_unlock_irqrestore(&chan->vc.lock, flags);
316 static int hisi_dma_terminate_all(struct dma_chan *c)
318 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
322 spin_lock_irqsave(&chan->vc.lock, flags);
324 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
326 vchan_terminate_vdesc(&chan->desc->vd);
330 vchan_get_all_descriptors(&chan->vc, &head);
332 spin_unlock_irqrestore(&chan->vc.lock, flags);
334 vchan_dma_desc_free_list(&chan->vc, &head);
335 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
340 static void hisi_dma_synchronize(struct dma_chan *c)
342 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
344 vchan_synchronize(&chan->vc);
347 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
349 size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
350 size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
351 struct device *dev = &hdma_dev->pdev->dev;
352 struct hisi_dma_chan *chan;
355 for (i = 0; i < hdma_dev->chan_num; i++) {
356 chan = &hdma_dev->chan[i];
357 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
362 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
371 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
373 struct hisi_dma_chan *chan = &hdma_dev->chan[index];
374 u32 hw_depth = hdma_dev->chan_depth - 1;
375 void __iomem *base = hdma_dev->base;
377 /* set sq, cq base */
378 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
379 lower_32_bits(chan->sq_dma));
380 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
381 upper_32_bits(chan->sq_dma));
382 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
383 lower_32_bits(chan->cq_dma));
384 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
385 upper_32_bits(chan->cq_dma));
387 /* set sq, cq depth */
388 hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
389 hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
391 /* init sq tail and cq head */
392 hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
393 hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
396 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
398 hisi_dma_init_hw_qp(hdma_dev, qp_index);
399 hisi_dma_unmask_irq(hdma_dev, qp_index);
400 hisi_dma_enable_dma(hdma_dev, qp_index, true);
403 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
405 hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
408 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
412 for (i = 0; i < hdma_dev->chan_num; i++) {
413 hdma_dev->chan[i].qp_num = i;
414 hdma_dev->chan[i].hdma_dev = hdma_dev;
415 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
416 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
417 hisi_dma_enable_qp(hdma_dev, i);
421 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
425 for (i = 0; i < hdma_dev->chan_num; i++) {
426 hisi_dma_disable_qp(hdma_dev, i);
427 tasklet_kill(&hdma_dev->chan[i].vc.task);
431 static irqreturn_t hisi_dma_irq(int irq, void *data)
433 struct hisi_dma_chan *chan = data;
434 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
435 struct hisi_dma_desc *desc;
436 struct hisi_dma_cqe *cqe;
439 spin_lock_irqsave(&chan->vc.lock, flags);
442 cqe = chan->cq + chan->cq_head;
444 chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
445 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR,
446 chan->qp_num, chan->cq_head);
447 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
448 vchan_cookie_complete(&desc->vd);
449 hisi_dma_start_transfer(chan);
451 dev_err(&hdma_dev->pdev->dev, "task error!\n");
455 spin_unlock_irqrestore(&chan->vc.lock, flags);
460 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
462 struct pci_dev *pdev = hdma_dev->pdev;
465 for (i = 0; i < hdma_dev->chan_num; i++) {
466 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
467 hisi_dma_irq, IRQF_SHARED, "hisi_dma",
476 /* This function enables all hw channels in a device */
477 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
481 ret = hisi_dma_alloc_qps_mem(hdma_dev);
483 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
487 ret = hisi_dma_request_qps_irq(hdma_dev);
489 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
493 hisi_dma_enable_qps(hdma_dev);
498 static void hisi_dma_disable_hw_channels(void *data)
500 hisi_dma_disable_qps(data);
503 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
504 enum hisi_dma_mode mode)
506 writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
509 static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
511 struct device *dev = &pdev->dev;
512 struct hisi_dma_dev *hdma_dev;
513 struct dma_device *dma_dev;
516 ret = pcim_enable_device(pdev);
518 dev_err(dev, "failed to enable device mem!\n");
522 ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
524 dev_err(dev, "failed to remap I/O region!\n");
528 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
532 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
536 hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
540 hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
541 hdma_dev->pdev = pdev;
542 hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
543 hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
545 pci_set_drvdata(pdev, hdma_dev);
546 pci_set_master(pdev);
548 ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
551 dev_err(dev, "Failed to allocate MSI vectors!\n");
555 ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
559 dma_dev = &hdma_dev->dma_dev;
560 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
561 dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
562 dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
563 dma_dev->device_tx_status = hisi_dma_tx_status;
564 dma_dev->device_issue_pending = hisi_dma_issue_pending;
565 dma_dev->device_terminate_all = hisi_dma_terminate_all;
566 dma_dev->device_synchronize = hisi_dma_synchronize;
567 dma_dev->directions = BIT(DMA_MEM_TO_MEM);
569 INIT_LIST_HEAD(&dma_dev->channels);
571 hisi_dma_set_mode(hdma_dev, RC);
573 ret = hisi_dma_enable_hw_channels(hdma_dev);
575 dev_err(dev, "failed to enable hw channel!\n");
579 ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
584 ret = dmaenginem_async_device_register(dma_dev);
586 dev_err(dev, "failed to register device!\n");
591 static const struct pci_device_id hisi_dma_pci_tbl[] = {
592 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
596 static struct pci_driver hisi_dma_pci_driver = {
598 .id_table = hisi_dma_pci_tbl,
599 .probe = hisi_dma_probe,
602 module_pci_driver(hisi_dma_pci_driver);
604 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
605 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
606 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
607 MODULE_LICENSE("GPL v2");
608 MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);