GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / dma / hisi_dma.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019 HiSilicon Limited. */
3 #include <linux/bitfield.h>
4 #include <linux/dmaengine.h>
5 #include <linux/init.h>
6 #include <linux/iopoll.h>
7 #include <linux/module.h>
8 #include <linux/pci.h>
9 #include <linux/spinlock.h>
10 #include "virt-dma.h"
11
12 #define HISI_DMA_SQ_BASE_L              0x0
13 #define HISI_DMA_SQ_BASE_H              0x4
14 #define HISI_DMA_SQ_DEPTH               0x8
15 #define HISI_DMA_SQ_TAIL_PTR            0xc
16 #define HISI_DMA_CQ_BASE_L              0x10
17 #define HISI_DMA_CQ_BASE_H              0x14
18 #define HISI_DMA_CQ_DEPTH               0x18
19 #define HISI_DMA_CQ_HEAD_PTR            0x1c
20 #define HISI_DMA_CTRL0                  0x20
21 #define HISI_DMA_CTRL0_QUEUE_EN_S       0
22 #define HISI_DMA_CTRL0_QUEUE_PAUSE_S    4
23 #define HISI_DMA_CTRL1                  0x24
24 #define HISI_DMA_CTRL1_QUEUE_RESET_S    0
25 #define HISI_DMA_Q_FSM_STS              0x30
26 #define HISI_DMA_FSM_STS_MASK           GENMASK(3, 0)
27 #define HISI_DMA_INT_STS                0x40
28 #define HISI_DMA_INT_STS_MASK           GENMASK(12, 0)
29 #define HISI_DMA_INT_MSK                0x44
30 #define HISI_DMA_MODE                   0x217c
31 #define HISI_DMA_OFFSET                 0x100
32
33 #define HISI_DMA_MSI_NUM                32
34 #define HISI_DMA_CHAN_NUM               30
35 #define HISI_DMA_Q_DEPTH_VAL            1024
36
37 #define PCI_BAR_2                       2
38
39 enum hisi_dma_mode {
40         EP = 0,
41         RC,
42 };
43
44 enum hisi_dma_chan_status {
45         DISABLE = -1,
46         IDLE = 0,
47         RUN,
48         CPL,
49         PAUSE,
50         HALT,
51         ABORT,
52         WAIT,
53         BUFFCLR,
54 };
55
56 struct hisi_dma_sqe {
57         __le32 dw0;
58 #define OPCODE_MASK                     GENMASK(3, 0)
59 #define OPCODE_SMALL_PACKAGE            0x1
60 #define OPCODE_M2M                      0x4
61 #define LOCAL_IRQ_EN                    BIT(8)
62 #define ATTR_SRC_MASK                   GENMASK(14, 12)
63         __le32 dw1;
64         __le32 dw2;
65 #define ATTR_DST_MASK                   GENMASK(26, 24)
66         __le32 length;
67         __le64 src_addr;
68         __le64 dst_addr;
69 };
70
71 struct hisi_dma_cqe {
72         __le32 rsv0;
73         __le32 rsv1;
74         __le16 sq_head;
75         __le16 rsv2;
76         __le16 rsv3;
77         __le16 w0;
78 #define STATUS_MASK                     GENMASK(15, 1)
79 #define STATUS_SUCC                     0x0
80 #define VALID_BIT                       BIT(0)
81 };
82
83 struct hisi_dma_desc {
84         struct virt_dma_desc vd;
85         struct hisi_dma_sqe sqe;
86 };
87
88 struct hisi_dma_chan {
89         struct virt_dma_chan vc;
90         struct hisi_dma_dev *hdma_dev;
91         struct hisi_dma_sqe *sq;
92         struct hisi_dma_cqe *cq;
93         dma_addr_t sq_dma;
94         dma_addr_t cq_dma;
95         u32 sq_tail;
96         u32 cq_head;
97         u32 qp_num;
98         enum hisi_dma_chan_status status;
99         struct hisi_dma_desc *desc;
100 };
101
102 struct hisi_dma_dev {
103         struct pci_dev *pdev;
104         void __iomem *base;
105         struct dma_device dma_dev;
106         u32 chan_num;
107         u32 chan_depth;
108         struct hisi_dma_chan chan[];
109 };
110
111 static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
112 {
113         return container_of(c, struct hisi_dma_chan, vc.chan);
114 }
115
116 static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
117 {
118         return container_of(vd, struct hisi_dma_desc, vd);
119 }
120
121 static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
122                                        u32 val)
123 {
124         writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
125 }
126
127 static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
128 {
129         u32 tmp;
130
131         tmp = readl_relaxed(addr);
132         tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
133         writel_relaxed(tmp, addr);
134 }
135
136 static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
137                                bool pause)
138 {
139         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
140                              HISI_DMA_OFFSET;
141
142         hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
143 }
144
145 static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
146                                 bool enable)
147 {
148         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
149                              HISI_DMA_OFFSET;
150
151         hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
152 }
153
154 static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
155 {
156         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
157                             HISI_DMA_INT_STS_MASK);
158 }
159
160 static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
161 {
162         void __iomem *base = hdma_dev->base;
163
164         hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
165                             HISI_DMA_INT_STS_MASK);
166         hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
167 }
168
169 static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
170 {
171         void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
172                              HISI_DMA_OFFSET;
173
174         hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
175 }
176
177 static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
178 {
179         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
180         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
181 }
182
183 static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
184 {
185         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
186         u32 index = chan->qp_num, tmp;
187         int ret;
188
189         hisi_dma_pause_dma(hdma_dev, index, true);
190         hisi_dma_enable_dma(hdma_dev, index, false);
191         hisi_dma_mask_irq(hdma_dev, index);
192
193         ret = readl_relaxed_poll_timeout(hdma_dev->base +
194                 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
195                 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
196         if (ret) {
197                 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
198                 WARN_ON(1);
199         }
200
201         hisi_dma_do_reset(hdma_dev, index);
202         hisi_dma_reset_qp_point(hdma_dev, index);
203         hisi_dma_pause_dma(hdma_dev, index, false);
204         hisi_dma_enable_dma(hdma_dev, index, true);
205         hisi_dma_unmask_irq(hdma_dev, index);
206
207         ret = readl_relaxed_poll_timeout(hdma_dev->base +
208                 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
209                 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
210         if (ret) {
211                 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
212                 WARN_ON(1);
213         }
214 }
215
216 static void hisi_dma_free_chan_resources(struct dma_chan *c)
217 {
218         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
219         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
220
221         hisi_dma_reset_hw_chan(chan);
222         vchan_free_chan_resources(&chan->vc);
223
224         memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
225         memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
226         chan->sq_tail = 0;
227         chan->cq_head = 0;
228         chan->status = DISABLE;
229 }
230
231 static void hisi_dma_desc_free(struct virt_dma_desc *vd)
232 {
233         kfree(to_hisi_dma_desc(vd));
234 }
235
236 static struct dma_async_tx_descriptor *
237 hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
238                          size_t len, unsigned long flags)
239 {
240         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
241         struct hisi_dma_desc *desc;
242
243         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
244         if (!desc)
245                 return NULL;
246
247         desc->sqe.length = cpu_to_le32(len);
248         desc->sqe.src_addr = cpu_to_le64(src);
249         desc->sqe.dst_addr = cpu_to_le64(dst);
250
251         return vchan_tx_prep(&chan->vc, &desc->vd, flags);
252 }
253
254 static enum dma_status
255 hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
256                    struct dma_tx_state *txstate)
257 {
258         return dma_cookie_status(c, cookie, txstate);
259 }
260
261 static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
262 {
263         struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
264         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
265         struct hisi_dma_desc *desc;
266         struct virt_dma_desc *vd;
267
268         vd = vchan_next_desc(&chan->vc);
269         if (!vd) {
270                 dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
271                 chan->desc = NULL;
272                 return;
273         }
274         list_del(&vd->node);
275         desc = to_hisi_dma_desc(vd);
276         chan->desc = desc;
277
278         memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
279
280         /* update other field in sqe */
281         sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
282         sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
283
284         /* make sure data has been updated in sqe */
285         wmb();
286
287         /* update sq tail, point to new sqe position */
288         chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
289
290         /* update sq_tail to trigger a new task */
291         hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
292                             chan->sq_tail);
293 }
294
295 static void hisi_dma_issue_pending(struct dma_chan *c)
296 {
297         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
298         unsigned long flags;
299
300         spin_lock_irqsave(&chan->vc.lock, flags);
301
302         if (vchan_issue_pending(&chan->vc))
303                 hisi_dma_start_transfer(chan);
304
305         spin_unlock_irqrestore(&chan->vc.lock, flags);
306 }
307
308 static int hisi_dma_terminate_all(struct dma_chan *c)
309 {
310         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
311         unsigned long flags;
312         LIST_HEAD(head);
313
314         spin_lock_irqsave(&chan->vc.lock, flags);
315
316         hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
317         if (chan->desc) {
318                 vchan_terminate_vdesc(&chan->desc->vd);
319                 chan->desc = NULL;
320         }
321
322         vchan_get_all_descriptors(&chan->vc, &head);
323
324         spin_unlock_irqrestore(&chan->vc.lock, flags);
325
326         vchan_dma_desc_free_list(&chan->vc, &head);
327         hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
328
329         return 0;
330 }
331
332 static void hisi_dma_synchronize(struct dma_chan *c)
333 {
334         struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
335
336         vchan_synchronize(&chan->vc);
337 }
338
339 static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
340 {
341         size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
342         size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
343         struct device *dev = &hdma_dev->pdev->dev;
344         struct hisi_dma_chan *chan;
345         int i;
346
347         for (i = 0; i < hdma_dev->chan_num; i++) {
348                 chan = &hdma_dev->chan[i];
349                 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
350                                                GFP_KERNEL);
351                 if (!chan->sq)
352                         return -ENOMEM;
353
354                 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
355                                                GFP_KERNEL);
356                 if (!chan->cq)
357                         return -ENOMEM;
358         }
359
360         return 0;
361 }
362
363 static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
364 {
365         struct hisi_dma_chan *chan = &hdma_dev->chan[index];
366         u32 hw_depth = hdma_dev->chan_depth - 1;
367         void __iomem *base = hdma_dev->base;
368
369         /* set sq, cq base */
370         hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
371                             lower_32_bits(chan->sq_dma));
372         hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
373                             upper_32_bits(chan->sq_dma));
374         hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
375                             lower_32_bits(chan->cq_dma));
376         hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
377                             upper_32_bits(chan->cq_dma));
378
379         /* set sq, cq depth */
380         hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
381         hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
382
383         /* init sq tail and cq head */
384         hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
385         hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
386 }
387
388 static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
389 {
390         hisi_dma_init_hw_qp(hdma_dev, qp_index);
391         hisi_dma_unmask_irq(hdma_dev, qp_index);
392         hisi_dma_enable_dma(hdma_dev, qp_index, true);
393 }
394
395 static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
396 {
397         hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
398 }
399
400 static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
401 {
402         int i;
403
404         for (i = 0; i < hdma_dev->chan_num; i++) {
405                 hdma_dev->chan[i].qp_num = i;
406                 hdma_dev->chan[i].hdma_dev = hdma_dev;
407                 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
408                 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
409                 hisi_dma_enable_qp(hdma_dev, i);
410         }
411 }
412
413 static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
414 {
415         int i;
416
417         for (i = 0; i < hdma_dev->chan_num; i++) {
418                 hisi_dma_disable_qp(hdma_dev, i);
419                 tasklet_kill(&hdma_dev->chan[i].vc.task);
420         }
421 }
422
423 static irqreturn_t hisi_dma_irq(int irq, void *data)
424 {
425         struct hisi_dma_chan *chan = data;
426         struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
427         struct hisi_dma_desc *desc;
428         struct hisi_dma_cqe *cqe;
429
430         spin_lock(&chan->vc.lock);
431
432         desc = chan->desc;
433         cqe = chan->cq + chan->cq_head;
434         if (desc) {
435                 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
436                         chan->cq_head = (chan->cq_head + 1) %
437                                         hdma_dev->chan_depth;
438                         hisi_dma_chan_write(hdma_dev->base,
439                                             HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
440                                             chan->cq_head);
441                         vchan_cookie_complete(&desc->vd);
442                 } else {
443                         dev_err(&hdma_dev->pdev->dev, "task error!\n");
444                 }
445
446                 chan->desc = NULL;
447         }
448
449         spin_unlock(&chan->vc.lock);
450
451         return IRQ_HANDLED;
452 }
453
454 static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
455 {
456         struct pci_dev *pdev = hdma_dev->pdev;
457         int i, ret;
458
459         for (i = 0; i < hdma_dev->chan_num; i++) {
460                 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
461                                        hisi_dma_irq, IRQF_SHARED, "hisi_dma",
462                                        &hdma_dev->chan[i]);
463                 if (ret)
464                         return ret;
465         }
466
467         return 0;
468 }
469
470 /* This function enables all hw channels in a device */
471 static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
472 {
473         int ret;
474
475         ret = hisi_dma_alloc_qps_mem(hdma_dev);
476         if (ret) {
477                 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
478                 return ret;
479         }
480
481         ret = hisi_dma_request_qps_irq(hdma_dev);
482         if (ret) {
483                 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
484                 return ret;
485         }
486
487         hisi_dma_enable_qps(hdma_dev);
488
489         return 0;
490 }
491
492 static void hisi_dma_disable_hw_channels(void *data)
493 {
494         hisi_dma_disable_qps(data);
495 }
496
497 static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
498                               enum hisi_dma_mode mode)
499 {
500         writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
501 }
502
503 static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
504 {
505         struct device *dev = &pdev->dev;
506         struct hisi_dma_dev *hdma_dev;
507         struct dma_device *dma_dev;
508         int ret;
509
510         ret = pcim_enable_device(pdev);
511         if (ret) {
512                 dev_err(dev, "failed to enable device mem!\n");
513                 return ret;
514         }
515
516         ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
517         if (ret) {
518                 dev_err(dev, "failed to remap I/O region!\n");
519                 return ret;
520         }
521
522         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
523         if (ret)
524                 return ret;
525
526         hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
527         if (!hdma_dev)
528                 return -EINVAL;
529
530         hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
531         hdma_dev->pdev = pdev;
532         hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
533         hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
534
535         pci_set_drvdata(pdev, hdma_dev);
536         pci_set_master(pdev);
537
538         /* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
539         ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
540                                     PCI_IRQ_MSI);
541         if (ret < 0) {
542                 dev_err(dev, "Failed to allocate MSI vectors!\n");
543                 return ret;
544         }
545
546         dma_dev = &hdma_dev->dma_dev;
547         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
548         dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
549         dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
550         dma_dev->device_tx_status = hisi_dma_tx_status;
551         dma_dev->device_issue_pending = hisi_dma_issue_pending;
552         dma_dev->device_terminate_all = hisi_dma_terminate_all;
553         dma_dev->device_synchronize = hisi_dma_synchronize;
554         dma_dev->directions = BIT(DMA_MEM_TO_MEM);
555         dma_dev->dev = dev;
556         INIT_LIST_HEAD(&dma_dev->channels);
557
558         hisi_dma_set_mode(hdma_dev, RC);
559
560         ret = hisi_dma_enable_hw_channels(hdma_dev);
561         if (ret < 0) {
562                 dev_err(dev, "failed to enable hw channel!\n");
563                 return ret;
564         }
565
566         ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
567                                        hdma_dev);
568         if (ret)
569                 return ret;
570
571         ret = dmaenginem_async_device_register(dma_dev);
572         if (ret < 0)
573                 dev_err(dev, "failed to register device!\n");
574
575         return ret;
576 }
577
578 static const struct pci_device_id hisi_dma_pci_tbl[] = {
579         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
580         { 0, }
581 };
582
583 static struct pci_driver hisi_dma_pci_driver = {
584         .name           = "hisi_dma",
585         .id_table       = hisi_dma_pci_tbl,
586         .probe          = hisi_dma_probe,
587 };
588
589 module_pci_driver(hisi_dma_pci_driver);
590
591 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
592 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
593 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
594 MODULE_LICENSE("GPL v2");
595 MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);