1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
4 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
7 #include <linux/kernel.h>
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/err.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/list.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/sys_soc.h>
21 #include <linux/of_dma.h>
22 #include <linux/of_device.h>
23 #include <linux/of_irq.h>
24 #include <linux/workqueue.h>
25 #include <linux/completion.h>
26 #include <linux/soc/ti/k3-ringacc.h>
27 #include <linux/soc/ti/ti_sci_protocol.h>
28 #include <linux/soc/ti/ti_sci_inta_msi.h>
29 #include <linux/dma/ti-cppi5.h>
31 #include "../virt-dma.h"
33 #include "k3-psil-priv.h"
35 struct udma_static_tr {
36 u8 elsize; /* RPSTR0 */
37 u16 elcnt; /* RPSTR0 */
38 u16 bstcnt; /* RPSTR1 */
41 #define K3_UDMA_MAX_RFLOWS 1024
42 #define K3_UDMA_DEFAULT_RING_SIZE 16
44 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */
45 #define UDMA_RFLOW_SRCTAG_NONE 0
46 #define UDMA_RFLOW_SRCTAG_CFG_TAG 1
47 #define UDMA_RFLOW_SRCTAG_FLOW_ID 2
48 #define UDMA_RFLOW_SRCTAG_SRC_TAG 4
50 #define UDMA_RFLOW_DSTTAG_NONE 0
51 #define UDMA_RFLOW_DSTTAG_CFG_TAG 1
52 #define UDMA_RFLOW_DSTTAG_FLOW_ID 2
53 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4
54 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5
65 static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" };
71 struct k3_ring *t_ring; /* Transmit ring */
72 struct k3_ring *tc_ring; /* Transmit Completion ring */
77 struct k3_ring *fd_ring; /* Free Descriptor ring */
78 struct k3_ring *r_ring; /* Receive ring */
87 #define UDMA_FLAG_PDMA_ACC32 BIT(0)
88 #define UDMA_FLAG_PDMA_BURST BIT(1)
90 struct udma_match_data {
92 bool enable_memcpy_support;
97 struct udma_soc_data {
102 size_t cppi5_desc_size;
103 void *cppi5_desc_vaddr;
104 dma_addr_t cppi5_desc_paddr;
106 /* TR descriptor internal pointers */
108 struct cppi5_tr_resp_t *tr_resp_base;
111 struct udma_rx_flush {
112 struct udma_hwdesc hwdescs[2];
116 dma_addr_t buffer_paddr;
120 struct dma_device ddev;
122 void __iomem *mmrs[MMR_LAST];
123 const struct udma_match_data *match_data;
124 const struct udma_soc_data *soc_data;
127 u32 tpl_start_idx[3];
129 size_t desc_align; /* alignment to use for descriptors */
131 struct udma_tisci_rm tisci_rm;
133 struct k3_ringacc *ringacc;
135 struct work_struct purge_work;
136 struct list_head desc_to_purge;
139 struct udma_rx_flush rx_flush;
145 unsigned long *tchan_map;
146 unsigned long *rchan_map;
147 unsigned long *rflow_gp_map;
148 unsigned long *rflow_gp_map_allocated;
149 unsigned long *rflow_in_use;
151 struct udma_tchan *tchans;
152 struct udma_rchan *rchans;
153 struct udma_rflow *rflows;
155 struct udma_chan *channels;
161 struct virt_dma_desc vd;
165 enum dma_transfer_direction dir;
167 struct udma_static_tr static_tr;
171 unsigned int desc_idx; /* Only used for cyclic in packet mode */
175 void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
177 unsigned int hwdesc_count;
178 struct udma_hwdesc hwdesc[];
181 enum udma_chan_state {
182 UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
183 UDMA_CHAN_IS_ACTIVE, /* Normal operation */
184 UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
187 struct udma_tx_drain {
188 struct delayed_work work;
193 struct udma_chan_config {
194 bool pkt_mode; /* TR or packet */
195 bool needs_epib; /* EPIB is needed for the communication or not */
196 u32 psd_size; /* size of Protocol Specific Data */
197 u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
198 u32 hdesc_size; /* Size of a packet descriptor in packet mode */
199 bool notdpkt; /* Suppress sending TDC packet */
200 int remote_thread_id;
204 enum psil_endpoint_type ep_type;
207 enum udma_tp_level channel_tpl; /* Channel Throughput Level */
209 enum dma_transfer_direction dir;
213 struct virt_dma_chan vc;
214 struct dma_slave_config cfg;
216 struct udma_desc *desc;
217 struct udma_desc *terminated_desc;
218 struct udma_static_tr static_tr;
221 struct udma_tchan *tchan;
222 struct udma_rchan *rchan;
223 struct udma_rflow *rflow;
233 enum udma_chan_state state;
234 struct completion teardown_completed;
236 struct udma_tx_drain tx_drain;
238 u32 bcnt; /* number of bytes completed since the start of the channel */
240 /* Channel configuration parameters */
241 struct udma_chan_config config;
243 /* dmapool for packet mode descriptors */
245 struct dma_pool *hdesc_pool;
250 static inline struct udma_dev *to_udma_dev(struct dma_device *d)
252 return container_of(d, struct udma_dev, ddev);
255 static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
257 return container_of(c, struct udma_chan, vc.chan);
260 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
262 return container_of(t, struct udma_desc, vd.tx);
265 /* Generic register access functions */
266 static inline u32 udma_read(void __iomem *base, int reg)
268 return readl(base + reg);
271 static inline void udma_write(void __iomem *base, int reg, u32 val)
273 writel(val, base + reg);
276 static inline void udma_update_bits(void __iomem *base, int reg,
281 orig = readl(base + reg);
286 writel(tmp, base + reg);
290 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)
294 return udma_read(uc->tchan->reg_rt, reg);
297 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)
301 udma_write(uc->tchan->reg_rt, reg, val);
304 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,
309 udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
313 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)
317 return udma_read(uc->rchan->reg_rt, reg);
320 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)
324 udma_write(uc->rchan->reg_rt, reg, val);
327 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,
332 udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
335 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
337 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
339 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
340 return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
341 tisci_rm->tisci_navss_dev_id,
342 src_thread, dst_thread);
345 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
348 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
350 dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
351 return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
352 tisci_rm->tisci_navss_dev_id,
353 src_thread, dst_thread);
356 static void udma_reset_uchan(struct udma_chan *uc)
358 memset(&uc->config, 0, sizeof(uc->config));
359 uc->config.remote_thread_id = -1;
360 uc->state = UDMA_CHAN_IS_IDLE;
363 static void udma_dump_chan_stdata(struct udma_chan *uc)
365 struct device *dev = uc->ud->dev;
369 if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
370 dev_dbg(dev, "TCHAN State data:\n");
371 for (i = 0; i < 32; i++) {
372 offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
373 dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
374 udma_tchanrt_read(uc, offset));
378 if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
379 dev_dbg(dev, "RCHAN State data:\n");
380 for (i = 0; i < 32; i++) {
381 offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
382 dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
383 udma_rchanrt_read(uc, offset));
388 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
391 return d->hwdesc[idx].cppi5_desc_paddr;
394 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
396 return d->hwdesc[idx].cppi5_desc_vaddr;
399 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
402 struct udma_desc *d = uc->terminated_desc;
405 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
408 if (desc_paddr != paddr)
415 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
418 if (desc_paddr != paddr)
426 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
428 if (uc->use_dma_pool) {
431 for (i = 0; i < d->hwdesc_count; i++) {
432 if (!d->hwdesc[i].cppi5_desc_vaddr)
435 dma_pool_free(uc->hdesc_pool,
436 d->hwdesc[i].cppi5_desc_vaddr,
437 d->hwdesc[i].cppi5_desc_paddr);
439 d->hwdesc[i].cppi5_desc_vaddr = NULL;
441 } else if (d->hwdesc[0].cppi5_desc_vaddr) {
442 struct udma_dev *ud = uc->ud;
444 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size,
445 d->hwdesc[0].cppi5_desc_vaddr,
446 d->hwdesc[0].cppi5_desc_paddr);
448 d->hwdesc[0].cppi5_desc_vaddr = NULL;
452 static void udma_purge_desc_work(struct work_struct *work)
454 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
455 struct virt_dma_desc *vd, *_vd;
459 spin_lock_irqsave(&ud->lock, flags);
460 list_splice_tail_init(&ud->desc_to_purge, &head);
461 spin_unlock_irqrestore(&ud->lock, flags);
463 list_for_each_entry_safe(vd, _vd, &head, node) {
464 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
465 struct udma_desc *d = to_udma_desc(&vd->tx);
467 udma_free_hwdesc(uc, d);
472 /* If more to purge, schedule the work again */
473 if (!list_empty(&ud->desc_to_purge))
474 schedule_work(&ud->purge_work);
477 static void udma_desc_free(struct virt_dma_desc *vd)
479 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
480 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
481 struct udma_desc *d = to_udma_desc(&vd->tx);
484 if (uc->terminated_desc == d)
485 uc->terminated_desc = NULL;
487 if (uc->use_dma_pool) {
488 udma_free_hwdesc(uc, d);
493 spin_lock_irqsave(&ud->lock, flags);
494 list_add_tail(&vd->node, &ud->desc_to_purge);
495 spin_unlock_irqrestore(&ud->lock, flags);
497 schedule_work(&ud->purge_work);
500 static bool udma_is_chan_running(struct udma_chan *uc)
506 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
508 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
510 if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
516 static bool udma_is_chan_paused(struct udma_chan *uc)
520 switch (uc->config.dir) {
522 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
523 pause_mask = UDMA_PEER_RT_EN_PAUSE;
526 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
527 pause_mask = UDMA_PEER_RT_EN_PAUSE;
530 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
531 pause_mask = UDMA_CHAN_RT_CTL_PAUSE;
537 if (val & pause_mask)
543 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
545 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
548 static int udma_push_to_ring(struct udma_chan *uc, int idx)
550 struct udma_desc *d = uc->desc;
551 struct k3_ring *ring = NULL;
554 switch (uc->config.dir) {
556 ring = uc->rflow->fd_ring;
560 ring = uc->tchan->t_ring;
566 /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
568 paddr = udma_get_rx_flush_hwdesc_paddr(uc);
570 paddr = udma_curr_cppi5_desc_paddr(d, idx);
572 wmb(); /* Ensure that writes are not moved over this point */
575 return k3_ringacc_ring_push(ring, &paddr);
578 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
580 if (uc->config.dir != DMA_DEV_TO_MEM)
583 if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
589 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
591 struct k3_ring *ring = NULL;
594 switch (uc->config.dir) {
596 ring = uc->rflow->r_ring;
600 ring = uc->tchan->tc_ring;
606 ret = k3_ringacc_ring_pop(ring, addr);
610 rmb(); /* Ensure that reads are not moved before this point */
612 /* Teardown completion */
613 if (cppi5_desc_is_tdcm(*addr))
616 /* Check for flush descriptor */
617 if (udma_desc_is_rx_flush(uc, *addr))
623 static void udma_reset_rings(struct udma_chan *uc)
625 struct k3_ring *ring1 = NULL;
626 struct k3_ring *ring2 = NULL;
628 switch (uc->config.dir) {
631 ring1 = uc->rflow->fd_ring;
632 ring2 = uc->rflow->r_ring;
638 ring1 = uc->tchan->t_ring;
639 ring2 = uc->tchan->tc_ring;
647 k3_ringacc_ring_reset_dma(ring1,
648 k3_ringacc_ring_get_occ(ring1));
650 k3_ringacc_ring_reset(ring2);
652 /* make sure we are not leaking memory by stalled descriptor */
653 if (uc->terminated_desc) {
654 udma_desc_free(&uc->terminated_desc->vd);
655 uc->terminated_desc = NULL;
659 static void udma_reset_counters(struct udma_chan *uc)
664 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
665 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
667 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
668 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
670 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
671 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
673 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
674 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
678 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
679 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
681 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
682 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
684 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
685 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
687 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
688 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
694 static int udma_reset_chan(struct udma_chan *uc, bool hard)
696 switch (uc->config.dir) {
698 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
699 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
702 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
703 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
706 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
707 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
713 /* Reset all counters */
714 udma_reset_counters(uc);
716 /* Hard reset: re-initialize the channel to reset */
718 struct udma_chan_config ucc_backup;
721 memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
722 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
724 /* restore the channel configuration */
725 memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
726 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
731 * Setting forced teardown after forced reset helps recovering
734 if (uc->config.dir == DMA_DEV_TO_MEM)
735 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
736 UDMA_CHAN_RT_CTL_EN |
737 UDMA_CHAN_RT_CTL_TDOWN |
738 UDMA_CHAN_RT_CTL_FTDOWN);
740 uc->state = UDMA_CHAN_IS_IDLE;
745 static void udma_start_desc(struct udma_chan *uc)
747 struct udma_chan_config *ucc = &uc->config;
749 if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
752 /* Push all descriptors to ring for packet mode cyclic or RX */
753 for (i = 0; i < uc->desc->sglen; i++)
754 udma_push_to_ring(uc, i);
756 udma_push_to_ring(uc, 0);
760 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
762 /* Only PDMAs have staticTR */
763 if (uc->config.ep_type == PSIL_EP_NATIVE)
766 /* Check if the staticTR configuration has changed for TX */
767 if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
773 static int udma_start(struct udma_chan *uc)
775 struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
784 uc->desc = to_udma_desc(&vd->tx);
786 /* Channel is already running and does not need reconfiguration */
787 if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
792 /* Make sure that we clear the teardown bit, if it is set */
793 udma_reset_chan(uc, false);
795 /* Push descriptors before we start the channel */
798 switch (uc->desc->dir) {
800 /* Config remote TR */
801 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
802 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
803 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
804 const struct udma_match_data *match_data =
807 if (uc->config.enable_acc32)
808 val |= PDMA_STATIC_TR_XY_ACC32;
809 if (uc->config.enable_burst)
810 val |= PDMA_STATIC_TR_XY_BURST;
812 udma_rchanrt_write(uc,
813 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
816 udma_rchanrt_write(uc,
817 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,
818 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
819 match_data->statictr_z_mask));
821 /* save the current staticTR configuration */
822 memcpy(&uc->static_tr, &uc->desc->static_tr,
823 sizeof(uc->static_tr));
826 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
827 UDMA_CHAN_RT_CTL_EN);
830 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
831 UDMA_PEER_RT_EN_ENABLE);
835 /* Config remote TR */
836 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
837 u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
838 PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
840 if (uc->config.enable_acc32)
841 val |= PDMA_STATIC_TR_XY_ACC32;
842 if (uc->config.enable_burst)
843 val |= PDMA_STATIC_TR_XY_BURST;
845 udma_tchanrt_write(uc,
846 UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
849 /* save the current staticTR configuration */
850 memcpy(&uc->static_tr, &uc->desc->static_tr,
851 sizeof(uc->static_tr));
855 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
856 UDMA_PEER_RT_EN_ENABLE);
858 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
859 UDMA_CHAN_RT_CTL_EN);
863 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
864 UDMA_CHAN_RT_CTL_EN);
865 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
866 UDMA_CHAN_RT_CTL_EN);
873 uc->state = UDMA_CHAN_IS_ACTIVE;
879 static int udma_stop(struct udma_chan *uc)
881 enum udma_chan_state old_state = uc->state;
883 uc->state = UDMA_CHAN_IS_TERMINATING;
884 reinit_completion(&uc->teardown_completed);
886 switch (uc->config.dir) {
888 if (!uc->cyclic && !uc->desc)
889 udma_push_to_ring(uc, -1);
891 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
892 UDMA_PEER_RT_EN_ENABLE |
893 UDMA_PEER_RT_EN_TEARDOWN);
896 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
897 UDMA_PEER_RT_EN_ENABLE |
898 UDMA_PEER_RT_EN_FLUSH);
899 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
900 UDMA_CHAN_RT_CTL_EN |
901 UDMA_CHAN_RT_CTL_TDOWN);
904 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
905 UDMA_CHAN_RT_CTL_EN |
906 UDMA_CHAN_RT_CTL_TDOWN);
909 uc->state = old_state;
910 complete_all(&uc->teardown_completed);
917 static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
919 struct udma_desc *d = uc->desc;
920 struct cppi5_host_desc_t *h_desc;
922 h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
923 cppi5_hdesc_reset_to_original(h_desc);
924 udma_push_to_ring(uc, d->desc_idx);
925 d->desc_idx = (d->desc_idx + 1) % d->sglen;
928 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
930 struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
932 memcpy(d->metadata, h_desc->epib, d->metadata_size);
935 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
939 /* Only TX towards PDMA is affected */
940 if (uc->config.ep_type == PSIL_EP_NATIVE ||
941 uc->config.dir != DMA_MEM_TO_DEV)
944 peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
945 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
947 /* Transfer is incomplete, store current residue and time stamp */
948 if (peer_bcnt < bcnt) {
949 uc->tx_drain.residue = bcnt - peer_bcnt;
950 uc->tx_drain.tstamp = ktime_get();
957 static void udma_check_tx_completion(struct work_struct *work)
959 struct udma_chan *uc = container_of(work, typeof(*uc),
961 bool desc_done = true;
968 /* Get previous residue and time stamp */
969 residue_diff = uc->tx_drain.residue;
970 time_diff = uc->tx_drain.tstamp;
972 * Get current residue and time stamp or see if
973 * transfer is complete
975 desc_done = udma_is_desc_really_done(uc, uc->desc);
980 * Find the time delta and residue delta w.r.t
983 time_diff = ktime_sub(uc->tx_drain.tstamp,
985 residue_diff -= uc->tx_drain.residue;
988 * Try to guess when we should check
989 * next time by calculating rate at
990 * which data is being drained at the
993 delay = (time_diff / residue_diff) *
994 uc->tx_drain.residue;
996 /* No progress, check again in 1 second */
997 schedule_delayed_work(&uc->tx_drain.work, HZ);
1001 usleep_range(ktime_to_us(delay),
1002 ktime_to_us(delay) + 10);
1007 struct udma_desc *d = uc->desc;
1009 uc->bcnt += d->residue;
1011 vchan_cookie_complete(&d->vd);
1019 static irqreturn_t udma_ring_irq_handler(int irq, void *data)
1021 struct udma_chan *uc = data;
1022 struct udma_desc *d;
1023 unsigned long flags;
1024 dma_addr_t paddr = 0;
1026 if (udma_pop_from_ring(uc, &paddr) || !paddr)
1029 spin_lock_irqsave(&uc->vc.lock, flags);
1031 /* Teardown completion message */
1032 if (cppi5_desc_is_tdcm(paddr)) {
1033 complete_all(&uc->teardown_completed);
1035 if (uc->terminated_desc) {
1036 udma_desc_free(&uc->terminated_desc->vd);
1037 uc->terminated_desc = NULL;
1046 d = udma_udma_desc_from_paddr(uc, paddr);
1049 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
1051 if (desc_paddr != paddr) {
1052 dev_err(uc->ud->dev, "not matching descriptors!\n");
1056 if (d == uc->desc) {
1057 /* active descriptor */
1059 udma_cyclic_packet_elapsed(uc);
1060 vchan_cyclic_callback(&d->vd);
1062 if (udma_is_desc_really_done(uc, d)) {
1063 uc->bcnt += d->residue;
1065 vchan_cookie_complete(&d->vd);
1067 schedule_delayed_work(&uc->tx_drain.work,
1073 * terminated descriptor, mark the descriptor as
1074 * completed to update the channel's cookie marker
1076 dma_cookie_complete(&d->vd.tx);
1080 spin_unlock_irqrestore(&uc->vc.lock, flags);
1085 static irqreturn_t udma_udma_irq_handler(int irq, void *data)
1087 struct udma_chan *uc = data;
1088 struct udma_desc *d;
1089 unsigned long flags;
1091 spin_lock_irqsave(&uc->vc.lock, flags);
1094 d->tr_idx = (d->tr_idx + 1) % d->sglen;
1097 vchan_cyclic_callback(&d->vd);
1099 /* TODO: figure out the real amount of data */
1100 uc->bcnt += d->residue;
1102 vchan_cookie_complete(&d->vd);
1106 spin_unlock_irqrestore(&uc->vc.lock, flags);
1112 * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
1114 * @from: Start the search from this flow id number
1115 * @cnt: Number of consecutive flow ids to allocate
1117 * Allocate range of RX flow ids for future use, those flows can be requested
1118 * only using explicit flow id number. if @from is set to -1 it will try to find
1119 * first free range. if @from is positive value it will force allocation only
1120 * of the specified range of flows.
1122 * Returns -ENOMEM if can't find free range.
1123 * -EEXIST if requested range is busy.
1124 * -EINVAL if wrong input values passed.
1125 * Returns flow id on success.
1127 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1129 int start, tmp_from;
1130 DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
1134 tmp_from = ud->rchan_cnt;
1135 /* default flows can't be allocated and accessible only by id */
1136 if (tmp_from < ud->rchan_cnt)
1139 if (tmp_from + cnt > ud->rflow_cnt)
1142 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
1145 start = bitmap_find_next_zero_area(tmp,
1148 if (start >= ud->rflow_cnt)
1151 if (from >= 0 && start != from)
1154 bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
1158 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1160 if (from < ud->rchan_cnt)
1162 if (from + cnt > ud->rflow_cnt)
1165 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
1169 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
1172 * Attempt to request rflow by ID can be made for any rflow
1173 * if not in use with assumption that caller knows what's doing.
1174 * TI-SCI FW will perform additional permission check ant way, it's
1178 if (id < 0 || id >= ud->rflow_cnt)
1179 return ERR_PTR(-ENOENT);
1181 if (test_bit(id, ud->rflow_in_use))
1182 return ERR_PTR(-ENOENT);
1184 /* GP rflow has to be allocated first */
1185 if (!test_bit(id, ud->rflow_gp_map) &&
1186 !test_bit(id, ud->rflow_gp_map_allocated))
1187 return ERR_PTR(-EINVAL);
1189 dev_dbg(ud->dev, "get rflow%d\n", id);
1190 set_bit(id, ud->rflow_in_use);
1191 return &ud->rflows[id];
1194 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
1196 if (!test_bit(rflow->id, ud->rflow_in_use)) {
1197 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
1201 dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
1202 clear_bit(rflow->id, ud->rflow_in_use);
1205 #define UDMA_RESERVE_RESOURCE(res) \
1206 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
1207 enum udma_tp_level tpl, \
1211 if (test_bit(id, ud->res##_map)) { \
1212 dev_err(ud->dev, "res##%d is in use\n", id); \
1213 return ERR_PTR(-ENOENT); \
1218 if (tpl >= ud->tpl_levels) \
1219 tpl = ud->tpl_levels - 1; \
1221 start = ud->tpl_start_idx[tpl]; \
1223 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \
1225 if (id == ud->res##_cnt) { \
1226 return ERR_PTR(-ENOENT); \
1230 set_bit(id, ud->res##_map); \
1231 return &ud->res##s[id]; \
1234 UDMA_RESERVE_RESOURCE(tchan);
1235 UDMA_RESERVE_RESOURCE(rchan);
1237 static int udma_get_tchan(struct udma_chan *uc)
1239 struct udma_dev *ud = uc->ud;
1242 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
1243 uc->id, uc->tchan->id);
1247 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1);
1249 return PTR_ERR_OR_ZERO(uc->tchan);
1252 static int udma_get_rchan(struct udma_chan *uc)
1254 struct udma_dev *ud = uc->ud;
1257 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
1258 uc->id, uc->rchan->id);
1262 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1);
1264 return PTR_ERR_OR_ZERO(uc->rchan);
1267 static int udma_get_chan_pair(struct udma_chan *uc)
1269 struct udma_dev *ud = uc->ud;
1272 if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
1273 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
1274 uc->id, uc->tchan->id);
1279 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
1280 uc->id, uc->tchan->id);
1282 } else if (uc->rchan) {
1283 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
1284 uc->id, uc->rchan->id);
1288 /* Can be optimized, but let's have it like this for now */
1289 end = min(ud->tchan_cnt, ud->rchan_cnt);
1290 /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */
1291 chan_id = ud->tpl_start_idx[ud->tpl_levels - 1];
1292 for (; chan_id < end; chan_id++) {
1293 if (!test_bit(chan_id, ud->tchan_map) &&
1294 !test_bit(chan_id, ud->rchan_map))
1301 set_bit(chan_id, ud->tchan_map);
1302 set_bit(chan_id, ud->rchan_map);
1303 uc->tchan = &ud->tchans[chan_id];
1304 uc->rchan = &ud->rchans[chan_id];
1309 static int udma_get_rflow(struct udma_chan *uc, int flow_id)
1311 struct udma_dev *ud = uc->ud;
1314 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
1319 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
1320 uc->id, uc->rflow->id);
1324 uc->rflow = __udma_get_rflow(ud, flow_id);
1326 return PTR_ERR_OR_ZERO(uc->rflow);
1329 static void udma_put_rchan(struct udma_chan *uc)
1331 struct udma_dev *ud = uc->ud;
1334 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
1336 clear_bit(uc->rchan->id, ud->rchan_map);
1341 static void udma_put_tchan(struct udma_chan *uc)
1343 struct udma_dev *ud = uc->ud;
1346 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
1348 clear_bit(uc->tchan->id, ud->tchan_map);
1353 static void udma_put_rflow(struct udma_chan *uc)
1355 struct udma_dev *ud = uc->ud;
1358 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
1360 __udma_put_rflow(ud, uc->rflow);
1365 static void udma_free_tx_resources(struct udma_chan *uc)
1370 k3_ringacc_ring_free(uc->tchan->t_ring);
1371 k3_ringacc_ring_free(uc->tchan->tc_ring);
1372 uc->tchan->t_ring = NULL;
1373 uc->tchan->tc_ring = NULL;
1378 static int udma_alloc_tx_resources(struct udma_chan *uc)
1380 struct k3_ring_cfg ring_cfg;
1381 struct udma_dev *ud = uc->ud;
1384 ret = udma_get_tchan(uc);
1388 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1,
1390 &uc->tchan->tc_ring);
1396 memset(&ring_cfg, 0, sizeof(ring_cfg));
1397 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1398 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1399 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1401 ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
1402 ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
1410 k3_ringacc_ring_free(uc->tchan->tc_ring);
1411 uc->tchan->tc_ring = NULL;
1412 k3_ringacc_ring_free(uc->tchan->t_ring);
1413 uc->tchan->t_ring = NULL;
1420 static void udma_free_rx_resources(struct udma_chan *uc)
1426 struct udma_rflow *rflow = uc->rflow;
1428 k3_ringacc_ring_free(rflow->fd_ring);
1429 k3_ringacc_ring_free(rflow->r_ring);
1430 rflow->fd_ring = NULL;
1431 rflow->r_ring = NULL;
1439 static int udma_alloc_rx_resources(struct udma_chan *uc)
1441 struct udma_dev *ud = uc->ud;
1442 struct k3_ring_cfg ring_cfg;
1443 struct udma_rflow *rflow;
1447 ret = udma_get_rchan(uc);
1451 /* For MEM_TO_MEM we don't need rflow or rings */
1452 if (uc->config.dir == DMA_MEM_TO_MEM)
1455 ret = udma_get_rflow(uc, uc->rchan->id);
1462 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id;
1463 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
1464 &rflow->fd_ring, &rflow->r_ring);
1470 memset(&ring_cfg, 0, sizeof(ring_cfg));
1472 if (uc->config.pkt_mode)
1473 ring_cfg.size = SG_MAX_SEGMENTS;
1475 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1477 ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1478 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1480 ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
1481 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1482 ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
1490 k3_ringacc_ring_free(rflow->r_ring);
1491 rflow->r_ring = NULL;
1492 k3_ringacc_ring_free(rflow->fd_ring);
1493 rflow->fd_ring = NULL;
1502 #define TISCI_TCHAN_VALID_PARAMS ( \
1503 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1504 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \
1505 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \
1506 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1507 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \
1508 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1509 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1510 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1512 #define TISCI_RCHAN_VALID_PARAMS ( \
1513 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
1514 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
1515 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
1516 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
1517 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \
1518 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \
1519 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \
1520 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \
1521 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1523 static int udma_tisci_m2m_channel_config(struct udma_chan *uc)
1525 struct udma_dev *ud = uc->ud;
1526 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1527 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1528 struct udma_tchan *tchan = uc->tchan;
1529 struct udma_rchan *rchan = uc->rchan;
1532 /* Non synchronized - mem to mem type of transfer */
1533 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1534 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1535 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1537 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS;
1538 req_tx.nav_id = tisci_rm->tisci_dev_id;
1539 req_tx.index = tchan->id;
1540 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1541 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1542 req_tx.txcq_qnum = tc_ring;
1543 req_tx.tx_atype = ud->atype;
1545 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1547 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1551 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS;
1552 req_rx.nav_id = tisci_rm->tisci_dev_id;
1553 req_rx.index = rchan->id;
1554 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1555 req_rx.rxcq_qnum = tc_ring;
1556 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1557 req_rx.rx_atype = ud->atype;
1559 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1561 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
1566 static int udma_tisci_tx_channel_config(struct udma_chan *uc)
1568 struct udma_dev *ud = uc->ud;
1569 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1570 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1571 struct udma_tchan *tchan = uc->tchan;
1572 int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1573 struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1574 u32 mode, fetch_size;
1577 if (uc->config.pkt_mode) {
1578 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1579 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1580 uc->config.psd_size, 0);
1582 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
1583 fetch_size = sizeof(struct cppi5_desc_hdr_t);
1586 req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS;
1587 req_tx.nav_id = tisci_rm->tisci_dev_id;
1588 req_tx.index = tchan->id;
1589 req_tx.tx_chan_type = mode;
1590 req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1591 req_tx.tx_fetch_size = fetch_size >> 2;
1592 req_tx.txcq_qnum = tc_ring;
1593 req_tx.tx_atype = uc->config.atype;
1595 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1597 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1602 static int udma_tisci_rx_channel_config(struct udma_chan *uc)
1604 struct udma_dev *ud = uc->ud;
1605 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1606 const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1607 struct udma_rchan *rchan = uc->rchan;
1608 int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
1609 int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
1610 struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1611 struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
1612 u32 mode, fetch_size;
1615 if (uc->config.pkt_mode) {
1616 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1617 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1618 uc->config.psd_size, 0);
1620 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
1621 fetch_size = sizeof(struct cppi5_desc_hdr_t);
1624 req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS;
1625 req_rx.nav_id = tisci_rm->tisci_dev_id;
1626 req_rx.index = rchan->id;
1627 req_rx.rx_fetch_size = fetch_size >> 2;
1628 req_rx.rxcq_qnum = rx_ring;
1629 req_rx.rx_chan_type = mode;
1630 req_rx.rx_atype = uc->config.atype;
1632 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1634 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
1638 flow_req.valid_params =
1639 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
1640 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
1641 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
1642 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
1643 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1644 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
1645 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
1646 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
1647 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
1648 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1649 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1650 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1651 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1653 flow_req.nav_id = tisci_rm->tisci_dev_id;
1654 flow_req.flow_index = rchan->id;
1656 if (uc->config.needs_epib)
1657 flow_req.rx_einfo_present = 1;
1659 flow_req.rx_einfo_present = 0;
1660 if (uc->config.psd_size)
1661 flow_req.rx_psinfo_present = 1;
1663 flow_req.rx_psinfo_present = 0;
1664 flow_req.rx_error_handling = 1;
1665 flow_req.rx_dest_qnum = rx_ring;
1666 flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;
1667 flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;
1668 flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;
1669 flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;
1670 flow_req.rx_fdq0_sz0_qnum = fd_ring;
1671 flow_req.rx_fdq1_qnum = fd_ring;
1672 flow_req.rx_fdq2_qnum = fd_ring;
1673 flow_req.rx_fdq3_qnum = fd_ring;
1675 ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
1678 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
1683 static int udma_alloc_chan_resources(struct dma_chan *chan)
1685 struct udma_chan *uc = to_udma_chan(chan);
1686 struct udma_dev *ud = to_udma_dev(chan->device);
1687 const struct udma_soc_data *soc_data = ud->soc_data;
1688 struct k3_ring *irq_ring;
1692 if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
1693 uc->use_dma_pool = true;
1694 /* in case of MEM_TO_MEM we have maximum of two TRs */
1695 if (uc->config.dir == DMA_MEM_TO_MEM) {
1696 uc->config.hdesc_size = cppi5_trdesc_calc_size(
1697 sizeof(struct cppi5_tr_type15_t), 2);
1698 uc->config.pkt_mode = false;
1702 if (uc->use_dma_pool) {
1703 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
1704 uc->config.hdesc_size,
1707 if (!uc->hdesc_pool) {
1708 dev_err(ud->ddev.dev,
1709 "Descriptor pool allocation failed\n");
1710 uc->use_dma_pool = false;
1717 * Make sure that the completion is in a known state:
1718 * No teardown, the channel is idle
1720 reinit_completion(&uc->teardown_completed);
1721 complete_all(&uc->teardown_completed);
1722 uc->state = UDMA_CHAN_IS_IDLE;
1724 switch (uc->config.dir) {
1725 case DMA_MEM_TO_MEM:
1726 /* Non synchronized - mem to mem type of transfer */
1727 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
1730 ret = udma_get_chan_pair(uc);
1734 ret = udma_alloc_tx_resources(uc);
1740 ret = udma_alloc_rx_resources(uc);
1742 udma_free_tx_resources(uc);
1746 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1747 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
1748 K3_PSIL_DST_THREAD_ID_OFFSET;
1750 irq_ring = uc->tchan->tc_ring;
1751 irq_udma_idx = uc->tchan->id;
1753 ret = udma_tisci_m2m_channel_config(uc);
1755 case DMA_MEM_TO_DEV:
1756 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
1757 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
1760 ret = udma_alloc_tx_resources(uc);
1764 uc->config.src_thread = ud->psil_base + uc->tchan->id;
1765 uc->config.dst_thread = uc->config.remote_thread_id;
1766 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
1768 irq_ring = uc->tchan->tc_ring;
1769 irq_udma_idx = uc->tchan->id;
1771 ret = udma_tisci_tx_channel_config(uc);
1773 case DMA_DEV_TO_MEM:
1774 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
1775 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
1778 ret = udma_alloc_rx_resources(uc);
1782 uc->config.src_thread = uc->config.remote_thread_id;
1783 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
1784 K3_PSIL_DST_THREAD_ID_OFFSET;
1786 irq_ring = uc->rflow->r_ring;
1787 irq_udma_idx = soc_data->rchan_oes_offset + uc->rchan->id;
1789 ret = udma_tisci_rx_channel_config(uc);
1792 /* Can not happen */
1793 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
1794 __func__, uc->id, uc->config.dir);
1800 /* check if the channel configuration was successful */
1804 if (udma_is_chan_running(uc)) {
1805 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
1806 udma_reset_chan(uc, false);
1807 if (udma_is_chan_running(uc)) {
1808 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
1815 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
1817 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
1818 uc->config.src_thread, uc->config.dst_thread);
1822 uc->psil_paired = true;
1824 uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
1825 if (uc->irq_num_ring <= 0) {
1826 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
1827 k3_ringacc_get_ring_id(irq_ring));
1832 ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
1833 IRQF_TRIGGER_HIGH, uc->name, uc);
1835 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
1839 /* Event from UDMA (TR events) only needed for slave TR mode channels */
1840 if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
1841 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev,
1843 if (uc->irq_num_udma <= 0) {
1844 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
1846 free_irq(uc->irq_num_ring, uc);
1851 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
1854 dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
1856 free_irq(uc->irq_num_ring, uc);
1860 uc->irq_num_udma = 0;
1863 udma_reset_rings(uc);
1868 uc->irq_num_ring = 0;
1869 uc->irq_num_udma = 0;
1871 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
1872 uc->psil_paired = false;
1874 udma_free_tx_resources(uc);
1875 udma_free_rx_resources(uc);
1877 udma_reset_uchan(uc);
1879 if (uc->use_dma_pool) {
1880 dma_pool_destroy(uc->hdesc_pool);
1881 uc->use_dma_pool = false;
1887 static int udma_slave_config(struct dma_chan *chan,
1888 struct dma_slave_config *cfg)
1890 struct udma_chan *uc = to_udma_chan(chan);
1892 memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
1897 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
1898 size_t tr_size, int tr_count,
1899 enum dma_transfer_direction dir)
1901 struct udma_hwdesc *hwdesc;
1902 struct cppi5_desc_hdr_t *tr_desc;
1903 struct udma_desc *d;
1904 u32 reload_count = 0;
1914 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
1918 /* We have only one descriptor containing multiple TRs */
1919 d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
1923 d->sglen = tr_count;
1925 d->hwdesc_count = 1;
1926 hwdesc = &d->hwdesc[0];
1928 /* Allocate memory for DMA ring descriptor */
1929 if (uc->use_dma_pool) {
1930 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
1931 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
1933 &hwdesc->cppi5_desc_paddr);
1935 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
1937 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
1938 uc->ud->desc_align);
1939 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
1940 hwdesc->cppi5_desc_size,
1941 &hwdesc->cppi5_desc_paddr,
1945 if (!hwdesc->cppi5_desc_vaddr) {
1950 /* Start of the TR req records */
1951 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
1952 /* Start address of the TR response array */
1953 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
1955 tr_desc = hwdesc->cppi5_desc_vaddr;
1958 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
1960 if (dir == DMA_DEV_TO_MEM)
1961 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
1963 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
1965 cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
1966 cppi5_desc_set_pktids(tr_desc, uc->id,
1967 CPPI5_INFO1_DESC_FLOWID_DEFAULT);
1968 cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
1974 * udma_get_tr_counters - calculate TR counters for a given length
1975 * @len: Length of the trasnfer
1976 * @align_to: Preferred alignment
1977 * @tr0_cnt0: First TR icnt0
1978 * @tr0_cnt1: First TR icnt1
1979 * @tr1_cnt0: Second (if used) TR icnt0
1981 * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
1982 * For len >= SZ_64K two TRs are used in a simple way:
1983 * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
1984 * Second TR: the remaining length (tr1_cnt0)
1986 * Returns the number of TRs the length needs (1 or 2)
1987 * -EINVAL if the length can not be supported
1989 static int udma_get_tr_counters(size_t len, unsigned long align_to,
1990 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
2003 *tr0_cnt0 = SZ_64K - BIT(align_to);
2004 if (len / *tr0_cnt0 >= SZ_64K) {
2012 *tr0_cnt1 = len / *tr0_cnt0;
2013 *tr1_cnt0 = len % *tr0_cnt0;
2018 static struct udma_desc *
2019 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
2020 unsigned int sglen, enum dma_transfer_direction dir,
2021 unsigned long tx_flags, void *context)
2023 struct scatterlist *sgent;
2024 struct udma_desc *d;
2025 struct cppi5_tr_type1_t *tr_req = NULL;
2026 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2032 /* estimate the number of TRs we will need */
2033 for_each_sg(sgl, sgent, sglen, i) {
2034 if (sg_dma_len(sgent) < SZ_64K)
2040 /* Now allocate and setup the descriptor. */
2041 tr_size = sizeof(struct cppi5_tr_type1_t);
2042 d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
2048 tr_req = d->hwdesc[0].tr_req_base;
2049 for_each_sg(sgl, sgent, sglen, i) {
2050 dma_addr_t sg_addr = sg_dma_address(sgent);
2052 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
2053 &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
2055 dev_err(uc->ud->dev, "size %u is not supported\n",
2057 udma_free_hwdesc(uc, d);
2062 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
2063 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2064 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
2066 tr_req[tr_idx].addr = sg_addr;
2067 tr_req[tr_idx].icnt0 = tr0_cnt0;
2068 tr_req[tr_idx].icnt1 = tr0_cnt1;
2069 tr_req[tr_idx].dim1 = tr0_cnt0;
2073 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
2075 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2076 cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2077 CPPI5_TR_CSF_SUPR_EVT);
2079 tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
2080 tr_req[tr_idx].icnt0 = tr1_cnt0;
2081 tr_req[tr_idx].icnt1 = 1;
2082 tr_req[tr_idx].dim1 = tr1_cnt0;
2086 d->residue += sg_dma_len(sgent);
2089 cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
2090 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
2095 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
2096 enum dma_slave_buswidth dev_width,
2099 if (uc->config.ep_type != PSIL_EP_PDMA_XY)
2102 /* Bus width translates to the element size (ES) */
2103 switch (dev_width) {
2104 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2105 d->static_tr.elsize = 0;
2107 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2108 d->static_tr.elsize = 1;
2110 case DMA_SLAVE_BUSWIDTH_3_BYTES:
2111 d->static_tr.elsize = 2;
2113 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2114 d->static_tr.elsize = 3;
2116 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2117 d->static_tr.elsize = 4;
2119 default: /* not reached */
2123 d->static_tr.elcnt = elcnt;
2126 * PDMA must to close the packet when the channel is in packet mode.
2127 * For TR mode when the channel is not cyclic we also need PDMA to close
2128 * the packet otherwise the transfer will stall because PDMA holds on
2129 * the data it has received from the peripheral.
2131 if (uc->config.pkt_mode || !uc->cyclic) {
2132 unsigned int div = dev_width * elcnt;
2135 d->static_tr.bstcnt = d->residue / d->sglen / div;
2137 d->static_tr.bstcnt = d->residue / div;
2139 if (uc->config.dir == DMA_DEV_TO_MEM &&
2140 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
2143 d->static_tr.bstcnt = 0;
2149 static struct udma_desc *
2150 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
2151 unsigned int sglen, enum dma_transfer_direction dir,
2152 unsigned long tx_flags, void *context)
2154 struct scatterlist *sgent;
2155 struct cppi5_host_desc_t *h_desc = NULL;
2156 struct udma_desc *d;
2160 d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
2165 d->hwdesc_count = sglen;
2167 if (dir == DMA_DEV_TO_MEM)
2168 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2170 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2172 for_each_sg(sgl, sgent, sglen, i) {
2173 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
2174 dma_addr_t sg_addr = sg_dma_address(sgent);
2175 struct cppi5_host_desc_t *desc;
2176 size_t sg_len = sg_dma_len(sgent);
2178 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2180 &hwdesc->cppi5_desc_paddr);
2181 if (!hwdesc->cppi5_desc_vaddr) {
2182 dev_err(uc->ud->dev,
2183 "descriptor%d allocation failed\n", i);
2185 udma_free_hwdesc(uc, d);
2190 d->residue += sg_len;
2191 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2192 desc = hwdesc->cppi5_desc_vaddr;
2195 cppi5_hdesc_init(desc, 0, 0);
2196 /* Flow and Packed ID */
2197 cppi5_desc_set_pktids(&desc->hdr, uc->id,
2198 CPPI5_INFO1_DESC_FLOWID_DEFAULT);
2199 cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
2201 cppi5_hdesc_reset_hbdesc(desc);
2202 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
2205 /* attach the sg buffer to the descriptor */
2206 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
2208 /* Attach link as host buffer descriptor */
2210 cppi5_hdesc_link_hbdesc(h_desc,
2211 hwdesc->cppi5_desc_paddr);
2213 if (dir == DMA_MEM_TO_DEV)
2217 if (d->residue >= SZ_4M) {
2218 dev_err(uc->ud->dev,
2219 "%s: Transfer size %u is over the supported 4M range\n",
2220 __func__, d->residue);
2221 udma_free_hwdesc(uc, d);
2226 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
2227 cppi5_hdesc_set_pktlen(h_desc, d->residue);
2232 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
2233 void *data, size_t len)
2235 struct udma_desc *d = to_udma_desc(desc);
2236 struct udma_chan *uc = to_udma_chan(desc->chan);
2237 struct cppi5_host_desc_t *h_desc;
2241 if (!uc->config.pkt_mode || !uc->config.metadata_size)
2244 if (!data || len > uc->config.metadata_size)
2247 if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
2250 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
2251 if (d->dir == DMA_MEM_TO_DEV)
2252 memcpy(h_desc->epib, data, len);
2254 if (uc->config.needs_epib)
2255 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
2258 d->metadata_size = len;
2259 if (uc->config.needs_epib)
2260 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
2262 cppi5_hdesc_update_flags(h_desc, flags);
2263 cppi5_hdesc_update_psdata_size(h_desc, psd_size);
2268 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
2269 size_t *payload_len, size_t *max_len)
2271 struct udma_desc *d = to_udma_desc(desc);
2272 struct udma_chan *uc = to_udma_chan(desc->chan);
2273 struct cppi5_host_desc_t *h_desc;
2275 if (!uc->config.pkt_mode || !uc->config.metadata_size)
2276 return ERR_PTR(-ENOTSUPP);
2278 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
2280 *max_len = uc->config.metadata_size;
2282 *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
2283 CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
2284 *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
2286 return h_desc->epib;
2289 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
2292 struct udma_desc *d = to_udma_desc(desc);
2293 struct udma_chan *uc = to_udma_chan(desc->chan);
2294 struct cppi5_host_desc_t *h_desc;
2295 u32 psd_size = payload_len;
2298 if (!uc->config.pkt_mode || !uc->config.metadata_size)
2301 if (payload_len > uc->config.metadata_size)
2304 if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
2307 h_desc = d->hwdesc[0].cppi5_desc_vaddr;
2309 if (uc->config.needs_epib) {
2310 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
2311 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
2314 cppi5_hdesc_update_flags(h_desc, flags);
2315 cppi5_hdesc_update_psdata_size(h_desc, psd_size);
2320 static struct dma_descriptor_metadata_ops metadata_ops = {
2321 .attach = udma_attach_metadata,
2322 .get_ptr = udma_get_metadata_ptr,
2323 .set_len = udma_set_metadata_len,
2326 static struct dma_async_tx_descriptor *
2327 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2328 unsigned int sglen, enum dma_transfer_direction dir,
2329 unsigned long tx_flags, void *context)
2331 struct udma_chan *uc = to_udma_chan(chan);
2332 enum dma_slave_buswidth dev_width;
2333 struct udma_desc *d;
2336 if (dir != uc->config.dir) {
2337 dev_err(chan->device->dev,
2338 "%s: chan%d is for %s, not supporting %s\n",
2340 dmaengine_get_direction_text(uc->config.dir),
2341 dmaengine_get_direction_text(dir));
2345 if (dir == DMA_DEV_TO_MEM) {
2346 dev_width = uc->cfg.src_addr_width;
2347 burst = uc->cfg.src_maxburst;
2348 } else if (dir == DMA_MEM_TO_DEV) {
2349 dev_width = uc->cfg.dst_addr_width;
2350 burst = uc->cfg.dst_maxburst;
2352 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
2359 if (uc->config.pkt_mode)
2360 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
2363 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
2373 /* static TR for remote PDMA */
2374 if (udma_configure_statictr(uc, d, dev_width, burst)) {
2375 dev_err(uc->ud->dev,
2376 "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
2377 __func__, d->static_tr.bstcnt);
2379 udma_free_hwdesc(uc, d);
2384 if (uc->config.metadata_size)
2385 d->vd.tx.metadata_ops = &metadata_ops;
2387 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
2390 static struct udma_desc *
2391 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
2392 size_t buf_len, size_t period_len,
2393 enum dma_transfer_direction dir, unsigned long flags)
2395 struct udma_desc *d;
2396 size_t tr_size, period_addr;
2397 struct cppi5_tr_type1_t *tr_req;
2398 unsigned int periods = buf_len / period_len;
2399 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2403 num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
2404 &tr0_cnt1, &tr1_cnt0);
2406 dev_err(uc->ud->dev, "size %zu is not supported\n",
2411 /* Now allocate and setup the descriptor. */
2412 tr_size = sizeof(struct cppi5_tr_type1_t);
2413 d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
2417 tr_req = d->hwdesc[0].tr_req_base;
2418 period_addr = buf_addr;
2419 for (i = 0; i < periods; i++) {
2420 int tr_idx = i * num_tr;
2422 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
2423 false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2425 tr_req[tr_idx].addr = period_addr;
2426 tr_req[tr_idx].icnt0 = tr0_cnt0;
2427 tr_req[tr_idx].icnt1 = tr0_cnt1;
2428 tr_req[tr_idx].dim1 = tr0_cnt0;
2431 cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2432 CPPI5_TR_CSF_SUPR_EVT);
2435 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
2437 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2439 tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
2440 tr_req[tr_idx].icnt0 = tr1_cnt0;
2441 tr_req[tr_idx].icnt1 = 1;
2442 tr_req[tr_idx].dim1 = tr1_cnt0;
2445 if (!(flags & DMA_PREP_INTERRUPT))
2446 cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2447 CPPI5_TR_CSF_SUPR_EVT);
2449 period_addr += period_len;
2455 static struct udma_desc *
2456 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
2457 size_t buf_len, size_t period_len,
2458 enum dma_transfer_direction dir, unsigned long flags)
2460 struct udma_desc *d;
2463 int periods = buf_len / period_len;
2465 if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
2468 if (period_len >= SZ_4M)
2471 d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
2475 d->hwdesc_count = periods;
2477 /* TODO: re-check this... */
2478 if (dir == DMA_DEV_TO_MEM)
2479 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2481 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2483 for (i = 0; i < periods; i++) {
2484 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
2485 dma_addr_t period_addr = buf_addr + (period_len * i);
2486 struct cppi5_host_desc_t *h_desc;
2488 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2490 &hwdesc->cppi5_desc_paddr);
2491 if (!hwdesc->cppi5_desc_vaddr) {
2492 dev_err(uc->ud->dev,
2493 "descriptor%d allocation failed\n", i);
2495 udma_free_hwdesc(uc, d);
2500 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2501 h_desc = hwdesc->cppi5_desc_vaddr;
2503 cppi5_hdesc_init(h_desc, 0, 0);
2504 cppi5_hdesc_set_pktlen(h_desc, period_len);
2506 /* Flow and Packed ID */
2507 cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
2508 CPPI5_INFO1_DESC_FLOWID_DEFAULT);
2509 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
2511 /* attach each period to a new descriptor */
2512 cppi5_hdesc_attach_buf(h_desc,
2513 period_addr, period_len,
2514 period_addr, period_len);
2520 static struct dma_async_tx_descriptor *
2521 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
2522 size_t period_len, enum dma_transfer_direction dir,
2523 unsigned long flags)
2525 struct udma_chan *uc = to_udma_chan(chan);
2526 enum dma_slave_buswidth dev_width;
2527 struct udma_desc *d;
2530 if (dir != uc->config.dir) {
2531 dev_err(chan->device->dev,
2532 "%s: chan%d is for %s, not supporting %s\n",
2534 dmaengine_get_direction_text(uc->config.dir),
2535 dmaengine_get_direction_text(dir));
2541 if (dir == DMA_DEV_TO_MEM) {
2542 dev_width = uc->cfg.src_addr_width;
2543 burst = uc->cfg.src_maxburst;
2544 } else if (dir == DMA_MEM_TO_DEV) {
2545 dev_width = uc->cfg.dst_addr_width;
2546 burst = uc->cfg.dst_maxburst;
2548 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
2555 if (uc->config.pkt_mode)
2556 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
2559 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
2565 d->sglen = buf_len / period_len;
2568 d->residue = buf_len;
2570 /* static TR for remote PDMA */
2571 if (udma_configure_statictr(uc, d, dev_width, burst)) {
2572 dev_err(uc->ud->dev,
2573 "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
2574 __func__, d->static_tr.bstcnt);
2576 udma_free_hwdesc(uc, d);
2581 if (uc->config.metadata_size)
2582 d->vd.tx.metadata_ops = &metadata_ops;
2584 return vchan_tx_prep(&uc->vc, &d->vd, flags);
2587 static struct dma_async_tx_descriptor *
2588 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
2589 size_t len, unsigned long tx_flags)
2591 struct udma_chan *uc = to_udma_chan(chan);
2592 struct udma_desc *d;
2593 struct cppi5_tr_type15_t *tr_req;
2595 size_t tr_size = sizeof(struct cppi5_tr_type15_t);
2596 u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2598 if (uc->config.dir != DMA_MEM_TO_MEM) {
2599 dev_err(chan->device->dev,
2600 "%s: chan%d is for %s, not supporting %s\n",
2602 dmaengine_get_direction_text(uc->config.dir),
2603 dmaengine_get_direction_text(DMA_MEM_TO_MEM));
2607 num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
2608 &tr0_cnt1, &tr1_cnt0);
2610 dev_err(uc->ud->dev, "size %zu is not supported\n",
2615 d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
2619 d->dir = DMA_MEM_TO_MEM;
2624 tr_req = d->hwdesc[0].tr_req_base;
2626 cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
2627 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2628 cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
2630 tr_req[0].addr = src;
2631 tr_req[0].icnt0 = tr0_cnt0;
2632 tr_req[0].icnt1 = tr0_cnt1;
2633 tr_req[0].icnt2 = 1;
2634 tr_req[0].icnt3 = 1;
2635 tr_req[0].dim1 = tr0_cnt0;
2637 tr_req[0].daddr = dest;
2638 tr_req[0].dicnt0 = tr0_cnt0;
2639 tr_req[0].dicnt1 = tr0_cnt1;
2640 tr_req[0].dicnt2 = 1;
2641 tr_req[0].dicnt3 = 1;
2642 tr_req[0].ddim1 = tr0_cnt0;
2645 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
2646 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2647 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
2649 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
2650 tr_req[1].icnt0 = tr1_cnt0;
2651 tr_req[1].icnt1 = 1;
2652 tr_req[1].icnt2 = 1;
2653 tr_req[1].icnt3 = 1;
2655 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
2656 tr_req[1].dicnt0 = tr1_cnt0;
2657 tr_req[1].dicnt1 = 1;
2658 tr_req[1].dicnt2 = 1;
2659 tr_req[1].dicnt3 = 1;
2662 cppi5_tr_csf_set(&tr_req[num_tr - 1].flags,
2663 CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
2665 if (uc->config.metadata_size)
2666 d->vd.tx.metadata_ops = &metadata_ops;
2668 return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
2671 static void udma_issue_pending(struct dma_chan *chan)
2673 struct udma_chan *uc = to_udma_chan(chan);
2674 unsigned long flags;
2676 spin_lock_irqsave(&uc->vc.lock, flags);
2678 /* If we have something pending and no active descriptor, then */
2679 if (vchan_issue_pending(&uc->vc) && !uc->desc) {
2681 * start a descriptor if the channel is NOT [marked as
2682 * terminating _and_ it is still running (teardown has not
2685 if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
2686 udma_is_chan_running(uc)))
2690 spin_unlock_irqrestore(&uc->vc.lock, flags);
2693 static enum dma_status udma_tx_status(struct dma_chan *chan,
2694 dma_cookie_t cookie,
2695 struct dma_tx_state *txstate)
2697 struct udma_chan *uc = to_udma_chan(chan);
2698 enum dma_status ret;
2699 unsigned long flags;
2701 spin_lock_irqsave(&uc->vc.lock, flags);
2703 ret = dma_cookie_status(chan, cookie, txstate);
2705 if (!udma_is_chan_running(uc))
2708 if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))
2711 if (ret == DMA_COMPLETE || !txstate)
2714 if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
2717 u32 residue = uc->desc->residue;
2720 if (uc->desc->dir == DMA_MEM_TO_DEV) {
2721 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
2723 if (uc->config.ep_type != PSIL_EP_NATIVE) {
2724 peer_bcnt = udma_tchanrt_read(uc,
2725 UDMA_CHAN_RT_PEER_BCNT_REG);
2727 if (bcnt > peer_bcnt)
2728 delay = bcnt - peer_bcnt;
2730 } else if (uc->desc->dir == DMA_DEV_TO_MEM) {
2731 bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
2733 if (uc->config.ep_type != PSIL_EP_NATIVE) {
2734 peer_bcnt = udma_rchanrt_read(uc,
2735 UDMA_CHAN_RT_PEER_BCNT_REG);
2737 if (peer_bcnt > bcnt)
2738 delay = peer_bcnt - bcnt;
2741 bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
2745 if (bcnt && !(bcnt % uc->desc->residue))
2748 residue -= bcnt % uc->desc->residue;
2750 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
2755 dma_set_residue(txstate, residue);
2756 dma_set_in_flight_bytes(txstate, delay);
2763 spin_unlock_irqrestore(&uc->vc.lock, flags);
2767 static int udma_pause(struct dma_chan *chan)
2769 struct udma_chan *uc = to_udma_chan(chan);
2771 /* pause the channel */
2772 switch (uc->config.dir) {
2773 case DMA_DEV_TO_MEM:
2774 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
2775 UDMA_PEER_RT_EN_PAUSE,
2776 UDMA_PEER_RT_EN_PAUSE);
2778 case DMA_MEM_TO_DEV:
2779 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
2780 UDMA_PEER_RT_EN_PAUSE,
2781 UDMA_PEER_RT_EN_PAUSE);
2783 case DMA_MEM_TO_MEM:
2784 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
2785 UDMA_CHAN_RT_CTL_PAUSE,
2786 UDMA_CHAN_RT_CTL_PAUSE);
2795 static int udma_resume(struct dma_chan *chan)
2797 struct udma_chan *uc = to_udma_chan(chan);
2799 /* resume the channel */
2800 switch (uc->config.dir) {
2801 case DMA_DEV_TO_MEM:
2802 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
2803 UDMA_PEER_RT_EN_PAUSE, 0);
2806 case DMA_MEM_TO_DEV:
2807 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
2808 UDMA_PEER_RT_EN_PAUSE, 0);
2810 case DMA_MEM_TO_MEM:
2811 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
2812 UDMA_CHAN_RT_CTL_PAUSE, 0);
2821 static int udma_terminate_all(struct dma_chan *chan)
2823 struct udma_chan *uc = to_udma_chan(chan);
2824 unsigned long flags;
2827 spin_lock_irqsave(&uc->vc.lock, flags);
2829 if (udma_is_chan_running(uc))
2833 uc->terminated_desc = uc->desc;
2835 uc->terminated_desc->terminated = true;
2836 cancel_delayed_work(&uc->tx_drain.work);
2841 vchan_get_all_descriptors(&uc->vc, &head);
2842 spin_unlock_irqrestore(&uc->vc.lock, flags);
2843 vchan_dma_desc_free_list(&uc->vc, &head);
2848 static void udma_synchronize(struct dma_chan *chan)
2850 struct udma_chan *uc = to_udma_chan(chan);
2851 unsigned long timeout = msecs_to_jiffies(1000);
2853 vchan_synchronize(&uc->vc);
2855 if (uc->state == UDMA_CHAN_IS_TERMINATING) {
2856 timeout = wait_for_completion_timeout(&uc->teardown_completed,
2859 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
2861 udma_dump_chan_stdata(uc);
2862 udma_reset_chan(uc, true);
2866 udma_reset_chan(uc, false);
2867 if (udma_is_chan_running(uc))
2868 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
2870 cancel_delayed_work_sync(&uc->tx_drain.work);
2871 udma_reset_rings(uc);
2874 static void udma_desc_pre_callback(struct virt_dma_chan *vc,
2875 struct virt_dma_desc *vd,
2876 struct dmaengine_result *result)
2878 struct udma_chan *uc = to_udma_chan(&vc->chan);
2879 struct udma_desc *d;
2884 d = to_udma_desc(&vd->tx);
2886 if (d->metadata_size)
2887 udma_fetch_epib(uc, d);
2889 /* Provide residue information for the client */
2891 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
2893 if (cppi5_desc_get_type(desc_vaddr) ==
2894 CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
2895 result->residue = d->residue -
2896 cppi5_hdesc_get_pktlen(desc_vaddr);
2897 if (result->residue)
2898 result->result = DMA_TRANS_ABORTED;
2900 result->result = DMA_TRANS_NOERROR;
2902 result->residue = 0;
2903 result->result = DMA_TRANS_NOERROR;
2909 * This tasklet handles the completion of a DMA descriptor by
2910 * calling its callback and freeing it.
2912 static void udma_vchan_complete(struct tasklet_struct *t)
2914 struct virt_dma_chan *vc = from_tasklet(vc, t, task);
2915 struct virt_dma_desc *vd, *_vd;
2916 struct dmaengine_desc_callback cb;
2919 spin_lock_irq(&vc->lock);
2920 list_splice_tail_init(&vc->desc_completed, &head);
2924 dmaengine_desc_get_callback(&vd->tx, &cb);
2926 memset(&cb, 0, sizeof(cb));
2928 spin_unlock_irq(&vc->lock);
2930 udma_desc_pre_callback(vc, vd, NULL);
2931 dmaengine_desc_callback_invoke(&cb, NULL);
2933 list_for_each_entry_safe(vd, _vd, &head, node) {
2934 struct dmaengine_result result;
2936 dmaengine_desc_get_callback(&vd->tx, &cb);
2938 list_del(&vd->node);
2940 udma_desc_pre_callback(vc, vd, &result);
2941 dmaengine_desc_callback_invoke(&cb, &result);
2943 vchan_vdesc_fini(vd);
2947 static void udma_free_chan_resources(struct dma_chan *chan)
2949 struct udma_chan *uc = to_udma_chan(chan);
2950 struct udma_dev *ud = to_udma_dev(chan->device);
2952 udma_terminate_all(chan);
2953 if (uc->terminated_desc) {
2954 udma_reset_chan(uc, false);
2955 udma_reset_rings(uc);
2958 cancel_delayed_work_sync(&uc->tx_drain.work);
2960 if (uc->irq_num_ring > 0) {
2961 free_irq(uc->irq_num_ring, uc);
2963 uc->irq_num_ring = 0;
2965 if (uc->irq_num_udma > 0) {
2966 free_irq(uc->irq_num_udma, uc);
2968 uc->irq_num_udma = 0;
2971 /* Release PSI-L pairing */
2972 if (uc->psil_paired) {
2973 navss_psil_unpair(ud, uc->config.src_thread,
2974 uc->config.dst_thread);
2975 uc->psil_paired = false;
2978 vchan_free_chan_resources(&uc->vc);
2979 tasklet_kill(&uc->vc.task);
2981 udma_free_tx_resources(uc);
2982 udma_free_rx_resources(uc);
2983 udma_reset_uchan(uc);
2985 if (uc->use_dma_pool) {
2986 dma_pool_destroy(uc->hdesc_pool);
2987 uc->use_dma_pool = false;
2991 static struct platform_driver udma_driver;
2993 struct udma_filter_param {
2994 int remote_thread_id;
2998 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
3000 struct udma_chan_config *ucc;
3001 struct psil_endpoint_config *ep_config;
3002 struct udma_filter_param *filter_param;
3003 struct udma_chan *uc;
3004 struct udma_dev *ud;
3006 if (chan->device->dev->driver != &udma_driver.driver)
3009 uc = to_udma_chan(chan);
3012 filter_param = param;
3014 if (filter_param->atype > 2) {
3015 dev_err(ud->dev, "Invalid channel atype: %u\n",
3016 filter_param->atype);
3020 ucc->remote_thread_id = filter_param->remote_thread_id;
3021 ucc->atype = filter_param->atype;
3023 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)
3024 ucc->dir = DMA_MEM_TO_DEV;
3026 ucc->dir = DMA_DEV_TO_MEM;
3028 ep_config = psil_get_ep_config(ucc->remote_thread_id);
3029 if (IS_ERR(ep_config)) {
3030 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
3031 ucc->remote_thread_id);
3032 ucc->dir = DMA_MEM_TO_MEM;
3033 ucc->remote_thread_id = -1;
3038 ucc->pkt_mode = ep_config->pkt_mode;
3039 ucc->channel_tpl = ep_config->channel_tpl;
3040 ucc->notdpkt = ep_config->notdpkt;
3041 ucc->ep_type = ep_config->ep_type;
3043 if (ucc->ep_type != PSIL_EP_NATIVE) {
3044 const struct udma_match_data *match_data = ud->match_data;
3046 if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
3047 ucc->enable_acc32 = ep_config->pdma_acc32;
3048 if (match_data->flags & UDMA_FLAG_PDMA_BURST)
3049 ucc->enable_burst = ep_config->pdma_burst;
3052 ucc->needs_epib = ep_config->needs_epib;
3053 ucc->psd_size = ep_config->psd_size;
3054 ucc->metadata_size =
3055 (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
3059 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
3060 ucc->metadata_size, ud->desc_align);
3062 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
3063 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
3068 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
3069 struct of_dma *ofdma)
3071 struct udma_dev *ud = ofdma->of_dma_data;
3072 dma_cap_mask_t mask = ud->ddev.cap_mask;
3073 struct udma_filter_param filter_param;
3074 struct dma_chan *chan;
3076 if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
3079 filter_param.remote_thread_id = dma_spec->args[0];
3080 if (dma_spec->args_count == 2)
3081 filter_param.atype = dma_spec->args[1];
3083 filter_param.atype = 0;
3085 chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,
3088 dev_err(ud->dev, "get channel fail in %s.\n", __func__);
3089 return ERR_PTR(-EINVAL);
3095 static struct udma_match_data am654_main_data = {
3096 .psil_base = 0x1000,
3097 .enable_memcpy_support = true,
3098 .statictr_z_mask = GENMASK(11, 0),
3101 static struct udma_match_data am654_mcu_data = {
3102 .psil_base = 0x6000,
3103 .enable_memcpy_support = false,
3104 .statictr_z_mask = GENMASK(11, 0),
3107 static struct udma_match_data j721e_main_data = {
3108 .psil_base = 0x1000,
3109 .enable_memcpy_support = true,
3110 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST,
3111 .statictr_z_mask = GENMASK(23, 0),
3114 static struct udma_match_data j721e_mcu_data = {
3115 .psil_base = 0x6000,
3116 .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
3117 .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST,
3118 .statictr_z_mask = GENMASK(23, 0),
3121 static const struct of_device_id udma_of_match[] = {
3123 .compatible = "ti,am654-navss-main-udmap",
3124 .data = &am654_main_data,
3127 .compatible = "ti,am654-navss-mcu-udmap",
3128 .data = &am654_mcu_data,
3130 .compatible = "ti,j721e-navss-main-udmap",
3131 .data = &j721e_main_data,
3133 .compatible = "ti,j721e-navss-mcu-udmap",
3134 .data = &j721e_mcu_data,
3139 static struct udma_soc_data am654_soc_data = {
3140 .rchan_oes_offset = 0x200,
3143 static struct udma_soc_data j721e_soc_data = {
3144 .rchan_oes_offset = 0x400,
3147 static struct udma_soc_data j7200_soc_data = {
3148 .rchan_oes_offset = 0x80,
3151 static const struct soc_device_attribute k3_soc_devices[] = {
3152 { .family = "AM65X", .data = &am654_soc_data },
3153 { .family = "J721E", .data = &j721e_soc_data },
3154 { .family = "J7200", .data = &j7200_soc_data },
3158 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
3162 for (i = 0; i < MMR_LAST; i++) {
3163 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
3164 if (IS_ERR(ud->mmrs[i]))
3165 return PTR_ERR(ud->mmrs[i]);
3171 static int udma_setup_resources(struct udma_dev *ud)
3173 struct device *dev = ud->dev;
3174 int ch_count, ret, i, j;
3176 struct ti_sci_resource_desc *rm_desc;
3177 struct ti_sci_resource *rm_res, irq_res;
3178 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
3179 static const char * const range_names[] = { "ti,sci-rm-range-tchan",
3180 "ti,sci-rm-range-rchan",
3181 "ti,sci-rm-range-rflow" };
3183 cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2));
3184 cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3));
3186 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
3187 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
3188 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
3189 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
3190 ch_count = ud->tchan_cnt + ud->rchan_cnt;
3192 /* Set up the throughput level start indexes */
3193 if (of_device_is_compatible(dev->of_node,
3194 "ti,am654-navss-main-udmap")) {
3196 ud->tpl_start_idx[0] = 8;
3197 } else if (of_device_is_compatible(dev->of_node,
3198 "ti,am654-navss-mcu-udmap")) {
3200 ud->tpl_start_idx[0] = 2;
3201 } else if (UDMA_CAP3_UCHAN_CNT(cap3)) {
3203 ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
3204 ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
3205 } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
3207 ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
3212 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
3213 sizeof(unsigned long), GFP_KERNEL);
3214 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
3216 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
3217 sizeof(unsigned long), GFP_KERNEL);
3218 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
3220 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
3221 sizeof(unsigned long),
3223 ud->rflow_gp_map_allocated = devm_kcalloc(dev,
3224 BITS_TO_LONGS(ud->rflow_cnt),
3225 sizeof(unsigned long),
3227 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
3228 sizeof(unsigned long),
3230 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
3233 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
3234 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
3235 !ud->rflows || !ud->rflow_in_use)
3239 * RX flows with the same Ids as RX channels are reserved to be used
3240 * as default flows if remote HW can't generate flow_ids. Those
3241 * RX flows can be requested only explicitly by id.
3243 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
3245 /* by default no GP rflows are assigned to Linux */
3246 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
3248 /* Get resource ranges from tisci */
3249 for (i = 0; i < RM_RANGE_LAST; i++)
3250 tisci_rm->rm_ranges[i] =
3251 devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
3252 tisci_rm->tisci_dev_id,
3253 (char *)range_names[i]);
3256 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
3257 if (IS_ERR(rm_res)) {
3258 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
3260 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
3261 for (i = 0; i < rm_res->sets; i++) {
3262 rm_desc = &rm_res->desc[i];
3263 bitmap_clear(ud->tchan_map, rm_desc->start,
3265 dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
3266 rm_desc->start, rm_desc->num);
3269 irq_res.sets = rm_res->sets;
3271 /* rchan and matching default flow ranges */
3272 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
3273 if (IS_ERR(rm_res)) {
3274 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
3276 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
3277 for (i = 0; i < rm_res->sets; i++) {
3278 rm_desc = &rm_res->desc[i];
3279 bitmap_clear(ud->rchan_map, rm_desc->start,
3281 dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
3282 rm_desc->start, rm_desc->num);
3286 irq_res.sets += rm_res->sets;
3287 irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
3288 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
3289 for (i = 0; i < rm_res->sets; i++) {
3290 irq_res.desc[i].start = rm_res->desc[i].start;
3291 irq_res.desc[i].num = rm_res->desc[i].num;
3293 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
3294 for (j = 0; j < rm_res->sets; j++, i++) {
3295 irq_res.desc[i].start = rm_res->desc[j].start +
3296 ud->soc_data->rchan_oes_offset;
3297 irq_res.desc[i].num = rm_res->desc[j].num;
3299 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
3300 kfree(irq_res.desc);
3302 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
3306 /* GP rflow ranges */
3307 rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
3308 if (IS_ERR(rm_res)) {
3309 /* all gp flows are assigned exclusively to Linux */
3310 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
3311 ud->rflow_cnt - ud->rchan_cnt);
3313 for (i = 0; i < rm_res->sets; i++) {
3314 rm_desc = &rm_res->desc[i];
3315 bitmap_clear(ud->rflow_gp_map, rm_desc->start,
3317 dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
3318 rm_desc->start, rm_desc->num);
3322 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
3323 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
3327 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
3332 dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
3334 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt),
3335 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt),
3336 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
3342 static int udma_setup_rx_flush(struct udma_dev *ud)
3344 struct udma_rx_flush *rx_flush = &ud->rx_flush;
3345 struct cppi5_desc_hdr_t *tr_desc;
3346 struct cppi5_tr_type1_t *tr_req;
3347 struct cppi5_host_desc_t *desc;
3348 struct device *dev = ud->dev;
3349 struct udma_hwdesc *hwdesc;
3352 /* Allocate 1K buffer for discarded data on RX channel teardown */
3353 rx_flush->buffer_size = SZ_1K;
3354 rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
3356 if (!rx_flush->buffer_vaddr)
3359 rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
3360 rx_flush->buffer_size,
3362 if (dma_mapping_error(dev, rx_flush->buffer_paddr))
3365 /* Set up descriptor to be used for TR mode */
3366 hwdesc = &rx_flush->hwdescs[0];
3367 tr_size = sizeof(struct cppi5_tr_type1_t);
3368 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
3369 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
3372 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
3374 if (!hwdesc->cppi5_desc_vaddr)
3377 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
3378 hwdesc->cppi5_desc_size,
3380 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
3383 /* Start of the TR req records */
3384 hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
3385 /* Start address of the TR response array */
3386 hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
3388 tr_desc = hwdesc->cppi5_desc_vaddr;
3389 cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
3390 cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3391 cppi5_desc_set_retpolicy(tr_desc, 0, 0);
3393 tr_req = hwdesc->tr_req_base;
3394 cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
3395 CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3396 cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
3398 tr_req->addr = rx_flush->buffer_paddr;
3399 tr_req->icnt0 = rx_flush->buffer_size;
3402 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
3403 hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
3405 /* Set up descriptor to be used for packet mode */
3406 hwdesc = &rx_flush->hwdescs[1];
3407 hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
3408 CPPI5_INFO0_HDESC_EPIB_SIZE +
3409 CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
3412 hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
3414 if (!hwdesc->cppi5_desc_vaddr)
3417 hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
3418 hwdesc->cppi5_desc_size,
3420 if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
3423 desc = hwdesc->cppi5_desc_vaddr;
3424 cppi5_hdesc_init(desc, 0, 0);
3425 cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3426 cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
3428 cppi5_hdesc_attach_buf(desc,
3429 rx_flush->buffer_paddr, rx_flush->buffer_size,
3430 rx_flush->buffer_paddr, rx_flush->buffer_size);
3432 dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
3433 hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
3437 #ifdef CONFIG_DEBUG_FS
3438 static void udma_dbg_summary_show_chan(struct seq_file *s,
3439 struct dma_chan *chan)
3441 struct udma_chan *uc = to_udma_chan(chan);
3442 struct udma_chan_config *ucc = &uc->config;
3444 seq_printf(s, " %-13s| %s", dma_chan_name(chan),
3445 chan->dbg_client_name ?: "in-use");
3446 seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir));
3448 switch (uc->config.dir) {
3449 case DMA_MEM_TO_MEM:
3450 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
3451 ucc->src_thread, ucc->dst_thread);
3453 case DMA_DEV_TO_MEM:
3454 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
3455 ucc->src_thread, ucc->dst_thread);
3457 case DMA_MEM_TO_DEV:
3458 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
3459 ucc->src_thread, ucc->dst_thread);
3462 seq_printf(s, ")\n");
3466 if (ucc->ep_type == PSIL_EP_NATIVE) {
3467 seq_printf(s, "PSI-L Native");
3468 if (ucc->metadata_size) {
3469 seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
3471 seq_printf(s, " PSDsize:%u", ucc->psd_size);
3472 seq_printf(s, " ]");
3475 seq_printf(s, "PDMA");
3476 if (ucc->enable_acc32 || ucc->enable_burst)
3477 seq_printf(s, "[%s%s ]",
3478 ucc->enable_acc32 ? " ACC32" : "",
3479 ucc->enable_burst ? " BURST" : "");
3482 seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
3485 static void udma_dbg_summary_show(struct seq_file *s,
3486 struct dma_device *dma_dev)
3488 struct dma_chan *chan;
3490 list_for_each_entry(chan, &dma_dev->channels, device_node) {
3491 if (chan->client_count)
3492 udma_dbg_summary_show_chan(s, chan);
3495 #endif /* CONFIG_DEBUG_FS */
3497 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
3498 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
3499 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
3500 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
3501 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
3503 static int udma_probe(struct platform_device *pdev)
3505 struct device_node *navss_node = pdev->dev.parent->of_node;
3506 const struct soc_device_attribute *soc;
3507 struct device *dev = &pdev->dev;
3508 struct udma_dev *ud;
3509 const struct of_device_id *match;
3513 ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
3515 dev_err(dev, "failed to set dma mask stuff\n");
3517 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
3521 ret = udma_get_mmrs(pdev, ud);
3525 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
3526 if (IS_ERR(ud->tisci_rm.tisci))
3527 return PTR_ERR(ud->tisci_rm.tisci);
3529 ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
3530 &ud->tisci_rm.tisci_dev_id);
3532 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
3535 pdev->id = ud->tisci_rm.tisci_dev_id;
3537 ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
3538 &ud->tisci_rm.tisci_navss_dev_id);
3540 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
3544 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype);
3545 if (!ret && ud->atype > 2) {
3546 dev_err(dev, "Invalid atype: %u\n", ud->atype);
3550 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
3551 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
3553 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
3554 if (IS_ERR(ud->ringacc))
3555 return PTR_ERR(ud->ringacc);
3557 dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
3558 DOMAIN_BUS_TI_SCI_INTA_MSI);
3559 if (!dev->msi_domain) {
3560 dev_err(dev, "Failed to get MSI domain\n");
3561 return -EPROBE_DEFER;
3564 match = of_match_node(udma_of_match, dev->of_node);
3566 dev_err(dev, "No compatible match found\n");
3569 ud->match_data = match->data;
3571 soc = soc_device_match(k3_soc_devices);
3573 dev_err(dev, "No compatible SoC found\n");
3576 ud->soc_data = soc->data;
3578 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
3579 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
3581 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources;
3582 ud->ddev.device_config = udma_slave_config;
3583 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
3584 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
3585 ud->ddev.device_issue_pending = udma_issue_pending;
3586 ud->ddev.device_tx_status = udma_tx_status;
3587 ud->ddev.device_pause = udma_pause;
3588 ud->ddev.device_resume = udma_resume;
3589 ud->ddev.device_terminate_all = udma_terminate_all;
3590 ud->ddev.device_synchronize = udma_synchronize;
3591 #ifdef CONFIG_DEBUG_FS
3592 ud->ddev.dbg_summary_show = udma_dbg_summary_show;
3595 ud->ddev.device_free_chan_resources = udma_free_chan_resources;
3596 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
3597 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
3598 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
3599 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
3600 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES;
3601 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
3602 DESC_METADATA_ENGINE;
3603 if (ud->match_data->enable_memcpy_support) {
3604 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
3605 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
3606 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
3611 ud->psil_base = ud->match_data->psil_base;
3613 INIT_LIST_HEAD(&ud->ddev.channels);
3614 INIT_LIST_HEAD(&ud->desc_to_purge);
3616 ch_count = udma_setup_resources(ud);
3620 spin_lock_init(&ud->lock);
3621 INIT_WORK(&ud->purge_work, udma_purge_desc_work);
3623 ud->desc_align = 64;
3624 if (ud->desc_align < dma_get_cache_alignment())
3625 ud->desc_align = dma_get_cache_alignment();
3627 ret = udma_setup_rx_flush(ud);
3631 for (i = 0; i < ud->tchan_cnt; i++) {
3632 struct udma_tchan *tchan = &ud->tchans[i];
3635 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
3638 for (i = 0; i < ud->rchan_cnt; i++) {
3639 struct udma_rchan *rchan = &ud->rchans[i];
3642 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
3645 for (i = 0; i < ud->rflow_cnt; i++) {
3646 struct udma_rflow *rflow = &ud->rflows[i];
3651 for (i = 0; i < ch_count; i++) {
3652 struct udma_chan *uc = &ud->channels[i];
3655 uc->vc.desc_free = udma_desc_free;
3659 uc->config.remote_thread_id = -1;
3660 uc->config.dir = DMA_MEM_TO_MEM;
3661 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
3664 vchan_init(&uc->vc, &ud->ddev);
3665 /* Use custom vchan completion handling */
3666 tasklet_setup(&uc->vc.task, udma_vchan_complete);
3667 init_completion(&uc->teardown_completed);
3668 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
3671 ret = dma_async_device_register(&ud->ddev);
3673 dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
3677 platform_set_drvdata(pdev, ud);
3679 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
3681 dev_err(dev, "failed to register of_dma controller\n");
3682 dma_async_device_unregister(&ud->ddev);
3688 static struct platform_driver udma_driver = {
3691 .of_match_table = udma_of_match,
3692 .suppress_bind_attrs = true,
3694 .probe = udma_probe,
3696 builtin_platform_driver(udma_driver);
3698 /* Private interfaces to UDMA */
3699 #include "k3-udma-private.c"