1 // SPDX-License-Identifier: GPL-2.0-only
3 * Core driver for the High Speed UART DMA
5 * Copyright (C) 2015 Intel Corporation
6 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
8 * Partially based on the bits found in drivers/tty/serial/mfd.c.
12 * DMA channel allocation:
13 * 1. Even number chans are used for DMA Read (UART TX), odd chans for DMA
15 * 2. 0/1 channel are assigned to port 0, 2/3 chan to port 1, 4/5 chan to
19 #include <linux/delay.h>
20 #include <linux/dmaengine.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/init.h>
23 #include <linux/module.h>
24 #include <linux/slab.h>
28 #define HSU_DMA_BUSWIDTHS \
29 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
30 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
31 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
32 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
33 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
34 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) | \
35 BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)
37 static inline void hsu_chan_disable(struct hsu_dma_chan *hsuc)
39 hsu_chan_writel(hsuc, HSU_CH_CR, 0);
42 static inline void hsu_chan_enable(struct hsu_dma_chan *hsuc)
44 u32 cr = HSU_CH_CR_CHA;
46 if (hsuc->direction == DMA_MEM_TO_DEV)
48 else if (hsuc->direction == DMA_DEV_TO_MEM)
51 hsu_chan_writel(hsuc, HSU_CH_CR, cr);
54 static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
56 struct dma_slave_config *config = &hsuc->config;
57 struct hsu_dma_desc *desc = hsuc->desc;
58 u32 bsr = 0, mtsr = 0; /* to shut the compiler up */
59 u32 dcr = HSU_CH_DCR_CHSOE | HSU_CH_DCR_CHEI;
60 unsigned int i, count;
62 if (hsuc->direction == DMA_MEM_TO_DEV) {
63 bsr = config->dst_maxburst;
64 mtsr = config->dst_addr_width;
65 } else if (hsuc->direction == DMA_DEV_TO_MEM) {
66 bsr = config->src_maxburst;
67 mtsr = config->src_addr_width;
70 hsu_chan_disable(hsuc);
72 hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
73 hsu_chan_writel(hsuc, HSU_CH_BSR, bsr);
74 hsu_chan_writel(hsuc, HSU_CH_MTSR, mtsr);
77 count = desc->nents - desc->active;
78 for (i = 0; i < count && i < HSU_DMA_CHAN_NR_DESC; i++) {
79 hsu_chan_writel(hsuc, HSU_CH_DxSAR(i), desc->sg[i].addr);
80 hsu_chan_writel(hsuc, HSU_CH_DxTSR(i), desc->sg[i].len);
82 /* Prepare value for DCR */
83 dcr |= HSU_CH_DCR_DESCA(i);
84 dcr |= HSU_CH_DCR_CHTOI(i); /* timeout bit, see HSU Errata 1 */
88 /* Only for the last descriptor in the chain */
89 dcr |= HSU_CH_DCR_CHSOD(count - 1);
90 dcr |= HSU_CH_DCR_CHDI(count - 1);
92 hsu_chan_writel(hsuc, HSU_CH_DCR, dcr);
94 hsu_chan_enable(hsuc);
97 static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc)
99 hsu_chan_disable(hsuc);
100 hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
103 static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc)
105 hsu_dma_chan_start(hsuc);
108 static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc)
110 struct virt_dma_desc *vdesc;
112 /* Get the next descriptor */
113 vdesc = vchan_next_desc(&hsuc->vchan);
119 list_del(&vdesc->node);
120 hsuc->desc = to_hsu_dma_desc(vdesc);
122 /* Start the channel with a new descriptor */
123 hsu_dma_start_channel(hsuc);
127 * hsu_dma_get_status() - get DMA channel status
128 * @chip: HSUART DMA chip
129 * @nr: DMA channel number
130 * @status: pointer for DMA Channel Status Register value
133 * The function reads and clears the DMA Channel Status Register, checks
134 * if it was a timeout interrupt and returns a corresponding value.
136 * Caller should provide a valid pointer for the DMA Channel Status
137 * Register value that will be returned in @status.
140 * 1 for DMA timeout status, 0 for other DMA status, or error code for
141 * invalid parameters or no interrupt pending.
143 int hsu_dma_get_status(struct hsu_dma_chip *chip, unsigned short nr,
146 struct hsu_dma_chan *hsuc;
151 if (nr >= chip->hsu->nr_channels)
154 hsuc = &chip->hsu->chan[nr];
157 * No matter what situation, need read clear the IRQ status
158 * There is a bug, see Errata 5, HSD 2900918
160 spin_lock_irqsave(&hsuc->vchan.lock, flags);
161 sr = hsu_chan_readl(hsuc, HSU_CH_SR);
162 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
164 /* Check if any interrupt is pending */
165 sr &= ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY);
169 /* Timeout IRQ, need wait some time, see Errata 2 */
170 if (sr & HSU_CH_SR_DESCTO_ANY)
174 * At this point, at least one of Descriptor Time Out, Channel Error
175 * or Descriptor Done bits must be set. Clear the Descriptor Time Out
176 * bits and if sr is still non-zero, it must be channel error or
177 * descriptor done which are higher priority than timeout and handled
178 * in hsu_dma_do_irq(). Else, it must be a timeout.
180 sr &= ~HSU_CH_SR_DESCTO_ANY;
186 EXPORT_SYMBOL_GPL(hsu_dma_get_status);
189 * hsu_dma_do_irq() - DMA interrupt handler
190 * @chip: HSUART DMA chip
191 * @nr: DMA channel number
192 * @status: Channel Status Register value
195 * This function handles Channel Error and Descriptor Done interrupts.
196 * This function should be called after determining that the DMA interrupt
197 * is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0.
200 * 0 for invalid channel number, 1 otherwise.
202 int hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, u32 status)
204 struct hsu_dma_chan *hsuc;
205 struct hsu_dma_desc *desc;
209 if (nr >= chip->hsu->nr_channels)
212 hsuc = &chip->hsu->chan[nr];
214 spin_lock_irqsave(&hsuc->vchan.lock, flags);
217 if (status & HSU_CH_SR_CHE) {
218 desc->status = DMA_ERROR;
219 } else if (desc->active < desc->nents) {
220 hsu_dma_start_channel(hsuc);
222 vchan_cookie_complete(&desc->vdesc);
223 desc->status = DMA_COMPLETE;
224 hsu_dma_start_transfer(hsuc);
227 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
231 EXPORT_SYMBOL_GPL(hsu_dma_do_irq);
233 static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents)
235 struct hsu_dma_desc *desc;
237 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
241 desc->sg = kcalloc(nents, sizeof(*desc->sg), GFP_NOWAIT);
250 static void hsu_dma_desc_free(struct virt_dma_desc *vdesc)
252 struct hsu_dma_desc *desc = to_hsu_dma_desc(vdesc);
258 static struct dma_async_tx_descriptor *hsu_dma_prep_slave_sg(
259 struct dma_chan *chan, struct scatterlist *sgl,
260 unsigned int sg_len, enum dma_transfer_direction direction,
261 unsigned long flags, void *context)
263 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
264 struct hsu_dma_desc *desc;
265 struct scatterlist *sg;
268 desc = hsu_dma_alloc_desc(sg_len);
272 for_each_sg(sgl, sg, sg_len, i) {
273 desc->sg[i].addr = sg_dma_address(sg);
274 desc->sg[i].len = sg_dma_len(sg);
276 desc->length += sg_dma_len(sg);
279 desc->nents = sg_len;
280 desc->direction = direction;
281 /* desc->active = 0 by kzalloc */
282 desc->status = DMA_IN_PROGRESS;
284 return vchan_tx_prep(&hsuc->vchan, &desc->vdesc, flags);
287 static void hsu_dma_issue_pending(struct dma_chan *chan)
289 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
292 spin_lock_irqsave(&hsuc->vchan.lock, flags);
293 if (vchan_issue_pending(&hsuc->vchan) && !hsuc->desc)
294 hsu_dma_start_transfer(hsuc);
295 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
298 static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc)
300 struct hsu_dma_desc *desc = hsuc->desc;
304 for (i = desc->active; i < desc->nents; i++)
305 bytes += desc->sg[i].len;
307 i = HSU_DMA_CHAN_NR_DESC - 1;
309 bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i));
315 static enum dma_status hsu_dma_tx_status(struct dma_chan *chan,
316 dma_cookie_t cookie, struct dma_tx_state *state)
318 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
319 struct virt_dma_desc *vdesc;
320 enum dma_status status;
324 status = dma_cookie_status(chan, cookie, state);
325 if (status == DMA_COMPLETE)
328 spin_lock_irqsave(&hsuc->vchan.lock, flags);
329 vdesc = vchan_find_desc(&hsuc->vchan, cookie);
330 if (hsuc->desc && cookie == hsuc->desc->vdesc.tx.cookie) {
331 bytes = hsu_dma_active_desc_size(hsuc);
332 dma_set_residue(state, bytes);
333 status = hsuc->desc->status;
335 bytes = to_hsu_dma_desc(vdesc)->length;
336 dma_set_residue(state, bytes);
338 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
343 static int hsu_dma_slave_config(struct dma_chan *chan,
344 struct dma_slave_config *config)
346 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
348 memcpy(&hsuc->config, config, sizeof(hsuc->config));
353 static int hsu_dma_pause(struct dma_chan *chan)
355 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
358 spin_lock_irqsave(&hsuc->vchan.lock, flags);
359 if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) {
360 hsu_chan_disable(hsuc);
361 hsuc->desc->status = DMA_PAUSED;
363 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
368 static int hsu_dma_resume(struct dma_chan *chan)
370 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
373 spin_lock_irqsave(&hsuc->vchan.lock, flags);
374 if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) {
375 hsuc->desc->status = DMA_IN_PROGRESS;
376 hsu_chan_enable(hsuc);
378 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
383 static int hsu_dma_terminate_all(struct dma_chan *chan)
385 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
389 spin_lock_irqsave(&hsuc->vchan.lock, flags);
391 hsu_dma_stop_channel(hsuc);
393 hsu_dma_desc_free(&hsuc->desc->vdesc);
397 vchan_get_all_descriptors(&hsuc->vchan, &head);
398 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
399 vchan_dma_desc_free_list(&hsuc->vchan, &head);
404 static void hsu_dma_free_chan_resources(struct dma_chan *chan)
406 vchan_free_chan_resources(to_virt_chan(chan));
409 static void hsu_dma_synchronize(struct dma_chan *chan)
411 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
413 vchan_synchronize(&hsuc->vchan);
416 int hsu_dma_probe(struct hsu_dma_chip *chip)
419 void __iomem *addr = chip->regs + chip->offset;
423 hsu = devm_kzalloc(chip->dev, sizeof(*hsu), GFP_KERNEL);
429 /* Calculate nr_channels from the IO space length */
430 hsu->nr_channels = (chip->length - chip->offset) / HSU_DMA_CHAN_LENGTH;
432 hsu->chan = devm_kcalloc(chip->dev, hsu->nr_channels,
433 sizeof(*hsu->chan), GFP_KERNEL);
437 INIT_LIST_HEAD(&hsu->dma.channels);
438 for (i = 0; i < hsu->nr_channels; i++) {
439 struct hsu_dma_chan *hsuc = &hsu->chan[i];
441 hsuc->vchan.desc_free = hsu_dma_desc_free;
442 vchan_init(&hsuc->vchan, &hsu->dma);
444 hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
445 hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH;
448 dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask);
449 dma_cap_set(DMA_PRIVATE, hsu->dma.cap_mask);
451 hsu->dma.device_free_chan_resources = hsu_dma_free_chan_resources;
453 hsu->dma.device_prep_slave_sg = hsu_dma_prep_slave_sg;
455 hsu->dma.device_issue_pending = hsu_dma_issue_pending;
456 hsu->dma.device_tx_status = hsu_dma_tx_status;
458 hsu->dma.device_config = hsu_dma_slave_config;
459 hsu->dma.device_pause = hsu_dma_pause;
460 hsu->dma.device_resume = hsu_dma_resume;
461 hsu->dma.device_terminate_all = hsu_dma_terminate_all;
462 hsu->dma.device_synchronize = hsu_dma_synchronize;
464 hsu->dma.src_addr_widths = HSU_DMA_BUSWIDTHS;
465 hsu->dma.dst_addr_widths = HSU_DMA_BUSWIDTHS;
466 hsu->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
467 hsu->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
469 hsu->dma.dev = chip->dev;
471 dma_set_max_seg_size(hsu->dma.dev, HSU_CH_DxTSR_MASK);
473 ret = dma_async_device_register(&hsu->dma);
477 dev_info(chip->dev, "Found HSU DMA, %d channels\n", hsu->nr_channels);
480 EXPORT_SYMBOL_GPL(hsu_dma_probe);
482 int hsu_dma_remove(struct hsu_dma_chip *chip)
484 struct hsu_dma *hsu = chip->hsu;
487 dma_async_device_unregister(&hsu->dma);
489 for (i = 0; i < hsu->nr_channels; i++) {
490 struct hsu_dma_chan *hsuc = &hsu->chan[i];
492 tasklet_kill(&hsuc->vchan.task);
497 EXPORT_SYMBOL_GPL(hsu_dma_remove);
499 MODULE_LICENSE("GPL v2");
500 MODULE_DESCRIPTION("High Speed UART DMA core driver");
501 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");