1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Altera TSE SGDMA and MSGDMA Linux driver
3 * Copyright (C) 2014 Altera Corporation. All rights reserved
6 #include <linux/list.h>
7 #include "altera_utils.h"
8 #include "altera_tse.h"
9 #include "altera_sgdmahw.h"
10 #include "altera_sgdma.h"
12 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
13 struct sgdma_descrip __iomem *ndesc,
14 dma_addr_t ndesc_phys,
22 static int sgdma_async_write(struct altera_tse_private *priv,
23 struct sgdma_descrip __iomem *desc);
25 static int sgdma_async_read(struct altera_tse_private *priv);
28 sgdma_txphysaddr(struct altera_tse_private *priv,
29 struct sgdma_descrip __iomem *desc);
32 sgdma_rxphysaddr(struct altera_tse_private *priv,
33 struct sgdma_descrip __iomem *desc);
35 static int sgdma_txbusy(struct altera_tse_private *priv);
37 static int sgdma_rxbusy(struct altera_tse_private *priv);
40 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer);
43 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer);
45 static struct tse_buffer *
46 dequeue_tx(struct altera_tse_private *priv);
48 static struct tse_buffer *
49 dequeue_rx(struct altera_tse_private *priv);
51 static struct tse_buffer *
52 queue_rx_peekhead(struct altera_tse_private *priv);
54 int sgdma_initialize(struct altera_tse_private *priv)
56 priv->txctrlreg = SGDMA_CTRLREG_ILASTD |
59 priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP |
63 INIT_LIST_HEAD(&priv->txlisthd);
64 INIT_LIST_HEAD(&priv->rxlisthd);
66 priv->rxdescphys = (dma_addr_t) 0;
67 priv->txdescphys = (dma_addr_t) 0;
69 priv->rxdescphys = dma_map_single(priv->device,
70 (void __force *)priv->rx_dma_desc,
71 priv->rxdescmem, DMA_BIDIRECTIONAL);
73 if (dma_mapping_error(priv->device, priv->rxdescphys)) {
74 sgdma_uninitialize(priv);
75 netdev_err(priv->dev, "error mapping rx descriptor memory\n");
79 priv->txdescphys = dma_map_single(priv->device,
80 (void __force *)priv->tx_dma_desc,
81 priv->txdescmem, DMA_TO_DEVICE);
83 if (dma_mapping_error(priv->device, priv->txdescphys)) {
84 sgdma_uninitialize(priv);
85 netdev_err(priv->dev, "error mapping tx descriptor memory\n");
89 /* Initialize descriptor memory to all 0's, sync memory to cache */
90 memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
91 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);
93 dma_sync_single_for_device(priv->device, priv->txdescphys,
94 priv->txdescmem, DMA_TO_DEVICE);
96 dma_sync_single_for_device(priv->device, priv->rxdescphys,
97 priv->rxdescmem, DMA_TO_DEVICE);
102 void sgdma_uninitialize(struct altera_tse_private *priv)
104 if (priv->rxdescphys)
105 dma_unmap_single(priv->device, priv->rxdescphys,
106 priv->rxdescmem, DMA_BIDIRECTIONAL);
108 if (priv->txdescphys)
109 dma_unmap_single(priv->device, priv->txdescphys,
110 priv->txdescmem, DMA_TO_DEVICE);
113 /* This function resets the SGDMA controller and clears the
114 * descriptor memory used for transmits and receives.
116 void sgdma_reset(struct altera_tse_private *priv)
118 /* Initialize descriptor memory to 0 */
119 memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
120 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);
122 csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control));
123 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));
125 csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control));
126 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
129 /* For SGDMA, interrupts remain enabled after initially enabling,
130 * so no need to provide implementations for abstract enable
134 void sgdma_enable_rxirq(struct altera_tse_private *priv)
138 void sgdma_enable_txirq(struct altera_tse_private *priv)
142 void sgdma_disable_rxirq(struct altera_tse_private *priv)
146 void sgdma_disable_txirq(struct altera_tse_private *priv)
150 void sgdma_clear_rxirq(struct altera_tse_private *priv)
152 tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control),
153 SGDMA_CTRLREG_CLRINT);
156 void sgdma_clear_txirq(struct altera_tse_private *priv)
158 tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control),
159 SGDMA_CTRLREG_CLRINT);
162 /* transmits buffer through SGDMA. Returns number of buffers
163 * transmitted, 0 if not possible.
165 * tx_lock is held by the caller
167 int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer)
169 struct sgdma_descrip __iomem *descbase =
170 (struct sgdma_descrip __iomem *)priv->tx_dma_desc;
172 struct sgdma_descrip __iomem *cdesc = &descbase[0];
173 struct sgdma_descrip __iomem *ndesc = &descbase[1];
175 /* wait 'til the tx sgdma is ready for the next transmit request */
176 if (sgdma_txbusy(priv))
179 sgdma_setup_descrip(cdesc, /* current descriptor */
180 ndesc, /* next descriptor */
181 sgdma_txphysaddr(priv, ndesc),
182 buffer->dma_addr, /* address of packet to xmit */
183 0, /* write addr 0 for tx dma */
184 buffer->len, /* length of packet */
185 SGDMA_CONTROL_EOP, /* Generate EOP */
187 SGDMA_CONTROL_WR_FIXED); /* Generate SOP */
189 sgdma_async_write(priv, cdesc);
191 /* enqueue the request to the pending transmit queue */
192 queue_tx(priv, buffer);
198 /* tx_lock held to protect access to queued tx list
200 u32 sgdma_tx_completions(struct altera_tse_private *priv)
204 if (!sgdma_txbusy(priv) &&
205 ((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control))
206 & SGDMA_CONTROL_HW_OWNED) == 0) &&
207 (dequeue_tx(priv))) {
214 void sgdma_start_rxdma(struct altera_tse_private *priv)
216 sgdma_async_read(priv);
219 void sgdma_add_rx_desc(struct altera_tse_private *priv,
220 struct tse_buffer *rxbuffer)
222 queue_rx(priv, rxbuffer);
225 /* status is returned on upper 16 bits,
226 * length is returned in lower 16 bits
228 u32 sgdma_rx_status(struct altera_tse_private *priv)
230 struct sgdma_descrip __iomem *base =
231 (struct sgdma_descrip __iomem *)priv->rx_dma_desc;
232 struct sgdma_descrip __iomem *desc = NULL;
233 struct tse_buffer *rxbuffer = NULL;
234 unsigned int rxstatus = 0;
236 u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status));
239 if (sts & SGDMA_STSREG_EOP) {
240 unsigned int pktlength = 0;
241 unsigned int pktstatus = 0;
242 dma_sync_single_for_cpu(priv->device,
247 pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred));
248 pktstatus = csrrd8(desc, sgdma_descroffs(status));
249 rxstatus = pktstatus & ~SGDMA_STATUS_EOP;
250 rxstatus = rxstatus << 16;
251 rxstatus |= (pktlength & 0xffff);
254 csrwr8(0, desc, sgdma_descroffs(status));
256 rxbuffer = dequeue_rx(priv);
257 if (rxbuffer == NULL)
258 netdev_info(priv->dev,
259 "sgdma rx and rx queue empty!\n");
262 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
264 csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status));
266 /* kick the rx sgdma after reaping this descriptor */
267 sgdma_async_read(priv);
270 /* If the SGDMA indicated an end of packet on recv,
271 * then it's expected that the rxstatus from the
272 * descriptor is non-zero - meaning a valid packet
273 * with a nonzero length, or an error has been
274 * indicated. if not, then all we can do is signal
275 * an error and return no packet received. Most likely
276 * there is a system design error, or an error in the
277 * underlying kernel (cache or cache management problem)
279 netdev_err(priv->dev,
280 "SGDMA RX Error Info: %x, %x, %x\n",
281 sts, csrrd8(desc, sgdma_descroffs(status)),
284 } else if (sts == 0) {
285 sgdma_async_read(priv);
292 /* Private functions */
293 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
294 struct sgdma_descrip __iomem *ndesc,
295 dma_addr_t ndesc_phys,
303 /* Clear the next descriptor as not owned by hardware */
305 u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control));
306 ctrl &= ~SGDMA_CONTROL_HW_OWNED;
307 csrwr8(ctrl, ndesc, sgdma_descroffs(control));
309 ctrl = SGDMA_CONTROL_HW_OWNED;
310 ctrl |= generate_eop;
314 /* Channel is implicitly zero, initialized to 0 by default */
315 csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr));
316 csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr));
318 csrwr32(0, desc, sgdma_descroffs(pad1));
319 csrwr32(0, desc, sgdma_descroffs(pad2));
320 csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next));
322 csrwr8(ctrl, desc, sgdma_descroffs(control));
323 csrwr8(0, desc, sgdma_descroffs(status));
324 csrwr8(0, desc, sgdma_descroffs(wburst));
325 csrwr8(0, desc, sgdma_descroffs(rburst));
326 csrwr16(length, desc, sgdma_descroffs(bytes));
327 csrwr16(0, desc, sgdma_descroffs(bytes_xferred));
330 /* If hardware is busy, don't restart async read.
331 * if status register is 0 - meaning initial state, restart async read,
332 * probably for the first time when populating a receive buffer.
333 * If read status indicate not busy and a status, restart the async
336 static int sgdma_async_read(struct altera_tse_private *priv)
338 struct sgdma_descrip __iomem *descbase =
339 (struct sgdma_descrip __iomem *)priv->rx_dma_desc;
341 struct sgdma_descrip __iomem *cdesc = &descbase[0];
342 struct sgdma_descrip __iomem *ndesc = &descbase[1];
343 struct tse_buffer *rxbuffer = NULL;
345 if (!sgdma_rxbusy(priv)) {
346 rxbuffer = queue_rx_peekhead(priv);
347 if (rxbuffer == NULL) {
348 netdev_err(priv->dev, "no rx buffers available\n");
352 sgdma_setup_descrip(cdesc, /* current descriptor */
353 ndesc, /* next descriptor */
354 sgdma_rxphysaddr(priv, ndesc),
355 0, /* read addr 0 for rx dma */
356 rxbuffer->dma_addr, /* write addr for rx dma */
357 0, /* read 'til EOP */
358 0, /* EOP: NA for rx dma */
359 0, /* read fixed: NA for rx dma */
360 0); /* SOP: NA for rx DMA */
362 dma_sync_single_for_device(priv->device,
367 csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)),
369 sgdma_csroffs(next_descrip));
371 csrwr32((priv->rxctrlreg | SGDMA_CTRLREG_START),
373 sgdma_csroffs(control));
381 static int sgdma_async_write(struct altera_tse_private *priv,
382 struct sgdma_descrip __iomem *desc)
384 if (sgdma_txbusy(priv))
387 /* clear control and status */
388 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));
389 csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status));
391 dma_sync_single_for_device(priv->device, priv->txdescphys,
392 SGDMA_DESC_LEN, DMA_TO_DEVICE);
394 csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)),
396 sgdma_csroffs(next_descrip));
398 csrwr32((priv->txctrlreg | SGDMA_CTRLREG_START),
400 sgdma_csroffs(control));
406 sgdma_txphysaddr(struct altera_tse_private *priv,
407 struct sgdma_descrip __iomem *desc)
409 dma_addr_t paddr = priv->txdescmem_busaddr;
410 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc;
411 return (dma_addr_t)((uintptr_t)paddr + offs);
415 sgdma_rxphysaddr(struct altera_tse_private *priv,
416 struct sgdma_descrip __iomem *desc)
418 dma_addr_t paddr = priv->rxdescmem_busaddr;
419 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc;
420 return (dma_addr_t)((uintptr_t)paddr + offs);
423 #define list_remove_head(list, entry, type, member) \
426 if (!list_empty(list)) { \
427 entry = list_entry((list)->next, type, member); \
428 list_del_init(&entry->member); \
432 #define list_peek_head(list, entry, type, member) \
435 if (!list_empty(list)) { \
436 entry = list_entry((list)->next, type, member); \
440 /* adds a tse_buffer to the tail of a tx buffer list.
441 * assumes the caller is managing and holding a mutual exclusion
442 * primitive to avoid simultaneous pushes/pops to the list.
445 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer)
447 list_add_tail(&buffer->lh, &priv->txlisthd);
451 /* adds a tse_buffer to the tail of a rx buffer list
452 * assumes the caller is managing and holding a mutual exclusion
453 * primitive to avoid simultaneous pushes/pops to the list.
456 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer)
458 list_add_tail(&buffer->lh, &priv->rxlisthd);
461 /* dequeues a tse_buffer from the transmit buffer list, otherwise
462 * returns NULL if empty.
463 * assumes the caller is managing and holding a mutual exclusion
464 * primitive to avoid simultaneous pushes/pops to the list.
466 static struct tse_buffer *
467 dequeue_tx(struct altera_tse_private *priv)
469 struct tse_buffer *buffer = NULL;
470 list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh);
474 /* dequeues a tse_buffer from the receive buffer list, otherwise
475 * returns NULL if empty
476 * assumes the caller is managing and holding a mutual exclusion
477 * primitive to avoid simultaneous pushes/pops to the list.
479 static struct tse_buffer *
480 dequeue_rx(struct altera_tse_private *priv)
482 struct tse_buffer *buffer = NULL;
483 list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
487 /* dequeues a tse_buffer from the receive buffer list, otherwise
488 * returns NULL if empty
489 * assumes the caller is managing and holding a mutual exclusion
490 * primitive to avoid simultaneous pushes/pops to the list while the
491 * head is being examined.
493 static struct tse_buffer *
494 queue_rx_peekhead(struct altera_tse_private *priv)
496 struct tse_buffer *buffer = NULL;
497 list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
501 /* check and return rx sgdma status without polling
503 static int sgdma_rxbusy(struct altera_tse_private *priv)
505 return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status))
509 /* waits for the tx sgdma to finish it's current operation, returns 0
510 * when it transitions to nonbusy, returns 1 if the operation times out
512 static int sgdma_txbusy(struct altera_tse_private *priv)
516 /* if DMA is busy, wait for current transaction to finish */
517 while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
518 & SGDMA_STSREG_BUSY) && (delay++ < 100))
521 if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
522 & SGDMA_STSREG_BUSY) {
523 netdev_err(priv->dev, "timeout waiting for tx dma\n");