1 #include <linux/device.h>
2 #include <linux/dma-mapping.h>
3 #include <linux/dmaengine.h>
4 #include <linux/sizes.h>
5 #include <linux/platform_device.h>
10 #include "musb_trace.h"
12 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
14 #define EP_MODE_AUTOREQ_NONE 0
15 #define EP_MODE_AUTOREQ_ALL_NEOP 1
16 #define EP_MODE_AUTOREQ_ALWAYS 3
18 #define EP_MODE_DMA_TRANSPARENT 0
19 #define EP_MODE_DMA_RNDIS 1
20 #define EP_MODE_DMA_GEN_RNDIS 3
22 #define USB_CTRL_TX_MODE 0x70
23 #define USB_CTRL_RX_MODE 0x74
24 #define USB_CTRL_AUTOREQ 0xd0
25 #define USB_TDOWN 0xd8
27 #define MUSB_DMA_NUM_CHANNELS 15
29 #define DA8XX_USB_MODE 0x10
30 #define DA8XX_USB_AUTOREQ 0x14
31 #define DA8XX_USB_TEARDOWN 0x1c
33 #define DA8XX_DMA_NUM_CHANNELS 4
35 struct cppi41_dma_controller {
36 struct dma_controller controller;
37 struct cppi41_dma_channel *rx_channel;
38 struct cppi41_dma_channel *tx_channel;
39 struct hrtimer early_tx;
40 struct list_head early_tx_list;
48 void (*set_dma_mode)(struct cppi41_dma_channel *cppi41_channel,
53 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
58 if (cppi41_channel->is_tx)
60 if (!is_host_active(cppi41_channel->controller->controller.musb))
63 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
64 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
66 cppi41_channel->usb_toggle = toggle;
69 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
71 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
72 struct musb *musb = hw_ep->musb;
76 if (cppi41_channel->is_tx)
78 if (!is_host_active(musb))
81 musb_ep_select(musb->mregs, hw_ep->epnum);
82 csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
83 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
86 * AM335x Advisory 1.0.13: Due to internal synchronisation error the
87 * data toggle may reset from DATA1 to DATA0 during receiving data from
88 * more than one endpoint.
90 if (!toggle && toggle == cppi41_channel->usb_toggle) {
91 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
92 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
93 musb_dbg(musb, "Restoring DATA1 toggle.");
96 cppi41_channel->usb_toggle = toggle;
99 static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep)
101 u8 epnum = hw_ep->epnum;
102 struct musb *musb = hw_ep->musb;
103 void __iomem *epio = musb->endpoints[epnum].regs;
106 musb_ep_select(musb->mregs, hw_ep->epnum);
107 csr = musb_readw(epio, MUSB_TXCSR);
108 if (csr & MUSB_TXCSR_TXPKTRDY)
113 static void cppi41_dma_callback(void *private_data,
114 const struct dmaengine_result *result);
116 static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)
118 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
119 struct musb *musb = hw_ep->musb;
120 void __iomem *epio = hw_ep->regs;
123 if (!cppi41_channel->prog_len ||
124 (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) {
127 cppi41_channel->channel.actual_len =
128 cppi41_channel->transferred;
129 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
130 cppi41_channel->channel.rx_packet_done = true;
133 * transmit ZLP using PIO mode for transfers which size is
134 * multiple of EP packet size.
136 if (cppi41_channel->tx_zlp && (cppi41_channel->transferred %
137 cppi41_channel->packet_sz) == 0) {
138 musb_ep_select(musb->mregs, hw_ep->epnum);
139 csr = MUSB_TXCSR_MODE | MUSB_TXCSR_TXPKTRDY;
140 musb_writew(epio, MUSB_TXCSR, csr);
143 trace_musb_cppi41_done(cppi41_channel);
144 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
146 /* next iteration, reload */
147 struct dma_chan *dc = cppi41_channel->dc;
148 struct dma_async_tx_descriptor *dma_desc;
149 enum dma_transfer_direction direction;
152 cppi41_channel->buf_addr += cppi41_channel->packet_sz;
154 remain_bytes = cppi41_channel->total_len;
155 remain_bytes -= cppi41_channel->transferred;
156 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
157 cppi41_channel->prog_len = remain_bytes;
159 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
161 dma_desc = dmaengine_prep_slave_single(dc,
162 cppi41_channel->buf_addr,
165 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
166 if (WARN_ON(!dma_desc))
169 dma_desc->callback_result = cppi41_dma_callback;
170 dma_desc->callback_param = &cppi41_channel->channel;
171 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
172 trace_musb_cppi41_cont(cppi41_channel);
173 dma_async_issue_pending(dc);
175 if (!cppi41_channel->is_tx) {
176 musb_ep_select(musb->mregs, hw_ep->epnum);
177 csr = musb_readw(epio, MUSB_RXCSR);
178 csr |= MUSB_RXCSR_H_REQPKT;
179 musb_writew(epio, MUSB_RXCSR, csr);
184 static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer)
186 struct cppi41_dma_controller *controller;
187 struct cppi41_dma_channel *cppi41_channel, *n;
190 enum hrtimer_restart ret = HRTIMER_NORESTART;
192 controller = container_of(timer, struct cppi41_dma_controller,
194 musb = controller->controller.musb;
196 spin_lock_irqsave(&musb->lock, flags);
197 list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,
200 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
202 empty = musb_is_tx_fifo_empty(hw_ep);
204 list_del_init(&cppi41_channel->tx_check);
205 cppi41_trans_done(cppi41_channel);
209 if (!list_empty(&controller->early_tx_list) &&
210 !hrtimer_is_queued(&controller->early_tx)) {
211 ret = HRTIMER_RESTART;
212 hrtimer_forward_now(&controller->early_tx, 20 * NSEC_PER_USEC);
215 spin_unlock_irqrestore(&musb->lock, flags);
219 static void cppi41_dma_callback(void *private_data,
220 const struct dmaengine_result *result)
222 struct dma_channel *channel = private_data;
223 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
224 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
225 struct cppi41_dma_controller *controller;
226 struct musb *musb = hw_ep->musb;
228 struct dma_tx_state txstate;
233 controller = cppi41_channel->controller;
234 if (controller->controller.dma_callback)
235 controller->controller.dma_callback(&controller->controller);
237 if (result->result == DMA_TRANS_ABORTED)
240 spin_lock_irqsave(&musb->lock, flags);
242 dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
244 transferred = cppi41_channel->prog_len - txstate.residue;
245 cppi41_channel->transferred += transferred;
247 trace_musb_cppi41_gb(cppi41_channel);
248 update_rx_toggle(cppi41_channel);
250 if (cppi41_channel->transferred == cppi41_channel->total_len ||
251 transferred < cppi41_channel->packet_sz)
252 cppi41_channel->prog_len = 0;
254 if (cppi41_channel->is_tx) {
257 if (is_host_active(musb))
258 type = hw_ep->out_qh->type;
260 type = hw_ep->ep_in.type;
262 if (type == USB_ENDPOINT_XFER_ISOC)
264 * Don't use the early-TX-interrupt workaround below
265 * for Isoch transfter. Since Isoch are periodic
266 * transfer, by the time the next transfer is
267 * scheduled, the current one should be done already.
269 * This avoids audio playback underrun issue.
273 empty = musb_is_tx_fifo_empty(hw_ep);
276 if (!cppi41_channel->is_tx || empty) {
277 cppi41_trans_done(cppi41_channel);
282 * On AM335x it has been observed that the TX interrupt fires
283 * too early that means the TXFIFO is not yet empty but the DMA
284 * engine says that it is done with the transfer. We don't
285 * receive a FIFO empty interrupt so the only thing we can do is
286 * to poll for the bit. On HS it usually takes 2us, on FS around
287 * 110us - 150us depending on the transfer size.
288 * We spin on HS (no longer than than 25us and setup a timer on
289 * FS to check for the bit and complete the transfer.
291 if (is_host_active(musb)) {
292 if (musb->port1_status & USB_PORT_STAT_HIGH_SPEED)
295 if (musb->g.speed == USB_SPEED_HIGH)
302 empty = musb_is_tx_fifo_empty(hw_ep);
304 cppi41_trans_done(cppi41_channel);
313 list_add_tail(&cppi41_channel->tx_check,
314 &controller->early_tx_list);
315 if (!hrtimer_is_queued(&controller->early_tx)) {
316 unsigned long usecs = cppi41_channel->total_len / 10;
318 hrtimer_start_range_ns(&controller->early_tx,
319 usecs * NSEC_PER_USEC,
325 spin_unlock_irqrestore(&musb->lock, flags);
328 static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
332 shift = (ep - 1) * 2;
333 old &= ~(3 << shift);
334 old |= mode << shift;
338 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
341 struct cppi41_dma_controller *controller = cppi41_channel->controller;
342 struct musb *musb = controller->controller.musb;
347 if (cppi41_channel->is_tx)
348 old_mode = controller->tx_mode;
350 old_mode = controller->rx_mode;
351 port = cppi41_channel->port_num;
352 new_mode = update_ep_mode(port, mode, old_mode);
354 if (new_mode == old_mode)
356 if (cppi41_channel->is_tx) {
357 controller->tx_mode = new_mode;
358 musb_writel(musb->ctrl_base, USB_CTRL_TX_MODE, new_mode);
360 controller->rx_mode = new_mode;
361 musb_writel(musb->ctrl_base, USB_CTRL_RX_MODE, new_mode);
365 static void da8xx_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
368 struct cppi41_dma_controller *controller = cppi41_channel->controller;
369 struct musb *musb = controller->controller.musb;
375 old_mode = controller->tx_mode;
376 port = cppi41_channel->port_num;
378 shift = (port - 1) * 4;
379 if (!cppi41_channel->is_tx)
381 new_mode = old_mode & ~(3 << shift);
382 new_mode |= mode << shift;
384 if (new_mode == old_mode)
386 controller->tx_mode = new_mode;
387 musb_writel(musb->ctrl_base, DA8XX_USB_MODE, new_mode);
391 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
394 struct cppi41_dma_controller *controller = cppi41_channel->controller;
399 old_mode = controller->auto_req;
400 port = cppi41_channel->port_num;
401 new_mode = update_ep_mode(port, mode, old_mode);
403 if (new_mode == old_mode)
405 controller->auto_req = new_mode;
406 musb_writel(controller->controller.musb->ctrl_base,
407 controller->autoreq_reg, new_mode);
410 static bool cppi41_configure_channel(struct dma_channel *channel,
411 u16 packet_sz, u8 mode,
412 dma_addr_t dma_addr, u32 len)
414 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
415 struct cppi41_dma_controller *controller = cppi41_channel->controller;
416 struct dma_chan *dc = cppi41_channel->dc;
417 struct dma_async_tx_descriptor *dma_desc;
418 enum dma_transfer_direction direction;
419 struct musb *musb = cppi41_channel->controller->controller.musb;
420 unsigned use_gen_rndis = 0;
422 cppi41_channel->buf_addr = dma_addr;
423 cppi41_channel->total_len = len;
424 cppi41_channel->transferred = 0;
425 cppi41_channel->packet_sz = packet_sz;
426 cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0;
429 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
430 * than max packet size at a time.
432 if (cppi41_channel->is_tx)
437 if (len > packet_sz) {
438 musb_writel(musb->ctrl_base,
439 RNDIS_REG(cppi41_channel->port_num), len);
441 controller->set_dma_mode(cppi41_channel,
442 EP_MODE_DMA_GEN_RNDIS);
445 cppi41_set_autoreq_mode(cppi41_channel,
446 EP_MODE_AUTOREQ_ALL_NEOP);
448 musb_writel(musb->ctrl_base,
449 RNDIS_REG(cppi41_channel->port_num), 0);
450 controller->set_dma_mode(cppi41_channel,
451 EP_MODE_DMA_TRANSPARENT);
452 cppi41_set_autoreq_mode(cppi41_channel,
453 EP_MODE_AUTOREQ_NONE);
457 controller->set_dma_mode(cppi41_channel,
458 EP_MODE_DMA_TRANSPARENT);
459 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
460 len = min_t(u32, packet_sz, len);
462 cppi41_channel->prog_len = len;
463 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
464 dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
465 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
469 dma_desc->callback_result = cppi41_dma_callback;
470 dma_desc->callback_param = channel;
471 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
472 cppi41_channel->channel.rx_packet_done = false;
474 trace_musb_cppi41_config(cppi41_channel);
476 save_rx_toggle(cppi41_channel);
477 dma_async_issue_pending(dc);
481 static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
482 struct musb_hw_ep *hw_ep, u8 is_tx)
484 struct cppi41_dma_controller *controller = container_of(c,
485 struct cppi41_dma_controller, controller);
486 struct cppi41_dma_channel *cppi41_channel = NULL;
487 u8 ch_num = hw_ep->epnum - 1;
489 if (ch_num >= controller->num_channels)
493 cppi41_channel = &controller->tx_channel[ch_num];
495 cppi41_channel = &controller->rx_channel[ch_num];
497 if (!cppi41_channel->dc)
500 if (cppi41_channel->is_allocated)
503 cppi41_channel->hw_ep = hw_ep;
504 cppi41_channel->is_allocated = 1;
506 trace_musb_cppi41_alloc(cppi41_channel);
507 return &cppi41_channel->channel;
510 static void cppi41_dma_channel_release(struct dma_channel *channel)
512 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
514 trace_musb_cppi41_free(cppi41_channel);
515 if (cppi41_channel->is_allocated) {
516 cppi41_channel->is_allocated = 0;
517 channel->status = MUSB_DMA_STATUS_FREE;
518 channel->actual_len = 0;
522 static int cppi41_dma_channel_program(struct dma_channel *channel,
523 u16 packet_sz, u8 mode,
524 dma_addr_t dma_addr, u32 len)
527 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
530 BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
531 channel->status == MUSB_DMA_STATUS_BUSY);
533 if (is_host_active(cppi41_channel->controller->controller.musb)) {
534 if (cppi41_channel->is_tx)
535 hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;
537 hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;
540 channel->status = MUSB_DMA_STATUS_BUSY;
541 channel->actual_len = 0;
544 packet_sz = hb_mult * (packet_sz & 0x7FF);
546 ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
548 channel->status = MUSB_DMA_STATUS_FREE;
553 static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
554 void *buf, u32 length)
556 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
557 struct cppi41_dma_controller *controller = cppi41_channel->controller;
558 struct musb *musb = controller->controller.musb;
560 if (is_host_active(musb)) {
564 if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)
566 if (cppi41_channel->is_tx)
568 /* AM335x Advisory 1.0.13. No workaround for device RX mode */
572 static int cppi41_dma_channel_abort(struct dma_channel *channel)
574 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
575 struct cppi41_dma_controller *controller = cppi41_channel->controller;
576 struct musb *musb = controller->controller.musb;
577 void __iomem *epio = cppi41_channel->hw_ep->regs;
583 is_tx = cppi41_channel->is_tx;
584 trace_musb_cppi41_abort(cppi41_channel);
586 if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
589 list_del_init(&cppi41_channel->tx_check);
591 csr = musb_readw(epio, MUSB_TXCSR);
592 csr &= ~MUSB_TXCSR_DMAENAB;
593 musb_writew(epio, MUSB_TXCSR, csr);
595 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
597 /* delay to drain to cppi dma pipeline for isoch */
600 csr = musb_readw(epio, MUSB_RXCSR);
601 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
602 musb_writew(epio, MUSB_RXCSR, csr);
604 /* wait to drain cppi dma pipe line */
607 csr = musb_readw(epio, MUSB_RXCSR);
608 if (csr & MUSB_RXCSR_RXPKTRDY) {
609 csr |= MUSB_RXCSR_FLUSHFIFO;
610 musb_writew(epio, MUSB_RXCSR, csr);
611 musb_writew(epio, MUSB_RXCSR, csr);
615 /* DA8xx Advisory 2.3.27: wait 250 ms before to start the teardown */
616 if (musb->io.quirks & MUSB_DA8XX)
619 tdbit = 1 << cppi41_channel->port_num;
625 musb_writel(musb->ctrl_base, controller->tdown_reg,
627 ret = dmaengine_terminate_all(cppi41_channel->dc);
628 } while (ret == -EAGAIN);
631 musb_writel(musb->ctrl_base, controller->tdown_reg, tdbit);
633 csr = musb_readw(epio, MUSB_TXCSR);
634 if (csr & MUSB_TXCSR_TXPKTRDY) {
635 csr |= MUSB_TXCSR_FLUSHFIFO;
636 musb_writew(epio, MUSB_TXCSR, csr);
640 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
644 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
649 for (i = 0; i < ctrl->num_channels; i++) {
650 dc = ctrl->tx_channel[i].dc;
652 dma_release_channel(dc);
653 dc = ctrl->rx_channel[i].dc;
655 dma_release_channel(dc);
659 static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
661 cppi41_release_all_dma_chans(controller);
664 static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
666 struct musb *musb = controller->controller.musb;
667 struct device *dev = musb->controller;
668 struct device_node *np = dev->parent->of_node;
669 struct cppi41_dma_channel *cppi41_channel;
674 count = of_property_count_strings(np, "dma-names");
678 for (i = 0; i < count; i++) {
680 struct dma_channel *musb_dma;
685 ret = of_property_read_string_index(np, "dma-names", i, &str);
688 if (strstarts(str, "tx"))
690 else if (strstarts(str, "rx"))
693 dev_err(dev, "Wrong dmatype %s\n", str);
696 ret = kstrtouint(str + 2, 0, &port);
701 if (port > controller->num_channels || !port)
704 cppi41_channel = &controller->tx_channel[port - 1];
706 cppi41_channel = &controller->rx_channel[port - 1];
708 cppi41_channel->controller = controller;
709 cppi41_channel->port_num = port;
710 cppi41_channel->is_tx = is_tx;
711 INIT_LIST_HEAD(&cppi41_channel->tx_check);
713 musb_dma = &cppi41_channel->channel;
714 musb_dma->private_data = cppi41_channel;
715 musb_dma->status = MUSB_DMA_STATUS_FREE;
716 musb_dma->max_len = SZ_4M;
718 dc = dma_request_chan(dev->parent, str);
721 if (ret != -EPROBE_DEFER)
722 dev_err(dev, "Failed to request %s: %d.\n",
727 cppi41_channel->dc = dc;
731 cppi41_release_all_dma_chans(controller);
735 void cppi41_dma_controller_destroy(struct dma_controller *c)
737 struct cppi41_dma_controller *controller = container_of(c,
738 struct cppi41_dma_controller, controller);
740 hrtimer_cancel(&controller->early_tx);
741 cppi41_dma_controller_stop(controller);
742 kfree(controller->rx_channel);
743 kfree(controller->tx_channel);
746 EXPORT_SYMBOL_GPL(cppi41_dma_controller_destroy);
748 struct dma_controller *
749 cppi41_dma_controller_create(struct musb *musb, void __iomem *base)
751 struct cppi41_dma_controller *controller;
755 if (!musb->controller->parent->of_node) {
756 dev_err(musb->controller, "Need DT for the DMA engine.\n");
760 controller = kzalloc(sizeof(*controller), GFP_KERNEL);
764 hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
765 controller->early_tx.function = cppi41_recheck_tx_req;
766 INIT_LIST_HEAD(&controller->early_tx_list);
768 controller->controller.channel_alloc = cppi41_dma_channel_allocate;
769 controller->controller.channel_release = cppi41_dma_channel_release;
770 controller->controller.channel_program = cppi41_dma_channel_program;
771 controller->controller.channel_abort = cppi41_dma_channel_abort;
772 controller->controller.is_compatible = cppi41_is_compatible;
773 controller->controller.musb = musb;
775 if (musb->io.quirks & MUSB_DA8XX) {
776 controller->tdown_reg = DA8XX_USB_TEARDOWN;
777 controller->autoreq_reg = DA8XX_USB_AUTOREQ;
778 controller->set_dma_mode = da8xx_set_dma_mode;
779 controller->num_channels = DA8XX_DMA_NUM_CHANNELS;
781 controller->tdown_reg = USB_TDOWN;
782 controller->autoreq_reg = USB_CTRL_AUTOREQ;
783 controller->set_dma_mode = cppi41_set_dma_mode;
784 controller->num_channels = MUSB_DMA_NUM_CHANNELS;
787 channel_size = controller->num_channels *
788 sizeof(struct cppi41_dma_channel);
789 controller->rx_channel = kzalloc(channel_size, GFP_KERNEL);
790 if (!controller->rx_channel)
791 goto rx_channel_alloc_fail;
792 controller->tx_channel = kzalloc(channel_size, GFP_KERNEL);
793 if (!controller->tx_channel)
794 goto tx_channel_alloc_fail;
796 ret = cppi41_dma_controller_start(controller);
799 return &controller->controller;
802 kfree(controller->tx_channel);
803 tx_channel_alloc_fail:
804 kfree(controller->rx_channel);
805 rx_channel_alloc_fail:
808 if (ret == -EPROBE_DEFER)
812 EXPORT_SYMBOL_GPL(cppi41_dma_controller_create);