drivers/usb/musb/musb_cppi41.c

   1 #include <linux/device.h>
   2 #include <linux/dma-mapping.h>
   3 #include <linux/dmaengine.h>
   4 #include <linux/sizes.h>
   5 #include <linux/platform_device.h>
   6 #include <linux/of.h>
   7
   8 #include "cppi_dma.h"
   9 #include "musb_core.h"
  10 #include "musb_trace.h"
  11
  12 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
  13
  14 #define EP_MODE_AUTOREQ_NONE            0
  15 #define EP_MODE_AUTOREQ_ALL_NEOP        1
  16 #define EP_MODE_AUTOREQ_ALWAYS          3
  17
  18 #define EP_MODE_DMA_TRANSPARENT         0
  19 #define EP_MODE_DMA_RNDIS               1
  20 #define EP_MODE_DMA_GEN_RNDIS           3
  21
  22 #define USB_CTRL_TX_MODE        0x70
  23 #define USB_CTRL_RX_MODE        0x74
  24 #define USB_CTRL_AUTOREQ        0xd0
  25 #define USB_TDOWN               0xd8
  26
  27 #define MUSB_DMA_NUM_CHANNELS 15
  28
  29 struct cppi41_dma_controller {
  30         struct dma_controller controller;
  31         struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS];
  32         struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS];
  33         struct musb *musb;
  34         struct hrtimer early_tx;
  35         struct list_head early_tx_list;
  36         u32 rx_mode;
  37         u32 tx_mode;
  38         u32 auto_req;
  39 };
  40
  41 static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  42 {
  43         u16 csr;
  44         u8 toggle;
  45
  46         if (cppi41_channel->is_tx)
  47                 return;
  48         if (!is_host_active(cppi41_channel->controller->musb))
  49                 return;
  50
  51         csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
  52         toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  53
  54         cppi41_channel->usb_toggle = toggle;
  55 }
  56
  57 static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  58 {
  59         struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
  60         struct musb *musb = hw_ep->musb;
  61         u16 csr;
  62         u8 toggle;
  63
  64         if (cppi41_channel->is_tx)
  65                 return;
  66         if (!is_host_active(musb))
  67                 return;
  68
  69         musb_ep_select(musb->mregs, hw_ep->epnum);
  70         csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
  71         toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  72
  73         /*
  74          * AM335x Advisory 1.0.13: Due to internal synchronisation error the
  75          * data toggle may reset from DATA1 to DATA0 during receiving data from
  76          * more than one endpoint.
  77          */
  78         if (!toggle && toggle == cppi41_channel->usb_toggle) {
  79                 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
  80                 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
  81                 musb_dbg(cppi41_channel->controller->musb,
  82                                 "Restoring DATA1 toggle.");
  83         }
  84
  85         cppi41_channel->usb_toggle = toggle;
  86 }
  87
  88 static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep)
  89 {
  90         u8              epnum = hw_ep->epnum;
  91         struct musb     *musb = hw_ep->musb;
  92         void __iomem    *epio = musb->endpoints[epnum].regs;
  93         u16             csr;
  94
  95         musb_ep_select(musb->mregs, hw_ep->epnum);
  96         csr = musb_readw(epio, MUSB_TXCSR);
  97         if (csr & MUSB_TXCSR_TXPKTRDY)
  98                 return false;
  99         return true;
 100 }
 101
 102 static void cppi41_dma_callback(void *private_data);
 103
 104 static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)
 105 {
 106         struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
 107         struct musb *musb = hw_ep->musb;
 108         void __iomem *epio = hw_ep->regs;
 109         u16 csr;
 110
 111         if (!cppi41_channel->prog_len ||
 112             (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) {
 113
 114                 /* done, complete */
 115                 cppi41_channel->channel.actual_len =
 116                         cppi41_channel->transferred;
 117                 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
 118                 cppi41_channel->channel.rx_packet_done = true;
 119
 120                 /*
 121                  * transmit ZLP using PIO mode for transfers which size is
 122                  * multiple of EP packet size.
 123                  */
 124                 if (cppi41_channel->tx_zlp && (cppi41_channel->transferred %
 125                                         cppi41_channel->packet_sz) == 0) {
 126                         musb_ep_select(musb->mregs, hw_ep->epnum);
 127                         csr = MUSB_TXCSR_MODE | MUSB_TXCSR_TXPKTRDY;
 128                         musb_writew(epio, MUSB_TXCSR, csr);
 129                 }
 130
 131                 trace_musb_cppi41_done(cppi41_channel);
 132                 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
 133         } else {
 134                 /* next iteration, reload */
 135                 struct dma_chan *dc = cppi41_channel->dc;
 136                 struct dma_async_tx_descriptor *dma_desc;
 137                 enum dma_transfer_direction direction;
 138                 u32 remain_bytes;
 139
 140                 cppi41_channel->buf_addr += cppi41_channel->packet_sz;
 141
 142                 remain_bytes = cppi41_channel->total_len;
 143                 remain_bytes -= cppi41_channel->transferred;
 144                 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
 145                 cppi41_channel->prog_len = remain_bytes;
 146
 147                 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
 148                         : DMA_DEV_TO_MEM;
 149                 dma_desc = dmaengine_prep_slave_single(dc,
 150                                 cppi41_channel->buf_addr,
 151                                 remain_bytes,
 152                                 direction,
 153                                 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 154                 if (WARN_ON(!dma_desc))
 155                         return;
 156
 157                 dma_desc->callback = cppi41_dma_callback;
 158                 dma_desc->callback_param = &cppi41_channel->channel;
 159                 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
 160                 trace_musb_cppi41_cont(cppi41_channel);
 161                 dma_async_issue_pending(dc);
 162
 163                 if (!cppi41_channel->is_tx) {
 164                         musb_ep_select(musb->mregs, hw_ep->epnum);
 165                         csr = musb_readw(epio, MUSB_RXCSR);
 166                         csr |= MUSB_RXCSR_H_REQPKT;
 167                         musb_writew(epio, MUSB_RXCSR, csr);
 168                 }
 169         }
 170 }
 171
 172 static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer)
 173 {
 174         struct cppi41_dma_controller *controller;
 175         struct cppi41_dma_channel *cppi41_channel, *n;
 176         struct musb *musb;
 177         unsigned long flags;
 178         enum hrtimer_restart ret = HRTIMER_NORESTART;
 179
 180         controller = container_of(timer, struct cppi41_dma_controller,
 181                         early_tx);
 182         musb = controller->musb;
 183
 184         spin_lock_irqsave(&musb->lock, flags);
 185         list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,
 186                         tx_check) {
 187                 bool empty;
 188                 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
 189
 190                 empty = musb_is_tx_fifo_empty(hw_ep);
 191                 if (empty) {
 192                         list_del_init(&cppi41_channel->tx_check);
 193                         cppi41_trans_done(cppi41_channel);
 194                 }
 195         }
 196
 197         if (!list_empty(&controller->early_tx_list) &&
 198             !hrtimer_is_queued(&controller->early_tx)) {
 199                 ret = HRTIMER_RESTART;
 200                 hrtimer_forward_now(&controller->early_tx,
 201                                 ktime_set(0, 20 * NSEC_PER_USEC));
 202         }
 203
 204         spin_unlock_irqrestore(&musb->lock, flags);
 205         return ret;
 206 }
 207
 208 static void cppi41_dma_callback(void *private_data)
 209 {
 210         struct dma_channel *channel = private_data;
 211         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 212         struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
 213         struct cppi41_dma_controller *controller;
 214         struct musb *musb = hw_ep->musb;
 215         unsigned long flags;
 216         struct dma_tx_state txstate;
 217         u32 transferred;
 218         int is_hs = 0;
 219         bool empty;
 220
 221         spin_lock_irqsave(&musb->lock, flags);
 222
 223         dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
 224                         &txstate);
 225         transferred = cppi41_channel->prog_len - txstate.residue;
 226         cppi41_channel->transferred += transferred;
 227
 228         trace_musb_cppi41_gb(cppi41_channel);
 229         update_rx_toggle(cppi41_channel);
 230
 231         if (cppi41_channel->transferred == cppi41_channel->total_len ||
 232                         transferred < cppi41_channel->packet_sz)
 233                 cppi41_channel->prog_len = 0;
 234
 235         if (cppi41_channel->is_tx) {
 236                 u8 type;
 237
 238                 if (is_host_active(musb))
 239                         type = hw_ep->out_qh->type;
 240                 else
 241                         type = hw_ep->ep_in.type;
 242
 243                 if (type == USB_ENDPOINT_XFER_ISOC)
 244                         /*
 245                          * Don't use the early-TX-interrupt workaround below
 246                          * for Isoch transfter. Since Isoch are periodic
 247                          * transfer, by the time the next transfer is
 248                          * scheduled, the current one should be done already.
 249                          *
 250                          * This avoids audio playback underrun issue.
 251                          */
 252                         empty = true;
 253                 else
 254                         empty = musb_is_tx_fifo_empty(hw_ep);
 255         }
 256
 257         if (!cppi41_channel->is_tx || empty) {
 258                 cppi41_trans_done(cppi41_channel);
 259                 goto out;
 260         }
 261
 262         /*
 263          * On AM335x it has been observed that the TX interrupt fires
 264          * too early that means the TXFIFO is not yet empty but the DMA
 265          * engine says that it is done with the transfer. We don't
 266          * receive a FIFO empty interrupt so the only thing we can do is
 267          * to poll for the bit. On HS it usually takes 2us, on FS around
 268          * 110us - 150us depending on the transfer size.
 269          * We spin on HS (no longer than than 25us and setup a timer on
 270          * FS to check for the bit and complete the transfer.
 271          */
 272         controller = cppi41_channel->controller;
 273
 274         if (is_host_active(musb)) {
 275                 if (musb->port1_status & USB_PORT_STAT_HIGH_SPEED)
 276                         is_hs = 1;
 277         } else {
 278                 if (musb->g.speed == USB_SPEED_HIGH)
 279                         is_hs = 1;
 280         }
 281         if (is_hs) {
 282                 unsigned wait = 25;
 283
 284                 do {
 285                         empty = musb_is_tx_fifo_empty(hw_ep);
 286                         if (empty) {
 287                                 cppi41_trans_done(cppi41_channel);
 288                                 goto out;
 289                         }
 290                         wait--;
 291                         if (!wait)
 292                                 break;
 293                         cpu_relax();
 294                 } while (1);
 295         }
 296         list_add_tail(&cppi41_channel->tx_check,
 297                         &controller->early_tx_list);
 298         if (!hrtimer_is_queued(&controller->early_tx)) {
 299                 unsigned long usecs = cppi41_channel->total_len / 10;
 300
 301                 hrtimer_start_range_ns(&controller->early_tx,
 302                                 ktime_set(0, usecs * NSEC_PER_USEC),
 303                                 20 * NSEC_PER_USEC,
 304                                 HRTIMER_MODE_REL);
 305         }
 306
 307 out:
 308         spin_unlock_irqrestore(&musb->lock, flags);
 309 }
 310
 311 static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
 312 {
 313         unsigned shift;
 314
 315         shift = (ep - 1) * 2;
 316         old &= ~(3 << shift);
 317         old |= mode << shift;
 318         return old;
 319 }
 320
 321 static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
 322                 unsigned mode)
 323 {
 324         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 325         u32 port;
 326         u32 new_mode;
 327         u32 old_mode;
 328
 329         if (cppi41_channel->is_tx)
 330                 old_mode = controller->tx_mode;
 331         else
 332                 old_mode = controller->rx_mode;
 333         port = cppi41_channel->port_num;
 334         new_mode = update_ep_mode(port, mode, old_mode);
 335
 336         if (new_mode == old_mode)
 337                 return;
 338         if (cppi41_channel->is_tx) {
 339                 controller->tx_mode = new_mode;
 340                 musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE,
 341                                 new_mode);
 342         } else {
 343                 controller->rx_mode = new_mode;
 344                 musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE,
 345                                 new_mode);
 346         }
 347 }
 348
 349 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
 350                 unsigned mode)
 351 {
 352         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 353         u32 port;
 354         u32 new_mode;
 355         u32 old_mode;
 356
 357         old_mode = controller->auto_req;
 358         port = cppi41_channel->port_num;
 359         new_mode = update_ep_mode(port, mode, old_mode);
 360
 361         if (new_mode == old_mode)
 362                 return;
 363         controller->auto_req = new_mode;
 364         musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode);
 365 }
 366
 367 static bool cppi41_configure_channel(struct dma_channel *channel,
 368                                 u16 packet_sz, u8 mode,
 369                                 dma_addr_t dma_addr, u32 len)
 370 {
 371         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 372         struct dma_chan *dc = cppi41_channel->dc;
 373         struct dma_async_tx_descriptor *dma_desc;
 374         enum dma_transfer_direction direction;
 375         struct musb *musb = cppi41_channel->controller->musb;
 376         unsigned use_gen_rndis = 0;
 377
 378         cppi41_channel->buf_addr = dma_addr;
 379         cppi41_channel->total_len = len;
 380         cppi41_channel->transferred = 0;
 381         cppi41_channel->packet_sz = packet_sz;
 382         cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0;
 383
 384         /*
 385          * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
 386          * than max packet size at a time.
 387          */
 388         if (cppi41_channel->is_tx)
 389                 use_gen_rndis = 1;
 390
 391         if (use_gen_rndis) {
 392                 /* RNDIS mode */
 393                 if (len > packet_sz) {
 394                         musb_writel(musb->ctrl_base,
 395                                 RNDIS_REG(cppi41_channel->port_num), len);
 396                         /* gen rndis */
 397                         cppi41_set_dma_mode(cppi41_channel,
 398                                         EP_MODE_DMA_GEN_RNDIS);
 399
 400                         /* auto req */
 401                         cppi41_set_autoreq_mode(cppi41_channel,
 402                                         EP_MODE_AUTOREQ_ALL_NEOP);
 403                 } else {
 404                         musb_writel(musb->ctrl_base,
 405                                         RNDIS_REG(cppi41_channel->port_num), 0);
 406                         cppi41_set_dma_mode(cppi41_channel,
 407                                         EP_MODE_DMA_TRANSPARENT);
 408                         cppi41_set_autoreq_mode(cppi41_channel,
 409                                         EP_MODE_AUTOREQ_NONE);
 410                 }
 411         } else {
 412                 /* fallback mode */
 413                 cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT);
 414                 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
 415                 len = min_t(u32, packet_sz, len);
 416         }
 417         cppi41_channel->prog_len = len;
 418         direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
 419         dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
 420                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 421         if (!dma_desc)
 422                 return false;
 423
 424         dma_desc->callback = cppi41_dma_callback;
 425         dma_desc->callback_param = channel;
 426         cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
 427         cppi41_channel->channel.rx_packet_done = false;
 428
 429         trace_musb_cppi41_config(cppi41_channel);
 430
 431         save_rx_toggle(cppi41_channel);
 432         dma_async_issue_pending(dc);
 433         return true;
 434 }
 435
 436 static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
 437                                 struct musb_hw_ep *hw_ep, u8 is_tx)
 438 {
 439         struct cppi41_dma_controller *controller = container_of(c,
 440                         struct cppi41_dma_controller, controller);
 441         struct cppi41_dma_channel *cppi41_channel = NULL;
 442         u8 ch_num = hw_ep->epnum - 1;
 443
 444         if (ch_num >= MUSB_DMA_NUM_CHANNELS)
 445                 return NULL;
 446
 447         if (is_tx)
 448                 cppi41_channel = &controller->tx_channel[ch_num];
 449         else
 450                 cppi41_channel = &controller->rx_channel[ch_num];
 451
 452         if (!cppi41_channel->dc)
 453                 return NULL;
 454
 455         if (cppi41_channel->is_allocated)
 456                 return NULL;
 457
 458         cppi41_channel->hw_ep = hw_ep;
 459         cppi41_channel->is_allocated = 1;
 460
 461         trace_musb_cppi41_alloc(cppi41_channel);
 462         return &cppi41_channel->channel;
 463 }
 464
 465 static void cppi41_dma_channel_release(struct dma_channel *channel)
 466 {
 467         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 468
 469         trace_musb_cppi41_free(cppi41_channel);
 470         if (cppi41_channel->is_allocated) {
 471                 cppi41_channel->is_allocated = 0;
 472                 channel->status = MUSB_DMA_STATUS_FREE;
 473                 channel->actual_len = 0;
 474         }
 475 }
 476
 477 static int cppi41_dma_channel_program(struct dma_channel *channel,
 478                                 u16 packet_sz, u8 mode,
 479                                 dma_addr_t dma_addr, u32 len)
 480 {
 481         int ret;
 482         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 483         int hb_mult = 0;
 484
 485         BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
 486                 channel->status == MUSB_DMA_STATUS_BUSY);
 487
 488         if (is_host_active(cppi41_channel->controller->musb)) {
 489                 if (cppi41_channel->is_tx)
 490                         hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;
 491                 else
 492                         hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;
 493         }
 494
 495         channel->status = MUSB_DMA_STATUS_BUSY;
 496         channel->actual_len = 0;
 497
 498         if (hb_mult)
 499                 packet_sz = hb_mult * (packet_sz & 0x7FF);
 500
 501         ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
 502         if (!ret)
 503                 channel->status = MUSB_DMA_STATUS_FREE;
 504
 505         return ret;
 506 }
 507
 508 static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
 509                 void *buf, u32 length)
 510 {
 511         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 512         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 513         struct musb *musb = controller->musb;
 514
 515         if (is_host_active(musb)) {
 516                 WARN_ON(1);
 517                 return 1;
 518         }
 519         if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)
 520                 return 0;
 521         if (cppi41_channel->is_tx)
 522                 return 1;
 523         /* AM335x Advisory 1.0.13. No workaround for device RX mode */
 524         return 0;
 525 }
 526
 527 static int cppi41_dma_channel_abort(struct dma_channel *channel)
 528 {
 529         struct cppi41_dma_channel *cppi41_channel = channel->private_data;
 530         struct cppi41_dma_controller *controller = cppi41_channel->controller;
 531         struct musb *musb = controller->musb;
 532         void __iomem *epio = cppi41_channel->hw_ep->regs;
 533         int tdbit;
 534         int ret;
 535         unsigned is_tx;
 536         u16 csr;
 537
 538         is_tx = cppi41_channel->is_tx;
 539         trace_musb_cppi41_abort(cppi41_channel);
 540
 541         if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
 542                 return 0;
 543
 544         list_del_init(&cppi41_channel->tx_check);
 545         if (is_tx) {
 546                 csr = musb_readw(epio, MUSB_TXCSR);
 547                 csr &= ~MUSB_TXCSR_DMAENAB;
 548                 musb_writew(epio, MUSB_TXCSR, csr);
 549         } else {
 550                 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
 551
 552                 /* delay to drain to cppi dma pipeline for isoch */
 553                 udelay(250);
 554
 555                 csr = musb_readw(epio, MUSB_RXCSR);
 556                 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
 557                 musb_writew(epio, MUSB_RXCSR, csr);
 558
 559                 /* wait to drain cppi dma pipe line */
 560                 udelay(50);
 561
 562                 csr = musb_readw(epio, MUSB_RXCSR);
 563                 if (csr & MUSB_RXCSR_RXPKTRDY) {
 564                         csr |= MUSB_RXCSR_FLUSHFIFO;
 565                         musb_writew(epio, MUSB_RXCSR, csr);
 566                         musb_writew(epio, MUSB_RXCSR, csr);
 567                 }
 568         }
 569
 570         tdbit = 1 << cppi41_channel->port_num;
 571         if (is_tx)
 572                 tdbit <<= 16;
 573
 574         do {
 575                 if (is_tx)
 576                         musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
 577                 ret = dmaengine_terminate_all(cppi41_channel->dc);
 578         } while (ret == -EAGAIN);
 579
 580         if (is_tx) {
 581                 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
 582
 583                 csr = musb_readw(epio, MUSB_TXCSR);
 584                 if (csr & MUSB_TXCSR_TXPKTRDY) {
 585                         csr |= MUSB_TXCSR_FLUSHFIFO;
 586                         musb_writew(epio, MUSB_TXCSR, csr);
 587                 }
 588         }
 589
 590         cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
 591         return 0;
 592 }
 593
 594 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
 595 {
 596         struct dma_chan *dc;
 597         int i;
 598
 599         for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) {
 600                 dc = ctrl->tx_channel[i].dc;
 601                 if (dc)
 602                         dma_release_channel(dc);
 603                 dc = ctrl->rx_channel[i].dc;
 604                 if (dc)
 605                         dma_release_channel(dc);
 606         }
 607 }
 608
 609 static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
 610 {
 611         cppi41_release_all_dma_chans(controller);
 612 }
 613
 614 static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
 615 {
 616         struct musb *musb = controller->musb;
 617         struct device *dev = musb->controller;
 618         struct device_node *np = dev->parent->of_node;
 619         struct cppi41_dma_channel *cppi41_channel;
 620         int count;
 621         int i;
 622         int ret;
 623
 624         count = of_property_count_strings(np, "dma-names");
 625         if (count < 0)
 626                 return count;
 627
 628         for (i = 0; i < count; i++) {
 629                 struct dma_chan *dc;
 630                 struct dma_channel *musb_dma;
 631                 const char *str;
 632                 unsigned is_tx;
 633                 unsigned int port;
 634
 635                 ret = of_property_read_string_index(np, "dma-names", i, &str);
 636                 if (ret)
 637                         goto err;
 638                 if (strstarts(str, "tx"))
 639                         is_tx = 1;
 640                 else if (strstarts(str, "rx"))
 641                         is_tx = 0;
 642                 else {
 643                         dev_err(dev, "Wrong dmatype %s\n", str);
 644                         goto err;
 645                 }
 646                 ret = kstrtouint(str + 2, 0, &port);
 647                 if (ret)
 648                         goto err;
 649
 650                 ret = -EINVAL;
 651                 if (port > MUSB_DMA_NUM_CHANNELS || !port)
 652                         goto err;
 653                 if (is_tx)
 654                         cppi41_channel = &controller->tx_channel[port - 1];
 655                 else
 656                         cppi41_channel = &controller->rx_channel[port - 1];
 657
 658                 cppi41_channel->controller = controller;
 659                 cppi41_channel->port_num = port;
 660                 cppi41_channel->is_tx = is_tx;
 661                 INIT_LIST_HEAD(&cppi41_channel->tx_check);
 662
 663                 musb_dma = &cppi41_channel->channel;
 664                 musb_dma->private_data = cppi41_channel;
 665                 musb_dma->status = MUSB_DMA_STATUS_FREE;
 666                 musb_dma->max_len = SZ_4M;
 667
 668                 dc = dma_request_slave_channel(dev->parent, str);
 669                 if (!dc) {
 670                         dev_err(dev, "Failed to request %s.\n", str);
 671                         ret = -EPROBE_DEFER;
 672                         goto err;
 673                 }
 674                 cppi41_channel->dc = dc;
 675         }
 676         return 0;
 677 err:
 678         cppi41_release_all_dma_chans(controller);
 679         return ret;
 680 }
 681
 682 void cppi41_dma_controller_destroy(struct dma_controller *c)
 683 {
 684         struct cppi41_dma_controller *controller = container_of(c,
 685                         struct cppi41_dma_controller, controller);
 686
 687         hrtimer_cancel(&controller->early_tx);
 688         cppi41_dma_controller_stop(controller);
 689         kfree(controller);
 690 }
 691 EXPORT_SYMBOL_GPL(cppi41_dma_controller_destroy);
 692
 693 struct dma_controller *
 694 cppi41_dma_controller_create(struct musb *musb, void __iomem *base)
 695 {
 696         struct cppi41_dma_controller *controller;
 697         int ret = 0;
 698
 699         if (!musb->controller->parent->of_node) {
 700                 dev_err(musb->controller, "Need DT for the DMA engine.\n");
 701                 return NULL;
 702         }
 703
 704         controller = kzalloc(sizeof(*controller), GFP_KERNEL);
 705         if (!controller)
 706                 goto kzalloc_fail;
 707
 708         hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 709         controller->early_tx.function = cppi41_recheck_tx_req;
 710         INIT_LIST_HEAD(&controller->early_tx_list);
 711         controller->musb = musb;
 712
 713         controller->controller.channel_alloc = cppi41_dma_channel_allocate;
 714         controller->controller.channel_release = cppi41_dma_channel_release;
 715         controller->controller.channel_program = cppi41_dma_channel_program;
 716         controller->controller.channel_abort = cppi41_dma_channel_abort;
 717         controller->controller.is_compatible = cppi41_is_compatible;
 718
 719         ret = cppi41_dma_controller_start(controller);
 720         if (ret)
 721                 goto plat_get_fail;
 722         return &controller->controller;
 723
 724 plat_get_fail:
 725         kfree(controller);
 726 kzalloc_fail:
 727         if (ret == -EPROBE_DEFER)
 728                 return ERR_PTR(ret);
 729         return NULL;
 730 }
 731 EXPORT_SYMBOL_GPL(cppi41_dma_controller_create);