drivers/dma/mv_xor.c

   1 /*
   2  * offload engine driver for the Marvell XOR engine
   3  * Copyright (C) 2007, 2008, Marvell International Ltd.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #include <linux/init.h>
  16 #include <linux/slab.h>
  17 #include <linux/delay.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/of_device.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/memory.h>
  24 #include <linux/clk.h>
  25 #include <linux/of.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/irqdomain.h>
  28 #include <linux/cpumask.h>
  29 #include <linux/platform_data/dma-mv_xor.h>
  30
  31 #include "dmaengine.h"
  32 #include "mv_xor.h"
  33
  34 enum mv_xor_type {
  35         XOR_ORION,
  36         XOR_ARMADA_38X,
  37         XOR_ARMADA_37XX,
  38 };
  39
  40 enum mv_xor_mode {
  41         XOR_MODE_IN_REG,
  42         XOR_MODE_IN_DESC,
  43 };
  44
  45 static void mv_xor_issue_pending(struct dma_chan *chan);
  46
  47 #define to_mv_xor_chan(chan)            \
  48         container_of(chan, struct mv_xor_chan, dmachan)
  49
  50 #define to_mv_xor_slot(tx)              \
  51         container_of(tx, struct mv_xor_desc_slot, async_tx)
  52
  53 #define mv_chan_to_devp(chan)           \
  54         ((chan)->dmadev.dev)
  55
  56 static void mv_desc_init(struct mv_xor_desc_slot *desc,
  57                          dma_addr_t addr, u32 byte_count,
  58                          enum dma_ctrl_flags flags)
  59 {
  60         struct mv_xor_desc *hw_desc = desc->hw_desc;
  61
  62         hw_desc->status = XOR_DESC_DMA_OWNED;
  63         hw_desc->phy_next_desc = 0;
  64         /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
  65         hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
  66                                 XOR_DESC_EOD_INT_EN : 0;
  67         hw_desc->phy_dest_addr = addr;
  68         hw_desc->byte_count = byte_count;
  69 }
  70
  71 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
  72 {
  73         struct mv_xor_desc *hw_desc = desc->hw_desc;
  74
  75         switch (desc->type) {
  76         case DMA_XOR:
  77         case DMA_INTERRUPT:
  78                 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
  79                 break;
  80         case DMA_MEMCPY:
  81                 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
  82                 break;
  83         default:
  84                 BUG();
  85                 return;
  86         }
  87 }
  88
  89 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
  90                                   u32 next_desc_addr)
  91 {
  92         struct mv_xor_desc *hw_desc = desc->hw_desc;
  93         BUG_ON(hw_desc->phy_next_desc);
  94         hw_desc->phy_next_desc = next_desc_addr;
  95 }
  96
  97 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
  98                                  int index, dma_addr_t addr)
  99 {
 100         struct mv_xor_desc *hw_desc = desc->hw_desc;
 101         hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
 102         if (desc->type == DMA_XOR)
 103                 hw_desc->desc_command |= (1 << index);
 104 }
 105
 106 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
 107 {
 108         return readl_relaxed(XOR_CURR_DESC(chan));
 109 }
 110
 111 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
 112                                         u32 next_desc_addr)
 113 {
 114         writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
 115 }
 116
 117 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
 118 {
 119         u32 val = readl_relaxed(XOR_INTR_MASK(chan));
 120         val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
 121         writel_relaxed(val, XOR_INTR_MASK(chan));
 122 }
 123
 124 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
 125 {
 126         u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
 127         intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
 128         return intr_cause;
 129 }
 130
 131 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
 132 {
 133         u32 val;
 134
 135         val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
 136         val = ~(val << (chan->idx * 16));
 137         dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
 138         writel_relaxed(val, XOR_INTR_CAUSE(chan));
 139 }
 140
 141 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
 142 {
 143         u32 val = 0xFFFF0000 >> (chan->idx * 16);
 144         writel_relaxed(val, XOR_INTR_CAUSE(chan));
 145 }
 146
 147 static void mv_chan_set_mode(struct mv_xor_chan *chan,
 148                              u32 op_mode)
 149 {
 150         u32 config = readl_relaxed(XOR_CONFIG(chan));
 151
 152         config &= ~0x7;
 153         config |= op_mode;
 154
 155 #if defined(__BIG_ENDIAN)
 156         config |= XOR_DESCRIPTOR_SWAP;
 157 #else
 158         config &= ~XOR_DESCRIPTOR_SWAP;
 159 #endif
 160
 161         writel_relaxed(config, XOR_CONFIG(chan));
 162 }
 163
 164 static void mv_chan_activate(struct mv_xor_chan *chan)
 165 {
 166         dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
 167
 168         /* writel ensures all descriptors are flushed before activation */
 169         writel(BIT(0), XOR_ACTIVATION(chan));
 170 }
 171
 172 static char mv_chan_is_busy(struct mv_xor_chan *chan)
 173 {
 174         u32 state = readl_relaxed(XOR_ACTIVATION(chan));
 175
 176         state = (state >> 4) & 0x3;
 177
 178         return (state == 1) ? 1 : 0;
 179 }
 180
 181 /*
 182  * mv_chan_start_new_chain - program the engine to operate on new
 183  * chain headed by sw_desc
 184  * Caller must hold &mv_chan->lock while calling this function
 185  */
 186 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
 187                                     struct mv_xor_desc_slot *sw_desc)
 188 {
 189         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
 190                 __func__, __LINE__, sw_desc);
 191
 192         /* set the hardware chain */
 193         mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
 194
 195         mv_chan->pending++;
 196         mv_xor_issue_pending(&mv_chan->dmachan);
 197 }
 198
 199 static dma_cookie_t
 200 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
 201                                 struct mv_xor_chan *mv_chan,
 202                                 dma_cookie_t cookie)
 203 {
 204         BUG_ON(desc->async_tx.cookie < 0);
 205
 206         if (desc->async_tx.cookie > 0) {
 207                 cookie = desc->async_tx.cookie;
 208
 209                 dma_descriptor_unmap(&desc->async_tx);
 210                 /* call the callback (must not sleep or submit new
 211                  * operations to this channel)
 212                  */
 213                 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
 214         }
 215
 216         /* run dependent operations */
 217         dma_run_dependencies(&desc->async_tx);
 218
 219         return cookie;
 220 }
 221
 222 static int
 223 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
 224 {
 225         struct mv_xor_desc_slot *iter, *_iter;
 226
 227         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
 228         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
 229                                  node) {
 230
 231                 if (async_tx_test_ack(&iter->async_tx))
 232                         list_move_tail(&iter->node, &mv_chan->free_slots);
 233         }
 234         return 0;
 235 }
 236
 237 static int
 238 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
 239                    struct mv_xor_chan *mv_chan)
 240 {
 241         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
 242                 __func__, __LINE__, desc, desc->async_tx.flags);
 243
 244         /* the client is allowed to attach dependent operations
 245          * until 'ack' is set
 246          */
 247         if (!async_tx_test_ack(&desc->async_tx))
 248                 /* move this slot to the completed_slots */
 249                 list_move_tail(&desc->node, &mv_chan->completed_slots);
 250         else
 251                 list_move_tail(&desc->node, &mv_chan->free_slots);
 252
 253         return 0;
 254 }
 255
 256 /* This function must be called with the mv_xor_chan spinlock held */
 257 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
 258 {
 259         struct mv_xor_desc_slot *iter, *_iter;
 260         dma_cookie_t cookie = 0;
 261         int busy = mv_chan_is_busy(mv_chan);
 262         u32 current_desc = mv_chan_get_current_desc(mv_chan);
 263         int current_cleaned = 0;
 264         struct mv_xor_desc *hw_desc;
 265
 266         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
 267         dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
 268         mv_chan_clean_completed_slots(mv_chan);
 269
 270         /* free completed slots from the chain starting with
 271          * the oldest descriptor
 272          */
 273
 274         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
 275                                  node) {
 276
 277                 /* clean finished descriptors */
 278                 hw_desc = iter->hw_desc;
 279                 if (hw_desc->status & XOR_DESC_SUCCESS) {
 280                         cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
 281                                                                  cookie);
 282
 283                         /* done processing desc, clean slot */
 284                         mv_desc_clean_slot(iter, mv_chan);
 285
 286                         /* break if we did cleaned the current */
 287                         if (iter->async_tx.phys == current_desc) {
 288                                 current_cleaned = 1;
 289                                 break;
 290                         }
 291                 } else {
 292                         if (iter->async_tx.phys == current_desc) {
 293                                 current_cleaned = 0;
 294                                 break;
 295                         }
 296                 }
 297         }
 298
 299         if ((busy == 0) && !list_empty(&mv_chan->chain)) {
 300                 if (current_cleaned) {
 301                         /*
 302                          * current descriptor cleaned and removed, run
 303                          * from list head
 304                          */
 305                         iter = list_entry(mv_chan->chain.next,
 306                                           struct mv_xor_desc_slot,
 307                                           node);
 308                         mv_chan_start_new_chain(mv_chan, iter);
 309                 } else {
 310                         if (!list_is_last(&iter->node, &mv_chan->chain)) {
 311                                 /*
 312                                  * descriptors are still waiting after
 313                                  * current, trigger them
 314                                  */
 315                                 iter = list_entry(iter->node.next,
 316                                                   struct mv_xor_desc_slot,
 317                                                   node);
 318                                 mv_chan_start_new_chain(mv_chan, iter);
 319                         } else {
 320                                 /*
 321                                  * some descriptors are still waiting
 322                                  * to be cleaned
 323                                  */
 324                                 tasklet_schedule(&mv_chan->irq_tasklet);
 325                         }
 326                 }
 327         }
 328
 329         if (cookie > 0)
 330                 mv_chan->dmachan.completed_cookie = cookie;
 331 }
 332
 333 static void mv_xor_tasklet(unsigned long data)
 334 {
 335         struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
 336
 337         spin_lock_bh(&chan->lock);
 338         mv_chan_slot_cleanup(chan);
 339         spin_unlock_bh(&chan->lock);
 340 }
 341
 342 static struct mv_xor_desc_slot *
 343 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
 344 {
 345         struct mv_xor_desc_slot *iter;
 346
 347         spin_lock_bh(&mv_chan->lock);
 348
 349         if (!list_empty(&mv_chan->free_slots)) {
 350                 iter = list_first_entry(&mv_chan->free_slots,
 351                                         struct mv_xor_desc_slot,
 352                                         node);
 353
 354                 list_move_tail(&iter->node, &mv_chan->allocated_slots);
 355
 356                 spin_unlock_bh(&mv_chan->lock);
 357
 358                 /* pre-ack descriptor */
 359                 async_tx_ack(&iter->async_tx);
 360                 iter->async_tx.cookie = -EBUSY;
 361
 362                 return iter;
 363
 364         }
 365
 366         spin_unlock_bh(&mv_chan->lock);
 367
 368         /* try to free some slots if the allocation fails */
 369         tasklet_schedule(&mv_chan->irq_tasklet);
 370
 371         return NULL;
 372 }
 373
 374 /************************ DMA engine API functions ****************************/
 375 static dma_cookie_t
 376 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
 377 {
 378         struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
 379         struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
 380         struct mv_xor_desc_slot *old_chain_tail;
 381         dma_cookie_t cookie;
 382         int new_hw_chain = 1;
 383
 384         dev_dbg(mv_chan_to_devp(mv_chan),
 385                 "%s sw_desc %p: async_tx %p\n",
 386                 __func__, sw_desc, &sw_desc->async_tx);
 387
 388         spin_lock_bh(&mv_chan->lock);
 389         cookie = dma_cookie_assign(tx);
 390
 391         if (list_empty(&mv_chan->chain))
 392                 list_move_tail(&sw_desc->node, &mv_chan->chain);
 393         else {
 394                 new_hw_chain = 0;
 395
 396                 old_chain_tail = list_entry(mv_chan->chain.prev,
 397                                             struct mv_xor_desc_slot,
 398                                             node);
 399                 list_move_tail(&sw_desc->node, &mv_chan->chain);
 400
 401                 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
 402                         &old_chain_tail->async_tx.phys);
 403
 404                 /* fix up the hardware chain */
 405                 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
 406
 407                 /* if the channel is not busy */
 408                 if (!mv_chan_is_busy(mv_chan)) {
 409                         u32 current_desc = mv_chan_get_current_desc(mv_chan);
 410                         /*
 411                          * and the curren desc is the end of the chain before
 412                          * the append, then we need to start the channel
 413                          */
 414                         if (current_desc == old_chain_tail->async_tx.phys)
 415                                 new_hw_chain = 1;
 416                 }
 417         }
 418
 419         if (new_hw_chain)
 420                 mv_chan_start_new_chain(mv_chan, sw_desc);
 421
 422         spin_unlock_bh(&mv_chan->lock);
 423
 424         return cookie;
 425 }
 426
 427 /* returns the number of allocated descriptors */
 428 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
 429 {
 430         void *virt_desc;
 431         dma_addr_t dma_desc;
 432         int idx;
 433         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 434         struct mv_xor_desc_slot *slot = NULL;
 435         int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
 436
 437         /* Allocate descriptor slots */
 438         idx = mv_chan->slots_allocated;
 439         while (idx < num_descs_in_pool) {
 440                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 441                 if (!slot) {
 442                         dev_info(mv_chan_to_devp(mv_chan),
 443                                  "channel only initialized %d descriptor slots",
 444                                  idx);
 445                         break;
 446                 }
 447                 virt_desc = mv_chan->dma_desc_pool_virt;
 448                 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
 449
 450                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 451                 slot->async_tx.tx_submit = mv_xor_tx_submit;
 452                 INIT_LIST_HEAD(&slot->node);
 453                 dma_desc = mv_chan->dma_desc_pool;
 454                 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
 455                 slot->idx = idx++;
 456
 457                 spin_lock_bh(&mv_chan->lock);
 458                 mv_chan->slots_allocated = idx;
 459                 list_add_tail(&slot->node, &mv_chan->free_slots);
 460                 spin_unlock_bh(&mv_chan->lock);
 461         }
 462
 463         dev_dbg(mv_chan_to_devp(mv_chan),
 464                 "allocated %d descriptor slots\n",
 465                 mv_chan->slots_allocated);
 466
 467         return mv_chan->slots_allocated ? : -ENOMEM;
 468 }
 469
 470 /*
 471  * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
 472  * a new MBus window if necessary. Use a cache for these check so that
 473  * the MMIO mapped registers don't have to be accessed for this check
 474  * to speed up this process.
 475  */
 476 static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
 477 {
 478         struct mv_xor_device *xordev = mv_chan->xordev;
 479         void __iomem *base = mv_chan->mmr_high_base;
 480         u32 win_enable;
 481         u32 size;
 482         u8 target, attr;
 483         int ret;
 484         int i;
 485
 486         /* Nothing needs to get done for the Armada 3700 */
 487         if (xordev->xor_type == XOR_ARMADA_37XX)
 488                 return 0;
 489
 490         /*
 491          * Loop over the cached windows to check, if the requested area
 492          * is already mapped. If this the case, nothing needs to be done
 493          * and we can return.
 494          */
 495         for (i = 0; i < WINDOW_COUNT; i++) {
 496                 if (addr >= xordev->win_start[i] &&
 497                     addr <= xordev->win_end[i]) {
 498                         /* Window is already mapped */
 499                         return 0;
 500                 }
 501         }
 502
 503         /*
 504          * The window is not mapped, so we need to create the new mapping
 505          */
 506
 507         /* If no IO window is found that addr has to be located in SDRAM */
 508         ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
 509         if (ret < 0)
 510                 return 0;
 511
 512         /*
 513          * Mask the base addr 'addr' according to 'size' read back from the
 514          * MBus window. Otherwise we might end up with an address located
 515          * somewhere in the middle of this area here.
 516          */
 517         size -= 1;
 518         addr &= ~size;
 519
 520         /*
 521          * Reading one of both enabled register is enough, as they are always
 522          * programmed to the identical values
 523          */
 524         win_enable = readl(base + WINDOW_BAR_ENABLE(0));
 525
 526         /* Set 'i' to the first free window to write the new values to */
 527         i = ffs(~win_enable) - 1;
 528         if (i >= WINDOW_COUNT)
 529                 return -ENOMEM;
 530
 531         writel((addr & 0xffff0000) | (attr << 8) | target,
 532                base + WINDOW_BASE(i));
 533         writel(size & 0xffff0000, base + WINDOW_SIZE(i));
 534
 535         /* Fill the caching variables for later use */
 536         xordev->win_start[i] = addr;
 537         xordev->win_end[i] = addr + size;
 538
 539         win_enable |= (1 << i);
 540         win_enable |= 3 << (16 + (2 * i));
 541         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
 542         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
 543
 544         return 0;
 545 }
 546
 547 static struct dma_async_tx_descriptor *
 548 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 549                     unsigned int src_cnt, size_t len, unsigned long flags)
 550 {
 551         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 552         struct mv_xor_desc_slot *sw_desc;
 553         int ret;
 554
 555         if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
 556                 return NULL;
 557
 558         BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
 559
 560         dev_dbg(mv_chan_to_devp(mv_chan),
 561                 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
 562                 __func__, src_cnt, len, &dest, flags);
 563
 564         /* Check if a new window needs to get added for 'dest' */
 565         ret = mv_xor_add_io_win(mv_chan, dest);
 566         if (ret)
 567                 return NULL;
 568
 569         sw_desc = mv_chan_alloc_slot(mv_chan);
 570         if (sw_desc) {
 571                 sw_desc->type = DMA_XOR;
 572                 sw_desc->async_tx.flags = flags;
 573                 mv_desc_init(sw_desc, dest, len, flags);
 574                 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
 575                         mv_desc_set_mode(sw_desc);
 576                 while (src_cnt--) {
 577                         /* Check if a new window needs to get added for 'src' */
 578                         ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
 579                         if (ret)
 580                                 return NULL;
 581                         mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
 582                 }
 583         }
 584
 585         dev_dbg(mv_chan_to_devp(mv_chan),
 586                 "%s sw_desc %p async_tx %p \n",
 587                 __func__, sw_desc, &sw_desc->async_tx);
 588         return sw_desc ? &sw_desc->async_tx : NULL;
 589 }
 590
 591 static struct dma_async_tx_descriptor *
 592 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 593                 size_t len, unsigned long flags)
 594 {
 595         /*
 596          * A MEMCPY operation is identical to an XOR operation with only
 597          * a single source address.
 598          */
 599         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 600 }
 601
 602 static struct dma_async_tx_descriptor *
 603 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 604 {
 605         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 606         dma_addr_t src, dest;
 607         size_t len;
 608
 609         src = mv_chan->dummy_src_addr;
 610         dest = mv_chan->dummy_dst_addr;
 611         len = MV_XOR_MIN_BYTE_COUNT;
 612
 613         /*
 614          * We implement the DMA_INTERRUPT operation as a minimum sized
 615          * XOR operation with a single dummy source address.
 616          */
 617         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 618 }
 619
 620 static void mv_xor_free_chan_resources(struct dma_chan *chan)
 621 {
 622         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 623         struct mv_xor_desc_slot *iter, *_iter;
 624         int in_use_descs = 0;
 625
 626         spin_lock_bh(&mv_chan->lock);
 627
 628         mv_chan_slot_cleanup(mv_chan);
 629
 630         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
 631                                         node) {
 632                 in_use_descs++;
 633                 list_move_tail(&iter->node, &mv_chan->free_slots);
 634         }
 635         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
 636                                  node) {
 637                 in_use_descs++;
 638                 list_move_tail(&iter->node, &mv_chan->free_slots);
 639         }
 640         list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
 641                                  node) {
 642                 in_use_descs++;
 643                 list_move_tail(&iter->node, &mv_chan->free_slots);
 644         }
 645         list_for_each_entry_safe_reverse(
 646                 iter, _iter, &mv_chan->free_slots, node) {
 647                 list_del(&iter->node);
 648                 kfree(iter);
 649                 mv_chan->slots_allocated--;
 650         }
 651
 652         dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
 653                 __func__, mv_chan->slots_allocated);
 654         spin_unlock_bh(&mv_chan->lock);
 655
 656         if (in_use_descs)
 657                 dev_err(mv_chan_to_devp(mv_chan),
 658                         "freeing %d in use descriptors!\n", in_use_descs);
 659 }
 660
 661 /**
 662  * mv_xor_status - poll the status of an XOR transaction
 663  * @chan: XOR channel handle
 664  * @cookie: XOR transaction identifier
 665  * @txstate: XOR transactions state holder (or NULL)
 666  */
 667 static enum dma_status mv_xor_status(struct dma_chan *chan,
 668                                           dma_cookie_t cookie,
 669                                           struct dma_tx_state *txstate)
 670 {
 671         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 672         enum dma_status ret;
 673
 674         ret = dma_cookie_status(chan, cookie, txstate);
 675         if (ret == DMA_COMPLETE)
 676                 return ret;
 677
 678         spin_lock_bh(&mv_chan->lock);
 679         mv_chan_slot_cleanup(mv_chan);
 680         spin_unlock_bh(&mv_chan->lock);
 681
 682         return dma_cookie_status(chan, cookie, txstate);
 683 }
 684
 685 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
 686 {
 687         u32 val;
 688
 689         val = readl_relaxed(XOR_CONFIG(chan));
 690         dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
 691
 692         val = readl_relaxed(XOR_ACTIVATION(chan));
 693         dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
 694
 695         val = readl_relaxed(XOR_INTR_CAUSE(chan));
 696         dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
 697
 698         val = readl_relaxed(XOR_INTR_MASK(chan));
 699         dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
 700
 701         val = readl_relaxed(XOR_ERROR_CAUSE(chan));
 702         dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
 703
 704         val = readl_relaxed(XOR_ERROR_ADDR(chan));
 705         dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
 706 }
 707
 708 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
 709                                           u32 intr_cause)
 710 {
 711         if (intr_cause & XOR_INT_ERR_DECODE) {
 712                 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
 713                 return;
 714         }
 715
 716         dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
 717                 chan->idx, intr_cause);
 718
 719         mv_chan_dump_regs(chan);
 720         WARN_ON(1);
 721 }
 722
 723 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
 724 {
 725         struct mv_xor_chan *chan = data;
 726         u32 intr_cause = mv_chan_get_intr_cause(chan);
 727
 728         dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
 729
 730         if (intr_cause & XOR_INTR_ERRORS)
 731                 mv_chan_err_interrupt_handler(chan, intr_cause);
 732
 733         tasklet_schedule(&chan->irq_tasklet);
 734
 735         mv_chan_clear_eoc_cause(chan);
 736
 737         return IRQ_HANDLED;
 738 }
 739
 740 static void mv_xor_issue_pending(struct dma_chan *chan)
 741 {
 742         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 743
 744         if (mv_chan->pending >= MV_XOR_THRESHOLD) {
 745                 mv_chan->pending = 0;
 746                 mv_chan_activate(mv_chan);
 747         }
 748 }
 749
 750 /*
 751  * Perform a transaction to verify the HW works.
 752  */
 753
 754 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
 755 {
 756         int i, ret;
 757         void *src, *dest;
 758         dma_addr_t src_dma, dest_dma;
 759         struct dma_chan *dma_chan;
 760         dma_cookie_t cookie;
 761         struct dma_async_tx_descriptor *tx;
 762         struct dmaengine_unmap_data *unmap;
 763         int err = 0;
 764
 765         src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
 766         if (!src)
 767                 return -ENOMEM;
 768
 769         dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
 770         if (!dest) {
 771                 kfree(src);
 772                 return -ENOMEM;
 773         }
 774
 775         /* Fill in src buffer */
 776         for (i = 0; i < PAGE_SIZE; i++)
 777                 ((u8 *) src)[i] = (u8)i;
 778
 779         dma_chan = &mv_chan->dmachan;
 780         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
 781                 err = -ENODEV;
 782                 goto out;
 783         }
 784
 785         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
 786         if (!unmap) {
 787                 err = -ENOMEM;
 788                 goto free_resources;
 789         }
 790
 791         src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
 792                                (size_t)src & ~PAGE_MASK, PAGE_SIZE,
 793                                DMA_TO_DEVICE);
 794         unmap->addr[0] = src_dma;
 795
 796         ret = dma_mapping_error(dma_chan->device->dev, src_dma);
 797         if (ret) {
 798                 err = -ENOMEM;
 799                 goto free_resources;
 800         }
 801         unmap->to_cnt = 1;
 802
 803         dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
 804                                 (size_t)dest & ~PAGE_MASK, PAGE_SIZE,
 805                                 DMA_FROM_DEVICE);
 806         unmap->addr[1] = dest_dma;
 807
 808         ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
 809         if (ret) {
 810                 err = -ENOMEM;
 811                 goto free_resources;
 812         }
 813         unmap->from_cnt = 1;
 814         unmap->len = PAGE_SIZE;
 815
 816         tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
 817                                     PAGE_SIZE, 0);
 818         if (!tx) {
 819                 dev_err(dma_chan->device->dev,
 820                         "Self-test cannot prepare operation, disabling\n");
 821                 err = -ENODEV;
 822                 goto free_resources;
 823         }
 824
 825         cookie = mv_xor_tx_submit(tx);
 826         if (dma_submit_error(cookie)) {
 827                 dev_err(dma_chan->device->dev,
 828                         "Self-test submit error, disabling\n");
 829                 err = -ENODEV;
 830                 goto free_resources;
 831         }
 832
 833         mv_xor_issue_pending(dma_chan);
 834         async_tx_ack(tx);
 835         msleep(1);
 836
 837         if (mv_xor_status(dma_chan, cookie, NULL) !=
 838             DMA_COMPLETE) {
 839                 dev_err(dma_chan->device->dev,
 840                         "Self-test copy timed out, disabling\n");
 841                 err = -ENODEV;
 842                 goto free_resources;
 843         }
 844
 845         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
 846                                 PAGE_SIZE, DMA_FROM_DEVICE);
 847         if (memcmp(src, dest, PAGE_SIZE)) {
 848                 dev_err(dma_chan->device->dev,
 849                         "Self-test copy failed compare, disabling\n");
 850                 err = -ENODEV;
 851                 goto free_resources;
 852         }
 853
 854 free_resources:
 855         dmaengine_unmap_put(unmap);
 856         mv_xor_free_chan_resources(dma_chan);
 857 out:
 858         kfree(src);
 859         kfree(dest);
 860         return err;
 861 }
 862
 863 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
 864 static int
 865 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
 866 {
 867         int i, src_idx, ret;
 868         struct page *dest;
 869         struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
 870         dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
 871         dma_addr_t dest_dma;
 872         struct dma_async_tx_descriptor *tx;
 873         struct dmaengine_unmap_data *unmap;
 874         struct dma_chan *dma_chan;
 875         dma_cookie_t cookie;
 876         u8 cmp_byte = 0;
 877         u32 cmp_word;
 878         int err = 0;
 879         int src_count = MV_XOR_NUM_SRC_TEST;
 880
 881         for (src_idx = 0; src_idx < src_count; src_idx++) {
 882                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
 883                 if (!xor_srcs[src_idx]) {
 884                         while (src_idx--)
 885                                 __free_page(xor_srcs[src_idx]);
 886                         return -ENOMEM;
 887                 }
 888         }
 889
 890         dest = alloc_page(GFP_KERNEL);
 891         if (!dest) {
 892                 while (src_idx--)
 893                         __free_page(xor_srcs[src_idx]);
 894                 return -ENOMEM;
 895         }
 896
 897         /* Fill in src buffers */
 898         for (src_idx = 0; src_idx < src_count; src_idx++) {
 899                 u8 *ptr = page_address(xor_srcs[src_idx]);
 900                 for (i = 0; i < PAGE_SIZE; i++)
 901                         ptr[i] = (1 << src_idx);
 902         }
 903
 904         for (src_idx = 0; src_idx < src_count; src_idx++)
 905                 cmp_byte ^= (u8) (1 << src_idx);
 906
 907         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
 908                 (cmp_byte << 8) | cmp_byte;
 909
 910         memset(page_address(dest), 0, PAGE_SIZE);
 911
 912         dma_chan = &mv_chan->dmachan;
 913         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
 914                 err = -ENODEV;
 915                 goto out;
 916         }
 917
 918         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
 919                                          GFP_KERNEL);
 920         if (!unmap) {
 921                 err = -ENOMEM;
 922                 goto free_resources;
 923         }
 924
 925         /* test xor */
 926         for (i = 0; i < src_count; i++) {
 927                 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
 928                                               0, PAGE_SIZE, DMA_TO_DEVICE);
 929                 dma_srcs[i] = unmap->addr[i];
 930                 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
 931                 if (ret) {
 932                         err = -ENOMEM;
 933                         goto free_resources;
 934                 }
 935                 unmap->to_cnt++;
 936         }
 937
 938         unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
 939                                       DMA_FROM_DEVICE);
 940         dest_dma = unmap->addr[src_count];
 941         ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
 942         if (ret) {
 943                 err = -ENOMEM;
 944                 goto free_resources;
 945         }
 946         unmap->from_cnt = 1;
 947         unmap->len = PAGE_SIZE;
 948
 949         tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
 950                                  src_count, PAGE_SIZE, 0);
 951         if (!tx) {
 952                 dev_err(dma_chan->device->dev,
 953                         "Self-test cannot prepare operation, disabling\n");
 954                 err = -ENODEV;
 955                 goto free_resources;
 956         }
 957
 958         cookie = mv_xor_tx_submit(tx);
 959         if (dma_submit_error(cookie)) {
 960                 dev_err(dma_chan->device->dev,
 961                         "Self-test submit error, disabling\n");
 962                 err = -ENODEV;
 963                 goto free_resources;
 964         }
 965
 966         mv_xor_issue_pending(dma_chan);
 967         async_tx_ack(tx);
 968         msleep(8);
 969
 970         if (mv_xor_status(dma_chan, cookie, NULL) !=
 971             DMA_COMPLETE) {
 972                 dev_err(dma_chan->device->dev,
 973                         "Self-test xor timed out, disabling\n");
 974                 err = -ENODEV;
 975                 goto free_resources;
 976         }
 977
 978         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
 979                                 PAGE_SIZE, DMA_FROM_DEVICE);
 980         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
 981                 u32 *ptr = page_address(dest);
 982                 if (ptr[i] != cmp_word) {
 983                         dev_err(dma_chan->device->dev,
 984                                 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
 985                                 i, ptr[i], cmp_word);
 986                         err = -ENODEV;
 987                         goto free_resources;
 988                 }
 989         }
 990
 991 free_resources:
 992         dmaengine_unmap_put(unmap);
 993         mv_xor_free_chan_resources(dma_chan);
 994 out:
 995         src_idx = src_count;
 996         while (src_idx--)
 997                 __free_page(xor_srcs[src_idx]);
 998         __free_page(dest);
 999         return err;
1000 }
1001
1002 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
1003 {
1004         struct dma_chan *chan, *_chan;
1005         struct device *dev = mv_chan->dmadev.dev;
1006
1007         dma_async_device_unregister(&mv_chan->dmadev);
1008
1009         dma_free_coherent(dev, MV_XOR_POOL_SIZE,
1010                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1011         dma_unmap_single(dev, mv_chan->dummy_src_addr,
1012                          MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1013         dma_unmap_single(dev, mv_chan->dummy_dst_addr,
1014                          MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1015
1016         list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
1017                                  device_node) {
1018                 list_del(&chan->device_node);
1019         }
1020
1021         free_irq(mv_chan->irq, mv_chan);
1022
1023         return 0;
1024 }
1025
1026 static struct mv_xor_chan *
1027 mv_xor_channel_add(struct mv_xor_device *xordev,
1028                    struct platform_device *pdev,
1029                    int idx, dma_cap_mask_t cap_mask, int irq)
1030 {
1031         int ret = 0;
1032         struct mv_xor_chan *mv_chan;
1033         struct dma_device *dma_dev;
1034
1035         mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
1036         if (!mv_chan)
1037                 return ERR_PTR(-ENOMEM);
1038
1039         mv_chan->idx = idx;
1040         mv_chan->irq = irq;
1041         if (xordev->xor_type == XOR_ORION)
1042                 mv_chan->op_in_desc = XOR_MODE_IN_REG;
1043         else
1044                 mv_chan->op_in_desc = XOR_MODE_IN_DESC;
1045
1046         dma_dev = &mv_chan->dmadev;
1047         dma_dev->dev = &pdev->dev;
1048         mv_chan->xordev = xordev;
1049
1050         /*
1051          * These source and destination dummy buffers are used to implement
1052          * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
1053          * Hence, we only need to map the buffers at initialization-time.
1054          */
1055         mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
1056                 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1057         mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
1058                 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1059
1060         /* allocate coherent memory for hardware descriptors
1061          * note: writecombine gives slightly better performance, but
1062          * requires that we explicitly flush the writes
1063          */
1064         mv_chan->dma_desc_pool_virt =
1065           dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
1066                        GFP_KERNEL);
1067         if (!mv_chan->dma_desc_pool_virt)
1068                 return ERR_PTR(-ENOMEM);
1069
1070         /* discover transaction capabilites from the platform data */
1071         dma_dev->cap_mask = cap_mask;
1072
1073         INIT_LIST_HEAD(&dma_dev->channels);
1074
1075         /* set base routines */
1076         dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1077         dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1078         dma_dev->device_tx_status = mv_xor_status;
1079         dma_dev->device_issue_pending = mv_xor_issue_pending;
1080
1081         /* set prep routines based on capability */
1082         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1083                 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1084         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1085                 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1086         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1087                 dma_dev->max_xor = 8;
1088                 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1089         }
1090
1091         mv_chan->mmr_base = xordev->xor_base;
1092         mv_chan->mmr_high_base = xordev->xor_high_base;
1093         tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1094                      mv_chan);
1095
1096         /* clear errors before enabling interrupts */
1097         mv_chan_clear_err_status(mv_chan);
1098
1099         ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1100                           0, dev_name(&pdev->dev), mv_chan);
1101         if (ret)
1102                 goto err_free_dma;
1103
1104         mv_chan_unmask_interrupts(mv_chan);
1105
1106         if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1107                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1108         else
1109                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1110
1111         spin_lock_init(&mv_chan->lock);
1112         INIT_LIST_HEAD(&mv_chan->chain);
1113         INIT_LIST_HEAD(&mv_chan->completed_slots);
1114         INIT_LIST_HEAD(&mv_chan->free_slots);
1115         INIT_LIST_HEAD(&mv_chan->allocated_slots);
1116         mv_chan->dmachan.device = dma_dev;
1117         dma_cookie_init(&mv_chan->dmachan);
1118
1119         list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1120
1121         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1122                 ret = mv_chan_memcpy_self_test(mv_chan);
1123                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1124                 if (ret)
1125                         goto err_free_irq;
1126         }
1127
1128         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1129                 ret = mv_chan_xor_self_test(mv_chan);
1130                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1131                 if (ret)
1132                         goto err_free_irq;
1133         }
1134
1135         dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1136                  mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1137                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1138                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1139                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1140
1141         dma_async_device_register(dma_dev);
1142         return mv_chan;
1143
1144 err_free_irq:
1145         free_irq(mv_chan->irq, mv_chan);
1146 err_free_dma:
1147         dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1148                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1149         return ERR_PTR(ret);
1150 }
1151
1152 static void
1153 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1154                          const struct mbus_dram_target_info *dram)
1155 {
1156         void __iomem *base = xordev->xor_high_base;
1157         u32 win_enable = 0;
1158         int i;
1159
1160         for (i = 0; i < 8; i++) {
1161                 writel(0, base + WINDOW_BASE(i));
1162                 writel(0, base + WINDOW_SIZE(i));
1163                 if (i < 4)
1164                         writel(0, base + WINDOW_REMAP_HIGH(i));
1165         }
1166
1167         for (i = 0; i < dram->num_cs; i++) {
1168                 const struct mbus_dram_window *cs = dram->cs + i;
1169
1170                 writel((cs->base & 0xffff0000) |
1171                        (cs->mbus_attr << 8) |
1172                        dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1173                 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1174
1175                 /* Fill the caching variables for later use */
1176                 xordev->win_start[i] = cs->base;
1177                 xordev->win_end[i] = cs->base + cs->size - 1;
1178
1179                 win_enable |= (1 << i);
1180                 win_enable |= 3 << (16 + (2 * i));
1181         }
1182
1183         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1184         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1185         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1186         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1187 }
1188
1189 static void
1190 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1191 {
1192         void __iomem *base = xordev->xor_high_base;
1193         u32 win_enable = 0;
1194         int i;
1195
1196         for (i = 0; i < 8; i++) {
1197                 writel(0, base + WINDOW_BASE(i));
1198                 writel(0, base + WINDOW_SIZE(i));
1199                 if (i < 4)
1200                         writel(0, base + WINDOW_REMAP_HIGH(i));
1201         }
1202         /*
1203          * For Armada3700 open default 4GB Mbus window. The dram
1204          * related configuration are done at AXIS level.
1205          */
1206         writel(0xffff0000, base + WINDOW_SIZE(0));
1207         win_enable |= 1;
1208         win_enable |= 3 << 16;
1209
1210         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1211         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1212         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1213         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1214 }
1215
1216 /*
1217  * Since this XOR driver is basically used only for RAID5, we don't
1218  * need to care about synchronizing ->suspend with DMA activity,
1219  * because the DMA engine will naturally be quiet due to the block
1220  * devices being suspended.
1221  */
1222 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1223 {
1224         struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1225         int i;
1226
1227         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1228                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1229
1230                 if (!mv_chan)
1231                         continue;
1232
1233                 mv_chan->saved_config_reg =
1234                         readl_relaxed(XOR_CONFIG(mv_chan));
1235                 mv_chan->saved_int_mask_reg =
1236                         readl_relaxed(XOR_INTR_MASK(mv_chan));
1237         }
1238
1239         return 0;
1240 }
1241
1242 static int mv_xor_resume(struct platform_device *dev)
1243 {
1244         struct mv_xor_device *xordev = platform_get_drvdata(dev);
1245         const struct mbus_dram_target_info *dram;
1246         int i;
1247
1248         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1249                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1250
1251                 if (!mv_chan)
1252                         continue;
1253
1254                 writel_relaxed(mv_chan->saved_config_reg,
1255                                XOR_CONFIG(mv_chan));
1256                 writel_relaxed(mv_chan->saved_int_mask_reg,
1257                                XOR_INTR_MASK(mv_chan));
1258         }
1259
1260         if (xordev->xor_type == XOR_ARMADA_37XX) {
1261                 mv_xor_conf_mbus_windows_a3700(xordev);
1262                 return 0;
1263         }
1264
1265         dram = mv_mbus_dram_info();
1266         if (dram)
1267                 mv_xor_conf_mbus_windows(xordev, dram);
1268
1269         return 0;
1270 }
1271
1272 static const struct of_device_id mv_xor_dt_ids[] = {
1273         { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1274         { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1275         { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1276         {},
1277 };
1278
1279 static unsigned int mv_xor_engine_count;
1280
1281 static int mv_xor_probe(struct platform_device *pdev)
1282 {
1283         const struct mbus_dram_target_info *dram;
1284         struct mv_xor_device *xordev;
1285         struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1286         struct resource *res;
1287         unsigned int max_engines, max_channels;
1288         int i, ret;
1289
1290         dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1291
1292         xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1293         if (!xordev)
1294                 return -ENOMEM;
1295
1296         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1297         if (!res)
1298                 return -ENODEV;
1299
1300         xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1301                                         resource_size(res));
1302         if (!xordev->xor_base)
1303                 return -EBUSY;
1304
1305         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1306         if (!res)
1307                 return -ENODEV;
1308
1309         xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1310                                              resource_size(res));
1311         if (!xordev->xor_high_base)
1312                 return -EBUSY;
1313
1314         platform_set_drvdata(pdev, xordev);
1315
1316
1317         /*
1318          * We need to know which type of XOR device we use before
1319          * setting up. In non-dt case it can only be the legacy one.
1320          */
1321         xordev->xor_type = XOR_ORION;
1322         if (pdev->dev.of_node) {
1323                 const struct of_device_id *of_id =
1324                         of_match_device(mv_xor_dt_ids,
1325                                         &pdev->dev);
1326
1327                 xordev->xor_type = (uintptr_t)of_id->data;
1328         }
1329
1330         /*
1331          * (Re-)program MBUS remapping windows if we are asked to.
1332          */
1333         if (xordev->xor_type == XOR_ARMADA_37XX) {
1334                 mv_xor_conf_mbus_windows_a3700(xordev);
1335         } else {
1336                 dram = mv_mbus_dram_info();
1337                 if (dram)
1338                         mv_xor_conf_mbus_windows(xordev, dram);
1339         }
1340
1341         /* Not all platforms can gate the clock, so it is not
1342          * an error if the clock does not exists.
1343          */
1344         xordev->clk = clk_get(&pdev->dev, NULL);
1345         if (!IS_ERR(xordev->clk))
1346                 clk_prepare_enable(xordev->clk);
1347
1348         /*
1349          * We don't want to have more than one channel per CPU in
1350          * order for async_tx to perform well. So we limit the number
1351          * of engines and channels so that we take into account this
1352          * constraint. Note that we also want to use channels from
1353          * separate engines when possible.  For dual-CPU Armada 3700
1354          * SoC with single XOR engine allow using its both channels.
1355          */
1356         max_engines = num_present_cpus();
1357         if (xordev->xor_type == XOR_ARMADA_37XX)
1358                 max_channels =  num_present_cpus();
1359         else
1360                 max_channels = min_t(unsigned int,
1361                                      MV_XOR_MAX_CHANNELS,
1362                                      DIV_ROUND_UP(num_present_cpus(), 2));
1363
1364         if (mv_xor_engine_count >= max_engines)
1365                 return 0;
1366
1367         if (pdev->dev.of_node) {
1368                 struct device_node *np;
1369                 int i = 0;
1370
1371                 for_each_child_of_node(pdev->dev.of_node, np) {
1372                         struct mv_xor_chan *chan;
1373                         dma_cap_mask_t cap_mask;
1374                         int irq;
1375
1376                         if (i >= max_channels)
1377                                 continue;
1378
1379                         dma_cap_zero(cap_mask);
1380                         dma_cap_set(DMA_MEMCPY, cap_mask);
1381                         dma_cap_set(DMA_XOR, cap_mask);
1382                         dma_cap_set(DMA_INTERRUPT, cap_mask);
1383
1384                         irq = irq_of_parse_and_map(np, 0);
1385                         if (!irq) {
1386                                 ret = -ENODEV;
1387                                 goto err_channel_add;
1388                         }
1389
1390                         chan = mv_xor_channel_add(xordev, pdev, i,
1391                                                   cap_mask, irq);
1392                         if (IS_ERR(chan)) {
1393                                 ret = PTR_ERR(chan);
1394                                 irq_dispose_mapping(irq);
1395                                 goto err_channel_add;
1396                         }
1397
1398                         xordev->channels[i] = chan;
1399                         i++;
1400                 }
1401         } else if (pdata && pdata->channels) {
1402                 for (i = 0; i < max_channels; i++) {
1403                         struct mv_xor_channel_data *cd;
1404                         struct mv_xor_chan *chan;
1405                         int irq;
1406
1407                         cd = &pdata->channels[i];
1408                         if (!cd) {
1409                                 ret = -ENODEV;
1410                                 goto err_channel_add;
1411                         }
1412
1413                         irq = platform_get_irq(pdev, i);
1414                         if (irq < 0) {
1415                                 ret = irq;
1416                                 goto err_channel_add;
1417                         }
1418
1419                         chan = mv_xor_channel_add(xordev, pdev, i,
1420                                                   cd->cap_mask, irq);
1421                         if (IS_ERR(chan)) {
1422                                 ret = PTR_ERR(chan);
1423                                 goto err_channel_add;
1424                         }
1425
1426                         xordev->channels[i] = chan;
1427                 }
1428         }
1429
1430         return 0;
1431
1432 err_channel_add:
1433         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1434                 if (xordev->channels[i]) {
1435                         mv_xor_channel_remove(xordev->channels[i]);
1436                         if (pdev->dev.of_node)
1437                                 irq_dispose_mapping(xordev->channels[i]->irq);
1438                 }
1439
1440         if (!IS_ERR(xordev->clk)) {
1441                 clk_disable_unprepare(xordev->clk);
1442                 clk_put(xordev->clk);
1443         }
1444
1445         return ret;
1446 }
1447
1448 static struct platform_driver mv_xor_driver = {
1449         .probe          = mv_xor_probe,
1450         .suspend        = mv_xor_suspend,
1451         .resume         = mv_xor_resume,
1452         .driver         = {
1453                 .name           = MV_XOR_NAME,
1454                 .of_match_table = of_match_ptr(mv_xor_dt_ids),
1455         },
1456 };
1457
1458
1459 static int __init mv_xor_init(void)
1460 {
1461         return platform_driver_register(&mv_xor_driver);
1462 }
1463 device_initcall(mv_xor_init);
1464
1465 /*
1466 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1467 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1468 MODULE_LICENSE("GPL");
1469 */