GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / dma / mv_xor.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * offload engine driver for the Marvell XOR engine
4  * Copyright (C) 2007, 2008, Marvell International Ltd.
5  */
6
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/spinlock.h>
12 #include <linux/interrupt.h>
13 #include <linux/of_device.h>
14 #include <linux/platform_device.h>
15 #include <linux/memory.h>
16 #include <linux/clk.h>
17 #include <linux/of.h>
18 #include <linux/of_irq.h>
19 #include <linux/irqdomain.h>
20 #include <linux/cpumask.h>
21 #include <linux/platform_data/dma-mv_xor.h>
22
23 #include "dmaengine.h"
24 #include "mv_xor.h"
25
26 enum mv_xor_type {
27         XOR_ORION,
28         XOR_ARMADA_38X,
29         XOR_ARMADA_37XX,
30 };
31
32 enum mv_xor_mode {
33         XOR_MODE_IN_REG,
34         XOR_MODE_IN_DESC,
35 };
36
37 static void mv_xor_issue_pending(struct dma_chan *chan);
38
39 #define to_mv_xor_chan(chan)            \
40         container_of(chan, struct mv_xor_chan, dmachan)
41
42 #define to_mv_xor_slot(tx)              \
43         container_of(tx, struct mv_xor_desc_slot, async_tx)
44
45 #define mv_chan_to_devp(chan)           \
46         ((chan)->dmadev.dev)
47
48 static void mv_desc_init(struct mv_xor_desc_slot *desc,
49                          dma_addr_t addr, u32 byte_count,
50                          enum dma_ctrl_flags flags)
51 {
52         struct mv_xor_desc *hw_desc = desc->hw_desc;
53
54         hw_desc->status = XOR_DESC_DMA_OWNED;
55         hw_desc->phy_next_desc = 0;
56         /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
57         hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
58                                 XOR_DESC_EOD_INT_EN : 0;
59         hw_desc->phy_dest_addr = addr;
60         hw_desc->byte_count = byte_count;
61 }
62
63 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
64 {
65         struct mv_xor_desc *hw_desc = desc->hw_desc;
66
67         switch (desc->type) {
68         case DMA_XOR:
69         case DMA_INTERRUPT:
70                 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
71                 break;
72         case DMA_MEMCPY:
73                 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
74                 break;
75         default:
76                 BUG();
77                 return;
78         }
79 }
80
81 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
82                                   u32 next_desc_addr)
83 {
84         struct mv_xor_desc *hw_desc = desc->hw_desc;
85         BUG_ON(hw_desc->phy_next_desc);
86         hw_desc->phy_next_desc = next_desc_addr;
87 }
88
89 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
90                                  int index, dma_addr_t addr)
91 {
92         struct mv_xor_desc *hw_desc = desc->hw_desc;
93         hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
94         if (desc->type == DMA_XOR)
95                 hw_desc->desc_command |= (1 << index);
96 }
97
98 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
99 {
100         return readl_relaxed(XOR_CURR_DESC(chan));
101 }
102
103 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
104                                         u32 next_desc_addr)
105 {
106         writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
107 }
108
109 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
110 {
111         u32 val = readl_relaxed(XOR_INTR_MASK(chan));
112         val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
113         writel_relaxed(val, XOR_INTR_MASK(chan));
114 }
115
116 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
117 {
118         u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
119         intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
120         return intr_cause;
121 }
122
123 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
124 {
125         u32 val;
126
127         val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
128         val = ~(val << (chan->idx * 16));
129         dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
130         writel_relaxed(val, XOR_INTR_CAUSE(chan));
131 }
132
133 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
134 {
135         u32 val = 0xFFFF0000 >> (chan->idx * 16);
136         writel_relaxed(val, XOR_INTR_CAUSE(chan));
137 }
138
139 static void mv_chan_set_mode(struct mv_xor_chan *chan,
140                              u32 op_mode)
141 {
142         u32 config = readl_relaxed(XOR_CONFIG(chan));
143
144         config &= ~0x7;
145         config |= op_mode;
146
147 #if defined(__BIG_ENDIAN)
148         config |= XOR_DESCRIPTOR_SWAP;
149 #else
150         config &= ~XOR_DESCRIPTOR_SWAP;
151 #endif
152
153         writel_relaxed(config, XOR_CONFIG(chan));
154 }
155
156 static void mv_chan_activate(struct mv_xor_chan *chan)
157 {
158         dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
159
160         /* writel ensures all descriptors are flushed before activation */
161         writel(BIT(0), XOR_ACTIVATION(chan));
162 }
163
164 static char mv_chan_is_busy(struct mv_xor_chan *chan)
165 {
166         u32 state = readl_relaxed(XOR_ACTIVATION(chan));
167
168         state = (state >> 4) & 0x3;
169
170         return (state == 1) ? 1 : 0;
171 }
172
173 /*
174  * mv_chan_start_new_chain - program the engine to operate on new
175  * chain headed by sw_desc
176  * Caller must hold &mv_chan->lock while calling this function
177  */
178 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
179                                     struct mv_xor_desc_slot *sw_desc)
180 {
181         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
182                 __func__, __LINE__, sw_desc);
183
184         /* set the hardware chain */
185         mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
186
187         mv_chan->pending++;
188         mv_xor_issue_pending(&mv_chan->dmachan);
189 }
190
191 static dma_cookie_t
192 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
193                                 struct mv_xor_chan *mv_chan,
194                                 dma_cookie_t cookie)
195 {
196         BUG_ON(desc->async_tx.cookie < 0);
197
198         if (desc->async_tx.cookie > 0) {
199                 cookie = desc->async_tx.cookie;
200
201                 dma_descriptor_unmap(&desc->async_tx);
202                 /* call the callback (must not sleep or submit new
203                  * operations to this channel)
204                  */
205                 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
206         }
207
208         /* run dependent operations */
209         dma_run_dependencies(&desc->async_tx);
210
211         return cookie;
212 }
213
214 static int
215 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
216 {
217         struct mv_xor_desc_slot *iter, *_iter;
218
219         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
220         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
221                                  node) {
222
223                 if (async_tx_test_ack(&iter->async_tx)) {
224                         list_move_tail(&iter->node, &mv_chan->free_slots);
225                         if (!list_empty(&iter->sg_tx_list)) {
226                                 list_splice_tail_init(&iter->sg_tx_list,
227                                                         &mv_chan->free_slots);
228                         }
229                 }
230         }
231         return 0;
232 }
233
234 static int
235 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
236                    struct mv_xor_chan *mv_chan)
237 {
238         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
239                 __func__, __LINE__, desc, desc->async_tx.flags);
240
241         /* the client is allowed to attach dependent operations
242          * until 'ack' is set
243          */
244         if (!async_tx_test_ack(&desc->async_tx)) {
245                 /* move this slot to the completed_slots */
246                 list_move_tail(&desc->node, &mv_chan->completed_slots);
247                 if (!list_empty(&desc->sg_tx_list)) {
248                         list_splice_tail_init(&desc->sg_tx_list,
249                                               &mv_chan->completed_slots);
250                 }
251         } else {
252                 list_move_tail(&desc->node, &mv_chan->free_slots);
253                 if (!list_empty(&desc->sg_tx_list)) {
254                         list_splice_tail_init(&desc->sg_tx_list,
255                                               &mv_chan->free_slots);
256                 }
257         }
258
259         return 0;
260 }
261
262 /* This function must be called with the mv_xor_chan spinlock held */
263 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
264 {
265         struct mv_xor_desc_slot *iter, *_iter;
266         dma_cookie_t cookie = 0;
267         int busy = mv_chan_is_busy(mv_chan);
268         u32 current_desc = mv_chan_get_current_desc(mv_chan);
269         int current_cleaned = 0;
270         struct mv_xor_desc *hw_desc;
271
272         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
273         dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
274         mv_chan_clean_completed_slots(mv_chan);
275
276         /* free completed slots from the chain starting with
277          * the oldest descriptor
278          */
279
280         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
281                                  node) {
282
283                 /* clean finished descriptors */
284                 hw_desc = iter->hw_desc;
285                 if (hw_desc->status & XOR_DESC_SUCCESS) {
286                         cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
287                                                                  cookie);
288
289                         /* done processing desc, clean slot */
290                         mv_desc_clean_slot(iter, mv_chan);
291
292                         /* break if we did cleaned the current */
293                         if (iter->async_tx.phys == current_desc) {
294                                 current_cleaned = 1;
295                                 break;
296                         }
297                 } else {
298                         if (iter->async_tx.phys == current_desc) {
299                                 current_cleaned = 0;
300                                 break;
301                         }
302                 }
303         }
304
305         if ((busy == 0) && !list_empty(&mv_chan->chain)) {
306                 if (current_cleaned) {
307                         /*
308                          * current descriptor cleaned and removed, run
309                          * from list head
310                          */
311                         iter = list_entry(mv_chan->chain.next,
312                                           struct mv_xor_desc_slot,
313                                           node);
314                         mv_chan_start_new_chain(mv_chan, iter);
315                 } else {
316                         if (!list_is_last(&iter->node, &mv_chan->chain)) {
317                                 /*
318                                  * descriptors are still waiting after
319                                  * current, trigger them
320                                  */
321                                 iter = list_entry(iter->node.next,
322                                                   struct mv_xor_desc_slot,
323                                                   node);
324                                 mv_chan_start_new_chain(mv_chan, iter);
325                         } else {
326                                 /*
327                                  * some descriptors are still waiting
328                                  * to be cleaned
329                                  */
330                                 tasklet_schedule(&mv_chan->irq_tasklet);
331                         }
332                 }
333         }
334
335         if (cookie > 0)
336                 mv_chan->dmachan.completed_cookie = cookie;
337 }
338
339 static void mv_xor_tasklet(struct tasklet_struct *t)
340 {
341         struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet);
342
343         spin_lock(&chan->lock);
344         mv_chan_slot_cleanup(chan);
345         spin_unlock(&chan->lock);
346 }
347
348 static struct mv_xor_desc_slot *
349 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
350 {
351         struct mv_xor_desc_slot *iter;
352
353         spin_lock_bh(&mv_chan->lock);
354
355         if (!list_empty(&mv_chan->free_slots)) {
356                 iter = list_first_entry(&mv_chan->free_slots,
357                                         struct mv_xor_desc_slot,
358                                         node);
359
360                 list_move_tail(&iter->node, &mv_chan->allocated_slots);
361
362                 spin_unlock_bh(&mv_chan->lock);
363
364                 /* pre-ack descriptor */
365                 async_tx_ack(&iter->async_tx);
366                 iter->async_tx.cookie = -EBUSY;
367
368                 return iter;
369
370         }
371
372         spin_unlock_bh(&mv_chan->lock);
373
374         /* try to free some slots if the allocation fails */
375         tasklet_schedule(&mv_chan->irq_tasklet);
376
377         return NULL;
378 }
379
380 /************************ DMA engine API functions ****************************/
381 static dma_cookie_t
382 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
383 {
384         struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
385         struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
386         struct mv_xor_desc_slot *old_chain_tail;
387         dma_cookie_t cookie;
388         int new_hw_chain = 1;
389
390         dev_dbg(mv_chan_to_devp(mv_chan),
391                 "%s sw_desc %p: async_tx %p\n",
392                 __func__, sw_desc, &sw_desc->async_tx);
393
394         spin_lock_bh(&mv_chan->lock);
395         cookie = dma_cookie_assign(tx);
396
397         if (list_empty(&mv_chan->chain))
398                 list_move_tail(&sw_desc->node, &mv_chan->chain);
399         else {
400                 new_hw_chain = 0;
401
402                 old_chain_tail = list_entry(mv_chan->chain.prev,
403                                             struct mv_xor_desc_slot,
404                                             node);
405                 list_move_tail(&sw_desc->node, &mv_chan->chain);
406
407                 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
408                         &old_chain_tail->async_tx.phys);
409
410                 /* fix up the hardware chain */
411                 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
412
413                 /* if the channel is not busy */
414                 if (!mv_chan_is_busy(mv_chan)) {
415                         u32 current_desc = mv_chan_get_current_desc(mv_chan);
416                         /*
417                          * and the curren desc is the end of the chain before
418                          * the append, then we need to start the channel
419                          */
420                         if (current_desc == old_chain_tail->async_tx.phys)
421                                 new_hw_chain = 1;
422                 }
423         }
424
425         if (new_hw_chain)
426                 mv_chan_start_new_chain(mv_chan, sw_desc);
427
428         spin_unlock_bh(&mv_chan->lock);
429
430         return cookie;
431 }
432
433 /* returns the number of allocated descriptors */
434 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
435 {
436         void *virt_desc;
437         dma_addr_t dma_desc;
438         int idx;
439         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
440         struct mv_xor_desc_slot *slot = NULL;
441         int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
442
443         /* Allocate descriptor slots */
444         idx = mv_chan->slots_allocated;
445         while (idx < num_descs_in_pool) {
446                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
447                 if (!slot) {
448                         dev_info(mv_chan_to_devp(mv_chan),
449                                  "channel only initialized %d descriptor slots",
450                                  idx);
451                         break;
452                 }
453                 virt_desc = mv_chan->dma_desc_pool_virt;
454                 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
455
456                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
457                 slot->async_tx.tx_submit = mv_xor_tx_submit;
458                 INIT_LIST_HEAD(&slot->node);
459                 INIT_LIST_HEAD(&slot->sg_tx_list);
460                 dma_desc = mv_chan->dma_desc_pool;
461                 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
462                 slot->idx = idx++;
463
464                 spin_lock_bh(&mv_chan->lock);
465                 mv_chan->slots_allocated = idx;
466                 list_add_tail(&slot->node, &mv_chan->free_slots);
467                 spin_unlock_bh(&mv_chan->lock);
468         }
469
470         dev_dbg(mv_chan_to_devp(mv_chan),
471                 "allocated %d descriptor slots\n",
472                 mv_chan->slots_allocated);
473
474         return mv_chan->slots_allocated ? : -ENOMEM;
475 }
476
477 /*
478  * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
479  * a new MBus window if necessary. Use a cache for these check so that
480  * the MMIO mapped registers don't have to be accessed for this check
481  * to speed up this process.
482  */
483 static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
484 {
485         struct mv_xor_device *xordev = mv_chan->xordev;
486         void __iomem *base = mv_chan->mmr_high_base;
487         u32 win_enable;
488         u32 size;
489         u8 target, attr;
490         int ret;
491         int i;
492
493         /* Nothing needs to get done for the Armada 3700 */
494         if (xordev->xor_type == XOR_ARMADA_37XX)
495                 return 0;
496
497         /*
498          * Loop over the cached windows to check, if the requested area
499          * is already mapped. If this the case, nothing needs to be done
500          * and we can return.
501          */
502         for (i = 0; i < WINDOW_COUNT; i++) {
503                 if (addr >= xordev->win_start[i] &&
504                     addr <= xordev->win_end[i]) {
505                         /* Window is already mapped */
506                         return 0;
507                 }
508         }
509
510         /*
511          * The window is not mapped, so we need to create the new mapping
512          */
513
514         /* If no IO window is found that addr has to be located in SDRAM */
515         ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
516         if (ret < 0)
517                 return 0;
518
519         /*
520          * Mask the base addr 'addr' according to 'size' read back from the
521          * MBus window. Otherwise we might end up with an address located
522          * somewhere in the middle of this area here.
523          */
524         size -= 1;
525         addr &= ~size;
526
527         /*
528          * Reading one of both enabled register is enough, as they are always
529          * programmed to the identical values
530          */
531         win_enable = readl(base + WINDOW_BAR_ENABLE(0));
532
533         /* Set 'i' to the first free window to write the new values to */
534         i = ffs(~win_enable) - 1;
535         if (i >= WINDOW_COUNT)
536                 return -ENOMEM;
537
538         writel((addr & 0xffff0000) | (attr << 8) | target,
539                base + WINDOW_BASE(i));
540         writel(size & 0xffff0000, base + WINDOW_SIZE(i));
541
542         /* Fill the caching variables for later use */
543         xordev->win_start[i] = addr;
544         xordev->win_end[i] = addr + size;
545
546         win_enable |= (1 << i);
547         win_enable |= 3 << (16 + (2 * i));
548         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
549         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
550
551         return 0;
552 }
553
554 static struct dma_async_tx_descriptor *
555 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
556                     unsigned int src_cnt, size_t len, unsigned long flags)
557 {
558         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
559         struct mv_xor_desc_slot *sw_desc;
560         int ret;
561
562         if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
563                 return NULL;
564
565         BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
566
567         dev_dbg(mv_chan_to_devp(mv_chan),
568                 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
569                 __func__, src_cnt, len, &dest, flags);
570
571         /* Check if a new window needs to get added for 'dest' */
572         ret = mv_xor_add_io_win(mv_chan, dest);
573         if (ret)
574                 return NULL;
575
576         sw_desc = mv_chan_alloc_slot(mv_chan);
577         if (sw_desc) {
578                 sw_desc->type = DMA_XOR;
579                 sw_desc->async_tx.flags = flags;
580                 mv_desc_init(sw_desc, dest, len, flags);
581                 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
582                         mv_desc_set_mode(sw_desc);
583                 while (src_cnt--) {
584                         /* Check if a new window needs to get added for 'src' */
585                         ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
586                         if (ret)
587                                 return NULL;
588                         mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
589                 }
590         }
591
592         dev_dbg(mv_chan_to_devp(mv_chan),
593                 "%s sw_desc %p async_tx %p \n",
594                 __func__, sw_desc, &sw_desc->async_tx);
595         return sw_desc ? &sw_desc->async_tx : NULL;
596 }
597
598 static struct dma_async_tx_descriptor *
599 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
600                 size_t len, unsigned long flags)
601 {
602         /*
603          * A MEMCPY operation is identical to an XOR operation with only
604          * a single source address.
605          */
606         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
607 }
608
609 static struct dma_async_tx_descriptor *
610 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
611 {
612         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
613         dma_addr_t src, dest;
614         size_t len;
615
616         src = mv_chan->dummy_src_addr;
617         dest = mv_chan->dummy_dst_addr;
618         len = MV_XOR_MIN_BYTE_COUNT;
619
620         /*
621          * We implement the DMA_INTERRUPT operation as a minimum sized
622          * XOR operation with a single dummy source address.
623          */
624         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
625 }
626
627 static void mv_xor_free_chan_resources(struct dma_chan *chan)
628 {
629         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
630         struct mv_xor_desc_slot *iter, *_iter;
631         int in_use_descs = 0;
632
633         spin_lock_bh(&mv_chan->lock);
634
635         mv_chan_slot_cleanup(mv_chan);
636
637         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
638                                         node) {
639                 in_use_descs++;
640                 list_move_tail(&iter->node, &mv_chan->free_slots);
641         }
642         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
643                                  node) {
644                 in_use_descs++;
645                 list_move_tail(&iter->node, &mv_chan->free_slots);
646         }
647         list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
648                                  node) {
649                 in_use_descs++;
650                 list_move_tail(&iter->node, &mv_chan->free_slots);
651         }
652         list_for_each_entry_safe_reverse(
653                 iter, _iter, &mv_chan->free_slots, node) {
654                 list_del(&iter->node);
655                 kfree(iter);
656                 mv_chan->slots_allocated--;
657         }
658
659         dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
660                 __func__, mv_chan->slots_allocated);
661         spin_unlock_bh(&mv_chan->lock);
662
663         if (in_use_descs)
664                 dev_err(mv_chan_to_devp(mv_chan),
665                         "freeing %d in use descriptors!\n", in_use_descs);
666 }
667
668 /**
669  * mv_xor_status - poll the status of an XOR transaction
670  * @chan: XOR channel handle
671  * @cookie: XOR transaction identifier
672  * @txstate: XOR transactions state holder (or NULL)
673  */
674 static enum dma_status mv_xor_status(struct dma_chan *chan,
675                                           dma_cookie_t cookie,
676                                           struct dma_tx_state *txstate)
677 {
678         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
679         enum dma_status ret;
680
681         ret = dma_cookie_status(chan, cookie, txstate);
682         if (ret == DMA_COMPLETE)
683                 return ret;
684
685         spin_lock_bh(&mv_chan->lock);
686         mv_chan_slot_cleanup(mv_chan);
687         spin_unlock_bh(&mv_chan->lock);
688
689         return dma_cookie_status(chan, cookie, txstate);
690 }
691
692 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
693 {
694         u32 val;
695
696         val = readl_relaxed(XOR_CONFIG(chan));
697         dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
698
699         val = readl_relaxed(XOR_ACTIVATION(chan));
700         dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
701
702         val = readl_relaxed(XOR_INTR_CAUSE(chan));
703         dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
704
705         val = readl_relaxed(XOR_INTR_MASK(chan));
706         dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
707
708         val = readl_relaxed(XOR_ERROR_CAUSE(chan));
709         dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
710
711         val = readl_relaxed(XOR_ERROR_ADDR(chan));
712         dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
713 }
714
715 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
716                                           u32 intr_cause)
717 {
718         if (intr_cause & XOR_INT_ERR_DECODE) {
719                 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
720                 return;
721         }
722
723         dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
724                 chan->idx, intr_cause);
725
726         mv_chan_dump_regs(chan);
727         WARN_ON(1);
728 }
729
730 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
731 {
732         struct mv_xor_chan *chan = data;
733         u32 intr_cause = mv_chan_get_intr_cause(chan);
734
735         dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
736
737         if (intr_cause & XOR_INTR_ERRORS)
738                 mv_chan_err_interrupt_handler(chan, intr_cause);
739
740         tasklet_schedule(&chan->irq_tasklet);
741
742         mv_chan_clear_eoc_cause(chan);
743
744         return IRQ_HANDLED;
745 }
746
747 static void mv_xor_issue_pending(struct dma_chan *chan)
748 {
749         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
750
751         if (mv_chan->pending >= MV_XOR_THRESHOLD) {
752                 mv_chan->pending = 0;
753                 mv_chan_activate(mv_chan);
754         }
755 }
756
757 /*
758  * Perform a transaction to verify the HW works.
759  */
760
761 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
762 {
763         int i, ret;
764         void *src, *dest;
765         dma_addr_t src_dma, dest_dma;
766         struct dma_chan *dma_chan;
767         dma_cookie_t cookie;
768         struct dma_async_tx_descriptor *tx;
769         struct dmaengine_unmap_data *unmap;
770         int err = 0;
771
772         src = kmalloc(PAGE_SIZE, GFP_KERNEL);
773         if (!src)
774                 return -ENOMEM;
775
776         dest = kzalloc(PAGE_SIZE, GFP_KERNEL);
777         if (!dest) {
778                 kfree(src);
779                 return -ENOMEM;
780         }
781
782         /* Fill in src buffer */
783         for (i = 0; i < PAGE_SIZE; i++)
784                 ((u8 *) src)[i] = (u8)i;
785
786         dma_chan = &mv_chan->dmachan;
787         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
788                 err = -ENODEV;
789                 goto out;
790         }
791
792         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
793         if (!unmap) {
794                 err = -ENOMEM;
795                 goto free_resources;
796         }
797
798         src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
799                                offset_in_page(src), PAGE_SIZE,
800                                DMA_TO_DEVICE);
801         unmap->addr[0] = src_dma;
802
803         ret = dma_mapping_error(dma_chan->device->dev, src_dma);
804         if (ret) {
805                 err = -ENOMEM;
806                 goto free_resources;
807         }
808         unmap->to_cnt = 1;
809
810         dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
811                                 offset_in_page(dest), PAGE_SIZE,
812                                 DMA_FROM_DEVICE);
813         unmap->addr[1] = dest_dma;
814
815         ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
816         if (ret) {
817                 err = -ENOMEM;
818                 goto free_resources;
819         }
820         unmap->from_cnt = 1;
821         unmap->len = PAGE_SIZE;
822
823         tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
824                                     PAGE_SIZE, 0);
825         if (!tx) {
826                 dev_err(dma_chan->device->dev,
827                         "Self-test cannot prepare operation, disabling\n");
828                 err = -ENODEV;
829                 goto free_resources;
830         }
831
832         cookie = mv_xor_tx_submit(tx);
833         if (dma_submit_error(cookie)) {
834                 dev_err(dma_chan->device->dev,
835                         "Self-test submit error, disabling\n");
836                 err = -ENODEV;
837                 goto free_resources;
838         }
839
840         mv_xor_issue_pending(dma_chan);
841         async_tx_ack(tx);
842         msleep(1);
843
844         if (mv_xor_status(dma_chan, cookie, NULL) !=
845             DMA_COMPLETE) {
846                 dev_err(dma_chan->device->dev,
847                         "Self-test copy timed out, disabling\n");
848                 err = -ENODEV;
849                 goto free_resources;
850         }
851
852         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
853                                 PAGE_SIZE, DMA_FROM_DEVICE);
854         if (memcmp(src, dest, PAGE_SIZE)) {
855                 dev_err(dma_chan->device->dev,
856                         "Self-test copy failed compare, disabling\n");
857                 err = -ENODEV;
858                 goto free_resources;
859         }
860
861 free_resources:
862         dmaengine_unmap_put(unmap);
863         mv_xor_free_chan_resources(dma_chan);
864 out:
865         kfree(src);
866         kfree(dest);
867         return err;
868 }
869
870 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
871 static int
872 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
873 {
874         int i, src_idx, ret;
875         struct page *dest;
876         struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
877         dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
878         dma_addr_t dest_dma;
879         struct dma_async_tx_descriptor *tx;
880         struct dmaengine_unmap_data *unmap;
881         struct dma_chan *dma_chan;
882         dma_cookie_t cookie;
883         u8 cmp_byte = 0;
884         u32 cmp_word;
885         int err = 0;
886         int src_count = MV_XOR_NUM_SRC_TEST;
887
888         for (src_idx = 0; src_idx < src_count; src_idx++) {
889                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
890                 if (!xor_srcs[src_idx]) {
891                         while (src_idx--)
892                                 __free_page(xor_srcs[src_idx]);
893                         return -ENOMEM;
894                 }
895         }
896
897         dest = alloc_page(GFP_KERNEL);
898         if (!dest) {
899                 while (src_idx--)
900                         __free_page(xor_srcs[src_idx]);
901                 return -ENOMEM;
902         }
903
904         /* Fill in src buffers */
905         for (src_idx = 0; src_idx < src_count; src_idx++) {
906                 u8 *ptr = page_address(xor_srcs[src_idx]);
907                 for (i = 0; i < PAGE_SIZE; i++)
908                         ptr[i] = (1 << src_idx);
909         }
910
911         for (src_idx = 0; src_idx < src_count; src_idx++)
912                 cmp_byte ^= (u8) (1 << src_idx);
913
914         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
915                 (cmp_byte << 8) | cmp_byte;
916
917         memset(page_address(dest), 0, PAGE_SIZE);
918
919         dma_chan = &mv_chan->dmachan;
920         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
921                 err = -ENODEV;
922                 goto out;
923         }
924
925         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
926                                          GFP_KERNEL);
927         if (!unmap) {
928                 err = -ENOMEM;
929                 goto free_resources;
930         }
931
932         /* test xor */
933         for (i = 0; i < src_count; i++) {
934                 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
935                                               0, PAGE_SIZE, DMA_TO_DEVICE);
936                 dma_srcs[i] = unmap->addr[i];
937                 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
938                 if (ret) {
939                         err = -ENOMEM;
940                         goto free_resources;
941                 }
942                 unmap->to_cnt++;
943         }
944
945         unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
946                                       DMA_FROM_DEVICE);
947         dest_dma = unmap->addr[src_count];
948         ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
949         if (ret) {
950                 err = -ENOMEM;
951                 goto free_resources;
952         }
953         unmap->from_cnt = 1;
954         unmap->len = PAGE_SIZE;
955
956         tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
957                                  src_count, PAGE_SIZE, 0);
958         if (!tx) {
959                 dev_err(dma_chan->device->dev,
960                         "Self-test cannot prepare operation, disabling\n");
961                 err = -ENODEV;
962                 goto free_resources;
963         }
964
965         cookie = mv_xor_tx_submit(tx);
966         if (dma_submit_error(cookie)) {
967                 dev_err(dma_chan->device->dev,
968                         "Self-test submit error, disabling\n");
969                 err = -ENODEV;
970                 goto free_resources;
971         }
972
973         mv_xor_issue_pending(dma_chan);
974         async_tx_ack(tx);
975         msleep(8);
976
977         if (mv_xor_status(dma_chan, cookie, NULL) !=
978             DMA_COMPLETE) {
979                 dev_err(dma_chan->device->dev,
980                         "Self-test xor timed out, disabling\n");
981                 err = -ENODEV;
982                 goto free_resources;
983         }
984
985         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
986                                 PAGE_SIZE, DMA_FROM_DEVICE);
987         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
988                 u32 *ptr = page_address(dest);
989                 if (ptr[i] != cmp_word) {
990                         dev_err(dma_chan->device->dev,
991                                 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
992                                 i, ptr[i], cmp_word);
993                         err = -ENODEV;
994                         goto free_resources;
995                 }
996         }
997
998 free_resources:
999         dmaengine_unmap_put(unmap);
1000         mv_xor_free_chan_resources(dma_chan);
1001 out:
1002         src_idx = src_count;
1003         while (src_idx--)
1004                 __free_page(xor_srcs[src_idx]);
1005         __free_page(dest);
1006         return err;
1007 }
1008
1009 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
1010 {
1011         struct dma_chan *chan, *_chan;
1012         struct device *dev = mv_chan->dmadev.dev;
1013
1014         dma_async_device_unregister(&mv_chan->dmadev);
1015
1016         dma_free_coherent(dev, MV_XOR_POOL_SIZE,
1017                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1018         dma_unmap_single(dev, mv_chan->dummy_src_addr,
1019                          MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1020         dma_unmap_single(dev, mv_chan->dummy_dst_addr,
1021                          MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1022
1023         list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
1024                                  device_node) {
1025                 list_del(&chan->device_node);
1026         }
1027
1028         free_irq(mv_chan->irq, mv_chan);
1029
1030         return 0;
1031 }
1032
1033 static struct mv_xor_chan *
1034 mv_xor_channel_add(struct mv_xor_device *xordev,
1035                    struct platform_device *pdev,
1036                    int idx, dma_cap_mask_t cap_mask, int irq)
1037 {
1038         int ret = 0;
1039         struct mv_xor_chan *mv_chan;
1040         struct dma_device *dma_dev;
1041
1042         mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
1043         if (!mv_chan)
1044                 return ERR_PTR(-ENOMEM);
1045
1046         mv_chan->idx = idx;
1047         mv_chan->irq = irq;
1048         if (xordev->xor_type == XOR_ORION)
1049                 mv_chan->op_in_desc = XOR_MODE_IN_REG;
1050         else
1051                 mv_chan->op_in_desc = XOR_MODE_IN_DESC;
1052
1053         dma_dev = &mv_chan->dmadev;
1054         dma_dev->dev = &pdev->dev;
1055         mv_chan->xordev = xordev;
1056
1057         /*
1058          * These source and destination dummy buffers are used to implement
1059          * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
1060          * Hence, we only need to map the buffers at initialization-time.
1061          */
1062         mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
1063                 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1064         mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
1065                 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1066
1067         /* allocate coherent memory for hardware descriptors
1068          * note: writecombine gives slightly better performance, but
1069          * requires that we explicitly flush the writes
1070          */
1071         mv_chan->dma_desc_pool_virt =
1072           dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
1073                        GFP_KERNEL);
1074         if (!mv_chan->dma_desc_pool_virt)
1075                 return ERR_PTR(-ENOMEM);
1076
1077         /* discover transaction capabilites from the platform data */
1078         dma_dev->cap_mask = cap_mask;
1079
1080         INIT_LIST_HEAD(&dma_dev->channels);
1081
1082         /* set base routines */
1083         dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1084         dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1085         dma_dev->device_tx_status = mv_xor_status;
1086         dma_dev->device_issue_pending = mv_xor_issue_pending;
1087
1088         /* set prep routines based on capability */
1089         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1090                 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1091         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1092                 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1093         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1094                 dma_dev->max_xor = 8;
1095                 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1096         }
1097
1098         mv_chan->mmr_base = xordev->xor_base;
1099         mv_chan->mmr_high_base = xordev->xor_high_base;
1100         tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet);
1101
1102         /* clear errors before enabling interrupts */
1103         mv_chan_clear_err_status(mv_chan);
1104
1105         ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1106                           0, dev_name(&pdev->dev), mv_chan);
1107         if (ret)
1108                 goto err_free_dma;
1109
1110         mv_chan_unmask_interrupts(mv_chan);
1111
1112         if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1113                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1114         else
1115                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1116
1117         spin_lock_init(&mv_chan->lock);
1118         INIT_LIST_HEAD(&mv_chan->chain);
1119         INIT_LIST_HEAD(&mv_chan->completed_slots);
1120         INIT_LIST_HEAD(&mv_chan->free_slots);
1121         INIT_LIST_HEAD(&mv_chan->allocated_slots);
1122         mv_chan->dmachan.device = dma_dev;
1123         dma_cookie_init(&mv_chan->dmachan);
1124
1125         list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1126
1127         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1128                 ret = mv_chan_memcpy_self_test(mv_chan);
1129                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1130                 if (ret)
1131                         goto err_free_irq;
1132         }
1133
1134         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1135                 ret = mv_chan_xor_self_test(mv_chan);
1136                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1137                 if (ret)
1138                         goto err_free_irq;
1139         }
1140
1141         dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1142                  mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1143                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1144                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1145                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1146
1147         ret = dma_async_device_register(dma_dev);
1148         if (ret)
1149                 goto err_free_irq;
1150
1151         return mv_chan;
1152
1153 err_free_irq:
1154         free_irq(mv_chan->irq, mv_chan);
1155 err_free_dma:
1156         dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1157                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1158         return ERR_PTR(ret);
1159 }
1160
1161 static void
1162 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1163                          const struct mbus_dram_target_info *dram)
1164 {
1165         void __iomem *base = xordev->xor_high_base;
1166         u32 win_enable = 0;
1167         int i;
1168
1169         for (i = 0; i < 8; i++) {
1170                 writel(0, base + WINDOW_BASE(i));
1171                 writel(0, base + WINDOW_SIZE(i));
1172                 if (i < 4)
1173                         writel(0, base + WINDOW_REMAP_HIGH(i));
1174         }
1175
1176         for (i = 0; i < dram->num_cs; i++) {
1177                 const struct mbus_dram_window *cs = dram->cs + i;
1178
1179                 writel((cs->base & 0xffff0000) |
1180                        (cs->mbus_attr << 8) |
1181                        dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1182                 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1183
1184                 /* Fill the caching variables for later use */
1185                 xordev->win_start[i] = cs->base;
1186                 xordev->win_end[i] = cs->base + cs->size - 1;
1187
1188                 win_enable |= (1 << i);
1189                 win_enable |= 3 << (16 + (2 * i));
1190         }
1191
1192         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1193         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1194         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1195         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1196 }
1197
1198 static void
1199 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1200 {
1201         void __iomem *base = xordev->xor_high_base;
1202         u32 win_enable = 0;
1203         int i;
1204
1205         for (i = 0; i < 8; i++) {
1206                 writel(0, base + WINDOW_BASE(i));
1207                 writel(0, base + WINDOW_SIZE(i));
1208                 if (i < 4)
1209                         writel(0, base + WINDOW_REMAP_HIGH(i));
1210         }
1211         /*
1212          * For Armada3700 open default 4GB Mbus window. The dram
1213          * related configuration are done at AXIS level.
1214          */
1215         writel(0xffff0000, base + WINDOW_SIZE(0));
1216         win_enable |= 1;
1217         win_enable |= 3 << 16;
1218
1219         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1220         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1221         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1222         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1223 }
1224
1225 /*
1226  * Since this XOR driver is basically used only for RAID5, we don't
1227  * need to care about synchronizing ->suspend with DMA activity,
1228  * because the DMA engine will naturally be quiet due to the block
1229  * devices being suspended.
1230  */
1231 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1232 {
1233         struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1234         int i;
1235
1236         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1237                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1238
1239                 if (!mv_chan)
1240                         continue;
1241
1242                 mv_chan->saved_config_reg =
1243                         readl_relaxed(XOR_CONFIG(mv_chan));
1244                 mv_chan->saved_int_mask_reg =
1245                         readl_relaxed(XOR_INTR_MASK(mv_chan));
1246         }
1247
1248         return 0;
1249 }
1250
1251 static int mv_xor_resume(struct platform_device *dev)
1252 {
1253         struct mv_xor_device *xordev = platform_get_drvdata(dev);
1254         const struct mbus_dram_target_info *dram;
1255         int i;
1256
1257         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1258                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1259
1260                 if (!mv_chan)
1261                         continue;
1262
1263                 writel_relaxed(mv_chan->saved_config_reg,
1264                                XOR_CONFIG(mv_chan));
1265                 writel_relaxed(mv_chan->saved_int_mask_reg,
1266                                XOR_INTR_MASK(mv_chan));
1267         }
1268
1269         if (xordev->xor_type == XOR_ARMADA_37XX) {
1270                 mv_xor_conf_mbus_windows_a3700(xordev);
1271                 return 0;
1272         }
1273
1274         dram = mv_mbus_dram_info();
1275         if (dram)
1276                 mv_xor_conf_mbus_windows(xordev, dram);
1277
1278         return 0;
1279 }
1280
1281 static const struct of_device_id mv_xor_dt_ids[] = {
1282         { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1283         { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1284         { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1285         {},
1286 };
1287
1288 static unsigned int mv_xor_engine_count;
1289
1290 static int mv_xor_probe(struct platform_device *pdev)
1291 {
1292         const struct mbus_dram_target_info *dram;
1293         struct mv_xor_device *xordev;
1294         struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1295         struct resource *res;
1296         unsigned int max_engines, max_channels;
1297         int i, ret;
1298
1299         dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1300
1301         xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1302         if (!xordev)
1303                 return -ENOMEM;
1304
1305         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1306         if (!res)
1307                 return -ENODEV;
1308
1309         xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1310                                         resource_size(res));
1311         if (!xordev->xor_base)
1312                 return -EBUSY;
1313
1314         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1315         if (!res)
1316                 return -ENODEV;
1317
1318         xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1319                                              resource_size(res));
1320         if (!xordev->xor_high_base)
1321                 return -EBUSY;
1322
1323         platform_set_drvdata(pdev, xordev);
1324
1325
1326         /*
1327          * We need to know which type of XOR device we use before
1328          * setting up. In non-dt case it can only be the legacy one.
1329          */
1330         xordev->xor_type = XOR_ORION;
1331         if (pdev->dev.of_node) {
1332                 const struct of_device_id *of_id =
1333                         of_match_device(mv_xor_dt_ids,
1334                                         &pdev->dev);
1335
1336                 xordev->xor_type = (uintptr_t)of_id->data;
1337         }
1338
1339         /*
1340          * (Re-)program MBUS remapping windows if we are asked to.
1341          */
1342         if (xordev->xor_type == XOR_ARMADA_37XX) {
1343                 mv_xor_conf_mbus_windows_a3700(xordev);
1344         } else {
1345                 dram = mv_mbus_dram_info();
1346                 if (dram)
1347                         mv_xor_conf_mbus_windows(xordev, dram);
1348         }
1349
1350         /* Not all platforms can gate the clock, so it is not
1351          * an error if the clock does not exists.
1352          */
1353         xordev->clk = clk_get(&pdev->dev, NULL);
1354         if (!IS_ERR(xordev->clk))
1355                 clk_prepare_enable(xordev->clk);
1356
1357         /*
1358          * We don't want to have more than one channel per CPU in
1359          * order for async_tx to perform well. So we limit the number
1360          * of engines and channels so that we take into account this
1361          * constraint. Note that we also want to use channels from
1362          * separate engines when possible.  For dual-CPU Armada 3700
1363          * SoC with single XOR engine allow using its both channels.
1364          */
1365         max_engines = num_present_cpus();
1366         if (xordev->xor_type == XOR_ARMADA_37XX)
1367                 max_channels =  num_present_cpus();
1368         else
1369                 max_channels = min_t(unsigned int,
1370                                      MV_XOR_MAX_CHANNELS,
1371                                      DIV_ROUND_UP(num_present_cpus(), 2));
1372
1373         if (mv_xor_engine_count >= max_engines)
1374                 return 0;
1375
1376         if (pdev->dev.of_node) {
1377                 struct device_node *np;
1378                 int i = 0;
1379
1380                 for_each_child_of_node(pdev->dev.of_node, np) {
1381                         struct mv_xor_chan *chan;
1382                         dma_cap_mask_t cap_mask;
1383                         int irq;
1384
1385                         if (i >= max_channels)
1386                                 continue;
1387
1388                         dma_cap_zero(cap_mask);
1389                         dma_cap_set(DMA_MEMCPY, cap_mask);
1390                         dma_cap_set(DMA_XOR, cap_mask);
1391                         dma_cap_set(DMA_INTERRUPT, cap_mask);
1392
1393                         irq = irq_of_parse_and_map(np, 0);
1394                         if (!irq) {
1395                                 ret = -ENODEV;
1396                                 goto err_channel_add;
1397                         }
1398
1399                         chan = mv_xor_channel_add(xordev, pdev, i,
1400                                                   cap_mask, irq);
1401                         if (IS_ERR(chan)) {
1402                                 ret = PTR_ERR(chan);
1403                                 irq_dispose_mapping(irq);
1404                                 goto err_channel_add;
1405                         }
1406
1407                         xordev->channels[i] = chan;
1408                         i++;
1409                 }
1410         } else if (pdata && pdata->channels) {
1411                 for (i = 0; i < max_channels; i++) {
1412                         struct mv_xor_channel_data *cd;
1413                         struct mv_xor_chan *chan;
1414                         int irq;
1415
1416                         cd = &pdata->channels[i];
1417                         irq = platform_get_irq(pdev, i);
1418                         if (irq < 0) {
1419                                 ret = irq;
1420                                 goto err_channel_add;
1421                         }
1422
1423                         chan = mv_xor_channel_add(xordev, pdev, i,
1424                                                   cd->cap_mask, irq);
1425                         if (IS_ERR(chan)) {
1426                                 ret = PTR_ERR(chan);
1427                                 goto err_channel_add;
1428                         }
1429
1430                         xordev->channels[i] = chan;
1431                 }
1432         }
1433
1434         return 0;
1435
1436 err_channel_add:
1437         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1438                 if (xordev->channels[i]) {
1439                         mv_xor_channel_remove(xordev->channels[i]);
1440                         if (pdev->dev.of_node)
1441                                 irq_dispose_mapping(xordev->channels[i]->irq);
1442                 }
1443
1444         if (!IS_ERR(xordev->clk)) {
1445                 clk_disable_unprepare(xordev->clk);
1446                 clk_put(xordev->clk);
1447         }
1448
1449         return ret;
1450 }
1451
1452 static struct platform_driver mv_xor_driver = {
1453         .probe          = mv_xor_probe,
1454         .suspend        = mv_xor_suspend,
1455         .resume         = mv_xor_resume,
1456         .driver         = {
1457                 .name           = MV_XOR_NAME,
1458                 .of_match_table = mv_xor_dt_ids,
1459         },
1460 };
1461
1462 builtin_platform_driver(mv_xor_driver);
1463
1464 /*
1465 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1466 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1467 MODULE_LICENSE("GPL");
1468 */