GNU Linux-libre 4.4.289-gnu1
[releases.git] / drivers / dma / mv_xor.c
1 /*
2  * offload engine driver for the Marvell XOR engine
3  * Copyright (C) 2007, 2008, Marvell International Ltd.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  */
14
15 #include <linux/init.h>
16 #include <linux/slab.h>
17 #include <linux/delay.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/of_device.h>
22 #include <linux/platform_device.h>
23 #include <linux/memory.h>
24 #include <linux/clk.h>
25 #include <linux/of.h>
26 #include <linux/of_irq.h>
27 #include <linux/irqdomain.h>
28 #include <linux/cpumask.h>
29 #include <linux/platform_data/dma-mv_xor.h>
30
31 #include "dmaengine.h"
32 #include "mv_xor.h"
33
34 enum mv_xor_mode {
35         XOR_MODE_IN_REG,
36         XOR_MODE_IN_DESC,
37 };
38
39 static void mv_xor_issue_pending(struct dma_chan *chan);
40
41 #define to_mv_xor_chan(chan)            \
42         container_of(chan, struct mv_xor_chan, dmachan)
43
44 #define to_mv_xor_slot(tx)              \
45         container_of(tx, struct mv_xor_desc_slot, async_tx)
46
47 #define mv_chan_to_devp(chan)           \
48         ((chan)->dmadev.dev)
49
50 static void mv_desc_init(struct mv_xor_desc_slot *desc,
51                          dma_addr_t addr, u32 byte_count,
52                          enum dma_ctrl_flags flags)
53 {
54         struct mv_xor_desc *hw_desc = desc->hw_desc;
55
56         hw_desc->status = XOR_DESC_DMA_OWNED;
57         hw_desc->phy_next_desc = 0;
58         /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
59         hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
60                                 XOR_DESC_EOD_INT_EN : 0;
61         hw_desc->phy_dest_addr = addr;
62         hw_desc->byte_count = byte_count;
63 }
64
65 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
66 {
67         struct mv_xor_desc *hw_desc = desc->hw_desc;
68
69         switch (desc->type) {
70         case DMA_XOR:
71         case DMA_INTERRUPT:
72                 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
73                 break;
74         case DMA_MEMCPY:
75                 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
76                 break;
77         default:
78                 BUG();
79                 return;
80         }
81 }
82
83 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
84                                   u32 next_desc_addr)
85 {
86         struct mv_xor_desc *hw_desc = desc->hw_desc;
87         BUG_ON(hw_desc->phy_next_desc);
88         hw_desc->phy_next_desc = next_desc_addr;
89 }
90
91 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
92                                  int index, dma_addr_t addr)
93 {
94         struct mv_xor_desc *hw_desc = desc->hw_desc;
95         hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
96         if (desc->type == DMA_XOR)
97                 hw_desc->desc_command |= (1 << index);
98 }
99
100 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
101 {
102         return readl_relaxed(XOR_CURR_DESC(chan));
103 }
104
105 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
106                                         u32 next_desc_addr)
107 {
108         writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
109 }
110
111 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
112 {
113         u32 val = readl_relaxed(XOR_INTR_MASK(chan));
114         val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
115         writel_relaxed(val, XOR_INTR_MASK(chan));
116 }
117
118 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
119 {
120         u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
121         intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
122         return intr_cause;
123 }
124
125 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
126 {
127         u32 val;
128
129         val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
130         val = ~(val << (chan->idx * 16));
131         dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
132         writel_relaxed(val, XOR_INTR_CAUSE(chan));
133 }
134
135 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
136 {
137         u32 val = 0xFFFF0000 >> (chan->idx * 16);
138         writel_relaxed(val, XOR_INTR_CAUSE(chan));
139 }
140
141 static void mv_chan_set_mode(struct mv_xor_chan *chan,
142                              enum dma_transaction_type type)
143 {
144         u32 op_mode;
145         u32 config = readl_relaxed(XOR_CONFIG(chan));
146
147         switch (type) {
148         case DMA_XOR:
149                 op_mode = XOR_OPERATION_MODE_XOR;
150                 break;
151         case DMA_MEMCPY:
152                 op_mode = XOR_OPERATION_MODE_MEMCPY;
153                 break;
154         default:
155                 dev_err(mv_chan_to_devp(chan),
156                         "error: unsupported operation %d\n",
157                         type);
158                 BUG();
159                 return;
160         }
161
162         config &= ~0x7;
163         config |= op_mode;
164
165 #if defined(__BIG_ENDIAN)
166         config |= XOR_DESCRIPTOR_SWAP;
167 #else
168         config &= ~XOR_DESCRIPTOR_SWAP;
169 #endif
170
171         writel_relaxed(config, XOR_CONFIG(chan));
172         chan->current_type = type;
173 }
174
175 static void mv_chan_set_mode_to_desc(struct mv_xor_chan *chan)
176 {
177         u32 op_mode;
178         u32 config = readl_relaxed(XOR_CONFIG(chan));
179
180         op_mode = XOR_OPERATION_MODE_IN_DESC;
181
182         config &= ~0x7;
183         config |= op_mode;
184
185 #if defined(__BIG_ENDIAN)
186         config |= XOR_DESCRIPTOR_SWAP;
187 #else
188         config &= ~XOR_DESCRIPTOR_SWAP;
189 #endif
190
191         writel_relaxed(config, XOR_CONFIG(chan));
192 }
193
194 static void mv_chan_activate(struct mv_xor_chan *chan)
195 {
196         dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
197
198         /* writel ensures all descriptors are flushed before activation */
199         writel(BIT(0), XOR_ACTIVATION(chan));
200 }
201
202 static char mv_chan_is_busy(struct mv_xor_chan *chan)
203 {
204         u32 state = readl_relaxed(XOR_ACTIVATION(chan));
205
206         state = (state >> 4) & 0x3;
207
208         return (state == 1) ? 1 : 0;
209 }
210
211 /*
212  * mv_chan_start_new_chain - program the engine to operate on new
213  * chain headed by sw_desc
214  * Caller must hold &mv_chan->lock while calling this function
215  */
216 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
217                                     struct mv_xor_desc_slot *sw_desc)
218 {
219         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
220                 __func__, __LINE__, sw_desc);
221
222         /* set the hardware chain */
223         mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
224
225         mv_chan->pending++;
226         mv_xor_issue_pending(&mv_chan->dmachan);
227 }
228
229 static dma_cookie_t
230 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
231                                 struct mv_xor_chan *mv_chan,
232                                 dma_cookie_t cookie)
233 {
234         BUG_ON(desc->async_tx.cookie < 0);
235
236         if (desc->async_tx.cookie > 0) {
237                 cookie = desc->async_tx.cookie;
238
239                 /* call the callback (must not sleep or submit new
240                  * operations to this channel)
241                  */
242                 if (desc->async_tx.callback)
243                         desc->async_tx.callback(
244                                 desc->async_tx.callback_param);
245
246                 dma_descriptor_unmap(&desc->async_tx);
247         }
248
249         /* run dependent operations */
250         dma_run_dependencies(&desc->async_tx);
251
252         return cookie;
253 }
254
255 static int
256 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
257 {
258         struct mv_xor_desc_slot *iter, *_iter;
259
260         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
261         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
262                                  node) {
263
264                 if (async_tx_test_ack(&iter->async_tx))
265                         list_move_tail(&iter->node, &mv_chan->free_slots);
266         }
267         return 0;
268 }
269
270 static int
271 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
272                    struct mv_xor_chan *mv_chan)
273 {
274         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
275                 __func__, __LINE__, desc, desc->async_tx.flags);
276
277         /* the client is allowed to attach dependent operations
278          * until 'ack' is set
279          */
280         if (!async_tx_test_ack(&desc->async_tx))
281                 /* move this slot to the completed_slots */
282                 list_move_tail(&desc->node, &mv_chan->completed_slots);
283         else
284                 list_move_tail(&desc->node, &mv_chan->free_slots);
285
286         return 0;
287 }
288
289 /* This function must be called with the mv_xor_chan spinlock held */
290 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
291 {
292         struct mv_xor_desc_slot *iter, *_iter;
293         dma_cookie_t cookie = 0;
294         int busy = mv_chan_is_busy(mv_chan);
295         u32 current_desc = mv_chan_get_current_desc(mv_chan);
296         int current_cleaned = 0;
297         struct mv_xor_desc *hw_desc;
298
299         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
300         dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
301         mv_chan_clean_completed_slots(mv_chan);
302
303         /* free completed slots from the chain starting with
304          * the oldest descriptor
305          */
306
307         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
308                                  node) {
309
310                 /* clean finished descriptors */
311                 hw_desc = iter->hw_desc;
312                 if (hw_desc->status & XOR_DESC_SUCCESS) {
313                         cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
314                                                                  cookie);
315
316                         /* done processing desc, clean slot */
317                         mv_desc_clean_slot(iter, mv_chan);
318
319                         /* break if we did cleaned the current */
320                         if (iter->async_tx.phys == current_desc) {
321                                 current_cleaned = 1;
322                                 break;
323                         }
324                 } else {
325                         if (iter->async_tx.phys == current_desc) {
326                                 current_cleaned = 0;
327                                 break;
328                         }
329                 }
330         }
331
332         if ((busy == 0) && !list_empty(&mv_chan->chain)) {
333                 if (current_cleaned) {
334                         /*
335                          * current descriptor cleaned and removed, run
336                          * from list head
337                          */
338                         iter = list_entry(mv_chan->chain.next,
339                                           struct mv_xor_desc_slot,
340                                           node);
341                         mv_chan_start_new_chain(mv_chan, iter);
342                 } else {
343                         if (!list_is_last(&iter->node, &mv_chan->chain)) {
344                                 /*
345                                  * descriptors are still waiting after
346                                  * current, trigger them
347                                  */
348                                 iter = list_entry(iter->node.next,
349                                                   struct mv_xor_desc_slot,
350                                                   node);
351                                 mv_chan_start_new_chain(mv_chan, iter);
352                         } else {
353                                 /*
354                                  * some descriptors are still waiting
355                                  * to be cleaned
356                                  */
357                                 tasklet_schedule(&mv_chan->irq_tasklet);
358                         }
359                 }
360         }
361
362         if (cookie > 0)
363                 mv_chan->dmachan.completed_cookie = cookie;
364 }
365
366 static void mv_xor_tasklet(unsigned long data)
367 {
368         struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
369
370         spin_lock_bh(&chan->lock);
371         mv_chan_slot_cleanup(chan);
372         spin_unlock_bh(&chan->lock);
373 }
374
375 static struct mv_xor_desc_slot *
376 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
377 {
378         struct mv_xor_desc_slot *iter;
379
380         spin_lock_bh(&mv_chan->lock);
381
382         if (!list_empty(&mv_chan->free_slots)) {
383                 iter = list_first_entry(&mv_chan->free_slots,
384                                         struct mv_xor_desc_slot,
385                                         node);
386
387                 list_move_tail(&iter->node, &mv_chan->allocated_slots);
388
389                 spin_unlock_bh(&mv_chan->lock);
390
391                 /* pre-ack descriptor */
392                 async_tx_ack(&iter->async_tx);
393                 iter->async_tx.cookie = -EBUSY;
394
395                 return iter;
396
397         }
398
399         spin_unlock_bh(&mv_chan->lock);
400
401         /* try to free some slots if the allocation fails */
402         tasklet_schedule(&mv_chan->irq_tasklet);
403
404         return NULL;
405 }
406
407 /************************ DMA engine API functions ****************************/
408 static dma_cookie_t
409 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
410 {
411         struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
412         struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
413         struct mv_xor_desc_slot *old_chain_tail;
414         dma_cookie_t cookie;
415         int new_hw_chain = 1;
416
417         dev_dbg(mv_chan_to_devp(mv_chan),
418                 "%s sw_desc %p: async_tx %p\n",
419                 __func__, sw_desc, &sw_desc->async_tx);
420
421         spin_lock_bh(&mv_chan->lock);
422         cookie = dma_cookie_assign(tx);
423
424         if (list_empty(&mv_chan->chain))
425                 list_move_tail(&sw_desc->node, &mv_chan->chain);
426         else {
427                 new_hw_chain = 0;
428
429                 old_chain_tail = list_entry(mv_chan->chain.prev,
430                                             struct mv_xor_desc_slot,
431                                             node);
432                 list_move_tail(&sw_desc->node, &mv_chan->chain);
433
434                 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
435                         &old_chain_tail->async_tx.phys);
436
437                 /* fix up the hardware chain */
438                 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
439
440                 /* if the channel is not busy */
441                 if (!mv_chan_is_busy(mv_chan)) {
442                         u32 current_desc = mv_chan_get_current_desc(mv_chan);
443                         /*
444                          * and the curren desc is the end of the chain before
445                          * the append, then we need to start the channel
446                          */
447                         if (current_desc == old_chain_tail->async_tx.phys)
448                                 new_hw_chain = 1;
449                 }
450         }
451
452         if (new_hw_chain)
453                 mv_chan_start_new_chain(mv_chan, sw_desc);
454
455         spin_unlock_bh(&mv_chan->lock);
456
457         return cookie;
458 }
459
460 /* returns the number of allocated descriptors */
461 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
462 {
463         void *virt_desc;
464         dma_addr_t dma_desc;
465         int idx;
466         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
467         struct mv_xor_desc_slot *slot = NULL;
468         int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
469
470         /* Allocate descriptor slots */
471         idx = mv_chan->slots_allocated;
472         while (idx < num_descs_in_pool) {
473                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
474                 if (!slot) {
475                         dev_info(mv_chan_to_devp(mv_chan),
476                                  "channel only initialized %d descriptor slots",
477                                  idx);
478                         break;
479                 }
480                 virt_desc = mv_chan->dma_desc_pool_virt;
481                 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
482
483                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
484                 slot->async_tx.tx_submit = mv_xor_tx_submit;
485                 INIT_LIST_HEAD(&slot->node);
486                 dma_desc = mv_chan->dma_desc_pool;
487                 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
488                 slot->idx = idx++;
489
490                 spin_lock_bh(&mv_chan->lock);
491                 mv_chan->slots_allocated = idx;
492                 list_add_tail(&slot->node, &mv_chan->free_slots);
493                 spin_unlock_bh(&mv_chan->lock);
494         }
495
496         dev_dbg(mv_chan_to_devp(mv_chan),
497                 "allocated %d descriptor slots\n",
498                 mv_chan->slots_allocated);
499
500         return mv_chan->slots_allocated ? : -ENOMEM;
501 }
502
503 static struct dma_async_tx_descriptor *
504 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
505                     unsigned int src_cnt, size_t len, unsigned long flags)
506 {
507         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
508         struct mv_xor_desc_slot *sw_desc;
509
510         if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
511                 return NULL;
512
513         BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
514
515         dev_dbg(mv_chan_to_devp(mv_chan),
516                 "%s src_cnt: %d len: %u dest %pad flags: %ld\n",
517                 __func__, src_cnt, len, &dest, flags);
518
519         sw_desc = mv_chan_alloc_slot(mv_chan);
520         if (sw_desc) {
521                 sw_desc->type = DMA_XOR;
522                 sw_desc->async_tx.flags = flags;
523                 mv_desc_init(sw_desc, dest, len, flags);
524                 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
525                         mv_desc_set_mode(sw_desc);
526                 while (src_cnt--)
527                         mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
528         }
529
530         dev_dbg(mv_chan_to_devp(mv_chan),
531                 "%s sw_desc %p async_tx %p \n",
532                 __func__, sw_desc, &sw_desc->async_tx);
533         return sw_desc ? &sw_desc->async_tx : NULL;
534 }
535
536 static struct dma_async_tx_descriptor *
537 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
538                 size_t len, unsigned long flags)
539 {
540         /*
541          * A MEMCPY operation is identical to an XOR operation with only
542          * a single source address.
543          */
544         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
545 }
546
547 static struct dma_async_tx_descriptor *
548 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
549 {
550         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
551         dma_addr_t src, dest;
552         size_t len;
553
554         src = mv_chan->dummy_src_addr;
555         dest = mv_chan->dummy_dst_addr;
556         len = MV_XOR_MIN_BYTE_COUNT;
557
558         /*
559          * We implement the DMA_INTERRUPT operation as a minimum sized
560          * XOR operation with a single dummy source address.
561          */
562         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
563 }
564
565 static void mv_xor_free_chan_resources(struct dma_chan *chan)
566 {
567         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
568         struct mv_xor_desc_slot *iter, *_iter;
569         int in_use_descs = 0;
570
571         spin_lock_bh(&mv_chan->lock);
572
573         mv_chan_slot_cleanup(mv_chan);
574
575         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
576                                         node) {
577                 in_use_descs++;
578                 list_move_tail(&iter->node, &mv_chan->free_slots);
579         }
580         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
581                                  node) {
582                 in_use_descs++;
583                 list_move_tail(&iter->node, &mv_chan->free_slots);
584         }
585         list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
586                                  node) {
587                 in_use_descs++;
588                 list_move_tail(&iter->node, &mv_chan->free_slots);
589         }
590         list_for_each_entry_safe_reverse(
591                 iter, _iter, &mv_chan->free_slots, node) {
592                 list_del(&iter->node);
593                 kfree(iter);
594                 mv_chan->slots_allocated--;
595         }
596
597         dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
598                 __func__, mv_chan->slots_allocated);
599         spin_unlock_bh(&mv_chan->lock);
600
601         if (in_use_descs)
602                 dev_err(mv_chan_to_devp(mv_chan),
603                         "freeing %d in use descriptors!\n", in_use_descs);
604 }
605
606 /**
607  * mv_xor_status - poll the status of an XOR transaction
608  * @chan: XOR channel handle
609  * @cookie: XOR transaction identifier
610  * @txstate: XOR transactions state holder (or NULL)
611  */
612 static enum dma_status mv_xor_status(struct dma_chan *chan,
613                                           dma_cookie_t cookie,
614                                           struct dma_tx_state *txstate)
615 {
616         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
617         enum dma_status ret;
618
619         ret = dma_cookie_status(chan, cookie, txstate);
620         if (ret == DMA_COMPLETE)
621                 return ret;
622
623         spin_lock_bh(&mv_chan->lock);
624         mv_chan_slot_cleanup(mv_chan);
625         spin_unlock_bh(&mv_chan->lock);
626
627         return dma_cookie_status(chan, cookie, txstate);
628 }
629
630 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
631 {
632         u32 val;
633
634         val = readl_relaxed(XOR_CONFIG(chan));
635         dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
636
637         val = readl_relaxed(XOR_ACTIVATION(chan));
638         dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
639
640         val = readl_relaxed(XOR_INTR_CAUSE(chan));
641         dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
642
643         val = readl_relaxed(XOR_INTR_MASK(chan));
644         dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
645
646         val = readl_relaxed(XOR_ERROR_CAUSE(chan));
647         dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
648
649         val = readl_relaxed(XOR_ERROR_ADDR(chan));
650         dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
651 }
652
653 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
654                                           u32 intr_cause)
655 {
656         if (intr_cause & XOR_INT_ERR_DECODE) {
657                 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
658                 return;
659         }
660
661         dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
662                 chan->idx, intr_cause);
663
664         mv_chan_dump_regs(chan);
665         WARN_ON(1);
666 }
667
668 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
669 {
670         struct mv_xor_chan *chan = data;
671         u32 intr_cause = mv_chan_get_intr_cause(chan);
672
673         dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
674
675         if (intr_cause & XOR_INTR_ERRORS)
676                 mv_chan_err_interrupt_handler(chan, intr_cause);
677
678         tasklet_schedule(&chan->irq_tasklet);
679
680         mv_chan_clear_eoc_cause(chan);
681
682         return IRQ_HANDLED;
683 }
684
685 static void mv_xor_issue_pending(struct dma_chan *chan)
686 {
687         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
688
689         if (mv_chan->pending >= MV_XOR_THRESHOLD) {
690                 mv_chan->pending = 0;
691                 mv_chan_activate(mv_chan);
692         }
693 }
694
695 /*
696  * Perform a transaction to verify the HW works.
697  */
698
699 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
700 {
701         int i, ret;
702         void *src, *dest;
703         dma_addr_t src_dma, dest_dma;
704         struct dma_chan *dma_chan;
705         dma_cookie_t cookie;
706         struct dma_async_tx_descriptor *tx;
707         struct dmaengine_unmap_data *unmap;
708         int err = 0;
709
710         src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
711         if (!src)
712                 return -ENOMEM;
713
714         dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
715         if (!dest) {
716                 kfree(src);
717                 return -ENOMEM;
718         }
719
720         /* Fill in src buffer */
721         for (i = 0; i < PAGE_SIZE; i++)
722                 ((u8 *) src)[i] = (u8)i;
723
724         dma_chan = &mv_chan->dmachan;
725         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
726                 err = -ENODEV;
727                 goto out;
728         }
729
730         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
731         if (!unmap) {
732                 err = -ENOMEM;
733                 goto free_resources;
734         }
735
736         src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 0,
737                                  PAGE_SIZE, DMA_TO_DEVICE);
738         unmap->addr[0] = src_dma;
739
740         ret = dma_mapping_error(dma_chan->device->dev, src_dma);
741         if (ret) {
742                 err = -ENOMEM;
743                 goto free_resources;
744         }
745         unmap->to_cnt = 1;
746
747         dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 0,
748                                   PAGE_SIZE, DMA_FROM_DEVICE);
749         unmap->addr[1] = dest_dma;
750
751         ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
752         if (ret) {
753                 err = -ENOMEM;
754                 goto free_resources;
755         }
756         unmap->from_cnt = 1;
757         unmap->len = PAGE_SIZE;
758
759         tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
760                                     PAGE_SIZE, 0);
761         if (!tx) {
762                 dev_err(dma_chan->device->dev,
763                         "Self-test cannot prepare operation, disabling\n");
764                 err = -ENODEV;
765                 goto free_resources;
766         }
767
768         cookie = mv_xor_tx_submit(tx);
769         if (dma_submit_error(cookie)) {
770                 dev_err(dma_chan->device->dev,
771                         "Self-test submit error, disabling\n");
772                 err = -ENODEV;
773                 goto free_resources;
774         }
775
776         mv_xor_issue_pending(dma_chan);
777         async_tx_ack(tx);
778         msleep(1);
779
780         if (mv_xor_status(dma_chan, cookie, NULL) !=
781             DMA_COMPLETE) {
782                 dev_err(dma_chan->device->dev,
783                         "Self-test copy timed out, disabling\n");
784                 err = -ENODEV;
785                 goto free_resources;
786         }
787
788         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
789                                 PAGE_SIZE, DMA_FROM_DEVICE);
790         if (memcmp(src, dest, PAGE_SIZE)) {
791                 dev_err(dma_chan->device->dev,
792                         "Self-test copy failed compare, disabling\n");
793                 err = -ENODEV;
794                 goto free_resources;
795         }
796
797 free_resources:
798         dmaengine_unmap_put(unmap);
799         mv_xor_free_chan_resources(dma_chan);
800 out:
801         kfree(src);
802         kfree(dest);
803         return err;
804 }
805
806 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
807 static int
808 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
809 {
810         int i, src_idx, ret;
811         struct page *dest;
812         struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
813         dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
814         dma_addr_t dest_dma;
815         struct dma_async_tx_descriptor *tx;
816         struct dmaengine_unmap_data *unmap;
817         struct dma_chan *dma_chan;
818         dma_cookie_t cookie;
819         u8 cmp_byte = 0;
820         u32 cmp_word;
821         int err = 0;
822         int src_count = MV_XOR_NUM_SRC_TEST;
823
824         for (src_idx = 0; src_idx < src_count; src_idx++) {
825                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
826                 if (!xor_srcs[src_idx]) {
827                         while (src_idx--)
828                                 __free_page(xor_srcs[src_idx]);
829                         return -ENOMEM;
830                 }
831         }
832
833         dest = alloc_page(GFP_KERNEL);
834         if (!dest) {
835                 while (src_idx--)
836                         __free_page(xor_srcs[src_idx]);
837                 return -ENOMEM;
838         }
839
840         /* Fill in src buffers */
841         for (src_idx = 0; src_idx < src_count; src_idx++) {
842                 u8 *ptr = page_address(xor_srcs[src_idx]);
843                 for (i = 0; i < PAGE_SIZE; i++)
844                         ptr[i] = (1 << src_idx);
845         }
846
847         for (src_idx = 0; src_idx < src_count; src_idx++)
848                 cmp_byte ^= (u8) (1 << src_idx);
849
850         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
851                 (cmp_byte << 8) | cmp_byte;
852
853         memset(page_address(dest), 0, PAGE_SIZE);
854
855         dma_chan = &mv_chan->dmachan;
856         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
857                 err = -ENODEV;
858                 goto out;
859         }
860
861         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
862                                          GFP_KERNEL);
863         if (!unmap) {
864                 err = -ENOMEM;
865                 goto free_resources;
866         }
867
868         /* test xor */
869         for (i = 0; i < src_count; i++) {
870                 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
871                                               0, PAGE_SIZE, DMA_TO_DEVICE);
872                 dma_srcs[i] = unmap->addr[i];
873                 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
874                 if (ret) {
875                         err = -ENOMEM;
876                         goto free_resources;
877                 }
878                 unmap->to_cnt++;
879         }
880
881         unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
882                                       DMA_FROM_DEVICE);
883         dest_dma = unmap->addr[src_count];
884         ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
885         if (ret) {
886                 err = -ENOMEM;
887                 goto free_resources;
888         }
889         unmap->from_cnt = 1;
890         unmap->len = PAGE_SIZE;
891
892         tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
893                                  src_count, PAGE_SIZE, 0);
894         if (!tx) {
895                 dev_err(dma_chan->device->dev,
896                         "Self-test cannot prepare operation, disabling\n");
897                 err = -ENODEV;
898                 goto free_resources;
899         }
900
901         cookie = mv_xor_tx_submit(tx);
902         if (dma_submit_error(cookie)) {
903                 dev_err(dma_chan->device->dev,
904                         "Self-test submit error, disabling\n");
905                 err = -ENODEV;
906                 goto free_resources;
907         }
908
909         mv_xor_issue_pending(dma_chan);
910         async_tx_ack(tx);
911         msleep(8);
912
913         if (mv_xor_status(dma_chan, cookie, NULL) !=
914             DMA_COMPLETE) {
915                 dev_err(dma_chan->device->dev,
916                         "Self-test xor timed out, disabling\n");
917                 err = -ENODEV;
918                 goto free_resources;
919         }
920
921         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
922                                 PAGE_SIZE, DMA_FROM_DEVICE);
923         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
924                 u32 *ptr = page_address(dest);
925                 if (ptr[i] != cmp_word) {
926                         dev_err(dma_chan->device->dev,
927                                 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
928                                 i, ptr[i], cmp_word);
929                         err = -ENODEV;
930                         goto free_resources;
931                 }
932         }
933
934 free_resources:
935         dmaengine_unmap_put(unmap);
936         mv_xor_free_chan_resources(dma_chan);
937 out:
938         src_idx = src_count;
939         while (src_idx--)
940                 __free_page(xor_srcs[src_idx]);
941         __free_page(dest);
942         return err;
943 }
944
945 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
946 {
947         struct dma_chan *chan, *_chan;
948         struct device *dev = mv_chan->dmadev.dev;
949
950         dma_async_device_unregister(&mv_chan->dmadev);
951
952         dma_free_coherent(dev, MV_XOR_POOL_SIZE,
953                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
954         dma_unmap_single(dev, mv_chan->dummy_src_addr,
955                          MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
956         dma_unmap_single(dev, mv_chan->dummy_dst_addr,
957                          MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
958
959         list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
960                                  device_node) {
961                 list_del(&chan->device_node);
962         }
963
964         free_irq(mv_chan->irq, mv_chan);
965
966         return 0;
967 }
968
969 static struct mv_xor_chan *
970 mv_xor_channel_add(struct mv_xor_device *xordev,
971                    struct platform_device *pdev,
972                    int idx, dma_cap_mask_t cap_mask, int irq, int op_in_desc)
973 {
974         int ret = 0;
975         struct mv_xor_chan *mv_chan;
976         struct dma_device *dma_dev;
977
978         mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
979         if (!mv_chan)
980                 return ERR_PTR(-ENOMEM);
981
982         mv_chan->idx = idx;
983         mv_chan->irq = irq;
984         mv_chan->op_in_desc = op_in_desc;
985
986         dma_dev = &mv_chan->dmadev;
987
988         /*
989          * These source and destination dummy buffers are used to implement
990          * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
991          * Hence, we only need to map the buffers at initialization-time.
992          */
993         mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
994                 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
995         mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
996                 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
997
998         /* allocate coherent memory for hardware descriptors
999          * note: writecombine gives slightly better performance, but
1000          * requires that we explicitly flush the writes
1001          */
1002         mv_chan->dma_desc_pool_virt =
1003           dma_alloc_writecombine(&pdev->dev, MV_XOR_POOL_SIZE,
1004                                  &mv_chan->dma_desc_pool, GFP_KERNEL);
1005         if (!mv_chan->dma_desc_pool_virt)
1006                 return ERR_PTR(-ENOMEM);
1007
1008         /* discover transaction capabilites from the platform data */
1009         dma_dev->cap_mask = cap_mask;
1010
1011         INIT_LIST_HEAD(&dma_dev->channels);
1012
1013         /* set base routines */
1014         dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1015         dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1016         dma_dev->device_tx_status = mv_xor_status;
1017         dma_dev->device_issue_pending = mv_xor_issue_pending;
1018         dma_dev->dev = &pdev->dev;
1019
1020         /* set prep routines based on capability */
1021         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1022                 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1023         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1024                 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1025         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1026                 dma_dev->max_xor = 8;
1027                 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1028         }
1029
1030         mv_chan->mmr_base = xordev->xor_base;
1031         mv_chan->mmr_high_base = xordev->xor_high_base;
1032         tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1033                      mv_chan);
1034
1035         /* clear errors before enabling interrupts */
1036         mv_chan_clear_err_status(mv_chan);
1037
1038         ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1039                           0, dev_name(&pdev->dev), mv_chan);
1040         if (ret)
1041                 goto err_free_dma;
1042
1043         mv_chan_unmask_interrupts(mv_chan);
1044
1045         if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1046                 mv_chan_set_mode_to_desc(mv_chan);
1047         else
1048                 mv_chan_set_mode(mv_chan, DMA_XOR);
1049
1050         spin_lock_init(&mv_chan->lock);
1051         INIT_LIST_HEAD(&mv_chan->chain);
1052         INIT_LIST_HEAD(&mv_chan->completed_slots);
1053         INIT_LIST_HEAD(&mv_chan->free_slots);
1054         INIT_LIST_HEAD(&mv_chan->allocated_slots);
1055         mv_chan->dmachan.device = dma_dev;
1056         dma_cookie_init(&mv_chan->dmachan);
1057
1058         list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1059
1060         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1061                 ret = mv_chan_memcpy_self_test(mv_chan);
1062                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1063                 if (ret)
1064                         goto err_free_irq;
1065         }
1066
1067         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1068                 ret = mv_chan_xor_self_test(mv_chan);
1069                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1070                 if (ret)
1071                         goto err_free_irq;
1072         }
1073
1074         dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1075                  mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1076                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1077                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1078                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1079
1080         dma_async_device_register(dma_dev);
1081         return mv_chan;
1082
1083 err_free_irq:
1084         free_irq(mv_chan->irq, mv_chan);
1085  err_free_dma:
1086         dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1087                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1088         return ERR_PTR(ret);
1089 }
1090
1091 static void
1092 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1093                          const struct mbus_dram_target_info *dram)
1094 {
1095         void __iomem *base = xordev->xor_high_base;
1096         u32 win_enable = 0;
1097         int i;
1098
1099         for (i = 0; i < 8; i++) {
1100                 writel(0, base + WINDOW_BASE(i));
1101                 writel(0, base + WINDOW_SIZE(i));
1102                 if (i < 4)
1103                         writel(0, base + WINDOW_REMAP_HIGH(i));
1104         }
1105
1106         for (i = 0; i < dram->num_cs; i++) {
1107                 const struct mbus_dram_window *cs = dram->cs + i;
1108
1109                 writel((cs->base & 0xffff0000) |
1110                        (cs->mbus_attr << 8) |
1111                        dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1112                 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1113
1114                 win_enable |= (1 << i);
1115                 win_enable |= 3 << (16 + (2 * i));
1116         }
1117
1118         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1119         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1120         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1121         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1122 }
1123
1124 static const struct of_device_id mv_xor_dt_ids[] = {
1125         { .compatible = "marvell,orion-xor", .data = (void *)XOR_MODE_IN_REG },
1126         { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC },
1127         {},
1128 };
1129
1130 static unsigned int mv_xor_engine_count;
1131
1132 static int mv_xor_probe(struct platform_device *pdev)
1133 {
1134         const struct mbus_dram_target_info *dram;
1135         struct mv_xor_device *xordev;
1136         struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1137         struct resource *res;
1138         unsigned int max_engines, max_channels;
1139         int i, ret;
1140         int op_in_desc;
1141
1142         dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1143
1144         xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1145         if (!xordev)
1146                 return -ENOMEM;
1147
1148         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1149         if (!res)
1150                 return -ENODEV;
1151
1152         xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1153                                         resource_size(res));
1154         if (!xordev->xor_base)
1155                 return -EBUSY;
1156
1157         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1158         if (!res)
1159                 return -ENODEV;
1160
1161         xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1162                                              resource_size(res));
1163         if (!xordev->xor_high_base)
1164                 return -EBUSY;
1165
1166         platform_set_drvdata(pdev, xordev);
1167
1168         /*
1169          * (Re-)program MBUS remapping windows if we are asked to.
1170          */
1171         dram = mv_mbus_dram_info();
1172         if (dram)
1173                 mv_xor_conf_mbus_windows(xordev, dram);
1174
1175         /* Not all platforms can gate the clock, so it is not
1176          * an error if the clock does not exists.
1177          */
1178         xordev->clk = clk_get(&pdev->dev, NULL);
1179         if (!IS_ERR(xordev->clk))
1180                 clk_prepare_enable(xordev->clk);
1181
1182         /*
1183          * We don't want to have more than one channel per CPU in
1184          * order for async_tx to perform well. So we limit the number
1185          * of engines and channels so that we take into account this
1186          * constraint. Note that we also want to use channels from
1187          * separate engines when possible.
1188          */
1189         max_engines = num_present_cpus();
1190         max_channels = min_t(unsigned int,
1191                              MV_XOR_MAX_CHANNELS,
1192                              DIV_ROUND_UP(num_present_cpus(), 2));
1193
1194         if (mv_xor_engine_count >= max_engines)
1195                 return 0;
1196
1197         if (pdev->dev.of_node) {
1198                 struct device_node *np;
1199                 int i = 0;
1200                 const struct of_device_id *of_id =
1201                         of_match_device(mv_xor_dt_ids,
1202                                         &pdev->dev);
1203
1204                 for_each_child_of_node(pdev->dev.of_node, np) {
1205                         struct mv_xor_chan *chan;
1206                         dma_cap_mask_t cap_mask;
1207                         int irq;
1208                         op_in_desc = (int)of_id->data;
1209
1210                         if (i >= max_channels)
1211                                 continue;
1212
1213                         dma_cap_zero(cap_mask);
1214                         dma_cap_set(DMA_MEMCPY, cap_mask);
1215                         dma_cap_set(DMA_XOR, cap_mask);
1216                         dma_cap_set(DMA_INTERRUPT, cap_mask);
1217
1218                         irq = irq_of_parse_and_map(np, 0);
1219                         if (!irq) {
1220                                 ret = -ENODEV;
1221                                 goto err_channel_add;
1222                         }
1223
1224                         chan = mv_xor_channel_add(xordev, pdev, i,
1225                                                   cap_mask, irq, op_in_desc);
1226                         if (IS_ERR(chan)) {
1227                                 ret = PTR_ERR(chan);
1228                                 irq_dispose_mapping(irq);
1229                                 goto err_channel_add;
1230                         }
1231
1232                         xordev->channels[i] = chan;
1233                         i++;
1234                 }
1235         } else if (pdata && pdata->channels) {
1236                 for (i = 0; i < max_channels; i++) {
1237                         struct mv_xor_channel_data *cd;
1238                         struct mv_xor_chan *chan;
1239                         int irq;
1240
1241                         cd = &pdata->channels[i];
1242                         if (!cd) {
1243                                 ret = -ENODEV;
1244                                 goto err_channel_add;
1245                         }
1246
1247                         irq = platform_get_irq(pdev, i);
1248                         if (irq < 0) {
1249                                 ret = irq;
1250                                 goto err_channel_add;
1251                         }
1252
1253                         chan = mv_xor_channel_add(xordev, pdev, i,
1254                                                   cd->cap_mask, irq,
1255                                                   XOR_MODE_IN_REG);
1256                         if (IS_ERR(chan)) {
1257                                 ret = PTR_ERR(chan);
1258                                 goto err_channel_add;
1259                         }
1260
1261                         xordev->channels[i] = chan;
1262                 }
1263         }
1264
1265         return 0;
1266
1267 err_channel_add:
1268         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1269                 if (xordev->channels[i]) {
1270                         mv_xor_channel_remove(xordev->channels[i]);
1271                         if (pdev->dev.of_node)
1272                                 irq_dispose_mapping(xordev->channels[i]->irq);
1273                 }
1274
1275         if (!IS_ERR(xordev->clk)) {
1276                 clk_disable_unprepare(xordev->clk);
1277                 clk_put(xordev->clk);
1278         }
1279
1280         return ret;
1281 }
1282
1283 static struct platform_driver mv_xor_driver = {
1284         .probe          = mv_xor_probe,
1285         .driver         = {
1286                 .name           = MV_XOR_NAME,
1287                 .of_match_table = of_match_ptr(mv_xor_dt_ids),
1288         },
1289 };
1290
1291
1292 static int __init mv_xor_init(void)
1293 {
1294         return platform_driver_register(&mv_xor_driver);
1295 }
1296 device_initcall(mv_xor_init);
1297
1298 /*
1299 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1300 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1301 MODULE_LICENSE("GPL");
1302 */