1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) 2006-2009 DENX Software Engineering.
5 * Author: Yuri Tikhonov <yur@emcraft.com>
7 * Further porting to arch/powerpc by
8 * Anatolij Gustschin <agust@denx.de>
12 * This driver supports the asynchrounous DMA copy and RAID engines available
13 * on the AMCC PPC440SPe Processors.
14 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
15 * ADMA driver written by D.Williams.
18 #include <linux/init.h>
19 #include <linux/module.h>
20 #include <linux/async_tx.h>
21 #include <linux/delay.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/spinlock.h>
24 #include <linux/interrupt.h>
25 #include <linux/slab.h>
26 #include <linux/uaccess.h>
27 #include <linux/proc_fs.h>
29 #include <linux/of_address.h>
30 #include <linux/of_irq.h>
31 #include <linux/of_platform.h>
33 #include <asm/dcr-regs.h>
35 #include "../dmaengine.h"
37 enum ppc_adma_init_code {
42 PPC_ADMA_INIT_COHERENT,
43 PPC_ADMA_INIT_CHANNEL,
46 PPC_ADMA_INIT_REGISTER
49 static char *ppc_adma_errors[] = {
50 [PPC_ADMA_INIT_OK] = "ok",
51 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
52 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
53 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
55 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
56 "hardware descriptors",
57 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
58 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
59 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
60 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
63 static enum ppc_adma_init_code
64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
66 struct ppc_dma_chan_ref {
67 struct dma_chan *chan;
68 struct list_head node;
71 /* The list of channels exported by ppc440spe ADMA */
73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
75 /* This flag is set when want to refetch the xor chain in the interrupt
78 static u32 do_xor_refetch;
80 /* Pointer to DMA0, DMA1 CP/CS FIFO */
81 static void *ppc440spe_dma_fifo_buf;
83 /* Pointers to last submitted to DMA0, DMA1 CDBs */
84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
87 /* Pointer to last linked and submitted xor CB */
88 static struct ppc440spe_adma_desc_slot *xor_last_linked;
89 static struct ppc440spe_adma_desc_slot *xor_last_submit;
91 /* This array is used in data-check operations for storing a pattern */
92 static char ppc440spe_qword[16];
94 static atomic_t ppc440spe_adma_err_irq_ref;
95 static dcr_host_t ppc440spe_mq_dcr_host;
96 static unsigned int ppc440spe_mq_dcr_len;
98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
99 * the block size in transactions, then we do not allow to activate more than
100 * only one RXOR transactions simultaneously. So use this var to store
101 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
102 * set) or not (PPC440SPE_RXOR_RUN is clear).
104 static unsigned long ppc440spe_rxor_state;
106 /* These are used in enable & check routines
108 static u32 ppc440spe_r6_enabled;
109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
110 static struct completion ppc440spe_r6_test_comp;
112 static int ppc440spe_adma_dma2rxor_prep_src(
113 struct ppc440spe_adma_desc_slot *desc,
114 struct ppc440spe_rxor *cursor, int index,
115 int src_cnt, u32 addr);
116 static void ppc440spe_adma_dma2rxor_set_src(
117 struct ppc440spe_adma_desc_slot *desc,
118 int index, dma_addr_t addr);
119 static void ppc440spe_adma_dma2rxor_set_mult(
120 struct ppc440spe_adma_desc_slot *desc,
124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
135 switch (chan->device->id) {
140 pr_debug("CDB at %p [%d]:\n"
141 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
142 "\t sg1u 0x%08x sg1l 0x%08x\n"
143 "\t sg2u 0x%08x sg2l 0x%08x\n"
144 "\t sg3u 0x%08x sg3l 0x%08x\n",
145 cdb, chan->device->id,
146 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
147 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
148 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
149 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
155 pr_debug("CB at %p [%d]:\n"
156 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
157 "\t cbtah 0x%08x cbtal 0x%08x\n"
158 "\t cblah 0x%08x cblal 0x%08x\n",
159 cb, chan->device->id,
160 cb->cbc, cb->cbbc, cb->cbs,
161 cb->cbtah, cb->cbtal,
162 cb->cblah, cb->cblal);
163 for (i = 0; i < 16; i++) {
164 if (i && !cb->ops[i].h && !cb->ops[i].l)
166 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
167 i, cb->ops[i].h, cb->ops[i].l);
173 static void print_cb_list(struct ppc440spe_adma_chan *chan,
174 struct ppc440spe_adma_desc_slot *iter)
176 for (; iter; iter = iter->hw_next)
177 print_cb(chan, iter->hw_desc);
180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
181 unsigned int src_cnt)
185 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
186 for (i = 0; i < src_cnt; i++)
187 pr_debug("\t0x%016llx ", src[i]);
188 pr_debug("dst:\n\t0x%016llx\n", dst);
191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
192 unsigned int src_cnt)
196 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
197 for (i = 0; i < src_cnt; i++)
198 pr_debug("\t0x%016llx ", src[i]);
200 for (i = 0; i < 2; i++)
201 pr_debug("\t0x%016llx ", dst[i]);
204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
205 unsigned int src_cnt,
206 const unsigned char *scf)
210 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
212 for (i = 0; i < src_cnt; i++)
213 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
215 for (i = 0; i < src_cnt; i++)
216 pr_debug("\t0x%016llx(no) ", src[i]);
220 for (i = 0; i < 2; i++)
221 pr_debug("\t0x%016llx ", src[src_cnt + i]);
224 /******************************************************************************
225 * Command (Descriptor) Blocks low-level routines
226 ******************************************************************************/
228 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
232 struct ppc440spe_adma_chan *chan)
236 switch (chan->device->id) {
237 case PPC440SPE_XOR_ID:
239 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
240 /* NOP with Command Block Complete Enable */
241 p->cbc = XOR_CBCR_CBCE_BIT;
243 case PPC440SPE_DMA0_ID:
244 case PPC440SPE_DMA1_ID:
245 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
246 /* NOP with interrupt */
247 set_bit(PPC440SPE_DESC_INT, &desc->flags);
250 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
257 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
262 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
263 desc->hw_next = NULL;
269 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
272 int src_cnt, unsigned long flags)
274 struct xor_cb *hw_desc = desc->hw_desc;
276 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
277 desc->hw_next = NULL;
278 desc->src_cnt = src_cnt;
281 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
282 if (flags & DMA_PREP_INTERRUPT)
283 /* Enable interrupt on completion */
284 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
288 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
289 * operation in DMA2 controller
291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
292 int dst_cnt, int src_cnt, unsigned long flags)
294 struct xor_cb *hw_desc = desc->hw_desc;
296 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
297 desc->hw_next = NULL;
298 desc->src_cnt = src_cnt;
299 desc->dst_cnt = dst_cnt;
300 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
301 desc->descs_per_op = 0;
303 hw_desc->cbc = XOR_CBCR_TGT_BIT;
304 if (flags & DMA_PREP_INTERRUPT)
305 /* Enable interrupt on completion */
306 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
309 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
310 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
311 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
314 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
318 int dst_cnt, int src_cnt, unsigned long flags,
321 struct dma_cdb *hw_desc;
322 struct ppc440spe_adma_desc_slot *iter;
325 /* Common initialization of a PQ descriptors chain */
326 set_bits(op, &desc->flags);
327 desc->src_cnt = src_cnt;
328 desc->dst_cnt = dst_cnt;
330 /* WXOR MULTICAST if both P and Q are being computed
331 * MV_SG1_SG2 if Q only
333 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
334 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
336 list_for_each_entry(iter, &desc->group_list, chain_node) {
337 hw_desc = iter->hw_desc;
338 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
340 if (likely(!list_is_last(&iter->chain_node,
341 &desc->group_list))) {
342 /* set 'next' pointer */
343 iter->hw_next = list_entry(iter->chain_node.next,
344 struct ppc440spe_adma_desc_slot, chain_node);
345 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
347 /* this is the last descriptor.
348 * this slot will be pasted from ADMA level
349 * each time it wants to configure parameters
350 * of the transaction (src, dst, ...)
352 iter->hw_next = NULL;
353 if (flags & DMA_PREP_INTERRUPT)
354 set_bit(PPC440SPE_DESC_INT, &iter->flags);
356 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
360 /* Set OPS depending on WXOR/RXOR type of operation */
361 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
362 /* This is a WXOR only chain:
363 * - first descriptors are for zeroing destinations
364 * if PPC440SPE_ZERO_P/Q set;
365 * - descriptors remained are for GF-XOR operations.
367 iter = list_first_entry(&desc->group_list,
368 struct ppc440spe_adma_desc_slot,
371 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
372 hw_desc = iter->hw_desc;
373 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
374 iter = list_first_entry(&iter->chain_node,
375 struct ppc440spe_adma_desc_slot,
379 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
380 hw_desc = iter->hw_desc;
381 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
382 iter = list_first_entry(&iter->chain_node,
383 struct ppc440spe_adma_desc_slot,
387 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
388 hw_desc = iter->hw_desc;
392 /* This is either RXOR-only or mixed RXOR/WXOR */
394 /* The first 1 or 2 slots in chain are always RXOR,
395 * if need to calculate P & Q, then there are two
396 * RXOR slots; if only P or only Q, then there is one
398 iter = list_first_entry(&desc->group_list,
399 struct ppc440spe_adma_desc_slot,
401 hw_desc = iter->hw_desc;
402 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
404 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
405 iter = list_first_entry(&iter->chain_node,
406 struct ppc440spe_adma_desc_slot,
408 hw_desc = iter->hw_desc;
409 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
412 /* The remaining descs (if any) are WXORs */
413 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
414 iter = list_first_entry(&iter->chain_node,
415 struct ppc440spe_adma_desc_slot,
417 list_for_each_entry_from(iter, &desc->group_list,
419 hw_desc = iter->hw_desc;
427 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
428 * for PQ_ZERO_SUM operation
430 static void ppc440spe_desc_init_dma01pqzero_sum(
431 struct ppc440spe_adma_desc_slot *desc,
432 int dst_cnt, int src_cnt)
434 struct dma_cdb *hw_desc;
435 struct ppc440spe_adma_desc_slot *iter;
437 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
438 DMA_CDB_OPC_MV_SG1_SG2;
440 * Initialize starting from 2nd or 3rd descriptor dependent
441 * on dst_cnt. First one or two slots are for cloning P
442 * and/or Q to chan->pdest and/or chan->qdest as we have
443 * to preserve original P/Q.
445 iter = list_first_entry(&desc->group_list,
446 struct ppc440spe_adma_desc_slot, chain_node);
447 iter = list_entry(iter->chain_node.next,
448 struct ppc440spe_adma_desc_slot, chain_node);
451 iter = list_entry(iter->chain_node.next,
452 struct ppc440spe_adma_desc_slot, chain_node);
454 /* initialize each source descriptor in chain */
455 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
456 hw_desc = iter->hw_desc;
457 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
461 /* This is a ZERO_SUM operation:
462 * - <src_cnt> descriptors starting from 2nd or 3rd
463 * descriptor are for GF-XOR operations;
464 * - remaining <dst_cnt> descriptors are for checking the result
467 /* MV_SG1_SG2 if only Q is being verified
468 * MULTICAST if both P and Q are being verified
472 /* DMA_CDB_OPC_DCHECK128 operation */
473 hw_desc->opc = DMA_CDB_OPC_DCHECK128;
475 if (likely(!list_is_last(&iter->chain_node,
476 &desc->group_list))) {
477 /* set 'next' pointer */
478 iter->hw_next = list_entry(iter->chain_node.next,
479 struct ppc440spe_adma_desc_slot,
482 /* this is the last descriptor.
483 * this slot will be pasted from ADMA level
484 * each time it wants to configure parameters
485 * of the transaction (src, dst, ...)
487 iter->hw_next = NULL;
488 /* always enable interrupt generation since we get
489 * the status of pqzero from the handler
491 set_bit(PPC440SPE_DESC_INT, &iter->flags);
494 desc->src_cnt = src_cnt;
495 desc->dst_cnt = dst_cnt;
499 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
504 struct dma_cdb *hw_desc = desc->hw_desc;
506 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
507 desc->hw_next = NULL;
511 if (flags & DMA_PREP_INTERRUPT)
512 set_bit(PPC440SPE_DESC_INT, &desc->flags);
514 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
516 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
520 * ppc440spe_desc_set_src_addr - set source address into the descriptor
522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
523 struct ppc440spe_adma_chan *chan,
524 int src_idx, dma_addr_t addrh,
527 struct dma_cdb *dma_hw_desc;
528 struct xor_cb *xor_hw_desc;
529 phys_addr_t addr64, tmplow, tmphi;
531 switch (chan->device->id) {
532 case PPC440SPE_DMA0_ID:
533 case PPC440SPE_DMA1_ID:
536 tmphi = (addr64 >> 32);
537 tmplow = (addr64 & 0xFFFFFFFF);
542 dma_hw_desc = desc->hw_desc;
543 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
544 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
546 case PPC440SPE_XOR_ID:
547 xor_hw_desc = desc->hw_desc;
548 xor_hw_desc->ops[src_idx].l = addrl;
549 xor_hw_desc->ops[src_idx].h |= addrh;
555 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
558 struct ppc440spe_adma_chan *chan, u32 mult_index,
559 int sg_index, unsigned char mult_value)
561 struct dma_cdb *dma_hw_desc;
562 struct xor_cb *xor_hw_desc;
565 switch (chan->device->id) {
566 case PPC440SPE_DMA0_ID:
567 case PPC440SPE_DMA1_ID:
568 dma_hw_desc = desc->hw_desc;
571 /* for RXOR operations set multiplier
572 * into source cued address
575 psgu = &dma_hw_desc->sg1u;
577 /* for WXOR operations set multiplier
578 * into destination cued address(es)
580 case DMA_CDB_SG_DST1:
581 psgu = &dma_hw_desc->sg2u;
583 case DMA_CDB_SG_DST2:
584 psgu = &dma_hw_desc->sg3u;
590 *psgu |= cpu_to_le32(mult_value << mult_index);
592 case PPC440SPE_XOR_ID:
593 xor_hw_desc = desc->hw_desc;
601 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
603 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
604 struct ppc440spe_adma_chan *chan,
605 dma_addr_t addrh, dma_addr_t addrl,
608 struct dma_cdb *dma_hw_desc;
609 struct xor_cb *xor_hw_desc;
610 phys_addr_t addr64, tmphi, tmplow;
613 switch (chan->device->id) {
614 case PPC440SPE_DMA0_ID:
615 case PPC440SPE_DMA1_ID:
618 tmphi = (addr64 >> 32);
619 tmplow = (addr64 & 0xFFFFFFFF);
624 dma_hw_desc = desc->hw_desc;
626 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
627 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
629 *psgl = cpu_to_le32((u32)tmplow);
630 *psgu |= cpu_to_le32((u32)tmphi);
632 case PPC440SPE_XOR_ID:
633 xor_hw_desc = desc->hw_desc;
634 xor_hw_desc->cbtal = addrl;
635 xor_hw_desc->cbtah |= addrh;
641 * ppc440spe_desc_set_byte_count - set number of data bytes involved
644 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
645 struct ppc440spe_adma_chan *chan,
648 struct dma_cdb *dma_hw_desc;
649 struct xor_cb *xor_hw_desc;
651 switch (chan->device->id) {
652 case PPC440SPE_DMA0_ID:
653 case PPC440SPE_DMA1_ID:
654 dma_hw_desc = desc->hw_desc;
655 dma_hw_desc->cnt = cpu_to_le32(byte_count);
657 case PPC440SPE_XOR_ID:
658 xor_hw_desc = desc->hw_desc;
659 xor_hw_desc->cbbc = byte_count;
665 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
667 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
669 /* assume that byte_count is aligned on the 512-boundary;
670 * thus write it directly to the register (bits 23:31 are
673 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
677 * ppc440spe_desc_set_dcheck - set CHECK pattern
679 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
680 struct ppc440spe_adma_chan *chan, u8 *qword)
682 struct dma_cdb *dma_hw_desc;
684 switch (chan->device->id) {
685 case PPC440SPE_DMA0_ID:
686 case PPC440SPE_DMA1_ID:
687 dma_hw_desc = desc->hw_desc;
688 iowrite32(qword[0], &dma_hw_desc->sg3l);
689 iowrite32(qword[4], &dma_hw_desc->sg3u);
690 iowrite32(qword[8], &dma_hw_desc->sg2l);
691 iowrite32(qword[12], &dma_hw_desc->sg2u);
699 * ppc440spe_xor_set_link - set link address in xor CB
701 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
702 struct ppc440spe_adma_desc_slot *next_desc)
704 struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
706 if (unlikely(!next_desc || !(next_desc->phys))) {
707 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
709 next_desc ? next_desc->phys : 0);
713 xor_hw_desc->cbs = 0;
714 xor_hw_desc->cblal = next_desc->phys;
715 xor_hw_desc->cblah = 0;
716 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
720 * ppc440spe_desc_set_link - set the address of descriptor following this
721 * descriptor in chain
723 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
724 struct ppc440spe_adma_desc_slot *prev_desc,
725 struct ppc440spe_adma_desc_slot *next_desc)
728 struct ppc440spe_adma_desc_slot *tail = next_desc;
730 if (unlikely(!prev_desc || !next_desc ||
731 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
732 /* If previous next is overwritten something is wrong.
733 * though we may refetch from append to initiate list
734 * processing; in this case - it's ok.
736 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
737 "prev->hw_next=0x%p\n", __func__, prev_desc,
738 next_desc, prev_desc ? prev_desc->hw_next : 0);
742 local_irq_save(flags);
744 /* do s/w chaining both for DMA and XOR descriptors */
745 prev_desc->hw_next = next_desc;
747 switch (chan->device->id) {
748 case PPC440SPE_DMA0_ID:
749 case PPC440SPE_DMA1_ID:
751 case PPC440SPE_XOR_ID:
752 /* bind descriptor to the chain */
753 while (tail->hw_next)
754 tail = tail->hw_next;
755 xor_last_linked = tail;
757 if (prev_desc == xor_last_submit)
758 /* do not link to the last submitted CB */
760 ppc440spe_xor_set_link(prev_desc, next_desc);
764 local_irq_restore(flags);
768 * ppc440spe_desc_get_link - get the address of the descriptor that
771 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
772 struct ppc440spe_adma_chan *chan)
777 return desc->hw_next->phys;
781 * ppc440spe_desc_is_aligned - check alignment
783 static inline int ppc440spe_desc_is_aligned(
784 struct ppc440spe_adma_desc_slot *desc, int num_slots)
786 return (desc->idx & (num_slots - 1)) ? 0 : 1;
790 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
793 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
798 /* each XOR descriptor provides up to 16 source operands */
799 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
801 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
804 printk(KERN_ERR "%s: len %d > max %d !!\n",
805 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
811 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
814 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
815 int src_cnt, size_t len)
817 signed long long order = 0;
821 for (i = 1; i < src_cnt; i++) {
822 dma_addr_t cur_addr = srcs[i];
823 dma_addr_t old_addr = srcs[i-1];
826 if (cur_addr == old_addr + len) {
832 } else if (old_addr == cur_addr + len) {
843 if (i == src_cnt-2 || (order == -1
844 && cur_addr != old_addr - len)) {
848 } else if (cur_addr == old_addr + len*order) {
852 } else if (cur_addr == old_addr + 2*len) {
856 } else if (cur_addr == old_addr + 3*len) {
875 if (src_cnt <= 1 || (state != 1 && state != 2)) {
876 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
877 __func__, src_cnt, state, addr_count, order);
878 for (i = 0; i < src_cnt; i++)
879 pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
883 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
887 /******************************************************************************
888 * ADMA channel low-level routines
889 ******************************************************************************/
892 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
893 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
896 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
898 static void ppc440spe_adma_device_clear_eot_status(
899 struct ppc440spe_adma_chan *chan)
901 struct dma_regs *dma_reg;
902 struct xor_regs *xor_reg;
903 u8 *p = chan->device->dma_desc_pool_virt;
907 switch (chan->device->id) {
908 case PPC440SPE_DMA0_ID:
909 case PPC440SPE_DMA1_ID:
910 /* read FIFO to ack */
911 dma_reg = chan->device->dma_reg;
912 while ((rv = ioread32(&dma_reg->csfpl))) {
913 i = rv & DMA_CDB_ADDR_MSK;
914 cdb = (struct dma_cdb *)&p[i -
915 (u32)chan->device->dma_desc_pool];
917 /* Clear opcode to ack. This is necessary for
918 * ZeroSum operations only
922 if (test_bit(PPC440SPE_RXOR_RUN,
923 &ppc440spe_rxor_state)) {
924 /* probably this is a completed RXOR op,
925 * get pointer to CDB using the fact that
926 * physical and virtual addresses of CDB
927 * in pools have the same offsets
929 if (le32_to_cpu(cdb->sg1u) &
932 clear_bit(PPC440SPE_RXOR_RUN,
933 &ppc440spe_rxor_state);
937 if (rv & DMA_CDB_STATUS_MSK) {
938 /* ZeroSum check failed
940 struct ppc440spe_adma_desc_slot *iter;
941 dma_addr_t phys = rv & ~DMA_CDB_MSK;
944 * Update the status of corresponding
947 list_for_each_entry(iter, &chan->chain,
949 if (iter->phys == phys)
953 * if cannot find the corresponding
956 BUG_ON(&iter->chain_node == &chan->chain);
958 if (iter->xor_check_result) {
959 if (test_bit(PPC440SPE_DESC_PCHECK,
961 *iter->xor_check_result |=
964 if (test_bit(PPC440SPE_DESC_QCHECK,
966 *iter->xor_check_result |=
974 rv = ioread32(&dma_reg->dsts);
976 pr_err("DMA%d err status: 0x%x\n",
977 chan->device->id, rv);
978 /* write back to clear */
979 iowrite32(rv, &dma_reg->dsts);
982 case PPC440SPE_XOR_ID:
983 /* reset status bits to ack */
984 xor_reg = chan->device->xor_reg;
985 rv = ioread32be(&xor_reg->sr);
986 iowrite32be(rv, &xor_reg->sr);
988 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
989 if (rv & XOR_IE_RPTIE_BIT) {
990 /* Read PLB Timeout Error.
991 * Try to resubmit the CB
993 u32 val = ioread32be(&xor_reg->ccbalr);
995 iowrite32be(val, &xor_reg->cblalr);
997 val = ioread32be(&xor_reg->crsr);
998 iowrite32be(val | XOR_CRSR_XAE_BIT,
1001 pr_err("XOR ERR 0x%x status\n", rv);
1005 /* if the XORcore is idle, but there are unprocessed CBs
1006 * then refetch the s/w chain here
1008 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1010 ppc440spe_chan_append(chan);
1016 * ppc440spe_chan_is_busy - get the channel status
1018 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1020 struct dma_regs *dma_reg;
1021 struct xor_regs *xor_reg;
1024 switch (chan->device->id) {
1025 case PPC440SPE_DMA0_ID:
1026 case PPC440SPE_DMA1_ID:
1027 dma_reg = chan->device->dma_reg;
1028 /* if command FIFO's head and tail pointers are equal and
1029 * status tail is the same as command, then channel is free
1031 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1032 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1035 case PPC440SPE_XOR_ID:
1036 /* use the special status bit for the XORcore
1038 xor_reg = chan->device->xor_reg;
1039 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1047 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1049 static void ppc440spe_chan_set_first_xor_descriptor(
1050 struct ppc440spe_adma_chan *chan,
1051 struct ppc440spe_adma_desc_slot *next_desc)
1053 struct xor_regs *xor_reg = chan->device->xor_reg;
1055 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1056 printk(KERN_INFO "%s: Warn: XORcore is running "
1057 "when try to set the first CDB!\n",
1060 xor_last_submit = xor_last_linked = next_desc;
1062 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1064 iowrite32be(next_desc->phys, &xor_reg->cblalr);
1065 iowrite32be(0, &xor_reg->cblahr);
1066 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1069 chan->hw_chain_inited = 1;
1073 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1074 * called with irqs disabled
1076 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1077 struct ppc440spe_adma_desc_slot *desc)
1080 struct dma_regs *dma_reg = chan->device->dma_reg;
1083 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1084 pcdb |= DMA_CDB_NO_INT;
1086 chan_last_sub[chan->device->id] = desc;
1088 ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1090 iowrite32(pcdb, &dma_reg->cpfpl);
1094 * ppc440spe_chan_append - update the h/w chain in the channel
1096 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1098 struct xor_regs *xor_reg;
1099 struct ppc440spe_adma_desc_slot *iter;
1102 unsigned long flags;
1104 local_irq_save(flags);
1106 switch (chan->device->id) {
1107 case PPC440SPE_DMA0_ID:
1108 case PPC440SPE_DMA1_ID:
1109 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1111 if (likely(cur_desc)) {
1112 iter = chan_last_sub[chan->device->id];
1116 iter = chan_first_cdb[chan->device->id];
1118 ppc440spe_dma_put_desc(chan, iter);
1119 chan->hw_chain_inited = 1;
1122 /* is there something new to append */
1126 /* flush descriptors from the s/w queue to fifo */
1127 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1128 ppc440spe_dma_put_desc(chan, iter);
1133 case PPC440SPE_XOR_ID:
1134 /* update h/w links and refetch */
1135 if (!xor_last_submit->hw_next)
1138 xor_reg = chan->device->xor_reg;
1139 /* the last linked CDB has to generate an interrupt
1140 * that we'd be able to append the next lists to h/w
1141 * regardless of the XOR engine state at the moment of
1142 * appending of these next lists
1144 xcb = xor_last_linked->hw_desc;
1145 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1147 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1148 /* XORcore is idle. Refetch now */
1150 ppc440spe_xor_set_link(xor_last_submit,
1151 xor_last_submit->hw_next);
1153 ADMA_LL_DBG(print_cb_list(chan,
1154 xor_last_submit->hw_next));
1156 xor_last_submit = xor_last_linked;
1157 iowrite32be(ioread32be(&xor_reg->crsr) |
1158 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1161 /* XORcore is running. Refetch later in the handler */
1168 local_irq_restore(flags);
1172 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1175 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1177 struct dma_regs *dma_reg;
1178 struct xor_regs *xor_reg;
1180 if (unlikely(!chan->hw_chain_inited))
1181 /* h/w descriptor chain is not initialized yet */
1184 switch (chan->device->id) {
1185 case PPC440SPE_DMA0_ID:
1186 case PPC440SPE_DMA1_ID:
1187 dma_reg = chan->device->dma_reg;
1188 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1189 case PPC440SPE_XOR_ID:
1190 xor_reg = chan->device->xor_reg;
1191 return ioread32be(&xor_reg->ccbalr);
1197 * ppc440spe_chan_run - enable the channel
1199 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1201 struct xor_regs *xor_reg;
1203 switch (chan->device->id) {
1204 case PPC440SPE_DMA0_ID:
1205 case PPC440SPE_DMA1_ID:
1206 /* DMAs are always enabled, do nothing */
1208 case PPC440SPE_XOR_ID:
1209 /* drain write buffer */
1210 xor_reg = chan->device->xor_reg;
1212 /* fetch descriptor pointed to in <link> */
1213 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1219 /******************************************************************************
1221 ******************************************************************************/
1223 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1224 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1227 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1229 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1230 dma_addr_t addr, int index);
1232 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1233 dma_addr_t addr, int index);
1236 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1237 dma_addr_t *paddr, unsigned long flags);
1239 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1240 dma_addr_t addr, int index);
1242 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1243 unsigned char mult, int index, int dst_pos);
1245 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1246 dma_addr_t paddr, dma_addr_t qaddr);
1248 static struct page *ppc440spe_rxor_srcs[32];
1251 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1253 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1255 int i, order = 0, state = 0;
1258 if (unlikely(!(src_cnt > 1)))
1261 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1263 /* Skip holes in the source list before checking */
1264 for (i = 0; i < src_cnt; i++) {
1267 ppc440spe_rxor_srcs[idx++] = srcs[i];
1271 for (i = 1; i < src_cnt; i++) {
1272 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1273 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1277 if (cur_addr == old_addr + len) {
1281 } else if (old_addr == cur_addr + len) {
1289 if ((i == src_cnt - 2) ||
1290 (order == -1 && cur_addr != old_addr - len)) {
1293 } else if ((cur_addr == old_addr + len * order) ||
1294 (cur_addr == old_addr + 2 * len) ||
1295 (cur_addr == old_addr + 3 * len)) {
1310 if (state == 1 || state == 2)
1317 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1318 * the operation given on this channel. It's assumed that 'chan' is
1319 * capable to process 'cap' type of operation.
1320 * @chan: channel to use
1321 * @cap: type of transaction
1322 * @dst_lst: array of destination pointers
1323 * @dst_cnt: number of destination operands
1324 * @src_lst: array of source pointers
1325 * @src_cnt: number of source operands
1326 * @src_sz: size of each source operand
1328 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1329 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1330 struct page **src_lst, int src_cnt, size_t src_sz)
1334 if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1335 /* If RAID-6 capabilities were not activated don't try
1338 if (unlikely(!ppc440spe_r6_enabled))
1341 /* In the current implementation of ppc440spe ADMA driver it
1342 * makes sense to pick out only pq case, because it may be
1344 * (1) either using Biskup method on DMA2;
1346 * Thus we give a favour to (1) if the sources are suitable;
1347 * else let it be processed on one of the DMA0/1 engines.
1348 * In the sum_product case where destination is also the
1349 * source process it on DMA0/1 only.
1351 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1353 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1354 ef = 0; /* sum_product case, process on DMA0/1 */
1355 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1356 ef = 3; /* override (DMA0/1 + idle) */
1358 ef = 0; /* can't process on DMA2 if !rxor */
1361 /* channel idleness increases the priority */
1363 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1370 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1371 struct page **dst_lst, int dst_cnt, struct page **src_lst,
1372 int src_cnt, size_t src_sz)
1374 struct dma_chan *best_chan = NULL;
1375 struct ppc_dma_chan_ref *ref;
1378 if (unlikely(!src_sz))
1380 if (src_sz > PAGE_SIZE) {
1382 * should a user of the api ever pass > PAGE_SIZE requests
1383 * we sort out cases where temporary page-sized buffers
1388 if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1390 if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1401 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1402 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1405 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1406 dst_cnt, src_lst, src_cnt, src_sz);
1407 if (rank > best_rank) {
1409 best_chan = ref->chan;
1416 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1419 * ppc440spe_get_group_entry - get group entry with index idx
1420 * @tdesc: is the last allocated slot in the group.
1422 static struct ppc440spe_adma_desc_slot *
1423 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1425 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1428 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1429 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1430 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1434 list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1435 if (i++ == entry_idx)
1442 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1443 * @slot: Slot to free
1444 * Caller must hold &ppc440spe_chan->lock while calling this function
1446 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1447 struct ppc440spe_adma_chan *chan)
1449 int stride = slot->slots_per_op;
1452 slot->slots_per_op = 0;
1453 slot = list_entry(slot->slot_node.next,
1454 struct ppc440spe_adma_desc_slot,
1460 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1463 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1464 struct ppc440spe_adma_desc_slot *desc,
1465 struct ppc440spe_adma_chan *chan,
1466 dma_cookie_t cookie)
1468 BUG_ON(desc->async_tx.cookie < 0);
1469 if (desc->async_tx.cookie > 0) {
1470 cookie = desc->async_tx.cookie;
1471 desc->async_tx.cookie = 0;
1473 dma_descriptor_unmap(&desc->async_tx);
1474 /* call the callback (must not sleep or submit new
1475 * operations to this channel)
1477 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1480 /* run dependent operations */
1481 dma_run_dependencies(&desc->async_tx);
1487 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1489 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1490 struct ppc440spe_adma_chan *chan)
1492 /* the client is allowed to attach dependent operations
1493 * until 'ack' is set
1495 if (!async_tx_test_ack(&desc->async_tx))
1498 /* leave the last descriptor in the chain
1499 * so we can append to it
1501 if (list_is_last(&desc->chain_node, &chan->chain) ||
1502 desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1505 if (chan->device->id != PPC440SPE_XOR_ID) {
1506 /* our DMA interrupt handler clears opc field of
1507 * each processed descriptor. For all types of
1508 * operations except for ZeroSum we do not actually
1509 * need ack from the interrupt handler. ZeroSum is a
1510 * special case since the result of this operation
1511 * is available from the handler only, so if we see
1512 * such type of descriptor (which is unprocessed yet)
1513 * then leave it in chain.
1515 struct dma_cdb *cdb = desc->hw_desc;
1516 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1520 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1521 desc->phys, desc->idx, desc->slots_per_op);
1523 list_del(&desc->chain_node);
1524 ppc440spe_adma_free_slots(desc, chan);
1529 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1530 * which runs through the channel CDBs list until reach the descriptor
1531 * currently processed. When routine determines that all CDBs of group
1532 * are completed then corresponding callbacks (if any) are called and slots
1535 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1537 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1538 dma_cookie_t cookie = 0;
1539 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1540 int busy = ppc440spe_chan_is_busy(chan);
1541 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1543 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1544 chan->device->id, __func__);
1546 if (!current_desc) {
1547 /* There were no transactions yet, so
1553 /* free completed slots from the chain starting with
1554 * the oldest descriptor
1556 list_for_each_entry_safe(iter, _iter, &chan->chain,
1558 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1559 "busy: %d this_desc: %#llx next_desc: %#x "
1560 "cur: %#x ack: %d\n",
1561 iter->async_tx.cookie, iter->idx, busy, iter->phys,
1562 ppc440spe_desc_get_link(iter, chan), current_desc,
1563 async_tx_test_ack(&iter->async_tx));
1565 prefetch(&_iter->async_tx);
1567 /* do not advance past the current descriptor loaded into the
1568 * hardware channel,subsequent descriptors are either in process
1569 * or have not been submitted
1574 /* stop the search if we reach the current descriptor and the
1575 * channel is busy, or if it appears that the current descriptor
1576 * needs to be re-read (i.e. has been appended to)
1578 if (iter->phys == current_desc) {
1579 BUG_ON(seen_current++);
1580 if (busy || ppc440spe_desc_get_link(iter, chan)) {
1581 /* not all descriptors of the group have
1582 * been completed; exit.
1588 /* detect the start of a group transaction */
1589 if (!slot_cnt && !slots_per_op) {
1590 slot_cnt = iter->slot_cnt;
1591 slots_per_op = iter->slots_per_op;
1592 if (slot_cnt <= slots_per_op) {
1601 slot_cnt -= slots_per_op;
1604 /* all the members of a group are complete */
1605 if (slots_per_op != 0 && slot_cnt == 0) {
1606 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1607 int end_of_chain = 0;
1609 /* clean up the group */
1610 slot_cnt = group_start->slot_cnt;
1611 grp_iter = group_start;
1612 list_for_each_entry_safe_from(grp_iter, _grp_iter,
1613 &chan->chain, chain_node) {
1615 cookie = ppc440spe_adma_run_tx_complete_actions(
1616 grp_iter, chan, cookie);
1618 slot_cnt -= slots_per_op;
1619 end_of_chain = ppc440spe_adma_clean_slot(
1621 if (end_of_chain && slot_cnt) {
1622 /* Should wait for ZeroSum completion */
1624 chan->common.completed_cookie = cookie;
1628 if (slot_cnt == 0 || end_of_chain)
1632 /* the group should be complete at this point */
1641 } else if (slots_per_op) /* wait for group completion */
1644 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1647 if (ppc440spe_adma_clean_slot(iter, chan))
1651 BUG_ON(!seen_current);
1654 chan->common.completed_cookie = cookie;
1655 pr_debug("\tcompleted cookie %d\n", cookie);
1661 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1663 static void ppc440spe_adma_tasklet(unsigned long data)
1665 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1667 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1668 __ppc440spe_adma_slot_cleanup(chan);
1669 spin_unlock(&chan->lock);
1673 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1675 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1677 spin_lock_bh(&chan->lock);
1678 __ppc440spe_adma_slot_cleanup(chan);
1679 spin_unlock_bh(&chan->lock);
1683 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1685 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1686 struct ppc440spe_adma_chan *chan, int num_slots,
1689 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1690 struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1691 struct list_head chain = LIST_HEAD_INIT(chain);
1692 int slots_found, retry = 0;
1695 BUG_ON(!num_slots || !slots_per_op);
1696 /* start search from the last allocated descrtiptor
1697 * if a contiguous allocation can not be found start searching
1698 * from the beginning of the list
1703 iter = chan->last_used;
1705 iter = list_entry(&chan->all_slots,
1706 struct ppc440spe_adma_desc_slot,
1708 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1711 prefetch(&_iter->async_tx);
1712 if (iter->slots_per_op) {
1717 /* start the allocation if the slot is correctly aligned */
1721 if (slots_found == num_slots) {
1722 struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1723 struct ppc440spe_adma_desc_slot *last_used = NULL;
1728 /* pre-ack all but the last descriptor */
1729 if (num_slots != slots_per_op)
1730 async_tx_ack(&iter->async_tx);
1732 list_add_tail(&iter->chain_node, &chain);
1734 iter->async_tx.cookie = 0;
1735 iter->hw_next = NULL;
1737 iter->slot_cnt = num_slots;
1738 iter->xor_check_result = NULL;
1739 for (i = 0; i < slots_per_op; i++) {
1740 iter->slots_per_op = slots_per_op - i;
1742 iter = list_entry(iter->slot_node.next,
1743 struct ppc440spe_adma_desc_slot,
1746 num_slots -= slots_per_op;
1748 alloc_tail->group_head = alloc_start;
1749 alloc_tail->async_tx.cookie = -EBUSY;
1750 list_splice(&chain, &alloc_tail->group_list);
1751 chan->last_used = last_used;
1758 /* try to free some slots if the allocation fails */
1759 tasklet_schedule(&chan->irq_tasklet);
1764 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
1766 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1768 struct ppc440spe_adma_chan *ppc440spe_chan;
1769 struct ppc440spe_adma_desc_slot *slot = NULL;
1774 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1775 init = ppc440spe_chan->slots_allocated ? 0 : 1;
1776 chan->chan_id = ppc440spe_chan->device->id;
1778 /* Allocate descriptor slots */
1779 i = ppc440spe_chan->slots_allocated;
1780 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1781 db_sz = sizeof(struct dma_cdb);
1783 db_sz = sizeof(struct xor_cb);
1785 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1786 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1789 printk(KERN_INFO "SPE ADMA Channel only initialized"
1790 " %d descriptor slots", i--);
1794 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1795 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1796 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1797 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1798 INIT_LIST_HEAD(&slot->chain_node);
1799 INIT_LIST_HEAD(&slot->slot_node);
1800 INIT_LIST_HEAD(&slot->group_list);
1801 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1804 spin_lock_bh(&ppc440spe_chan->lock);
1805 ppc440spe_chan->slots_allocated++;
1806 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1807 spin_unlock_bh(&ppc440spe_chan->lock);
1810 if (i && !ppc440spe_chan->last_used) {
1811 ppc440spe_chan->last_used =
1812 list_entry(ppc440spe_chan->all_slots.next,
1813 struct ppc440spe_adma_desc_slot,
1817 dev_dbg(ppc440spe_chan->device->common.dev,
1818 "ppc440spe adma%d: allocated %d descriptor slots\n",
1819 ppc440spe_chan->device->id, i);
1821 /* initialize the channel and the chain with a null operation */
1823 switch (ppc440spe_chan->device->id) {
1824 case PPC440SPE_DMA0_ID:
1825 case PPC440SPE_DMA1_ID:
1826 ppc440spe_chan->hw_chain_inited = 0;
1827 /* Use WXOR for self-testing */
1828 if (!ppc440spe_r6_tchan)
1829 ppc440spe_r6_tchan = ppc440spe_chan;
1831 case PPC440SPE_XOR_ID:
1832 ppc440spe_chan_start_null_xor(ppc440spe_chan);
1837 ppc440spe_chan->needs_unmap = 1;
1840 return (i > 0) ? i : -ENOMEM;
1844 * ppc440spe_rxor_set_region_data -
1846 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1847 u8 xor_arg_no, u32 mask)
1849 struct xor_cb *xcb = desc->hw_desc;
1851 xcb->ops[xor_arg_no].h |= mask;
1855 * ppc440spe_rxor_set_src -
1857 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1858 u8 xor_arg_no, dma_addr_t addr)
1860 struct xor_cb *xcb = desc->hw_desc;
1862 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1863 xcb->ops[xor_arg_no].l = addr;
1867 * ppc440spe_rxor_set_mult -
1869 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1870 u8 xor_arg_no, u8 idx, u8 mult)
1872 struct xor_cb *xcb = desc->hw_desc;
1874 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1878 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1881 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1883 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1884 chan->device->id, chan->pending);
1886 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1888 ppc440spe_chan_append(chan);
1893 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1894 * (it's not necessary that descriptors will be submitted to the h/w
1895 * chains too right now)
1897 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1899 struct ppc440spe_adma_desc_slot *sw_desc;
1900 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1901 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1904 dma_cookie_t cookie;
1906 sw_desc = tx_to_ppc440spe_adma_slot(tx);
1908 group_start = sw_desc->group_head;
1909 slot_cnt = group_start->slot_cnt;
1910 slots_per_op = group_start->slots_per_op;
1912 spin_lock_bh(&chan->lock);
1913 cookie = dma_cookie_assign(tx);
1915 if (unlikely(list_empty(&chan->chain))) {
1917 list_splice_init(&sw_desc->group_list, &chan->chain);
1918 chan_first_cdb[chan->device->id] = group_start;
1920 /* isn't first peer, bind CDBs to chain */
1921 old_chain_tail = list_entry(chan->chain.prev,
1922 struct ppc440spe_adma_desc_slot,
1924 list_splice_init(&sw_desc->group_list,
1925 &old_chain_tail->chain_node);
1926 /* fix up the hardware chain */
1927 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1930 /* increment the pending count by the number of operations */
1931 chan->pending += slot_cnt / slots_per_op;
1932 ppc440spe_adma_check_threshold(chan);
1933 spin_unlock_bh(&chan->lock);
1935 dev_dbg(chan->device->common.dev,
1936 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1937 chan->device->id, __func__,
1938 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1944 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1946 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1947 struct dma_chan *chan, unsigned long flags)
1949 struct ppc440spe_adma_chan *ppc440spe_chan;
1950 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1951 int slot_cnt, slots_per_op;
1953 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1955 dev_dbg(ppc440spe_chan->device->common.dev,
1956 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1959 spin_lock_bh(&ppc440spe_chan->lock);
1960 slot_cnt = slots_per_op = 1;
1961 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1964 group_start = sw_desc->group_head;
1965 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1966 group_start->unmap_len = 0;
1967 sw_desc->async_tx.flags = flags;
1969 spin_unlock_bh(&ppc440spe_chan->lock);
1971 return sw_desc ? &sw_desc->async_tx : NULL;
1975 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1977 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1978 struct dma_chan *chan, dma_addr_t dma_dest,
1979 dma_addr_t dma_src, size_t len, unsigned long flags)
1981 struct ppc440spe_adma_chan *ppc440spe_chan;
1982 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1983 int slot_cnt, slots_per_op;
1985 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1990 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1992 spin_lock_bh(&ppc440spe_chan->lock);
1994 dev_dbg(ppc440spe_chan->device->common.dev,
1995 "ppc440spe adma%d: %s len: %u int_en %d\n",
1996 ppc440spe_chan->device->id, __func__, len,
1997 flags & DMA_PREP_INTERRUPT ? 1 : 0);
1998 slot_cnt = slots_per_op = 1;
1999 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2002 group_start = sw_desc->group_head;
2003 ppc440spe_desc_init_memcpy(group_start, flags);
2004 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2005 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2006 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2007 sw_desc->unmap_len = len;
2008 sw_desc->async_tx.flags = flags;
2010 spin_unlock_bh(&ppc440spe_chan->lock);
2012 return sw_desc ? &sw_desc->async_tx : NULL;
2016 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2018 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2019 struct dma_chan *chan, dma_addr_t dma_dest,
2020 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2021 unsigned long flags)
2023 struct ppc440spe_adma_chan *ppc440spe_chan;
2024 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2025 int slot_cnt, slots_per_op;
2027 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2029 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2030 dma_dest, dma_src, src_cnt));
2033 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2035 dev_dbg(ppc440spe_chan->device->common.dev,
2036 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2037 ppc440spe_chan->device->id, __func__, src_cnt, len,
2038 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2040 spin_lock_bh(&ppc440spe_chan->lock);
2041 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2042 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2045 group_start = sw_desc->group_head;
2046 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2047 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2049 ppc440spe_adma_memcpy_xor_set_src(group_start,
2050 dma_src[src_cnt], src_cnt);
2051 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2052 sw_desc->unmap_len = len;
2053 sw_desc->async_tx.flags = flags;
2055 spin_unlock_bh(&ppc440spe_chan->lock);
2057 return sw_desc ? &sw_desc->async_tx : NULL;
2061 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2063 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2066 * ppc440spe_adma_init_dma2rxor_slot -
2068 static void ppc440spe_adma_init_dma2rxor_slot(
2069 struct ppc440spe_adma_desc_slot *desc,
2070 dma_addr_t *src, int src_cnt)
2074 /* initialize CDB */
2075 for (i = 0; i < src_cnt; i++) {
2076 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2077 desc->src_cnt, (u32)src[i]);
2082 * ppc440spe_dma01_prep_mult -
2083 * for Q operation where destination is also the source
2085 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2086 struct ppc440spe_adma_chan *ppc440spe_chan,
2087 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2088 const unsigned char *scf, size_t len, unsigned long flags)
2090 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2091 unsigned long op = 0;
2094 set_bit(PPC440SPE_DESC_WXOR, &op);
2097 spin_lock_bh(&ppc440spe_chan->lock);
2099 /* use WXOR, each descriptor occupies one slot */
2100 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2102 struct ppc440spe_adma_chan *chan;
2103 struct ppc440spe_adma_desc_slot *iter;
2104 struct dma_cdb *hw_desc;
2106 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2107 set_bits(op, &sw_desc->flags);
2108 sw_desc->src_cnt = src_cnt;
2109 sw_desc->dst_cnt = dst_cnt;
2110 /* First descriptor, zero data in the destination and copy it
2111 * to q page using MULTICAST transfer.
2113 iter = list_first_entry(&sw_desc->group_list,
2114 struct ppc440spe_adma_desc_slot,
2116 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2117 /* set 'next' pointer */
2118 iter->hw_next = list_entry(iter->chain_node.next,
2119 struct ppc440spe_adma_desc_slot,
2121 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2122 hw_desc = iter->hw_desc;
2123 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2125 ppc440spe_desc_set_dest_addr(iter, chan,
2126 DMA_CUED_XOR_BASE, dst[0], 0);
2127 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2128 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2130 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2131 iter->unmap_len = len;
2134 * Second descriptor, multiply data from the q page
2135 * and store the result in real destination.
2137 iter = list_first_entry(&iter->chain_node,
2138 struct ppc440spe_adma_desc_slot,
2140 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2141 iter->hw_next = NULL;
2142 if (flags & DMA_PREP_INTERRUPT)
2143 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2145 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2147 hw_desc = iter->hw_desc;
2148 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2149 ppc440spe_desc_set_src_addr(iter, chan, 0,
2150 DMA_CUED_XOR_HB, dst[1]);
2151 ppc440spe_desc_set_dest_addr(iter, chan,
2152 DMA_CUED_XOR_BASE, dst[0], 0);
2154 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2155 DMA_CDB_SG_DST1, scf[0]);
2156 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2157 iter->unmap_len = len;
2158 sw_desc->async_tx.flags = flags;
2161 spin_unlock_bh(&ppc440spe_chan->lock);
2167 * ppc440spe_dma01_prep_sum_product -
2168 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2171 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2172 struct ppc440spe_adma_chan *ppc440spe_chan,
2173 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2174 const unsigned char *scf, size_t len, unsigned long flags)
2176 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2177 unsigned long op = 0;
2180 set_bit(PPC440SPE_DESC_WXOR, &op);
2183 spin_lock_bh(&ppc440spe_chan->lock);
2185 /* WXOR, each descriptor occupies one slot */
2186 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2188 struct ppc440spe_adma_chan *chan;
2189 struct ppc440spe_adma_desc_slot *iter;
2190 struct dma_cdb *hw_desc;
2192 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2193 set_bits(op, &sw_desc->flags);
2194 sw_desc->src_cnt = src_cnt;
2195 sw_desc->dst_cnt = 1;
2196 /* 1st descriptor, src[1] data to q page and zero destination */
2197 iter = list_first_entry(&sw_desc->group_list,
2198 struct ppc440spe_adma_desc_slot,
2200 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2201 iter->hw_next = list_entry(iter->chain_node.next,
2202 struct ppc440spe_adma_desc_slot,
2204 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2205 hw_desc = iter->hw_desc;
2206 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2208 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2210 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2211 ppc440spe_chan->qdest, 1);
2212 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2214 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2215 iter->unmap_len = len;
2217 /* 2nd descriptor, multiply src[1] data and store the
2218 * result in destination */
2219 iter = list_first_entry(&iter->chain_node,
2220 struct ppc440spe_adma_desc_slot,
2222 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2223 /* set 'next' pointer */
2224 iter->hw_next = list_entry(iter->chain_node.next,
2225 struct ppc440spe_adma_desc_slot,
2227 if (flags & DMA_PREP_INTERRUPT)
2228 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2230 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2232 hw_desc = iter->hw_desc;
2233 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2234 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2235 ppc440spe_chan->qdest);
2236 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2238 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2239 DMA_CDB_SG_DST1, scf[1]);
2240 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2241 iter->unmap_len = len;
2244 * 3rd descriptor, multiply src[0] data and xor it
2247 iter = list_first_entry(&iter->chain_node,
2248 struct ppc440spe_adma_desc_slot,
2250 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2251 iter->hw_next = NULL;
2252 if (flags & DMA_PREP_INTERRUPT)
2253 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2255 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2257 hw_desc = iter->hw_desc;
2258 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2259 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2261 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2263 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2264 DMA_CDB_SG_DST1, scf[0]);
2265 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2266 iter->unmap_len = len;
2267 sw_desc->async_tx.flags = flags;
2270 spin_unlock_bh(&ppc440spe_chan->lock);
2275 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2276 struct ppc440spe_adma_chan *ppc440spe_chan,
2277 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2278 const unsigned char *scf, size_t len, unsigned long flags)
2281 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2282 unsigned long op = 0;
2283 unsigned char mult = 1;
2285 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2286 __func__, dst_cnt, src_cnt, len);
2287 /* select operations WXOR/RXOR depending on the
2288 * source addresses of operators and the number
2289 * of destinations (RXOR support only Q-parity calculations)
2291 set_bit(PPC440SPE_DESC_WXOR, &op);
2292 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2295 * - there are more than 1 source,
2296 * - len is aligned on 512-byte boundary,
2297 * - source addresses fit to one of 4 possible regions.
2300 !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2301 (src[0] + len) == src[1]) {
2302 /* may do RXOR R1 R2 */
2303 set_bit(PPC440SPE_DESC_RXOR, &op);
2305 /* may try to enhance region of RXOR */
2306 if ((src[1] + len) == src[2]) {
2307 /* do RXOR R1 R2 R3 */
2308 set_bit(PPC440SPE_DESC_RXOR123,
2310 } else if ((src[1] + len * 2) == src[2]) {
2311 /* do RXOR R1 R2 R4 */
2312 set_bit(PPC440SPE_DESC_RXOR124, &op);
2313 } else if ((src[1] + len * 3) == src[2]) {
2314 /* do RXOR R1 R2 R5 */
2315 set_bit(PPC440SPE_DESC_RXOR125,
2319 set_bit(PPC440SPE_DESC_RXOR12,
2324 set_bit(PPC440SPE_DESC_RXOR12, &op);
2328 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2329 /* can not do this operation with RXOR */
2330 clear_bit(PPC440SPE_RXOR_RUN,
2331 &ppc440spe_rxor_state);
2333 /* can do; set block size right now */
2334 ppc440spe_desc_set_rxor_block_size(len);
2338 /* Number of necessary slots depends on operation type selected */
2339 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2340 /* This is a WXOR only chain. Need descriptors for each
2341 * source to GF-XOR them with WXOR, and need descriptors
2342 * for each destination to zero them with WXOR
2346 if (flags & DMA_PREP_ZERO_P) {
2348 set_bit(PPC440SPE_ZERO_P, &op);
2350 if (flags & DMA_PREP_ZERO_Q) {
2352 set_bit(PPC440SPE_ZERO_Q, &op);
2355 /* Need 1/2 descriptor for RXOR operation, and
2356 * need (src_cnt - (2 or 3)) for WXOR of sources
2361 if (flags & DMA_PREP_ZERO_P)
2362 set_bit(PPC440SPE_ZERO_P, &op);
2363 if (flags & DMA_PREP_ZERO_Q)
2364 set_bit(PPC440SPE_ZERO_Q, &op);
2366 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2367 slot_cnt += src_cnt - 2;
2369 slot_cnt += src_cnt - 3;
2371 /* Thus we have either RXOR only chain or
2374 if (slot_cnt == dst_cnt)
2375 /* RXOR only chain */
2376 clear_bit(PPC440SPE_DESC_WXOR, &op);
2379 spin_lock_bh(&ppc440spe_chan->lock);
2380 /* for both RXOR/WXOR each descriptor occupies one slot */
2381 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2383 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2386 /* setup dst/src/mult */
2387 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2388 __func__, dst[0], dst[1]);
2389 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2391 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2394 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2395 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2396 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2397 * leads to zeroing source data after RXOR.
2398 * So, for P case set-up mult=1 explicitly.
2400 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2401 mult = scf[src_cnt];
2402 ppc440spe_adma_pq_set_src_mult(sw_desc,
2403 mult, src_cnt, dst_cnt - 1);
2406 /* Setup byte count foreach slot just allocated */
2407 sw_desc->async_tx.flags = flags;
2408 list_for_each_entry(iter, &sw_desc->group_list,
2410 ppc440spe_desc_set_byte_count(iter,
2411 ppc440spe_chan, len);
2412 iter->unmap_len = len;
2415 spin_unlock_bh(&ppc440spe_chan->lock);
2420 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2421 struct ppc440spe_adma_chan *ppc440spe_chan,
2422 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2423 const unsigned char *scf, size_t len, unsigned long flags)
2425 int slot_cnt, descs_per_op;
2426 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2427 unsigned long op = 0;
2428 unsigned char mult = 1;
2431 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2432 __func__, dst_cnt, src_cnt, len);*/
2434 spin_lock_bh(&ppc440spe_chan->lock);
2435 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2436 if (descs_per_op < 0) {
2437 spin_unlock_bh(&ppc440spe_chan->lock);
2441 /* depending on number of sources we have 1 or 2 RXOR chains */
2442 slot_cnt = descs_per_op * dst_cnt;
2444 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2447 sw_desc->async_tx.flags = flags;
2448 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2449 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2451 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2453 iter->unmap_len = len;
2455 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2456 iter->rxor_cursor.len = len;
2457 iter->descs_per_op = descs_per_op;
2460 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2462 if (op % descs_per_op == 0)
2463 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2465 if (likely(!list_is_last(&iter->chain_node,
2466 &sw_desc->group_list))) {
2467 /* set 'next' pointer */
2469 list_entry(iter->chain_node.next,
2470 struct ppc440spe_adma_desc_slot,
2472 ppc440spe_xor_set_link(iter, iter->hw_next);
2474 /* this is the last descriptor. */
2475 iter->hw_next = NULL;
2479 /* fixup head descriptor */
2480 sw_desc->dst_cnt = dst_cnt;
2481 if (flags & DMA_PREP_ZERO_P)
2482 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2483 if (flags & DMA_PREP_ZERO_Q)
2484 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2486 /* setup dst/src/mult */
2487 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2490 /* handle descriptors (if dst_cnt == 2) inside
2491 * the ppc440spe_adma_pq_set_srcxxx() functions
2493 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2495 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2496 mult = scf[src_cnt];
2497 ppc440spe_adma_pq_set_src_mult(sw_desc,
2498 mult, src_cnt, dst_cnt - 1);
2501 spin_unlock_bh(&ppc440spe_chan->lock);
2502 ppc440spe_desc_set_rxor_block_size(len);
2507 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2509 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2510 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2511 unsigned int src_cnt, const unsigned char *scf,
2512 size_t len, unsigned long flags)
2514 struct ppc440spe_adma_chan *ppc440spe_chan;
2515 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2518 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2520 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2521 dst, src, src_cnt));
2523 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2526 if (src_cnt == 1 && dst[1] == src[0]) {
2529 /* dst[1] is real destination (Q) */
2531 /* this is the page to multicast source data to */
2532 dest[1] = ppc440spe_chan->qdest;
2533 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2534 dest, 2, src, src_cnt, scf, len, flags);
2535 return sw_desc ? &sw_desc->async_tx : NULL;
2538 if (src_cnt == 2 && dst[1] == src[1]) {
2539 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2540 &dst[1], src, 2, scf, len, flags);
2541 return sw_desc ? &sw_desc->async_tx : NULL;
2544 if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2547 flags |= DMA_PREP_ZERO_P;
2550 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2553 flags |= DMA_PREP_ZERO_Q;
2558 dev_dbg(ppc440spe_chan->device->common.dev,
2559 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2560 ppc440spe_chan->device->id, __func__, src_cnt, len,
2561 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2563 switch (ppc440spe_chan->device->id) {
2564 case PPC440SPE_DMA0_ID:
2565 case PPC440SPE_DMA1_ID:
2566 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2567 dst, dst_cnt, src, src_cnt, scf,
2571 case PPC440SPE_XOR_ID:
2572 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2573 dst, dst_cnt, src, src_cnt, scf,
2578 return sw_desc ? &sw_desc->async_tx : NULL;
2582 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2583 * a PQ_ZERO_SUM operation
2585 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2586 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2587 unsigned int src_cnt, const unsigned char *scf, size_t len,
2588 enum sum_check_flags *pqres, unsigned long flags)
2590 struct ppc440spe_adma_chan *ppc440spe_chan;
2591 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2592 dma_addr_t pdest, qdest;
2593 int slot_cnt, slots_per_op, idst, dst_cnt;
2595 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2597 if (flags & DMA_PREP_PQ_DISABLE_P)
2602 if (flags & DMA_PREP_PQ_DISABLE_Q)
2607 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2608 src, src_cnt, scf));
2610 /* Always use WXOR for P/Q calculations (two destinations).
2611 * Need 1 or 2 extra slots to verify results are zero.
2613 idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2615 /* One additional slot per destination to clone P/Q
2616 * before calculation (we have to preserve destinations).
2618 slot_cnt = src_cnt + dst_cnt * 2;
2621 spin_lock_bh(&ppc440spe_chan->lock);
2622 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2625 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2627 /* Setup byte count for each slot just allocated */
2628 sw_desc->async_tx.flags = flags;
2629 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2630 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2632 iter->unmap_len = len;
2636 struct dma_cdb *hw_desc;
2637 struct ppc440spe_adma_chan *chan;
2639 iter = sw_desc->group_head;
2640 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2641 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2642 iter->hw_next = list_entry(iter->chain_node.next,
2643 struct ppc440spe_adma_desc_slot,
2645 hw_desc = iter->hw_desc;
2646 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2649 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2650 ppc440spe_chan->pdest, 0);
2651 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2652 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2654 iter->unmap_len = 0;
2655 /* override pdest to preserve original P */
2656 pdest = ppc440spe_chan->pdest;
2659 struct dma_cdb *hw_desc;
2660 struct ppc440spe_adma_chan *chan;
2662 iter = list_first_entry(&sw_desc->group_list,
2663 struct ppc440spe_adma_desc_slot,
2665 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2668 iter = list_entry(iter->chain_node.next,
2669 struct ppc440spe_adma_desc_slot,
2673 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2674 iter->hw_next = list_entry(iter->chain_node.next,
2675 struct ppc440spe_adma_desc_slot,
2677 hw_desc = iter->hw_desc;
2678 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2681 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2682 ppc440spe_chan->qdest, 0);
2683 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2684 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2686 iter->unmap_len = 0;
2687 /* override qdest to preserve original Q */
2688 qdest = ppc440spe_chan->qdest;
2691 /* Setup destinations for P/Q ops */
2692 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2694 /* Setup zero QWORDs into DCHECK CDBs */
2696 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2699 * The last CDB corresponds to Q-parity check,
2700 * the one before last CDB corresponds
2703 if (idst == DMA_DEST_MAX_NUM) {
2704 if (idst == dst_cnt) {
2705 set_bit(PPC440SPE_DESC_QCHECK,
2708 set_bit(PPC440SPE_DESC_PCHECK,
2713 set_bit(PPC440SPE_DESC_QCHECK,
2716 set_bit(PPC440SPE_DESC_PCHECK,
2720 iter->xor_check_result = pqres;
2723 * set it to zero, if check fail then result will
2726 *iter->xor_check_result = 0;
2727 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2734 /* Setup sources and mults for P/Q ops */
2735 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2737 struct ppc440spe_adma_chan *chan;
2740 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2741 ppc440spe_desc_set_src_addr(iter, chan, 0,
2745 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2747 ppc440spe_desc_set_src_mult(iter, chan,
2756 spin_unlock_bh(&ppc440spe_chan->lock);
2757 return sw_desc ? &sw_desc->async_tx : NULL;
2761 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2762 * XOR ZERO_SUM operation
2764 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2765 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2766 size_t len, enum sum_check_flags *result, unsigned long flags)
2768 struct dma_async_tx_descriptor *tx;
2771 /* validate P, disable Q */
2774 flags |= DMA_PREP_PQ_DISABLE_Q;
2776 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2777 src_cnt - 1, 0, len,
2783 * ppc440spe_adma_set_dest - set destination address into descriptor
2785 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2786 dma_addr_t addr, int index)
2788 struct ppc440spe_adma_chan *chan;
2790 BUG_ON(index >= sw_desc->dst_cnt);
2792 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2794 switch (chan->device->id) {
2795 case PPC440SPE_DMA0_ID:
2796 case PPC440SPE_DMA1_ID:
2797 /* to do: support transfers lengths >
2798 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2800 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2801 chan, 0, addr, index);
2803 case PPC440SPE_XOR_ID:
2804 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2805 ppc440spe_desc_set_dest_addr(sw_desc,
2806 chan, 0, addr, index);
2811 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2812 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2814 /* To clear destinations update the descriptor
2815 * (P or Q depending on index) as follows:
2816 * addr is destination (0 corresponds to SG2):
2818 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2820 /* ... and the addr is source: */
2821 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2823 /* addr is always SG2 then the mult is always DST1 */
2824 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2825 DMA_CDB_SG_DST1, 1);
2829 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2830 * for the PQXOR operation
2832 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2833 dma_addr_t *addrs, unsigned long flags)
2835 struct ppc440spe_adma_desc_slot *iter;
2836 struct ppc440spe_adma_chan *chan;
2837 dma_addr_t paddr, qaddr;
2838 dma_addr_t addr = 0, ppath, qpath;
2841 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2843 if (flags & DMA_PREP_PQ_DISABLE_P)
2848 if (flags & DMA_PREP_PQ_DISABLE_Q)
2853 if (!paddr || !qaddr)
2854 addr = paddr ? paddr : qaddr;
2856 switch (chan->device->id) {
2857 case PPC440SPE_DMA0_ID:
2858 case PPC440SPE_DMA1_ID:
2859 /* walk through the WXOR source list and set P/Q-destinations
2862 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2863 /* This is WXOR-only chain; may have 1/2 zero descs */
2864 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2866 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2869 iter = ppc440spe_get_group_entry(sw_desc, index);
2871 /* one destination */
2872 list_for_each_entry_from(iter,
2873 &sw_desc->group_list, chain_node)
2874 ppc440spe_desc_set_dest_addr(iter, chan,
2875 DMA_CUED_XOR_BASE, addr, 0);
2877 /* two destinations */
2878 list_for_each_entry_from(iter,
2879 &sw_desc->group_list, chain_node) {
2880 ppc440spe_desc_set_dest_addr(iter, chan,
2881 DMA_CUED_XOR_BASE, paddr, 0);
2882 ppc440spe_desc_set_dest_addr(iter, chan,
2883 DMA_CUED_XOR_BASE, qaddr, 1);
2888 /* To clear destinations update the descriptor
2889 * (1st,2nd, or both depending on flags)
2892 if (test_bit(PPC440SPE_ZERO_P,
2894 iter = ppc440spe_get_group_entry(
2896 ppc440spe_adma_pq_zero_op(iter, chan,
2900 if (test_bit(PPC440SPE_ZERO_Q,
2902 iter = ppc440spe_get_group_entry(
2904 ppc440spe_adma_pq_zero_op(iter, chan,
2911 /* This is RXOR-only or RXOR/WXOR mixed chain */
2913 /* If we want to include destination into calculations,
2914 * then make dest addresses cued with mult=1 (XOR).
2916 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2919 (1 << DMA_CUED_MULT1_OFF);
2920 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2923 (1 << DMA_CUED_MULT1_OFF);
2925 /* Setup destination(s) in RXOR slot(s) */
2926 iter = ppc440spe_get_group_entry(sw_desc, index++);
2927 ppc440spe_desc_set_dest_addr(iter, chan,
2928 paddr ? ppath : qpath,
2929 paddr ? paddr : qaddr, 0);
2931 /* two destinations */
2932 iter = ppc440spe_get_group_entry(sw_desc,
2934 ppc440spe_desc_set_dest_addr(iter, chan,
2938 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2939 /* Setup destination(s) in remaining WXOR
2942 iter = ppc440spe_get_group_entry(sw_desc,
2945 /* one destination */
2946 list_for_each_entry_from(iter,
2947 &sw_desc->group_list,
2949 ppc440spe_desc_set_dest_addr(
2955 /* two destinations */
2956 list_for_each_entry_from(iter,
2957 &sw_desc->group_list,
2959 ppc440spe_desc_set_dest_addr(
2963 ppc440spe_desc_set_dest_addr(
2974 case PPC440SPE_XOR_ID:
2975 /* DMA2 descriptors have only 1 destination, so there are
2976 * two chains - one for each dest.
2977 * If we want to include destination into calculations,
2978 * then make dest addresses cued with mult=1 (XOR).
2980 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2983 (1 << DMA_CUED_MULT1_OFF);
2985 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2988 (1 << DMA_CUED_MULT1_OFF);
2990 iter = ppc440spe_get_group_entry(sw_desc, 0);
2991 for (i = 0; i < sw_desc->descs_per_op; i++) {
2992 ppc440spe_desc_set_dest_addr(iter, chan,
2993 paddr ? ppath : qpath,
2994 paddr ? paddr : qaddr, 0);
2995 iter = list_entry(iter->chain_node.next,
2996 struct ppc440spe_adma_desc_slot,
3001 /* Two destinations; setup Q here */
3002 iter = ppc440spe_get_group_entry(sw_desc,
3003 sw_desc->descs_per_op);
3004 for (i = 0; i < sw_desc->descs_per_op; i++) {
3005 ppc440spe_desc_set_dest_addr(iter,
3006 chan, qpath, qaddr, 0);
3007 iter = list_entry(iter->chain_node.next,
3008 struct ppc440spe_adma_desc_slot,
3018 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3019 * for the PQ_ZERO_SUM operation
3021 static void ppc440spe_adma_pqzero_sum_set_dest(
3022 struct ppc440spe_adma_desc_slot *sw_desc,
3023 dma_addr_t paddr, dma_addr_t qaddr)
3025 struct ppc440spe_adma_desc_slot *iter, *end;
3026 struct ppc440spe_adma_chan *chan;
3027 dma_addr_t addr = 0;
3030 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3032 /* walk through the WXOR source list and set P/Q-destinations
3035 idx = (paddr && qaddr) ? 2 : 1;
3037 list_for_each_entry_reverse(end, &sw_desc->group_list,
3043 idx = (paddr && qaddr) ? 2 : 1;
3044 iter = ppc440spe_get_group_entry(sw_desc, idx);
3046 if (paddr && qaddr) {
3047 /* two destinations */
3048 list_for_each_entry_from(iter, &sw_desc->group_list,
3050 if (unlikely(iter == end))
3052 ppc440spe_desc_set_dest_addr(iter, chan,
3053 DMA_CUED_XOR_BASE, paddr, 0);
3054 ppc440spe_desc_set_dest_addr(iter, chan,
3055 DMA_CUED_XOR_BASE, qaddr, 1);
3058 /* one destination */
3059 addr = paddr ? paddr : qaddr;
3060 list_for_each_entry_from(iter, &sw_desc->group_list,
3062 if (unlikely(iter == end))
3064 ppc440spe_desc_set_dest_addr(iter, chan,
3065 DMA_CUED_XOR_BASE, addr, 0);
3069 /* The remaining descriptors are DATACHECK. These have no need in
3070 * destination. Actually, these destinations are used there
3071 * as sources for check operation. So, set addr as source.
3073 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3076 end = list_entry(end->chain_node.next,
3077 struct ppc440spe_adma_desc_slot, chain_node);
3078 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3083 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3085 static inline void ppc440spe_desc_set_xor_src_cnt(
3086 struct ppc440spe_adma_desc_slot *desc,
3089 struct xor_cb *hw_desc = desc->hw_desc;
3091 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3092 hw_desc->cbc |= src_cnt;
3096 * ppc440spe_adma_pq_set_src - set source address into descriptor
3098 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3099 dma_addr_t addr, int index)
3101 struct ppc440spe_adma_chan *chan;
3102 dma_addr_t haddr = 0;
3103 struct ppc440spe_adma_desc_slot *iter = NULL;
3105 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3107 switch (chan->device->id) {
3108 case PPC440SPE_DMA0_ID:
3109 case PPC440SPE_DMA1_ID:
3110 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3112 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3113 /* RXOR-only or RXOR/WXOR operation */
3114 int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3115 &sw_desc->flags) ? 2 : 3;
3118 /* 1st slot (RXOR) */
3119 /* setup sources region (R1-2-3, R1-2-4,
3122 if (test_bit(PPC440SPE_DESC_RXOR12,
3124 haddr = DMA_RXOR12 <<
3125 DMA_CUED_REGION_OFF;
3126 else if (test_bit(PPC440SPE_DESC_RXOR123,
3128 haddr = DMA_RXOR123 <<
3129 DMA_CUED_REGION_OFF;
3130 else if (test_bit(PPC440SPE_DESC_RXOR124,
3132 haddr = DMA_RXOR124 <<
3133 DMA_CUED_REGION_OFF;
3134 else if (test_bit(PPC440SPE_DESC_RXOR125,
3136 haddr = DMA_RXOR125 <<
3137 DMA_CUED_REGION_OFF;
3140 haddr |= DMA_CUED_XOR_BASE;
3141 iter = ppc440spe_get_group_entry(sw_desc, 0);
3142 } else if (index < iskip) {
3144 * shall actually set source address only once
3145 * instead of first <iskip>
3149 /* 2nd/3d and next slots (WXOR);
3150 * skip first slot with RXOR
3152 haddr = DMA_CUED_XOR_HB;
3153 iter = ppc440spe_get_group_entry(sw_desc,
3154 index - iskip + sw_desc->dst_cnt);
3159 /* WXOR-only operation; skip first slots with
3160 * zeroing destinations
3162 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3164 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3167 haddr = DMA_CUED_XOR_HB;
3168 iter = ppc440spe_get_group_entry(sw_desc,
3173 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3176 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3177 sw_desc->dst_cnt == 2) {
3178 /* if we have two destinations for RXOR, then
3179 * setup source in the second descr too
3181 iter = ppc440spe_get_group_entry(sw_desc, 1);
3182 ppc440spe_desc_set_src_addr(iter, chan, 0,
3188 case PPC440SPE_XOR_ID:
3189 /* DMA2 may do Biskup */
3190 iter = sw_desc->group_head;
3191 if (iter->dst_cnt == 2) {
3192 /* both P & Q calculations required; set P src here */
3193 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3196 iter = ppc440spe_get_group_entry(sw_desc,
3197 sw_desc->descs_per_op);
3199 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3205 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3207 static void ppc440spe_adma_memcpy_xor_set_src(
3208 struct ppc440spe_adma_desc_slot *sw_desc,
3209 dma_addr_t addr, int index)
3211 struct ppc440spe_adma_chan *chan;
3213 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3214 sw_desc = sw_desc->group_head;
3216 if (likely(sw_desc))
3217 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3221 * ppc440spe_adma_dma2rxor_inc_addr -
3223 static void ppc440spe_adma_dma2rxor_inc_addr(
3224 struct ppc440spe_adma_desc_slot *desc,
3225 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3227 cursor->addr_count++;
3228 if (index == src_cnt - 1) {
3229 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230 } else if (cursor->addr_count == XOR_MAX_OPS) {
3231 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3232 cursor->addr_count = 0;
3233 cursor->desc_count++;
3238 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3240 static int ppc440spe_adma_dma2rxor_prep_src(
3241 struct ppc440spe_adma_desc_slot *hdesc,
3242 struct ppc440spe_rxor *cursor, int index,
3243 int src_cnt, u32 addr)
3247 struct ppc440spe_adma_desc_slot *desc = hdesc;
3250 for (i = 0; i < cursor->desc_count; i++) {
3251 desc = list_entry(hdesc->chain_node.next,
3252 struct ppc440spe_adma_desc_slot,
3256 switch (cursor->state) {
3258 if (addr == cursor->addrl + cursor->len) {
3261 cursor->xor_count++;
3262 if (index == src_cnt-1) {
3263 ppc440spe_rxor_set_region(desc,
3265 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3266 ppc440spe_adma_dma2rxor_inc_addr(
3267 desc, cursor, index, src_cnt);
3269 } else if (cursor->addrl == addr + cursor->len) {
3272 cursor->xor_count++;
3273 set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3274 if (index == src_cnt-1) {
3275 ppc440spe_rxor_set_region(desc,
3277 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3278 ppc440spe_adma_dma2rxor_inc_addr(
3279 desc, cursor, index, src_cnt);
3282 printk(KERN_ERR "Cannot build "
3283 "DMA2 RXOR command block.\n");
3288 sign = test_bit(cursor->addr_count,
3289 desc->reverse_flags)
3291 if (index == src_cnt-2 || (sign == -1
3292 && addr != cursor->addrl - 2*cursor->len)) {
3294 cursor->xor_count = 1;
3295 cursor->addrl = addr;
3296 ppc440spe_rxor_set_region(desc,
3298 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3299 ppc440spe_adma_dma2rxor_inc_addr(
3300 desc, cursor, index, src_cnt);
3301 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3303 cursor->xor_count = 0;
3304 ppc440spe_rxor_set_region(desc,
3306 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3307 if (index == src_cnt-1) {
3308 ppc440spe_adma_dma2rxor_inc_addr(
3309 desc, cursor, index, src_cnt);
3311 } else if (addr == cursor->addrl + 3*cursor->len) {
3313 cursor->xor_count = 0;
3314 ppc440spe_rxor_set_region(desc,
3316 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3317 if (index == src_cnt-1) {
3318 ppc440spe_adma_dma2rxor_inc_addr(
3319 desc, cursor, index, src_cnt);
3321 } else if (addr == cursor->addrl + 4*cursor->len) {
3323 cursor->xor_count = 0;
3324 ppc440spe_rxor_set_region(desc,
3326 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3327 if (index == src_cnt-1) {
3328 ppc440spe_adma_dma2rxor_inc_addr(
3329 desc, cursor, index, src_cnt);
3333 cursor->xor_count = 1;
3334 cursor->addrl = addr;
3335 ppc440spe_rxor_set_region(desc,
3337 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3338 ppc440spe_adma_dma2rxor_inc_addr(
3339 desc, cursor, index, src_cnt);
3344 cursor->addrl = addr;
3345 cursor->xor_count++;
3347 ppc440spe_adma_dma2rxor_inc_addr(
3348 desc, cursor, index, src_cnt);
3357 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3358 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3360 static void ppc440spe_adma_dma2rxor_set_src(
3361 struct ppc440spe_adma_desc_slot *desc,
3362 int index, dma_addr_t addr)
3364 struct xor_cb *xcb = desc->hw_desc;
3365 int k = 0, op = 0, lop = 0;
3367 /* get the RXOR operand which corresponds to index addr */
3368 while (op <= index) {
3370 if (k == XOR_MAX_OPS) {
3372 desc = list_entry(desc->chain_node.next,
3373 struct ppc440spe_adma_desc_slot, chain_node);
3374 xcb = desc->hw_desc;
3377 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3378 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3386 if (test_bit(k-1, desc->reverse_flags)) {
3387 /* reverse operand order; put last op in RXOR group */
3388 if (index == op - 1)
3389 ppc440spe_rxor_set_src(desc, k - 1, addr);
3391 /* direct operand order; put first op in RXOR group */
3393 ppc440spe_rxor_set_src(desc, k - 1, addr);
3398 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3399 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3401 static void ppc440spe_adma_dma2rxor_set_mult(
3402 struct ppc440spe_adma_desc_slot *desc,
3405 struct xor_cb *xcb = desc->hw_desc;
3406 int k = 0, op = 0, lop = 0;
3408 /* get the RXOR operand which corresponds to index mult */
3409 while (op <= index) {
3411 if (k == XOR_MAX_OPS) {
3413 desc = list_entry(desc->chain_node.next,
3414 struct ppc440spe_adma_desc_slot,
3416 xcb = desc->hw_desc;
3419 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3420 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3427 if (test_bit(k-1, desc->reverse_flags)) {
3429 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3432 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3437 * ppc440spe_init_rxor_cursor -
3439 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3441 memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3446 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3447 * descriptor for the PQXOR operation
3449 static void ppc440spe_adma_pq_set_src_mult(
3450 struct ppc440spe_adma_desc_slot *sw_desc,
3451 unsigned char mult, int index, int dst_pos)
3453 struct ppc440spe_adma_chan *chan;
3454 u32 mult_idx, mult_dst;
3455 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3457 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3459 switch (chan->device->id) {
3460 case PPC440SPE_DMA0_ID:
3461 case PPC440SPE_DMA1_ID:
3462 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3463 int region = test_bit(PPC440SPE_DESC_RXOR12,
3464 &sw_desc->flags) ? 2 : 3;
3466 if (index < region) {
3467 /* RXOR multipliers */
3468 iter = ppc440spe_get_group_entry(sw_desc,
3469 sw_desc->dst_cnt - 1);
3470 if (sw_desc->dst_cnt == 2)
3471 iter1 = ppc440spe_get_group_entry(
3474 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3475 mult_dst = DMA_CDB_SG_SRC;
3477 /* WXOR multiplier */
3478 iter = ppc440spe_get_group_entry(sw_desc,
3481 mult_idx = DMA_CUED_MULT1_OFF;
3482 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3489 * skip first slots with destinations (if ZERO_DST has
3492 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3494 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3497 iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3498 mult_idx = DMA_CUED_MULT1_OFF;
3499 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3503 ppc440spe_desc_set_src_mult(iter, chan,
3504 mult_idx, mult_dst, mult);
3506 if (unlikely(iter1)) {
3507 /* if we have two destinations for RXOR, then
3508 * we've just set Q mult. Set-up P now.
3510 ppc440spe_desc_set_src_mult(iter1, chan,
3511 mult_idx, mult_dst, 1);
3517 case PPC440SPE_XOR_ID:
3518 iter = sw_desc->group_head;
3519 if (sw_desc->dst_cnt == 2) {
3520 /* both P & Q calculations required; set P mult here */
3521 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3523 /* and then set Q mult */
3524 iter = ppc440spe_get_group_entry(sw_desc,
3525 sw_desc->descs_per_op);
3527 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3533 * ppc440spe_adma_free_chan_resources - free the resources allocated
3535 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3537 struct ppc440spe_adma_chan *ppc440spe_chan;
3538 struct ppc440spe_adma_desc_slot *iter, *_iter;
3539 int in_use_descs = 0;
3541 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3542 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3544 spin_lock_bh(&ppc440spe_chan->lock);
3545 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3548 list_del(&iter->chain_node);
3550 list_for_each_entry_safe_reverse(iter, _iter,
3551 &ppc440spe_chan->all_slots, slot_node) {
3552 list_del(&iter->slot_node);
3554 ppc440spe_chan->slots_allocated--;
3556 ppc440spe_chan->last_used = NULL;
3558 dev_dbg(ppc440spe_chan->device->common.dev,
3559 "ppc440spe adma%d %s slots_allocated %d\n",
3560 ppc440spe_chan->device->id,
3561 __func__, ppc440spe_chan->slots_allocated);
3562 spin_unlock_bh(&ppc440spe_chan->lock);
3564 /* one is ok since we left it on there on purpose */
3565 if (in_use_descs > 1)
3566 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3571 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3572 * @chan: ADMA channel handle
3573 * @cookie: ADMA transaction identifier
3574 * @txstate: a holder for the current state of the channel
3576 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3577 dma_cookie_t cookie, struct dma_tx_state *txstate)
3579 struct ppc440spe_adma_chan *ppc440spe_chan;
3580 enum dma_status ret;
3582 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3583 ret = dma_cookie_status(chan, cookie, txstate);
3584 if (ret == DMA_COMPLETE)
3587 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3589 return dma_cookie_status(chan, cookie, txstate);
3593 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3595 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3597 struct ppc440spe_adma_chan *chan = data;
3599 dev_dbg(chan->device->common.dev,
3600 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3602 tasklet_schedule(&chan->irq_tasklet);
3603 ppc440spe_adma_device_clear_eot_status(chan);
3609 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3610 * do the same things as a eot handler
3612 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3614 struct ppc440spe_adma_chan *chan = data;
3616 dev_dbg(chan->device->common.dev,
3617 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3619 tasklet_schedule(&chan->irq_tasklet);
3620 ppc440spe_adma_device_clear_eot_status(chan);
3626 * ppc440spe_test_callback - called when test operation has been done
3628 static void ppc440spe_test_callback(void *unused)
3630 complete(&ppc440spe_r6_test_comp);
3634 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3636 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3638 struct ppc440spe_adma_chan *ppc440spe_chan;
3640 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3641 dev_dbg(ppc440spe_chan->device->common.dev,
3642 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3643 __func__, ppc440spe_chan->pending);
3645 if (ppc440spe_chan->pending) {
3646 ppc440spe_chan->pending = 0;
3647 ppc440spe_chan_append(ppc440spe_chan);
3652 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3653 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
3654 * specific operation)
3656 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3658 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3659 dma_cookie_t cookie;
3660 int slot_cnt, slots_per_op;
3662 dev_dbg(chan->device->common.dev,
3663 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3665 spin_lock_bh(&chan->lock);
3666 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3667 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3669 group_start = sw_desc->group_head;
3670 list_splice_init(&sw_desc->group_list, &chan->chain);
3671 async_tx_ack(&sw_desc->async_tx);
3672 ppc440spe_desc_init_null_xor(group_start);
3674 cookie = dma_cookie_assign(&sw_desc->async_tx);
3676 /* initialize the completed cookie to be less than
3677 * the most recently used cookie
3679 chan->common.completed_cookie = cookie - 1;
3681 /* channel should not be busy */
3682 BUG_ON(ppc440spe_chan_is_busy(chan));
3684 /* set the descriptor address */
3685 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3687 /* run the descriptor */
3688 ppc440spe_chan_run(chan);
3690 printk(KERN_ERR "ppc440spe adma%d"
3691 " failed to allocate null descriptor\n",
3693 spin_unlock_bh(&chan->lock);
3697 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3698 * For this we just perform one WXOR operation with the same source
3699 * and destination addresses, the GF-multiplier is 1; so if RAID-6
3700 * capabilities are enabled then we'll get src/dst filled with zero.
3702 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3704 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3707 dma_addr_t dma_addr, addrs[2];
3708 unsigned long op = 0;
3711 set_bit(PPC440SPE_DESC_WXOR, &op);
3713 pg = alloc_page(GFP_KERNEL);
3717 spin_lock_bh(&chan->lock);
3718 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3720 /* 1 src, 1 dsr, int_ena, WXOR */
3721 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3722 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3723 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3724 iter->unmap_len = PAGE_SIZE;
3728 spin_unlock_bh(&chan->lock);
3731 spin_unlock_bh(&chan->lock);
3733 /* Fill the test page with ones */
3734 memset(page_address(pg), 0xFF, PAGE_SIZE);
3735 dma_addr = dma_map_page(chan->device->dev, pg, 0,
3736 PAGE_SIZE, DMA_BIDIRECTIONAL);
3738 /* Setup addresses */
3739 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3740 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3741 addrs[0] = dma_addr;
3743 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3745 async_tx_ack(&sw_desc->async_tx);
3746 sw_desc->async_tx.callback = ppc440spe_test_callback;
3747 sw_desc->async_tx.callback_param = NULL;
3749 init_completion(&ppc440spe_r6_test_comp);
3751 ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3752 ppc440spe_adma_issue_pending(&chan->common);
3754 wait_for_completion(&ppc440spe_r6_test_comp);
3756 /* Now check if the test page is zeroed */
3757 a = page_address(pg);
3758 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3759 /* page is zero - RAID-6 enabled */
3762 /* RAID-6 was not enabled */
3770 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3773 case PPC440SPE_DMA0_ID:
3774 case PPC440SPE_DMA1_ID:
3775 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3776 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3777 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3778 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3779 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3781 case PPC440SPE_XOR_ID:
3782 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3783 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3784 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3785 adev->common.cap_mask = adev->common.cap_mask;
3789 /* Set base routines */
3790 adev->common.device_alloc_chan_resources =
3791 ppc440spe_adma_alloc_chan_resources;
3792 adev->common.device_free_chan_resources =
3793 ppc440spe_adma_free_chan_resources;
3794 adev->common.device_tx_status = ppc440spe_adma_tx_status;
3795 adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3797 /* Set prep routines based on capability */
3798 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3799 adev->common.device_prep_dma_memcpy =
3800 ppc440spe_adma_prep_dma_memcpy;
3802 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3803 adev->common.max_xor = XOR_MAX_OPS;
3804 adev->common.device_prep_dma_xor =
3805 ppc440spe_adma_prep_dma_xor;
3807 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3809 case PPC440SPE_DMA0_ID:
3810 dma_set_maxpq(&adev->common,
3811 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3813 case PPC440SPE_DMA1_ID:
3814 dma_set_maxpq(&adev->common,
3815 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3817 case PPC440SPE_XOR_ID:
3818 adev->common.max_pq = XOR_MAX_OPS * 3;
3821 adev->common.device_prep_dma_pq =
3822 ppc440spe_adma_prep_dma_pq;
3824 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3826 case PPC440SPE_DMA0_ID:
3827 adev->common.max_pq = DMA0_FIFO_SIZE /
3828 sizeof(struct dma_cdb);
3830 case PPC440SPE_DMA1_ID:
3831 adev->common.max_pq = DMA1_FIFO_SIZE /
3832 sizeof(struct dma_cdb);
3835 adev->common.device_prep_dma_pq_val =
3836 ppc440spe_adma_prep_dma_pqzero_sum;
3838 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3840 case PPC440SPE_DMA0_ID:
3841 adev->common.max_xor = DMA0_FIFO_SIZE /
3842 sizeof(struct dma_cdb);
3844 case PPC440SPE_DMA1_ID:
3845 adev->common.max_xor = DMA1_FIFO_SIZE /
3846 sizeof(struct dma_cdb);
3849 adev->common.device_prep_dma_xor_val =
3850 ppc440spe_adma_prep_dma_xor_zero_sum;
3852 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3853 adev->common.device_prep_dma_interrupt =
3854 ppc440spe_adma_prep_dma_interrupt;
3856 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3857 "( %s%s%s%s%s%s)\n",
3858 dev_name(adev->dev),
3859 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3860 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3861 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3862 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3863 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3864 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3867 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3868 struct ppc440spe_adma_chan *chan,
3871 struct platform_device *ofdev;
3872 struct device_node *np;
3875 ofdev = container_of(adev->dev, struct platform_device, dev);
3876 np = ofdev->dev.of_node;
3877 if (adev->id != PPC440SPE_XOR_ID) {
3878 adev->err_irq = irq_of_parse_and_map(np, 1);
3879 if (!adev->err_irq) {
3880 dev_warn(adev->dev, "no err irq resource?\n");
3881 *initcode = PPC_ADMA_INIT_IRQ2;
3882 adev->err_irq = -ENXIO;
3884 atomic_inc(&ppc440spe_adma_err_irq_ref);
3886 adev->err_irq = -ENXIO;
3889 adev->irq = irq_of_parse_and_map(np, 0);
3891 dev_err(adev->dev, "no irq resource\n");
3892 *initcode = PPC_ADMA_INIT_IRQ1;
3896 dev_dbg(adev->dev, "irq %d, err irq %d\n",
3897 adev->irq, adev->err_irq);
3899 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3900 0, dev_driver_string(adev->dev), chan);
3902 dev_err(adev->dev, "can't request irq %d\n",
3904 *initcode = PPC_ADMA_INIT_IRQ1;
3909 /* only DMA engines have a separate error IRQ
3910 * so it's Ok if err_irq < 0 in XOR engine case.
3912 if (adev->err_irq > 0) {
3913 /* both DMA engines share common error IRQ */
3914 ret = request_irq(adev->err_irq,
3915 ppc440spe_adma_err_handler,
3917 dev_driver_string(adev->dev),
3920 dev_err(adev->dev, "can't request irq %d\n",
3922 *initcode = PPC_ADMA_INIT_IRQ2;
3928 if (adev->id == PPC440SPE_XOR_ID) {
3929 /* enable XOR engine interrupts */
3930 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3931 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3932 &adev->xor_reg->ier);
3936 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3938 pr_err("%s: can't find I2O device tree node\n",
3943 adev->i2o_reg = of_iomap(np, 0);
3944 if (!adev->i2o_reg) {
3945 pr_err("%s: failed to map I2O registers\n", __func__);
3951 /* Unmask 'CS FIFO Attention' interrupts and
3952 * enable generating interrupts on errors
3954 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3955 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3956 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3957 mask = ioread32(&adev->i2o_reg->iopim) & enable;
3958 iowrite32(mask, &adev->i2o_reg->iopim);
3963 free_irq(adev->irq, chan);
3965 irq_dispose_mapping(adev->irq);
3967 if (adev->err_irq > 0) {
3968 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3969 irq_dispose_mapping(adev->err_irq);
3974 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3975 struct ppc440spe_adma_chan *chan)
3979 if (adev->id == PPC440SPE_XOR_ID) {
3980 /* disable XOR engine interrupts */
3981 mask = ioread32be(&adev->xor_reg->ier);
3982 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3983 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3984 iowrite32be(mask, &adev->xor_reg->ier);
3986 /* disable DMAx engine interrupts */
3987 disable = (adev->id == PPC440SPE_DMA0_ID) ?
3988 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3989 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3990 mask = ioread32(&adev->i2o_reg->iopim) | disable;
3991 iowrite32(mask, &adev->i2o_reg->iopim);
3993 free_irq(adev->irq, chan);
3994 irq_dispose_mapping(adev->irq);
3995 if (adev->err_irq > 0) {
3996 free_irq(adev->err_irq, chan);
3997 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3998 irq_dispose_mapping(adev->err_irq);
3999 iounmap(adev->i2o_reg);
4005 * ppc440spe_adma_probe - probe the asynch device
4007 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4009 struct device_node *np = ofdev->dev.of_node;
4010 struct resource res;
4011 struct ppc440spe_adma_device *adev;
4012 struct ppc440spe_adma_chan *chan;
4013 struct ppc_dma_chan_ref *ref, *_ref;
4014 int ret = 0, initcode = PPC_ADMA_INIT_OK;
4020 if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4021 id = PPC440SPE_XOR_ID;
4022 /* As far as the XOR engine is concerned, it does not
4023 * use FIFOs but uses linked list. So there is no dependency
4024 * between pool size to allocate and the engine configuration.
4026 pool_size = PAGE_SIZE << 1;
4028 /* it is DMA0 or DMA1 */
4029 idx = of_get_property(np, "cell-index", &len);
4030 if (!idx || (len != sizeof(u32))) {
4031 dev_err(&ofdev->dev, "Device node %pOF has missing "
4032 "or invalid cell-index property\n",
4037 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4038 * should allocate the pool accordingly to size of this
4039 * FIFO. Thus, the pool size depends on the FIFO depth:
4040 * how much CDBs pointers the FIFO may contain then so
4041 * much CDBs we should provide in the pool.
4044 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4045 * Pool size = CDBs number * CDB size =
4046 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4048 pool_size = (id == PPC440SPE_DMA0_ID) ?
4049 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4053 if (of_address_to_resource(np, 0, &res)) {
4054 dev_err(&ofdev->dev, "failed to get memory resource\n");
4055 initcode = PPC_ADMA_INIT_MEMRES;
4060 if (!request_mem_region(res.start, resource_size(&res),
4061 dev_driver_string(&ofdev->dev))) {
4062 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4064 initcode = PPC_ADMA_INIT_MEMREG;
4069 /* create a device */
4070 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4072 initcode = PPC_ADMA_INIT_ALLOC;
4074 goto err_adev_alloc;
4078 adev->pool_size = pool_size;
4079 /* allocate coherent memory for hardware descriptors */
4080 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4081 adev->pool_size, &adev->dma_desc_pool,
4083 if (adev->dma_desc_pool_virt == NULL) {
4084 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4085 "memory for hardware descriptors\n",
4087 initcode = PPC_ADMA_INIT_COHERENT;
4091 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4092 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4094 regs = ioremap(res.start, resource_size(&res));
4096 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4098 goto err_regs_alloc;
4101 if (adev->id == PPC440SPE_XOR_ID) {
4102 adev->xor_reg = regs;
4104 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4105 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4107 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4108 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4109 adev->dma_reg = regs;
4110 /* DMAx_FIFO_SIZE is defined in bytes,
4111 * <fsiz> - is defined in number of CDB pointers (8byte).
4112 * DMA FIFO Length = CSlength + CPlength, where
4113 * CSlength = CPlength = (fsiz + 1) * 8.
4115 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4116 &adev->dma_reg->fsiz);
4117 /* Configure DMA engine */
4118 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4119 &adev->dma_reg->cfg);
4121 iowrite32(~0, &adev->dma_reg->dsts);
4124 adev->dev = &ofdev->dev;
4125 adev->common.dev = &ofdev->dev;
4126 INIT_LIST_HEAD(&adev->common.channels);
4127 platform_set_drvdata(ofdev, adev);
4129 /* create a channel */
4130 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4132 initcode = PPC_ADMA_INIT_CHANNEL;
4134 goto err_chan_alloc;
4137 spin_lock_init(&chan->lock);
4138 INIT_LIST_HEAD(&chan->chain);
4139 INIT_LIST_HEAD(&chan->all_slots);
4140 chan->device = adev;
4141 chan->common.device = &adev->common;
4142 dma_cookie_init(&chan->common);
4143 list_add_tail(&chan->common.device_node, &adev->common.channels);
4144 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4145 (unsigned long)chan);
4147 /* allocate and map helper pages for async validation or
4148 * async_mult/async_sum_product operations on DMA0/1.
4150 if (adev->id != PPC440SPE_XOR_ID) {
4151 chan->pdest_page = alloc_page(GFP_KERNEL);
4152 chan->qdest_page = alloc_page(GFP_KERNEL);
4153 if (!chan->pdest_page ||
4154 !chan->qdest_page) {
4155 if (chan->pdest_page)
4156 __free_page(chan->pdest_page);
4157 if (chan->qdest_page)
4158 __free_page(chan->qdest_page);
4160 goto err_page_alloc;
4162 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4163 PAGE_SIZE, DMA_BIDIRECTIONAL);
4164 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4165 PAGE_SIZE, DMA_BIDIRECTIONAL);
4168 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4170 ref->chan = &chan->common;
4171 INIT_LIST_HEAD(&ref->node);
4172 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4174 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4179 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4183 ppc440spe_adma_init_capabilities(adev);
4185 ret = dma_async_device_register(&adev->common);
4187 initcode = PPC_ADMA_INIT_REGISTER;
4188 dev_err(&ofdev->dev, "failed to register dma device\n");
4195 ppc440spe_adma_release_irqs(adev, chan);
4197 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4198 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4199 list_del(&ref->node);
4204 if (adev->id != PPC440SPE_XOR_ID) {
4205 dma_unmap_page(&ofdev->dev, chan->pdest,
4206 PAGE_SIZE, DMA_BIDIRECTIONAL);
4207 dma_unmap_page(&ofdev->dev, chan->qdest,
4208 PAGE_SIZE, DMA_BIDIRECTIONAL);
4209 __free_page(chan->pdest_page);
4210 __free_page(chan->qdest_page);
4215 if (adev->id == PPC440SPE_XOR_ID)
4216 iounmap(adev->xor_reg);
4218 iounmap(adev->dma_reg);
4220 dma_free_coherent(adev->dev, adev->pool_size,
4221 adev->dma_desc_pool_virt,
4222 adev->dma_desc_pool);
4226 release_mem_region(res.start, resource_size(&res));
4228 if (id < PPC440SPE_ADMA_ENGINES_NUM)
4229 ppc440spe_adma_devices[id] = initcode;
4235 * ppc440spe_adma_remove - remove the asynch device
4237 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4239 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4240 struct device_node *np = ofdev->dev.of_node;
4241 struct resource res;
4242 struct dma_chan *chan, *_chan;
4243 struct ppc_dma_chan_ref *ref, *_ref;
4244 struct ppc440spe_adma_chan *ppc440spe_chan;
4246 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4247 ppc440spe_adma_devices[adev->id] = -1;
4249 dma_async_device_unregister(&adev->common);
4251 list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4253 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4254 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4255 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4256 if (adev->id != PPC440SPE_XOR_ID) {
4257 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4258 PAGE_SIZE, DMA_BIDIRECTIONAL);
4259 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4260 PAGE_SIZE, DMA_BIDIRECTIONAL);
4261 __free_page(ppc440spe_chan->pdest_page);
4262 __free_page(ppc440spe_chan->qdest_page);
4264 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4266 if (ppc440spe_chan ==
4267 to_ppc440spe_adma_chan(ref->chan)) {
4268 list_del(&ref->node);
4272 list_del(&chan->device_node);
4273 kfree(ppc440spe_chan);
4276 dma_free_coherent(adev->dev, adev->pool_size,
4277 adev->dma_desc_pool_virt, adev->dma_desc_pool);
4278 if (adev->id == PPC440SPE_XOR_ID)
4279 iounmap(adev->xor_reg);
4281 iounmap(adev->dma_reg);
4282 of_address_to_resource(np, 0, &res);
4283 release_mem_region(res.start, resource_size(&res));
4289 * /sys driver interface to enable h/w RAID-6 capabilities
4290 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4291 * directory are "devices", "enable" and "poly".
4292 * "devices" shows available engines.
4293 * "enable" is used to enable RAID-6 capabilities or to check
4294 * whether these has been activated.
4295 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4298 static ssize_t devices_show(struct device_driver *dev, char *buf)
4303 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4304 if (ppc440spe_adma_devices[i] == -1)
4306 size += snprintf(buf + size, PAGE_SIZE - size,
4307 "PPC440SP(E)-ADMA.%d: %s\n", i,
4308 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4312 static DRIVER_ATTR_RO(devices);
4314 static ssize_t enable_show(struct device_driver *dev, char *buf)
4316 return snprintf(buf, PAGE_SIZE,
4317 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4318 ppc440spe_r6_enabled ? "EN" : "DIS");
4321 static ssize_t enable_store(struct device_driver *dev, const char *buf,
4326 if (!count || count > 11)
4329 if (!ppc440spe_r6_tchan)
4333 sscanf(buf, "%lx", &val);
4334 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4337 /* Verify whether it really works now */
4338 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4339 pr_info("PPC440SP(e) RAID-6 has been activated "
4341 ppc440spe_r6_enabled = 1;
4343 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4345 ppc440spe_r6_enabled = 0;
4349 static DRIVER_ATTR_RW(enable);
4351 static ssize_t poly_show(struct device_driver *dev, char *buf)
4357 /* 440SP has fixed polynomial */
4360 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4361 reg >>= MQ0_CFBHL_POLY;
4365 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4366 "uses 0x1%02x polynomial.\n", reg);
4370 static ssize_t poly_store(struct device_driver *dev, const char *buf,
4373 unsigned long reg, val;
4376 /* 440SP uses default 0x14D polynomial only */
4380 if (!count || count > 6)
4383 /* e.g., 0x14D or 0x11D */
4384 sscanf(buf, "%lx", &val);
4390 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4391 reg &= ~(0xFF << MQ0_CFBHL_POLY);
4392 reg |= val << MQ0_CFBHL_POLY;
4393 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4397 static DRIVER_ATTR_RW(poly);
4400 * Common initialisation for RAID engines; allocate memory for
4401 * DMAx FIFOs, perform configuration common for all DMA engines.
4402 * Further DMA engine specific configuration is done at probe time.
4404 static int ppc440spe_configure_raid_devices(void)
4406 struct device_node *np;
4407 struct resource i2o_res;
4408 struct i2o_regs __iomem *i2o_reg;
4409 dcr_host_t i2o_dcr_host;
4410 unsigned int dcr_base, dcr_len;
4413 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4415 pr_err("%s: can't find I2O device tree node\n",
4420 if (of_address_to_resource(np, 0, &i2o_res)) {
4425 i2o_reg = of_iomap(np, 0);
4427 pr_err("%s: failed to map I2O registers\n", __func__);
4432 /* Get I2O DCRs base */
4433 dcr_base = dcr_resource_start(np, 0);
4434 dcr_len = dcr_resource_len(np, 0);
4435 if (!dcr_base && !dcr_len) {
4436 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4442 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4443 if (!DCR_MAP_OK(i2o_dcr_host)) {
4444 pr_err("%pOF: failed to map DCRs!\n", np);
4451 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4452 * the base address of FIFO memory space.
4453 * Actually we need twice more physical memory than programmed in the
4454 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4456 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4458 if (!ppc440spe_dma_fifo_buf) {
4459 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4461 dcr_unmap(i2o_dcr_host, dcr_len);
4469 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4470 mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4472 /* Setup the base address of mmaped registers */
4473 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4474 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4476 dcr_unmap(i2o_dcr_host, dcr_len);
4478 /* Setup FIFO memory space base address */
4479 iowrite32(0, &i2o_reg->ifbah);
4480 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4482 /* set zero FIFO size for I2O, so the whole
4483 * ppc440spe_dma_fifo_buf is used by DMAs.
4484 * DMAx_FIFOs will be configured while probe.
4486 iowrite32(0, &i2o_reg->ifsiz);
4489 /* To prepare WXOR/RXOR functionality we need access to
4490 * Memory Queue Module DCRs (finally it will be enabled
4491 * via /sys interface of the ppc440spe ADMA driver).
4493 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4495 pr_err("%s: can't find MQ device tree node\n",
4501 /* Get MQ DCRs base */
4502 dcr_base = dcr_resource_start(np, 0);
4503 dcr_len = dcr_resource_len(np, 0);
4504 if (!dcr_base && !dcr_len) {
4505 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4510 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4511 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4512 pr_err("%pOF: failed to map DCRs!\n", np);
4517 ppc440spe_mq_dcr_len = dcr_len;
4520 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4523 * - LL transaction passing limit to 1;
4524 * - Memory controller cycle limit to 1;
4525 * - Galois Polynomial to 0x14d (default)
4527 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4528 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4529 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4531 atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4532 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4533 ppc440spe_adma_devices[i] = -1;
4540 kfree(ppc440spe_dma_fifo_buf);
4544 static const struct of_device_id ppc440spe_adma_of_match[] = {
4545 { .compatible = "ibm,dma-440spe", },
4546 { .compatible = "amcc,xor-accelerator", },
4549 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4551 static struct platform_driver ppc440spe_adma_driver = {
4552 .probe = ppc440spe_adma_probe,
4553 .remove = ppc440spe_adma_remove,
4555 .name = "PPC440SP(E)-ADMA",
4556 .of_match_table = ppc440spe_adma_of_match,
4560 static __init int ppc440spe_adma_init(void)
4564 ret = ppc440spe_configure_raid_devices();
4568 ret = platform_driver_register(&ppc440spe_adma_driver);
4570 pr_err("%s: failed to register platform driver\n",
4575 /* Initialization status */
4576 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4577 &driver_attr_devices);
4581 /* RAID-6 h/w enable entry */
4582 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4583 &driver_attr_enable);
4587 /* GF polynomial to use */
4588 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4593 driver_remove_file(&ppc440spe_adma_driver.driver,
4594 &driver_attr_enable);
4596 driver_remove_file(&ppc440spe_adma_driver.driver,
4597 &driver_attr_devices);
4599 /* User will not be able to enable h/w RAID-6 */
4600 pr_err("%s: failed to create RAID-6 driver interface\n",
4602 platform_driver_unregister(&ppc440spe_adma_driver);
4604 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4605 kfree(ppc440spe_dma_fifo_buf);
4609 static void __exit ppc440spe_adma_exit(void)
4611 driver_remove_file(&ppc440spe_adma_driver.driver,
4613 driver_remove_file(&ppc440spe_adma_driver.driver,
4614 &driver_attr_enable);
4615 driver_remove_file(&ppc440spe_adma_driver.driver,
4616 &driver_attr_devices);
4617 platform_driver_unregister(&ppc440spe_adma_driver);
4618 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4619 kfree(ppc440spe_dma_fifo_buf);
4622 arch_initcall(ppc440spe_adma_init);
4623 module_exit(ppc440spe_adma_exit);
4625 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4626 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4627 MODULE_LICENSE("GPL");