2 * Copyright (C) 2006-2009 DENX Software Engineering.
4 * Author: Yuri Tikhonov <yur@emcraft.com>
6 * Further porting to arch/powerpc by
7 * Anatolij Gustschin <agust@denx.de>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option)
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23 * The full GNU General Public License is included in this distribution in the
24 * file called COPYING.
28 * This driver supports the asynchrounous DMA copy and RAID engines available
29 * on the AMCC PPC440SPe Processors.
30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
31 * ADMA driver written by D.Williams.
34 #include <linux/init.h>
35 #include <linux/module.h>
36 #include <linux/async_tx.h>
37 #include <linux/delay.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/spinlock.h>
40 #include <linux/interrupt.h>
41 #include <linux/slab.h>
42 #include <linux/uaccess.h>
43 #include <linux/proc_fs.h>
45 #include <linux/of_platform.h>
47 #include <asm/dcr-regs.h>
49 #include "../dmaengine.h"
51 enum ppc_adma_init_code
{
56 PPC_ADMA_INIT_COHERENT
,
57 PPC_ADMA_INIT_CHANNEL
,
60 PPC_ADMA_INIT_REGISTER
63 static char *ppc_adma_errors
[] = {
64 [PPC_ADMA_INIT_OK
] = "ok",
65 [PPC_ADMA_INIT_MEMRES
] = "failed to get memory resource",
66 [PPC_ADMA_INIT_MEMREG
] = "failed to request memory region",
67 [PPC_ADMA_INIT_ALLOC
] = "failed to allocate memory for adev "
69 [PPC_ADMA_INIT_COHERENT
] = "failed to allocate coherent memory for "
70 "hardware descriptors",
71 [PPC_ADMA_INIT_CHANNEL
] = "failed to allocate memory for channel",
72 [PPC_ADMA_INIT_IRQ1
] = "failed to request first irq",
73 [PPC_ADMA_INIT_IRQ2
] = "failed to request second irq",
74 [PPC_ADMA_INIT_REGISTER
] = "failed to register dma async device",
77 static enum ppc_adma_init_code
78 ppc440spe_adma_devices
[PPC440SPE_ADMA_ENGINES_NUM
];
80 struct ppc_dma_chan_ref
{
81 struct dma_chan
*chan
;
82 struct list_head node
;
85 /* The list of channels exported by ppc440spe ADMA */
87 ppc440spe_adma_chan_list
= LIST_HEAD_INIT(ppc440spe_adma_chan_list
);
89 /* This flag is set when want to refetch the xor chain in the interrupt
92 static u32 do_xor_refetch
;
94 /* Pointer to DMA0, DMA1 CP/CS FIFO */
95 static void *ppc440spe_dma_fifo_buf
;
97 /* Pointers to last submitted to DMA0, DMA1 CDBs */
98 static struct ppc440spe_adma_desc_slot
*chan_last_sub
[3];
99 static struct ppc440spe_adma_desc_slot
*chan_first_cdb
[3];
101 /* Pointer to last linked and submitted xor CB */
102 static struct ppc440spe_adma_desc_slot
*xor_last_linked
;
103 static struct ppc440spe_adma_desc_slot
*xor_last_submit
;
105 /* This array is used in data-check operations for storing a pattern */
106 static char ppc440spe_qword
[16];
108 static atomic_t ppc440spe_adma_err_irq_ref
;
109 static dcr_host_t ppc440spe_mq_dcr_host
;
110 static unsigned int ppc440spe_mq_dcr_len
;
112 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
113 * the block size in transactions, then we do not allow to activate more than
114 * only one RXOR transactions simultaneously. So use this var to store
115 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
116 * set) or not (PPC440SPE_RXOR_RUN is clear).
118 static unsigned long ppc440spe_rxor_state
;
120 /* These are used in enable & check routines
122 static u32 ppc440spe_r6_enabled
;
123 static struct ppc440spe_adma_chan
*ppc440spe_r6_tchan
;
124 static struct completion ppc440spe_r6_test_comp
;
126 static int ppc440spe_adma_dma2rxor_prep_src(
127 struct ppc440spe_adma_desc_slot
*desc
,
128 struct ppc440spe_rxor
*cursor
, int index
,
129 int src_cnt
, u32 addr
);
130 static void ppc440spe_adma_dma2rxor_set_src(
131 struct ppc440spe_adma_desc_slot
*desc
,
132 int index
, dma_addr_t addr
);
133 static void ppc440spe_adma_dma2rxor_set_mult(
134 struct ppc440spe_adma_desc_slot
*desc
,
138 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
140 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
143 static void print_cb(struct ppc440spe_adma_chan
*chan
, void *block
)
149 switch (chan
->device
->id
) {
154 pr_debug("CDB at %p [%d]:\n"
155 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
156 "\t sg1u 0x%08x sg1l 0x%08x\n"
157 "\t sg2u 0x%08x sg2l 0x%08x\n"
158 "\t sg3u 0x%08x sg3l 0x%08x\n",
159 cdb
, chan
->device
->id
,
160 cdb
->attr
, cdb
->opc
, le32_to_cpu(cdb
->cnt
),
161 le32_to_cpu(cdb
->sg1u
), le32_to_cpu(cdb
->sg1l
),
162 le32_to_cpu(cdb
->sg2u
), le32_to_cpu(cdb
->sg2l
),
163 le32_to_cpu(cdb
->sg3u
), le32_to_cpu(cdb
->sg3l
)
169 pr_debug("CB at %p [%d]:\n"
170 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
171 "\t cbtah 0x%08x cbtal 0x%08x\n"
172 "\t cblah 0x%08x cblal 0x%08x\n",
173 cb
, chan
->device
->id
,
174 cb
->cbc
, cb
->cbbc
, cb
->cbs
,
175 cb
->cbtah
, cb
->cbtal
,
176 cb
->cblah
, cb
->cblal
);
177 for (i
= 0; i
< 16; i
++) {
178 if (i
&& !cb
->ops
[i
].h
&& !cb
->ops
[i
].l
)
180 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
181 i
, cb
->ops
[i
].h
, cb
->ops
[i
].l
);
187 static void print_cb_list(struct ppc440spe_adma_chan
*chan
,
188 struct ppc440spe_adma_desc_slot
*iter
)
190 for (; iter
; iter
= iter
->hw_next
)
191 print_cb(chan
, iter
->hw_desc
);
194 static void prep_dma_xor_dbg(int id
, dma_addr_t dst
, dma_addr_t
*src
,
195 unsigned int src_cnt
)
199 pr_debug("\n%s(%d):\nsrc: ", __func__
, id
);
200 for (i
= 0; i
< src_cnt
; i
++)
201 pr_debug("\t0x%016llx ", src
[i
]);
202 pr_debug("dst:\n\t0x%016llx\n", dst
);
205 static void prep_dma_pq_dbg(int id
, dma_addr_t
*dst
, dma_addr_t
*src
,
206 unsigned int src_cnt
)
210 pr_debug("\n%s(%d):\nsrc: ", __func__
, id
);
211 for (i
= 0; i
< src_cnt
; i
++)
212 pr_debug("\t0x%016llx ", src
[i
]);
214 for (i
= 0; i
< 2; i
++)
215 pr_debug("\t0x%016llx ", dst
[i
]);
218 static void prep_dma_pqzero_sum_dbg(int id
, dma_addr_t
*src
,
219 unsigned int src_cnt
,
220 const unsigned char *scf
)
224 pr_debug("\n%s(%d):\nsrc(coef): ", __func__
, id
);
226 for (i
= 0; i
< src_cnt
; i
++)
227 pr_debug("\t0x%016llx(0x%02x) ", src
[i
], scf
[i
]);
229 for (i
= 0; i
< src_cnt
; i
++)
230 pr_debug("\t0x%016llx(no) ", src
[i
]);
234 for (i
= 0; i
< 2; i
++)
235 pr_debug("\t0x%016llx ", src
[src_cnt
+ i
]);
238 /******************************************************************************
239 * Command (Descriptor) Blocks low-level routines
240 ******************************************************************************/
242 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
245 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot
*desc
,
246 struct ppc440spe_adma_chan
*chan
)
250 switch (chan
->device
->id
) {
251 case PPC440SPE_XOR_ID
:
253 memset(desc
->hw_desc
, 0, sizeof(struct xor_cb
));
254 /* NOP with Command Block Complete Enable */
255 p
->cbc
= XOR_CBCR_CBCE_BIT
;
257 case PPC440SPE_DMA0_ID
:
258 case PPC440SPE_DMA1_ID
:
259 memset(desc
->hw_desc
, 0, sizeof(struct dma_cdb
));
260 /* NOP with interrupt */
261 set_bit(PPC440SPE_DESC_INT
, &desc
->flags
);
264 printk(KERN_ERR
"Unsupported id %d in %s\n", chan
->device
->id
,
271 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
274 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot
*desc
)
276 memset(desc
->hw_desc
, 0, sizeof(struct xor_cb
));
277 desc
->hw_next
= NULL
;
283 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
285 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot
*desc
,
286 int src_cnt
, unsigned long flags
)
288 struct xor_cb
*hw_desc
= desc
->hw_desc
;
290 memset(desc
->hw_desc
, 0, sizeof(struct xor_cb
));
291 desc
->hw_next
= NULL
;
292 desc
->src_cnt
= src_cnt
;
295 hw_desc
->cbc
= XOR_CBCR_TGT_BIT
| src_cnt
;
296 if (flags
& DMA_PREP_INTERRUPT
)
297 /* Enable interrupt on completion */
298 hw_desc
->cbc
|= XOR_CBCR_CBCE_BIT
;
302 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
303 * operation in DMA2 controller
305 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot
*desc
,
306 int dst_cnt
, int src_cnt
, unsigned long flags
)
308 struct xor_cb
*hw_desc
= desc
->hw_desc
;
310 memset(desc
->hw_desc
, 0, sizeof(struct xor_cb
));
311 desc
->hw_next
= NULL
;
312 desc
->src_cnt
= src_cnt
;
313 desc
->dst_cnt
= dst_cnt
;
314 memset(desc
->reverse_flags
, 0, sizeof(desc
->reverse_flags
));
315 desc
->descs_per_op
= 0;
317 hw_desc
->cbc
= XOR_CBCR_TGT_BIT
;
318 if (flags
& DMA_PREP_INTERRUPT
)
319 /* Enable interrupt on completion */
320 hw_desc
->cbc
|= XOR_CBCR_CBCE_BIT
;
323 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
324 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
325 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
328 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
331 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot
*desc
,
332 int dst_cnt
, int src_cnt
, unsigned long flags
,
335 struct dma_cdb
*hw_desc
;
336 struct ppc440spe_adma_desc_slot
*iter
;
339 /* Common initialization of a PQ descriptors chain */
340 set_bits(op
, &desc
->flags
);
341 desc
->src_cnt
= src_cnt
;
342 desc
->dst_cnt
= dst_cnt
;
344 /* WXOR MULTICAST if both P and Q are being computed
345 * MV_SG1_SG2 if Q only
347 dopc
= (desc
->dst_cnt
== DMA_DEST_MAX_NUM
) ?
348 DMA_CDB_OPC_MULTICAST
: DMA_CDB_OPC_MV_SG1_SG2
;
350 list_for_each_entry(iter
, &desc
->group_list
, chain_node
) {
351 hw_desc
= iter
->hw_desc
;
352 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
354 if (likely(!list_is_last(&iter
->chain_node
,
355 &desc
->group_list
))) {
356 /* set 'next' pointer */
357 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
358 struct ppc440spe_adma_desc_slot
, chain_node
);
359 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
361 /* this is the last descriptor.
362 * this slot will be pasted from ADMA level
363 * each time it wants to configure parameters
364 * of the transaction (src, dst, ...)
366 iter
->hw_next
= NULL
;
367 if (flags
& DMA_PREP_INTERRUPT
)
368 set_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
370 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
374 /* Set OPS depending on WXOR/RXOR type of operation */
375 if (!test_bit(PPC440SPE_DESC_RXOR
, &desc
->flags
)) {
376 /* This is a WXOR only chain:
377 * - first descriptors are for zeroing destinations
378 * if PPC440SPE_ZERO_P/Q set;
379 * - descriptors remained are for GF-XOR operations.
381 iter
= list_first_entry(&desc
->group_list
,
382 struct ppc440spe_adma_desc_slot
,
385 if (test_bit(PPC440SPE_ZERO_P
, &desc
->flags
)) {
386 hw_desc
= iter
->hw_desc
;
387 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
388 iter
= list_first_entry(&iter
->chain_node
,
389 struct ppc440spe_adma_desc_slot
,
393 if (test_bit(PPC440SPE_ZERO_Q
, &desc
->flags
)) {
394 hw_desc
= iter
->hw_desc
;
395 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
396 iter
= list_first_entry(&iter
->chain_node
,
397 struct ppc440spe_adma_desc_slot
,
401 list_for_each_entry_from(iter
, &desc
->group_list
, chain_node
) {
402 hw_desc
= iter
->hw_desc
;
406 /* This is either RXOR-only or mixed RXOR/WXOR */
408 /* The first 1 or 2 slots in chain are always RXOR,
409 * if need to calculate P & Q, then there are two
410 * RXOR slots; if only P or only Q, then there is one
412 iter
= list_first_entry(&desc
->group_list
,
413 struct ppc440spe_adma_desc_slot
,
415 hw_desc
= iter
->hw_desc
;
416 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
418 if (desc
->dst_cnt
== DMA_DEST_MAX_NUM
) {
419 iter
= list_first_entry(&iter
->chain_node
,
420 struct ppc440spe_adma_desc_slot
,
422 hw_desc
= iter
->hw_desc
;
423 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
426 /* The remaining descs (if any) are WXORs */
427 if (test_bit(PPC440SPE_DESC_WXOR
, &desc
->flags
)) {
428 iter
= list_first_entry(&iter
->chain_node
,
429 struct ppc440spe_adma_desc_slot
,
431 list_for_each_entry_from(iter
, &desc
->group_list
,
433 hw_desc
= iter
->hw_desc
;
441 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
442 * for PQ_ZERO_SUM operation
444 static void ppc440spe_desc_init_dma01pqzero_sum(
445 struct ppc440spe_adma_desc_slot
*desc
,
446 int dst_cnt
, int src_cnt
)
448 struct dma_cdb
*hw_desc
;
449 struct ppc440spe_adma_desc_slot
*iter
;
451 u8 dopc
= (dst_cnt
== 2) ? DMA_CDB_OPC_MULTICAST
:
452 DMA_CDB_OPC_MV_SG1_SG2
;
454 * Initialize starting from 2nd or 3rd descriptor dependent
455 * on dst_cnt. First one or two slots are for cloning P
456 * and/or Q to chan->pdest and/or chan->qdest as we have
457 * to preserve original P/Q.
459 iter
= list_first_entry(&desc
->group_list
,
460 struct ppc440spe_adma_desc_slot
, chain_node
);
461 iter
= list_entry(iter
->chain_node
.next
,
462 struct ppc440spe_adma_desc_slot
, chain_node
);
465 iter
= list_entry(iter
->chain_node
.next
,
466 struct ppc440spe_adma_desc_slot
, chain_node
);
468 /* initialize each source descriptor in chain */
469 list_for_each_entry_from(iter
, &desc
->group_list
, chain_node
) {
470 hw_desc
= iter
->hw_desc
;
471 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
475 /* This is a ZERO_SUM operation:
476 * - <src_cnt> descriptors starting from 2nd or 3rd
477 * descriptor are for GF-XOR operations;
478 * - remaining <dst_cnt> descriptors are for checking the result
481 /* MV_SG1_SG2 if only Q is being verified
482 * MULTICAST if both P and Q are being verified
486 /* DMA_CDB_OPC_DCHECK128 operation */
487 hw_desc
->opc
= DMA_CDB_OPC_DCHECK128
;
489 if (likely(!list_is_last(&iter
->chain_node
,
490 &desc
->group_list
))) {
491 /* set 'next' pointer */
492 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
493 struct ppc440spe_adma_desc_slot
,
496 /* this is the last descriptor.
497 * this slot will be pasted from ADMA level
498 * each time it wants to configure parameters
499 * of the transaction (src, dst, ...)
501 iter
->hw_next
= NULL
;
502 /* always enable interrupt generation since we get
503 * the status of pqzero from the handler
505 set_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
508 desc
->src_cnt
= src_cnt
;
509 desc
->dst_cnt
= dst_cnt
;
513 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
515 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot
*desc
,
518 struct dma_cdb
*hw_desc
= desc
->hw_desc
;
520 memset(desc
->hw_desc
, 0, sizeof(struct dma_cdb
));
521 desc
->hw_next
= NULL
;
525 if (flags
& DMA_PREP_INTERRUPT
)
526 set_bit(PPC440SPE_DESC_INT
, &desc
->flags
);
528 clear_bit(PPC440SPE_DESC_INT
, &desc
->flags
);
530 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
534 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
536 static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot
*desc
,
537 int value
, unsigned long flags
)
539 struct dma_cdb
*hw_desc
= desc
->hw_desc
;
541 memset(desc
->hw_desc
, 0, sizeof(struct dma_cdb
));
542 desc
->hw_next
= NULL
;
546 if (flags
& DMA_PREP_INTERRUPT
)
547 set_bit(PPC440SPE_DESC_INT
, &desc
->flags
);
549 clear_bit(PPC440SPE_DESC_INT
, &desc
->flags
);
551 hw_desc
->sg1u
= hw_desc
->sg1l
= cpu_to_le32((u32
)value
);
552 hw_desc
->sg3u
= hw_desc
->sg3l
= cpu_to_le32((u32
)value
);
553 hw_desc
->opc
= DMA_CDB_OPC_DFILL128
;
557 * ppc440spe_desc_set_src_addr - set source address into the descriptor
559 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot
*desc
,
560 struct ppc440spe_adma_chan
*chan
,
561 int src_idx
, dma_addr_t addrh
,
564 struct dma_cdb
*dma_hw_desc
;
565 struct xor_cb
*xor_hw_desc
;
566 phys_addr_t addr64
, tmplow
, tmphi
;
568 switch (chan
->device
->id
) {
569 case PPC440SPE_DMA0_ID
:
570 case PPC440SPE_DMA1_ID
:
573 tmphi
= (addr64
>> 32);
574 tmplow
= (addr64
& 0xFFFFFFFF);
579 dma_hw_desc
= desc
->hw_desc
;
580 dma_hw_desc
->sg1l
= cpu_to_le32((u32
)tmplow
);
581 dma_hw_desc
->sg1u
|= cpu_to_le32((u32
)tmphi
);
583 case PPC440SPE_XOR_ID
:
584 xor_hw_desc
= desc
->hw_desc
;
585 xor_hw_desc
->ops
[src_idx
].l
= addrl
;
586 xor_hw_desc
->ops
[src_idx
].h
|= addrh
;
592 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
594 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot
*desc
,
595 struct ppc440spe_adma_chan
*chan
, u32 mult_index
,
596 int sg_index
, unsigned char mult_value
)
598 struct dma_cdb
*dma_hw_desc
;
599 struct xor_cb
*xor_hw_desc
;
602 switch (chan
->device
->id
) {
603 case PPC440SPE_DMA0_ID
:
604 case PPC440SPE_DMA1_ID
:
605 dma_hw_desc
= desc
->hw_desc
;
608 /* for RXOR operations set multiplier
609 * into source cued address
612 psgu
= &dma_hw_desc
->sg1u
;
614 /* for WXOR operations set multiplier
615 * into destination cued address(es)
617 case DMA_CDB_SG_DST1
:
618 psgu
= &dma_hw_desc
->sg2u
;
620 case DMA_CDB_SG_DST2
:
621 psgu
= &dma_hw_desc
->sg3u
;
627 *psgu
|= cpu_to_le32(mult_value
<< mult_index
);
629 case PPC440SPE_XOR_ID
:
630 xor_hw_desc
= desc
->hw_desc
;
638 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
640 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot
*desc
,
641 struct ppc440spe_adma_chan
*chan
,
642 dma_addr_t addrh
, dma_addr_t addrl
,
645 struct dma_cdb
*dma_hw_desc
;
646 struct xor_cb
*xor_hw_desc
;
647 phys_addr_t addr64
, tmphi
, tmplow
;
650 switch (chan
->device
->id
) {
651 case PPC440SPE_DMA0_ID
:
652 case PPC440SPE_DMA1_ID
:
655 tmphi
= (addr64
>> 32);
656 tmplow
= (addr64
& 0xFFFFFFFF);
661 dma_hw_desc
= desc
->hw_desc
;
663 psgu
= dst_idx
? &dma_hw_desc
->sg3u
: &dma_hw_desc
->sg2u
;
664 psgl
= dst_idx
? &dma_hw_desc
->sg3l
: &dma_hw_desc
->sg2l
;
666 *psgl
= cpu_to_le32((u32
)tmplow
);
667 *psgu
|= cpu_to_le32((u32
)tmphi
);
669 case PPC440SPE_XOR_ID
:
670 xor_hw_desc
= desc
->hw_desc
;
671 xor_hw_desc
->cbtal
= addrl
;
672 xor_hw_desc
->cbtah
|= addrh
;
678 * ppc440spe_desc_set_byte_count - set number of data bytes involved
681 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot
*desc
,
682 struct ppc440spe_adma_chan
*chan
,
685 struct dma_cdb
*dma_hw_desc
;
686 struct xor_cb
*xor_hw_desc
;
688 switch (chan
->device
->id
) {
689 case PPC440SPE_DMA0_ID
:
690 case PPC440SPE_DMA1_ID
:
691 dma_hw_desc
= desc
->hw_desc
;
692 dma_hw_desc
->cnt
= cpu_to_le32(byte_count
);
694 case PPC440SPE_XOR_ID
:
695 xor_hw_desc
= desc
->hw_desc
;
696 xor_hw_desc
->cbbc
= byte_count
;
702 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
704 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count
)
706 /* assume that byte_count is aligned on the 512-boundary;
707 * thus write it directly to the register (bits 23:31 are
710 dcr_write(ppc440spe_mq_dcr_host
, DCRN_MQ0_CF2H
, byte_count
);
714 * ppc440spe_desc_set_dcheck - set CHECK pattern
716 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot
*desc
,
717 struct ppc440spe_adma_chan
*chan
, u8
*qword
)
719 struct dma_cdb
*dma_hw_desc
;
721 switch (chan
->device
->id
) {
722 case PPC440SPE_DMA0_ID
:
723 case PPC440SPE_DMA1_ID
:
724 dma_hw_desc
= desc
->hw_desc
;
725 iowrite32(qword
[0], &dma_hw_desc
->sg3l
);
726 iowrite32(qword
[4], &dma_hw_desc
->sg3u
);
727 iowrite32(qword
[8], &dma_hw_desc
->sg2l
);
728 iowrite32(qword
[12], &dma_hw_desc
->sg2u
);
736 * ppc440spe_xor_set_link - set link address in xor CB
738 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot
*prev_desc
,
739 struct ppc440spe_adma_desc_slot
*next_desc
)
741 struct xor_cb
*xor_hw_desc
= prev_desc
->hw_desc
;
743 if (unlikely(!next_desc
|| !(next_desc
->phys
))) {
744 printk(KERN_ERR
"%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
746 next_desc
? next_desc
->phys
: 0);
750 xor_hw_desc
->cbs
= 0;
751 xor_hw_desc
->cblal
= next_desc
->phys
;
752 xor_hw_desc
->cblah
= 0;
753 xor_hw_desc
->cbc
|= XOR_CBCR_LNK_BIT
;
757 * ppc440spe_desc_set_link - set the address of descriptor following this
758 * descriptor in chain
760 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan
*chan
,
761 struct ppc440spe_adma_desc_slot
*prev_desc
,
762 struct ppc440spe_adma_desc_slot
*next_desc
)
765 struct ppc440spe_adma_desc_slot
*tail
= next_desc
;
767 if (unlikely(!prev_desc
|| !next_desc
||
768 (prev_desc
->hw_next
&& prev_desc
->hw_next
!= next_desc
))) {
769 /* If previous next is overwritten something is wrong.
770 * though we may refetch from append to initiate list
771 * processing; in this case - it's ok.
773 printk(KERN_ERR
"%s: prev_desc=0x%p; next_desc=0x%p; "
774 "prev->hw_next=0x%p\n", __func__
, prev_desc
,
775 next_desc
, prev_desc
? prev_desc
->hw_next
: 0);
779 local_irq_save(flags
);
781 /* do s/w chaining both for DMA and XOR descriptors */
782 prev_desc
->hw_next
= next_desc
;
784 switch (chan
->device
->id
) {
785 case PPC440SPE_DMA0_ID
:
786 case PPC440SPE_DMA1_ID
:
788 case PPC440SPE_XOR_ID
:
789 /* bind descriptor to the chain */
790 while (tail
->hw_next
)
791 tail
= tail
->hw_next
;
792 xor_last_linked
= tail
;
794 if (prev_desc
== xor_last_submit
)
795 /* do not link to the last submitted CB */
797 ppc440spe_xor_set_link(prev_desc
, next_desc
);
801 local_irq_restore(flags
);
805 * ppc440spe_desc_get_link - get the address of the descriptor that
808 static inline u32
ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot
*desc
,
809 struct ppc440spe_adma_chan
*chan
)
814 return desc
->hw_next
->phys
;
818 * ppc440spe_desc_is_aligned - check alignment
820 static inline int ppc440spe_desc_is_aligned(
821 struct ppc440spe_adma_desc_slot
*desc
, int num_slots
)
823 return (desc
->idx
& (num_slots
- 1)) ? 0 : 1;
827 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
830 static int ppc440spe_chan_xor_slot_count(size_t len
, int src_cnt
,
835 /* each XOR descriptor provides up to 16 source operands */
836 slot_cnt
= *slots_per_op
= (src_cnt
+ XOR_MAX_OPS
- 1)/XOR_MAX_OPS
;
838 if (likely(len
<= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
))
841 printk(KERN_ERR
"%s: len %d > max %d !!\n",
842 __func__
, len
, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
);
848 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
851 static int ppc440spe_dma2_pq_slot_count(dma_addr_t
*srcs
,
852 int src_cnt
, size_t len
)
854 signed long long order
= 0;
858 for (i
= 1; i
< src_cnt
; i
++) {
859 dma_addr_t cur_addr
= srcs
[i
];
860 dma_addr_t old_addr
= srcs
[i
-1];
863 if (cur_addr
== old_addr
+ len
) {
869 } else if (old_addr
== cur_addr
+ len
) {
880 if (i
== src_cnt
-2 || (order
== -1
881 && cur_addr
!= old_addr
- len
)) {
885 } else if (cur_addr
== old_addr
+ len
*order
) {
889 } else if (cur_addr
== old_addr
+ 2*len
) {
893 } else if (cur_addr
== old_addr
+ 3*len
) {
912 if (src_cnt
<= 1 || (state
!= 1 && state
!= 2)) {
913 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
914 __func__
, src_cnt
, state
, addr_count
, order
);
915 for (i
= 0; i
< src_cnt
; i
++)
916 pr_err("\t[%d] 0x%llx \n", i
, srcs
[i
]);
920 return (addr_count
+ XOR_MAX_OPS
- 1) / XOR_MAX_OPS
;
924 /******************************************************************************
925 * ADMA channel low-level routines
926 ******************************************************************************/
929 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan
*chan
);
930 static void ppc440spe_chan_append(struct ppc440spe_adma_chan
*chan
);
933 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
935 static void ppc440spe_adma_device_clear_eot_status(
936 struct ppc440spe_adma_chan
*chan
)
938 struct dma_regs
*dma_reg
;
939 struct xor_regs
*xor_reg
;
940 u8
*p
= chan
->device
->dma_desc_pool_virt
;
944 switch (chan
->device
->id
) {
945 case PPC440SPE_DMA0_ID
:
946 case PPC440SPE_DMA1_ID
:
947 /* read FIFO to ack */
948 dma_reg
= chan
->device
->dma_reg
;
949 while ((rv
= ioread32(&dma_reg
->csfpl
))) {
950 i
= rv
& DMA_CDB_ADDR_MSK
;
951 cdb
= (struct dma_cdb
*)&p
[i
-
952 (u32
)chan
->device
->dma_desc_pool
];
954 /* Clear opcode to ack. This is necessary for
955 * ZeroSum operations only
959 if (test_bit(PPC440SPE_RXOR_RUN
,
960 &ppc440spe_rxor_state
)) {
961 /* probably this is a completed RXOR op,
962 * get pointer to CDB using the fact that
963 * physical and virtual addresses of CDB
964 * in pools have the same offsets
966 if (le32_to_cpu(cdb
->sg1u
) &
969 clear_bit(PPC440SPE_RXOR_RUN
,
970 &ppc440spe_rxor_state
);
974 if (rv
& DMA_CDB_STATUS_MSK
) {
975 /* ZeroSum check failed
977 struct ppc440spe_adma_desc_slot
*iter
;
978 dma_addr_t phys
= rv
& ~DMA_CDB_MSK
;
981 * Update the status of corresponding
984 list_for_each_entry(iter
, &chan
->chain
,
986 if (iter
->phys
== phys
)
990 * if cannot find the corresponding
993 BUG_ON(&iter
->chain_node
== &chan
->chain
);
995 if (iter
->xor_check_result
) {
996 if (test_bit(PPC440SPE_DESC_PCHECK
,
998 *iter
->xor_check_result
|=
1001 if (test_bit(PPC440SPE_DESC_QCHECK
,
1003 *iter
->xor_check_result
|=
1011 rv
= ioread32(&dma_reg
->dsts
);
1013 pr_err("DMA%d err status: 0x%x\n",
1014 chan
->device
->id
, rv
);
1015 /* write back to clear */
1016 iowrite32(rv
, &dma_reg
->dsts
);
1019 case PPC440SPE_XOR_ID
:
1020 /* reset status bits to ack */
1021 xor_reg
= chan
->device
->xor_reg
;
1022 rv
= ioread32be(&xor_reg
->sr
);
1023 iowrite32be(rv
, &xor_reg
->sr
);
1025 if (rv
& (XOR_IE_ICBIE_BIT
|XOR_IE_ICIE_BIT
|XOR_IE_RPTIE_BIT
)) {
1026 if (rv
& XOR_IE_RPTIE_BIT
) {
1027 /* Read PLB Timeout Error.
1028 * Try to resubmit the CB
1030 u32 val
= ioread32be(&xor_reg
->ccbalr
);
1032 iowrite32be(val
, &xor_reg
->cblalr
);
1034 val
= ioread32be(&xor_reg
->crsr
);
1035 iowrite32be(val
| XOR_CRSR_XAE_BIT
,
1038 pr_err("XOR ERR 0x%x status\n", rv
);
1042 /* if the XORcore is idle, but there are unprocessed CBs
1043 * then refetch the s/w chain here
1045 if (!(ioread32be(&xor_reg
->sr
) & XOR_SR_XCP_BIT
) &&
1047 ppc440spe_chan_append(chan
);
1053 * ppc440spe_chan_is_busy - get the channel status
1055 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan
*chan
)
1057 struct dma_regs
*dma_reg
;
1058 struct xor_regs
*xor_reg
;
1061 switch (chan
->device
->id
) {
1062 case PPC440SPE_DMA0_ID
:
1063 case PPC440SPE_DMA1_ID
:
1064 dma_reg
= chan
->device
->dma_reg
;
1065 /* if command FIFO's head and tail pointers are equal and
1066 * status tail is the same as command, then channel is free
1068 if (ioread16(&dma_reg
->cpfhp
) != ioread16(&dma_reg
->cpftp
) ||
1069 ioread16(&dma_reg
->cpftp
) != ioread16(&dma_reg
->csftp
))
1072 case PPC440SPE_XOR_ID
:
1073 /* use the special status bit for the XORcore
1075 xor_reg
= chan
->device
->xor_reg
;
1076 busy
= (ioread32be(&xor_reg
->sr
) & XOR_SR_XCP_BIT
) ? 1 : 0;
1084 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1086 static void ppc440spe_chan_set_first_xor_descriptor(
1087 struct ppc440spe_adma_chan
*chan
,
1088 struct ppc440spe_adma_desc_slot
*next_desc
)
1090 struct xor_regs
*xor_reg
= chan
->device
->xor_reg
;
1092 if (ioread32be(&xor_reg
->sr
) & XOR_SR_XCP_BIT
)
1093 printk(KERN_INFO
"%s: Warn: XORcore is running "
1094 "when try to set the first CDB!\n",
1097 xor_last_submit
= xor_last_linked
= next_desc
;
1099 iowrite32be(XOR_CRSR_64BA_BIT
, &xor_reg
->crsr
);
1101 iowrite32be(next_desc
->phys
, &xor_reg
->cblalr
);
1102 iowrite32be(0, &xor_reg
->cblahr
);
1103 iowrite32be(ioread32be(&xor_reg
->cbcr
) | XOR_CBCR_LNK_BIT
,
1106 chan
->hw_chain_inited
= 1;
1110 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1111 * called with irqs disabled
1113 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan
*chan
,
1114 struct ppc440spe_adma_desc_slot
*desc
)
1117 struct dma_regs
*dma_reg
= chan
->device
->dma_reg
;
1120 if (!test_bit(PPC440SPE_DESC_INT
, &desc
->flags
))
1121 pcdb
|= DMA_CDB_NO_INT
;
1123 chan_last_sub
[chan
->device
->id
] = desc
;
1125 ADMA_LL_DBG(print_cb(chan
, desc
->hw_desc
));
1127 iowrite32(pcdb
, &dma_reg
->cpfpl
);
1131 * ppc440spe_chan_append - update the h/w chain in the channel
1133 static void ppc440spe_chan_append(struct ppc440spe_adma_chan
*chan
)
1135 struct xor_regs
*xor_reg
;
1136 struct ppc440spe_adma_desc_slot
*iter
;
1139 unsigned long flags
;
1141 local_irq_save(flags
);
1143 switch (chan
->device
->id
) {
1144 case PPC440SPE_DMA0_ID
:
1145 case PPC440SPE_DMA1_ID
:
1146 cur_desc
= ppc440spe_chan_get_current_descriptor(chan
);
1148 if (likely(cur_desc
)) {
1149 iter
= chan_last_sub
[chan
->device
->id
];
1153 iter
= chan_first_cdb
[chan
->device
->id
];
1155 ppc440spe_dma_put_desc(chan
, iter
);
1156 chan
->hw_chain_inited
= 1;
1159 /* is there something new to append */
1163 /* flush descriptors from the s/w queue to fifo */
1164 list_for_each_entry_continue(iter
, &chan
->chain
, chain_node
) {
1165 ppc440spe_dma_put_desc(chan
, iter
);
1170 case PPC440SPE_XOR_ID
:
1171 /* update h/w links and refetch */
1172 if (!xor_last_submit
->hw_next
)
1175 xor_reg
= chan
->device
->xor_reg
;
1176 /* the last linked CDB has to generate an interrupt
1177 * that we'd be able to append the next lists to h/w
1178 * regardless of the XOR engine state at the moment of
1179 * appending of these next lists
1181 xcb
= xor_last_linked
->hw_desc
;
1182 xcb
->cbc
|= XOR_CBCR_CBCE_BIT
;
1184 if (!(ioread32be(&xor_reg
->sr
) & XOR_SR_XCP_BIT
)) {
1185 /* XORcore is idle. Refetch now */
1187 ppc440spe_xor_set_link(xor_last_submit
,
1188 xor_last_submit
->hw_next
);
1190 ADMA_LL_DBG(print_cb_list(chan
,
1191 xor_last_submit
->hw_next
));
1193 xor_last_submit
= xor_last_linked
;
1194 iowrite32be(ioread32be(&xor_reg
->crsr
) |
1195 XOR_CRSR_RCBE_BIT
| XOR_CRSR_64BA_BIT
,
1198 /* XORcore is running. Refetch later in the handler */
1205 local_irq_restore(flags
);
1209 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1212 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan
*chan
)
1214 struct dma_regs
*dma_reg
;
1215 struct xor_regs
*xor_reg
;
1217 if (unlikely(!chan
->hw_chain_inited
))
1218 /* h/w descriptor chain is not initialized yet */
1221 switch (chan
->device
->id
) {
1222 case PPC440SPE_DMA0_ID
:
1223 case PPC440SPE_DMA1_ID
:
1224 dma_reg
= chan
->device
->dma_reg
;
1225 return ioread32(&dma_reg
->acpl
) & (~DMA_CDB_MSK
);
1226 case PPC440SPE_XOR_ID
:
1227 xor_reg
= chan
->device
->xor_reg
;
1228 return ioread32be(&xor_reg
->ccbalr
);
1234 * ppc440spe_chan_run - enable the channel
1236 static void ppc440spe_chan_run(struct ppc440spe_adma_chan
*chan
)
1238 struct xor_regs
*xor_reg
;
1240 switch (chan
->device
->id
) {
1241 case PPC440SPE_DMA0_ID
:
1242 case PPC440SPE_DMA1_ID
:
1243 /* DMAs are always enabled, do nothing */
1245 case PPC440SPE_XOR_ID
:
1246 /* drain write buffer */
1247 xor_reg
= chan
->device
->xor_reg
;
1249 /* fetch descriptor pointed to in <link> */
1250 iowrite32be(XOR_CRSR_64BA_BIT
| XOR_CRSR_XAE_BIT
,
1256 /******************************************************************************
1258 ******************************************************************************/
1260 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan
*chan
);
1261 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan
*chan
);
1264 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor
*tx
);
1266 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot
*tx
,
1267 dma_addr_t addr
, int index
);
1269 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot
*tx
,
1270 dma_addr_t addr
, int index
);
1273 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot
*tx
,
1274 dma_addr_t
*paddr
, unsigned long flags
);
1276 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot
*tx
,
1277 dma_addr_t addr
, int index
);
1279 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot
*tx
,
1280 unsigned char mult
, int index
, int dst_pos
);
1282 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot
*tx
,
1283 dma_addr_t paddr
, dma_addr_t qaddr
);
1285 static struct page
*ppc440spe_rxor_srcs
[32];
1288 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1290 static int ppc440spe_can_rxor(struct page
**srcs
, int src_cnt
, size_t len
)
1292 int i
, order
= 0, state
= 0;
1295 if (unlikely(!(src_cnt
> 1)))
1298 BUG_ON(src_cnt
> ARRAY_SIZE(ppc440spe_rxor_srcs
));
1300 /* Skip holes in the source list before checking */
1301 for (i
= 0; i
< src_cnt
; i
++) {
1304 ppc440spe_rxor_srcs
[idx
++] = srcs
[i
];
1308 for (i
= 1; i
< src_cnt
; i
++) {
1309 char *cur_addr
= page_address(ppc440spe_rxor_srcs
[i
]);
1310 char *old_addr
= page_address(ppc440spe_rxor_srcs
[i
- 1]);
1314 if (cur_addr
== old_addr
+ len
) {
1318 } else if (old_addr
== cur_addr
+ len
) {
1326 if ((i
== src_cnt
- 2) ||
1327 (order
== -1 && cur_addr
!= old_addr
- len
)) {
1330 } else if ((cur_addr
== old_addr
+ len
* order
) ||
1331 (cur_addr
== old_addr
+ 2 * len
) ||
1332 (cur_addr
== old_addr
+ 3 * len
)) {
1347 if (state
== 1 || state
== 2)
1354 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1355 * the operation given on this channel. It's assumed that 'chan' is
1356 * capable to process 'cap' type of operation.
1357 * @chan: channel to use
1358 * @cap: type of transaction
1359 * @dst_lst: array of destination pointers
1360 * @dst_cnt: number of destination operands
1361 * @src_lst: array of source pointers
1362 * @src_cnt: number of source operands
1363 * @src_sz: size of each source operand
1365 static int ppc440spe_adma_estimate(struct dma_chan
*chan
,
1366 enum dma_transaction_type cap
, struct page
**dst_lst
, int dst_cnt
,
1367 struct page
**src_lst
, int src_cnt
, size_t src_sz
)
1371 if (cap
== DMA_PQ
|| cap
== DMA_PQ_VAL
) {
1372 /* If RAID-6 capabilities were not activated don't try
1375 if (unlikely(!ppc440spe_r6_enabled
))
1378 /* In the current implementation of ppc440spe ADMA driver it
1379 * makes sense to pick out only pq case, because it may be
1381 * (1) either using Biskup method on DMA2;
1383 * Thus we give a favour to (1) if the sources are suitable;
1384 * else let it be processed on one of the DMA0/1 engines.
1385 * In the sum_product case where destination is also the
1386 * source process it on DMA0/1 only.
1388 if (cap
== DMA_PQ
&& chan
->chan_id
== PPC440SPE_XOR_ID
) {
1390 if (dst_cnt
== 1 && src_cnt
== 2 && dst_lst
[0] == src_lst
[1])
1391 ef
= 0; /* sum_product case, process on DMA0/1 */
1392 else if (ppc440spe_can_rxor(src_lst
, src_cnt
, src_sz
))
1393 ef
= 3; /* override (DMA0/1 + idle) */
1395 ef
= 0; /* can't process on DMA2 if !rxor */
1398 /* channel idleness increases the priority */
1400 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan
)))
1407 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap
,
1408 struct page
**dst_lst
, int dst_cnt
, struct page
**src_lst
,
1409 int src_cnt
, size_t src_sz
)
1411 struct dma_chan
*best_chan
= NULL
;
1412 struct ppc_dma_chan_ref
*ref
;
1415 if (unlikely(!src_sz
))
1417 if (src_sz
> PAGE_SIZE
) {
1419 * should a user of the api ever pass > PAGE_SIZE requests
1420 * we sort out cases where temporary page-sized buffers
1425 if (src_cnt
== 1 && dst_lst
[1] == src_lst
[0])
1427 if (src_cnt
== 2 && dst_lst
[1] == src_lst
[1])
1438 list_for_each_entry(ref
, &ppc440spe_adma_chan_list
, node
) {
1439 if (dma_has_cap(cap
, ref
->chan
->device
->cap_mask
)) {
1442 rank
= ppc440spe_adma_estimate(ref
->chan
, cap
, dst_lst
,
1443 dst_cnt
, src_lst
, src_cnt
, src_sz
);
1444 if (rank
> best_rank
) {
1446 best_chan
= ref
->chan
;
1453 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel
);
1456 * ppc440spe_get_group_entry - get group entry with index idx
1457 * @tdesc: is the last allocated slot in the group.
1459 static struct ppc440spe_adma_desc_slot
*
1460 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot
*tdesc
, u32 entry_idx
)
1462 struct ppc440spe_adma_desc_slot
*iter
= tdesc
->group_head
;
1465 if (entry_idx
< 0 || entry_idx
>= (tdesc
->src_cnt
+ tdesc
->dst_cnt
)) {
1466 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1467 __func__
, entry_idx
, tdesc
->src_cnt
, tdesc
->dst_cnt
);
1471 list_for_each_entry(iter
, &tdesc
->group_list
, chain_node
) {
1472 if (i
++ == entry_idx
)
1479 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1480 * @slot: Slot to free
1481 * Caller must hold &ppc440spe_chan->lock while calling this function
1483 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot
*slot
,
1484 struct ppc440spe_adma_chan
*chan
)
1486 int stride
= slot
->slots_per_op
;
1489 slot
->slots_per_op
= 0;
1490 slot
= list_entry(slot
->slot_node
.next
,
1491 struct ppc440spe_adma_desc_slot
,
1497 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1500 static dma_cookie_t
ppc440spe_adma_run_tx_complete_actions(
1501 struct ppc440spe_adma_desc_slot
*desc
,
1502 struct ppc440spe_adma_chan
*chan
,
1503 dma_cookie_t cookie
)
1507 BUG_ON(desc
->async_tx
.cookie
< 0);
1508 if (desc
->async_tx
.cookie
> 0) {
1509 cookie
= desc
->async_tx
.cookie
;
1510 desc
->async_tx
.cookie
= 0;
1512 /* call the callback (must not sleep or submit new
1513 * operations to this channel)
1515 if (desc
->async_tx
.callback
)
1516 desc
->async_tx
.callback(
1517 desc
->async_tx
.callback_param
);
1519 dma_descriptor_unmap(&desc
->async_tx
);
1522 /* run dependent operations */
1523 dma_run_dependencies(&desc
->async_tx
);
1529 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1531 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot
*desc
,
1532 struct ppc440spe_adma_chan
*chan
)
1534 /* the client is allowed to attach dependent operations
1535 * until 'ack' is set
1537 if (!async_tx_test_ack(&desc
->async_tx
))
1540 /* leave the last descriptor in the chain
1541 * so we can append to it
1543 if (list_is_last(&desc
->chain_node
, &chan
->chain
) ||
1544 desc
->phys
== ppc440spe_chan_get_current_descriptor(chan
))
1547 if (chan
->device
->id
!= PPC440SPE_XOR_ID
) {
1548 /* our DMA interrupt handler clears opc field of
1549 * each processed descriptor. For all types of
1550 * operations except for ZeroSum we do not actually
1551 * need ack from the interrupt handler. ZeroSum is a
1552 * special case since the result of this operation
1553 * is available from the handler only, so if we see
1554 * such type of descriptor (which is unprocessed yet)
1555 * then leave it in chain.
1557 struct dma_cdb
*cdb
= desc
->hw_desc
;
1558 if (cdb
->opc
== DMA_CDB_OPC_DCHECK128
)
1562 dev_dbg(chan
->device
->common
.dev
, "\tfree slot %llx: %d stride: %d\n",
1563 desc
->phys
, desc
->idx
, desc
->slots_per_op
);
1565 list_del(&desc
->chain_node
);
1566 ppc440spe_adma_free_slots(desc
, chan
);
1571 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1572 * which runs through the channel CDBs list until reach the descriptor
1573 * currently processed. When routine determines that all CDBs of group
1574 * are completed then corresponding callbacks (if any) are called and slots
1577 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan
*chan
)
1579 struct ppc440spe_adma_desc_slot
*iter
, *_iter
, *group_start
= NULL
;
1580 dma_cookie_t cookie
= 0;
1581 u32 current_desc
= ppc440spe_chan_get_current_descriptor(chan
);
1582 int busy
= ppc440spe_chan_is_busy(chan
);
1583 int seen_current
= 0, slot_cnt
= 0, slots_per_op
= 0;
1585 dev_dbg(chan
->device
->common
.dev
, "ppc440spe adma%d: %s\n",
1586 chan
->device
->id
, __func__
);
1588 if (!current_desc
) {
1589 /* There were no transactions yet, so
1595 /* free completed slots from the chain starting with
1596 * the oldest descriptor
1598 list_for_each_entry_safe(iter
, _iter
, &chan
->chain
,
1600 dev_dbg(chan
->device
->common
.dev
, "\tcookie: %d slot: %d "
1601 "busy: %d this_desc: %#llx next_desc: %#x "
1602 "cur: %#x ack: %d\n",
1603 iter
->async_tx
.cookie
, iter
->idx
, busy
, iter
->phys
,
1604 ppc440spe_desc_get_link(iter
, chan
), current_desc
,
1605 async_tx_test_ack(&iter
->async_tx
));
1607 prefetch(&_iter
->async_tx
);
1609 /* do not advance past the current descriptor loaded into the
1610 * hardware channel,subsequent descriptors are either in process
1611 * or have not been submitted
1616 /* stop the search if we reach the current descriptor and the
1617 * channel is busy, or if it appears that the current descriptor
1618 * needs to be re-read (i.e. has been appended to)
1620 if (iter
->phys
== current_desc
) {
1621 BUG_ON(seen_current
++);
1622 if (busy
|| ppc440spe_desc_get_link(iter
, chan
)) {
1623 /* not all descriptors of the group have
1624 * been completed; exit.
1630 /* detect the start of a group transaction */
1631 if (!slot_cnt
&& !slots_per_op
) {
1632 slot_cnt
= iter
->slot_cnt
;
1633 slots_per_op
= iter
->slots_per_op
;
1634 if (slot_cnt
<= slots_per_op
) {
1643 slot_cnt
-= slots_per_op
;
1646 /* all the members of a group are complete */
1647 if (slots_per_op
!= 0 && slot_cnt
== 0) {
1648 struct ppc440spe_adma_desc_slot
*grp_iter
, *_grp_iter
;
1649 int end_of_chain
= 0;
1651 /* clean up the group */
1652 slot_cnt
= group_start
->slot_cnt
;
1653 grp_iter
= group_start
;
1654 list_for_each_entry_safe_from(grp_iter
, _grp_iter
,
1655 &chan
->chain
, chain_node
) {
1657 cookie
= ppc440spe_adma_run_tx_complete_actions(
1658 grp_iter
, chan
, cookie
);
1660 slot_cnt
-= slots_per_op
;
1661 end_of_chain
= ppc440spe_adma_clean_slot(
1663 if (end_of_chain
&& slot_cnt
) {
1664 /* Should wait for ZeroSum completion */
1666 chan
->common
.completed_cookie
= cookie
;
1670 if (slot_cnt
== 0 || end_of_chain
)
1674 /* the group should be complete at this point */
1683 } else if (slots_per_op
) /* wait for group completion */
1686 cookie
= ppc440spe_adma_run_tx_complete_actions(iter
, chan
,
1689 if (ppc440spe_adma_clean_slot(iter
, chan
))
1693 BUG_ON(!seen_current
);
1696 chan
->common
.completed_cookie
= cookie
;
1697 pr_debug("\tcompleted cookie %d\n", cookie
);
1703 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1705 static void ppc440spe_adma_tasklet(unsigned long data
)
1707 struct ppc440spe_adma_chan
*chan
= (struct ppc440spe_adma_chan
*) data
;
1709 spin_lock_nested(&chan
->lock
, SINGLE_DEPTH_NESTING
);
1710 __ppc440spe_adma_slot_cleanup(chan
);
1711 spin_unlock(&chan
->lock
);
1715 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1717 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan
*chan
)
1719 spin_lock_bh(&chan
->lock
);
1720 __ppc440spe_adma_slot_cleanup(chan
);
1721 spin_unlock_bh(&chan
->lock
);
1725 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1727 static struct ppc440spe_adma_desc_slot
*ppc440spe_adma_alloc_slots(
1728 struct ppc440spe_adma_chan
*chan
, int num_slots
,
1731 struct ppc440spe_adma_desc_slot
*iter
= NULL
, *_iter
;
1732 struct ppc440spe_adma_desc_slot
*alloc_start
= NULL
;
1733 struct list_head chain
= LIST_HEAD_INIT(chain
);
1734 int slots_found
, retry
= 0;
1737 BUG_ON(!num_slots
|| !slots_per_op
);
1738 /* start search from the last allocated descrtiptor
1739 * if a contiguous allocation can not be found start searching
1740 * from the beginning of the list
1745 iter
= chan
->last_used
;
1747 iter
= list_entry(&chan
->all_slots
,
1748 struct ppc440spe_adma_desc_slot
,
1750 list_for_each_entry_safe_continue(iter
, _iter
, &chan
->all_slots
,
1753 prefetch(&_iter
->async_tx
);
1754 if (iter
->slots_per_op
) {
1759 /* start the allocation if the slot is correctly aligned */
1763 if (slots_found
== num_slots
) {
1764 struct ppc440spe_adma_desc_slot
*alloc_tail
= NULL
;
1765 struct ppc440spe_adma_desc_slot
*last_used
= NULL
;
1770 /* pre-ack all but the last descriptor */
1771 if (num_slots
!= slots_per_op
)
1772 async_tx_ack(&iter
->async_tx
);
1774 list_add_tail(&iter
->chain_node
, &chain
);
1776 iter
->async_tx
.cookie
= 0;
1777 iter
->hw_next
= NULL
;
1779 iter
->slot_cnt
= num_slots
;
1780 iter
->xor_check_result
= NULL
;
1781 for (i
= 0; i
< slots_per_op
; i
++) {
1782 iter
->slots_per_op
= slots_per_op
- i
;
1784 iter
= list_entry(iter
->slot_node
.next
,
1785 struct ppc440spe_adma_desc_slot
,
1788 num_slots
-= slots_per_op
;
1790 alloc_tail
->group_head
= alloc_start
;
1791 alloc_tail
->async_tx
.cookie
= -EBUSY
;
1792 list_splice(&chain
, &alloc_tail
->group_list
);
1793 chan
->last_used
= last_used
;
1800 /* try to free some slots if the allocation fails */
1801 tasklet_schedule(&chan
->irq_tasklet
);
1806 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
1808 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan
*chan
)
1810 struct ppc440spe_adma_chan
*ppc440spe_chan
;
1811 struct ppc440spe_adma_desc_slot
*slot
= NULL
;
1816 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
1817 init
= ppc440spe_chan
->slots_allocated
? 0 : 1;
1818 chan
->chan_id
= ppc440spe_chan
->device
->id
;
1820 /* Allocate descriptor slots */
1821 i
= ppc440spe_chan
->slots_allocated
;
1822 if (ppc440spe_chan
->device
->id
!= PPC440SPE_XOR_ID
)
1823 db_sz
= sizeof(struct dma_cdb
);
1825 db_sz
= sizeof(struct xor_cb
);
1827 for (; i
< (ppc440spe_chan
->device
->pool_size
/ db_sz
); i
++) {
1828 slot
= kzalloc(sizeof(struct ppc440spe_adma_desc_slot
),
1831 printk(KERN_INFO
"SPE ADMA Channel only initialized"
1832 " %d descriptor slots", i
--);
1836 hw_desc
= (char *) ppc440spe_chan
->device
->dma_desc_pool_virt
;
1837 slot
->hw_desc
= (void *) &hw_desc
[i
* db_sz
];
1838 dma_async_tx_descriptor_init(&slot
->async_tx
, chan
);
1839 slot
->async_tx
.tx_submit
= ppc440spe_adma_tx_submit
;
1840 INIT_LIST_HEAD(&slot
->chain_node
);
1841 INIT_LIST_HEAD(&slot
->slot_node
);
1842 INIT_LIST_HEAD(&slot
->group_list
);
1843 slot
->phys
= ppc440spe_chan
->device
->dma_desc_pool
+ i
* db_sz
;
1846 spin_lock_bh(&ppc440spe_chan
->lock
);
1847 ppc440spe_chan
->slots_allocated
++;
1848 list_add_tail(&slot
->slot_node
, &ppc440spe_chan
->all_slots
);
1849 spin_unlock_bh(&ppc440spe_chan
->lock
);
1852 if (i
&& !ppc440spe_chan
->last_used
) {
1853 ppc440spe_chan
->last_used
=
1854 list_entry(ppc440spe_chan
->all_slots
.next
,
1855 struct ppc440spe_adma_desc_slot
,
1859 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
1860 "ppc440spe adma%d: allocated %d descriptor slots\n",
1861 ppc440spe_chan
->device
->id
, i
);
1863 /* initialize the channel and the chain with a null operation */
1865 switch (ppc440spe_chan
->device
->id
) {
1866 case PPC440SPE_DMA0_ID
:
1867 case PPC440SPE_DMA1_ID
:
1868 ppc440spe_chan
->hw_chain_inited
= 0;
1869 /* Use WXOR for self-testing */
1870 if (!ppc440spe_r6_tchan
)
1871 ppc440spe_r6_tchan
= ppc440spe_chan
;
1873 case PPC440SPE_XOR_ID
:
1874 ppc440spe_chan_start_null_xor(ppc440spe_chan
);
1879 ppc440spe_chan
->needs_unmap
= 1;
1882 return (i
> 0) ? i
: -ENOMEM
;
1886 * ppc440spe_rxor_set_region_data -
1888 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot
*desc
,
1889 u8 xor_arg_no
, u32 mask
)
1891 struct xor_cb
*xcb
= desc
->hw_desc
;
1893 xcb
->ops
[xor_arg_no
].h
|= mask
;
1897 * ppc440spe_rxor_set_src -
1899 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot
*desc
,
1900 u8 xor_arg_no
, dma_addr_t addr
)
1902 struct xor_cb
*xcb
= desc
->hw_desc
;
1904 xcb
->ops
[xor_arg_no
].h
|= DMA_CUED_XOR_BASE
;
1905 xcb
->ops
[xor_arg_no
].l
= addr
;
1909 * ppc440spe_rxor_set_mult -
1911 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot
*desc
,
1912 u8 xor_arg_no
, u8 idx
, u8 mult
)
1914 struct xor_cb
*xcb
= desc
->hw_desc
;
1916 xcb
->ops
[xor_arg_no
].h
|= mult
<< (DMA_CUED_MULT1_OFF
+ idx
* 8);
1920 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1923 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan
*chan
)
1925 dev_dbg(chan
->device
->common
.dev
, "ppc440spe adma%d: pending: %d\n",
1926 chan
->device
->id
, chan
->pending
);
1928 if (chan
->pending
>= PPC440SPE_ADMA_THRESHOLD
) {
1930 ppc440spe_chan_append(chan
);
1935 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1936 * (it's not necessary that descriptors will be submitted to the h/w
1937 * chains too right now)
1939 static dma_cookie_t
ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor
*tx
)
1941 struct ppc440spe_adma_desc_slot
*sw_desc
;
1942 struct ppc440spe_adma_chan
*chan
= to_ppc440spe_adma_chan(tx
->chan
);
1943 struct ppc440spe_adma_desc_slot
*group_start
, *old_chain_tail
;
1946 dma_cookie_t cookie
;
1948 sw_desc
= tx_to_ppc440spe_adma_slot(tx
);
1950 group_start
= sw_desc
->group_head
;
1951 slot_cnt
= group_start
->slot_cnt
;
1952 slots_per_op
= group_start
->slots_per_op
;
1954 spin_lock_bh(&chan
->lock
);
1955 cookie
= dma_cookie_assign(tx
);
1957 if (unlikely(list_empty(&chan
->chain
))) {
1959 list_splice_init(&sw_desc
->group_list
, &chan
->chain
);
1960 chan_first_cdb
[chan
->device
->id
] = group_start
;
1962 /* isn't first peer, bind CDBs to chain */
1963 old_chain_tail
= list_entry(chan
->chain
.prev
,
1964 struct ppc440spe_adma_desc_slot
,
1966 list_splice_init(&sw_desc
->group_list
,
1967 &old_chain_tail
->chain_node
);
1968 /* fix up the hardware chain */
1969 ppc440spe_desc_set_link(chan
, old_chain_tail
, group_start
);
1972 /* increment the pending count by the number of operations */
1973 chan
->pending
+= slot_cnt
/ slots_per_op
;
1974 ppc440spe_adma_check_threshold(chan
);
1975 spin_unlock_bh(&chan
->lock
);
1977 dev_dbg(chan
->device
->common
.dev
,
1978 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1979 chan
->device
->id
, __func__
,
1980 sw_desc
->async_tx
.cookie
, sw_desc
->idx
, sw_desc
);
1986 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1988 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_interrupt(
1989 struct dma_chan
*chan
, unsigned long flags
)
1991 struct ppc440spe_adma_chan
*ppc440spe_chan
;
1992 struct ppc440spe_adma_desc_slot
*sw_desc
, *group_start
;
1993 int slot_cnt
, slots_per_op
;
1995 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
1997 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
1998 "ppc440spe adma%d: %s\n", ppc440spe_chan
->device
->id
,
2001 spin_lock_bh(&ppc440spe_chan
->lock
);
2002 slot_cnt
= slots_per_op
= 1;
2003 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
,
2006 group_start
= sw_desc
->group_head
;
2007 ppc440spe_desc_init_interrupt(group_start
, ppc440spe_chan
);
2008 group_start
->unmap_len
= 0;
2009 sw_desc
->async_tx
.flags
= flags
;
2011 spin_unlock_bh(&ppc440spe_chan
->lock
);
2013 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2017 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2019 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_memcpy(
2020 struct dma_chan
*chan
, dma_addr_t dma_dest
,
2021 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
2023 struct ppc440spe_adma_chan
*ppc440spe_chan
;
2024 struct ppc440spe_adma_desc_slot
*sw_desc
, *group_start
;
2025 int slot_cnt
, slots_per_op
;
2027 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
2032 BUG_ON(len
> PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT
);
2034 spin_lock_bh(&ppc440spe_chan
->lock
);
2036 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
2037 "ppc440spe adma%d: %s len: %u int_en %d\n",
2038 ppc440spe_chan
->device
->id
, __func__
, len
,
2039 flags
& DMA_PREP_INTERRUPT
? 1 : 0);
2040 slot_cnt
= slots_per_op
= 1;
2041 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
,
2044 group_start
= sw_desc
->group_head
;
2045 ppc440spe_desc_init_memcpy(group_start
, flags
);
2046 ppc440spe_adma_set_dest(group_start
, dma_dest
, 0);
2047 ppc440spe_adma_memcpy_xor_set_src(group_start
, dma_src
, 0);
2048 ppc440spe_desc_set_byte_count(group_start
, ppc440spe_chan
, len
);
2049 sw_desc
->unmap_len
= len
;
2050 sw_desc
->async_tx
.flags
= flags
;
2052 spin_unlock_bh(&ppc440spe_chan
->lock
);
2054 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2058 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2060 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_xor(
2061 struct dma_chan
*chan
, dma_addr_t dma_dest
,
2062 dma_addr_t
*dma_src
, u32 src_cnt
, size_t len
,
2063 unsigned long flags
)
2065 struct ppc440spe_adma_chan
*ppc440spe_chan
;
2066 struct ppc440spe_adma_desc_slot
*sw_desc
, *group_start
;
2067 int slot_cnt
, slots_per_op
;
2069 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
2071 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan
->device
->id
,
2072 dma_dest
, dma_src
, src_cnt
));
2075 BUG_ON(len
> PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
);
2077 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
2078 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2079 ppc440spe_chan
->device
->id
, __func__
, src_cnt
, len
,
2080 flags
& DMA_PREP_INTERRUPT
? 1 : 0);
2082 spin_lock_bh(&ppc440spe_chan
->lock
);
2083 slot_cnt
= ppc440spe_chan_xor_slot_count(len
, src_cnt
, &slots_per_op
);
2084 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
,
2087 group_start
= sw_desc
->group_head
;
2088 ppc440spe_desc_init_xor(group_start
, src_cnt
, flags
);
2089 ppc440spe_adma_set_dest(group_start
, dma_dest
, 0);
2091 ppc440spe_adma_memcpy_xor_set_src(group_start
,
2092 dma_src
[src_cnt
], src_cnt
);
2093 ppc440spe_desc_set_byte_count(group_start
, ppc440spe_chan
, len
);
2094 sw_desc
->unmap_len
= len
;
2095 sw_desc
->async_tx
.flags
= flags
;
2097 spin_unlock_bh(&ppc440spe_chan
->lock
);
2099 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2103 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot
*desc
,
2105 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor
*cursor
);
2108 * ppc440spe_adma_init_dma2rxor_slot -
2110 static void ppc440spe_adma_init_dma2rxor_slot(
2111 struct ppc440spe_adma_desc_slot
*desc
,
2112 dma_addr_t
*src
, int src_cnt
)
2116 /* initialize CDB */
2117 for (i
= 0; i
< src_cnt
; i
++) {
2118 ppc440spe_adma_dma2rxor_prep_src(desc
, &desc
->rxor_cursor
, i
,
2119 desc
->src_cnt
, (u32
)src
[i
]);
2124 * ppc440spe_dma01_prep_mult -
2125 * for Q operation where destination is also the source
2127 static struct ppc440spe_adma_desc_slot
*ppc440spe_dma01_prep_mult(
2128 struct ppc440spe_adma_chan
*ppc440spe_chan
,
2129 dma_addr_t
*dst
, int dst_cnt
, dma_addr_t
*src
, int src_cnt
,
2130 const unsigned char *scf
, size_t len
, unsigned long flags
)
2132 struct ppc440spe_adma_desc_slot
*sw_desc
= NULL
;
2133 unsigned long op
= 0;
2136 set_bit(PPC440SPE_DESC_WXOR
, &op
);
2139 spin_lock_bh(&ppc440spe_chan
->lock
);
2141 /* use WXOR, each descriptor occupies one slot */
2142 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
, 1);
2144 struct ppc440spe_adma_chan
*chan
;
2145 struct ppc440spe_adma_desc_slot
*iter
;
2146 struct dma_cdb
*hw_desc
;
2148 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
2149 set_bits(op
, &sw_desc
->flags
);
2150 sw_desc
->src_cnt
= src_cnt
;
2151 sw_desc
->dst_cnt
= dst_cnt
;
2152 /* First descriptor, zero data in the destination and copy it
2153 * to q page using MULTICAST transfer.
2155 iter
= list_first_entry(&sw_desc
->group_list
,
2156 struct ppc440spe_adma_desc_slot
,
2158 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2159 /* set 'next' pointer */
2160 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
2161 struct ppc440spe_adma_desc_slot
,
2163 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2164 hw_desc
= iter
->hw_desc
;
2165 hw_desc
->opc
= DMA_CDB_OPC_MULTICAST
;
2167 ppc440spe_desc_set_dest_addr(iter
, chan
,
2168 DMA_CUED_XOR_BASE
, dst
[0], 0);
2169 ppc440spe_desc_set_dest_addr(iter
, chan
, 0, dst
[1], 1);
2170 ppc440spe_desc_set_src_addr(iter
, chan
, 0, DMA_CUED_XOR_HB
,
2172 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
, len
);
2173 iter
->unmap_len
= len
;
2176 * Second descriptor, multiply data from the q page
2177 * and store the result in real destination.
2179 iter
= list_first_entry(&iter
->chain_node
,
2180 struct ppc440spe_adma_desc_slot
,
2182 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2183 iter
->hw_next
= NULL
;
2184 if (flags
& DMA_PREP_INTERRUPT
)
2185 set_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2187 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2189 hw_desc
= iter
->hw_desc
;
2190 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
2191 ppc440spe_desc_set_src_addr(iter
, chan
, 0,
2192 DMA_CUED_XOR_HB
, dst
[1]);
2193 ppc440spe_desc_set_dest_addr(iter
, chan
,
2194 DMA_CUED_XOR_BASE
, dst
[0], 0);
2196 ppc440spe_desc_set_src_mult(iter
, chan
, DMA_CUED_MULT1_OFF
,
2197 DMA_CDB_SG_DST1
, scf
[0]);
2198 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
, len
);
2199 iter
->unmap_len
= len
;
2200 sw_desc
->async_tx
.flags
= flags
;
2203 spin_unlock_bh(&ppc440spe_chan
->lock
);
2209 * ppc440spe_dma01_prep_sum_product -
2210 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2213 static struct ppc440spe_adma_desc_slot
*ppc440spe_dma01_prep_sum_product(
2214 struct ppc440spe_adma_chan
*ppc440spe_chan
,
2215 dma_addr_t
*dst
, dma_addr_t
*src
, int src_cnt
,
2216 const unsigned char *scf
, size_t len
, unsigned long flags
)
2218 struct ppc440spe_adma_desc_slot
*sw_desc
= NULL
;
2219 unsigned long op
= 0;
2222 set_bit(PPC440SPE_DESC_WXOR
, &op
);
2225 spin_lock_bh(&ppc440spe_chan
->lock
);
2227 /* WXOR, each descriptor occupies one slot */
2228 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
, 1);
2230 struct ppc440spe_adma_chan
*chan
;
2231 struct ppc440spe_adma_desc_slot
*iter
;
2232 struct dma_cdb
*hw_desc
;
2234 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
2235 set_bits(op
, &sw_desc
->flags
);
2236 sw_desc
->src_cnt
= src_cnt
;
2237 sw_desc
->dst_cnt
= 1;
2238 /* 1st descriptor, src[1] data to q page and zero destination */
2239 iter
= list_first_entry(&sw_desc
->group_list
,
2240 struct ppc440spe_adma_desc_slot
,
2242 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2243 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
2244 struct ppc440spe_adma_desc_slot
,
2246 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2247 hw_desc
= iter
->hw_desc
;
2248 hw_desc
->opc
= DMA_CDB_OPC_MULTICAST
;
2250 ppc440spe_desc_set_dest_addr(iter
, chan
, DMA_CUED_XOR_BASE
,
2252 ppc440spe_desc_set_dest_addr(iter
, chan
, 0,
2253 ppc440spe_chan
->qdest
, 1);
2254 ppc440spe_desc_set_src_addr(iter
, chan
, 0, DMA_CUED_XOR_HB
,
2256 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
, len
);
2257 iter
->unmap_len
= len
;
2259 /* 2nd descriptor, multiply src[1] data and store the
2260 * result in destination */
2261 iter
= list_first_entry(&iter
->chain_node
,
2262 struct ppc440spe_adma_desc_slot
,
2264 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2265 /* set 'next' pointer */
2266 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
2267 struct ppc440spe_adma_desc_slot
,
2269 if (flags
& DMA_PREP_INTERRUPT
)
2270 set_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2272 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2274 hw_desc
= iter
->hw_desc
;
2275 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
2276 ppc440spe_desc_set_src_addr(iter
, chan
, 0, DMA_CUED_XOR_HB
,
2277 ppc440spe_chan
->qdest
);
2278 ppc440spe_desc_set_dest_addr(iter
, chan
, DMA_CUED_XOR_BASE
,
2280 ppc440spe_desc_set_src_mult(iter
, chan
, DMA_CUED_MULT1_OFF
,
2281 DMA_CDB_SG_DST1
, scf
[1]);
2282 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
, len
);
2283 iter
->unmap_len
= len
;
2286 * 3rd descriptor, multiply src[0] data and xor it
2289 iter
= list_first_entry(&iter
->chain_node
,
2290 struct ppc440spe_adma_desc_slot
,
2292 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2293 iter
->hw_next
= NULL
;
2294 if (flags
& DMA_PREP_INTERRUPT
)
2295 set_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2297 clear_bit(PPC440SPE_DESC_INT
, &iter
->flags
);
2299 hw_desc
= iter
->hw_desc
;
2300 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
2301 ppc440spe_desc_set_src_addr(iter
, chan
, 0, DMA_CUED_XOR_HB
,
2303 ppc440spe_desc_set_dest_addr(iter
, chan
, DMA_CUED_XOR_BASE
,
2305 ppc440spe_desc_set_src_mult(iter
, chan
, DMA_CUED_MULT1_OFF
,
2306 DMA_CDB_SG_DST1
, scf
[0]);
2307 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
, len
);
2308 iter
->unmap_len
= len
;
2309 sw_desc
->async_tx
.flags
= flags
;
2312 spin_unlock_bh(&ppc440spe_chan
->lock
);
2317 static struct ppc440spe_adma_desc_slot
*ppc440spe_dma01_prep_pq(
2318 struct ppc440spe_adma_chan
*ppc440spe_chan
,
2319 dma_addr_t
*dst
, int dst_cnt
, dma_addr_t
*src
, int src_cnt
,
2320 const unsigned char *scf
, size_t len
, unsigned long flags
)
2323 struct ppc440spe_adma_desc_slot
*sw_desc
= NULL
, *iter
;
2324 unsigned long op
= 0;
2325 unsigned char mult
= 1;
2327 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2328 __func__
, dst_cnt
, src_cnt
, len
);
2329 /* select operations WXOR/RXOR depending on the
2330 * source addresses of operators and the number
2331 * of destinations (RXOR support only Q-parity calculations)
2333 set_bit(PPC440SPE_DESC_WXOR
, &op
);
2334 if (!test_and_set_bit(PPC440SPE_RXOR_RUN
, &ppc440spe_rxor_state
)) {
2337 * - there are more than 1 source,
2338 * - len is aligned on 512-byte boundary,
2339 * - source addresses fit to one of 4 possible regions.
2342 !(len
& MQ0_CF2H_RXOR_BS_MASK
) &&
2343 (src
[0] + len
) == src
[1]) {
2344 /* may do RXOR R1 R2 */
2345 set_bit(PPC440SPE_DESC_RXOR
, &op
);
2347 /* may try to enhance region of RXOR */
2348 if ((src
[1] + len
) == src
[2]) {
2349 /* do RXOR R1 R2 R3 */
2350 set_bit(PPC440SPE_DESC_RXOR123
,
2352 } else if ((src
[1] + len
* 2) == src
[2]) {
2353 /* do RXOR R1 R2 R4 */
2354 set_bit(PPC440SPE_DESC_RXOR124
, &op
);
2355 } else if ((src
[1] + len
* 3) == src
[2]) {
2356 /* do RXOR R1 R2 R5 */
2357 set_bit(PPC440SPE_DESC_RXOR125
,
2361 set_bit(PPC440SPE_DESC_RXOR12
,
2366 set_bit(PPC440SPE_DESC_RXOR12
, &op
);
2370 if (!test_bit(PPC440SPE_DESC_RXOR
, &op
)) {
2371 /* can not do this operation with RXOR */
2372 clear_bit(PPC440SPE_RXOR_RUN
,
2373 &ppc440spe_rxor_state
);
2375 /* can do; set block size right now */
2376 ppc440spe_desc_set_rxor_block_size(len
);
2380 /* Number of necessary slots depends on operation type selected */
2381 if (!test_bit(PPC440SPE_DESC_RXOR
, &op
)) {
2382 /* This is a WXOR only chain. Need descriptors for each
2383 * source to GF-XOR them with WXOR, and need descriptors
2384 * for each destination to zero them with WXOR
2388 if (flags
& DMA_PREP_ZERO_P
) {
2390 set_bit(PPC440SPE_ZERO_P
, &op
);
2392 if (flags
& DMA_PREP_ZERO_Q
) {
2394 set_bit(PPC440SPE_ZERO_Q
, &op
);
2397 /* Need 1/2 descriptor for RXOR operation, and
2398 * need (src_cnt - (2 or 3)) for WXOR of sources
2403 if (flags
& DMA_PREP_ZERO_P
)
2404 set_bit(PPC440SPE_ZERO_P
, &op
);
2405 if (flags
& DMA_PREP_ZERO_Q
)
2406 set_bit(PPC440SPE_ZERO_Q
, &op
);
2408 if (test_bit(PPC440SPE_DESC_RXOR12
, &op
))
2409 slot_cnt
+= src_cnt
- 2;
2411 slot_cnt
+= src_cnt
- 3;
2413 /* Thus we have either RXOR only chain or
2416 if (slot_cnt
== dst_cnt
)
2417 /* RXOR only chain */
2418 clear_bit(PPC440SPE_DESC_WXOR
, &op
);
2421 spin_lock_bh(&ppc440spe_chan
->lock
);
2422 /* for both RXOR/WXOR each descriptor occupies one slot */
2423 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
, 1);
2425 ppc440spe_desc_init_dma01pq(sw_desc
, dst_cnt
, src_cnt
,
2428 /* setup dst/src/mult */
2429 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2430 __func__
, dst
[0], dst
[1]);
2431 ppc440spe_adma_pq_set_dest(sw_desc
, dst
, flags
);
2433 ppc440spe_adma_pq_set_src(sw_desc
, src
[src_cnt
],
2436 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2437 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2438 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2439 * leads to zeroing source data after RXOR.
2440 * So, for P case set-up mult=1 explicitly.
2442 if (!(flags
& DMA_PREP_PQ_DISABLE_Q
))
2443 mult
= scf
[src_cnt
];
2444 ppc440spe_adma_pq_set_src_mult(sw_desc
,
2445 mult
, src_cnt
, dst_cnt
- 1);
2448 /* Setup byte count foreach slot just allocated */
2449 sw_desc
->async_tx
.flags
= flags
;
2450 list_for_each_entry(iter
, &sw_desc
->group_list
,
2452 ppc440spe_desc_set_byte_count(iter
,
2453 ppc440spe_chan
, len
);
2454 iter
->unmap_len
= len
;
2457 spin_unlock_bh(&ppc440spe_chan
->lock
);
2462 static struct ppc440spe_adma_desc_slot
*ppc440spe_dma2_prep_pq(
2463 struct ppc440spe_adma_chan
*ppc440spe_chan
,
2464 dma_addr_t
*dst
, int dst_cnt
, dma_addr_t
*src
, int src_cnt
,
2465 const unsigned char *scf
, size_t len
, unsigned long flags
)
2467 int slot_cnt
, descs_per_op
;
2468 struct ppc440spe_adma_desc_slot
*sw_desc
= NULL
, *iter
;
2469 unsigned long op
= 0;
2470 unsigned char mult
= 1;
2473 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2474 __func__, dst_cnt, src_cnt, len);*/
2476 spin_lock_bh(&ppc440spe_chan
->lock
);
2477 descs_per_op
= ppc440spe_dma2_pq_slot_count(src
, src_cnt
, len
);
2478 if (descs_per_op
< 0) {
2479 spin_unlock_bh(&ppc440spe_chan
->lock
);
2483 /* depending on number of sources we have 1 or 2 RXOR chains */
2484 slot_cnt
= descs_per_op
* dst_cnt
;
2486 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
, 1);
2489 sw_desc
->async_tx
.flags
= flags
;
2490 list_for_each_entry(iter
, &sw_desc
->group_list
, chain_node
) {
2491 ppc440spe_desc_init_dma2pq(iter
, dst_cnt
, src_cnt
,
2493 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
,
2495 iter
->unmap_len
= len
;
2497 ppc440spe_init_rxor_cursor(&(iter
->rxor_cursor
));
2498 iter
->rxor_cursor
.len
= len
;
2499 iter
->descs_per_op
= descs_per_op
;
2502 list_for_each_entry(iter
, &sw_desc
->group_list
, chain_node
) {
2504 if (op
% descs_per_op
== 0)
2505 ppc440spe_adma_init_dma2rxor_slot(iter
, src
,
2507 if (likely(!list_is_last(&iter
->chain_node
,
2508 &sw_desc
->group_list
))) {
2509 /* set 'next' pointer */
2511 list_entry(iter
->chain_node
.next
,
2512 struct ppc440spe_adma_desc_slot
,
2514 ppc440spe_xor_set_link(iter
, iter
->hw_next
);
2516 /* this is the last descriptor. */
2517 iter
->hw_next
= NULL
;
2521 /* fixup head descriptor */
2522 sw_desc
->dst_cnt
= dst_cnt
;
2523 if (flags
& DMA_PREP_ZERO_P
)
2524 set_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
);
2525 if (flags
& DMA_PREP_ZERO_Q
)
2526 set_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
);
2528 /* setup dst/src/mult */
2529 ppc440spe_adma_pq_set_dest(sw_desc
, dst
, flags
);
2532 /* handle descriptors (if dst_cnt == 2) inside
2533 * the ppc440spe_adma_pq_set_srcxxx() functions
2535 ppc440spe_adma_pq_set_src(sw_desc
, src
[src_cnt
],
2537 if (!(flags
& DMA_PREP_PQ_DISABLE_Q
))
2538 mult
= scf
[src_cnt
];
2539 ppc440spe_adma_pq_set_src_mult(sw_desc
,
2540 mult
, src_cnt
, dst_cnt
- 1);
2543 spin_unlock_bh(&ppc440spe_chan
->lock
);
2544 ppc440spe_desc_set_rxor_block_size(len
);
2549 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2551 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_pq(
2552 struct dma_chan
*chan
, dma_addr_t
*dst
, dma_addr_t
*src
,
2553 unsigned int src_cnt
, const unsigned char *scf
,
2554 size_t len
, unsigned long flags
)
2556 struct ppc440spe_adma_chan
*ppc440spe_chan
;
2557 struct ppc440spe_adma_desc_slot
*sw_desc
= NULL
;
2560 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
2562 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan
->device
->id
,
2563 dst
, src
, src_cnt
));
2565 BUG_ON(len
> PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
);
2568 if (src_cnt
== 1 && dst
[1] == src
[0]) {
2571 /* dst[1] is real destination (Q) */
2573 /* this is the page to multicast source data to */
2574 dest
[1] = ppc440spe_chan
->qdest
;
2575 sw_desc
= ppc440spe_dma01_prep_mult(ppc440spe_chan
,
2576 dest
, 2, src
, src_cnt
, scf
, len
, flags
);
2577 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2580 if (src_cnt
== 2 && dst
[1] == src
[1]) {
2581 sw_desc
= ppc440spe_dma01_prep_sum_product(ppc440spe_chan
,
2582 &dst
[1], src
, 2, scf
, len
, flags
);
2583 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2586 if (!(flags
& DMA_PREP_PQ_DISABLE_P
)) {
2589 flags
|= DMA_PREP_ZERO_P
;
2592 if (!(flags
& DMA_PREP_PQ_DISABLE_Q
)) {
2595 flags
|= DMA_PREP_ZERO_Q
;
2600 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
2601 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2602 ppc440spe_chan
->device
->id
, __func__
, src_cnt
, len
,
2603 flags
& DMA_PREP_INTERRUPT
? 1 : 0);
2605 switch (ppc440spe_chan
->device
->id
) {
2606 case PPC440SPE_DMA0_ID
:
2607 case PPC440SPE_DMA1_ID
:
2608 sw_desc
= ppc440spe_dma01_prep_pq(ppc440spe_chan
,
2609 dst
, dst_cnt
, src
, src_cnt
, scf
,
2613 case PPC440SPE_XOR_ID
:
2614 sw_desc
= ppc440spe_dma2_prep_pq(ppc440spe_chan
,
2615 dst
, dst_cnt
, src
, src_cnt
, scf
,
2620 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2624 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2625 * a PQ_ZERO_SUM operation
2627 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_pqzero_sum(
2628 struct dma_chan
*chan
, dma_addr_t
*pq
, dma_addr_t
*src
,
2629 unsigned int src_cnt
, const unsigned char *scf
, size_t len
,
2630 enum sum_check_flags
*pqres
, unsigned long flags
)
2632 struct ppc440spe_adma_chan
*ppc440spe_chan
;
2633 struct ppc440spe_adma_desc_slot
*sw_desc
, *iter
;
2634 dma_addr_t pdest
, qdest
;
2635 int slot_cnt
, slots_per_op
, idst
, dst_cnt
;
2637 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
2639 if (flags
& DMA_PREP_PQ_DISABLE_P
)
2644 if (flags
& DMA_PREP_PQ_DISABLE_Q
)
2649 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan
->device
->id
,
2650 src
, src_cnt
, scf
));
2652 /* Always use WXOR for P/Q calculations (two destinations).
2653 * Need 1 or 2 extra slots to verify results are zero.
2655 idst
= dst_cnt
= (pdest
&& qdest
) ? 2 : 1;
2657 /* One additional slot per destination to clone P/Q
2658 * before calculation (we have to preserve destinations).
2660 slot_cnt
= src_cnt
+ dst_cnt
* 2;
2663 spin_lock_bh(&ppc440spe_chan
->lock
);
2664 sw_desc
= ppc440spe_adma_alloc_slots(ppc440spe_chan
, slot_cnt
,
2667 ppc440spe_desc_init_dma01pqzero_sum(sw_desc
, dst_cnt
, src_cnt
);
2669 /* Setup byte count for each slot just allocated */
2670 sw_desc
->async_tx
.flags
= flags
;
2671 list_for_each_entry(iter
, &sw_desc
->group_list
, chain_node
) {
2672 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
,
2674 iter
->unmap_len
= len
;
2678 struct dma_cdb
*hw_desc
;
2679 struct ppc440spe_adma_chan
*chan
;
2681 iter
= sw_desc
->group_head
;
2682 chan
= to_ppc440spe_adma_chan(iter
->async_tx
.chan
);
2683 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2684 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
2685 struct ppc440spe_adma_desc_slot
,
2687 hw_desc
= iter
->hw_desc
;
2688 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
2691 ppc440spe_desc_set_dest_addr(iter
, chan
, 0,
2692 ppc440spe_chan
->pdest
, 0);
2693 ppc440spe_desc_set_src_addr(iter
, chan
, 0, 0, pdest
);
2694 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
,
2696 iter
->unmap_len
= 0;
2697 /* override pdest to preserve original P */
2698 pdest
= ppc440spe_chan
->pdest
;
2701 struct dma_cdb
*hw_desc
;
2702 struct ppc440spe_adma_chan
*chan
;
2704 iter
= list_first_entry(&sw_desc
->group_list
,
2705 struct ppc440spe_adma_desc_slot
,
2707 chan
= to_ppc440spe_adma_chan(iter
->async_tx
.chan
);
2710 iter
= list_entry(iter
->chain_node
.next
,
2711 struct ppc440spe_adma_desc_slot
,
2715 memset(iter
->hw_desc
, 0, sizeof(struct dma_cdb
));
2716 iter
->hw_next
= list_entry(iter
->chain_node
.next
,
2717 struct ppc440spe_adma_desc_slot
,
2719 hw_desc
= iter
->hw_desc
;
2720 hw_desc
->opc
= DMA_CDB_OPC_MV_SG1_SG2
;
2723 ppc440spe_desc_set_dest_addr(iter
, chan
, 0,
2724 ppc440spe_chan
->qdest
, 0);
2725 ppc440spe_desc_set_src_addr(iter
, chan
, 0, 0, qdest
);
2726 ppc440spe_desc_set_byte_count(iter
, ppc440spe_chan
,
2728 iter
->unmap_len
= 0;
2729 /* override qdest to preserve original Q */
2730 qdest
= ppc440spe_chan
->qdest
;
2733 /* Setup destinations for P/Q ops */
2734 ppc440spe_adma_pqzero_sum_set_dest(sw_desc
, pdest
, qdest
);
2736 /* Setup zero QWORDs into DCHECK CDBs */
2738 list_for_each_entry_reverse(iter
, &sw_desc
->group_list
,
2741 * The last CDB corresponds to Q-parity check,
2742 * the one before last CDB corresponds
2745 if (idst
== DMA_DEST_MAX_NUM
) {
2746 if (idst
== dst_cnt
) {
2747 set_bit(PPC440SPE_DESC_QCHECK
,
2750 set_bit(PPC440SPE_DESC_PCHECK
,
2755 set_bit(PPC440SPE_DESC_QCHECK
,
2758 set_bit(PPC440SPE_DESC_PCHECK
,
2762 iter
->xor_check_result
= pqres
;
2765 * set it to zero, if check fail then result will
2768 *iter
->xor_check_result
= 0;
2769 ppc440spe_desc_set_dcheck(iter
, ppc440spe_chan
,
2776 /* Setup sources and mults for P/Q ops */
2777 list_for_each_entry_continue_reverse(iter
, &sw_desc
->group_list
,
2779 struct ppc440spe_adma_chan
*chan
;
2782 chan
= to_ppc440spe_adma_chan(iter
->async_tx
.chan
);
2783 ppc440spe_desc_set_src_addr(iter
, chan
, 0,
2787 mult_dst
= (dst_cnt
- 1) ? DMA_CDB_SG_DST2
:
2789 ppc440spe_desc_set_src_mult(iter
, chan
,
2798 spin_unlock_bh(&ppc440spe_chan
->lock
);
2799 return sw_desc
? &sw_desc
->async_tx
: NULL
;
2803 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2804 * XOR ZERO_SUM operation
2806 static struct dma_async_tx_descriptor
*ppc440spe_adma_prep_dma_xor_zero_sum(
2807 struct dma_chan
*chan
, dma_addr_t
*src
, unsigned int src_cnt
,
2808 size_t len
, enum sum_check_flags
*result
, unsigned long flags
)
2810 struct dma_async_tx_descriptor
*tx
;
2813 /* validate P, disable Q */
2816 flags
|= DMA_PREP_PQ_DISABLE_Q
;
2818 tx
= ppc440spe_adma_prep_dma_pqzero_sum(chan
, pq
, &src
[1],
2819 src_cnt
- 1, 0, len
,
2825 * ppc440spe_adma_set_dest - set destination address into descriptor
2827 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot
*sw_desc
,
2828 dma_addr_t addr
, int index
)
2830 struct ppc440spe_adma_chan
*chan
;
2832 BUG_ON(index
>= sw_desc
->dst_cnt
);
2834 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
2836 switch (chan
->device
->id
) {
2837 case PPC440SPE_DMA0_ID
:
2838 case PPC440SPE_DMA1_ID
:
2839 /* to do: support transfers lengths >
2840 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2842 ppc440spe_desc_set_dest_addr(sw_desc
->group_head
,
2843 chan
, 0, addr
, index
);
2845 case PPC440SPE_XOR_ID
:
2846 sw_desc
= ppc440spe_get_group_entry(sw_desc
, index
);
2847 ppc440spe_desc_set_dest_addr(sw_desc
,
2848 chan
, 0, addr
, index
);
2853 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot
*iter
,
2854 struct ppc440spe_adma_chan
*chan
, dma_addr_t addr
)
2856 /* To clear destinations update the descriptor
2857 * (P or Q depending on index) as follows:
2858 * addr is destination (0 corresponds to SG2):
2860 ppc440spe_desc_set_dest_addr(iter
, chan
, DMA_CUED_XOR_BASE
, addr
, 0);
2862 /* ... and the addr is source: */
2863 ppc440spe_desc_set_src_addr(iter
, chan
, 0, DMA_CUED_XOR_HB
, addr
);
2865 /* addr is always SG2 then the mult is always DST1 */
2866 ppc440spe_desc_set_src_mult(iter
, chan
, DMA_CUED_MULT1_OFF
,
2867 DMA_CDB_SG_DST1
, 1);
2871 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2872 * for the PQXOR operation
2874 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot
*sw_desc
,
2875 dma_addr_t
*addrs
, unsigned long flags
)
2877 struct ppc440spe_adma_desc_slot
*iter
;
2878 struct ppc440spe_adma_chan
*chan
;
2879 dma_addr_t paddr
, qaddr
;
2880 dma_addr_t addr
= 0, ppath
, qpath
;
2883 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
2885 if (flags
& DMA_PREP_PQ_DISABLE_P
)
2890 if (flags
& DMA_PREP_PQ_DISABLE_Q
)
2895 if (!paddr
|| !qaddr
)
2896 addr
= paddr
? paddr
: qaddr
;
2898 switch (chan
->device
->id
) {
2899 case PPC440SPE_DMA0_ID
:
2900 case PPC440SPE_DMA1_ID
:
2901 /* walk through the WXOR source list and set P/Q-destinations
2904 if (!test_bit(PPC440SPE_DESC_RXOR
, &sw_desc
->flags
)) {
2905 /* This is WXOR-only chain; may have 1/2 zero descs */
2906 if (test_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
))
2908 if (test_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
))
2911 iter
= ppc440spe_get_group_entry(sw_desc
, index
);
2913 /* one destination */
2914 list_for_each_entry_from(iter
,
2915 &sw_desc
->group_list
, chain_node
)
2916 ppc440spe_desc_set_dest_addr(iter
, chan
,
2917 DMA_CUED_XOR_BASE
, addr
, 0);
2919 /* two destinations */
2920 list_for_each_entry_from(iter
,
2921 &sw_desc
->group_list
, chain_node
) {
2922 ppc440spe_desc_set_dest_addr(iter
, chan
,
2923 DMA_CUED_XOR_BASE
, paddr
, 0);
2924 ppc440spe_desc_set_dest_addr(iter
, chan
,
2925 DMA_CUED_XOR_BASE
, qaddr
, 1);
2930 /* To clear destinations update the descriptor
2931 * (1st,2nd, or both depending on flags)
2934 if (test_bit(PPC440SPE_ZERO_P
,
2936 iter
= ppc440spe_get_group_entry(
2938 ppc440spe_adma_pq_zero_op(iter
, chan
,
2942 if (test_bit(PPC440SPE_ZERO_Q
,
2944 iter
= ppc440spe_get_group_entry(
2946 ppc440spe_adma_pq_zero_op(iter
, chan
,
2953 /* This is RXOR-only or RXOR/WXOR mixed chain */
2955 /* If we want to include destination into calculations,
2956 * then make dest addresses cued with mult=1 (XOR).
2958 ppath
= test_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
) ?
2961 (1 << DMA_CUED_MULT1_OFF
);
2962 qpath
= test_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
) ?
2965 (1 << DMA_CUED_MULT1_OFF
);
2967 /* Setup destination(s) in RXOR slot(s) */
2968 iter
= ppc440spe_get_group_entry(sw_desc
, index
++);
2969 ppc440spe_desc_set_dest_addr(iter
, chan
,
2970 paddr
? ppath
: qpath
,
2971 paddr
? paddr
: qaddr
, 0);
2973 /* two destinations */
2974 iter
= ppc440spe_get_group_entry(sw_desc
,
2976 ppc440spe_desc_set_dest_addr(iter
, chan
,
2980 if (test_bit(PPC440SPE_DESC_WXOR
, &sw_desc
->flags
)) {
2981 /* Setup destination(s) in remaining WXOR
2984 iter
= ppc440spe_get_group_entry(sw_desc
,
2987 /* one destination */
2988 list_for_each_entry_from(iter
,
2989 &sw_desc
->group_list
,
2991 ppc440spe_desc_set_dest_addr(
2997 /* two destinations */
2998 list_for_each_entry_from(iter
,
2999 &sw_desc
->group_list
,
3001 ppc440spe_desc_set_dest_addr(
3005 ppc440spe_desc_set_dest_addr(
3016 case PPC440SPE_XOR_ID
:
3017 /* DMA2 descriptors have only 1 destination, so there are
3018 * two chains - one for each dest.
3019 * If we want to include destination into calculations,
3020 * then make dest addresses cued with mult=1 (XOR).
3022 ppath
= test_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
) ?
3025 (1 << DMA_CUED_MULT1_OFF
);
3027 qpath
= test_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
) ?
3030 (1 << DMA_CUED_MULT1_OFF
);
3032 iter
= ppc440spe_get_group_entry(sw_desc
, 0);
3033 for (i
= 0; i
< sw_desc
->descs_per_op
; i
++) {
3034 ppc440spe_desc_set_dest_addr(iter
, chan
,
3035 paddr
? ppath
: qpath
,
3036 paddr
? paddr
: qaddr
, 0);
3037 iter
= list_entry(iter
->chain_node
.next
,
3038 struct ppc440spe_adma_desc_slot
,
3043 /* Two destinations; setup Q here */
3044 iter
= ppc440spe_get_group_entry(sw_desc
,
3045 sw_desc
->descs_per_op
);
3046 for (i
= 0; i
< sw_desc
->descs_per_op
; i
++) {
3047 ppc440spe_desc_set_dest_addr(iter
,
3048 chan
, qpath
, qaddr
, 0);
3049 iter
= list_entry(iter
->chain_node
.next
,
3050 struct ppc440spe_adma_desc_slot
,
3060 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3061 * for the PQ_ZERO_SUM operation
3063 static void ppc440spe_adma_pqzero_sum_set_dest(
3064 struct ppc440spe_adma_desc_slot
*sw_desc
,
3065 dma_addr_t paddr
, dma_addr_t qaddr
)
3067 struct ppc440spe_adma_desc_slot
*iter
, *end
;
3068 struct ppc440spe_adma_chan
*chan
;
3069 dma_addr_t addr
= 0;
3072 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
3074 /* walk through the WXOR source list and set P/Q-destinations
3077 idx
= (paddr
&& qaddr
) ? 2 : 1;
3079 list_for_each_entry_reverse(end
, &sw_desc
->group_list
,
3085 idx
= (paddr
&& qaddr
) ? 2 : 1;
3086 iter
= ppc440spe_get_group_entry(sw_desc
, idx
);
3088 if (paddr
&& qaddr
) {
3089 /* two destinations */
3090 list_for_each_entry_from(iter
, &sw_desc
->group_list
,
3092 if (unlikely(iter
== end
))
3094 ppc440spe_desc_set_dest_addr(iter
, chan
,
3095 DMA_CUED_XOR_BASE
, paddr
, 0);
3096 ppc440spe_desc_set_dest_addr(iter
, chan
,
3097 DMA_CUED_XOR_BASE
, qaddr
, 1);
3100 /* one destination */
3101 addr
= paddr
? paddr
: qaddr
;
3102 list_for_each_entry_from(iter
, &sw_desc
->group_list
,
3104 if (unlikely(iter
== end
))
3106 ppc440spe_desc_set_dest_addr(iter
, chan
,
3107 DMA_CUED_XOR_BASE
, addr
, 0);
3111 /* The remaining descriptors are DATACHECK. These have no need in
3112 * destination. Actually, these destinations are used there
3113 * as sources for check operation. So, set addr as source.
3115 ppc440spe_desc_set_src_addr(end
, chan
, 0, 0, addr
? addr
: paddr
);
3118 end
= list_entry(end
->chain_node
.next
,
3119 struct ppc440spe_adma_desc_slot
, chain_node
);
3120 ppc440spe_desc_set_src_addr(end
, chan
, 0, 0, qaddr
);
3125 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3127 static inline void ppc440spe_desc_set_xor_src_cnt(
3128 struct ppc440spe_adma_desc_slot
*desc
,
3131 struct xor_cb
*hw_desc
= desc
->hw_desc
;
3133 hw_desc
->cbc
&= ~XOR_CDCR_OAC_MSK
;
3134 hw_desc
->cbc
|= src_cnt
;
3138 * ppc440spe_adma_pq_set_src - set source address into descriptor
3140 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot
*sw_desc
,
3141 dma_addr_t addr
, int index
)
3143 struct ppc440spe_adma_chan
*chan
;
3144 dma_addr_t haddr
= 0;
3145 struct ppc440spe_adma_desc_slot
*iter
= NULL
;
3147 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
3149 switch (chan
->device
->id
) {
3150 case PPC440SPE_DMA0_ID
:
3151 case PPC440SPE_DMA1_ID
:
3152 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3154 if (test_bit(PPC440SPE_DESC_RXOR
, &sw_desc
->flags
)) {
3155 /* RXOR-only or RXOR/WXOR operation */
3156 int iskip
= test_bit(PPC440SPE_DESC_RXOR12
,
3157 &sw_desc
->flags
) ? 2 : 3;
3160 /* 1st slot (RXOR) */
3161 /* setup sources region (R1-2-3, R1-2-4,
3164 if (test_bit(PPC440SPE_DESC_RXOR12
,
3166 haddr
= DMA_RXOR12
<<
3167 DMA_CUED_REGION_OFF
;
3168 else if (test_bit(PPC440SPE_DESC_RXOR123
,
3170 haddr
= DMA_RXOR123
<<
3171 DMA_CUED_REGION_OFF
;
3172 else if (test_bit(PPC440SPE_DESC_RXOR124
,
3174 haddr
= DMA_RXOR124
<<
3175 DMA_CUED_REGION_OFF
;
3176 else if (test_bit(PPC440SPE_DESC_RXOR125
,
3178 haddr
= DMA_RXOR125
<<
3179 DMA_CUED_REGION_OFF
;
3182 haddr
|= DMA_CUED_XOR_BASE
;
3183 iter
= ppc440spe_get_group_entry(sw_desc
, 0);
3184 } else if (index
< iskip
) {
3186 * shall actually set source address only once
3187 * instead of first <iskip>
3191 /* 2nd/3d and next slots (WXOR);
3192 * skip first slot with RXOR
3194 haddr
= DMA_CUED_XOR_HB
;
3195 iter
= ppc440spe_get_group_entry(sw_desc
,
3196 index
- iskip
+ sw_desc
->dst_cnt
);
3201 /* WXOR-only operation; skip first slots with
3202 * zeroing destinations
3204 if (test_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
))
3206 if (test_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
))
3209 haddr
= DMA_CUED_XOR_HB
;
3210 iter
= ppc440spe_get_group_entry(sw_desc
,
3215 ppc440spe_desc_set_src_addr(iter
, chan
, 0, haddr
, addr
);
3218 test_bit(PPC440SPE_DESC_RXOR
, &sw_desc
->flags
) &&
3219 sw_desc
->dst_cnt
== 2) {
3220 /* if we have two destinations for RXOR, then
3221 * setup source in the second descr too
3223 iter
= ppc440spe_get_group_entry(sw_desc
, 1);
3224 ppc440spe_desc_set_src_addr(iter
, chan
, 0,
3230 case PPC440SPE_XOR_ID
:
3231 /* DMA2 may do Biskup */
3232 iter
= sw_desc
->group_head
;
3233 if (iter
->dst_cnt
== 2) {
3234 /* both P & Q calculations required; set P src here */
3235 ppc440spe_adma_dma2rxor_set_src(iter
, index
, addr
);
3238 iter
= ppc440spe_get_group_entry(sw_desc
,
3239 sw_desc
->descs_per_op
);
3241 ppc440spe_adma_dma2rxor_set_src(iter
, index
, addr
);
3247 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3249 static void ppc440spe_adma_memcpy_xor_set_src(
3250 struct ppc440spe_adma_desc_slot
*sw_desc
,
3251 dma_addr_t addr
, int index
)
3253 struct ppc440spe_adma_chan
*chan
;
3255 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
3256 sw_desc
= sw_desc
->group_head
;
3258 if (likely(sw_desc
))
3259 ppc440spe_desc_set_src_addr(sw_desc
, chan
, index
, 0, addr
);
3263 * ppc440spe_adma_dma2rxor_inc_addr -
3265 static void ppc440spe_adma_dma2rxor_inc_addr(
3266 struct ppc440spe_adma_desc_slot
*desc
,
3267 struct ppc440spe_rxor
*cursor
, int index
, int src_cnt
)
3269 cursor
->addr_count
++;
3270 if (index
== src_cnt
- 1) {
3271 ppc440spe_desc_set_xor_src_cnt(desc
, cursor
->addr_count
);
3272 } else if (cursor
->addr_count
== XOR_MAX_OPS
) {
3273 ppc440spe_desc_set_xor_src_cnt(desc
, cursor
->addr_count
);
3274 cursor
->addr_count
= 0;
3275 cursor
->desc_count
++;
3280 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3282 static int ppc440spe_adma_dma2rxor_prep_src(
3283 struct ppc440spe_adma_desc_slot
*hdesc
,
3284 struct ppc440spe_rxor
*cursor
, int index
,
3285 int src_cnt
, u32 addr
)
3289 struct ppc440spe_adma_desc_slot
*desc
= hdesc
;
3292 for (i
= 0; i
< cursor
->desc_count
; i
++) {
3293 desc
= list_entry(hdesc
->chain_node
.next
,
3294 struct ppc440spe_adma_desc_slot
,
3298 switch (cursor
->state
) {
3300 if (addr
== cursor
->addrl
+ cursor
->len
) {
3303 cursor
->xor_count
++;
3304 if (index
== src_cnt
-1) {
3305 ppc440spe_rxor_set_region(desc
,
3307 DMA_RXOR12
<< DMA_CUED_REGION_OFF
);
3308 ppc440spe_adma_dma2rxor_inc_addr(
3309 desc
, cursor
, index
, src_cnt
);
3311 } else if (cursor
->addrl
== addr
+ cursor
->len
) {
3314 cursor
->xor_count
++;
3315 set_bit(cursor
->addr_count
, &desc
->reverse_flags
[0]);
3316 if (index
== src_cnt
-1) {
3317 ppc440spe_rxor_set_region(desc
,
3319 DMA_RXOR12
<< DMA_CUED_REGION_OFF
);
3320 ppc440spe_adma_dma2rxor_inc_addr(
3321 desc
, cursor
, index
, src_cnt
);
3324 printk(KERN_ERR
"Cannot build "
3325 "DMA2 RXOR command block.\n");
3330 sign
= test_bit(cursor
->addr_count
,
3331 desc
->reverse_flags
)
3333 if (index
== src_cnt
-2 || (sign
== -1
3334 && addr
!= cursor
->addrl
- 2*cursor
->len
)) {
3336 cursor
->xor_count
= 1;
3337 cursor
->addrl
= addr
;
3338 ppc440spe_rxor_set_region(desc
,
3340 DMA_RXOR12
<< DMA_CUED_REGION_OFF
);
3341 ppc440spe_adma_dma2rxor_inc_addr(
3342 desc
, cursor
, index
, src_cnt
);
3343 } else if (addr
== cursor
->addrl
+ 2*sign
*cursor
->len
) {
3345 cursor
->xor_count
= 0;
3346 ppc440spe_rxor_set_region(desc
,
3348 DMA_RXOR123
<< DMA_CUED_REGION_OFF
);
3349 if (index
== src_cnt
-1) {
3350 ppc440spe_adma_dma2rxor_inc_addr(
3351 desc
, cursor
, index
, src_cnt
);
3353 } else if (addr
== cursor
->addrl
+ 3*cursor
->len
) {
3355 cursor
->xor_count
= 0;
3356 ppc440spe_rxor_set_region(desc
,
3358 DMA_RXOR124
<< DMA_CUED_REGION_OFF
);
3359 if (index
== src_cnt
-1) {
3360 ppc440spe_adma_dma2rxor_inc_addr(
3361 desc
, cursor
, index
, src_cnt
);
3363 } else if (addr
== cursor
->addrl
+ 4*cursor
->len
) {
3365 cursor
->xor_count
= 0;
3366 ppc440spe_rxor_set_region(desc
,
3368 DMA_RXOR125
<< DMA_CUED_REGION_OFF
);
3369 if (index
== src_cnt
-1) {
3370 ppc440spe_adma_dma2rxor_inc_addr(
3371 desc
, cursor
, index
, src_cnt
);
3375 cursor
->xor_count
= 1;
3376 cursor
->addrl
= addr
;
3377 ppc440spe_rxor_set_region(desc
,
3379 DMA_RXOR12
<< DMA_CUED_REGION_OFF
);
3380 ppc440spe_adma_dma2rxor_inc_addr(
3381 desc
, cursor
, index
, src_cnt
);
3386 cursor
->addrl
= addr
;
3387 cursor
->xor_count
++;
3389 ppc440spe_adma_dma2rxor_inc_addr(
3390 desc
, cursor
, index
, src_cnt
);
3399 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3400 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3402 static void ppc440spe_adma_dma2rxor_set_src(
3403 struct ppc440spe_adma_desc_slot
*desc
,
3404 int index
, dma_addr_t addr
)
3406 struct xor_cb
*xcb
= desc
->hw_desc
;
3407 int k
= 0, op
= 0, lop
= 0;
3409 /* get the RXOR operand which corresponds to index addr */
3410 while (op
<= index
) {
3412 if (k
== XOR_MAX_OPS
) {
3414 desc
= list_entry(desc
->chain_node
.next
,
3415 struct ppc440spe_adma_desc_slot
, chain_node
);
3416 xcb
= desc
->hw_desc
;
3419 if ((xcb
->ops
[k
++].h
& (DMA_RXOR12
<< DMA_CUED_REGION_OFF
)) ==
3420 (DMA_RXOR12
<< DMA_CUED_REGION_OFF
))
3428 if (test_bit(k
-1, desc
->reverse_flags
)) {
3429 /* reverse operand order; put last op in RXOR group */
3430 if (index
== op
- 1)
3431 ppc440spe_rxor_set_src(desc
, k
- 1, addr
);
3433 /* direct operand order; put first op in RXOR group */
3435 ppc440spe_rxor_set_src(desc
, k
- 1, addr
);
3440 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3441 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3443 static void ppc440spe_adma_dma2rxor_set_mult(
3444 struct ppc440spe_adma_desc_slot
*desc
,
3447 struct xor_cb
*xcb
= desc
->hw_desc
;
3448 int k
= 0, op
= 0, lop
= 0;
3450 /* get the RXOR operand which corresponds to index mult */
3451 while (op
<= index
) {
3453 if (k
== XOR_MAX_OPS
) {
3455 desc
= list_entry(desc
->chain_node
.next
,
3456 struct ppc440spe_adma_desc_slot
,
3458 xcb
= desc
->hw_desc
;
3461 if ((xcb
->ops
[k
++].h
& (DMA_RXOR12
<< DMA_CUED_REGION_OFF
)) ==
3462 (DMA_RXOR12
<< DMA_CUED_REGION_OFF
))
3469 if (test_bit(k
-1, desc
->reverse_flags
)) {
3471 ppc440spe_rxor_set_mult(desc
, k
- 1, op
- index
- 1, mult
);
3474 ppc440spe_rxor_set_mult(desc
, k
- 1, index
- lop
, mult
);
3479 * ppc440spe_init_rxor_cursor -
3481 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor
*cursor
)
3483 memset(cursor
, 0, sizeof(struct ppc440spe_rxor
));
3488 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3489 * descriptor for the PQXOR operation
3491 static void ppc440spe_adma_pq_set_src_mult(
3492 struct ppc440spe_adma_desc_slot
*sw_desc
,
3493 unsigned char mult
, int index
, int dst_pos
)
3495 struct ppc440spe_adma_chan
*chan
;
3496 u32 mult_idx
, mult_dst
;
3497 struct ppc440spe_adma_desc_slot
*iter
= NULL
, *iter1
= NULL
;
3499 chan
= to_ppc440spe_adma_chan(sw_desc
->async_tx
.chan
);
3501 switch (chan
->device
->id
) {
3502 case PPC440SPE_DMA0_ID
:
3503 case PPC440SPE_DMA1_ID
:
3504 if (test_bit(PPC440SPE_DESC_RXOR
, &sw_desc
->flags
)) {
3505 int region
= test_bit(PPC440SPE_DESC_RXOR12
,
3506 &sw_desc
->flags
) ? 2 : 3;
3508 if (index
< region
) {
3509 /* RXOR multipliers */
3510 iter
= ppc440spe_get_group_entry(sw_desc
,
3511 sw_desc
->dst_cnt
- 1);
3512 if (sw_desc
->dst_cnt
== 2)
3513 iter1
= ppc440spe_get_group_entry(
3516 mult_idx
= DMA_CUED_MULT1_OFF
+ (index
<< 3);
3517 mult_dst
= DMA_CDB_SG_SRC
;
3519 /* WXOR multiplier */
3520 iter
= ppc440spe_get_group_entry(sw_desc
,
3523 mult_idx
= DMA_CUED_MULT1_OFF
;
3524 mult_dst
= dst_pos
? DMA_CDB_SG_DST2
:
3531 * skip first slots with destinations (if ZERO_DST has
3534 if (test_bit(PPC440SPE_ZERO_P
, &sw_desc
->flags
))
3536 if (test_bit(PPC440SPE_ZERO_Q
, &sw_desc
->flags
))
3539 iter
= ppc440spe_get_group_entry(sw_desc
, index
+ znum
);
3540 mult_idx
= DMA_CUED_MULT1_OFF
;
3541 mult_dst
= dst_pos
? DMA_CDB_SG_DST2
: DMA_CDB_SG_DST1
;
3545 ppc440spe_desc_set_src_mult(iter
, chan
,
3546 mult_idx
, mult_dst
, mult
);
3548 if (unlikely(iter1
)) {
3549 /* if we have two destinations for RXOR, then
3550 * we've just set Q mult. Set-up P now.
3552 ppc440spe_desc_set_src_mult(iter1
, chan
,
3553 mult_idx
, mult_dst
, 1);
3559 case PPC440SPE_XOR_ID
:
3560 iter
= sw_desc
->group_head
;
3561 if (sw_desc
->dst_cnt
== 2) {
3562 /* both P & Q calculations required; set P mult here */
3563 ppc440spe_adma_dma2rxor_set_mult(iter
, index
, 1);
3565 /* and then set Q mult */
3566 iter
= ppc440spe_get_group_entry(sw_desc
,
3567 sw_desc
->descs_per_op
);
3569 ppc440spe_adma_dma2rxor_set_mult(iter
, index
, mult
);
3575 * ppc440spe_adma_free_chan_resources - free the resources allocated
3577 static void ppc440spe_adma_free_chan_resources(struct dma_chan
*chan
)
3579 struct ppc440spe_adma_chan
*ppc440spe_chan
;
3580 struct ppc440spe_adma_desc_slot
*iter
, *_iter
;
3581 int in_use_descs
= 0;
3583 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
3584 ppc440spe_adma_slot_cleanup(ppc440spe_chan
);
3586 spin_lock_bh(&ppc440spe_chan
->lock
);
3587 list_for_each_entry_safe(iter
, _iter
, &ppc440spe_chan
->chain
,
3590 list_del(&iter
->chain_node
);
3592 list_for_each_entry_safe_reverse(iter
, _iter
,
3593 &ppc440spe_chan
->all_slots
, slot_node
) {
3594 list_del(&iter
->slot_node
);
3596 ppc440spe_chan
->slots_allocated
--;
3598 ppc440spe_chan
->last_used
= NULL
;
3600 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
3601 "ppc440spe adma%d %s slots_allocated %d\n",
3602 ppc440spe_chan
->device
->id
,
3603 __func__
, ppc440spe_chan
->slots_allocated
);
3604 spin_unlock_bh(&ppc440spe_chan
->lock
);
3606 /* one is ok since we left it on there on purpose */
3607 if (in_use_descs
> 1)
3608 printk(KERN_ERR
"SPE: Freeing %d in use descriptors!\n",
3613 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3614 * @chan: ADMA channel handle
3615 * @cookie: ADMA transaction identifier
3616 * @txstate: a holder for the current state of the channel
3618 static enum dma_status
ppc440spe_adma_tx_status(struct dma_chan
*chan
,
3619 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
3621 struct ppc440spe_adma_chan
*ppc440spe_chan
;
3622 enum dma_status ret
;
3624 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
3625 ret
= dma_cookie_status(chan
, cookie
, txstate
);
3626 if (ret
== DMA_SUCCESS
)
3629 ppc440spe_adma_slot_cleanup(ppc440spe_chan
);
3631 return dma_cookie_status(chan
, cookie
, txstate
);
3635 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3637 static irqreturn_t
ppc440spe_adma_eot_handler(int irq
, void *data
)
3639 struct ppc440spe_adma_chan
*chan
= data
;
3641 dev_dbg(chan
->device
->common
.dev
,
3642 "ppc440spe adma%d: %s\n", chan
->device
->id
, __func__
);
3644 tasklet_schedule(&chan
->irq_tasklet
);
3645 ppc440spe_adma_device_clear_eot_status(chan
);
3651 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3652 * do the same things as a eot handler
3654 static irqreturn_t
ppc440spe_adma_err_handler(int irq
, void *data
)
3656 struct ppc440spe_adma_chan
*chan
= data
;
3658 dev_dbg(chan
->device
->common
.dev
,
3659 "ppc440spe adma%d: %s\n", chan
->device
->id
, __func__
);
3661 tasklet_schedule(&chan
->irq_tasklet
);
3662 ppc440spe_adma_device_clear_eot_status(chan
);
3668 * ppc440spe_test_callback - called when test operation has been done
3670 static void ppc440spe_test_callback(void *unused
)
3672 complete(&ppc440spe_r6_test_comp
);
3676 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3678 static void ppc440spe_adma_issue_pending(struct dma_chan
*chan
)
3680 struct ppc440spe_adma_chan
*ppc440spe_chan
;
3682 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
3683 dev_dbg(ppc440spe_chan
->device
->common
.dev
,
3684 "ppc440spe adma%d: %s %d \n", ppc440spe_chan
->device
->id
,
3685 __func__
, ppc440spe_chan
->pending
);
3687 if (ppc440spe_chan
->pending
) {
3688 ppc440spe_chan
->pending
= 0;
3689 ppc440spe_chan_append(ppc440spe_chan
);
3694 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3695 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
3696 * specific operation)
3698 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan
*chan
)
3700 struct ppc440spe_adma_desc_slot
*sw_desc
, *group_start
;
3701 dma_cookie_t cookie
;
3702 int slot_cnt
, slots_per_op
;
3704 dev_dbg(chan
->device
->common
.dev
,
3705 "ppc440spe adma%d: %s\n", chan
->device
->id
, __func__
);
3707 spin_lock_bh(&chan
->lock
);
3708 slot_cnt
= ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op
);
3709 sw_desc
= ppc440spe_adma_alloc_slots(chan
, slot_cnt
, slots_per_op
);
3711 group_start
= sw_desc
->group_head
;
3712 list_splice_init(&sw_desc
->group_list
, &chan
->chain
);
3713 async_tx_ack(&sw_desc
->async_tx
);
3714 ppc440spe_desc_init_null_xor(group_start
);
3716 cookie
= dma_cookie_assign(&sw_desc
->async_tx
);
3718 /* initialize the completed cookie to be less than
3719 * the most recently used cookie
3721 chan
->common
.completed_cookie
= cookie
- 1;
3723 /* channel should not be busy */
3724 BUG_ON(ppc440spe_chan_is_busy(chan
));
3726 /* set the descriptor address */
3727 ppc440spe_chan_set_first_xor_descriptor(chan
, sw_desc
);
3729 /* run the descriptor */
3730 ppc440spe_chan_run(chan
);
3732 printk(KERN_ERR
"ppc440spe adma%d"
3733 " failed to allocate null descriptor\n",
3735 spin_unlock_bh(&chan
->lock
);
3739 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3740 * For this we just perform one WXOR operation with the same source
3741 * and destination addresses, the GF-multiplier is 1; so if RAID-6
3742 * capabilities are enabled then we'll get src/dst filled with zero.
3744 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan
*chan
)
3746 struct ppc440spe_adma_desc_slot
*sw_desc
, *iter
;
3749 dma_addr_t dma_addr
, addrs
[2];
3750 unsigned long op
= 0;
3753 set_bit(PPC440SPE_DESC_WXOR
, &op
);
3755 pg
= alloc_page(GFP_KERNEL
);
3759 spin_lock_bh(&chan
->lock
);
3760 sw_desc
= ppc440spe_adma_alloc_slots(chan
, 1, 1);
3762 /* 1 src, 1 dsr, int_ena, WXOR */
3763 ppc440spe_desc_init_dma01pq(sw_desc
, 1, 1, 1, op
);
3764 list_for_each_entry(iter
, &sw_desc
->group_list
, chain_node
) {
3765 ppc440spe_desc_set_byte_count(iter
, chan
, PAGE_SIZE
);
3766 iter
->unmap_len
= PAGE_SIZE
;
3770 spin_unlock_bh(&chan
->lock
);
3773 spin_unlock_bh(&chan
->lock
);
3775 /* Fill the test page with ones */
3776 memset(page_address(pg
), 0xFF, PAGE_SIZE
);
3777 dma_addr
= dma_map_page(chan
->device
->dev
, pg
, 0,
3778 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
3780 /* Setup addresses */
3781 ppc440spe_adma_pq_set_src(sw_desc
, dma_addr
, 0);
3782 ppc440spe_adma_pq_set_src_mult(sw_desc
, 1, 0, 0);
3783 addrs
[0] = dma_addr
;
3785 ppc440spe_adma_pq_set_dest(sw_desc
, addrs
, DMA_PREP_PQ_DISABLE_Q
);
3787 async_tx_ack(&sw_desc
->async_tx
);
3788 sw_desc
->async_tx
.callback
= ppc440spe_test_callback
;
3789 sw_desc
->async_tx
.callback_param
= NULL
;
3791 init_completion(&ppc440spe_r6_test_comp
);
3793 ppc440spe_adma_tx_submit(&sw_desc
->async_tx
);
3794 ppc440spe_adma_issue_pending(&chan
->common
);
3796 wait_for_completion(&ppc440spe_r6_test_comp
);
3798 /* Now check if the test page is zeroed */
3799 a
= page_address(pg
);
3800 if ((*(u32
*)a
) == 0 && memcmp(a
, a
+4, PAGE_SIZE
-4) == 0) {
3801 /* page is zero - RAID-6 enabled */
3804 /* RAID-6 was not enabled */
3812 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device
*adev
)
3815 case PPC440SPE_DMA0_ID
:
3816 case PPC440SPE_DMA1_ID
:
3817 dma_cap_set(DMA_MEMCPY
, adev
->common
.cap_mask
);
3818 dma_cap_set(DMA_INTERRUPT
, adev
->common
.cap_mask
);
3819 dma_cap_set(DMA_PQ
, adev
->common
.cap_mask
);
3820 dma_cap_set(DMA_PQ_VAL
, adev
->common
.cap_mask
);
3821 dma_cap_set(DMA_XOR_VAL
, adev
->common
.cap_mask
);
3823 case PPC440SPE_XOR_ID
:
3824 dma_cap_set(DMA_XOR
, adev
->common
.cap_mask
);
3825 dma_cap_set(DMA_PQ
, adev
->common
.cap_mask
);
3826 dma_cap_set(DMA_INTERRUPT
, adev
->common
.cap_mask
);
3827 adev
->common
.cap_mask
= adev
->common
.cap_mask
;
3831 /* Set base routines */
3832 adev
->common
.device_alloc_chan_resources
=
3833 ppc440spe_adma_alloc_chan_resources
;
3834 adev
->common
.device_free_chan_resources
=
3835 ppc440spe_adma_free_chan_resources
;
3836 adev
->common
.device_tx_status
= ppc440spe_adma_tx_status
;
3837 adev
->common
.device_issue_pending
= ppc440spe_adma_issue_pending
;
3839 /* Set prep routines based on capability */
3840 if (dma_has_cap(DMA_MEMCPY
, adev
->common
.cap_mask
)) {
3841 adev
->common
.device_prep_dma_memcpy
=
3842 ppc440spe_adma_prep_dma_memcpy
;
3844 if (dma_has_cap(DMA_XOR
, adev
->common
.cap_mask
)) {
3845 adev
->common
.max_xor
= XOR_MAX_OPS
;
3846 adev
->common
.device_prep_dma_xor
=
3847 ppc440spe_adma_prep_dma_xor
;
3849 if (dma_has_cap(DMA_PQ
, adev
->common
.cap_mask
)) {
3851 case PPC440SPE_DMA0_ID
:
3852 dma_set_maxpq(&adev
->common
,
3853 DMA0_FIFO_SIZE
/ sizeof(struct dma_cdb
), 0);
3855 case PPC440SPE_DMA1_ID
:
3856 dma_set_maxpq(&adev
->common
,
3857 DMA1_FIFO_SIZE
/ sizeof(struct dma_cdb
), 0);
3859 case PPC440SPE_XOR_ID
:
3860 adev
->common
.max_pq
= XOR_MAX_OPS
* 3;
3863 adev
->common
.device_prep_dma_pq
=
3864 ppc440spe_adma_prep_dma_pq
;
3866 if (dma_has_cap(DMA_PQ_VAL
, adev
->common
.cap_mask
)) {
3868 case PPC440SPE_DMA0_ID
:
3869 adev
->common
.max_pq
= DMA0_FIFO_SIZE
/
3870 sizeof(struct dma_cdb
);
3872 case PPC440SPE_DMA1_ID
:
3873 adev
->common
.max_pq
= DMA1_FIFO_SIZE
/
3874 sizeof(struct dma_cdb
);
3877 adev
->common
.device_prep_dma_pq_val
=
3878 ppc440spe_adma_prep_dma_pqzero_sum
;
3880 if (dma_has_cap(DMA_XOR_VAL
, adev
->common
.cap_mask
)) {
3882 case PPC440SPE_DMA0_ID
:
3883 adev
->common
.max_xor
= DMA0_FIFO_SIZE
/
3884 sizeof(struct dma_cdb
);
3886 case PPC440SPE_DMA1_ID
:
3887 adev
->common
.max_xor
= DMA1_FIFO_SIZE
/
3888 sizeof(struct dma_cdb
);
3891 adev
->common
.device_prep_dma_xor_val
=
3892 ppc440spe_adma_prep_dma_xor_zero_sum
;
3894 if (dma_has_cap(DMA_INTERRUPT
, adev
->common
.cap_mask
)) {
3895 adev
->common
.device_prep_dma_interrupt
=
3896 ppc440spe_adma_prep_dma_interrupt
;
3898 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3899 "( %s%s%s%s%s%s%s)\n",
3900 dev_name(adev
->dev
),
3901 dma_has_cap(DMA_PQ
, adev
->common
.cap_mask
) ? "pq " : "",
3902 dma_has_cap(DMA_PQ_VAL
, adev
->common
.cap_mask
) ? "pq_val " : "",
3903 dma_has_cap(DMA_XOR
, adev
->common
.cap_mask
) ? "xor " : "",
3904 dma_has_cap(DMA_XOR_VAL
, adev
->common
.cap_mask
) ? "xor_val " : "",
3905 dma_has_cap(DMA_MEMCPY
, adev
->common
.cap_mask
) ? "memcpy " : "",
3906 dma_has_cap(DMA_INTERRUPT
, adev
->common
.cap_mask
) ? "intr " : "");
3909 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device
*adev
,
3910 struct ppc440spe_adma_chan
*chan
,
3913 struct platform_device
*ofdev
;
3914 struct device_node
*np
;
3917 ofdev
= container_of(adev
->dev
, struct platform_device
, dev
);
3918 np
= ofdev
->dev
.of_node
;
3919 if (adev
->id
!= PPC440SPE_XOR_ID
) {
3920 adev
->err_irq
= irq_of_parse_and_map(np
, 1);
3921 if (adev
->err_irq
== NO_IRQ
) {
3922 dev_warn(adev
->dev
, "no err irq resource?\n");
3923 *initcode
= PPC_ADMA_INIT_IRQ2
;
3924 adev
->err_irq
= -ENXIO
;
3926 atomic_inc(&ppc440spe_adma_err_irq_ref
);
3928 adev
->err_irq
= -ENXIO
;
3931 adev
->irq
= irq_of_parse_and_map(np
, 0);
3932 if (adev
->irq
== NO_IRQ
) {
3933 dev_err(adev
->dev
, "no irq resource\n");
3934 *initcode
= PPC_ADMA_INIT_IRQ1
;
3938 dev_dbg(adev
->dev
, "irq %d, err irq %d\n",
3939 adev
->irq
, adev
->err_irq
);
3941 ret
= request_irq(adev
->irq
, ppc440spe_adma_eot_handler
,
3942 0, dev_driver_string(adev
->dev
), chan
);
3944 dev_err(adev
->dev
, "can't request irq %d\n",
3946 *initcode
= PPC_ADMA_INIT_IRQ1
;
3951 /* only DMA engines have a separate error IRQ
3952 * so it's Ok if err_irq < 0 in XOR engine case.
3954 if (adev
->err_irq
> 0) {
3955 /* both DMA engines share common error IRQ */
3956 ret
= request_irq(adev
->err_irq
,
3957 ppc440spe_adma_err_handler
,
3959 dev_driver_string(adev
->dev
),
3962 dev_err(adev
->dev
, "can't request irq %d\n",
3964 *initcode
= PPC_ADMA_INIT_IRQ2
;
3970 if (adev
->id
== PPC440SPE_XOR_ID
) {
3971 /* enable XOR engine interrupts */
3972 iowrite32be(XOR_IE_CBCIE_BIT
| XOR_IE_ICBIE_BIT
|
3973 XOR_IE_ICIE_BIT
| XOR_IE_RPTIE_BIT
,
3974 &adev
->xor_reg
->ier
);
3978 np
= of_find_compatible_node(NULL
, NULL
, "ibm,i2o-440spe");
3980 pr_err("%s: can't find I2O device tree node\n",
3985 adev
->i2o_reg
= of_iomap(np
, 0);
3986 if (!adev
->i2o_reg
) {
3987 pr_err("%s: failed to map I2O registers\n", __func__
);
3993 /* Unmask 'CS FIFO Attention' interrupts and
3994 * enable generating interrupts on errors
3996 enable
= (adev
->id
== PPC440SPE_DMA0_ID
) ?
3997 ~(I2O_IOPIM_P0SNE
| I2O_IOPIM_P0EM
) :
3998 ~(I2O_IOPIM_P1SNE
| I2O_IOPIM_P1EM
);
3999 mask
= ioread32(&adev
->i2o_reg
->iopim
) & enable
;
4000 iowrite32(mask
, &adev
->i2o_reg
->iopim
);
4005 free_irq(adev
->irq
, chan
);
4007 irq_dispose_mapping(adev
->irq
);
4009 if (adev
->err_irq
> 0) {
4010 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref
))
4011 irq_dispose_mapping(adev
->err_irq
);
4016 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device
*adev
,
4017 struct ppc440spe_adma_chan
*chan
)
4021 if (adev
->id
== PPC440SPE_XOR_ID
) {
4022 /* disable XOR engine interrupts */
4023 mask
= ioread32be(&adev
->xor_reg
->ier
);
4024 mask
&= ~(XOR_IE_CBCIE_BIT
| XOR_IE_ICBIE_BIT
|
4025 XOR_IE_ICIE_BIT
| XOR_IE_RPTIE_BIT
);
4026 iowrite32be(mask
, &adev
->xor_reg
->ier
);
4028 /* disable DMAx engine interrupts */
4029 disable
= (adev
->id
== PPC440SPE_DMA0_ID
) ?
4030 (I2O_IOPIM_P0SNE
| I2O_IOPIM_P0EM
) :
4031 (I2O_IOPIM_P1SNE
| I2O_IOPIM_P1EM
);
4032 mask
= ioread32(&adev
->i2o_reg
->iopim
) | disable
;
4033 iowrite32(mask
, &adev
->i2o_reg
->iopim
);
4035 free_irq(adev
->irq
, chan
);
4036 irq_dispose_mapping(adev
->irq
);
4037 if (adev
->err_irq
> 0) {
4038 free_irq(adev
->err_irq
, chan
);
4039 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref
)) {
4040 irq_dispose_mapping(adev
->err_irq
);
4041 iounmap(adev
->i2o_reg
);
4047 * ppc440spe_adma_probe - probe the asynch device
4049 static int ppc440spe_adma_probe(struct platform_device
*ofdev
)
4051 struct device_node
*np
= ofdev
->dev
.of_node
;
4052 struct resource res
;
4053 struct ppc440spe_adma_device
*adev
;
4054 struct ppc440spe_adma_chan
*chan
;
4055 struct ppc_dma_chan_ref
*ref
, *_ref
;
4056 int ret
= 0, initcode
= PPC_ADMA_INIT_OK
;
4062 if (of_device_is_compatible(np
, "amcc,xor-accelerator")) {
4063 id
= PPC440SPE_XOR_ID
;
4064 /* As far as the XOR engine is concerned, it does not
4065 * use FIFOs but uses linked list. So there is no dependency
4066 * between pool size to allocate and the engine configuration.
4068 pool_size
= PAGE_SIZE
<< 1;
4070 /* it is DMA0 or DMA1 */
4071 idx
= of_get_property(np
, "cell-index", &len
);
4072 if (!idx
|| (len
!= sizeof(u32
))) {
4073 dev_err(&ofdev
->dev
, "Device node %s has missing "
4074 "or invalid cell-index property\n",
4079 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4080 * should allocate the pool accordingly to size of this
4081 * FIFO. Thus, the pool size depends on the FIFO depth:
4082 * how much CDBs pointers the FIFO may contain then so
4083 * much CDBs we should provide in the pool.
4086 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4087 * Pool size = CDBs number * CDB size =
4088 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4090 pool_size
= (id
== PPC440SPE_DMA0_ID
) ?
4091 DMA0_FIFO_SIZE
: DMA1_FIFO_SIZE
;
4095 if (of_address_to_resource(np
, 0, &res
)) {
4096 dev_err(&ofdev
->dev
, "failed to get memory resource\n");
4097 initcode
= PPC_ADMA_INIT_MEMRES
;
4102 if (!request_mem_region(res
.start
, resource_size(&res
),
4103 dev_driver_string(&ofdev
->dev
))) {
4104 dev_err(&ofdev
->dev
, "failed to request memory region %pR\n",
4106 initcode
= PPC_ADMA_INIT_MEMREG
;
4111 /* create a device */
4112 adev
= kzalloc(sizeof(*adev
), GFP_KERNEL
);
4114 dev_err(&ofdev
->dev
, "failed to allocate device\n");
4115 initcode
= PPC_ADMA_INIT_ALLOC
;
4117 goto err_adev_alloc
;
4121 adev
->pool_size
= pool_size
;
4122 /* allocate coherent memory for hardware descriptors */
4123 adev
->dma_desc_pool_virt
= dma_alloc_coherent(&ofdev
->dev
,
4124 adev
->pool_size
, &adev
->dma_desc_pool
,
4126 if (adev
->dma_desc_pool_virt
== NULL
) {
4127 dev_err(&ofdev
->dev
, "failed to allocate %d bytes of coherent "
4128 "memory for hardware descriptors\n",
4130 initcode
= PPC_ADMA_INIT_COHERENT
;
4134 dev_dbg(&ofdev
->dev
, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4135 adev
->dma_desc_pool_virt
, (u64
)adev
->dma_desc_pool
);
4137 regs
= ioremap(res
.start
, resource_size(&res
));
4139 dev_err(&ofdev
->dev
, "failed to ioremap regs!\n");
4140 goto err_regs_alloc
;
4143 if (adev
->id
== PPC440SPE_XOR_ID
) {
4144 adev
->xor_reg
= regs
;
4146 iowrite32be(XOR_CRSR_XASR_BIT
, &adev
->xor_reg
->crsr
);
4147 iowrite32be(XOR_CRSR_64BA_BIT
, &adev
->xor_reg
->crrr
);
4149 size_t fifo_size
= (adev
->id
== PPC440SPE_DMA0_ID
) ?
4150 DMA0_FIFO_SIZE
: DMA1_FIFO_SIZE
;
4151 adev
->dma_reg
= regs
;
4152 /* DMAx_FIFO_SIZE is defined in bytes,
4153 * <fsiz> - is defined in number of CDB pointers (8byte).
4154 * DMA FIFO Length = CSlength + CPlength, where
4155 * CSlength = CPlength = (fsiz + 1) * 8.
4157 iowrite32(DMA_FIFO_ENABLE
| ((fifo_size
>> 3) - 2),
4158 &adev
->dma_reg
->fsiz
);
4159 /* Configure DMA engine */
4160 iowrite32(DMA_CFG_DXEPR_HP
| DMA_CFG_DFMPP_HP
| DMA_CFG_FALGN
,
4161 &adev
->dma_reg
->cfg
);
4163 iowrite32(~0, &adev
->dma_reg
->dsts
);
4166 adev
->dev
= &ofdev
->dev
;
4167 adev
->common
.dev
= &ofdev
->dev
;
4168 INIT_LIST_HEAD(&adev
->common
.channels
);
4169 platform_set_drvdata(ofdev
, adev
);
4171 /* create a channel */
4172 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
4174 dev_err(&ofdev
->dev
, "can't allocate channel structure\n");
4175 initcode
= PPC_ADMA_INIT_CHANNEL
;
4177 goto err_chan_alloc
;
4180 spin_lock_init(&chan
->lock
);
4181 INIT_LIST_HEAD(&chan
->chain
);
4182 INIT_LIST_HEAD(&chan
->all_slots
);
4183 chan
->device
= adev
;
4184 chan
->common
.device
= &adev
->common
;
4185 dma_cookie_init(&chan
->common
);
4186 list_add_tail(&chan
->common
.device_node
, &adev
->common
.channels
);
4187 tasklet_init(&chan
->irq_tasklet
, ppc440spe_adma_tasklet
,
4188 (unsigned long)chan
);
4190 /* allocate and map helper pages for async validation or
4191 * async_mult/async_sum_product operations on DMA0/1.
4193 if (adev
->id
!= PPC440SPE_XOR_ID
) {
4194 chan
->pdest_page
= alloc_page(GFP_KERNEL
);
4195 chan
->qdest_page
= alloc_page(GFP_KERNEL
);
4196 if (!chan
->pdest_page
||
4197 !chan
->qdest_page
) {
4198 if (chan
->pdest_page
)
4199 __free_page(chan
->pdest_page
);
4200 if (chan
->qdest_page
)
4201 __free_page(chan
->qdest_page
);
4203 goto err_page_alloc
;
4205 chan
->pdest
= dma_map_page(&ofdev
->dev
, chan
->pdest_page
, 0,
4206 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4207 chan
->qdest
= dma_map_page(&ofdev
->dev
, chan
->qdest_page
, 0,
4208 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4211 ref
= kmalloc(sizeof(*ref
), GFP_KERNEL
);
4213 ref
->chan
= &chan
->common
;
4214 INIT_LIST_HEAD(&ref
->node
);
4215 list_add_tail(&ref
->node
, &ppc440spe_adma_chan_list
);
4217 dev_err(&ofdev
->dev
, "failed to allocate channel reference!\n");
4222 ret
= ppc440spe_adma_setup_irqs(adev
, chan
, &initcode
);
4226 ppc440spe_adma_init_capabilities(adev
);
4228 ret
= dma_async_device_register(&adev
->common
);
4230 initcode
= PPC_ADMA_INIT_REGISTER
;
4231 dev_err(&ofdev
->dev
, "failed to register dma device\n");
4238 ppc440spe_adma_release_irqs(adev
, chan
);
4240 list_for_each_entry_safe(ref
, _ref
, &ppc440spe_adma_chan_list
, node
) {
4241 if (chan
== to_ppc440spe_adma_chan(ref
->chan
)) {
4242 list_del(&ref
->node
);
4247 if (adev
->id
!= PPC440SPE_XOR_ID
) {
4248 dma_unmap_page(&ofdev
->dev
, chan
->pdest
,
4249 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4250 dma_unmap_page(&ofdev
->dev
, chan
->qdest
,
4251 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4252 __free_page(chan
->pdest_page
);
4253 __free_page(chan
->qdest_page
);
4258 if (adev
->id
== PPC440SPE_XOR_ID
)
4259 iounmap(adev
->xor_reg
);
4261 iounmap(adev
->dma_reg
);
4263 dma_free_coherent(adev
->dev
, adev
->pool_size
,
4264 adev
->dma_desc_pool_virt
,
4265 adev
->dma_desc_pool
);
4269 release_mem_region(res
.start
, resource_size(&res
));
4271 if (id
< PPC440SPE_ADMA_ENGINES_NUM
)
4272 ppc440spe_adma_devices
[id
] = initcode
;
4278 * ppc440spe_adma_remove - remove the asynch device
4280 static int ppc440spe_adma_remove(struct platform_device
*ofdev
)
4282 struct ppc440spe_adma_device
*adev
= platform_get_drvdata(ofdev
);
4283 struct device_node
*np
= ofdev
->dev
.of_node
;
4284 struct resource res
;
4285 struct dma_chan
*chan
, *_chan
;
4286 struct ppc_dma_chan_ref
*ref
, *_ref
;
4287 struct ppc440spe_adma_chan
*ppc440spe_chan
;
4289 if (adev
->id
< PPC440SPE_ADMA_ENGINES_NUM
)
4290 ppc440spe_adma_devices
[adev
->id
] = -1;
4292 dma_async_device_unregister(&adev
->common
);
4294 list_for_each_entry_safe(chan
, _chan
, &adev
->common
.channels
,
4296 ppc440spe_chan
= to_ppc440spe_adma_chan(chan
);
4297 ppc440spe_adma_release_irqs(adev
, ppc440spe_chan
);
4298 tasklet_kill(&ppc440spe_chan
->irq_tasklet
);
4299 if (adev
->id
!= PPC440SPE_XOR_ID
) {
4300 dma_unmap_page(&ofdev
->dev
, ppc440spe_chan
->pdest
,
4301 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4302 dma_unmap_page(&ofdev
->dev
, ppc440spe_chan
->qdest
,
4303 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
4304 __free_page(ppc440spe_chan
->pdest_page
);
4305 __free_page(ppc440spe_chan
->qdest_page
);
4307 list_for_each_entry_safe(ref
, _ref
, &ppc440spe_adma_chan_list
,
4309 if (ppc440spe_chan
==
4310 to_ppc440spe_adma_chan(ref
->chan
)) {
4311 list_del(&ref
->node
);
4315 list_del(&chan
->device_node
);
4316 kfree(ppc440spe_chan
);
4319 dma_free_coherent(adev
->dev
, adev
->pool_size
,
4320 adev
->dma_desc_pool_virt
, adev
->dma_desc_pool
);
4321 if (adev
->id
== PPC440SPE_XOR_ID
)
4322 iounmap(adev
->xor_reg
);
4324 iounmap(adev
->dma_reg
);
4325 of_address_to_resource(np
, 0, &res
);
4326 release_mem_region(res
.start
, resource_size(&res
));
4332 * /sys driver interface to enable h/w RAID-6 capabilities
4333 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4334 * directory are "devices", "enable" and "poly".
4335 * "devices" shows available engines.
4336 * "enable" is used to enable RAID-6 capabilities or to check
4337 * whether these has been activated.
4338 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4341 static ssize_t
show_ppc440spe_devices(struct device_driver
*dev
, char *buf
)
4346 for (i
= 0; i
< PPC440SPE_ADMA_ENGINES_NUM
; i
++) {
4347 if (ppc440spe_adma_devices
[i
] == -1)
4349 size
+= snprintf(buf
+ size
, PAGE_SIZE
- size
,
4350 "PPC440SP(E)-ADMA.%d: %s\n", i
,
4351 ppc_adma_errors
[ppc440spe_adma_devices
[i
]]);
4356 static ssize_t
show_ppc440spe_r6enable(struct device_driver
*dev
, char *buf
)
4358 return snprintf(buf
, PAGE_SIZE
,
4359 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4360 ppc440spe_r6_enabled
? "EN" : "DIS");
4363 static ssize_t
store_ppc440spe_r6enable(struct device_driver
*dev
,
4364 const char *buf
, size_t count
)
4368 if (!count
|| count
> 11)
4371 if (!ppc440spe_r6_tchan
)
4375 sscanf(buf
, "%lx", &val
);
4376 dcr_write(ppc440spe_mq_dcr_host
, DCRN_MQ0_XORBA
, val
);
4379 /* Verify whether it really works now */
4380 if (ppc440spe_test_raid6(ppc440spe_r6_tchan
) == 0) {
4381 pr_info("PPC440SP(e) RAID-6 has been activated "
4383 ppc440spe_r6_enabled
= 1;
4385 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4387 ppc440spe_r6_enabled
= 0;
4392 static ssize_t
show_ppc440spe_r6poly(struct device_driver
*dev
, char *buf
)
4398 /* 440SP has fixed polynomial */
4401 reg
= dcr_read(ppc440spe_mq_dcr_host
, DCRN_MQ0_CFBHL
);
4402 reg
>>= MQ0_CFBHL_POLY
;
4406 size
= snprintf(buf
, PAGE_SIZE
, "PPC440SP(e) RAID-6 driver "
4407 "uses 0x1%02x polynomial.\n", reg
);
4411 static ssize_t
store_ppc440spe_r6poly(struct device_driver
*dev
,
4412 const char *buf
, size_t count
)
4414 unsigned long reg
, val
;
4417 /* 440SP uses default 0x14D polynomial only */
4421 if (!count
|| count
> 6)
4424 /* e.g., 0x14D or 0x11D */
4425 sscanf(buf
, "%lx", &val
);
4431 reg
= dcr_read(ppc440spe_mq_dcr_host
, DCRN_MQ0_CFBHL
);
4432 reg
&= ~(0xFF << MQ0_CFBHL_POLY
);
4433 reg
|= val
<< MQ0_CFBHL_POLY
;
4434 dcr_write(ppc440spe_mq_dcr_host
, DCRN_MQ0_CFBHL
, reg
);
4439 static DRIVER_ATTR(devices
, S_IRUGO
, show_ppc440spe_devices
, NULL
);
4440 static DRIVER_ATTR(enable
, S_IRUGO
| S_IWUSR
, show_ppc440spe_r6enable
,
4441 store_ppc440spe_r6enable
);
4442 static DRIVER_ATTR(poly
, S_IRUGO
| S_IWUSR
, show_ppc440spe_r6poly
,
4443 store_ppc440spe_r6poly
);
4446 * Common initialisation for RAID engines; allocate memory for
4447 * DMAx FIFOs, perform configuration common for all DMA engines.
4448 * Further DMA engine specific configuration is done at probe time.
4450 static int ppc440spe_configure_raid_devices(void)
4452 struct device_node
*np
;
4453 struct resource i2o_res
;
4454 struct i2o_regs __iomem
*i2o_reg
;
4455 dcr_host_t i2o_dcr_host
;
4456 unsigned int dcr_base
, dcr_len
;
4459 np
= of_find_compatible_node(NULL
, NULL
, "ibm,i2o-440spe");
4461 pr_err("%s: can't find I2O device tree node\n",
4466 if (of_address_to_resource(np
, 0, &i2o_res
)) {
4471 i2o_reg
= of_iomap(np
, 0);
4473 pr_err("%s: failed to map I2O registers\n", __func__
);
4478 /* Get I2O DCRs base */
4479 dcr_base
= dcr_resource_start(np
, 0);
4480 dcr_len
= dcr_resource_len(np
, 0);
4481 if (!dcr_base
&& !dcr_len
) {
4482 pr_err("%s: can't get DCR registers base/len!\n",
4489 i2o_dcr_host
= dcr_map(np
, dcr_base
, dcr_len
);
4490 if (!DCR_MAP_OK(i2o_dcr_host
)) {
4491 pr_err("%s: failed to map DCRs!\n", np
->full_name
);
4498 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4499 * the base address of FIFO memory space.
4500 * Actually we need twice more physical memory than programmed in the
4501 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4503 ppc440spe_dma_fifo_buf
= kmalloc((DMA0_FIFO_SIZE
+ DMA1_FIFO_SIZE
) << 1,
4505 if (!ppc440spe_dma_fifo_buf
) {
4506 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__
);
4508 dcr_unmap(i2o_dcr_host
, dcr_len
);
4516 mtdcri(SDR0
, DCRN_SDR0_SRST
, DCRN_SDR0_SRST_I2ODMA
);
4517 mtdcri(SDR0
, DCRN_SDR0_SRST
, 0);
4519 /* Setup the base address of mmaped registers */
4520 dcr_write(i2o_dcr_host
, DCRN_I2O0_IBAH
, (u32
)(i2o_res
.start
>> 32));
4521 dcr_write(i2o_dcr_host
, DCRN_I2O0_IBAL
, (u32
)(i2o_res
.start
) |
4523 dcr_unmap(i2o_dcr_host
, dcr_len
);
4525 /* Setup FIFO memory space base address */
4526 iowrite32(0, &i2o_reg
->ifbah
);
4527 iowrite32(((u32
)__pa(ppc440spe_dma_fifo_buf
)), &i2o_reg
->ifbal
);
4529 /* set zero FIFO size for I2O, so the whole
4530 * ppc440spe_dma_fifo_buf is used by DMAs.
4531 * DMAx_FIFOs will be configured while probe.
4533 iowrite32(0, &i2o_reg
->ifsiz
);
4536 /* To prepare WXOR/RXOR functionality we need access to
4537 * Memory Queue Module DCRs (finally it will be enabled
4538 * via /sys interface of the ppc440spe ADMA driver).
4540 np
= of_find_compatible_node(NULL
, NULL
, "ibm,mq-440spe");
4542 pr_err("%s: can't find MQ device tree node\n",
4548 /* Get MQ DCRs base */
4549 dcr_base
= dcr_resource_start(np
, 0);
4550 dcr_len
= dcr_resource_len(np
, 0);
4551 if (!dcr_base
&& !dcr_len
) {
4552 pr_err("%s: can't get DCR registers base/len!\n",
4558 ppc440spe_mq_dcr_host
= dcr_map(np
, dcr_base
, dcr_len
);
4559 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host
)) {
4560 pr_err("%s: failed to map DCRs!\n", np
->full_name
);
4565 ppc440spe_mq_dcr_len
= dcr_len
;
4568 dcr_write(ppc440spe_mq_dcr_host
, DCRN_MQ0_BAUH
, DMA_CUED_XOR_HB
);
4571 * - LL transaction passing limit to 1;
4572 * - Memory controller cycle limit to 1;
4573 * - Galois Polynomial to 0x14d (default)
4575 dcr_write(ppc440spe_mq_dcr_host
, DCRN_MQ0_CFBHL
,
4576 (1 << MQ0_CFBHL_TPLM
) | (1 << MQ0_CFBHL_HBCL
) |
4577 (PPC440SPE_DEFAULT_POLY
<< MQ0_CFBHL_POLY
));
4579 atomic_set(&ppc440spe_adma_err_irq_ref
, 0);
4580 for (i
= 0; i
< PPC440SPE_ADMA_ENGINES_NUM
; i
++)
4581 ppc440spe_adma_devices
[i
] = -1;
4588 kfree(ppc440spe_dma_fifo_buf
);
4592 static const struct of_device_id ppc440spe_adma_of_match
[] = {
4593 { .compatible
= "ibm,dma-440spe", },
4594 { .compatible
= "amcc,xor-accelerator", },
4597 MODULE_DEVICE_TABLE(of
, ppc440spe_adma_of_match
);
4599 static struct platform_driver ppc440spe_adma_driver
= {
4600 .probe
= ppc440spe_adma_probe
,
4601 .remove
= ppc440spe_adma_remove
,
4603 .name
= "PPC440SP(E)-ADMA",
4604 .owner
= THIS_MODULE
,
4605 .of_match_table
= ppc440spe_adma_of_match
,
4609 static __init
int ppc440spe_adma_init(void)
4613 ret
= ppc440spe_configure_raid_devices();
4617 ret
= platform_driver_register(&ppc440spe_adma_driver
);
4619 pr_err("%s: failed to register platform driver\n",
4624 /* Initialization status */
4625 ret
= driver_create_file(&ppc440spe_adma_driver
.driver
,
4626 &driver_attr_devices
);
4630 /* RAID-6 h/w enable entry */
4631 ret
= driver_create_file(&ppc440spe_adma_driver
.driver
,
4632 &driver_attr_enable
);
4636 /* GF polynomial to use */
4637 ret
= driver_create_file(&ppc440spe_adma_driver
.driver
,
4642 driver_remove_file(&ppc440spe_adma_driver
.driver
,
4643 &driver_attr_enable
);
4645 driver_remove_file(&ppc440spe_adma_driver
.driver
,
4646 &driver_attr_devices
);
4648 /* User will not be able to enable h/w RAID-6 */
4649 pr_err("%s: failed to create RAID-6 driver interface\n",
4651 platform_driver_unregister(&ppc440spe_adma_driver
);
4653 dcr_unmap(ppc440spe_mq_dcr_host
, ppc440spe_mq_dcr_len
);
4654 kfree(ppc440spe_dma_fifo_buf
);
4658 static void __exit
ppc440spe_adma_exit(void)
4660 driver_remove_file(&ppc440spe_adma_driver
.driver
,
4662 driver_remove_file(&ppc440spe_adma_driver
.driver
,
4663 &driver_attr_enable
);
4664 driver_remove_file(&ppc440spe_adma_driver
.driver
,
4665 &driver_attr_devices
);
4666 platform_driver_unregister(&ppc440spe_adma_driver
);
4667 dcr_unmap(ppc440spe_mq_dcr_host
, ppc440spe_mq_dcr_len
);
4668 kfree(ppc440spe_dma_fifo_buf
);
4671 arch_initcall(ppc440spe_adma_init
);
4672 module_exit(ppc440spe_adma_exit
);
4674 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4675 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4676 MODULE_LICENSE("GPL");