2 * Copyright(c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 #include <linux/types.h>
22 #include <mach/hardware.h>
23 #include <asm/hardware/iop_adma.h>
25 #define ADMA_ACCR(chan) (chan->mmr_base + 0x0)
26 #define ADMA_ACSR(chan) (chan->mmr_base + 0x4)
27 #define ADMA_ADAR(chan) (chan->mmr_base + 0x8)
28 #define ADMA_IIPCR(chan) (chan->mmr_base + 0x18)
29 #define ADMA_IIPAR(chan) (chan->mmr_base + 0x1c)
30 #define ADMA_IIPUAR(chan) (chan->mmr_base + 0x20)
31 #define ADMA_ANDAR(chan) (chan->mmr_base + 0x24)
32 #define ADMA_ADCR(chan) (chan->mmr_base + 0x28)
33 #define ADMA_CARMD(chan) (chan->mmr_base + 0x2c)
34 #define ADMA_ABCR(chan) (chan->mmr_base + 0x30)
35 #define ADMA_DLADR(chan) (chan->mmr_base + 0x34)
36 #define ADMA_DUADR(chan) (chan->mmr_base + 0x38)
37 #define ADMA_SLAR(src, chan) (chan->mmr_base + (0x3c + (src << 3)))
38 #define ADMA_SUAR(src, chan) (chan->mmr_base + (0x40 + (src << 3)))
40 struct iop13xx_adma_src
{
45 unsigned int pq_upper_src_addr
:24;
46 unsigned int pq_dmlt
:8;
51 struct iop13xx_adma_desc_ctrl
{
52 unsigned int int_en
:1;
53 unsigned int xfer_dir
:2;
54 unsigned int src_select
:4;
55 unsigned int zero_result
:1;
56 unsigned int block_fill_en
:1;
57 unsigned int crc_gen_en
:1;
58 unsigned int crc_xfer_dis
:1;
59 unsigned int crc_seed_fetch_dis
:1;
60 unsigned int status_write_back_en
:1;
61 unsigned int endian_swap_en
:1;
62 unsigned int reserved0
:2;
63 unsigned int pq_update_xfer_en
:1;
64 unsigned int dual_xor_en
:1;
65 unsigned int pq_xfer_en
:1;
66 unsigned int p_xfer_dis
:1;
67 unsigned int reserved1
:10;
68 unsigned int relax_order_en
:1;
69 unsigned int no_snoop_en
:1;
72 struct iop13xx_adma_byte_count
{
73 unsigned int byte_count
:24;
74 unsigned int host_if
:3;
75 unsigned int reserved
:2;
76 unsigned int zero_result_err_q
:1;
77 unsigned int zero_result_err
:1;
78 unsigned int tx_complete
:1;
81 struct iop13xx_adma_desc_hw
{
85 struct iop13xx_adma_desc_ctrl desc_ctrl_field
;
94 struct iop13xx_adma_byte_count byte_count_field
;
102 u32 pq_upper_dest_addr
;
104 struct iop13xx_adma_src src
[1];
107 struct iop13xx_adma_desc_dual_xor
{
113 u32 h_upper_dest_addr
;
119 u32 h_upper_src_addr
;
121 u32 d_upper_src_addr
;
123 u32 d_upper_dest_addr
;
126 struct iop13xx_adma_desc_pq_update
{
132 u32 p_upper_dest_addr
;
138 u32 p_upper_src_addr
;
141 unsigned int q_upper_src_addr
:24;
142 unsigned int q_dmlt
:8;
145 u32 q_upper_dest_addr
;
148 static inline int iop_adma_get_max_xor(void)
153 #define iop_adma_get_max_pq iop_adma_get_max_xor
155 static inline u32
iop_chan_get_current_descriptor(struct iop_adma_chan
*chan
)
157 return __raw_readl(ADMA_ADAR(chan
));
160 static inline void iop_chan_set_next_descriptor(struct iop_adma_chan
*chan
,
163 __raw_writel(next_desc_addr
, ADMA_ANDAR(chan
));
166 #define ADMA_STATUS_BUSY (1 << 13)
168 static inline char iop_chan_is_busy(struct iop_adma_chan
*chan
)
170 if (__raw_readl(ADMA_ACSR(chan
)) &
178 iop_chan_get_desc_align(struct iop_adma_chan
*chan
, int num_slots
)
182 #define iop_desc_is_aligned(x, y) 1
185 iop_chan_memcpy_slot_count(size_t len
, int *slots_per_op
)
191 #define iop_chan_interrupt_slot_count(s, c) iop_chan_memcpy_slot_count(0, s)
194 iop_chan_memset_slot_count(size_t len
, int *slots_per_op
)
201 iop_chan_xor_slot_count(size_t len
, int src_cnt
, int *slots_per_op
)
203 static const char slot_count_table
[] = { 1, 2, 2, 2,
208 *slots_per_op
= slot_count_table
[src_cnt
- 1];
209 return *slots_per_op
;
212 #define ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
213 #define IOP_ADMA_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
214 #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
215 #define IOP_ADMA_XOR_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
216 #define IOP_ADMA_PQ_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
217 #define iop_chan_zero_sum_slot_count(l, s, o) iop_chan_xor_slot_count(l, s, o)
218 #define iop_chan_pq_slot_count iop_chan_xor_slot_count
219 #define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count
221 static inline u32
iop_desc_get_byte_count(struct iop_adma_desc_slot
*desc
,
222 struct iop_adma_chan
*chan
)
224 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
225 return hw_desc
->byte_count_field
.byte_count
;
228 static inline u32
iop_desc_get_src_addr(struct iop_adma_desc_slot
*desc
,
229 struct iop_adma_chan
*chan
,
232 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
233 return hw_desc
->src
[src_idx
].src_addr
;
236 static inline u32
iop_desc_get_src_count(struct iop_adma_desc_slot
*desc
,
237 struct iop_adma_chan
*chan
)
239 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
240 return hw_desc
->desc_ctrl_field
.src_select
+ 1;
244 iop_desc_init_memcpy(struct iop_adma_desc_slot
*desc
, unsigned long flags
)
246 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
249 struct iop13xx_adma_desc_ctrl field
;
252 u_desc_ctrl
.value
= 0;
253 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
254 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
255 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
256 hw_desc
->crc_addr
= 0;
260 iop_desc_init_memset(struct iop_adma_desc_slot
*desc
, unsigned long flags
)
262 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
265 struct iop13xx_adma_desc_ctrl field
;
268 u_desc_ctrl
.value
= 0;
269 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
270 u_desc_ctrl
.field
.block_fill_en
= 1;
271 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
272 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
273 hw_desc
->crc_addr
= 0;
276 /* to do: support buffers larger than ADMA_MAX_BYTE_COUNT */
278 iop_desc_init_xor(struct iop_adma_desc_slot
*desc
, int src_cnt
,
281 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
284 struct iop13xx_adma_desc_ctrl field
;
287 u_desc_ctrl
.value
= 0;
288 u_desc_ctrl
.field
.src_select
= src_cnt
- 1;
289 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
290 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
291 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
292 hw_desc
->crc_addr
= 0;
295 #define iop_desc_init_null_xor(d, s, i) iop_desc_init_xor(d, s, i)
297 /* to do: support buffers larger than ADMA_MAX_BYTE_COUNT */
299 iop_desc_init_zero_sum(struct iop_adma_desc_slot
*desc
, int src_cnt
,
302 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
305 struct iop13xx_adma_desc_ctrl field
;
308 u_desc_ctrl
.value
= 0;
309 u_desc_ctrl
.field
.src_select
= src_cnt
- 1;
310 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
311 u_desc_ctrl
.field
.zero_result
= 1;
312 u_desc_ctrl
.field
.status_write_back_en
= 1;
313 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
314 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
315 hw_desc
->crc_addr
= 0;
321 iop_desc_init_pq(struct iop_adma_desc_slot
*desc
, int src_cnt
,
324 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
327 struct iop13xx_adma_desc_ctrl field
;
330 u_desc_ctrl
.value
= 0;
331 u_desc_ctrl
.field
.src_select
= src_cnt
- 1;
332 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
333 u_desc_ctrl
.field
.pq_xfer_en
= 1;
334 u_desc_ctrl
.field
.p_xfer_dis
= !!(flags
& DMA_PREP_PQ_DISABLE_P
);
335 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
336 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
340 iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot
*desc
, int src_cnt
,
343 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
346 struct iop13xx_adma_desc_ctrl field
;
349 u_desc_ctrl
.value
= 0;
350 u_desc_ctrl
.field
.src_select
= src_cnt
- 1;
351 u_desc_ctrl
.field
.xfer_dir
= 3; /* local to internal bus */
352 u_desc_ctrl
.field
.zero_result
= 1;
353 u_desc_ctrl
.field
.status_write_back_en
= 1;
354 u_desc_ctrl
.field
.pq_xfer_en
= 1;
355 u_desc_ctrl
.field
.p_xfer_dis
= !!(flags
& DMA_PREP_PQ_DISABLE_P
);
356 u_desc_ctrl
.field
.int_en
= flags
& DMA_PREP_INTERRUPT
;
357 hw_desc
->desc_ctrl
= u_desc_ctrl
.value
;
360 static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot
*desc
,
361 struct iop_adma_chan
*chan
,
364 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
365 hw_desc
->byte_count
= byte_count
;
369 iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot
*desc
, u32 len
)
371 int slots_per_op
= desc
->slots_per_op
;
372 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
, *iter
;
375 if (len
<= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT
) {
376 hw_desc
->byte_count
= len
;
379 iter
= iop_hw_desc_slot_idx(hw_desc
, i
);
380 iter
->byte_count
= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT
;
381 len
-= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT
;
383 } while (len
> IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT
);
386 iter
= iop_hw_desc_slot_idx(hw_desc
, i
);
387 iter
->byte_count
= len
;
392 #define iop_desc_set_pq_zero_sum_byte_count iop_desc_set_zero_sum_byte_count
394 static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot
*desc
,
395 struct iop_adma_chan
*chan
,
398 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
399 hw_desc
->dest_addr
= addr
;
400 hw_desc
->upper_dest_addr
= 0;
404 iop_desc_set_pq_addr(struct iop_adma_desc_slot
*desc
, dma_addr_t
*addr
)
406 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
408 hw_desc
->dest_addr
= addr
[0];
409 hw_desc
->q_dest_addr
= addr
[1];
410 hw_desc
->upper_dest_addr
= 0;
413 static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot
*desc
,
416 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
417 hw_desc
->src
[0].src_addr
= addr
;
418 hw_desc
->src
[0].upper_src_addr
= 0;
421 static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot
*desc
,
422 int src_idx
, dma_addr_t addr
)
424 int slot_cnt
= desc
->slot_cnt
, slots_per_op
= desc
->slots_per_op
;
425 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
, *iter
;
429 iter
= iop_hw_desc_slot_idx(hw_desc
, i
);
430 iter
->src
[src_idx
].src_addr
= addr
;
431 iter
->src
[src_idx
].upper_src_addr
= 0;
432 slot_cnt
-= slots_per_op
;
435 addr
+= IOP_ADMA_XOR_MAX_BYTE_COUNT
;
441 iop_desc_set_pq_src_addr(struct iop_adma_desc_slot
*desc
, int src_idx
,
442 dma_addr_t addr
, unsigned char coef
)
444 int slot_cnt
= desc
->slot_cnt
, slots_per_op
= desc
->slots_per_op
;
445 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
, *iter
;
446 struct iop13xx_adma_src
*src
;
450 iter
= iop_hw_desc_slot_idx(hw_desc
, i
);
451 src
= &iter
->src
[src_idx
];
452 src
->src_addr
= addr
;
453 src
->pq_upper_src_addr
= 0;
455 slot_cnt
-= slots_per_op
;
458 addr
+= IOP_ADMA_PQ_MAX_BYTE_COUNT
;
464 iop_desc_init_interrupt(struct iop_adma_desc_slot
*desc
,
465 struct iop_adma_chan
*chan
)
467 iop_desc_init_memcpy(desc
, 1);
468 iop_desc_set_byte_count(desc
, chan
, 0);
469 iop_desc_set_dest_addr(desc
, chan
, 0);
470 iop_desc_set_memcpy_src_addr(desc
, 0);
473 #define iop_desc_set_zero_sum_src_addr iop_desc_set_xor_src_addr
474 #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
477 iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot
*desc
, int pq_idx
,
480 iop_desc_set_xor_src_addr(desc
, pq_idx
, src
[pq_idx
]);
481 iop_desc_set_xor_src_addr(desc
, pq_idx
+1, src
[pq_idx
+1]);
484 static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot
*desc
,
487 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
489 iop_paranoia(hw_desc
->next_desc
);
490 hw_desc
->next_desc
= next_desc_addr
;
493 static inline u32
iop_desc_get_next_desc(struct iop_adma_desc_slot
*desc
)
495 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
496 return hw_desc
->next_desc
;
499 static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot
*desc
)
501 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
502 hw_desc
->next_desc
= 0;
505 static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot
*desc
,
508 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
509 hw_desc
->block_fill_data
= val
;
512 static inline enum sum_check_flags
513 iop_desc_get_zero_result(struct iop_adma_desc_slot
*desc
)
515 struct iop13xx_adma_desc_hw
*hw_desc
= desc
->hw_desc
;
516 struct iop13xx_adma_desc_ctrl desc_ctrl
= hw_desc
->desc_ctrl_field
;
517 struct iop13xx_adma_byte_count byte_count
= hw_desc
->byte_count_field
;
518 enum sum_check_flags flags
;
520 BUG_ON(!(byte_count
.tx_complete
&& desc_ctrl
.zero_result
));
522 flags
= byte_count
.zero_result_err_q
<< SUM_CHECK_Q
;
523 flags
|= byte_count
.zero_result_err
<< SUM_CHECK_P
;
528 static inline void iop_chan_append(struct iop_adma_chan
*chan
)
532 adma_accr
= __raw_readl(ADMA_ACCR(chan
));
534 __raw_writel(adma_accr
, ADMA_ACCR(chan
));
537 static inline u32
iop_chan_get_status(struct iop_adma_chan
*chan
)
539 return __raw_readl(ADMA_ACSR(chan
));
542 static inline void iop_chan_disable(struct iop_adma_chan
*chan
)
544 u32 adma_chan_ctrl
= __raw_readl(ADMA_ACCR(chan
));
545 adma_chan_ctrl
&= ~0x1;
546 __raw_writel(adma_chan_ctrl
, ADMA_ACCR(chan
));
549 static inline void iop_chan_enable(struct iop_adma_chan
*chan
)
553 adma_chan_ctrl
= __raw_readl(ADMA_ACCR(chan
));
554 adma_chan_ctrl
|= 0x1;
555 __raw_writel(adma_chan_ctrl
, ADMA_ACCR(chan
));
558 static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan
*chan
)
560 u32 status
= __raw_readl(ADMA_ACSR(chan
));
562 __raw_writel(status
, ADMA_ACSR(chan
));
565 static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan
*chan
)
567 u32 status
= __raw_readl(ADMA_ACSR(chan
));
569 __raw_writel(status
, ADMA_ACSR(chan
));
572 static inline void iop_adma_device_clear_err_status(struct iop_adma_chan
*chan
)
574 u32 status
= __raw_readl(ADMA_ACSR(chan
));
575 status
&= (1 << 9) | (1 << 5) | (1 << 4) | (1 << 3);
576 __raw_writel(status
, ADMA_ACSR(chan
));
580 iop_is_err_int_parity(unsigned long status
, struct iop_adma_chan
*chan
)
582 return test_bit(9, &status
);
586 iop_is_err_mcu_abort(unsigned long status
, struct iop_adma_chan
*chan
)
588 return test_bit(5, &status
);
592 iop_is_err_int_tabort(unsigned long status
, struct iop_adma_chan
*chan
)
594 return test_bit(4, &status
);
598 iop_is_err_int_mabort(unsigned long status
, struct iop_adma_chan
*chan
)
600 return test_bit(3, &status
);
604 iop_is_err_pci_tabort(unsigned long status
, struct iop_adma_chan
*chan
)
610 iop_is_err_pci_mabort(unsigned long status
, struct iop_adma_chan
*chan
)
616 iop_is_err_split_tx(unsigned long status
, struct iop_adma_chan
*chan
)