2 * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
5 * Copyright (C) 2007-2008 Atmel Corporation
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #include <linux/clk.h>
12 #include <linux/delay.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
23 #include "dw_dmac_regs.h"
26 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
27 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
28 * of which use ARM any more). See the "Databook" from Synopsys for
29 * information beyond what licensees probably provide.
31 * The driver has currently been tested only with the Atmel AT32AP7000,
32 * which does not support descriptor writeback.
35 /* NOTE: DMS+SMS is system-specific. We should get this information
36 * from the platform code somehow.
38 #define DWC_DEFAULT_CTLLO (DWC_CTLL_DST_MSIZE(0) \
39 | DWC_CTLL_SRC_MSIZE(0) \
46 * This is configuration-dependent and usually a funny size like 4095.
47 * Let's round it down to the nearest power of two.
49 * Note that this is a transfer count, i.e. if we transfer 32-bit
50 * words, we can do 8192 bytes per descriptor.
52 * This parameter is also system-specific.
54 #define DWC_MAX_COUNT 2048U
57 * Number of descriptors to allocate for each channel. This should be
58 * made configurable somehow; preferably, the clients (at least the
59 * ones using slave transfers) should be able to give us a hint.
61 #define NR_DESCS_PER_CHANNEL 64
63 /*----------------------------------------------------------------------*/
66 * Because we're not relying on writeback from the controller (it may not
67 * even be configured into the core!) we don't need to use dma_pool. These
68 * descriptors -- and associated data -- are cacheable. We do need to make
69 * sure their dcache entries are written back before handing them off to
70 * the controller, though.
73 static struct dw_desc
*dwc_first_active(struct dw_dma_chan
*dwc
)
75 return list_entry(dwc
->active_list
.next
, struct dw_desc
, desc_node
);
78 static struct dw_desc
*dwc_first_queued(struct dw_dma_chan
*dwc
)
80 return list_entry(dwc
->queue
.next
, struct dw_desc
, desc_node
);
83 static struct dw_desc
*dwc_desc_get(struct dw_dma_chan
*dwc
)
85 struct dw_desc
*desc
, *_desc
;
86 struct dw_desc
*ret
= NULL
;
89 spin_lock_bh(&dwc
->lock
);
90 list_for_each_entry_safe(desc
, _desc
, &dwc
->free_list
, desc_node
) {
91 if (async_tx_test_ack(&desc
->txd
)) {
92 list_del(&desc
->desc_node
);
96 dev_dbg(&dwc
->chan
.dev
, "desc %p not ACKed\n", desc
);
99 spin_unlock_bh(&dwc
->lock
);
101 dev_vdbg(&dwc
->chan
.dev
, "scanned %u descriptors on freelist\n", i
);
106 static void dwc_sync_desc_for_cpu(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
108 struct dw_desc
*child
;
110 list_for_each_entry(child
, &desc
->txd
.tx_list
, desc_node
)
111 dma_sync_single_for_cpu(dwc
->chan
.dev
.parent
,
112 child
->txd
.phys
, sizeof(child
->lli
),
114 dma_sync_single_for_cpu(dwc
->chan
.dev
.parent
,
115 desc
->txd
.phys
, sizeof(desc
->lli
),
120 * Move a descriptor, including any children, to the free list.
121 * `desc' must not be on any lists.
123 static void dwc_desc_put(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
126 struct dw_desc
*child
;
128 dwc_sync_desc_for_cpu(dwc
, desc
);
130 spin_lock_bh(&dwc
->lock
);
131 list_for_each_entry(child
, &desc
->txd
.tx_list
, desc_node
)
132 dev_vdbg(&dwc
->chan
.dev
,
133 "moving child desc %p to freelist\n",
135 list_splice_init(&desc
->txd
.tx_list
, &dwc
->free_list
);
136 dev_vdbg(&dwc
->chan
.dev
, "moving desc %p to freelist\n", desc
);
137 list_add(&desc
->desc_node
, &dwc
->free_list
);
138 spin_unlock_bh(&dwc
->lock
);
142 /* Called with dwc->lock held and bh disabled */
144 dwc_assign_cookie(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
146 dma_cookie_t cookie
= dwc
->chan
.cookie
;
151 dwc
->chan
.cookie
= cookie
;
152 desc
->txd
.cookie
= cookie
;
157 /*----------------------------------------------------------------------*/
159 /* Called with dwc->lock held and bh disabled */
160 static void dwc_dostart(struct dw_dma_chan
*dwc
, struct dw_desc
*first
)
162 struct dw_dma
*dw
= to_dw_dma(dwc
->chan
.device
);
164 /* ASSERT: channel is idle */
165 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
166 dev_err(&dwc
->chan
.dev
,
167 "BUG: Attempted to start non-idle channel\n");
168 dev_err(&dwc
->chan
.dev
,
169 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
170 channel_readl(dwc
, SAR
),
171 channel_readl(dwc
, DAR
),
172 channel_readl(dwc
, LLP
),
173 channel_readl(dwc
, CTL_HI
),
174 channel_readl(dwc
, CTL_LO
));
176 /* The tasklet will hopefully advance the queue... */
180 channel_writel(dwc
, LLP
, first
->txd
.phys
);
181 channel_writel(dwc
, CTL_LO
,
182 DWC_CTLL_LLP_D_EN
| DWC_CTLL_LLP_S_EN
);
183 channel_writel(dwc
, CTL_HI
, 0);
184 channel_set_bit(dw
, CH_EN
, dwc
->mask
);
187 /*----------------------------------------------------------------------*/
190 dwc_descriptor_complete(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
192 dma_async_tx_callback callback
;
194 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
196 dev_vdbg(&dwc
->chan
.dev
, "descriptor %u complete\n", txd
->cookie
);
198 dwc
->completed
= txd
->cookie
;
199 callback
= txd
->callback
;
200 param
= txd
->callback_param
;
202 dwc_sync_desc_for_cpu(dwc
, desc
);
203 list_splice_init(&txd
->tx_list
, &dwc
->free_list
);
204 list_move(&desc
->desc_node
, &dwc
->free_list
);
207 * We use dma_unmap_page() regardless of how the buffers were
208 * mapped before they were submitted...
210 if (!(txd
->flags
& DMA_COMPL_SKIP_DEST_UNMAP
))
211 dma_unmap_page(dwc
->chan
.dev
.parent
, desc
->lli
.dar
, desc
->len
,
213 if (!(txd
->flags
& DMA_COMPL_SKIP_SRC_UNMAP
))
214 dma_unmap_page(dwc
->chan
.dev
.parent
, desc
->lli
.sar
, desc
->len
,
218 * The API requires that no submissions are done from a
219 * callback, so we don't need to drop the lock here
225 static void dwc_complete_all(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
227 struct dw_desc
*desc
, *_desc
;
230 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
231 dev_err(&dwc
->chan
.dev
,
232 "BUG: XFER bit set, but channel not idle!\n");
234 /* Try to continue after resetting the channel... */
235 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
236 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
241 * Submit queued descriptors ASAP, i.e. before we go through
242 * the completed ones.
244 if (!list_empty(&dwc
->queue
))
245 dwc_dostart(dwc
, dwc_first_queued(dwc
));
246 list_splice_init(&dwc
->active_list
, &list
);
247 list_splice_init(&dwc
->queue
, &dwc
->active_list
);
249 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
250 dwc_descriptor_complete(dwc
, desc
);
253 static void dwc_scan_descriptors(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
256 struct dw_desc
*desc
, *_desc
;
257 struct dw_desc
*child
;
261 * Clear block interrupt flag before scanning so that we don't
262 * miss any, and read LLP before RAW_XFER to ensure it is
263 * valid if we decide to scan the list.
265 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
266 llp
= channel_readl(dwc
, LLP
);
267 status_xfer
= dma_readl(dw
, RAW
.XFER
);
269 if (status_xfer
& dwc
->mask
) {
270 /* Everything we've submitted is done */
271 dma_writel(dw
, CLEAR
.XFER
, dwc
->mask
);
272 dwc_complete_all(dw
, dwc
);
276 dev_vdbg(&dwc
->chan
.dev
, "scan_descriptors: llp=0x%x\n", llp
);
278 list_for_each_entry_safe(desc
, _desc
, &dwc
->active_list
, desc_node
) {
279 if (desc
->lli
.llp
== llp
)
280 /* This one is currently in progress */
283 list_for_each_entry(child
, &desc
->txd
.tx_list
, desc_node
)
284 if (child
->lli
.llp
== llp
)
285 /* Currently in progress */
289 * No descriptors so far seem to be in progress, i.e.
290 * this one must be done.
292 dwc_descriptor_complete(dwc
, desc
);
295 dev_err(&dwc
->chan
.dev
,
296 "BUG: All descriptors done, but channel not idle!\n");
298 /* Try to continue after resetting the channel... */
299 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
300 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
303 if (!list_empty(&dwc
->queue
)) {
304 dwc_dostart(dwc
, dwc_first_queued(dwc
));
305 list_splice_init(&dwc
->queue
, &dwc
->active_list
);
309 static void dwc_dump_lli(struct dw_dma_chan
*dwc
, struct dw_lli
*lli
)
311 dev_printk(KERN_CRIT
, &dwc
->chan
.dev
,
312 " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
313 lli
->sar
, lli
->dar
, lli
->llp
,
314 lli
->ctlhi
, lli
->ctllo
);
317 static void dwc_handle_error(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
319 struct dw_desc
*bad_desc
;
320 struct dw_desc
*child
;
322 dwc_scan_descriptors(dw
, dwc
);
325 * The descriptor currently at the head of the active list is
326 * borked. Since we don't have any way to report errors, we'll
327 * just have to scream loudly and try to carry on.
329 bad_desc
= dwc_first_active(dwc
);
330 list_del_init(&bad_desc
->desc_node
);
331 list_splice_init(&dwc
->queue
, dwc
->active_list
.prev
);
333 /* Clear the error flag and try to restart the controller */
334 dma_writel(dw
, CLEAR
.ERROR
, dwc
->mask
);
335 if (!list_empty(&dwc
->active_list
))
336 dwc_dostart(dwc
, dwc_first_active(dwc
));
339 * KERN_CRITICAL may seem harsh, but since this only happens
340 * when someone submits a bad physical address in a
341 * descriptor, we should consider ourselves lucky that the
342 * controller flagged an error instead of scribbling over
343 * random memory locations.
345 dev_printk(KERN_CRIT
, &dwc
->chan
.dev
,
346 "Bad descriptor submitted for DMA!\n");
347 dev_printk(KERN_CRIT
, &dwc
->chan
.dev
,
348 " cookie: %d\n", bad_desc
->txd
.cookie
);
349 dwc_dump_lli(dwc
, &bad_desc
->lli
);
350 list_for_each_entry(child
, &bad_desc
->txd
.tx_list
, desc_node
)
351 dwc_dump_lli(dwc
, &child
->lli
);
353 /* Pretend the descriptor completed successfully */
354 dwc_descriptor_complete(dwc
, bad_desc
);
357 static void dw_dma_tasklet(unsigned long data
)
359 struct dw_dma
*dw
= (struct dw_dma
*)data
;
360 struct dw_dma_chan
*dwc
;
366 status_block
= dma_readl(dw
, RAW
.BLOCK
);
367 status_xfer
= dma_readl(dw
, RAW
.XFER
);
368 status_err
= dma_readl(dw
, RAW
.ERROR
);
370 dev_vdbg(dw
->dma
.dev
, "tasklet: status_block=%x status_err=%x\n",
371 status_block
, status_err
);
373 for (i
= 0; i
< dw
->dma
.chancnt
; i
++) {
375 spin_lock(&dwc
->lock
);
376 if (status_err
& (1 << i
))
377 dwc_handle_error(dw
, dwc
);
378 else if ((status_block
| status_xfer
) & (1 << i
))
379 dwc_scan_descriptors(dw
, dwc
);
380 spin_unlock(&dwc
->lock
);
384 * Re-enable interrupts. Block Complete interrupts are only
385 * enabled if the INT_EN bit in the descriptor is set. This
386 * will trigger a scan before the whole list is done.
388 channel_set_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
389 channel_set_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
390 channel_set_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
393 static irqreturn_t
dw_dma_interrupt(int irq
, void *dev_id
)
395 struct dw_dma
*dw
= dev_id
;
398 dev_vdbg(dw
->dma
.dev
, "interrupt: status=0x%x\n",
399 dma_readl(dw
, STATUS_INT
));
402 * Just disable the interrupts. We'll turn them back on in the
405 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
406 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
407 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
409 status
= dma_readl(dw
, STATUS_INT
);
412 "BUG: Unexpected interrupts pending: 0x%x\n",
416 channel_clear_bit(dw
, MASK
.XFER
, (1 << 8) - 1);
417 channel_clear_bit(dw
, MASK
.BLOCK
, (1 << 8) - 1);
418 channel_clear_bit(dw
, MASK
.SRC_TRAN
, (1 << 8) - 1);
419 channel_clear_bit(dw
, MASK
.DST_TRAN
, (1 << 8) - 1);
420 channel_clear_bit(dw
, MASK
.ERROR
, (1 << 8) - 1);
423 tasklet_schedule(&dw
->tasklet
);
428 /*----------------------------------------------------------------------*/
430 static dma_cookie_t
dwc_tx_submit(struct dma_async_tx_descriptor
*tx
)
432 struct dw_desc
*desc
= txd_to_dw_desc(tx
);
433 struct dw_dma_chan
*dwc
= to_dw_dma_chan(tx
->chan
);
436 spin_lock_bh(&dwc
->lock
);
437 cookie
= dwc_assign_cookie(dwc
, desc
);
440 * REVISIT: We should attempt to chain as many descriptors as
441 * possible, perhaps even appending to those already submitted
442 * for DMA. But this is hard to do in a race-free manner.
444 if (list_empty(&dwc
->active_list
)) {
445 dev_vdbg(&tx
->chan
->dev
, "tx_submit: started %u\n",
447 dwc_dostart(dwc
, desc
);
448 list_add_tail(&desc
->desc_node
, &dwc
->active_list
);
450 dev_vdbg(&tx
->chan
->dev
, "tx_submit: queued %u\n",
453 list_add_tail(&desc
->desc_node
, &dwc
->queue
);
456 spin_unlock_bh(&dwc
->lock
);
461 static struct dma_async_tx_descriptor
*
462 dwc_prep_dma_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
463 size_t len
, unsigned long flags
)
465 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
466 struct dw_desc
*desc
;
467 struct dw_desc
*first
;
468 struct dw_desc
*prev
;
471 unsigned int src_width
;
472 unsigned int dst_width
;
475 dev_vdbg(&chan
->dev
, "prep_dma_memcpy d0x%x s0x%x l0x%zx f0x%lx\n",
476 dest
, src
, len
, flags
);
478 if (unlikely(!len
)) {
479 dev_dbg(&chan
->dev
, "prep_dma_memcpy: length is zero!\n");
484 * We can be a lot more clever here, but this should take care
485 * of the most common optimization.
487 if (!((src
| dest
| len
) & 3))
488 src_width
= dst_width
= 2;
489 else if (!((src
| dest
| len
) & 1))
490 src_width
= dst_width
= 1;
492 src_width
= dst_width
= 0;
494 ctllo
= DWC_DEFAULT_CTLLO
495 | DWC_CTLL_DST_WIDTH(dst_width
)
496 | DWC_CTLL_SRC_WIDTH(src_width
)
502 for (offset
= 0; offset
< len
; offset
+= xfer_count
<< src_width
) {
503 xfer_count
= min_t(size_t, (len
- offset
) >> src_width
,
506 desc
= dwc_desc_get(dwc
);
510 desc
->lli
.sar
= src
+ offset
;
511 desc
->lli
.dar
= dest
+ offset
;
512 desc
->lli
.ctllo
= ctllo
;
513 desc
->lli
.ctlhi
= xfer_count
;
518 prev
->lli
.llp
= desc
->txd
.phys
;
519 dma_sync_single_for_device(chan
->dev
.parent
,
520 prev
->txd
.phys
, sizeof(prev
->lli
),
522 list_add_tail(&desc
->desc_node
,
523 &first
->txd
.tx_list
);
529 if (flags
& DMA_PREP_INTERRUPT
)
530 /* Trigger interrupt after last block */
531 prev
->lli
.ctllo
|= DWC_CTLL_INT_EN
;
534 dma_sync_single_for_device(chan
->dev
.parent
,
535 prev
->txd
.phys
, sizeof(prev
->lli
),
538 first
->txd
.flags
= flags
;
544 dwc_desc_put(dwc
, first
);
548 static struct dma_async_tx_descriptor
*
549 dwc_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
550 unsigned int sg_len
, enum dma_data_direction direction
,
553 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
554 struct dw_dma_slave
*dws
= dwc
->dws
;
555 struct dw_desc
*prev
;
556 struct dw_desc
*first
;
559 unsigned int reg_width
;
560 unsigned int mem_width
;
562 struct scatterlist
*sg
;
563 size_t total_len
= 0;
565 dev_vdbg(&chan
->dev
, "prep_dma_slave\n");
567 if (unlikely(!dws
|| !sg_len
))
570 reg_width
= dws
->reg_width
;
573 sg_len
= dma_map_sg(chan
->dev
.parent
, sgl
, sg_len
, direction
);
577 ctllo
= (DWC_DEFAULT_CTLLO
578 | DWC_CTLL_DST_WIDTH(reg_width
)
583 for_each_sg(sgl
, sg
, sg_len
, i
) {
584 struct dw_desc
*desc
;
588 desc
= dwc_desc_get(dwc
);
591 "not enough descriptors available\n");
596 len
= sg_dma_len(sg
);
598 if (unlikely(mem
& 3 || len
& 3))
603 desc
->lli
.ctllo
= ctllo
| DWC_CTLL_SRC_WIDTH(mem_width
);
604 desc
->lli
.ctlhi
= len
>> mem_width
;
609 prev
->lli
.llp
= desc
->txd
.phys
;
610 dma_sync_single_for_device(chan
->dev
.parent
,
614 list_add_tail(&desc
->desc_node
,
615 &first
->txd
.tx_list
);
621 case DMA_FROM_DEVICE
:
622 ctllo
= (DWC_DEFAULT_CTLLO
623 | DWC_CTLL_SRC_WIDTH(reg_width
)
629 for_each_sg(sgl
, sg
, sg_len
, i
) {
630 struct dw_desc
*desc
;
634 desc
= dwc_desc_get(dwc
);
637 "not enough descriptors available\n");
642 len
= sg_dma_len(sg
);
644 if (unlikely(mem
& 3 || len
& 3))
649 desc
->lli
.ctllo
= ctllo
| DWC_CTLL_DST_WIDTH(mem_width
);
650 desc
->lli
.ctlhi
= len
>> reg_width
;
655 prev
->lli
.llp
= desc
->txd
.phys
;
656 dma_sync_single_for_device(chan
->dev
.parent
,
660 list_add_tail(&desc
->desc_node
,
661 &first
->txd
.tx_list
);
671 if (flags
& DMA_PREP_INTERRUPT
)
672 /* Trigger interrupt after last block */
673 prev
->lli
.ctllo
|= DWC_CTLL_INT_EN
;
676 dma_sync_single_for_device(chan
->dev
.parent
,
677 prev
->txd
.phys
, sizeof(prev
->lli
),
680 first
->len
= total_len
;
685 dwc_desc_put(dwc
, first
);
689 static void dwc_terminate_all(struct dma_chan
*chan
)
691 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
692 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
693 struct dw_desc
*desc
, *_desc
;
697 * This is only called when something went wrong elsewhere, so
698 * we don't really care about the data. Just disable the
699 * channel. We still have to poll the channel enable bit due
700 * to AHB/HSB limitations.
702 spin_lock_bh(&dwc
->lock
);
704 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
706 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
709 /* active_list entries will end up before queued entries */
710 list_splice_init(&dwc
->queue
, &list
);
711 list_splice_init(&dwc
->active_list
, &list
);
713 spin_unlock_bh(&dwc
->lock
);
715 /* Flush all pending and queued descriptors */
716 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
717 dwc_descriptor_complete(dwc
, desc
);
720 static enum dma_status
721 dwc_is_tx_complete(struct dma_chan
*chan
,
723 dma_cookie_t
*done
, dma_cookie_t
*used
)
725 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
726 dma_cookie_t last_used
;
727 dma_cookie_t last_complete
;
730 last_complete
= dwc
->completed
;
731 last_used
= chan
->cookie
;
733 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
734 if (ret
!= DMA_SUCCESS
) {
735 dwc_scan_descriptors(to_dw_dma(chan
->device
), dwc
);
737 last_complete
= dwc
->completed
;
738 last_used
= chan
->cookie
;
740 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
744 *done
= last_complete
;
751 static void dwc_issue_pending(struct dma_chan
*chan
)
753 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
755 spin_lock_bh(&dwc
->lock
);
756 if (!list_empty(&dwc
->queue
))
757 dwc_scan_descriptors(to_dw_dma(chan
->device
), dwc
);
758 spin_unlock_bh(&dwc
->lock
);
761 static int dwc_alloc_chan_resources(struct dma_chan
*chan
)
763 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
764 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
765 struct dw_desc
*desc
;
766 struct dw_dma_slave
*dws
;
771 dev_vdbg(&chan
->dev
, "alloc_chan_resources\n");
773 /* ASSERT: channel is idle */
774 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
775 dev_dbg(&chan
->dev
, "DMA channel not idle?\n");
779 dwc
->completed
= chan
->cookie
= 1;
781 cfghi
= DWC_CFGH_FIFO_MODE
;
787 * We need controller-specific data to set up slave
790 BUG_ON(!dws
->dma_dev
|| dws
->dma_dev
!= dw
->dma
.dev
);
795 channel_writel(dwc
, CFG_LO
, cfglo
);
796 channel_writel(dwc
, CFG_HI
, cfghi
);
799 * NOTE: some controllers may have additional features that we
800 * need to initialize here, like "scatter-gather" (which
801 * doesn't mean what you think it means), and status writeback.
804 spin_lock_bh(&dwc
->lock
);
805 i
= dwc
->descs_allocated
;
806 while (dwc
->descs_allocated
< NR_DESCS_PER_CHANNEL
) {
807 spin_unlock_bh(&dwc
->lock
);
809 desc
= kzalloc(sizeof(struct dw_desc
), GFP_KERNEL
);
812 "only allocated %d descriptors\n", i
);
813 spin_lock_bh(&dwc
->lock
);
817 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
818 desc
->txd
.tx_submit
= dwc_tx_submit
;
819 desc
->txd
.flags
= DMA_CTRL_ACK
;
820 INIT_LIST_HEAD(&desc
->txd
.tx_list
);
821 desc
->txd
.phys
= dma_map_single(chan
->dev
.parent
, &desc
->lli
,
822 sizeof(desc
->lli
), DMA_TO_DEVICE
);
823 dwc_desc_put(dwc
, desc
);
825 spin_lock_bh(&dwc
->lock
);
826 i
= ++dwc
->descs_allocated
;
829 /* Enable interrupts */
830 channel_set_bit(dw
, MASK
.XFER
, dwc
->mask
);
831 channel_set_bit(dw
, MASK
.BLOCK
, dwc
->mask
);
832 channel_set_bit(dw
, MASK
.ERROR
, dwc
->mask
);
834 spin_unlock_bh(&dwc
->lock
);
837 "alloc_chan_resources allocated %d descriptors\n", i
);
842 static void dwc_free_chan_resources(struct dma_chan
*chan
)
844 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
845 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
846 struct dw_desc
*desc
, *_desc
;
849 dev_dbg(&chan
->dev
, "free_chan_resources (descs allocated=%u)\n",
850 dwc
->descs_allocated
);
852 /* ASSERT: channel is idle */
853 BUG_ON(!list_empty(&dwc
->active_list
));
854 BUG_ON(!list_empty(&dwc
->queue
));
855 BUG_ON(dma_readl(to_dw_dma(chan
->device
), CH_EN
) & dwc
->mask
);
857 spin_lock_bh(&dwc
->lock
);
858 list_splice_init(&dwc
->free_list
, &list
);
859 dwc
->descs_allocated
= 0;
862 /* Disable interrupts */
863 channel_clear_bit(dw
, MASK
.XFER
, dwc
->mask
);
864 channel_clear_bit(dw
, MASK
.BLOCK
, dwc
->mask
);
865 channel_clear_bit(dw
, MASK
.ERROR
, dwc
->mask
);
867 spin_unlock_bh(&dwc
->lock
);
869 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
) {
870 dev_vdbg(&chan
->dev
, " freeing descriptor %p\n", desc
);
871 dma_unmap_single(chan
->dev
.parent
, desc
->txd
.phys
,
872 sizeof(desc
->lli
), DMA_TO_DEVICE
);
876 dev_vdbg(&chan
->dev
, "free_chan_resources done\n");
879 /*----------------------------------------------------------------------*/
881 static void dw_dma_off(struct dw_dma
*dw
)
883 dma_writel(dw
, CFG
, 0);
885 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
886 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
887 channel_clear_bit(dw
, MASK
.SRC_TRAN
, dw
->all_chan_mask
);
888 channel_clear_bit(dw
, MASK
.DST_TRAN
, dw
->all_chan_mask
);
889 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
891 while (dma_readl(dw
, CFG
) & DW_CFG_DMA_EN
)
895 static int __init
dw_probe(struct platform_device
*pdev
)
897 struct dw_dma_platform_data
*pdata
;
905 pdata
= pdev
->dev
.platform_data
;
906 if (!pdata
|| pdata
->nr_channels
> DW_DMA_MAX_NR_CHANNELS
)
909 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
913 irq
= platform_get_irq(pdev
, 0);
917 size
= sizeof(struct dw_dma
);
918 size
+= pdata
->nr_channels
* sizeof(struct dw_dma_chan
);
919 dw
= kzalloc(size
, GFP_KERNEL
);
923 if (!request_mem_region(io
->start
, DW_REGLEN
, pdev
->dev
.driver
->name
)) {
928 memset(dw
, 0, sizeof *dw
);
930 dw
->regs
= ioremap(io
->start
, DW_REGLEN
);
936 dw
->clk
= clk_get(&pdev
->dev
, "hclk");
937 if (IS_ERR(dw
->clk
)) {
938 err
= PTR_ERR(dw
->clk
);
943 /* force dma off, just in case */
946 err
= request_irq(irq
, dw_dma_interrupt
, 0, "dw_dmac", dw
);
950 platform_set_drvdata(pdev
, dw
);
952 tasklet_init(&dw
->tasklet
, dw_dma_tasklet
, (unsigned long)dw
);
954 dw
->all_chan_mask
= (1 << pdata
->nr_channels
) - 1;
956 INIT_LIST_HEAD(&dw
->dma
.channels
);
957 for (i
= 0; i
< pdata
->nr_channels
; i
++, dw
->dma
.chancnt
++) {
958 struct dw_dma_chan
*dwc
= &dw
->chan
[i
];
960 dwc
->chan
.device
= &dw
->dma
;
961 dwc
->chan
.cookie
= dwc
->completed
= 1;
962 dwc
->chan
.chan_id
= i
;
963 list_add_tail(&dwc
->chan
.device_node
, &dw
->dma
.channels
);
965 dwc
->ch_regs
= &__dw_regs(dw
)->CHAN
[i
];
966 spin_lock_init(&dwc
->lock
);
969 INIT_LIST_HEAD(&dwc
->active_list
);
970 INIT_LIST_HEAD(&dwc
->queue
);
971 INIT_LIST_HEAD(&dwc
->free_list
);
973 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
976 /* Clear/disable all interrupts on all channels. */
977 dma_writel(dw
, CLEAR
.XFER
, dw
->all_chan_mask
);
978 dma_writel(dw
, CLEAR
.BLOCK
, dw
->all_chan_mask
);
979 dma_writel(dw
, CLEAR
.SRC_TRAN
, dw
->all_chan_mask
);
980 dma_writel(dw
, CLEAR
.DST_TRAN
, dw
->all_chan_mask
);
981 dma_writel(dw
, CLEAR
.ERROR
, dw
->all_chan_mask
);
983 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
984 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
985 channel_clear_bit(dw
, MASK
.SRC_TRAN
, dw
->all_chan_mask
);
986 channel_clear_bit(dw
, MASK
.DST_TRAN
, dw
->all_chan_mask
);
987 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
989 dma_cap_set(DMA_MEMCPY
, dw
->dma
.cap_mask
);
990 dma_cap_set(DMA_SLAVE
, dw
->dma
.cap_mask
);
991 dw
->dma
.dev
= &pdev
->dev
;
992 dw
->dma
.device_alloc_chan_resources
= dwc_alloc_chan_resources
;
993 dw
->dma
.device_free_chan_resources
= dwc_free_chan_resources
;
995 dw
->dma
.device_prep_dma_memcpy
= dwc_prep_dma_memcpy
;
997 dw
->dma
.device_prep_slave_sg
= dwc_prep_slave_sg
;
998 dw
->dma
.device_terminate_all
= dwc_terminate_all
;
1000 dw
->dma
.device_is_tx_complete
= dwc_is_tx_complete
;
1001 dw
->dma
.device_issue_pending
= dwc_issue_pending
;
1003 dma_writel(dw
, CFG
, DW_CFG_DMA_EN
);
1005 printk(KERN_INFO
"%s: DesignWare DMA Controller, %d channels\n",
1006 pdev
->dev
.bus_id
, dw
->dma
.chancnt
);
1008 dma_async_device_register(&dw
->dma
);
1013 clk_disable(dw
->clk
);
1019 release_resource(io
);
1025 static int __exit
dw_remove(struct platform_device
*pdev
)
1027 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1028 struct dw_dma_chan
*dwc
, *_dwc
;
1029 struct resource
*io
;
1032 dma_async_device_unregister(&dw
->dma
);
1034 free_irq(platform_get_irq(pdev
, 0), dw
);
1035 tasklet_kill(&dw
->tasklet
);
1037 list_for_each_entry_safe(dwc
, _dwc
, &dw
->dma
.channels
,
1039 list_del(&dwc
->chan
.device_node
);
1040 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
1043 clk_disable(dw
->clk
);
1049 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1050 release_mem_region(io
->start
, DW_REGLEN
);
1057 static void dw_shutdown(struct platform_device
*pdev
)
1059 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1061 dw_dma_off(platform_get_drvdata(pdev
));
1062 clk_disable(dw
->clk
);
1065 static int dw_suspend_late(struct platform_device
*pdev
, pm_message_t mesg
)
1067 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1069 dw_dma_off(platform_get_drvdata(pdev
));
1070 clk_disable(dw
->clk
);
1074 static int dw_resume_early(struct platform_device
*pdev
)
1076 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1078 clk_enable(dw
->clk
);
1079 dma_writel(dw
, CFG
, DW_CFG_DMA_EN
);
1084 static struct platform_driver dw_driver
= {
1085 .remove
= __exit_p(dw_remove
),
1086 .shutdown
= dw_shutdown
,
1087 .suspend_late
= dw_suspend_late
,
1088 .resume_early
= dw_resume_early
,
1094 static int __init
dw_init(void)
1096 return platform_driver_probe(&dw_driver
, dw_probe
);
1098 module_init(dw_init
);
1100 static void __exit
dw_exit(void)
1102 platform_driver_unregister(&dw_driver
);
1104 module_exit(dw_exit
);
1106 MODULE_LICENSE("GPL v2");
1107 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1108 MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");