2 * Freescale MPC85xx, MPC83xx DMA Engine support
4 * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007
11 * DMA engine driver for Freescale MPC8540 DMA controller, which is
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA controller is also added.
15 * This driver instructs the DMA controller to issue the PCI Read Multiple
16 * command for PCI read operations, instead of using the default PCI Read Line
17 * command. Please be aware that this setting may result in read pre-fetching
20 * This is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/slab.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/dmapool.h>
36 #include <linux/of_address.h>
37 #include <linux/of_irq.h>
38 #include <linux/of_platform.h>
39 #include <linux/fsldma.h>
40 #include "dmaengine.h"
43 #define chan_dbg(chan, fmt, arg...) \
44 dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
45 #define chan_err(chan, fmt, arg...) \
46 dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
48 static const char msg_ld_oom
[] = "No free memory for link descriptor";
54 static void set_sr(struct fsldma_chan
*chan
, u32 val
)
56 DMA_OUT(chan
, &chan
->regs
->sr
, val
, 32);
59 static u32
get_sr(struct fsldma_chan
*chan
)
61 return DMA_IN(chan
, &chan
->regs
->sr
, 32);
64 static void set_mr(struct fsldma_chan
*chan
, u32 val
)
66 DMA_OUT(chan
, &chan
->regs
->mr
, val
, 32);
69 static u32
get_mr(struct fsldma_chan
*chan
)
71 return DMA_IN(chan
, &chan
->regs
->mr
, 32);
74 static void set_cdar(struct fsldma_chan
*chan
, dma_addr_t addr
)
76 DMA_OUT(chan
, &chan
->regs
->cdar
, addr
| FSL_DMA_SNEN
, 64);
79 static dma_addr_t
get_cdar(struct fsldma_chan
*chan
)
81 return DMA_IN(chan
, &chan
->regs
->cdar
, 64) & ~FSL_DMA_SNEN
;
84 static void set_bcr(struct fsldma_chan
*chan
, u32 val
)
86 DMA_OUT(chan
, &chan
->regs
->bcr
, val
, 32);
89 static u32
get_bcr(struct fsldma_chan
*chan
)
91 return DMA_IN(chan
, &chan
->regs
->bcr
, 32);
98 static void set_desc_cnt(struct fsldma_chan
*chan
,
99 struct fsl_dma_ld_hw
*hw
, u32 count
)
101 hw
->count
= CPU_TO_DMA(chan
, count
, 32);
104 static void set_desc_src(struct fsldma_chan
*chan
,
105 struct fsl_dma_ld_hw
*hw
, dma_addr_t src
)
109 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
110 ? ((u64
)FSL_DMA_SATR_SREADTYPE_SNOOP_READ
<< 32) : 0;
111 hw
->src_addr
= CPU_TO_DMA(chan
, snoop_bits
| src
, 64);
114 static void set_desc_dst(struct fsldma_chan
*chan
,
115 struct fsl_dma_ld_hw
*hw
, dma_addr_t dst
)
119 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
)
120 ? ((u64
)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE
<< 32) : 0;
121 hw
->dst_addr
= CPU_TO_DMA(chan
, snoop_bits
| dst
, 64);
124 static void set_desc_next(struct fsldma_chan
*chan
,
125 struct fsl_dma_ld_hw
*hw
, dma_addr_t next
)
129 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
131 hw
->next_ln_addr
= CPU_TO_DMA(chan
, snoop_bits
| next
, 64);
134 static void set_ld_eol(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
138 snoop_bits
= ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_83XX
)
141 desc
->hw
.next_ln_addr
= CPU_TO_DMA(chan
,
142 DMA_TO_CPU(chan
, desc
->hw
.next_ln_addr
, 64) | FSL_DMA_EOL
147 * DMA Engine Hardware Control Helpers
150 static void dma_init(struct fsldma_chan
*chan
)
152 /* Reset the channel */
155 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
156 case FSL_DMA_IP_85XX
:
157 /* Set the channel to below modes:
158 * EIE - Error interrupt enable
159 * EOLNIE - End of links interrupt enable
160 * BWC - Bandwidth sharing among channels
162 set_mr(chan
, FSL_DMA_MR_BWC
| FSL_DMA_MR_EIE
163 | FSL_DMA_MR_EOLNIE
);
165 case FSL_DMA_IP_83XX
:
166 /* Set the channel to below modes:
167 * EOTIE - End-of-transfer interrupt enable
168 * PRC_RM - PCI read multiple
170 set_mr(chan
, FSL_DMA_MR_EOTIE
| FSL_DMA_MR_PRC_RM
);
175 static int dma_is_idle(struct fsldma_chan
*chan
)
177 u32 sr
= get_sr(chan
);
178 return (!(sr
& FSL_DMA_SR_CB
)) || (sr
& FSL_DMA_SR_CH
);
182 * Start the DMA controller
185 * - the CDAR register must point to the start descriptor
186 * - the MRn[CS] bit must be cleared
188 static void dma_start(struct fsldma_chan
*chan
)
194 if (chan
->feature
& FSL_DMA_CHAN_PAUSE_EXT
) {
196 mode
|= FSL_DMA_MR_EMP_EN
;
198 mode
&= ~FSL_DMA_MR_EMP_EN
;
201 if (chan
->feature
& FSL_DMA_CHAN_START_EXT
) {
202 mode
|= FSL_DMA_MR_EMS_EN
;
204 mode
&= ~FSL_DMA_MR_EMS_EN
;
205 mode
|= FSL_DMA_MR_CS
;
211 static void dma_halt(struct fsldma_chan
*chan
)
216 /* read the mode register */
220 * The 85xx controller supports channel abort, which will stop
221 * the current transfer. On 83xx, this bit is the transfer error
222 * mask bit, which should not be changed.
224 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
225 mode
|= FSL_DMA_MR_CA
;
228 mode
&= ~FSL_DMA_MR_CA
;
231 /* stop the DMA controller */
232 mode
&= ~(FSL_DMA_MR_CS
| FSL_DMA_MR_EMS_EN
);
235 /* wait for the DMA controller to become idle */
236 for (i
= 0; i
< 100; i
++) {
237 if (dma_is_idle(chan
))
243 if (!dma_is_idle(chan
))
244 chan_err(chan
, "DMA halt timeout!\n");
248 * fsl_chan_set_src_loop_size - Set source address hold transfer size
249 * @chan : Freescale DMA channel
250 * @size : Address loop size, 0 for disable loop
252 * The set source address hold transfer size. The source
253 * address hold or loop transfer size is when the DMA transfer
254 * data from source address (SA), if the loop size is 4, the DMA will
255 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
256 * SA + 1 ... and so on.
258 static void fsl_chan_set_src_loop_size(struct fsldma_chan
*chan
, int size
)
266 mode
&= ~FSL_DMA_MR_SAHE
;
272 mode
&= ~FSL_DMA_MR_SAHTS_MASK
;
273 mode
|= FSL_DMA_MR_SAHE
| (__ilog2(size
) << 14);
281 * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
282 * @chan : Freescale DMA channel
283 * @size : Address loop size, 0 for disable loop
285 * The set destination address hold transfer size. The destination
286 * address hold or loop transfer size is when the DMA transfer
287 * data to destination address (TA), if the loop size is 4, the DMA will
288 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
289 * TA + 1 ... and so on.
291 static void fsl_chan_set_dst_loop_size(struct fsldma_chan
*chan
, int size
)
299 mode
&= ~FSL_DMA_MR_DAHE
;
305 mode
&= ~FSL_DMA_MR_DAHTS_MASK
;
306 mode
|= FSL_DMA_MR_DAHE
| (__ilog2(size
) << 16);
314 * fsl_chan_set_request_count - Set DMA Request Count for external control
315 * @chan : Freescale DMA channel
316 * @size : Number of bytes to transfer in a single request
318 * The Freescale DMA channel can be controlled by the external signal DREQ#.
319 * The DMA request count is how many bytes are allowed to transfer before
320 * pausing the channel, after which a new assertion of DREQ# resumes channel
323 * A size of 0 disables external pause control. The maximum size is 1024.
325 static void fsl_chan_set_request_count(struct fsldma_chan
*chan
, int size
)
332 mode
&= ~FSL_DMA_MR_BWC_MASK
;
333 mode
|= (__ilog2(size
) << 24) & FSL_DMA_MR_BWC_MASK
;
339 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
340 * @chan : Freescale DMA channel
341 * @enable : 0 is disabled, 1 is enabled.
343 * The Freescale DMA channel can be controlled by the external signal DREQ#.
344 * The DMA Request Count feature should be used in addition to this feature
345 * to set the number of bytes to transfer before pausing the channel.
347 static void fsl_chan_toggle_ext_pause(struct fsldma_chan
*chan
, int enable
)
350 chan
->feature
|= FSL_DMA_CHAN_PAUSE_EXT
;
352 chan
->feature
&= ~FSL_DMA_CHAN_PAUSE_EXT
;
356 * fsl_chan_toggle_ext_start - Toggle channel external start status
357 * @chan : Freescale DMA channel
358 * @enable : 0 is disabled, 1 is enabled.
360 * If enable the external start, the channel can be started by an
361 * external DMA start pin. So the dma_start() does not start the
362 * transfer immediately. The DMA channel will wait for the
363 * control pin asserted.
365 static void fsl_chan_toggle_ext_start(struct fsldma_chan
*chan
, int enable
)
368 chan
->feature
|= FSL_DMA_CHAN_START_EXT
;
370 chan
->feature
&= ~FSL_DMA_CHAN_START_EXT
;
373 int fsl_dma_external_start(struct dma_chan
*dchan
, int enable
)
375 struct fsldma_chan
*chan
;
380 chan
= to_fsl_chan(dchan
);
382 fsl_chan_toggle_ext_start(chan
, enable
);
385 EXPORT_SYMBOL_GPL(fsl_dma_external_start
);
387 static void append_ld_queue(struct fsldma_chan
*chan
, struct fsl_desc_sw
*desc
)
389 struct fsl_desc_sw
*tail
= to_fsl_desc(chan
->ld_pending
.prev
);
391 if (list_empty(&chan
->ld_pending
))
395 * Add the hardware descriptor to the chain of hardware descriptors
396 * that already exists in memory.
398 * This will un-set the EOL bit of the existing transaction, and the
399 * last link in this transaction will become the EOL descriptor.
401 set_desc_next(chan
, &tail
->hw
, desc
->async_tx
.phys
);
404 * Add the software descriptor and all children to the list
405 * of pending transactions
408 list_splice_tail_init(&desc
->tx_list
, &chan
->ld_pending
);
411 static dma_cookie_t
fsl_dma_tx_submit(struct dma_async_tx_descriptor
*tx
)
413 struct fsldma_chan
*chan
= to_fsl_chan(tx
->chan
);
414 struct fsl_desc_sw
*desc
= tx_to_fsl_desc(tx
);
415 struct fsl_desc_sw
*child
;
416 dma_cookie_t cookie
= -EINVAL
;
418 spin_lock_bh(&chan
->desc_lock
);
421 if (unlikely(chan
->pm_state
!= RUNNING
)) {
422 chan_dbg(chan
, "cannot submit due to suspend\n");
423 spin_unlock_bh(&chan
->desc_lock
);
429 * assign cookies to all of the software descriptors
430 * that make up this transaction
432 list_for_each_entry(child
, &desc
->tx_list
, node
) {
433 cookie
= dma_cookie_assign(&child
->async_tx
);
436 /* put this transaction onto the tail of the pending queue */
437 append_ld_queue(chan
, desc
);
439 spin_unlock_bh(&chan
->desc_lock
);
445 * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool.
446 * @chan : Freescale DMA channel
447 * @desc: descriptor to be freed
449 static void fsl_dma_free_descriptor(struct fsldma_chan
*chan
,
450 struct fsl_desc_sw
*desc
)
452 list_del(&desc
->node
);
453 chan_dbg(chan
, "LD %p free\n", desc
);
454 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
458 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
459 * @chan : Freescale DMA channel
461 * Return - The descriptor allocated. NULL for failed.
463 static struct fsl_desc_sw
*fsl_dma_alloc_descriptor(struct fsldma_chan
*chan
)
465 struct fsl_desc_sw
*desc
;
468 desc
= dma_pool_zalloc(chan
->desc_pool
, GFP_ATOMIC
, &pdesc
);
470 chan_dbg(chan
, "out of memory for link descriptor\n");
474 INIT_LIST_HEAD(&desc
->tx_list
);
475 dma_async_tx_descriptor_init(&desc
->async_tx
, &chan
->common
);
476 desc
->async_tx
.tx_submit
= fsl_dma_tx_submit
;
477 desc
->async_tx
.phys
= pdesc
;
479 chan_dbg(chan
, "LD %p allocated\n", desc
);
485 * fsldma_clean_completed_descriptor - free all descriptors which
486 * has been completed and acked
487 * @chan: Freescale DMA channel
489 * This function is used on all completed and acked descriptors.
490 * All descriptors should only be freed in this function.
492 static void fsldma_clean_completed_descriptor(struct fsldma_chan
*chan
)
494 struct fsl_desc_sw
*desc
, *_desc
;
496 /* Run the callback for each descriptor, in order */
497 list_for_each_entry_safe(desc
, _desc
, &chan
->ld_completed
, node
)
498 if (async_tx_test_ack(&desc
->async_tx
))
499 fsl_dma_free_descriptor(chan
, desc
);
503 * fsldma_run_tx_complete_actions - cleanup a single link descriptor
504 * @chan: Freescale DMA channel
505 * @desc: descriptor to cleanup and free
506 * @cookie: Freescale DMA transaction identifier
508 * This function is used on a descriptor which has been executed by the DMA
509 * controller. It will run any callbacks, submit any dependencies.
511 static dma_cookie_t
fsldma_run_tx_complete_actions(struct fsldma_chan
*chan
,
512 struct fsl_desc_sw
*desc
, dma_cookie_t cookie
)
514 struct dma_async_tx_descriptor
*txd
= &desc
->async_tx
;
515 dma_cookie_t ret
= cookie
;
517 BUG_ON(txd
->cookie
< 0);
519 if (txd
->cookie
> 0) {
522 dma_descriptor_unmap(txd
);
523 /* Run the link descriptor callback function */
524 dmaengine_desc_get_callback_invoke(txd
, NULL
);
527 /* Run any dependencies */
528 dma_run_dependencies(txd
);
534 * fsldma_clean_running_descriptor - move the completed descriptor from
535 * ld_running to ld_completed
536 * @chan: Freescale DMA channel
537 * @desc: the descriptor which is completed
539 * Free the descriptor directly if acked by async_tx api, or move it to
540 * queue ld_completed.
542 static void fsldma_clean_running_descriptor(struct fsldma_chan
*chan
,
543 struct fsl_desc_sw
*desc
)
545 /* Remove from the list of transactions */
546 list_del(&desc
->node
);
549 * the client is allowed to attach dependent operations
552 if (!async_tx_test_ack(&desc
->async_tx
)) {
554 * Move this descriptor to the list of descriptors which is
555 * completed, but still awaiting the 'ack' bit to be set.
557 list_add_tail(&desc
->node
, &chan
->ld_completed
);
561 dma_pool_free(chan
->desc_pool
, desc
, desc
->async_tx
.phys
);
565 * fsl_chan_xfer_ld_queue - transfer any pending transactions
566 * @chan : Freescale DMA channel
568 * HARDWARE STATE: idle
569 * LOCKING: must hold chan->desc_lock
571 static void fsl_chan_xfer_ld_queue(struct fsldma_chan
*chan
)
573 struct fsl_desc_sw
*desc
;
576 * If the list of pending descriptors is empty, then we
577 * don't need to do any work at all
579 if (list_empty(&chan
->ld_pending
)) {
580 chan_dbg(chan
, "no pending LDs\n");
585 * The DMA controller is not idle, which means that the interrupt
586 * handler will start any queued transactions when it runs after
587 * this transaction finishes
590 chan_dbg(chan
, "DMA controller still busy\n");
595 * If there are some link descriptors which have not been
596 * transferred, we need to start the controller
600 * Move all elements from the queue of pending transactions
601 * onto the list of running transactions
603 chan_dbg(chan
, "idle, starting controller\n");
604 desc
= list_first_entry(&chan
->ld_pending
, struct fsl_desc_sw
, node
);
605 list_splice_tail_init(&chan
->ld_pending
, &chan
->ld_running
);
608 * The 85xx DMA controller doesn't clear the channel start bit
609 * automatically at the end of a transfer. Therefore we must clear
610 * it in software before starting the transfer.
612 if ((chan
->feature
& FSL_DMA_IP_MASK
) == FSL_DMA_IP_85XX
) {
616 mode
&= ~FSL_DMA_MR_CS
;
621 * Program the descriptor's address into the DMA controller,
622 * then start the DMA transaction
624 set_cdar(chan
, desc
->async_tx
.phys
);
632 * fsldma_cleanup_descriptors - cleanup link descriptors which are completed
633 * and move them to ld_completed to free until flag 'ack' is set
634 * @chan: Freescale DMA channel
636 * This function is used on descriptors which have been executed by the DMA
637 * controller. It will run any callbacks, submit any dependencies, then
638 * free these descriptors if flag 'ack' is set.
640 static void fsldma_cleanup_descriptors(struct fsldma_chan
*chan
)
642 struct fsl_desc_sw
*desc
, *_desc
;
643 dma_cookie_t cookie
= 0;
644 dma_addr_t curr_phys
= get_cdar(chan
);
645 int seen_current
= 0;
647 fsldma_clean_completed_descriptor(chan
);
649 /* Run the callback for each descriptor, in order */
650 list_for_each_entry_safe(desc
, _desc
, &chan
->ld_running
, node
) {
652 * do not advance past the current descriptor loaded into the
653 * hardware channel, subsequent descriptors are either in
654 * process or have not been submitted
660 * stop the search if we reach the current descriptor and the
663 if (desc
->async_tx
.phys
== curr_phys
) {
665 if (!dma_is_idle(chan
))
669 cookie
= fsldma_run_tx_complete_actions(chan
, desc
, cookie
);
671 fsldma_clean_running_descriptor(chan
, desc
);
675 * Start any pending transactions automatically
677 * In the ideal case, we keep the DMA controller busy while we go
678 * ahead and free the descriptors below.
680 fsl_chan_xfer_ld_queue(chan
);
683 chan
->common
.completed_cookie
= cookie
;
687 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
688 * @chan : Freescale DMA channel
690 * This function will create a dma pool for descriptor allocation.
692 * Return - The number of descriptors allocated.
694 static int fsl_dma_alloc_chan_resources(struct dma_chan
*dchan
)
696 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
698 /* Has this channel already been allocated? */
703 * We need the descriptor to be aligned to 32bytes
704 * for meeting FSL DMA specification requirement.
706 chan
->desc_pool
= dma_pool_create(chan
->name
, chan
->dev
,
707 sizeof(struct fsl_desc_sw
),
708 __alignof__(struct fsl_desc_sw
), 0);
709 if (!chan
->desc_pool
) {
710 chan_err(chan
, "unable to allocate descriptor pool\n");
714 /* there is at least one descriptor free to be allocated */
719 * fsldma_free_desc_list - Free all descriptors in a queue
720 * @chan: Freescae DMA channel
721 * @list: the list to free
723 * LOCKING: must hold chan->desc_lock
725 static void fsldma_free_desc_list(struct fsldma_chan
*chan
,
726 struct list_head
*list
)
728 struct fsl_desc_sw
*desc
, *_desc
;
730 list_for_each_entry_safe(desc
, _desc
, list
, node
)
731 fsl_dma_free_descriptor(chan
, desc
);
734 static void fsldma_free_desc_list_reverse(struct fsldma_chan
*chan
,
735 struct list_head
*list
)
737 struct fsl_desc_sw
*desc
, *_desc
;
739 list_for_each_entry_safe_reverse(desc
, _desc
, list
, node
)
740 fsl_dma_free_descriptor(chan
, desc
);
744 * fsl_dma_free_chan_resources - Free all resources of the channel.
745 * @chan : Freescale DMA channel
747 static void fsl_dma_free_chan_resources(struct dma_chan
*dchan
)
749 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
751 chan_dbg(chan
, "free all channel resources\n");
752 spin_lock_bh(&chan
->desc_lock
);
753 fsldma_cleanup_descriptors(chan
);
754 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
755 fsldma_free_desc_list(chan
, &chan
->ld_running
);
756 fsldma_free_desc_list(chan
, &chan
->ld_completed
);
757 spin_unlock_bh(&chan
->desc_lock
);
759 dma_pool_destroy(chan
->desc_pool
);
760 chan
->desc_pool
= NULL
;
763 static struct dma_async_tx_descriptor
*
764 fsl_dma_prep_memcpy(struct dma_chan
*dchan
,
765 dma_addr_t dma_dst
, dma_addr_t dma_src
,
766 size_t len
, unsigned long flags
)
768 struct fsldma_chan
*chan
;
769 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new;
778 chan
= to_fsl_chan(dchan
);
782 /* Allocate the link descriptor from DMA pool */
783 new = fsl_dma_alloc_descriptor(chan
);
785 chan_err(chan
, "%s\n", msg_ld_oom
);
789 copy
= min(len
, (size_t)FSL_DMA_BCR_MAX_CNT
);
791 set_desc_cnt(chan
, &new->hw
, copy
);
792 set_desc_src(chan
, &new->hw
, dma_src
);
793 set_desc_dst(chan
, &new->hw
, dma_dst
);
798 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
800 new->async_tx
.cookie
= 0;
801 async_tx_ack(&new->async_tx
);
808 /* Insert the link descriptor to the LD ring */
809 list_add_tail(&new->node
, &first
->tx_list
);
812 new->async_tx
.flags
= flags
; /* client is in control of this ack */
813 new->async_tx
.cookie
= -EBUSY
;
815 /* Set End-of-link to the last link descriptor of new list */
816 set_ld_eol(chan
, new);
818 return &first
->async_tx
;
824 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
828 static struct dma_async_tx_descriptor
*fsl_dma_prep_sg(struct dma_chan
*dchan
,
829 struct scatterlist
*dst_sg
, unsigned int dst_nents
,
830 struct scatterlist
*src_sg
, unsigned int src_nents
,
833 struct fsl_desc_sw
*first
= NULL
, *prev
= NULL
, *new = NULL
;
834 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
835 size_t dst_avail
, src_avail
;
839 /* basic sanity checks */
840 if (dst_nents
== 0 || src_nents
== 0)
843 if (dst_sg
== NULL
|| src_sg
== NULL
)
847 * TODO: should we check that both scatterlists have the same
848 * TODO: number of bytes in total? Is that really an error?
851 /* get prepared for the loop */
852 dst_avail
= sg_dma_len(dst_sg
);
853 src_avail
= sg_dma_len(src_sg
);
855 /* run until we are out of scatterlist entries */
858 /* create the largest transaction possible */
859 len
= min_t(size_t, src_avail
, dst_avail
);
860 len
= min_t(size_t, len
, FSL_DMA_BCR_MAX_CNT
);
864 dst
= sg_dma_address(dst_sg
) + sg_dma_len(dst_sg
) - dst_avail
;
865 src
= sg_dma_address(src_sg
) + sg_dma_len(src_sg
) - src_avail
;
867 /* allocate and populate the descriptor */
868 new = fsl_dma_alloc_descriptor(chan
);
870 chan_err(chan
, "%s\n", msg_ld_oom
);
874 set_desc_cnt(chan
, &new->hw
, len
);
875 set_desc_src(chan
, &new->hw
, src
);
876 set_desc_dst(chan
, &new->hw
, dst
);
881 set_desc_next(chan
, &prev
->hw
, new->async_tx
.phys
);
883 new->async_tx
.cookie
= 0;
884 async_tx_ack(&new->async_tx
);
887 /* Insert the link descriptor to the LD ring */
888 list_add_tail(&new->node
, &first
->tx_list
);
890 /* update metadata */
895 /* fetch the next dst scatterlist entry */
896 if (dst_avail
== 0) {
898 /* no more entries: we're done */
902 /* fetch the next entry: if there are no more: done */
903 dst_sg
= sg_next(dst_sg
);
908 dst_avail
= sg_dma_len(dst_sg
);
911 /* fetch the next src scatterlist entry */
912 if (src_avail
== 0) {
914 /* no more entries: we're done */
918 /* fetch the next entry: if there are no more: done */
919 src_sg
= sg_next(src_sg
);
924 src_avail
= sg_dma_len(src_sg
);
928 new->async_tx
.flags
= flags
; /* client is in control of this ack */
929 new->async_tx
.cookie
= -EBUSY
;
931 /* Set End-of-link to the last link descriptor of new list */
932 set_ld_eol(chan
, new);
934 return &first
->async_tx
;
940 fsldma_free_desc_list_reverse(chan
, &first
->tx_list
);
944 static int fsl_dma_device_terminate_all(struct dma_chan
*dchan
)
946 struct fsldma_chan
*chan
;
951 chan
= to_fsl_chan(dchan
);
953 spin_lock_bh(&chan
->desc_lock
);
955 /* Halt the DMA engine */
958 /* Remove and free all of the descriptors in the LD queue */
959 fsldma_free_desc_list(chan
, &chan
->ld_pending
);
960 fsldma_free_desc_list(chan
, &chan
->ld_running
);
961 fsldma_free_desc_list(chan
, &chan
->ld_completed
);
964 spin_unlock_bh(&chan
->desc_lock
);
968 static int fsl_dma_device_config(struct dma_chan
*dchan
,
969 struct dma_slave_config
*config
)
971 struct fsldma_chan
*chan
;
977 chan
= to_fsl_chan(dchan
);
979 /* make sure the channel supports setting burst size */
980 if (!chan
->set_request_count
)
983 /* we set the controller burst size depending on direction */
984 if (config
->direction
== DMA_MEM_TO_DEV
)
985 size
= config
->dst_addr_width
* config
->dst_maxburst
;
987 size
= config
->src_addr_width
* config
->src_maxburst
;
989 chan
->set_request_count(chan
, size
);
995 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
996 * @chan : Freescale DMA channel
998 static void fsl_dma_memcpy_issue_pending(struct dma_chan
*dchan
)
1000 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
1002 spin_lock_bh(&chan
->desc_lock
);
1003 fsl_chan_xfer_ld_queue(chan
);
1004 spin_unlock_bh(&chan
->desc_lock
);
1008 * fsl_tx_status - Determine the DMA status
1009 * @chan : Freescale DMA channel
1011 static enum dma_status
fsl_tx_status(struct dma_chan
*dchan
,
1012 dma_cookie_t cookie
,
1013 struct dma_tx_state
*txstate
)
1015 struct fsldma_chan
*chan
= to_fsl_chan(dchan
);
1016 enum dma_status ret
;
1018 ret
= dma_cookie_status(dchan
, cookie
, txstate
);
1019 if (ret
== DMA_COMPLETE
)
1022 spin_lock_bh(&chan
->desc_lock
);
1023 fsldma_cleanup_descriptors(chan
);
1024 spin_unlock_bh(&chan
->desc_lock
);
1026 return dma_cookie_status(dchan
, cookie
, txstate
);
1029 /*----------------------------------------------------------------------------*/
1030 /* Interrupt Handling */
1031 /*----------------------------------------------------------------------------*/
1033 static irqreturn_t
fsldma_chan_irq(int irq
, void *data
)
1035 struct fsldma_chan
*chan
= data
;
1038 /* save and clear the status register */
1039 stat
= get_sr(chan
);
1041 chan_dbg(chan
, "irq: stat = 0x%x\n", stat
);
1043 /* check that this was really our device */
1044 stat
&= ~(FSL_DMA_SR_CB
| FSL_DMA_SR_CH
);
1048 if (stat
& FSL_DMA_SR_TE
)
1049 chan_err(chan
, "Transfer Error!\n");
1053 * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1054 * trigger a PE interrupt.
1056 if (stat
& FSL_DMA_SR_PE
) {
1057 chan_dbg(chan
, "irq: Programming Error INT\n");
1058 stat
&= ~FSL_DMA_SR_PE
;
1059 if (get_bcr(chan
) != 0)
1060 chan_err(chan
, "Programming Error!\n");
1064 * For MPC8349, EOCDI event need to update cookie
1065 * and start the next transfer if it exist.
1067 if (stat
& FSL_DMA_SR_EOCDI
) {
1068 chan_dbg(chan
, "irq: End-of-Chain link INT\n");
1069 stat
&= ~FSL_DMA_SR_EOCDI
;
1073 * If it current transfer is the end-of-transfer,
1074 * we should clear the Channel Start bit for
1075 * prepare next transfer.
1077 if (stat
& FSL_DMA_SR_EOLNI
) {
1078 chan_dbg(chan
, "irq: End-of-link INT\n");
1079 stat
&= ~FSL_DMA_SR_EOLNI
;
1082 /* check that the DMA controller is really idle */
1083 if (!dma_is_idle(chan
))
1084 chan_err(chan
, "irq: controller not idle!\n");
1086 /* check that we handled all of the bits */
1088 chan_err(chan
, "irq: unhandled sr 0x%08x\n", stat
);
1091 * Schedule the tasklet to handle all cleanup of the current
1092 * transaction. It will start a new transaction if there is
1095 tasklet_schedule(&chan
->tasklet
);
1096 chan_dbg(chan
, "irq: Exit\n");
1100 static void dma_do_tasklet(unsigned long data
)
1102 struct fsldma_chan
*chan
= (struct fsldma_chan
*)data
;
1104 chan_dbg(chan
, "tasklet entry\n");
1106 spin_lock_bh(&chan
->desc_lock
);
1108 /* the hardware is now idle and ready for more */
1111 /* Run all cleanup for descriptors which have been completed */
1112 fsldma_cleanup_descriptors(chan
);
1114 spin_unlock_bh(&chan
->desc_lock
);
1116 chan_dbg(chan
, "tasklet exit\n");
1119 static irqreturn_t
fsldma_ctrl_irq(int irq
, void *data
)
1121 struct fsldma_device
*fdev
= data
;
1122 struct fsldma_chan
*chan
;
1123 unsigned int handled
= 0;
1127 gsr
= (fdev
->feature
& FSL_DMA_BIG_ENDIAN
) ? in_be32(fdev
->regs
)
1128 : in_le32(fdev
->regs
);
1130 dev_dbg(fdev
->dev
, "IRQ: gsr 0x%.8x\n", gsr
);
1132 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1133 chan
= fdev
->chan
[i
];
1138 dev_dbg(fdev
->dev
, "IRQ: chan %d\n", chan
->id
);
1139 fsldma_chan_irq(irq
, chan
);
1147 return IRQ_RETVAL(handled
);
1150 static void fsldma_free_irqs(struct fsldma_device
*fdev
)
1152 struct fsldma_chan
*chan
;
1156 dev_dbg(fdev
->dev
, "free per-controller IRQ\n");
1157 free_irq(fdev
->irq
, fdev
);
1161 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1162 chan
= fdev
->chan
[i
];
1163 if (chan
&& chan
->irq
) {
1164 chan_dbg(chan
, "free per-channel IRQ\n");
1165 free_irq(chan
->irq
, chan
);
1170 static int fsldma_request_irqs(struct fsldma_device
*fdev
)
1172 struct fsldma_chan
*chan
;
1176 /* if we have a per-controller IRQ, use that */
1178 dev_dbg(fdev
->dev
, "request per-controller IRQ\n");
1179 ret
= request_irq(fdev
->irq
, fsldma_ctrl_irq
, IRQF_SHARED
,
1180 "fsldma-controller", fdev
);
1184 /* no per-controller IRQ, use the per-channel IRQs */
1185 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1186 chan
= fdev
->chan
[i
];
1191 chan_err(chan
, "interrupts property missing in device tree\n");
1196 chan_dbg(chan
, "request per-channel IRQ\n");
1197 ret
= request_irq(chan
->irq
, fsldma_chan_irq
, IRQF_SHARED
,
1198 "fsldma-chan", chan
);
1200 chan_err(chan
, "unable to request per-channel IRQ\n");
1208 for (/* none */; i
>= 0; i
--) {
1209 chan
= fdev
->chan
[i
];
1216 free_irq(chan
->irq
, chan
);
1222 /*----------------------------------------------------------------------------*/
1223 /* OpenFirmware Subsystem */
1224 /*----------------------------------------------------------------------------*/
1226 static int fsl_dma_chan_probe(struct fsldma_device
*fdev
,
1227 struct device_node
*node
, u32 feature
, const char *compatible
)
1229 struct fsldma_chan
*chan
;
1230 struct resource res
;
1234 chan
= kzalloc(sizeof(*chan
), GFP_KERNEL
);
1240 /* ioremap registers for use */
1241 chan
->regs
= of_iomap(node
, 0);
1243 dev_err(fdev
->dev
, "unable to ioremap registers\n");
1248 err
= of_address_to_resource(node
, 0, &res
);
1250 dev_err(fdev
->dev
, "unable to find 'reg' property\n");
1251 goto out_iounmap_regs
;
1254 chan
->feature
= feature
;
1256 fdev
->feature
= chan
->feature
;
1259 * If the DMA device's feature is different than the feature
1260 * of its channels, report the bug
1262 WARN_ON(fdev
->feature
!= chan
->feature
);
1264 chan
->dev
= fdev
->dev
;
1265 chan
->id
= (res
.start
& 0xfff) < 0x300 ?
1266 ((res
.start
- 0x100) & 0xfff) >> 7 :
1267 ((res
.start
- 0x200) & 0xfff) >> 7;
1268 if (chan
->id
>= FSL_DMA_MAX_CHANS_PER_DEVICE
) {
1269 dev_err(fdev
->dev
, "too many channels for device\n");
1271 goto out_iounmap_regs
;
1274 fdev
->chan
[chan
->id
] = chan
;
1275 tasklet_init(&chan
->tasklet
, dma_do_tasklet
, (unsigned long)chan
);
1276 snprintf(chan
->name
, sizeof(chan
->name
), "chan%d", chan
->id
);
1278 /* Initialize the channel */
1281 /* Clear cdar registers */
1284 switch (chan
->feature
& FSL_DMA_IP_MASK
) {
1285 case FSL_DMA_IP_85XX
:
1286 chan
->toggle_ext_pause
= fsl_chan_toggle_ext_pause
;
1287 case FSL_DMA_IP_83XX
:
1288 chan
->toggle_ext_start
= fsl_chan_toggle_ext_start
;
1289 chan
->set_src_loop_size
= fsl_chan_set_src_loop_size
;
1290 chan
->set_dst_loop_size
= fsl_chan_set_dst_loop_size
;
1291 chan
->set_request_count
= fsl_chan_set_request_count
;
1294 spin_lock_init(&chan
->desc_lock
);
1295 INIT_LIST_HEAD(&chan
->ld_pending
);
1296 INIT_LIST_HEAD(&chan
->ld_running
);
1297 INIT_LIST_HEAD(&chan
->ld_completed
);
1300 chan
->pm_state
= RUNNING
;
1303 chan
->common
.device
= &fdev
->common
;
1304 dma_cookie_init(&chan
->common
);
1306 /* find the IRQ line, if it exists in the device tree */
1307 chan
->irq
= irq_of_parse_and_map(node
, 0);
1309 /* Add the channel to DMA device channel list */
1310 list_add_tail(&chan
->common
.device_node
, &fdev
->common
.channels
);
1312 dev_info(fdev
->dev
, "#%d (%s), irq %d\n", chan
->id
, compatible
,
1313 chan
->irq
? chan
->irq
: fdev
->irq
);
1318 iounmap(chan
->regs
);
1325 static void fsl_dma_chan_remove(struct fsldma_chan
*chan
)
1327 irq_dispose_mapping(chan
->irq
);
1328 list_del(&chan
->common
.device_node
);
1329 iounmap(chan
->regs
);
1333 static int fsldma_of_probe(struct platform_device
*op
)
1335 struct fsldma_device
*fdev
;
1336 struct device_node
*child
;
1339 fdev
= kzalloc(sizeof(*fdev
), GFP_KERNEL
);
1345 fdev
->dev
= &op
->dev
;
1346 INIT_LIST_HEAD(&fdev
->common
.channels
);
1348 /* ioremap the registers for use */
1349 fdev
->regs
= of_iomap(op
->dev
.of_node
, 0);
1351 dev_err(&op
->dev
, "unable to ioremap registers\n");
1356 /* map the channel IRQ if it exists, but don't hookup the handler yet */
1357 fdev
->irq
= irq_of_parse_and_map(op
->dev
.of_node
, 0);
1359 dma_cap_set(DMA_MEMCPY
, fdev
->common
.cap_mask
);
1360 dma_cap_set(DMA_SG
, fdev
->common
.cap_mask
);
1361 dma_cap_set(DMA_SLAVE
, fdev
->common
.cap_mask
);
1362 fdev
->common
.device_alloc_chan_resources
= fsl_dma_alloc_chan_resources
;
1363 fdev
->common
.device_free_chan_resources
= fsl_dma_free_chan_resources
;
1364 fdev
->common
.device_prep_dma_memcpy
= fsl_dma_prep_memcpy
;
1365 fdev
->common
.device_prep_dma_sg
= fsl_dma_prep_sg
;
1366 fdev
->common
.device_tx_status
= fsl_tx_status
;
1367 fdev
->common
.device_issue_pending
= fsl_dma_memcpy_issue_pending
;
1368 fdev
->common
.device_config
= fsl_dma_device_config
;
1369 fdev
->common
.device_terminate_all
= fsl_dma_device_terminate_all
;
1370 fdev
->common
.dev
= &op
->dev
;
1372 fdev
->common
.src_addr_widths
= FSL_DMA_BUSWIDTHS
;
1373 fdev
->common
.dst_addr_widths
= FSL_DMA_BUSWIDTHS
;
1374 fdev
->common
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
1375 fdev
->common
.residue_granularity
= DMA_RESIDUE_GRANULARITY_DESCRIPTOR
;
1377 dma_set_mask(&(op
->dev
), DMA_BIT_MASK(36));
1379 platform_set_drvdata(op
, fdev
);
1382 * We cannot use of_platform_bus_probe() because there is no
1383 * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1386 for_each_child_of_node(op
->dev
.of_node
, child
) {
1387 if (of_device_is_compatible(child
, "fsl,eloplus-dma-channel")) {
1388 fsl_dma_chan_probe(fdev
, child
,
1389 FSL_DMA_IP_85XX
| FSL_DMA_BIG_ENDIAN
,
1390 "fsl,eloplus-dma-channel");
1393 if (of_device_is_compatible(child
, "fsl,elo-dma-channel")) {
1394 fsl_dma_chan_probe(fdev
, child
,
1395 FSL_DMA_IP_83XX
| FSL_DMA_LITTLE_ENDIAN
,
1396 "fsl,elo-dma-channel");
1401 * Hookup the IRQ handler(s)
1403 * If we have a per-controller interrupt, we prefer that to the
1404 * per-channel interrupts to reduce the number of shared interrupt
1405 * handlers on the same IRQ line
1407 err
= fsldma_request_irqs(fdev
);
1409 dev_err(fdev
->dev
, "unable to request IRQs\n");
1413 dma_async_device_register(&fdev
->common
);
1417 irq_dispose_mapping(fdev
->irq
);
1418 iounmap(fdev
->regs
);
1425 static int fsldma_of_remove(struct platform_device
*op
)
1427 struct fsldma_device
*fdev
;
1430 fdev
= platform_get_drvdata(op
);
1431 dma_async_device_unregister(&fdev
->common
);
1433 fsldma_free_irqs(fdev
);
1435 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1437 fsl_dma_chan_remove(fdev
->chan
[i
]);
1440 iounmap(fdev
->regs
);
1447 static int fsldma_suspend_late(struct device
*dev
)
1449 struct platform_device
*pdev
= to_platform_device(dev
);
1450 struct fsldma_device
*fdev
= platform_get_drvdata(pdev
);
1451 struct fsldma_chan
*chan
;
1454 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1455 chan
= fdev
->chan
[i
];
1459 spin_lock_bh(&chan
->desc_lock
);
1460 if (unlikely(!chan
->idle
))
1462 chan
->regs_save
.mr
= get_mr(chan
);
1463 chan
->pm_state
= SUSPENDED
;
1464 spin_unlock_bh(&chan
->desc_lock
);
1469 for (; i
>= 0; i
--) {
1470 chan
= fdev
->chan
[i
];
1473 chan
->pm_state
= RUNNING
;
1474 spin_unlock_bh(&chan
->desc_lock
);
1479 static int fsldma_resume_early(struct device
*dev
)
1481 struct platform_device
*pdev
= to_platform_device(dev
);
1482 struct fsldma_device
*fdev
= platform_get_drvdata(pdev
);
1483 struct fsldma_chan
*chan
;
1487 for (i
= 0; i
< FSL_DMA_MAX_CHANS_PER_DEVICE
; i
++) {
1488 chan
= fdev
->chan
[i
];
1492 spin_lock_bh(&chan
->desc_lock
);
1493 mode
= chan
->regs_save
.mr
1494 & ~FSL_DMA_MR_CS
& ~FSL_DMA_MR_CC
& ~FSL_DMA_MR_CA
;
1496 chan
->pm_state
= RUNNING
;
1497 spin_unlock_bh(&chan
->desc_lock
);
1503 static const struct dev_pm_ops fsldma_pm_ops
= {
1504 .suspend_late
= fsldma_suspend_late
,
1505 .resume_early
= fsldma_resume_early
,
1509 static const struct of_device_id fsldma_of_ids
[] = {
1510 { .compatible
= "fsl,elo3-dma", },
1511 { .compatible
= "fsl,eloplus-dma", },
1512 { .compatible
= "fsl,elo-dma", },
1515 MODULE_DEVICE_TABLE(of
, fsldma_of_ids
);
1517 static struct platform_driver fsldma_of_driver
= {
1519 .name
= "fsl-elo-dma",
1520 .of_match_table
= fsldma_of_ids
,
1522 .pm
= &fsldma_pm_ops
,
1525 .probe
= fsldma_of_probe
,
1526 .remove
= fsldma_of_remove
,
1529 /*----------------------------------------------------------------------------*/
1530 /* Module Init / Exit */
1531 /*----------------------------------------------------------------------------*/
1533 static __init
int fsldma_init(void)
1535 pr_info("Freescale Elo series DMA driver\n");
1536 return platform_driver_register(&fsldma_of_driver
);
1539 static void __exit
fsldma_exit(void)
1541 platform_driver_unregister(&fsldma_of_driver
);
1544 subsys_initcall(fsldma_init
);
1545 module_exit(fsldma_exit
);
1547 MODULE_DESCRIPTION("Freescale Elo series DMA driver");
1548 MODULE_LICENSE("GPL");