1 // SPDX-License-Identifier: GPL-2.0+
3 // Actions Semi Owl SoCs DMA driver
5 // Copyright (c) 2014 Actions Semi Inc.
6 // Author: David Liu <liuwei@actions-semi.com>
8 // Copyright (c) 2018 Linaro Ltd.
9 // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/err.h>
18 #include <linux/init.h>
19 #include <linux/interrupt.h>
22 #include <linux/module.h>
23 #include <linux/of_device.h>
24 #include <linux/of_dma.h>
25 #include <linux/slab.h>
28 #define OWL_DMA_FRAME_MAX_LENGTH 0xfffff
30 /* Global DMA Controller Registers */
31 #define OWL_DMA_IRQ_PD0 0x00
32 #define OWL_DMA_IRQ_PD1 0x04
33 #define OWL_DMA_IRQ_PD2 0x08
34 #define OWL_DMA_IRQ_PD3 0x0C
35 #define OWL_DMA_IRQ_EN0 0x10
36 #define OWL_DMA_IRQ_EN1 0x14
37 #define OWL_DMA_IRQ_EN2 0x18
38 #define OWL_DMA_IRQ_EN3 0x1C
39 #define OWL_DMA_SECURE_ACCESS_CTL 0x20
40 #define OWL_DMA_NIC_QOS 0x24
41 #define OWL_DMA_DBGSEL 0x28
42 #define OWL_DMA_IDLE_STAT 0x2C
44 /* Channel Registers */
45 #define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100)
46 #define OWL_DMAX_MODE 0x00
47 #define OWL_DMAX_SOURCE 0x04
48 #define OWL_DMAX_DESTINATION 0x08
49 #define OWL_DMAX_FRAME_LEN 0x0C
50 #define OWL_DMAX_FRAME_CNT 0x10
51 #define OWL_DMAX_REMAIN_FRAME_CNT 0x14
52 #define OWL_DMAX_REMAIN_CNT 0x18
53 #define OWL_DMAX_SOURCE_STRIDE 0x1C
54 #define OWL_DMAX_DESTINATION_STRIDE 0x20
55 #define OWL_DMAX_START 0x24
56 #define OWL_DMAX_PAUSE 0x28
57 #define OWL_DMAX_CHAINED_CTL 0x2C
58 #define OWL_DMAX_CONSTANT 0x30
59 #define OWL_DMAX_LINKLIST_CTL 0x34
60 #define OWL_DMAX_NEXT_DESCRIPTOR 0x38
61 #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C
62 #define OWL_DMAX_INT_CTL 0x40
63 #define OWL_DMAX_INT_STATUS 0x44
64 #define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48
65 #define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C
67 /* OWL_DMAX_MODE Bits */
68 #define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0)
69 #define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8)
70 #define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0)
71 #define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2)
72 #define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3)
73 #define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10)
74 #define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0)
75 #define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2)
76 #define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3)
77 #define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16)
78 #define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0)
79 #define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1)
80 #define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2)
81 #define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18)
82 #define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0)
83 #define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1)
84 #define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2)
85 #define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20)
86 #define OWL_DMA_MODE_CB BIT(23)
87 #define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28)
88 #define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0)
89 #define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1)
90 #define OWL_DMA_MODE_CFE BIT(29)
91 #define OWL_DMA_MODE_LME BIT(30)
92 #define OWL_DMA_MODE_CME BIT(31)
94 /* OWL_DMAX_LINKLIST_CTL Bits */
95 #define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8)
96 #define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0)
97 #define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1)
98 #define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2)
99 #define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10)
100 #define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0)
101 #define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1)
102 #define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2)
103 #define OWL_DMA_LLC_SUSPEND BIT(16)
105 /* OWL_DMAX_INT_CTL Bits */
106 #define OWL_DMA_INTCTL_BLOCK BIT(0)
107 #define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1)
108 #define OWL_DMA_INTCTL_FRAME BIT(2)
109 #define OWL_DMA_INTCTL_HALF_FRAME BIT(3)
110 #define OWL_DMA_INTCTL_LAST_FRAME BIT(4)
112 /* OWL_DMAX_INT_STATUS Bits */
113 #define OWL_DMA_INTSTAT_BLOCK BIT(0)
114 #define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1)
115 #define OWL_DMA_INTSTAT_FRAME BIT(2)
116 #define OWL_DMA_INTSTAT_HALF_FRAME BIT(3)
117 #define OWL_DMA_INTSTAT_LAST_FRAME BIT(4)
119 /* Pack shift and newshift in a single word */
120 #define BIT_FIELD(val, width, shift, newshift) \
121 ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))
124 * struct owl_dma_lli_hw - Hardware link list for dma transfer
125 * @next_lli: physical address of the next link list
126 * @saddr: source physical address
127 * @daddr: destination physical address
128 * @flen: frame length
130 * @src_stride: source stride
131 * @dst_stride: destination stride
132 * @ctrla: dma_mode and linklist ctrl config
133 * @ctrlb: interrupt config
134 * @const_num: data for constant fill
136 struct owl_dma_lli_hw
{
150 * struct owl_dma_lli - Link list for dma transfer
151 * @hw: hardware link list
152 * @phys: physical address of hardware link list
153 * @node: node for txd's lli_list
156 struct owl_dma_lli_hw hw
;
158 struct list_head node
;
162 * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
163 * @vd: virtual DMA descriptor
164 * @lli_list: link list of lli nodes
165 * @cyclic: flag to indicate cyclic transfers
168 struct virt_dma_desc vd
;
169 struct list_head lli_list
;
174 * struct owl_dma_pchan - Holder for the physical channels
175 * @id: physical index to this channel
176 * @base: virtual memory base for the dma channel
177 * @vchan: the virtual channel currently being served by this physical channel
179 struct owl_dma_pchan
{
182 struct owl_dma_vchan
*vchan
;
186 * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
187 * @vc: wrappped virtual channel
188 * @pchan: the physical channel utilized by this channel
189 * @txd: active transaction on this channel
190 * @cfg: slave configuration for this channel
191 * @drq: physical DMA request ID for this channel
193 struct owl_dma_vchan
{
194 struct virt_dma_chan vc
;
195 struct owl_dma_pchan
*pchan
;
196 struct owl_dma_txd
*txd
;
197 struct dma_slave_config cfg
;
202 * struct owl_dma - Holder for the Owl DMA controller
203 * @dma: dma engine for this instance
204 * @base: virtual memory base for the DMA controller
205 * @clk: clock for the DMA controller
206 * @lock: a lock to use when change DMA controller global register
207 * @lli_pool: a pool for the LLI descriptors
208 * @irq: interrupt ID for the DMA controller
209 * @nr_pchans: the number of physical channels
210 * @pchans: array of data for the physical channels
211 * @nr_vchans: the number of physical channels
212 * @vchans: array of data for the physical channels
215 struct dma_device dma
;
219 struct dma_pool
*lli_pool
;
222 unsigned int nr_pchans
;
223 struct owl_dma_pchan
*pchans
;
225 unsigned int nr_vchans
;
226 struct owl_dma_vchan
*vchans
;
229 static void pchan_update(struct owl_dma_pchan
*pchan
, u32 reg
,
234 regval
= readl(pchan
->base
+ reg
);
241 writel(val
, pchan
->base
+ reg
);
244 static void pchan_writel(struct owl_dma_pchan
*pchan
, u32 reg
, u32 data
)
246 writel(data
, pchan
->base
+ reg
);
249 static u32
pchan_readl(struct owl_dma_pchan
*pchan
, u32 reg
)
251 return readl(pchan
->base
+ reg
);
254 static void dma_update(struct owl_dma
*od
, u32 reg
, u32 val
, bool state
)
258 regval
= readl(od
->base
+ reg
);
265 writel(val
, od
->base
+ reg
);
268 static void dma_writel(struct owl_dma
*od
, u32 reg
, u32 data
)
270 writel(data
, od
->base
+ reg
);
273 static u32
dma_readl(struct owl_dma
*od
, u32 reg
)
275 return readl(od
->base
+ reg
);
278 static inline struct owl_dma
*to_owl_dma(struct dma_device
*dd
)
280 return container_of(dd
, struct owl_dma
, dma
);
283 static struct device
*chan2dev(struct dma_chan
*chan
)
285 return &chan
->dev
->device
;
288 static inline struct owl_dma_vchan
*to_owl_vchan(struct dma_chan
*chan
)
290 return container_of(chan
, struct owl_dma_vchan
, vc
.chan
);
293 static inline struct owl_dma_txd
*to_owl_txd(struct dma_async_tx_descriptor
*tx
)
295 return container_of(tx
, struct owl_dma_txd
, vd
.tx
);
298 static inline u32
llc_hw_ctrla(u32 mode
, u32 llc_ctl
)
302 ctl
= BIT_FIELD(mode
, 4, 28, 28) |
303 BIT_FIELD(mode
, 8, 16, 20) |
304 BIT_FIELD(mode
, 4, 8, 16) |
305 BIT_FIELD(mode
, 6, 0, 10) |
306 BIT_FIELD(llc_ctl
, 2, 10, 8) |
307 BIT_FIELD(llc_ctl
, 2, 8, 6);
312 static inline u32
llc_hw_ctrlb(u32 int_ctl
)
316 ctl
= BIT_FIELD(int_ctl
, 7, 0, 18);
321 static void owl_dma_free_lli(struct owl_dma
*od
,
322 struct owl_dma_lli
*lli
)
324 list_del(&lli
->node
);
325 dma_pool_free(od
->lli_pool
, lli
, lli
->phys
);
328 static struct owl_dma_lli
*owl_dma_alloc_lli(struct owl_dma
*od
)
330 struct owl_dma_lli
*lli
;
333 lli
= dma_pool_alloc(od
->lli_pool
, GFP_NOWAIT
, &phys
);
337 INIT_LIST_HEAD(&lli
->node
);
343 static struct owl_dma_lli
*owl_dma_add_lli(struct owl_dma_txd
*txd
,
344 struct owl_dma_lli
*prev
,
345 struct owl_dma_lli
*next
,
349 list_add_tail(&next
->node
, &txd
->lli_list
);
352 prev
->hw
.next_lli
= next
->phys
;
353 prev
->hw
.ctrla
|= llc_hw_ctrla(OWL_DMA_MODE_LME
, 0);
359 static inline int owl_dma_cfg_lli(struct owl_dma_vchan
*vchan
,
360 struct owl_dma_lli
*lli
,
361 dma_addr_t src
, dma_addr_t dst
,
362 u32 len
, enum dma_transfer_direction dir
,
363 struct dma_slave_config
*sconfig
,
366 struct owl_dma_lli_hw
*hw
= &lli
->hw
;
369 mode
= OWL_DMA_MODE_PW(0);
373 mode
|= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU
|
374 OWL_DMA_MODE_DT_DCU
| OWL_DMA_MODE_SAM_INC
|
375 OWL_DMA_MODE_DAM_INC
;
379 mode
|= OWL_DMA_MODE_TS(vchan
->drq
)
380 | OWL_DMA_MODE_ST_DCU
| OWL_DMA_MODE_DT_DEV
381 | OWL_DMA_MODE_SAM_INC
| OWL_DMA_MODE_DAM_CONST
;
384 * Hardware only supports 32bit and 8bit buswidth. Since the
385 * default is 32bit, select 8bit only when requested.
387 if (sconfig
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_1_BYTE
)
388 mode
|= OWL_DMA_MODE_NDDBW_8BIT
;
392 mode
|= OWL_DMA_MODE_TS(vchan
->drq
)
393 | OWL_DMA_MODE_ST_DEV
| OWL_DMA_MODE_DT_DCU
394 | OWL_DMA_MODE_SAM_CONST
| OWL_DMA_MODE_DAM_INC
;
397 * Hardware only supports 32bit and 8bit buswidth. Since the
398 * default is 32bit, select 8bit only when requested.
400 if (sconfig
->src_addr_width
== DMA_SLAVE_BUSWIDTH_1_BYTE
)
401 mode
|= OWL_DMA_MODE_NDDBW_8BIT
;
408 hw
->next_lli
= 0; /* One link list by default */
412 hw
->fcnt
= 1; /* Frame count fixed as 1 */
413 hw
->flen
= len
; /* Max frame length is 1MB */
416 hw
->ctrla
= llc_hw_ctrla(mode
,
417 OWL_DMA_LLC_SAV_LOAD_NEXT
|
418 OWL_DMA_LLC_DAV_LOAD_NEXT
);
421 hw
->ctrlb
= llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK
);
423 hw
->ctrlb
= llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK
);
428 static struct owl_dma_pchan
*owl_dma_get_pchan(struct owl_dma
*od
,
429 struct owl_dma_vchan
*vchan
)
431 struct owl_dma_pchan
*pchan
= NULL
;
435 for (i
= 0; i
< od
->nr_pchans
; i
++) {
436 pchan
= &od
->pchans
[i
];
438 spin_lock_irqsave(&od
->lock
, flags
);
440 pchan
->vchan
= vchan
;
441 spin_unlock_irqrestore(&od
->lock
, flags
);
445 spin_unlock_irqrestore(&od
->lock
, flags
);
451 static int owl_dma_pchan_busy(struct owl_dma
*od
, struct owl_dma_pchan
*pchan
)
455 val
= dma_readl(od
, OWL_DMA_IDLE_STAT
);
457 return !(val
& (1 << pchan
->id
));
460 static void owl_dma_terminate_pchan(struct owl_dma
*od
,
461 struct owl_dma_pchan
*pchan
)
466 pchan_writel(pchan
, OWL_DMAX_START
, 0);
467 pchan_update(pchan
, OWL_DMAX_INT_STATUS
, 0xff, false);
469 spin_lock_irqsave(&od
->lock
, flags
);
470 dma_update(od
, OWL_DMA_IRQ_EN0
, (1 << pchan
->id
), false);
472 irq_pd
= dma_readl(od
, OWL_DMA_IRQ_PD0
);
473 if (irq_pd
& (1 << pchan
->id
)) {
474 dev_warn(od
->dma
.dev
,
475 "terminating pchan %d that still has pending irq\n",
477 dma_writel(od
, OWL_DMA_IRQ_PD0
, (1 << pchan
->id
));
482 spin_unlock_irqrestore(&od
->lock
, flags
);
485 static void owl_dma_pause_pchan(struct owl_dma_pchan
*pchan
)
487 pchan_writel(pchan
, 1, OWL_DMAX_PAUSE
);
490 static void owl_dma_resume_pchan(struct owl_dma_pchan
*pchan
)
492 pchan_writel(pchan
, 0, OWL_DMAX_PAUSE
);
495 static int owl_dma_start_next_txd(struct owl_dma_vchan
*vchan
)
497 struct owl_dma
*od
= to_owl_dma(vchan
->vc
.chan
.device
);
498 struct virt_dma_desc
*vd
= vchan_next_desc(&vchan
->vc
);
499 struct owl_dma_pchan
*pchan
= vchan
->pchan
;
500 struct owl_dma_txd
*txd
= to_owl_txd(&vd
->tx
);
501 struct owl_dma_lli
*lli
;
509 /* Wait for channel inactive */
510 while (owl_dma_pchan_busy(od
, pchan
))
513 lli
= list_first_entry(&txd
->lli_list
,
514 struct owl_dma_lli
, node
);
517 int_ctl
= OWL_DMA_INTCTL_BLOCK
;
519 int_ctl
= OWL_DMA_INTCTL_SUPER_BLOCK
;
521 pchan_writel(pchan
, OWL_DMAX_MODE
, OWL_DMA_MODE_LME
);
522 pchan_writel(pchan
, OWL_DMAX_LINKLIST_CTL
,
523 OWL_DMA_LLC_SAV_LOAD_NEXT
| OWL_DMA_LLC_DAV_LOAD_NEXT
);
524 pchan_writel(pchan
, OWL_DMAX_NEXT_DESCRIPTOR
, lli
->phys
);
525 pchan_writel(pchan
, OWL_DMAX_INT_CTL
, int_ctl
);
527 /* Clear IRQ status for this pchan */
528 pchan_update(pchan
, OWL_DMAX_INT_STATUS
, 0xff, false);
530 spin_lock_irqsave(&od
->lock
, flags
);
532 dma_update(od
, OWL_DMA_IRQ_EN0
, (1 << pchan
->id
), true);
534 spin_unlock_irqrestore(&od
->lock
, flags
);
536 dev_dbg(chan2dev(&vchan
->vc
.chan
), "starting pchan %d\n", pchan
->id
);
538 /* Start DMA transfer for this pchan */
539 pchan_writel(pchan
, OWL_DMAX_START
, 0x1);
544 static void owl_dma_phy_free(struct owl_dma
*od
, struct owl_dma_vchan
*vchan
)
546 /* Ensure that the physical channel is stopped */
547 owl_dma_terminate_pchan(od
, vchan
->pchan
);
552 static irqreturn_t
owl_dma_interrupt(int irq
, void *dev_id
)
554 struct owl_dma
*od
= dev_id
;
555 struct owl_dma_vchan
*vchan
;
556 struct owl_dma_pchan
*pchan
;
557 unsigned long pending
;
559 unsigned int global_irq_pending
, chan_irq_pending
;
561 spin_lock(&od
->lock
);
563 pending
= dma_readl(od
, OWL_DMA_IRQ_PD0
);
565 /* Clear IRQ status for each pchan */
566 for_each_set_bit(i
, &pending
, od
->nr_pchans
) {
567 pchan
= &od
->pchans
[i
];
568 pchan_update(pchan
, OWL_DMAX_INT_STATUS
, 0xff, false);
571 /* Clear pending IRQ */
572 dma_writel(od
, OWL_DMA_IRQ_PD0
, pending
);
574 /* Check missed pending IRQ */
575 for (i
= 0; i
< od
->nr_pchans
; i
++) {
576 pchan
= &od
->pchans
[i
];
577 chan_irq_pending
= pchan_readl(pchan
, OWL_DMAX_INT_CTL
) &
578 pchan_readl(pchan
, OWL_DMAX_INT_STATUS
);
580 /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
581 dma_readl(od
, OWL_DMA_IRQ_PD0
);
583 global_irq_pending
= dma_readl(od
, OWL_DMA_IRQ_PD0
);
585 if (chan_irq_pending
&& !(global_irq_pending
& BIT(i
))) {
587 "global and channel IRQ pending match err\n");
589 /* Clear IRQ status for this pchan */
590 pchan_update(pchan
, OWL_DMAX_INT_STATUS
,
593 /* Update global IRQ pending */
598 spin_unlock(&od
->lock
);
600 for_each_set_bit(i
, &pending
, od
->nr_pchans
) {
601 struct owl_dma_txd
*txd
;
603 pchan
= &od
->pchans
[i
];
605 vchan
= pchan
->vchan
;
607 dev_warn(od
->dma
.dev
, "no vchan attached on pchan %d\n",
612 spin_lock(&vchan
->vc
.lock
);
618 vchan_cookie_complete(&txd
->vd
);
621 * Start the next descriptor (if any),
622 * otherwise free this channel.
624 if (vchan_next_desc(&vchan
->vc
))
625 owl_dma_start_next_txd(vchan
);
627 owl_dma_phy_free(od
, vchan
);
630 spin_unlock(&vchan
->vc
.lock
);
636 static void owl_dma_free_txd(struct owl_dma
*od
, struct owl_dma_txd
*txd
)
638 struct owl_dma_lli
*lli
, *_lli
;
643 list_for_each_entry_safe(lli
, _lli
, &txd
->lli_list
, node
)
644 owl_dma_free_lli(od
, lli
);
649 static void owl_dma_desc_free(struct virt_dma_desc
*vd
)
651 struct owl_dma
*od
= to_owl_dma(vd
->tx
.chan
->device
);
652 struct owl_dma_txd
*txd
= to_owl_txd(&vd
->tx
);
654 owl_dma_free_txd(od
, txd
);
657 static int owl_dma_terminate_all(struct dma_chan
*chan
)
659 struct owl_dma
*od
= to_owl_dma(chan
->device
);
660 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
664 spin_lock_irqsave(&vchan
->vc
.lock
, flags
);
667 owl_dma_phy_free(od
, vchan
);
670 owl_dma_desc_free(&vchan
->txd
->vd
);
674 vchan_get_all_descriptors(&vchan
->vc
, &head
);
676 spin_unlock_irqrestore(&vchan
->vc
.lock
, flags
);
678 vchan_dma_desc_free_list(&vchan
->vc
, &head
);
683 static int owl_dma_config(struct dma_chan
*chan
,
684 struct dma_slave_config
*config
)
686 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
688 /* Reject definitely invalid configurations */
689 if (config
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
690 config
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
693 memcpy(&vchan
->cfg
, config
, sizeof(struct dma_slave_config
));
698 static int owl_dma_pause(struct dma_chan
*chan
)
700 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
703 spin_lock_irqsave(&vchan
->vc
.lock
, flags
);
705 owl_dma_pause_pchan(vchan
->pchan
);
707 spin_unlock_irqrestore(&vchan
->vc
.lock
, flags
);
712 static int owl_dma_resume(struct dma_chan
*chan
)
714 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
717 if (!vchan
->pchan
&& !vchan
->txd
)
720 dev_dbg(chan2dev(chan
), "vchan %p: resume\n", &vchan
->vc
);
722 spin_lock_irqsave(&vchan
->vc
.lock
, flags
);
724 owl_dma_resume_pchan(vchan
->pchan
);
726 spin_unlock_irqrestore(&vchan
->vc
.lock
, flags
);
731 static u32
owl_dma_getbytes_chan(struct owl_dma_vchan
*vchan
)
733 struct owl_dma_pchan
*pchan
;
734 struct owl_dma_txd
*txd
;
735 struct owl_dma_lli
*lli
;
736 unsigned int next_lli_phy
;
739 pchan
= vchan
->pchan
;
745 /* Get remain count of current node in link list */
746 bytes
= pchan_readl(pchan
, OWL_DMAX_REMAIN_CNT
);
748 /* Loop through the preceding nodes to get total remaining bytes */
749 if (pchan_readl(pchan
, OWL_DMAX_MODE
) & OWL_DMA_MODE_LME
) {
750 next_lli_phy
= pchan_readl(pchan
, OWL_DMAX_NEXT_DESCRIPTOR
);
751 list_for_each_entry(lli
, &txd
->lli_list
, node
) {
752 /* Start from the next active node */
753 if (lli
->phys
== next_lli_phy
) {
754 list_for_each_entry(lli
, &txd
->lli_list
, node
)
755 bytes
+= lli
->hw
.flen
;
764 static enum dma_status
owl_dma_tx_status(struct dma_chan
*chan
,
766 struct dma_tx_state
*state
)
768 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
769 struct owl_dma_lli
*lli
;
770 struct virt_dma_desc
*vd
;
771 struct owl_dma_txd
*txd
;
776 ret
= dma_cookie_status(chan
, cookie
, state
);
777 if (ret
== DMA_COMPLETE
|| !state
)
780 spin_lock_irqsave(&vchan
->vc
.lock
, flags
);
782 vd
= vchan_find_desc(&vchan
->vc
, cookie
);
784 txd
= to_owl_txd(&vd
->tx
);
785 list_for_each_entry(lli
, &txd
->lli_list
, node
)
786 bytes
+= lli
->hw
.flen
;
788 bytes
= owl_dma_getbytes_chan(vchan
);
791 spin_unlock_irqrestore(&vchan
->vc
.lock
, flags
);
793 dma_set_residue(state
, bytes
);
798 static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan
*vchan
)
800 struct owl_dma
*od
= to_owl_dma(vchan
->vc
.chan
.device
);
801 struct owl_dma_pchan
*pchan
;
803 pchan
= owl_dma_get_pchan(od
, vchan
);
807 dev_dbg(od
->dma
.dev
, "allocated pchan %d\n", pchan
->id
);
809 vchan
->pchan
= pchan
;
810 owl_dma_start_next_txd(vchan
);
813 static void owl_dma_issue_pending(struct dma_chan
*chan
)
815 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
818 spin_lock_irqsave(&vchan
->vc
.lock
, flags
);
819 if (vchan_issue_pending(&vchan
->vc
)) {
821 owl_dma_phy_alloc_and_start(vchan
);
823 spin_unlock_irqrestore(&vchan
->vc
.lock
, flags
);
826 static struct dma_async_tx_descriptor
827 *owl_dma_prep_memcpy(struct dma_chan
*chan
,
828 dma_addr_t dst
, dma_addr_t src
,
829 size_t len
, unsigned long flags
)
831 struct owl_dma
*od
= to_owl_dma(chan
->device
);
832 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
833 struct owl_dma_txd
*txd
;
834 struct owl_dma_lli
*lli
, *prev
= NULL
;
835 size_t offset
, bytes
;
841 txd
= kzalloc(sizeof(*txd
), GFP_NOWAIT
);
845 INIT_LIST_HEAD(&txd
->lli_list
);
847 /* Process the transfer as frame by frame */
848 for (offset
= 0; offset
< len
; offset
+= bytes
) {
849 lli
= owl_dma_alloc_lli(od
);
851 dev_warn(chan2dev(chan
), "failed to allocate lli\n");
855 bytes
= min_t(size_t, (len
- offset
), OWL_DMA_FRAME_MAX_LENGTH
);
857 ret
= owl_dma_cfg_lli(vchan
, lli
, src
+ offset
, dst
+ offset
,
858 bytes
, DMA_MEM_TO_MEM
,
859 &vchan
->cfg
, txd
->cyclic
);
861 dev_warn(chan2dev(chan
), "failed to config lli\n");
865 prev
= owl_dma_add_lli(txd
, prev
, lli
, false);
868 return vchan_tx_prep(&vchan
->vc
, &txd
->vd
, flags
);
871 owl_dma_free_txd(od
, txd
);
875 static struct dma_async_tx_descriptor
876 *owl_dma_prep_slave_sg(struct dma_chan
*chan
,
877 struct scatterlist
*sgl
,
879 enum dma_transfer_direction dir
,
880 unsigned long flags
, void *context
)
882 struct owl_dma
*od
= to_owl_dma(chan
->device
);
883 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
884 struct dma_slave_config
*sconfig
= &vchan
->cfg
;
885 struct owl_dma_txd
*txd
;
886 struct owl_dma_lli
*lli
, *prev
= NULL
;
887 struct scatterlist
*sg
;
888 dma_addr_t addr
, src
= 0, dst
= 0;
892 txd
= kzalloc(sizeof(*txd
), GFP_NOWAIT
);
896 INIT_LIST_HEAD(&txd
->lli_list
);
898 for_each_sg(sgl
, sg
, sg_len
, i
) {
899 addr
= sg_dma_address(sg
);
900 len
= sg_dma_len(sg
);
902 if (len
> OWL_DMA_FRAME_MAX_LENGTH
) {
904 "frame length exceeds max supported length");
908 lli
= owl_dma_alloc_lli(od
);
910 dev_err(chan2dev(chan
), "failed to allocate lli");
914 if (dir
== DMA_MEM_TO_DEV
) {
916 dst
= sconfig
->dst_addr
;
918 src
= sconfig
->src_addr
;
922 ret
= owl_dma_cfg_lli(vchan
, lli
, src
, dst
, len
, dir
, sconfig
,
925 dev_warn(chan2dev(chan
), "failed to config lli");
929 prev
= owl_dma_add_lli(txd
, prev
, lli
, false);
932 return vchan_tx_prep(&vchan
->vc
, &txd
->vd
, flags
);
935 owl_dma_free_txd(od
, txd
);
940 static struct dma_async_tx_descriptor
941 *owl_prep_dma_cyclic(struct dma_chan
*chan
,
942 dma_addr_t buf_addr
, size_t buf_len
,
944 enum dma_transfer_direction dir
,
947 struct owl_dma
*od
= to_owl_dma(chan
->device
);
948 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
949 struct dma_slave_config
*sconfig
= &vchan
->cfg
;
950 struct owl_dma_txd
*txd
;
951 struct owl_dma_lli
*lli
, *prev
= NULL
, *first
= NULL
;
952 dma_addr_t src
= 0, dst
= 0;
953 unsigned int periods
= buf_len
/ period_len
;
956 txd
= kzalloc(sizeof(*txd
), GFP_NOWAIT
);
960 INIT_LIST_HEAD(&txd
->lli_list
);
963 for (i
= 0; i
< periods
; i
++) {
964 lli
= owl_dma_alloc_lli(od
);
966 dev_warn(chan2dev(chan
), "failed to allocate lli");
970 if (dir
== DMA_MEM_TO_DEV
) {
971 src
= buf_addr
+ (period_len
* i
);
972 dst
= sconfig
->dst_addr
;
973 } else if (dir
== DMA_DEV_TO_MEM
) {
974 src
= sconfig
->src_addr
;
975 dst
= buf_addr
+ (period_len
* i
);
978 ret
= owl_dma_cfg_lli(vchan
, lli
, src
, dst
, period_len
,
979 dir
, sconfig
, txd
->cyclic
);
981 dev_warn(chan2dev(chan
), "failed to config lli");
988 prev
= owl_dma_add_lli(txd
, prev
, lli
, false);
991 /* close the cyclic list */
992 owl_dma_add_lli(txd
, prev
, first
, true);
994 return vchan_tx_prep(&vchan
->vc
, &txd
->vd
, flags
);
997 owl_dma_free_txd(od
, txd
);
1002 static void owl_dma_free_chan_resources(struct dma_chan
*chan
)
1004 struct owl_dma_vchan
*vchan
= to_owl_vchan(chan
);
1006 /* Ensure all queued descriptors are freed */
1007 vchan_free_chan_resources(&vchan
->vc
);
1010 static inline void owl_dma_free(struct owl_dma
*od
)
1012 struct owl_dma_vchan
*vchan
= NULL
;
1013 struct owl_dma_vchan
*next
;
1015 list_for_each_entry_safe(vchan
,
1016 next
, &od
->dma
.channels
, vc
.chan
.device_node
) {
1017 list_del(&vchan
->vc
.chan
.device_node
);
1018 tasklet_kill(&vchan
->vc
.task
);
1022 static struct dma_chan
*owl_dma_of_xlate(struct of_phandle_args
*dma_spec
,
1023 struct of_dma
*ofdma
)
1025 struct owl_dma
*od
= ofdma
->of_dma_data
;
1026 struct owl_dma_vchan
*vchan
;
1027 struct dma_chan
*chan
;
1028 u8 drq
= dma_spec
->args
[0];
1030 if (drq
> od
->nr_vchans
)
1033 chan
= dma_get_any_slave_channel(&od
->dma
);
1037 vchan
= to_owl_vchan(chan
);
1043 static int owl_dma_probe(struct platform_device
*pdev
)
1045 struct device_node
*np
= pdev
->dev
.of_node
;
1047 int ret
, i
, nr_channels
, nr_requests
;
1049 od
= devm_kzalloc(&pdev
->dev
, sizeof(*od
), GFP_KERNEL
);
1053 od
->base
= devm_platform_ioremap_resource(pdev
, 0);
1054 if (IS_ERR(od
->base
))
1055 return PTR_ERR(od
->base
);
1057 ret
= of_property_read_u32(np
, "dma-channels", &nr_channels
);
1059 dev_err(&pdev
->dev
, "can't get dma-channels\n");
1063 ret
= of_property_read_u32(np
, "dma-requests", &nr_requests
);
1065 dev_err(&pdev
->dev
, "can't get dma-requests\n");
1069 dev_info(&pdev
->dev
, "dma-channels %d, dma-requests %d\n",
1070 nr_channels
, nr_requests
);
1072 od
->nr_pchans
= nr_channels
;
1073 od
->nr_vchans
= nr_requests
;
1075 pdev
->dev
.coherent_dma_mask
= DMA_BIT_MASK(32);
1077 platform_set_drvdata(pdev
, od
);
1078 spin_lock_init(&od
->lock
);
1080 dma_cap_set(DMA_MEMCPY
, od
->dma
.cap_mask
);
1081 dma_cap_set(DMA_SLAVE
, od
->dma
.cap_mask
);
1082 dma_cap_set(DMA_CYCLIC
, od
->dma
.cap_mask
);
1084 od
->dma
.dev
= &pdev
->dev
;
1085 od
->dma
.device_free_chan_resources
= owl_dma_free_chan_resources
;
1086 od
->dma
.device_tx_status
= owl_dma_tx_status
;
1087 od
->dma
.device_issue_pending
= owl_dma_issue_pending
;
1088 od
->dma
.device_prep_dma_memcpy
= owl_dma_prep_memcpy
;
1089 od
->dma
.device_prep_slave_sg
= owl_dma_prep_slave_sg
;
1090 od
->dma
.device_prep_dma_cyclic
= owl_prep_dma_cyclic
;
1091 od
->dma
.device_config
= owl_dma_config
;
1092 od
->dma
.device_pause
= owl_dma_pause
;
1093 od
->dma
.device_resume
= owl_dma_resume
;
1094 od
->dma
.device_terminate_all
= owl_dma_terminate_all
;
1095 od
->dma
.src_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
1096 od
->dma
.dst_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
1097 od
->dma
.directions
= BIT(DMA_MEM_TO_MEM
);
1098 od
->dma
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
1100 INIT_LIST_HEAD(&od
->dma
.channels
);
1102 od
->clk
= devm_clk_get(&pdev
->dev
, NULL
);
1103 if (IS_ERR(od
->clk
)) {
1104 dev_err(&pdev
->dev
, "unable to get clock\n");
1105 return PTR_ERR(od
->clk
);
1109 * Eventhough the DMA controller is capable of generating 4
1110 * IRQ's for DMA priority feature, we only use 1 IRQ for
1113 od
->irq
= platform_get_irq(pdev
, 0);
1114 ret
= devm_request_irq(&pdev
->dev
, od
->irq
, owl_dma_interrupt
, 0,
1115 dev_name(&pdev
->dev
), od
);
1117 dev_err(&pdev
->dev
, "unable to request IRQ\n");
1121 /* Init physical channel */
1122 od
->pchans
= devm_kcalloc(&pdev
->dev
, od
->nr_pchans
,
1123 sizeof(struct owl_dma_pchan
), GFP_KERNEL
);
1127 for (i
= 0; i
< od
->nr_pchans
; i
++) {
1128 struct owl_dma_pchan
*pchan
= &od
->pchans
[i
];
1131 pchan
->base
= od
->base
+ OWL_DMA_CHAN_BASE(i
);
1134 /* Init virtual channel */
1135 od
->vchans
= devm_kcalloc(&pdev
->dev
, od
->nr_vchans
,
1136 sizeof(struct owl_dma_vchan
), GFP_KERNEL
);
1140 for (i
= 0; i
< od
->nr_vchans
; i
++) {
1141 struct owl_dma_vchan
*vchan
= &od
->vchans
[i
];
1143 vchan
->vc
.desc_free
= owl_dma_desc_free
;
1144 vchan_init(&vchan
->vc
, &od
->dma
);
1147 /* Create a pool of consistent memory blocks for hardware descriptors */
1148 od
->lli_pool
= dma_pool_create(dev_name(od
->dma
.dev
), od
->dma
.dev
,
1149 sizeof(struct owl_dma_lli
),
1150 __alignof__(struct owl_dma_lli
),
1152 if (!od
->lli_pool
) {
1153 dev_err(&pdev
->dev
, "unable to allocate DMA descriptor pool\n");
1157 clk_prepare_enable(od
->clk
);
1159 ret
= dma_async_device_register(&od
->dma
);
1161 dev_err(&pdev
->dev
, "failed to register DMA engine device\n");
1165 /* Device-tree DMA controller registration */
1166 ret
= of_dma_controller_register(pdev
->dev
.of_node
,
1167 owl_dma_of_xlate
, od
);
1169 dev_err(&pdev
->dev
, "of_dma_controller_register failed\n");
1170 goto err_dma_unregister
;
1176 dma_async_device_unregister(&od
->dma
);
1178 clk_disable_unprepare(od
->clk
);
1179 dma_pool_destroy(od
->lli_pool
);
1184 static int owl_dma_remove(struct platform_device
*pdev
)
1186 struct owl_dma
*od
= platform_get_drvdata(pdev
);
1188 of_dma_controller_free(pdev
->dev
.of_node
);
1189 dma_async_device_unregister(&od
->dma
);
1191 /* Mask all interrupts for this execution environment */
1192 dma_writel(od
, OWL_DMA_IRQ_EN0
, 0x0);
1194 /* Make sure we won't have any further interrupts */
1195 devm_free_irq(od
->dma
.dev
, od
->irq
, od
);
1199 clk_disable_unprepare(od
->clk
);
1204 static const struct of_device_id owl_dma_match
[] = {
1205 { .compatible
= "actions,s900-dma", },
1208 MODULE_DEVICE_TABLE(of
, owl_dma_match
);
1210 static struct platform_driver owl_dma_driver
= {
1211 .probe
= owl_dma_probe
,
1212 .remove
= owl_dma_remove
,
1215 .of_match_table
= of_match_ptr(owl_dma_match
),
1219 static int owl_dma_init(void)
1221 return platform_driver_register(&owl_dma_driver
);
1223 subsys_initcall(owl_dma_init
);
1225 static void __exit
owl_dma_exit(void)
1227 platform_driver_unregister(&owl_dma_driver
);
1229 module_exit(owl_dma_exit
);
1231 MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
1232 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
1233 MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
1234 MODULE_LICENSE("GPL");