2 * OMAP DMAengine support
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/err.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/omap-dma.h>
18 #include <linux/platform_device.h>
19 #include <linux/slab.h>
20 #include <linux/spinlock.h>
21 #include <linux/of_dma.h>
22 #include <linux/of_device.h>
26 #define OMAP_SDMA_REQUESTS 127
27 #define OMAP_SDMA_CHANNELS 32
30 struct dma_device ddev
;
33 const struct omap_dma_reg
*reg_map
;
34 struct omap_system_dma_plat_info
*plat
;
37 struct dma_pool
*desc_pool
;
38 unsigned dma_requests
;
40 uint32_t irq_enable_mask
;
41 struct omap_chan
**lch_map
;
45 struct virt_dma_chan vc
;
46 void __iomem
*channel_base
;
47 const struct omap_dma_reg
*reg_map
;
50 struct dma_slave_config cfg
;
57 struct omap_desc
*desc
;
61 #define DESC_NXT_SV_REFRESH (0x1 << 24)
62 #define DESC_NXT_SV_REUSE (0x2 << 24)
63 #define DESC_NXT_DV_REFRESH (0x1 << 26)
64 #define DESC_NXT_DV_REUSE (0x2 << 26)
65 #define DESC_NTYPE_TYPE2 (0x2 << 29)
67 /* Type 2 descriptor with Source or Destination address update */
68 struct omap_type2_desc
{
71 uint32_t addr
; /* src or dst */
82 uint32_t en
; /* number of elements (24-bit) */
83 uint32_t fn
; /* number of frames (16-bit) */
84 int32_t fi
; /* for double indexing */
85 int16_t ei
; /* for double indexing */
88 struct omap_type2_desc
*t2_desc
;
89 dma_addr_t t2_desc_paddr
;
93 struct virt_dma_desc vd
;
95 enum dma_transfer_direction dir
;
98 int32_t fi
; /* for OMAP_DMA_SYNC_PACKET / double indexing */
99 int16_t ei
; /* for double indexing */
100 uint8_t es
; /* CSDP_DATA_TYPE_xxx */
101 uint32_t ccr
; /* CCR value */
102 uint16_t clnk_ctrl
; /* CLNK_CTRL value */
103 uint16_t cicr
; /* CICR value */
104 uint32_t csdp
; /* CSDP value */
107 struct omap_sg sg
[0];
111 CAPS_0_SUPPORT_LL123
= BIT(20), /* Linked List type1/2/3 */
112 CAPS_0_SUPPORT_LL4
= BIT(21), /* Linked List type4 */
115 CCR_READ_PRIORITY
= BIT(6),
117 CCR_AUTO_INIT
= BIT(8), /* OMAP1 only */
118 CCR_REPEAT
= BIT(9), /* OMAP1 only */
119 CCR_OMAP31_DISABLE
= BIT(10), /* OMAP1 only */
120 CCR_SUSPEND_SENSITIVE
= BIT(8), /* OMAP2+ only */
121 CCR_RD_ACTIVE
= BIT(9), /* OMAP2+ only */
122 CCR_WR_ACTIVE
= BIT(10), /* OMAP2+ only */
123 CCR_SRC_AMODE_CONSTANT
= 0 << 12,
124 CCR_SRC_AMODE_POSTINC
= 1 << 12,
125 CCR_SRC_AMODE_SGLIDX
= 2 << 12,
126 CCR_SRC_AMODE_DBLIDX
= 3 << 12,
127 CCR_DST_AMODE_CONSTANT
= 0 << 14,
128 CCR_DST_AMODE_POSTINC
= 1 << 14,
129 CCR_DST_AMODE_SGLIDX
= 2 << 14,
130 CCR_DST_AMODE_DBLIDX
= 3 << 14,
131 CCR_CONSTANT_FILL
= BIT(16),
132 CCR_TRANSPARENT_COPY
= BIT(17),
134 CCR_SUPERVISOR
= BIT(22),
135 CCR_PREFETCH
= BIT(23),
136 CCR_TRIGGER_SRC
= BIT(24),
137 CCR_BUFFERING_DISABLE
= BIT(25),
138 CCR_WRITE_PRIORITY
= BIT(26),
139 CCR_SYNC_ELEMENT
= 0,
140 CCR_SYNC_FRAME
= CCR_FS
,
141 CCR_SYNC_BLOCK
= CCR_BS
,
142 CCR_SYNC_PACKET
= CCR_BS
| CCR_FS
,
144 CSDP_DATA_TYPE_8
= 0,
145 CSDP_DATA_TYPE_16
= 1,
146 CSDP_DATA_TYPE_32
= 2,
147 CSDP_SRC_PORT_EMIFF
= 0 << 2, /* OMAP1 only */
148 CSDP_SRC_PORT_EMIFS
= 1 << 2, /* OMAP1 only */
149 CSDP_SRC_PORT_OCP_T1
= 2 << 2, /* OMAP1 only */
150 CSDP_SRC_PORT_TIPB
= 3 << 2, /* OMAP1 only */
151 CSDP_SRC_PORT_OCP_T2
= 4 << 2, /* OMAP1 only */
152 CSDP_SRC_PORT_MPUI
= 5 << 2, /* OMAP1 only */
153 CSDP_SRC_PACKED
= BIT(6),
154 CSDP_SRC_BURST_1
= 0 << 7,
155 CSDP_SRC_BURST_16
= 1 << 7,
156 CSDP_SRC_BURST_32
= 2 << 7,
157 CSDP_SRC_BURST_64
= 3 << 7,
158 CSDP_DST_PORT_EMIFF
= 0 << 9, /* OMAP1 only */
159 CSDP_DST_PORT_EMIFS
= 1 << 9, /* OMAP1 only */
160 CSDP_DST_PORT_OCP_T1
= 2 << 9, /* OMAP1 only */
161 CSDP_DST_PORT_TIPB
= 3 << 9, /* OMAP1 only */
162 CSDP_DST_PORT_OCP_T2
= 4 << 9, /* OMAP1 only */
163 CSDP_DST_PORT_MPUI
= 5 << 9, /* OMAP1 only */
164 CSDP_DST_PACKED
= BIT(13),
165 CSDP_DST_BURST_1
= 0 << 14,
166 CSDP_DST_BURST_16
= 1 << 14,
167 CSDP_DST_BURST_32
= 2 << 14,
168 CSDP_DST_BURST_64
= 3 << 14,
169 CSDP_WRITE_NON_POSTED
= 0 << 16,
170 CSDP_WRITE_POSTED
= 1 << 16,
171 CSDP_WRITE_LAST_NON_POSTED
= 2 << 16,
173 CICR_TOUT_IE
= BIT(0), /* OMAP1 only */
174 CICR_DROP_IE
= BIT(1),
175 CICR_HALF_IE
= BIT(2),
176 CICR_FRAME_IE
= BIT(3),
177 CICR_LAST_IE
= BIT(4),
178 CICR_BLOCK_IE
= BIT(5),
179 CICR_PKT_IE
= BIT(7), /* OMAP2+ only */
180 CICR_TRANS_ERR_IE
= BIT(8), /* OMAP2+ only */
181 CICR_SUPERVISOR_ERR_IE
= BIT(10), /* OMAP2+ only */
182 CICR_MISALIGNED_ERR_IE
= BIT(11), /* OMAP2+ only */
183 CICR_DRAIN_IE
= BIT(12), /* OMAP2+ only */
184 CICR_SUPER_BLOCK_IE
= BIT(14), /* OMAP2+ only */
186 CLNK_CTRL_ENABLE_LNK
= BIT(15),
188 CDP_DST_VALID_INC
= 0 << 0,
189 CDP_DST_VALID_RELOAD
= 1 << 0,
190 CDP_DST_VALID_REUSE
= 2 << 0,
191 CDP_SRC_VALID_INC
= 0 << 2,
192 CDP_SRC_VALID_RELOAD
= 1 << 2,
193 CDP_SRC_VALID_REUSE
= 2 << 2,
194 CDP_NTYPE_TYPE1
= 1 << 4,
195 CDP_NTYPE_TYPE2
= 2 << 4,
196 CDP_NTYPE_TYPE3
= 3 << 4,
197 CDP_TMODE_NORMAL
= 0 << 8,
198 CDP_TMODE_LLIST
= 1 << 8,
202 static const unsigned es_bytes
[] = {
203 [CSDP_DATA_TYPE_8
] = 1,
204 [CSDP_DATA_TYPE_16
] = 2,
205 [CSDP_DATA_TYPE_32
] = 4,
208 static struct of_dma_filter_info omap_dma_info
= {
209 .filter_fn
= omap_dma_filter_fn
,
212 static inline struct omap_dmadev
*to_omap_dma_dev(struct dma_device
*d
)
214 return container_of(d
, struct omap_dmadev
, ddev
);
217 static inline struct omap_chan
*to_omap_dma_chan(struct dma_chan
*c
)
219 return container_of(c
, struct omap_chan
, vc
.chan
);
222 static inline struct omap_desc
*to_omap_dma_desc(struct dma_async_tx_descriptor
*t
)
224 return container_of(t
, struct omap_desc
, vd
.tx
);
227 static void omap_dma_desc_free(struct virt_dma_desc
*vd
)
229 struct omap_desc
*d
= to_omap_dma_desc(&vd
->tx
);
232 struct omap_dmadev
*od
= to_omap_dma_dev(vd
->tx
.chan
->device
);
235 for (i
= 0; i
< d
->sglen
; i
++) {
236 if (d
->sg
[i
].t2_desc
)
237 dma_pool_free(od
->desc_pool
, d
->sg
[i
].t2_desc
,
238 d
->sg
[i
].t2_desc_paddr
);
245 static void omap_dma_fill_type2_desc(struct omap_desc
*d
, int idx
,
246 enum dma_transfer_direction dir
, bool last
)
248 struct omap_sg
*sg
= &d
->sg
[idx
];
249 struct omap_type2_desc
*t2_desc
= sg
->t2_desc
;
252 d
->sg
[idx
- 1].t2_desc
->next_desc
= sg
->t2_desc_paddr
;
254 t2_desc
->next_desc
= 0xfffffffc;
256 t2_desc
->en
= sg
->en
;
257 t2_desc
->addr
= sg
->addr
;
258 t2_desc
->fn
= sg
->fn
& 0xffff;
259 t2_desc
->cicr
= d
->cicr
;
261 t2_desc
->cicr
&= ~CICR_BLOCK_IE
;
265 t2_desc
->cdei
= sg
->ei
;
266 t2_desc
->csei
= d
->ei
;
267 t2_desc
->cdfi
= sg
->fi
;
268 t2_desc
->csfi
= d
->fi
;
270 t2_desc
->en
|= DESC_NXT_DV_REFRESH
;
271 t2_desc
->en
|= DESC_NXT_SV_REUSE
;
274 t2_desc
->cdei
= d
->ei
;
275 t2_desc
->csei
= sg
->ei
;
276 t2_desc
->cdfi
= d
->fi
;
277 t2_desc
->csfi
= sg
->fi
;
279 t2_desc
->en
|= DESC_NXT_SV_REFRESH
;
280 t2_desc
->en
|= DESC_NXT_DV_REUSE
;
286 t2_desc
->en
|= DESC_NTYPE_TYPE2
;
289 static void omap_dma_write(uint32_t val
, unsigned type
, void __iomem
*addr
)
292 case OMAP_DMA_REG_16BIT
:
293 writew_relaxed(val
, addr
);
295 case OMAP_DMA_REG_2X16BIT
:
296 writew_relaxed(val
, addr
);
297 writew_relaxed(val
>> 16, addr
+ 2);
299 case OMAP_DMA_REG_32BIT
:
300 writel_relaxed(val
, addr
);
307 static unsigned omap_dma_read(unsigned type
, void __iomem
*addr
)
312 case OMAP_DMA_REG_16BIT
:
313 val
= readw_relaxed(addr
);
315 case OMAP_DMA_REG_2X16BIT
:
316 val
= readw_relaxed(addr
);
317 val
|= readw_relaxed(addr
+ 2) << 16;
319 case OMAP_DMA_REG_32BIT
:
320 val
= readl_relaxed(addr
);
330 static void omap_dma_glbl_write(struct omap_dmadev
*od
, unsigned reg
, unsigned val
)
332 const struct omap_dma_reg
*r
= od
->reg_map
+ reg
;
336 omap_dma_write(val
, r
->type
, od
->base
+ r
->offset
);
339 static unsigned omap_dma_glbl_read(struct omap_dmadev
*od
, unsigned reg
)
341 const struct omap_dma_reg
*r
= od
->reg_map
+ reg
;
345 return omap_dma_read(r
->type
, od
->base
+ r
->offset
);
348 static void omap_dma_chan_write(struct omap_chan
*c
, unsigned reg
, unsigned val
)
350 const struct omap_dma_reg
*r
= c
->reg_map
+ reg
;
352 omap_dma_write(val
, r
->type
, c
->channel_base
+ r
->offset
);
355 static unsigned omap_dma_chan_read(struct omap_chan
*c
, unsigned reg
)
357 const struct omap_dma_reg
*r
= c
->reg_map
+ reg
;
359 return omap_dma_read(r
->type
, c
->channel_base
+ r
->offset
);
362 static void omap_dma_clear_csr(struct omap_chan
*c
)
365 omap_dma_chan_read(c
, CSR
);
367 omap_dma_chan_write(c
, CSR
, ~0);
370 static unsigned omap_dma_get_csr(struct omap_chan
*c
)
372 unsigned val
= omap_dma_chan_read(c
, CSR
);
375 omap_dma_chan_write(c
, CSR
, val
);
380 static void omap_dma_assign(struct omap_dmadev
*od
, struct omap_chan
*c
,
383 c
->channel_base
= od
->base
+ od
->plat
->channel_stride
* lch
;
385 od
->lch_map
[lch
] = c
;
388 static void omap_dma_start(struct omap_chan
*c
, struct omap_desc
*d
)
390 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
391 uint16_t cicr
= d
->cicr
;
393 if (__dma_omap15xx(od
->plat
->dma_attr
))
394 omap_dma_chan_write(c
, CPC
, 0);
396 omap_dma_chan_write(c
, CDAC
, 0);
398 omap_dma_clear_csr(c
);
401 uint32_t cdp
= CDP_TMODE_LLIST
| CDP_NTYPE_TYPE2
| CDP_FAST
;
403 if (d
->dir
== DMA_DEV_TO_MEM
)
404 cdp
|= (CDP_DST_VALID_RELOAD
| CDP_SRC_VALID_REUSE
);
406 cdp
|= (CDP_DST_VALID_REUSE
| CDP_SRC_VALID_RELOAD
);
407 omap_dma_chan_write(c
, CDP
, cdp
);
409 omap_dma_chan_write(c
, CNDP
, d
->sg
[0].t2_desc_paddr
);
410 omap_dma_chan_write(c
, CCDN
, 0);
411 omap_dma_chan_write(c
, CCFN
, 0xffff);
412 omap_dma_chan_write(c
, CCEN
, 0xffffff);
414 cicr
&= ~CICR_BLOCK_IE
;
415 } else if (od
->ll123_supported
) {
416 omap_dma_chan_write(c
, CDP
, 0);
419 /* Enable interrupts */
420 omap_dma_chan_write(c
, CICR
, cicr
);
423 omap_dma_chan_write(c
, CCR
, d
->ccr
| CCR_ENABLE
);
428 static void omap_dma_drain_chan(struct omap_chan
*c
)
433 /* Wait for sDMA FIFO to drain */
435 val
= omap_dma_chan_read(c
, CCR
);
436 if (!(val
& (CCR_RD_ACTIVE
| CCR_WR_ACTIVE
)))
445 if (val
& (CCR_RD_ACTIVE
| CCR_WR_ACTIVE
))
446 dev_err(c
->vc
.chan
.device
->dev
,
447 "DMA drain did not complete on lch %d\n",
451 static int omap_dma_stop(struct omap_chan
*c
)
453 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
457 omap_dma_chan_write(c
, CICR
, 0);
459 omap_dma_clear_csr(c
);
461 val
= omap_dma_chan_read(c
, CCR
);
462 if (od
->plat
->errata
& DMA_ERRATA_i541
&& val
& CCR_TRIGGER_SRC
) {
465 sysconfig
= omap_dma_glbl_read(od
, OCP_SYSCONFIG
);
466 val
= sysconfig
& ~DMA_SYSCONFIG_MIDLEMODE_MASK
;
467 val
|= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE
);
468 omap_dma_glbl_write(od
, OCP_SYSCONFIG
, val
);
470 val
= omap_dma_chan_read(c
, CCR
);
472 omap_dma_chan_write(c
, CCR
, val
);
474 if (!(c
->ccr
& CCR_BUFFERING_DISABLE
))
475 omap_dma_drain_chan(c
);
477 omap_dma_glbl_write(od
, OCP_SYSCONFIG
, sysconfig
);
479 if (!(val
& CCR_ENABLE
))
483 omap_dma_chan_write(c
, CCR
, val
);
485 if (!(c
->ccr
& CCR_BUFFERING_DISABLE
))
486 omap_dma_drain_chan(c
);
491 if (!__dma_omap15xx(od
->plat
->dma_attr
) && c
->cyclic
) {
492 val
= omap_dma_chan_read(c
, CLNK_CTRL
);
495 val
|= 1 << 14; /* set the STOP_LNK bit */
497 val
&= ~CLNK_CTRL_ENABLE_LNK
;
499 omap_dma_chan_write(c
, CLNK_CTRL
, val
);
505 static void omap_dma_start_sg(struct omap_chan
*c
, struct omap_desc
*d
)
507 struct omap_sg
*sg
= d
->sg
+ c
->sgidx
;
508 unsigned cxsa
, cxei
, cxfi
;
510 if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
) {
520 omap_dma_chan_write(c
, cxsa
, sg
->addr
);
521 omap_dma_chan_write(c
, cxei
, sg
->ei
);
522 omap_dma_chan_write(c
, cxfi
, sg
->fi
);
523 omap_dma_chan_write(c
, CEN
, sg
->en
);
524 omap_dma_chan_write(c
, CFN
, sg
->fn
);
526 omap_dma_start(c
, d
);
530 static void omap_dma_start_desc(struct omap_chan
*c
)
532 struct virt_dma_desc
*vd
= vchan_next_desc(&c
->vc
);
534 unsigned cxsa
, cxei
, cxfi
;
543 c
->desc
= d
= to_omap_dma_desc(&vd
->tx
);
547 * This provides the necessary barrier to ensure data held in
548 * DMA coherent memory is visible to the DMA engine prior to
549 * the transfer starting.
553 omap_dma_chan_write(c
, CCR
, d
->ccr
);
555 omap_dma_chan_write(c
, CCR2
, d
->ccr
>> 16);
557 if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
) {
567 omap_dma_chan_write(c
, cxsa
, d
->dev_addr
);
568 omap_dma_chan_write(c
, cxei
, d
->ei
);
569 omap_dma_chan_write(c
, cxfi
, d
->fi
);
570 omap_dma_chan_write(c
, CSDP
, d
->csdp
);
571 omap_dma_chan_write(c
, CLNK_CTRL
, d
->clnk_ctrl
);
573 omap_dma_start_sg(c
, d
);
576 static void omap_dma_callback(int ch
, u16 status
, void *data
)
578 struct omap_chan
*c
= data
;
582 spin_lock_irqsave(&c
->vc
.lock
, flags
);
586 vchan_cyclic_callback(&d
->vd
);
587 } else if (d
->using_ll
|| c
->sgidx
== d
->sglen
) {
588 omap_dma_start_desc(c
);
589 vchan_cookie_complete(&d
->vd
);
591 omap_dma_start_sg(c
, d
);
594 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
597 static irqreturn_t
omap_dma_irq(int irq
, void *devid
)
599 struct omap_dmadev
*od
= devid
;
600 unsigned status
, channel
;
602 spin_lock(&od
->irq_lock
);
604 status
= omap_dma_glbl_read(od
, IRQSTATUS_L1
);
605 status
&= od
->irq_enable_mask
;
607 spin_unlock(&od
->irq_lock
);
611 while ((channel
= ffs(status
)) != 0) {
619 c
= od
->lch_map
[channel
];
621 /* This should never happen */
622 dev_err(od
->ddev
.dev
, "invalid channel %u\n", channel
);
626 csr
= omap_dma_get_csr(c
);
627 omap_dma_glbl_write(od
, IRQSTATUS_L1
, mask
);
629 omap_dma_callback(channel
, csr
, c
);
632 spin_unlock(&od
->irq_lock
);
637 static int omap_dma_alloc_chan_resources(struct dma_chan
*chan
)
639 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
640 struct omap_chan
*c
= to_omap_dma_chan(chan
);
641 struct device
*dev
= od
->ddev
.dev
;
645 ret
= omap_request_dma(c
->dma_sig
, "DMA engine",
646 omap_dma_callback
, c
, &c
->dma_ch
);
648 ret
= omap_request_dma(c
->dma_sig
, "DMA engine", NULL
, NULL
,
652 dev_dbg(dev
, "allocating channel %u for %u\n", c
->dma_ch
, c
->dma_sig
);
655 omap_dma_assign(od
, c
, c
->dma_ch
);
660 spin_lock_irq(&od
->irq_lock
);
661 val
= BIT(c
->dma_ch
);
662 omap_dma_glbl_write(od
, IRQSTATUS_L1
, val
);
663 od
->irq_enable_mask
|= val
;
664 omap_dma_glbl_write(od
, IRQENABLE_L1
, od
->irq_enable_mask
);
666 val
= omap_dma_glbl_read(od
, IRQENABLE_L0
);
667 val
&= ~BIT(c
->dma_ch
);
668 omap_dma_glbl_write(od
, IRQENABLE_L0
, val
);
669 spin_unlock_irq(&od
->irq_lock
);
674 if (__dma_omap16xx(od
->plat
->dma_attr
)) {
675 c
->ccr
= CCR_OMAP31_DISABLE
;
676 /* Duplicate what plat-omap/dma.c does */
677 c
->ccr
|= c
->dma_ch
+ 1;
679 c
->ccr
= c
->dma_sig
& 0x1f;
682 c
->ccr
= c
->dma_sig
& 0x1f;
683 c
->ccr
|= (c
->dma_sig
& ~0x1f) << 14;
685 if (od
->plat
->errata
& DMA_ERRATA_IFRAME_BUFFERING
)
686 c
->ccr
|= CCR_BUFFERING_DISABLE
;
691 static void omap_dma_free_chan_resources(struct dma_chan
*chan
)
693 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
694 struct omap_chan
*c
= to_omap_dma_chan(chan
);
697 spin_lock_irq(&od
->irq_lock
);
698 od
->irq_enable_mask
&= ~BIT(c
->dma_ch
);
699 omap_dma_glbl_write(od
, IRQENABLE_L1
, od
->irq_enable_mask
);
700 spin_unlock_irq(&od
->irq_lock
);
703 c
->channel_base
= NULL
;
704 od
->lch_map
[c
->dma_ch
] = NULL
;
705 vchan_free_chan_resources(&c
->vc
);
706 omap_free_dma(c
->dma_ch
);
708 dev_dbg(od
->ddev
.dev
, "freeing channel %u used for %u\n", c
->dma_ch
,
713 static size_t omap_dma_sg_size(struct omap_sg
*sg
)
715 return sg
->en
* sg
->fn
;
718 static size_t omap_dma_desc_size(struct omap_desc
*d
)
723 for (size
= i
= 0; i
< d
->sglen
; i
++)
724 size
+= omap_dma_sg_size(&d
->sg
[i
]);
726 return size
* es_bytes
[d
->es
];
729 static size_t omap_dma_desc_size_pos(struct omap_desc
*d
, dma_addr_t addr
)
732 size_t size
, es_size
= es_bytes
[d
->es
];
734 for (size
= i
= 0; i
< d
->sglen
; i
++) {
735 size_t this_size
= omap_dma_sg_size(&d
->sg
[i
]) * es_size
;
739 else if (addr
>= d
->sg
[i
].addr
&&
740 addr
< d
->sg
[i
].addr
+ this_size
)
741 size
+= d
->sg
[i
].addr
+ this_size
- addr
;
747 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
748 * read before the DMA controller finished disabling the channel.
750 static uint32_t omap_dma_chan_read_3_3(struct omap_chan
*c
, unsigned reg
)
752 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
755 val
= omap_dma_chan_read(c
, reg
);
756 if (val
== 0 && od
->plat
->errata
& DMA_ERRATA_3_3
)
757 val
= omap_dma_chan_read(c
, reg
);
762 static dma_addr_t
omap_dma_get_src_pos(struct omap_chan
*c
)
764 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
765 dma_addr_t addr
, cdac
;
767 if (__dma_omap15xx(od
->plat
->dma_attr
)) {
768 addr
= omap_dma_chan_read(c
, CPC
);
770 addr
= omap_dma_chan_read_3_3(c
, CSAC
);
771 cdac
= omap_dma_chan_read_3_3(c
, CDAC
);
774 * CDAC == 0 indicates that the DMA transfer on the channel has
775 * not been started (no data has been transferred so far).
776 * Return the programmed source start address in this case.
779 addr
= omap_dma_chan_read(c
, CSSA
);
783 addr
|= omap_dma_chan_read(c
, CSSA
) & 0xffff0000;
788 static dma_addr_t
omap_dma_get_dst_pos(struct omap_chan
*c
)
790 struct omap_dmadev
*od
= to_omap_dma_dev(c
->vc
.chan
.device
);
793 if (__dma_omap15xx(od
->plat
->dma_attr
)) {
794 addr
= omap_dma_chan_read(c
, CPC
);
796 addr
= omap_dma_chan_read_3_3(c
, CDAC
);
799 * CDAC == 0 indicates that the DMA transfer on the channel
800 * has not been started (no data has been transferred so
801 * far). Return the programmed destination start address in
805 addr
= omap_dma_chan_read(c
, CDSA
);
809 addr
|= omap_dma_chan_read(c
, CDSA
) & 0xffff0000;
814 static enum dma_status
omap_dma_tx_status(struct dma_chan
*chan
,
815 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
817 struct omap_chan
*c
= to_omap_dma_chan(chan
);
818 struct virt_dma_desc
*vd
;
822 ret
= dma_cookie_status(chan
, cookie
, txstate
);
824 if (!c
->paused
&& c
->running
) {
825 uint32_t ccr
= omap_dma_chan_read(c
, CCR
);
827 * The channel is no longer active, set the return value
830 if (!(ccr
& CCR_ENABLE
))
834 if (ret
== DMA_COMPLETE
|| !txstate
)
837 spin_lock_irqsave(&c
->vc
.lock
, flags
);
838 vd
= vchan_find_desc(&c
->vc
, cookie
);
840 txstate
->residue
= omap_dma_desc_size(to_omap_dma_desc(&vd
->tx
));
841 } else if (c
->desc
&& c
->desc
->vd
.tx
.cookie
== cookie
) {
842 struct omap_desc
*d
= c
->desc
;
845 if (d
->dir
== DMA_MEM_TO_DEV
)
846 pos
= omap_dma_get_src_pos(c
);
847 else if (d
->dir
== DMA_DEV_TO_MEM
|| d
->dir
== DMA_MEM_TO_MEM
)
848 pos
= omap_dma_get_dst_pos(c
);
852 txstate
->residue
= omap_dma_desc_size_pos(d
, pos
);
854 txstate
->residue
= 0;
856 if (ret
== DMA_IN_PROGRESS
&& c
->paused
)
858 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
863 static void omap_dma_issue_pending(struct dma_chan
*chan
)
865 struct omap_chan
*c
= to_omap_dma_chan(chan
);
868 spin_lock_irqsave(&c
->vc
.lock
, flags
);
869 if (vchan_issue_pending(&c
->vc
) && !c
->desc
)
870 omap_dma_start_desc(c
);
871 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
874 static struct dma_async_tx_descriptor
*omap_dma_prep_slave_sg(
875 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned sglen
,
876 enum dma_transfer_direction dir
, unsigned long tx_flags
, void *context
)
878 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
879 struct omap_chan
*c
= to_omap_dma_chan(chan
);
880 enum dma_slave_buswidth dev_width
;
881 struct scatterlist
*sgent
;
884 unsigned i
, es
, en
, frame_bytes
;
885 bool ll_failed
= false;
887 u32 port_window
, port_window_bytes
;
889 if (dir
== DMA_DEV_TO_MEM
) {
890 dev_addr
= c
->cfg
.src_addr
;
891 dev_width
= c
->cfg
.src_addr_width
;
892 burst
= c
->cfg
.src_maxburst
;
893 port_window
= c
->cfg
.src_port_window_size
;
894 } else if (dir
== DMA_MEM_TO_DEV
) {
895 dev_addr
= c
->cfg
.dst_addr
;
896 dev_width
= c
->cfg
.dst_addr_width
;
897 burst
= c
->cfg
.dst_maxburst
;
898 port_window
= c
->cfg
.dst_port_window_size
;
900 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
904 /* Bus width translates to the element size (ES) */
906 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
907 es
= CSDP_DATA_TYPE_8
;
909 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
910 es
= CSDP_DATA_TYPE_16
;
912 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
913 es
= CSDP_DATA_TYPE_32
;
915 default: /* not reached */
919 /* Now allocate and setup the descriptor. */
920 d
= kzalloc(struct_size(d
, sg
, sglen
), GFP_ATOMIC
);
925 d
->dev_addr
= dev_addr
;
928 /* When the port_window is used, one frame must cover the window */
931 port_window_bytes
= port_window
* es_bytes
[es
];
935 * One frame covers the port_window and by configure
936 * the source frame index to be -1 * (port_window - 1)
937 * we instruct the sDMA that after a frame is processed
938 * it should move back to the start of the window.
940 d
->fi
= -(port_window_bytes
- 1);
943 d
->ccr
= c
->ccr
| CCR_SYNC_FRAME
;
944 if (dir
== DMA_DEV_TO_MEM
) {
945 d
->csdp
= CSDP_DST_BURST_64
| CSDP_DST_PACKED
;
947 d
->ccr
|= CCR_DST_AMODE_POSTINC
;
949 d
->ccr
|= CCR_SRC_AMODE_DBLIDX
;
951 if (port_window_bytes
>= 64)
952 d
->csdp
|= CSDP_SRC_BURST_64
;
953 else if (port_window_bytes
>= 32)
954 d
->csdp
|= CSDP_SRC_BURST_32
;
955 else if (port_window_bytes
>= 16)
956 d
->csdp
|= CSDP_SRC_BURST_16
;
959 d
->ccr
|= CCR_SRC_AMODE_CONSTANT
;
962 d
->csdp
= CSDP_SRC_BURST_64
| CSDP_SRC_PACKED
;
964 d
->ccr
|= CCR_SRC_AMODE_POSTINC
;
966 d
->ccr
|= CCR_DST_AMODE_DBLIDX
;
968 if (port_window_bytes
>= 64)
969 d
->csdp
|= CSDP_DST_BURST_64
;
970 else if (port_window_bytes
>= 32)
971 d
->csdp
|= CSDP_DST_BURST_32
;
972 else if (port_window_bytes
>= 16)
973 d
->csdp
|= CSDP_DST_BURST_16
;
975 d
->ccr
|= CCR_DST_AMODE_CONSTANT
;
979 d
->cicr
= CICR_DROP_IE
| CICR_BLOCK_IE
;
983 d
->cicr
|= CICR_TOUT_IE
;
985 if (dir
== DMA_DEV_TO_MEM
)
986 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_TIPB
;
988 d
->csdp
|= CSDP_DST_PORT_TIPB
| CSDP_SRC_PORT_EMIFF
;
990 if (dir
== DMA_DEV_TO_MEM
)
991 d
->ccr
|= CCR_TRIGGER_SRC
;
993 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
996 d
->csdp
|= CSDP_WRITE_LAST_NON_POSTED
;
998 if (od
->plat
->errata
& DMA_ERRATA_PARALLEL_CHANNELS
)
999 d
->clnk_ctrl
= c
->dma_ch
;
1002 * Build our scatterlist entries: each contains the address,
1003 * the number of elements (EN) in each frame, and the number of
1004 * frames (FN). Number of bytes for this entry = ES * EN * FN.
1006 * Burst size translates to number of elements with frame sync.
1007 * Note: DMA engine defines burst to be the number of dev-width
1011 frame_bytes
= es_bytes
[es
] * en
;
1014 d
->using_ll
= od
->ll123_supported
;
1016 for_each_sg(sgl
, sgent
, sglen
, i
) {
1017 struct omap_sg
*osg
= &d
->sg
[i
];
1019 osg
->addr
= sg_dma_address(sgent
);
1021 osg
->fn
= sg_dma_len(sgent
) / frame_bytes
;
1024 osg
->t2_desc
= dma_pool_alloc(od
->desc_pool
, GFP_ATOMIC
,
1025 &osg
->t2_desc_paddr
);
1026 if (!osg
->t2_desc
) {
1027 dev_err(chan
->device
->dev
,
1028 "t2_desc[%d] allocation failed\n", i
);
1030 d
->using_ll
= false;
1034 omap_dma_fill_type2_desc(d
, i
, dir
, (i
== sglen
- 1));
1040 /* Release the dma_pool entries if one allocation failed */
1042 for (i
= 0; i
< d
->sglen
; i
++) {
1043 struct omap_sg
*osg
= &d
->sg
[i
];
1046 dma_pool_free(od
->desc_pool
, osg
->t2_desc
,
1047 osg
->t2_desc_paddr
);
1048 osg
->t2_desc
= NULL
;
1053 return vchan_tx_prep(&c
->vc
, &d
->vd
, tx_flags
);
1056 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_cyclic(
1057 struct dma_chan
*chan
, dma_addr_t buf_addr
, size_t buf_len
,
1058 size_t period_len
, enum dma_transfer_direction dir
, unsigned long flags
)
1060 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
1061 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1062 enum dma_slave_buswidth dev_width
;
1063 struct omap_desc
*d
;
1064 dma_addr_t dev_addr
;
1068 if (dir
== DMA_DEV_TO_MEM
) {
1069 dev_addr
= c
->cfg
.src_addr
;
1070 dev_width
= c
->cfg
.src_addr_width
;
1071 burst
= c
->cfg
.src_maxburst
;
1072 } else if (dir
== DMA_MEM_TO_DEV
) {
1073 dev_addr
= c
->cfg
.dst_addr
;
1074 dev_width
= c
->cfg
.dst_addr_width
;
1075 burst
= c
->cfg
.dst_maxburst
;
1077 dev_err(chan
->device
->dev
, "%s: bad direction?\n", __func__
);
1081 /* Bus width translates to the element size (ES) */
1082 switch (dev_width
) {
1083 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
1084 es
= CSDP_DATA_TYPE_8
;
1086 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
1087 es
= CSDP_DATA_TYPE_16
;
1089 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
1090 es
= CSDP_DATA_TYPE_32
;
1092 default: /* not reached */
1096 /* Now allocate and setup the descriptor. */
1097 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
1102 d
->dev_addr
= dev_addr
;
1105 d
->sg
[0].addr
= buf_addr
;
1106 d
->sg
[0].en
= period_len
/ es_bytes
[es
];
1107 d
->sg
[0].fn
= buf_len
/ period_len
;
1111 if (dir
== DMA_DEV_TO_MEM
)
1112 d
->ccr
|= CCR_DST_AMODE_POSTINC
| CCR_SRC_AMODE_CONSTANT
;
1114 d
->ccr
|= CCR_DST_AMODE_CONSTANT
| CCR_SRC_AMODE_POSTINC
;
1116 d
->cicr
= CICR_DROP_IE
;
1117 if (flags
& DMA_PREP_INTERRUPT
)
1118 d
->cicr
|= CICR_FRAME_IE
;
1123 d
->cicr
|= CICR_TOUT_IE
;
1125 if (dir
== DMA_DEV_TO_MEM
)
1126 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_MPUI
;
1128 d
->csdp
|= CSDP_DST_PORT_MPUI
| CSDP_SRC_PORT_EMIFF
;
1131 d
->ccr
|= CCR_SYNC_PACKET
;
1133 d
->ccr
|= CCR_SYNC_ELEMENT
;
1135 if (dir
== DMA_DEV_TO_MEM
) {
1136 d
->ccr
|= CCR_TRIGGER_SRC
;
1137 d
->csdp
|= CSDP_DST_PACKED
;
1139 d
->csdp
|= CSDP_SRC_PACKED
;
1142 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
1144 d
->csdp
|= CSDP_DST_BURST_64
| CSDP_SRC_BURST_64
;
1147 if (__dma_omap15xx(od
->plat
->dma_attr
))
1148 d
->ccr
|= CCR_AUTO_INIT
| CCR_REPEAT
;
1150 d
->clnk_ctrl
= c
->dma_ch
| CLNK_CTRL_ENABLE_LNK
;
1154 return vchan_tx_prep(&c
->vc
, &d
->vd
, flags
);
1157 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_memcpy(
1158 struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
1159 size_t len
, unsigned long tx_flags
)
1161 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1162 struct omap_desc
*d
;
1165 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
1169 data_type
= __ffs((src
| dest
| len
));
1170 if (data_type
> CSDP_DATA_TYPE_32
)
1171 data_type
= CSDP_DATA_TYPE_32
;
1173 d
->dir
= DMA_MEM_TO_MEM
;
1177 d
->sg
[0].en
= len
/ BIT(data_type
);
1179 d
->sg
[0].addr
= dest
;
1182 d
->ccr
|= CCR_DST_AMODE_POSTINC
| CCR_SRC_AMODE_POSTINC
;
1184 d
->cicr
= CICR_DROP_IE
| CICR_FRAME_IE
;
1186 d
->csdp
= data_type
;
1189 d
->cicr
|= CICR_TOUT_IE
;
1190 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_EMIFF
;
1192 d
->csdp
|= CSDP_DST_PACKED
| CSDP_SRC_PACKED
;
1193 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
1194 d
->csdp
|= CSDP_DST_BURST_64
| CSDP_SRC_BURST_64
;
1197 return vchan_tx_prep(&c
->vc
, &d
->vd
, tx_flags
);
1200 static struct dma_async_tx_descriptor
*omap_dma_prep_dma_interleaved(
1201 struct dma_chan
*chan
, struct dma_interleaved_template
*xt
,
1202 unsigned long flags
)
1204 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1205 struct omap_desc
*d
;
1208 size_t src_icg
, dst_icg
;
1210 /* Slave mode is not supported */
1211 if (is_slave_direction(xt
->dir
))
1214 if (xt
->frame_size
!= 1 || xt
->numf
== 0)
1217 d
= kzalloc(sizeof(*d
) + sizeof(d
->sg
[0]), GFP_ATOMIC
);
1221 data_type
= __ffs((xt
->src_start
| xt
->dst_start
| xt
->sgl
[0].size
));
1222 if (data_type
> CSDP_DATA_TYPE_32
)
1223 data_type
= CSDP_DATA_TYPE_32
;
1226 d
->dir
= DMA_MEM_TO_MEM
;
1227 d
->dev_addr
= xt
->src_start
;
1229 sg
->en
= xt
->sgl
[0].size
/ BIT(data_type
);
1231 sg
->addr
= xt
->dst_start
;
1235 src_icg
= dmaengine_get_src_icg(xt
, &xt
->sgl
[0]);
1236 dst_icg
= dmaengine_get_dst_icg(xt
, &xt
->sgl
[0]);
1238 d
->ccr
|= CCR_SRC_AMODE_DBLIDX
;
1241 } else if (xt
->src_inc
) {
1242 d
->ccr
|= CCR_SRC_AMODE_POSTINC
;
1245 dev_err(chan
->device
->dev
,
1246 "%s: SRC constant addressing is not supported\n",
1253 d
->ccr
|= CCR_DST_AMODE_DBLIDX
;
1256 } else if (xt
->dst_inc
) {
1257 d
->ccr
|= CCR_DST_AMODE_POSTINC
;
1260 dev_err(chan
->device
->dev
,
1261 "%s: DST constant addressing is not supported\n",
1267 d
->cicr
= CICR_DROP_IE
| CICR_FRAME_IE
;
1269 d
->csdp
= data_type
;
1272 d
->cicr
|= CICR_TOUT_IE
;
1273 d
->csdp
|= CSDP_DST_PORT_EMIFF
| CSDP_SRC_PORT_EMIFF
;
1275 d
->csdp
|= CSDP_DST_PACKED
| CSDP_SRC_PACKED
;
1276 d
->cicr
|= CICR_MISALIGNED_ERR_IE
| CICR_TRANS_ERR_IE
;
1277 d
->csdp
|= CSDP_DST_BURST_64
| CSDP_SRC_BURST_64
;
1280 return vchan_tx_prep(&c
->vc
, &d
->vd
, flags
);
1283 static int omap_dma_slave_config(struct dma_chan
*chan
, struct dma_slave_config
*cfg
)
1285 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1287 if (cfg
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
1288 cfg
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
1291 if (cfg
->src_maxburst
> chan
->device
->max_burst
||
1292 cfg
->dst_maxburst
> chan
->device
->max_burst
)
1295 memcpy(&c
->cfg
, cfg
, sizeof(c
->cfg
));
1300 static int omap_dma_terminate_all(struct dma_chan
*chan
)
1302 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1303 unsigned long flags
;
1306 spin_lock_irqsave(&c
->vc
.lock
, flags
);
1309 * Stop DMA activity: we assume the callback will not be called
1310 * after omap_dma_stop() returns (even if it does, it will see
1311 * c->desc is NULL and exit.)
1314 vchan_terminate_vdesc(&c
->desc
->vd
);
1316 /* Avoid stopping the dma twice */
1324 vchan_get_all_descriptors(&c
->vc
, &head
);
1325 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
1326 vchan_dma_desc_free_list(&c
->vc
, &head
);
1331 static void omap_dma_synchronize(struct dma_chan
*chan
)
1333 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1335 vchan_synchronize(&c
->vc
);
1338 static int omap_dma_pause(struct dma_chan
*chan
)
1340 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1341 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
1342 unsigned long flags
;
1344 bool can_pause
= false;
1346 spin_lock_irqsave(&od
->irq_lock
, flags
);
1355 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1356 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1357 * "When a channel is disabled during a transfer, the channel undergoes
1358 * an abort, unless it is hardware-source-synchronized …".
1359 * A source-synchronised channel is one where the fetching of data is
1360 * under control of the device. In other words, a device-to-memory
1361 * transfer. So, a destination-synchronised channel (which would be a
1362 * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1364 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1365 * aborts immediately after completion of current read/write
1366 * transactions and then the FIFO is cleaned up." The term "cleaned up"
1367 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1368 * are both clear _before_ disabling the channel, otherwise data loss
1370 * The problem is that if the channel is active, then device activity
1371 * can result in DMA activity starting between reading those as both
1372 * clear and the write to DMA_CCR to clear the enable bit hitting the
1373 * hardware. If the DMA hardware can't drain the data in its FIFO to the
1374 * destination, then data loss "might" occur (say if we write to an UART
1375 * and the UART is not accepting any further data).
1377 else if (c
->desc
->dir
== DMA_DEV_TO_MEM
)
1380 if (can_pause
&& !c
->paused
) {
1381 ret
= omap_dma_stop(c
);
1386 spin_unlock_irqrestore(&od
->irq_lock
, flags
);
1391 static int omap_dma_resume(struct dma_chan
*chan
)
1393 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1394 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
1395 unsigned long flags
;
1398 spin_lock_irqsave(&od
->irq_lock
, flags
);
1400 if (c
->paused
&& c
->desc
) {
1403 /* Restore channel link register */
1404 omap_dma_chan_write(c
, CLNK_CTRL
, c
->desc
->clnk_ctrl
);
1406 omap_dma_start(c
, c
->desc
);
1410 spin_unlock_irqrestore(&od
->irq_lock
, flags
);
1415 static int omap_dma_chan_init(struct omap_dmadev
*od
)
1417 struct omap_chan
*c
;
1419 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
1423 c
->reg_map
= od
->reg_map
;
1424 c
->vc
.desc_free
= omap_dma_desc_free
;
1425 vchan_init(&c
->vc
, &od
->ddev
);
1430 static void omap_dma_free(struct omap_dmadev
*od
)
1432 while (!list_empty(&od
->ddev
.channels
)) {
1433 struct omap_chan
*c
= list_first_entry(&od
->ddev
.channels
,
1434 struct omap_chan
, vc
.chan
.device_node
);
1436 list_del(&c
->vc
.chan
.device_node
);
1437 tasklet_kill(&c
->vc
.task
);
1442 #define OMAP_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1443 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1444 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1446 static int omap_dma_probe(struct platform_device
*pdev
)
1448 struct omap_dmadev
*od
;
1449 struct resource
*res
;
1453 od
= devm_kzalloc(&pdev
->dev
, sizeof(*od
), GFP_KERNEL
);
1457 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1458 od
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1459 if (IS_ERR(od
->base
))
1460 return PTR_ERR(od
->base
);
1462 od
->plat
= omap_get_plat_info();
1464 return -EPROBE_DEFER
;
1466 od
->reg_map
= od
->plat
->reg_map
;
1468 dma_cap_set(DMA_SLAVE
, od
->ddev
.cap_mask
);
1469 dma_cap_set(DMA_CYCLIC
, od
->ddev
.cap_mask
);
1470 dma_cap_set(DMA_MEMCPY
, od
->ddev
.cap_mask
);
1471 dma_cap_set(DMA_INTERLEAVE
, od
->ddev
.cap_mask
);
1472 od
->ddev
.device_alloc_chan_resources
= omap_dma_alloc_chan_resources
;
1473 od
->ddev
.device_free_chan_resources
= omap_dma_free_chan_resources
;
1474 od
->ddev
.device_tx_status
= omap_dma_tx_status
;
1475 od
->ddev
.device_issue_pending
= omap_dma_issue_pending
;
1476 od
->ddev
.device_prep_slave_sg
= omap_dma_prep_slave_sg
;
1477 od
->ddev
.device_prep_dma_cyclic
= omap_dma_prep_dma_cyclic
;
1478 od
->ddev
.device_prep_dma_memcpy
= omap_dma_prep_dma_memcpy
;
1479 od
->ddev
.device_prep_interleaved_dma
= omap_dma_prep_dma_interleaved
;
1480 od
->ddev
.device_config
= omap_dma_slave_config
;
1481 od
->ddev
.device_pause
= omap_dma_pause
;
1482 od
->ddev
.device_resume
= omap_dma_resume
;
1483 od
->ddev
.device_terminate_all
= omap_dma_terminate_all
;
1484 od
->ddev
.device_synchronize
= omap_dma_synchronize
;
1485 od
->ddev
.src_addr_widths
= OMAP_DMA_BUSWIDTHS
;
1486 od
->ddev
.dst_addr_widths
= OMAP_DMA_BUSWIDTHS
;
1487 od
->ddev
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
1488 od
->ddev
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
1489 od
->ddev
.max_burst
= SZ_16M
- 1; /* CCEN: 24bit unsigned */
1490 od
->ddev
.dev
= &pdev
->dev
;
1491 INIT_LIST_HEAD(&od
->ddev
.channels
);
1492 spin_lock_init(&od
->lock
);
1493 spin_lock_init(&od
->irq_lock
);
1495 /* Number of DMA requests */
1496 od
->dma_requests
= OMAP_SDMA_REQUESTS
;
1497 if (pdev
->dev
.of_node
&& of_property_read_u32(pdev
->dev
.of_node
,
1499 &od
->dma_requests
)) {
1500 dev_info(&pdev
->dev
,
1501 "Missing dma-requests property, using %u.\n",
1502 OMAP_SDMA_REQUESTS
);
1505 /* Number of available logical channels */
1506 if (!pdev
->dev
.of_node
) {
1507 lch_count
= od
->plat
->dma_attr
->lch_count
;
1508 if (unlikely(!lch_count
))
1509 lch_count
= OMAP_SDMA_CHANNELS
;
1510 } else if (of_property_read_u32(pdev
->dev
.of_node
, "dma-channels",
1512 dev_info(&pdev
->dev
,
1513 "Missing dma-channels property, using %u.\n",
1514 OMAP_SDMA_CHANNELS
);
1515 lch_count
= OMAP_SDMA_CHANNELS
;
1518 od
->lch_map
= devm_kcalloc(&pdev
->dev
, lch_count
, sizeof(*od
->lch_map
),
1523 for (i
= 0; i
< od
->dma_requests
; i
++) {
1524 rc
= omap_dma_chan_init(od
);
1531 irq
= platform_get_irq(pdev
, 1);
1533 dev_info(&pdev
->dev
, "failed to get L1 IRQ: %d\n", irq
);
1536 /* Disable all interrupts */
1537 od
->irq_enable_mask
= 0;
1538 omap_dma_glbl_write(od
, IRQENABLE_L1
, 0);
1540 rc
= devm_request_irq(&pdev
->dev
, irq
, omap_dma_irq
,
1541 IRQF_SHARED
, "omap-dma-engine", od
);
1546 if (omap_dma_glbl_read(od
, CAPS_0
) & CAPS_0_SUPPORT_LL123
)
1547 od
->ll123_supported
= true;
1549 od
->ddev
.filter
.map
= od
->plat
->slave_map
;
1550 od
->ddev
.filter
.mapcnt
= od
->plat
->slavecnt
;
1551 od
->ddev
.filter
.fn
= omap_dma_filter_fn
;
1553 if (od
->ll123_supported
) {
1554 od
->desc_pool
= dma_pool_create(dev_name(&pdev
->dev
),
1556 sizeof(struct omap_type2_desc
),
1558 if (!od
->desc_pool
) {
1560 "unable to allocate descriptor pool\n");
1561 od
->ll123_supported
= false;
1565 rc
= dma_async_device_register(&od
->ddev
);
1567 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1573 platform_set_drvdata(pdev
, od
);
1575 if (pdev
->dev
.of_node
) {
1576 omap_dma_info
.dma_cap
= od
->ddev
.cap_mask
;
1578 /* Device-tree DMA controller registration */
1579 rc
= of_dma_controller_register(pdev
->dev
.of_node
,
1580 of_dma_simple_xlate
, &omap_dma_info
);
1582 pr_warn("OMAP-DMA: failed to register DMA controller\n");
1583 dma_async_device_unregister(&od
->ddev
);
1588 dev_info(&pdev
->dev
, "OMAP DMA engine driver%s\n",
1589 od
->ll123_supported
? " (LinkedList1/2/3 supported)" : "");
1594 static int omap_dma_remove(struct platform_device
*pdev
)
1596 struct omap_dmadev
*od
= platform_get_drvdata(pdev
);
1599 if (pdev
->dev
.of_node
)
1600 of_dma_controller_free(pdev
->dev
.of_node
);
1602 irq
= platform_get_irq(pdev
, 1);
1603 devm_free_irq(&pdev
->dev
, irq
, od
);
1605 dma_async_device_unregister(&od
->ddev
);
1608 /* Disable all interrupts */
1609 omap_dma_glbl_write(od
, IRQENABLE_L0
, 0);
1612 if (od
->ll123_supported
)
1613 dma_pool_destroy(od
->desc_pool
);
1620 static const struct of_device_id omap_dma_match
[] = {
1621 { .compatible
= "ti,omap2420-sdma", },
1622 { .compatible
= "ti,omap2430-sdma", },
1623 { .compatible
= "ti,omap3430-sdma", },
1624 { .compatible
= "ti,omap3630-sdma", },
1625 { .compatible
= "ti,omap4430-sdma", },
1628 MODULE_DEVICE_TABLE(of
, omap_dma_match
);
1630 static struct platform_driver omap_dma_driver
= {
1631 .probe
= omap_dma_probe
,
1632 .remove
= omap_dma_remove
,
1634 .name
= "omap-dma-engine",
1635 .of_match_table
= of_match_ptr(omap_dma_match
),
1639 bool omap_dma_filter_fn(struct dma_chan
*chan
, void *param
)
1641 if (chan
->device
->dev
->driver
== &omap_dma_driver
.driver
) {
1642 struct omap_dmadev
*od
= to_omap_dma_dev(chan
->device
);
1643 struct omap_chan
*c
= to_omap_dma_chan(chan
);
1644 unsigned req
= *(unsigned *)param
;
1646 if (req
<= od
->dma_requests
) {
1653 EXPORT_SYMBOL_GPL(omap_dma_filter_fn
);
1655 static int omap_dma_init(void)
1657 return platform_driver_register(&omap_dma_driver
);
1659 subsys_initcall(omap_dma_init
);
1661 static void __exit
omap_dma_exit(void)
1663 platform_driver_unregister(&omap_dma_driver
);
1665 module_exit(omap_dma_exit
);
1667 MODULE_AUTHOR("Russell King");
1668 MODULE_LICENSE("GPL");