2 * Copyright 2015 Linaro.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/sched.h>
9 #include <linux/device.h>
10 #include <linux/dmaengine.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmapool.h>
13 #include <linux/init.h>
14 #include <linux/interrupt.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/platform_device.h>
18 #include <linux/slab.h>
19 #include <linux/spinlock.h>
20 #include <linux/of_device.h>
22 #include <linux/clk.h>
23 #include <linux/of_dma.h>
27 #define DRIVER_NAME "zx-dma"
29 #define DMA_MAX_SIZE (0x10000 - PAGE_SIZE)
30 #define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
32 #define REG_ZX_SRC_ADDR 0x00
33 #define REG_ZX_DST_ADDR 0x04
34 #define REG_ZX_TX_X_COUNT 0x08
35 #define REG_ZX_TX_ZY_COUNT 0x0c
36 #define REG_ZX_SRC_ZY_STEP 0x10
37 #define REG_ZX_DST_ZY_STEP 0x14
38 #define REG_ZX_LLI_ADDR 0x1c
39 #define REG_ZX_CTRL 0x20
40 #define REG_ZX_TC_IRQ 0x800
41 #define REG_ZX_SRC_ERR_IRQ 0x804
42 #define REG_ZX_DST_ERR_IRQ 0x808
43 #define REG_ZX_CFG_ERR_IRQ 0x80c
44 #define REG_ZX_TC_IRQ_RAW 0x810
45 #define REG_ZX_SRC_ERR_IRQ_RAW 0x814
46 #define REG_ZX_DST_ERR_IRQ_RAW 0x818
47 #define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
48 #define REG_ZX_STATUS 0x820
49 #define REG_ZX_DMA_GRP_PRIO 0x824
50 #define REG_ZX_DMA_ARB 0x828
52 #define ZX_FORCE_CLOSE BIT(31)
53 #define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
54 #define ZX_MAX_BURST_LEN 16
55 #define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
56 #define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
57 #define ZX_IRQ_ENABLE_ALL (3 << 4)
58 #define ZX_DST_FIFO_MODE BIT(3)
59 #define ZX_SRC_FIFO_MODE BIT(2)
60 #define ZX_SOFT_REQ BIT(1)
61 #define ZX_CH_ENABLE BIT(0)
63 #define ZX_DMA_BUSWIDTHS \
64 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
65 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
66 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
67 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
68 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
70 enum zx_dma_burst_width
{
71 ZX_DMA_WIDTH_8BIT
= 0,
72 ZX_DMA_WIDTH_16BIT
= 1,
73 ZX_DMA_WIDTH_32BIT
= 2,
74 ZX_DMA_WIDTH_64BIT
= 3,
87 u32 reserved
[7]; /* pack as hardware registers region size */
90 struct zx_dma_desc_sw
{
91 struct virt_dma_desc vd
;
92 dma_addr_t desc_hw_lli
;
95 struct zx_desc_hw
*desc_hw
;
101 struct dma_slave_config slave_cfg
;
102 int id
; /* Request phy chan id */
104 struct virt_dma_chan vc
;
105 struct zx_dma_phy
*phy
;
106 struct list_head node
;
108 enum dma_status status
;
114 struct zx_dma_chan
*vchan
;
115 struct zx_dma_desc_sw
*ds_run
;
116 struct zx_dma_desc_sw
*ds_done
;
120 struct dma_device slave
;
122 spinlock_t lock
; /* lock for ch and phy */
123 struct list_head chan_pending
;
124 struct zx_dma_phy
*phy
;
125 struct zx_dma_chan
*chans
;
127 struct dma_pool
*pool
;
133 #define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
135 static struct zx_dma_chan
*to_zx_chan(struct dma_chan
*chan
)
137 return container_of(chan
, struct zx_dma_chan
, vc
.chan
);
140 static void zx_dma_terminate_chan(struct zx_dma_phy
*phy
, struct zx_dma_dev
*d
)
144 val
= readl_relaxed(phy
->base
+ REG_ZX_CTRL
);
145 val
&= ~ZX_CH_ENABLE
;
146 writel_relaxed(val
, phy
->base
+ REG_ZX_CTRL
);
148 val
= 0x1 << phy
->idx
;
149 writel_relaxed(val
, d
->base
+ REG_ZX_TC_IRQ_RAW
);
150 writel_relaxed(val
, d
->base
+ REG_ZX_SRC_ERR_IRQ_RAW
);
151 writel_relaxed(val
, d
->base
+ REG_ZX_DST_ERR_IRQ_RAW
);
152 writel_relaxed(val
, d
->base
+ REG_ZX_CFG_ERR_IRQ_RAW
);
155 static void zx_dma_set_desc(struct zx_dma_phy
*phy
, struct zx_desc_hw
*hw
)
157 writel_relaxed(hw
->saddr
, phy
->base
+ REG_ZX_SRC_ADDR
);
158 writel_relaxed(hw
->daddr
, phy
->base
+ REG_ZX_DST_ADDR
);
159 writel_relaxed(hw
->src_x
, phy
->base
+ REG_ZX_TX_X_COUNT
);
160 writel_relaxed(0, phy
->base
+ REG_ZX_TX_ZY_COUNT
);
161 writel_relaxed(0, phy
->base
+ REG_ZX_SRC_ZY_STEP
);
162 writel_relaxed(0, phy
->base
+ REG_ZX_DST_ZY_STEP
);
163 writel_relaxed(hw
->lli
, phy
->base
+ REG_ZX_LLI_ADDR
);
164 writel_relaxed(hw
->ctr
, phy
->base
+ REG_ZX_CTRL
);
167 static u32
zx_dma_get_curr_lli(struct zx_dma_phy
*phy
)
169 return readl_relaxed(phy
->base
+ REG_ZX_LLI_ADDR
);
172 static u32
zx_dma_get_chan_stat(struct zx_dma_dev
*d
)
174 return readl_relaxed(d
->base
+ REG_ZX_STATUS
);
177 static void zx_dma_init_state(struct zx_dma_dev
*d
)
179 /* set same priority */
180 writel_relaxed(0x0, d
->base
+ REG_ZX_DMA_ARB
);
182 writel_relaxed(0xffffffff, d
->base
+ REG_ZX_TC_IRQ_RAW
);
183 writel_relaxed(0xffffffff, d
->base
+ REG_ZX_SRC_ERR_IRQ_RAW
);
184 writel_relaxed(0xffffffff, d
->base
+ REG_ZX_DST_ERR_IRQ_RAW
);
185 writel_relaxed(0xffffffff, d
->base
+ REG_ZX_CFG_ERR_IRQ_RAW
);
188 static int zx_dma_start_txd(struct zx_dma_chan
*c
)
190 struct zx_dma_dev
*d
= to_zx_dma(c
->vc
.chan
.device
);
191 struct virt_dma_desc
*vd
= vchan_next_desc(&c
->vc
);
196 if (BIT(c
->phy
->idx
) & zx_dma_get_chan_stat(d
))
200 struct zx_dma_desc_sw
*ds
=
201 container_of(vd
, struct zx_dma_desc_sw
, vd
);
203 * fetch and remove request from vc->desc_issued
204 * so vc->desc_issued only contains desc pending
206 list_del(&ds
->vd
.node
);
208 c
->phy
->ds_done
= NULL
;
210 zx_dma_set_desc(c
->phy
, ds
->desc_hw
);
213 c
->phy
->ds_done
= NULL
;
214 c
->phy
->ds_run
= NULL
;
218 static void zx_dma_task(struct zx_dma_dev
*d
)
220 struct zx_dma_phy
*p
;
221 struct zx_dma_chan
*c
, *cn
;
222 unsigned pch
, pch_alloc
= 0;
225 /* check new dma request of running channel in vc->desc_issued */
226 list_for_each_entry_safe(c
, cn
, &d
->slave
.channels
,
227 vc
.chan
.device_node
) {
228 spin_lock_irqsave(&c
->vc
.lock
, flags
);
230 if (p
&& p
->ds_done
&& zx_dma_start_txd(c
)) {
231 /* No current txd associated with this channel */
232 dev_dbg(d
->slave
.dev
, "pchan %u: free\n", p
->idx
);
233 /* Mark this channel free */
237 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
240 /* check new channel request in d->chan_pending */
241 spin_lock_irqsave(&d
->lock
, flags
);
242 while (!list_empty(&d
->chan_pending
)) {
243 c
= list_first_entry(&d
->chan_pending
,
244 struct zx_dma_chan
, node
);
247 /* remove from d->chan_pending */
248 list_del_init(&c
->node
);
249 pch_alloc
|= 1 << c
->id
;
250 /* Mark this channel allocated */
254 dev_dbg(d
->slave
.dev
, "pchan %u: busy!\n", c
->id
);
257 spin_unlock_irqrestore(&d
->lock
, flags
);
259 for (pch
= 0; pch
< d
->dma_channels
; pch
++) {
260 if (pch_alloc
& (1 << pch
)) {
264 spin_lock_irqsave(&c
->vc
.lock
, flags
);
266 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
272 static irqreturn_t
zx_dma_int_handler(int irq
, void *dev_id
)
274 struct zx_dma_dev
*d
= (struct zx_dma_dev
*)dev_id
;
275 struct zx_dma_phy
*p
;
276 struct zx_dma_chan
*c
;
277 u32 tc
= readl_relaxed(d
->base
+ REG_ZX_TC_IRQ
);
278 u32 serr
= readl_relaxed(d
->base
+ REG_ZX_SRC_ERR_IRQ
);
279 u32 derr
= readl_relaxed(d
->base
+ REG_ZX_DST_ERR_IRQ
);
280 u32 cfg
= readl_relaxed(d
->base
+ REG_ZX_CFG_ERR_IRQ
);
291 spin_lock_irqsave(&c
->vc
.lock
, flags
);
292 vchan_cookie_complete(&p
->ds_run
->vd
);
293 p
->ds_done
= p
->ds_run
;
294 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
299 if (serr
|| derr
|| cfg
)
300 dev_warn(d
->slave
.dev
, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n",
303 writel_relaxed(irq_chan
, d
->base
+ REG_ZX_TC_IRQ_RAW
);
304 writel_relaxed(serr
, d
->base
+ REG_ZX_SRC_ERR_IRQ_RAW
);
305 writel_relaxed(derr
, d
->base
+ REG_ZX_DST_ERR_IRQ_RAW
);
306 writel_relaxed(cfg
, d
->base
+ REG_ZX_CFG_ERR_IRQ_RAW
);
316 static void zx_dma_free_chan_resources(struct dma_chan
*chan
)
318 struct zx_dma_chan
*c
= to_zx_chan(chan
);
319 struct zx_dma_dev
*d
= to_zx_dma(chan
->device
);
322 spin_lock_irqsave(&d
->lock
, flags
);
323 list_del_init(&c
->node
);
324 spin_unlock_irqrestore(&d
->lock
, flags
);
326 vchan_free_chan_resources(&c
->vc
);
330 static enum dma_status
zx_dma_tx_status(struct dma_chan
*chan
,
332 struct dma_tx_state
*state
)
334 struct zx_dma_chan
*c
= to_zx_chan(chan
);
335 struct zx_dma_phy
*p
;
336 struct virt_dma_desc
*vd
;
341 ret
= dma_cookie_status(&c
->vc
.chan
, cookie
, state
);
342 if (ret
== DMA_COMPLETE
|| !state
)
345 spin_lock_irqsave(&c
->vc
.lock
, flags
);
350 * If the cookie is on our issue queue, then the residue is
353 vd
= vchan_find_desc(&c
->vc
, cookie
);
355 bytes
= container_of(vd
, struct zx_dma_desc_sw
, vd
)->size
;
356 } else if ((!p
) || (!p
->ds_run
)) {
359 struct zx_dma_desc_sw
*ds
= p
->ds_run
;
360 u32 clli
= 0, index
= 0;
363 clli
= zx_dma_get_curr_lli(p
);
364 index
= (clli
- ds
->desc_hw_lli
) / sizeof(struct zx_desc_hw
);
365 for (; index
< ds
->desc_num
; index
++) {
366 bytes
+= ds
->desc_hw
[index
].src_x
;
368 if (!ds
->desc_hw
[index
].lli
)
372 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
373 dma_set_residue(state
, bytes
);
377 static void zx_dma_issue_pending(struct dma_chan
*chan
)
379 struct zx_dma_chan
*c
= to_zx_chan(chan
);
380 struct zx_dma_dev
*d
= to_zx_dma(chan
->device
);
384 spin_lock_irqsave(&c
->vc
.lock
, flags
);
385 /* add request to vc->desc_issued */
386 if (vchan_issue_pending(&c
->vc
)) {
388 if (!c
->phy
&& list_empty(&c
->node
)) {
389 /* if new channel, add chan_pending */
390 list_add_tail(&c
->node
, &d
->chan_pending
);
392 dev_dbg(d
->slave
.dev
, "vchan %p: issued\n", &c
->vc
);
394 spin_unlock(&d
->lock
);
396 dev_dbg(d
->slave
.dev
, "vchan %p: nothing to issue\n", &c
->vc
);
398 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
404 static void zx_dma_fill_desc(struct zx_dma_desc_sw
*ds
, dma_addr_t dst
,
405 dma_addr_t src
, size_t len
, u32 num
, u32 ccfg
)
407 if ((num
+ 1) < ds
->desc_num
)
408 ds
->desc_hw
[num
].lli
= ds
->desc_hw_lli
+ (num
+ 1) *
409 sizeof(struct zx_desc_hw
);
410 ds
->desc_hw
[num
].saddr
= src
;
411 ds
->desc_hw
[num
].daddr
= dst
;
412 ds
->desc_hw
[num
].src_x
= len
;
413 ds
->desc_hw
[num
].ctr
= ccfg
;
416 static struct zx_dma_desc_sw
*zx_alloc_desc_resource(int num
,
417 struct dma_chan
*chan
)
419 struct zx_dma_chan
*c
= to_zx_chan(chan
);
420 struct zx_dma_desc_sw
*ds
;
421 struct zx_dma_dev
*d
= to_zx_dma(chan
->device
);
422 int lli_limit
= LLI_BLOCK_SIZE
/ sizeof(struct zx_desc_hw
);
424 if (num
> lli_limit
) {
425 dev_dbg(chan
->device
->dev
, "vch %p: sg num %d exceed max %d\n",
426 &c
->vc
, num
, lli_limit
);
430 ds
= kzalloc(sizeof(*ds
), GFP_ATOMIC
);
434 ds
->desc_hw
= dma_pool_alloc(d
->pool
, GFP_NOWAIT
, &ds
->desc_hw_lli
);
436 dev_dbg(chan
->device
->dev
, "vch %p: dma alloc fail\n", &c
->vc
);
440 memset(ds
->desc_hw
, sizeof(struct zx_desc_hw
) * num
, 0);
445 static enum zx_dma_burst_width
zx_dma_burst_width(enum dma_slave_buswidth width
)
448 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
449 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
450 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
451 case DMA_SLAVE_BUSWIDTH_8_BYTES
:
452 return ffs(width
) - 1;
454 return ZX_DMA_WIDTH_32BIT
;
458 static int zx_pre_config(struct zx_dma_chan
*c
, enum dma_transfer_direction dir
)
460 struct dma_slave_config
*cfg
= &c
->slave_cfg
;
461 enum zx_dma_burst_width src_width
;
462 enum zx_dma_burst_width dst_width
;
467 c
->ccfg
= ZX_CH_ENABLE
| ZX_SOFT_REQ
468 | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN
- 1)
469 | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT
)
470 | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT
);
473 c
->dev_addr
= cfg
->dst_addr
;
474 /* dst len is calculated from src width, len and dst width.
475 * We need make sure dst len not exceed MAX LEN.
477 dst_width
= zx_dma_burst_width(cfg
->dst_addr_width
);
478 maxburst
= cfg
->dst_maxburst
* cfg
->dst_addr_width
479 / DMA_SLAVE_BUSWIDTH_8_BYTES
;
480 maxburst
= maxburst
< ZX_MAX_BURST_LEN
?
481 maxburst
: ZX_MAX_BURST_LEN
;
482 c
->ccfg
= ZX_DST_FIFO_MODE
| ZX_CH_ENABLE
483 | ZX_SRC_BURST_LEN(maxburst
- 1)
484 | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_64BIT
)
485 | ZX_DST_BURST_WIDTH(dst_width
);
488 c
->dev_addr
= cfg
->src_addr
;
489 src_width
= zx_dma_burst_width(cfg
->src_addr_width
);
490 maxburst
= cfg
->src_maxburst
;
491 maxburst
= maxburst
< ZX_MAX_BURST_LEN
?
492 maxburst
: ZX_MAX_BURST_LEN
;
493 c
->ccfg
= ZX_SRC_FIFO_MODE
| ZX_CH_ENABLE
494 | ZX_SRC_BURST_LEN(maxburst
- 1)
495 | ZX_SRC_BURST_WIDTH(src_width
)
496 | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_64BIT
);
504 static struct dma_async_tx_descriptor
*zx_dma_prep_memcpy(
505 struct dma_chan
*chan
, dma_addr_t dst
, dma_addr_t src
,
506 size_t len
, unsigned long flags
)
508 struct zx_dma_chan
*c
= to_zx_chan(chan
);
509 struct zx_dma_desc_sw
*ds
;
516 if (zx_pre_config(c
, DMA_MEM_TO_MEM
))
519 num
= DIV_ROUND_UP(len
, DMA_MAX_SIZE
);
521 ds
= zx_alloc_desc_resource(num
, chan
);
529 copy
= min_t(size_t, len
, DMA_MAX_SIZE
);
530 zx_dma_fill_desc(ds
, dst
, src
, copy
, num
++, c
->ccfg
);
537 ds
->desc_hw
[num
- 1].lli
= 0; /* end of link */
538 ds
->desc_hw
[num
- 1].ctr
|= ZX_IRQ_ENABLE_ALL
;
539 return vchan_tx_prep(&c
->vc
, &ds
->vd
, flags
);
542 static struct dma_async_tx_descriptor
*zx_dma_prep_slave_sg(
543 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned int sglen
,
544 enum dma_transfer_direction dir
, unsigned long flags
, void *context
)
546 struct zx_dma_chan
*c
= to_zx_chan(chan
);
547 struct zx_dma_desc_sw
*ds
;
548 size_t len
, avail
, total
= 0;
549 struct scatterlist
*sg
;
550 dma_addr_t addr
, src
= 0, dst
= 0;
556 if (zx_pre_config(c
, dir
))
559 for_each_sg(sgl
, sg
, sglen
, i
) {
560 avail
= sg_dma_len(sg
);
561 if (avail
> DMA_MAX_SIZE
)
562 num
+= DIV_ROUND_UP(avail
, DMA_MAX_SIZE
) - 1;
565 ds
= zx_alloc_desc_resource(num
, chan
);
570 for_each_sg(sgl
, sg
, sglen
, i
) {
571 addr
= sg_dma_address(sg
);
572 avail
= sg_dma_len(sg
);
576 len
= min_t(size_t, avail
, DMA_MAX_SIZE
);
578 if (dir
== DMA_MEM_TO_DEV
) {
581 } else if (dir
== DMA_DEV_TO_MEM
) {
586 zx_dma_fill_desc(ds
, dst
, src
, len
, num
++, c
->ccfg
);
593 ds
->desc_hw
[num
- 1].lli
= 0; /* end of link */
594 ds
->desc_hw
[num
- 1].ctr
|= ZX_IRQ_ENABLE_ALL
;
596 return vchan_tx_prep(&c
->vc
, &ds
->vd
, flags
);
599 static int zx_dma_config(struct dma_chan
*chan
,
600 struct dma_slave_config
*cfg
)
602 struct zx_dma_chan
*c
= to_zx_chan(chan
);
607 memcpy(&c
->slave_cfg
, cfg
, sizeof(*cfg
));
612 static int zx_dma_terminate_all(struct dma_chan
*chan
)
614 struct zx_dma_chan
*c
= to_zx_chan(chan
);
615 struct zx_dma_dev
*d
= to_zx_dma(chan
->device
);
616 struct zx_dma_phy
*p
= c
->phy
;
620 dev_dbg(d
->slave
.dev
, "vchan %p: terminate all\n", &c
->vc
);
622 /* Prevent this channel being scheduled */
624 list_del_init(&c
->node
);
625 spin_unlock(&d
->lock
);
627 /* Clear the tx descriptor lists */
628 spin_lock_irqsave(&c
->vc
.lock
, flags
);
629 vchan_get_all_descriptors(&c
->vc
, &head
);
631 /* vchan is assigned to a pchan - stop the channel */
632 zx_dma_terminate_chan(p
, d
);
638 spin_unlock_irqrestore(&c
->vc
.lock
, flags
);
639 vchan_dma_desc_free_list(&c
->vc
, &head
);
644 static void zx_dma_free_desc(struct virt_dma_desc
*vd
)
646 struct zx_dma_desc_sw
*ds
=
647 container_of(vd
, struct zx_dma_desc_sw
, vd
);
648 struct zx_dma_dev
*d
= to_zx_dma(vd
->tx
.chan
->device
);
650 dma_pool_free(d
->pool
, ds
->desc_hw
, ds
->desc_hw_lli
);
654 static const struct of_device_id zx6702_dma_dt_ids
[] = {
655 { .compatible
= "zte,zx296702-dma", },
658 MODULE_DEVICE_TABLE(of
, zx6702_dma_dt_ids
);
660 static struct dma_chan
*zx_of_dma_simple_xlate(struct of_phandle_args
*dma_spec
,
661 struct of_dma
*ofdma
)
663 struct zx_dma_dev
*d
= ofdma
->of_dma_data
;
664 unsigned int request
= dma_spec
->args
[0];
665 struct dma_chan
*chan
;
666 struct zx_dma_chan
*c
;
668 if (request
> d
->dma_requests
)
671 chan
= dma_get_any_slave_channel(&d
->slave
);
673 dev_err(d
->slave
.dev
, "get channel fail in %s.\n", __func__
);
676 c
= to_zx_chan(chan
);
678 dev_info(d
->slave
.dev
, "zx_dma: pchan %u: alloc vchan %p\n",
683 static int zx_dma_probe(struct platform_device
*op
)
685 struct zx_dma_dev
*d
;
686 struct resource
*iores
;
689 iores
= platform_get_resource(op
, IORESOURCE_MEM
, 0);
693 d
= devm_kzalloc(&op
->dev
, sizeof(*d
), GFP_KERNEL
);
697 d
->base
= devm_ioremap_resource(&op
->dev
, iores
);
699 return PTR_ERR(d
->base
);
701 of_property_read_u32((&op
->dev
)->of_node
,
702 "dma-channels", &d
->dma_channels
);
703 of_property_read_u32((&op
->dev
)->of_node
,
704 "dma-requests", &d
->dma_requests
);
705 if (!d
->dma_requests
|| !d
->dma_channels
)
708 d
->clk
= devm_clk_get(&op
->dev
, NULL
);
709 if (IS_ERR(d
->clk
)) {
710 dev_err(&op
->dev
, "no dma clk\n");
711 return PTR_ERR(d
->clk
);
714 d
->irq
= platform_get_irq(op
, 0);
715 ret
= devm_request_irq(&op
->dev
, d
->irq
, zx_dma_int_handler
,
720 /* A DMA memory pool for LLIs, align on 32-byte boundary */
721 d
->pool
= dmam_pool_create(DRIVER_NAME
, &op
->dev
,
722 LLI_BLOCK_SIZE
, 32, 0);
726 /* init phy channel */
727 d
->phy
= devm_kzalloc(&op
->dev
,
728 d
->dma_channels
* sizeof(struct zx_dma_phy
), GFP_KERNEL
);
732 for (i
= 0; i
< d
->dma_channels
; i
++) {
733 struct zx_dma_phy
*p
= &d
->phy
[i
];
736 p
->base
= d
->base
+ i
* 0x40;
739 INIT_LIST_HEAD(&d
->slave
.channels
);
740 dma_cap_set(DMA_SLAVE
, d
->slave
.cap_mask
);
741 dma_cap_set(DMA_MEMCPY
, d
->slave
.cap_mask
);
742 dma_cap_set(DMA_PRIVATE
, d
->slave
.cap_mask
);
743 d
->slave
.dev
= &op
->dev
;
744 d
->slave
.device_free_chan_resources
= zx_dma_free_chan_resources
;
745 d
->slave
.device_tx_status
= zx_dma_tx_status
;
746 d
->slave
.device_prep_dma_memcpy
= zx_dma_prep_memcpy
;
747 d
->slave
.device_prep_slave_sg
= zx_dma_prep_slave_sg
;
748 d
->slave
.device_issue_pending
= zx_dma_issue_pending
;
749 d
->slave
.device_config
= zx_dma_config
;
750 d
->slave
.device_terminate_all
= zx_dma_terminate_all
;
751 d
->slave
.copy_align
= DMA_ALIGN
;
752 d
->slave
.src_addr_widths
= ZX_DMA_BUSWIDTHS
;
753 d
->slave
.dst_addr_widths
= ZX_DMA_BUSWIDTHS
;
754 d
->slave
.directions
= BIT(DMA_MEM_TO_MEM
) | BIT(DMA_MEM_TO_DEV
)
755 | BIT(DMA_DEV_TO_MEM
);
756 d
->slave
.residue_granularity
= DMA_RESIDUE_GRANULARITY_SEGMENT
;
758 /* init virtual channel */
759 d
->chans
= devm_kzalloc(&op
->dev
,
760 d
->dma_requests
* sizeof(struct zx_dma_chan
), GFP_KERNEL
);
764 for (i
= 0; i
< d
->dma_requests
; i
++) {
765 struct zx_dma_chan
*c
= &d
->chans
[i
];
767 c
->status
= DMA_IN_PROGRESS
;
768 INIT_LIST_HEAD(&c
->node
);
769 c
->vc
.desc_free
= zx_dma_free_desc
;
770 vchan_init(&c
->vc
, &d
->slave
);
773 /* Enable clock before accessing registers */
774 ret
= clk_prepare_enable(d
->clk
);
776 dev_err(&op
->dev
, "clk_prepare_enable failed: %d\n", ret
);
780 zx_dma_init_state(d
);
782 spin_lock_init(&d
->lock
);
783 INIT_LIST_HEAD(&d
->chan_pending
);
784 platform_set_drvdata(op
, d
);
786 ret
= dma_async_device_register(&d
->slave
);
790 ret
= of_dma_controller_register((&op
->dev
)->of_node
,
791 zx_of_dma_simple_xlate
, d
);
793 goto of_dma_register_fail
;
795 dev_info(&op
->dev
, "initialized\n");
798 of_dma_register_fail
:
799 dma_async_device_unregister(&d
->slave
);
801 clk_disable_unprepare(d
->clk
);
806 static int zx_dma_remove(struct platform_device
*op
)
808 struct zx_dma_chan
*c
, *cn
;
809 struct zx_dma_dev
*d
= platform_get_drvdata(op
);
811 /* explictly free the irq */
812 devm_free_irq(&op
->dev
, d
->irq
, d
);
814 dma_async_device_unregister(&d
->slave
);
815 of_dma_controller_free((&op
->dev
)->of_node
);
817 list_for_each_entry_safe(c
, cn
, &d
->slave
.channels
,
818 vc
.chan
.device_node
) {
819 list_del(&c
->vc
.chan
.device_node
);
821 clk_disable_unprepare(d
->clk
);
822 dmam_pool_destroy(d
->pool
);
827 #ifdef CONFIG_PM_SLEEP
828 static int zx_dma_suspend_dev(struct device
*dev
)
830 struct zx_dma_dev
*d
= dev_get_drvdata(dev
);
833 stat
= zx_dma_get_chan_stat(d
);
835 dev_warn(d
->slave
.dev
,
836 "chan %d is running fail to suspend\n", stat
);
839 clk_disable_unprepare(d
->clk
);
843 static int zx_dma_resume_dev(struct device
*dev
)
845 struct zx_dma_dev
*d
= dev_get_drvdata(dev
);
848 ret
= clk_prepare_enable(d
->clk
);
850 dev_err(d
->slave
.dev
, "clk_prepare_enable failed: %d\n", ret
);
853 zx_dma_init_state(d
);
858 static SIMPLE_DEV_PM_OPS(zx_dma_pmops
, zx_dma_suspend_dev
, zx_dma_resume_dev
);
860 static struct platform_driver zx_pdma_driver
= {
864 .of_match_table
= zx6702_dma_dt_ids
,
866 .probe
= zx_dma_probe
,
867 .remove
= zx_dma_remove
,
870 module_platform_driver(zx_pdma_driver
);
872 MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver");
873 MODULE_AUTHOR("Jun Nie jun.nie@linaro.org");
874 MODULE_LICENSE("GPL v2");