1 #include <linux/delay.h>
2 #include <linux/dmaengine.h>
3 #include <linux/dma-mapping.h>
4 #include <linux/platform_device.h>
5 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/of_dma.h>
9 #include <linux/of_irq.h>
10 #include <linux/dmapool.h>
11 #include <linux/interrupt.h>
12 #include <linux/of_address.h>
13 #include <linux/pm_runtime.h>
14 #include "dmaengine.h"
17 #define DESC_TYPE_HOST 0x10
18 #define DESC_TYPE_TEARD 0x13
20 #define TD_DESC_IS_RX (1 << 16)
21 #define TD_DESC_DMA_NUM 10
23 #define DESC_LENGTH_BITS_NUM 21
25 #define DESC_TYPE_USB (5 << 26)
26 #define DESC_PD_COMPLETE (1 << 31)
30 #define DMA_TXGCR(x) (0x800 + (x) * 0x20)
31 #define DMA_RXGCR(x) (0x808 + (x) * 0x20)
34 #define GCR_CHAN_ENABLE (1 << 31)
35 #define GCR_TEARDOWN (1 << 30)
36 #define GCR_STARV_RETRY (1 << 24)
37 #define GCR_DESC_TYPE_HOST (1 << 14)
40 #define DMA_SCHED_CTRL 0
41 #define DMA_SCHED_CTRL_EN (1 << 31)
42 #define DMA_SCHED_WORD(x) ((x) * 4 + 0x800)
44 #define SCHED_ENTRY0_CHAN(x) ((x) << 0)
45 #define SCHED_ENTRY0_IS_RX (1 << 7)
47 #define SCHED_ENTRY1_CHAN(x) ((x) << 8)
48 #define SCHED_ENTRY1_IS_RX (1 << 15)
50 #define SCHED_ENTRY2_CHAN(x) ((x) << 16)
51 #define SCHED_ENTRY2_IS_RX (1 << 23)
53 #define SCHED_ENTRY3_CHAN(x) ((x) << 24)
54 #define SCHED_ENTRY3_IS_RX (1 << 31)
57 /* 4 KiB of memory for descriptors, 2 for each endpoint */
58 #define ALLOC_DECS_NUM 128
60 #define TOTAL_DESCS_NUM (ALLOC_DECS_NUM * DESCS_AREAS)
61 #define QMGR_SCRATCH_SIZE (TOTAL_DESCS_NUM * 4)
63 #define QMGR_LRAM0_BASE 0x80
64 #define QMGR_LRAM_SIZE 0x84
65 #define QMGR_LRAM1_BASE 0x88
66 #define QMGR_MEMBASE(x) (0x1000 + (x) * 0x10)
67 #define QMGR_MEMCTRL(x) (0x1004 + (x) * 0x10)
68 #define QMGR_MEMCTRL_IDX_SH 16
69 #define QMGR_MEMCTRL_DESC_SH 8
71 #define QMGR_NUM_PEND 5
72 #define QMGR_PEND(x) (0x90 + (x) * 4)
74 #define QMGR_PENDING_SLOT_Q(x) (x / 32)
75 #define QMGR_PENDING_BIT_Q(x) (x % 32)
77 #define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10)
78 #define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10)
79 #define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
80 #define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
82 /* Glue layer specific */
83 /* USBSS / USB AM335x */
84 #define USBSS_IRQ_STATUS 0x28
85 #define USBSS_IRQ_ENABLER 0x2c
86 #define USBSS_IRQ_CLEARR 0x30
88 #define USBSS_IRQ_PD_COMP (1 << 2)
90 /* Packet Descriptor */
91 #define PD2_ZERO_LENGTH (1 << 19)
93 struct cppi41_channel
{
95 struct dma_async_tx_descriptor txd
;
96 struct cppi41_dd
*cdd
;
97 struct cppi41_desc
*desc
;
99 void __iomem
*gcr_reg
;
104 unsigned int q_comp_num
;
105 unsigned int port_num
;
108 unsigned td_queued
:1;
110 unsigned td_desc_seen
:1;
112 struct list_head node
; /* Node for pending list */
132 struct dma_device ddev
;
135 dma_addr_t scratch_phys
;
137 struct cppi41_desc
*cd
;
138 dma_addr_t descs_phys
;
140 struct cppi41_channel
*chan_busy
[ALLOC_DECS_NUM
];
142 void __iomem
*usbss_mem
;
143 void __iomem
*ctrl_mem
;
144 void __iomem
*sched_mem
;
145 void __iomem
*qmgr_mem
;
147 const struct chan_queues
*queues_rx
;
148 const struct chan_queues
*queues_tx
;
149 struct chan_queues td_queue
;
151 struct list_head pending
; /* Pending queued transfers */
152 spinlock_t lock
; /* Lock for pending list */
154 /* context for suspend/resume */
155 unsigned int dma_tdfdq
;
158 #define FIST_COMPLETION_QUEUE 93
159 static struct chan_queues usb_queues_tx
[] = {
161 [ 0] = { .submit
= 32, .complete
= 93},
162 [ 1] = { .submit
= 34, .complete
= 94},
163 [ 2] = { .submit
= 36, .complete
= 95},
164 [ 3] = { .submit
= 38, .complete
= 96},
165 [ 4] = { .submit
= 40, .complete
= 97},
166 [ 5] = { .submit
= 42, .complete
= 98},
167 [ 6] = { .submit
= 44, .complete
= 99},
168 [ 7] = { .submit
= 46, .complete
= 100},
169 [ 8] = { .submit
= 48, .complete
= 101},
170 [ 9] = { .submit
= 50, .complete
= 102},
171 [10] = { .submit
= 52, .complete
= 103},
172 [11] = { .submit
= 54, .complete
= 104},
173 [12] = { .submit
= 56, .complete
= 105},
174 [13] = { .submit
= 58, .complete
= 106},
175 [14] = { .submit
= 60, .complete
= 107},
178 [15] = { .submit
= 62, .complete
= 125},
179 [16] = { .submit
= 64, .complete
= 126},
180 [17] = { .submit
= 66, .complete
= 127},
181 [18] = { .submit
= 68, .complete
= 128},
182 [19] = { .submit
= 70, .complete
= 129},
183 [20] = { .submit
= 72, .complete
= 130},
184 [21] = { .submit
= 74, .complete
= 131},
185 [22] = { .submit
= 76, .complete
= 132},
186 [23] = { .submit
= 78, .complete
= 133},
187 [24] = { .submit
= 80, .complete
= 134},
188 [25] = { .submit
= 82, .complete
= 135},
189 [26] = { .submit
= 84, .complete
= 136},
190 [27] = { .submit
= 86, .complete
= 137},
191 [28] = { .submit
= 88, .complete
= 138},
192 [29] = { .submit
= 90, .complete
= 139},
195 static const struct chan_queues usb_queues_rx
[] = {
197 [ 0] = { .submit
= 1, .complete
= 109},
198 [ 1] = { .submit
= 2, .complete
= 110},
199 [ 2] = { .submit
= 3, .complete
= 111},
200 [ 3] = { .submit
= 4, .complete
= 112},
201 [ 4] = { .submit
= 5, .complete
= 113},
202 [ 5] = { .submit
= 6, .complete
= 114},
203 [ 6] = { .submit
= 7, .complete
= 115},
204 [ 7] = { .submit
= 8, .complete
= 116},
205 [ 8] = { .submit
= 9, .complete
= 117},
206 [ 9] = { .submit
= 10, .complete
= 118},
207 [10] = { .submit
= 11, .complete
= 119},
208 [11] = { .submit
= 12, .complete
= 120},
209 [12] = { .submit
= 13, .complete
= 121},
210 [13] = { .submit
= 14, .complete
= 122},
211 [14] = { .submit
= 15, .complete
= 123},
214 [15] = { .submit
= 16, .complete
= 141},
215 [16] = { .submit
= 17, .complete
= 142},
216 [17] = { .submit
= 18, .complete
= 143},
217 [18] = { .submit
= 19, .complete
= 144},
218 [19] = { .submit
= 20, .complete
= 145},
219 [20] = { .submit
= 21, .complete
= 146},
220 [21] = { .submit
= 22, .complete
= 147},
221 [22] = { .submit
= 23, .complete
= 148},
222 [23] = { .submit
= 24, .complete
= 149},
223 [24] = { .submit
= 25, .complete
= 150},
224 [25] = { .submit
= 26, .complete
= 151},
225 [26] = { .submit
= 27, .complete
= 152},
226 [27] = { .submit
= 28, .complete
= 153},
227 [28] = { .submit
= 29, .complete
= 154},
228 [29] = { .submit
= 30, .complete
= 155},
231 struct cppi_glue_infos
{
232 irqreturn_t (*isr
)(int irq
, void *data
);
233 const struct chan_queues
*queues_rx
;
234 const struct chan_queues
*queues_tx
;
235 struct chan_queues td_queue
;
238 static struct cppi41_channel
*to_cpp41_chan(struct dma_chan
*c
)
240 return container_of(c
, struct cppi41_channel
, chan
);
243 static struct cppi41_channel
*desc_to_chan(struct cppi41_dd
*cdd
, u32 desc
)
245 struct cppi41_channel
*c
;
249 descs_size
= sizeof(struct cppi41_desc
) * ALLOC_DECS_NUM
;
251 if (!((desc
>= cdd
->descs_phys
) &&
252 (desc
< (cdd
->descs_phys
+ descs_size
)))) {
256 desc_num
= (desc
- cdd
->descs_phys
) / sizeof(struct cppi41_desc
);
257 BUG_ON(desc_num
>= ALLOC_DECS_NUM
);
258 c
= cdd
->chan_busy
[desc_num
];
259 cdd
->chan_busy
[desc_num
] = NULL
;
263 static void cppi_writel(u32 val
, void *__iomem
*mem
)
265 __raw_writel(val
, mem
);
268 static u32
cppi_readl(void *__iomem
*mem
)
270 return __raw_readl(mem
);
273 static u32
pd_trans_len(u32 val
)
275 return val
& ((1 << (DESC_LENGTH_BITS_NUM
+ 1)) - 1);
278 static u32
cppi41_pop_desc(struct cppi41_dd
*cdd
, unsigned queue_num
)
282 desc
= cppi_readl(cdd
->qmgr_mem
+ QMGR_QUEUE_D(queue_num
));
287 static irqreturn_t
cppi41_irq(int irq
, void *data
)
289 struct cppi41_dd
*cdd
= data
;
290 struct cppi41_channel
*c
;
294 status
= cppi_readl(cdd
->usbss_mem
+ USBSS_IRQ_STATUS
);
295 if (!(status
& USBSS_IRQ_PD_COMP
))
297 cppi_writel(status
, cdd
->usbss_mem
+ USBSS_IRQ_STATUS
);
299 for (i
= QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE
); i
< QMGR_NUM_PEND
;
304 val
= cppi_readl(cdd
->qmgr_mem
+ QMGR_PEND(i
));
305 if (i
== QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE
) && val
) {
307 /* set corresponding bit for completetion Q 93 */
308 mask
= 1 << QMGR_PENDING_BIT_Q(FIST_COMPLETION_QUEUE
);
309 /* not set all bits for queues less than Q 93 */
311 /* now invert and keep only Q 93+ set */
322 val
&= ~(1 << q_num
);
324 desc
= cppi41_pop_desc(cdd
, q_num
);
325 c
= desc_to_chan(cdd
, desc
);
327 pr_err("%s() q %d desc %08x\n", __func__
,
332 if (c
->desc
->pd2
& PD2_ZERO_LENGTH
)
335 len
= pd_trans_len(c
->desc
->pd0
);
337 c
->residue
= pd_trans_len(c
->desc
->pd6
) - len
;
338 dma_cookie_complete(&c
->txd
);
339 c
->txd
.callback(c
->txd
.callback_param
);
341 /* Paired with cppi41_dma_issue_pending */
342 pm_runtime_mark_last_busy(cdd
->ddev
.dev
);
343 pm_runtime_put_autosuspend(cdd
->ddev
.dev
);
349 static dma_cookie_t
cppi41_tx_submit(struct dma_async_tx_descriptor
*tx
)
353 cookie
= dma_cookie_assign(tx
);
358 static int cppi41_dma_alloc_chan_resources(struct dma_chan
*chan
)
360 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
361 struct cppi41_dd
*cdd
= c
->cdd
;
364 error
= pm_runtime_get_sync(cdd
->ddev
.dev
);
368 dma_cookie_init(chan
);
369 dma_async_tx_descriptor_init(&c
->txd
, chan
);
370 c
->txd
.tx_submit
= cppi41_tx_submit
;
373 cppi_writel(c
->q_num
, c
->gcr_reg
+ RXHPCRA0
);
375 pm_runtime_mark_last_busy(cdd
->ddev
.dev
);
376 pm_runtime_put_autosuspend(cdd
->ddev
.dev
);
381 static void cppi41_dma_free_chan_resources(struct dma_chan
*chan
)
383 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
384 struct cppi41_dd
*cdd
= c
->cdd
;
387 error
= pm_runtime_get_sync(cdd
->ddev
.dev
);
391 WARN_ON(!list_empty(&cdd
->pending
));
393 pm_runtime_mark_last_busy(cdd
->ddev
.dev
);
394 pm_runtime_put_autosuspend(cdd
->ddev
.dev
);
397 static enum dma_status
cppi41_dma_tx_status(struct dma_chan
*chan
,
398 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
400 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
404 ret
= dma_cookie_status(chan
, cookie
, txstate
);
405 if (txstate
&& ret
== DMA_COMPLETE
)
406 txstate
->residue
= c
->residue
;
412 static void push_desc_queue(struct cppi41_channel
*c
)
414 struct cppi41_dd
*cdd
= c
->cdd
;
421 reg
= GCR_CHAN_ENABLE
;
423 reg
|= GCR_STARV_RETRY
;
424 reg
|= GCR_DESC_TYPE_HOST
;
425 reg
|= c
->q_comp_num
;
428 cppi_writel(reg
, c
->gcr_reg
);
431 * We don't use writel() but __raw_writel() so we have to make sure
432 * that the DMA descriptor in coherent memory made to the main memory
433 * before starting the dma engine.
437 desc_phys
= lower_32_bits(c
->desc_phys
);
438 desc_num
= (desc_phys
- cdd
->descs_phys
) / sizeof(struct cppi41_desc
);
439 WARN_ON(cdd
->chan_busy
[desc_num
]);
440 cdd
->chan_busy
[desc_num
] = c
;
442 reg
= (sizeof(struct cppi41_desc
) - 24) / 4;
444 cppi_writel(reg
, cdd
->qmgr_mem
+ QMGR_QUEUE_D(c
->q_num
));
447 static void pending_desc(struct cppi41_channel
*c
)
449 struct cppi41_dd
*cdd
= c
->cdd
;
452 spin_lock_irqsave(&cdd
->lock
, flags
);
453 list_add_tail(&c
->node
, &cdd
->pending
);
454 spin_unlock_irqrestore(&cdd
->lock
, flags
);
457 static void cppi41_dma_issue_pending(struct dma_chan
*chan
)
459 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
460 struct cppi41_dd
*cdd
= c
->cdd
;
463 /* PM runtime paired with dmaengine_desc_get_callback_invoke */
464 error
= pm_runtime_get(cdd
->ddev
.dev
);
466 dev_err(cdd
->ddev
.dev
, "Failed to pm_runtime_get: %i\n",
472 if (likely(pm_runtime_active(cdd
->ddev
.dev
)))
478 static u32
get_host_pd0(u32 length
)
482 reg
= DESC_TYPE_HOST
<< DESC_TYPE
;
488 static u32
get_host_pd1(struct cppi41_channel
*c
)
497 static u32
get_host_pd2(struct cppi41_channel
*c
)
502 reg
|= c
->q_comp_num
;
507 static u32
get_host_pd3(u32 length
)
511 /* PD3 = packet size */
517 static u32
get_host_pd6(u32 length
)
521 /* PD6 buffer size */
522 reg
= DESC_PD_COMPLETE
;
528 static u32
get_host_pd4_or_7(u32 addr
)
537 static u32
get_host_pd5(void)
546 static struct dma_async_tx_descriptor
*cppi41_dma_prep_slave_sg(
547 struct dma_chan
*chan
, struct scatterlist
*sgl
, unsigned sg_len
,
548 enum dma_transfer_direction dir
, unsigned long tx_flags
, void *context
)
550 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
551 struct cppi41_desc
*d
;
552 struct scatterlist
*sg
;
556 for_each_sg(sgl
, sg
, sg_len
, i
) {
560 /* We need to use more than one desc once musb supports sg */
561 addr
= lower_32_bits(sg_dma_address(sg
));
562 len
= sg_dma_len(sg
);
564 d
->pd0
= get_host_pd0(len
);
565 d
->pd1
= get_host_pd1(c
);
566 d
->pd2
= get_host_pd2(c
);
567 d
->pd3
= get_host_pd3(len
);
568 d
->pd4
= get_host_pd4_or_7(addr
);
569 d
->pd5
= get_host_pd5();
570 d
->pd6
= get_host_pd6(len
);
571 d
->pd7
= get_host_pd4_or_7(addr
);
579 static void cppi41_compute_td_desc(struct cppi41_desc
*d
)
581 d
->pd0
= DESC_TYPE_TEARD
<< DESC_TYPE
;
584 static int cppi41_tear_down_chan(struct cppi41_channel
*c
)
586 struct cppi41_dd
*cdd
= c
->cdd
;
587 struct cppi41_desc
*td
;
593 td
+= cdd
->first_td_desc
;
595 td_desc_phys
= cdd
->descs_phys
;
596 td_desc_phys
+= cdd
->first_td_desc
* sizeof(struct cppi41_desc
);
599 cppi41_compute_td_desc(td
);
602 reg
= (sizeof(struct cppi41_desc
) - 24) / 4;
604 cppi_writel(reg
, cdd
->qmgr_mem
+
605 QMGR_QUEUE_D(cdd
->td_queue
.submit
));
607 reg
= GCR_CHAN_ENABLE
;
609 reg
|= GCR_STARV_RETRY
;
610 reg
|= GCR_DESC_TYPE_HOST
;
611 reg
|= c
->q_comp_num
;
614 cppi_writel(reg
, c
->gcr_reg
);
619 if (!c
->td_seen
|| !c
->td_desc_seen
) {
621 desc_phys
= cppi41_pop_desc(cdd
, cdd
->td_queue
.complete
);
623 desc_phys
= cppi41_pop_desc(cdd
, c
->q_comp_num
);
625 if (desc_phys
== c
->desc_phys
) {
628 } else if (desc_phys
== td_desc_phys
) {
633 WARN_ON((pd0
>> DESC_TYPE
) != DESC_TYPE_TEARD
);
634 WARN_ON(!c
->is_tx
&& !(pd0
& TD_DESC_IS_RX
));
635 WARN_ON((pd0
& 0x1f) != c
->port_num
);
637 } else if (desc_phys
) {
643 * If the TX descriptor / channel is in use, the caller needs to poke
644 * his TD bit multiple times. After that he hardware releases the
645 * transfer descriptor followed by TD descriptor. Waiting seems not to
646 * cause any difference.
647 * RX seems to be thrown out right away. However once the TearDown
648 * descriptor gets through we are done. If we have seens the transfer
649 * descriptor before the TD we fetch it from enqueue, it has to be
650 * there waiting for us.
652 if (!c
->td_seen
&& c
->td_retry
) {
656 WARN_ON(!c
->td_retry
);
658 if (!c
->td_desc_seen
) {
659 desc_phys
= cppi41_pop_desc(cdd
, c
->q_num
);
661 desc_phys
= cppi41_pop_desc(cdd
, c
->q_comp_num
);
668 cppi_writel(0, c
->gcr_reg
);
672 static int cppi41_stop_chan(struct dma_chan
*chan
)
674 struct cppi41_channel
*c
= to_cpp41_chan(chan
);
675 struct cppi41_dd
*cdd
= c
->cdd
;
680 desc_phys
= lower_32_bits(c
->desc_phys
);
681 desc_num
= (desc_phys
- cdd
->descs_phys
) / sizeof(struct cppi41_desc
);
682 if (!cdd
->chan_busy
[desc_num
])
685 ret
= cppi41_tear_down_chan(c
);
689 WARN_ON(!cdd
->chan_busy
[desc_num
]);
690 cdd
->chan_busy
[desc_num
] = NULL
;
695 static void cleanup_chans(struct cppi41_dd
*cdd
)
697 while (!list_empty(&cdd
->ddev
.channels
)) {
698 struct cppi41_channel
*cchan
;
700 cchan
= list_first_entry(&cdd
->ddev
.channels
,
701 struct cppi41_channel
, chan
.device_node
);
702 list_del(&cchan
->chan
.device_node
);
707 static int cppi41_add_chans(struct device
*dev
, struct cppi41_dd
*cdd
)
709 struct cppi41_channel
*cchan
;
714 ret
= of_property_read_u32(dev
->of_node
, "#dma-channels",
719 * The channels can only be used as TX or as RX. So we add twice
720 * that much dma channels because USB can only do RX or TX.
724 for (i
= 0; i
< n_chans
; i
++) {
725 cchan
= kzalloc(sizeof(*cchan
), GFP_KERNEL
);
731 cchan
->gcr_reg
= cdd
->ctrl_mem
+ DMA_TXGCR(i
>> 1);
734 cchan
->gcr_reg
= cdd
->ctrl_mem
+ DMA_RXGCR(i
>> 1);
737 cchan
->port_num
= i
>> 1;
738 cchan
->desc
= &cdd
->cd
[i
];
739 cchan
->desc_phys
= cdd
->descs_phys
;
740 cchan
->desc_phys
+= i
* sizeof(struct cppi41_desc
);
741 cchan
->chan
.device
= &cdd
->ddev
;
742 list_add_tail(&cchan
->chan
.device_node
, &cdd
->ddev
.channels
);
744 cdd
->first_td_desc
= n_chans
;
752 static void purge_descs(struct device
*dev
, struct cppi41_dd
*cdd
)
754 unsigned int mem_decs
;
757 mem_decs
= ALLOC_DECS_NUM
* sizeof(struct cppi41_desc
);
759 for (i
= 0; i
< DESCS_AREAS
; i
++) {
761 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_MEMBASE(i
));
762 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_MEMCTRL(i
));
764 dma_free_coherent(dev
, mem_decs
, cdd
->cd
,
769 static void disable_sched(struct cppi41_dd
*cdd
)
771 cppi_writel(0, cdd
->sched_mem
+ DMA_SCHED_CTRL
);
774 static void deinit_cppi41(struct device
*dev
, struct cppi41_dd
*cdd
)
778 purge_descs(dev
, cdd
);
780 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_LRAM0_BASE
);
781 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_LRAM0_BASE
);
782 dma_free_coherent(dev
, QMGR_SCRATCH_SIZE
, cdd
->qmgr_scratch
,
786 static int init_descs(struct device
*dev
, struct cppi41_dd
*cdd
)
788 unsigned int desc_size
;
789 unsigned int mem_decs
;
794 BUILD_BUG_ON(sizeof(struct cppi41_desc
) &
795 (sizeof(struct cppi41_desc
) - 1));
796 BUILD_BUG_ON(sizeof(struct cppi41_desc
) < 32);
797 BUILD_BUG_ON(ALLOC_DECS_NUM
< 32);
799 desc_size
= sizeof(struct cppi41_desc
);
800 mem_decs
= ALLOC_DECS_NUM
* desc_size
;
803 for (i
= 0; i
< DESCS_AREAS
; i
++) {
805 reg
= idx
<< QMGR_MEMCTRL_IDX_SH
;
806 reg
|= (ilog2(desc_size
) - 5) << QMGR_MEMCTRL_DESC_SH
;
807 reg
|= ilog2(ALLOC_DECS_NUM
) - 5;
809 BUILD_BUG_ON(DESCS_AREAS
!= 1);
810 cdd
->cd
= dma_alloc_coherent(dev
, mem_decs
,
811 &cdd
->descs_phys
, GFP_KERNEL
);
815 cppi_writel(cdd
->descs_phys
, cdd
->qmgr_mem
+ QMGR_MEMBASE(i
));
816 cppi_writel(reg
, cdd
->qmgr_mem
+ QMGR_MEMCTRL(i
));
818 idx
+= ALLOC_DECS_NUM
;
823 static void init_sched(struct cppi41_dd
*cdd
)
830 cppi_writel(0, cdd
->sched_mem
+ DMA_SCHED_CTRL
);
831 for (ch
= 0; ch
< 15 * 2; ch
+= 2) {
833 reg
= SCHED_ENTRY0_CHAN(ch
);
834 reg
|= SCHED_ENTRY1_CHAN(ch
) | SCHED_ENTRY1_IS_RX
;
836 reg
|= SCHED_ENTRY2_CHAN(ch
+ 1);
837 reg
|= SCHED_ENTRY3_CHAN(ch
+ 1) | SCHED_ENTRY3_IS_RX
;
838 cppi_writel(reg
, cdd
->sched_mem
+ DMA_SCHED_WORD(word
));
841 reg
= 15 * 2 * 2 - 1;
842 reg
|= DMA_SCHED_CTRL_EN
;
843 cppi_writel(reg
, cdd
->sched_mem
+ DMA_SCHED_CTRL
);
846 static int init_cppi41(struct device
*dev
, struct cppi41_dd
*cdd
)
850 BUILD_BUG_ON(QMGR_SCRATCH_SIZE
> ((1 << 14) - 1));
851 cdd
->qmgr_scratch
= dma_alloc_coherent(dev
, QMGR_SCRATCH_SIZE
,
852 &cdd
->scratch_phys
, GFP_KERNEL
);
853 if (!cdd
->qmgr_scratch
)
856 cppi_writel(cdd
->scratch_phys
, cdd
->qmgr_mem
+ QMGR_LRAM0_BASE
);
857 cppi_writel(QMGR_SCRATCH_SIZE
, cdd
->qmgr_mem
+ QMGR_LRAM_SIZE
);
858 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_LRAM1_BASE
);
860 ret
= init_descs(dev
, cdd
);
864 cppi_writel(cdd
->td_queue
.submit
, cdd
->ctrl_mem
+ DMA_TDFDQ
);
868 deinit_cppi41(dev
, cdd
);
872 static struct platform_driver cpp41_dma_driver
;
874 * The param format is:
882 static bool cpp41_dma_filter_fn(struct dma_chan
*chan
, void *param
)
884 struct cppi41_channel
*cchan
;
885 struct cppi41_dd
*cdd
;
886 const struct chan_queues
*queues
;
889 if (chan
->device
->dev
->driver
!= &cpp41_dma_driver
.driver
)
892 cchan
= to_cpp41_chan(chan
);
894 if (cchan
->port_num
!= num
[INFO_PORT
])
897 if (cchan
->is_tx
&& !num
[INFO_IS_TX
])
901 queues
= cdd
->queues_tx
;
903 queues
= cdd
->queues_rx
;
905 BUILD_BUG_ON(ARRAY_SIZE(usb_queues_rx
) != ARRAY_SIZE(usb_queues_tx
));
906 if (WARN_ON(cchan
->port_num
> ARRAY_SIZE(usb_queues_rx
)))
909 cchan
->q_num
= queues
[cchan
->port_num
].submit
;
910 cchan
->q_comp_num
= queues
[cchan
->port_num
].complete
;
914 static struct of_dma_filter_info cpp41_dma_info
= {
915 .filter_fn
= cpp41_dma_filter_fn
,
918 static struct dma_chan
*cppi41_dma_xlate(struct of_phandle_args
*dma_spec
,
919 struct of_dma
*ofdma
)
921 int count
= dma_spec
->args_count
;
922 struct of_dma_filter_info
*info
= ofdma
->of_dma_data
;
924 if (!info
|| !info
->filter_fn
)
930 return dma_request_channel(info
->dma_cap
, info
->filter_fn
,
934 static const struct cppi_glue_infos usb_infos
= {
936 .queues_rx
= usb_queues_rx
,
937 .queues_tx
= usb_queues_tx
,
938 .td_queue
= { .submit
= 31, .complete
= 0 },
941 static const struct of_device_id cppi41_dma_ids
[] = {
942 { .compatible
= "ti,am3359-cppi41", .data
= &usb_infos
},
945 MODULE_DEVICE_TABLE(of
, cppi41_dma_ids
);
947 static const struct cppi_glue_infos
*get_glue_info(struct device
*dev
)
949 const struct of_device_id
*of_id
;
951 of_id
= of_match_node(cppi41_dma_ids
, dev
->of_node
);
957 #define CPPI41_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
958 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
959 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
960 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
962 static int cppi41_dma_probe(struct platform_device
*pdev
)
964 struct cppi41_dd
*cdd
;
965 struct device
*dev
= &pdev
->dev
;
966 const struct cppi_glue_infos
*glue_info
;
970 glue_info
= get_glue_info(dev
);
974 cdd
= devm_kzalloc(&pdev
->dev
, sizeof(*cdd
), GFP_KERNEL
);
978 dma_cap_set(DMA_SLAVE
, cdd
->ddev
.cap_mask
);
979 cdd
->ddev
.device_alloc_chan_resources
= cppi41_dma_alloc_chan_resources
;
980 cdd
->ddev
.device_free_chan_resources
= cppi41_dma_free_chan_resources
;
981 cdd
->ddev
.device_tx_status
= cppi41_dma_tx_status
;
982 cdd
->ddev
.device_issue_pending
= cppi41_dma_issue_pending
;
983 cdd
->ddev
.device_prep_slave_sg
= cppi41_dma_prep_slave_sg
;
984 cdd
->ddev
.device_terminate_all
= cppi41_stop_chan
;
985 cdd
->ddev
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
986 cdd
->ddev
.src_addr_widths
= CPPI41_DMA_BUSWIDTHS
;
987 cdd
->ddev
.dst_addr_widths
= CPPI41_DMA_BUSWIDTHS
;
988 cdd
->ddev
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
990 INIT_LIST_HEAD(&cdd
->ddev
.channels
);
991 cpp41_dma_info
.dma_cap
= cdd
->ddev
.cap_mask
;
993 cdd
->usbss_mem
= of_iomap(dev
->of_node
, 0);
994 cdd
->ctrl_mem
= of_iomap(dev
->of_node
, 1);
995 cdd
->sched_mem
= of_iomap(dev
->of_node
, 2);
996 cdd
->qmgr_mem
= of_iomap(dev
->of_node
, 3);
997 spin_lock_init(&cdd
->lock
);
998 INIT_LIST_HEAD(&cdd
->pending
);
1000 platform_set_drvdata(pdev
, cdd
);
1002 if (!cdd
->usbss_mem
|| !cdd
->ctrl_mem
|| !cdd
->sched_mem
||
1006 pm_runtime_enable(dev
);
1007 pm_runtime_set_autosuspend_delay(dev
, 100);
1008 pm_runtime_use_autosuspend(dev
);
1009 ret
= pm_runtime_get_sync(dev
);
1013 cdd
->queues_rx
= glue_info
->queues_rx
;
1014 cdd
->queues_tx
= glue_info
->queues_tx
;
1015 cdd
->td_queue
= glue_info
->td_queue
;
1017 ret
= init_cppi41(dev
, cdd
);
1021 ret
= cppi41_add_chans(dev
, cdd
);
1025 irq
= irq_of_parse_and_map(dev
->of_node
, 0);
1031 cppi_writel(USBSS_IRQ_PD_COMP
, cdd
->usbss_mem
+ USBSS_IRQ_ENABLER
);
1033 ret
= devm_request_irq(&pdev
->dev
, irq
, glue_info
->isr
, IRQF_SHARED
,
1034 dev_name(dev
), cdd
);
1039 ret
= dma_async_device_register(&cdd
->ddev
);
1043 ret
= of_dma_controller_register(dev
->of_node
,
1044 cppi41_dma_xlate
, &cpp41_dma_info
);
1048 pm_runtime_mark_last_busy(dev
);
1049 pm_runtime_put_autosuspend(dev
);
1053 dma_async_device_unregister(&cdd
->ddev
);
1056 cppi_writel(0, cdd
->usbss_mem
+ USBSS_IRQ_CLEARR
);
1059 deinit_cppi41(dev
, cdd
);
1061 pm_runtime_dont_use_autosuspend(dev
);
1062 pm_runtime_put_sync(dev
);
1064 pm_runtime_disable(dev
);
1065 iounmap(cdd
->usbss_mem
);
1066 iounmap(cdd
->ctrl_mem
);
1067 iounmap(cdd
->sched_mem
);
1068 iounmap(cdd
->qmgr_mem
);
1072 static int cppi41_dma_remove(struct platform_device
*pdev
)
1074 struct cppi41_dd
*cdd
= platform_get_drvdata(pdev
);
1076 of_dma_controller_free(pdev
->dev
.of_node
);
1077 dma_async_device_unregister(&cdd
->ddev
);
1079 cppi_writel(0, cdd
->usbss_mem
+ USBSS_IRQ_CLEARR
);
1080 devm_free_irq(&pdev
->dev
, cdd
->irq
, cdd
);
1082 deinit_cppi41(&pdev
->dev
, cdd
);
1083 iounmap(cdd
->usbss_mem
);
1084 iounmap(cdd
->ctrl_mem
);
1085 iounmap(cdd
->sched_mem
);
1086 iounmap(cdd
->qmgr_mem
);
1087 pm_runtime_dont_use_autosuspend(&pdev
->dev
);
1088 pm_runtime_put_sync(&pdev
->dev
);
1089 pm_runtime_disable(&pdev
->dev
);
1093 #ifdef CONFIG_PM_SLEEP
1094 static int cppi41_suspend(struct device
*dev
)
1096 struct cppi41_dd
*cdd
= dev_get_drvdata(dev
);
1098 cdd
->dma_tdfdq
= cppi_readl(cdd
->ctrl_mem
+ DMA_TDFDQ
);
1099 cppi_writel(0, cdd
->usbss_mem
+ USBSS_IRQ_CLEARR
);
1105 static int cppi41_resume(struct device
*dev
)
1107 struct cppi41_dd
*cdd
= dev_get_drvdata(dev
);
1108 struct cppi41_channel
*c
;
1111 for (i
= 0; i
< DESCS_AREAS
; i
++)
1112 cppi_writel(cdd
->descs_phys
, cdd
->qmgr_mem
+ QMGR_MEMBASE(i
));
1114 list_for_each_entry(c
, &cdd
->ddev
.channels
, chan
.device_node
)
1116 cppi_writel(c
->q_num
, c
->gcr_reg
+ RXHPCRA0
);
1120 cppi_writel(cdd
->dma_tdfdq
, cdd
->ctrl_mem
+ DMA_TDFDQ
);
1121 cppi_writel(cdd
->scratch_phys
, cdd
->qmgr_mem
+ QMGR_LRAM0_BASE
);
1122 cppi_writel(QMGR_SCRATCH_SIZE
, cdd
->qmgr_mem
+ QMGR_LRAM_SIZE
);
1123 cppi_writel(0, cdd
->qmgr_mem
+ QMGR_LRAM1_BASE
);
1125 cppi_writel(USBSS_IRQ_PD_COMP
, cdd
->usbss_mem
+ USBSS_IRQ_ENABLER
);
1130 static int cppi41_runtime_suspend(struct device
*dev
)
1132 struct cppi41_dd
*cdd
= dev_get_drvdata(dev
);
1134 WARN_ON(!list_empty(&cdd
->pending
));
1139 static int cppi41_runtime_resume(struct device
*dev
)
1141 struct cppi41_dd
*cdd
= dev_get_drvdata(dev
);
1142 struct cppi41_channel
*c
, *_c
;
1143 unsigned long flags
;
1145 spin_lock_irqsave(&cdd
->lock
, flags
);
1146 list_for_each_entry_safe(c
, _c
, &cdd
->pending
, node
) {
1150 spin_unlock_irqrestore(&cdd
->lock
, flags
);
1156 static const struct dev_pm_ops cppi41_pm_ops
= {
1157 SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend
, cppi41_resume
)
1158 SET_RUNTIME_PM_OPS(cppi41_runtime_suspend
,
1159 cppi41_runtime_resume
,
1163 static struct platform_driver cpp41_dma_driver
= {
1164 .probe
= cppi41_dma_probe
,
1165 .remove
= cppi41_dma_remove
,
1167 .name
= "cppi41-dma-engine",
1168 .pm
= &cppi41_pm_ops
,
1169 .of_match_table
= of_match_ptr(cppi41_dma_ids
),
1173 module_platform_driver(cpp41_dma_driver
);
1174 MODULE_LICENSE("GPL");
1175 MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");