1 #include <linux/device.h>
2 #include <linux/dma-mapping.h>
3 #include <linux/dmaengine.h>
4 #include <linux/sizes.h>
5 #include <linux/platform_device.h>
10 #include "musb_trace.h"
12 #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
14 #define EP_MODE_AUTOREQ_NONE 0
15 #define EP_MODE_AUTOREQ_ALL_NEOP 1
16 #define EP_MODE_AUTOREQ_ALWAYS 3
18 #define EP_MODE_DMA_TRANSPARENT 0
19 #define EP_MODE_DMA_RNDIS 1
20 #define EP_MODE_DMA_GEN_RNDIS 3
22 #define USB_CTRL_TX_MODE 0x70
23 #define USB_CTRL_RX_MODE 0x74
24 #define USB_CTRL_AUTOREQ 0xd0
25 #define USB_TDOWN 0xd8
27 #define MUSB_DMA_NUM_CHANNELS 15
29 struct cppi41_dma_controller
{
30 struct dma_controller controller
;
31 struct cppi41_dma_channel rx_channel
[MUSB_DMA_NUM_CHANNELS
];
32 struct cppi41_dma_channel tx_channel
[MUSB_DMA_NUM_CHANNELS
];
33 struct hrtimer early_tx
;
34 struct list_head early_tx_list
;
40 static void save_rx_toggle(struct cppi41_dma_channel
*cppi41_channel
)
45 if (cppi41_channel
->is_tx
)
47 if (!is_host_active(cppi41_channel
->controller
->controller
.musb
))
50 csr
= musb_readw(cppi41_channel
->hw_ep
->regs
, MUSB_RXCSR
);
51 toggle
= csr
& MUSB_RXCSR_H_DATATOGGLE
? 1 : 0;
53 cppi41_channel
->usb_toggle
= toggle
;
56 static void update_rx_toggle(struct cppi41_dma_channel
*cppi41_channel
)
58 struct musb_hw_ep
*hw_ep
= cppi41_channel
->hw_ep
;
59 struct musb
*musb
= hw_ep
->musb
;
63 if (cppi41_channel
->is_tx
)
65 if (!is_host_active(musb
))
68 musb_ep_select(musb
->mregs
, hw_ep
->epnum
);
69 csr
= musb_readw(hw_ep
->regs
, MUSB_RXCSR
);
70 toggle
= csr
& MUSB_RXCSR_H_DATATOGGLE
? 1 : 0;
73 * AM335x Advisory 1.0.13: Due to internal synchronisation error the
74 * data toggle may reset from DATA1 to DATA0 during receiving data from
75 * more than one endpoint.
77 if (!toggle
&& toggle
== cppi41_channel
->usb_toggle
) {
78 csr
|= MUSB_RXCSR_H_DATATOGGLE
| MUSB_RXCSR_H_WR_DATATOGGLE
;
79 musb_writew(cppi41_channel
->hw_ep
->regs
, MUSB_RXCSR
, csr
);
80 musb_dbg(musb
, "Restoring DATA1 toggle.");
83 cppi41_channel
->usb_toggle
= toggle
;
86 static bool musb_is_tx_fifo_empty(struct musb_hw_ep
*hw_ep
)
88 u8 epnum
= hw_ep
->epnum
;
89 struct musb
*musb
= hw_ep
->musb
;
90 void __iomem
*epio
= musb
->endpoints
[epnum
].regs
;
93 musb_ep_select(musb
->mregs
, hw_ep
->epnum
);
94 csr
= musb_readw(epio
, MUSB_TXCSR
);
95 if (csr
& MUSB_TXCSR_TXPKTRDY
)
100 static void cppi41_dma_callback(void *private_data
,
101 const struct dmaengine_result
*result
);
103 static void cppi41_trans_done(struct cppi41_dma_channel
*cppi41_channel
)
105 struct musb_hw_ep
*hw_ep
= cppi41_channel
->hw_ep
;
106 struct musb
*musb
= hw_ep
->musb
;
107 void __iomem
*epio
= hw_ep
->regs
;
110 if (!cppi41_channel
->prog_len
||
111 (cppi41_channel
->channel
.status
== MUSB_DMA_STATUS_FREE
)) {
114 cppi41_channel
->channel
.actual_len
=
115 cppi41_channel
->transferred
;
116 cppi41_channel
->channel
.status
= MUSB_DMA_STATUS_FREE
;
117 cppi41_channel
->channel
.rx_packet_done
= true;
120 * transmit ZLP using PIO mode for transfers which size is
121 * multiple of EP packet size.
123 if (cppi41_channel
->tx_zlp
&& (cppi41_channel
->transferred
%
124 cppi41_channel
->packet_sz
) == 0) {
125 musb_ep_select(musb
->mregs
, hw_ep
->epnum
);
126 csr
= MUSB_TXCSR_MODE
| MUSB_TXCSR_TXPKTRDY
;
127 musb_writew(epio
, MUSB_TXCSR
, csr
);
130 trace_musb_cppi41_done(cppi41_channel
);
131 musb_dma_completion(musb
, hw_ep
->epnum
, cppi41_channel
->is_tx
);
133 /* next iteration, reload */
134 struct dma_chan
*dc
= cppi41_channel
->dc
;
135 struct dma_async_tx_descriptor
*dma_desc
;
136 enum dma_transfer_direction direction
;
139 cppi41_channel
->buf_addr
+= cppi41_channel
->packet_sz
;
141 remain_bytes
= cppi41_channel
->total_len
;
142 remain_bytes
-= cppi41_channel
->transferred
;
143 remain_bytes
= min(remain_bytes
, cppi41_channel
->packet_sz
);
144 cppi41_channel
->prog_len
= remain_bytes
;
146 direction
= cppi41_channel
->is_tx
? DMA_MEM_TO_DEV
148 dma_desc
= dmaengine_prep_slave_single(dc
,
149 cppi41_channel
->buf_addr
,
152 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
153 if (WARN_ON(!dma_desc
))
156 dma_desc
->callback_result
= cppi41_dma_callback
;
157 dma_desc
->callback_param
= &cppi41_channel
->channel
;
158 cppi41_channel
->cookie
= dma_desc
->tx_submit(dma_desc
);
159 trace_musb_cppi41_cont(cppi41_channel
);
160 dma_async_issue_pending(dc
);
162 if (!cppi41_channel
->is_tx
) {
163 musb_ep_select(musb
->mregs
, hw_ep
->epnum
);
164 csr
= musb_readw(epio
, MUSB_RXCSR
);
165 csr
|= MUSB_RXCSR_H_REQPKT
;
166 musb_writew(epio
, MUSB_RXCSR
, csr
);
171 static enum hrtimer_restart
cppi41_recheck_tx_req(struct hrtimer
*timer
)
173 struct cppi41_dma_controller
*controller
;
174 struct cppi41_dma_channel
*cppi41_channel
, *n
;
177 enum hrtimer_restart ret
= HRTIMER_NORESTART
;
179 controller
= container_of(timer
, struct cppi41_dma_controller
,
181 musb
= controller
->controller
.musb
;
183 spin_lock_irqsave(&musb
->lock
, flags
);
184 list_for_each_entry_safe(cppi41_channel
, n
, &controller
->early_tx_list
,
187 struct musb_hw_ep
*hw_ep
= cppi41_channel
->hw_ep
;
189 empty
= musb_is_tx_fifo_empty(hw_ep
);
191 list_del_init(&cppi41_channel
->tx_check
);
192 cppi41_trans_done(cppi41_channel
);
196 if (!list_empty(&controller
->early_tx_list
) &&
197 !hrtimer_is_queued(&controller
->early_tx
)) {
198 ret
= HRTIMER_RESTART
;
199 hrtimer_forward_now(&controller
->early_tx
, 20 * NSEC_PER_USEC
);
202 spin_unlock_irqrestore(&musb
->lock
, flags
);
206 static void cppi41_dma_callback(void *private_data
,
207 const struct dmaengine_result
*result
)
209 struct dma_channel
*channel
= private_data
;
210 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
211 struct musb_hw_ep
*hw_ep
= cppi41_channel
->hw_ep
;
212 struct cppi41_dma_controller
*controller
;
213 struct musb
*musb
= hw_ep
->musb
;
215 struct dma_tx_state txstate
;
220 controller
= cppi41_channel
->controller
;
221 if (controller
->controller
.dma_callback
)
222 controller
->controller
.dma_callback(&controller
->controller
);
224 if (result
->result
== DMA_TRANS_ABORTED
)
227 spin_lock_irqsave(&musb
->lock
, flags
);
229 dmaengine_tx_status(cppi41_channel
->dc
, cppi41_channel
->cookie
,
231 transferred
= cppi41_channel
->prog_len
- txstate
.residue
;
232 cppi41_channel
->transferred
+= transferred
;
234 trace_musb_cppi41_gb(cppi41_channel
);
235 update_rx_toggle(cppi41_channel
);
237 if (cppi41_channel
->transferred
== cppi41_channel
->total_len
||
238 transferred
< cppi41_channel
->packet_sz
)
239 cppi41_channel
->prog_len
= 0;
241 if (cppi41_channel
->is_tx
)
242 empty
= musb_is_tx_fifo_empty(hw_ep
);
244 if (!cppi41_channel
->is_tx
|| empty
) {
245 cppi41_trans_done(cppi41_channel
);
250 * On AM335x it has been observed that the TX interrupt fires
251 * too early that means the TXFIFO is not yet empty but the DMA
252 * engine says that it is done with the transfer. We don't
253 * receive a FIFO empty interrupt so the only thing we can do is
254 * to poll for the bit. On HS it usually takes 2us, on FS around
255 * 110us - 150us depending on the transfer size.
256 * We spin on HS (no longer than than 25us and setup a timer on
257 * FS to check for the bit and complete the transfer.
259 if (is_host_active(musb
)) {
260 if (musb
->port1_status
& USB_PORT_STAT_HIGH_SPEED
)
263 if (musb
->g
.speed
== USB_SPEED_HIGH
)
270 empty
= musb_is_tx_fifo_empty(hw_ep
);
272 cppi41_trans_done(cppi41_channel
);
281 list_add_tail(&cppi41_channel
->tx_check
,
282 &controller
->early_tx_list
);
283 if (!hrtimer_is_queued(&controller
->early_tx
)) {
284 unsigned long usecs
= cppi41_channel
->total_len
/ 10;
286 hrtimer_start_range_ns(&controller
->early_tx
,
287 usecs
* NSEC_PER_USEC
,
293 spin_unlock_irqrestore(&musb
->lock
, flags
);
296 static u32
update_ep_mode(unsigned ep
, unsigned mode
, u32 old
)
300 shift
= (ep
- 1) * 2;
301 old
&= ~(3 << shift
);
302 old
|= mode
<< shift
;
306 static void cppi41_set_dma_mode(struct cppi41_dma_channel
*cppi41_channel
,
309 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
310 struct musb
*musb
= controller
->controller
.musb
;
315 if (cppi41_channel
->is_tx
)
316 old_mode
= controller
->tx_mode
;
318 old_mode
= controller
->rx_mode
;
319 port
= cppi41_channel
->port_num
;
320 new_mode
= update_ep_mode(port
, mode
, old_mode
);
322 if (new_mode
== old_mode
)
324 if (cppi41_channel
->is_tx
) {
325 controller
->tx_mode
= new_mode
;
326 musb_writel(musb
->ctrl_base
, USB_CTRL_TX_MODE
, new_mode
);
328 controller
->rx_mode
= new_mode
;
329 musb_writel(musb
->ctrl_base
, USB_CTRL_RX_MODE
, new_mode
);
333 static void cppi41_set_autoreq_mode(struct cppi41_dma_channel
*cppi41_channel
,
336 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
341 old_mode
= controller
->auto_req
;
342 port
= cppi41_channel
->port_num
;
343 new_mode
= update_ep_mode(port
, mode
, old_mode
);
345 if (new_mode
== old_mode
)
347 controller
->auto_req
= new_mode
;
348 musb_writel(controller
->controller
.musb
->ctrl_base
, USB_CTRL_AUTOREQ
,
352 static bool cppi41_configure_channel(struct dma_channel
*channel
,
353 u16 packet_sz
, u8 mode
,
354 dma_addr_t dma_addr
, u32 len
)
356 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
357 struct dma_chan
*dc
= cppi41_channel
->dc
;
358 struct dma_async_tx_descriptor
*dma_desc
;
359 enum dma_transfer_direction direction
;
360 struct musb
*musb
= cppi41_channel
->controller
->controller
.musb
;
361 unsigned use_gen_rndis
= 0;
363 cppi41_channel
->buf_addr
= dma_addr
;
364 cppi41_channel
->total_len
= len
;
365 cppi41_channel
->transferred
= 0;
366 cppi41_channel
->packet_sz
= packet_sz
;
367 cppi41_channel
->tx_zlp
= (cppi41_channel
->is_tx
&& mode
) ? 1 : 0;
370 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
371 * than max packet size at a time.
373 if (cppi41_channel
->is_tx
)
378 if (len
> packet_sz
) {
379 musb_writel(musb
->ctrl_base
,
380 RNDIS_REG(cppi41_channel
->port_num
), len
);
382 cppi41_set_dma_mode(cppi41_channel
,
383 EP_MODE_DMA_GEN_RNDIS
);
386 cppi41_set_autoreq_mode(cppi41_channel
,
387 EP_MODE_AUTOREQ_ALL_NEOP
);
389 musb_writel(musb
->ctrl_base
,
390 RNDIS_REG(cppi41_channel
->port_num
), 0);
391 cppi41_set_dma_mode(cppi41_channel
,
392 EP_MODE_DMA_TRANSPARENT
);
393 cppi41_set_autoreq_mode(cppi41_channel
,
394 EP_MODE_AUTOREQ_NONE
);
398 cppi41_set_dma_mode(cppi41_channel
, EP_MODE_DMA_TRANSPARENT
);
399 cppi41_set_autoreq_mode(cppi41_channel
, EP_MODE_AUTOREQ_NONE
);
400 len
= min_t(u32
, packet_sz
, len
);
402 cppi41_channel
->prog_len
= len
;
403 direction
= cppi41_channel
->is_tx
? DMA_MEM_TO_DEV
: DMA_DEV_TO_MEM
;
404 dma_desc
= dmaengine_prep_slave_single(dc
, dma_addr
, len
, direction
,
405 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
409 dma_desc
->callback_result
= cppi41_dma_callback
;
410 dma_desc
->callback_param
= channel
;
411 cppi41_channel
->cookie
= dma_desc
->tx_submit(dma_desc
);
412 cppi41_channel
->channel
.rx_packet_done
= false;
414 trace_musb_cppi41_config(cppi41_channel
);
416 save_rx_toggle(cppi41_channel
);
417 dma_async_issue_pending(dc
);
421 static struct dma_channel
*cppi41_dma_channel_allocate(struct dma_controller
*c
,
422 struct musb_hw_ep
*hw_ep
, u8 is_tx
)
424 struct cppi41_dma_controller
*controller
= container_of(c
,
425 struct cppi41_dma_controller
, controller
);
426 struct cppi41_dma_channel
*cppi41_channel
= NULL
;
427 u8 ch_num
= hw_ep
->epnum
- 1;
429 if (ch_num
>= MUSB_DMA_NUM_CHANNELS
)
433 cppi41_channel
= &controller
->tx_channel
[ch_num
];
435 cppi41_channel
= &controller
->rx_channel
[ch_num
];
437 if (!cppi41_channel
->dc
)
440 if (cppi41_channel
->is_allocated
)
443 cppi41_channel
->hw_ep
= hw_ep
;
444 cppi41_channel
->is_allocated
= 1;
446 trace_musb_cppi41_alloc(cppi41_channel
);
447 return &cppi41_channel
->channel
;
450 static void cppi41_dma_channel_release(struct dma_channel
*channel
)
452 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
454 trace_musb_cppi41_free(cppi41_channel
);
455 if (cppi41_channel
->is_allocated
) {
456 cppi41_channel
->is_allocated
= 0;
457 channel
->status
= MUSB_DMA_STATUS_FREE
;
458 channel
->actual_len
= 0;
462 static int cppi41_dma_channel_program(struct dma_channel
*channel
,
463 u16 packet_sz
, u8 mode
,
464 dma_addr_t dma_addr
, u32 len
)
467 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
470 BUG_ON(channel
->status
== MUSB_DMA_STATUS_UNKNOWN
||
471 channel
->status
== MUSB_DMA_STATUS_BUSY
);
473 if (is_host_active(cppi41_channel
->controller
->controller
.musb
)) {
474 if (cppi41_channel
->is_tx
)
475 hb_mult
= cppi41_channel
->hw_ep
->out_qh
->hb_mult
;
477 hb_mult
= cppi41_channel
->hw_ep
->in_qh
->hb_mult
;
480 channel
->status
= MUSB_DMA_STATUS_BUSY
;
481 channel
->actual_len
= 0;
484 packet_sz
= hb_mult
* (packet_sz
& 0x7FF);
486 ret
= cppi41_configure_channel(channel
, packet_sz
, mode
, dma_addr
, len
);
488 channel
->status
= MUSB_DMA_STATUS_FREE
;
493 static int cppi41_is_compatible(struct dma_channel
*channel
, u16 maxpacket
,
494 void *buf
, u32 length
)
496 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
497 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
498 struct musb
*musb
= controller
->controller
.musb
;
500 if (is_host_active(musb
)) {
504 if (cppi41_channel
->hw_ep
->ep_in
.type
!= USB_ENDPOINT_XFER_BULK
)
506 if (cppi41_channel
->is_tx
)
508 /* AM335x Advisory 1.0.13. No workaround for device RX mode */
512 static int cppi41_dma_channel_abort(struct dma_channel
*channel
)
514 struct cppi41_dma_channel
*cppi41_channel
= channel
->private_data
;
515 struct cppi41_dma_controller
*controller
= cppi41_channel
->controller
;
516 struct musb
*musb
= controller
->controller
.musb
;
517 void __iomem
*epio
= cppi41_channel
->hw_ep
->regs
;
523 is_tx
= cppi41_channel
->is_tx
;
524 trace_musb_cppi41_abort(cppi41_channel
);
526 if (cppi41_channel
->channel
.status
== MUSB_DMA_STATUS_FREE
)
529 list_del_init(&cppi41_channel
->tx_check
);
531 csr
= musb_readw(epio
, MUSB_TXCSR
);
532 csr
&= ~MUSB_TXCSR_DMAENAB
;
533 musb_writew(epio
, MUSB_TXCSR
, csr
);
535 cppi41_set_autoreq_mode(cppi41_channel
, EP_MODE_AUTOREQ_NONE
);
537 /* delay to drain to cppi dma pipeline for isoch */
540 csr
= musb_readw(epio
, MUSB_RXCSR
);
541 csr
&= ~(MUSB_RXCSR_H_REQPKT
| MUSB_RXCSR_DMAENAB
);
542 musb_writew(epio
, MUSB_RXCSR
, csr
);
544 /* wait to drain cppi dma pipe line */
547 csr
= musb_readw(epio
, MUSB_RXCSR
);
548 if (csr
& MUSB_RXCSR_RXPKTRDY
) {
549 csr
|= MUSB_RXCSR_FLUSHFIFO
;
550 musb_writew(epio
, MUSB_RXCSR
, csr
);
551 musb_writew(epio
, MUSB_RXCSR
, csr
);
555 tdbit
= 1 << cppi41_channel
->port_num
;
561 musb_writel(musb
->ctrl_base
, USB_TDOWN
, tdbit
);
562 ret
= dmaengine_terminate_all(cppi41_channel
->dc
);
563 } while (ret
== -EAGAIN
);
566 musb_writel(musb
->ctrl_base
, USB_TDOWN
, tdbit
);
568 csr
= musb_readw(epio
, MUSB_TXCSR
);
569 if (csr
& MUSB_TXCSR_TXPKTRDY
) {
570 csr
|= MUSB_TXCSR_FLUSHFIFO
;
571 musb_writew(epio
, MUSB_TXCSR
, csr
);
575 cppi41_channel
->channel
.status
= MUSB_DMA_STATUS_FREE
;
579 static void cppi41_release_all_dma_chans(struct cppi41_dma_controller
*ctrl
)
584 for (i
= 0; i
< MUSB_DMA_NUM_CHANNELS
; i
++) {
585 dc
= ctrl
->tx_channel
[i
].dc
;
587 dma_release_channel(dc
);
588 dc
= ctrl
->rx_channel
[i
].dc
;
590 dma_release_channel(dc
);
594 static void cppi41_dma_controller_stop(struct cppi41_dma_controller
*controller
)
596 cppi41_release_all_dma_chans(controller
);
599 static int cppi41_dma_controller_start(struct cppi41_dma_controller
*controller
)
601 struct musb
*musb
= controller
->controller
.musb
;
602 struct device
*dev
= musb
->controller
;
603 struct device_node
*np
= dev
->parent
->of_node
;
604 struct cppi41_dma_channel
*cppi41_channel
;
609 count
= of_property_count_strings(np
, "dma-names");
613 for (i
= 0; i
< count
; i
++) {
615 struct dma_channel
*musb_dma
;
620 ret
= of_property_read_string_index(np
, "dma-names", i
, &str
);
623 if (strstarts(str
, "tx"))
625 else if (strstarts(str
, "rx"))
628 dev_err(dev
, "Wrong dmatype %s\n", str
);
631 ret
= kstrtouint(str
+ 2, 0, &port
);
636 if (port
> MUSB_DMA_NUM_CHANNELS
|| !port
)
639 cppi41_channel
= &controller
->tx_channel
[port
- 1];
641 cppi41_channel
= &controller
->rx_channel
[port
- 1];
643 cppi41_channel
->controller
= controller
;
644 cppi41_channel
->port_num
= port
;
645 cppi41_channel
->is_tx
= is_tx
;
646 INIT_LIST_HEAD(&cppi41_channel
->tx_check
);
648 musb_dma
= &cppi41_channel
->channel
;
649 musb_dma
->private_data
= cppi41_channel
;
650 musb_dma
->status
= MUSB_DMA_STATUS_FREE
;
651 musb_dma
->max_len
= SZ_4M
;
653 dc
= dma_request_slave_channel(dev
->parent
, str
);
655 dev_err(dev
, "Failed to request %s.\n", str
);
659 cppi41_channel
->dc
= dc
;
663 cppi41_release_all_dma_chans(controller
);
667 void cppi41_dma_controller_destroy(struct dma_controller
*c
)
669 struct cppi41_dma_controller
*controller
= container_of(c
,
670 struct cppi41_dma_controller
, controller
);
672 hrtimer_cancel(&controller
->early_tx
);
673 cppi41_dma_controller_stop(controller
);
676 EXPORT_SYMBOL_GPL(cppi41_dma_controller_destroy
);
678 struct dma_controller
*
679 cppi41_dma_controller_create(struct musb
*musb
, void __iomem
*base
)
681 struct cppi41_dma_controller
*controller
;
684 if (!musb
->controller
->parent
->of_node
) {
685 dev_err(musb
->controller
, "Need DT for the DMA engine.\n");
689 controller
= kzalloc(sizeof(*controller
), GFP_KERNEL
);
693 hrtimer_init(&controller
->early_tx
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
694 controller
->early_tx
.function
= cppi41_recheck_tx_req
;
695 INIT_LIST_HEAD(&controller
->early_tx_list
);
697 controller
->controller
.channel_alloc
= cppi41_dma_channel_allocate
;
698 controller
->controller
.channel_release
= cppi41_dma_channel_release
;
699 controller
->controller
.channel_program
= cppi41_dma_channel_program
;
700 controller
->controller
.channel_abort
= cppi41_dma_channel_abort
;
701 controller
->controller
.is_compatible
= cppi41_is_compatible
;
702 controller
->controller
.musb
= musb
;
704 ret
= cppi41_dma_controller_start(controller
);
707 return &controller
->controller
;
712 if (ret
== -EPROBE_DEFER
)
716 EXPORT_SYMBOL_GPL(cppi41_dma_controller_create
);