]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/dma/dw_dmac.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-3.7' into next
[mirror_ubuntu-artful-kernel.git] / drivers / dma / dw_dmac.c
1 /*
2 * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
3 * AVR32 systems.)
4 *
5 * Copyright (C) 2007-2008 Atmel Corporation
6 * Copyright (C) 2010-2011 ST Microelectronics
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12 #include <linux/bitops.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dmaengine.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/of.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25
26 #include "dw_dmac_regs.h"
27 #include "dmaengine.h"
28
29 /*
30 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
31 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
32 * of which use ARM any more). See the "Databook" from Synopsys for
33 * information beyond what licensees probably provide.
34 *
35 * The driver has currently been tested only with the Atmel AT32AP7000,
36 * which does not support descriptor writeback.
37 */
38
39 #define DWC_DEFAULT_CTLLO(_chan) ({ \
40 struct dw_dma_slave *__slave = (_chan->private); \
41 struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \
42 struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
43 int _dms = __slave ? __slave->dst_master : 0; \
44 int _sms = __slave ? __slave->src_master : 1; \
45 u8 _smsize = __slave ? _sconfig->src_maxburst : \
46 DW_DMA_MSIZE_16; \
47 u8 _dmsize = __slave ? _sconfig->dst_maxburst : \
48 DW_DMA_MSIZE_16; \
49 \
50 (DWC_CTLL_DST_MSIZE(_dmsize) \
51 | DWC_CTLL_SRC_MSIZE(_smsize) \
52 | DWC_CTLL_LLP_D_EN \
53 | DWC_CTLL_LLP_S_EN \
54 | DWC_CTLL_DMS(_dms) \
55 | DWC_CTLL_SMS(_sms)); \
56 })
57
58 /*
59 * This is configuration-dependent and usually a funny size like 4095.
60 *
61 * Note that this is a transfer count, i.e. if we transfer 32-bit
62 * words, we can do 16380 bytes per descriptor.
63 *
64 * This parameter is also system-specific.
65 */
66 #define DWC_MAX_COUNT 4095U
67
68 /*
69 * Number of descriptors to allocate for each channel. This should be
70 * made configurable somehow; preferably, the clients (at least the
71 * ones using slave transfers) should be able to give us a hint.
72 */
73 #define NR_DESCS_PER_CHANNEL 64
74
75 /*----------------------------------------------------------------------*/
76
77 /*
78 * Because we're not relying on writeback from the controller (it may not
79 * even be configured into the core!) we don't need to use dma_pool. These
80 * descriptors -- and associated data -- are cacheable. We do need to make
81 * sure their dcache entries are written back before handing them off to
82 * the controller, though.
83 */
84
85 static struct device *chan2dev(struct dma_chan *chan)
86 {
87 return &chan->dev->device;
88 }
89 static struct device *chan2parent(struct dma_chan *chan)
90 {
91 return chan->dev->device.parent;
92 }
93
94 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
95 {
96 return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
97 }
98
99 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
100 {
101 struct dw_desc *desc, *_desc;
102 struct dw_desc *ret = NULL;
103 unsigned int i = 0;
104 unsigned long flags;
105
106 spin_lock_irqsave(&dwc->lock, flags);
107 list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
108 i++;
109 if (async_tx_test_ack(&desc->txd)) {
110 list_del(&desc->desc_node);
111 ret = desc;
112 break;
113 }
114 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
115 }
116 spin_unlock_irqrestore(&dwc->lock, flags);
117
118 dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
119
120 return ret;
121 }
122
123 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
124 {
125 struct dw_desc *child;
126
127 list_for_each_entry(child, &desc->tx_list, desc_node)
128 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
129 child->txd.phys, sizeof(child->lli),
130 DMA_TO_DEVICE);
131 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
132 desc->txd.phys, sizeof(desc->lli),
133 DMA_TO_DEVICE);
134 }
135
136 /*
137 * Move a descriptor, including any children, to the free list.
138 * `desc' must not be on any lists.
139 */
140 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
141 {
142 unsigned long flags;
143
144 if (desc) {
145 struct dw_desc *child;
146
147 dwc_sync_desc_for_cpu(dwc, desc);
148
149 spin_lock_irqsave(&dwc->lock, flags);
150 list_for_each_entry(child, &desc->tx_list, desc_node)
151 dev_vdbg(chan2dev(&dwc->chan),
152 "moving child desc %p to freelist\n",
153 child);
154 list_splice_init(&desc->tx_list, &dwc->free_list);
155 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
156 list_add(&desc->desc_node, &dwc->free_list);
157 spin_unlock_irqrestore(&dwc->lock, flags);
158 }
159 }
160
161 static void dwc_initialize(struct dw_dma_chan *dwc)
162 {
163 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
164 struct dw_dma_slave *dws = dwc->chan.private;
165 u32 cfghi = DWC_CFGH_FIFO_MODE;
166 u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
167
168 if (dwc->initialized == true)
169 return;
170
171 if (dws) {
172 /*
173 * We need controller-specific data to set up slave
174 * transfers.
175 */
176 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
177
178 cfghi = dws->cfg_hi;
179 cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
180 }
181
182 channel_writel(dwc, CFG_LO, cfglo);
183 channel_writel(dwc, CFG_HI, cfghi);
184
185 /* Enable interrupts */
186 channel_set_bit(dw, MASK.XFER, dwc->mask);
187 channel_set_bit(dw, MASK.ERROR, dwc->mask);
188
189 dwc->initialized = true;
190 }
191
192 /*----------------------------------------------------------------------*/
193
194 static inline unsigned int dwc_fast_fls(unsigned long long v)
195 {
196 /*
197 * We can be a lot more clever here, but this should take care
198 * of the most common optimization.
199 */
200 if (!(v & 7))
201 return 3;
202 else if (!(v & 3))
203 return 2;
204 else if (!(v & 1))
205 return 1;
206 return 0;
207 }
208
209 static void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
210 {
211 dev_err(chan2dev(&dwc->chan),
212 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
213 channel_readl(dwc, SAR),
214 channel_readl(dwc, DAR),
215 channel_readl(dwc, LLP),
216 channel_readl(dwc, CTL_HI),
217 channel_readl(dwc, CTL_LO));
218 }
219
220
221 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
222 {
223 channel_clear_bit(dw, CH_EN, dwc->mask);
224 while (dma_readl(dw, CH_EN) & dwc->mask)
225 cpu_relax();
226 }
227
228 /*----------------------------------------------------------------------*/
229
230 /* Called with dwc->lock held and bh disabled */
231 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
232 {
233 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
234
235 /* ASSERT: channel is idle */
236 if (dma_readl(dw, CH_EN) & dwc->mask) {
237 dev_err(chan2dev(&dwc->chan),
238 "BUG: Attempted to start non-idle channel\n");
239 dwc_dump_chan_regs(dwc);
240
241 /* The tasklet will hopefully advance the queue... */
242 return;
243 }
244
245 dwc_initialize(dwc);
246
247 channel_writel(dwc, LLP, first->txd.phys);
248 channel_writel(dwc, CTL_LO,
249 DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
250 channel_writel(dwc, CTL_HI, 0);
251 channel_set_bit(dw, CH_EN, dwc->mask);
252 }
253
254 /*----------------------------------------------------------------------*/
255
256 static void
257 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
258 bool callback_required)
259 {
260 dma_async_tx_callback callback = NULL;
261 void *param = NULL;
262 struct dma_async_tx_descriptor *txd = &desc->txd;
263 struct dw_desc *child;
264 unsigned long flags;
265
266 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
267
268 spin_lock_irqsave(&dwc->lock, flags);
269 dma_cookie_complete(txd);
270 if (callback_required) {
271 callback = txd->callback;
272 param = txd->callback_param;
273 }
274
275 dwc_sync_desc_for_cpu(dwc, desc);
276
277 /* async_tx_ack */
278 list_for_each_entry(child, &desc->tx_list, desc_node)
279 async_tx_ack(&child->txd);
280 async_tx_ack(&desc->txd);
281
282 list_splice_init(&desc->tx_list, &dwc->free_list);
283 list_move(&desc->desc_node, &dwc->free_list);
284
285 if (!dwc->chan.private) {
286 struct device *parent = chan2parent(&dwc->chan);
287 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
288 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
289 dma_unmap_single(parent, desc->lli.dar,
290 desc->len, DMA_FROM_DEVICE);
291 else
292 dma_unmap_page(parent, desc->lli.dar,
293 desc->len, DMA_FROM_DEVICE);
294 }
295 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
296 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
297 dma_unmap_single(parent, desc->lli.sar,
298 desc->len, DMA_TO_DEVICE);
299 else
300 dma_unmap_page(parent, desc->lli.sar,
301 desc->len, DMA_TO_DEVICE);
302 }
303 }
304
305 spin_unlock_irqrestore(&dwc->lock, flags);
306
307 if (callback_required && callback)
308 callback(param);
309 }
310
311 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
312 {
313 struct dw_desc *desc, *_desc;
314 LIST_HEAD(list);
315 unsigned long flags;
316
317 spin_lock_irqsave(&dwc->lock, flags);
318 if (dma_readl(dw, CH_EN) & dwc->mask) {
319 dev_err(chan2dev(&dwc->chan),
320 "BUG: XFER bit set, but channel not idle!\n");
321
322 /* Try to continue after resetting the channel... */
323 dwc_chan_disable(dw, dwc);
324 }
325
326 /*
327 * Submit queued descriptors ASAP, i.e. before we go through
328 * the completed ones.
329 */
330 list_splice_init(&dwc->active_list, &list);
331 if (!list_empty(&dwc->queue)) {
332 list_move(dwc->queue.next, &dwc->active_list);
333 dwc_dostart(dwc, dwc_first_active(dwc));
334 }
335
336 spin_unlock_irqrestore(&dwc->lock, flags);
337
338 list_for_each_entry_safe(desc, _desc, &list, desc_node)
339 dwc_descriptor_complete(dwc, desc, true);
340 }
341
342 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
343 {
344 dma_addr_t llp;
345 struct dw_desc *desc, *_desc;
346 struct dw_desc *child;
347 u32 status_xfer;
348 unsigned long flags;
349
350 spin_lock_irqsave(&dwc->lock, flags);
351 llp = channel_readl(dwc, LLP);
352 status_xfer = dma_readl(dw, RAW.XFER);
353
354 if (status_xfer & dwc->mask) {
355 /* Everything we've submitted is done */
356 dma_writel(dw, CLEAR.XFER, dwc->mask);
357 spin_unlock_irqrestore(&dwc->lock, flags);
358
359 dwc_complete_all(dw, dwc);
360 return;
361 }
362
363 if (list_empty(&dwc->active_list)) {
364 spin_unlock_irqrestore(&dwc->lock, flags);
365 return;
366 }
367
368 dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__,
369 (unsigned long long)llp);
370
371 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
372 /* check first descriptors addr */
373 if (desc->txd.phys == llp) {
374 spin_unlock_irqrestore(&dwc->lock, flags);
375 return;
376 }
377
378 /* check first descriptors llp */
379 if (desc->lli.llp == llp) {
380 /* This one is currently in progress */
381 spin_unlock_irqrestore(&dwc->lock, flags);
382 return;
383 }
384
385 list_for_each_entry(child, &desc->tx_list, desc_node)
386 if (child->lli.llp == llp) {
387 /* Currently in progress */
388 spin_unlock_irqrestore(&dwc->lock, flags);
389 return;
390 }
391
392 /*
393 * No descriptors so far seem to be in progress, i.e.
394 * this one must be done.
395 */
396 spin_unlock_irqrestore(&dwc->lock, flags);
397 dwc_descriptor_complete(dwc, desc, true);
398 spin_lock_irqsave(&dwc->lock, flags);
399 }
400
401 dev_err(chan2dev(&dwc->chan),
402 "BUG: All descriptors done, but channel not idle!\n");
403
404 /* Try to continue after resetting the channel... */
405 dwc_chan_disable(dw, dwc);
406
407 if (!list_empty(&dwc->queue)) {
408 list_move(dwc->queue.next, &dwc->active_list);
409 dwc_dostart(dwc, dwc_first_active(dwc));
410 }
411 spin_unlock_irqrestore(&dwc->lock, flags);
412 }
413
414 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
415 {
416 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
417 " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
418 lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
419 }
420
421 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
422 {
423 struct dw_desc *bad_desc;
424 struct dw_desc *child;
425 unsigned long flags;
426
427 dwc_scan_descriptors(dw, dwc);
428
429 spin_lock_irqsave(&dwc->lock, flags);
430
431 /*
432 * The descriptor currently at the head of the active list is
433 * borked. Since we don't have any way to report errors, we'll
434 * just have to scream loudly and try to carry on.
435 */
436 bad_desc = dwc_first_active(dwc);
437 list_del_init(&bad_desc->desc_node);
438 list_move(dwc->queue.next, dwc->active_list.prev);
439
440 /* Clear the error flag and try to restart the controller */
441 dma_writel(dw, CLEAR.ERROR, dwc->mask);
442 if (!list_empty(&dwc->active_list))
443 dwc_dostart(dwc, dwc_first_active(dwc));
444
445 /*
446 * KERN_CRITICAL may seem harsh, but since this only happens
447 * when someone submits a bad physical address in a
448 * descriptor, we should consider ourselves lucky that the
449 * controller flagged an error instead of scribbling over
450 * random memory locations.
451 */
452 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
453 "Bad descriptor submitted for DMA!\n");
454 dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
455 " cookie: %d\n", bad_desc->txd.cookie);
456 dwc_dump_lli(dwc, &bad_desc->lli);
457 list_for_each_entry(child, &bad_desc->tx_list, desc_node)
458 dwc_dump_lli(dwc, &child->lli);
459
460 spin_unlock_irqrestore(&dwc->lock, flags);
461
462 /* Pretend the descriptor completed successfully */
463 dwc_descriptor_complete(dwc, bad_desc, true);
464 }
465
466 /* --------------------- Cyclic DMA API extensions -------------------- */
467
468 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
469 {
470 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
471 return channel_readl(dwc, SAR);
472 }
473 EXPORT_SYMBOL(dw_dma_get_src_addr);
474
475 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
476 {
477 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
478 return channel_readl(dwc, DAR);
479 }
480 EXPORT_SYMBOL(dw_dma_get_dst_addr);
481
482 /* called with dwc->lock held and all DMAC interrupts disabled */
483 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
484 u32 status_err, u32 status_xfer)
485 {
486 unsigned long flags;
487
488 if (dwc->mask) {
489 void (*callback)(void *param);
490 void *callback_param;
491
492 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
493 channel_readl(dwc, LLP));
494
495 callback = dwc->cdesc->period_callback;
496 callback_param = dwc->cdesc->period_callback_param;
497
498 if (callback)
499 callback(callback_param);
500 }
501
502 /*
503 * Error and transfer complete are highly unlikely, and will most
504 * likely be due to a configuration error by the user.
505 */
506 if (unlikely(status_err & dwc->mask) ||
507 unlikely(status_xfer & dwc->mask)) {
508 int i;
509
510 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
511 "interrupt, stopping DMA transfer\n",
512 status_xfer ? "xfer" : "error");
513
514 spin_lock_irqsave(&dwc->lock, flags);
515
516 dwc_dump_chan_regs(dwc);
517
518 dwc_chan_disable(dw, dwc);
519
520 /* make sure DMA does not restart by loading a new list */
521 channel_writel(dwc, LLP, 0);
522 channel_writel(dwc, CTL_LO, 0);
523 channel_writel(dwc, CTL_HI, 0);
524
525 dma_writel(dw, CLEAR.ERROR, dwc->mask);
526 dma_writel(dw, CLEAR.XFER, dwc->mask);
527
528 for (i = 0; i < dwc->cdesc->periods; i++)
529 dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
530
531 spin_unlock_irqrestore(&dwc->lock, flags);
532 }
533 }
534
535 /* ------------------------------------------------------------------------- */
536
537 static void dw_dma_tasklet(unsigned long data)
538 {
539 struct dw_dma *dw = (struct dw_dma *)data;
540 struct dw_dma_chan *dwc;
541 u32 status_xfer;
542 u32 status_err;
543 int i;
544
545 status_xfer = dma_readl(dw, RAW.XFER);
546 status_err = dma_readl(dw, RAW.ERROR);
547
548 dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
549
550 for (i = 0; i < dw->dma.chancnt; i++) {
551 dwc = &dw->chan[i];
552 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
553 dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
554 else if (status_err & (1 << i))
555 dwc_handle_error(dw, dwc);
556 else if (status_xfer & (1 << i))
557 dwc_scan_descriptors(dw, dwc);
558 }
559
560 /*
561 * Re-enable interrupts.
562 */
563 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
564 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
565 }
566
567 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
568 {
569 struct dw_dma *dw = dev_id;
570 u32 status;
571
572 dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__,
573 dma_readl(dw, STATUS_INT));
574
575 /*
576 * Just disable the interrupts. We'll turn them back on in the
577 * softirq handler.
578 */
579 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
580 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
581
582 status = dma_readl(dw, STATUS_INT);
583 if (status) {
584 dev_err(dw->dma.dev,
585 "BUG: Unexpected interrupts pending: 0x%x\n",
586 status);
587
588 /* Try to recover */
589 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
590 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
591 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
592 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
593 }
594
595 tasklet_schedule(&dw->tasklet);
596
597 return IRQ_HANDLED;
598 }
599
600 /*----------------------------------------------------------------------*/
601
602 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
603 {
604 struct dw_desc *desc = txd_to_dw_desc(tx);
605 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan);
606 dma_cookie_t cookie;
607 unsigned long flags;
608
609 spin_lock_irqsave(&dwc->lock, flags);
610 cookie = dma_cookie_assign(tx);
611
612 /*
613 * REVISIT: We should attempt to chain as many descriptors as
614 * possible, perhaps even appending to those already submitted
615 * for DMA. But this is hard to do in a race-free manner.
616 */
617 if (list_empty(&dwc->active_list)) {
618 dev_vdbg(chan2dev(tx->chan), "%s: started %u\n", __func__,
619 desc->txd.cookie);
620 list_add_tail(&desc->desc_node, &dwc->active_list);
621 dwc_dostart(dwc, dwc_first_active(dwc));
622 } else {
623 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__,
624 desc->txd.cookie);
625
626 list_add_tail(&desc->desc_node, &dwc->queue);
627 }
628
629 spin_unlock_irqrestore(&dwc->lock, flags);
630
631 return cookie;
632 }
633
634 static struct dma_async_tx_descriptor *
635 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
636 size_t len, unsigned long flags)
637 {
638 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
639 struct dw_desc *desc;
640 struct dw_desc *first;
641 struct dw_desc *prev;
642 size_t xfer_count;
643 size_t offset;
644 unsigned int src_width;
645 unsigned int dst_width;
646 u32 ctllo;
647
648 dev_vdbg(chan2dev(chan),
649 "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__,
650 (unsigned long long)dest, (unsigned long long)src,
651 len, flags);
652
653 if (unlikely(!len)) {
654 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
655 return NULL;
656 }
657
658 src_width = dst_width = dwc_fast_fls(src | dest | len);
659
660 ctllo = DWC_DEFAULT_CTLLO(chan)
661 | DWC_CTLL_DST_WIDTH(dst_width)
662 | DWC_CTLL_SRC_WIDTH(src_width)
663 | DWC_CTLL_DST_INC
664 | DWC_CTLL_SRC_INC
665 | DWC_CTLL_FC_M2M;
666 prev = first = NULL;
667
668 for (offset = 0; offset < len; offset += xfer_count << src_width) {
669 xfer_count = min_t(size_t, (len - offset) >> src_width,
670 DWC_MAX_COUNT);
671
672 desc = dwc_desc_get(dwc);
673 if (!desc)
674 goto err_desc_get;
675
676 desc->lli.sar = src + offset;
677 desc->lli.dar = dest + offset;
678 desc->lli.ctllo = ctllo;
679 desc->lli.ctlhi = xfer_count;
680
681 if (!first) {
682 first = desc;
683 } else {
684 prev->lli.llp = desc->txd.phys;
685 dma_sync_single_for_device(chan2parent(chan),
686 prev->txd.phys, sizeof(prev->lli),
687 DMA_TO_DEVICE);
688 list_add_tail(&desc->desc_node,
689 &first->tx_list);
690 }
691 prev = desc;
692 }
693
694
695 if (flags & DMA_PREP_INTERRUPT)
696 /* Trigger interrupt after last block */
697 prev->lli.ctllo |= DWC_CTLL_INT_EN;
698
699 prev->lli.llp = 0;
700 dma_sync_single_for_device(chan2parent(chan),
701 prev->txd.phys, sizeof(prev->lli),
702 DMA_TO_DEVICE);
703
704 first->txd.flags = flags;
705 first->len = len;
706
707 return &first->txd;
708
709 err_desc_get:
710 dwc_desc_put(dwc, first);
711 return NULL;
712 }
713
714 static struct dma_async_tx_descriptor *
715 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
716 unsigned int sg_len, enum dma_transfer_direction direction,
717 unsigned long flags, void *context)
718 {
719 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
720 struct dw_dma_slave *dws = chan->private;
721 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
722 struct dw_desc *prev;
723 struct dw_desc *first;
724 u32 ctllo;
725 dma_addr_t reg;
726 unsigned int reg_width;
727 unsigned int mem_width;
728 unsigned int i;
729 struct scatterlist *sg;
730 size_t total_len = 0;
731
732 dev_vdbg(chan2dev(chan), "%s\n", __func__);
733
734 if (unlikely(!dws || !sg_len))
735 return NULL;
736
737 prev = first = NULL;
738
739 switch (direction) {
740 case DMA_MEM_TO_DEV:
741 reg_width = __fls(sconfig->dst_addr_width);
742 reg = sconfig->dst_addr;
743 ctllo = (DWC_DEFAULT_CTLLO(chan)
744 | DWC_CTLL_DST_WIDTH(reg_width)
745 | DWC_CTLL_DST_FIX
746 | DWC_CTLL_SRC_INC);
747
748 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
749 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
750
751 for_each_sg(sgl, sg, sg_len, i) {
752 struct dw_desc *desc;
753 u32 len, dlen, mem;
754
755 mem = sg_dma_address(sg);
756 len = sg_dma_len(sg);
757
758 mem_width = dwc_fast_fls(mem | len);
759
760 slave_sg_todev_fill_desc:
761 desc = dwc_desc_get(dwc);
762 if (!desc) {
763 dev_err(chan2dev(chan),
764 "not enough descriptors available\n");
765 goto err_desc_get;
766 }
767
768 desc->lli.sar = mem;
769 desc->lli.dar = reg;
770 desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
771 if ((len >> mem_width) > DWC_MAX_COUNT) {
772 dlen = DWC_MAX_COUNT << mem_width;
773 mem += dlen;
774 len -= dlen;
775 } else {
776 dlen = len;
777 len = 0;
778 }
779
780 desc->lli.ctlhi = dlen >> mem_width;
781
782 if (!first) {
783 first = desc;
784 } else {
785 prev->lli.llp = desc->txd.phys;
786 dma_sync_single_for_device(chan2parent(chan),
787 prev->txd.phys,
788 sizeof(prev->lli),
789 DMA_TO_DEVICE);
790 list_add_tail(&desc->desc_node,
791 &first->tx_list);
792 }
793 prev = desc;
794 total_len += dlen;
795
796 if (len)
797 goto slave_sg_todev_fill_desc;
798 }
799 break;
800 case DMA_DEV_TO_MEM:
801 reg_width = __fls(sconfig->src_addr_width);
802 reg = sconfig->src_addr;
803 ctllo = (DWC_DEFAULT_CTLLO(chan)
804 | DWC_CTLL_SRC_WIDTH(reg_width)
805 | DWC_CTLL_DST_INC
806 | DWC_CTLL_SRC_FIX);
807
808 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
809 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
810
811 for_each_sg(sgl, sg, sg_len, i) {
812 struct dw_desc *desc;
813 u32 len, dlen, mem;
814
815 mem = sg_dma_address(sg);
816 len = sg_dma_len(sg);
817
818 mem_width = dwc_fast_fls(mem | len);
819
820 slave_sg_fromdev_fill_desc:
821 desc = dwc_desc_get(dwc);
822 if (!desc) {
823 dev_err(chan2dev(chan),
824 "not enough descriptors available\n");
825 goto err_desc_get;
826 }
827
828 desc->lli.sar = reg;
829 desc->lli.dar = mem;
830 desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
831 if ((len >> reg_width) > DWC_MAX_COUNT) {
832 dlen = DWC_MAX_COUNT << reg_width;
833 mem += dlen;
834 len -= dlen;
835 } else {
836 dlen = len;
837 len = 0;
838 }
839 desc->lli.ctlhi = dlen >> reg_width;
840
841 if (!first) {
842 first = desc;
843 } else {
844 prev->lli.llp = desc->txd.phys;
845 dma_sync_single_for_device(chan2parent(chan),
846 prev->txd.phys,
847 sizeof(prev->lli),
848 DMA_TO_DEVICE);
849 list_add_tail(&desc->desc_node,
850 &first->tx_list);
851 }
852 prev = desc;
853 total_len += dlen;
854
855 if (len)
856 goto slave_sg_fromdev_fill_desc;
857 }
858 break;
859 default:
860 return NULL;
861 }
862
863 if (flags & DMA_PREP_INTERRUPT)
864 /* Trigger interrupt after last block */
865 prev->lli.ctllo |= DWC_CTLL_INT_EN;
866
867 prev->lli.llp = 0;
868 dma_sync_single_for_device(chan2parent(chan),
869 prev->txd.phys, sizeof(prev->lli),
870 DMA_TO_DEVICE);
871
872 first->len = total_len;
873
874 return &first->txd;
875
876 err_desc_get:
877 dwc_desc_put(dwc, first);
878 return NULL;
879 }
880
881 /*
882 * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
883 * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
884 *
885 * NOTE: burst size 2 is not supported by controller.
886 *
887 * This can be done by finding least significant bit set: n & (n - 1)
888 */
889 static inline void convert_burst(u32 *maxburst)
890 {
891 if (*maxburst > 1)
892 *maxburst = fls(*maxburst) - 2;
893 else
894 *maxburst = 0;
895 }
896
897 static int
898 set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
899 {
900 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
901
902 /* Check if it is chan is configured for slave transfers */
903 if (!chan->private)
904 return -EINVAL;
905
906 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
907
908 convert_burst(&dwc->dma_sconfig.src_maxburst);
909 convert_burst(&dwc->dma_sconfig.dst_maxburst);
910
911 return 0;
912 }
913
914 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
915 unsigned long arg)
916 {
917 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
918 struct dw_dma *dw = to_dw_dma(chan->device);
919 struct dw_desc *desc, *_desc;
920 unsigned long flags;
921 u32 cfglo;
922 LIST_HEAD(list);
923
924 if (cmd == DMA_PAUSE) {
925 spin_lock_irqsave(&dwc->lock, flags);
926
927 cfglo = channel_readl(dwc, CFG_LO);
928 channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
929 while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY))
930 cpu_relax();
931
932 dwc->paused = true;
933 spin_unlock_irqrestore(&dwc->lock, flags);
934 } else if (cmd == DMA_RESUME) {
935 if (!dwc->paused)
936 return 0;
937
938 spin_lock_irqsave(&dwc->lock, flags);
939
940 cfglo = channel_readl(dwc, CFG_LO);
941 channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
942 dwc->paused = false;
943
944 spin_unlock_irqrestore(&dwc->lock, flags);
945 } else if (cmd == DMA_TERMINATE_ALL) {
946 spin_lock_irqsave(&dwc->lock, flags);
947
948 dwc_chan_disable(dw, dwc);
949
950 dwc->paused = false;
951
952 /* active_list entries will end up before queued entries */
953 list_splice_init(&dwc->queue, &list);
954 list_splice_init(&dwc->active_list, &list);
955
956 spin_unlock_irqrestore(&dwc->lock, flags);
957
958 /* Flush all pending and queued descriptors */
959 list_for_each_entry_safe(desc, _desc, &list, desc_node)
960 dwc_descriptor_complete(dwc, desc, false);
961 } else if (cmd == DMA_SLAVE_CONFIG) {
962 return set_runtime_config(chan, (struct dma_slave_config *)arg);
963 } else {
964 return -ENXIO;
965 }
966
967 return 0;
968 }
969
970 static enum dma_status
971 dwc_tx_status(struct dma_chan *chan,
972 dma_cookie_t cookie,
973 struct dma_tx_state *txstate)
974 {
975 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
976 enum dma_status ret;
977
978 ret = dma_cookie_status(chan, cookie, txstate);
979 if (ret != DMA_SUCCESS) {
980 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
981
982 ret = dma_cookie_status(chan, cookie, txstate);
983 }
984
985 if (ret != DMA_SUCCESS)
986 dma_set_residue(txstate, dwc_first_active(dwc)->len);
987
988 if (dwc->paused)
989 return DMA_PAUSED;
990
991 return ret;
992 }
993
994 static void dwc_issue_pending(struct dma_chan *chan)
995 {
996 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
997
998 if (!list_empty(&dwc->queue))
999 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1000 }
1001
1002 static int dwc_alloc_chan_resources(struct dma_chan *chan)
1003 {
1004 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1005 struct dw_dma *dw = to_dw_dma(chan->device);
1006 struct dw_desc *desc;
1007 int i;
1008 unsigned long flags;
1009
1010 dev_vdbg(chan2dev(chan), "%s\n", __func__);
1011
1012 /* ASSERT: channel is idle */
1013 if (dma_readl(dw, CH_EN) & dwc->mask) {
1014 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
1015 return -EIO;
1016 }
1017
1018 dma_cookie_init(chan);
1019
1020 /*
1021 * NOTE: some controllers may have additional features that we
1022 * need to initialize here, like "scatter-gather" (which
1023 * doesn't mean what you think it means), and status writeback.
1024 */
1025
1026 spin_lock_irqsave(&dwc->lock, flags);
1027 i = dwc->descs_allocated;
1028 while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
1029 spin_unlock_irqrestore(&dwc->lock, flags);
1030
1031 desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
1032 if (!desc) {
1033 dev_info(chan2dev(chan),
1034 "only allocated %d descriptors\n", i);
1035 spin_lock_irqsave(&dwc->lock, flags);
1036 break;
1037 }
1038
1039 INIT_LIST_HEAD(&desc->tx_list);
1040 dma_async_tx_descriptor_init(&desc->txd, chan);
1041 desc->txd.tx_submit = dwc_tx_submit;
1042 desc->txd.flags = DMA_CTRL_ACK;
1043 desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
1044 sizeof(desc->lli), DMA_TO_DEVICE);
1045 dwc_desc_put(dwc, desc);
1046
1047 spin_lock_irqsave(&dwc->lock, flags);
1048 i = ++dwc->descs_allocated;
1049 }
1050
1051 spin_unlock_irqrestore(&dwc->lock, flags);
1052
1053 dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1054
1055 return i;
1056 }
1057
1058 static void dwc_free_chan_resources(struct dma_chan *chan)
1059 {
1060 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1061 struct dw_dma *dw = to_dw_dma(chan->device);
1062 struct dw_desc *desc, *_desc;
1063 unsigned long flags;
1064 LIST_HEAD(list);
1065
1066 dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1067 dwc->descs_allocated);
1068
1069 /* ASSERT: channel is idle */
1070 BUG_ON(!list_empty(&dwc->active_list));
1071 BUG_ON(!list_empty(&dwc->queue));
1072 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1073
1074 spin_lock_irqsave(&dwc->lock, flags);
1075 list_splice_init(&dwc->free_list, &list);
1076 dwc->descs_allocated = 0;
1077 dwc->initialized = false;
1078
1079 /* Disable interrupts */
1080 channel_clear_bit(dw, MASK.XFER, dwc->mask);
1081 channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1082
1083 spin_unlock_irqrestore(&dwc->lock, flags);
1084
1085 list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1086 dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
1087 dma_unmap_single(chan2parent(chan), desc->txd.phys,
1088 sizeof(desc->lli), DMA_TO_DEVICE);
1089 kfree(desc);
1090 }
1091
1092 dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1093 }
1094
1095 /* --------------------- Cyclic DMA API extensions -------------------- */
1096
1097 /**
1098 * dw_dma_cyclic_start - start the cyclic DMA transfer
1099 * @chan: the DMA channel to start
1100 *
1101 * Must be called with soft interrupts disabled. Returns zero on success or
1102 * -errno on failure.
1103 */
1104 int dw_dma_cyclic_start(struct dma_chan *chan)
1105 {
1106 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1107 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1108 unsigned long flags;
1109
1110 if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1111 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1112 return -ENODEV;
1113 }
1114
1115 spin_lock_irqsave(&dwc->lock, flags);
1116
1117 /* assert channel is idle */
1118 if (dma_readl(dw, CH_EN) & dwc->mask) {
1119 dev_err(chan2dev(&dwc->chan),
1120 "BUG: Attempted to start non-idle channel\n");
1121 dwc_dump_chan_regs(dwc);
1122 spin_unlock_irqrestore(&dwc->lock, flags);
1123 return -EBUSY;
1124 }
1125
1126 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1127 dma_writel(dw, CLEAR.XFER, dwc->mask);
1128
1129 /* setup DMAC channel registers */
1130 channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1131 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1132 channel_writel(dwc, CTL_HI, 0);
1133
1134 channel_set_bit(dw, CH_EN, dwc->mask);
1135
1136 spin_unlock_irqrestore(&dwc->lock, flags);
1137
1138 return 0;
1139 }
1140 EXPORT_SYMBOL(dw_dma_cyclic_start);
1141
1142 /**
1143 * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1144 * @chan: the DMA channel to stop
1145 *
1146 * Must be called with soft interrupts disabled.
1147 */
1148 void dw_dma_cyclic_stop(struct dma_chan *chan)
1149 {
1150 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1151 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1152 unsigned long flags;
1153
1154 spin_lock_irqsave(&dwc->lock, flags);
1155
1156 dwc_chan_disable(dw, dwc);
1157
1158 spin_unlock_irqrestore(&dwc->lock, flags);
1159 }
1160 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1161
1162 /**
1163 * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1164 * @chan: the DMA channel to prepare
1165 * @buf_addr: physical DMA address where the buffer starts
1166 * @buf_len: total number of bytes for the entire buffer
1167 * @period_len: number of bytes for each period
1168 * @direction: transfer direction, to or from device
1169 *
1170 * Must be called before trying to start the transfer. Returns a valid struct
1171 * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1172 */
1173 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1174 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1175 enum dma_transfer_direction direction)
1176 {
1177 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1178 struct dma_slave_config *sconfig = &dwc->dma_sconfig;
1179 struct dw_cyclic_desc *cdesc;
1180 struct dw_cyclic_desc *retval = NULL;
1181 struct dw_desc *desc;
1182 struct dw_desc *last = NULL;
1183 unsigned long was_cyclic;
1184 unsigned int reg_width;
1185 unsigned int periods;
1186 unsigned int i;
1187 unsigned long flags;
1188
1189 spin_lock_irqsave(&dwc->lock, flags);
1190 if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1191 spin_unlock_irqrestore(&dwc->lock, flags);
1192 dev_dbg(chan2dev(&dwc->chan),
1193 "queue and/or active list are not empty\n");
1194 return ERR_PTR(-EBUSY);
1195 }
1196
1197 was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1198 spin_unlock_irqrestore(&dwc->lock, flags);
1199 if (was_cyclic) {
1200 dev_dbg(chan2dev(&dwc->chan),
1201 "channel already prepared for cyclic DMA\n");
1202 return ERR_PTR(-EBUSY);
1203 }
1204
1205 retval = ERR_PTR(-EINVAL);
1206
1207 if (direction == DMA_MEM_TO_DEV)
1208 reg_width = __ffs(sconfig->dst_addr_width);
1209 else
1210 reg_width = __ffs(sconfig->src_addr_width);
1211
1212 periods = buf_len / period_len;
1213
1214 /* Check for too big/unaligned periods and unaligned DMA buffer. */
1215 if (period_len > (DWC_MAX_COUNT << reg_width))
1216 goto out_err;
1217 if (unlikely(period_len & ((1 << reg_width) - 1)))
1218 goto out_err;
1219 if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1220 goto out_err;
1221 if (unlikely(!(direction & (DMA_MEM_TO_DEV | DMA_DEV_TO_MEM))))
1222 goto out_err;
1223
1224 retval = ERR_PTR(-ENOMEM);
1225
1226 if (periods > NR_DESCS_PER_CHANNEL)
1227 goto out_err;
1228
1229 cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1230 if (!cdesc)
1231 goto out_err;
1232
1233 cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1234 if (!cdesc->desc)
1235 goto out_err_alloc;
1236
1237 for (i = 0; i < periods; i++) {
1238 desc = dwc_desc_get(dwc);
1239 if (!desc)
1240 goto out_err_desc_get;
1241
1242 switch (direction) {
1243 case DMA_MEM_TO_DEV:
1244 desc->lli.dar = sconfig->dst_addr;
1245 desc->lli.sar = buf_addr + (period_len * i);
1246 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1247 | DWC_CTLL_DST_WIDTH(reg_width)
1248 | DWC_CTLL_SRC_WIDTH(reg_width)
1249 | DWC_CTLL_DST_FIX
1250 | DWC_CTLL_SRC_INC
1251 | DWC_CTLL_INT_EN);
1252
1253 desc->lli.ctllo |= sconfig->device_fc ?
1254 DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
1255 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
1256
1257 break;
1258 case DMA_DEV_TO_MEM:
1259 desc->lli.dar = buf_addr + (period_len * i);
1260 desc->lli.sar = sconfig->src_addr;
1261 desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1262 | DWC_CTLL_SRC_WIDTH(reg_width)
1263 | DWC_CTLL_DST_WIDTH(reg_width)
1264 | DWC_CTLL_DST_INC
1265 | DWC_CTLL_SRC_FIX
1266 | DWC_CTLL_INT_EN);
1267
1268 desc->lli.ctllo |= sconfig->device_fc ?
1269 DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
1270 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
1271
1272 break;
1273 default:
1274 break;
1275 }
1276
1277 desc->lli.ctlhi = (period_len >> reg_width);
1278 cdesc->desc[i] = desc;
1279
1280 if (last) {
1281 last->lli.llp = desc->txd.phys;
1282 dma_sync_single_for_device(chan2parent(chan),
1283 last->txd.phys, sizeof(last->lli),
1284 DMA_TO_DEVICE);
1285 }
1286
1287 last = desc;
1288 }
1289
1290 /* lets make a cyclic list */
1291 last->lli.llp = cdesc->desc[0]->txd.phys;
1292 dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
1293 sizeof(last->lli), DMA_TO_DEVICE);
1294
1295 dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
1296 "period %zu periods %d\n", (unsigned long long)buf_addr,
1297 buf_len, period_len, periods);
1298
1299 cdesc->periods = periods;
1300 dwc->cdesc = cdesc;
1301
1302 return cdesc;
1303
1304 out_err_desc_get:
1305 while (i--)
1306 dwc_desc_put(dwc, cdesc->desc[i]);
1307 out_err_alloc:
1308 kfree(cdesc);
1309 out_err:
1310 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1311 return (struct dw_cyclic_desc *)retval;
1312 }
1313 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1314
1315 /**
1316 * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1317 * @chan: the DMA channel to free
1318 */
1319 void dw_dma_cyclic_free(struct dma_chan *chan)
1320 {
1321 struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
1322 struct dw_dma *dw = to_dw_dma(dwc->chan.device);
1323 struct dw_cyclic_desc *cdesc = dwc->cdesc;
1324 int i;
1325 unsigned long flags;
1326
1327 dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
1328
1329 if (!cdesc)
1330 return;
1331
1332 spin_lock_irqsave(&dwc->lock, flags);
1333
1334 dwc_chan_disable(dw, dwc);
1335
1336 dma_writel(dw, CLEAR.ERROR, dwc->mask);
1337 dma_writel(dw, CLEAR.XFER, dwc->mask);
1338
1339 spin_unlock_irqrestore(&dwc->lock, flags);
1340
1341 for (i = 0; i < cdesc->periods; i++)
1342 dwc_desc_put(dwc, cdesc->desc[i]);
1343
1344 kfree(cdesc->desc);
1345 kfree(cdesc);
1346
1347 clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1348 }
1349 EXPORT_SYMBOL(dw_dma_cyclic_free);
1350
1351 /*----------------------------------------------------------------------*/
1352
1353 static void dw_dma_off(struct dw_dma *dw)
1354 {
1355 int i;
1356
1357 dma_writel(dw, CFG, 0);
1358
1359 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1360 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1361 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1362 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1363
1364 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1365 cpu_relax();
1366
1367 for (i = 0; i < dw->dma.chancnt; i++)
1368 dw->chan[i].initialized = false;
1369 }
1370
1371 static int __devinit dw_probe(struct platform_device *pdev)
1372 {
1373 struct dw_dma_platform_data *pdata;
1374 struct resource *io;
1375 struct dw_dma *dw;
1376 size_t size;
1377 int irq;
1378 int err;
1379 int i;
1380
1381 pdata = dev_get_platdata(&pdev->dev);
1382 if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1383 return -EINVAL;
1384
1385 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1386 if (!io)
1387 return -EINVAL;
1388
1389 irq = platform_get_irq(pdev, 0);
1390 if (irq < 0)
1391 return irq;
1392
1393 size = sizeof(struct dw_dma);
1394 size += pdata->nr_channels * sizeof(struct dw_dma_chan);
1395 dw = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1396 if (!dw)
1397 return -ENOMEM;
1398
1399 dw->regs = devm_request_and_ioremap(&pdev->dev, io);
1400 if (!dw->regs)
1401 return -EBUSY;
1402
1403 dw->clk = devm_clk_get(&pdev->dev, "hclk");
1404 if (IS_ERR(dw->clk))
1405 return PTR_ERR(dw->clk);
1406 clk_prepare_enable(dw->clk);
1407
1408 /* Calculate all channel mask before DMA setup */
1409 dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
1410
1411 /* force dma off, just in case */
1412 dw_dma_off(dw);
1413
1414 /* disable BLOCK interrupts as well */
1415 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1416
1417 err = devm_request_irq(&pdev->dev, irq, dw_dma_interrupt, 0,
1418 "dw_dmac", dw);
1419 if (err)
1420 return err;
1421
1422 platform_set_drvdata(pdev, dw);
1423
1424 tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1425
1426 INIT_LIST_HEAD(&dw->dma.channels);
1427 for (i = 0; i < pdata->nr_channels; i++) {
1428 struct dw_dma_chan *dwc = &dw->chan[i];
1429
1430 dwc->chan.device = &dw->dma;
1431 dma_cookie_init(&dwc->chan);
1432 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1433 list_add_tail(&dwc->chan.device_node,
1434 &dw->dma.channels);
1435 else
1436 list_add(&dwc->chan.device_node, &dw->dma.channels);
1437
1438 /* 7 is highest priority & 0 is lowest. */
1439 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1440 dwc->priority = pdata->nr_channels - i - 1;
1441 else
1442 dwc->priority = i;
1443
1444 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1445 spin_lock_init(&dwc->lock);
1446 dwc->mask = 1 << i;
1447
1448 INIT_LIST_HEAD(&dwc->active_list);
1449 INIT_LIST_HEAD(&dwc->queue);
1450 INIT_LIST_HEAD(&dwc->free_list);
1451
1452 channel_clear_bit(dw, CH_EN, dwc->mask);
1453 }
1454
1455 /* Clear all interrupts on all channels. */
1456 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1457 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1458 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1459 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1460 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1461
1462 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1463 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1464 if (pdata->is_private)
1465 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1466 dw->dma.dev = &pdev->dev;
1467 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1468 dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1469
1470 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1471
1472 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1473 dw->dma.device_control = dwc_control;
1474
1475 dw->dma.device_tx_status = dwc_tx_status;
1476 dw->dma.device_issue_pending = dwc_issue_pending;
1477
1478 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1479
1480 printk(KERN_INFO "%s: DesignWare DMA Controller, %d channels\n",
1481 dev_name(&pdev->dev), pdata->nr_channels);
1482
1483 dma_async_device_register(&dw->dma);
1484
1485 return 0;
1486 }
1487
1488 static int __devexit dw_remove(struct platform_device *pdev)
1489 {
1490 struct dw_dma *dw = platform_get_drvdata(pdev);
1491 struct dw_dma_chan *dwc, *_dwc;
1492
1493 dw_dma_off(dw);
1494 dma_async_device_unregister(&dw->dma);
1495
1496 tasklet_kill(&dw->tasklet);
1497
1498 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1499 chan.device_node) {
1500 list_del(&dwc->chan.device_node);
1501 channel_clear_bit(dw, CH_EN, dwc->mask);
1502 }
1503
1504 return 0;
1505 }
1506
1507 static void dw_shutdown(struct platform_device *pdev)
1508 {
1509 struct dw_dma *dw = platform_get_drvdata(pdev);
1510
1511 dw_dma_off(platform_get_drvdata(pdev));
1512 clk_disable_unprepare(dw->clk);
1513 }
1514
1515 static int dw_suspend_noirq(struct device *dev)
1516 {
1517 struct platform_device *pdev = to_platform_device(dev);
1518 struct dw_dma *dw = platform_get_drvdata(pdev);
1519
1520 dw_dma_off(platform_get_drvdata(pdev));
1521 clk_disable_unprepare(dw->clk);
1522
1523 return 0;
1524 }
1525
1526 static int dw_resume_noirq(struct device *dev)
1527 {
1528 struct platform_device *pdev = to_platform_device(dev);
1529 struct dw_dma *dw = platform_get_drvdata(pdev);
1530
1531 clk_prepare_enable(dw->clk);
1532 dma_writel(dw, CFG, DW_CFG_DMA_EN);
1533 return 0;
1534 }
1535
1536 static const struct dev_pm_ops dw_dev_pm_ops = {
1537 .suspend_noirq = dw_suspend_noirq,
1538 .resume_noirq = dw_resume_noirq,
1539 .freeze_noirq = dw_suspend_noirq,
1540 .thaw_noirq = dw_resume_noirq,
1541 .restore_noirq = dw_resume_noirq,
1542 .poweroff_noirq = dw_suspend_noirq,
1543 };
1544
1545 #ifdef CONFIG_OF
1546 static const struct of_device_id dw_dma_id_table[] = {
1547 { .compatible = "snps,dma-spear1340" },
1548 {}
1549 };
1550 MODULE_DEVICE_TABLE(of, dw_dma_id_table);
1551 #endif
1552
1553 static struct platform_driver dw_driver = {
1554 .remove = __devexit_p(dw_remove),
1555 .shutdown = dw_shutdown,
1556 .driver = {
1557 .name = "dw_dmac",
1558 .pm = &dw_dev_pm_ops,
1559 .of_match_table = of_match_ptr(dw_dma_id_table),
1560 },
1561 };
1562
1563 static int __init dw_init(void)
1564 {
1565 return platform_driver_probe(&dw_driver, dw_probe);
1566 }
1567 subsys_initcall(dw_init);
1568
1569 static void __exit dw_exit(void)
1570 {
1571 platform_driver_unregister(&dw_driver);
1572 }
1573 module_exit(dw_exit);
1574
1575 MODULE_LICENSE("GPL v2");
1576 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1577 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1578 MODULE_AUTHOR("Viresh Kumar <viresh.linux@gmail.com>");
Ubuntu Artful kernel mirror